1	// SPDX-License-Identifier: GPL-2.0
2
3	// Generated by scripts/atomic/gen-atomic-instrumented.sh
4	// DO NOT MODIFY THIS FILE DIRECTLY
5
6	/*
7	* This file provides wrappers with KASAN instrumentation for atomic operations.
8	* To use this functionality an arch's atomic.h file needs to define all
9	* atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
10	* this file at the end. This file provides atomic_read() that forwards to
11	* arch_atomic_read() for actual atomic operation.
12	* Note: if an arch atomic operation is implemented by means of other atomic
13	* operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
14	* arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15	* double instrumentation.
16	*/
17	#ifndef _LINUX_ATOMIC_INSTRUMENTED_H
18	#define _LINUX_ATOMIC_INSTRUMENTED_H
19
20	#include <linux/build_bug.h>
21	#include <linux/compiler.h>
22	#include <linux/instrumented.h>
23
24	static __always_inline int
25	atomic_read(const atomic_t *v)
26	{
27	instrument_atomic_read(v, sizeof(*v));
28	return arch_atomic_read(v);
29	}
30
31	static __always_inline int
32	atomic_read_acquire(const atomic_t *v)
33	{
34	instrument_atomic_read(v, sizeof(*v));
35	return arch_atomic_read_acquire(v);
36	}
37
38	static __always_inline void
39	atomic_set(atomic_t *v, int i)
40	{
41	instrument_atomic_write(v, sizeof(*v));
42	arch_atomic_set(v, i);
43	}
44
45	static __always_inline void
46	atomic_set_release(atomic_t *v, int i)
47	{
48	kcsan_release();
49	instrument_atomic_write(v, sizeof(*v));
50	arch_atomic_set_release(v, i);
51	}
52
53	static __always_inline void
54	atomic_add(int i, atomic_t *v)
55	{
56	instrument_atomic_read_write(v, sizeof(*v));
57	arch_atomic_add(i, v);
58	}
59
60	static __always_inline int
61	atomic_add_return(int i, atomic_t *v)
62	{
63	kcsan_mb();
64	instrument_atomic_read_write(v, sizeof(*v));
65	return arch_atomic_add_return(i, v);
66	}
67
68	static __always_inline int
69	atomic_add_return_acquire(int i, atomic_t *v)
70	{
71	instrument_atomic_read_write(v, sizeof(*v));
72	return arch_atomic_add_return_acquire(i, v);
73	}
74
75	static __always_inline int
76	atomic_add_return_release(int i, atomic_t *v)
77	{
78	kcsan_release();
79	instrument_atomic_read_write(v, sizeof(*v));
80	return arch_atomic_add_return_release(i, v);
81	}
82
83	static __always_inline int
84	atomic_add_return_relaxed(int i, atomic_t *v)
85	{
86	instrument_atomic_read_write(v, sizeof(*v));
87	return arch_atomic_add_return_relaxed(i, v);
88	}
89
90	static __always_inline int
91	atomic_fetch_add(int i, atomic_t *v)
92	{
93	kcsan_mb();
94	instrument_atomic_read_write(v, sizeof(*v));
95	return arch_atomic_fetch_add(i, v);
96	}
97
98	static __always_inline int
99	atomic_fetch_add_acquire(int i, atomic_t *v)
100	{
101	instrument_atomic_read_write(v, sizeof(*v));
102	return arch_atomic_fetch_add_acquire(i, v);
103	}
104
105	static __always_inline int
106	atomic_fetch_add_release(int i, atomic_t *v)
107	{
108	kcsan_release();
109	instrument_atomic_read_write(v, sizeof(*v));
110	return arch_atomic_fetch_add_release(i, v);
111	}
112
113	static __always_inline int
114	atomic_fetch_add_relaxed(int i, atomic_t *v)
115	{
116	instrument_atomic_read_write(v, sizeof(*v));
117	return arch_atomic_fetch_add_relaxed(i, v);
118	}
119
120	static __always_inline void
121	atomic_sub(int i, atomic_t *v)
122	{
123	instrument_atomic_read_write(v, sizeof(*v));
124	arch_atomic_sub(i, v);
125	}
126
127	static __always_inline int
128	atomic_sub_return(int i, atomic_t *v)
129	{
130	kcsan_mb();
131	instrument_atomic_read_write(v, sizeof(*v));
132	return arch_atomic_sub_return(i, v);
133	}
134
135	static __always_inline int
136	atomic_sub_return_acquire(int i, atomic_t *v)
137	{
138	instrument_atomic_read_write(v, sizeof(*v));
139	return arch_atomic_sub_return_acquire(i, v);
140	}
141
142	static __always_inline int
143	atomic_sub_return_release(int i, atomic_t *v)
144	{
145	kcsan_release();
146	instrument_atomic_read_write(v, sizeof(*v));
147	return arch_atomic_sub_return_release(i, v);
148	}
149
150	static __always_inline int
151	atomic_sub_return_relaxed(int i, atomic_t *v)
152	{
153	instrument_atomic_read_write(v, sizeof(*v));
154	return arch_atomic_sub_return_relaxed(i, v);
155	}
156
157	static __always_inline int
158	atomic_fetch_sub(int i, atomic_t *v)
159	{
160	kcsan_mb();
161	instrument_atomic_read_write(v, sizeof(*v));
162	return arch_atomic_fetch_sub(i, v);
163	}
164
165	static __always_inline int
166	atomic_fetch_sub_acquire(int i, atomic_t *v)
167	{
168	instrument_atomic_read_write(v, sizeof(*v));
169	return arch_atomic_fetch_sub_acquire(i, v);
170	}
171
172	static __always_inline int
173	atomic_fetch_sub_release(int i, atomic_t *v)
174	{
175	kcsan_release();
176	instrument_atomic_read_write(v, sizeof(*v));
177	return arch_atomic_fetch_sub_release(i, v);
178	}
179
180	static __always_inline int
181	atomic_fetch_sub_relaxed(int i, atomic_t *v)
182	{
183	instrument_atomic_read_write(v, sizeof(*v));
184	return arch_atomic_fetch_sub_relaxed(i, v);
185	}
186
187	static __always_inline void
188	atomic_inc(atomic_t *v)
189	{
190	instrument_atomic_read_write(v, sizeof(*v));
191	arch_atomic_inc(v);
192	}
193
194	static __always_inline int
195	atomic_inc_return(atomic_t *v)
196	{
197	kcsan_mb();
198	instrument_atomic_read_write(v, sizeof(*v));
199	return arch_atomic_inc_return(v);
200	}
201
202	static __always_inline int
203	atomic_inc_return_acquire(atomic_t *v)
204	{
205	instrument_atomic_read_write(v, sizeof(*v));
206	return arch_atomic_inc_return_acquire(v);
207	}
208
209	static __always_inline int
210	atomic_inc_return_release(atomic_t *v)
211	{
212	kcsan_release();
213	instrument_atomic_read_write(v, sizeof(*v));
214	return arch_atomic_inc_return_release(v);
215	}
216
217	static __always_inline int
218	atomic_inc_return_relaxed(atomic_t *v)
219	{
220	instrument_atomic_read_write(v, sizeof(*v));
221	return arch_atomic_inc_return_relaxed(v);
222	}
223
224	static __always_inline int
225	atomic_fetch_inc(atomic_t *v)
226	{
227	kcsan_mb();
228	instrument_atomic_read_write(v, sizeof(*v));
229	return arch_atomic_fetch_inc(v);
230	}
231
232	static __always_inline int
233	atomic_fetch_inc_acquire(atomic_t *v)
234	{
235	instrument_atomic_read_write(v, sizeof(*v));
236	return arch_atomic_fetch_inc_acquire(v);
237	}
238
239	static __always_inline int
240	atomic_fetch_inc_release(atomic_t *v)
241	{
242	kcsan_release();
243	instrument_atomic_read_write(v, sizeof(*v));
244	return arch_atomic_fetch_inc_release(v);
245	}
246
247	static __always_inline int
248	atomic_fetch_inc_relaxed(atomic_t *v)
249	{
250	instrument_atomic_read_write(v, sizeof(*v));
251	return arch_atomic_fetch_inc_relaxed(v);
252	}
253
254	static __always_inline void
255	atomic_dec(atomic_t *v)
256	{
257	instrument_atomic_read_write(v, sizeof(*v));
258	arch_atomic_dec(v);
259	}
260
261	static __always_inline int
262	atomic_dec_return(atomic_t *v)
263	{
264	kcsan_mb();
265	instrument_atomic_read_write(v, sizeof(*v));
266	return arch_atomic_dec_return(v);
267	}
268
269	static __always_inline int
270	atomic_dec_return_acquire(atomic_t *v)
271	{
272	instrument_atomic_read_write(v, sizeof(*v));
273	return arch_atomic_dec_return_acquire(v);
274	}
275
276	static __always_inline int
277	atomic_dec_return_release(atomic_t *v)
278	{
279	kcsan_release();
280	instrument_atomic_read_write(v, sizeof(*v));
281	return arch_atomic_dec_return_release(v);
282	}
283
284	static __always_inline int
285	atomic_dec_return_relaxed(atomic_t *v)
286	{
287	instrument_atomic_read_write(v, sizeof(*v));
288	return arch_atomic_dec_return_relaxed(v);
289	}
290
291	static __always_inline int
292	atomic_fetch_dec(atomic_t *v)
293	{
294	kcsan_mb();
295	instrument_atomic_read_write(v, sizeof(*v));
296	return arch_atomic_fetch_dec(v);
297	}
298
299	static __always_inline int
300	atomic_fetch_dec_acquire(atomic_t *v)
301	{
302	instrument_atomic_read_write(v, sizeof(*v));
303	return arch_atomic_fetch_dec_acquire(v);
304	}
305
306	static __always_inline int
307	atomic_fetch_dec_release(atomic_t *v)
308	{
309	kcsan_release();
310	instrument_atomic_read_write(v, sizeof(*v));
311	return arch_atomic_fetch_dec_release(v);
312	}
313
314	static __always_inline int
315	atomic_fetch_dec_relaxed(atomic_t *v)
316	{
317	instrument_atomic_read_write(v, sizeof(*v));
318	return arch_atomic_fetch_dec_relaxed(v);
319	}
320
321	static __always_inline void
322	atomic_and(int i, atomic_t *v)
323	{
324	instrument_atomic_read_write(v, sizeof(*v));
325	arch_atomic_and(i, v);
326	}
327
328	static __always_inline int
329	atomic_fetch_and(int i, atomic_t *v)
330	{
331	kcsan_mb();
332	instrument_atomic_read_write(v, sizeof(*v));
333	return arch_atomic_fetch_and(i, v);
334	}
335
336	static __always_inline int
337	atomic_fetch_and_acquire(int i, atomic_t *v)
338	{
339	instrument_atomic_read_write(v, sizeof(*v));
340	return arch_atomic_fetch_and_acquire(i, v);
341	}
342
343	static __always_inline int
344	atomic_fetch_and_release(int i, atomic_t *v)
345	{
346	kcsan_release();
347	instrument_atomic_read_write(v, sizeof(*v));
348	return arch_atomic_fetch_and_release(i, v);
349	}
350
351	static __always_inline int
352	atomic_fetch_and_relaxed(int i, atomic_t *v)
353	{
354	instrument_atomic_read_write(v, sizeof(*v));
355	return arch_atomic_fetch_and_relaxed(i, v);
356	}
357
358	static __always_inline void
359	atomic_andnot(int i, atomic_t *v)
360	{
361	instrument_atomic_read_write(v, sizeof(*v));
362	arch_atomic_andnot(i, v);
363	}
364
365	static __always_inline int
366	atomic_fetch_andnot(int i, atomic_t *v)
367	{
368	kcsan_mb();
369	instrument_atomic_read_write(v, sizeof(*v));
370	return arch_atomic_fetch_andnot(i, v);
371	}
372
373	static __always_inline int
374	atomic_fetch_andnot_acquire(int i, atomic_t *v)
375	{
376	instrument_atomic_read_write(v, sizeof(*v));
377	return arch_atomic_fetch_andnot_acquire(i, v);
378	}
379
380	static __always_inline int
381	atomic_fetch_andnot_release(int i, atomic_t *v)
382	{
383	kcsan_release();
384	instrument_atomic_read_write(v, sizeof(*v));
385	return arch_atomic_fetch_andnot_release(i, v);
386	}
387
388	static __always_inline int
389	atomic_fetch_andnot_relaxed(int i, atomic_t *v)
390	{
391	instrument_atomic_read_write(v, sizeof(*v));
392	return arch_atomic_fetch_andnot_relaxed(i, v);
393	}
394
395	static __always_inline void
396	atomic_or(int i, atomic_t *v)
397	{
398	instrument_atomic_read_write(v, sizeof(*v));
399	arch_atomic_or(i, v);
400	}
401
402	static __always_inline int
403	atomic_fetch_or(int i, atomic_t *v)
404	{
405	kcsan_mb();
406	instrument_atomic_read_write(v, sizeof(*v));
407	return arch_atomic_fetch_or(i, v);
408	}
409
410	static __always_inline int
411	atomic_fetch_or_acquire(int i, atomic_t *v)
412	{
413	instrument_atomic_read_write(v, sizeof(*v));
414	return arch_atomic_fetch_or_acquire(i, v);
415	}
416
417	static __always_inline int
418	atomic_fetch_or_release(int i, atomic_t *v)
419	{
420	kcsan_release();
421	instrument_atomic_read_write(v, sizeof(*v));
422	return arch_atomic_fetch_or_release(i, v);
423	}
424
425	static __always_inline int
426	atomic_fetch_or_relaxed(int i, atomic_t *v)
427	{
428	instrument_atomic_read_write(v, sizeof(*v));
429	return arch_atomic_fetch_or_relaxed(i, v);
430	}
431
432	static __always_inline void
433	atomic_xor(int i, atomic_t *v)
434	{
435	instrument_atomic_read_write(v, sizeof(*v));
436	arch_atomic_xor(i, v);
437	}
438
439	static __always_inline int
440	atomic_fetch_xor(int i, atomic_t *v)
441	{
442	kcsan_mb();
443	instrument_atomic_read_write(v, sizeof(*v));
444	return arch_atomic_fetch_xor(i, v);
445	}
446
447	static __always_inline int
448	atomic_fetch_xor_acquire(int i, atomic_t *v)
449	{
450	instrument_atomic_read_write(v, sizeof(*v));
451	return arch_atomic_fetch_xor_acquire(i, v);
452	}
453
454	static __always_inline int
455	atomic_fetch_xor_release(int i, atomic_t *v)
456	{
457	kcsan_release();
458	instrument_atomic_read_write(v, sizeof(*v));
459	return arch_atomic_fetch_xor_release(i, v);
460	}
461
462	static __always_inline int
463	atomic_fetch_xor_relaxed(int i, atomic_t *v)
464	{
465	instrument_atomic_read_write(v, sizeof(*v));
466	return arch_atomic_fetch_xor_relaxed(i, v);
467	}
468
469	static __always_inline int
470	atomic_xchg(atomic_t *v, int i)
471	{
472	kcsan_mb();
473	instrument_atomic_read_write(v, sizeof(*v));
474	return arch_atomic_xchg(v, i);
475	}
476
477	static __always_inline int
478	atomic_xchg_acquire(atomic_t *v, int i)
479	{
480	instrument_atomic_read_write(v, sizeof(*v));
481	return arch_atomic_xchg_acquire(v, i);
482	}
483
484	static __always_inline int
485	atomic_xchg_release(atomic_t *v, int i)
486	{
487	kcsan_release();
488	instrument_atomic_read_write(v, sizeof(*v));
489	return arch_atomic_xchg_release(v, i);
490	}
491
492	static __always_inline int
493	atomic_xchg_relaxed(atomic_t *v, int i)
494	{
495	instrument_atomic_read_write(v, sizeof(*v));
496	return arch_atomic_xchg_relaxed(v, i);
497	}
498
499	static __always_inline int
500	atomic_cmpxchg(atomic_t *v, int old, int new)
501	{
502	kcsan_mb();
503	instrument_atomic_read_write(v, sizeof(*v));
504	return arch_atomic_cmpxchg(v, old, new);
505	}
506
507	static __always_inline int
508	atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
509	{
510	instrument_atomic_read_write(v, sizeof(*v));
511	return arch_atomic_cmpxchg_acquire(v, old, new);
512	}
513
514	static __always_inline int
515	atomic_cmpxchg_release(atomic_t *v, int old, int new)
516	{
517	kcsan_release();
518	instrument_atomic_read_write(v, sizeof(*v));
519	return arch_atomic_cmpxchg_release(v, old, new);
520	}
521
522	static __always_inline int
523	atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
524	{
525	instrument_atomic_read_write(v, sizeof(*v));
526	return arch_atomic_cmpxchg_relaxed(v, old, new);
527	}
528
529	static __always_inline bool
530	atomic_try_cmpxchg(atomic_t *v, int *old, int new)
531	{
532	kcsan_mb();
533	instrument_atomic_read_write(v, sizeof(*v));
534	instrument_atomic_read_write(old, sizeof(*old));
535	return arch_atomic_try_cmpxchg(v, old, new);
536	}
537
538	static __always_inline bool
539	atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
540	{
541	instrument_atomic_read_write(v, sizeof(*v));
542	instrument_atomic_read_write(old, sizeof(*old));
543	return arch_atomic_try_cmpxchg_acquire(v, old, new);
544	}
545
546	static __always_inline bool
547	atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
548	{
549	kcsan_release();
550	instrument_atomic_read_write(v, sizeof(*v));
551	instrument_atomic_read_write(old, sizeof(*old));
552	return arch_atomic_try_cmpxchg_release(v, old, new);
553	}
554
555	static __always_inline bool
556	atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
557	{
558	instrument_atomic_read_write(v, sizeof(*v));
559	instrument_atomic_read_write(old, sizeof(*old));
560	return arch_atomic_try_cmpxchg_relaxed(v, old, new);
561	}
562
563	static __always_inline bool
564	atomic_sub_and_test(int i, atomic_t *v)
565	{
566	kcsan_mb();
567	instrument_atomic_read_write(v, sizeof(*v));
568	return arch_atomic_sub_and_test(i, v);
569	}
570
571	static __always_inline bool
572	atomic_dec_and_test(atomic_t *v)
573	{
574	kcsan_mb();
575	instrument_atomic_read_write(v, sizeof(*v));
576	return arch_atomic_dec_and_test(v);
577	}
578
579	static __always_inline bool
580	atomic_inc_and_test(atomic_t *v)
581	{
582	kcsan_mb();
583	instrument_atomic_read_write(v, sizeof(*v));
584	return arch_atomic_inc_and_test(v);
585	}
586
587	static __always_inline bool
588	atomic_add_negative(int i, atomic_t *v)
589	{
590	kcsan_mb();
591	instrument_atomic_read_write(v, sizeof(*v));
592	return arch_atomic_add_negative(i, v);
593	}
594
595	static __always_inline int
596	atomic_fetch_add_unless(atomic_t *v, int a, int u)
597	{
598	kcsan_mb();
599	instrument_atomic_read_write(v, sizeof(*v));
600	return arch_atomic_fetch_add_unless(v, a, u);
601	}
602
603	static __always_inline bool
604	atomic_add_unless(atomic_t *v, int a, int u)
605	{
606	kcsan_mb();
607	instrument_atomic_read_write(v, sizeof(*v));
608	return arch_atomic_add_unless(v, a, u);
609	}
610
611	static __always_inline bool
612	atomic_inc_not_zero(atomic_t *v)
613	{
614	kcsan_mb();
615	instrument_atomic_read_write(v, sizeof(*v));
616	return arch_atomic_inc_not_zero(v);
617	}
618
619	static __always_inline bool
620	atomic_inc_unless_negative(atomic_t *v)
621	{
622	kcsan_mb();
623	instrument_atomic_read_write(v, sizeof(*v));
624	return arch_atomic_inc_unless_negative(v);
625	}
626
627	static __always_inline bool
628	atomic_dec_unless_positive(atomic_t *v)
629	{
630	kcsan_mb();
631	instrument_atomic_read_write(v, sizeof(*v));
632	return arch_atomic_dec_unless_positive(v);
633	}
634
635	static __always_inline int
636	atomic_dec_if_positive(atomic_t *v)
637	{
638	kcsan_mb();
639	instrument_atomic_read_write(v, sizeof(*v));
640	return arch_atomic_dec_if_positive(v);
641	}
642
643	static __always_inline s64
644	atomic64_read(const atomic64_t *v)
645	{
646	instrument_atomic_read(v, sizeof(*v));
647	return arch_atomic64_read(v);
648	}
649
650	static __always_inline s64
651	atomic64_read_acquire(const atomic64_t *v)
652	{
653	instrument_atomic_read(v, sizeof(*v));
654	return arch_atomic64_read_acquire(v);
655	}
656
657	static __always_inline void
658	atomic64_set(atomic64_t *v, s64 i)
659	{
660	instrument_atomic_write(v, sizeof(*v));
661	arch_atomic64_set(v, i);
662	}
663
664	static __always_inline void
665	atomic64_set_release(atomic64_t *v, s64 i)
666	{
667	kcsan_release();
668	instrument_atomic_write(v, sizeof(*v));
669	arch_atomic64_set_release(v, i);
670	}
671
672	static __always_inline void
673	atomic64_add(s64 i, atomic64_t *v)
674	{
675	instrument_atomic_read_write(v, sizeof(*v));
676	arch_atomic64_add(i, v);
677	}
678
679	static __always_inline s64
680	atomic64_add_return(s64 i, atomic64_t *v)
681	{
682	kcsan_mb();
683	instrument_atomic_read_write(v, sizeof(*v));
684	return arch_atomic64_add_return(i, v);
685	}
686
687	static __always_inline s64
688	atomic64_add_return_acquire(s64 i, atomic64_t *v)
689	{
690	instrument_atomic_read_write(v, sizeof(*v));
691	return arch_atomic64_add_return_acquire(i, v);
692	}
693
694	static __always_inline s64
695	atomic64_add_return_release(s64 i, atomic64_t *v)
696	{
697	kcsan_release();
698	instrument_atomic_read_write(v, sizeof(*v));
699	return arch_atomic64_add_return_release(i, v);
700	}
701
702	static __always_inline s64
703	atomic64_add_return_relaxed(s64 i, atomic64_t *v)
704	{
705	instrument_atomic_read_write(v, sizeof(*v));
706	return arch_atomic64_add_return_relaxed(i, v);
707	}
708
709	static __always_inline s64
710	atomic64_fetch_add(s64 i, atomic64_t *v)
711	{
712	kcsan_mb();
713	instrument_atomic_read_write(v, sizeof(*v));
714	return arch_atomic64_fetch_add(i, v);
715	}
716
717	static __always_inline s64
718	atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
719	{
720	instrument_atomic_read_write(v, sizeof(*v));
721	return arch_atomic64_fetch_add_acquire(i, v);
722	}
723
724	static __always_inline s64
725	atomic64_fetch_add_release(s64 i, atomic64_t *v)
726	{
727	kcsan_release();
728	instrument_atomic_read_write(v, sizeof(*v));
729	return arch_atomic64_fetch_add_release(i, v);
730	}
731
732	static __always_inline s64
733	atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
734	{
735	instrument_atomic_read_write(v, sizeof(*v));
736	return arch_atomic64_fetch_add_relaxed(i, v);
737	}
738
739	static __always_inline void
740	atomic64_sub(s64 i, atomic64_t *v)
741	{
742	instrument_atomic_read_write(v, sizeof(*v));
743	arch_atomic64_sub(i, v);
744	}
745
746	static __always_inline s64
747	atomic64_sub_return(s64 i, atomic64_t *v)
748	{
749	kcsan_mb();
750	instrument_atomic_read_write(v, sizeof(*v));
751	return arch_atomic64_sub_return(i, v);
752	}
753
754	static __always_inline s64
755	atomic64_sub_return_acquire(s64 i, atomic64_t *v)
756	{
757	instrument_atomic_read_write(v, sizeof(*v));
758	return arch_atomic64_sub_return_acquire(i, v);
759	}
760
761	static __always_inline s64
762	atomic64_sub_return_release(s64 i, atomic64_t *v)
763	{
764	kcsan_release();
765	instrument_atomic_read_write(v, sizeof(*v));
766	return arch_atomic64_sub_return_release(i, v);
767	}
768
769	static __always_inline s64
770	atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
771	{
772	instrument_atomic_read_write(v, sizeof(*v));
773	return arch_atomic64_sub_return_relaxed(i, v);
774	}
775
776	static __always_inline s64
777	atomic64_fetch_sub(s64 i, atomic64_t *v)
778	{
779	kcsan_mb();
780	instrument_atomic_read_write(v, sizeof(*v));
781	return arch_atomic64_fetch_sub(i, v);
782	}
783
784	static __always_inline s64
785	atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
786	{
787	instrument_atomic_read_write(v, sizeof(*v));
788	return arch_atomic64_fetch_sub_acquire(i, v);
789	}
790
791	static __always_inline s64
792	atomic64_fetch_sub_release(s64 i, atomic64_t *v)
793	{
794	kcsan_release();
795	instrument_atomic_read_write(v, sizeof(*v));
796	return arch_atomic64_fetch_sub_release(i, v);
797	}
798
799	static __always_inline s64
800	atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
801	{
802	instrument_atomic_read_write(v, sizeof(*v));
803	return arch_atomic64_fetch_sub_relaxed(i, v);
804	}
805
806	static __always_inline void
807	atomic64_inc(atomic64_t *v)
808	{
809	instrument_atomic_read_write(v, sizeof(*v));
810	arch_atomic64_inc(v);
811	}
812
813	static __always_inline s64
814	atomic64_inc_return(atomic64_t *v)
815	{
816	kcsan_mb();
817	instrument_atomic_read_write(v, sizeof(*v));
818	return arch_atomic64_inc_return(v);
819	}
820
821	static __always_inline s64
822	atomic64_inc_return_acquire(atomic64_t *v)
823	{
824	instrument_atomic_read_write(v, sizeof(*v));
825	return arch_atomic64_inc_return_acquire(v);
826	}
827
828	static __always_inline s64
829	atomic64_inc_return_release(atomic64_t *v)
830	{
831	kcsan_release();
832	instrument_atomic_read_write(v, sizeof(*v));
833	return arch_atomic64_inc_return_release(v);
834	}
835
836	static __always_inline s64
837	atomic64_inc_return_relaxed(atomic64_t *v)
838	{
839	instrument_atomic_read_write(v, sizeof(*v));
840	return arch_atomic64_inc_return_relaxed(v);
841	}
842
843	static __always_inline s64
844	atomic64_fetch_inc(atomic64_t *v)
845	{
846	kcsan_mb();
847	instrument_atomic_read_write(v, sizeof(*v));
848	return arch_atomic64_fetch_inc(v);
849	}
850
851	static __always_inline s64
852	atomic64_fetch_inc_acquire(atomic64_t *v)
853	{
854	instrument_atomic_read_write(v, sizeof(*v));
855	return arch_atomic64_fetch_inc_acquire(v);
856	}
857
858	static __always_inline s64
859	atomic64_fetch_inc_release(atomic64_t *v)
860	{
861	kcsan_release();
862	instrument_atomic_read_write(v, sizeof(*v));
863	return arch_atomic64_fetch_inc_release(v);
864	}
865
866	static __always_inline s64
867	atomic64_fetch_inc_relaxed(atomic64_t *v)
868	{
869	instrument_atomic_read_write(v, sizeof(*v));
870	return arch_atomic64_fetch_inc_relaxed(v);
871	}
872
873	static __always_inline void
874	atomic64_dec(atomic64_t *v)
875	{
876	instrument_atomic_read_write(v, sizeof(*v));
877	arch_atomic64_dec(v);
878	}
879
880	static __always_inline s64
881	atomic64_dec_return(atomic64_t *v)
882	{
883	kcsan_mb();
884	instrument_atomic_read_write(v, sizeof(*v));
885	return arch_atomic64_dec_return(v);
886	}
887
888	static __always_inline s64
889	atomic64_dec_return_acquire(atomic64_t *v)
890	{
891	instrument_atomic_read_write(v, sizeof(*v));
892	return arch_atomic64_dec_return_acquire(v);
893	}
894
895	static __always_inline s64
896	atomic64_dec_return_release(atomic64_t *v)
897	{
898	kcsan_release();
899	instrument_atomic_read_write(v, sizeof(*v));
900	return arch_atomic64_dec_return_release(v);
901	}
902
903	static __always_inline s64
904	atomic64_dec_return_relaxed(atomic64_t *v)
905	{
906	instrument_atomic_read_write(v, sizeof(*v));
907	return arch_atomic64_dec_return_relaxed(v);
908	}
909
910	static __always_inline s64
911	atomic64_fetch_dec(atomic64_t *v)
912	{
913	kcsan_mb();
914	instrument_atomic_read_write(v, sizeof(*v));
915	return arch_atomic64_fetch_dec(v);
916	}
917
918	static __always_inline s64
919	atomic64_fetch_dec_acquire(atomic64_t *v)
920	{
921	instrument_atomic_read_write(v, sizeof(*v));
922	return arch_atomic64_fetch_dec_acquire(v);
923	}
924
925	static __always_inline s64
926	atomic64_fetch_dec_release(atomic64_t *v)
927	{
928	kcsan_release();
929	instrument_atomic_read_write(v, sizeof(*v));
930	return arch_atomic64_fetch_dec_release(v);
931	}
932
933	static __always_inline s64
934	atomic64_fetch_dec_relaxed(atomic64_t *v)
935	{
936	instrument_atomic_read_write(v, sizeof(*v));
937	return arch_atomic64_fetch_dec_relaxed(v);
938	}
939
940	static __always_inline void
941	atomic64_and(s64 i, atomic64_t *v)
942	{
943	instrument_atomic_read_write(v, sizeof(*v));
944	arch_atomic64_and(i, v);
945	}
946
947	static __always_inline s64
948	atomic64_fetch_and(s64 i, atomic64_t *v)
949	{
950	kcsan_mb();
951	instrument_atomic_read_write(v, sizeof(*v));
952	return arch_atomic64_fetch_and(i, v);
953	}
954
955	static __always_inline s64
956	atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
957	{
958	instrument_atomic_read_write(v, sizeof(*v));
959	return arch_atomic64_fetch_and_acquire(i, v);
960	}
961
962	static __always_inline s64
963	atomic64_fetch_and_release(s64 i, atomic64_t *v)
964	{
965	kcsan_release();
966	instrument_atomic_read_write(v, sizeof(*v));
967	return arch_atomic64_fetch_and_release(i, v);
968	}
969
970	static __always_inline s64
971	atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
972	{
973	instrument_atomic_read_write(v, sizeof(*v));
974	return arch_atomic64_fetch_and_relaxed(i, v);
975	}
976
977	static __always_inline void
978	atomic64_andnot(s64 i, atomic64_t *v)
979	{
980	instrument_atomic_read_write(v, sizeof(*v));
981	arch_atomic64_andnot(i, v);
982	}
983
984	static __always_inline s64
985	atomic64_fetch_andnot(s64 i, atomic64_t *v)
986	{
987	kcsan_mb();
988	instrument_atomic_read_write(v, sizeof(*v));
989	return arch_atomic64_fetch_andnot(i, v);
990	}
991
992	static __always_inline s64
993	atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
994	{
995	instrument_atomic_read_write(v, sizeof(*v));
996	return arch_atomic64_fetch_andnot_acquire(i, v);
997	}
998
999	static __always_inline s64
1000	atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1001	{
1002	kcsan_release();
1003	instrument_atomic_read_write(v, sizeof(*v));
1004	return arch_atomic64_fetch_andnot_release(i, v);
1005	}
1006
1007	static __always_inline s64
1008	atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1009	{
1010	instrument_atomic_read_write(v, sizeof(*v));
1011	return arch_atomic64_fetch_andnot_relaxed(i, v);
1012	}
1013
1014	static __always_inline void
1015	atomic64_or(s64 i, atomic64_t *v)
1016	{
1017	instrument_atomic_read_write(v, sizeof(*v));
1018	arch_atomic64_or(i, v);
1019	}
1020
1021	static __always_inline s64
1022	atomic64_fetch_or(s64 i, atomic64_t *v)
1023	{
1024	kcsan_mb();
1025	instrument_atomic_read_write(v, sizeof(*v));
1026	return arch_atomic64_fetch_or(i, v);
1027	}
1028
1029	static __always_inline s64
1030	atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1031	{
1032	instrument_atomic_read_write(v, sizeof(*v));
1033	return arch_atomic64_fetch_or_acquire(i, v);
1034	}
1035
1036	static __always_inline s64
1037	atomic64_fetch_or_release(s64 i, atomic64_t *v)
1038	{
1039	kcsan_release();
1040	instrument_atomic_read_write(v, sizeof(*v));
1041	return arch_atomic64_fetch_or_release(i, v);
1042	}
1043
1044	static __always_inline s64
1045	atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1046	{
1047	instrument_atomic_read_write(v, sizeof(*v));
1048	return arch_atomic64_fetch_or_relaxed(i, v);
1049	}
1050
1051	static __always_inline void
1052	atomic64_xor(s64 i, atomic64_t *v)
1053	{
1054	instrument_atomic_read_write(v, sizeof(*v));
1055	arch_atomic64_xor(i, v);
1056	}
1057
1058	static __always_inline s64
1059	atomic64_fetch_xor(s64 i, atomic64_t *v)
1060	{
1061	kcsan_mb();
1062	instrument_atomic_read_write(v, sizeof(*v));
1063	return arch_atomic64_fetch_xor(i, v);
1064	}
1065
1066	static __always_inline s64
1067	atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1068	{
1069	instrument_atomic_read_write(v, sizeof(*v));
1070	return arch_atomic64_fetch_xor_acquire(i, v);
1071	}
1072
1073	static __always_inline s64
1074	atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1075	{
1076	kcsan_release();
1077	instrument_atomic_read_write(v, sizeof(*v));
1078	return arch_atomic64_fetch_xor_release(i, v);
1079	}
1080
1081	static __always_inline s64
1082	atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1083	{
1084	instrument_atomic_read_write(v, sizeof(*v));
1085	return arch_atomic64_fetch_xor_relaxed(i, v);
1086	}
1087
1088	static __always_inline s64
1089	atomic64_xchg(atomic64_t *v, s64 i)
1090	{
1091	kcsan_mb();
1092	instrument_atomic_read_write(v, sizeof(*v));
1093	return arch_atomic64_xchg(v, i);
1094	}
1095
1096	static __always_inline s64
1097	atomic64_xchg_acquire(atomic64_t *v, s64 i)
1098	{
1099	instrument_atomic_read_write(v, sizeof(*v));
1100	return arch_atomic64_xchg_acquire(v, i);
1101	}
1102
1103	static __always_inline s64
1104	atomic64_xchg_release(atomic64_t *v, s64 i)
1105	{
1106	kcsan_release();
1107	instrument_atomic_read_write(v, sizeof(*v));
1108	return arch_atomic64_xchg_release(v, i);
1109	}
1110
1111	static __always_inline s64
1112	atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1113	{
1114	instrument_atomic_read_write(v, sizeof(*v));
1115	return arch_atomic64_xchg_relaxed(v, i);
1116	}
1117
1118	static __always_inline s64
1119	atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1120	{
1121	kcsan_mb();
1122	instrument_atomic_read_write(v, sizeof(*v));
1123	return arch_atomic64_cmpxchg(v, old, new);
1124	}
1125
1126	static __always_inline s64
1127	atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1128	{
1129	instrument_atomic_read_write(v, sizeof(*v));
1130	return arch_atomic64_cmpxchg_acquire(v, old, new);
1131	}
1132
1133	static __always_inline s64
1134	atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1135	{
1136	kcsan_release();
1137	instrument_atomic_read_write(v, sizeof(*v));
1138	return arch_atomic64_cmpxchg_release(v, old, new);
1139	}
1140
1141	static __always_inline s64
1142	atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1143	{
1144	instrument_atomic_read_write(v, sizeof(*v));
1145	return arch_atomic64_cmpxchg_relaxed(v, old, new);
1146	}
1147
1148	static __always_inline bool
1149	atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1150	{
1151	kcsan_mb();
1152	instrument_atomic_read_write(v, sizeof(*v));
1153	instrument_atomic_read_write(old, sizeof(*old));
1154	return arch_atomic64_try_cmpxchg(v, old, new);
1155	}
1156
1157	static __always_inline bool
1158	atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1159	{
1160	instrument_atomic_read_write(v, sizeof(*v));
1161	instrument_atomic_read_write(old, sizeof(*old));
1162	return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1163	}
1164
1165	static __always_inline bool
1166	atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1167	{
1168	kcsan_release();
1169	instrument_atomic_read_write(v, sizeof(*v));
1170	instrument_atomic_read_write(old, sizeof(*old));
1171	return arch_atomic64_try_cmpxchg_release(v, old, new);
1172	}
1173
1174	static __always_inline bool
1175	atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1176	{
1177	instrument_atomic_read_write(v, sizeof(*v));
1178	instrument_atomic_read_write(old, sizeof(*old));
1179	return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1180	}
1181
1182	static __always_inline bool
1183	atomic64_sub_and_test(s64 i, atomic64_t *v)
1184	{
1185	kcsan_mb();
1186	instrument_atomic_read_write(v, sizeof(*v));
1187	return arch_atomic64_sub_and_test(i, v);
1188	}
1189
1190	static __always_inline bool
1191	atomic64_dec_and_test(atomic64_t *v)
1192	{
1193	kcsan_mb();
1194	instrument_atomic_read_write(v, sizeof(*v));
1195	return arch_atomic64_dec_and_test(v);
1196	}
1197
1198	static __always_inline bool
1199	atomic64_inc_and_test(atomic64_t *v)
1200	{
1201	kcsan_mb();
1202	instrument_atomic_read_write(v, sizeof(*v));
1203	return arch_atomic64_inc_and_test(v);
1204	}
1205
1206	static __always_inline bool
1207	atomic64_add_negative(s64 i, atomic64_t *v)
1208	{
1209	kcsan_mb();
1210	instrument_atomic_read_write(v, sizeof(*v));
1211	return arch_atomic64_add_negative(i, v);
1212	}
1213
1214	static __always_inline s64
1215	atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1216	{
1217	kcsan_mb();
1218	instrument_atomic_read_write(v, sizeof(*v));
1219	return arch_atomic64_fetch_add_unless(v, a, u);
1220	}
1221
1222	static __always_inline bool
1223	atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1224	{
1225	kcsan_mb();
1226	instrument_atomic_read_write(v, sizeof(*v));
1227	return arch_atomic64_add_unless(v, a, u);
1228	}
1229
1230	static __always_inline bool
1231	atomic64_inc_not_zero(atomic64_t *v)
1232	{
1233	kcsan_mb();
1234	instrument_atomic_read_write(v, sizeof(*v));
1235	return arch_atomic64_inc_not_zero(v);
1236	}
1237
1238	static __always_inline bool
1239	atomic64_inc_unless_negative(atomic64_t *v)
1240	{
1241	kcsan_mb();
1242	instrument_atomic_read_write(v, sizeof(*v));
1243	return arch_atomic64_inc_unless_negative(v);
1244	}
1245
1246	static __always_inline bool
1247	atomic64_dec_unless_positive(atomic64_t *v)
1248	{
1249	kcsan_mb();
1250	instrument_atomic_read_write(v, sizeof(*v));
1251	return arch_atomic64_dec_unless_positive(v);
1252	}
1253
1254	static __always_inline s64
1255	atomic64_dec_if_positive(atomic64_t *v)
1256	{
1257	kcsan_mb();
1258	instrument_atomic_read_write(v, sizeof(*v));
1259	return arch_atomic64_dec_if_positive(v);
1260	}
1261
1262	static __always_inline long
1263	atomic_long_read(const atomic_long_t *v)
1264	{
1265	instrument_atomic_read(v, sizeof(*v));
1266	return arch_atomic_long_read(v);
1267	}
1268
1269	static __always_inline long
1270	atomic_long_read_acquire(const atomic_long_t *v)
1271	{
1272	instrument_atomic_read(v, sizeof(*v));
1273	return arch_atomic_long_read_acquire(v);
1274	}
1275
1276	static __always_inline void
1277	atomic_long_set(atomic_long_t *v, long i)
1278	{
1279	instrument_atomic_write(v, sizeof(*v));
1280	arch_atomic_long_set(v, i);
1281	}
1282
1283	static __always_inline void
1284	atomic_long_set_release(atomic_long_t *v, long i)
1285	{
1286	kcsan_release();
1287	instrument_atomic_write(v, sizeof(*v));
1288	arch_atomic_long_set_release(v, i);
1289	}
1290
1291	static __always_inline void
1292	atomic_long_add(long i, atomic_long_t *v)
1293	{
1294	instrument_atomic_read_write(v, sizeof(*v));
1295	arch_atomic_long_add(i, v);
1296	}
1297
1298	static __always_inline long
1299	atomic_long_add_return(long i, atomic_long_t *v)
1300	{
1301	kcsan_mb();
1302	instrument_atomic_read_write(v, sizeof(*v));
1303	return arch_atomic_long_add_return(i, v);
1304	}
1305
1306	static __always_inline long
1307	atomic_long_add_return_acquire(long i, atomic_long_t *v)
1308	{
1309	instrument_atomic_read_write(v, sizeof(*v));
1310	return arch_atomic_long_add_return_acquire(i, v);
1311	}
1312
1313	static __always_inline long
1314	atomic_long_add_return_release(long i, atomic_long_t *v)
1315	{
1316	kcsan_release();
1317	instrument_atomic_read_write(v, sizeof(*v));
1318	return arch_atomic_long_add_return_release(i, v);
1319	}
1320
1321	static __always_inline long
1322	atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1323	{
1324	instrument_atomic_read_write(v, sizeof(*v));
1325	return arch_atomic_long_add_return_relaxed(i, v);
1326	}
1327
1328	static __always_inline long
1329	atomic_long_fetch_add(long i, atomic_long_t *v)
1330	{
1331	kcsan_mb();
1332	instrument_atomic_read_write(v, sizeof(*v));
1333	return arch_atomic_long_fetch_add(i, v);
1334	}
1335
1336	static __always_inline long
1337	atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1338	{
1339	instrument_atomic_read_write(v, sizeof(*v));
1340	return arch_atomic_long_fetch_add_acquire(i, v);
1341	}
1342
1343	static __always_inline long
1344	atomic_long_fetch_add_release(long i, atomic_long_t *v)
1345	{
1346	kcsan_release();
1347	instrument_atomic_read_write(v, sizeof(*v));
1348	return arch_atomic_long_fetch_add_release(i, v);
1349	}
1350
1351	static __always_inline long
1352	atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1353	{
1354	instrument_atomic_read_write(v, sizeof(*v));
1355	return arch_atomic_long_fetch_add_relaxed(i, v);
1356	}
1357
1358	static __always_inline void
1359	atomic_long_sub(long i, atomic_long_t *v)
1360	{
1361	instrument_atomic_read_write(v, sizeof(*v));
1362	arch_atomic_long_sub(i, v);
1363	}
1364
1365	static __always_inline long
1366	atomic_long_sub_return(long i, atomic_long_t *v)
1367	{
1368	kcsan_mb();
1369	instrument_atomic_read_write(v, sizeof(*v));
1370	return arch_atomic_long_sub_return(i, v);
1371	}
1372
1373	static __always_inline long
1374	atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1375	{
1376	instrument_atomic_read_write(v, sizeof(*v));
1377	return arch_atomic_long_sub_return_acquire(i, v);
1378	}
1379
1380	static __always_inline long
1381	atomic_long_sub_return_release(long i, atomic_long_t *v)
1382	{
1383	kcsan_release();
1384	instrument_atomic_read_write(v, sizeof(*v));
1385	return arch_atomic_long_sub_return_release(i, v);
1386	}
1387
1388	static __always_inline long
1389	atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1390	{
1391	instrument_atomic_read_write(v, sizeof(*v));
1392	return arch_atomic_long_sub_return_relaxed(i, v);
1393	}
1394
1395	static __always_inline long
1396	atomic_long_fetch_sub(long i, atomic_long_t *v)
1397	{
1398	kcsan_mb();
1399	instrument_atomic_read_write(v, sizeof(*v));
1400	return arch_atomic_long_fetch_sub(i, v);
1401	}
1402
1403	static __always_inline long
1404	atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1405	{
1406	instrument_atomic_read_write(v, sizeof(*v));
1407	return arch_atomic_long_fetch_sub_acquire(i, v);
1408	}
1409
1410	static __always_inline long
1411	atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1412	{
1413	kcsan_release();
1414	instrument_atomic_read_write(v, sizeof(*v));
1415	return arch_atomic_long_fetch_sub_release(i, v);
1416	}
1417
1418	static __always_inline long
1419	atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1420	{
1421	instrument_atomic_read_write(v, sizeof(*v));
1422	return arch_atomic_long_fetch_sub_relaxed(i, v);
1423	}
1424
1425	static __always_inline void
1426	atomic_long_inc(atomic_long_t *v)
1427	{
1428	instrument_atomic_read_write(v, sizeof(*v));
1429	arch_atomic_long_inc(v);
1430	}
1431
1432	static __always_inline long
1433	atomic_long_inc_return(atomic_long_t *v)
1434	{
1435	kcsan_mb();
1436	instrument_atomic_read_write(v, sizeof(*v));
1437	return arch_atomic_long_inc_return(v);
1438	}
1439
1440	static __always_inline long
1441	atomic_long_inc_return_acquire(atomic_long_t *v)
1442	{
1443	instrument_atomic_read_write(v, sizeof(*v));
1444	return arch_atomic_long_inc_return_acquire(v);
1445	}
1446
1447	static __always_inline long
1448	atomic_long_inc_return_release(atomic_long_t *v)
1449	{
1450	kcsan_release();
1451	instrument_atomic_read_write(v, sizeof(*v));
1452	return arch_atomic_long_inc_return_release(v);
1453	}
1454
1455	static __always_inline long
1456	atomic_long_inc_return_relaxed(atomic_long_t *v)
1457	{
1458	instrument_atomic_read_write(v, sizeof(*v));
1459	return arch_atomic_long_inc_return_relaxed(v);
1460	}
1461
1462	static __always_inline long
1463	atomic_long_fetch_inc(atomic_long_t *v)
1464	{
1465	kcsan_mb();
1466	instrument_atomic_read_write(v, sizeof(*v));
1467	return arch_atomic_long_fetch_inc(v);
1468	}
1469
1470	static __always_inline long
1471	atomic_long_fetch_inc_acquire(atomic_long_t *v)
1472	{
1473	instrument_atomic_read_write(v, sizeof(*v));
1474	return arch_atomic_long_fetch_inc_acquire(v);
1475	}
1476
1477	static __always_inline long
1478	atomic_long_fetch_inc_release(atomic_long_t *v)
1479	{
1480	kcsan_release();
1481	instrument_atomic_read_write(v, sizeof(*v));
1482	return arch_atomic_long_fetch_inc_release(v);
1483	}
1484
1485	static __always_inline long
1486	atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1487	{
1488	instrument_atomic_read_write(v, sizeof(*v));
1489	return arch_atomic_long_fetch_inc_relaxed(v);
1490	}
1491
1492	static __always_inline void
1493	atomic_long_dec(atomic_long_t *v)
1494	{
1495	instrument_atomic_read_write(v, sizeof(*v));
1496	arch_atomic_long_dec(v);
1497	}
1498
1499	static __always_inline long
1500	atomic_long_dec_return(atomic_long_t *v)
1501	{
1502	kcsan_mb();
1503	instrument_atomic_read_write(v, sizeof(*v));
1504	return arch_atomic_long_dec_return(v);
1505	}
1506
1507	static __always_inline long
1508	atomic_long_dec_return_acquire(atomic_long_t *v)
1509	{
1510	instrument_atomic_read_write(v, sizeof(*v));
1511	return arch_atomic_long_dec_return_acquire(v);
1512	}
1513
1514	static __always_inline long
1515	atomic_long_dec_return_release(atomic_long_t *v)
1516	{
1517	kcsan_release();
1518	instrument_atomic_read_write(v, sizeof(*v));
1519	return arch_atomic_long_dec_return_release(v);
1520	}
1521
1522	static __always_inline long
1523	atomic_long_dec_return_relaxed(atomic_long_t *v)
1524	{
1525	instrument_atomic_read_write(v, sizeof(*v));
1526	return arch_atomic_long_dec_return_relaxed(v);
1527	}
1528
1529	static __always_inline long
1530	atomic_long_fetch_dec(atomic_long_t *v)
1531	{
1532	kcsan_mb();
1533	instrument_atomic_read_write(v, sizeof(*v));
1534	return arch_atomic_long_fetch_dec(v);
1535	}
1536
1537	static __always_inline long
1538	atomic_long_fetch_dec_acquire(atomic_long_t *v)
1539	{
1540	instrument_atomic_read_write(v, sizeof(*v));
1541	return arch_atomic_long_fetch_dec_acquire(v);
1542	}
1543
1544	static __always_inline long
1545	atomic_long_fetch_dec_release(atomic_long_t *v)
1546	{
1547	kcsan_release();
1548	instrument_atomic_read_write(v, sizeof(*v));
1549	return arch_atomic_long_fetch_dec_release(v);
1550	}
1551
1552	static __always_inline long
1553	atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1554	{
1555	instrument_atomic_read_write(v, sizeof(*v));
1556	return arch_atomic_long_fetch_dec_relaxed(v);
1557	}
1558
1559	static __always_inline void
1560	atomic_long_and(long i, atomic_long_t *v)
1561	{
1562	instrument_atomic_read_write(v, sizeof(*v));
1563	arch_atomic_long_and(i, v);
1564	}
1565
1566	static __always_inline long
1567	atomic_long_fetch_and(long i, atomic_long_t *v)
1568	{
1569	kcsan_mb();
1570	instrument_atomic_read_write(v, sizeof(*v));
1571	return arch_atomic_long_fetch_and(i, v);
1572	}
1573
1574	static __always_inline long
1575	atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1576	{
1577	instrument_atomic_read_write(v, sizeof(*v));
1578	return arch_atomic_long_fetch_and_acquire(i, v);
1579	}
1580
1581	static __always_inline long
1582	atomic_long_fetch_and_release(long i, atomic_long_t *v)
1583	{
1584	kcsan_release();
1585	instrument_atomic_read_write(v, sizeof(*v));
1586	return arch_atomic_long_fetch_and_release(i, v);
1587	}
1588
1589	static __always_inline long
1590	atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1591	{
1592	instrument_atomic_read_write(v, sizeof(*v));
1593	return arch_atomic_long_fetch_and_relaxed(i, v);
1594	}
1595
1596	static __always_inline void
1597	atomic_long_andnot(long i, atomic_long_t *v)
1598	{
1599	instrument_atomic_read_write(v, sizeof(*v));
1600	arch_atomic_long_andnot(i, v);
1601	}
1602
1603	static __always_inline long
1604	atomic_long_fetch_andnot(long i, atomic_long_t *v)
1605	{
1606	kcsan_mb();
1607	instrument_atomic_read_write(v, sizeof(*v));
1608	return arch_atomic_long_fetch_andnot(i, v);
1609	}
1610
1611	static __always_inline long
1612	atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1613	{
1614	instrument_atomic_read_write(v, sizeof(*v));
1615	return arch_atomic_long_fetch_andnot_acquire(i, v);
1616	}
1617
1618	static __always_inline long
1619	atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1620	{
1621	kcsan_release();
1622	instrument_atomic_read_write(v, sizeof(*v));
1623	return arch_atomic_long_fetch_andnot_release(i, v);
1624	}
1625
1626	static __always_inline long
1627	atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1628	{
1629	instrument_atomic_read_write(v, sizeof(*v));
1630	return arch_atomic_long_fetch_andnot_relaxed(i, v);
1631	}
1632
1633	static __always_inline void
1634	atomic_long_or(long i, atomic_long_t *v)
1635	{
1636	instrument_atomic_read_write(v, sizeof(*v));
1637	arch_atomic_long_or(i, v);
1638	}
1639
1640	static __always_inline long
1641	atomic_long_fetch_or(long i, atomic_long_t *v)
1642	{
1643	kcsan_mb();
1644	instrument_atomic_read_write(v, sizeof(*v));
1645	return arch_atomic_long_fetch_or(i, v);
1646	}
1647
1648	static __always_inline long
1649	atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1650	{
1651	instrument_atomic_read_write(v, sizeof(*v));
1652	return arch_atomic_long_fetch_or_acquire(i, v);
1653	}
1654
1655	static __always_inline long
1656	atomic_long_fetch_or_release(long i, atomic_long_t *v)
1657	{
1658	kcsan_release();
1659	instrument_atomic_read_write(v, sizeof(*v));
1660	return arch_atomic_long_fetch_or_release(i, v);
1661	}
1662
1663	static __always_inline long
1664	atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1665	{
1666	instrument_atomic_read_write(v, sizeof(*v));
1667	return arch_atomic_long_fetch_or_relaxed(i, v);
1668	}
1669
1670	static __always_inline void
1671	atomic_long_xor(long i, atomic_long_t *v)
1672	{
1673	instrument_atomic_read_write(v, sizeof(*v));
1674	arch_atomic_long_xor(i, v);
1675	}
1676
1677	static __always_inline long
1678	atomic_long_fetch_xor(long i, atomic_long_t *v)
1679	{
1680	kcsan_mb();
1681	instrument_atomic_read_write(v, sizeof(*v));
1682	return arch_atomic_long_fetch_xor(i, v);
1683	}
1684
1685	static __always_inline long
1686	atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1687	{
1688	instrument_atomic_read_write(v, sizeof(*v));
1689	return arch_atomic_long_fetch_xor_acquire(i, v);
1690	}
1691
1692	static __always_inline long
1693	atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1694	{
1695	kcsan_release();
1696	instrument_atomic_read_write(v, sizeof(*v));
1697	return arch_atomic_long_fetch_xor_release(i, v);
1698	}
1699
1700	static __always_inline long
1701	atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1702	{
1703	instrument_atomic_read_write(v, sizeof(*v));
1704	return arch_atomic_long_fetch_xor_relaxed(i, v);
1705	}
1706
1707	static __always_inline long
1708	atomic_long_xchg(atomic_long_t *v, long i)
1709	{
1710	kcsan_mb();
1711	instrument_atomic_read_write(v, sizeof(*v));
1712	return arch_atomic_long_xchg(v, i);
1713	}
1714
1715	static __always_inline long
1716	atomic_long_xchg_acquire(atomic_long_t *v, long i)
1717	{
1718	instrument_atomic_read_write(v, sizeof(*v));
1719	return arch_atomic_long_xchg_acquire(v, i);
1720	}
1721
1722	static __always_inline long
1723	atomic_long_xchg_release(atomic_long_t *v, long i)
1724	{
1725	kcsan_release();
1726	instrument_atomic_read_write(v, sizeof(*v));
1727	return arch_atomic_long_xchg_release(v, i);
1728	}
1729
1730	static __always_inline long
1731	atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1732	{
1733	instrument_atomic_read_write(v, sizeof(*v));
1734	return arch_atomic_long_xchg_relaxed(v, i);
1735	}
1736
1737	static __always_inline long
1738	atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1739	{
1740	kcsan_mb();
1741	instrument_atomic_read_write(v, sizeof(*v));
1742	return arch_atomic_long_cmpxchg(v, old, new);
1743	}
1744
1745	static __always_inline long
1746	atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1747	{
1748	instrument_atomic_read_write(v, sizeof(*v));
1749	return arch_atomic_long_cmpxchg_acquire(v, old, new);
1750	}
1751
1752	static __always_inline long
1753	atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1754	{
1755	kcsan_release();
1756	instrument_atomic_read_write(v, sizeof(*v));
1757	return arch_atomic_long_cmpxchg_release(v, old, new);
1758	}
1759
1760	static __always_inline long
1761	atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1762	{
1763	instrument_atomic_read_write(v, sizeof(*v));
1764	return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1765	}
1766
1767	static __always_inline bool
1768	atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1769	{
1770	kcsan_mb();
1771	instrument_atomic_read_write(v, sizeof(*v));
1772	instrument_atomic_read_write(old, sizeof(*old));
1773	return arch_atomic_long_try_cmpxchg(v, old, new);
1774	}
1775
1776	static __always_inline bool
1777	atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1778	{
1779	instrument_atomic_read_write(v, sizeof(*v));
1780	instrument_atomic_read_write(old, sizeof(*old));
1781	return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1782	}
1783
1784	static __always_inline bool
1785	atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1786	{
1787	kcsan_release();
1788	instrument_atomic_read_write(v, sizeof(*v));
1789	instrument_atomic_read_write(old, sizeof(*old));
1790	return arch_atomic_long_try_cmpxchg_release(v, old, new);
1791	}
1792
1793	static __always_inline bool
1794	atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1795	{
1796	instrument_atomic_read_write(v, sizeof(*v));
1797	instrument_atomic_read_write(old, sizeof(*old));
1798	return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1799	}
1800
1801	static __always_inline bool
1802	atomic_long_sub_and_test(long i, atomic_long_t *v)
1803	{
1804	kcsan_mb();
1805	instrument_atomic_read_write(v, sizeof(*v));
1806	return arch_atomic_long_sub_and_test(i, v);
1807	}
1808
1809	static __always_inline bool
1810	atomic_long_dec_and_test(atomic_long_t *v)
1811	{
1812	kcsan_mb();
1813	instrument_atomic_read_write(v, sizeof(*v));
1814	return arch_atomic_long_dec_and_test(v);
1815	}
1816
1817	static __always_inline bool
1818	atomic_long_inc_and_test(atomic_long_t *v)
1819	{
1820	kcsan_mb();
1821	instrument_atomic_read_write(v, sizeof(*v));
1822	return arch_atomic_long_inc_and_test(v);
1823	}
1824
1825	static __always_inline bool
1826	atomic_long_add_negative(long i, atomic_long_t *v)
1827	{
1828	kcsan_mb();
1829	instrument_atomic_read_write(v, sizeof(*v));
1830	return arch_atomic_long_add_negative(i, v);
1831	}
1832
1833	static __always_inline long
1834	atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1835	{
1836	kcsan_mb();
1837	instrument_atomic_read_write(v, sizeof(*v));
1838	return arch_atomic_long_fetch_add_unless(v, a, u);
1839	}
1840
1841	static __always_inline bool
1842	atomic_long_add_unless(atomic_long_t *v, long a, long u)
1843	{
1844	kcsan_mb();
1845	instrument_atomic_read_write(v, sizeof(*v));
1846	return arch_atomic_long_add_unless(v, a, u);
1847	}
1848
1849	static __always_inline bool
1850	atomic_long_inc_not_zero(atomic_long_t *v)
1851	{
1852	kcsan_mb();
1853	instrument_atomic_read_write(v, sizeof(*v));
1854	return arch_atomic_long_inc_not_zero(v);
1855	}
1856
1857	static __always_inline bool
1858	atomic_long_inc_unless_negative(atomic_long_t *v)
1859	{
1860	kcsan_mb();
1861	instrument_atomic_read_write(v, sizeof(*v));
1862	return arch_atomic_long_inc_unless_negative(v);
1863	}
1864
1865	static __always_inline bool
1866	atomic_long_dec_unless_positive(atomic_long_t *v)
1867	{
1868	kcsan_mb();
1869	instrument_atomic_read_write(v, sizeof(*v));
1870	return arch_atomic_long_dec_unless_positive(v);
1871	}
1872
1873	static __always_inline long
1874	atomic_long_dec_if_positive(atomic_long_t *v)
1875	{
1876	kcsan_mb();
1877	instrument_atomic_read_write(v, sizeof(*v));
1878	return arch_atomic_long_dec_if_positive(v);
1879	}
1880
1881	#define xchg(ptr, ...) \
1882	({ \
1883	typeof(ptr) __ai_ptr = (ptr); \
1884	kcsan_mb(); \
1885	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1886	arch_xchg(__ai_ptr, __VA_ARGS__); \
1887	})
1888
1889	#define xchg_acquire(ptr, ...) \
1890	({ \
1891	typeof(ptr) __ai_ptr = (ptr); \
1892	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1893	arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1894	})
1895
1896	#define xchg_release(ptr, ...) \
1897	({ \
1898	typeof(ptr) __ai_ptr = (ptr); \
1899	kcsan_release(); \
1900	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1901	arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1902	})
1903
1904	#define xchg_relaxed(ptr, ...) \
1905	({ \
1906	typeof(ptr) __ai_ptr = (ptr); \
1907	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1908	arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1909	})
1910
1911	#define cmpxchg(ptr, ...) \
1912	({ \
1913	typeof(ptr) __ai_ptr = (ptr); \
1914	kcsan_mb(); \
1915	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1916	arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1917	})
1918
1919	#define cmpxchg_acquire(ptr, ...) \
1920	({ \
1921	typeof(ptr) __ai_ptr = (ptr); \
1922	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1923	arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1924	})
1925
1926	#define cmpxchg_release(ptr, ...) \
1927	({ \
1928	typeof(ptr) __ai_ptr = (ptr); \
1929	kcsan_release(); \
1930	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1931	arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1932	})
1933
1934	#define cmpxchg_relaxed(ptr, ...) \
1935	({ \
1936	typeof(ptr) __ai_ptr = (ptr); \
1937	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1938	arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1939	})
1940
1941	#define cmpxchg64(ptr, ...) \
1942	({ \
1943	typeof(ptr) __ai_ptr = (ptr); \
1944	kcsan_mb(); \
1945	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1946	arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1947	})
1948
1949	#define cmpxchg64_acquire(ptr, ...) \
1950	({ \
1951	typeof(ptr) __ai_ptr = (ptr); \
1952	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1953	arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1954	})
1955
1956	#define cmpxchg64_release(ptr, ...) \
1957	({ \
1958	typeof(ptr) __ai_ptr = (ptr); \
1959	kcsan_release(); \
1960	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1961	arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1962	})
1963
1964	#define cmpxchg64_relaxed(ptr, ...) \
1965	({ \
1966	typeof(ptr) __ai_ptr = (ptr); \
1967	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1968	arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1969	})
1970
1971	#define try_cmpxchg(ptr, oldp, ...) \
1972	({ \
1973	typeof(ptr) __ai_ptr = (ptr); \
1974	typeof(oldp) __ai_oldp = (oldp); \
1975	kcsan_mb(); \
1976	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1977	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1978	arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1979	})
1980
1981	#define try_cmpxchg_acquire(ptr, oldp, ...) \
1982	({ \
1983	typeof(ptr) __ai_ptr = (ptr); \
1984	typeof(oldp) __ai_oldp = (oldp); \
1985	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1986	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1987	arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1988	})
1989
1990	#define try_cmpxchg_release(ptr, oldp, ...) \
1991	({ \
1992	typeof(ptr) __ai_ptr = (ptr); \
1993	typeof(oldp) __ai_oldp = (oldp); \
1994	kcsan_release(); \
1995	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1996	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1997	arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1998	})
1999
2000	#define try_cmpxchg_relaxed(ptr, oldp, ...) \
2001	({ \
2002	typeof(ptr) __ai_ptr = (ptr); \
2003	typeof(oldp) __ai_oldp = (oldp); \
2004	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2005	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2006	arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2007	})
2008
2009	#define try_cmpxchg64(ptr, oldp, ...) \
2010	({ \
2011	typeof(ptr) __ai_ptr = (ptr); \
2012	typeof(oldp) __ai_oldp = (oldp); \
2013	kcsan_mb(); \
2014	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2015	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2016	arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2017	})
2018
2019	#define try_cmpxchg64_acquire(ptr, oldp, ...) \
2020	({ \
2021	typeof(ptr) __ai_ptr = (ptr); \
2022	typeof(oldp) __ai_oldp = (oldp); \
2023	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2024	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2025	arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2026	})
2027
2028	#define try_cmpxchg64_release(ptr, oldp, ...) \
2029	({ \
2030	typeof(ptr) __ai_ptr = (ptr); \
2031	typeof(oldp) __ai_oldp = (oldp); \
2032	kcsan_release(); \
2033	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2034	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2035	arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2036	})
2037
2038	#define try_cmpxchg64_relaxed(ptr, oldp, ...) \
2039	({ \
2040	typeof(ptr) __ai_ptr = (ptr); \
2041	typeof(oldp) __ai_oldp = (oldp); \
2042	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043	instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2044	arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2045	})
2046
2047	#define cmpxchg_local(ptr, ...) \
2048	({ \
2049	typeof(ptr) __ai_ptr = (ptr); \
2050	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2051	arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
2052	})
2053
2054	#define cmpxchg64_local(ptr, ...) \
2055	({ \
2056	typeof(ptr) __ai_ptr = (ptr); \
2057	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2058	arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
2059	})
2060
2061	#define sync_cmpxchg(ptr, ...) \
2062	({ \
2063	typeof(ptr) __ai_ptr = (ptr); \
2064	kcsan_mb(); \
2065	instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2066	arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
2067	})
2068
2069	#define cmpxchg_double(ptr, ...) \
2070	({ \
2071	typeof(ptr) __ai_ptr = (ptr); \
2072	kcsan_mb(); \
2073	instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2074	arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2075	})
2076
2077
2078	#define cmpxchg_double_local(ptr, ...) \
2079	({ \
2080	typeof(ptr) __ai_ptr = (ptr); \
2081	instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2082	arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2083	})
2084
2085	#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2086	// 764f741eb77a7ad565dc8d99ce2837d5542e8aee
2087