1 | // SPDX-License-Identifier: GPL-2.0 |
2 | |
3 | // Generated by scripts/atomic/gen-atomic-fallback.sh |
4 | // DO NOT MODIFY THIS FILE DIRECTLY |
5 | |
6 | #ifndef _LINUX_ATOMIC_FALLBACK_H |
7 | #define _LINUX_ATOMIC_FALLBACK_H |
8 | |
9 | #include <linux/compiler.h> |
10 | |
11 | #if defined(arch_xchg) |
12 | #define raw_xchg arch_xchg |
13 | #elif defined(arch_xchg_relaxed) |
14 | #define raw_xchg(...) \ |
15 | __atomic_op_fence(arch_xchg, __VA_ARGS__) |
16 | #else |
17 | extern void raw_xchg_not_implemented(void); |
18 | #define raw_xchg(...) raw_xchg_not_implemented() |
19 | #endif |
20 | |
21 | #if defined(arch_xchg_acquire) |
22 | #define raw_xchg_acquire arch_xchg_acquire |
23 | #elif defined(arch_xchg_relaxed) |
24 | #define raw_xchg_acquire(...) \ |
25 | __atomic_op_acquire(arch_xchg, __VA_ARGS__) |
26 | #elif defined(arch_xchg) |
27 | #define raw_xchg_acquire arch_xchg |
28 | #else |
29 | extern void raw_xchg_acquire_not_implemented(void); |
30 | #define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented() |
31 | #endif |
32 | |
33 | #if defined(arch_xchg_release) |
34 | #define raw_xchg_release arch_xchg_release |
35 | #elif defined(arch_xchg_relaxed) |
36 | #define raw_xchg_release(...) \ |
37 | __atomic_op_release(arch_xchg, __VA_ARGS__) |
38 | #elif defined(arch_xchg) |
39 | #define raw_xchg_release arch_xchg |
40 | #else |
41 | extern void raw_xchg_release_not_implemented(void); |
42 | #define raw_xchg_release(...) raw_xchg_release_not_implemented() |
43 | #endif |
44 | |
45 | #if defined(arch_xchg_relaxed) |
46 | #define raw_xchg_relaxed arch_xchg_relaxed |
47 | #elif defined(arch_xchg) |
48 | #define raw_xchg_relaxed arch_xchg |
49 | #else |
50 | extern void raw_xchg_relaxed_not_implemented(void); |
51 | #define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented() |
52 | #endif |
53 | |
54 | #if defined(arch_cmpxchg) |
55 | #define raw_cmpxchg arch_cmpxchg |
56 | #elif defined(arch_cmpxchg_relaxed) |
57 | #define raw_cmpxchg(...) \ |
58 | __atomic_op_fence(arch_cmpxchg, __VA_ARGS__) |
59 | #else |
60 | extern void raw_cmpxchg_not_implemented(void); |
61 | #define raw_cmpxchg(...) raw_cmpxchg_not_implemented() |
62 | #endif |
63 | |
64 | #if defined(arch_cmpxchg_acquire) |
65 | #define raw_cmpxchg_acquire arch_cmpxchg_acquire |
66 | #elif defined(arch_cmpxchg_relaxed) |
67 | #define raw_cmpxchg_acquire(...) \ |
68 | __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__) |
69 | #elif defined(arch_cmpxchg) |
70 | #define raw_cmpxchg_acquire arch_cmpxchg |
71 | #else |
72 | extern void raw_cmpxchg_acquire_not_implemented(void); |
73 | #define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented() |
74 | #endif |
75 | |
76 | #if defined(arch_cmpxchg_release) |
77 | #define raw_cmpxchg_release arch_cmpxchg_release |
78 | #elif defined(arch_cmpxchg_relaxed) |
79 | #define raw_cmpxchg_release(...) \ |
80 | __atomic_op_release(arch_cmpxchg, __VA_ARGS__) |
81 | #elif defined(arch_cmpxchg) |
82 | #define raw_cmpxchg_release arch_cmpxchg |
83 | #else |
84 | extern void raw_cmpxchg_release_not_implemented(void); |
85 | #define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented() |
86 | #endif |
87 | |
88 | #if defined(arch_cmpxchg_relaxed) |
89 | #define raw_cmpxchg_relaxed arch_cmpxchg_relaxed |
90 | #elif defined(arch_cmpxchg) |
91 | #define raw_cmpxchg_relaxed arch_cmpxchg |
92 | #else |
93 | extern void raw_cmpxchg_relaxed_not_implemented(void); |
94 | #define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented() |
95 | #endif |
96 | |
97 | #if defined(arch_cmpxchg64) |
98 | #define raw_cmpxchg64 arch_cmpxchg64 |
99 | #elif defined(arch_cmpxchg64_relaxed) |
100 | #define raw_cmpxchg64(...) \ |
101 | __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__) |
102 | #else |
103 | extern void raw_cmpxchg64_not_implemented(void); |
104 | #define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented() |
105 | #endif |
106 | |
107 | #if defined(arch_cmpxchg64_acquire) |
108 | #define raw_cmpxchg64_acquire arch_cmpxchg64_acquire |
109 | #elif defined(arch_cmpxchg64_relaxed) |
110 | #define raw_cmpxchg64_acquire(...) \ |
111 | __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__) |
112 | #elif defined(arch_cmpxchg64) |
113 | #define raw_cmpxchg64_acquire arch_cmpxchg64 |
114 | #else |
115 | extern void raw_cmpxchg64_acquire_not_implemented(void); |
116 | #define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented() |
117 | #endif |
118 | |
119 | #if defined(arch_cmpxchg64_release) |
120 | #define raw_cmpxchg64_release arch_cmpxchg64_release |
121 | #elif defined(arch_cmpxchg64_relaxed) |
122 | #define raw_cmpxchg64_release(...) \ |
123 | __atomic_op_release(arch_cmpxchg64, __VA_ARGS__) |
124 | #elif defined(arch_cmpxchg64) |
125 | #define raw_cmpxchg64_release arch_cmpxchg64 |
126 | #else |
127 | extern void raw_cmpxchg64_release_not_implemented(void); |
128 | #define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented() |
129 | #endif |
130 | |
131 | #if defined(arch_cmpxchg64_relaxed) |
132 | #define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed |
133 | #elif defined(arch_cmpxchg64) |
134 | #define raw_cmpxchg64_relaxed arch_cmpxchg64 |
135 | #else |
136 | extern void raw_cmpxchg64_relaxed_not_implemented(void); |
137 | #define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented() |
138 | #endif |
139 | |
140 | #if defined(arch_cmpxchg128) |
141 | #define raw_cmpxchg128 arch_cmpxchg128 |
142 | #elif defined(arch_cmpxchg128_relaxed) |
143 | #define raw_cmpxchg128(...) \ |
144 | __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__) |
145 | #else |
146 | extern void raw_cmpxchg128_not_implemented(void); |
147 | #define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented() |
148 | #endif |
149 | |
150 | #if defined(arch_cmpxchg128_acquire) |
151 | #define raw_cmpxchg128_acquire arch_cmpxchg128_acquire |
152 | #elif defined(arch_cmpxchg128_relaxed) |
153 | #define raw_cmpxchg128_acquire(...) \ |
154 | __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__) |
155 | #elif defined(arch_cmpxchg128) |
156 | #define raw_cmpxchg128_acquire arch_cmpxchg128 |
157 | #else |
158 | extern void raw_cmpxchg128_acquire_not_implemented(void); |
159 | #define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented() |
160 | #endif |
161 | |
162 | #if defined(arch_cmpxchg128_release) |
163 | #define raw_cmpxchg128_release arch_cmpxchg128_release |
164 | #elif defined(arch_cmpxchg128_relaxed) |
165 | #define raw_cmpxchg128_release(...) \ |
166 | __atomic_op_release(arch_cmpxchg128, __VA_ARGS__) |
167 | #elif defined(arch_cmpxchg128) |
168 | #define raw_cmpxchg128_release arch_cmpxchg128 |
169 | #else |
170 | extern void raw_cmpxchg128_release_not_implemented(void); |
171 | #define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented() |
172 | #endif |
173 | |
174 | #if defined(arch_cmpxchg128_relaxed) |
175 | #define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed |
176 | #elif defined(arch_cmpxchg128) |
177 | #define raw_cmpxchg128_relaxed arch_cmpxchg128 |
178 | #else |
179 | extern void raw_cmpxchg128_relaxed_not_implemented(void); |
180 | #define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented() |
181 | #endif |
182 | |
183 | #if defined(arch_try_cmpxchg) |
184 | #define raw_try_cmpxchg arch_try_cmpxchg |
185 | #elif defined(arch_try_cmpxchg_relaxed) |
186 | #define raw_try_cmpxchg(...) \ |
187 | __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__) |
188 | #else |
189 | #define raw_try_cmpxchg(_ptr, _oldp, _new) \ |
190 | ({ \ |
191 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
192 | ___r = raw_cmpxchg((_ptr), ___o, (_new)); \ |
193 | if (unlikely(___r != ___o)) \ |
194 | *___op = ___r; \ |
195 | likely(___r == ___o); \ |
196 | }) |
197 | #endif |
198 | |
199 | #if defined(arch_try_cmpxchg_acquire) |
200 | #define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire |
201 | #elif defined(arch_try_cmpxchg_relaxed) |
202 | #define raw_try_cmpxchg_acquire(...) \ |
203 | __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__) |
204 | #elif defined(arch_try_cmpxchg) |
205 | #define raw_try_cmpxchg_acquire arch_try_cmpxchg |
206 | #else |
207 | #define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \ |
208 | ({ \ |
209 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
210 | ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \ |
211 | if (unlikely(___r != ___o)) \ |
212 | *___op = ___r; \ |
213 | likely(___r == ___o); \ |
214 | }) |
215 | #endif |
216 | |
217 | #if defined(arch_try_cmpxchg_release) |
218 | #define raw_try_cmpxchg_release arch_try_cmpxchg_release |
219 | #elif defined(arch_try_cmpxchg_relaxed) |
220 | #define raw_try_cmpxchg_release(...) \ |
221 | __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__) |
222 | #elif defined(arch_try_cmpxchg) |
223 | #define raw_try_cmpxchg_release arch_try_cmpxchg |
224 | #else |
225 | #define raw_try_cmpxchg_release(_ptr, _oldp, _new) \ |
226 | ({ \ |
227 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
228 | ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \ |
229 | if (unlikely(___r != ___o)) \ |
230 | *___op = ___r; \ |
231 | likely(___r == ___o); \ |
232 | }) |
233 | #endif |
234 | |
235 | #if defined(arch_try_cmpxchg_relaxed) |
236 | #define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed |
237 | #elif defined(arch_try_cmpxchg) |
238 | #define raw_try_cmpxchg_relaxed arch_try_cmpxchg |
239 | #else |
240 | #define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \ |
241 | ({ \ |
242 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
243 | ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \ |
244 | if (unlikely(___r != ___o)) \ |
245 | *___op = ___r; \ |
246 | likely(___r == ___o); \ |
247 | }) |
248 | #endif |
249 | |
250 | #if defined(arch_try_cmpxchg64) |
251 | #define raw_try_cmpxchg64 arch_try_cmpxchg64 |
252 | #elif defined(arch_try_cmpxchg64_relaxed) |
253 | #define raw_try_cmpxchg64(...) \ |
254 | __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__) |
255 | #else |
256 | #define raw_try_cmpxchg64(_ptr, _oldp, _new) \ |
257 | ({ \ |
258 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
259 | ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \ |
260 | if (unlikely(___r != ___o)) \ |
261 | *___op = ___r; \ |
262 | likely(___r == ___o); \ |
263 | }) |
264 | #endif |
265 | |
266 | #if defined(arch_try_cmpxchg64_acquire) |
267 | #define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire |
268 | #elif defined(arch_try_cmpxchg64_relaxed) |
269 | #define raw_try_cmpxchg64_acquire(...) \ |
270 | __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__) |
271 | #elif defined(arch_try_cmpxchg64) |
272 | #define raw_try_cmpxchg64_acquire arch_try_cmpxchg64 |
273 | #else |
274 | #define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \ |
275 | ({ \ |
276 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
277 | ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \ |
278 | if (unlikely(___r != ___o)) \ |
279 | *___op = ___r; \ |
280 | likely(___r == ___o); \ |
281 | }) |
282 | #endif |
283 | |
284 | #if defined(arch_try_cmpxchg64_release) |
285 | #define raw_try_cmpxchg64_release arch_try_cmpxchg64_release |
286 | #elif defined(arch_try_cmpxchg64_relaxed) |
287 | #define raw_try_cmpxchg64_release(...) \ |
288 | __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__) |
289 | #elif defined(arch_try_cmpxchg64) |
290 | #define raw_try_cmpxchg64_release arch_try_cmpxchg64 |
291 | #else |
292 | #define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \ |
293 | ({ \ |
294 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
295 | ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \ |
296 | if (unlikely(___r != ___o)) \ |
297 | *___op = ___r; \ |
298 | likely(___r == ___o); \ |
299 | }) |
300 | #endif |
301 | |
302 | #if defined(arch_try_cmpxchg64_relaxed) |
303 | #define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed |
304 | #elif defined(arch_try_cmpxchg64) |
305 | #define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64 |
306 | #else |
307 | #define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \ |
308 | ({ \ |
309 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
310 | ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \ |
311 | if (unlikely(___r != ___o)) \ |
312 | *___op = ___r; \ |
313 | likely(___r == ___o); \ |
314 | }) |
315 | #endif |
316 | |
317 | #if defined(arch_try_cmpxchg128) |
318 | #define raw_try_cmpxchg128 arch_try_cmpxchg128 |
319 | #elif defined(arch_try_cmpxchg128_relaxed) |
320 | #define raw_try_cmpxchg128(...) \ |
321 | __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__) |
322 | #else |
323 | #define raw_try_cmpxchg128(_ptr, _oldp, _new) \ |
324 | ({ \ |
325 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
326 | ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \ |
327 | if (unlikely(___r != ___o)) \ |
328 | *___op = ___r; \ |
329 | likely(___r == ___o); \ |
330 | }) |
331 | #endif |
332 | |
333 | #if defined(arch_try_cmpxchg128_acquire) |
334 | #define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire |
335 | #elif defined(arch_try_cmpxchg128_relaxed) |
336 | #define raw_try_cmpxchg128_acquire(...) \ |
337 | __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__) |
338 | #elif defined(arch_try_cmpxchg128) |
339 | #define raw_try_cmpxchg128_acquire arch_try_cmpxchg128 |
340 | #else |
341 | #define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \ |
342 | ({ \ |
343 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
344 | ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \ |
345 | if (unlikely(___r != ___o)) \ |
346 | *___op = ___r; \ |
347 | likely(___r == ___o); \ |
348 | }) |
349 | #endif |
350 | |
351 | #if defined(arch_try_cmpxchg128_release) |
352 | #define raw_try_cmpxchg128_release arch_try_cmpxchg128_release |
353 | #elif defined(arch_try_cmpxchg128_relaxed) |
354 | #define raw_try_cmpxchg128_release(...) \ |
355 | __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__) |
356 | #elif defined(arch_try_cmpxchg128) |
357 | #define raw_try_cmpxchg128_release arch_try_cmpxchg128 |
358 | #else |
359 | #define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \ |
360 | ({ \ |
361 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
362 | ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \ |
363 | if (unlikely(___r != ___o)) \ |
364 | *___op = ___r; \ |
365 | likely(___r == ___o); \ |
366 | }) |
367 | #endif |
368 | |
369 | #if defined(arch_try_cmpxchg128_relaxed) |
370 | #define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed |
371 | #elif defined(arch_try_cmpxchg128) |
372 | #define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128 |
373 | #else |
374 | #define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \ |
375 | ({ \ |
376 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
377 | ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \ |
378 | if (unlikely(___r != ___o)) \ |
379 | *___op = ___r; \ |
380 | likely(___r == ___o); \ |
381 | }) |
382 | #endif |
383 | |
384 | #define raw_cmpxchg_local arch_cmpxchg_local |
385 | |
386 | #ifdef arch_try_cmpxchg_local |
387 | #define raw_try_cmpxchg_local arch_try_cmpxchg_local |
388 | #else |
389 | #define raw_try_cmpxchg_local(_ptr, _oldp, _new) \ |
390 | ({ \ |
391 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
392 | ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \ |
393 | if (unlikely(___r != ___o)) \ |
394 | *___op = ___r; \ |
395 | likely(___r == ___o); \ |
396 | }) |
397 | #endif |
398 | |
399 | #define raw_cmpxchg64_local arch_cmpxchg64_local |
400 | |
401 | #ifdef arch_try_cmpxchg64_local |
402 | #define raw_try_cmpxchg64_local arch_try_cmpxchg64_local |
403 | #else |
404 | #define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \ |
405 | ({ \ |
406 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
407 | ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \ |
408 | if (unlikely(___r != ___o)) \ |
409 | *___op = ___r; \ |
410 | likely(___r == ___o); \ |
411 | }) |
412 | #endif |
413 | |
414 | #define raw_cmpxchg128_local arch_cmpxchg128_local |
415 | |
416 | #ifdef arch_try_cmpxchg128_local |
417 | #define raw_try_cmpxchg128_local arch_try_cmpxchg128_local |
418 | #else |
419 | #define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \ |
420 | ({ \ |
421 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
422 | ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \ |
423 | if (unlikely(___r != ___o)) \ |
424 | *___op = ___r; \ |
425 | likely(___r == ___o); \ |
426 | }) |
427 | #endif |
428 | |
429 | #define raw_sync_cmpxchg arch_sync_cmpxchg |
430 | |
431 | #ifdef arch_sync_try_cmpxchg |
432 | #define raw_sync_try_cmpxchg arch_sync_try_cmpxchg |
433 | #else |
434 | #define raw_sync_try_cmpxchg(_ptr, _oldp, _new) \ |
435 | ({ \ |
436 | typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ |
437 | ___r = raw_sync_cmpxchg((_ptr), ___o, (_new)); \ |
438 | if (unlikely(___r != ___o)) \ |
439 | *___op = ___r; \ |
440 | likely(___r == ___o); \ |
441 | }) |
442 | #endif |
443 | |
444 | /** |
445 | * raw_atomic_read() - atomic load with relaxed ordering |
446 | * @v: pointer to atomic_t |
447 | * |
448 | * Atomically loads the value of @v with relaxed ordering. |
449 | * |
450 | * Safe to use in noinstr code; prefer atomic_read() elsewhere. |
451 | * |
452 | * Return: The value loaded from @v. |
453 | */ |
454 | static __always_inline int |
455 | raw_atomic_read(const atomic_t *v) |
456 | { |
457 | return arch_atomic_read(v); |
458 | } |
459 | |
460 | /** |
461 | * raw_atomic_read_acquire() - atomic load with acquire ordering |
462 | * @v: pointer to atomic_t |
463 | * |
464 | * Atomically loads the value of @v with acquire ordering. |
465 | * |
466 | * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere. |
467 | * |
468 | * Return: The value loaded from @v. |
469 | */ |
470 | static __always_inline int |
471 | raw_atomic_read_acquire(const atomic_t *v) |
472 | { |
473 | #if defined(arch_atomic_read_acquire) |
474 | return arch_atomic_read_acquire(v); |
475 | #else |
476 | int ret; |
477 | |
478 | if (__native_word(atomic_t)) { |
479 | ret = smp_load_acquire(&(v)->counter); |
480 | } else { |
481 | ret = raw_atomic_read(v); |
482 | __atomic_acquire_fence(); |
483 | } |
484 | |
485 | return ret; |
486 | #endif |
487 | } |
488 | |
489 | /** |
490 | * raw_atomic_set() - atomic set with relaxed ordering |
491 | * @v: pointer to atomic_t |
492 | * @i: int value to assign |
493 | * |
494 | * Atomically sets @v to @i with relaxed ordering. |
495 | * |
496 | * Safe to use in noinstr code; prefer atomic_set() elsewhere. |
497 | * |
498 | * Return: Nothing. |
499 | */ |
500 | static __always_inline void |
501 | raw_atomic_set(atomic_t *v, int i) |
502 | { |
503 | arch_atomic_set(v, i); |
504 | } |
505 | |
506 | /** |
507 | * raw_atomic_set_release() - atomic set with release ordering |
508 | * @v: pointer to atomic_t |
509 | * @i: int value to assign |
510 | * |
511 | * Atomically sets @v to @i with release ordering. |
512 | * |
513 | * Safe to use in noinstr code; prefer atomic_set_release() elsewhere. |
514 | * |
515 | * Return: Nothing. |
516 | */ |
517 | static __always_inline void |
518 | raw_atomic_set_release(atomic_t *v, int i) |
519 | { |
520 | #if defined(arch_atomic_set_release) |
521 | arch_atomic_set_release(v, i); |
522 | #else |
523 | if (__native_word(atomic_t)) { |
524 | smp_store_release(&(v)->counter, i); |
525 | } else { |
526 | __atomic_release_fence(); |
527 | raw_atomic_set(v, i); |
528 | } |
529 | #endif |
530 | } |
531 | |
532 | /** |
533 | * raw_atomic_add() - atomic add with relaxed ordering |
534 | * @i: int value to add |
535 | * @v: pointer to atomic_t |
536 | * |
537 | * Atomically updates @v to (@v + @i) with relaxed ordering. |
538 | * |
539 | * Safe to use in noinstr code; prefer atomic_add() elsewhere. |
540 | * |
541 | * Return: Nothing. |
542 | */ |
543 | static __always_inline void |
544 | raw_atomic_add(int i, atomic_t *v) |
545 | { |
546 | arch_atomic_add(i, v); |
547 | } |
548 | |
549 | /** |
550 | * raw_atomic_add_return() - atomic add with full ordering |
551 | * @i: int value to add |
552 | * @v: pointer to atomic_t |
553 | * |
554 | * Atomically updates @v to (@v + @i) with full ordering. |
555 | * |
556 | * Safe to use in noinstr code; prefer atomic_add_return() elsewhere. |
557 | * |
558 | * Return: The updated value of @v. |
559 | */ |
560 | static __always_inline int |
561 | raw_atomic_add_return(int i, atomic_t *v) |
562 | { |
563 | #if defined(arch_atomic_add_return) |
564 | return arch_atomic_add_return(i, v); |
565 | #elif defined(arch_atomic_add_return_relaxed) |
566 | int ret; |
567 | __atomic_pre_full_fence(); |
568 | ret = arch_atomic_add_return_relaxed(i, v); |
569 | __atomic_post_full_fence(); |
570 | return ret; |
571 | #else |
572 | #error "Unable to define raw_atomic_add_return" |
573 | #endif |
574 | } |
575 | |
576 | /** |
577 | * raw_atomic_add_return_acquire() - atomic add with acquire ordering |
578 | * @i: int value to add |
579 | * @v: pointer to atomic_t |
580 | * |
581 | * Atomically updates @v to (@v + @i) with acquire ordering. |
582 | * |
583 | * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere. |
584 | * |
585 | * Return: The updated value of @v. |
586 | */ |
587 | static __always_inline int |
588 | raw_atomic_add_return_acquire(int i, atomic_t *v) |
589 | { |
590 | #if defined(arch_atomic_add_return_acquire) |
591 | return arch_atomic_add_return_acquire(i, v); |
592 | #elif defined(arch_atomic_add_return_relaxed) |
593 | int ret = arch_atomic_add_return_relaxed(i, v); |
594 | __atomic_acquire_fence(); |
595 | return ret; |
596 | #elif defined(arch_atomic_add_return) |
597 | return arch_atomic_add_return(i, v); |
598 | #else |
599 | #error "Unable to define raw_atomic_add_return_acquire" |
600 | #endif |
601 | } |
602 | |
603 | /** |
604 | * raw_atomic_add_return_release() - atomic add with release ordering |
605 | * @i: int value to add |
606 | * @v: pointer to atomic_t |
607 | * |
608 | * Atomically updates @v to (@v + @i) with release ordering. |
609 | * |
610 | * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere. |
611 | * |
612 | * Return: The updated value of @v. |
613 | */ |
614 | static __always_inline int |
615 | raw_atomic_add_return_release(int i, atomic_t *v) |
616 | { |
617 | #if defined(arch_atomic_add_return_release) |
618 | return arch_atomic_add_return_release(i, v); |
619 | #elif defined(arch_atomic_add_return_relaxed) |
620 | __atomic_release_fence(); |
621 | return arch_atomic_add_return_relaxed(i, v); |
622 | #elif defined(arch_atomic_add_return) |
623 | return arch_atomic_add_return(i, v); |
624 | #else |
625 | #error "Unable to define raw_atomic_add_return_release" |
626 | #endif |
627 | } |
628 | |
629 | /** |
630 | * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering |
631 | * @i: int value to add |
632 | * @v: pointer to atomic_t |
633 | * |
634 | * Atomically updates @v to (@v + @i) with relaxed ordering. |
635 | * |
636 | * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere. |
637 | * |
638 | * Return: The updated value of @v. |
639 | */ |
640 | static __always_inline int |
641 | raw_atomic_add_return_relaxed(int i, atomic_t *v) |
642 | { |
643 | #if defined(arch_atomic_add_return_relaxed) |
644 | return arch_atomic_add_return_relaxed(i, v); |
645 | #elif defined(arch_atomic_add_return) |
646 | return arch_atomic_add_return(i, v); |
647 | #else |
648 | #error "Unable to define raw_atomic_add_return_relaxed" |
649 | #endif |
650 | } |
651 | |
652 | /** |
653 | * raw_atomic_fetch_add() - atomic add with full ordering |
654 | * @i: int value to add |
655 | * @v: pointer to atomic_t |
656 | * |
657 | * Atomically updates @v to (@v + @i) with full ordering. |
658 | * |
659 | * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere. |
660 | * |
661 | * Return: The original value of @v. |
662 | */ |
663 | static __always_inline int |
664 | raw_atomic_fetch_add(int i, atomic_t *v) |
665 | { |
666 | #if defined(arch_atomic_fetch_add) |
667 | return arch_atomic_fetch_add(i, v); |
668 | #elif defined(arch_atomic_fetch_add_relaxed) |
669 | int ret; |
670 | __atomic_pre_full_fence(); |
671 | ret = arch_atomic_fetch_add_relaxed(i, v); |
672 | __atomic_post_full_fence(); |
673 | return ret; |
674 | #else |
675 | #error "Unable to define raw_atomic_fetch_add" |
676 | #endif |
677 | } |
678 | |
679 | /** |
680 | * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering |
681 | * @i: int value to add |
682 | * @v: pointer to atomic_t |
683 | * |
684 | * Atomically updates @v to (@v + @i) with acquire ordering. |
685 | * |
686 | * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere. |
687 | * |
688 | * Return: The original value of @v. |
689 | */ |
690 | static __always_inline int |
691 | raw_atomic_fetch_add_acquire(int i, atomic_t *v) |
692 | { |
693 | #if defined(arch_atomic_fetch_add_acquire) |
694 | return arch_atomic_fetch_add_acquire(i, v); |
695 | #elif defined(arch_atomic_fetch_add_relaxed) |
696 | int ret = arch_atomic_fetch_add_relaxed(i, v); |
697 | __atomic_acquire_fence(); |
698 | return ret; |
699 | #elif defined(arch_atomic_fetch_add) |
700 | return arch_atomic_fetch_add(i, v); |
701 | #else |
702 | #error "Unable to define raw_atomic_fetch_add_acquire" |
703 | #endif |
704 | } |
705 | |
706 | /** |
707 | * raw_atomic_fetch_add_release() - atomic add with release ordering |
708 | * @i: int value to add |
709 | * @v: pointer to atomic_t |
710 | * |
711 | * Atomically updates @v to (@v + @i) with release ordering. |
712 | * |
713 | * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere. |
714 | * |
715 | * Return: The original value of @v. |
716 | */ |
717 | static __always_inline int |
718 | raw_atomic_fetch_add_release(int i, atomic_t *v) |
719 | { |
720 | #if defined(arch_atomic_fetch_add_release) |
721 | return arch_atomic_fetch_add_release(i, v); |
722 | #elif defined(arch_atomic_fetch_add_relaxed) |
723 | __atomic_release_fence(); |
724 | return arch_atomic_fetch_add_relaxed(i, v); |
725 | #elif defined(arch_atomic_fetch_add) |
726 | return arch_atomic_fetch_add(i, v); |
727 | #else |
728 | #error "Unable to define raw_atomic_fetch_add_release" |
729 | #endif |
730 | } |
731 | |
732 | /** |
733 | * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering |
734 | * @i: int value to add |
735 | * @v: pointer to atomic_t |
736 | * |
737 | * Atomically updates @v to (@v + @i) with relaxed ordering. |
738 | * |
739 | * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere. |
740 | * |
741 | * Return: The original value of @v. |
742 | */ |
743 | static __always_inline int |
744 | raw_atomic_fetch_add_relaxed(int i, atomic_t *v) |
745 | { |
746 | #if defined(arch_atomic_fetch_add_relaxed) |
747 | return arch_atomic_fetch_add_relaxed(i, v); |
748 | #elif defined(arch_atomic_fetch_add) |
749 | return arch_atomic_fetch_add(i, v); |
750 | #else |
751 | #error "Unable to define raw_atomic_fetch_add_relaxed" |
752 | #endif |
753 | } |
754 | |
755 | /** |
756 | * raw_atomic_sub() - atomic subtract with relaxed ordering |
757 | * @i: int value to subtract |
758 | * @v: pointer to atomic_t |
759 | * |
760 | * Atomically updates @v to (@v - @i) with relaxed ordering. |
761 | * |
762 | * Safe to use in noinstr code; prefer atomic_sub() elsewhere. |
763 | * |
764 | * Return: Nothing. |
765 | */ |
766 | static __always_inline void |
767 | raw_atomic_sub(int i, atomic_t *v) |
768 | { |
769 | arch_atomic_sub(i, v); |
770 | } |
771 | |
772 | /** |
773 | * raw_atomic_sub_return() - atomic subtract with full ordering |
774 | * @i: int value to subtract |
775 | * @v: pointer to atomic_t |
776 | * |
777 | * Atomically updates @v to (@v - @i) with full ordering. |
778 | * |
779 | * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere. |
780 | * |
781 | * Return: The updated value of @v. |
782 | */ |
783 | static __always_inline int |
784 | raw_atomic_sub_return(int i, atomic_t *v) |
785 | { |
786 | #if defined(arch_atomic_sub_return) |
787 | return arch_atomic_sub_return(i, v); |
788 | #elif defined(arch_atomic_sub_return_relaxed) |
789 | int ret; |
790 | __atomic_pre_full_fence(); |
791 | ret = arch_atomic_sub_return_relaxed(i, v); |
792 | __atomic_post_full_fence(); |
793 | return ret; |
794 | #else |
795 | #error "Unable to define raw_atomic_sub_return" |
796 | #endif |
797 | } |
798 | |
799 | /** |
800 | * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering |
801 | * @i: int value to subtract |
802 | * @v: pointer to atomic_t |
803 | * |
804 | * Atomically updates @v to (@v - @i) with acquire ordering. |
805 | * |
806 | * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere. |
807 | * |
808 | * Return: The updated value of @v. |
809 | */ |
810 | static __always_inline int |
811 | raw_atomic_sub_return_acquire(int i, atomic_t *v) |
812 | { |
813 | #if defined(arch_atomic_sub_return_acquire) |
814 | return arch_atomic_sub_return_acquire(i, v); |
815 | #elif defined(arch_atomic_sub_return_relaxed) |
816 | int ret = arch_atomic_sub_return_relaxed(i, v); |
817 | __atomic_acquire_fence(); |
818 | return ret; |
819 | #elif defined(arch_atomic_sub_return) |
820 | return arch_atomic_sub_return(i, v); |
821 | #else |
822 | #error "Unable to define raw_atomic_sub_return_acquire" |
823 | #endif |
824 | } |
825 | |
826 | /** |
827 | * raw_atomic_sub_return_release() - atomic subtract with release ordering |
828 | * @i: int value to subtract |
829 | * @v: pointer to atomic_t |
830 | * |
831 | * Atomically updates @v to (@v - @i) with release ordering. |
832 | * |
833 | * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere. |
834 | * |
835 | * Return: The updated value of @v. |
836 | */ |
837 | static __always_inline int |
838 | raw_atomic_sub_return_release(int i, atomic_t *v) |
839 | { |
840 | #if defined(arch_atomic_sub_return_release) |
841 | return arch_atomic_sub_return_release(i, v); |
842 | #elif defined(arch_atomic_sub_return_relaxed) |
843 | __atomic_release_fence(); |
844 | return arch_atomic_sub_return_relaxed(i, v); |
845 | #elif defined(arch_atomic_sub_return) |
846 | return arch_atomic_sub_return(i, v); |
847 | #else |
848 | #error "Unable to define raw_atomic_sub_return_release" |
849 | #endif |
850 | } |
851 | |
852 | /** |
853 | * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering |
854 | * @i: int value to subtract |
855 | * @v: pointer to atomic_t |
856 | * |
857 | * Atomically updates @v to (@v - @i) with relaxed ordering. |
858 | * |
859 | * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere. |
860 | * |
861 | * Return: The updated value of @v. |
862 | */ |
863 | static __always_inline int |
864 | raw_atomic_sub_return_relaxed(int i, atomic_t *v) |
865 | { |
866 | #if defined(arch_atomic_sub_return_relaxed) |
867 | return arch_atomic_sub_return_relaxed(i, v); |
868 | #elif defined(arch_atomic_sub_return) |
869 | return arch_atomic_sub_return(i, v); |
870 | #else |
871 | #error "Unable to define raw_atomic_sub_return_relaxed" |
872 | #endif |
873 | } |
874 | |
875 | /** |
876 | * raw_atomic_fetch_sub() - atomic subtract with full ordering |
877 | * @i: int value to subtract |
878 | * @v: pointer to atomic_t |
879 | * |
880 | * Atomically updates @v to (@v - @i) with full ordering. |
881 | * |
882 | * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere. |
883 | * |
884 | * Return: The original value of @v. |
885 | */ |
886 | static __always_inline int |
887 | raw_atomic_fetch_sub(int i, atomic_t *v) |
888 | { |
889 | #if defined(arch_atomic_fetch_sub) |
890 | return arch_atomic_fetch_sub(i, v); |
891 | #elif defined(arch_atomic_fetch_sub_relaxed) |
892 | int ret; |
893 | __atomic_pre_full_fence(); |
894 | ret = arch_atomic_fetch_sub_relaxed(i, v); |
895 | __atomic_post_full_fence(); |
896 | return ret; |
897 | #else |
898 | #error "Unable to define raw_atomic_fetch_sub" |
899 | #endif |
900 | } |
901 | |
902 | /** |
903 | * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering |
904 | * @i: int value to subtract |
905 | * @v: pointer to atomic_t |
906 | * |
907 | * Atomically updates @v to (@v - @i) with acquire ordering. |
908 | * |
909 | * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere. |
910 | * |
911 | * Return: The original value of @v. |
912 | */ |
913 | static __always_inline int |
914 | raw_atomic_fetch_sub_acquire(int i, atomic_t *v) |
915 | { |
916 | #if defined(arch_atomic_fetch_sub_acquire) |
917 | return arch_atomic_fetch_sub_acquire(i, v); |
918 | #elif defined(arch_atomic_fetch_sub_relaxed) |
919 | int ret = arch_atomic_fetch_sub_relaxed(i, v); |
920 | __atomic_acquire_fence(); |
921 | return ret; |
922 | #elif defined(arch_atomic_fetch_sub) |
923 | return arch_atomic_fetch_sub(i, v); |
924 | #else |
925 | #error "Unable to define raw_atomic_fetch_sub_acquire" |
926 | #endif |
927 | } |
928 | |
929 | /** |
930 | * raw_atomic_fetch_sub_release() - atomic subtract with release ordering |
931 | * @i: int value to subtract |
932 | * @v: pointer to atomic_t |
933 | * |
934 | * Atomically updates @v to (@v - @i) with release ordering. |
935 | * |
936 | * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere. |
937 | * |
938 | * Return: The original value of @v. |
939 | */ |
940 | static __always_inline int |
941 | raw_atomic_fetch_sub_release(int i, atomic_t *v) |
942 | { |
943 | #if defined(arch_atomic_fetch_sub_release) |
944 | return arch_atomic_fetch_sub_release(i, v); |
945 | #elif defined(arch_atomic_fetch_sub_relaxed) |
946 | __atomic_release_fence(); |
947 | return arch_atomic_fetch_sub_relaxed(i, v); |
948 | #elif defined(arch_atomic_fetch_sub) |
949 | return arch_atomic_fetch_sub(i, v); |
950 | #else |
951 | #error "Unable to define raw_atomic_fetch_sub_release" |
952 | #endif |
953 | } |
954 | |
955 | /** |
956 | * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering |
957 | * @i: int value to subtract |
958 | * @v: pointer to atomic_t |
959 | * |
960 | * Atomically updates @v to (@v - @i) with relaxed ordering. |
961 | * |
962 | * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere. |
963 | * |
964 | * Return: The original value of @v. |
965 | */ |
966 | static __always_inline int |
967 | raw_atomic_fetch_sub_relaxed(int i, atomic_t *v) |
968 | { |
969 | #if defined(arch_atomic_fetch_sub_relaxed) |
970 | return arch_atomic_fetch_sub_relaxed(i, v); |
971 | #elif defined(arch_atomic_fetch_sub) |
972 | return arch_atomic_fetch_sub(i, v); |
973 | #else |
974 | #error "Unable to define raw_atomic_fetch_sub_relaxed" |
975 | #endif |
976 | } |
977 | |
978 | /** |
979 | * raw_atomic_inc() - atomic increment with relaxed ordering |
980 | * @v: pointer to atomic_t |
981 | * |
982 | * Atomically updates @v to (@v + 1) with relaxed ordering. |
983 | * |
984 | * Safe to use in noinstr code; prefer atomic_inc() elsewhere. |
985 | * |
986 | * Return: Nothing. |
987 | */ |
988 | static __always_inline void |
989 | raw_atomic_inc(atomic_t *v) |
990 | { |
991 | #if defined(arch_atomic_inc) |
992 | arch_atomic_inc(v); |
993 | #else |
994 | raw_atomic_add(1, v); |
995 | #endif |
996 | } |
997 | |
998 | /** |
999 | * raw_atomic_inc_return() - atomic increment with full ordering |
1000 | * @v: pointer to atomic_t |
1001 | * |
1002 | * Atomically updates @v to (@v + 1) with full ordering. |
1003 | * |
1004 | * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere. |
1005 | * |
1006 | * Return: The updated value of @v. |
1007 | */ |
1008 | static __always_inline int |
1009 | raw_atomic_inc_return(atomic_t *v) |
1010 | { |
1011 | #if defined(arch_atomic_inc_return) |
1012 | return arch_atomic_inc_return(v); |
1013 | #elif defined(arch_atomic_inc_return_relaxed) |
1014 | int ret; |
1015 | __atomic_pre_full_fence(); |
1016 | ret = arch_atomic_inc_return_relaxed(v); |
1017 | __atomic_post_full_fence(); |
1018 | return ret; |
1019 | #else |
1020 | return raw_atomic_add_return(i: 1, v); |
1021 | #endif |
1022 | } |
1023 | |
1024 | /** |
1025 | * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering |
1026 | * @v: pointer to atomic_t |
1027 | * |
1028 | * Atomically updates @v to (@v + 1) with acquire ordering. |
1029 | * |
1030 | * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere. |
1031 | * |
1032 | * Return: The updated value of @v. |
1033 | */ |
1034 | static __always_inline int |
1035 | raw_atomic_inc_return_acquire(atomic_t *v) |
1036 | { |
1037 | #if defined(arch_atomic_inc_return_acquire) |
1038 | return arch_atomic_inc_return_acquire(v); |
1039 | #elif defined(arch_atomic_inc_return_relaxed) |
1040 | int ret = arch_atomic_inc_return_relaxed(v); |
1041 | __atomic_acquire_fence(); |
1042 | return ret; |
1043 | #elif defined(arch_atomic_inc_return) |
1044 | return arch_atomic_inc_return(v); |
1045 | #else |
1046 | return raw_atomic_add_return_acquire(i: 1, v); |
1047 | #endif |
1048 | } |
1049 | |
1050 | /** |
1051 | * raw_atomic_inc_return_release() - atomic increment with release ordering |
1052 | * @v: pointer to atomic_t |
1053 | * |
1054 | * Atomically updates @v to (@v + 1) with release ordering. |
1055 | * |
1056 | * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere. |
1057 | * |
1058 | * Return: The updated value of @v. |
1059 | */ |
1060 | static __always_inline int |
1061 | raw_atomic_inc_return_release(atomic_t *v) |
1062 | { |
1063 | #if defined(arch_atomic_inc_return_release) |
1064 | return arch_atomic_inc_return_release(v); |
1065 | #elif defined(arch_atomic_inc_return_relaxed) |
1066 | __atomic_release_fence(); |
1067 | return arch_atomic_inc_return_relaxed(v); |
1068 | #elif defined(arch_atomic_inc_return) |
1069 | return arch_atomic_inc_return(v); |
1070 | #else |
1071 | return raw_atomic_add_return_release(i: 1, v); |
1072 | #endif |
1073 | } |
1074 | |
1075 | /** |
1076 | * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering |
1077 | * @v: pointer to atomic_t |
1078 | * |
1079 | * Atomically updates @v to (@v + 1) with relaxed ordering. |
1080 | * |
1081 | * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere. |
1082 | * |
1083 | * Return: The updated value of @v. |
1084 | */ |
1085 | static __always_inline int |
1086 | raw_atomic_inc_return_relaxed(atomic_t *v) |
1087 | { |
1088 | #if defined(arch_atomic_inc_return_relaxed) |
1089 | return arch_atomic_inc_return_relaxed(v); |
1090 | #elif defined(arch_atomic_inc_return) |
1091 | return arch_atomic_inc_return(v); |
1092 | #else |
1093 | return raw_atomic_add_return_relaxed(i: 1, v); |
1094 | #endif |
1095 | } |
1096 | |
1097 | /** |
1098 | * raw_atomic_fetch_inc() - atomic increment with full ordering |
1099 | * @v: pointer to atomic_t |
1100 | * |
1101 | * Atomically updates @v to (@v + 1) with full ordering. |
1102 | * |
1103 | * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere. |
1104 | * |
1105 | * Return: The original value of @v. |
1106 | */ |
1107 | static __always_inline int |
1108 | raw_atomic_fetch_inc(atomic_t *v) |
1109 | { |
1110 | #if defined(arch_atomic_fetch_inc) |
1111 | return arch_atomic_fetch_inc(v); |
1112 | #elif defined(arch_atomic_fetch_inc_relaxed) |
1113 | int ret; |
1114 | __atomic_pre_full_fence(); |
1115 | ret = arch_atomic_fetch_inc_relaxed(v); |
1116 | __atomic_post_full_fence(); |
1117 | return ret; |
1118 | #else |
1119 | return raw_atomic_fetch_add(i: 1, v); |
1120 | #endif |
1121 | } |
1122 | |
1123 | /** |
1124 | * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering |
1125 | * @v: pointer to atomic_t |
1126 | * |
1127 | * Atomically updates @v to (@v + 1) with acquire ordering. |
1128 | * |
1129 | * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere. |
1130 | * |
1131 | * Return: The original value of @v. |
1132 | */ |
1133 | static __always_inline int |
1134 | raw_atomic_fetch_inc_acquire(atomic_t *v) |
1135 | { |
1136 | #if defined(arch_atomic_fetch_inc_acquire) |
1137 | return arch_atomic_fetch_inc_acquire(v); |
1138 | #elif defined(arch_atomic_fetch_inc_relaxed) |
1139 | int ret = arch_atomic_fetch_inc_relaxed(v); |
1140 | __atomic_acquire_fence(); |
1141 | return ret; |
1142 | #elif defined(arch_atomic_fetch_inc) |
1143 | return arch_atomic_fetch_inc(v); |
1144 | #else |
1145 | return raw_atomic_fetch_add_acquire(i: 1, v); |
1146 | #endif |
1147 | } |
1148 | |
1149 | /** |
1150 | * raw_atomic_fetch_inc_release() - atomic increment with release ordering |
1151 | * @v: pointer to atomic_t |
1152 | * |
1153 | * Atomically updates @v to (@v + 1) with release ordering. |
1154 | * |
1155 | * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere. |
1156 | * |
1157 | * Return: The original value of @v. |
1158 | */ |
1159 | static __always_inline int |
1160 | raw_atomic_fetch_inc_release(atomic_t *v) |
1161 | { |
1162 | #if defined(arch_atomic_fetch_inc_release) |
1163 | return arch_atomic_fetch_inc_release(v); |
1164 | #elif defined(arch_atomic_fetch_inc_relaxed) |
1165 | __atomic_release_fence(); |
1166 | return arch_atomic_fetch_inc_relaxed(v); |
1167 | #elif defined(arch_atomic_fetch_inc) |
1168 | return arch_atomic_fetch_inc(v); |
1169 | #else |
1170 | return raw_atomic_fetch_add_release(i: 1, v); |
1171 | #endif |
1172 | } |
1173 | |
1174 | /** |
1175 | * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering |
1176 | * @v: pointer to atomic_t |
1177 | * |
1178 | * Atomically updates @v to (@v + 1) with relaxed ordering. |
1179 | * |
1180 | * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere. |
1181 | * |
1182 | * Return: The original value of @v. |
1183 | */ |
1184 | static __always_inline int |
1185 | raw_atomic_fetch_inc_relaxed(atomic_t *v) |
1186 | { |
1187 | #if defined(arch_atomic_fetch_inc_relaxed) |
1188 | return arch_atomic_fetch_inc_relaxed(v); |
1189 | #elif defined(arch_atomic_fetch_inc) |
1190 | return arch_atomic_fetch_inc(v); |
1191 | #else |
1192 | return raw_atomic_fetch_add_relaxed(i: 1, v); |
1193 | #endif |
1194 | } |
1195 | |
1196 | /** |
1197 | * raw_atomic_dec() - atomic decrement with relaxed ordering |
1198 | * @v: pointer to atomic_t |
1199 | * |
1200 | * Atomically updates @v to (@v - 1) with relaxed ordering. |
1201 | * |
1202 | * Safe to use in noinstr code; prefer atomic_dec() elsewhere. |
1203 | * |
1204 | * Return: Nothing. |
1205 | */ |
1206 | static __always_inline void |
1207 | raw_atomic_dec(atomic_t *v) |
1208 | { |
1209 | #if defined(arch_atomic_dec) |
1210 | arch_atomic_dec(v); |
1211 | #else |
1212 | raw_atomic_sub(1, v); |
1213 | #endif |
1214 | } |
1215 | |
1216 | /** |
1217 | * raw_atomic_dec_return() - atomic decrement with full ordering |
1218 | * @v: pointer to atomic_t |
1219 | * |
1220 | * Atomically updates @v to (@v - 1) with full ordering. |
1221 | * |
1222 | * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere. |
1223 | * |
1224 | * Return: The updated value of @v. |
1225 | */ |
1226 | static __always_inline int |
1227 | raw_atomic_dec_return(atomic_t *v) |
1228 | { |
1229 | #if defined(arch_atomic_dec_return) |
1230 | return arch_atomic_dec_return(v); |
1231 | #elif defined(arch_atomic_dec_return_relaxed) |
1232 | int ret; |
1233 | __atomic_pre_full_fence(); |
1234 | ret = arch_atomic_dec_return_relaxed(v); |
1235 | __atomic_post_full_fence(); |
1236 | return ret; |
1237 | #else |
1238 | return raw_atomic_sub_return(i: 1, v); |
1239 | #endif |
1240 | } |
1241 | |
1242 | /** |
1243 | * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering |
1244 | * @v: pointer to atomic_t |
1245 | * |
1246 | * Atomically updates @v to (@v - 1) with acquire ordering. |
1247 | * |
1248 | * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere. |
1249 | * |
1250 | * Return: The updated value of @v. |
1251 | */ |
1252 | static __always_inline int |
1253 | raw_atomic_dec_return_acquire(atomic_t *v) |
1254 | { |
1255 | #if defined(arch_atomic_dec_return_acquire) |
1256 | return arch_atomic_dec_return_acquire(v); |
1257 | #elif defined(arch_atomic_dec_return_relaxed) |
1258 | int ret = arch_atomic_dec_return_relaxed(v); |
1259 | __atomic_acquire_fence(); |
1260 | return ret; |
1261 | #elif defined(arch_atomic_dec_return) |
1262 | return arch_atomic_dec_return(v); |
1263 | #else |
1264 | return raw_atomic_sub_return_acquire(i: 1, v); |
1265 | #endif |
1266 | } |
1267 | |
1268 | /** |
1269 | * raw_atomic_dec_return_release() - atomic decrement with release ordering |
1270 | * @v: pointer to atomic_t |
1271 | * |
1272 | * Atomically updates @v to (@v - 1) with release ordering. |
1273 | * |
1274 | * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere. |
1275 | * |
1276 | * Return: The updated value of @v. |
1277 | */ |
1278 | static __always_inline int |
1279 | raw_atomic_dec_return_release(atomic_t *v) |
1280 | { |
1281 | #if defined(arch_atomic_dec_return_release) |
1282 | return arch_atomic_dec_return_release(v); |
1283 | #elif defined(arch_atomic_dec_return_relaxed) |
1284 | __atomic_release_fence(); |
1285 | return arch_atomic_dec_return_relaxed(v); |
1286 | #elif defined(arch_atomic_dec_return) |
1287 | return arch_atomic_dec_return(v); |
1288 | #else |
1289 | return raw_atomic_sub_return_release(i: 1, v); |
1290 | #endif |
1291 | } |
1292 | |
1293 | /** |
1294 | * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering |
1295 | * @v: pointer to atomic_t |
1296 | * |
1297 | * Atomically updates @v to (@v - 1) with relaxed ordering. |
1298 | * |
1299 | * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere. |
1300 | * |
1301 | * Return: The updated value of @v. |
1302 | */ |
1303 | static __always_inline int |
1304 | raw_atomic_dec_return_relaxed(atomic_t *v) |
1305 | { |
1306 | #if defined(arch_atomic_dec_return_relaxed) |
1307 | return arch_atomic_dec_return_relaxed(v); |
1308 | #elif defined(arch_atomic_dec_return) |
1309 | return arch_atomic_dec_return(v); |
1310 | #else |
1311 | return raw_atomic_sub_return_relaxed(i: 1, v); |
1312 | #endif |
1313 | } |
1314 | |
1315 | /** |
1316 | * raw_atomic_fetch_dec() - atomic decrement with full ordering |
1317 | * @v: pointer to atomic_t |
1318 | * |
1319 | * Atomically updates @v to (@v - 1) with full ordering. |
1320 | * |
1321 | * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere. |
1322 | * |
1323 | * Return: The original value of @v. |
1324 | */ |
1325 | static __always_inline int |
1326 | raw_atomic_fetch_dec(atomic_t *v) |
1327 | { |
1328 | #if defined(arch_atomic_fetch_dec) |
1329 | return arch_atomic_fetch_dec(v); |
1330 | #elif defined(arch_atomic_fetch_dec_relaxed) |
1331 | int ret; |
1332 | __atomic_pre_full_fence(); |
1333 | ret = arch_atomic_fetch_dec_relaxed(v); |
1334 | __atomic_post_full_fence(); |
1335 | return ret; |
1336 | #else |
1337 | return raw_atomic_fetch_sub(i: 1, v); |
1338 | #endif |
1339 | } |
1340 | |
1341 | /** |
1342 | * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering |
1343 | * @v: pointer to atomic_t |
1344 | * |
1345 | * Atomically updates @v to (@v - 1) with acquire ordering. |
1346 | * |
1347 | * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere. |
1348 | * |
1349 | * Return: The original value of @v. |
1350 | */ |
1351 | static __always_inline int |
1352 | raw_atomic_fetch_dec_acquire(atomic_t *v) |
1353 | { |
1354 | #if defined(arch_atomic_fetch_dec_acquire) |
1355 | return arch_atomic_fetch_dec_acquire(v); |
1356 | #elif defined(arch_atomic_fetch_dec_relaxed) |
1357 | int ret = arch_atomic_fetch_dec_relaxed(v); |
1358 | __atomic_acquire_fence(); |
1359 | return ret; |
1360 | #elif defined(arch_atomic_fetch_dec) |
1361 | return arch_atomic_fetch_dec(v); |
1362 | #else |
1363 | return raw_atomic_fetch_sub_acquire(i: 1, v); |
1364 | #endif |
1365 | } |
1366 | |
1367 | /** |
1368 | * raw_atomic_fetch_dec_release() - atomic decrement with release ordering |
1369 | * @v: pointer to atomic_t |
1370 | * |
1371 | * Atomically updates @v to (@v - 1) with release ordering. |
1372 | * |
1373 | * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere. |
1374 | * |
1375 | * Return: The original value of @v. |
1376 | */ |
1377 | static __always_inline int |
1378 | raw_atomic_fetch_dec_release(atomic_t *v) |
1379 | { |
1380 | #if defined(arch_atomic_fetch_dec_release) |
1381 | return arch_atomic_fetch_dec_release(v); |
1382 | #elif defined(arch_atomic_fetch_dec_relaxed) |
1383 | __atomic_release_fence(); |
1384 | return arch_atomic_fetch_dec_relaxed(v); |
1385 | #elif defined(arch_atomic_fetch_dec) |
1386 | return arch_atomic_fetch_dec(v); |
1387 | #else |
1388 | return raw_atomic_fetch_sub_release(i: 1, v); |
1389 | #endif |
1390 | } |
1391 | |
1392 | /** |
1393 | * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering |
1394 | * @v: pointer to atomic_t |
1395 | * |
1396 | * Atomically updates @v to (@v - 1) with relaxed ordering. |
1397 | * |
1398 | * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere. |
1399 | * |
1400 | * Return: The original value of @v. |
1401 | */ |
1402 | static __always_inline int |
1403 | raw_atomic_fetch_dec_relaxed(atomic_t *v) |
1404 | { |
1405 | #if defined(arch_atomic_fetch_dec_relaxed) |
1406 | return arch_atomic_fetch_dec_relaxed(v); |
1407 | #elif defined(arch_atomic_fetch_dec) |
1408 | return arch_atomic_fetch_dec(v); |
1409 | #else |
1410 | return raw_atomic_fetch_sub_relaxed(i: 1, v); |
1411 | #endif |
1412 | } |
1413 | |
1414 | /** |
1415 | * raw_atomic_and() - atomic bitwise AND with relaxed ordering |
1416 | * @i: int value |
1417 | * @v: pointer to atomic_t |
1418 | * |
1419 | * Atomically updates @v to (@v & @i) with relaxed ordering. |
1420 | * |
1421 | * Safe to use in noinstr code; prefer atomic_and() elsewhere. |
1422 | * |
1423 | * Return: Nothing. |
1424 | */ |
1425 | static __always_inline void |
1426 | raw_atomic_and(int i, atomic_t *v) |
1427 | { |
1428 | arch_atomic_and(i, v); |
1429 | } |
1430 | |
1431 | /** |
1432 | * raw_atomic_fetch_and() - atomic bitwise AND with full ordering |
1433 | * @i: int value |
1434 | * @v: pointer to atomic_t |
1435 | * |
1436 | * Atomically updates @v to (@v & @i) with full ordering. |
1437 | * |
1438 | * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere. |
1439 | * |
1440 | * Return: The original value of @v. |
1441 | */ |
1442 | static __always_inline int |
1443 | raw_atomic_fetch_and(int i, atomic_t *v) |
1444 | { |
1445 | #if defined(arch_atomic_fetch_and) |
1446 | return arch_atomic_fetch_and(i, v); |
1447 | #elif defined(arch_atomic_fetch_and_relaxed) |
1448 | int ret; |
1449 | __atomic_pre_full_fence(); |
1450 | ret = arch_atomic_fetch_and_relaxed(i, v); |
1451 | __atomic_post_full_fence(); |
1452 | return ret; |
1453 | #else |
1454 | #error "Unable to define raw_atomic_fetch_and" |
1455 | #endif |
1456 | } |
1457 | |
1458 | /** |
1459 | * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering |
1460 | * @i: int value |
1461 | * @v: pointer to atomic_t |
1462 | * |
1463 | * Atomically updates @v to (@v & @i) with acquire ordering. |
1464 | * |
1465 | * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere. |
1466 | * |
1467 | * Return: The original value of @v. |
1468 | */ |
1469 | static __always_inline int |
1470 | raw_atomic_fetch_and_acquire(int i, atomic_t *v) |
1471 | { |
1472 | #if defined(arch_atomic_fetch_and_acquire) |
1473 | return arch_atomic_fetch_and_acquire(i, v); |
1474 | #elif defined(arch_atomic_fetch_and_relaxed) |
1475 | int ret = arch_atomic_fetch_and_relaxed(i, v); |
1476 | __atomic_acquire_fence(); |
1477 | return ret; |
1478 | #elif defined(arch_atomic_fetch_and) |
1479 | return arch_atomic_fetch_and(i, v); |
1480 | #else |
1481 | #error "Unable to define raw_atomic_fetch_and_acquire" |
1482 | #endif |
1483 | } |
1484 | |
1485 | /** |
1486 | * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering |
1487 | * @i: int value |
1488 | * @v: pointer to atomic_t |
1489 | * |
1490 | * Atomically updates @v to (@v & @i) with release ordering. |
1491 | * |
1492 | * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere. |
1493 | * |
1494 | * Return: The original value of @v. |
1495 | */ |
1496 | static __always_inline int |
1497 | raw_atomic_fetch_and_release(int i, atomic_t *v) |
1498 | { |
1499 | #if defined(arch_atomic_fetch_and_release) |
1500 | return arch_atomic_fetch_and_release(i, v); |
1501 | #elif defined(arch_atomic_fetch_and_relaxed) |
1502 | __atomic_release_fence(); |
1503 | return arch_atomic_fetch_and_relaxed(i, v); |
1504 | #elif defined(arch_atomic_fetch_and) |
1505 | return arch_atomic_fetch_and(i, v); |
1506 | #else |
1507 | #error "Unable to define raw_atomic_fetch_and_release" |
1508 | #endif |
1509 | } |
1510 | |
1511 | /** |
1512 | * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering |
1513 | * @i: int value |
1514 | * @v: pointer to atomic_t |
1515 | * |
1516 | * Atomically updates @v to (@v & @i) with relaxed ordering. |
1517 | * |
1518 | * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere. |
1519 | * |
1520 | * Return: The original value of @v. |
1521 | */ |
1522 | static __always_inline int |
1523 | raw_atomic_fetch_and_relaxed(int i, atomic_t *v) |
1524 | { |
1525 | #if defined(arch_atomic_fetch_and_relaxed) |
1526 | return arch_atomic_fetch_and_relaxed(i, v); |
1527 | #elif defined(arch_atomic_fetch_and) |
1528 | return arch_atomic_fetch_and(i, v); |
1529 | #else |
1530 | #error "Unable to define raw_atomic_fetch_and_relaxed" |
1531 | #endif |
1532 | } |
1533 | |
1534 | /** |
1535 | * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering |
1536 | * @i: int value |
1537 | * @v: pointer to atomic_t |
1538 | * |
1539 | * Atomically updates @v to (@v & ~@i) with relaxed ordering. |
1540 | * |
1541 | * Safe to use in noinstr code; prefer atomic_andnot() elsewhere. |
1542 | * |
1543 | * Return: Nothing. |
1544 | */ |
1545 | static __always_inline void |
1546 | raw_atomic_andnot(int i, atomic_t *v) |
1547 | { |
1548 | #if defined(arch_atomic_andnot) |
1549 | arch_atomic_andnot(i, v); |
1550 | #else |
1551 | raw_atomic_and(i: ~i, v); |
1552 | #endif |
1553 | } |
1554 | |
1555 | /** |
1556 | * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering |
1557 | * @i: int value |
1558 | * @v: pointer to atomic_t |
1559 | * |
1560 | * Atomically updates @v to (@v & ~@i) with full ordering. |
1561 | * |
1562 | * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere. |
1563 | * |
1564 | * Return: The original value of @v. |
1565 | */ |
1566 | static __always_inline int |
1567 | raw_atomic_fetch_andnot(int i, atomic_t *v) |
1568 | { |
1569 | #if defined(arch_atomic_fetch_andnot) |
1570 | return arch_atomic_fetch_andnot(i, v); |
1571 | #elif defined(arch_atomic_fetch_andnot_relaxed) |
1572 | int ret; |
1573 | __atomic_pre_full_fence(); |
1574 | ret = arch_atomic_fetch_andnot_relaxed(i, v); |
1575 | __atomic_post_full_fence(); |
1576 | return ret; |
1577 | #else |
1578 | return raw_atomic_fetch_and(i: ~i, v); |
1579 | #endif |
1580 | } |
1581 | |
1582 | /** |
1583 | * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering |
1584 | * @i: int value |
1585 | * @v: pointer to atomic_t |
1586 | * |
1587 | * Atomically updates @v to (@v & ~@i) with acquire ordering. |
1588 | * |
1589 | * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere. |
1590 | * |
1591 | * Return: The original value of @v. |
1592 | */ |
1593 | static __always_inline int |
1594 | raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) |
1595 | { |
1596 | #if defined(arch_atomic_fetch_andnot_acquire) |
1597 | return arch_atomic_fetch_andnot_acquire(i, v); |
1598 | #elif defined(arch_atomic_fetch_andnot_relaxed) |
1599 | int ret = arch_atomic_fetch_andnot_relaxed(i, v); |
1600 | __atomic_acquire_fence(); |
1601 | return ret; |
1602 | #elif defined(arch_atomic_fetch_andnot) |
1603 | return arch_atomic_fetch_andnot(i, v); |
1604 | #else |
1605 | return raw_atomic_fetch_and_acquire(i: ~i, v); |
1606 | #endif |
1607 | } |
1608 | |
1609 | /** |
1610 | * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering |
1611 | * @i: int value |
1612 | * @v: pointer to atomic_t |
1613 | * |
1614 | * Atomically updates @v to (@v & ~@i) with release ordering. |
1615 | * |
1616 | * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere. |
1617 | * |
1618 | * Return: The original value of @v. |
1619 | */ |
1620 | static __always_inline int |
1621 | raw_atomic_fetch_andnot_release(int i, atomic_t *v) |
1622 | { |
1623 | #if defined(arch_atomic_fetch_andnot_release) |
1624 | return arch_atomic_fetch_andnot_release(i, v); |
1625 | #elif defined(arch_atomic_fetch_andnot_relaxed) |
1626 | __atomic_release_fence(); |
1627 | return arch_atomic_fetch_andnot_relaxed(i, v); |
1628 | #elif defined(arch_atomic_fetch_andnot) |
1629 | return arch_atomic_fetch_andnot(i, v); |
1630 | #else |
1631 | return raw_atomic_fetch_and_release(i: ~i, v); |
1632 | #endif |
1633 | } |
1634 | |
1635 | /** |
1636 | * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering |
1637 | * @i: int value |
1638 | * @v: pointer to atomic_t |
1639 | * |
1640 | * Atomically updates @v to (@v & ~@i) with relaxed ordering. |
1641 | * |
1642 | * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere. |
1643 | * |
1644 | * Return: The original value of @v. |
1645 | */ |
1646 | static __always_inline int |
1647 | raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v) |
1648 | { |
1649 | #if defined(arch_atomic_fetch_andnot_relaxed) |
1650 | return arch_atomic_fetch_andnot_relaxed(i, v); |
1651 | #elif defined(arch_atomic_fetch_andnot) |
1652 | return arch_atomic_fetch_andnot(i, v); |
1653 | #else |
1654 | return raw_atomic_fetch_and_relaxed(i: ~i, v); |
1655 | #endif |
1656 | } |
1657 | |
1658 | /** |
1659 | * raw_atomic_or() - atomic bitwise OR with relaxed ordering |
1660 | * @i: int value |
1661 | * @v: pointer to atomic_t |
1662 | * |
1663 | * Atomically updates @v to (@v | @i) with relaxed ordering. |
1664 | * |
1665 | * Safe to use in noinstr code; prefer atomic_or() elsewhere. |
1666 | * |
1667 | * Return: Nothing. |
1668 | */ |
1669 | static __always_inline void |
1670 | raw_atomic_or(int i, atomic_t *v) |
1671 | { |
1672 | arch_atomic_or(i, v); |
1673 | } |
1674 | |
1675 | /** |
1676 | * raw_atomic_fetch_or() - atomic bitwise OR with full ordering |
1677 | * @i: int value |
1678 | * @v: pointer to atomic_t |
1679 | * |
1680 | * Atomically updates @v to (@v | @i) with full ordering. |
1681 | * |
1682 | * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere. |
1683 | * |
1684 | * Return: The original value of @v. |
1685 | */ |
1686 | static __always_inline int |
1687 | raw_atomic_fetch_or(int i, atomic_t *v) |
1688 | { |
1689 | #if defined(arch_atomic_fetch_or) |
1690 | return arch_atomic_fetch_or(i, v); |
1691 | #elif defined(arch_atomic_fetch_or_relaxed) |
1692 | int ret; |
1693 | __atomic_pre_full_fence(); |
1694 | ret = arch_atomic_fetch_or_relaxed(i, v); |
1695 | __atomic_post_full_fence(); |
1696 | return ret; |
1697 | #else |
1698 | #error "Unable to define raw_atomic_fetch_or" |
1699 | #endif |
1700 | } |
1701 | |
1702 | /** |
1703 | * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering |
1704 | * @i: int value |
1705 | * @v: pointer to atomic_t |
1706 | * |
1707 | * Atomically updates @v to (@v | @i) with acquire ordering. |
1708 | * |
1709 | * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere. |
1710 | * |
1711 | * Return: The original value of @v. |
1712 | */ |
1713 | static __always_inline int |
1714 | raw_atomic_fetch_or_acquire(int i, atomic_t *v) |
1715 | { |
1716 | #if defined(arch_atomic_fetch_or_acquire) |
1717 | return arch_atomic_fetch_or_acquire(i, v); |
1718 | #elif defined(arch_atomic_fetch_or_relaxed) |
1719 | int ret = arch_atomic_fetch_or_relaxed(i, v); |
1720 | __atomic_acquire_fence(); |
1721 | return ret; |
1722 | #elif defined(arch_atomic_fetch_or) |
1723 | return arch_atomic_fetch_or(i, v); |
1724 | #else |
1725 | #error "Unable to define raw_atomic_fetch_or_acquire" |
1726 | #endif |
1727 | } |
1728 | |
1729 | /** |
1730 | * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering |
1731 | * @i: int value |
1732 | * @v: pointer to atomic_t |
1733 | * |
1734 | * Atomically updates @v to (@v | @i) with release ordering. |
1735 | * |
1736 | * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere. |
1737 | * |
1738 | * Return: The original value of @v. |
1739 | */ |
1740 | static __always_inline int |
1741 | raw_atomic_fetch_or_release(int i, atomic_t *v) |
1742 | { |
1743 | #if defined(arch_atomic_fetch_or_release) |
1744 | return arch_atomic_fetch_or_release(i, v); |
1745 | #elif defined(arch_atomic_fetch_or_relaxed) |
1746 | __atomic_release_fence(); |
1747 | return arch_atomic_fetch_or_relaxed(i, v); |
1748 | #elif defined(arch_atomic_fetch_or) |
1749 | return arch_atomic_fetch_or(i, v); |
1750 | #else |
1751 | #error "Unable to define raw_atomic_fetch_or_release" |
1752 | #endif |
1753 | } |
1754 | |
1755 | /** |
1756 | * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering |
1757 | * @i: int value |
1758 | * @v: pointer to atomic_t |
1759 | * |
1760 | * Atomically updates @v to (@v | @i) with relaxed ordering. |
1761 | * |
1762 | * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere. |
1763 | * |
1764 | * Return: The original value of @v. |
1765 | */ |
1766 | static __always_inline int |
1767 | raw_atomic_fetch_or_relaxed(int i, atomic_t *v) |
1768 | { |
1769 | #if defined(arch_atomic_fetch_or_relaxed) |
1770 | return arch_atomic_fetch_or_relaxed(i, v); |
1771 | #elif defined(arch_atomic_fetch_or) |
1772 | return arch_atomic_fetch_or(i, v); |
1773 | #else |
1774 | #error "Unable to define raw_atomic_fetch_or_relaxed" |
1775 | #endif |
1776 | } |
1777 | |
1778 | /** |
1779 | * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering |
1780 | * @i: int value |
1781 | * @v: pointer to atomic_t |
1782 | * |
1783 | * Atomically updates @v to (@v ^ @i) with relaxed ordering. |
1784 | * |
1785 | * Safe to use in noinstr code; prefer atomic_xor() elsewhere. |
1786 | * |
1787 | * Return: Nothing. |
1788 | */ |
1789 | static __always_inline void |
1790 | raw_atomic_xor(int i, atomic_t *v) |
1791 | { |
1792 | arch_atomic_xor(i, v); |
1793 | } |
1794 | |
1795 | /** |
1796 | * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering |
1797 | * @i: int value |
1798 | * @v: pointer to atomic_t |
1799 | * |
1800 | * Atomically updates @v to (@v ^ @i) with full ordering. |
1801 | * |
1802 | * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere. |
1803 | * |
1804 | * Return: The original value of @v. |
1805 | */ |
1806 | static __always_inline int |
1807 | raw_atomic_fetch_xor(int i, atomic_t *v) |
1808 | { |
1809 | #if defined(arch_atomic_fetch_xor) |
1810 | return arch_atomic_fetch_xor(i, v); |
1811 | #elif defined(arch_atomic_fetch_xor_relaxed) |
1812 | int ret; |
1813 | __atomic_pre_full_fence(); |
1814 | ret = arch_atomic_fetch_xor_relaxed(i, v); |
1815 | __atomic_post_full_fence(); |
1816 | return ret; |
1817 | #else |
1818 | #error "Unable to define raw_atomic_fetch_xor" |
1819 | #endif |
1820 | } |
1821 | |
1822 | /** |
1823 | * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering |
1824 | * @i: int value |
1825 | * @v: pointer to atomic_t |
1826 | * |
1827 | * Atomically updates @v to (@v ^ @i) with acquire ordering. |
1828 | * |
1829 | * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere. |
1830 | * |
1831 | * Return: The original value of @v. |
1832 | */ |
1833 | static __always_inline int |
1834 | raw_atomic_fetch_xor_acquire(int i, atomic_t *v) |
1835 | { |
1836 | #if defined(arch_atomic_fetch_xor_acquire) |
1837 | return arch_atomic_fetch_xor_acquire(i, v); |
1838 | #elif defined(arch_atomic_fetch_xor_relaxed) |
1839 | int ret = arch_atomic_fetch_xor_relaxed(i, v); |
1840 | __atomic_acquire_fence(); |
1841 | return ret; |
1842 | #elif defined(arch_atomic_fetch_xor) |
1843 | return arch_atomic_fetch_xor(i, v); |
1844 | #else |
1845 | #error "Unable to define raw_atomic_fetch_xor_acquire" |
1846 | #endif |
1847 | } |
1848 | |
1849 | /** |
1850 | * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering |
1851 | * @i: int value |
1852 | * @v: pointer to atomic_t |
1853 | * |
1854 | * Atomically updates @v to (@v ^ @i) with release ordering. |
1855 | * |
1856 | * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere. |
1857 | * |
1858 | * Return: The original value of @v. |
1859 | */ |
1860 | static __always_inline int |
1861 | raw_atomic_fetch_xor_release(int i, atomic_t *v) |
1862 | { |
1863 | #if defined(arch_atomic_fetch_xor_release) |
1864 | return arch_atomic_fetch_xor_release(i, v); |
1865 | #elif defined(arch_atomic_fetch_xor_relaxed) |
1866 | __atomic_release_fence(); |
1867 | return arch_atomic_fetch_xor_relaxed(i, v); |
1868 | #elif defined(arch_atomic_fetch_xor) |
1869 | return arch_atomic_fetch_xor(i, v); |
1870 | #else |
1871 | #error "Unable to define raw_atomic_fetch_xor_release" |
1872 | #endif |
1873 | } |
1874 | |
1875 | /** |
1876 | * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering |
1877 | * @i: int value |
1878 | * @v: pointer to atomic_t |
1879 | * |
1880 | * Atomically updates @v to (@v ^ @i) with relaxed ordering. |
1881 | * |
1882 | * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere. |
1883 | * |
1884 | * Return: The original value of @v. |
1885 | */ |
1886 | static __always_inline int |
1887 | raw_atomic_fetch_xor_relaxed(int i, atomic_t *v) |
1888 | { |
1889 | #if defined(arch_atomic_fetch_xor_relaxed) |
1890 | return arch_atomic_fetch_xor_relaxed(i, v); |
1891 | #elif defined(arch_atomic_fetch_xor) |
1892 | return arch_atomic_fetch_xor(i, v); |
1893 | #else |
1894 | #error "Unable to define raw_atomic_fetch_xor_relaxed" |
1895 | #endif |
1896 | } |
1897 | |
1898 | /** |
1899 | * raw_atomic_xchg() - atomic exchange with full ordering |
1900 | * @v: pointer to atomic_t |
1901 | * @new: int value to assign |
1902 | * |
1903 | * Atomically updates @v to @new with full ordering. |
1904 | * |
1905 | * Safe to use in noinstr code; prefer atomic_xchg() elsewhere. |
1906 | * |
1907 | * Return: The original value of @v. |
1908 | */ |
1909 | static __always_inline int |
1910 | raw_atomic_xchg(atomic_t *v, int new) |
1911 | { |
1912 | #if defined(arch_atomic_xchg) |
1913 | return arch_atomic_xchg(v, new); |
1914 | #elif defined(arch_atomic_xchg_relaxed) |
1915 | int ret; |
1916 | __atomic_pre_full_fence(); |
1917 | ret = arch_atomic_xchg_relaxed(v, new); |
1918 | __atomic_post_full_fence(); |
1919 | return ret; |
1920 | #else |
1921 | return raw_xchg(&v->counter, new); |
1922 | #endif |
1923 | } |
1924 | |
1925 | /** |
1926 | * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering |
1927 | * @v: pointer to atomic_t |
1928 | * @new: int value to assign |
1929 | * |
1930 | * Atomically updates @v to @new with acquire ordering. |
1931 | * |
1932 | * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere. |
1933 | * |
1934 | * Return: The original value of @v. |
1935 | */ |
1936 | static __always_inline int |
1937 | raw_atomic_xchg_acquire(atomic_t *v, int new) |
1938 | { |
1939 | #if defined(arch_atomic_xchg_acquire) |
1940 | return arch_atomic_xchg_acquire(v, new); |
1941 | #elif defined(arch_atomic_xchg_relaxed) |
1942 | int ret = arch_atomic_xchg_relaxed(v, new); |
1943 | __atomic_acquire_fence(); |
1944 | return ret; |
1945 | #elif defined(arch_atomic_xchg) |
1946 | return arch_atomic_xchg(v, new); |
1947 | #else |
1948 | return raw_xchg_acquire(&v->counter, new); |
1949 | #endif |
1950 | } |
1951 | |
1952 | /** |
1953 | * raw_atomic_xchg_release() - atomic exchange with release ordering |
1954 | * @v: pointer to atomic_t |
1955 | * @new: int value to assign |
1956 | * |
1957 | * Atomically updates @v to @new with release ordering. |
1958 | * |
1959 | * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere. |
1960 | * |
1961 | * Return: The original value of @v. |
1962 | */ |
1963 | static __always_inline int |
1964 | raw_atomic_xchg_release(atomic_t *v, int new) |
1965 | { |
1966 | #if defined(arch_atomic_xchg_release) |
1967 | return arch_atomic_xchg_release(v, new); |
1968 | #elif defined(arch_atomic_xchg_relaxed) |
1969 | __atomic_release_fence(); |
1970 | return arch_atomic_xchg_relaxed(v, new); |
1971 | #elif defined(arch_atomic_xchg) |
1972 | return arch_atomic_xchg(v, new); |
1973 | #else |
1974 | return raw_xchg_release(&v->counter, new); |
1975 | #endif |
1976 | } |
1977 | |
1978 | /** |
1979 | * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering |
1980 | * @v: pointer to atomic_t |
1981 | * @new: int value to assign |
1982 | * |
1983 | * Atomically updates @v to @new with relaxed ordering. |
1984 | * |
1985 | * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere. |
1986 | * |
1987 | * Return: The original value of @v. |
1988 | */ |
1989 | static __always_inline int |
1990 | raw_atomic_xchg_relaxed(atomic_t *v, int new) |
1991 | { |
1992 | #if defined(arch_atomic_xchg_relaxed) |
1993 | return arch_atomic_xchg_relaxed(v, new); |
1994 | #elif defined(arch_atomic_xchg) |
1995 | return arch_atomic_xchg(v, new); |
1996 | #else |
1997 | return raw_xchg_relaxed(&v->counter, new); |
1998 | #endif |
1999 | } |
2000 | |
2001 | /** |
2002 | * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering |
2003 | * @v: pointer to atomic_t |
2004 | * @old: int value to compare with |
2005 | * @new: int value to assign |
2006 | * |
2007 | * If (@v == @old), atomically updates @v to @new with full ordering. |
2008 | * |
2009 | * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere. |
2010 | * |
2011 | * Return: The original value of @v. |
2012 | */ |
2013 | static __always_inline int |
2014 | raw_atomic_cmpxchg(atomic_t *v, int old, int new) |
2015 | { |
2016 | #if defined(arch_atomic_cmpxchg) |
2017 | return arch_atomic_cmpxchg(v, old, new); |
2018 | #elif defined(arch_atomic_cmpxchg_relaxed) |
2019 | int ret; |
2020 | __atomic_pre_full_fence(); |
2021 | ret = arch_atomic_cmpxchg_relaxed(v, old, new); |
2022 | __atomic_post_full_fence(); |
2023 | return ret; |
2024 | #else |
2025 | return raw_cmpxchg(&v->counter, old, new); |
2026 | #endif |
2027 | } |
2028 | |
2029 | /** |
2030 | * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering |
2031 | * @v: pointer to atomic_t |
2032 | * @old: int value to compare with |
2033 | * @new: int value to assign |
2034 | * |
2035 | * If (@v == @old), atomically updates @v to @new with acquire ordering. |
2036 | * |
2037 | * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere. |
2038 | * |
2039 | * Return: The original value of @v. |
2040 | */ |
2041 | static __always_inline int |
2042 | raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) |
2043 | { |
2044 | #if defined(arch_atomic_cmpxchg_acquire) |
2045 | return arch_atomic_cmpxchg_acquire(v, old, new); |
2046 | #elif defined(arch_atomic_cmpxchg_relaxed) |
2047 | int ret = arch_atomic_cmpxchg_relaxed(v, old, new); |
2048 | __atomic_acquire_fence(); |
2049 | return ret; |
2050 | #elif defined(arch_atomic_cmpxchg) |
2051 | return arch_atomic_cmpxchg(v, old, new); |
2052 | #else |
2053 | return raw_cmpxchg_acquire(&v->counter, old, new); |
2054 | #endif |
2055 | } |
2056 | |
2057 | /** |
2058 | * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering |
2059 | * @v: pointer to atomic_t |
2060 | * @old: int value to compare with |
2061 | * @new: int value to assign |
2062 | * |
2063 | * If (@v == @old), atomically updates @v to @new with release ordering. |
2064 | * |
2065 | * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere. |
2066 | * |
2067 | * Return: The original value of @v. |
2068 | */ |
2069 | static __always_inline int |
2070 | raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) |
2071 | { |
2072 | #if defined(arch_atomic_cmpxchg_release) |
2073 | return arch_atomic_cmpxchg_release(v, old, new); |
2074 | #elif defined(arch_atomic_cmpxchg_relaxed) |
2075 | __atomic_release_fence(); |
2076 | return arch_atomic_cmpxchg_relaxed(v, old, new); |
2077 | #elif defined(arch_atomic_cmpxchg) |
2078 | return arch_atomic_cmpxchg(v, old, new); |
2079 | #else |
2080 | return raw_cmpxchg_release(&v->counter, old, new); |
2081 | #endif |
2082 | } |
2083 | |
2084 | /** |
2085 | * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering |
2086 | * @v: pointer to atomic_t |
2087 | * @old: int value to compare with |
2088 | * @new: int value to assign |
2089 | * |
2090 | * If (@v == @old), atomically updates @v to @new with relaxed ordering. |
2091 | * |
2092 | * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere. |
2093 | * |
2094 | * Return: The original value of @v. |
2095 | */ |
2096 | static __always_inline int |
2097 | raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) |
2098 | { |
2099 | #if defined(arch_atomic_cmpxchg_relaxed) |
2100 | return arch_atomic_cmpxchg_relaxed(v, old, new); |
2101 | #elif defined(arch_atomic_cmpxchg) |
2102 | return arch_atomic_cmpxchg(v, old, new); |
2103 | #else |
2104 | return raw_cmpxchg_relaxed(&v->counter, old, new); |
2105 | #endif |
2106 | } |
2107 | |
2108 | /** |
2109 | * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering |
2110 | * @v: pointer to atomic_t |
2111 | * @old: pointer to int value to compare with |
2112 | * @new: int value to assign |
2113 | * |
2114 | * If (@v == @old), atomically updates @v to @new with full ordering. |
2115 | * Otherwise, updates @old to the current value of @v. |
2116 | * |
2117 | * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere. |
2118 | * |
2119 | * Return: @true if the exchange occured, @false otherwise. |
2120 | */ |
2121 | static __always_inline bool |
2122 | raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) |
2123 | { |
2124 | #if defined(arch_atomic_try_cmpxchg) |
2125 | return arch_atomic_try_cmpxchg(v, old, new); |
2126 | #elif defined(arch_atomic_try_cmpxchg_relaxed) |
2127 | bool ret; |
2128 | __atomic_pre_full_fence(); |
2129 | ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); |
2130 | __atomic_post_full_fence(); |
2131 | return ret; |
2132 | #else |
2133 | int r, o = *old; |
2134 | r = raw_atomic_cmpxchg(v, o, new); |
2135 | if (unlikely(r != o)) |
2136 | *old = r; |
2137 | return likely(r == o); |
2138 | #endif |
2139 | } |
2140 | |
2141 | /** |
2142 | * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering |
2143 | * @v: pointer to atomic_t |
2144 | * @old: pointer to int value to compare with |
2145 | * @new: int value to assign |
2146 | * |
2147 | * If (@v == @old), atomically updates @v to @new with acquire ordering. |
2148 | * Otherwise, updates @old to the current value of @v. |
2149 | * |
2150 | * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere. |
2151 | * |
2152 | * Return: @true if the exchange occured, @false otherwise. |
2153 | */ |
2154 | static __always_inline bool |
2155 | raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) |
2156 | { |
2157 | #if defined(arch_atomic_try_cmpxchg_acquire) |
2158 | return arch_atomic_try_cmpxchg_acquire(v, old, new); |
2159 | #elif defined(arch_atomic_try_cmpxchg_relaxed) |
2160 | bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); |
2161 | __atomic_acquire_fence(); |
2162 | return ret; |
2163 | #elif defined(arch_atomic_try_cmpxchg) |
2164 | return arch_atomic_try_cmpxchg(v, old, new); |
2165 | #else |
2166 | int r, o = *old; |
2167 | r = raw_atomic_cmpxchg_acquire(v, o, new); |
2168 | if (unlikely(r != o)) |
2169 | *old = r; |
2170 | return likely(r == o); |
2171 | #endif |
2172 | } |
2173 | |
2174 | /** |
2175 | * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering |
2176 | * @v: pointer to atomic_t |
2177 | * @old: pointer to int value to compare with |
2178 | * @new: int value to assign |
2179 | * |
2180 | * If (@v == @old), atomically updates @v to @new with release ordering. |
2181 | * Otherwise, updates @old to the current value of @v. |
2182 | * |
2183 | * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere. |
2184 | * |
2185 | * Return: @true if the exchange occured, @false otherwise. |
2186 | */ |
2187 | static __always_inline bool |
2188 | raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) |
2189 | { |
2190 | #if defined(arch_atomic_try_cmpxchg_release) |
2191 | return arch_atomic_try_cmpxchg_release(v, old, new); |
2192 | #elif defined(arch_atomic_try_cmpxchg_relaxed) |
2193 | __atomic_release_fence(); |
2194 | return arch_atomic_try_cmpxchg_relaxed(v, old, new); |
2195 | #elif defined(arch_atomic_try_cmpxchg) |
2196 | return arch_atomic_try_cmpxchg(v, old, new); |
2197 | #else |
2198 | int r, o = *old; |
2199 | r = raw_atomic_cmpxchg_release(v, o, new); |
2200 | if (unlikely(r != o)) |
2201 | *old = r; |
2202 | return likely(r == o); |
2203 | #endif |
2204 | } |
2205 | |
2206 | /** |
2207 | * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering |
2208 | * @v: pointer to atomic_t |
2209 | * @old: pointer to int value to compare with |
2210 | * @new: int value to assign |
2211 | * |
2212 | * If (@v == @old), atomically updates @v to @new with relaxed ordering. |
2213 | * Otherwise, updates @old to the current value of @v. |
2214 | * |
2215 | * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere. |
2216 | * |
2217 | * Return: @true if the exchange occured, @false otherwise. |
2218 | */ |
2219 | static __always_inline bool |
2220 | raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) |
2221 | { |
2222 | #if defined(arch_atomic_try_cmpxchg_relaxed) |
2223 | return arch_atomic_try_cmpxchg_relaxed(v, old, new); |
2224 | #elif defined(arch_atomic_try_cmpxchg) |
2225 | return arch_atomic_try_cmpxchg(v, old, new); |
2226 | #else |
2227 | int r, o = *old; |
2228 | r = raw_atomic_cmpxchg_relaxed(v, o, new); |
2229 | if (unlikely(r != o)) |
2230 | *old = r; |
2231 | return likely(r == o); |
2232 | #endif |
2233 | } |
2234 | |
2235 | /** |
2236 | * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering |
2237 | * @i: int value to add |
2238 | * @v: pointer to atomic_t |
2239 | * |
2240 | * Atomically updates @v to (@v - @i) with full ordering. |
2241 | * |
2242 | * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere. |
2243 | * |
2244 | * Return: @true if the resulting value of @v is zero, @false otherwise. |
2245 | */ |
2246 | static __always_inline bool |
2247 | raw_atomic_sub_and_test(int i, atomic_t *v) |
2248 | { |
2249 | #if defined(arch_atomic_sub_and_test) |
2250 | return arch_atomic_sub_and_test(i, v); |
2251 | #else |
2252 | return raw_atomic_sub_return(i, v) == 0; |
2253 | #endif |
2254 | } |
2255 | |
2256 | /** |
2257 | * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering |
2258 | * @v: pointer to atomic_t |
2259 | * |
2260 | * Atomically updates @v to (@v - 1) with full ordering. |
2261 | * |
2262 | * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere. |
2263 | * |
2264 | * Return: @true if the resulting value of @v is zero, @false otherwise. |
2265 | */ |
2266 | static __always_inline bool |
2267 | raw_atomic_dec_and_test(atomic_t *v) |
2268 | { |
2269 | #if defined(arch_atomic_dec_and_test) |
2270 | return arch_atomic_dec_and_test(v); |
2271 | #else |
2272 | return raw_atomic_dec_return(v) == 0; |
2273 | #endif |
2274 | } |
2275 | |
2276 | /** |
2277 | * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering |
2278 | * @v: pointer to atomic_t |
2279 | * |
2280 | * Atomically updates @v to (@v + 1) with full ordering. |
2281 | * |
2282 | * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere. |
2283 | * |
2284 | * Return: @true if the resulting value of @v is zero, @false otherwise. |
2285 | */ |
2286 | static __always_inline bool |
2287 | raw_atomic_inc_and_test(atomic_t *v) |
2288 | { |
2289 | #if defined(arch_atomic_inc_and_test) |
2290 | return arch_atomic_inc_and_test(v); |
2291 | #else |
2292 | return raw_atomic_inc_return(v) == 0; |
2293 | #endif |
2294 | } |
2295 | |
2296 | /** |
2297 | * raw_atomic_add_negative() - atomic add and test if negative with full ordering |
2298 | * @i: int value to add |
2299 | * @v: pointer to atomic_t |
2300 | * |
2301 | * Atomically updates @v to (@v + @i) with full ordering. |
2302 | * |
2303 | * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere. |
2304 | * |
2305 | * Return: @true if the resulting value of @v is negative, @false otherwise. |
2306 | */ |
2307 | static __always_inline bool |
2308 | raw_atomic_add_negative(int i, atomic_t *v) |
2309 | { |
2310 | #if defined(arch_atomic_add_negative) |
2311 | return arch_atomic_add_negative(i, v); |
2312 | #elif defined(arch_atomic_add_negative_relaxed) |
2313 | bool ret; |
2314 | __atomic_pre_full_fence(); |
2315 | ret = arch_atomic_add_negative_relaxed(i, v); |
2316 | __atomic_post_full_fence(); |
2317 | return ret; |
2318 | #else |
2319 | return raw_atomic_add_return(i, v) < 0; |
2320 | #endif |
2321 | } |
2322 | |
2323 | /** |
2324 | * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering |
2325 | * @i: int value to add |
2326 | * @v: pointer to atomic_t |
2327 | * |
2328 | * Atomically updates @v to (@v + @i) with acquire ordering. |
2329 | * |
2330 | * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere. |
2331 | * |
2332 | * Return: @true if the resulting value of @v is negative, @false otherwise. |
2333 | */ |
2334 | static __always_inline bool |
2335 | raw_atomic_add_negative_acquire(int i, atomic_t *v) |
2336 | { |
2337 | #if defined(arch_atomic_add_negative_acquire) |
2338 | return arch_atomic_add_negative_acquire(i, v); |
2339 | #elif defined(arch_atomic_add_negative_relaxed) |
2340 | bool ret = arch_atomic_add_negative_relaxed(i, v); |
2341 | __atomic_acquire_fence(); |
2342 | return ret; |
2343 | #elif defined(arch_atomic_add_negative) |
2344 | return arch_atomic_add_negative(i, v); |
2345 | #else |
2346 | return raw_atomic_add_return_acquire(i, v) < 0; |
2347 | #endif |
2348 | } |
2349 | |
2350 | /** |
2351 | * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering |
2352 | * @i: int value to add |
2353 | * @v: pointer to atomic_t |
2354 | * |
2355 | * Atomically updates @v to (@v + @i) with release ordering. |
2356 | * |
2357 | * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere. |
2358 | * |
2359 | * Return: @true if the resulting value of @v is negative, @false otherwise. |
2360 | */ |
2361 | static __always_inline bool |
2362 | raw_atomic_add_negative_release(int i, atomic_t *v) |
2363 | { |
2364 | #if defined(arch_atomic_add_negative_release) |
2365 | return arch_atomic_add_negative_release(i, v); |
2366 | #elif defined(arch_atomic_add_negative_relaxed) |
2367 | __atomic_release_fence(); |
2368 | return arch_atomic_add_negative_relaxed(i, v); |
2369 | #elif defined(arch_atomic_add_negative) |
2370 | return arch_atomic_add_negative(i, v); |
2371 | #else |
2372 | return raw_atomic_add_return_release(i, v) < 0; |
2373 | #endif |
2374 | } |
2375 | |
2376 | /** |
2377 | * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering |
2378 | * @i: int value to add |
2379 | * @v: pointer to atomic_t |
2380 | * |
2381 | * Atomically updates @v to (@v + @i) with relaxed ordering. |
2382 | * |
2383 | * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere. |
2384 | * |
2385 | * Return: @true if the resulting value of @v is negative, @false otherwise. |
2386 | */ |
2387 | static __always_inline bool |
2388 | raw_atomic_add_negative_relaxed(int i, atomic_t *v) |
2389 | { |
2390 | #if defined(arch_atomic_add_negative_relaxed) |
2391 | return arch_atomic_add_negative_relaxed(i, v); |
2392 | #elif defined(arch_atomic_add_negative) |
2393 | return arch_atomic_add_negative(i, v); |
2394 | #else |
2395 | return raw_atomic_add_return_relaxed(i, v) < 0; |
2396 | #endif |
2397 | } |
2398 | |
2399 | /** |
2400 | * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering |
2401 | * @v: pointer to atomic_t |
2402 | * @a: int value to add |
2403 | * @u: int value to compare with |
2404 | * |
2405 | * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. |
2406 | * |
2407 | * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere. |
2408 | * |
2409 | * Return: The original value of @v. |
2410 | */ |
2411 | static __always_inline int |
2412 | raw_atomic_fetch_add_unless(atomic_t *v, int a, int u) |
2413 | { |
2414 | #if defined(arch_atomic_fetch_add_unless) |
2415 | return arch_atomic_fetch_add_unless(v, a, u); |
2416 | #else |
2417 | int c = raw_atomic_read(v); |
2418 | |
2419 | do { |
2420 | if (unlikely(c == u)) |
2421 | break; |
2422 | } while (!raw_atomic_try_cmpxchg(v, old: &c, new: c + a)); |
2423 | |
2424 | return c; |
2425 | #endif |
2426 | } |
2427 | |
2428 | /** |
2429 | * raw_atomic_add_unless() - atomic add unless value with full ordering |
2430 | * @v: pointer to atomic_t |
2431 | * @a: int value to add |
2432 | * @u: int value to compare with |
2433 | * |
2434 | * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. |
2435 | * |
2436 | * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere. |
2437 | * |
2438 | * Return: @true if @v was updated, @false otherwise. |
2439 | */ |
2440 | static __always_inline bool |
2441 | raw_atomic_add_unless(atomic_t *v, int a, int u) |
2442 | { |
2443 | #if defined(arch_atomic_add_unless) |
2444 | return arch_atomic_add_unless(v, a, u); |
2445 | #else |
2446 | return raw_atomic_fetch_add_unless(v, a, u) != u; |
2447 | #endif |
2448 | } |
2449 | |
2450 | /** |
2451 | * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering |
2452 | * @v: pointer to atomic_t |
2453 | * |
2454 | * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. |
2455 | * |
2456 | * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere. |
2457 | * |
2458 | * Return: @true if @v was updated, @false otherwise. |
2459 | */ |
2460 | static __always_inline bool |
2461 | raw_atomic_inc_not_zero(atomic_t *v) |
2462 | { |
2463 | #if defined(arch_atomic_inc_not_zero) |
2464 | return arch_atomic_inc_not_zero(v); |
2465 | #else |
2466 | return raw_atomic_add_unless(v, a: 1, u: 0); |
2467 | #endif |
2468 | } |
2469 | |
2470 | /** |
2471 | * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering |
2472 | * @v: pointer to atomic_t |
2473 | * |
2474 | * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. |
2475 | * |
2476 | * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere. |
2477 | * |
2478 | * Return: @true if @v was updated, @false otherwise. |
2479 | */ |
2480 | static __always_inline bool |
2481 | raw_atomic_inc_unless_negative(atomic_t *v) |
2482 | { |
2483 | #if defined(arch_atomic_inc_unless_negative) |
2484 | return arch_atomic_inc_unless_negative(v); |
2485 | #else |
2486 | int c = raw_atomic_read(v); |
2487 | |
2488 | do { |
2489 | if (unlikely(c < 0)) |
2490 | return false; |
2491 | } while (!raw_atomic_try_cmpxchg(v, old: &c, new: c + 1)); |
2492 | |
2493 | return true; |
2494 | #endif |
2495 | } |
2496 | |
2497 | /** |
2498 | * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering |
2499 | * @v: pointer to atomic_t |
2500 | * |
2501 | * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. |
2502 | * |
2503 | * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere. |
2504 | * |
2505 | * Return: @true if @v was updated, @false otherwise. |
2506 | */ |
2507 | static __always_inline bool |
2508 | raw_atomic_dec_unless_positive(atomic_t *v) |
2509 | { |
2510 | #if defined(arch_atomic_dec_unless_positive) |
2511 | return arch_atomic_dec_unless_positive(v); |
2512 | #else |
2513 | int c = raw_atomic_read(v); |
2514 | |
2515 | do { |
2516 | if (unlikely(c > 0)) |
2517 | return false; |
2518 | } while (!raw_atomic_try_cmpxchg(v, old: &c, new: c - 1)); |
2519 | |
2520 | return true; |
2521 | #endif |
2522 | } |
2523 | |
2524 | /** |
2525 | * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering |
2526 | * @v: pointer to atomic_t |
2527 | * |
2528 | * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. |
2529 | * |
2530 | * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere. |
2531 | * |
2532 | * Return: The old value of (@v - 1), regardless of whether @v was updated. |
2533 | */ |
2534 | static __always_inline int |
2535 | raw_atomic_dec_if_positive(atomic_t *v) |
2536 | { |
2537 | #if defined(arch_atomic_dec_if_positive) |
2538 | return arch_atomic_dec_if_positive(v); |
2539 | #else |
2540 | int dec, c = raw_atomic_read(v); |
2541 | |
2542 | do { |
2543 | dec = c - 1; |
2544 | if (unlikely(dec < 0)) |
2545 | break; |
2546 | } while (!raw_atomic_try_cmpxchg(v, old: &c, new: dec)); |
2547 | |
2548 | return dec; |
2549 | #endif |
2550 | } |
2551 | |
2552 | #ifdef CONFIG_GENERIC_ATOMIC64 |
2553 | #include <asm-generic/atomic64.h> |
2554 | #endif |
2555 | |
2556 | /** |
2557 | * raw_atomic64_read() - atomic load with relaxed ordering |
2558 | * @v: pointer to atomic64_t |
2559 | * |
2560 | * Atomically loads the value of @v with relaxed ordering. |
2561 | * |
2562 | * Safe to use in noinstr code; prefer atomic64_read() elsewhere. |
2563 | * |
2564 | * Return: The value loaded from @v. |
2565 | */ |
2566 | static __always_inline s64 |
2567 | raw_atomic64_read(const atomic64_t *v) |
2568 | { |
2569 | return arch_atomic64_read(v); |
2570 | } |
2571 | |
2572 | /** |
2573 | * raw_atomic64_read_acquire() - atomic load with acquire ordering |
2574 | * @v: pointer to atomic64_t |
2575 | * |
2576 | * Atomically loads the value of @v with acquire ordering. |
2577 | * |
2578 | * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere. |
2579 | * |
2580 | * Return: The value loaded from @v. |
2581 | */ |
2582 | static __always_inline s64 |
2583 | raw_atomic64_read_acquire(const atomic64_t *v) |
2584 | { |
2585 | #if defined(arch_atomic64_read_acquire) |
2586 | return arch_atomic64_read_acquire(v); |
2587 | #else |
2588 | s64 ret; |
2589 | |
2590 | if (__native_word(atomic64_t)) { |
2591 | ret = smp_load_acquire(&(v)->counter); |
2592 | } else { |
2593 | ret = raw_atomic64_read(v); |
2594 | __atomic_acquire_fence(); |
2595 | } |
2596 | |
2597 | return ret; |
2598 | #endif |
2599 | } |
2600 | |
2601 | /** |
2602 | * raw_atomic64_set() - atomic set with relaxed ordering |
2603 | * @v: pointer to atomic64_t |
2604 | * @i: s64 value to assign |
2605 | * |
2606 | * Atomically sets @v to @i with relaxed ordering. |
2607 | * |
2608 | * Safe to use in noinstr code; prefer atomic64_set() elsewhere. |
2609 | * |
2610 | * Return: Nothing. |
2611 | */ |
2612 | static __always_inline void |
2613 | raw_atomic64_set(atomic64_t *v, s64 i) |
2614 | { |
2615 | arch_atomic64_set(v, i); |
2616 | } |
2617 | |
2618 | /** |
2619 | * raw_atomic64_set_release() - atomic set with release ordering |
2620 | * @v: pointer to atomic64_t |
2621 | * @i: s64 value to assign |
2622 | * |
2623 | * Atomically sets @v to @i with release ordering. |
2624 | * |
2625 | * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere. |
2626 | * |
2627 | * Return: Nothing. |
2628 | */ |
2629 | static __always_inline void |
2630 | raw_atomic64_set_release(atomic64_t *v, s64 i) |
2631 | { |
2632 | #if defined(arch_atomic64_set_release) |
2633 | arch_atomic64_set_release(v, i); |
2634 | #else |
2635 | if (__native_word(atomic64_t)) { |
2636 | smp_store_release(&(v)->counter, i); |
2637 | } else { |
2638 | __atomic_release_fence(); |
2639 | raw_atomic64_set(v, i); |
2640 | } |
2641 | #endif |
2642 | } |
2643 | |
2644 | /** |
2645 | * raw_atomic64_add() - atomic add with relaxed ordering |
2646 | * @i: s64 value to add |
2647 | * @v: pointer to atomic64_t |
2648 | * |
2649 | * Atomically updates @v to (@v + @i) with relaxed ordering. |
2650 | * |
2651 | * Safe to use in noinstr code; prefer atomic64_add() elsewhere. |
2652 | * |
2653 | * Return: Nothing. |
2654 | */ |
2655 | static __always_inline void |
2656 | raw_atomic64_add(s64 i, atomic64_t *v) |
2657 | { |
2658 | arch_atomic64_add(i, v); |
2659 | } |
2660 | |
2661 | /** |
2662 | * raw_atomic64_add_return() - atomic add with full ordering |
2663 | * @i: s64 value to add |
2664 | * @v: pointer to atomic64_t |
2665 | * |
2666 | * Atomically updates @v to (@v + @i) with full ordering. |
2667 | * |
2668 | * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere. |
2669 | * |
2670 | * Return: The updated value of @v. |
2671 | */ |
2672 | static __always_inline s64 |
2673 | raw_atomic64_add_return(s64 i, atomic64_t *v) |
2674 | { |
2675 | #if defined(arch_atomic64_add_return) |
2676 | return arch_atomic64_add_return(i, v); |
2677 | #elif defined(arch_atomic64_add_return_relaxed) |
2678 | s64 ret; |
2679 | __atomic_pre_full_fence(); |
2680 | ret = arch_atomic64_add_return_relaxed(i, v); |
2681 | __atomic_post_full_fence(); |
2682 | return ret; |
2683 | #else |
2684 | #error "Unable to define raw_atomic64_add_return" |
2685 | #endif |
2686 | } |
2687 | |
2688 | /** |
2689 | * raw_atomic64_add_return_acquire() - atomic add with acquire ordering |
2690 | * @i: s64 value to add |
2691 | * @v: pointer to atomic64_t |
2692 | * |
2693 | * Atomically updates @v to (@v + @i) with acquire ordering. |
2694 | * |
2695 | * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere. |
2696 | * |
2697 | * Return: The updated value of @v. |
2698 | */ |
2699 | static __always_inline s64 |
2700 | raw_atomic64_add_return_acquire(s64 i, atomic64_t *v) |
2701 | { |
2702 | #if defined(arch_atomic64_add_return_acquire) |
2703 | return arch_atomic64_add_return_acquire(i, v); |
2704 | #elif defined(arch_atomic64_add_return_relaxed) |
2705 | s64 ret = arch_atomic64_add_return_relaxed(i, v); |
2706 | __atomic_acquire_fence(); |
2707 | return ret; |
2708 | #elif defined(arch_atomic64_add_return) |
2709 | return arch_atomic64_add_return(i, v); |
2710 | #else |
2711 | #error "Unable to define raw_atomic64_add_return_acquire" |
2712 | #endif |
2713 | } |
2714 | |
2715 | /** |
2716 | * raw_atomic64_add_return_release() - atomic add with release ordering |
2717 | * @i: s64 value to add |
2718 | * @v: pointer to atomic64_t |
2719 | * |
2720 | * Atomically updates @v to (@v + @i) with release ordering. |
2721 | * |
2722 | * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere. |
2723 | * |
2724 | * Return: The updated value of @v. |
2725 | */ |
2726 | static __always_inline s64 |
2727 | raw_atomic64_add_return_release(s64 i, atomic64_t *v) |
2728 | { |
2729 | #if defined(arch_atomic64_add_return_release) |
2730 | return arch_atomic64_add_return_release(i, v); |
2731 | #elif defined(arch_atomic64_add_return_relaxed) |
2732 | __atomic_release_fence(); |
2733 | return arch_atomic64_add_return_relaxed(i, v); |
2734 | #elif defined(arch_atomic64_add_return) |
2735 | return arch_atomic64_add_return(i, v); |
2736 | #else |
2737 | #error "Unable to define raw_atomic64_add_return_release" |
2738 | #endif |
2739 | } |
2740 | |
2741 | /** |
2742 | * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering |
2743 | * @i: s64 value to add |
2744 | * @v: pointer to atomic64_t |
2745 | * |
2746 | * Atomically updates @v to (@v + @i) with relaxed ordering. |
2747 | * |
2748 | * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere. |
2749 | * |
2750 | * Return: The updated value of @v. |
2751 | */ |
2752 | static __always_inline s64 |
2753 | raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v) |
2754 | { |
2755 | #if defined(arch_atomic64_add_return_relaxed) |
2756 | return arch_atomic64_add_return_relaxed(i, v); |
2757 | #elif defined(arch_atomic64_add_return) |
2758 | return arch_atomic64_add_return(i, v); |
2759 | #else |
2760 | #error "Unable to define raw_atomic64_add_return_relaxed" |
2761 | #endif |
2762 | } |
2763 | |
2764 | /** |
2765 | * raw_atomic64_fetch_add() - atomic add with full ordering |
2766 | * @i: s64 value to add |
2767 | * @v: pointer to atomic64_t |
2768 | * |
2769 | * Atomically updates @v to (@v + @i) with full ordering. |
2770 | * |
2771 | * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere. |
2772 | * |
2773 | * Return: The original value of @v. |
2774 | */ |
2775 | static __always_inline s64 |
2776 | raw_atomic64_fetch_add(s64 i, atomic64_t *v) |
2777 | { |
2778 | #if defined(arch_atomic64_fetch_add) |
2779 | return arch_atomic64_fetch_add(i, v); |
2780 | #elif defined(arch_atomic64_fetch_add_relaxed) |
2781 | s64 ret; |
2782 | __atomic_pre_full_fence(); |
2783 | ret = arch_atomic64_fetch_add_relaxed(i, v); |
2784 | __atomic_post_full_fence(); |
2785 | return ret; |
2786 | #else |
2787 | #error "Unable to define raw_atomic64_fetch_add" |
2788 | #endif |
2789 | } |
2790 | |
2791 | /** |
2792 | * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering |
2793 | * @i: s64 value to add |
2794 | * @v: pointer to atomic64_t |
2795 | * |
2796 | * Atomically updates @v to (@v + @i) with acquire ordering. |
2797 | * |
2798 | * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere. |
2799 | * |
2800 | * Return: The original value of @v. |
2801 | */ |
2802 | static __always_inline s64 |
2803 | raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) |
2804 | { |
2805 | #if defined(arch_atomic64_fetch_add_acquire) |
2806 | return arch_atomic64_fetch_add_acquire(i, v); |
2807 | #elif defined(arch_atomic64_fetch_add_relaxed) |
2808 | s64 ret = arch_atomic64_fetch_add_relaxed(i, v); |
2809 | __atomic_acquire_fence(); |
2810 | return ret; |
2811 | #elif defined(arch_atomic64_fetch_add) |
2812 | return arch_atomic64_fetch_add(i, v); |
2813 | #else |
2814 | #error "Unable to define raw_atomic64_fetch_add_acquire" |
2815 | #endif |
2816 | } |
2817 | |
2818 | /** |
2819 | * raw_atomic64_fetch_add_release() - atomic add with release ordering |
2820 | * @i: s64 value to add |
2821 | * @v: pointer to atomic64_t |
2822 | * |
2823 | * Atomically updates @v to (@v + @i) with release ordering. |
2824 | * |
2825 | * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere. |
2826 | * |
2827 | * Return: The original value of @v. |
2828 | */ |
2829 | static __always_inline s64 |
2830 | raw_atomic64_fetch_add_release(s64 i, atomic64_t *v) |
2831 | { |
2832 | #if defined(arch_atomic64_fetch_add_release) |
2833 | return arch_atomic64_fetch_add_release(i, v); |
2834 | #elif defined(arch_atomic64_fetch_add_relaxed) |
2835 | __atomic_release_fence(); |
2836 | return arch_atomic64_fetch_add_relaxed(i, v); |
2837 | #elif defined(arch_atomic64_fetch_add) |
2838 | return arch_atomic64_fetch_add(i, v); |
2839 | #else |
2840 | #error "Unable to define raw_atomic64_fetch_add_release" |
2841 | #endif |
2842 | } |
2843 | |
2844 | /** |
2845 | * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering |
2846 | * @i: s64 value to add |
2847 | * @v: pointer to atomic64_t |
2848 | * |
2849 | * Atomically updates @v to (@v + @i) with relaxed ordering. |
2850 | * |
2851 | * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere. |
2852 | * |
2853 | * Return: The original value of @v. |
2854 | */ |
2855 | static __always_inline s64 |
2856 | raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) |
2857 | { |
2858 | #if defined(arch_atomic64_fetch_add_relaxed) |
2859 | return arch_atomic64_fetch_add_relaxed(i, v); |
2860 | #elif defined(arch_atomic64_fetch_add) |
2861 | return arch_atomic64_fetch_add(i, v); |
2862 | #else |
2863 | #error "Unable to define raw_atomic64_fetch_add_relaxed" |
2864 | #endif |
2865 | } |
2866 | |
2867 | /** |
2868 | * raw_atomic64_sub() - atomic subtract with relaxed ordering |
2869 | * @i: s64 value to subtract |
2870 | * @v: pointer to atomic64_t |
2871 | * |
2872 | * Atomically updates @v to (@v - @i) with relaxed ordering. |
2873 | * |
2874 | * Safe to use in noinstr code; prefer atomic64_sub() elsewhere. |
2875 | * |
2876 | * Return: Nothing. |
2877 | */ |
2878 | static __always_inline void |
2879 | raw_atomic64_sub(s64 i, atomic64_t *v) |
2880 | { |
2881 | arch_atomic64_sub(i, v); |
2882 | } |
2883 | |
2884 | /** |
2885 | * raw_atomic64_sub_return() - atomic subtract with full ordering |
2886 | * @i: s64 value to subtract |
2887 | * @v: pointer to atomic64_t |
2888 | * |
2889 | * Atomically updates @v to (@v - @i) with full ordering. |
2890 | * |
2891 | * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere. |
2892 | * |
2893 | * Return: The updated value of @v. |
2894 | */ |
2895 | static __always_inline s64 |
2896 | raw_atomic64_sub_return(s64 i, atomic64_t *v) |
2897 | { |
2898 | #if defined(arch_atomic64_sub_return) |
2899 | return arch_atomic64_sub_return(i, v); |
2900 | #elif defined(arch_atomic64_sub_return_relaxed) |
2901 | s64 ret; |
2902 | __atomic_pre_full_fence(); |
2903 | ret = arch_atomic64_sub_return_relaxed(i, v); |
2904 | __atomic_post_full_fence(); |
2905 | return ret; |
2906 | #else |
2907 | #error "Unable to define raw_atomic64_sub_return" |
2908 | #endif |
2909 | } |
2910 | |
2911 | /** |
2912 | * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering |
2913 | * @i: s64 value to subtract |
2914 | * @v: pointer to atomic64_t |
2915 | * |
2916 | * Atomically updates @v to (@v - @i) with acquire ordering. |
2917 | * |
2918 | * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere. |
2919 | * |
2920 | * Return: The updated value of @v. |
2921 | */ |
2922 | static __always_inline s64 |
2923 | raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v) |
2924 | { |
2925 | #if defined(arch_atomic64_sub_return_acquire) |
2926 | return arch_atomic64_sub_return_acquire(i, v); |
2927 | #elif defined(arch_atomic64_sub_return_relaxed) |
2928 | s64 ret = arch_atomic64_sub_return_relaxed(i, v); |
2929 | __atomic_acquire_fence(); |
2930 | return ret; |
2931 | #elif defined(arch_atomic64_sub_return) |
2932 | return arch_atomic64_sub_return(i, v); |
2933 | #else |
2934 | #error "Unable to define raw_atomic64_sub_return_acquire" |
2935 | #endif |
2936 | } |
2937 | |
2938 | /** |
2939 | * raw_atomic64_sub_return_release() - atomic subtract with release ordering |
2940 | * @i: s64 value to subtract |
2941 | * @v: pointer to atomic64_t |
2942 | * |
2943 | * Atomically updates @v to (@v - @i) with release ordering. |
2944 | * |
2945 | * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere. |
2946 | * |
2947 | * Return: The updated value of @v. |
2948 | */ |
2949 | static __always_inline s64 |
2950 | raw_atomic64_sub_return_release(s64 i, atomic64_t *v) |
2951 | { |
2952 | #if defined(arch_atomic64_sub_return_release) |
2953 | return arch_atomic64_sub_return_release(i, v); |
2954 | #elif defined(arch_atomic64_sub_return_relaxed) |
2955 | __atomic_release_fence(); |
2956 | return arch_atomic64_sub_return_relaxed(i, v); |
2957 | #elif defined(arch_atomic64_sub_return) |
2958 | return arch_atomic64_sub_return(i, v); |
2959 | #else |
2960 | #error "Unable to define raw_atomic64_sub_return_release" |
2961 | #endif |
2962 | } |
2963 | |
2964 | /** |
2965 | * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering |
2966 | * @i: s64 value to subtract |
2967 | * @v: pointer to atomic64_t |
2968 | * |
2969 | * Atomically updates @v to (@v - @i) with relaxed ordering. |
2970 | * |
2971 | * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere. |
2972 | * |
2973 | * Return: The updated value of @v. |
2974 | */ |
2975 | static __always_inline s64 |
2976 | raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v) |
2977 | { |
2978 | #if defined(arch_atomic64_sub_return_relaxed) |
2979 | return arch_atomic64_sub_return_relaxed(i, v); |
2980 | #elif defined(arch_atomic64_sub_return) |
2981 | return arch_atomic64_sub_return(i, v); |
2982 | #else |
2983 | #error "Unable to define raw_atomic64_sub_return_relaxed" |
2984 | #endif |
2985 | } |
2986 | |
2987 | /** |
2988 | * raw_atomic64_fetch_sub() - atomic subtract with full ordering |
2989 | * @i: s64 value to subtract |
2990 | * @v: pointer to atomic64_t |
2991 | * |
2992 | * Atomically updates @v to (@v - @i) with full ordering. |
2993 | * |
2994 | * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere. |
2995 | * |
2996 | * Return: The original value of @v. |
2997 | */ |
2998 | static __always_inline s64 |
2999 | raw_atomic64_fetch_sub(s64 i, atomic64_t *v) |
3000 | { |
3001 | #if defined(arch_atomic64_fetch_sub) |
3002 | return arch_atomic64_fetch_sub(i, v); |
3003 | #elif defined(arch_atomic64_fetch_sub_relaxed) |
3004 | s64 ret; |
3005 | __atomic_pre_full_fence(); |
3006 | ret = arch_atomic64_fetch_sub_relaxed(i, v); |
3007 | __atomic_post_full_fence(); |
3008 | return ret; |
3009 | #else |
3010 | #error "Unable to define raw_atomic64_fetch_sub" |
3011 | #endif |
3012 | } |
3013 | |
3014 | /** |
3015 | * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering |
3016 | * @i: s64 value to subtract |
3017 | * @v: pointer to atomic64_t |
3018 | * |
3019 | * Atomically updates @v to (@v - @i) with acquire ordering. |
3020 | * |
3021 | * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere. |
3022 | * |
3023 | * Return: The original value of @v. |
3024 | */ |
3025 | static __always_inline s64 |
3026 | raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) |
3027 | { |
3028 | #if defined(arch_atomic64_fetch_sub_acquire) |
3029 | return arch_atomic64_fetch_sub_acquire(i, v); |
3030 | #elif defined(arch_atomic64_fetch_sub_relaxed) |
3031 | s64 ret = arch_atomic64_fetch_sub_relaxed(i, v); |
3032 | __atomic_acquire_fence(); |
3033 | return ret; |
3034 | #elif defined(arch_atomic64_fetch_sub) |
3035 | return arch_atomic64_fetch_sub(i, v); |
3036 | #else |
3037 | #error "Unable to define raw_atomic64_fetch_sub_acquire" |
3038 | #endif |
3039 | } |
3040 | |
3041 | /** |
3042 | * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering |
3043 | * @i: s64 value to subtract |
3044 | * @v: pointer to atomic64_t |
3045 | * |
3046 | * Atomically updates @v to (@v - @i) with release ordering. |
3047 | * |
3048 | * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere. |
3049 | * |
3050 | * Return: The original value of @v. |
3051 | */ |
3052 | static __always_inline s64 |
3053 | raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v) |
3054 | { |
3055 | #if defined(arch_atomic64_fetch_sub_release) |
3056 | return arch_atomic64_fetch_sub_release(i, v); |
3057 | #elif defined(arch_atomic64_fetch_sub_relaxed) |
3058 | __atomic_release_fence(); |
3059 | return arch_atomic64_fetch_sub_relaxed(i, v); |
3060 | #elif defined(arch_atomic64_fetch_sub) |
3061 | return arch_atomic64_fetch_sub(i, v); |
3062 | #else |
3063 | #error "Unable to define raw_atomic64_fetch_sub_release" |
3064 | #endif |
3065 | } |
3066 | |
3067 | /** |
3068 | * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering |
3069 | * @i: s64 value to subtract |
3070 | * @v: pointer to atomic64_t |
3071 | * |
3072 | * Atomically updates @v to (@v - @i) with relaxed ordering. |
3073 | * |
3074 | * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere. |
3075 | * |
3076 | * Return: The original value of @v. |
3077 | */ |
3078 | static __always_inline s64 |
3079 | raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) |
3080 | { |
3081 | #if defined(arch_atomic64_fetch_sub_relaxed) |
3082 | return arch_atomic64_fetch_sub_relaxed(i, v); |
3083 | #elif defined(arch_atomic64_fetch_sub) |
3084 | return arch_atomic64_fetch_sub(i, v); |
3085 | #else |
3086 | #error "Unable to define raw_atomic64_fetch_sub_relaxed" |
3087 | #endif |
3088 | } |
3089 | |
3090 | /** |
3091 | * raw_atomic64_inc() - atomic increment with relaxed ordering |
3092 | * @v: pointer to atomic64_t |
3093 | * |
3094 | * Atomically updates @v to (@v + 1) with relaxed ordering. |
3095 | * |
3096 | * Safe to use in noinstr code; prefer atomic64_inc() elsewhere. |
3097 | * |
3098 | * Return: Nothing. |
3099 | */ |
3100 | static __always_inline void |
3101 | raw_atomic64_inc(atomic64_t *v) |
3102 | { |
3103 | #if defined(arch_atomic64_inc) |
3104 | arch_atomic64_inc(v); |
3105 | #else |
3106 | raw_atomic64_add(1, v); |
3107 | #endif |
3108 | } |
3109 | |
3110 | /** |
3111 | * raw_atomic64_inc_return() - atomic increment with full ordering |
3112 | * @v: pointer to atomic64_t |
3113 | * |
3114 | * Atomically updates @v to (@v + 1) with full ordering. |
3115 | * |
3116 | * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere. |
3117 | * |
3118 | * Return: The updated value of @v. |
3119 | */ |
3120 | static __always_inline s64 |
3121 | raw_atomic64_inc_return(atomic64_t *v) |
3122 | { |
3123 | #if defined(arch_atomic64_inc_return) |
3124 | return arch_atomic64_inc_return(v); |
3125 | #elif defined(arch_atomic64_inc_return_relaxed) |
3126 | s64 ret; |
3127 | __atomic_pre_full_fence(); |
3128 | ret = arch_atomic64_inc_return_relaxed(v); |
3129 | __atomic_post_full_fence(); |
3130 | return ret; |
3131 | #else |
3132 | return raw_atomic64_add_return(i: 1, v); |
3133 | #endif |
3134 | } |
3135 | |
3136 | /** |
3137 | * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering |
3138 | * @v: pointer to atomic64_t |
3139 | * |
3140 | * Atomically updates @v to (@v + 1) with acquire ordering. |
3141 | * |
3142 | * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere. |
3143 | * |
3144 | * Return: The updated value of @v. |
3145 | */ |
3146 | static __always_inline s64 |
3147 | raw_atomic64_inc_return_acquire(atomic64_t *v) |
3148 | { |
3149 | #if defined(arch_atomic64_inc_return_acquire) |
3150 | return arch_atomic64_inc_return_acquire(v); |
3151 | #elif defined(arch_atomic64_inc_return_relaxed) |
3152 | s64 ret = arch_atomic64_inc_return_relaxed(v); |
3153 | __atomic_acquire_fence(); |
3154 | return ret; |
3155 | #elif defined(arch_atomic64_inc_return) |
3156 | return arch_atomic64_inc_return(v); |
3157 | #else |
3158 | return raw_atomic64_add_return_acquire(i: 1, v); |
3159 | #endif |
3160 | } |
3161 | |
3162 | /** |
3163 | * raw_atomic64_inc_return_release() - atomic increment with release ordering |
3164 | * @v: pointer to atomic64_t |
3165 | * |
3166 | * Atomically updates @v to (@v + 1) with release ordering. |
3167 | * |
3168 | * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere. |
3169 | * |
3170 | * Return: The updated value of @v. |
3171 | */ |
3172 | static __always_inline s64 |
3173 | raw_atomic64_inc_return_release(atomic64_t *v) |
3174 | { |
3175 | #if defined(arch_atomic64_inc_return_release) |
3176 | return arch_atomic64_inc_return_release(v); |
3177 | #elif defined(arch_atomic64_inc_return_relaxed) |
3178 | __atomic_release_fence(); |
3179 | return arch_atomic64_inc_return_relaxed(v); |
3180 | #elif defined(arch_atomic64_inc_return) |
3181 | return arch_atomic64_inc_return(v); |
3182 | #else |
3183 | return raw_atomic64_add_return_release(i: 1, v); |
3184 | #endif |
3185 | } |
3186 | |
3187 | /** |
3188 | * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering |
3189 | * @v: pointer to atomic64_t |
3190 | * |
3191 | * Atomically updates @v to (@v + 1) with relaxed ordering. |
3192 | * |
3193 | * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere. |
3194 | * |
3195 | * Return: The updated value of @v. |
3196 | */ |
3197 | static __always_inline s64 |
3198 | raw_atomic64_inc_return_relaxed(atomic64_t *v) |
3199 | { |
3200 | #if defined(arch_atomic64_inc_return_relaxed) |
3201 | return arch_atomic64_inc_return_relaxed(v); |
3202 | #elif defined(arch_atomic64_inc_return) |
3203 | return arch_atomic64_inc_return(v); |
3204 | #else |
3205 | return raw_atomic64_add_return_relaxed(i: 1, v); |
3206 | #endif |
3207 | } |
3208 | |
3209 | /** |
3210 | * raw_atomic64_fetch_inc() - atomic increment with full ordering |
3211 | * @v: pointer to atomic64_t |
3212 | * |
3213 | * Atomically updates @v to (@v + 1) with full ordering. |
3214 | * |
3215 | * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere. |
3216 | * |
3217 | * Return: The original value of @v. |
3218 | */ |
3219 | static __always_inline s64 |
3220 | raw_atomic64_fetch_inc(atomic64_t *v) |
3221 | { |
3222 | #if defined(arch_atomic64_fetch_inc) |
3223 | return arch_atomic64_fetch_inc(v); |
3224 | #elif defined(arch_atomic64_fetch_inc_relaxed) |
3225 | s64 ret; |
3226 | __atomic_pre_full_fence(); |
3227 | ret = arch_atomic64_fetch_inc_relaxed(v); |
3228 | __atomic_post_full_fence(); |
3229 | return ret; |
3230 | #else |
3231 | return raw_atomic64_fetch_add(i: 1, v); |
3232 | #endif |
3233 | } |
3234 | |
3235 | /** |
3236 | * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering |
3237 | * @v: pointer to atomic64_t |
3238 | * |
3239 | * Atomically updates @v to (@v + 1) with acquire ordering. |
3240 | * |
3241 | * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere. |
3242 | * |
3243 | * Return: The original value of @v. |
3244 | */ |
3245 | static __always_inline s64 |
3246 | raw_atomic64_fetch_inc_acquire(atomic64_t *v) |
3247 | { |
3248 | #if defined(arch_atomic64_fetch_inc_acquire) |
3249 | return arch_atomic64_fetch_inc_acquire(v); |
3250 | #elif defined(arch_atomic64_fetch_inc_relaxed) |
3251 | s64 ret = arch_atomic64_fetch_inc_relaxed(v); |
3252 | __atomic_acquire_fence(); |
3253 | return ret; |
3254 | #elif defined(arch_atomic64_fetch_inc) |
3255 | return arch_atomic64_fetch_inc(v); |
3256 | #else |
3257 | return raw_atomic64_fetch_add_acquire(i: 1, v); |
3258 | #endif |
3259 | } |
3260 | |
3261 | /** |
3262 | * raw_atomic64_fetch_inc_release() - atomic increment with release ordering |
3263 | * @v: pointer to atomic64_t |
3264 | * |
3265 | * Atomically updates @v to (@v + 1) with release ordering. |
3266 | * |
3267 | * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere. |
3268 | * |
3269 | * Return: The original value of @v. |
3270 | */ |
3271 | static __always_inline s64 |
3272 | raw_atomic64_fetch_inc_release(atomic64_t *v) |
3273 | { |
3274 | #if defined(arch_atomic64_fetch_inc_release) |
3275 | return arch_atomic64_fetch_inc_release(v); |
3276 | #elif defined(arch_atomic64_fetch_inc_relaxed) |
3277 | __atomic_release_fence(); |
3278 | return arch_atomic64_fetch_inc_relaxed(v); |
3279 | #elif defined(arch_atomic64_fetch_inc) |
3280 | return arch_atomic64_fetch_inc(v); |
3281 | #else |
3282 | return raw_atomic64_fetch_add_release(i: 1, v); |
3283 | #endif |
3284 | } |
3285 | |
3286 | /** |
3287 | * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering |
3288 | * @v: pointer to atomic64_t |
3289 | * |
3290 | * Atomically updates @v to (@v + 1) with relaxed ordering. |
3291 | * |
3292 | * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere. |
3293 | * |
3294 | * Return: The original value of @v. |
3295 | */ |
3296 | static __always_inline s64 |
3297 | raw_atomic64_fetch_inc_relaxed(atomic64_t *v) |
3298 | { |
3299 | #if defined(arch_atomic64_fetch_inc_relaxed) |
3300 | return arch_atomic64_fetch_inc_relaxed(v); |
3301 | #elif defined(arch_atomic64_fetch_inc) |
3302 | return arch_atomic64_fetch_inc(v); |
3303 | #else |
3304 | return raw_atomic64_fetch_add_relaxed(i: 1, v); |
3305 | #endif |
3306 | } |
3307 | |
3308 | /** |
3309 | * raw_atomic64_dec() - atomic decrement with relaxed ordering |
3310 | * @v: pointer to atomic64_t |
3311 | * |
3312 | * Atomically updates @v to (@v - 1) with relaxed ordering. |
3313 | * |
3314 | * Safe to use in noinstr code; prefer atomic64_dec() elsewhere. |
3315 | * |
3316 | * Return: Nothing. |
3317 | */ |
3318 | static __always_inline void |
3319 | raw_atomic64_dec(atomic64_t *v) |
3320 | { |
3321 | #if defined(arch_atomic64_dec) |
3322 | arch_atomic64_dec(v); |
3323 | #else |
3324 | raw_atomic64_sub(1, v); |
3325 | #endif |
3326 | } |
3327 | |
3328 | /** |
3329 | * raw_atomic64_dec_return() - atomic decrement with full ordering |
3330 | * @v: pointer to atomic64_t |
3331 | * |
3332 | * Atomically updates @v to (@v - 1) with full ordering. |
3333 | * |
3334 | * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere. |
3335 | * |
3336 | * Return: The updated value of @v. |
3337 | */ |
3338 | static __always_inline s64 |
3339 | raw_atomic64_dec_return(atomic64_t *v) |
3340 | { |
3341 | #if defined(arch_atomic64_dec_return) |
3342 | return arch_atomic64_dec_return(v); |
3343 | #elif defined(arch_atomic64_dec_return_relaxed) |
3344 | s64 ret; |
3345 | __atomic_pre_full_fence(); |
3346 | ret = arch_atomic64_dec_return_relaxed(v); |
3347 | __atomic_post_full_fence(); |
3348 | return ret; |
3349 | #else |
3350 | return raw_atomic64_sub_return(i: 1, v); |
3351 | #endif |
3352 | } |
3353 | |
3354 | /** |
3355 | * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering |
3356 | * @v: pointer to atomic64_t |
3357 | * |
3358 | * Atomically updates @v to (@v - 1) with acquire ordering. |
3359 | * |
3360 | * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere. |
3361 | * |
3362 | * Return: The updated value of @v. |
3363 | */ |
3364 | static __always_inline s64 |
3365 | raw_atomic64_dec_return_acquire(atomic64_t *v) |
3366 | { |
3367 | #if defined(arch_atomic64_dec_return_acquire) |
3368 | return arch_atomic64_dec_return_acquire(v); |
3369 | #elif defined(arch_atomic64_dec_return_relaxed) |
3370 | s64 ret = arch_atomic64_dec_return_relaxed(v); |
3371 | __atomic_acquire_fence(); |
3372 | return ret; |
3373 | #elif defined(arch_atomic64_dec_return) |
3374 | return arch_atomic64_dec_return(v); |
3375 | #else |
3376 | return raw_atomic64_sub_return_acquire(i: 1, v); |
3377 | #endif |
3378 | } |
3379 | |
3380 | /** |
3381 | * raw_atomic64_dec_return_release() - atomic decrement with release ordering |
3382 | * @v: pointer to atomic64_t |
3383 | * |
3384 | * Atomically updates @v to (@v - 1) with release ordering. |
3385 | * |
3386 | * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere. |
3387 | * |
3388 | * Return: The updated value of @v. |
3389 | */ |
3390 | static __always_inline s64 |
3391 | raw_atomic64_dec_return_release(atomic64_t *v) |
3392 | { |
3393 | #if defined(arch_atomic64_dec_return_release) |
3394 | return arch_atomic64_dec_return_release(v); |
3395 | #elif defined(arch_atomic64_dec_return_relaxed) |
3396 | __atomic_release_fence(); |
3397 | return arch_atomic64_dec_return_relaxed(v); |
3398 | #elif defined(arch_atomic64_dec_return) |
3399 | return arch_atomic64_dec_return(v); |
3400 | #else |
3401 | return raw_atomic64_sub_return_release(i: 1, v); |
3402 | #endif |
3403 | } |
3404 | |
3405 | /** |
3406 | * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering |
3407 | * @v: pointer to atomic64_t |
3408 | * |
3409 | * Atomically updates @v to (@v - 1) with relaxed ordering. |
3410 | * |
3411 | * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere. |
3412 | * |
3413 | * Return: The updated value of @v. |
3414 | */ |
3415 | static __always_inline s64 |
3416 | raw_atomic64_dec_return_relaxed(atomic64_t *v) |
3417 | { |
3418 | #if defined(arch_atomic64_dec_return_relaxed) |
3419 | return arch_atomic64_dec_return_relaxed(v); |
3420 | #elif defined(arch_atomic64_dec_return) |
3421 | return arch_atomic64_dec_return(v); |
3422 | #else |
3423 | return raw_atomic64_sub_return_relaxed(i: 1, v); |
3424 | #endif |
3425 | } |
3426 | |
3427 | /** |
3428 | * raw_atomic64_fetch_dec() - atomic decrement with full ordering |
3429 | * @v: pointer to atomic64_t |
3430 | * |
3431 | * Atomically updates @v to (@v - 1) with full ordering. |
3432 | * |
3433 | * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere. |
3434 | * |
3435 | * Return: The original value of @v. |
3436 | */ |
3437 | static __always_inline s64 |
3438 | raw_atomic64_fetch_dec(atomic64_t *v) |
3439 | { |
3440 | #if defined(arch_atomic64_fetch_dec) |
3441 | return arch_atomic64_fetch_dec(v); |
3442 | #elif defined(arch_atomic64_fetch_dec_relaxed) |
3443 | s64 ret; |
3444 | __atomic_pre_full_fence(); |
3445 | ret = arch_atomic64_fetch_dec_relaxed(v); |
3446 | __atomic_post_full_fence(); |
3447 | return ret; |
3448 | #else |
3449 | return raw_atomic64_fetch_sub(i: 1, v); |
3450 | #endif |
3451 | } |
3452 | |
3453 | /** |
3454 | * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering |
3455 | * @v: pointer to atomic64_t |
3456 | * |
3457 | * Atomically updates @v to (@v - 1) with acquire ordering. |
3458 | * |
3459 | * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere. |
3460 | * |
3461 | * Return: The original value of @v. |
3462 | */ |
3463 | static __always_inline s64 |
3464 | raw_atomic64_fetch_dec_acquire(atomic64_t *v) |
3465 | { |
3466 | #if defined(arch_atomic64_fetch_dec_acquire) |
3467 | return arch_atomic64_fetch_dec_acquire(v); |
3468 | #elif defined(arch_atomic64_fetch_dec_relaxed) |
3469 | s64 ret = arch_atomic64_fetch_dec_relaxed(v); |
3470 | __atomic_acquire_fence(); |
3471 | return ret; |
3472 | #elif defined(arch_atomic64_fetch_dec) |
3473 | return arch_atomic64_fetch_dec(v); |
3474 | #else |
3475 | return raw_atomic64_fetch_sub_acquire(i: 1, v); |
3476 | #endif |
3477 | } |
3478 | |
3479 | /** |
3480 | * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering |
3481 | * @v: pointer to atomic64_t |
3482 | * |
3483 | * Atomically updates @v to (@v - 1) with release ordering. |
3484 | * |
3485 | * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere. |
3486 | * |
3487 | * Return: The original value of @v. |
3488 | */ |
3489 | static __always_inline s64 |
3490 | raw_atomic64_fetch_dec_release(atomic64_t *v) |
3491 | { |
3492 | #if defined(arch_atomic64_fetch_dec_release) |
3493 | return arch_atomic64_fetch_dec_release(v); |
3494 | #elif defined(arch_atomic64_fetch_dec_relaxed) |
3495 | __atomic_release_fence(); |
3496 | return arch_atomic64_fetch_dec_relaxed(v); |
3497 | #elif defined(arch_atomic64_fetch_dec) |
3498 | return arch_atomic64_fetch_dec(v); |
3499 | #else |
3500 | return raw_atomic64_fetch_sub_release(i: 1, v); |
3501 | #endif |
3502 | } |
3503 | |
3504 | /** |
3505 | * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering |
3506 | * @v: pointer to atomic64_t |
3507 | * |
3508 | * Atomically updates @v to (@v - 1) with relaxed ordering. |
3509 | * |
3510 | * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere. |
3511 | * |
3512 | * Return: The original value of @v. |
3513 | */ |
3514 | static __always_inline s64 |
3515 | raw_atomic64_fetch_dec_relaxed(atomic64_t *v) |
3516 | { |
3517 | #if defined(arch_atomic64_fetch_dec_relaxed) |
3518 | return arch_atomic64_fetch_dec_relaxed(v); |
3519 | #elif defined(arch_atomic64_fetch_dec) |
3520 | return arch_atomic64_fetch_dec(v); |
3521 | #else |
3522 | return raw_atomic64_fetch_sub_relaxed(i: 1, v); |
3523 | #endif |
3524 | } |
3525 | |
3526 | /** |
3527 | * raw_atomic64_and() - atomic bitwise AND with relaxed ordering |
3528 | * @i: s64 value |
3529 | * @v: pointer to atomic64_t |
3530 | * |
3531 | * Atomically updates @v to (@v & @i) with relaxed ordering. |
3532 | * |
3533 | * Safe to use in noinstr code; prefer atomic64_and() elsewhere. |
3534 | * |
3535 | * Return: Nothing. |
3536 | */ |
3537 | static __always_inline void |
3538 | raw_atomic64_and(s64 i, atomic64_t *v) |
3539 | { |
3540 | arch_atomic64_and(i, v); |
3541 | } |
3542 | |
3543 | /** |
3544 | * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering |
3545 | * @i: s64 value |
3546 | * @v: pointer to atomic64_t |
3547 | * |
3548 | * Atomically updates @v to (@v & @i) with full ordering. |
3549 | * |
3550 | * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere. |
3551 | * |
3552 | * Return: The original value of @v. |
3553 | */ |
3554 | static __always_inline s64 |
3555 | raw_atomic64_fetch_and(s64 i, atomic64_t *v) |
3556 | { |
3557 | #if defined(arch_atomic64_fetch_and) |
3558 | return arch_atomic64_fetch_and(i, v); |
3559 | #elif defined(arch_atomic64_fetch_and_relaxed) |
3560 | s64 ret; |
3561 | __atomic_pre_full_fence(); |
3562 | ret = arch_atomic64_fetch_and_relaxed(i, v); |
3563 | __atomic_post_full_fence(); |
3564 | return ret; |
3565 | #else |
3566 | #error "Unable to define raw_atomic64_fetch_and" |
3567 | #endif |
3568 | } |
3569 | |
3570 | /** |
3571 | * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering |
3572 | * @i: s64 value |
3573 | * @v: pointer to atomic64_t |
3574 | * |
3575 | * Atomically updates @v to (@v & @i) with acquire ordering. |
3576 | * |
3577 | * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere. |
3578 | * |
3579 | * Return: The original value of @v. |
3580 | */ |
3581 | static __always_inline s64 |
3582 | raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) |
3583 | { |
3584 | #if defined(arch_atomic64_fetch_and_acquire) |
3585 | return arch_atomic64_fetch_and_acquire(i, v); |
3586 | #elif defined(arch_atomic64_fetch_and_relaxed) |
3587 | s64 ret = arch_atomic64_fetch_and_relaxed(i, v); |
3588 | __atomic_acquire_fence(); |
3589 | return ret; |
3590 | #elif defined(arch_atomic64_fetch_and) |
3591 | return arch_atomic64_fetch_and(i, v); |
3592 | #else |
3593 | #error "Unable to define raw_atomic64_fetch_and_acquire" |
3594 | #endif |
3595 | } |
3596 | |
3597 | /** |
3598 | * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering |
3599 | * @i: s64 value |
3600 | * @v: pointer to atomic64_t |
3601 | * |
3602 | * Atomically updates @v to (@v & @i) with release ordering. |
3603 | * |
3604 | * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere. |
3605 | * |
3606 | * Return: The original value of @v. |
3607 | */ |
3608 | static __always_inline s64 |
3609 | raw_atomic64_fetch_and_release(s64 i, atomic64_t *v) |
3610 | { |
3611 | #if defined(arch_atomic64_fetch_and_release) |
3612 | return arch_atomic64_fetch_and_release(i, v); |
3613 | #elif defined(arch_atomic64_fetch_and_relaxed) |
3614 | __atomic_release_fence(); |
3615 | return arch_atomic64_fetch_and_relaxed(i, v); |
3616 | #elif defined(arch_atomic64_fetch_and) |
3617 | return arch_atomic64_fetch_and(i, v); |
3618 | #else |
3619 | #error "Unable to define raw_atomic64_fetch_and_release" |
3620 | #endif |
3621 | } |
3622 | |
3623 | /** |
3624 | * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering |
3625 | * @i: s64 value |
3626 | * @v: pointer to atomic64_t |
3627 | * |
3628 | * Atomically updates @v to (@v & @i) with relaxed ordering. |
3629 | * |
3630 | * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere. |
3631 | * |
3632 | * Return: The original value of @v. |
3633 | */ |
3634 | static __always_inline s64 |
3635 | raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) |
3636 | { |
3637 | #if defined(arch_atomic64_fetch_and_relaxed) |
3638 | return arch_atomic64_fetch_and_relaxed(i, v); |
3639 | #elif defined(arch_atomic64_fetch_and) |
3640 | return arch_atomic64_fetch_and(i, v); |
3641 | #else |
3642 | #error "Unable to define raw_atomic64_fetch_and_relaxed" |
3643 | #endif |
3644 | } |
3645 | |
3646 | /** |
3647 | * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering |
3648 | * @i: s64 value |
3649 | * @v: pointer to atomic64_t |
3650 | * |
3651 | * Atomically updates @v to (@v & ~@i) with relaxed ordering. |
3652 | * |
3653 | * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere. |
3654 | * |
3655 | * Return: Nothing. |
3656 | */ |
3657 | static __always_inline void |
3658 | raw_atomic64_andnot(s64 i, atomic64_t *v) |
3659 | { |
3660 | #if defined(arch_atomic64_andnot) |
3661 | arch_atomic64_andnot(i, v); |
3662 | #else |
3663 | raw_atomic64_and(i: ~i, v); |
3664 | #endif |
3665 | } |
3666 | |
3667 | /** |
3668 | * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering |
3669 | * @i: s64 value |
3670 | * @v: pointer to atomic64_t |
3671 | * |
3672 | * Atomically updates @v to (@v & ~@i) with full ordering. |
3673 | * |
3674 | * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere. |
3675 | * |
3676 | * Return: The original value of @v. |
3677 | */ |
3678 | static __always_inline s64 |
3679 | raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) |
3680 | { |
3681 | #if defined(arch_atomic64_fetch_andnot) |
3682 | return arch_atomic64_fetch_andnot(i, v); |
3683 | #elif defined(arch_atomic64_fetch_andnot_relaxed) |
3684 | s64 ret; |
3685 | __atomic_pre_full_fence(); |
3686 | ret = arch_atomic64_fetch_andnot_relaxed(i, v); |
3687 | __atomic_post_full_fence(); |
3688 | return ret; |
3689 | #else |
3690 | return raw_atomic64_fetch_and(i: ~i, v); |
3691 | #endif |
3692 | } |
3693 | |
3694 | /** |
3695 | * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering |
3696 | * @i: s64 value |
3697 | * @v: pointer to atomic64_t |
3698 | * |
3699 | * Atomically updates @v to (@v & ~@i) with acquire ordering. |
3700 | * |
3701 | * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere. |
3702 | * |
3703 | * Return: The original value of @v. |
3704 | */ |
3705 | static __always_inline s64 |
3706 | raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) |
3707 | { |
3708 | #if defined(arch_atomic64_fetch_andnot_acquire) |
3709 | return arch_atomic64_fetch_andnot_acquire(i, v); |
3710 | #elif defined(arch_atomic64_fetch_andnot_relaxed) |
3711 | s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v); |
3712 | __atomic_acquire_fence(); |
3713 | return ret; |
3714 | #elif defined(arch_atomic64_fetch_andnot) |
3715 | return arch_atomic64_fetch_andnot(i, v); |
3716 | #else |
3717 | return raw_atomic64_fetch_and_acquire(i: ~i, v); |
3718 | #endif |
3719 | } |
3720 | |
3721 | /** |
3722 | * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering |
3723 | * @i: s64 value |
3724 | * @v: pointer to atomic64_t |
3725 | * |
3726 | * Atomically updates @v to (@v & ~@i) with release ordering. |
3727 | * |
3728 | * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere. |
3729 | * |
3730 | * Return: The original value of @v. |
3731 | */ |
3732 | static __always_inline s64 |
3733 | raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) |
3734 | { |
3735 | #if defined(arch_atomic64_fetch_andnot_release) |
3736 | return arch_atomic64_fetch_andnot_release(i, v); |
3737 | #elif defined(arch_atomic64_fetch_andnot_relaxed) |
3738 | __atomic_release_fence(); |
3739 | return arch_atomic64_fetch_andnot_relaxed(i, v); |
3740 | #elif defined(arch_atomic64_fetch_andnot) |
3741 | return arch_atomic64_fetch_andnot(i, v); |
3742 | #else |
3743 | return raw_atomic64_fetch_and_release(i: ~i, v); |
3744 | #endif |
3745 | } |
3746 | |
3747 | /** |
3748 | * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering |
3749 | * @i: s64 value |
3750 | * @v: pointer to atomic64_t |
3751 | * |
3752 | * Atomically updates @v to (@v & ~@i) with relaxed ordering. |
3753 | * |
3754 | * Safe to use in noinstr code; prefer atomic64_fetch_andnot_relaxed() elsewhere. |
3755 | * |
3756 | * Return: The original value of @v. |
3757 | */ |
3758 | static __always_inline s64 |
3759 | raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) |
3760 | { |
3761 | #if defined(arch_atomic64_fetch_andnot_relaxed) |
3762 | return arch_atomic64_fetch_andnot_relaxed(i, v); |
3763 | #elif defined(arch_atomic64_fetch_andnot) |
3764 | return arch_atomic64_fetch_andnot(i, v); |
3765 | #else |
3766 | return raw_atomic64_fetch_and_relaxed(i: ~i, v); |
3767 | #endif |
3768 | } |
3769 | |
3770 | /** |
3771 | * raw_atomic64_or() - atomic bitwise OR with relaxed ordering |
3772 | * @i: s64 value |
3773 | * @v: pointer to atomic64_t |
3774 | * |
3775 | * Atomically updates @v to (@v | @i) with relaxed ordering. |
3776 | * |
3777 | * Safe to use in noinstr code; prefer atomic64_or() elsewhere. |
3778 | * |
3779 | * Return: Nothing. |
3780 | */ |
3781 | static __always_inline void |
3782 | raw_atomic64_or(s64 i, atomic64_t *v) |
3783 | { |
3784 | arch_atomic64_or(i, v); |
3785 | } |
3786 | |
3787 | /** |
3788 | * raw_atomic64_fetch_or() - atomic bitwise OR with full ordering |
3789 | * @i: s64 value |
3790 | * @v: pointer to atomic64_t |
3791 | * |
3792 | * Atomically updates @v to (@v | @i) with full ordering. |
3793 | * |
3794 | * Safe to use in noinstr code; prefer atomic64_fetch_or() elsewhere. |
3795 | * |
3796 | * Return: The original value of @v. |
3797 | */ |
3798 | static __always_inline s64 |
3799 | raw_atomic64_fetch_or(s64 i, atomic64_t *v) |
3800 | { |
3801 | #if defined(arch_atomic64_fetch_or) |
3802 | return arch_atomic64_fetch_or(i, v); |
3803 | #elif defined(arch_atomic64_fetch_or_relaxed) |
3804 | s64 ret; |
3805 | __atomic_pre_full_fence(); |
3806 | ret = arch_atomic64_fetch_or_relaxed(i, v); |
3807 | __atomic_post_full_fence(); |
3808 | return ret; |
3809 | #else |
3810 | #error "Unable to define raw_atomic64_fetch_or" |
3811 | #endif |
3812 | } |
3813 | |
3814 | /** |
3815 | * raw_atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering |
3816 | * @i: s64 value |
3817 | * @v: pointer to atomic64_t |
3818 | * |
3819 | * Atomically updates @v to (@v | @i) with acquire ordering. |
3820 | * |
3821 | * Safe to use in noinstr code; prefer atomic64_fetch_or_acquire() elsewhere. |
3822 | * |
3823 | * Return: The original value of @v. |
3824 | */ |
3825 | static __always_inline s64 |
3826 | raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) |
3827 | { |
3828 | #if defined(arch_atomic64_fetch_or_acquire) |
3829 | return arch_atomic64_fetch_or_acquire(i, v); |
3830 | #elif defined(arch_atomic64_fetch_or_relaxed) |
3831 | s64 ret = arch_atomic64_fetch_or_relaxed(i, v); |
3832 | __atomic_acquire_fence(); |
3833 | return ret; |
3834 | #elif defined(arch_atomic64_fetch_or) |
3835 | return arch_atomic64_fetch_or(i, v); |
3836 | #else |
3837 | #error "Unable to define raw_atomic64_fetch_or_acquire" |
3838 | #endif |
3839 | } |
3840 | |
3841 | /** |
3842 | * raw_atomic64_fetch_or_release() - atomic bitwise OR with release ordering |
3843 | * @i: s64 value |
3844 | * @v: pointer to atomic64_t |
3845 | * |
3846 | * Atomically updates @v to (@v | @i) with release ordering. |
3847 | * |
3848 | * Safe to use in noinstr code; prefer atomic64_fetch_or_release() elsewhere. |
3849 | * |
3850 | * Return: The original value of @v. |
3851 | */ |
3852 | static __always_inline s64 |
3853 | raw_atomic64_fetch_or_release(s64 i, atomic64_t *v) |
3854 | { |
3855 | #if defined(arch_atomic64_fetch_or_release) |
3856 | return arch_atomic64_fetch_or_release(i, v); |
3857 | #elif defined(arch_atomic64_fetch_or_relaxed) |
3858 | __atomic_release_fence(); |
3859 | return arch_atomic64_fetch_or_relaxed(i, v); |
3860 | #elif defined(arch_atomic64_fetch_or) |
3861 | return arch_atomic64_fetch_or(i, v); |
3862 | #else |
3863 | #error "Unable to define raw_atomic64_fetch_or_release" |
3864 | #endif |
3865 | } |
3866 | |
3867 | /** |
3868 | * raw_atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering |
3869 | * @i: s64 value |
3870 | * @v: pointer to atomic64_t |
3871 | * |
3872 | * Atomically updates @v to (@v | @i) with relaxed ordering. |
3873 | * |
3874 | * Safe to use in noinstr code; prefer atomic64_fetch_or_relaxed() elsewhere. |
3875 | * |
3876 | * Return: The original value of @v. |
3877 | */ |
3878 | static __always_inline s64 |
3879 | raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) |
3880 | { |
3881 | #if defined(arch_atomic64_fetch_or_relaxed) |
3882 | return arch_atomic64_fetch_or_relaxed(i, v); |
3883 | #elif defined(arch_atomic64_fetch_or) |
3884 | return arch_atomic64_fetch_or(i, v); |
3885 | #else |
3886 | #error "Unable to define raw_atomic64_fetch_or_relaxed" |
3887 | #endif |
3888 | } |
3889 | |
3890 | /** |
3891 | * raw_atomic64_xor() - atomic bitwise XOR with relaxed ordering |
3892 | * @i: s64 value |
3893 | * @v: pointer to atomic64_t |
3894 | * |
3895 | * Atomically updates @v to (@v ^ @i) with relaxed ordering. |
3896 | * |
3897 | * Safe to use in noinstr code; prefer atomic64_xor() elsewhere. |
3898 | * |
3899 | * Return: Nothing. |
3900 | */ |
3901 | static __always_inline void |
3902 | raw_atomic64_xor(s64 i, atomic64_t *v) |
3903 | { |
3904 | arch_atomic64_xor(i, v); |
3905 | } |
3906 | |
3907 | /** |
3908 | * raw_atomic64_fetch_xor() - atomic bitwise XOR with full ordering |
3909 | * @i: s64 value |
3910 | * @v: pointer to atomic64_t |
3911 | * |
3912 | * Atomically updates @v to (@v ^ @i) with full ordering. |
3913 | * |
3914 | * Safe to use in noinstr code; prefer atomic64_fetch_xor() elsewhere. |
3915 | * |
3916 | * Return: The original value of @v. |
3917 | */ |
3918 | static __always_inline s64 |
3919 | raw_atomic64_fetch_xor(s64 i, atomic64_t *v) |
3920 | { |
3921 | #if defined(arch_atomic64_fetch_xor) |
3922 | return arch_atomic64_fetch_xor(i, v); |
3923 | #elif defined(arch_atomic64_fetch_xor_relaxed) |
3924 | s64 ret; |
3925 | __atomic_pre_full_fence(); |
3926 | ret = arch_atomic64_fetch_xor_relaxed(i, v); |
3927 | __atomic_post_full_fence(); |
3928 | return ret; |
3929 | #else |
3930 | #error "Unable to define raw_atomic64_fetch_xor" |
3931 | #endif |
3932 | } |
3933 | |
3934 | /** |
3935 | * raw_atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering |
3936 | * @i: s64 value |
3937 | * @v: pointer to atomic64_t |
3938 | * |
3939 | * Atomically updates @v to (@v ^ @i) with acquire ordering. |
3940 | * |
3941 | * Safe to use in noinstr code; prefer atomic64_fetch_xor_acquire() elsewhere. |
3942 | * |
3943 | * Return: The original value of @v. |
3944 | */ |
3945 | static __always_inline s64 |
3946 | raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) |
3947 | { |
3948 | #if defined(arch_atomic64_fetch_xor_acquire) |
3949 | return arch_atomic64_fetch_xor_acquire(i, v); |
3950 | #elif defined(arch_atomic64_fetch_xor_relaxed) |
3951 | s64 ret = arch_atomic64_fetch_xor_relaxed(i, v); |
3952 | __atomic_acquire_fence(); |
3953 | return ret; |
3954 | #elif defined(arch_atomic64_fetch_xor) |
3955 | return arch_atomic64_fetch_xor(i, v); |
3956 | #else |
3957 | #error "Unable to define raw_atomic64_fetch_xor_acquire" |
3958 | #endif |
3959 | } |
3960 | |
3961 | /** |
3962 | * raw_atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering |
3963 | * @i: s64 value |
3964 | * @v: pointer to atomic64_t |
3965 | * |
3966 | * Atomically updates @v to (@v ^ @i) with release ordering. |
3967 | * |
3968 | * Safe to use in noinstr code; prefer atomic64_fetch_xor_release() elsewhere. |
3969 | * |
3970 | * Return: The original value of @v. |
3971 | */ |
3972 | static __always_inline s64 |
3973 | raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v) |
3974 | { |
3975 | #if defined(arch_atomic64_fetch_xor_release) |
3976 | return arch_atomic64_fetch_xor_release(i, v); |
3977 | #elif defined(arch_atomic64_fetch_xor_relaxed) |
3978 | __atomic_release_fence(); |
3979 | return arch_atomic64_fetch_xor_relaxed(i, v); |
3980 | #elif defined(arch_atomic64_fetch_xor) |
3981 | return arch_atomic64_fetch_xor(i, v); |
3982 | #else |
3983 | #error "Unable to define raw_atomic64_fetch_xor_release" |
3984 | #endif |
3985 | } |
3986 | |
3987 | /** |
3988 | * raw_atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering |
3989 | * @i: s64 value |
3990 | * @v: pointer to atomic64_t |
3991 | * |
3992 | * Atomically updates @v to (@v ^ @i) with relaxed ordering. |
3993 | * |
3994 | * Safe to use in noinstr code; prefer atomic64_fetch_xor_relaxed() elsewhere. |
3995 | * |
3996 | * Return: The original value of @v. |
3997 | */ |
3998 | static __always_inline s64 |
3999 | raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) |
4000 | { |
4001 | #if defined(arch_atomic64_fetch_xor_relaxed) |
4002 | return arch_atomic64_fetch_xor_relaxed(i, v); |
4003 | #elif defined(arch_atomic64_fetch_xor) |
4004 | return arch_atomic64_fetch_xor(i, v); |
4005 | #else |
4006 | #error "Unable to define raw_atomic64_fetch_xor_relaxed" |
4007 | #endif |
4008 | } |
4009 | |
4010 | /** |
4011 | * raw_atomic64_xchg() - atomic exchange with full ordering |
4012 | * @v: pointer to atomic64_t |
4013 | * @new: s64 value to assign |
4014 | * |
4015 | * Atomically updates @v to @new with full ordering. |
4016 | * |
4017 | * Safe to use in noinstr code; prefer atomic64_xchg() elsewhere. |
4018 | * |
4019 | * Return: The original value of @v. |
4020 | */ |
4021 | static __always_inline s64 |
4022 | raw_atomic64_xchg(atomic64_t *v, s64 new) |
4023 | { |
4024 | #if defined(arch_atomic64_xchg) |
4025 | return arch_atomic64_xchg(v, new); |
4026 | #elif defined(arch_atomic64_xchg_relaxed) |
4027 | s64 ret; |
4028 | __atomic_pre_full_fence(); |
4029 | ret = arch_atomic64_xchg_relaxed(v, new); |
4030 | __atomic_post_full_fence(); |
4031 | return ret; |
4032 | #else |
4033 | return raw_xchg(&v->counter, new); |
4034 | #endif |
4035 | } |
4036 | |
4037 | /** |
4038 | * raw_atomic64_xchg_acquire() - atomic exchange with acquire ordering |
4039 | * @v: pointer to atomic64_t |
4040 | * @new: s64 value to assign |
4041 | * |
4042 | * Atomically updates @v to @new with acquire ordering. |
4043 | * |
4044 | * Safe to use in noinstr code; prefer atomic64_xchg_acquire() elsewhere. |
4045 | * |
4046 | * Return: The original value of @v. |
4047 | */ |
4048 | static __always_inline s64 |
4049 | raw_atomic64_xchg_acquire(atomic64_t *v, s64 new) |
4050 | { |
4051 | #if defined(arch_atomic64_xchg_acquire) |
4052 | return arch_atomic64_xchg_acquire(v, new); |
4053 | #elif defined(arch_atomic64_xchg_relaxed) |
4054 | s64 ret = arch_atomic64_xchg_relaxed(v, new); |
4055 | __atomic_acquire_fence(); |
4056 | return ret; |
4057 | #elif defined(arch_atomic64_xchg) |
4058 | return arch_atomic64_xchg(v, new); |
4059 | #else |
4060 | return raw_xchg_acquire(&v->counter, new); |
4061 | #endif |
4062 | } |
4063 | |
4064 | /** |
4065 | * raw_atomic64_xchg_release() - atomic exchange with release ordering |
4066 | * @v: pointer to atomic64_t |
4067 | * @new: s64 value to assign |
4068 | * |
4069 | * Atomically updates @v to @new with release ordering. |
4070 | * |
4071 | * Safe to use in noinstr code; prefer atomic64_xchg_release() elsewhere. |
4072 | * |
4073 | * Return: The original value of @v. |
4074 | */ |
4075 | static __always_inline s64 |
4076 | raw_atomic64_xchg_release(atomic64_t *v, s64 new) |
4077 | { |
4078 | #if defined(arch_atomic64_xchg_release) |
4079 | return arch_atomic64_xchg_release(v, new); |
4080 | #elif defined(arch_atomic64_xchg_relaxed) |
4081 | __atomic_release_fence(); |
4082 | return arch_atomic64_xchg_relaxed(v, new); |
4083 | #elif defined(arch_atomic64_xchg) |
4084 | return arch_atomic64_xchg(v, new); |
4085 | #else |
4086 | return raw_xchg_release(&v->counter, new); |
4087 | #endif |
4088 | } |
4089 | |
4090 | /** |
4091 | * raw_atomic64_xchg_relaxed() - atomic exchange with relaxed ordering |
4092 | * @v: pointer to atomic64_t |
4093 | * @new: s64 value to assign |
4094 | * |
4095 | * Atomically updates @v to @new with relaxed ordering. |
4096 | * |
4097 | * Safe to use in noinstr code; prefer atomic64_xchg_relaxed() elsewhere. |
4098 | * |
4099 | * Return: The original value of @v. |
4100 | */ |
4101 | static __always_inline s64 |
4102 | raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new) |
4103 | { |
4104 | #if defined(arch_atomic64_xchg_relaxed) |
4105 | return arch_atomic64_xchg_relaxed(v, new); |
4106 | #elif defined(arch_atomic64_xchg) |
4107 | return arch_atomic64_xchg(v, new); |
4108 | #else |
4109 | return raw_xchg_relaxed(&v->counter, new); |
4110 | #endif |
4111 | } |
4112 | |
4113 | /** |
4114 | * raw_atomic64_cmpxchg() - atomic compare and exchange with full ordering |
4115 | * @v: pointer to atomic64_t |
4116 | * @old: s64 value to compare with |
4117 | * @new: s64 value to assign |
4118 | * |
4119 | * If (@v == @old), atomically updates @v to @new with full ordering. |
4120 | * |
4121 | * Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere. |
4122 | * |
4123 | * Return: The original value of @v. |
4124 | */ |
4125 | static __always_inline s64 |
4126 | raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) |
4127 | { |
4128 | #if defined(arch_atomic64_cmpxchg) |
4129 | return arch_atomic64_cmpxchg(v, old, new); |
4130 | #elif defined(arch_atomic64_cmpxchg_relaxed) |
4131 | s64 ret; |
4132 | __atomic_pre_full_fence(); |
4133 | ret = arch_atomic64_cmpxchg_relaxed(v, old, new); |
4134 | __atomic_post_full_fence(); |
4135 | return ret; |
4136 | #else |
4137 | return raw_cmpxchg(&v->counter, old, new); |
4138 | #endif |
4139 | } |
4140 | |
4141 | /** |
4142 | * raw_atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering |
4143 | * @v: pointer to atomic64_t |
4144 | * @old: s64 value to compare with |
4145 | * @new: s64 value to assign |
4146 | * |
4147 | * If (@v == @old), atomically updates @v to @new with acquire ordering. |
4148 | * |
4149 | * Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere. |
4150 | * |
4151 | * Return: The original value of @v. |
4152 | */ |
4153 | static __always_inline s64 |
4154 | raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) |
4155 | { |
4156 | #if defined(arch_atomic64_cmpxchg_acquire) |
4157 | return arch_atomic64_cmpxchg_acquire(v, old, new); |
4158 | #elif defined(arch_atomic64_cmpxchg_relaxed) |
4159 | s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); |
4160 | __atomic_acquire_fence(); |
4161 | return ret; |
4162 | #elif defined(arch_atomic64_cmpxchg) |
4163 | return arch_atomic64_cmpxchg(v, old, new); |
4164 | #else |
4165 | return raw_cmpxchg_acquire(&v->counter, old, new); |
4166 | #endif |
4167 | } |
4168 | |
4169 | /** |
4170 | * raw_atomic64_cmpxchg_release() - atomic compare and exchange with release ordering |
4171 | * @v: pointer to atomic64_t |
4172 | * @old: s64 value to compare with |
4173 | * @new: s64 value to assign |
4174 | * |
4175 | * If (@v == @old), atomically updates @v to @new with release ordering. |
4176 | * |
4177 | * Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere. |
4178 | * |
4179 | * Return: The original value of @v. |
4180 | */ |
4181 | static __always_inline s64 |
4182 | raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) |
4183 | { |
4184 | #if defined(arch_atomic64_cmpxchg_release) |
4185 | return arch_atomic64_cmpxchg_release(v, old, new); |
4186 | #elif defined(arch_atomic64_cmpxchg_relaxed) |
4187 | __atomic_release_fence(); |
4188 | return arch_atomic64_cmpxchg_relaxed(v, old, new); |
4189 | #elif defined(arch_atomic64_cmpxchg) |
4190 | return arch_atomic64_cmpxchg(v, old, new); |
4191 | #else |
4192 | return raw_cmpxchg_release(&v->counter, old, new); |
4193 | #endif |
4194 | } |
4195 | |
4196 | /** |
4197 | * raw_atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering |
4198 | * @v: pointer to atomic64_t |
4199 | * @old: s64 value to compare with |
4200 | * @new: s64 value to assign |
4201 | * |
4202 | * If (@v == @old), atomically updates @v to @new with relaxed ordering. |
4203 | * |
4204 | * Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere. |
4205 | * |
4206 | * Return: The original value of @v. |
4207 | */ |
4208 | static __always_inline s64 |
4209 | raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) |
4210 | { |
4211 | #if defined(arch_atomic64_cmpxchg_relaxed) |
4212 | return arch_atomic64_cmpxchg_relaxed(v, old, new); |
4213 | #elif defined(arch_atomic64_cmpxchg) |
4214 | return arch_atomic64_cmpxchg(v, old, new); |
4215 | #else |
4216 | return raw_cmpxchg_relaxed(&v->counter, old, new); |
4217 | #endif |
4218 | } |
4219 | |
4220 | /** |
4221 | * raw_atomic64_try_cmpxchg() - atomic compare and exchange with full ordering |
4222 | * @v: pointer to atomic64_t |
4223 | * @old: pointer to s64 value to compare with |
4224 | * @new: s64 value to assign |
4225 | * |
4226 | * If (@v == @old), atomically updates @v to @new with full ordering. |
4227 | * Otherwise, updates @old to the current value of @v. |
4228 | * |
4229 | * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere. |
4230 | * |
4231 | * Return: @true if the exchange occured, @false otherwise. |
4232 | */ |
4233 | static __always_inline bool |
4234 | raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) |
4235 | { |
4236 | #if defined(arch_atomic64_try_cmpxchg) |
4237 | return arch_atomic64_try_cmpxchg(v, old, new); |
4238 | #elif defined(arch_atomic64_try_cmpxchg_relaxed) |
4239 | bool ret; |
4240 | __atomic_pre_full_fence(); |
4241 | ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); |
4242 | __atomic_post_full_fence(); |
4243 | return ret; |
4244 | #else |
4245 | s64 r, o = *old; |
4246 | r = raw_atomic64_cmpxchg(v, o, new); |
4247 | if (unlikely(r != o)) |
4248 | *old = r; |
4249 | return likely(r == o); |
4250 | #endif |
4251 | } |
4252 | |
4253 | /** |
4254 | * raw_atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering |
4255 | * @v: pointer to atomic64_t |
4256 | * @old: pointer to s64 value to compare with |
4257 | * @new: s64 value to assign |
4258 | * |
4259 | * If (@v == @old), atomically updates @v to @new with acquire ordering. |
4260 | * Otherwise, updates @old to the current value of @v. |
4261 | * |
4262 | * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere. |
4263 | * |
4264 | * Return: @true if the exchange occured, @false otherwise. |
4265 | */ |
4266 | static __always_inline bool |
4267 | raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) |
4268 | { |
4269 | #if defined(arch_atomic64_try_cmpxchg_acquire) |
4270 | return arch_atomic64_try_cmpxchg_acquire(v, old, new); |
4271 | #elif defined(arch_atomic64_try_cmpxchg_relaxed) |
4272 | bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); |
4273 | __atomic_acquire_fence(); |
4274 | return ret; |
4275 | #elif defined(arch_atomic64_try_cmpxchg) |
4276 | return arch_atomic64_try_cmpxchg(v, old, new); |
4277 | #else |
4278 | s64 r, o = *old; |
4279 | r = raw_atomic64_cmpxchg_acquire(v, o, new); |
4280 | if (unlikely(r != o)) |
4281 | *old = r; |
4282 | return likely(r == o); |
4283 | #endif |
4284 | } |
4285 | |
4286 | /** |
4287 | * raw_atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering |
4288 | * @v: pointer to atomic64_t |
4289 | * @old: pointer to s64 value to compare with |
4290 | * @new: s64 value to assign |
4291 | * |
4292 | * If (@v == @old), atomically updates @v to @new with release ordering. |
4293 | * Otherwise, updates @old to the current value of @v. |
4294 | * |
4295 | * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere. |
4296 | * |
4297 | * Return: @true if the exchange occured, @false otherwise. |
4298 | */ |
4299 | static __always_inline bool |
4300 | raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) |
4301 | { |
4302 | #if defined(arch_atomic64_try_cmpxchg_release) |
4303 | return arch_atomic64_try_cmpxchg_release(v, old, new); |
4304 | #elif defined(arch_atomic64_try_cmpxchg_relaxed) |
4305 | __atomic_release_fence(); |
4306 | return arch_atomic64_try_cmpxchg_relaxed(v, old, new); |
4307 | #elif defined(arch_atomic64_try_cmpxchg) |
4308 | return arch_atomic64_try_cmpxchg(v, old, new); |
4309 | #else |
4310 | s64 r, o = *old; |
4311 | r = raw_atomic64_cmpxchg_release(v, o, new); |
4312 | if (unlikely(r != o)) |
4313 | *old = r; |
4314 | return likely(r == o); |
4315 | #endif |
4316 | } |
4317 | |
4318 | /** |
4319 | * raw_atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering |
4320 | * @v: pointer to atomic64_t |
4321 | * @old: pointer to s64 value to compare with |
4322 | * @new: s64 value to assign |
4323 | * |
4324 | * If (@v == @old), atomically updates @v to @new with relaxed ordering. |
4325 | * Otherwise, updates @old to the current value of @v. |
4326 | * |
4327 | * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere. |
4328 | * |
4329 | * Return: @true if the exchange occured, @false otherwise. |
4330 | */ |
4331 | static __always_inline bool |
4332 | raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) |
4333 | { |
4334 | #if defined(arch_atomic64_try_cmpxchg_relaxed) |
4335 | return arch_atomic64_try_cmpxchg_relaxed(v, old, new); |
4336 | #elif defined(arch_atomic64_try_cmpxchg) |
4337 | return arch_atomic64_try_cmpxchg(v, old, new); |
4338 | #else |
4339 | s64 r, o = *old; |
4340 | r = raw_atomic64_cmpxchg_relaxed(v, o, new); |
4341 | if (unlikely(r != o)) |
4342 | *old = r; |
4343 | return likely(r == o); |
4344 | #endif |
4345 | } |
4346 | |
4347 | /** |
4348 | * raw_atomic64_sub_and_test() - atomic subtract and test if zero with full ordering |
4349 | * @i: s64 value to add |
4350 | * @v: pointer to atomic64_t |
4351 | * |
4352 | * Atomically updates @v to (@v - @i) with full ordering. |
4353 | * |
4354 | * Safe to use in noinstr code; prefer atomic64_sub_and_test() elsewhere. |
4355 | * |
4356 | * Return: @true if the resulting value of @v is zero, @false otherwise. |
4357 | */ |
4358 | static __always_inline bool |
4359 | raw_atomic64_sub_and_test(s64 i, atomic64_t *v) |
4360 | { |
4361 | #if defined(arch_atomic64_sub_and_test) |
4362 | return arch_atomic64_sub_and_test(i, v); |
4363 | #else |
4364 | return raw_atomic64_sub_return(i, v) == 0; |
4365 | #endif |
4366 | } |
4367 | |
4368 | /** |
4369 | * raw_atomic64_dec_and_test() - atomic decrement and test if zero with full ordering |
4370 | * @v: pointer to atomic64_t |
4371 | * |
4372 | * Atomically updates @v to (@v - 1) with full ordering. |
4373 | * |
4374 | * Safe to use in noinstr code; prefer atomic64_dec_and_test() elsewhere. |
4375 | * |
4376 | * Return: @true if the resulting value of @v is zero, @false otherwise. |
4377 | */ |
4378 | static __always_inline bool |
4379 | raw_atomic64_dec_and_test(atomic64_t *v) |
4380 | { |
4381 | #if defined(arch_atomic64_dec_and_test) |
4382 | return arch_atomic64_dec_and_test(v); |
4383 | #else |
4384 | return raw_atomic64_dec_return(v) == 0; |
4385 | #endif |
4386 | } |
4387 | |
4388 | /** |
4389 | * raw_atomic64_inc_and_test() - atomic increment and test if zero with full ordering |
4390 | * @v: pointer to atomic64_t |
4391 | * |
4392 | * Atomically updates @v to (@v + 1) with full ordering. |
4393 | * |
4394 | * Safe to use in noinstr code; prefer atomic64_inc_and_test() elsewhere. |
4395 | * |
4396 | * Return: @true if the resulting value of @v is zero, @false otherwise. |
4397 | */ |
4398 | static __always_inline bool |
4399 | raw_atomic64_inc_and_test(atomic64_t *v) |
4400 | { |
4401 | #if defined(arch_atomic64_inc_and_test) |
4402 | return arch_atomic64_inc_and_test(v); |
4403 | #else |
4404 | return raw_atomic64_inc_return(v) == 0; |
4405 | #endif |
4406 | } |
4407 | |
4408 | /** |
4409 | * raw_atomic64_add_negative() - atomic add and test if negative with full ordering |
4410 | * @i: s64 value to add |
4411 | * @v: pointer to atomic64_t |
4412 | * |
4413 | * Atomically updates @v to (@v + @i) with full ordering. |
4414 | * |
4415 | * Safe to use in noinstr code; prefer atomic64_add_negative() elsewhere. |
4416 | * |
4417 | * Return: @true if the resulting value of @v is negative, @false otherwise. |
4418 | */ |
4419 | static __always_inline bool |
4420 | raw_atomic64_add_negative(s64 i, atomic64_t *v) |
4421 | { |
4422 | #if defined(arch_atomic64_add_negative) |
4423 | return arch_atomic64_add_negative(i, v); |
4424 | #elif defined(arch_atomic64_add_negative_relaxed) |
4425 | bool ret; |
4426 | __atomic_pre_full_fence(); |
4427 | ret = arch_atomic64_add_negative_relaxed(i, v); |
4428 | __atomic_post_full_fence(); |
4429 | return ret; |
4430 | #else |
4431 | return raw_atomic64_add_return(i, v) < 0; |
4432 | #endif |
4433 | } |
4434 | |
4435 | /** |
4436 | * raw_atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering |
4437 | * @i: s64 value to add |
4438 | * @v: pointer to atomic64_t |
4439 | * |
4440 | * Atomically updates @v to (@v + @i) with acquire ordering. |
4441 | * |
4442 | * Safe to use in noinstr code; prefer atomic64_add_negative_acquire() elsewhere. |
4443 | * |
4444 | * Return: @true if the resulting value of @v is negative, @false otherwise. |
4445 | */ |
4446 | static __always_inline bool |
4447 | raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) |
4448 | { |
4449 | #if defined(arch_atomic64_add_negative_acquire) |
4450 | return arch_atomic64_add_negative_acquire(i, v); |
4451 | #elif defined(arch_atomic64_add_negative_relaxed) |
4452 | bool ret = arch_atomic64_add_negative_relaxed(i, v); |
4453 | __atomic_acquire_fence(); |
4454 | return ret; |
4455 | #elif defined(arch_atomic64_add_negative) |
4456 | return arch_atomic64_add_negative(i, v); |
4457 | #else |
4458 | return raw_atomic64_add_return_acquire(i, v) < 0; |
4459 | #endif |
4460 | } |
4461 | |
4462 | /** |
4463 | * raw_atomic64_add_negative_release() - atomic add and test if negative with release ordering |
4464 | * @i: s64 value to add |
4465 | * @v: pointer to atomic64_t |
4466 | * |
4467 | * Atomically updates @v to (@v + @i) with release ordering. |
4468 | * |
4469 | * Safe to use in noinstr code; prefer atomic64_add_negative_release() elsewhere. |
4470 | * |
4471 | * Return: @true if the resulting value of @v is negative, @false otherwise. |
4472 | */ |
4473 | static __always_inline bool |
4474 | raw_atomic64_add_negative_release(s64 i, atomic64_t *v) |
4475 | { |
4476 | #if defined(arch_atomic64_add_negative_release) |
4477 | return arch_atomic64_add_negative_release(i, v); |
4478 | #elif defined(arch_atomic64_add_negative_relaxed) |
4479 | __atomic_release_fence(); |
4480 | return arch_atomic64_add_negative_relaxed(i, v); |
4481 | #elif defined(arch_atomic64_add_negative) |
4482 | return arch_atomic64_add_negative(i, v); |
4483 | #else |
4484 | return raw_atomic64_add_return_release(i, v) < 0; |
4485 | #endif |
4486 | } |
4487 | |
4488 | /** |
4489 | * raw_atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering |
4490 | * @i: s64 value to add |
4491 | * @v: pointer to atomic64_t |
4492 | * |
4493 | * Atomically updates @v to (@v + @i) with relaxed ordering. |
4494 | * |
4495 | * Safe to use in noinstr code; prefer atomic64_add_negative_relaxed() elsewhere. |
4496 | * |
4497 | * Return: @true if the resulting value of @v is negative, @false otherwise. |
4498 | */ |
4499 | static __always_inline bool |
4500 | raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) |
4501 | { |
4502 | #if defined(arch_atomic64_add_negative_relaxed) |
4503 | return arch_atomic64_add_negative_relaxed(i, v); |
4504 | #elif defined(arch_atomic64_add_negative) |
4505 | return arch_atomic64_add_negative(i, v); |
4506 | #else |
4507 | return raw_atomic64_add_return_relaxed(i, v) < 0; |
4508 | #endif |
4509 | } |
4510 | |
4511 | /** |
4512 | * raw_atomic64_fetch_add_unless() - atomic add unless value with full ordering |
4513 | * @v: pointer to atomic64_t |
4514 | * @a: s64 value to add |
4515 | * @u: s64 value to compare with |
4516 | * |
4517 | * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. |
4518 | * |
4519 | * Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere. |
4520 | * |
4521 | * Return: The original value of @v. |
4522 | */ |
4523 | static __always_inline s64 |
4524 | raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) |
4525 | { |
4526 | #if defined(arch_atomic64_fetch_add_unless) |
4527 | return arch_atomic64_fetch_add_unless(v, a, u); |
4528 | #else |
4529 | s64 c = raw_atomic64_read(v); |
4530 | |
4531 | do { |
4532 | if (unlikely(c == u)) |
4533 | break; |
4534 | } while (!raw_atomic64_try_cmpxchg(v, old: &c, new: c + a)); |
4535 | |
4536 | return c; |
4537 | #endif |
4538 | } |
4539 | |
4540 | /** |
4541 | * raw_atomic64_add_unless() - atomic add unless value with full ordering |
4542 | * @v: pointer to atomic64_t |
4543 | * @a: s64 value to add |
4544 | * @u: s64 value to compare with |
4545 | * |
4546 | * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. |
4547 | * |
4548 | * Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere. |
4549 | * |
4550 | * Return: @true if @v was updated, @false otherwise. |
4551 | */ |
4552 | static __always_inline bool |
4553 | raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) |
4554 | { |
4555 | #if defined(arch_atomic64_add_unless) |
4556 | return arch_atomic64_add_unless(v, a, u); |
4557 | #else |
4558 | return raw_atomic64_fetch_add_unless(v, a, u) != u; |
4559 | #endif |
4560 | } |
4561 | |
4562 | /** |
4563 | * raw_atomic64_inc_not_zero() - atomic increment unless zero with full ordering |
4564 | * @v: pointer to atomic64_t |
4565 | * |
4566 | * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. |
4567 | * |
4568 | * Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere. |
4569 | * |
4570 | * Return: @true if @v was updated, @false otherwise. |
4571 | */ |
4572 | static __always_inline bool |
4573 | raw_atomic64_inc_not_zero(atomic64_t *v) |
4574 | { |
4575 | #if defined(arch_atomic64_inc_not_zero) |
4576 | return arch_atomic64_inc_not_zero(v); |
4577 | #else |
4578 | return raw_atomic64_add_unless(v, a: 1, u: 0); |
4579 | #endif |
4580 | } |
4581 | |
4582 | /** |
4583 | * raw_atomic64_inc_unless_negative() - atomic increment unless negative with full ordering |
4584 | * @v: pointer to atomic64_t |
4585 | * |
4586 | * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. |
4587 | * |
4588 | * Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere. |
4589 | * |
4590 | * Return: @true if @v was updated, @false otherwise. |
4591 | */ |
4592 | static __always_inline bool |
4593 | raw_atomic64_inc_unless_negative(atomic64_t *v) |
4594 | { |
4595 | #if defined(arch_atomic64_inc_unless_negative) |
4596 | return arch_atomic64_inc_unless_negative(v); |
4597 | #else |
4598 | s64 c = raw_atomic64_read(v); |
4599 | |
4600 | do { |
4601 | if (unlikely(c < 0)) |
4602 | return false; |
4603 | } while (!raw_atomic64_try_cmpxchg(v, old: &c, new: c + 1)); |
4604 | |
4605 | return true; |
4606 | #endif |
4607 | } |
4608 | |
4609 | /** |
4610 | * raw_atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering |
4611 | * @v: pointer to atomic64_t |
4612 | * |
4613 | * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. |
4614 | * |
4615 | * Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere. |
4616 | * |
4617 | * Return: @true if @v was updated, @false otherwise. |
4618 | */ |
4619 | static __always_inline bool |
4620 | raw_atomic64_dec_unless_positive(atomic64_t *v) |
4621 | { |
4622 | #if defined(arch_atomic64_dec_unless_positive) |
4623 | return arch_atomic64_dec_unless_positive(v); |
4624 | #else |
4625 | s64 c = raw_atomic64_read(v); |
4626 | |
4627 | do { |
4628 | if (unlikely(c > 0)) |
4629 | return false; |
4630 | } while (!raw_atomic64_try_cmpxchg(v, old: &c, new: c - 1)); |
4631 | |
4632 | return true; |
4633 | #endif |
4634 | } |
4635 | |
4636 | /** |
4637 | * raw_atomic64_dec_if_positive() - atomic decrement if positive with full ordering |
4638 | * @v: pointer to atomic64_t |
4639 | * |
4640 | * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. |
4641 | * |
4642 | * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere. |
4643 | * |
4644 | * Return: The old value of (@v - 1), regardless of whether @v was updated. |
4645 | */ |
4646 | static __always_inline s64 |
4647 | raw_atomic64_dec_if_positive(atomic64_t *v) |
4648 | { |
4649 | #if defined(arch_atomic64_dec_if_positive) |
4650 | return arch_atomic64_dec_if_positive(v); |
4651 | #else |
4652 | s64 dec, c = raw_atomic64_read(v); |
4653 | |
4654 | do { |
4655 | dec = c - 1; |
4656 | if (unlikely(dec < 0)) |
4657 | break; |
4658 | } while (!raw_atomic64_try_cmpxchg(v, old: &c, new: dec)); |
4659 | |
4660 | return dec; |
4661 | #endif |
4662 | } |
4663 | |
4664 | #endif /* _LINUX_ATOMIC_FALLBACK_H */ |
4665 | // eec048affea735b8464f58e6d96992101f8f85f1 |
4666 | |