1// SPDX-License-Identifier: GPL-2.0
2
3// Generated by scripts/atomic/gen-atomic-fallback.sh
4// DO NOT MODIFY THIS FILE DIRECTLY
5
6#ifndef _LINUX_ATOMIC_FALLBACK_H
7#define _LINUX_ATOMIC_FALLBACK_H
8
9#include <linux/compiler.h>
10
11#if defined(arch_xchg)
12#define raw_xchg arch_xchg
13#elif defined(arch_xchg_relaxed)
14#define raw_xchg(...) \
15 __atomic_op_fence(arch_xchg, __VA_ARGS__)
16#else
17extern void raw_xchg_not_implemented(void);
18#define raw_xchg(...) raw_xchg_not_implemented()
19#endif
20
21#if defined(arch_xchg_acquire)
22#define raw_xchg_acquire arch_xchg_acquire
23#elif defined(arch_xchg_relaxed)
24#define raw_xchg_acquire(...) \
25 __atomic_op_acquire(arch_xchg, __VA_ARGS__)
26#elif defined(arch_xchg)
27#define raw_xchg_acquire arch_xchg
28#else
29extern void raw_xchg_acquire_not_implemented(void);
30#define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented()
31#endif
32
33#if defined(arch_xchg_release)
34#define raw_xchg_release arch_xchg_release
35#elif defined(arch_xchg_relaxed)
36#define raw_xchg_release(...) \
37 __atomic_op_release(arch_xchg, __VA_ARGS__)
38#elif defined(arch_xchg)
39#define raw_xchg_release arch_xchg
40#else
41extern void raw_xchg_release_not_implemented(void);
42#define raw_xchg_release(...) raw_xchg_release_not_implemented()
43#endif
44
45#if defined(arch_xchg_relaxed)
46#define raw_xchg_relaxed arch_xchg_relaxed
47#elif defined(arch_xchg)
48#define raw_xchg_relaxed arch_xchg
49#else
50extern void raw_xchg_relaxed_not_implemented(void);
51#define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented()
52#endif
53
54#if defined(arch_cmpxchg)
55#define raw_cmpxchg arch_cmpxchg
56#elif defined(arch_cmpxchg_relaxed)
57#define raw_cmpxchg(...) \
58 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
59#else
60extern void raw_cmpxchg_not_implemented(void);
61#define raw_cmpxchg(...) raw_cmpxchg_not_implemented()
62#endif
63
64#if defined(arch_cmpxchg_acquire)
65#define raw_cmpxchg_acquire arch_cmpxchg_acquire
66#elif defined(arch_cmpxchg_relaxed)
67#define raw_cmpxchg_acquire(...) \
68 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
69#elif defined(arch_cmpxchg)
70#define raw_cmpxchg_acquire arch_cmpxchg
71#else
72extern void raw_cmpxchg_acquire_not_implemented(void);
73#define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented()
74#endif
75
76#if defined(arch_cmpxchg_release)
77#define raw_cmpxchg_release arch_cmpxchg_release
78#elif defined(arch_cmpxchg_relaxed)
79#define raw_cmpxchg_release(...) \
80 __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
81#elif defined(arch_cmpxchg)
82#define raw_cmpxchg_release arch_cmpxchg
83#else
84extern void raw_cmpxchg_release_not_implemented(void);
85#define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented()
86#endif
87
88#if defined(arch_cmpxchg_relaxed)
89#define raw_cmpxchg_relaxed arch_cmpxchg_relaxed
90#elif defined(arch_cmpxchg)
91#define raw_cmpxchg_relaxed arch_cmpxchg
92#else
93extern void raw_cmpxchg_relaxed_not_implemented(void);
94#define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented()
95#endif
96
97#if defined(arch_cmpxchg64)
98#define raw_cmpxchg64 arch_cmpxchg64
99#elif defined(arch_cmpxchg64_relaxed)
100#define raw_cmpxchg64(...) \
101 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
102#else
103extern void raw_cmpxchg64_not_implemented(void);
104#define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented()
105#endif
106
107#if defined(arch_cmpxchg64_acquire)
108#define raw_cmpxchg64_acquire arch_cmpxchg64_acquire
109#elif defined(arch_cmpxchg64_relaxed)
110#define raw_cmpxchg64_acquire(...) \
111 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
112#elif defined(arch_cmpxchg64)
113#define raw_cmpxchg64_acquire arch_cmpxchg64
114#else
115extern void raw_cmpxchg64_acquire_not_implemented(void);
116#define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented()
117#endif
118
119#if defined(arch_cmpxchg64_release)
120#define raw_cmpxchg64_release arch_cmpxchg64_release
121#elif defined(arch_cmpxchg64_relaxed)
122#define raw_cmpxchg64_release(...) \
123 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
124#elif defined(arch_cmpxchg64)
125#define raw_cmpxchg64_release arch_cmpxchg64
126#else
127extern void raw_cmpxchg64_release_not_implemented(void);
128#define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented()
129#endif
130
131#if defined(arch_cmpxchg64_relaxed)
132#define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed
133#elif defined(arch_cmpxchg64)
134#define raw_cmpxchg64_relaxed arch_cmpxchg64
135#else
136extern void raw_cmpxchg64_relaxed_not_implemented(void);
137#define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented()
138#endif
139
140#if defined(arch_cmpxchg128)
141#define raw_cmpxchg128 arch_cmpxchg128
142#elif defined(arch_cmpxchg128_relaxed)
143#define raw_cmpxchg128(...) \
144 __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)
145#else
146extern void raw_cmpxchg128_not_implemented(void);
147#define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented()
148#endif
149
150#if defined(arch_cmpxchg128_acquire)
151#define raw_cmpxchg128_acquire arch_cmpxchg128_acquire
152#elif defined(arch_cmpxchg128_relaxed)
153#define raw_cmpxchg128_acquire(...) \
154 __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)
155#elif defined(arch_cmpxchg128)
156#define raw_cmpxchg128_acquire arch_cmpxchg128
157#else
158extern void raw_cmpxchg128_acquire_not_implemented(void);
159#define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented()
160#endif
161
162#if defined(arch_cmpxchg128_release)
163#define raw_cmpxchg128_release arch_cmpxchg128_release
164#elif defined(arch_cmpxchg128_relaxed)
165#define raw_cmpxchg128_release(...) \
166 __atomic_op_release(arch_cmpxchg128, __VA_ARGS__)
167#elif defined(arch_cmpxchg128)
168#define raw_cmpxchg128_release arch_cmpxchg128
169#else
170extern void raw_cmpxchg128_release_not_implemented(void);
171#define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented()
172#endif
173
174#if defined(arch_cmpxchg128_relaxed)
175#define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed
176#elif defined(arch_cmpxchg128)
177#define raw_cmpxchg128_relaxed arch_cmpxchg128
178#else
179extern void raw_cmpxchg128_relaxed_not_implemented(void);
180#define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented()
181#endif
182
183#if defined(arch_try_cmpxchg)
184#define raw_try_cmpxchg arch_try_cmpxchg
185#elif defined(arch_try_cmpxchg_relaxed)
186#define raw_try_cmpxchg(...) \
187 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
188#else
189#define raw_try_cmpxchg(_ptr, _oldp, _new) \
190({ \
191 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
192 ___r = raw_cmpxchg((_ptr), ___o, (_new)); \
193 if (unlikely(___r != ___o)) \
194 *___op = ___r; \
195 likely(___r == ___o); \
196})
197#endif
198
199#if defined(arch_try_cmpxchg_acquire)
200#define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire
201#elif defined(arch_try_cmpxchg_relaxed)
202#define raw_try_cmpxchg_acquire(...) \
203 __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
204#elif defined(arch_try_cmpxchg)
205#define raw_try_cmpxchg_acquire arch_try_cmpxchg
206#else
207#define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \
208({ \
209 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
210 ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \
211 if (unlikely(___r != ___o)) \
212 *___op = ___r; \
213 likely(___r == ___o); \
214})
215#endif
216
217#if defined(arch_try_cmpxchg_release)
218#define raw_try_cmpxchg_release arch_try_cmpxchg_release
219#elif defined(arch_try_cmpxchg_relaxed)
220#define raw_try_cmpxchg_release(...) \
221 __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
222#elif defined(arch_try_cmpxchg)
223#define raw_try_cmpxchg_release arch_try_cmpxchg
224#else
225#define raw_try_cmpxchg_release(_ptr, _oldp, _new) \
226({ \
227 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
228 ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \
229 if (unlikely(___r != ___o)) \
230 *___op = ___r; \
231 likely(___r == ___o); \
232})
233#endif
234
235#if defined(arch_try_cmpxchg_relaxed)
236#define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed
237#elif defined(arch_try_cmpxchg)
238#define raw_try_cmpxchg_relaxed arch_try_cmpxchg
239#else
240#define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
241({ \
242 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
243 ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \
244 if (unlikely(___r != ___o)) \
245 *___op = ___r; \
246 likely(___r == ___o); \
247})
248#endif
249
250#if defined(arch_try_cmpxchg64)
251#define raw_try_cmpxchg64 arch_try_cmpxchg64
252#elif defined(arch_try_cmpxchg64_relaxed)
253#define raw_try_cmpxchg64(...) \
254 __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
255#else
256#define raw_try_cmpxchg64(_ptr, _oldp, _new) \
257({ \
258 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
259 ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \
260 if (unlikely(___r != ___o)) \
261 *___op = ___r; \
262 likely(___r == ___o); \
263})
264#endif
265
266#if defined(arch_try_cmpxchg64_acquire)
267#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire
268#elif defined(arch_try_cmpxchg64_relaxed)
269#define raw_try_cmpxchg64_acquire(...) \
270 __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
271#elif defined(arch_try_cmpxchg64)
272#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64
273#else
274#define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
275({ \
276 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
277 ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \
278 if (unlikely(___r != ___o)) \
279 *___op = ___r; \
280 likely(___r == ___o); \
281})
282#endif
283
284#if defined(arch_try_cmpxchg64_release)
285#define raw_try_cmpxchg64_release arch_try_cmpxchg64_release
286#elif defined(arch_try_cmpxchg64_relaxed)
287#define raw_try_cmpxchg64_release(...) \
288 __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
289#elif defined(arch_try_cmpxchg64)
290#define raw_try_cmpxchg64_release arch_try_cmpxchg64
291#else
292#define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \
293({ \
294 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
295 ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \
296 if (unlikely(___r != ___o)) \
297 *___op = ___r; \
298 likely(___r == ___o); \
299})
300#endif
301
302#if defined(arch_try_cmpxchg64_relaxed)
303#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed
304#elif defined(arch_try_cmpxchg64)
305#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64
306#else
307#define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
308({ \
309 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
310 ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
311 if (unlikely(___r != ___o)) \
312 *___op = ___r; \
313 likely(___r == ___o); \
314})
315#endif
316
317#if defined(arch_try_cmpxchg128)
318#define raw_try_cmpxchg128 arch_try_cmpxchg128
319#elif defined(arch_try_cmpxchg128_relaxed)
320#define raw_try_cmpxchg128(...) \
321 __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)
322#else
323#define raw_try_cmpxchg128(_ptr, _oldp, _new) \
324({ \
325 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
326 ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \
327 if (unlikely(___r != ___o)) \
328 *___op = ___r; \
329 likely(___r == ___o); \
330})
331#endif
332
333#if defined(arch_try_cmpxchg128_acquire)
334#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire
335#elif defined(arch_try_cmpxchg128_relaxed)
336#define raw_try_cmpxchg128_acquire(...) \
337 __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)
338#elif defined(arch_try_cmpxchg128)
339#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128
340#else
341#define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \
342({ \
343 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
344 ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \
345 if (unlikely(___r != ___o)) \
346 *___op = ___r; \
347 likely(___r == ___o); \
348})
349#endif
350
351#if defined(arch_try_cmpxchg128_release)
352#define raw_try_cmpxchg128_release arch_try_cmpxchg128_release
353#elif defined(arch_try_cmpxchg128_relaxed)
354#define raw_try_cmpxchg128_release(...) \
355 __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)
356#elif defined(arch_try_cmpxchg128)
357#define raw_try_cmpxchg128_release arch_try_cmpxchg128
358#else
359#define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \
360({ \
361 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
362 ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \
363 if (unlikely(___r != ___o)) \
364 *___op = ___r; \
365 likely(___r == ___o); \
366})
367#endif
368
369#if defined(arch_try_cmpxchg128_relaxed)
370#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed
371#elif defined(arch_try_cmpxchg128)
372#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128
373#else
374#define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \
375({ \
376 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
377 ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \
378 if (unlikely(___r != ___o)) \
379 *___op = ___r; \
380 likely(___r == ___o); \
381})
382#endif
383
384#define raw_cmpxchg_local arch_cmpxchg_local
385
386#ifdef arch_try_cmpxchg_local
387#define raw_try_cmpxchg_local arch_try_cmpxchg_local
388#else
389#define raw_try_cmpxchg_local(_ptr, _oldp, _new) \
390({ \
391 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
392 ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \
393 if (unlikely(___r != ___o)) \
394 *___op = ___r; \
395 likely(___r == ___o); \
396})
397#endif
398
399#define raw_cmpxchg64_local arch_cmpxchg64_local
400
401#ifdef arch_try_cmpxchg64_local
402#define raw_try_cmpxchg64_local arch_try_cmpxchg64_local
403#else
404#define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \
405({ \
406 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
407 ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \
408 if (unlikely(___r != ___o)) \
409 *___op = ___r; \
410 likely(___r == ___o); \
411})
412#endif
413
414#define raw_cmpxchg128_local arch_cmpxchg128_local
415
416#ifdef arch_try_cmpxchg128_local
417#define raw_try_cmpxchg128_local arch_try_cmpxchg128_local
418#else
419#define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \
420({ \
421 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
422 ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \
423 if (unlikely(___r != ___o)) \
424 *___op = ___r; \
425 likely(___r == ___o); \
426})
427#endif
428
429#define raw_sync_cmpxchg arch_sync_cmpxchg
430
431#ifdef arch_sync_try_cmpxchg
432#define raw_sync_try_cmpxchg arch_sync_try_cmpxchg
433#else
434#define raw_sync_try_cmpxchg(_ptr, _oldp, _new) \
435({ \
436 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
437 ___r = raw_sync_cmpxchg((_ptr), ___o, (_new)); \
438 if (unlikely(___r != ___o)) \
439 *___op = ___r; \
440 likely(___r == ___o); \
441})
442#endif
443
444/**
445 * raw_atomic_read() - atomic load with relaxed ordering
446 * @v: pointer to atomic_t
447 *
448 * Atomically loads the value of @v with relaxed ordering.
449 *
450 * Safe to use in noinstr code; prefer atomic_read() elsewhere.
451 *
452 * Return: The value loaded from @v.
453 */
454static __always_inline int
455raw_atomic_read(const atomic_t *v)
456{
457 return arch_atomic_read(v);
458}
459
460/**
461 * raw_atomic_read_acquire() - atomic load with acquire ordering
462 * @v: pointer to atomic_t
463 *
464 * Atomically loads the value of @v with acquire ordering.
465 *
466 * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere.
467 *
468 * Return: The value loaded from @v.
469 */
470static __always_inline int
471raw_atomic_read_acquire(const atomic_t *v)
472{
473#if defined(arch_atomic_read_acquire)
474 return arch_atomic_read_acquire(v);
475#else
476 int ret;
477
478 if (__native_word(atomic_t)) {
479 ret = smp_load_acquire(&(v)->counter);
480 } else {
481 ret = raw_atomic_read(v);
482 __atomic_acquire_fence();
483 }
484
485 return ret;
486#endif
487}
488
489/**
490 * raw_atomic_set() - atomic set with relaxed ordering
491 * @v: pointer to atomic_t
492 * @i: int value to assign
493 *
494 * Atomically sets @v to @i with relaxed ordering.
495 *
496 * Safe to use in noinstr code; prefer atomic_set() elsewhere.
497 *
498 * Return: Nothing.
499 */
500static __always_inline void
501raw_atomic_set(atomic_t *v, int i)
502{
503 arch_atomic_set(v, i);
504}
505
506/**
507 * raw_atomic_set_release() - atomic set with release ordering
508 * @v: pointer to atomic_t
509 * @i: int value to assign
510 *
511 * Atomically sets @v to @i with release ordering.
512 *
513 * Safe to use in noinstr code; prefer atomic_set_release() elsewhere.
514 *
515 * Return: Nothing.
516 */
517static __always_inline void
518raw_atomic_set_release(atomic_t *v, int i)
519{
520#if defined(arch_atomic_set_release)
521 arch_atomic_set_release(v, i);
522#else
523 if (__native_word(atomic_t)) {
524 smp_store_release(&(v)->counter, i);
525 } else {
526 __atomic_release_fence();
527 raw_atomic_set(v, i);
528 }
529#endif
530}
531
532/**
533 * raw_atomic_add() - atomic add with relaxed ordering
534 * @i: int value to add
535 * @v: pointer to atomic_t
536 *
537 * Atomically updates @v to (@v + @i) with relaxed ordering.
538 *
539 * Safe to use in noinstr code; prefer atomic_add() elsewhere.
540 *
541 * Return: Nothing.
542 */
543static __always_inline void
544raw_atomic_add(int i, atomic_t *v)
545{
546 arch_atomic_add(i, v);
547}
548
549/**
550 * raw_atomic_add_return() - atomic add with full ordering
551 * @i: int value to add
552 * @v: pointer to atomic_t
553 *
554 * Atomically updates @v to (@v + @i) with full ordering.
555 *
556 * Safe to use in noinstr code; prefer atomic_add_return() elsewhere.
557 *
558 * Return: The updated value of @v.
559 */
560static __always_inline int
561raw_atomic_add_return(int i, atomic_t *v)
562{
563#if defined(arch_atomic_add_return)
564 return arch_atomic_add_return(i, v);
565#elif defined(arch_atomic_add_return_relaxed)
566 int ret;
567 __atomic_pre_full_fence();
568 ret = arch_atomic_add_return_relaxed(i, v);
569 __atomic_post_full_fence();
570 return ret;
571#else
572#error "Unable to define raw_atomic_add_return"
573#endif
574}
575
576/**
577 * raw_atomic_add_return_acquire() - atomic add with acquire ordering
578 * @i: int value to add
579 * @v: pointer to atomic_t
580 *
581 * Atomically updates @v to (@v + @i) with acquire ordering.
582 *
583 * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere.
584 *
585 * Return: The updated value of @v.
586 */
587static __always_inline int
588raw_atomic_add_return_acquire(int i, atomic_t *v)
589{
590#if defined(arch_atomic_add_return_acquire)
591 return arch_atomic_add_return_acquire(i, v);
592#elif defined(arch_atomic_add_return_relaxed)
593 int ret = arch_atomic_add_return_relaxed(i, v);
594 __atomic_acquire_fence();
595 return ret;
596#elif defined(arch_atomic_add_return)
597 return arch_atomic_add_return(i, v);
598#else
599#error "Unable to define raw_atomic_add_return_acquire"
600#endif
601}
602
603/**
604 * raw_atomic_add_return_release() - atomic add with release ordering
605 * @i: int value to add
606 * @v: pointer to atomic_t
607 *
608 * Atomically updates @v to (@v + @i) with release ordering.
609 *
610 * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere.
611 *
612 * Return: The updated value of @v.
613 */
614static __always_inline int
615raw_atomic_add_return_release(int i, atomic_t *v)
616{
617#if defined(arch_atomic_add_return_release)
618 return arch_atomic_add_return_release(i, v);
619#elif defined(arch_atomic_add_return_relaxed)
620 __atomic_release_fence();
621 return arch_atomic_add_return_relaxed(i, v);
622#elif defined(arch_atomic_add_return)
623 return arch_atomic_add_return(i, v);
624#else
625#error "Unable to define raw_atomic_add_return_release"
626#endif
627}
628
629/**
630 * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering
631 * @i: int value to add
632 * @v: pointer to atomic_t
633 *
634 * Atomically updates @v to (@v + @i) with relaxed ordering.
635 *
636 * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere.
637 *
638 * Return: The updated value of @v.
639 */
640static __always_inline int
641raw_atomic_add_return_relaxed(int i, atomic_t *v)
642{
643#if defined(arch_atomic_add_return_relaxed)
644 return arch_atomic_add_return_relaxed(i, v);
645#elif defined(arch_atomic_add_return)
646 return arch_atomic_add_return(i, v);
647#else
648#error "Unable to define raw_atomic_add_return_relaxed"
649#endif
650}
651
652/**
653 * raw_atomic_fetch_add() - atomic add with full ordering
654 * @i: int value to add
655 * @v: pointer to atomic_t
656 *
657 * Atomically updates @v to (@v + @i) with full ordering.
658 *
659 * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere.
660 *
661 * Return: The original value of @v.
662 */
663static __always_inline int
664raw_atomic_fetch_add(int i, atomic_t *v)
665{
666#if defined(arch_atomic_fetch_add)
667 return arch_atomic_fetch_add(i, v);
668#elif defined(arch_atomic_fetch_add_relaxed)
669 int ret;
670 __atomic_pre_full_fence();
671 ret = arch_atomic_fetch_add_relaxed(i, v);
672 __atomic_post_full_fence();
673 return ret;
674#else
675#error "Unable to define raw_atomic_fetch_add"
676#endif
677}
678
679/**
680 * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering
681 * @i: int value to add
682 * @v: pointer to atomic_t
683 *
684 * Atomically updates @v to (@v + @i) with acquire ordering.
685 *
686 * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere.
687 *
688 * Return: The original value of @v.
689 */
690static __always_inline int
691raw_atomic_fetch_add_acquire(int i, atomic_t *v)
692{
693#if defined(arch_atomic_fetch_add_acquire)
694 return arch_atomic_fetch_add_acquire(i, v);
695#elif defined(arch_atomic_fetch_add_relaxed)
696 int ret = arch_atomic_fetch_add_relaxed(i, v);
697 __atomic_acquire_fence();
698 return ret;
699#elif defined(arch_atomic_fetch_add)
700 return arch_atomic_fetch_add(i, v);
701#else
702#error "Unable to define raw_atomic_fetch_add_acquire"
703#endif
704}
705
706/**
707 * raw_atomic_fetch_add_release() - atomic add with release ordering
708 * @i: int value to add
709 * @v: pointer to atomic_t
710 *
711 * Atomically updates @v to (@v + @i) with release ordering.
712 *
713 * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere.
714 *
715 * Return: The original value of @v.
716 */
717static __always_inline int
718raw_atomic_fetch_add_release(int i, atomic_t *v)
719{
720#if defined(arch_atomic_fetch_add_release)
721 return arch_atomic_fetch_add_release(i, v);
722#elif defined(arch_atomic_fetch_add_relaxed)
723 __atomic_release_fence();
724 return arch_atomic_fetch_add_relaxed(i, v);
725#elif defined(arch_atomic_fetch_add)
726 return arch_atomic_fetch_add(i, v);
727#else
728#error "Unable to define raw_atomic_fetch_add_release"
729#endif
730}
731
732/**
733 * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering
734 * @i: int value to add
735 * @v: pointer to atomic_t
736 *
737 * Atomically updates @v to (@v + @i) with relaxed ordering.
738 *
739 * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere.
740 *
741 * Return: The original value of @v.
742 */
743static __always_inline int
744raw_atomic_fetch_add_relaxed(int i, atomic_t *v)
745{
746#if defined(arch_atomic_fetch_add_relaxed)
747 return arch_atomic_fetch_add_relaxed(i, v);
748#elif defined(arch_atomic_fetch_add)
749 return arch_atomic_fetch_add(i, v);
750#else
751#error "Unable to define raw_atomic_fetch_add_relaxed"
752#endif
753}
754
755/**
756 * raw_atomic_sub() - atomic subtract with relaxed ordering
757 * @i: int value to subtract
758 * @v: pointer to atomic_t
759 *
760 * Atomically updates @v to (@v - @i) with relaxed ordering.
761 *
762 * Safe to use in noinstr code; prefer atomic_sub() elsewhere.
763 *
764 * Return: Nothing.
765 */
766static __always_inline void
767raw_atomic_sub(int i, atomic_t *v)
768{
769 arch_atomic_sub(i, v);
770}
771
772/**
773 * raw_atomic_sub_return() - atomic subtract with full ordering
774 * @i: int value to subtract
775 * @v: pointer to atomic_t
776 *
777 * Atomically updates @v to (@v - @i) with full ordering.
778 *
779 * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere.
780 *
781 * Return: The updated value of @v.
782 */
783static __always_inline int
784raw_atomic_sub_return(int i, atomic_t *v)
785{
786#if defined(arch_atomic_sub_return)
787 return arch_atomic_sub_return(i, v);
788#elif defined(arch_atomic_sub_return_relaxed)
789 int ret;
790 __atomic_pre_full_fence();
791 ret = arch_atomic_sub_return_relaxed(i, v);
792 __atomic_post_full_fence();
793 return ret;
794#else
795#error "Unable to define raw_atomic_sub_return"
796#endif
797}
798
799/**
800 * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering
801 * @i: int value to subtract
802 * @v: pointer to atomic_t
803 *
804 * Atomically updates @v to (@v - @i) with acquire ordering.
805 *
806 * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere.
807 *
808 * Return: The updated value of @v.
809 */
810static __always_inline int
811raw_atomic_sub_return_acquire(int i, atomic_t *v)
812{
813#if defined(arch_atomic_sub_return_acquire)
814 return arch_atomic_sub_return_acquire(i, v);
815#elif defined(arch_atomic_sub_return_relaxed)
816 int ret = arch_atomic_sub_return_relaxed(i, v);
817 __atomic_acquire_fence();
818 return ret;
819#elif defined(arch_atomic_sub_return)
820 return arch_atomic_sub_return(i, v);
821#else
822#error "Unable to define raw_atomic_sub_return_acquire"
823#endif
824}
825
826/**
827 * raw_atomic_sub_return_release() - atomic subtract with release ordering
828 * @i: int value to subtract
829 * @v: pointer to atomic_t
830 *
831 * Atomically updates @v to (@v - @i) with release ordering.
832 *
833 * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere.
834 *
835 * Return: The updated value of @v.
836 */
837static __always_inline int
838raw_atomic_sub_return_release(int i, atomic_t *v)
839{
840#if defined(arch_atomic_sub_return_release)
841 return arch_atomic_sub_return_release(i, v);
842#elif defined(arch_atomic_sub_return_relaxed)
843 __atomic_release_fence();
844 return arch_atomic_sub_return_relaxed(i, v);
845#elif defined(arch_atomic_sub_return)
846 return arch_atomic_sub_return(i, v);
847#else
848#error "Unable to define raw_atomic_sub_return_release"
849#endif
850}
851
852/**
853 * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
854 * @i: int value to subtract
855 * @v: pointer to atomic_t
856 *
857 * Atomically updates @v to (@v - @i) with relaxed ordering.
858 *
859 * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere.
860 *
861 * Return: The updated value of @v.
862 */
863static __always_inline int
864raw_atomic_sub_return_relaxed(int i, atomic_t *v)
865{
866#if defined(arch_atomic_sub_return_relaxed)
867 return arch_atomic_sub_return_relaxed(i, v);
868#elif defined(arch_atomic_sub_return)
869 return arch_atomic_sub_return(i, v);
870#else
871#error "Unable to define raw_atomic_sub_return_relaxed"
872#endif
873}
874
875/**
876 * raw_atomic_fetch_sub() - atomic subtract with full ordering
877 * @i: int value to subtract
878 * @v: pointer to atomic_t
879 *
880 * Atomically updates @v to (@v - @i) with full ordering.
881 *
882 * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere.
883 *
884 * Return: The original value of @v.
885 */
886static __always_inline int
887raw_atomic_fetch_sub(int i, atomic_t *v)
888{
889#if defined(arch_atomic_fetch_sub)
890 return arch_atomic_fetch_sub(i, v);
891#elif defined(arch_atomic_fetch_sub_relaxed)
892 int ret;
893 __atomic_pre_full_fence();
894 ret = arch_atomic_fetch_sub_relaxed(i, v);
895 __atomic_post_full_fence();
896 return ret;
897#else
898#error "Unable to define raw_atomic_fetch_sub"
899#endif
900}
901
902/**
903 * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
904 * @i: int value to subtract
905 * @v: pointer to atomic_t
906 *
907 * Atomically updates @v to (@v - @i) with acquire ordering.
908 *
909 * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere.
910 *
911 * Return: The original value of @v.
912 */
913static __always_inline int
914raw_atomic_fetch_sub_acquire(int i, atomic_t *v)
915{
916#if defined(arch_atomic_fetch_sub_acquire)
917 return arch_atomic_fetch_sub_acquire(i, v);
918#elif defined(arch_atomic_fetch_sub_relaxed)
919 int ret = arch_atomic_fetch_sub_relaxed(i, v);
920 __atomic_acquire_fence();
921 return ret;
922#elif defined(arch_atomic_fetch_sub)
923 return arch_atomic_fetch_sub(i, v);
924#else
925#error "Unable to define raw_atomic_fetch_sub_acquire"
926#endif
927}
928
929/**
930 * raw_atomic_fetch_sub_release() - atomic subtract with release ordering
931 * @i: int value to subtract
932 * @v: pointer to atomic_t
933 *
934 * Atomically updates @v to (@v - @i) with release ordering.
935 *
936 * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere.
937 *
938 * Return: The original value of @v.
939 */
940static __always_inline int
941raw_atomic_fetch_sub_release(int i, atomic_t *v)
942{
943#if defined(arch_atomic_fetch_sub_release)
944 return arch_atomic_fetch_sub_release(i, v);
945#elif defined(arch_atomic_fetch_sub_relaxed)
946 __atomic_release_fence();
947 return arch_atomic_fetch_sub_relaxed(i, v);
948#elif defined(arch_atomic_fetch_sub)
949 return arch_atomic_fetch_sub(i, v);
950#else
951#error "Unable to define raw_atomic_fetch_sub_release"
952#endif
953}
954
955/**
956 * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
957 * @i: int value to subtract
958 * @v: pointer to atomic_t
959 *
960 * Atomically updates @v to (@v - @i) with relaxed ordering.
961 *
962 * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere.
963 *
964 * Return: The original value of @v.
965 */
966static __always_inline int
967raw_atomic_fetch_sub_relaxed(int i, atomic_t *v)
968{
969#if defined(arch_atomic_fetch_sub_relaxed)
970 return arch_atomic_fetch_sub_relaxed(i, v);
971#elif defined(arch_atomic_fetch_sub)
972 return arch_atomic_fetch_sub(i, v);
973#else
974#error "Unable to define raw_atomic_fetch_sub_relaxed"
975#endif
976}
977
978/**
979 * raw_atomic_inc() - atomic increment with relaxed ordering
980 * @v: pointer to atomic_t
981 *
982 * Atomically updates @v to (@v + 1) with relaxed ordering.
983 *
984 * Safe to use in noinstr code; prefer atomic_inc() elsewhere.
985 *
986 * Return: Nothing.
987 */
988static __always_inline void
989raw_atomic_inc(atomic_t *v)
990{
991#if defined(arch_atomic_inc)
992 arch_atomic_inc(v);
993#else
994 raw_atomic_add(1, v);
995#endif
996}
997
998/**
999 * raw_atomic_inc_return() - atomic increment with full ordering
1000 * @v: pointer to atomic_t
1001 *
1002 * Atomically updates @v to (@v + 1) with full ordering.
1003 *
1004 * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere.
1005 *
1006 * Return: The updated value of @v.
1007 */
1008static __always_inline int
1009raw_atomic_inc_return(atomic_t *v)
1010{
1011#if defined(arch_atomic_inc_return)
1012 return arch_atomic_inc_return(v);
1013#elif defined(arch_atomic_inc_return_relaxed)
1014 int ret;
1015 __atomic_pre_full_fence();
1016 ret = arch_atomic_inc_return_relaxed(v);
1017 __atomic_post_full_fence();
1018 return ret;
1019#else
1020 return raw_atomic_add_return(i: 1, v);
1021#endif
1022}
1023
1024/**
1025 * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering
1026 * @v: pointer to atomic_t
1027 *
1028 * Atomically updates @v to (@v + 1) with acquire ordering.
1029 *
1030 * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere.
1031 *
1032 * Return: The updated value of @v.
1033 */
1034static __always_inline int
1035raw_atomic_inc_return_acquire(atomic_t *v)
1036{
1037#if defined(arch_atomic_inc_return_acquire)
1038 return arch_atomic_inc_return_acquire(v);
1039#elif defined(arch_atomic_inc_return_relaxed)
1040 int ret = arch_atomic_inc_return_relaxed(v);
1041 __atomic_acquire_fence();
1042 return ret;
1043#elif defined(arch_atomic_inc_return)
1044 return arch_atomic_inc_return(v);
1045#else
1046 return raw_atomic_add_return_acquire(i: 1, v);
1047#endif
1048}
1049
1050/**
1051 * raw_atomic_inc_return_release() - atomic increment with release ordering
1052 * @v: pointer to atomic_t
1053 *
1054 * Atomically updates @v to (@v + 1) with release ordering.
1055 *
1056 * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere.
1057 *
1058 * Return: The updated value of @v.
1059 */
1060static __always_inline int
1061raw_atomic_inc_return_release(atomic_t *v)
1062{
1063#if defined(arch_atomic_inc_return_release)
1064 return arch_atomic_inc_return_release(v);
1065#elif defined(arch_atomic_inc_return_relaxed)
1066 __atomic_release_fence();
1067 return arch_atomic_inc_return_relaxed(v);
1068#elif defined(arch_atomic_inc_return)
1069 return arch_atomic_inc_return(v);
1070#else
1071 return raw_atomic_add_return_release(i: 1, v);
1072#endif
1073}
1074
1075/**
1076 * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering
1077 * @v: pointer to atomic_t
1078 *
1079 * Atomically updates @v to (@v + 1) with relaxed ordering.
1080 *
1081 * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere.
1082 *
1083 * Return: The updated value of @v.
1084 */
1085static __always_inline int
1086raw_atomic_inc_return_relaxed(atomic_t *v)
1087{
1088#if defined(arch_atomic_inc_return_relaxed)
1089 return arch_atomic_inc_return_relaxed(v);
1090#elif defined(arch_atomic_inc_return)
1091 return arch_atomic_inc_return(v);
1092#else
1093 return raw_atomic_add_return_relaxed(i: 1, v);
1094#endif
1095}
1096
1097/**
1098 * raw_atomic_fetch_inc() - atomic increment with full ordering
1099 * @v: pointer to atomic_t
1100 *
1101 * Atomically updates @v to (@v + 1) with full ordering.
1102 *
1103 * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere.
1104 *
1105 * Return: The original value of @v.
1106 */
1107static __always_inline int
1108raw_atomic_fetch_inc(atomic_t *v)
1109{
1110#if defined(arch_atomic_fetch_inc)
1111 return arch_atomic_fetch_inc(v);
1112#elif defined(arch_atomic_fetch_inc_relaxed)
1113 int ret;
1114 __atomic_pre_full_fence();
1115 ret = arch_atomic_fetch_inc_relaxed(v);
1116 __atomic_post_full_fence();
1117 return ret;
1118#else
1119 return raw_atomic_fetch_add(i: 1, v);
1120#endif
1121}
1122
1123/**
1124 * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering
1125 * @v: pointer to atomic_t
1126 *
1127 * Atomically updates @v to (@v + 1) with acquire ordering.
1128 *
1129 * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere.
1130 *
1131 * Return: The original value of @v.
1132 */
1133static __always_inline int
1134raw_atomic_fetch_inc_acquire(atomic_t *v)
1135{
1136#if defined(arch_atomic_fetch_inc_acquire)
1137 return arch_atomic_fetch_inc_acquire(v);
1138#elif defined(arch_atomic_fetch_inc_relaxed)
1139 int ret = arch_atomic_fetch_inc_relaxed(v);
1140 __atomic_acquire_fence();
1141 return ret;
1142#elif defined(arch_atomic_fetch_inc)
1143 return arch_atomic_fetch_inc(v);
1144#else
1145 return raw_atomic_fetch_add_acquire(i: 1, v);
1146#endif
1147}
1148
1149/**
1150 * raw_atomic_fetch_inc_release() - atomic increment with release ordering
1151 * @v: pointer to atomic_t
1152 *
1153 * Atomically updates @v to (@v + 1) with release ordering.
1154 *
1155 * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere.
1156 *
1157 * Return: The original value of @v.
1158 */
1159static __always_inline int
1160raw_atomic_fetch_inc_release(atomic_t *v)
1161{
1162#if defined(arch_atomic_fetch_inc_release)
1163 return arch_atomic_fetch_inc_release(v);
1164#elif defined(arch_atomic_fetch_inc_relaxed)
1165 __atomic_release_fence();
1166 return arch_atomic_fetch_inc_relaxed(v);
1167#elif defined(arch_atomic_fetch_inc)
1168 return arch_atomic_fetch_inc(v);
1169#else
1170 return raw_atomic_fetch_add_release(i: 1, v);
1171#endif
1172}
1173
1174/**
1175 * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
1176 * @v: pointer to atomic_t
1177 *
1178 * Atomically updates @v to (@v + 1) with relaxed ordering.
1179 *
1180 * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere.
1181 *
1182 * Return: The original value of @v.
1183 */
1184static __always_inline int
1185raw_atomic_fetch_inc_relaxed(atomic_t *v)
1186{
1187#if defined(arch_atomic_fetch_inc_relaxed)
1188 return arch_atomic_fetch_inc_relaxed(v);
1189#elif defined(arch_atomic_fetch_inc)
1190 return arch_atomic_fetch_inc(v);
1191#else
1192 return raw_atomic_fetch_add_relaxed(i: 1, v);
1193#endif
1194}
1195
1196/**
1197 * raw_atomic_dec() - atomic decrement with relaxed ordering
1198 * @v: pointer to atomic_t
1199 *
1200 * Atomically updates @v to (@v - 1) with relaxed ordering.
1201 *
1202 * Safe to use in noinstr code; prefer atomic_dec() elsewhere.
1203 *
1204 * Return: Nothing.
1205 */
1206static __always_inline void
1207raw_atomic_dec(atomic_t *v)
1208{
1209#if defined(arch_atomic_dec)
1210 arch_atomic_dec(v);
1211#else
1212 raw_atomic_sub(1, v);
1213#endif
1214}
1215
1216/**
1217 * raw_atomic_dec_return() - atomic decrement with full ordering
1218 * @v: pointer to atomic_t
1219 *
1220 * Atomically updates @v to (@v - 1) with full ordering.
1221 *
1222 * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere.
1223 *
1224 * Return: The updated value of @v.
1225 */
1226static __always_inline int
1227raw_atomic_dec_return(atomic_t *v)
1228{
1229#if defined(arch_atomic_dec_return)
1230 return arch_atomic_dec_return(v);
1231#elif defined(arch_atomic_dec_return_relaxed)
1232 int ret;
1233 __atomic_pre_full_fence();
1234 ret = arch_atomic_dec_return_relaxed(v);
1235 __atomic_post_full_fence();
1236 return ret;
1237#else
1238 return raw_atomic_sub_return(i: 1, v);
1239#endif
1240}
1241
1242/**
1243 * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering
1244 * @v: pointer to atomic_t
1245 *
1246 * Atomically updates @v to (@v - 1) with acquire ordering.
1247 *
1248 * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere.
1249 *
1250 * Return: The updated value of @v.
1251 */
1252static __always_inline int
1253raw_atomic_dec_return_acquire(atomic_t *v)
1254{
1255#if defined(arch_atomic_dec_return_acquire)
1256 return arch_atomic_dec_return_acquire(v);
1257#elif defined(arch_atomic_dec_return_relaxed)
1258 int ret = arch_atomic_dec_return_relaxed(v);
1259 __atomic_acquire_fence();
1260 return ret;
1261#elif defined(arch_atomic_dec_return)
1262 return arch_atomic_dec_return(v);
1263#else
1264 return raw_atomic_sub_return_acquire(i: 1, v);
1265#endif
1266}
1267
1268/**
1269 * raw_atomic_dec_return_release() - atomic decrement with release ordering
1270 * @v: pointer to atomic_t
1271 *
1272 * Atomically updates @v to (@v - 1) with release ordering.
1273 *
1274 * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere.
1275 *
1276 * Return: The updated value of @v.
1277 */
1278static __always_inline int
1279raw_atomic_dec_return_release(atomic_t *v)
1280{
1281#if defined(arch_atomic_dec_return_release)
1282 return arch_atomic_dec_return_release(v);
1283#elif defined(arch_atomic_dec_return_relaxed)
1284 __atomic_release_fence();
1285 return arch_atomic_dec_return_relaxed(v);
1286#elif defined(arch_atomic_dec_return)
1287 return arch_atomic_dec_return(v);
1288#else
1289 return raw_atomic_sub_return_release(i: 1, v);
1290#endif
1291}
1292
1293/**
1294 * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
1295 * @v: pointer to atomic_t
1296 *
1297 * Atomically updates @v to (@v - 1) with relaxed ordering.
1298 *
1299 * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere.
1300 *
1301 * Return: The updated value of @v.
1302 */
1303static __always_inline int
1304raw_atomic_dec_return_relaxed(atomic_t *v)
1305{
1306#if defined(arch_atomic_dec_return_relaxed)
1307 return arch_atomic_dec_return_relaxed(v);
1308#elif defined(arch_atomic_dec_return)
1309 return arch_atomic_dec_return(v);
1310#else
1311 return raw_atomic_sub_return_relaxed(i: 1, v);
1312#endif
1313}
1314
1315/**
1316 * raw_atomic_fetch_dec() - atomic decrement with full ordering
1317 * @v: pointer to atomic_t
1318 *
1319 * Atomically updates @v to (@v - 1) with full ordering.
1320 *
1321 * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere.
1322 *
1323 * Return: The original value of @v.
1324 */
1325static __always_inline int
1326raw_atomic_fetch_dec(atomic_t *v)
1327{
1328#if defined(arch_atomic_fetch_dec)
1329 return arch_atomic_fetch_dec(v);
1330#elif defined(arch_atomic_fetch_dec_relaxed)
1331 int ret;
1332 __atomic_pre_full_fence();
1333 ret = arch_atomic_fetch_dec_relaxed(v);
1334 __atomic_post_full_fence();
1335 return ret;
1336#else
1337 return raw_atomic_fetch_sub(i: 1, v);
1338#endif
1339}
1340
1341/**
1342 * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
1343 * @v: pointer to atomic_t
1344 *
1345 * Atomically updates @v to (@v - 1) with acquire ordering.
1346 *
1347 * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere.
1348 *
1349 * Return: The original value of @v.
1350 */
1351static __always_inline int
1352raw_atomic_fetch_dec_acquire(atomic_t *v)
1353{
1354#if defined(arch_atomic_fetch_dec_acquire)
1355 return arch_atomic_fetch_dec_acquire(v);
1356#elif defined(arch_atomic_fetch_dec_relaxed)
1357 int ret = arch_atomic_fetch_dec_relaxed(v);
1358 __atomic_acquire_fence();
1359 return ret;
1360#elif defined(arch_atomic_fetch_dec)
1361 return arch_atomic_fetch_dec(v);
1362#else
1363 return raw_atomic_fetch_sub_acquire(i: 1, v);
1364#endif
1365}
1366
1367/**
1368 * raw_atomic_fetch_dec_release() - atomic decrement with release ordering
1369 * @v: pointer to atomic_t
1370 *
1371 * Atomically updates @v to (@v - 1) with release ordering.
1372 *
1373 * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere.
1374 *
1375 * Return: The original value of @v.
1376 */
1377static __always_inline int
1378raw_atomic_fetch_dec_release(atomic_t *v)
1379{
1380#if defined(arch_atomic_fetch_dec_release)
1381 return arch_atomic_fetch_dec_release(v);
1382#elif defined(arch_atomic_fetch_dec_relaxed)
1383 __atomic_release_fence();
1384 return arch_atomic_fetch_dec_relaxed(v);
1385#elif defined(arch_atomic_fetch_dec)
1386 return arch_atomic_fetch_dec(v);
1387#else
1388 return raw_atomic_fetch_sub_release(i: 1, v);
1389#endif
1390}
1391
1392/**
1393 * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
1394 * @v: pointer to atomic_t
1395 *
1396 * Atomically updates @v to (@v - 1) with relaxed ordering.
1397 *
1398 * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere.
1399 *
1400 * Return: The original value of @v.
1401 */
1402static __always_inline int
1403raw_atomic_fetch_dec_relaxed(atomic_t *v)
1404{
1405#if defined(arch_atomic_fetch_dec_relaxed)
1406 return arch_atomic_fetch_dec_relaxed(v);
1407#elif defined(arch_atomic_fetch_dec)
1408 return arch_atomic_fetch_dec(v);
1409#else
1410 return raw_atomic_fetch_sub_relaxed(i: 1, v);
1411#endif
1412}
1413
1414/**
1415 * raw_atomic_and() - atomic bitwise AND with relaxed ordering
1416 * @i: int value
1417 * @v: pointer to atomic_t
1418 *
1419 * Atomically updates @v to (@v & @i) with relaxed ordering.
1420 *
1421 * Safe to use in noinstr code; prefer atomic_and() elsewhere.
1422 *
1423 * Return: Nothing.
1424 */
1425static __always_inline void
1426raw_atomic_and(int i, atomic_t *v)
1427{
1428 arch_atomic_and(i, v);
1429}
1430
1431/**
1432 * raw_atomic_fetch_and() - atomic bitwise AND with full ordering
1433 * @i: int value
1434 * @v: pointer to atomic_t
1435 *
1436 * Atomically updates @v to (@v & @i) with full ordering.
1437 *
1438 * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere.
1439 *
1440 * Return: The original value of @v.
1441 */
1442static __always_inline int
1443raw_atomic_fetch_and(int i, atomic_t *v)
1444{
1445#if defined(arch_atomic_fetch_and)
1446 return arch_atomic_fetch_and(i, v);
1447#elif defined(arch_atomic_fetch_and_relaxed)
1448 int ret;
1449 __atomic_pre_full_fence();
1450 ret = arch_atomic_fetch_and_relaxed(i, v);
1451 __atomic_post_full_fence();
1452 return ret;
1453#else
1454#error "Unable to define raw_atomic_fetch_and"
1455#endif
1456}
1457
1458/**
1459 * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
1460 * @i: int value
1461 * @v: pointer to atomic_t
1462 *
1463 * Atomically updates @v to (@v & @i) with acquire ordering.
1464 *
1465 * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere.
1466 *
1467 * Return: The original value of @v.
1468 */
1469static __always_inline int
1470raw_atomic_fetch_and_acquire(int i, atomic_t *v)
1471{
1472#if defined(arch_atomic_fetch_and_acquire)
1473 return arch_atomic_fetch_and_acquire(i, v);
1474#elif defined(arch_atomic_fetch_and_relaxed)
1475 int ret = arch_atomic_fetch_and_relaxed(i, v);
1476 __atomic_acquire_fence();
1477 return ret;
1478#elif defined(arch_atomic_fetch_and)
1479 return arch_atomic_fetch_and(i, v);
1480#else
1481#error "Unable to define raw_atomic_fetch_and_acquire"
1482#endif
1483}
1484
1485/**
1486 * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering
1487 * @i: int value
1488 * @v: pointer to atomic_t
1489 *
1490 * Atomically updates @v to (@v & @i) with release ordering.
1491 *
1492 * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere.
1493 *
1494 * Return: The original value of @v.
1495 */
1496static __always_inline int
1497raw_atomic_fetch_and_release(int i, atomic_t *v)
1498{
1499#if defined(arch_atomic_fetch_and_release)
1500 return arch_atomic_fetch_and_release(i, v);
1501#elif defined(arch_atomic_fetch_and_relaxed)
1502 __atomic_release_fence();
1503 return arch_atomic_fetch_and_relaxed(i, v);
1504#elif defined(arch_atomic_fetch_and)
1505 return arch_atomic_fetch_and(i, v);
1506#else
1507#error "Unable to define raw_atomic_fetch_and_release"
1508#endif
1509}
1510
1511/**
1512 * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
1513 * @i: int value
1514 * @v: pointer to atomic_t
1515 *
1516 * Atomically updates @v to (@v & @i) with relaxed ordering.
1517 *
1518 * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere.
1519 *
1520 * Return: The original value of @v.
1521 */
1522static __always_inline int
1523raw_atomic_fetch_and_relaxed(int i, atomic_t *v)
1524{
1525#if defined(arch_atomic_fetch_and_relaxed)
1526 return arch_atomic_fetch_and_relaxed(i, v);
1527#elif defined(arch_atomic_fetch_and)
1528 return arch_atomic_fetch_and(i, v);
1529#else
1530#error "Unable to define raw_atomic_fetch_and_relaxed"
1531#endif
1532}
1533
1534/**
1535 * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
1536 * @i: int value
1537 * @v: pointer to atomic_t
1538 *
1539 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1540 *
1541 * Safe to use in noinstr code; prefer atomic_andnot() elsewhere.
1542 *
1543 * Return: Nothing.
1544 */
1545static __always_inline void
1546raw_atomic_andnot(int i, atomic_t *v)
1547{
1548#if defined(arch_atomic_andnot)
1549 arch_atomic_andnot(i, v);
1550#else
1551 raw_atomic_and(i: ~i, v);
1552#endif
1553}
1554
1555/**
1556 * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
1557 * @i: int value
1558 * @v: pointer to atomic_t
1559 *
1560 * Atomically updates @v to (@v & ~@i) with full ordering.
1561 *
1562 * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere.
1563 *
1564 * Return: The original value of @v.
1565 */
1566static __always_inline int
1567raw_atomic_fetch_andnot(int i, atomic_t *v)
1568{
1569#if defined(arch_atomic_fetch_andnot)
1570 return arch_atomic_fetch_andnot(i, v);
1571#elif defined(arch_atomic_fetch_andnot_relaxed)
1572 int ret;
1573 __atomic_pre_full_fence();
1574 ret = arch_atomic_fetch_andnot_relaxed(i, v);
1575 __atomic_post_full_fence();
1576 return ret;
1577#else
1578 return raw_atomic_fetch_and(i: ~i, v);
1579#endif
1580}
1581
1582/**
1583 * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
1584 * @i: int value
1585 * @v: pointer to atomic_t
1586 *
1587 * Atomically updates @v to (@v & ~@i) with acquire ordering.
1588 *
1589 * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere.
1590 *
1591 * Return: The original value of @v.
1592 */
1593static __always_inline int
1594raw_atomic_fetch_andnot_acquire(int i, atomic_t *v)
1595{
1596#if defined(arch_atomic_fetch_andnot_acquire)
1597 return arch_atomic_fetch_andnot_acquire(i, v);
1598#elif defined(arch_atomic_fetch_andnot_relaxed)
1599 int ret = arch_atomic_fetch_andnot_relaxed(i, v);
1600 __atomic_acquire_fence();
1601 return ret;
1602#elif defined(arch_atomic_fetch_andnot)
1603 return arch_atomic_fetch_andnot(i, v);
1604#else
1605 return raw_atomic_fetch_and_acquire(i: ~i, v);
1606#endif
1607}
1608
1609/**
1610 * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
1611 * @i: int value
1612 * @v: pointer to atomic_t
1613 *
1614 * Atomically updates @v to (@v & ~@i) with release ordering.
1615 *
1616 * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere.
1617 *
1618 * Return: The original value of @v.
1619 */
1620static __always_inline int
1621raw_atomic_fetch_andnot_release(int i, atomic_t *v)
1622{
1623#if defined(arch_atomic_fetch_andnot_release)
1624 return arch_atomic_fetch_andnot_release(i, v);
1625#elif defined(arch_atomic_fetch_andnot_relaxed)
1626 __atomic_release_fence();
1627 return arch_atomic_fetch_andnot_relaxed(i, v);
1628#elif defined(arch_atomic_fetch_andnot)
1629 return arch_atomic_fetch_andnot(i, v);
1630#else
1631 return raw_atomic_fetch_and_release(i: ~i, v);
1632#endif
1633}
1634
1635/**
1636 * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
1637 * @i: int value
1638 * @v: pointer to atomic_t
1639 *
1640 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1641 *
1642 * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere.
1643 *
1644 * Return: The original value of @v.
1645 */
1646static __always_inline int
1647raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
1648{
1649#if defined(arch_atomic_fetch_andnot_relaxed)
1650 return arch_atomic_fetch_andnot_relaxed(i, v);
1651#elif defined(arch_atomic_fetch_andnot)
1652 return arch_atomic_fetch_andnot(i, v);
1653#else
1654 return raw_atomic_fetch_and_relaxed(i: ~i, v);
1655#endif
1656}
1657
1658/**
1659 * raw_atomic_or() - atomic bitwise OR with relaxed ordering
1660 * @i: int value
1661 * @v: pointer to atomic_t
1662 *
1663 * Atomically updates @v to (@v | @i) with relaxed ordering.
1664 *
1665 * Safe to use in noinstr code; prefer atomic_or() elsewhere.
1666 *
1667 * Return: Nothing.
1668 */
1669static __always_inline void
1670raw_atomic_or(int i, atomic_t *v)
1671{
1672 arch_atomic_or(i, v);
1673}
1674
1675/**
1676 * raw_atomic_fetch_or() - atomic bitwise OR with full ordering
1677 * @i: int value
1678 * @v: pointer to atomic_t
1679 *
1680 * Atomically updates @v to (@v | @i) with full ordering.
1681 *
1682 * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere.
1683 *
1684 * Return: The original value of @v.
1685 */
1686static __always_inline int
1687raw_atomic_fetch_or(int i, atomic_t *v)
1688{
1689#if defined(arch_atomic_fetch_or)
1690 return arch_atomic_fetch_or(i, v);
1691#elif defined(arch_atomic_fetch_or_relaxed)
1692 int ret;
1693 __atomic_pre_full_fence();
1694 ret = arch_atomic_fetch_or_relaxed(i, v);
1695 __atomic_post_full_fence();
1696 return ret;
1697#else
1698#error "Unable to define raw_atomic_fetch_or"
1699#endif
1700}
1701
1702/**
1703 * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
1704 * @i: int value
1705 * @v: pointer to atomic_t
1706 *
1707 * Atomically updates @v to (@v | @i) with acquire ordering.
1708 *
1709 * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere.
1710 *
1711 * Return: The original value of @v.
1712 */
1713static __always_inline int
1714raw_atomic_fetch_or_acquire(int i, atomic_t *v)
1715{
1716#if defined(arch_atomic_fetch_or_acquire)
1717 return arch_atomic_fetch_or_acquire(i, v);
1718#elif defined(arch_atomic_fetch_or_relaxed)
1719 int ret = arch_atomic_fetch_or_relaxed(i, v);
1720 __atomic_acquire_fence();
1721 return ret;
1722#elif defined(arch_atomic_fetch_or)
1723 return arch_atomic_fetch_or(i, v);
1724#else
1725#error "Unable to define raw_atomic_fetch_or_acquire"
1726#endif
1727}
1728
1729/**
1730 * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering
1731 * @i: int value
1732 * @v: pointer to atomic_t
1733 *
1734 * Atomically updates @v to (@v | @i) with release ordering.
1735 *
1736 * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere.
1737 *
1738 * Return: The original value of @v.
1739 */
1740static __always_inline int
1741raw_atomic_fetch_or_release(int i, atomic_t *v)
1742{
1743#if defined(arch_atomic_fetch_or_release)
1744 return arch_atomic_fetch_or_release(i, v);
1745#elif defined(arch_atomic_fetch_or_relaxed)
1746 __atomic_release_fence();
1747 return arch_atomic_fetch_or_relaxed(i, v);
1748#elif defined(arch_atomic_fetch_or)
1749 return arch_atomic_fetch_or(i, v);
1750#else
1751#error "Unable to define raw_atomic_fetch_or_release"
1752#endif
1753}
1754
1755/**
1756 * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
1757 * @i: int value
1758 * @v: pointer to atomic_t
1759 *
1760 * Atomically updates @v to (@v | @i) with relaxed ordering.
1761 *
1762 * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere.
1763 *
1764 * Return: The original value of @v.
1765 */
1766static __always_inline int
1767raw_atomic_fetch_or_relaxed(int i, atomic_t *v)
1768{
1769#if defined(arch_atomic_fetch_or_relaxed)
1770 return arch_atomic_fetch_or_relaxed(i, v);
1771#elif defined(arch_atomic_fetch_or)
1772 return arch_atomic_fetch_or(i, v);
1773#else
1774#error "Unable to define raw_atomic_fetch_or_relaxed"
1775#endif
1776}
1777
1778/**
1779 * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering
1780 * @i: int value
1781 * @v: pointer to atomic_t
1782 *
1783 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1784 *
1785 * Safe to use in noinstr code; prefer atomic_xor() elsewhere.
1786 *
1787 * Return: Nothing.
1788 */
1789static __always_inline void
1790raw_atomic_xor(int i, atomic_t *v)
1791{
1792 arch_atomic_xor(i, v);
1793}
1794
1795/**
1796 * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering
1797 * @i: int value
1798 * @v: pointer to atomic_t
1799 *
1800 * Atomically updates @v to (@v ^ @i) with full ordering.
1801 *
1802 * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere.
1803 *
1804 * Return: The original value of @v.
1805 */
1806static __always_inline int
1807raw_atomic_fetch_xor(int i, atomic_t *v)
1808{
1809#if defined(arch_atomic_fetch_xor)
1810 return arch_atomic_fetch_xor(i, v);
1811#elif defined(arch_atomic_fetch_xor_relaxed)
1812 int ret;
1813 __atomic_pre_full_fence();
1814 ret = arch_atomic_fetch_xor_relaxed(i, v);
1815 __atomic_post_full_fence();
1816 return ret;
1817#else
1818#error "Unable to define raw_atomic_fetch_xor"
1819#endif
1820}
1821
1822/**
1823 * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
1824 * @i: int value
1825 * @v: pointer to atomic_t
1826 *
1827 * Atomically updates @v to (@v ^ @i) with acquire ordering.
1828 *
1829 * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere.
1830 *
1831 * Return: The original value of @v.
1832 */
1833static __always_inline int
1834raw_atomic_fetch_xor_acquire(int i, atomic_t *v)
1835{
1836#if defined(arch_atomic_fetch_xor_acquire)
1837 return arch_atomic_fetch_xor_acquire(i, v);
1838#elif defined(arch_atomic_fetch_xor_relaxed)
1839 int ret = arch_atomic_fetch_xor_relaxed(i, v);
1840 __atomic_acquire_fence();
1841 return ret;
1842#elif defined(arch_atomic_fetch_xor)
1843 return arch_atomic_fetch_xor(i, v);
1844#else
1845#error "Unable to define raw_atomic_fetch_xor_acquire"
1846#endif
1847}
1848
1849/**
1850 * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
1851 * @i: int value
1852 * @v: pointer to atomic_t
1853 *
1854 * Atomically updates @v to (@v ^ @i) with release ordering.
1855 *
1856 * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere.
1857 *
1858 * Return: The original value of @v.
1859 */
1860static __always_inline int
1861raw_atomic_fetch_xor_release(int i, atomic_t *v)
1862{
1863#if defined(arch_atomic_fetch_xor_release)
1864 return arch_atomic_fetch_xor_release(i, v);
1865#elif defined(arch_atomic_fetch_xor_relaxed)
1866 __atomic_release_fence();
1867 return arch_atomic_fetch_xor_relaxed(i, v);
1868#elif defined(arch_atomic_fetch_xor)
1869 return arch_atomic_fetch_xor(i, v);
1870#else
1871#error "Unable to define raw_atomic_fetch_xor_release"
1872#endif
1873}
1874
1875/**
1876 * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
1877 * @i: int value
1878 * @v: pointer to atomic_t
1879 *
1880 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1881 *
1882 * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere.
1883 *
1884 * Return: The original value of @v.
1885 */
1886static __always_inline int
1887raw_atomic_fetch_xor_relaxed(int i, atomic_t *v)
1888{
1889#if defined(arch_atomic_fetch_xor_relaxed)
1890 return arch_atomic_fetch_xor_relaxed(i, v);
1891#elif defined(arch_atomic_fetch_xor)
1892 return arch_atomic_fetch_xor(i, v);
1893#else
1894#error "Unable to define raw_atomic_fetch_xor_relaxed"
1895#endif
1896}
1897
1898/**
1899 * raw_atomic_xchg() - atomic exchange with full ordering
1900 * @v: pointer to atomic_t
1901 * @new: int value to assign
1902 *
1903 * Atomically updates @v to @new with full ordering.
1904 *
1905 * Safe to use in noinstr code; prefer atomic_xchg() elsewhere.
1906 *
1907 * Return: The original value of @v.
1908 */
1909static __always_inline int
1910raw_atomic_xchg(atomic_t *v, int new)
1911{
1912#if defined(arch_atomic_xchg)
1913 return arch_atomic_xchg(v, new);
1914#elif defined(arch_atomic_xchg_relaxed)
1915 int ret;
1916 __atomic_pre_full_fence();
1917 ret = arch_atomic_xchg_relaxed(v, new);
1918 __atomic_post_full_fence();
1919 return ret;
1920#else
1921 return raw_xchg(&v->counter, new);
1922#endif
1923}
1924
1925/**
1926 * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering
1927 * @v: pointer to atomic_t
1928 * @new: int value to assign
1929 *
1930 * Atomically updates @v to @new with acquire ordering.
1931 *
1932 * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere.
1933 *
1934 * Return: The original value of @v.
1935 */
1936static __always_inline int
1937raw_atomic_xchg_acquire(atomic_t *v, int new)
1938{
1939#if defined(arch_atomic_xchg_acquire)
1940 return arch_atomic_xchg_acquire(v, new);
1941#elif defined(arch_atomic_xchg_relaxed)
1942 int ret = arch_atomic_xchg_relaxed(v, new);
1943 __atomic_acquire_fence();
1944 return ret;
1945#elif defined(arch_atomic_xchg)
1946 return arch_atomic_xchg(v, new);
1947#else
1948 return raw_xchg_acquire(&v->counter, new);
1949#endif
1950}
1951
1952/**
1953 * raw_atomic_xchg_release() - atomic exchange with release ordering
1954 * @v: pointer to atomic_t
1955 * @new: int value to assign
1956 *
1957 * Atomically updates @v to @new with release ordering.
1958 *
1959 * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere.
1960 *
1961 * Return: The original value of @v.
1962 */
1963static __always_inline int
1964raw_atomic_xchg_release(atomic_t *v, int new)
1965{
1966#if defined(arch_atomic_xchg_release)
1967 return arch_atomic_xchg_release(v, new);
1968#elif defined(arch_atomic_xchg_relaxed)
1969 __atomic_release_fence();
1970 return arch_atomic_xchg_relaxed(v, new);
1971#elif defined(arch_atomic_xchg)
1972 return arch_atomic_xchg(v, new);
1973#else
1974 return raw_xchg_release(&v->counter, new);
1975#endif
1976}
1977
1978/**
1979 * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering
1980 * @v: pointer to atomic_t
1981 * @new: int value to assign
1982 *
1983 * Atomically updates @v to @new with relaxed ordering.
1984 *
1985 * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere.
1986 *
1987 * Return: The original value of @v.
1988 */
1989static __always_inline int
1990raw_atomic_xchg_relaxed(atomic_t *v, int new)
1991{
1992#if defined(arch_atomic_xchg_relaxed)
1993 return arch_atomic_xchg_relaxed(v, new);
1994#elif defined(arch_atomic_xchg)
1995 return arch_atomic_xchg(v, new);
1996#else
1997 return raw_xchg_relaxed(&v->counter, new);
1998#endif
1999}
2000
2001/**
2002 * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering
2003 * @v: pointer to atomic_t
2004 * @old: int value to compare with
2005 * @new: int value to assign
2006 *
2007 * If (@v == @old), atomically updates @v to @new with full ordering.
2008 * Otherwise, @v is not modified and relaxed ordering is provided.
2009 *
2010 * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere.
2011 *
2012 * Return: The original value of @v.
2013 */
2014static __always_inline int
2015raw_atomic_cmpxchg(atomic_t *v, int old, int new)
2016{
2017#if defined(arch_atomic_cmpxchg)
2018 return arch_atomic_cmpxchg(v, old, new);
2019#elif defined(arch_atomic_cmpxchg_relaxed)
2020 int ret;
2021 __atomic_pre_full_fence();
2022 ret = arch_atomic_cmpxchg_relaxed(v, old, new);
2023 __atomic_post_full_fence();
2024 return ret;
2025#else
2026 return raw_cmpxchg(&v->counter, old, new);
2027#endif
2028}
2029
2030/**
2031 * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2032 * @v: pointer to atomic_t
2033 * @old: int value to compare with
2034 * @new: int value to assign
2035 *
2036 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2037 * Otherwise, @v is not modified and relaxed ordering is provided.
2038 *
2039 * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere.
2040 *
2041 * Return: The original value of @v.
2042 */
2043static __always_inline int
2044raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
2045{
2046#if defined(arch_atomic_cmpxchg_acquire)
2047 return arch_atomic_cmpxchg_acquire(v, old, new);
2048#elif defined(arch_atomic_cmpxchg_relaxed)
2049 int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
2050 __atomic_acquire_fence();
2051 return ret;
2052#elif defined(arch_atomic_cmpxchg)
2053 return arch_atomic_cmpxchg(v, old, new);
2054#else
2055 return raw_cmpxchg_acquire(&v->counter, old, new);
2056#endif
2057}
2058
2059/**
2060 * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering
2061 * @v: pointer to atomic_t
2062 * @old: int value to compare with
2063 * @new: int value to assign
2064 *
2065 * If (@v == @old), atomically updates @v to @new with release ordering.
2066 * Otherwise, @v is not modified and relaxed ordering is provided.
2067 *
2068 * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere.
2069 *
2070 * Return: The original value of @v.
2071 */
2072static __always_inline int
2073raw_atomic_cmpxchg_release(atomic_t *v, int old, int new)
2074{
2075#if defined(arch_atomic_cmpxchg_release)
2076 return arch_atomic_cmpxchg_release(v, old, new);
2077#elif defined(arch_atomic_cmpxchg_relaxed)
2078 __atomic_release_fence();
2079 return arch_atomic_cmpxchg_relaxed(v, old, new);
2080#elif defined(arch_atomic_cmpxchg)
2081 return arch_atomic_cmpxchg(v, old, new);
2082#else
2083 return raw_cmpxchg_release(&v->counter, old, new);
2084#endif
2085}
2086
2087/**
2088 * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2089 * @v: pointer to atomic_t
2090 * @old: int value to compare with
2091 * @new: int value to assign
2092 *
2093 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2094 * Otherwise, @v is not modified and relaxed ordering is provided.
2095 *
2096 * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere.
2097 *
2098 * Return: The original value of @v.
2099 */
2100static __always_inline int
2101raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
2102{
2103#if defined(arch_atomic_cmpxchg_relaxed)
2104 return arch_atomic_cmpxchg_relaxed(v, old, new);
2105#elif defined(arch_atomic_cmpxchg)
2106 return arch_atomic_cmpxchg(v, old, new);
2107#else
2108 return raw_cmpxchg_relaxed(&v->counter, old, new);
2109#endif
2110}
2111
2112/**
2113 * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering
2114 * @v: pointer to atomic_t
2115 * @old: pointer to int value to compare with
2116 * @new: int value to assign
2117 *
2118 * If (@v == @old), atomically updates @v to @new with full ordering.
2119 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2120 * and relaxed ordering is provided.
2121 *
2122 * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere.
2123 *
2124 * Return: @true if the exchange occured, @false otherwise.
2125 */
2126static __always_inline bool
2127raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
2128{
2129#if defined(arch_atomic_try_cmpxchg)
2130 return arch_atomic_try_cmpxchg(v, old, new);
2131#elif defined(arch_atomic_try_cmpxchg_relaxed)
2132 bool ret;
2133 __atomic_pre_full_fence();
2134 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2135 __atomic_post_full_fence();
2136 return ret;
2137#else
2138 int r, o = *old;
2139 r = raw_atomic_cmpxchg(v, o, new);
2140 if (unlikely(r != o))
2141 *old = r;
2142 return likely(r == o);
2143#endif
2144}
2145
2146/**
2147 * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2148 * @v: pointer to atomic_t
2149 * @old: pointer to int value to compare with
2150 * @new: int value to assign
2151 *
2152 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2153 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2154 * and relaxed ordering is provided.
2155 *
2156 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere.
2157 *
2158 * Return: @true if the exchange occured, @false otherwise.
2159 */
2160static __always_inline bool
2161raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
2162{
2163#if defined(arch_atomic_try_cmpxchg_acquire)
2164 return arch_atomic_try_cmpxchg_acquire(v, old, new);
2165#elif defined(arch_atomic_try_cmpxchg_relaxed)
2166 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2167 __atomic_acquire_fence();
2168 return ret;
2169#elif defined(arch_atomic_try_cmpxchg)
2170 return arch_atomic_try_cmpxchg(v, old, new);
2171#else
2172 int r, o = *old;
2173 r = raw_atomic_cmpxchg_acquire(v, o, new);
2174 if (unlikely(r != o))
2175 *old = r;
2176 return likely(r == o);
2177#endif
2178}
2179
2180/**
2181 * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
2182 * @v: pointer to atomic_t
2183 * @old: pointer to int value to compare with
2184 * @new: int value to assign
2185 *
2186 * If (@v == @old), atomically updates @v to @new with release ordering.
2187 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2188 * and relaxed ordering is provided.
2189 *
2190 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere.
2191 *
2192 * Return: @true if the exchange occured, @false otherwise.
2193 */
2194static __always_inline bool
2195raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
2196{
2197#if defined(arch_atomic_try_cmpxchg_release)
2198 return arch_atomic_try_cmpxchg_release(v, old, new);
2199#elif defined(arch_atomic_try_cmpxchg_relaxed)
2200 __atomic_release_fence();
2201 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2202#elif defined(arch_atomic_try_cmpxchg)
2203 return arch_atomic_try_cmpxchg(v, old, new);
2204#else
2205 int r, o = *old;
2206 r = raw_atomic_cmpxchg_release(v, o, new);
2207 if (unlikely(r != o))
2208 *old = r;
2209 return likely(r == o);
2210#endif
2211}
2212
2213/**
2214 * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2215 * @v: pointer to atomic_t
2216 * @old: pointer to int value to compare with
2217 * @new: int value to assign
2218 *
2219 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2220 * Otherwise, @v is not modified, @old is updated to the current value of @v,
2221 * and relaxed ordering is provided.
2222 *
2223 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere.
2224 *
2225 * Return: @true if the exchange occured, @false otherwise.
2226 */
2227static __always_inline bool
2228raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
2229{
2230#if defined(arch_atomic_try_cmpxchg_relaxed)
2231 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2232#elif defined(arch_atomic_try_cmpxchg)
2233 return arch_atomic_try_cmpxchg(v, old, new);
2234#else
2235 int r, o = *old;
2236 r = raw_atomic_cmpxchg_relaxed(v, o, new);
2237 if (unlikely(r != o))
2238 *old = r;
2239 return likely(r == o);
2240#endif
2241}
2242
2243/**
2244 * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering
2245 * @i: int value to subtract
2246 * @v: pointer to atomic_t
2247 *
2248 * Atomically updates @v to (@v - @i) with full ordering.
2249 *
2250 * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere.
2251 *
2252 * Return: @true if the resulting value of @v is zero, @false otherwise.
2253 */
2254static __always_inline bool
2255raw_atomic_sub_and_test(int i, atomic_t *v)
2256{
2257#if defined(arch_atomic_sub_and_test)
2258 return arch_atomic_sub_and_test(i, v);
2259#else
2260 return raw_atomic_sub_return(i, v) == 0;
2261#endif
2262}
2263
2264/**
2265 * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering
2266 * @v: pointer to atomic_t
2267 *
2268 * Atomically updates @v to (@v - 1) with full ordering.
2269 *
2270 * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere.
2271 *
2272 * Return: @true if the resulting value of @v is zero, @false otherwise.
2273 */
2274static __always_inline bool
2275raw_atomic_dec_and_test(atomic_t *v)
2276{
2277#if defined(arch_atomic_dec_and_test)
2278 return arch_atomic_dec_and_test(v);
2279#else
2280 return raw_atomic_dec_return(v) == 0;
2281#endif
2282}
2283
2284/**
2285 * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering
2286 * @v: pointer to atomic_t
2287 *
2288 * Atomically updates @v to (@v + 1) with full ordering.
2289 *
2290 * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere.
2291 *
2292 * Return: @true if the resulting value of @v is zero, @false otherwise.
2293 */
2294static __always_inline bool
2295raw_atomic_inc_and_test(atomic_t *v)
2296{
2297#if defined(arch_atomic_inc_and_test)
2298 return arch_atomic_inc_and_test(v);
2299#else
2300 return raw_atomic_inc_return(v) == 0;
2301#endif
2302}
2303
2304/**
2305 * raw_atomic_add_negative() - atomic add and test if negative with full ordering
2306 * @i: int value to add
2307 * @v: pointer to atomic_t
2308 *
2309 * Atomically updates @v to (@v + @i) with full ordering.
2310 *
2311 * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere.
2312 *
2313 * Return: @true if the resulting value of @v is negative, @false otherwise.
2314 */
2315static __always_inline bool
2316raw_atomic_add_negative(int i, atomic_t *v)
2317{
2318#if defined(arch_atomic_add_negative)
2319 return arch_atomic_add_negative(i, v);
2320#elif defined(arch_atomic_add_negative_relaxed)
2321 bool ret;
2322 __atomic_pre_full_fence();
2323 ret = arch_atomic_add_negative_relaxed(i, v);
2324 __atomic_post_full_fence();
2325 return ret;
2326#else
2327 return raw_atomic_add_return(i, v) < 0;
2328#endif
2329}
2330
2331/**
2332 * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
2333 * @i: int value to add
2334 * @v: pointer to atomic_t
2335 *
2336 * Atomically updates @v to (@v + @i) with acquire ordering.
2337 *
2338 * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere.
2339 *
2340 * Return: @true if the resulting value of @v is negative, @false otherwise.
2341 */
2342static __always_inline bool
2343raw_atomic_add_negative_acquire(int i, atomic_t *v)
2344{
2345#if defined(arch_atomic_add_negative_acquire)
2346 return arch_atomic_add_negative_acquire(i, v);
2347#elif defined(arch_atomic_add_negative_relaxed)
2348 bool ret = arch_atomic_add_negative_relaxed(i, v);
2349 __atomic_acquire_fence();
2350 return ret;
2351#elif defined(arch_atomic_add_negative)
2352 return arch_atomic_add_negative(i, v);
2353#else
2354 return raw_atomic_add_return_acquire(i, v) < 0;
2355#endif
2356}
2357
2358/**
2359 * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering
2360 * @i: int value to add
2361 * @v: pointer to atomic_t
2362 *
2363 * Atomically updates @v to (@v + @i) with release ordering.
2364 *
2365 * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere.
2366 *
2367 * Return: @true if the resulting value of @v is negative, @false otherwise.
2368 */
2369static __always_inline bool
2370raw_atomic_add_negative_release(int i, atomic_t *v)
2371{
2372#if defined(arch_atomic_add_negative_release)
2373 return arch_atomic_add_negative_release(i, v);
2374#elif defined(arch_atomic_add_negative_relaxed)
2375 __atomic_release_fence();
2376 return arch_atomic_add_negative_relaxed(i, v);
2377#elif defined(arch_atomic_add_negative)
2378 return arch_atomic_add_negative(i, v);
2379#else
2380 return raw_atomic_add_return_release(i, v) < 0;
2381#endif
2382}
2383
2384/**
2385 * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
2386 * @i: int value to add
2387 * @v: pointer to atomic_t
2388 *
2389 * Atomically updates @v to (@v + @i) with relaxed ordering.
2390 *
2391 * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere.
2392 *
2393 * Return: @true if the resulting value of @v is negative, @false otherwise.
2394 */
2395static __always_inline bool
2396raw_atomic_add_negative_relaxed(int i, atomic_t *v)
2397{
2398#if defined(arch_atomic_add_negative_relaxed)
2399 return arch_atomic_add_negative_relaxed(i, v);
2400#elif defined(arch_atomic_add_negative)
2401 return arch_atomic_add_negative(i, v);
2402#else
2403 return raw_atomic_add_return_relaxed(i, v) < 0;
2404#endif
2405}
2406
2407/**
2408 * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering
2409 * @v: pointer to atomic_t
2410 * @a: int value to add
2411 * @u: int value to compare with
2412 *
2413 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2414 * Otherwise, @v is not modified and relaxed ordering is provided.
2415 *
2416 * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere.
2417 *
2418 * Return: The original value of @v.
2419 */
2420static __always_inline int
2421raw_atomic_fetch_add_unless(atomic_t *v, int a, int u)
2422{
2423#if defined(arch_atomic_fetch_add_unless)
2424 return arch_atomic_fetch_add_unless(v, a, u);
2425#else
2426 int c = raw_atomic_read(v);
2427
2428 do {
2429 if (unlikely(c == u))
2430 break;
2431 } while (!raw_atomic_try_cmpxchg(v, old: &c, new: c + a));
2432
2433 return c;
2434#endif
2435}
2436
2437/**
2438 * raw_atomic_add_unless() - atomic add unless value with full ordering
2439 * @v: pointer to atomic_t
2440 * @a: int value to add
2441 * @u: int value to compare with
2442 *
2443 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2444 * Otherwise, @v is not modified and relaxed ordering is provided.
2445 *
2446 * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere.
2447 *
2448 * Return: @true if @v was updated, @false otherwise.
2449 */
2450static __always_inline bool
2451raw_atomic_add_unless(atomic_t *v, int a, int u)
2452{
2453#if defined(arch_atomic_add_unless)
2454 return arch_atomic_add_unless(v, a, u);
2455#else
2456 return raw_atomic_fetch_add_unless(v, a, u) != u;
2457#endif
2458}
2459
2460/**
2461 * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering
2462 * @v: pointer to atomic_t
2463 *
2464 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
2465 * Otherwise, @v is not modified and relaxed ordering is provided.
2466 *
2467 * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere.
2468 *
2469 * Return: @true if @v was updated, @false otherwise.
2470 */
2471static __always_inline bool
2472raw_atomic_inc_not_zero(atomic_t *v)
2473{
2474#if defined(arch_atomic_inc_not_zero)
2475 return arch_atomic_inc_not_zero(v);
2476#else
2477 return raw_atomic_add_unless(v, a: 1, u: 0);
2478#endif
2479}
2480
2481/**
2482 * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering
2483 * @v: pointer to atomic_t
2484 *
2485 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
2486 * Otherwise, @v is not modified and relaxed ordering is provided.
2487 *
2488 * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere.
2489 *
2490 * Return: @true if @v was updated, @false otherwise.
2491 */
2492static __always_inline bool
2493raw_atomic_inc_unless_negative(atomic_t *v)
2494{
2495#if defined(arch_atomic_inc_unless_negative)
2496 return arch_atomic_inc_unless_negative(v);
2497#else
2498 int c = raw_atomic_read(v);
2499
2500 do {
2501 if (unlikely(c < 0))
2502 return false;
2503 } while (!raw_atomic_try_cmpxchg(v, old: &c, new: c + 1));
2504
2505 return true;
2506#endif
2507}
2508
2509/**
2510 * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
2511 * @v: pointer to atomic_t
2512 *
2513 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
2514 * Otherwise, @v is not modified and relaxed ordering is provided.
2515 *
2516 * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere.
2517 *
2518 * Return: @true if @v was updated, @false otherwise.
2519 */
2520static __always_inline bool
2521raw_atomic_dec_unless_positive(atomic_t *v)
2522{
2523#if defined(arch_atomic_dec_unless_positive)
2524 return arch_atomic_dec_unless_positive(v);
2525#else
2526 int c = raw_atomic_read(v);
2527
2528 do {
2529 if (unlikely(c > 0))
2530 return false;
2531 } while (!raw_atomic_try_cmpxchg(v, old: &c, new: c - 1));
2532
2533 return true;
2534#endif
2535}
2536
2537/**
2538 * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering
2539 * @v: pointer to atomic_t
2540 *
2541 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
2542 * Otherwise, @v is not modified and relaxed ordering is provided.
2543 *
2544 * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere.
2545 *
2546 * Return: The old value of (@v - 1), regardless of whether @v was updated.
2547 */
2548static __always_inline int
2549raw_atomic_dec_if_positive(atomic_t *v)
2550{
2551#if defined(arch_atomic_dec_if_positive)
2552 return arch_atomic_dec_if_positive(v);
2553#else
2554 int dec, c = raw_atomic_read(v);
2555
2556 do {
2557 dec = c - 1;
2558 if (unlikely(dec < 0))
2559 break;
2560 } while (!raw_atomic_try_cmpxchg(v, old: &c, new: dec));
2561
2562 return dec;
2563#endif
2564}
2565
2566#ifdef CONFIG_GENERIC_ATOMIC64
2567#include <asm-generic/atomic64.h>
2568#endif
2569
2570/**
2571 * raw_atomic64_read() - atomic load with relaxed ordering
2572 * @v: pointer to atomic64_t
2573 *
2574 * Atomically loads the value of @v with relaxed ordering.
2575 *
2576 * Safe to use in noinstr code; prefer atomic64_read() elsewhere.
2577 *
2578 * Return: The value loaded from @v.
2579 */
2580static __always_inline s64
2581raw_atomic64_read(const atomic64_t *v)
2582{
2583 return arch_atomic64_read(v);
2584}
2585
2586/**
2587 * raw_atomic64_read_acquire() - atomic load with acquire ordering
2588 * @v: pointer to atomic64_t
2589 *
2590 * Atomically loads the value of @v with acquire ordering.
2591 *
2592 * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere.
2593 *
2594 * Return: The value loaded from @v.
2595 */
2596static __always_inline s64
2597raw_atomic64_read_acquire(const atomic64_t *v)
2598{
2599#if defined(arch_atomic64_read_acquire)
2600 return arch_atomic64_read_acquire(v);
2601#else
2602 s64 ret;
2603
2604 if (__native_word(atomic64_t)) {
2605 ret = smp_load_acquire(&(v)->counter);
2606 } else {
2607 ret = raw_atomic64_read(v);
2608 __atomic_acquire_fence();
2609 }
2610
2611 return ret;
2612#endif
2613}
2614
2615/**
2616 * raw_atomic64_set() - atomic set with relaxed ordering
2617 * @v: pointer to atomic64_t
2618 * @i: s64 value to assign
2619 *
2620 * Atomically sets @v to @i with relaxed ordering.
2621 *
2622 * Safe to use in noinstr code; prefer atomic64_set() elsewhere.
2623 *
2624 * Return: Nothing.
2625 */
2626static __always_inline void
2627raw_atomic64_set(atomic64_t *v, s64 i)
2628{
2629 arch_atomic64_set(v, i);
2630}
2631
2632/**
2633 * raw_atomic64_set_release() - atomic set with release ordering
2634 * @v: pointer to atomic64_t
2635 * @i: s64 value to assign
2636 *
2637 * Atomically sets @v to @i with release ordering.
2638 *
2639 * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere.
2640 *
2641 * Return: Nothing.
2642 */
2643static __always_inline void
2644raw_atomic64_set_release(atomic64_t *v, s64 i)
2645{
2646#if defined(arch_atomic64_set_release)
2647 arch_atomic64_set_release(v, i);
2648#else
2649 if (__native_word(atomic64_t)) {
2650 smp_store_release(&(v)->counter, i);
2651 } else {
2652 __atomic_release_fence();
2653 raw_atomic64_set(v, i);
2654 }
2655#endif
2656}
2657
2658/**
2659 * raw_atomic64_add() - atomic add with relaxed ordering
2660 * @i: s64 value to add
2661 * @v: pointer to atomic64_t
2662 *
2663 * Atomically updates @v to (@v + @i) with relaxed ordering.
2664 *
2665 * Safe to use in noinstr code; prefer atomic64_add() elsewhere.
2666 *
2667 * Return: Nothing.
2668 */
2669static __always_inline void
2670raw_atomic64_add(s64 i, atomic64_t *v)
2671{
2672 arch_atomic64_add(i, v);
2673}
2674
2675/**
2676 * raw_atomic64_add_return() - atomic add with full ordering
2677 * @i: s64 value to add
2678 * @v: pointer to atomic64_t
2679 *
2680 * Atomically updates @v to (@v + @i) with full ordering.
2681 *
2682 * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere.
2683 *
2684 * Return: The updated value of @v.
2685 */
2686static __always_inline s64
2687raw_atomic64_add_return(s64 i, atomic64_t *v)
2688{
2689#if defined(arch_atomic64_add_return)
2690 return arch_atomic64_add_return(i, v);
2691#elif defined(arch_atomic64_add_return_relaxed)
2692 s64 ret;
2693 __atomic_pre_full_fence();
2694 ret = arch_atomic64_add_return_relaxed(i, v);
2695 __atomic_post_full_fence();
2696 return ret;
2697#else
2698#error "Unable to define raw_atomic64_add_return"
2699#endif
2700}
2701
2702/**
2703 * raw_atomic64_add_return_acquire() - atomic add with acquire ordering
2704 * @i: s64 value to add
2705 * @v: pointer to atomic64_t
2706 *
2707 * Atomically updates @v to (@v + @i) with acquire ordering.
2708 *
2709 * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere.
2710 *
2711 * Return: The updated value of @v.
2712 */
2713static __always_inline s64
2714raw_atomic64_add_return_acquire(s64 i, atomic64_t *v)
2715{
2716#if defined(arch_atomic64_add_return_acquire)
2717 return arch_atomic64_add_return_acquire(i, v);
2718#elif defined(arch_atomic64_add_return_relaxed)
2719 s64 ret = arch_atomic64_add_return_relaxed(i, v);
2720 __atomic_acquire_fence();
2721 return ret;
2722#elif defined(arch_atomic64_add_return)
2723 return arch_atomic64_add_return(i, v);
2724#else
2725#error "Unable to define raw_atomic64_add_return_acquire"
2726#endif
2727}
2728
2729/**
2730 * raw_atomic64_add_return_release() - atomic add with release ordering
2731 * @i: s64 value to add
2732 * @v: pointer to atomic64_t
2733 *
2734 * Atomically updates @v to (@v + @i) with release ordering.
2735 *
2736 * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere.
2737 *
2738 * Return: The updated value of @v.
2739 */
2740static __always_inline s64
2741raw_atomic64_add_return_release(s64 i, atomic64_t *v)
2742{
2743#if defined(arch_atomic64_add_return_release)
2744 return arch_atomic64_add_return_release(i, v);
2745#elif defined(arch_atomic64_add_return_relaxed)
2746 __atomic_release_fence();
2747 return arch_atomic64_add_return_relaxed(i, v);
2748#elif defined(arch_atomic64_add_return)
2749 return arch_atomic64_add_return(i, v);
2750#else
2751#error "Unable to define raw_atomic64_add_return_release"
2752#endif
2753}
2754
2755/**
2756 * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering
2757 * @i: s64 value to add
2758 * @v: pointer to atomic64_t
2759 *
2760 * Atomically updates @v to (@v + @i) with relaxed ordering.
2761 *
2762 * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere.
2763 *
2764 * Return: The updated value of @v.
2765 */
2766static __always_inline s64
2767raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v)
2768{
2769#if defined(arch_atomic64_add_return_relaxed)
2770 return arch_atomic64_add_return_relaxed(i, v);
2771#elif defined(arch_atomic64_add_return)
2772 return arch_atomic64_add_return(i, v);
2773#else
2774#error "Unable to define raw_atomic64_add_return_relaxed"
2775#endif
2776}
2777
2778/**
2779 * raw_atomic64_fetch_add() - atomic add with full ordering
2780 * @i: s64 value to add
2781 * @v: pointer to atomic64_t
2782 *
2783 * Atomically updates @v to (@v + @i) with full ordering.
2784 *
2785 * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere.
2786 *
2787 * Return: The original value of @v.
2788 */
2789static __always_inline s64
2790raw_atomic64_fetch_add(s64 i, atomic64_t *v)
2791{
2792#if defined(arch_atomic64_fetch_add)
2793 return arch_atomic64_fetch_add(i, v);
2794#elif defined(arch_atomic64_fetch_add_relaxed)
2795 s64 ret;
2796 __atomic_pre_full_fence();
2797 ret = arch_atomic64_fetch_add_relaxed(i, v);
2798 __atomic_post_full_fence();
2799 return ret;
2800#else
2801#error "Unable to define raw_atomic64_fetch_add"
2802#endif
2803}
2804
2805/**
2806 * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering
2807 * @i: s64 value to add
2808 * @v: pointer to atomic64_t
2809 *
2810 * Atomically updates @v to (@v + @i) with acquire ordering.
2811 *
2812 * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere.
2813 *
2814 * Return: The original value of @v.
2815 */
2816static __always_inline s64
2817raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
2818{
2819#if defined(arch_atomic64_fetch_add_acquire)
2820 return arch_atomic64_fetch_add_acquire(i, v);
2821#elif defined(arch_atomic64_fetch_add_relaxed)
2822 s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
2823 __atomic_acquire_fence();
2824 return ret;
2825#elif defined(arch_atomic64_fetch_add)
2826 return arch_atomic64_fetch_add(i, v);
2827#else
2828#error "Unable to define raw_atomic64_fetch_add_acquire"
2829#endif
2830}
2831
2832/**
2833 * raw_atomic64_fetch_add_release() - atomic add with release ordering
2834 * @i: s64 value to add
2835 * @v: pointer to atomic64_t
2836 *
2837 * Atomically updates @v to (@v + @i) with release ordering.
2838 *
2839 * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere.
2840 *
2841 * Return: The original value of @v.
2842 */
2843static __always_inline s64
2844raw_atomic64_fetch_add_release(s64 i, atomic64_t *v)
2845{
2846#if defined(arch_atomic64_fetch_add_release)
2847 return arch_atomic64_fetch_add_release(i, v);
2848#elif defined(arch_atomic64_fetch_add_relaxed)
2849 __atomic_release_fence();
2850 return arch_atomic64_fetch_add_relaxed(i, v);
2851#elif defined(arch_atomic64_fetch_add)
2852 return arch_atomic64_fetch_add(i, v);
2853#else
2854#error "Unable to define raw_atomic64_fetch_add_release"
2855#endif
2856}
2857
2858/**
2859 * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
2860 * @i: s64 value to add
2861 * @v: pointer to atomic64_t
2862 *
2863 * Atomically updates @v to (@v + @i) with relaxed ordering.
2864 *
2865 * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere.
2866 *
2867 * Return: The original value of @v.
2868 */
2869static __always_inline s64
2870raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
2871{
2872#if defined(arch_atomic64_fetch_add_relaxed)
2873 return arch_atomic64_fetch_add_relaxed(i, v);
2874#elif defined(arch_atomic64_fetch_add)
2875 return arch_atomic64_fetch_add(i, v);
2876#else
2877#error "Unable to define raw_atomic64_fetch_add_relaxed"
2878#endif
2879}
2880
2881/**
2882 * raw_atomic64_sub() - atomic subtract with relaxed ordering
2883 * @i: s64 value to subtract
2884 * @v: pointer to atomic64_t
2885 *
2886 * Atomically updates @v to (@v - @i) with relaxed ordering.
2887 *
2888 * Safe to use in noinstr code; prefer atomic64_sub() elsewhere.
2889 *
2890 * Return: Nothing.
2891 */
2892static __always_inline void
2893raw_atomic64_sub(s64 i, atomic64_t *v)
2894{
2895 arch_atomic64_sub(i, v);
2896}
2897
2898/**
2899 * raw_atomic64_sub_return() - atomic subtract with full ordering
2900 * @i: s64 value to subtract
2901 * @v: pointer to atomic64_t
2902 *
2903 * Atomically updates @v to (@v - @i) with full ordering.
2904 *
2905 * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere.
2906 *
2907 * Return: The updated value of @v.
2908 */
2909static __always_inline s64
2910raw_atomic64_sub_return(s64 i, atomic64_t *v)
2911{
2912#if defined(arch_atomic64_sub_return)
2913 return arch_atomic64_sub_return(i, v);
2914#elif defined(arch_atomic64_sub_return_relaxed)
2915 s64 ret;
2916 __atomic_pre_full_fence();
2917 ret = arch_atomic64_sub_return_relaxed(i, v);
2918 __atomic_post_full_fence();
2919 return ret;
2920#else
2921#error "Unable to define raw_atomic64_sub_return"
2922#endif
2923}
2924
2925/**
2926 * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering
2927 * @i: s64 value to subtract
2928 * @v: pointer to atomic64_t
2929 *
2930 * Atomically updates @v to (@v - @i) with acquire ordering.
2931 *
2932 * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere.
2933 *
2934 * Return: The updated value of @v.
2935 */
2936static __always_inline s64
2937raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
2938{
2939#if defined(arch_atomic64_sub_return_acquire)
2940 return arch_atomic64_sub_return_acquire(i, v);
2941#elif defined(arch_atomic64_sub_return_relaxed)
2942 s64 ret = arch_atomic64_sub_return_relaxed(i, v);
2943 __atomic_acquire_fence();
2944 return ret;
2945#elif defined(arch_atomic64_sub_return)
2946 return arch_atomic64_sub_return(i, v);
2947#else
2948#error "Unable to define raw_atomic64_sub_return_acquire"
2949#endif
2950}
2951
2952/**
2953 * raw_atomic64_sub_return_release() - atomic subtract with release ordering
2954 * @i: s64 value to subtract
2955 * @v: pointer to atomic64_t
2956 *
2957 * Atomically updates @v to (@v - @i) with release ordering.
2958 *
2959 * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere.
2960 *
2961 * Return: The updated value of @v.
2962 */
2963static __always_inline s64
2964raw_atomic64_sub_return_release(s64 i, atomic64_t *v)
2965{
2966#if defined(arch_atomic64_sub_return_release)
2967 return arch_atomic64_sub_return_release(i, v);
2968#elif defined(arch_atomic64_sub_return_relaxed)
2969 __atomic_release_fence();
2970 return arch_atomic64_sub_return_relaxed(i, v);
2971#elif defined(arch_atomic64_sub_return)
2972 return arch_atomic64_sub_return(i, v);
2973#else
2974#error "Unable to define raw_atomic64_sub_return_release"
2975#endif
2976}
2977
2978/**
2979 * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
2980 * @i: s64 value to subtract
2981 * @v: pointer to atomic64_t
2982 *
2983 * Atomically updates @v to (@v - @i) with relaxed ordering.
2984 *
2985 * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere.
2986 *
2987 * Return: The updated value of @v.
2988 */
2989static __always_inline s64
2990raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
2991{
2992#if defined(arch_atomic64_sub_return_relaxed)
2993 return arch_atomic64_sub_return_relaxed(i, v);
2994#elif defined(arch_atomic64_sub_return)
2995 return arch_atomic64_sub_return(i, v);
2996#else
2997#error "Unable to define raw_atomic64_sub_return_relaxed"
2998#endif
2999}
3000
3001/**
3002 * raw_atomic64_fetch_sub() - atomic subtract with full ordering
3003 * @i: s64 value to subtract
3004 * @v: pointer to atomic64_t
3005 *
3006 * Atomically updates @v to (@v - @i) with full ordering.
3007 *
3008 * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere.
3009 *
3010 * Return: The original value of @v.
3011 */
3012static __always_inline s64
3013raw_atomic64_fetch_sub(s64 i, atomic64_t *v)
3014{
3015#if defined(arch_atomic64_fetch_sub)
3016 return arch_atomic64_fetch_sub(i, v);
3017#elif defined(arch_atomic64_fetch_sub_relaxed)
3018 s64 ret;
3019 __atomic_pre_full_fence();
3020 ret = arch_atomic64_fetch_sub_relaxed(i, v);
3021 __atomic_post_full_fence();
3022 return ret;
3023#else
3024#error "Unable to define raw_atomic64_fetch_sub"
3025#endif
3026}
3027
3028/**
3029 * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
3030 * @i: s64 value to subtract
3031 * @v: pointer to atomic64_t
3032 *
3033 * Atomically updates @v to (@v - @i) with acquire ordering.
3034 *
3035 * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere.
3036 *
3037 * Return: The original value of @v.
3038 */
3039static __always_inline s64
3040raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
3041{
3042#if defined(arch_atomic64_fetch_sub_acquire)
3043 return arch_atomic64_fetch_sub_acquire(i, v);
3044#elif defined(arch_atomic64_fetch_sub_relaxed)
3045 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
3046 __atomic_acquire_fence();
3047 return ret;
3048#elif defined(arch_atomic64_fetch_sub)
3049 return arch_atomic64_fetch_sub(i, v);
3050#else
3051#error "Unable to define raw_atomic64_fetch_sub_acquire"
3052#endif
3053}
3054
3055/**
3056 * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering
3057 * @i: s64 value to subtract
3058 * @v: pointer to atomic64_t
3059 *
3060 * Atomically updates @v to (@v - @i) with release ordering.
3061 *
3062 * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere.
3063 *
3064 * Return: The original value of @v.
3065 */
3066static __always_inline s64
3067raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
3068{
3069#if defined(arch_atomic64_fetch_sub_release)
3070 return arch_atomic64_fetch_sub_release(i, v);
3071#elif defined(arch_atomic64_fetch_sub_relaxed)
3072 __atomic_release_fence();
3073 return arch_atomic64_fetch_sub_relaxed(i, v);
3074#elif defined(arch_atomic64_fetch_sub)
3075 return arch_atomic64_fetch_sub(i, v);
3076#else
3077#error "Unable to define raw_atomic64_fetch_sub_release"
3078#endif
3079}
3080
3081/**
3082 * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
3083 * @i: s64 value to subtract
3084 * @v: pointer to atomic64_t
3085 *
3086 * Atomically updates @v to (@v - @i) with relaxed ordering.
3087 *
3088 * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere.
3089 *
3090 * Return: The original value of @v.
3091 */
3092static __always_inline s64
3093raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
3094{
3095#if defined(arch_atomic64_fetch_sub_relaxed)
3096 return arch_atomic64_fetch_sub_relaxed(i, v);
3097#elif defined(arch_atomic64_fetch_sub)
3098 return arch_atomic64_fetch_sub(i, v);
3099#else
3100#error "Unable to define raw_atomic64_fetch_sub_relaxed"
3101#endif
3102}
3103
3104/**
3105 * raw_atomic64_inc() - atomic increment with relaxed ordering
3106 * @v: pointer to atomic64_t
3107 *
3108 * Atomically updates @v to (@v + 1) with relaxed ordering.
3109 *
3110 * Safe to use in noinstr code; prefer atomic64_inc() elsewhere.
3111 *
3112 * Return: Nothing.
3113 */
3114static __always_inline void
3115raw_atomic64_inc(atomic64_t *v)
3116{
3117#if defined(arch_atomic64_inc)
3118 arch_atomic64_inc(v);
3119#else
3120 raw_atomic64_add(1, v);
3121#endif
3122}
3123
3124/**
3125 * raw_atomic64_inc_return() - atomic increment with full ordering
3126 * @v: pointer to atomic64_t
3127 *
3128 * Atomically updates @v to (@v + 1) with full ordering.
3129 *
3130 * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere.
3131 *
3132 * Return: The updated value of @v.
3133 */
3134static __always_inline s64
3135raw_atomic64_inc_return(atomic64_t *v)
3136{
3137#if defined(arch_atomic64_inc_return)
3138 return arch_atomic64_inc_return(v);
3139#elif defined(arch_atomic64_inc_return_relaxed)
3140 s64 ret;
3141 __atomic_pre_full_fence();
3142 ret = arch_atomic64_inc_return_relaxed(v);
3143 __atomic_post_full_fence();
3144 return ret;
3145#else
3146 return raw_atomic64_add_return(i: 1, v);
3147#endif
3148}
3149
3150/**
3151 * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering
3152 * @v: pointer to atomic64_t
3153 *
3154 * Atomically updates @v to (@v + 1) with acquire ordering.
3155 *
3156 * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere.
3157 *
3158 * Return: The updated value of @v.
3159 */
3160static __always_inline s64
3161raw_atomic64_inc_return_acquire(atomic64_t *v)
3162{
3163#if defined(arch_atomic64_inc_return_acquire)
3164 return arch_atomic64_inc_return_acquire(v);
3165#elif defined(arch_atomic64_inc_return_relaxed)
3166 s64 ret = arch_atomic64_inc_return_relaxed(v);
3167 __atomic_acquire_fence();
3168 return ret;
3169#elif defined(arch_atomic64_inc_return)
3170 return arch_atomic64_inc_return(v);
3171#else
3172 return raw_atomic64_add_return_acquire(i: 1, v);
3173#endif
3174}
3175
3176/**
3177 * raw_atomic64_inc_return_release() - atomic increment with release ordering
3178 * @v: pointer to atomic64_t
3179 *
3180 * Atomically updates @v to (@v + 1) with release ordering.
3181 *
3182 * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere.
3183 *
3184 * Return: The updated value of @v.
3185 */
3186static __always_inline s64
3187raw_atomic64_inc_return_release(atomic64_t *v)
3188{
3189#if defined(arch_atomic64_inc_return_release)
3190 return arch_atomic64_inc_return_release(v);
3191#elif defined(arch_atomic64_inc_return_relaxed)
3192 __atomic_release_fence();
3193 return arch_atomic64_inc_return_relaxed(v);
3194#elif defined(arch_atomic64_inc_return)
3195 return arch_atomic64_inc_return(v);
3196#else
3197 return raw_atomic64_add_return_release(i: 1, v);
3198#endif
3199}
3200
3201/**
3202 * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
3203 * @v: pointer to atomic64_t
3204 *
3205 * Atomically updates @v to (@v + 1) with relaxed ordering.
3206 *
3207 * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere.
3208 *
3209 * Return: The updated value of @v.
3210 */
3211static __always_inline s64
3212raw_atomic64_inc_return_relaxed(atomic64_t *v)
3213{
3214#if defined(arch_atomic64_inc_return_relaxed)
3215 return arch_atomic64_inc_return_relaxed(v);
3216#elif defined(arch_atomic64_inc_return)
3217 return arch_atomic64_inc_return(v);
3218#else
3219 return raw_atomic64_add_return_relaxed(i: 1, v);
3220#endif
3221}
3222
3223/**
3224 * raw_atomic64_fetch_inc() - atomic increment with full ordering
3225 * @v: pointer to atomic64_t
3226 *
3227 * Atomically updates @v to (@v + 1) with full ordering.
3228 *
3229 * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere.
3230 *
3231 * Return: The original value of @v.
3232 */
3233static __always_inline s64
3234raw_atomic64_fetch_inc(atomic64_t *v)
3235{
3236#if defined(arch_atomic64_fetch_inc)
3237 return arch_atomic64_fetch_inc(v);
3238#elif defined(arch_atomic64_fetch_inc_relaxed)
3239 s64 ret;
3240 __atomic_pre_full_fence();
3241 ret = arch_atomic64_fetch_inc_relaxed(v);
3242 __atomic_post_full_fence();
3243 return ret;
3244#else
3245 return raw_atomic64_fetch_add(i: 1, v);
3246#endif
3247}
3248
3249/**
3250 * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
3251 * @v: pointer to atomic64_t
3252 *
3253 * Atomically updates @v to (@v + 1) with acquire ordering.
3254 *
3255 * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere.
3256 *
3257 * Return: The original value of @v.
3258 */
3259static __always_inline s64
3260raw_atomic64_fetch_inc_acquire(atomic64_t *v)
3261{
3262#if defined(arch_atomic64_fetch_inc_acquire)
3263 return arch_atomic64_fetch_inc_acquire(v);
3264#elif defined(arch_atomic64_fetch_inc_relaxed)
3265 s64 ret = arch_atomic64_fetch_inc_relaxed(v);
3266 __atomic_acquire_fence();
3267 return ret;
3268#elif defined(arch_atomic64_fetch_inc)
3269 return arch_atomic64_fetch_inc(v);
3270#else
3271 return raw_atomic64_fetch_add_acquire(i: 1, v);
3272#endif
3273}
3274
3275/**
3276 * raw_atomic64_fetch_inc_release() - atomic increment with release ordering
3277 * @v: pointer to atomic64_t
3278 *
3279 * Atomically updates @v to (@v + 1) with release ordering.
3280 *
3281 * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere.
3282 *
3283 * Return: The original value of @v.
3284 */
3285static __always_inline s64
3286raw_atomic64_fetch_inc_release(atomic64_t *v)
3287{
3288#if defined(arch_atomic64_fetch_inc_release)
3289 return arch_atomic64_fetch_inc_release(v);
3290#elif defined(arch_atomic64_fetch_inc_relaxed)
3291 __atomic_release_fence();
3292 return arch_atomic64_fetch_inc_relaxed(v);
3293#elif defined(arch_atomic64_fetch_inc)
3294 return arch_atomic64_fetch_inc(v);
3295#else
3296 return raw_atomic64_fetch_add_release(i: 1, v);
3297#endif
3298}
3299
3300/**
3301 * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
3302 * @v: pointer to atomic64_t
3303 *
3304 * Atomically updates @v to (@v + 1) with relaxed ordering.
3305 *
3306 * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere.
3307 *
3308 * Return: The original value of @v.
3309 */
3310static __always_inline s64
3311raw_atomic64_fetch_inc_relaxed(atomic64_t *v)
3312{
3313#if defined(arch_atomic64_fetch_inc_relaxed)
3314 return arch_atomic64_fetch_inc_relaxed(v);
3315#elif defined(arch_atomic64_fetch_inc)
3316 return arch_atomic64_fetch_inc(v);
3317#else
3318 return raw_atomic64_fetch_add_relaxed(i: 1, v);
3319#endif
3320}
3321
3322/**
3323 * raw_atomic64_dec() - atomic decrement with relaxed ordering
3324 * @v: pointer to atomic64_t
3325 *
3326 * Atomically updates @v to (@v - 1) with relaxed ordering.
3327 *
3328 * Safe to use in noinstr code; prefer atomic64_dec() elsewhere.
3329 *
3330 * Return: Nothing.
3331 */
3332static __always_inline void
3333raw_atomic64_dec(atomic64_t *v)
3334{
3335#if defined(arch_atomic64_dec)
3336 arch_atomic64_dec(v);
3337#else
3338 raw_atomic64_sub(1, v);
3339#endif
3340}
3341
3342/**
3343 * raw_atomic64_dec_return() - atomic decrement with full ordering
3344 * @v: pointer to atomic64_t
3345 *
3346 * Atomically updates @v to (@v - 1) with full ordering.
3347 *
3348 * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere.
3349 *
3350 * Return: The updated value of @v.
3351 */
3352static __always_inline s64
3353raw_atomic64_dec_return(atomic64_t *v)
3354{
3355#if defined(arch_atomic64_dec_return)
3356 return arch_atomic64_dec_return(v);
3357#elif defined(arch_atomic64_dec_return_relaxed)
3358 s64 ret;
3359 __atomic_pre_full_fence();
3360 ret = arch_atomic64_dec_return_relaxed(v);
3361 __atomic_post_full_fence();
3362 return ret;
3363#else
3364 return raw_atomic64_sub_return(i: 1, v);
3365#endif
3366}
3367
3368/**
3369 * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering
3370 * @v: pointer to atomic64_t
3371 *
3372 * Atomically updates @v to (@v - 1) with acquire ordering.
3373 *
3374 * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere.
3375 *
3376 * Return: The updated value of @v.
3377 */
3378static __always_inline s64
3379raw_atomic64_dec_return_acquire(atomic64_t *v)
3380{
3381#if defined(arch_atomic64_dec_return_acquire)
3382 return arch_atomic64_dec_return_acquire(v);
3383#elif defined(arch_atomic64_dec_return_relaxed)
3384 s64 ret = arch_atomic64_dec_return_relaxed(v);
3385 __atomic_acquire_fence();
3386 return ret;
3387#elif defined(arch_atomic64_dec_return)
3388 return arch_atomic64_dec_return(v);
3389#else
3390 return raw_atomic64_sub_return_acquire(i: 1, v);
3391#endif
3392}
3393
3394/**
3395 * raw_atomic64_dec_return_release() - atomic decrement with release ordering
3396 * @v: pointer to atomic64_t
3397 *
3398 * Atomically updates @v to (@v - 1) with release ordering.
3399 *
3400 * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere.
3401 *
3402 * Return: The updated value of @v.
3403 */
3404static __always_inline s64
3405raw_atomic64_dec_return_release(atomic64_t *v)
3406{
3407#if defined(arch_atomic64_dec_return_release)
3408 return arch_atomic64_dec_return_release(v);
3409#elif defined(arch_atomic64_dec_return_relaxed)
3410 __atomic_release_fence();
3411 return arch_atomic64_dec_return_relaxed(v);
3412#elif defined(arch_atomic64_dec_return)
3413 return arch_atomic64_dec_return(v);
3414#else
3415 return raw_atomic64_sub_return_release(i: 1, v);
3416#endif
3417}
3418
3419/**
3420 * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
3421 * @v: pointer to atomic64_t
3422 *
3423 * Atomically updates @v to (@v - 1) with relaxed ordering.
3424 *
3425 * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere.
3426 *
3427 * Return: The updated value of @v.
3428 */
3429static __always_inline s64
3430raw_atomic64_dec_return_relaxed(atomic64_t *v)
3431{
3432#if defined(arch_atomic64_dec_return_relaxed)
3433 return arch_atomic64_dec_return_relaxed(v);
3434#elif defined(arch_atomic64_dec_return)
3435 return arch_atomic64_dec_return(v);
3436#else
3437 return raw_atomic64_sub_return_relaxed(i: 1, v);
3438#endif
3439}
3440
3441/**
3442 * raw_atomic64_fetch_dec() - atomic decrement with full ordering
3443 * @v: pointer to atomic64_t
3444 *
3445 * Atomically updates @v to (@v - 1) with full ordering.
3446 *
3447 * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere.
3448 *
3449 * Return: The original value of @v.
3450 */
3451static __always_inline s64
3452raw_atomic64_fetch_dec(atomic64_t *v)
3453{
3454#if defined(arch_atomic64_fetch_dec)
3455 return arch_atomic64_fetch_dec(v);
3456#elif defined(arch_atomic64_fetch_dec_relaxed)
3457 s64 ret;
3458 __atomic_pre_full_fence();
3459 ret = arch_atomic64_fetch_dec_relaxed(v);
3460 __atomic_post_full_fence();
3461 return ret;
3462#else
3463 return raw_atomic64_fetch_sub(i: 1, v);
3464#endif
3465}
3466
3467/**
3468 * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
3469 * @v: pointer to atomic64_t
3470 *
3471 * Atomically updates @v to (@v - 1) with acquire ordering.
3472 *
3473 * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere.
3474 *
3475 * Return: The original value of @v.
3476 */
3477static __always_inline s64
3478raw_atomic64_fetch_dec_acquire(atomic64_t *v)
3479{
3480#if defined(arch_atomic64_fetch_dec_acquire)
3481 return arch_atomic64_fetch_dec_acquire(v);
3482#elif defined(arch_atomic64_fetch_dec_relaxed)
3483 s64 ret = arch_atomic64_fetch_dec_relaxed(v);
3484 __atomic_acquire_fence();
3485 return ret;
3486#elif defined(arch_atomic64_fetch_dec)
3487 return arch_atomic64_fetch_dec(v);
3488#else
3489 return raw_atomic64_fetch_sub_acquire(i: 1, v);
3490#endif
3491}
3492
3493/**
3494 * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering
3495 * @v: pointer to atomic64_t
3496 *
3497 * Atomically updates @v to (@v - 1) with release ordering.
3498 *
3499 * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere.
3500 *
3501 * Return: The original value of @v.
3502 */
3503static __always_inline s64
3504raw_atomic64_fetch_dec_release(atomic64_t *v)
3505{
3506#if defined(arch_atomic64_fetch_dec_release)
3507 return arch_atomic64_fetch_dec_release(v);
3508#elif defined(arch_atomic64_fetch_dec_relaxed)
3509 __atomic_release_fence();
3510 return arch_atomic64_fetch_dec_relaxed(v);
3511#elif defined(arch_atomic64_fetch_dec)
3512 return arch_atomic64_fetch_dec(v);
3513#else
3514 return raw_atomic64_fetch_sub_release(i: 1, v);
3515#endif
3516}
3517
3518/**
3519 * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
3520 * @v: pointer to atomic64_t
3521 *
3522 * Atomically updates @v to (@v - 1) with relaxed ordering.
3523 *
3524 * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere.
3525 *
3526 * Return: The original value of @v.
3527 */
3528static __always_inline s64
3529raw_atomic64_fetch_dec_relaxed(atomic64_t *v)
3530{
3531#if defined(arch_atomic64_fetch_dec_relaxed)
3532 return arch_atomic64_fetch_dec_relaxed(v);
3533#elif defined(arch_atomic64_fetch_dec)
3534 return arch_atomic64_fetch_dec(v);
3535#else
3536 return raw_atomic64_fetch_sub_relaxed(i: 1, v);
3537#endif
3538}
3539
3540/**
3541 * raw_atomic64_and() - atomic bitwise AND with relaxed ordering
3542 * @i: s64 value
3543 * @v: pointer to atomic64_t
3544 *
3545 * Atomically updates @v to (@v & @i) with relaxed ordering.
3546 *
3547 * Safe to use in noinstr code; prefer atomic64_and() elsewhere.
3548 *
3549 * Return: Nothing.
3550 */
3551static __always_inline void
3552raw_atomic64_and(s64 i, atomic64_t *v)
3553{
3554 arch_atomic64_and(i, v);
3555}
3556
3557/**
3558 * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering
3559 * @i: s64 value
3560 * @v: pointer to atomic64_t
3561 *
3562 * Atomically updates @v to (@v & @i) with full ordering.
3563 *
3564 * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere.
3565 *
3566 * Return: The original value of @v.
3567 */
3568static __always_inline s64
3569raw_atomic64_fetch_and(s64 i, atomic64_t *v)
3570{
3571#if defined(arch_atomic64_fetch_and)
3572 return arch_atomic64_fetch_and(i, v);
3573#elif defined(arch_atomic64_fetch_and_relaxed)
3574 s64 ret;
3575 __atomic_pre_full_fence();
3576 ret = arch_atomic64_fetch_and_relaxed(i, v);
3577 __atomic_post_full_fence();
3578 return ret;
3579#else
3580#error "Unable to define raw_atomic64_fetch_and"
3581#endif
3582}
3583
3584/**
3585 * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
3586 * @i: s64 value
3587 * @v: pointer to atomic64_t
3588 *
3589 * Atomically updates @v to (@v & @i) with acquire ordering.
3590 *
3591 * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere.
3592 *
3593 * Return: The original value of @v.
3594 */
3595static __always_inline s64
3596raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
3597{
3598#if defined(arch_atomic64_fetch_and_acquire)
3599 return arch_atomic64_fetch_and_acquire(i, v);
3600#elif defined(arch_atomic64_fetch_and_relaxed)
3601 s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
3602 __atomic_acquire_fence();
3603 return ret;
3604#elif defined(arch_atomic64_fetch_and)
3605 return arch_atomic64_fetch_and(i, v);
3606#else
3607#error "Unable to define raw_atomic64_fetch_and_acquire"
3608#endif
3609}
3610
3611/**
3612 * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering
3613 * @i: s64 value
3614 * @v: pointer to atomic64_t
3615 *
3616 * Atomically updates @v to (@v & @i) with release ordering.
3617 *
3618 * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere.
3619 *
3620 * Return: The original value of @v.
3621 */
3622static __always_inline s64
3623raw_atomic64_fetch_and_release(s64 i, atomic64_t *v)
3624{
3625#if defined(arch_atomic64_fetch_and_release)
3626 return arch_atomic64_fetch_and_release(i, v);
3627#elif defined(arch_atomic64_fetch_and_relaxed)
3628 __atomic_release_fence();
3629 return arch_atomic64_fetch_and_relaxed(i, v);
3630#elif defined(arch_atomic64_fetch_and)
3631 return arch_atomic64_fetch_and(i, v);
3632#else
3633#error "Unable to define raw_atomic64_fetch_and_release"
3634#endif
3635}
3636
3637/**
3638 * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
3639 * @i: s64 value
3640 * @v: pointer to atomic64_t
3641 *
3642 * Atomically updates @v to (@v & @i) with relaxed ordering.
3643 *
3644 * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere.
3645 *
3646 * Return: The original value of @v.
3647 */
3648static __always_inline s64
3649raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
3650{
3651#if defined(arch_atomic64_fetch_and_relaxed)
3652 return arch_atomic64_fetch_and_relaxed(i, v);
3653#elif defined(arch_atomic64_fetch_and)
3654 return arch_atomic64_fetch_and(i, v);
3655#else
3656#error "Unable to define raw_atomic64_fetch_and_relaxed"
3657#endif
3658}
3659
3660/**
3661 * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
3662 * @i: s64 value
3663 * @v: pointer to atomic64_t
3664 *
3665 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3666 *
3667 * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere.
3668 *
3669 * Return: Nothing.
3670 */
3671static __always_inline void
3672raw_atomic64_andnot(s64 i, atomic64_t *v)
3673{
3674#if defined(arch_atomic64_andnot)
3675 arch_atomic64_andnot(i, v);
3676#else
3677 raw_atomic64_and(i: ~i, v);
3678#endif
3679}
3680
3681/**
3682 * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
3683 * @i: s64 value
3684 * @v: pointer to atomic64_t
3685 *
3686 * Atomically updates @v to (@v & ~@i) with full ordering.
3687 *
3688 * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere.
3689 *
3690 * Return: The original value of @v.
3691 */
3692static __always_inline s64
3693raw_atomic64_fetch_andnot(s64 i, atomic64_t *v)
3694{
3695#if defined(arch_atomic64_fetch_andnot)
3696 return arch_atomic64_fetch_andnot(i, v);
3697#elif defined(arch_atomic64_fetch_andnot_relaxed)
3698 s64 ret;
3699 __atomic_pre_full_fence();
3700 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
3701 __atomic_post_full_fence();
3702 return ret;
3703#else
3704 return raw_atomic64_fetch_and(i: ~i, v);
3705#endif
3706}
3707
3708/**
3709 * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
3710 * @i: s64 value
3711 * @v: pointer to atomic64_t
3712 *
3713 * Atomically updates @v to (@v & ~@i) with acquire ordering.
3714 *
3715 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere.
3716 *
3717 * Return: The original value of @v.
3718 */
3719static __always_inline s64
3720raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
3721{
3722#if defined(arch_atomic64_fetch_andnot_acquire)
3723 return arch_atomic64_fetch_andnot_acquire(i, v);
3724#elif defined(arch_atomic64_fetch_andnot_relaxed)
3725 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
3726 __atomic_acquire_fence();
3727 return ret;
3728#elif defined(arch_atomic64_fetch_andnot)
3729 return arch_atomic64_fetch_andnot(i, v);
3730#else
3731 return raw_atomic64_fetch_and_acquire(i: ~i, v);
3732#endif
3733}
3734
3735/**
3736 * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
3737 * @i: s64 value
3738 * @v: pointer to atomic64_t
3739 *
3740 * Atomically updates @v to (@v & ~@i) with release ordering.
3741 *
3742 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere.
3743 *
3744 * Return: The original value of @v.
3745 */
3746static __always_inline s64
3747raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
3748{
3749#if defined(arch_atomic64_fetch_andnot_release)
3750 return arch_atomic64_fetch_andnot_release(i, v);
3751#elif defined(arch_atomic64_fetch_andnot_relaxed)
3752 __atomic_release_fence();
3753 return arch_atomic64_fetch_andnot_relaxed(i, v);
3754#elif defined(arch_atomic64_fetch_andnot)
3755 return arch_atomic64_fetch_andnot(i, v);
3756#else
3757 return raw_atomic64_fetch_and_release(i: ~i, v);
3758#endif
3759}
3760
3761/**
3762 * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
3763 * @i: s64 value
3764 * @v: pointer to atomic64_t
3765 *
3766 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3767 *
3768 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_relaxed() elsewhere.
3769 *
3770 * Return: The original value of @v.
3771 */
3772static __always_inline s64
3773raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
3774{
3775#if defined(arch_atomic64_fetch_andnot_relaxed)
3776 return arch_atomic64_fetch_andnot_relaxed(i, v);
3777#elif defined(arch_atomic64_fetch_andnot)
3778 return arch_atomic64_fetch_andnot(i, v);
3779#else
3780 return raw_atomic64_fetch_and_relaxed(i: ~i, v);
3781#endif
3782}
3783
3784/**
3785 * raw_atomic64_or() - atomic bitwise OR with relaxed ordering
3786 * @i: s64 value
3787 * @v: pointer to atomic64_t
3788 *
3789 * Atomically updates @v to (@v | @i) with relaxed ordering.
3790 *
3791 * Safe to use in noinstr code; prefer atomic64_or() elsewhere.
3792 *
3793 * Return: Nothing.
3794 */
3795static __always_inline void
3796raw_atomic64_or(s64 i, atomic64_t *v)
3797{
3798 arch_atomic64_or(i, v);
3799}
3800
3801/**
3802 * raw_atomic64_fetch_or() - atomic bitwise OR with full ordering
3803 * @i: s64 value
3804 * @v: pointer to atomic64_t
3805 *
3806 * Atomically updates @v to (@v | @i) with full ordering.
3807 *
3808 * Safe to use in noinstr code; prefer atomic64_fetch_or() elsewhere.
3809 *
3810 * Return: The original value of @v.
3811 */
3812static __always_inline s64
3813raw_atomic64_fetch_or(s64 i, atomic64_t *v)
3814{
3815#if defined(arch_atomic64_fetch_or)
3816 return arch_atomic64_fetch_or(i, v);
3817#elif defined(arch_atomic64_fetch_or_relaxed)
3818 s64 ret;
3819 __atomic_pre_full_fence();
3820 ret = arch_atomic64_fetch_or_relaxed(i, v);
3821 __atomic_post_full_fence();
3822 return ret;
3823#else
3824#error "Unable to define raw_atomic64_fetch_or"
3825#endif
3826}
3827
3828/**
3829 * raw_atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
3830 * @i: s64 value
3831 * @v: pointer to atomic64_t
3832 *
3833 * Atomically updates @v to (@v | @i) with acquire ordering.
3834 *
3835 * Safe to use in noinstr code; prefer atomic64_fetch_or_acquire() elsewhere.
3836 *
3837 * Return: The original value of @v.
3838 */
3839static __always_inline s64
3840raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
3841{
3842#if defined(arch_atomic64_fetch_or_acquire)
3843 return arch_atomic64_fetch_or_acquire(i, v);
3844#elif defined(arch_atomic64_fetch_or_relaxed)
3845 s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
3846 __atomic_acquire_fence();
3847 return ret;
3848#elif defined(arch_atomic64_fetch_or)
3849 return arch_atomic64_fetch_or(i, v);
3850#else
3851#error "Unable to define raw_atomic64_fetch_or_acquire"
3852#endif
3853}
3854
3855/**
3856 * raw_atomic64_fetch_or_release() - atomic bitwise OR with release ordering
3857 * @i: s64 value
3858 * @v: pointer to atomic64_t
3859 *
3860 * Atomically updates @v to (@v | @i) with release ordering.
3861 *
3862 * Safe to use in noinstr code; prefer atomic64_fetch_or_release() elsewhere.
3863 *
3864 * Return: The original value of @v.
3865 */
3866static __always_inline s64
3867raw_atomic64_fetch_or_release(s64 i, atomic64_t *v)
3868{
3869#if defined(arch_atomic64_fetch_or_release)
3870 return arch_atomic64_fetch_or_release(i, v);
3871#elif defined(arch_atomic64_fetch_or_relaxed)
3872 __atomic_release_fence();
3873 return arch_atomic64_fetch_or_relaxed(i, v);
3874#elif defined(arch_atomic64_fetch_or)
3875 return arch_atomic64_fetch_or(i, v);
3876#else
3877#error "Unable to define raw_atomic64_fetch_or_release"
3878#endif
3879}
3880
3881/**
3882 * raw_atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
3883 * @i: s64 value
3884 * @v: pointer to atomic64_t
3885 *
3886 * Atomically updates @v to (@v | @i) with relaxed ordering.
3887 *
3888 * Safe to use in noinstr code; prefer atomic64_fetch_or_relaxed() elsewhere.
3889 *
3890 * Return: The original value of @v.
3891 */
3892static __always_inline s64
3893raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
3894{
3895#if defined(arch_atomic64_fetch_or_relaxed)
3896 return arch_atomic64_fetch_or_relaxed(i, v);
3897#elif defined(arch_atomic64_fetch_or)
3898 return arch_atomic64_fetch_or(i, v);
3899#else
3900#error "Unable to define raw_atomic64_fetch_or_relaxed"
3901#endif
3902}
3903
3904/**
3905 * raw_atomic64_xor() - atomic bitwise XOR with relaxed ordering
3906 * @i: s64 value
3907 * @v: pointer to atomic64_t
3908 *
3909 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
3910 *
3911 * Safe to use in noinstr code; prefer atomic64_xor() elsewhere.
3912 *
3913 * Return: Nothing.
3914 */
3915static __always_inline void
3916raw_atomic64_xor(s64 i, atomic64_t *v)
3917{
3918 arch_atomic64_xor(i, v);
3919}
3920
3921/**
3922 * raw_atomic64_fetch_xor() - atomic bitwise XOR with full ordering
3923 * @i: s64 value
3924 * @v: pointer to atomic64_t
3925 *
3926 * Atomically updates @v to (@v ^ @i) with full ordering.
3927 *
3928 * Safe to use in noinstr code; prefer atomic64_fetch_xor() elsewhere.
3929 *
3930 * Return: The original value of @v.
3931 */
3932static __always_inline s64
3933raw_atomic64_fetch_xor(s64 i, atomic64_t *v)
3934{
3935#if defined(arch_atomic64_fetch_xor)
3936 return arch_atomic64_fetch_xor(i, v);
3937#elif defined(arch_atomic64_fetch_xor_relaxed)
3938 s64 ret;
3939 __atomic_pre_full_fence();
3940 ret = arch_atomic64_fetch_xor_relaxed(i, v);
3941 __atomic_post_full_fence();
3942 return ret;
3943#else
3944#error "Unable to define raw_atomic64_fetch_xor"
3945#endif
3946}
3947
3948/**
3949 * raw_atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
3950 * @i: s64 value
3951 * @v: pointer to atomic64_t
3952 *
3953 * Atomically updates @v to (@v ^ @i) with acquire ordering.
3954 *
3955 * Safe to use in noinstr code; prefer atomic64_fetch_xor_acquire() elsewhere.
3956 *
3957 * Return: The original value of @v.
3958 */
3959static __always_inline s64
3960raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
3961{
3962#if defined(arch_atomic64_fetch_xor_acquire)
3963 return arch_atomic64_fetch_xor_acquire(i, v);
3964#elif defined(arch_atomic64_fetch_xor_relaxed)
3965 s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
3966 __atomic_acquire_fence();
3967 return ret;
3968#elif defined(arch_atomic64_fetch_xor)
3969 return arch_atomic64_fetch_xor(i, v);
3970#else
3971#error "Unable to define raw_atomic64_fetch_xor_acquire"
3972#endif
3973}
3974
3975/**
3976 * raw_atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
3977 * @i: s64 value
3978 * @v: pointer to atomic64_t
3979 *
3980 * Atomically updates @v to (@v ^ @i) with release ordering.
3981 *
3982 * Safe to use in noinstr code; prefer atomic64_fetch_xor_release() elsewhere.
3983 *
3984 * Return: The original value of @v.
3985 */
3986static __always_inline s64
3987raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
3988{
3989#if defined(arch_atomic64_fetch_xor_release)
3990 return arch_atomic64_fetch_xor_release(i, v);
3991#elif defined(arch_atomic64_fetch_xor_relaxed)
3992 __atomic_release_fence();
3993 return arch_atomic64_fetch_xor_relaxed(i, v);
3994#elif defined(arch_atomic64_fetch_xor)
3995 return arch_atomic64_fetch_xor(i, v);
3996#else
3997#error "Unable to define raw_atomic64_fetch_xor_release"
3998#endif
3999}
4000
4001/**
4002 * raw_atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
4003 * @i: s64 value
4004 * @v: pointer to atomic64_t
4005 *
4006 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
4007 *
4008 * Safe to use in noinstr code; prefer atomic64_fetch_xor_relaxed() elsewhere.
4009 *
4010 * Return: The original value of @v.
4011 */
4012static __always_inline s64
4013raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
4014{
4015#if defined(arch_atomic64_fetch_xor_relaxed)
4016 return arch_atomic64_fetch_xor_relaxed(i, v);
4017#elif defined(arch_atomic64_fetch_xor)
4018 return arch_atomic64_fetch_xor(i, v);
4019#else
4020#error "Unable to define raw_atomic64_fetch_xor_relaxed"
4021#endif
4022}
4023
4024/**
4025 * raw_atomic64_xchg() - atomic exchange with full ordering
4026 * @v: pointer to atomic64_t
4027 * @new: s64 value to assign
4028 *
4029 * Atomically updates @v to @new with full ordering.
4030 *
4031 * Safe to use in noinstr code; prefer atomic64_xchg() elsewhere.
4032 *
4033 * Return: The original value of @v.
4034 */
4035static __always_inline s64
4036raw_atomic64_xchg(atomic64_t *v, s64 new)
4037{
4038#if defined(arch_atomic64_xchg)
4039 return arch_atomic64_xchg(v, new);
4040#elif defined(arch_atomic64_xchg_relaxed)
4041 s64 ret;
4042 __atomic_pre_full_fence();
4043 ret = arch_atomic64_xchg_relaxed(v, new);
4044 __atomic_post_full_fence();
4045 return ret;
4046#else
4047 return raw_xchg(&v->counter, new);
4048#endif
4049}
4050
4051/**
4052 * raw_atomic64_xchg_acquire() - atomic exchange with acquire ordering
4053 * @v: pointer to atomic64_t
4054 * @new: s64 value to assign
4055 *
4056 * Atomically updates @v to @new with acquire ordering.
4057 *
4058 * Safe to use in noinstr code; prefer atomic64_xchg_acquire() elsewhere.
4059 *
4060 * Return: The original value of @v.
4061 */
4062static __always_inline s64
4063raw_atomic64_xchg_acquire(atomic64_t *v, s64 new)
4064{
4065#if defined(arch_atomic64_xchg_acquire)
4066 return arch_atomic64_xchg_acquire(v, new);
4067#elif defined(arch_atomic64_xchg_relaxed)
4068 s64 ret = arch_atomic64_xchg_relaxed(v, new);
4069 __atomic_acquire_fence();
4070 return ret;
4071#elif defined(arch_atomic64_xchg)
4072 return arch_atomic64_xchg(v, new);
4073#else
4074 return raw_xchg_acquire(&v->counter, new);
4075#endif
4076}
4077
4078/**
4079 * raw_atomic64_xchg_release() - atomic exchange with release ordering
4080 * @v: pointer to atomic64_t
4081 * @new: s64 value to assign
4082 *
4083 * Atomically updates @v to @new with release ordering.
4084 *
4085 * Safe to use in noinstr code; prefer atomic64_xchg_release() elsewhere.
4086 *
4087 * Return: The original value of @v.
4088 */
4089static __always_inline s64
4090raw_atomic64_xchg_release(atomic64_t *v, s64 new)
4091{
4092#if defined(arch_atomic64_xchg_release)
4093 return arch_atomic64_xchg_release(v, new);
4094#elif defined(arch_atomic64_xchg_relaxed)
4095 __atomic_release_fence();
4096 return arch_atomic64_xchg_relaxed(v, new);
4097#elif defined(arch_atomic64_xchg)
4098 return arch_atomic64_xchg(v, new);
4099#else
4100 return raw_xchg_release(&v->counter, new);
4101#endif
4102}
4103
4104/**
4105 * raw_atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
4106 * @v: pointer to atomic64_t
4107 * @new: s64 value to assign
4108 *
4109 * Atomically updates @v to @new with relaxed ordering.
4110 *
4111 * Safe to use in noinstr code; prefer atomic64_xchg_relaxed() elsewhere.
4112 *
4113 * Return: The original value of @v.
4114 */
4115static __always_inline s64
4116raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new)
4117{
4118#if defined(arch_atomic64_xchg_relaxed)
4119 return arch_atomic64_xchg_relaxed(v, new);
4120#elif defined(arch_atomic64_xchg)
4121 return arch_atomic64_xchg(v, new);
4122#else
4123 return raw_xchg_relaxed(&v->counter, new);
4124#endif
4125}
4126
4127/**
4128 * raw_atomic64_cmpxchg() - atomic compare and exchange with full ordering
4129 * @v: pointer to atomic64_t
4130 * @old: s64 value to compare with
4131 * @new: s64 value to assign
4132 *
4133 * If (@v == @old), atomically updates @v to @new with full ordering.
4134 * Otherwise, @v is not modified and relaxed ordering is provided.
4135 *
4136 * Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere.
4137 *
4138 * Return: The original value of @v.
4139 */
4140static __always_inline s64
4141raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
4142{
4143#if defined(arch_atomic64_cmpxchg)
4144 return arch_atomic64_cmpxchg(v, old, new);
4145#elif defined(arch_atomic64_cmpxchg_relaxed)
4146 s64 ret;
4147 __atomic_pre_full_fence();
4148 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
4149 __atomic_post_full_fence();
4150 return ret;
4151#else
4152 return raw_cmpxchg(&v->counter, old, new);
4153#endif
4154}
4155
4156/**
4157 * raw_atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4158 * @v: pointer to atomic64_t
4159 * @old: s64 value to compare with
4160 * @new: s64 value to assign
4161 *
4162 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4163 * Otherwise, @v is not modified and relaxed ordering is provided.
4164 *
4165 * Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere.
4166 *
4167 * Return: The original value of @v.
4168 */
4169static __always_inline s64
4170raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
4171{
4172#if defined(arch_atomic64_cmpxchg_acquire)
4173 return arch_atomic64_cmpxchg_acquire(v, old, new);
4174#elif defined(arch_atomic64_cmpxchg_relaxed)
4175 s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
4176 __atomic_acquire_fence();
4177 return ret;
4178#elif defined(arch_atomic64_cmpxchg)
4179 return arch_atomic64_cmpxchg(v, old, new);
4180#else
4181 return raw_cmpxchg_acquire(&v->counter, old, new);
4182#endif
4183}
4184
4185/**
4186 * raw_atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
4187 * @v: pointer to atomic64_t
4188 * @old: s64 value to compare with
4189 * @new: s64 value to assign
4190 *
4191 * If (@v == @old), atomically updates @v to @new with release ordering.
4192 * Otherwise, @v is not modified and relaxed ordering is provided.
4193 *
4194 * Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere.
4195 *
4196 * Return: The original value of @v.
4197 */
4198static __always_inline s64
4199raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
4200{
4201#if defined(arch_atomic64_cmpxchg_release)
4202 return arch_atomic64_cmpxchg_release(v, old, new);
4203#elif defined(arch_atomic64_cmpxchg_relaxed)
4204 __atomic_release_fence();
4205 return arch_atomic64_cmpxchg_relaxed(v, old, new);
4206#elif defined(arch_atomic64_cmpxchg)
4207 return arch_atomic64_cmpxchg(v, old, new);
4208#else
4209 return raw_cmpxchg_release(&v->counter, old, new);
4210#endif
4211}
4212
4213/**
4214 * raw_atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4215 * @v: pointer to atomic64_t
4216 * @old: s64 value to compare with
4217 * @new: s64 value to assign
4218 *
4219 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4220 * Otherwise, @v is not modified and relaxed ordering is provided.
4221 *
4222 * Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere.
4223 *
4224 * Return: The original value of @v.
4225 */
4226static __always_inline s64
4227raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
4228{
4229#if defined(arch_atomic64_cmpxchg_relaxed)
4230 return arch_atomic64_cmpxchg_relaxed(v, old, new);
4231#elif defined(arch_atomic64_cmpxchg)
4232 return arch_atomic64_cmpxchg(v, old, new);
4233#else
4234 return raw_cmpxchg_relaxed(&v->counter, old, new);
4235#endif
4236}
4237
4238/**
4239 * raw_atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
4240 * @v: pointer to atomic64_t
4241 * @old: pointer to s64 value to compare with
4242 * @new: s64 value to assign
4243 *
4244 * If (@v == @old), atomically updates @v to @new with full ordering.
4245 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4246 * and relaxed ordering is provided.
4247 *
4248 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere.
4249 *
4250 * Return: @true if the exchange occured, @false otherwise.
4251 */
4252static __always_inline bool
4253raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
4254{
4255#if defined(arch_atomic64_try_cmpxchg)
4256 return arch_atomic64_try_cmpxchg(v, old, new);
4257#elif defined(arch_atomic64_try_cmpxchg_relaxed)
4258 bool ret;
4259 __atomic_pre_full_fence();
4260 ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4261 __atomic_post_full_fence();
4262 return ret;
4263#else
4264 s64 r, o = *old;
4265 r = raw_atomic64_cmpxchg(v, o, new);
4266 if (unlikely(r != o))
4267 *old = r;
4268 return likely(r == o);
4269#endif
4270}
4271
4272/**
4273 * raw_atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4274 * @v: pointer to atomic64_t
4275 * @old: pointer to s64 value to compare with
4276 * @new: s64 value to assign
4277 *
4278 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4279 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4280 * and relaxed ordering is provided.
4281 *
4282 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere.
4283 *
4284 * Return: @true if the exchange occured, @false otherwise.
4285 */
4286static __always_inline bool
4287raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
4288{
4289#if defined(arch_atomic64_try_cmpxchg_acquire)
4290 return arch_atomic64_try_cmpxchg_acquire(v, old, new);
4291#elif defined(arch_atomic64_try_cmpxchg_relaxed)
4292 bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4293 __atomic_acquire_fence();
4294 return ret;
4295#elif defined(arch_atomic64_try_cmpxchg)
4296 return arch_atomic64_try_cmpxchg(v, old, new);
4297#else
4298 s64 r, o = *old;
4299 r = raw_atomic64_cmpxchg_acquire(v, o, new);
4300 if (unlikely(r != o))
4301 *old = r;
4302 return likely(r == o);
4303#endif
4304}
4305
4306/**
4307 * raw_atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
4308 * @v: pointer to atomic64_t
4309 * @old: pointer to s64 value to compare with
4310 * @new: s64 value to assign
4311 *
4312 * If (@v == @old), atomically updates @v to @new with release ordering.
4313 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4314 * and relaxed ordering is provided.
4315 *
4316 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere.
4317 *
4318 * Return: @true if the exchange occured, @false otherwise.
4319 */
4320static __always_inline bool
4321raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
4322{
4323#if defined(arch_atomic64_try_cmpxchg_release)
4324 return arch_atomic64_try_cmpxchg_release(v, old, new);
4325#elif defined(arch_atomic64_try_cmpxchg_relaxed)
4326 __atomic_release_fence();
4327 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4328#elif defined(arch_atomic64_try_cmpxchg)
4329 return arch_atomic64_try_cmpxchg(v, old, new);
4330#else
4331 s64 r, o = *old;
4332 r = raw_atomic64_cmpxchg_release(v, o, new);
4333 if (unlikely(r != o))
4334 *old = r;
4335 return likely(r == o);
4336#endif
4337}
4338
4339/**
4340 * raw_atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4341 * @v: pointer to atomic64_t
4342 * @old: pointer to s64 value to compare with
4343 * @new: s64 value to assign
4344 *
4345 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4346 * Otherwise, @v is not modified, @old is updated to the current value of @v,
4347 * and relaxed ordering is provided.
4348 *
4349 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere.
4350 *
4351 * Return: @true if the exchange occured, @false otherwise.
4352 */
4353static __always_inline bool
4354raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
4355{
4356#if defined(arch_atomic64_try_cmpxchg_relaxed)
4357 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4358#elif defined(arch_atomic64_try_cmpxchg)
4359 return arch_atomic64_try_cmpxchg(v, old, new);
4360#else
4361 s64 r, o = *old;
4362 r = raw_atomic64_cmpxchg_relaxed(v, o, new);
4363 if (unlikely(r != o))
4364 *old = r;
4365 return likely(r == o);
4366#endif
4367}
4368
4369/**
4370 * raw_atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
4371 * @i: s64 value to subtract
4372 * @v: pointer to atomic64_t
4373 *
4374 * Atomically updates @v to (@v - @i) with full ordering.
4375 *
4376 * Safe to use in noinstr code; prefer atomic64_sub_and_test() elsewhere.
4377 *
4378 * Return: @true if the resulting value of @v is zero, @false otherwise.
4379 */
4380static __always_inline bool
4381raw_atomic64_sub_and_test(s64 i, atomic64_t *v)
4382{
4383#if defined(arch_atomic64_sub_and_test)
4384 return arch_atomic64_sub_and_test(i, v);
4385#else
4386 return raw_atomic64_sub_return(i, v) == 0;
4387#endif
4388}
4389
4390/**
4391 * raw_atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
4392 * @v: pointer to atomic64_t
4393 *
4394 * Atomically updates @v to (@v - 1) with full ordering.
4395 *
4396 * Safe to use in noinstr code; prefer atomic64_dec_and_test() elsewhere.
4397 *
4398 * Return: @true if the resulting value of @v is zero, @false otherwise.
4399 */
4400static __always_inline bool
4401raw_atomic64_dec_and_test(atomic64_t *v)
4402{
4403#if defined(arch_atomic64_dec_and_test)
4404 return arch_atomic64_dec_and_test(v);
4405#else
4406 return raw_atomic64_dec_return(v) == 0;
4407#endif
4408}
4409
4410/**
4411 * raw_atomic64_inc_and_test() - atomic increment and test if zero with full ordering
4412 * @v: pointer to atomic64_t
4413 *
4414 * Atomically updates @v to (@v + 1) with full ordering.
4415 *
4416 * Safe to use in noinstr code; prefer atomic64_inc_and_test() elsewhere.
4417 *
4418 * Return: @true if the resulting value of @v is zero, @false otherwise.
4419 */
4420static __always_inline bool
4421raw_atomic64_inc_and_test(atomic64_t *v)
4422{
4423#if defined(arch_atomic64_inc_and_test)
4424 return arch_atomic64_inc_and_test(v);
4425#else
4426 return raw_atomic64_inc_return(v) == 0;
4427#endif
4428}
4429
4430/**
4431 * raw_atomic64_add_negative() - atomic add and test if negative with full ordering
4432 * @i: s64 value to add
4433 * @v: pointer to atomic64_t
4434 *
4435 * Atomically updates @v to (@v + @i) with full ordering.
4436 *
4437 * Safe to use in noinstr code; prefer atomic64_add_negative() elsewhere.
4438 *
4439 * Return: @true if the resulting value of @v is negative, @false otherwise.
4440 */
4441static __always_inline bool
4442raw_atomic64_add_negative(s64 i, atomic64_t *v)
4443{
4444#if defined(arch_atomic64_add_negative)
4445 return arch_atomic64_add_negative(i, v);
4446#elif defined(arch_atomic64_add_negative_relaxed)
4447 bool ret;
4448 __atomic_pre_full_fence();
4449 ret = arch_atomic64_add_negative_relaxed(i, v);
4450 __atomic_post_full_fence();
4451 return ret;
4452#else
4453 return raw_atomic64_add_return(i, v) < 0;
4454#endif
4455}
4456
4457/**
4458 * raw_atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
4459 * @i: s64 value to add
4460 * @v: pointer to atomic64_t
4461 *
4462 * Atomically updates @v to (@v + @i) with acquire ordering.
4463 *
4464 * Safe to use in noinstr code; prefer atomic64_add_negative_acquire() elsewhere.
4465 *
4466 * Return: @true if the resulting value of @v is negative, @false otherwise.
4467 */
4468static __always_inline bool
4469raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
4470{
4471#if defined(arch_atomic64_add_negative_acquire)
4472 return arch_atomic64_add_negative_acquire(i, v);
4473#elif defined(arch_atomic64_add_negative_relaxed)
4474 bool ret = arch_atomic64_add_negative_relaxed(i, v);
4475 __atomic_acquire_fence();
4476 return ret;
4477#elif defined(arch_atomic64_add_negative)
4478 return arch_atomic64_add_negative(i, v);
4479#else
4480 return raw_atomic64_add_return_acquire(i, v) < 0;
4481#endif
4482}
4483
4484/**
4485 * raw_atomic64_add_negative_release() - atomic add and test if negative with release ordering
4486 * @i: s64 value to add
4487 * @v: pointer to atomic64_t
4488 *
4489 * Atomically updates @v to (@v + @i) with release ordering.
4490 *
4491 * Safe to use in noinstr code; prefer atomic64_add_negative_release() elsewhere.
4492 *
4493 * Return: @true if the resulting value of @v is negative, @false otherwise.
4494 */
4495static __always_inline bool
4496raw_atomic64_add_negative_release(s64 i, atomic64_t *v)
4497{
4498#if defined(arch_atomic64_add_negative_release)
4499 return arch_atomic64_add_negative_release(i, v);
4500#elif defined(arch_atomic64_add_negative_relaxed)
4501 __atomic_release_fence();
4502 return arch_atomic64_add_negative_relaxed(i, v);
4503#elif defined(arch_atomic64_add_negative)
4504 return arch_atomic64_add_negative(i, v);
4505#else
4506 return raw_atomic64_add_return_release(i, v) < 0;
4507#endif
4508}
4509
4510/**
4511 * raw_atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
4512 * @i: s64 value to add
4513 * @v: pointer to atomic64_t
4514 *
4515 * Atomically updates @v to (@v + @i) with relaxed ordering.
4516 *
4517 * Safe to use in noinstr code; prefer atomic64_add_negative_relaxed() elsewhere.
4518 *
4519 * Return: @true if the resulting value of @v is negative, @false otherwise.
4520 */
4521static __always_inline bool
4522raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
4523{
4524#if defined(arch_atomic64_add_negative_relaxed)
4525 return arch_atomic64_add_negative_relaxed(i, v);
4526#elif defined(arch_atomic64_add_negative)
4527 return arch_atomic64_add_negative(i, v);
4528#else
4529 return raw_atomic64_add_return_relaxed(i, v) < 0;
4530#endif
4531}
4532
4533/**
4534 * raw_atomic64_fetch_add_unless() - atomic add unless value with full ordering
4535 * @v: pointer to atomic64_t
4536 * @a: s64 value to add
4537 * @u: s64 value to compare with
4538 *
4539 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4540 * Otherwise, @v is not modified and relaxed ordering is provided.
4541 *
4542 * Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere.
4543 *
4544 * Return: The original value of @v.
4545 */
4546static __always_inline s64
4547raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
4548{
4549#if defined(arch_atomic64_fetch_add_unless)
4550 return arch_atomic64_fetch_add_unless(v, a, u);
4551#else
4552 s64 c = raw_atomic64_read(v);
4553
4554 do {
4555 if (unlikely(c == u))
4556 break;
4557 } while (!raw_atomic64_try_cmpxchg(v, old: &c, new: c + a));
4558
4559 return c;
4560#endif
4561}
4562
4563/**
4564 * raw_atomic64_add_unless() - atomic add unless value with full ordering
4565 * @v: pointer to atomic64_t
4566 * @a: s64 value to add
4567 * @u: s64 value to compare with
4568 *
4569 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4570 * Otherwise, @v is not modified and relaxed ordering is provided.
4571 *
4572 * Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere.
4573 *
4574 * Return: @true if @v was updated, @false otherwise.
4575 */
4576static __always_inline bool
4577raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
4578{
4579#if defined(arch_atomic64_add_unless)
4580 return arch_atomic64_add_unless(v, a, u);
4581#else
4582 return raw_atomic64_fetch_add_unless(v, a, u) != u;
4583#endif
4584}
4585
4586/**
4587 * raw_atomic64_inc_not_zero() - atomic increment unless zero with full ordering
4588 * @v: pointer to atomic64_t
4589 *
4590 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
4591 * Otherwise, @v is not modified and relaxed ordering is provided.
4592 *
4593 * Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere.
4594 *
4595 * Return: @true if @v was updated, @false otherwise.
4596 */
4597static __always_inline bool
4598raw_atomic64_inc_not_zero(atomic64_t *v)
4599{
4600#if defined(arch_atomic64_inc_not_zero)
4601 return arch_atomic64_inc_not_zero(v);
4602#else
4603 return raw_atomic64_add_unless(v, a: 1, u: 0);
4604#endif
4605}
4606
4607/**
4608 * raw_atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
4609 * @v: pointer to atomic64_t
4610 *
4611 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
4612 * Otherwise, @v is not modified and relaxed ordering is provided.
4613 *
4614 * Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere.
4615 *
4616 * Return: @true if @v was updated, @false otherwise.
4617 */
4618static __always_inline bool
4619raw_atomic64_inc_unless_negative(atomic64_t *v)
4620{
4621#if defined(arch_atomic64_inc_unless_negative)
4622 return arch_atomic64_inc_unless_negative(v);
4623#else
4624 s64 c = raw_atomic64_read(v);
4625
4626 do {
4627 if (unlikely(c < 0))
4628 return false;
4629 } while (!raw_atomic64_try_cmpxchg(v, old: &c, new: c + 1));
4630
4631 return true;
4632#endif
4633}
4634
4635/**
4636 * raw_atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
4637 * @v: pointer to atomic64_t
4638 *
4639 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
4640 * Otherwise, @v is not modified and relaxed ordering is provided.
4641 *
4642 * Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere.
4643 *
4644 * Return: @true if @v was updated, @false otherwise.
4645 */
4646static __always_inline bool
4647raw_atomic64_dec_unless_positive(atomic64_t *v)
4648{
4649#if defined(arch_atomic64_dec_unless_positive)
4650 return arch_atomic64_dec_unless_positive(v);
4651#else
4652 s64 c = raw_atomic64_read(v);
4653
4654 do {
4655 if (unlikely(c > 0))
4656 return false;
4657 } while (!raw_atomic64_try_cmpxchg(v, old: &c, new: c - 1));
4658
4659 return true;
4660#endif
4661}
4662
4663/**
4664 * raw_atomic64_dec_if_positive() - atomic decrement if positive with full ordering
4665 * @v: pointer to atomic64_t
4666 *
4667 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
4668 * Otherwise, @v is not modified and relaxed ordering is provided.
4669 *
4670 * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere.
4671 *
4672 * Return: The old value of (@v - 1), regardless of whether @v was updated.
4673 */
4674static __always_inline s64
4675raw_atomic64_dec_if_positive(atomic64_t *v)
4676{
4677#if defined(arch_atomic64_dec_if_positive)
4678 return arch_atomic64_dec_if_positive(v);
4679#else
4680 s64 dec, c = raw_atomic64_read(v);
4681
4682 do {
4683 dec = c - 1;
4684 if (unlikely(dec < 0))
4685 break;
4686 } while (!raw_atomic64_try_cmpxchg(v, old: &c, new: dec));
4687
4688 return dec;
4689#endif
4690}
4691
4692#endif /* _LINUX_ATOMIC_FALLBACK_H */
4693// b565db590afeeff0d7c9485ccbca5bb6e155749f
4694