1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __LINUX_COOKIE_H
3#define __LINUX_COOKIE_H
4
5#include <linux/atomic.h>
6#include <linux/percpu.h>
7#include <asm/local.h>
8
9struct pcpu_gen_cookie {
10 local_t nesting;
11 u64 last;
12} __aligned(16);
13
14struct gen_cookie {
15 struct pcpu_gen_cookie __percpu *local;
16 atomic64_t forward_last ____cacheline_aligned_in_smp;
17 atomic64_t reverse_last;
18};
19
20#define COOKIE_LOCAL_BATCH 4096
21
22#define DEFINE_COOKIE(name) \
23 static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name); \
24 static struct gen_cookie name = { \
25 .local = &__##name, \
26 .forward_last = ATOMIC64_INIT(0), \
27 .reverse_last = ATOMIC64_INIT(0), \
28 }
29
30static __always_inline u64 gen_cookie_next(struct gen_cookie *gc)
31{
32 struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local);
33 u64 val;
34
35 if (likely(local_inc_return(&local->nesting) == 1)) {
36 val = local->last;
37 if (__is_defined(CONFIG_SMP) &&
38 unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) {
39 s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH,
40 v: &gc->forward_last);
41 val = next - COOKIE_LOCAL_BATCH;
42 }
43 local->last = ++val;
44 } else {
45 val = atomic64_dec_return(v: &gc->reverse_last);
46 }
47 local_dec(l: &local->nesting);
48 return val;
49}
50
51#endif /* __LINUX_COOKIE_H */
52