xref: /linux/include/linux/cookie.h (revision 92acdc58) 
1*92acdc58SDaniel Borkmann /* SPDX-License-Identifier: GPL-2.0 */
2*92acdc58SDaniel Borkmann #ifndef __LINUX_COOKIE_H
3*92acdc58SDaniel Borkmann #define __LINUX_COOKIE_H
4*92acdc58SDaniel Borkmann 
5*92acdc58SDaniel Borkmann #include <linux/atomic.h>
6*92acdc58SDaniel Borkmann #include <linux/percpu.h>
7*92acdc58SDaniel Borkmann #include <asm/local.h>
8*92acdc58SDaniel Borkmann 
9*92acdc58SDaniel Borkmann struct pcpu_gen_cookie {
10*92acdc58SDaniel Borkmann 	local_t nesting;
11*92acdc58SDaniel Borkmann 	u64 last;
12*92acdc58SDaniel Borkmann } __aligned(16);
13*92acdc58SDaniel Borkmann 
14*92acdc58SDaniel Borkmann struct gen_cookie {
15*92acdc58SDaniel Borkmann 	struct pcpu_gen_cookie __percpu *local;
16*92acdc58SDaniel Borkmann 	atomic64_t forward_last ____cacheline_aligned_in_smp;
17*92acdc58SDaniel Borkmann 	atomic64_t reverse_last;
18*92acdc58SDaniel Borkmann };
19*92acdc58SDaniel Borkmann 
20*92acdc58SDaniel Borkmann #define COOKIE_LOCAL_BATCH	4096
21*92acdc58SDaniel Borkmann 
22*92acdc58SDaniel Borkmann #define DEFINE_COOKIE(name)						\
23*92acdc58SDaniel Borkmann 	static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name);	\
24*92acdc58SDaniel Borkmann 	static struct gen_cookie name = {				\
25*92acdc58SDaniel Borkmann 		.local		= &__##name,				\
26*92acdc58SDaniel Borkmann 		.forward_last	= ATOMIC64_INIT(0),			\
27*92acdc58SDaniel Borkmann 		.reverse_last	= ATOMIC64_INIT(0),			\
28*92acdc58SDaniel Borkmann 	}
29*92acdc58SDaniel Borkmann 
gen_cookie_next(struct gen_cookie * gc)30*92acdc58SDaniel Borkmann static __always_inline u64 gen_cookie_next(struct gen_cookie *gc)
31*92acdc58SDaniel Borkmann {
32*92acdc58SDaniel Borkmann 	struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local);
33*92acdc58SDaniel Borkmann 	u64 val;
34*92acdc58SDaniel Borkmann 
35*92acdc58SDaniel Borkmann 	if (likely(local_inc_return(&local->nesting) == 1)) {
36*92acdc58SDaniel Borkmann 		val = local->last;
37*92acdc58SDaniel Borkmann 		if (__is_defined(CONFIG_SMP) &&
38*92acdc58SDaniel Borkmann 		    unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) {
39*92acdc58SDaniel Borkmann 			s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH,
40*92acdc58SDaniel Borkmann 						       &gc->forward_last);
41*92acdc58SDaniel Borkmann 			val = next - COOKIE_LOCAL_BATCH;
42*92acdc58SDaniel Borkmann 		}
43*92acdc58SDaniel Borkmann 		local->last = ++val;
44*92acdc58SDaniel Borkmann 	} else {
45*92acdc58SDaniel Borkmann 		val = atomic64_dec_return(&gc->reverse_last);
46*92acdc58SDaniel Borkmann 	}
47*92acdc58SDaniel Borkmann 	local_dec(&local->nesting);
48*92acdc58SDaniel Borkmann 	return val;
49*92acdc58SDaniel Borkmann }
50*92acdc58SDaniel Borkmann 
51*92acdc58SDaniel Borkmann #endif /* __LINUX_COOKIE_H */
52