1*bf0d449cSmpi /* $OpenBSD: percpu.h,v 1.9 2023/09/16 09:33:27 mpi Exp $ */
204e271afSdlg
304e271afSdlg /*
404e271afSdlg * Copyright (c) 2016 David Gwynne <dlg@openbsd.org>
504e271afSdlg *
604e271afSdlg * Permission to use, copy, modify, and distribute this software for any
704e271afSdlg * purpose with or without fee is hereby granted, provided that the above
804e271afSdlg * copyright notice and this permission notice appear in all copies.
904e271afSdlg *
1004e271afSdlg * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
1104e271afSdlg * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
1204e271afSdlg * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
1304e271afSdlg * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
1404e271afSdlg * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
1504e271afSdlg * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
1604e271afSdlg * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
1704e271afSdlg */
1804e271afSdlg
1904e271afSdlg #ifndef _SYS_PERCPU_H_
2004e271afSdlg #define _SYS_PERCPU_H_
2104e271afSdlg
2204e271afSdlg #ifndef CACHELINESIZE
2304e271afSdlg #define CACHELINESIZE 64
2404e271afSdlg #endif
2504e271afSdlg
2604e271afSdlg #ifndef __upunused /* this should go in param.h */
2704e271afSdlg #ifdef MULTIPROCESSOR
2804e271afSdlg #define __upunused
2904e271afSdlg #else
3004e271afSdlg #define __upunused __attribute__((__unused__))
3104e271afSdlg #endif
3204e271afSdlg #endif
3304e271afSdlg
3404e271afSdlg struct cpumem {
3504e271afSdlg void *mem;
3604e271afSdlg };
3704e271afSdlg
3804e271afSdlg struct cpumem_iter {
3904e271afSdlg unsigned int cpu;
4004e271afSdlg } __upunused;
4104e271afSdlg
4204e271afSdlg struct counters_ref {
4304e271afSdlg uint64_t g;
4404e271afSdlg uint64_t *c;
4504e271afSdlg };
4604e271afSdlg
4704e271afSdlg #ifdef _KERNEL
4804e271afSdlg
4904e271afSdlg #include <sys/atomic.h>
5004e271afSdlg
5104e271afSdlg struct pool;
5204e271afSdlg
5304e271afSdlg struct cpumem *cpumem_get(struct pool *);
5404e271afSdlg void cpumem_put(struct pool *, struct cpumem *);
5504e271afSdlg
5604e271afSdlg struct cpumem *cpumem_malloc(size_t, int);
57780021b3Sdlg struct cpumem *cpumem_malloc_ncpus(struct cpumem *, size_t, int);
5804e271afSdlg void cpumem_free(struct cpumem *, int, size_t);
5904e271afSdlg
606c8f19e1Sderaadt void *cpumem_first(struct cpumem_iter *, struct cpumem *);
616c8f19e1Sderaadt void *cpumem_next(struct cpumem_iter *, struct cpumem *);
626c8f19e1Sderaadt
6304e271afSdlg static inline void *
cpumem_enter(struct cpumem * cm)6404e271afSdlg cpumem_enter(struct cpumem *cm)
6504e271afSdlg {
666c8f19e1Sderaadt #ifdef MULTIPROCESSOR
6704e271afSdlg return (cm[cpu_number()].mem);
686c8f19e1Sderaadt #else
696c8f19e1Sderaadt return (cm);
706c8f19e1Sderaadt #endif
7104e271afSdlg }
7204e271afSdlg
7304e271afSdlg static inline void
cpumem_leave(struct cpumem * cm,void * mem)7404e271afSdlg cpumem_leave(struct cpumem *cm, void *mem)
7504e271afSdlg {
7604e271afSdlg /* KDASSERT? */
7704e271afSdlg }
7804e271afSdlg
796c8f19e1Sderaadt #ifdef MULTIPROCESSOR
8004e271afSdlg
8104e271afSdlg #define CPUMEM_BOOT_MEMORY(_name, _sz) \
8204e271afSdlg static struct { \
8304e271afSdlg unsigned char mem[_sz]; \
8404e271afSdlg struct cpumem cpumem; \
8504e271afSdlg } __aligned(CACHELINESIZE) _name##_boot_cpumem = { \
8604e271afSdlg .cpumem = { _name##_boot_cpumem.mem } \
8704e271afSdlg }
8804e271afSdlg
8904e271afSdlg #define CPUMEM_BOOT_INITIALIZER(_name) \
9004e271afSdlg { &_name##_boot_cpumem.cpumem }
9104e271afSdlg
9204e271afSdlg #else /* MULTIPROCESSOR */
9304e271afSdlg
9404e271afSdlg #define CPUMEM_BOOT_MEMORY(_name, _sz) \
9504e271afSdlg static struct { \
9604e271afSdlg unsigned char mem[_sz]; \
9701f82b0bSdlg } __aligned(sizeof(uint64_t)) _name##_boot_cpumem
9804e271afSdlg
9904e271afSdlg #define CPUMEM_BOOT_INITIALIZER(_name) \
10004e271afSdlg { (struct cpumem *)&_name##_boot_cpumem.mem }
10104e271afSdlg
10204e271afSdlg #endif /* MULTIPROCESSOR */
10304e271afSdlg
10404e271afSdlg #define CPUMEM_FOREACH(_var, _iter, _cpumem) \
10504e271afSdlg for ((_var) = cpumem_first((_iter), (_cpumem)); \
10604e271afSdlg (_var) != NULL; \
10704e271afSdlg (_var) = cpumem_next((_iter), (_cpumem)))
10804e271afSdlg
1096c8f19e1Sderaadt /*
1106c8f19e1Sderaadt * per cpu counters
1116c8f19e1Sderaadt */
1126c8f19e1Sderaadt
113599d0588Sjca struct cpumem *counters_alloc(unsigned int);
114599d0588Sjca struct cpumem *counters_alloc_ncpus(struct cpumem *, unsigned int);
115599d0588Sjca void counters_free(struct cpumem *, unsigned int);
116*bf0d449cSmpi void counters_read(struct cpumem *, uint64_t *, unsigned int,
117*bf0d449cSmpi uint64_t *);
11804e271afSdlg void counters_zero(struct cpumem *, unsigned int);
11904e271afSdlg
12004e271afSdlg static inline uint64_t *
counters_enter(struct counters_ref * ref,struct cpumem * cm)12104e271afSdlg counters_enter(struct counters_ref *ref, struct cpumem *cm)
12204e271afSdlg {
12304e271afSdlg ref->c = cpumem_enter(cm);
1246c8f19e1Sderaadt #ifdef MULTIPROCESSOR
12504e271afSdlg ref->g = ++(*ref->c); /* make the generation number odd */
126298d98f1Sbluhm membar_producer();
12704e271afSdlg return (ref->c + 1);
1286c8f19e1Sderaadt #else
1296c8f19e1Sderaadt return (ref->c);
1306c8f19e1Sderaadt #endif
13104e271afSdlg }
13204e271afSdlg
13304e271afSdlg static inline void
counters_leave(struct counters_ref * ref,struct cpumem * cm)13404e271afSdlg counters_leave(struct counters_ref *ref, struct cpumem *cm)
13504e271afSdlg {
1366c8f19e1Sderaadt #ifdef MULTIPROCESSOR
13704e271afSdlg membar_producer();
13804e271afSdlg (*ref->c) = ++ref->g; /* make the generation number even again */
1396c8f19e1Sderaadt #endif
14004e271afSdlg cpumem_leave(cm, ref->c);
14104e271afSdlg }
1426c8f19e1Sderaadt
143274986c5Sdlg static inline void
counters_inc(struct cpumem * cm,unsigned int c)144274986c5Sdlg counters_inc(struct cpumem *cm, unsigned int c)
145274986c5Sdlg {
146274986c5Sdlg struct counters_ref ref;
147274986c5Sdlg uint64_t *counters;
148274986c5Sdlg
149274986c5Sdlg counters = counters_enter(&ref, cm);
150274986c5Sdlg counters[c]++;
151274986c5Sdlg counters_leave(&ref, cm);
152274986c5Sdlg }
153274986c5Sdlg
154274986c5Sdlg static inline void
counters_dec(struct cpumem * cm,unsigned int c)1552edaa7baSmpi counters_dec(struct cpumem *cm, unsigned int c)
1562edaa7baSmpi {
1572edaa7baSmpi struct counters_ref ref;
1582edaa7baSmpi uint64_t *counters;
1592edaa7baSmpi
1602edaa7baSmpi counters = counters_enter(&ref, cm);
1612edaa7baSmpi counters[c]--;
1622edaa7baSmpi counters_leave(&ref, cm);
1632edaa7baSmpi }
1642edaa7baSmpi
1652edaa7baSmpi static inline void
counters_add(struct cpumem * cm,unsigned int c,uint64_t v)166274986c5Sdlg counters_add(struct cpumem *cm, unsigned int c, uint64_t v)
167274986c5Sdlg {
168274986c5Sdlg struct counters_ref ref;
169274986c5Sdlg uint64_t *counters;
170274986c5Sdlg
171274986c5Sdlg counters = counters_enter(&ref, cm);
172274986c5Sdlg counters[c] += v;
173274986c5Sdlg counters_leave(&ref, cm);
174274986c5Sdlg }
175274986c5Sdlg
176274986c5Sdlg static inline void
counters_pkt(struct cpumem * cm,unsigned int c,unsigned int b,uint64_t v)177274986c5Sdlg counters_pkt(struct cpumem *cm, unsigned int c, unsigned int b, uint64_t v)
178274986c5Sdlg {
179274986c5Sdlg struct counters_ref ref;
180274986c5Sdlg uint64_t *counters;
181274986c5Sdlg
182274986c5Sdlg counters = counters_enter(&ref, cm);
183274986c5Sdlg counters[c]++;
184274986c5Sdlg counters[b] += v;
185274986c5Sdlg counters_leave(&ref, cm);
186274986c5Sdlg }
187274986c5Sdlg
1886c8f19e1Sderaadt #ifdef MULTIPROCESSOR
18904e271afSdlg #define COUNTERS_BOOT_MEMORY(_name, _n) \
19004e271afSdlg CPUMEM_BOOT_MEMORY(_name, ((_n) + 1) * sizeof(uint64_t))
19104e271afSdlg #else
19204e271afSdlg #define COUNTERS_BOOT_MEMORY(_name, _n) \
19304e271afSdlg CPUMEM_BOOT_MEMORY(_name, (_n) * sizeof(uint64_t))
19404e271afSdlg #endif
19504e271afSdlg
19604e271afSdlg #define COUNTERS_BOOT_INITIALIZER(_name) CPUMEM_BOOT_INITIALIZER(_name)
19704e271afSdlg
19804e271afSdlg #endif /* _KERNEL */
19904e271afSdlg #endif /* _SYS_PERCPU_H_ */
200