18d7e7a98SRuslan Bukin /*-
28d7e7a98SRuslan Bukin * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
38d7e7a98SRuslan Bukin * All rights reserved.
48d7e7a98SRuslan Bukin *
58d7e7a98SRuslan Bukin * Redistribution and use in source and binary forms, with or without
68d7e7a98SRuslan Bukin * modification, are permitted provided that the following conditions
78d7e7a98SRuslan Bukin * are met:
88d7e7a98SRuslan Bukin * 1. Redistributions of source code must retain the above copyright
98d7e7a98SRuslan Bukin * notice, this list of conditions and the following disclaimer.
108d7e7a98SRuslan Bukin * 2. Redistributions in binary form must reproduce the above copyright
118d7e7a98SRuslan Bukin * notice, this list of conditions and the following disclaimer in the
128d7e7a98SRuslan Bukin * documentation and/or other materials provided with the distribution.
138d7e7a98SRuslan Bukin *
148d7e7a98SRuslan Bukin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
158d7e7a98SRuslan Bukin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
168d7e7a98SRuslan Bukin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
178d7e7a98SRuslan Bukin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
188d7e7a98SRuslan Bukin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
198d7e7a98SRuslan Bukin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
208d7e7a98SRuslan Bukin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
218d7e7a98SRuslan Bukin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
228d7e7a98SRuslan Bukin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
238d7e7a98SRuslan Bukin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
248d7e7a98SRuslan Bukin * SUCH DAMAGE.
258d7e7a98SRuslan Bukin */
268d7e7a98SRuslan Bukin
278d7e7a98SRuslan Bukin #ifndef _MACHINE_COUNTER_H_
288d7e7a98SRuslan Bukin #define _MACHINE_COUNTER_H_
298d7e7a98SRuslan Bukin
308d7e7a98SRuslan Bukin #include <sys/pcpu.h>
31f7cc0eaeSMitchell Horne #include <machine/atomic.h>
328d7e7a98SRuslan Bukin
334d3b6bd5SRuslan Bukin #define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter
344d3b6bd5SRuslan Bukin
35f7cc0eaeSMitchell Horne #define counter_enter() do {} while (0)
36f7cc0eaeSMitchell Horne #define counter_exit() do {} while (0)
378d7e7a98SRuslan Bukin
388d7e7a98SRuslan Bukin #ifdef IN_SUBR_COUNTER_C
398d7e7a98SRuslan Bukin static inline uint64_t
counter_u64_read_one(uint64_t * p,int cpu)408d7e7a98SRuslan Bukin counter_u64_read_one(uint64_t *p, int cpu)
418d7e7a98SRuslan Bukin {
428d7e7a98SRuslan Bukin
43ab3059a8SMatt Macy return (*(uint64_t *)((char *)p + UMA_PCPU_ALLOC_SIZE * cpu));
448d7e7a98SRuslan Bukin }
458d7e7a98SRuslan Bukin
468d7e7a98SRuslan Bukin static inline uint64_t
counter_u64_fetch_inline(uint64_t * p)478d7e7a98SRuslan Bukin counter_u64_fetch_inline(uint64_t *p)
488d7e7a98SRuslan Bukin {
498d7e7a98SRuslan Bukin uint64_t r;
508d7e7a98SRuslan Bukin int i;
518d7e7a98SRuslan Bukin
528d7e7a98SRuslan Bukin r = 0;
53f7cc0eaeSMitchell Horne CPU_FOREACH(i)
54f7cc0eaeSMitchell Horne r += counter_u64_read_one(p, i);
558d7e7a98SRuslan Bukin
568d7e7a98SRuslan Bukin return (r);
578d7e7a98SRuslan Bukin }
588d7e7a98SRuslan Bukin
598d7e7a98SRuslan Bukin static void
counter_u64_zero_one_cpu(void * arg)608d7e7a98SRuslan Bukin counter_u64_zero_one_cpu(void *arg)
618d7e7a98SRuslan Bukin {
628d7e7a98SRuslan Bukin
63f7cc0eaeSMitchell Horne *(zpcpu_get((counter_u64_t *)arg)) = 0;
648d7e7a98SRuslan Bukin }
658d7e7a98SRuslan Bukin
668d7e7a98SRuslan Bukin static inline void
counter_u64_zero_inline(counter_u64_t c)678d7e7a98SRuslan Bukin counter_u64_zero_inline(counter_u64_t c)
688d7e7a98SRuslan Bukin {
698d7e7a98SRuslan Bukin
7067d955aaSPatrick Kelsey smp_rendezvous(smp_no_rendezvous_barrier, counter_u64_zero_one_cpu,
7167d955aaSPatrick Kelsey smp_no_rendezvous_barrier, c);
728d7e7a98SRuslan Bukin }
738d7e7a98SRuslan Bukin #endif
748d7e7a98SRuslan Bukin
75f7cc0eaeSMitchell Horne #define counter_u64_add_protected(c, inc) counter_u64_add(c, inc)
768d7e7a98SRuslan Bukin
778d7e7a98SRuslan Bukin static inline void
counter_u64_add(counter_u64_t c,int64_t inc)788d7e7a98SRuslan Bukin counter_u64_add(counter_u64_t c, int64_t inc)
798d7e7a98SRuslan Bukin {
808d7e7a98SRuslan Bukin
81f7cc0eaeSMitchell Horne atomic_add_64((uint64_t *)zpcpu_get(c), inc);
828d7e7a98SRuslan Bukin }
838d7e7a98SRuslan Bukin
848d7e7a98SRuslan Bukin #endif /* ! _MACHINE_COUNTER_H_ */
85