14e76af6aSGleb Smirnoff /*- 2af3dc4a7SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3af3dc4a7SPedro F. Giffuni * 44e76af6aSGleb Smirnoff * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> 54e76af6aSGleb Smirnoff * All rights reserved. 64e76af6aSGleb Smirnoff * 74e76af6aSGleb Smirnoff * Redistribution and use in source and binary forms, with or without 84e76af6aSGleb Smirnoff * modification, are permitted provided that the following conditions 94e76af6aSGleb Smirnoff * are met: 104e76af6aSGleb Smirnoff * 1. Redistributions of source code must retain the above copyright 114e76af6aSGleb Smirnoff * notice, this list of conditions and the following disclaimer. 124e76af6aSGleb Smirnoff * 2. Redistributions in binary form must reproduce the above copyright 134e76af6aSGleb Smirnoff * notice, this list of conditions and the following disclaimer in the 144e76af6aSGleb Smirnoff * documentation and/or other materials provided with the distribution. 154e76af6aSGleb Smirnoff * 164e76af6aSGleb Smirnoff * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 174e76af6aSGleb Smirnoff * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 184e76af6aSGleb Smirnoff * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 194e76af6aSGleb Smirnoff * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 204e76af6aSGleb Smirnoff * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 214e76af6aSGleb Smirnoff * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 224e76af6aSGleb Smirnoff * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 234e76af6aSGleb Smirnoff * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 244e76af6aSGleb Smirnoff * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 254e76af6aSGleb Smirnoff * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 264e76af6aSGleb Smirnoff * SUCH DAMAGE. 274e76af6aSGleb Smirnoff * 284e76af6aSGleb Smirnoff * $FreeBSD$ 294e76af6aSGleb Smirnoff */ 304e76af6aSGleb Smirnoff 314e76af6aSGleb Smirnoff #ifndef __MACHINE_COUNTER_H__ 324e76af6aSGleb Smirnoff #define __MACHINE_COUNTER_H__ 334e76af6aSGleb Smirnoff 344e76af6aSGleb Smirnoff #include <sys/pcpu.h> 35eaa4e276SIan Lepore #include <machine/atomic.h> 364e76af6aSGleb Smirnoff 3783c9dea1SGleb Smirnoff #define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter 3883c9dea1SGleb Smirnoff 393b7a388bSKonstantin Belousov #define counter_enter() do {} while (0) 403b7a388bSKonstantin Belousov #define counter_exit() do {} while (0) 414e76af6aSGleb Smirnoff 4270a7dd5dSKonstantin Belousov #ifdef IN_SUBR_COUNTER_C 43eaa4e276SIan Lepore 4470a7dd5dSKonstantin Belousov static inline uint64_t 4570a7dd5dSKonstantin Belousov counter_u64_read_one(uint64_t *p, int cpu) 4670a7dd5dSKonstantin Belousov { 4770a7dd5dSKonstantin Belousov 4878e3a168SMateusz Guzik return (atomic_load_64((uint64_t *)zpcpu_get_cpu(p, cpu))); 4970a7dd5dSKonstantin Belousov } 5070a7dd5dSKonstantin Belousov 5170a7dd5dSKonstantin Belousov static inline uint64_t 5270a7dd5dSKonstantin Belousov counter_u64_fetch_inline(uint64_t *p) 5370a7dd5dSKonstantin Belousov { 5470a7dd5dSKonstantin Belousov uint64_t r; 5570a7dd5dSKonstantin Belousov int i; 5670a7dd5dSKonstantin Belousov 5770a7dd5dSKonstantin Belousov r = 0; 583b7a388bSKonstantin Belousov CPU_FOREACH(i) 5970a7dd5dSKonstantin Belousov r += counter_u64_read_one((uint64_t *)p, i); 6070a7dd5dSKonstantin Belousov 6170a7dd5dSKonstantin Belousov return (r); 6270a7dd5dSKonstantin Belousov } 6370a7dd5dSKonstantin Belousov 6470a7dd5dSKonstantin Belousov static void 6570a7dd5dSKonstantin Belousov counter_u64_zero_one_cpu(void *arg) 6670a7dd5dSKonstantin Belousov { 6770a7dd5dSKonstantin Belousov 6878e3a168SMateusz Guzik atomic_store_64((uint64_t *)zpcpu_get(arg), 0); 6970a7dd5dSKonstantin Belousov } 7070a7dd5dSKonstantin Belousov 7170a7dd5dSKonstantin Belousov static inline void 7270a7dd5dSKonstantin Belousov counter_u64_zero_inline(counter_u64_t c) 7370a7dd5dSKonstantin Belousov { 7470a7dd5dSKonstantin Belousov 7567d955aaSPatrick Kelsey smp_rendezvous(smp_no_rendezvous_barrier, counter_u64_zero_one_cpu, 7667d955aaSPatrick Kelsey smp_no_rendezvous_barrier, c); 7770a7dd5dSKonstantin Belousov } 7870a7dd5dSKonstantin Belousov #endif 7970a7dd5dSKonstantin Belousov 803b7a388bSKonstantin Belousov #define counter_u64_add_protected(c, inc) counter_u64_add(c, inc) 814e76af6aSGleb Smirnoff 824e76af6aSGleb Smirnoff static inline void 834e76af6aSGleb Smirnoff counter_u64_add(counter_u64_t c, int64_t inc) 844e76af6aSGleb Smirnoff { 854e76af6aSGleb Smirnoff 863b7a388bSKonstantin Belousov atomic_add_64((uint64_t *)zpcpu_get(c), inc); 874e76af6aSGleb Smirnoff } 884e76af6aSGleb Smirnoff 894e76af6aSGleb Smirnoff #endif /* ! __MACHINE_COUNTER_H__ */ 90