18d7e7a98SRuslan Bukin /*- 28d7e7a98SRuslan Bukin * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> 38d7e7a98SRuslan Bukin * All rights reserved. 48d7e7a98SRuslan Bukin * 58d7e7a98SRuslan Bukin * Redistribution and use in source and binary forms, with or without 68d7e7a98SRuslan Bukin * modification, are permitted provided that the following conditions 78d7e7a98SRuslan Bukin * are met: 88d7e7a98SRuslan Bukin * 1. Redistributions of source code must retain the above copyright 98d7e7a98SRuslan Bukin * notice, this list of conditions and the following disclaimer. 108d7e7a98SRuslan Bukin * 2. Redistributions in binary form must reproduce the above copyright 118d7e7a98SRuslan Bukin * notice, this list of conditions and the following disclaimer in the 128d7e7a98SRuslan Bukin * documentation and/or other materials provided with the distribution. 138d7e7a98SRuslan Bukin * 148d7e7a98SRuslan Bukin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 158d7e7a98SRuslan Bukin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 168d7e7a98SRuslan Bukin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 178d7e7a98SRuslan Bukin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 188d7e7a98SRuslan Bukin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 198d7e7a98SRuslan Bukin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 208d7e7a98SRuslan Bukin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 218d7e7a98SRuslan Bukin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 228d7e7a98SRuslan Bukin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 238d7e7a98SRuslan Bukin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 248d7e7a98SRuslan Bukin * SUCH DAMAGE. 258d7e7a98SRuslan Bukin * 268d7e7a98SRuslan Bukin * $FreeBSD$ 278d7e7a98SRuslan Bukin */ 288d7e7a98SRuslan Bukin 298d7e7a98SRuslan Bukin #ifndef _MACHINE_COUNTER_H_ 308d7e7a98SRuslan Bukin #define _MACHINE_COUNTER_H_ 318d7e7a98SRuslan Bukin 328d7e7a98SRuslan Bukin #include <sys/pcpu.h> 33f7cc0eaeSMitchell Horne #include <machine/atomic.h> 348d7e7a98SRuslan Bukin 354d3b6bd5SRuslan Bukin #define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter 364d3b6bd5SRuslan Bukin 37f7cc0eaeSMitchell Horne #define counter_enter() do {} while (0) 38f7cc0eaeSMitchell Horne #define counter_exit() do {} while (0) 398d7e7a98SRuslan Bukin 408d7e7a98SRuslan Bukin #ifdef IN_SUBR_COUNTER_C 418d7e7a98SRuslan Bukin static inline uint64_t 428d7e7a98SRuslan Bukin counter_u64_read_one(uint64_t *p, int cpu) 438d7e7a98SRuslan Bukin { 448d7e7a98SRuslan Bukin 45ab3059a8SMatt Macy return (*(uint64_t *)((char *)p + UMA_PCPU_ALLOC_SIZE * cpu)); 468d7e7a98SRuslan Bukin } 478d7e7a98SRuslan Bukin 488d7e7a98SRuslan Bukin static inline uint64_t 498d7e7a98SRuslan Bukin counter_u64_fetch_inline(uint64_t *p) 508d7e7a98SRuslan Bukin { 518d7e7a98SRuslan Bukin uint64_t r; 528d7e7a98SRuslan Bukin int i; 538d7e7a98SRuslan Bukin 548d7e7a98SRuslan Bukin r = 0; 55f7cc0eaeSMitchell Horne CPU_FOREACH(i) 56f7cc0eaeSMitchell Horne r += counter_u64_read_one(p, i); 578d7e7a98SRuslan Bukin 588d7e7a98SRuslan Bukin return (r); 598d7e7a98SRuslan Bukin } 608d7e7a98SRuslan Bukin 618d7e7a98SRuslan Bukin static void 628d7e7a98SRuslan Bukin counter_u64_zero_one_cpu(void *arg) 638d7e7a98SRuslan Bukin { 648d7e7a98SRuslan Bukin 65f7cc0eaeSMitchell Horne *(zpcpu_get((counter_u64_t *)arg)) = 0; 668d7e7a98SRuslan Bukin } 678d7e7a98SRuslan Bukin 688d7e7a98SRuslan Bukin static inline void 698d7e7a98SRuslan Bukin counter_u64_zero_inline(counter_u64_t c) 708d7e7a98SRuslan Bukin { 718d7e7a98SRuslan Bukin 7267d955aaSPatrick Kelsey smp_rendezvous(smp_no_rendezvous_barrier, counter_u64_zero_one_cpu, 7367d955aaSPatrick Kelsey smp_no_rendezvous_barrier, c); 748d7e7a98SRuslan Bukin } 758d7e7a98SRuslan Bukin #endif 768d7e7a98SRuslan Bukin 77f7cc0eaeSMitchell Horne #define counter_u64_add_protected(c, inc) counter_u64_add(c, inc) 788d7e7a98SRuslan Bukin 798d7e7a98SRuslan Bukin static inline void 808d7e7a98SRuslan Bukin counter_u64_add(counter_u64_t c, int64_t inc) 818d7e7a98SRuslan Bukin { 828d7e7a98SRuslan Bukin 83f7cc0eaeSMitchell Horne atomic_add_64((uint64_t *)zpcpu_get(c), inc); 848d7e7a98SRuslan Bukin } 858d7e7a98SRuslan Bukin 868d7e7a98SRuslan Bukin #endif /* ! _MACHINE_COUNTER_H_ */ 87