1 /*- 2 * Copyright (c) 1990 The Regents of the University of California. 3 * Copyright (c) 2008 The DragonFly Project. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to Berkeley by 7 * William Jolitz and Don Ahn. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. Neither the name of the University nor the names of its contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 * 33 * from: @(#)clock.c 7.2 (Berkeley) 5/12/91 34 * $FreeBSD: src/sys/i386/isa/clock.c,v 1.149.2.6 2002/11/02 04:41:50 iwasaki Exp $ 35 */ 36 37 /* 38 * TSC cputimer. 39 */ 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/kernel.h> 44 #include <sys/sysctl.h> 45 #include <sys/systimer.h> 46 #include <sys/globaldata.h> 47 48 #include <machine/clock.h> 49 #include <machine/cputypes.h> 50 51 static sysclock_t tsc_cputimer_count_mfence(void); 52 static sysclock_t tsc_cputimer_count_lfence(void); 53 static void tsc_cputimer_construct(struct cputimer *, sysclock_t); 54 55 static struct cputimer tsc_cputimer = { 56 .next = SLIST_ENTRY_INITIALIZER, 57 .name = "TSC", 58 .pri = CPUTIMER_PRI_TSC, 59 .type = CPUTIMER_TSC, 60 .count = NULL, /* determined later */ 61 .fromhz = cputimer_default_fromhz, 62 .fromus = cputimer_default_fromus, 63 .construct = tsc_cputimer_construct, 64 .destruct = cputimer_default_destruct, 65 .freq = 0 /* determined later */ 66 }; 67 68 static struct cpucounter tsc_cpucounter = { 69 .freq = 0, /* determined later */ 70 .count = NULL, /* determined later */ 71 .flags = 0, /* adjusted later */ 72 .prio = CPUCOUNTER_PRIO_TSC, 73 .type = CPUCOUNTER_TSC 74 }; 75 76 static void 77 tsc_cputimer_construct(struct cputimer *timer, sysclock_t oldclock) 78 { 79 timer->base = 0; 80 timer->base = oldclock - timer->count(); 81 } 82 83 static __inline sysclock_t 84 tsc_cputimer_count(void) 85 { 86 uint64_t tsc; 87 88 tsc = rdtsc(); 89 90 return (tsc + tsc_cputimer.base); 91 } 92 93 static sysclock_t 94 tsc_cputimer_count_lfence(void) 95 { 96 cpu_lfence(); 97 return tsc_cputimer_count(); 98 } 99 100 static sysclock_t 101 tsc_cputimer_count_mfence(void) 102 { 103 cpu_mfence(); 104 return tsc_cputimer_count(); 105 } 106 107 static uint64_t 108 tsc_cpucounter_count_lfence(void) 109 { 110 cpu_lfence(); 111 return (rdtsc()); 112 } 113 114 static uint64_t 115 tsc_cpucounter_count_mfence(void) 116 { 117 cpu_mfence(); 118 return (rdtsc()); 119 } 120 121 static void 122 tsc_cputimer_register(void) 123 { 124 uint64_t freq; 125 int enable = 1; 126 127 if (!tsc_mpsync) { 128 #ifndef _KERNEL_VIRTUAL 129 if (tsc_invariant) { 130 /* Per-cpu cpucounter still works. */ 131 goto regcnt; 132 } 133 #endif 134 return; 135 } 136 137 TUNABLE_INT_FETCH("hw.tsc_cputimer_enable", &enable); 138 if (!enable) 139 return; 140 141 /* 142 * NOTE: We no longer shift the tsc to limit the reported 143 * frequency. We use the actual tsc frequency as-is. 144 */ 145 freq = tsc_frequency; 146 tsc_cputimer.freq = freq; 147 148 kprintf("TSC: cputimer freq %ju\n", (uintmax_t)freq); 149 150 if (cpu_vendor_id == CPU_VENDOR_INTEL) 151 tsc_cputimer.count = tsc_cputimer_count_lfence; 152 else 153 tsc_cputimer.count = tsc_cputimer_count_mfence; /* safe bet */ 154 155 cputimer_register(&tsc_cputimer); 156 cputimer_select(&tsc_cputimer, 0); 157 158 tsc_cpucounter.flags |= CPUCOUNTER_FLAG_MPSYNC; 159 #ifndef _KERNEL_VIRTUAL 160 regcnt: 161 #endif 162 tsc_cpucounter.freq = tsc_frequency; 163 if (cpu_vendor_id == CPU_VENDOR_INTEL) { 164 tsc_cpucounter.count = 165 tsc_cpucounter_count_lfence; 166 } else { 167 tsc_cpucounter.count = 168 tsc_cpucounter_count_mfence; /* safe bet */ 169 } 170 cpucounter_register(&tsc_cpucounter); 171 } 172 SYSINIT(tsc_cputimer_reg, SI_BOOT2_POST_SMP, SI_ORDER_FIRST, 173 tsc_cputimer_register, NULL); 174