1dd7d207dSJung-uk Kim /*- 2ebf5747bSPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3ebf5747bSPedro F. Giffuni * 4dd7d207dSJung-uk Kim * Copyright (c) 1998-2003 Poul-Henning Kamp 5dd7d207dSJung-uk Kim * All rights reserved. 6dd7d207dSJung-uk Kim * 7dd7d207dSJung-uk Kim * Redistribution and use in source and binary forms, with or without 8dd7d207dSJung-uk Kim * modification, are permitted provided that the following conditions 9dd7d207dSJung-uk Kim * are met: 10dd7d207dSJung-uk Kim * 1. Redistributions of source code must retain the above copyright 11dd7d207dSJung-uk Kim * notice, this list of conditions and the following disclaimer. 12dd7d207dSJung-uk Kim * 2. Redistributions in binary form must reproduce the above copyright 13dd7d207dSJung-uk Kim * notice, this list of conditions and the following disclaimer in the 14dd7d207dSJung-uk Kim * documentation and/or other materials provided with the distribution. 15dd7d207dSJung-uk Kim * 16dd7d207dSJung-uk Kim * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17dd7d207dSJung-uk Kim * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18dd7d207dSJung-uk Kim * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19dd7d207dSJung-uk Kim * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20dd7d207dSJung-uk Kim * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21dd7d207dSJung-uk Kim * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22dd7d207dSJung-uk Kim * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23dd7d207dSJung-uk Kim * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24dd7d207dSJung-uk Kim * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25dd7d207dSJung-uk Kim * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26dd7d207dSJung-uk Kim * SUCH DAMAGE. 27dd7d207dSJung-uk Kim */ 28dd7d207dSJung-uk Kim 29dd7d207dSJung-uk Kim #include <sys/cdefs.h> 30dd7d207dSJung-uk Kim __FBSDID("$FreeBSD$"); 31dd7d207dSJung-uk Kim 32dd7d207dSJung-uk Kim #include "opt_clock.h" 33dd7d207dSJung-uk Kim 34dd7d207dSJung-uk Kim #include <sys/param.h> 35dd7d207dSJung-uk Kim #include <sys/bus.h> 36dd7d207dSJung-uk Kim #include <sys/cpu.h> 37e2e050c8SConrad Meyer #include <sys/eventhandler.h> 385da5812bSJung-uk Kim #include <sys/limits.h> 39dd7d207dSJung-uk Kim #include <sys/malloc.h> 40dd7d207dSJung-uk Kim #include <sys/systm.h> 41dd7d207dSJung-uk Kim #include <sys/sysctl.h> 42dd7d207dSJung-uk Kim #include <sys/time.h> 43dd7d207dSJung-uk Kim #include <sys/timetc.h> 44dd7d207dSJung-uk Kim #include <sys/kernel.h> 45dd7d207dSJung-uk Kim #include <sys/power.h> 46dd7d207dSJung-uk Kim #include <sys/smp.h> 47aea81038SKonstantin Belousov #include <sys/vdso.h> 48dd7d207dSJung-uk Kim #include <machine/clock.h> 49dd7d207dSJung-uk Kim #include <machine/cputypes.h> 50dd7d207dSJung-uk Kim #include <machine/md_var.h> 51dd7d207dSJung-uk Kim #include <machine/specialreg.h> 5201e1933dSJohn Baldwin #include <x86/vmware.h> 5316808549SKonstantin Belousov #include <dev/acpica/acpi_hpet.h> 54ce3bf750SKonstantin Belousov #include <contrib/dev/acpica/include/acpi.h> 55dd7d207dSJung-uk Kim 56dd7d207dSJung-uk Kim #include "cpufreq_if.h" 57dd7d207dSJung-uk Kim 58dd7d207dSJung-uk Kim uint64_t tsc_freq; 59dd7d207dSJung-uk Kim int tsc_is_invariant; 60155094d7SJung-uk Kim int tsc_perf_stat; 61155094d7SJung-uk Kim 62dd7d207dSJung-uk Kim static eventhandler_tag tsc_levels_tag, tsc_pre_tag, tsc_post_tag; 63dd7d207dSJung-uk Kim 64dd7d207dSJung-uk Kim SYSCTL_INT(_kern_timecounter, OID_AUTO, invariant_tsc, CTLFLAG_RDTUN, 65dd7d207dSJung-uk Kim &tsc_is_invariant, 0, "Indicates whether the TSC is P-state invariant"); 66dd7d207dSJung-uk Kim 67dd7d207dSJung-uk Kim #ifdef SMP 681472b87fSNeel Natu int smp_tsc; 69dd7d207dSJung-uk Kim SYSCTL_INT(_kern_timecounter, OID_AUTO, smp_tsc, CTLFLAG_RDTUN, &smp_tsc, 0, 70dd7d207dSJung-uk Kim "Indicates whether the TSC is safe to use in SMP mode"); 71b2c63698SAlexander Motin 72b2c63698SAlexander Motin int smp_tsc_adjust = 0; 73b2c63698SAlexander Motin SYSCTL_INT(_kern_timecounter, OID_AUTO, smp_tsc_adjust, CTLFLAG_RDTUN, 74b2c63698SAlexander Motin &smp_tsc_adjust, 0, "Try to adjust TSC on APs to match BSP"); 75dd7d207dSJung-uk Kim #endif 76dd7d207dSJung-uk Kim 77e7f1427dSKonstantin Belousov static int tsc_shift = 1; 78e7f1427dSKonstantin Belousov SYSCTL_INT(_kern_timecounter, OID_AUTO, tsc_shift, CTLFLAG_RDTUN, 79e7f1427dSKonstantin Belousov &tsc_shift, 0, "Shift to pre-apply for the maximum TSC frequency"); 80e7f1427dSKonstantin Belousov 8179422085SJung-uk Kim static int tsc_disabled; 8279422085SJung-uk Kim SYSCTL_INT(_machdep, OID_AUTO, disable_tsc, CTLFLAG_RDTUN, &tsc_disabled, 0, 8379422085SJung-uk Kim "Disable x86 Time Stamp Counter"); 8479422085SJung-uk Kim 85a4e4127fSJung-uk Kim static int tsc_skip_calibration; 86ab23c278SKonstantin Belousov SYSCTL_INT(_machdep, OID_AUTO, disable_tsc_calibration, CTLFLAG_RDTUN, 87ab23c278SKonstantin Belousov &tsc_skip_calibration, 0, 88ce3bf750SKonstantin Belousov "Disable TSC frequency calibration"); 89a4e4127fSJung-uk Kim 90dd7d207dSJung-uk Kim static void tsc_freq_changed(void *arg, const struct cf_level *level, 91dd7d207dSJung-uk Kim int status); 92dd7d207dSJung-uk Kim static void tsc_freq_changing(void *arg, const struct cf_level *level, 93dd7d207dSJung-uk Kim int *status); 94826fc3ccSKonstantin Belousov static u_int tsc_get_timecount(struct timecounter *tc); 95826fc3ccSKonstantin Belousov static inline u_int tsc_get_timecount_low(struct timecounter *tc); 96826fc3ccSKonstantin Belousov static u_int tsc_get_timecount_lfence(struct timecounter *tc); 97826fc3ccSKonstantin Belousov static u_int tsc_get_timecount_low_lfence(struct timecounter *tc); 98826fc3ccSKonstantin Belousov static u_int tsc_get_timecount_mfence(struct timecounter *tc); 99826fc3ccSKonstantin Belousov static u_int tsc_get_timecount_low_mfence(struct timecounter *tc); 100dd7d207dSJung-uk Kim static void tsc_levels_changed(void *arg, int unit); 10116808549SKonstantin Belousov static uint32_t x86_tsc_vdso_timehands(struct vdso_timehands *vdso_th, 10216808549SKonstantin Belousov struct timecounter *tc); 10316808549SKonstantin Belousov #ifdef COMPAT_FREEBSD32 10416808549SKonstantin Belousov static uint32_t x86_tsc_vdso_timehands32(struct vdso_timehands32 *vdso_th32, 10516808549SKonstantin Belousov struct timecounter *tc); 10616808549SKonstantin Belousov #endif 107dd7d207dSJung-uk Kim 108dd7d207dSJung-uk Kim static struct timecounter tsc_timecounter = { 10916808549SKonstantin Belousov .tc_get_timecount = tsc_get_timecount, 11016808549SKonstantin Belousov .tc_counter_mask = ~0u, 11116808549SKonstantin Belousov .tc_name = "TSC", 11216808549SKonstantin Belousov .tc_quality = 800, /* adjusted in code */ 11316808549SKonstantin Belousov .tc_fill_vdso_timehands = x86_tsc_vdso_timehands, 11416808549SKonstantin Belousov #ifdef COMPAT_FREEBSD32 11516808549SKonstantin Belousov .tc_fill_vdso_timehands32 = x86_tsc_vdso_timehands32, 11616808549SKonstantin Belousov #endif 117dd7d207dSJung-uk Kim }; 118dd7d207dSJung-uk Kim 11901e1933dSJohn Baldwin static void 1205da5812bSJung-uk Kim tsc_freq_vmware(void) 1215da5812bSJung-uk Kim { 1225da5812bSJung-uk Kim u_int regs[4]; 1235da5812bSJung-uk Kim 1245da5812bSJung-uk Kim if (hv_high >= 0x40000010) { 1255da5812bSJung-uk Kim do_cpuid(0x40000010, regs); 1265da5812bSJung-uk Kim tsc_freq = regs[0] * 1000; 1275da5812bSJung-uk Kim } else { 1285da5812bSJung-uk Kim vmware_hvcall(VMW_HVCMD_GETHZ, regs); 1295da5812bSJung-uk Kim if (regs[1] != UINT_MAX) 1305da5812bSJung-uk Kim tsc_freq = regs[0] | ((uint64_t)regs[1] << 32); 1315da5812bSJung-uk Kim } 1325da5812bSJung-uk Kim tsc_is_invariant = 1; 1335da5812bSJung-uk Kim } 1345da5812bSJung-uk Kim 135506a906cSKonstantin Belousov /* 136506a906cSKonstantin Belousov * Calculate TSC frequency using information from the CPUID leaf 0x15 137a9d0e007SKonstantin Belousov * 'Time Stamp Counter and Nominal Core Crystal Clock'. If leaf 0x15 138a9d0e007SKonstantin Belousov * is not functional, as it is on Skylake/Kabylake, try 0x16 'Processor 139a9d0e007SKonstantin Belousov * Frequency Information'. Leaf 0x16 is described in the SDM as 140a9d0e007SKonstantin Belousov * informational only, but if 0x15 did not work, and TSC calibration 141a9d0e007SKonstantin Belousov * is disabled, it is the best we can get at all. It should still be 142506a906cSKonstantin Belousov * an improvement over the parsing of the CPU model name in 143506a906cSKonstantin Belousov * tsc_freq_intel(), when available. 144506a906cSKonstantin Belousov */ 145506a906cSKonstantin Belousov static bool 146bd8a359fSKonstantin Belousov tsc_freq_cpuid(uint64_t *res) 147506a906cSKonstantin Belousov { 148506a906cSKonstantin Belousov u_int regs[4]; 149506a906cSKonstantin Belousov 150506a906cSKonstantin Belousov if (cpu_high < 0x15) 151506a906cSKonstantin Belousov return (false); 152506a906cSKonstantin Belousov do_cpuid(0x15, regs); 153a9d0e007SKonstantin Belousov if (regs[0] != 0 && regs[1] != 0 && regs[2] != 0) { 154bd8a359fSKonstantin Belousov *res = (uint64_t)regs[2] * regs[1] / regs[0]; 155506a906cSKonstantin Belousov return (true); 156506a906cSKonstantin Belousov } 157506a906cSKonstantin Belousov 158a9d0e007SKonstantin Belousov if (cpu_high < 0x16) 159a9d0e007SKonstantin Belousov return (false); 160a9d0e007SKonstantin Belousov do_cpuid(0x16, regs); 161a9d0e007SKonstantin Belousov if (regs[0] != 0) { 162bd8a359fSKonstantin Belousov *res = (uint64_t)regs[0] * 1000000; 163a9d0e007SKonstantin Belousov return (true); 164a9d0e007SKonstantin Belousov } 165a9d0e007SKonstantin Belousov 166a9d0e007SKonstantin Belousov return (false); 167a9d0e007SKonstantin Belousov } 168a9d0e007SKonstantin Belousov 169a4e4127fSJung-uk Kim static void 170a4e4127fSJung-uk Kim tsc_freq_intel(void) 171dd7d207dSJung-uk Kim { 172a4e4127fSJung-uk Kim char brand[48]; 173a4e4127fSJung-uk Kim u_int regs[4]; 174a4e4127fSJung-uk Kim uint64_t freq; 175a4e4127fSJung-uk Kim char *p; 176a4e4127fSJung-uk Kim u_int i; 177dd7d207dSJung-uk Kim 178a4e4127fSJung-uk Kim /* 179a4e4127fSJung-uk Kim * Intel Processor Identification and the CPUID Instruction 180a4e4127fSJung-uk Kim * Application Note 485. 181a4e4127fSJung-uk Kim * http://www.intel.com/assets/pdf/appnote/241618.pdf 182a4e4127fSJung-uk Kim */ 183a4e4127fSJung-uk Kim if (cpu_exthigh >= 0x80000004) { 184a4e4127fSJung-uk Kim p = brand; 185a4e4127fSJung-uk Kim for (i = 0x80000002; i < 0x80000005; i++) { 186a4e4127fSJung-uk Kim do_cpuid(i, regs); 187a4e4127fSJung-uk Kim memcpy(p, regs, sizeof(regs)); 188a4e4127fSJung-uk Kim p += sizeof(regs); 189a4e4127fSJung-uk Kim } 190a4e4127fSJung-uk Kim p = NULL; 191a4e4127fSJung-uk Kim for (i = 0; i < sizeof(brand) - 1; i++) 192a4e4127fSJung-uk Kim if (brand[i] == 'H' && brand[i + 1] == 'z') 193a4e4127fSJung-uk Kim p = brand + i; 194a4e4127fSJung-uk Kim if (p != NULL) { 195a4e4127fSJung-uk Kim p -= 5; 196a4e4127fSJung-uk Kim switch (p[4]) { 197a4e4127fSJung-uk Kim case 'M': 198a4e4127fSJung-uk Kim i = 1; 199a4e4127fSJung-uk Kim break; 200a4e4127fSJung-uk Kim case 'G': 201a4e4127fSJung-uk Kim i = 1000; 202a4e4127fSJung-uk Kim break; 203a4e4127fSJung-uk Kim case 'T': 204a4e4127fSJung-uk Kim i = 1000000; 205a4e4127fSJung-uk Kim break; 206a4e4127fSJung-uk Kim default: 207dd7d207dSJung-uk Kim return; 208a4e4127fSJung-uk Kim } 209a4e4127fSJung-uk Kim #define C2D(c) ((c) - '0') 210a4e4127fSJung-uk Kim if (p[1] == '.') { 211a4e4127fSJung-uk Kim freq = C2D(p[0]) * 1000; 212a4e4127fSJung-uk Kim freq += C2D(p[2]) * 100; 213a4e4127fSJung-uk Kim freq += C2D(p[3]) * 10; 214a4e4127fSJung-uk Kim freq *= i * 1000; 215a4e4127fSJung-uk Kim } else { 216a4e4127fSJung-uk Kim freq = C2D(p[0]) * 1000; 217a4e4127fSJung-uk Kim freq += C2D(p[1]) * 100; 218a4e4127fSJung-uk Kim freq += C2D(p[2]) * 10; 219a4e4127fSJung-uk Kim freq += C2D(p[3]); 220a4e4127fSJung-uk Kim freq *= i * 1000000; 221a4e4127fSJung-uk Kim } 222a4e4127fSJung-uk Kim #undef C2D 223a4e4127fSJung-uk Kim tsc_freq = freq; 224a4e4127fSJung-uk Kim } 225a4e4127fSJung-uk Kim } 226a4e4127fSJung-uk Kim } 227dd7d207dSJung-uk Kim 228a4e4127fSJung-uk Kim static void 229a4e4127fSJung-uk Kim probe_tsc_freq(void) 230a4e4127fSJung-uk Kim { 231bd8a359fSKonstantin Belousov uint64_t tmp_freq, tsc1, tsc2; 232bd8a359fSKonstantin Belousov int no_cpuid_override; 233dd7d207dSJung-uk Kim 234bb044eafSConrad Meyer if (cpu_power_ecx & CPUID_PERF_STAT) { 2355da5812bSJung-uk Kim /* 236bb044eafSConrad Meyer * XXX Some emulators expose host CPUID without actual support 237bb044eafSConrad Meyer * for these MSRs. We must test whether they really work. 2385da5812bSJung-uk Kim */ 2395da5812bSJung-uk Kim wrmsr(MSR_MPERF, 0); 2405da5812bSJung-uk Kim wrmsr(MSR_APERF, 0); 2415da5812bSJung-uk Kim DELAY(10); 2425da5812bSJung-uk Kim if (rdmsr(MSR_MPERF) > 0 && rdmsr(MSR_APERF) > 0) 2435da5812bSJung-uk Kim tsc_perf_stat = 1; 2445da5812bSJung-uk Kim } 2455da5812bSJung-uk Kim 24601e1933dSJohn Baldwin if (vm_guest == VM_GUEST_VMWARE) { 24701e1933dSJohn Baldwin tsc_freq_vmware(); 2485da5812bSJung-uk Kim return; 24901e1933dSJohn Baldwin } 2505da5812bSJung-uk Kim 251dd7d207dSJung-uk Kim switch (cpu_vendor_id) { 252dd7d207dSJung-uk Kim case CPU_VENDOR_AMD: 2532ee49facSKonstantin Belousov case CPU_VENDOR_HYGON: 254a106a27cSJung-uk Kim if ((amd_pminfo & AMDPM_TSC_INVARIANT) != 0 || 255a106a27cSJung-uk Kim (vm_guest == VM_GUEST_NO && 256a106a27cSJung-uk Kim CPUID_TO_FAMILY(cpu_id) >= 0x10)) 257dd7d207dSJung-uk Kim tsc_is_invariant = 1; 258814124c3SKonstantin Belousov if (cpu_feature & CPUID_SSE2) { 259814124c3SKonstantin Belousov tsc_timecounter.tc_get_timecount = 260814124c3SKonstantin Belousov tsc_get_timecount_mfence; 261814124c3SKonstantin Belousov } 262dd7d207dSJung-uk Kim break; 263dd7d207dSJung-uk Kim case CPU_VENDOR_INTEL: 264a106a27cSJung-uk Kim if ((amd_pminfo & AMDPM_TSC_INVARIANT) != 0 || 265a106a27cSJung-uk Kim (vm_guest == VM_GUEST_NO && 266a106a27cSJung-uk Kim ((CPUID_TO_FAMILY(cpu_id) == 0x6 && 267dd7d207dSJung-uk Kim CPUID_TO_MODEL(cpu_id) >= 0xe) || 268dd7d207dSJung-uk Kim (CPUID_TO_FAMILY(cpu_id) == 0xf && 269a106a27cSJung-uk Kim CPUID_TO_MODEL(cpu_id) >= 0x3)))) 270dd7d207dSJung-uk Kim tsc_is_invariant = 1; 271814124c3SKonstantin Belousov if (cpu_feature & CPUID_SSE2) { 272814124c3SKonstantin Belousov tsc_timecounter.tc_get_timecount = 273814124c3SKonstantin Belousov tsc_get_timecount_lfence; 274814124c3SKonstantin Belousov } 275dd7d207dSJung-uk Kim break; 276dd7d207dSJung-uk Kim case CPU_VENDOR_CENTAUR: 277a106a27cSJung-uk Kim if (vm_guest == VM_GUEST_NO && 278a106a27cSJung-uk Kim CPUID_TO_FAMILY(cpu_id) == 0x6 && 279dd7d207dSJung-uk Kim CPUID_TO_MODEL(cpu_id) >= 0xf && 280dd7d207dSJung-uk Kim (rdmsr(0x1203) & 0x100000000ULL) == 0) 281dd7d207dSJung-uk Kim tsc_is_invariant = 1; 282814124c3SKonstantin Belousov if (cpu_feature & CPUID_SSE2) { 283814124c3SKonstantin Belousov tsc_timecounter.tc_get_timecount = 284814124c3SKonstantin Belousov tsc_get_timecount_lfence; 285814124c3SKonstantin Belousov } 286dd7d207dSJung-uk Kim break; 287dd7d207dSJung-uk Kim } 288dd7d207dSJung-uk Kim 289a4e4127fSJung-uk Kim if (tsc_skip_calibration) { 290bd8a359fSKonstantin Belousov if (tsc_freq_cpuid(&tmp_freq)) 291bd8a359fSKonstantin Belousov tsc_freq = tmp_freq; 292506a906cSKonstantin Belousov else if (cpu_vendor_id == CPU_VENDOR_INTEL) 293a4e4127fSJung-uk Kim tsc_freq_intel(); 294ab23c278SKonstantin Belousov if (tsc_freq == 0) 295ab23c278SKonstantin Belousov tsc_disabled = 1; 296506a906cSKonstantin Belousov } else { 297a4e4127fSJung-uk Kim if (bootverbose) 298a4e4127fSJung-uk Kim printf("Calibrating TSC clock ... "); 299a4e4127fSJung-uk Kim tsc1 = rdtsc(); 300a4e4127fSJung-uk Kim DELAY(1000000); 301a4e4127fSJung-uk Kim tsc2 = rdtsc(); 302a4e4127fSJung-uk Kim tsc_freq = tsc2 - tsc1; 303bd8a359fSKonstantin Belousov 304bd8a359fSKonstantin Belousov /* 305bd8a359fSKonstantin Belousov * If the difference between calibrated frequency and 306bd8a359fSKonstantin Belousov * the frequency reported by CPUID 0x15/0x16 leafs 307bd8a359fSKonstantin Belousov * differ significantly, this probably means that 308bd8a359fSKonstantin Belousov * calibration is bogus. It happens on machines 309ab23c278SKonstantin Belousov * without 8254 timer. The BIOS rarely properly 310ab23c278SKonstantin Belousov * reports it in FADT boot flags, so just compare the 311ab23c278SKonstantin Belousov * frequencies directly. 312bd8a359fSKonstantin Belousov */ 313bd8a359fSKonstantin Belousov if (tsc_freq_cpuid(&tmp_freq) && qabs(tsc_freq - tmp_freq) > 314bd8a359fSKonstantin Belousov uqmin(tsc_freq, tmp_freq)) { 315bd8a359fSKonstantin Belousov no_cpuid_override = 0; 316bd8a359fSKonstantin Belousov TUNABLE_INT_FETCH("machdep.disable_tsc_cpuid_override", 317bd8a359fSKonstantin Belousov &no_cpuid_override); 318bd8a359fSKonstantin Belousov if (!no_cpuid_override) { 319bd8a359fSKonstantin Belousov if (bootverbose) { 320bd8a359fSKonstantin Belousov printf( 321bd8a359fSKonstantin Belousov "TSC clock: calibration freq %ju Hz, CPUID freq %ju Hz%s\n", 322bd8a359fSKonstantin Belousov (uintmax_t)tsc_freq, 323bd8a359fSKonstantin Belousov (uintmax_t)tmp_freq, 324bd8a359fSKonstantin Belousov no_cpuid_override ? "" : 325bd8a359fSKonstantin Belousov ", doing CPUID override"); 326bd8a359fSKonstantin Belousov } 327bd8a359fSKonstantin Belousov tsc_freq = tmp_freq; 328bd8a359fSKonstantin Belousov } 329bd8a359fSKonstantin Belousov } 330506a906cSKonstantin Belousov } 331a4e4127fSJung-uk Kim if (bootverbose) 332a4e4127fSJung-uk Kim printf("TSC clock: %ju Hz\n", (intmax_t)tsc_freq); 333a4e4127fSJung-uk Kim } 334a4e4127fSJung-uk Kim 335a4e4127fSJung-uk Kim void 336a4e4127fSJung-uk Kim init_TSC(void) 337a4e4127fSJung-uk Kim { 338a4e4127fSJung-uk Kim 339a4e4127fSJung-uk Kim if ((cpu_feature & CPUID_TSC) == 0 || tsc_disabled) 340a4e4127fSJung-uk Kim return; 341a4e4127fSJung-uk Kim 342fe760cfaSJohn Baldwin #ifdef __i386__ 343fe760cfaSJohn Baldwin /* The TSC is known to be broken on certain CPUs. */ 344fe760cfaSJohn Baldwin switch (cpu_vendor_id) { 345fe760cfaSJohn Baldwin case CPU_VENDOR_AMD: 346fe760cfaSJohn Baldwin switch (cpu_id & 0xFF0) { 347fe760cfaSJohn Baldwin case 0x500: 348fe760cfaSJohn Baldwin /* K5 Model 0 */ 349fe760cfaSJohn Baldwin return; 350fe760cfaSJohn Baldwin } 351fe760cfaSJohn Baldwin break; 352fe760cfaSJohn Baldwin case CPU_VENDOR_CENTAUR: 353fe760cfaSJohn Baldwin switch (cpu_id & 0xff0) { 354fe760cfaSJohn Baldwin case 0x540: 355fe760cfaSJohn Baldwin /* 356fe760cfaSJohn Baldwin * http://www.centtech.com/c6_data_sheet.pdf 357fe760cfaSJohn Baldwin * 358fe760cfaSJohn Baldwin * I-12 RDTSC may return incoherent values in EDX:EAX 359fe760cfaSJohn Baldwin * I-13 RDTSC hangs when certain event counters are used 360fe760cfaSJohn Baldwin */ 361fe760cfaSJohn Baldwin return; 362fe760cfaSJohn Baldwin } 363fe760cfaSJohn Baldwin break; 364fe760cfaSJohn Baldwin case CPU_VENDOR_NSC: 365fe760cfaSJohn Baldwin switch (cpu_id & 0xff0) { 366fe760cfaSJohn Baldwin case 0x540: 367fe760cfaSJohn Baldwin if ((cpu_id & CPUID_STEPPING) == 0) 368fe760cfaSJohn Baldwin return; 369fe760cfaSJohn Baldwin break; 370fe760cfaSJohn Baldwin } 371fe760cfaSJohn Baldwin break; 372fe760cfaSJohn Baldwin } 373fe760cfaSJohn Baldwin #endif 374fe760cfaSJohn Baldwin 375a4e4127fSJung-uk Kim probe_tsc_freq(); 376a4e4127fSJung-uk Kim 377dd7d207dSJung-uk Kim /* 378dd7d207dSJung-uk Kim * Inform CPU accounting about our boot-time clock rate. This will 379dd7d207dSJung-uk Kim * be updated if someone loads a cpufreq driver after boot that 380dd7d207dSJung-uk Kim * discovers a new max frequency. 381dd7d207dSJung-uk Kim */ 382a4e4127fSJung-uk Kim if (tsc_freq != 0) 3835ac44f72SJung-uk Kim set_cputicker(rdtsc, tsc_freq, !tsc_is_invariant); 384dd7d207dSJung-uk Kim 385dd7d207dSJung-uk Kim if (tsc_is_invariant) 386dd7d207dSJung-uk Kim return; 387dd7d207dSJung-uk Kim 388dd7d207dSJung-uk Kim /* Register to find out about changes in CPU frequency. */ 389dd7d207dSJung-uk Kim tsc_pre_tag = EVENTHANDLER_REGISTER(cpufreq_pre_change, 390dd7d207dSJung-uk Kim tsc_freq_changing, NULL, EVENTHANDLER_PRI_FIRST); 391dd7d207dSJung-uk Kim tsc_post_tag = EVENTHANDLER_REGISTER(cpufreq_post_change, 392dd7d207dSJung-uk Kim tsc_freq_changed, NULL, EVENTHANDLER_PRI_FIRST); 393dd7d207dSJung-uk Kim tsc_levels_tag = EVENTHANDLER_REGISTER(cpufreq_levels_changed, 394dd7d207dSJung-uk Kim tsc_levels_changed, NULL, EVENTHANDLER_PRI_ANY); 395dd7d207dSJung-uk Kim } 396dd7d207dSJung-uk Kim 39765e7d70bSJung-uk Kim #ifdef SMP 39865e7d70bSJung-uk Kim 399814124c3SKonstantin Belousov /* 400814124c3SKonstantin Belousov * RDTSC is not a serializing instruction, and does not drain 401814124c3SKonstantin Belousov * instruction stream, so we need to drain the stream before executing 402814124c3SKonstantin Belousov * it. It could be fixed by use of RDTSCP, except the instruction is 403814124c3SKonstantin Belousov * not available everywhere. 404814124c3SKonstantin Belousov * 405814124c3SKonstantin Belousov * Use CPUID for draining in the boot-time SMP constistency test. The 406814124c3SKonstantin Belousov * timecounters use MFENCE for AMD CPUs, and LFENCE for others (Intel 407814124c3SKonstantin Belousov * and VIA) when SSE2 is present, and nothing on older machines which 408814124c3SKonstantin Belousov * also do not issue RDTSC prematurely. There, testing for SSE2 and 409e1a18e46SKonstantin Belousov * vendor is too cumbersome, and we learn about TSC presence from CPUID. 410814124c3SKonstantin Belousov * 411814124c3SKonstantin Belousov * Do not use do_cpuid(), since we do not need CPUID results, which 412814124c3SKonstantin Belousov * have to be written into memory with do_cpuid(). 413814124c3SKonstantin Belousov */ 41465e7d70bSJung-uk Kim #define TSC_READ(x) \ 41565e7d70bSJung-uk Kim static void \ 41665e7d70bSJung-uk Kim tsc_read_##x(void *arg) \ 41765e7d70bSJung-uk Kim { \ 4187bfcb3bbSJim Harris uint64_t *tsc = arg; \ 41965e7d70bSJung-uk Kim u_int cpu = PCPU_GET(cpuid); \ 42065e7d70bSJung-uk Kim \ 421814124c3SKonstantin Belousov __asm __volatile("cpuid" : : : "eax", "ebx", "ecx", "edx"); \ 4227bfcb3bbSJim Harris tsc[cpu * 3 + x] = rdtsc(); \ 42365e7d70bSJung-uk Kim } 42465e7d70bSJung-uk Kim TSC_READ(0) 42565e7d70bSJung-uk Kim TSC_READ(1) 42665e7d70bSJung-uk Kim TSC_READ(2) 42765e7d70bSJung-uk Kim #undef TSC_READ 42865e7d70bSJung-uk Kim 42965e7d70bSJung-uk Kim #define N 1000 43065e7d70bSJung-uk Kim 43165e7d70bSJung-uk Kim static void 43265e7d70bSJung-uk Kim comp_smp_tsc(void *arg) 43365e7d70bSJung-uk Kim { 4347bfcb3bbSJim Harris uint64_t *tsc; 4357bfcb3bbSJim Harris int64_t d1, d2; 43665e7d70bSJung-uk Kim u_int cpu = PCPU_GET(cpuid); 43765e7d70bSJung-uk Kim u_int i, j, size; 43865e7d70bSJung-uk Kim 43965e7d70bSJung-uk Kim size = (mp_maxid + 1) * 3; 44065e7d70bSJung-uk Kim for (i = 0, tsc = arg; i < N; i++, tsc += size) 44165e7d70bSJung-uk Kim CPU_FOREACH(j) { 44265e7d70bSJung-uk Kim if (j == cpu) 44365e7d70bSJung-uk Kim continue; 44465e7d70bSJung-uk Kim d1 = tsc[cpu * 3 + 1] - tsc[j * 3]; 44565e7d70bSJung-uk Kim d2 = tsc[cpu * 3 + 2] - tsc[j * 3 + 1]; 44665e7d70bSJung-uk Kim if (d1 <= 0 || d2 <= 0) { 44765e7d70bSJung-uk Kim smp_tsc = 0; 44865e7d70bSJung-uk Kim return; 44965e7d70bSJung-uk Kim } 45065e7d70bSJung-uk Kim } 45165e7d70bSJung-uk Kim } 45265e7d70bSJung-uk Kim 453b2c63698SAlexander Motin static void 454b2c63698SAlexander Motin adj_smp_tsc(void *arg) 455b2c63698SAlexander Motin { 456b2c63698SAlexander Motin uint64_t *tsc; 457b2c63698SAlexander Motin int64_t d, min, max; 458b2c63698SAlexander Motin u_int cpu = PCPU_GET(cpuid); 459b2c63698SAlexander Motin u_int first, i, size; 460b2c63698SAlexander Motin 461b2c63698SAlexander Motin first = CPU_FIRST(); 462b2c63698SAlexander Motin if (cpu == first) 463b2c63698SAlexander Motin return; 464b2c63698SAlexander Motin min = INT64_MIN; 465b2c63698SAlexander Motin max = INT64_MAX; 466b2c63698SAlexander Motin size = (mp_maxid + 1) * 3; 467b2c63698SAlexander Motin for (i = 0, tsc = arg; i < N; i++, tsc += size) { 468b2c63698SAlexander Motin d = tsc[first * 3] - tsc[cpu * 3 + 1]; 469b2c63698SAlexander Motin if (d > min) 470b2c63698SAlexander Motin min = d; 471b2c63698SAlexander Motin d = tsc[first * 3 + 1] - tsc[cpu * 3 + 2]; 472b2c63698SAlexander Motin if (d > min) 473b2c63698SAlexander Motin min = d; 474b2c63698SAlexander Motin d = tsc[first * 3 + 1] - tsc[cpu * 3]; 475b2c63698SAlexander Motin if (d < max) 476b2c63698SAlexander Motin max = d; 477b2c63698SAlexander Motin d = tsc[first * 3 + 2] - tsc[cpu * 3 + 1]; 478b2c63698SAlexander Motin if (d < max) 479b2c63698SAlexander Motin max = d; 480b2c63698SAlexander Motin } 481b2c63698SAlexander Motin if (min > max) 482b2c63698SAlexander Motin return; 483b2c63698SAlexander Motin d = min / 2 + max / 2; 484b2c63698SAlexander Motin __asm __volatile ( 485b2c63698SAlexander Motin "movl $0x10, %%ecx\n\t" 486b2c63698SAlexander Motin "rdmsr\n\t" 487b2c63698SAlexander Motin "addl %%edi, %%eax\n\t" 488b2c63698SAlexander Motin "adcl %%esi, %%edx\n\t" 489b2c63698SAlexander Motin "wrmsr\n" 490b2c63698SAlexander Motin : /* No output */ 491b2c63698SAlexander Motin : "D" ((uint32_t)d), "S" ((uint32_t)(d >> 32)) 492b2c63698SAlexander Motin : "ax", "cx", "dx", "cc" 493b2c63698SAlexander Motin ); 494b2c63698SAlexander Motin } 495b2c63698SAlexander Motin 49665e7d70bSJung-uk Kim static int 497279be68bSAndriy Gapon test_tsc(int adj_max_count) 49865e7d70bSJung-uk Kim { 4997bfcb3bbSJim Harris uint64_t *data, *tsc; 500b2c63698SAlexander Motin u_int i, size, adj; 50165e7d70bSJung-uk Kim 50284eaf2ccSKonstantin Belousov if ((!smp_tsc && !tsc_is_invariant)) 50365e7d70bSJung-uk Kim return (-100); 50465e7d70bSJung-uk Kim size = (mp_maxid + 1) * 3; 50565e7d70bSJung-uk Kim data = malloc(sizeof(*data) * size * N, M_TEMP, M_WAITOK); 506b2c63698SAlexander Motin adj = 0; 507b2c63698SAlexander Motin retry: 50865e7d70bSJung-uk Kim for (i = 0, tsc = data; i < N; i++, tsc += size) 50965e7d70bSJung-uk Kim smp_rendezvous(tsc_read_0, tsc_read_1, tsc_read_2, tsc); 51065e7d70bSJung-uk Kim smp_tsc = 1; /* XXX */ 51167d955aaSPatrick Kelsey smp_rendezvous(smp_no_rendezvous_barrier, comp_smp_tsc, 51267d955aaSPatrick Kelsey smp_no_rendezvous_barrier, data); 513279be68bSAndriy Gapon if (!smp_tsc && adj < adj_max_count) { 514b2c63698SAlexander Motin adj++; 51567d955aaSPatrick Kelsey smp_rendezvous(smp_no_rendezvous_barrier, adj_smp_tsc, 51667d955aaSPatrick Kelsey smp_no_rendezvous_barrier, data); 517b2c63698SAlexander Motin goto retry; 518b2c63698SAlexander Motin } 51965e7d70bSJung-uk Kim free(data, M_TEMP); 52065e7d70bSJung-uk Kim if (bootverbose) 521b2c63698SAlexander Motin printf("SMP: %sed TSC synchronization test%s\n", 522b2c63698SAlexander Motin smp_tsc ? "pass" : "fail", 523b2c63698SAlexander Motin adj > 0 ? " after adjustment" : ""); 52426e6537aSJung-uk Kim if (smp_tsc && tsc_is_invariant) { 52526e6537aSJung-uk Kim switch (cpu_vendor_id) { 52626e6537aSJung-uk Kim case CPU_VENDOR_AMD: 5272ee49facSKonstantin Belousov case CPU_VENDOR_HYGON: 52826e6537aSJung-uk Kim /* 529450d86fcSJung-uk Kim * Processor Programming Reference (PPR) for AMD 530450d86fcSJung-uk Kim * Family 17h states that the TSC uses a common 531450d86fcSJung-uk Kim * reference for all sockets, cores and threads. 532450d86fcSJung-uk Kim */ 533450d86fcSJung-uk Kim if (CPUID_TO_FAMILY(cpu_id) >= 0x17) 534450d86fcSJung-uk Kim return (1000); 535450d86fcSJung-uk Kim /* 53626e6537aSJung-uk Kim * Starting with Family 15h processors, TSC clock 53726e6537aSJung-uk Kim * source is in the north bridge. Check whether 53826e6537aSJung-uk Kim * we have a single-socket/multi-core platform. 53926e6537aSJung-uk Kim * XXX Need more work for complex cases. 54026e6537aSJung-uk Kim */ 54126e6537aSJung-uk Kim if (CPUID_TO_FAMILY(cpu_id) < 0x15 || 54226e6537aSJung-uk Kim (amd_feature2 & AMDID2_CMP) == 0 || 54326e6537aSJung-uk Kim smp_cpus > (cpu_procinfo2 & AMDID_CMP_CORES) + 1) 54426e6537aSJung-uk Kim break; 54526e6537aSJung-uk Kim return (1000); 54626e6537aSJung-uk Kim case CPU_VENDOR_INTEL: 54726e6537aSJung-uk Kim /* 54826e6537aSJung-uk Kim * XXX Assume Intel platforms have synchronized TSCs. 54926e6537aSJung-uk Kim */ 55026e6537aSJung-uk Kim return (1000); 55126e6537aSJung-uk Kim } 55226e6537aSJung-uk Kim return (800); 55326e6537aSJung-uk Kim } 55426e6537aSJung-uk Kim return (-100); 55565e7d70bSJung-uk Kim } 55665e7d70bSJung-uk Kim 55765e7d70bSJung-uk Kim #undef N 55865e7d70bSJung-uk Kim 55965e7d70bSJung-uk Kim #endif /* SMP */ 56065e7d70bSJung-uk Kim 56165e7d70bSJung-uk Kim static void 562dd7d207dSJung-uk Kim init_TSC_tc(void) 563dd7d207dSJung-uk Kim { 56495f2f098SJung-uk Kim uint64_t max_freq; 56595f2f098SJung-uk Kim int shift; 566dd7d207dSJung-uk Kim 56738b8542cSJung-uk Kim if ((cpu_feature & CPUID_TSC) == 0 || tsc_disabled) 568dd7d207dSJung-uk Kim return; 569dd7d207dSJung-uk Kim 570dd7d207dSJung-uk Kim /* 57195f2f098SJung-uk Kim * Limit timecounter frequency to fit in an int and prevent it from 57295f2f098SJung-uk Kim * overflowing too fast. 57395f2f098SJung-uk Kim */ 57495f2f098SJung-uk Kim max_freq = UINT_MAX; 57595f2f098SJung-uk Kim 57695f2f098SJung-uk Kim /* 577dd7d207dSJung-uk Kim * We can not use the TSC if we support APM. Precise timekeeping 578dd7d207dSJung-uk Kim * on an APM'ed machine is at best a fools pursuit, since 579dd7d207dSJung-uk Kim * any and all of the time spent in various SMM code can't 580dd7d207dSJung-uk Kim * be reliably accounted for. Reading the RTC is your only 581dd7d207dSJung-uk Kim * source of reliable time info. The i8254 loses too, of course, 582dd7d207dSJung-uk Kim * but we need to have some kind of time... 583dd7d207dSJung-uk Kim * We don't know at this point whether APM is going to be used 584dd7d207dSJung-uk Kim * or not, nor when it might be activated. Play it safe. 585dd7d207dSJung-uk Kim */ 586dd7d207dSJung-uk Kim if (power_pm_get_type() == POWER_PM_TYPE_APM) { 587dd7d207dSJung-uk Kim tsc_timecounter.tc_quality = -1000; 588dd7d207dSJung-uk Kim if (bootverbose) 589dd7d207dSJung-uk Kim printf("TSC timecounter disabled: APM enabled.\n"); 59065e7d70bSJung-uk Kim goto init; 591dd7d207dSJung-uk Kim } 592dd7d207dSJung-uk Kim 593a49399a9SJung-uk Kim /* 59492597e06SJohn Baldwin * Intel CPUs without a C-state invariant TSC can stop the TSC 595d1411416SJohn Baldwin * in either C2 or C3. Disable use of C2 and C3 while using 596d1411416SJohn Baldwin * the TSC as the timecounter. The timecounter can be changed 597d1411416SJohn Baldwin * to enable C2 and C3. 598d1411416SJohn Baldwin * 599d1411416SJohn Baldwin * Note that the TSC is used as the cputicker for computing 600d1411416SJohn Baldwin * thread runtime regardless of the timecounter setting, so 601d1411416SJohn Baldwin * using an alternate timecounter and enabling C2 or C3 can 602d1411416SJohn Baldwin * result incorrect runtimes for kernel idle threads (but not 603d1411416SJohn Baldwin * for any non-idle threads). 604a49399a9SJung-uk Kim */ 6058cd59625SKonstantin Belousov if (cpu_vendor_id == CPU_VENDOR_INTEL && 606a49399a9SJung-uk Kim (amd_pminfo & AMDPM_TSC_INVARIANT) == 0) { 60792597e06SJohn Baldwin tsc_timecounter.tc_flags |= TC_FLAGS_C2STOP; 608a49399a9SJung-uk Kim if (bootverbose) 609d1411416SJohn Baldwin printf("TSC timecounter disables C2 and C3.\n"); 610a49399a9SJung-uk Kim } 611a49399a9SJung-uk Kim 612dd7d207dSJung-uk Kim /* 613e7f1427dSKonstantin Belousov * We can not use the TSC in SMP mode unless the TSCs on all CPUs 614e7f1427dSKonstantin Belousov * are synchronized. If the user is sure that the system has 615e7f1427dSKonstantin Belousov * synchronized TSCs, set kern.timecounter.smp_tsc tunable to a 616e7f1427dSKonstantin Belousov * non-zero value. The TSC seems unreliable in virtualized SMP 6175cf8ac1bSMike Silbersack * environments, so it is set to a negative quality in those cases. 618dd7d207dSJung-uk Kim */ 619ba79ab82SAndriy Gapon #ifdef SMP 620e7f1427dSKonstantin Belousov if (mp_ncpus > 1) 621279be68bSAndriy Gapon tsc_timecounter.tc_quality = test_tsc(smp_tsc_adjust); 622ba79ab82SAndriy Gapon else 623ba79ab82SAndriy Gapon #endif /* SMP */ 624ba79ab82SAndriy Gapon if (tsc_is_invariant) 62526e6537aSJung-uk Kim tsc_timecounter.tc_quality = 1000; 626e7f1427dSKonstantin Belousov max_freq >>= tsc_shift; 62726e6537aSJung-uk Kim 62865e7d70bSJung-uk Kim init: 629e7f1427dSKonstantin Belousov for (shift = 0; shift <= 31 && (tsc_freq >> shift) > max_freq; shift++) 63095f2f098SJung-uk Kim ; 631e7f1427dSKonstantin Belousov if ((cpu_feature & CPUID_SSE2) != 0 && mp_ncpus > 1) { 6322ee49facSKonstantin Belousov if (cpu_vendor_id == CPU_VENDOR_AMD || 6332ee49facSKonstantin Belousov cpu_vendor_id == CPU_VENDOR_HYGON) { 634e7f1427dSKonstantin Belousov tsc_timecounter.tc_get_timecount = shift > 0 ? 635e7f1427dSKonstantin Belousov tsc_get_timecount_low_mfence : 636e7f1427dSKonstantin Belousov tsc_get_timecount_mfence; 637814124c3SKonstantin Belousov } else { 638e7f1427dSKonstantin Belousov tsc_timecounter.tc_get_timecount = shift > 0 ? 639e7f1427dSKonstantin Belousov tsc_get_timecount_low_lfence : 640e7f1427dSKonstantin Belousov tsc_get_timecount_lfence; 641814124c3SKonstantin Belousov } 642e7f1427dSKonstantin Belousov } else { 643e7f1427dSKonstantin Belousov tsc_timecounter.tc_get_timecount = shift > 0 ? 644e7f1427dSKonstantin Belousov tsc_get_timecount_low : tsc_get_timecount; 645e7f1427dSKonstantin Belousov } 646e7f1427dSKonstantin Belousov if (shift > 0) { 64795f2f098SJung-uk Kim tsc_timecounter.tc_name = "TSC-low"; 64895f2f098SJung-uk Kim if (bootverbose) 649bc8e4ad2SJung-uk Kim printf("TSC timecounter discards lower %d bit(s)\n", 65095f2f098SJung-uk Kim shift); 65195f2f098SJung-uk Kim } 652bc34c87eSJung-uk Kim if (tsc_freq != 0) { 65395f2f098SJung-uk Kim tsc_timecounter.tc_frequency = tsc_freq >> shift; 65495f2f098SJung-uk Kim tsc_timecounter.tc_priv = (void *)(intptr_t)shift; 655dd7d207dSJung-uk Kim tc_init(&tsc_timecounter); 656dd7d207dSJung-uk Kim } 657dd7d207dSJung-uk Kim } 65865e7d70bSJung-uk Kim SYSINIT(tsc_tc, SI_SUB_SMP, SI_ORDER_ANY, init_TSC_tc, NULL); 659dd7d207dSJung-uk Kim 660279be68bSAndriy Gapon void 661279be68bSAndriy Gapon resume_TSC(void) 662279be68bSAndriy Gapon { 663ba79ab82SAndriy Gapon #ifdef SMP 664279be68bSAndriy Gapon int quality; 665279be68bSAndriy Gapon 666279be68bSAndriy Gapon /* If TSC was not good on boot, it is unlikely to become good now. */ 667279be68bSAndriy Gapon if (tsc_timecounter.tc_quality < 0) 668279be68bSAndriy Gapon return; 669279be68bSAndriy Gapon /* Nothing to do with UP. */ 670279be68bSAndriy Gapon if (mp_ncpus < 2) 671279be68bSAndriy Gapon return; 672279be68bSAndriy Gapon 673279be68bSAndriy Gapon /* 674279be68bSAndriy Gapon * If TSC was good, a single synchronization should be enough, 675279be68bSAndriy Gapon * but honour smp_tsc_adjust if it's set. 676279be68bSAndriy Gapon */ 677279be68bSAndriy Gapon quality = test_tsc(MAX(smp_tsc_adjust, 1)); 678279be68bSAndriy Gapon if (quality != tsc_timecounter.tc_quality) { 679279be68bSAndriy Gapon printf("TSC timecounter quality changed: %d -> %d\n", 680279be68bSAndriy Gapon tsc_timecounter.tc_quality, quality); 681279be68bSAndriy Gapon tsc_timecounter.tc_quality = quality; 682279be68bSAndriy Gapon } 683ba79ab82SAndriy Gapon #endif /* SMP */ 684279be68bSAndriy Gapon } 685279be68bSAndriy Gapon 686dd7d207dSJung-uk Kim /* 687dd7d207dSJung-uk Kim * When cpufreq levels change, find out about the (new) max frequency. We 688dd7d207dSJung-uk Kim * use this to update CPU accounting in case it got a lower estimate at boot. 689dd7d207dSJung-uk Kim */ 690dd7d207dSJung-uk Kim static void 691dd7d207dSJung-uk Kim tsc_levels_changed(void *arg, int unit) 692dd7d207dSJung-uk Kim { 693dd7d207dSJung-uk Kim device_t cf_dev; 694dd7d207dSJung-uk Kim struct cf_level *levels; 695dd7d207dSJung-uk Kim int count, error; 696dd7d207dSJung-uk Kim uint64_t max_freq; 697dd7d207dSJung-uk Kim 698dd7d207dSJung-uk Kim /* Only use values from the first CPU, assuming all are equal. */ 699dd7d207dSJung-uk Kim if (unit != 0) 700dd7d207dSJung-uk Kim return; 701dd7d207dSJung-uk Kim 702dd7d207dSJung-uk Kim /* Find the appropriate cpufreq device instance. */ 703dd7d207dSJung-uk Kim cf_dev = devclass_get_device(devclass_find("cpufreq"), unit); 704dd7d207dSJung-uk Kim if (cf_dev == NULL) { 705dd7d207dSJung-uk Kim printf("tsc_levels_changed() called but no cpufreq device?\n"); 706dd7d207dSJung-uk Kim return; 707dd7d207dSJung-uk Kim } 708dd7d207dSJung-uk Kim 709dd7d207dSJung-uk Kim /* Get settings from the device and find the max frequency. */ 710dd7d207dSJung-uk Kim count = 64; 711dd7d207dSJung-uk Kim levels = malloc(count * sizeof(*levels), M_TEMP, M_NOWAIT); 712dd7d207dSJung-uk Kim if (levels == NULL) 713dd7d207dSJung-uk Kim return; 714dd7d207dSJung-uk Kim error = CPUFREQ_LEVELS(cf_dev, levels, &count); 715dd7d207dSJung-uk Kim if (error == 0 && count != 0) { 716dd7d207dSJung-uk Kim max_freq = (uint64_t)levels[0].total_set.freq * 1000000; 717dd7d207dSJung-uk Kim set_cputicker(rdtsc, max_freq, 1); 718dd7d207dSJung-uk Kim } else 719dd7d207dSJung-uk Kim printf("tsc_levels_changed: no max freq found\n"); 720dd7d207dSJung-uk Kim free(levels, M_TEMP); 721dd7d207dSJung-uk Kim } 722dd7d207dSJung-uk Kim 723dd7d207dSJung-uk Kim /* 724dd7d207dSJung-uk Kim * If the TSC timecounter is in use, veto the pending change. It may be 725dd7d207dSJung-uk Kim * possible in the future to handle a dynamically-changing timecounter rate. 726dd7d207dSJung-uk Kim */ 727dd7d207dSJung-uk Kim static void 728dd7d207dSJung-uk Kim tsc_freq_changing(void *arg, const struct cf_level *level, int *status) 729dd7d207dSJung-uk Kim { 730dd7d207dSJung-uk Kim 731dd7d207dSJung-uk Kim if (*status != 0 || timecounter != &tsc_timecounter) 732dd7d207dSJung-uk Kim return; 733dd7d207dSJung-uk Kim 734dd7d207dSJung-uk Kim printf("timecounter TSC must not be in use when " 735dd7d207dSJung-uk Kim "changing frequencies; change denied\n"); 736dd7d207dSJung-uk Kim *status = EBUSY; 737dd7d207dSJung-uk Kim } 738dd7d207dSJung-uk Kim 739dd7d207dSJung-uk Kim /* Update TSC freq with the value indicated by the caller. */ 740dd7d207dSJung-uk Kim static void 741dd7d207dSJung-uk Kim tsc_freq_changed(void *arg, const struct cf_level *level, int status) 742dd7d207dSJung-uk Kim { 7433453537fSJung-uk Kim uint64_t freq; 744dd7d207dSJung-uk Kim 745dd7d207dSJung-uk Kim /* If there was an error during the transition, don't do anything. */ 74679422085SJung-uk Kim if (tsc_disabled || status != 0) 747dd7d207dSJung-uk Kim return; 748dd7d207dSJung-uk Kim 749dd7d207dSJung-uk Kim /* Total setting for this level gives the new frequency in MHz. */ 7503453537fSJung-uk Kim freq = (uint64_t)level->total_set.freq * 1000000; 7513453537fSJung-uk Kim atomic_store_rel_64(&tsc_freq, freq); 75295f2f098SJung-uk Kim tsc_timecounter.tc_frequency = 75395f2f098SJung-uk Kim freq >> (int)(intptr_t)tsc_timecounter.tc_priv; 754dd7d207dSJung-uk Kim } 755dd7d207dSJung-uk Kim 756dd7d207dSJung-uk Kim static int 757dd7d207dSJung-uk Kim sysctl_machdep_tsc_freq(SYSCTL_HANDLER_ARGS) 758dd7d207dSJung-uk Kim { 759dd7d207dSJung-uk Kim int error; 760dd7d207dSJung-uk Kim uint64_t freq; 761dd7d207dSJung-uk Kim 7623453537fSJung-uk Kim freq = atomic_load_acq_64(&tsc_freq); 7633453537fSJung-uk Kim if (freq == 0) 764dd7d207dSJung-uk Kim return (EOPNOTSUPP); 765cbc134adSMatthew D Fleming error = sysctl_handle_64(oidp, &freq, 0, req); 7667ebbcb21SJung-uk Kim if (error == 0 && req->newptr != NULL) { 7673453537fSJung-uk Kim atomic_store_rel_64(&tsc_freq, freq); 768bc8e4ad2SJung-uk Kim atomic_store_rel_64(&tsc_timecounter.tc_frequency, 769bc8e4ad2SJung-uk Kim freq >> (int)(intptr_t)tsc_timecounter.tc_priv); 7707ebbcb21SJung-uk Kim } 771dd7d207dSJung-uk Kim return (error); 772dd7d207dSJung-uk Kim } 773dd7d207dSJung-uk Kim 7747029da5cSPawel Biernacki SYSCTL_PROC(_machdep, OID_AUTO, tsc_freq, 7757029da5cSPawel Biernacki CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 7767029da5cSPawel Biernacki 0, 0, sysctl_machdep_tsc_freq, "QU", 7777029da5cSPawel Biernacki "Time Stamp Counter frequency"); 778dd7d207dSJung-uk Kim 779727c7b2dSJung-uk Kim static u_int 78095f2f098SJung-uk Kim tsc_get_timecount(struct timecounter *tc __unused) 781dd7d207dSJung-uk Kim { 782727c7b2dSJung-uk Kim 783727c7b2dSJung-uk Kim return (rdtsc32()); 784dd7d207dSJung-uk Kim } 78595f2f098SJung-uk Kim 786814124c3SKonstantin Belousov static inline u_int 787bc8e4ad2SJung-uk Kim tsc_get_timecount_low(struct timecounter *tc) 78895f2f098SJung-uk Kim { 7895df88f46SJung-uk Kim uint32_t rv; 79095f2f098SJung-uk Kim 7915df88f46SJung-uk Kim __asm __volatile("rdtsc; shrd %%cl, %%edx, %0" 7925df88f46SJung-uk Kim : "=a" (rv) : "c" ((int)(intptr_t)tc->tc_priv) : "edx"); 7935df88f46SJung-uk Kim return (rv); 79495f2f098SJung-uk Kim } 795aea81038SKonstantin Belousov 796814124c3SKonstantin Belousov static u_int 797814124c3SKonstantin Belousov tsc_get_timecount_lfence(struct timecounter *tc __unused) 798814124c3SKonstantin Belousov { 799814124c3SKonstantin Belousov 800814124c3SKonstantin Belousov lfence(); 801814124c3SKonstantin Belousov return (rdtsc32()); 802814124c3SKonstantin Belousov } 803814124c3SKonstantin Belousov 804814124c3SKonstantin Belousov static u_int 805814124c3SKonstantin Belousov tsc_get_timecount_low_lfence(struct timecounter *tc) 806814124c3SKonstantin Belousov { 807814124c3SKonstantin Belousov 808814124c3SKonstantin Belousov lfence(); 809814124c3SKonstantin Belousov return (tsc_get_timecount_low(tc)); 810814124c3SKonstantin Belousov } 811814124c3SKonstantin Belousov 812814124c3SKonstantin Belousov static u_int 813814124c3SKonstantin Belousov tsc_get_timecount_mfence(struct timecounter *tc __unused) 814814124c3SKonstantin Belousov { 815814124c3SKonstantin Belousov 816814124c3SKonstantin Belousov mfence(); 817814124c3SKonstantin Belousov return (rdtsc32()); 818814124c3SKonstantin Belousov } 819814124c3SKonstantin Belousov 820814124c3SKonstantin Belousov static u_int 821814124c3SKonstantin Belousov tsc_get_timecount_low_mfence(struct timecounter *tc) 822814124c3SKonstantin Belousov { 823814124c3SKonstantin Belousov 824814124c3SKonstantin Belousov mfence(); 825814124c3SKonstantin Belousov return (tsc_get_timecount_low(tc)); 826814124c3SKonstantin Belousov } 827814124c3SKonstantin Belousov 82816808549SKonstantin Belousov static uint32_t 82916808549SKonstantin Belousov x86_tsc_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) 830aea81038SKonstantin Belousov { 831aea81038SKonstantin Belousov 83216808549SKonstantin Belousov vdso_th->th_algo = VDSO_TH_ALGO_X86_TSC; 833d1b1b600SNeel Natu vdso_th->th_x86_shift = (int)(intptr_t)tc->tc_priv; 83416808549SKonstantin Belousov vdso_th->th_x86_hpet_idx = 0xffffffff; 835aea81038SKonstantin Belousov bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); 83616808549SKonstantin Belousov return (1); 837aea81038SKonstantin Belousov } 838aea81038SKonstantin Belousov 839aea81038SKonstantin Belousov #ifdef COMPAT_FREEBSD32 84016808549SKonstantin Belousov static uint32_t 84116808549SKonstantin Belousov x86_tsc_vdso_timehands32(struct vdso_timehands32 *vdso_th32, 842d1b1b600SNeel Natu struct timecounter *tc) 843aea81038SKonstantin Belousov { 844aea81038SKonstantin Belousov 84516808549SKonstantin Belousov vdso_th32->th_algo = VDSO_TH_ALGO_X86_TSC; 846d1b1b600SNeel Natu vdso_th32->th_x86_shift = (int)(intptr_t)tc->tc_priv; 84716808549SKonstantin Belousov vdso_th32->th_x86_hpet_idx = 0xffffffff; 848aea81038SKonstantin Belousov bzero(vdso_th32->th_res, sizeof(vdso_th32->th_res)); 84916808549SKonstantin Belousov return (1); 850aea81038SKonstantin Belousov } 851aea81038SKonstantin Belousov #endif 852