1dd7d207dSJung-uk Kim /*- 2dd7d207dSJung-uk Kim * Copyright (c) 1998-2003 Poul-Henning Kamp 3dd7d207dSJung-uk Kim * All rights reserved. 4dd7d207dSJung-uk Kim * 5dd7d207dSJung-uk Kim * Redistribution and use in source and binary forms, with or without 6dd7d207dSJung-uk Kim * modification, are permitted provided that the following conditions 7dd7d207dSJung-uk Kim * are met: 8dd7d207dSJung-uk Kim * 1. Redistributions of source code must retain the above copyright 9dd7d207dSJung-uk Kim * notice, this list of conditions and the following disclaimer. 10dd7d207dSJung-uk Kim * 2. Redistributions in binary form must reproduce the above copyright 11dd7d207dSJung-uk Kim * notice, this list of conditions and the following disclaimer in the 12dd7d207dSJung-uk Kim * documentation and/or other materials provided with the distribution. 13dd7d207dSJung-uk Kim * 14dd7d207dSJung-uk Kim * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15dd7d207dSJung-uk Kim * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16dd7d207dSJung-uk Kim * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17dd7d207dSJung-uk Kim * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18dd7d207dSJung-uk Kim * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19dd7d207dSJung-uk Kim * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20dd7d207dSJung-uk Kim * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21dd7d207dSJung-uk Kim * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22dd7d207dSJung-uk Kim * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23dd7d207dSJung-uk Kim * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24dd7d207dSJung-uk Kim * SUCH DAMAGE. 25dd7d207dSJung-uk Kim */ 26dd7d207dSJung-uk Kim 27dd7d207dSJung-uk Kim #include <sys/cdefs.h> 28dd7d207dSJung-uk Kim __FBSDID("$FreeBSD$"); 29dd7d207dSJung-uk Kim 30aea81038SKonstantin Belousov #include "opt_compat.h" 31dd7d207dSJung-uk Kim #include "opt_clock.h" 32dd7d207dSJung-uk Kim 33dd7d207dSJung-uk Kim #include <sys/param.h> 34dd7d207dSJung-uk Kim #include <sys/bus.h> 35dd7d207dSJung-uk Kim #include <sys/cpu.h> 365da5812bSJung-uk Kim #include <sys/limits.h> 37dd7d207dSJung-uk Kim #include <sys/malloc.h> 38dd7d207dSJung-uk Kim #include <sys/systm.h> 39dd7d207dSJung-uk Kim #include <sys/sysctl.h> 40dd7d207dSJung-uk Kim #include <sys/time.h> 41dd7d207dSJung-uk Kim #include <sys/timetc.h> 42dd7d207dSJung-uk Kim #include <sys/kernel.h> 43dd7d207dSJung-uk Kim #include <sys/power.h> 44dd7d207dSJung-uk Kim #include <sys/smp.h> 45aea81038SKonstantin Belousov #include <sys/vdso.h> 46dd7d207dSJung-uk Kim #include <machine/clock.h> 47dd7d207dSJung-uk Kim #include <machine/cputypes.h> 48dd7d207dSJung-uk Kim #include <machine/md_var.h> 49dd7d207dSJung-uk Kim #include <machine/specialreg.h> 50dd7d207dSJung-uk Kim 51dd7d207dSJung-uk Kim #include "cpufreq_if.h" 52dd7d207dSJung-uk Kim 53dd7d207dSJung-uk Kim uint64_t tsc_freq; 54dd7d207dSJung-uk Kim int tsc_is_invariant; 55155094d7SJung-uk Kim int tsc_perf_stat; 56155094d7SJung-uk Kim 57dd7d207dSJung-uk Kim static eventhandler_tag tsc_levels_tag, tsc_pre_tag, tsc_post_tag; 58dd7d207dSJung-uk Kim 59dd7d207dSJung-uk Kim SYSCTL_INT(_kern_timecounter, OID_AUTO, invariant_tsc, CTLFLAG_RDTUN, 60dd7d207dSJung-uk Kim &tsc_is_invariant, 0, "Indicates whether the TSC is P-state invariant"); 61dd7d207dSJung-uk Kim TUNABLE_INT("kern.timecounter.invariant_tsc", &tsc_is_invariant); 62dd7d207dSJung-uk Kim 63dd7d207dSJung-uk Kim #ifdef SMP 641472b87fSNeel Natu int smp_tsc; 65dd7d207dSJung-uk Kim SYSCTL_INT(_kern_timecounter, OID_AUTO, smp_tsc, CTLFLAG_RDTUN, &smp_tsc, 0, 66dd7d207dSJung-uk Kim "Indicates whether the TSC is safe to use in SMP mode"); 67dd7d207dSJung-uk Kim TUNABLE_INT("kern.timecounter.smp_tsc", &smp_tsc); 68b2c63698SAlexander Motin 69b2c63698SAlexander Motin int smp_tsc_adjust = 0; 70b2c63698SAlexander Motin SYSCTL_INT(_kern_timecounter, OID_AUTO, smp_tsc_adjust, CTLFLAG_RDTUN, 71b2c63698SAlexander Motin &smp_tsc_adjust, 0, "Try to adjust TSC on APs to match BSP"); 72b2c63698SAlexander Motin TUNABLE_INT("kern.timecounter.smp_tsc_adjust", &smp_tsc_adjust); 73dd7d207dSJung-uk Kim #endif 74dd7d207dSJung-uk Kim 75e7f1427dSKonstantin Belousov static int tsc_shift = 1; 76e7f1427dSKonstantin Belousov SYSCTL_INT(_kern_timecounter, OID_AUTO, tsc_shift, CTLFLAG_RDTUN, 77e7f1427dSKonstantin Belousov &tsc_shift, 0, "Shift to pre-apply for the maximum TSC frequency"); 78e7f1427dSKonstantin Belousov TUNABLE_INT("kern.timecounter.tsc_shift", &tsc_shift); 79e7f1427dSKonstantin Belousov 8079422085SJung-uk Kim static int tsc_disabled; 8179422085SJung-uk Kim SYSCTL_INT(_machdep, OID_AUTO, disable_tsc, CTLFLAG_RDTUN, &tsc_disabled, 0, 8279422085SJung-uk Kim "Disable x86 Time Stamp Counter"); 8379422085SJung-uk Kim TUNABLE_INT("machdep.disable_tsc", &tsc_disabled); 8479422085SJung-uk Kim 85a4e4127fSJung-uk Kim static int tsc_skip_calibration; 86a4e4127fSJung-uk Kim SYSCTL_INT(_machdep, OID_AUTO, disable_tsc_calibration, CTLFLAG_RDTUN, 87a4e4127fSJung-uk Kim &tsc_skip_calibration, 0, "Disable TSC frequency calibration"); 88a4e4127fSJung-uk Kim TUNABLE_INT("machdep.disable_tsc_calibration", &tsc_skip_calibration); 89a4e4127fSJung-uk Kim 90dd7d207dSJung-uk Kim static void tsc_freq_changed(void *arg, const struct cf_level *level, 91dd7d207dSJung-uk Kim int status); 92dd7d207dSJung-uk Kim static void tsc_freq_changing(void *arg, const struct cf_level *level, 93dd7d207dSJung-uk Kim int *status); 94dd7d207dSJung-uk Kim static unsigned tsc_get_timecount(struct timecounter *tc); 95814124c3SKonstantin Belousov static inline unsigned tsc_get_timecount_low(struct timecounter *tc); 96814124c3SKonstantin Belousov static unsigned tsc_get_timecount_lfence(struct timecounter *tc); 97814124c3SKonstantin Belousov static unsigned tsc_get_timecount_low_lfence(struct timecounter *tc); 98814124c3SKonstantin Belousov static unsigned tsc_get_timecount_mfence(struct timecounter *tc); 99814124c3SKonstantin Belousov static unsigned tsc_get_timecount_low_mfence(struct timecounter *tc); 100dd7d207dSJung-uk Kim static void tsc_levels_changed(void *arg, int unit); 101dd7d207dSJung-uk Kim 102dd7d207dSJung-uk Kim static struct timecounter tsc_timecounter = { 103dd7d207dSJung-uk Kim tsc_get_timecount, /* get_timecount */ 104dd7d207dSJung-uk Kim 0, /* no poll_pps */ 105dd7d207dSJung-uk Kim ~0u, /* counter_mask */ 106dd7d207dSJung-uk Kim 0, /* frequency */ 107dd7d207dSJung-uk Kim "TSC", /* name */ 108dd7d207dSJung-uk Kim 800, /* quality (adjusted in code) */ 109dd7d207dSJung-uk Kim }; 110dd7d207dSJung-uk Kim 1115da5812bSJung-uk Kim #define VMW_HVMAGIC 0x564d5868 1125da5812bSJung-uk Kim #define VMW_HVPORT 0x5658 1135da5812bSJung-uk Kim #define VMW_HVCMD_GETVERSION 10 1145da5812bSJung-uk Kim #define VMW_HVCMD_GETHZ 45 1155da5812bSJung-uk Kim 1165da5812bSJung-uk Kim static __inline void 1175da5812bSJung-uk Kim vmware_hvcall(u_int cmd, u_int *p) 1185da5812bSJung-uk Kim { 1195da5812bSJung-uk Kim 120a990fbf9SJung-uk Kim __asm __volatile("inl %w3, %0" 1215da5812bSJung-uk Kim : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 1225da5812bSJung-uk Kim : "0" (VMW_HVMAGIC), "1" (UINT_MAX), "2" (cmd), "3" (VMW_HVPORT) 1235da5812bSJung-uk Kim : "memory"); 1245da5812bSJung-uk Kim } 1255da5812bSJung-uk Kim 1265da5812bSJung-uk Kim static int 1275da5812bSJung-uk Kim tsc_freq_vmware(void) 1285da5812bSJung-uk Kim { 1295da5812bSJung-uk Kim char hv_sig[13]; 1305da5812bSJung-uk Kim u_int regs[4]; 1315da5812bSJung-uk Kim char *p; 1325da5812bSJung-uk Kim u_int hv_high; 1335da5812bSJung-uk Kim int i; 1345da5812bSJung-uk Kim 1355da5812bSJung-uk Kim /* 1365da5812bSJung-uk Kim * [RFC] CPUID usage for interaction between Hypervisors and Linux. 1375da5812bSJung-uk Kim * http://lkml.org/lkml/2008/10/1/246 1385da5812bSJung-uk Kim * 1395da5812bSJung-uk Kim * KB1009458: Mechanisms to determine if software is running in 1405da5812bSJung-uk Kim * a VMware virtual machine 1415da5812bSJung-uk Kim * http://kb.vmware.com/kb/1009458 1425da5812bSJung-uk Kim */ 1435da5812bSJung-uk Kim hv_high = 0; 1445da5812bSJung-uk Kim if ((cpu_feature2 & CPUID2_HV) != 0) { 1455da5812bSJung-uk Kim do_cpuid(0x40000000, regs); 1465da5812bSJung-uk Kim hv_high = regs[0]; 1475da5812bSJung-uk Kim for (i = 1, p = hv_sig; i < 4; i++, p += sizeof(regs) / 4) 1485da5812bSJung-uk Kim memcpy(p, ®s[i], sizeof(regs[i])); 1495da5812bSJung-uk Kim *p = '\0'; 1505da5812bSJung-uk Kim if (bootverbose) { 1515da5812bSJung-uk Kim /* 1525da5812bSJung-uk Kim * HV vendor ID string 1535da5812bSJung-uk Kim * ------------+-------------- 1545da5812bSJung-uk Kim * KVM "KVMKVMKVM" 1555da5812bSJung-uk Kim * Microsoft "Microsoft Hv" 1565da5812bSJung-uk Kim * VMware "VMwareVMware" 1575da5812bSJung-uk Kim * Xen "XenVMMXenVMM" 1585da5812bSJung-uk Kim */ 1595da5812bSJung-uk Kim printf("Hypervisor: Origin = \"%s\"\n", hv_sig); 1605da5812bSJung-uk Kim } 1615da5812bSJung-uk Kim if (strncmp(hv_sig, "VMwareVMware", 12) != 0) 1625da5812bSJung-uk Kim return (0); 1635da5812bSJung-uk Kim } else { 1645da5812bSJung-uk Kim p = getenv("smbios.system.serial"); 1655da5812bSJung-uk Kim if (p == NULL) 1665da5812bSJung-uk Kim return (0); 1675da5812bSJung-uk Kim if (strncmp(p, "VMware-", 7) != 0 && 1685da5812bSJung-uk Kim strncmp(p, "VMW", 3) != 0) { 1695da5812bSJung-uk Kim freeenv(p); 1705da5812bSJung-uk Kim return (0); 1715da5812bSJung-uk Kim } 1725da5812bSJung-uk Kim freeenv(p); 1735da5812bSJung-uk Kim vmware_hvcall(VMW_HVCMD_GETVERSION, regs); 1745da5812bSJung-uk Kim if (regs[1] != VMW_HVMAGIC) 1755da5812bSJung-uk Kim return (0); 1765da5812bSJung-uk Kim } 1775da5812bSJung-uk Kim if (hv_high >= 0x40000010) { 1785da5812bSJung-uk Kim do_cpuid(0x40000010, regs); 1795da5812bSJung-uk Kim tsc_freq = regs[0] * 1000; 1805da5812bSJung-uk Kim } else { 1815da5812bSJung-uk Kim vmware_hvcall(VMW_HVCMD_GETHZ, regs); 1825da5812bSJung-uk Kim if (regs[1] != UINT_MAX) 1835da5812bSJung-uk Kim tsc_freq = regs[0] | ((uint64_t)regs[1] << 32); 1845da5812bSJung-uk Kim } 1855da5812bSJung-uk Kim tsc_is_invariant = 1; 1865da5812bSJung-uk Kim return (1); 1875da5812bSJung-uk Kim } 1885da5812bSJung-uk Kim 189a4e4127fSJung-uk Kim static void 190a4e4127fSJung-uk Kim tsc_freq_intel(void) 191dd7d207dSJung-uk Kim { 192a4e4127fSJung-uk Kim char brand[48]; 193a4e4127fSJung-uk Kim u_int regs[4]; 194a4e4127fSJung-uk Kim uint64_t freq; 195a4e4127fSJung-uk Kim char *p; 196a4e4127fSJung-uk Kim u_int i; 197dd7d207dSJung-uk Kim 198a4e4127fSJung-uk Kim /* 199a4e4127fSJung-uk Kim * Intel Processor Identification and the CPUID Instruction 200a4e4127fSJung-uk Kim * Application Note 485. 201a4e4127fSJung-uk Kim * http://www.intel.com/assets/pdf/appnote/241618.pdf 202a4e4127fSJung-uk Kim */ 203a4e4127fSJung-uk Kim if (cpu_exthigh >= 0x80000004) { 204a4e4127fSJung-uk Kim p = brand; 205a4e4127fSJung-uk Kim for (i = 0x80000002; i < 0x80000005; i++) { 206a4e4127fSJung-uk Kim do_cpuid(i, regs); 207a4e4127fSJung-uk Kim memcpy(p, regs, sizeof(regs)); 208a4e4127fSJung-uk Kim p += sizeof(regs); 209a4e4127fSJung-uk Kim } 210a4e4127fSJung-uk Kim p = NULL; 211a4e4127fSJung-uk Kim for (i = 0; i < sizeof(brand) - 1; i++) 212a4e4127fSJung-uk Kim if (brand[i] == 'H' && brand[i + 1] == 'z') 213a4e4127fSJung-uk Kim p = brand + i; 214a4e4127fSJung-uk Kim if (p != NULL) { 215a4e4127fSJung-uk Kim p -= 5; 216a4e4127fSJung-uk Kim switch (p[4]) { 217a4e4127fSJung-uk Kim case 'M': 218a4e4127fSJung-uk Kim i = 1; 219a4e4127fSJung-uk Kim break; 220a4e4127fSJung-uk Kim case 'G': 221a4e4127fSJung-uk Kim i = 1000; 222a4e4127fSJung-uk Kim break; 223a4e4127fSJung-uk Kim case 'T': 224a4e4127fSJung-uk Kim i = 1000000; 225a4e4127fSJung-uk Kim break; 226a4e4127fSJung-uk Kim default: 227dd7d207dSJung-uk Kim return; 228a4e4127fSJung-uk Kim } 229a4e4127fSJung-uk Kim #define C2D(c) ((c) - '0') 230a4e4127fSJung-uk Kim if (p[1] == '.') { 231a4e4127fSJung-uk Kim freq = C2D(p[0]) * 1000; 232a4e4127fSJung-uk Kim freq += C2D(p[2]) * 100; 233a4e4127fSJung-uk Kim freq += C2D(p[3]) * 10; 234a4e4127fSJung-uk Kim freq *= i * 1000; 235a4e4127fSJung-uk Kim } else { 236a4e4127fSJung-uk Kim freq = C2D(p[0]) * 1000; 237a4e4127fSJung-uk Kim freq += C2D(p[1]) * 100; 238a4e4127fSJung-uk Kim freq += C2D(p[2]) * 10; 239a4e4127fSJung-uk Kim freq += C2D(p[3]); 240a4e4127fSJung-uk Kim freq *= i * 1000000; 241a4e4127fSJung-uk Kim } 242a4e4127fSJung-uk Kim #undef C2D 243a4e4127fSJung-uk Kim tsc_freq = freq; 244a4e4127fSJung-uk Kim } 245a4e4127fSJung-uk Kim } 246a4e4127fSJung-uk Kim } 247dd7d207dSJung-uk Kim 248a4e4127fSJung-uk Kim static void 249a4e4127fSJung-uk Kim probe_tsc_freq(void) 250a4e4127fSJung-uk Kim { 251155094d7SJung-uk Kim u_int regs[4]; 252a4e4127fSJung-uk Kim uint64_t tsc1, tsc2; 253dd7d207dSJung-uk Kim 2545da5812bSJung-uk Kim if (cpu_high >= 6) { 2555da5812bSJung-uk Kim do_cpuid(6, regs); 2565da5812bSJung-uk Kim if ((regs[2] & CPUID_PERF_STAT) != 0) { 2575da5812bSJung-uk Kim /* 2585da5812bSJung-uk Kim * XXX Some emulators expose host CPUID without actual 2595da5812bSJung-uk Kim * support for these MSRs. We must test whether they 2605da5812bSJung-uk Kim * really work. 2615da5812bSJung-uk Kim */ 2625da5812bSJung-uk Kim wrmsr(MSR_MPERF, 0); 2635da5812bSJung-uk Kim wrmsr(MSR_APERF, 0); 2645da5812bSJung-uk Kim DELAY(10); 2655da5812bSJung-uk Kim if (rdmsr(MSR_MPERF) > 0 && rdmsr(MSR_APERF) > 0) 2665da5812bSJung-uk Kim tsc_perf_stat = 1; 2675da5812bSJung-uk Kim } 2685da5812bSJung-uk Kim } 2695da5812bSJung-uk Kim 2705da5812bSJung-uk Kim if (tsc_freq_vmware()) 2715da5812bSJung-uk Kim return; 2725da5812bSJung-uk Kim 273dd7d207dSJung-uk Kim switch (cpu_vendor_id) { 274dd7d207dSJung-uk Kim case CPU_VENDOR_AMD: 275a106a27cSJung-uk Kim if ((amd_pminfo & AMDPM_TSC_INVARIANT) != 0 || 276a106a27cSJung-uk Kim (vm_guest == VM_GUEST_NO && 277a106a27cSJung-uk Kim CPUID_TO_FAMILY(cpu_id) >= 0x10)) 278dd7d207dSJung-uk Kim tsc_is_invariant = 1; 279814124c3SKonstantin Belousov if (cpu_feature & CPUID_SSE2) { 280814124c3SKonstantin Belousov tsc_timecounter.tc_get_timecount = 281814124c3SKonstantin Belousov tsc_get_timecount_mfence; 282814124c3SKonstantin Belousov } 283dd7d207dSJung-uk Kim break; 284dd7d207dSJung-uk Kim case CPU_VENDOR_INTEL: 285a106a27cSJung-uk Kim if ((amd_pminfo & AMDPM_TSC_INVARIANT) != 0 || 286a106a27cSJung-uk Kim (vm_guest == VM_GUEST_NO && 287a106a27cSJung-uk Kim ((CPUID_TO_FAMILY(cpu_id) == 0x6 && 288dd7d207dSJung-uk Kim CPUID_TO_MODEL(cpu_id) >= 0xe) || 289dd7d207dSJung-uk Kim (CPUID_TO_FAMILY(cpu_id) == 0xf && 290a106a27cSJung-uk Kim CPUID_TO_MODEL(cpu_id) >= 0x3)))) 291dd7d207dSJung-uk Kim tsc_is_invariant = 1; 292814124c3SKonstantin Belousov if (cpu_feature & CPUID_SSE2) { 293814124c3SKonstantin Belousov tsc_timecounter.tc_get_timecount = 294814124c3SKonstantin Belousov tsc_get_timecount_lfence; 295814124c3SKonstantin Belousov } 296dd7d207dSJung-uk Kim break; 297dd7d207dSJung-uk Kim case CPU_VENDOR_CENTAUR: 298a106a27cSJung-uk Kim if (vm_guest == VM_GUEST_NO && 299a106a27cSJung-uk Kim CPUID_TO_FAMILY(cpu_id) == 0x6 && 300dd7d207dSJung-uk Kim CPUID_TO_MODEL(cpu_id) >= 0xf && 301dd7d207dSJung-uk Kim (rdmsr(0x1203) & 0x100000000ULL) == 0) 302dd7d207dSJung-uk Kim tsc_is_invariant = 1; 303814124c3SKonstantin Belousov if (cpu_feature & CPUID_SSE2) { 304814124c3SKonstantin Belousov tsc_timecounter.tc_get_timecount = 305814124c3SKonstantin Belousov tsc_get_timecount_lfence; 306814124c3SKonstantin Belousov } 307dd7d207dSJung-uk Kim break; 308dd7d207dSJung-uk Kim } 309dd7d207dSJung-uk Kim 310a4e4127fSJung-uk Kim if (tsc_skip_calibration) { 311a4e4127fSJung-uk Kim if (cpu_vendor_id == CPU_VENDOR_INTEL) 312a4e4127fSJung-uk Kim tsc_freq_intel(); 313a4e4127fSJung-uk Kim return; 314a4e4127fSJung-uk Kim } 315a4e4127fSJung-uk Kim 316a4e4127fSJung-uk Kim if (bootverbose) 317a4e4127fSJung-uk Kim printf("Calibrating TSC clock ... "); 318a4e4127fSJung-uk Kim tsc1 = rdtsc(); 319a4e4127fSJung-uk Kim DELAY(1000000); 320a4e4127fSJung-uk Kim tsc2 = rdtsc(); 321a4e4127fSJung-uk Kim tsc_freq = tsc2 - tsc1; 322a4e4127fSJung-uk Kim if (bootverbose) 323a4e4127fSJung-uk Kim printf("TSC clock: %ju Hz\n", (intmax_t)tsc_freq); 324a4e4127fSJung-uk Kim } 325a4e4127fSJung-uk Kim 326a4e4127fSJung-uk Kim void 327a4e4127fSJung-uk Kim init_TSC(void) 328a4e4127fSJung-uk Kim { 329a4e4127fSJung-uk Kim 330a4e4127fSJung-uk Kim if ((cpu_feature & CPUID_TSC) == 0 || tsc_disabled) 331a4e4127fSJung-uk Kim return; 332a4e4127fSJung-uk Kim 333a4e4127fSJung-uk Kim probe_tsc_freq(); 334a4e4127fSJung-uk Kim 335dd7d207dSJung-uk Kim /* 336dd7d207dSJung-uk Kim * Inform CPU accounting about our boot-time clock rate. This will 337dd7d207dSJung-uk Kim * be updated if someone loads a cpufreq driver after boot that 338dd7d207dSJung-uk Kim * discovers a new max frequency. 339dd7d207dSJung-uk Kim */ 340a4e4127fSJung-uk Kim if (tsc_freq != 0) 3415ac44f72SJung-uk Kim set_cputicker(rdtsc, tsc_freq, !tsc_is_invariant); 342dd7d207dSJung-uk Kim 343dd7d207dSJung-uk Kim if (tsc_is_invariant) 344dd7d207dSJung-uk Kim return; 345dd7d207dSJung-uk Kim 346dd7d207dSJung-uk Kim /* Register to find out about changes in CPU frequency. */ 347dd7d207dSJung-uk Kim tsc_pre_tag = EVENTHANDLER_REGISTER(cpufreq_pre_change, 348dd7d207dSJung-uk Kim tsc_freq_changing, NULL, EVENTHANDLER_PRI_FIRST); 349dd7d207dSJung-uk Kim tsc_post_tag = EVENTHANDLER_REGISTER(cpufreq_post_change, 350dd7d207dSJung-uk Kim tsc_freq_changed, NULL, EVENTHANDLER_PRI_FIRST); 351dd7d207dSJung-uk Kim tsc_levels_tag = EVENTHANDLER_REGISTER(cpufreq_levels_changed, 352dd7d207dSJung-uk Kim tsc_levels_changed, NULL, EVENTHANDLER_PRI_ANY); 353dd7d207dSJung-uk Kim } 354dd7d207dSJung-uk Kim 35565e7d70bSJung-uk Kim #ifdef SMP 35665e7d70bSJung-uk Kim 357814124c3SKonstantin Belousov /* 358814124c3SKonstantin Belousov * RDTSC is not a serializing instruction, and does not drain 359814124c3SKonstantin Belousov * instruction stream, so we need to drain the stream before executing 360814124c3SKonstantin Belousov * it. It could be fixed by use of RDTSCP, except the instruction is 361814124c3SKonstantin Belousov * not available everywhere. 362814124c3SKonstantin Belousov * 363814124c3SKonstantin Belousov * Use CPUID for draining in the boot-time SMP constistency test. The 364814124c3SKonstantin Belousov * timecounters use MFENCE for AMD CPUs, and LFENCE for others (Intel 365814124c3SKonstantin Belousov * and VIA) when SSE2 is present, and nothing on older machines which 366814124c3SKonstantin Belousov * also do not issue RDTSC prematurely. There, testing for SSE2 and 367e1a18e46SKonstantin Belousov * vendor is too cumbersome, and we learn about TSC presence from CPUID. 368814124c3SKonstantin Belousov * 369814124c3SKonstantin Belousov * Do not use do_cpuid(), since we do not need CPUID results, which 370814124c3SKonstantin Belousov * have to be written into memory with do_cpuid(). 371814124c3SKonstantin Belousov */ 37265e7d70bSJung-uk Kim #define TSC_READ(x) \ 37365e7d70bSJung-uk Kim static void \ 37465e7d70bSJung-uk Kim tsc_read_##x(void *arg) \ 37565e7d70bSJung-uk Kim { \ 3767bfcb3bbSJim Harris uint64_t *tsc = arg; \ 37765e7d70bSJung-uk Kim u_int cpu = PCPU_GET(cpuid); \ 37865e7d70bSJung-uk Kim \ 379814124c3SKonstantin Belousov __asm __volatile("cpuid" : : : "eax", "ebx", "ecx", "edx"); \ 3807bfcb3bbSJim Harris tsc[cpu * 3 + x] = rdtsc(); \ 38165e7d70bSJung-uk Kim } 38265e7d70bSJung-uk Kim TSC_READ(0) 38365e7d70bSJung-uk Kim TSC_READ(1) 38465e7d70bSJung-uk Kim TSC_READ(2) 38565e7d70bSJung-uk Kim #undef TSC_READ 38665e7d70bSJung-uk Kim 38765e7d70bSJung-uk Kim #define N 1000 38865e7d70bSJung-uk Kim 38965e7d70bSJung-uk Kim static void 39065e7d70bSJung-uk Kim comp_smp_tsc(void *arg) 39165e7d70bSJung-uk Kim { 3927bfcb3bbSJim Harris uint64_t *tsc; 3937bfcb3bbSJim Harris int64_t d1, d2; 39465e7d70bSJung-uk Kim u_int cpu = PCPU_GET(cpuid); 39565e7d70bSJung-uk Kim u_int i, j, size; 39665e7d70bSJung-uk Kim 39765e7d70bSJung-uk Kim size = (mp_maxid + 1) * 3; 39865e7d70bSJung-uk Kim for (i = 0, tsc = arg; i < N; i++, tsc += size) 39965e7d70bSJung-uk Kim CPU_FOREACH(j) { 40065e7d70bSJung-uk Kim if (j == cpu) 40165e7d70bSJung-uk Kim continue; 40265e7d70bSJung-uk Kim d1 = tsc[cpu * 3 + 1] - tsc[j * 3]; 40365e7d70bSJung-uk Kim d2 = tsc[cpu * 3 + 2] - tsc[j * 3 + 1]; 40465e7d70bSJung-uk Kim if (d1 <= 0 || d2 <= 0) { 40565e7d70bSJung-uk Kim smp_tsc = 0; 40665e7d70bSJung-uk Kim return; 40765e7d70bSJung-uk Kim } 40865e7d70bSJung-uk Kim } 40965e7d70bSJung-uk Kim } 41065e7d70bSJung-uk Kim 411b2c63698SAlexander Motin static void 412b2c63698SAlexander Motin adj_smp_tsc(void *arg) 413b2c63698SAlexander Motin { 414b2c63698SAlexander Motin uint64_t *tsc; 415b2c63698SAlexander Motin int64_t d, min, max; 416b2c63698SAlexander Motin u_int cpu = PCPU_GET(cpuid); 417b2c63698SAlexander Motin u_int first, i, size; 418b2c63698SAlexander Motin 419b2c63698SAlexander Motin first = CPU_FIRST(); 420b2c63698SAlexander Motin if (cpu == first) 421b2c63698SAlexander Motin return; 422b2c63698SAlexander Motin min = INT64_MIN; 423b2c63698SAlexander Motin max = INT64_MAX; 424b2c63698SAlexander Motin size = (mp_maxid + 1) * 3; 425b2c63698SAlexander Motin for (i = 0, tsc = arg; i < N; i++, tsc += size) { 426b2c63698SAlexander Motin d = tsc[first * 3] - tsc[cpu * 3 + 1]; 427b2c63698SAlexander Motin if (d > min) 428b2c63698SAlexander Motin min = d; 429b2c63698SAlexander Motin d = tsc[first * 3 + 1] - tsc[cpu * 3 + 2]; 430b2c63698SAlexander Motin if (d > min) 431b2c63698SAlexander Motin min = d; 432b2c63698SAlexander Motin d = tsc[first * 3 + 1] - tsc[cpu * 3]; 433b2c63698SAlexander Motin if (d < max) 434b2c63698SAlexander Motin max = d; 435b2c63698SAlexander Motin d = tsc[first * 3 + 2] - tsc[cpu * 3 + 1]; 436b2c63698SAlexander Motin if (d < max) 437b2c63698SAlexander Motin max = d; 438b2c63698SAlexander Motin } 439b2c63698SAlexander Motin if (min > max) 440b2c63698SAlexander Motin return; 441b2c63698SAlexander Motin d = min / 2 + max / 2; 442b2c63698SAlexander Motin __asm __volatile ( 443b2c63698SAlexander Motin "movl $0x10, %%ecx\n\t" 444b2c63698SAlexander Motin "rdmsr\n\t" 445b2c63698SAlexander Motin "addl %%edi, %%eax\n\t" 446b2c63698SAlexander Motin "adcl %%esi, %%edx\n\t" 447b2c63698SAlexander Motin "wrmsr\n" 448b2c63698SAlexander Motin : /* No output */ 449b2c63698SAlexander Motin : "D" ((uint32_t)d), "S" ((uint32_t)(d >> 32)) 450b2c63698SAlexander Motin : "ax", "cx", "dx", "cc" 451b2c63698SAlexander Motin ); 452b2c63698SAlexander Motin } 453b2c63698SAlexander Motin 45465e7d70bSJung-uk Kim static int 455e7f1427dSKonstantin Belousov test_tsc(void) 45665e7d70bSJung-uk Kim { 4577bfcb3bbSJim Harris uint64_t *data, *tsc; 458b2c63698SAlexander Motin u_int i, size, adj; 45965e7d70bSJung-uk Kim 460e7f1427dSKonstantin Belousov if ((!smp_tsc && !tsc_is_invariant) || vm_guest) 46165e7d70bSJung-uk Kim return (-100); 46265e7d70bSJung-uk Kim size = (mp_maxid + 1) * 3; 46365e7d70bSJung-uk Kim data = malloc(sizeof(*data) * size * N, M_TEMP, M_WAITOK); 464b2c63698SAlexander Motin adj = 0; 465b2c63698SAlexander Motin retry: 46665e7d70bSJung-uk Kim for (i = 0, tsc = data; i < N; i++, tsc += size) 46765e7d70bSJung-uk Kim smp_rendezvous(tsc_read_0, tsc_read_1, tsc_read_2, tsc); 46865e7d70bSJung-uk Kim smp_tsc = 1; /* XXX */ 46965e7d70bSJung-uk Kim smp_rendezvous(smp_no_rendevous_barrier, comp_smp_tsc, 47065e7d70bSJung-uk Kim smp_no_rendevous_barrier, data); 471b2c63698SAlexander Motin if (!smp_tsc && adj < smp_tsc_adjust) { 472b2c63698SAlexander Motin adj++; 473b2c63698SAlexander Motin smp_rendezvous(smp_no_rendevous_barrier, adj_smp_tsc, 474b2c63698SAlexander Motin smp_no_rendevous_barrier, data); 475b2c63698SAlexander Motin goto retry; 476b2c63698SAlexander Motin } 47765e7d70bSJung-uk Kim free(data, M_TEMP); 47865e7d70bSJung-uk Kim if (bootverbose) 479b2c63698SAlexander Motin printf("SMP: %sed TSC synchronization test%s\n", 480b2c63698SAlexander Motin smp_tsc ? "pass" : "fail", 481b2c63698SAlexander Motin adj > 0 ? " after adjustment" : ""); 48226e6537aSJung-uk Kim if (smp_tsc && tsc_is_invariant) { 48326e6537aSJung-uk Kim switch (cpu_vendor_id) { 48426e6537aSJung-uk Kim case CPU_VENDOR_AMD: 48526e6537aSJung-uk Kim /* 48626e6537aSJung-uk Kim * Starting with Family 15h processors, TSC clock 48726e6537aSJung-uk Kim * source is in the north bridge. Check whether 48826e6537aSJung-uk Kim * we have a single-socket/multi-core platform. 48926e6537aSJung-uk Kim * XXX Need more work for complex cases. 49026e6537aSJung-uk Kim */ 49126e6537aSJung-uk Kim if (CPUID_TO_FAMILY(cpu_id) < 0x15 || 49226e6537aSJung-uk Kim (amd_feature2 & AMDID2_CMP) == 0 || 49326e6537aSJung-uk Kim smp_cpus > (cpu_procinfo2 & AMDID_CMP_CORES) + 1) 49426e6537aSJung-uk Kim break; 49526e6537aSJung-uk Kim return (1000); 49626e6537aSJung-uk Kim case CPU_VENDOR_INTEL: 49726e6537aSJung-uk Kim /* 49826e6537aSJung-uk Kim * XXX Assume Intel platforms have synchronized TSCs. 49926e6537aSJung-uk Kim */ 50026e6537aSJung-uk Kim return (1000); 50126e6537aSJung-uk Kim } 50226e6537aSJung-uk Kim return (800); 50326e6537aSJung-uk Kim } 50426e6537aSJung-uk Kim return (-100); 50565e7d70bSJung-uk Kim } 50665e7d70bSJung-uk Kim 50765e7d70bSJung-uk Kim #undef N 50865e7d70bSJung-uk Kim 509e7f1427dSKonstantin Belousov #else 510e7f1427dSKonstantin Belousov 511e7f1427dSKonstantin Belousov /* 512e7f1427dSKonstantin Belousov * The function is not called, it is provided to avoid linking failure 513e7f1427dSKonstantin Belousov * on uniprocessor kernel. 514e7f1427dSKonstantin Belousov */ 515e7f1427dSKonstantin Belousov static int 516e7f1427dSKonstantin Belousov test_tsc(void) 517e7f1427dSKonstantin Belousov { 518e7f1427dSKonstantin Belousov 519e7f1427dSKonstantin Belousov return (0); 520e7f1427dSKonstantin Belousov } 521e7f1427dSKonstantin Belousov 52265e7d70bSJung-uk Kim #endif /* SMP */ 52365e7d70bSJung-uk Kim 52465e7d70bSJung-uk Kim static void 525dd7d207dSJung-uk Kim init_TSC_tc(void) 526dd7d207dSJung-uk Kim { 52795f2f098SJung-uk Kim uint64_t max_freq; 52895f2f098SJung-uk Kim int shift; 529dd7d207dSJung-uk Kim 53038b8542cSJung-uk Kim if ((cpu_feature & CPUID_TSC) == 0 || tsc_disabled) 531dd7d207dSJung-uk Kim return; 532dd7d207dSJung-uk Kim 533dd7d207dSJung-uk Kim /* 53495f2f098SJung-uk Kim * Limit timecounter frequency to fit in an int and prevent it from 53595f2f098SJung-uk Kim * overflowing too fast. 53695f2f098SJung-uk Kim */ 53795f2f098SJung-uk Kim max_freq = UINT_MAX; 53895f2f098SJung-uk Kim 53995f2f098SJung-uk Kim /* 540dd7d207dSJung-uk Kim * We can not use the TSC if we support APM. Precise timekeeping 541dd7d207dSJung-uk Kim * on an APM'ed machine is at best a fools pursuit, since 542dd7d207dSJung-uk Kim * any and all of the time spent in various SMM code can't 543dd7d207dSJung-uk Kim * be reliably accounted for. Reading the RTC is your only 544dd7d207dSJung-uk Kim * source of reliable time info. The i8254 loses too, of course, 545dd7d207dSJung-uk Kim * but we need to have some kind of time... 546dd7d207dSJung-uk Kim * We don't know at this point whether APM is going to be used 547dd7d207dSJung-uk Kim * or not, nor when it might be activated. Play it safe. 548dd7d207dSJung-uk Kim */ 549dd7d207dSJung-uk Kim if (power_pm_get_type() == POWER_PM_TYPE_APM) { 550dd7d207dSJung-uk Kim tsc_timecounter.tc_quality = -1000; 551dd7d207dSJung-uk Kim if (bootverbose) 552dd7d207dSJung-uk Kim printf("TSC timecounter disabled: APM enabled.\n"); 55365e7d70bSJung-uk Kim goto init; 554dd7d207dSJung-uk Kim } 555dd7d207dSJung-uk Kim 556a49399a9SJung-uk Kim /* 557a49399a9SJung-uk Kim * We cannot use the TSC if it stops incrementing in deep sleep. 558a49399a9SJung-uk Kim * Currently only Intel CPUs are known for this problem unless 559a49399a9SJung-uk Kim * the invariant TSC bit is set. 560a49399a9SJung-uk Kim */ 561a49399a9SJung-uk Kim if (cpu_can_deep_sleep && cpu_vendor_id == CPU_VENDOR_INTEL && 562a49399a9SJung-uk Kim (amd_pminfo & AMDPM_TSC_INVARIANT) == 0) { 563a49399a9SJung-uk Kim tsc_timecounter.tc_quality = -1000; 56408e1b4f4SJung-uk Kim tsc_timecounter.tc_flags |= TC_FLAGS_C3STOP; 565a49399a9SJung-uk Kim if (bootverbose) 566a49399a9SJung-uk Kim printf("TSC timecounter disabled: C3 enabled.\n"); 567a49399a9SJung-uk Kim goto init; 568a49399a9SJung-uk Kim } 569a49399a9SJung-uk Kim 570dd7d207dSJung-uk Kim /* 571e7f1427dSKonstantin Belousov * We can not use the TSC in SMP mode unless the TSCs on all CPUs 572e7f1427dSKonstantin Belousov * are synchronized. If the user is sure that the system has 573e7f1427dSKonstantin Belousov * synchronized TSCs, set kern.timecounter.smp_tsc tunable to a 574e7f1427dSKonstantin Belousov * non-zero value. The TSC seems unreliable in virtualized SMP 5755cf8ac1bSMike Silbersack * environments, so it is set to a negative quality in those cases. 576dd7d207dSJung-uk Kim */ 577e7f1427dSKonstantin Belousov if (mp_ncpus > 1) 578e7f1427dSKonstantin Belousov tsc_timecounter.tc_quality = test_tsc(); 579e7f1427dSKonstantin Belousov else if (tsc_is_invariant) 58026e6537aSJung-uk Kim tsc_timecounter.tc_quality = 1000; 581e7f1427dSKonstantin Belousov max_freq >>= tsc_shift; 58226e6537aSJung-uk Kim 58365e7d70bSJung-uk Kim init: 584e7f1427dSKonstantin Belousov for (shift = 0; shift <= 31 && (tsc_freq >> shift) > max_freq; shift++) 58595f2f098SJung-uk Kim ; 586e7f1427dSKonstantin Belousov if ((cpu_feature & CPUID_SSE2) != 0 && mp_ncpus > 1) { 587814124c3SKonstantin Belousov if (cpu_vendor_id == CPU_VENDOR_AMD) { 588e7f1427dSKonstantin Belousov tsc_timecounter.tc_get_timecount = shift > 0 ? 589e7f1427dSKonstantin Belousov tsc_get_timecount_low_mfence : 590e7f1427dSKonstantin Belousov tsc_get_timecount_mfence; 591814124c3SKonstantin Belousov } else { 592e7f1427dSKonstantin Belousov tsc_timecounter.tc_get_timecount = shift > 0 ? 593e7f1427dSKonstantin Belousov tsc_get_timecount_low_lfence : 594e7f1427dSKonstantin Belousov tsc_get_timecount_lfence; 595814124c3SKonstantin Belousov } 596e7f1427dSKonstantin Belousov } else { 597e7f1427dSKonstantin Belousov tsc_timecounter.tc_get_timecount = shift > 0 ? 598e7f1427dSKonstantin Belousov tsc_get_timecount_low : tsc_get_timecount; 599e7f1427dSKonstantin Belousov } 600e7f1427dSKonstantin Belousov if (shift > 0) { 60195f2f098SJung-uk Kim tsc_timecounter.tc_name = "TSC-low"; 60295f2f098SJung-uk Kim if (bootverbose) 603bc8e4ad2SJung-uk Kim printf("TSC timecounter discards lower %d bit(s)\n", 60495f2f098SJung-uk Kim shift); 60595f2f098SJung-uk Kim } 606bc34c87eSJung-uk Kim if (tsc_freq != 0) { 60795f2f098SJung-uk Kim tsc_timecounter.tc_frequency = tsc_freq >> shift; 60895f2f098SJung-uk Kim tsc_timecounter.tc_priv = (void *)(intptr_t)shift; 609dd7d207dSJung-uk Kim tc_init(&tsc_timecounter); 610dd7d207dSJung-uk Kim } 611dd7d207dSJung-uk Kim } 61265e7d70bSJung-uk Kim SYSINIT(tsc_tc, SI_SUB_SMP, SI_ORDER_ANY, init_TSC_tc, NULL); 613dd7d207dSJung-uk Kim 614dd7d207dSJung-uk Kim /* 615dd7d207dSJung-uk Kim * When cpufreq levels change, find out about the (new) max frequency. We 616dd7d207dSJung-uk Kim * use this to update CPU accounting in case it got a lower estimate at boot. 617dd7d207dSJung-uk Kim */ 618dd7d207dSJung-uk Kim static void 619dd7d207dSJung-uk Kim tsc_levels_changed(void *arg, int unit) 620dd7d207dSJung-uk Kim { 621dd7d207dSJung-uk Kim device_t cf_dev; 622dd7d207dSJung-uk Kim struct cf_level *levels; 623dd7d207dSJung-uk Kim int count, error; 624dd7d207dSJung-uk Kim uint64_t max_freq; 625dd7d207dSJung-uk Kim 626dd7d207dSJung-uk Kim /* Only use values from the first CPU, assuming all are equal. */ 627dd7d207dSJung-uk Kim if (unit != 0) 628dd7d207dSJung-uk Kim return; 629dd7d207dSJung-uk Kim 630dd7d207dSJung-uk Kim /* Find the appropriate cpufreq device instance. */ 631dd7d207dSJung-uk Kim cf_dev = devclass_get_device(devclass_find("cpufreq"), unit); 632dd7d207dSJung-uk Kim if (cf_dev == NULL) { 633dd7d207dSJung-uk Kim printf("tsc_levels_changed() called but no cpufreq device?\n"); 634dd7d207dSJung-uk Kim return; 635dd7d207dSJung-uk Kim } 636dd7d207dSJung-uk Kim 637dd7d207dSJung-uk Kim /* Get settings from the device and find the max frequency. */ 638dd7d207dSJung-uk Kim count = 64; 639dd7d207dSJung-uk Kim levels = malloc(count * sizeof(*levels), M_TEMP, M_NOWAIT); 640dd7d207dSJung-uk Kim if (levels == NULL) 641dd7d207dSJung-uk Kim return; 642dd7d207dSJung-uk Kim error = CPUFREQ_LEVELS(cf_dev, levels, &count); 643dd7d207dSJung-uk Kim if (error == 0 && count != 0) { 644dd7d207dSJung-uk Kim max_freq = (uint64_t)levels[0].total_set.freq * 1000000; 645dd7d207dSJung-uk Kim set_cputicker(rdtsc, max_freq, 1); 646dd7d207dSJung-uk Kim } else 647dd7d207dSJung-uk Kim printf("tsc_levels_changed: no max freq found\n"); 648dd7d207dSJung-uk Kim free(levels, M_TEMP); 649dd7d207dSJung-uk Kim } 650dd7d207dSJung-uk Kim 651dd7d207dSJung-uk Kim /* 652dd7d207dSJung-uk Kim * If the TSC timecounter is in use, veto the pending change. It may be 653dd7d207dSJung-uk Kim * possible in the future to handle a dynamically-changing timecounter rate. 654dd7d207dSJung-uk Kim */ 655dd7d207dSJung-uk Kim static void 656dd7d207dSJung-uk Kim tsc_freq_changing(void *arg, const struct cf_level *level, int *status) 657dd7d207dSJung-uk Kim { 658dd7d207dSJung-uk Kim 659dd7d207dSJung-uk Kim if (*status != 0 || timecounter != &tsc_timecounter) 660dd7d207dSJung-uk Kim return; 661dd7d207dSJung-uk Kim 662dd7d207dSJung-uk Kim printf("timecounter TSC must not be in use when " 663dd7d207dSJung-uk Kim "changing frequencies; change denied\n"); 664dd7d207dSJung-uk Kim *status = EBUSY; 665dd7d207dSJung-uk Kim } 666dd7d207dSJung-uk Kim 667dd7d207dSJung-uk Kim /* Update TSC freq with the value indicated by the caller. */ 668dd7d207dSJung-uk Kim static void 669dd7d207dSJung-uk Kim tsc_freq_changed(void *arg, const struct cf_level *level, int status) 670dd7d207dSJung-uk Kim { 6713453537fSJung-uk Kim uint64_t freq; 672dd7d207dSJung-uk Kim 673dd7d207dSJung-uk Kim /* If there was an error during the transition, don't do anything. */ 67479422085SJung-uk Kim if (tsc_disabled || status != 0) 675dd7d207dSJung-uk Kim return; 676dd7d207dSJung-uk Kim 677dd7d207dSJung-uk Kim /* Total setting for this level gives the new frequency in MHz. */ 6783453537fSJung-uk Kim freq = (uint64_t)level->total_set.freq * 1000000; 6793453537fSJung-uk Kim atomic_store_rel_64(&tsc_freq, freq); 68095f2f098SJung-uk Kim tsc_timecounter.tc_frequency = 68195f2f098SJung-uk Kim freq >> (int)(intptr_t)tsc_timecounter.tc_priv; 682dd7d207dSJung-uk Kim } 683dd7d207dSJung-uk Kim 684dd7d207dSJung-uk Kim static int 685dd7d207dSJung-uk Kim sysctl_machdep_tsc_freq(SYSCTL_HANDLER_ARGS) 686dd7d207dSJung-uk Kim { 687dd7d207dSJung-uk Kim int error; 688dd7d207dSJung-uk Kim uint64_t freq; 689dd7d207dSJung-uk Kim 6903453537fSJung-uk Kim freq = atomic_load_acq_64(&tsc_freq); 6913453537fSJung-uk Kim if (freq == 0) 692dd7d207dSJung-uk Kim return (EOPNOTSUPP); 693cbc134adSMatthew D Fleming error = sysctl_handle_64(oidp, &freq, 0, req); 6947ebbcb21SJung-uk Kim if (error == 0 && req->newptr != NULL) { 6953453537fSJung-uk Kim atomic_store_rel_64(&tsc_freq, freq); 696bc8e4ad2SJung-uk Kim atomic_store_rel_64(&tsc_timecounter.tc_frequency, 697bc8e4ad2SJung-uk Kim freq >> (int)(intptr_t)tsc_timecounter.tc_priv); 6987ebbcb21SJung-uk Kim } 699dd7d207dSJung-uk Kim return (error); 700dd7d207dSJung-uk Kim } 701dd7d207dSJung-uk Kim 702cbc134adSMatthew D Fleming SYSCTL_PROC(_machdep, OID_AUTO, tsc_freq, CTLTYPE_U64 | CTLFLAG_RW, 7035331d61dSJung-uk Kim 0, 0, sysctl_machdep_tsc_freq, "QU", "Time Stamp Counter frequency"); 704dd7d207dSJung-uk Kim 705727c7b2dSJung-uk Kim static u_int 70695f2f098SJung-uk Kim tsc_get_timecount(struct timecounter *tc __unused) 707dd7d207dSJung-uk Kim { 708727c7b2dSJung-uk Kim 709727c7b2dSJung-uk Kim return (rdtsc32()); 710dd7d207dSJung-uk Kim } 71195f2f098SJung-uk Kim 712814124c3SKonstantin Belousov static inline u_int 713bc8e4ad2SJung-uk Kim tsc_get_timecount_low(struct timecounter *tc) 71495f2f098SJung-uk Kim { 7155df88f46SJung-uk Kim uint32_t rv; 71695f2f098SJung-uk Kim 7175df88f46SJung-uk Kim __asm __volatile("rdtsc; shrd %%cl, %%edx, %0" 7185df88f46SJung-uk Kim : "=a" (rv) : "c" ((int)(intptr_t)tc->tc_priv) : "edx"); 7195df88f46SJung-uk Kim return (rv); 72095f2f098SJung-uk Kim } 721aea81038SKonstantin Belousov 722814124c3SKonstantin Belousov static u_int 723814124c3SKonstantin Belousov tsc_get_timecount_lfence(struct timecounter *tc __unused) 724814124c3SKonstantin Belousov { 725814124c3SKonstantin Belousov 726814124c3SKonstantin Belousov lfence(); 727814124c3SKonstantin Belousov return (rdtsc32()); 728814124c3SKonstantin Belousov } 729814124c3SKonstantin Belousov 730814124c3SKonstantin Belousov static u_int 731814124c3SKonstantin Belousov tsc_get_timecount_low_lfence(struct timecounter *tc) 732814124c3SKonstantin Belousov { 733814124c3SKonstantin Belousov 734814124c3SKonstantin Belousov lfence(); 735814124c3SKonstantin Belousov return (tsc_get_timecount_low(tc)); 736814124c3SKonstantin Belousov } 737814124c3SKonstantin Belousov 738814124c3SKonstantin Belousov static u_int 739814124c3SKonstantin Belousov tsc_get_timecount_mfence(struct timecounter *tc __unused) 740814124c3SKonstantin Belousov { 741814124c3SKonstantin Belousov 742814124c3SKonstantin Belousov mfence(); 743814124c3SKonstantin Belousov return (rdtsc32()); 744814124c3SKonstantin Belousov } 745814124c3SKonstantin Belousov 746814124c3SKonstantin Belousov static u_int 747814124c3SKonstantin Belousov tsc_get_timecount_low_mfence(struct timecounter *tc) 748814124c3SKonstantin Belousov { 749814124c3SKonstantin Belousov 750814124c3SKonstantin Belousov mfence(); 751814124c3SKonstantin Belousov return (tsc_get_timecount_low(tc)); 752814124c3SKonstantin Belousov } 753814124c3SKonstantin Belousov 754aea81038SKonstantin Belousov uint32_t 755aea81038SKonstantin Belousov cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th) 756aea81038SKonstantin Belousov { 757aea81038SKonstantin Belousov 758aea81038SKonstantin Belousov vdso_th->th_x86_shift = (int)(intptr_t)timecounter->tc_priv; 759aea81038SKonstantin Belousov bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); 760aea81038SKonstantin Belousov return (timecounter == &tsc_timecounter); 761aea81038SKonstantin Belousov } 762aea81038SKonstantin Belousov 763aea81038SKonstantin Belousov #ifdef COMPAT_FREEBSD32 764aea81038SKonstantin Belousov uint32_t 765aea81038SKonstantin Belousov cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32) 766aea81038SKonstantin Belousov { 767aea81038SKonstantin Belousov 768aea81038SKonstantin Belousov vdso_th32->th_x86_shift = (int)(intptr_t)timecounter->tc_priv; 769aea81038SKonstantin Belousov bzero(vdso_th32->th_res, sizeof(vdso_th32->th_res)); 770aea81038SKonstantin Belousov return (timecounter == &tsc_timecounter); 771aea81038SKonstantin Belousov } 772aea81038SKonstantin Belousov #endif 773