19931033bSDmitry Chagin/*- 24d846d26SWarner Losh * SPDX-License-Identifier: BSD-2-Clause 39931033bSDmitry Chagin * 49931033bSDmitry Chagin * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> 59931033bSDmitry Chagin * Copyright (c) 2016, 2017, 2019 The FreeBSD Foundation 69931033bSDmitry Chagin * Copyright (c) 2021 Dmitry Chagin <dchagin@FreeBSD.org> 79931033bSDmitry Chagin * 89931033bSDmitry Chagin * Portions of this software were developed by Konstantin Belousov 99931033bSDmitry Chagin * under sponsorship from the FreeBSD Foundation. 109931033bSDmitry Chagin * 119931033bSDmitry Chagin * Redistribution and use in source and binary forms, with or without 129931033bSDmitry Chagin * modification, are permitted provided that the following conditions 139931033bSDmitry Chagin * are met: 149931033bSDmitry Chagin * 1. Redistributions of source code must retain the above copyright 159931033bSDmitry Chagin * notice, this list of conditions and the following disclaimer. 169931033bSDmitry Chagin * 2. Redistributions in binary form must reproduce the above copyright 179931033bSDmitry Chagin * notice, this list of conditions and the following disclaimer in the 189931033bSDmitry Chagin * documentation and/or other materials provided with the distribution. 199931033bSDmitry Chagin * 209931033bSDmitry Chagin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 219931033bSDmitry Chagin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 229931033bSDmitry Chagin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 239931033bSDmitry Chagin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 249931033bSDmitry Chagin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 259931033bSDmitry Chagin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 269931033bSDmitry Chagin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 279931033bSDmitry Chagin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 289931033bSDmitry Chagin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 299931033bSDmitry Chagin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 309931033bSDmitry Chagin * SUCH DAMAGE. 319931033bSDmitry Chagin */ 329931033bSDmitry Chagin 339931033bSDmitry Chaginstatic inline u_int 349931033bSDmitry Chaginrdtsc_low(const struct vdso_timehands *th) 359931033bSDmitry Chagin{ 369931033bSDmitry Chagin u_int rv; 379931033bSDmitry Chagin 389931033bSDmitry Chagin __asm __volatile("rdtsc; shrd %%cl, %%edx, %0" 399931033bSDmitry Chagin : "=a" (rv) : "c" (th->th_x86_shift) : "edx"); 409931033bSDmitry Chagin return (rv); 419931033bSDmitry Chagin} 429931033bSDmitry Chagin 439931033bSDmitry Chaginstatic inline u_int 449931033bSDmitry Chaginrdtscp_low(const struct vdso_timehands *th) 459931033bSDmitry Chagin{ 469931033bSDmitry Chagin u_int rv; 479931033bSDmitry Chagin 489931033bSDmitry Chagin __asm __volatile("rdtscp; movl %%edi,%%ecx; shrd %%cl, %%edx, %0" 499931033bSDmitry Chagin : "=a" (rv) : "D" (th->th_x86_shift) : "ecx", "edx"); 509931033bSDmitry Chagin return (rv); 519931033bSDmitry Chagin} 529931033bSDmitry Chagin 539931033bSDmitry Chaginstatic u_int 549931033bSDmitry Chaginrdtsc_low_mb_lfence(const struct vdso_timehands *th) 559931033bSDmitry Chagin{ 569931033bSDmitry Chagin lfence(); 579931033bSDmitry Chagin return (rdtsc_low(th)); 589931033bSDmitry Chagin} 599931033bSDmitry Chagin 609931033bSDmitry Chaginstatic u_int 619931033bSDmitry Chaginrdtsc_low_mb_mfence(const struct vdso_timehands *th) 629931033bSDmitry Chagin{ 639931033bSDmitry Chagin mfence(); 649931033bSDmitry Chagin return (rdtsc_low(th)); 659931033bSDmitry Chagin} 669931033bSDmitry Chagin 679931033bSDmitry Chaginstatic u_int 689931033bSDmitry Chaginrdtsc_low_mb_none(const struct vdso_timehands *th) 699931033bSDmitry Chagin{ 709931033bSDmitry Chagin return (rdtsc_low(th)); 719931033bSDmitry Chagin} 729931033bSDmitry Chagin 739931033bSDmitry Chaginstatic u_int 749931033bSDmitry Chaginrdtsc32_mb_lfence(void) 759931033bSDmitry Chagin{ 769931033bSDmitry Chagin lfence(); 779931033bSDmitry Chagin return (rdtsc32()); 789931033bSDmitry Chagin} 799931033bSDmitry Chagin 809931033bSDmitry Chaginstatic u_int 819931033bSDmitry Chaginrdtsc32_mb_mfence(void) 829931033bSDmitry Chagin{ 839931033bSDmitry Chagin mfence(); 849931033bSDmitry Chagin return (rdtsc32()); 859931033bSDmitry Chagin} 869931033bSDmitry Chagin 879931033bSDmitry Chaginstatic u_int 889931033bSDmitry Chaginrdtsc32_mb_none(void) 899931033bSDmitry Chagin{ 909931033bSDmitry Chagin return (rdtsc32()); 919931033bSDmitry Chagin} 929931033bSDmitry Chagin 939931033bSDmitry Chaginstatic u_int 949931033bSDmitry Chaginrdtscp32_(void) 959931033bSDmitry Chagin{ 969931033bSDmitry Chagin return (rdtscp32()); 979931033bSDmitry Chagin} 989931033bSDmitry Chagin 999931033bSDmitry Chaginstruct tsc_selector_tag { 1009931033bSDmitry Chagin u_int (*ts_rdtsc32)(void); 1019931033bSDmitry Chagin u_int (*ts_rdtsc_low)(const struct vdso_timehands *); 1029931033bSDmitry Chagin}; 1039931033bSDmitry Chagin 1049931033bSDmitry Chaginstatic const struct tsc_selector_tag tsc_selector[] = { 1059931033bSDmitry Chagin [0] = { /* Intel, LFENCE */ 1069931033bSDmitry Chagin .ts_rdtsc32 = rdtsc32_mb_lfence, 1079931033bSDmitry Chagin .ts_rdtsc_low = rdtsc_low_mb_lfence, 1089931033bSDmitry Chagin }, 1099931033bSDmitry Chagin [1] = { /* AMD, MFENCE */ 1109931033bSDmitry Chagin .ts_rdtsc32 = rdtsc32_mb_mfence, 1119931033bSDmitry Chagin .ts_rdtsc_low = rdtsc_low_mb_mfence, 1129931033bSDmitry Chagin }, 1139931033bSDmitry Chagin [2] = { /* No SSE2 */ 1149931033bSDmitry Chagin .ts_rdtsc32 = rdtsc32_mb_none, 1159931033bSDmitry Chagin .ts_rdtsc_low = rdtsc_low_mb_none, 1169931033bSDmitry Chagin }, 1179931033bSDmitry Chagin [3] = { /* RDTSCP */ 1189931033bSDmitry Chagin .ts_rdtsc32 = rdtscp32_, 1199931033bSDmitry Chagin .ts_rdtsc_low = rdtscp_low, 1209931033bSDmitry Chagin }, 1219931033bSDmitry Chagin}; 1229931033bSDmitry Chagin 1239931033bSDmitry Chaginstatic u_int 1249931033bSDmitry Chagin__vdso_gettc_rdtsc_low(const struct vdso_timehands *th) 1259931033bSDmitry Chagin{ 1269931033bSDmitry Chagin 1279931033bSDmitry Chagin return (tsc_selector[kern_tsc_selector].ts_rdtsc_low(th)); 1289931033bSDmitry Chagin} 1299931033bSDmitry Chagin 1309931033bSDmitry Chaginstatic u_int 1319931033bSDmitry Chagin__vdso_gettc_rdtsc32(void) 1329931033bSDmitry Chagin{ 1339931033bSDmitry Chagin 1349931033bSDmitry Chagin return (tsc_selector[kern_tsc_selector].ts_rdtsc32()); 1359931033bSDmitry Chagin} 1369931033bSDmitry Chagin 1379931033bSDmitry Chaginint 1389931033bSDmitry Chagin__vdso_gettc(const struct vdso_timehands *th, u_int *tc) 1399931033bSDmitry Chagin{ 1409931033bSDmitry Chagin 1419931033bSDmitry Chagin switch (th->th_algo) { 1429931033bSDmitry Chagin case VDSO_TH_ALGO_X86_TSC: 1439931033bSDmitry Chagin *tc = th->th_x86_shift > 0 ? __vdso_gettc_rdtsc_low(th) : 1449931033bSDmitry Chagin __vdso_gettc_rdtsc32(); 1459931033bSDmitry Chagin return (0); 1469931033bSDmitry Chagin case VDSO_TH_ALGO_X86_HPET: 1479931033bSDmitry Chagin /* TODO */ 1489931033bSDmitry Chagin default: 1499931033bSDmitry Chagin return (ENOSYS); 1509931033bSDmitry Chagin } 1519931033bSDmitry Chagin} 152