14577cf37SConrad Meyer /*- 24d846d26SWarner Losh * SPDX-License-Identifier: BSD-2-Clause 34577cf37SConrad Meyer * 44577cf37SConrad Meyer * Copyright (c) 2018 Intel Corporation 54577cf37SConrad Meyer * 64577cf37SConrad Meyer * Redistribution and use in source and binary forms, with or without 74577cf37SConrad Meyer * modification, are permitted providing that the following conditions 84577cf37SConrad Meyer * are met: 94577cf37SConrad Meyer * 1. Redistributions of source code must retain the above copyright 104577cf37SConrad Meyer * notice, this list of conditions and the following disclaimer. 114577cf37SConrad Meyer * 2. Redistributions in binary form must reproduce the above copyright 124577cf37SConrad Meyer * notice, this list of conditions and the following disclaimer in the 134577cf37SConrad Meyer * documentation and/or other materials provided with the distribution. 144577cf37SConrad Meyer * 154577cf37SConrad Meyer * THIS SOFTWARE IS PROVIDED BY THE AUTHOR``AS IS'' AND ANY EXPRESS OR 164577cf37SConrad Meyer * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 174577cf37SConrad Meyer * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 184577cf37SConrad Meyer * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 194577cf37SConrad Meyer * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 204577cf37SConrad Meyer * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 214577cf37SConrad Meyer * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 224577cf37SConrad Meyer * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 234577cf37SConrad Meyer * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 244577cf37SConrad Meyer * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 254577cf37SConrad Meyer * POSSIBILITY OF SUCH DAMAGE. 264577cf37SConrad Meyer */ 274577cf37SConrad Meyer 284577cf37SConrad Meyer #include <sys/cdefs.h> 294577cf37SConrad Meyer __FBSDID("$FreeBSD$"); 304577cf37SConrad Meyer 314577cf37SConrad Meyer #include <sys/types.h> 32df38ada2SBjoern A. Zeeb #include <sys/param.h> 334577cf37SConrad Meyer #include <sys/sbuf.h> 344577cf37SConrad Meyer #include <sys/module.h> 354577cf37SConrad Meyer #include <sys/systm.h> 364577cf37SConrad Meyer #include <sys/errno.h> 374577cf37SConrad Meyer #include <sys/param.h> 384577cf37SConrad Meyer #include <sys/kernel.h> 394577cf37SConrad Meyer #include <sys/bus.h> 404577cf37SConrad Meyer #include <sys/cpu.h> 414577cf37SConrad Meyer #include <sys/smp.h> 424577cf37SConrad Meyer #include <sys/proc.h> 434577cf37SConrad Meyer #include <sys/sched.h> 444577cf37SConrad Meyer 454577cf37SConrad Meyer #include <machine/cpu.h> 464577cf37SConrad Meyer #include <machine/md_var.h> 474577cf37SConrad Meyer #include <machine/cputypes.h> 484577cf37SConrad Meyer #include <machine/specialreg.h> 494577cf37SConrad Meyer 504577cf37SConrad Meyer #include <contrib/dev/acpica/include/acpi.h> 514577cf37SConrad Meyer 524577cf37SConrad Meyer #include <dev/acpica/acpivar.h> 534577cf37SConrad Meyer 544577cf37SConrad Meyer #include <x86/cpufreq/hwpstate_intel_internal.h> 554577cf37SConrad Meyer 564577cf37SConrad Meyer #include "acpi_if.h" 574577cf37SConrad Meyer #include "cpufreq_if.h" 584577cf37SConrad Meyer 594577cf37SConrad Meyer extern uint64_t tsc_freq; 604577cf37SConrad Meyer 614577cf37SConrad Meyer static int intel_hwpstate_probe(device_t dev); 624577cf37SConrad Meyer static int intel_hwpstate_attach(device_t dev); 634577cf37SConrad Meyer static int intel_hwpstate_detach(device_t dev); 644577cf37SConrad Meyer static int intel_hwpstate_suspend(device_t dev); 654577cf37SConrad Meyer static int intel_hwpstate_resume(device_t dev); 664577cf37SConrad Meyer 674577cf37SConrad Meyer static int intel_hwpstate_get(device_t dev, struct cf_setting *cf); 684577cf37SConrad Meyer static int intel_hwpstate_type(device_t dev, int *type); 694577cf37SConrad Meyer 704577cf37SConrad Meyer static device_method_t intel_hwpstate_methods[] = { 714577cf37SConrad Meyer /* Device interface */ 724577cf37SConrad Meyer DEVMETHOD(device_identify, intel_hwpstate_identify), 734577cf37SConrad Meyer DEVMETHOD(device_probe, intel_hwpstate_probe), 744577cf37SConrad Meyer DEVMETHOD(device_attach, intel_hwpstate_attach), 754577cf37SConrad Meyer DEVMETHOD(device_detach, intel_hwpstate_detach), 764577cf37SConrad Meyer DEVMETHOD(device_suspend, intel_hwpstate_suspend), 774577cf37SConrad Meyer DEVMETHOD(device_resume, intel_hwpstate_resume), 784577cf37SConrad Meyer 794577cf37SConrad Meyer /* cpufreq interface */ 804577cf37SConrad Meyer DEVMETHOD(cpufreq_drv_get, intel_hwpstate_get), 814577cf37SConrad Meyer DEVMETHOD(cpufreq_drv_type, intel_hwpstate_type), 824577cf37SConrad Meyer 834577cf37SConrad Meyer DEVMETHOD_END 844577cf37SConrad Meyer }; 854577cf37SConrad Meyer 864577cf37SConrad Meyer struct hwp_softc { 874577cf37SConrad Meyer device_t dev; 884577cf37SConrad Meyer bool hwp_notifications; 894577cf37SConrad Meyer bool hwp_activity_window; 904577cf37SConrad Meyer bool hwp_pref_ctrl; 914577cf37SConrad Meyer bool hwp_pkg_ctrl; 92cd4e43b2SConrad Meyer bool hwp_pkg_ctrl_en; 93556a1a0bSConrad Meyer bool hwp_perf_bias; 94e656fa70SConrad Meyer bool hwp_perf_bias_cached; 954577cf37SConrad Meyer 96e656fa70SConrad Meyer uint64_t req; /* Cached copy of HWP_REQUEST */ 97e656fa70SConrad Meyer uint64_t hwp_energy_perf_bias; /* Cache PERF_BIAS */ 984577cf37SConrad Meyer 994577cf37SConrad Meyer uint8_t high; 1004577cf37SConrad Meyer uint8_t guaranteed; 1014577cf37SConrad Meyer uint8_t efficient; 1024577cf37SConrad Meyer uint8_t low; 1034577cf37SConrad Meyer }; 1044577cf37SConrad Meyer 1054577cf37SConrad Meyer static driver_t hwpstate_intel_driver = { 1064577cf37SConrad Meyer "hwpstate_intel", 1074577cf37SConrad Meyer intel_hwpstate_methods, 1084577cf37SConrad Meyer sizeof(struct hwp_softc), 1094577cf37SConrad Meyer }; 1104577cf37SConrad Meyer 111b3407dccSJohn Baldwin DRIVER_MODULE(hwpstate_intel, cpu, hwpstate_intel_driver, NULL, NULL); 11208a220ddSConrad Meyer MODULE_VERSION(hwpstate_intel, 1); 1134577cf37SConrad Meyer 114cd4e43b2SConrad Meyer static bool hwpstate_pkg_ctrl_enable = true; 115cd4e43b2SConrad Meyer SYSCTL_BOOL(_machdep, OID_AUTO, hwpstate_pkg_ctrl, CTLFLAG_RDTUN, 116cd4e43b2SConrad Meyer &hwpstate_pkg_ctrl_enable, 0, 117cd4e43b2SConrad Meyer "Set 1 (default) to enable package-level control, 0 to disable"); 118cd4e43b2SConrad Meyer 1194577cf37SConrad Meyer static int 1204577cf37SConrad Meyer intel_hwp_dump_sysctl_handler(SYSCTL_HANDLER_ARGS) 1214577cf37SConrad Meyer { 1224577cf37SConrad Meyer device_t dev; 1234577cf37SConrad Meyer struct pcpu *pc; 1244577cf37SConrad Meyer struct sbuf *sb; 1254577cf37SConrad Meyer struct hwp_softc *sc; 1264577cf37SConrad Meyer uint64_t data, data2; 1274577cf37SConrad Meyer int ret; 1284577cf37SConrad Meyer 1294577cf37SConrad Meyer sc = (struct hwp_softc *)arg1; 1304577cf37SConrad Meyer dev = sc->dev; 1314577cf37SConrad Meyer 1324577cf37SConrad Meyer pc = cpu_get_pcpu(dev); 1334577cf37SConrad Meyer if (pc == NULL) 1344577cf37SConrad Meyer return (ENXIO); 1354577cf37SConrad Meyer 1364577cf37SConrad Meyer sb = sbuf_new(NULL, NULL, 1024, SBUF_FIXEDLEN | SBUF_INCLUDENUL); 1374577cf37SConrad Meyer sbuf_putc(sb, '\n'); 1384577cf37SConrad Meyer thread_lock(curthread); 1394577cf37SConrad Meyer sched_bind(curthread, pc->pc_cpuid); 1404577cf37SConrad Meyer thread_unlock(curthread); 1414577cf37SConrad Meyer 1424577cf37SConrad Meyer rdmsr_safe(MSR_IA32_PM_ENABLE, &data); 1434577cf37SConrad Meyer sbuf_printf(sb, "CPU%d: HWP %sabled\n", pc->pc_cpuid, 1444577cf37SConrad Meyer ((data & 1) ? "En" : "Dis")); 1454577cf37SConrad Meyer 1464577cf37SConrad Meyer if (data == 0) { 1474577cf37SConrad Meyer ret = 0; 1484577cf37SConrad Meyer goto out; 1494577cf37SConrad Meyer } 1504577cf37SConrad Meyer 1514577cf37SConrad Meyer rdmsr_safe(MSR_IA32_HWP_CAPABILITIES, &data); 152ca9fb12aSCy Schubert sbuf_printf(sb, "\tHighest Performance: %03ju\n", data & 0xff); 153ca9fb12aSCy Schubert sbuf_printf(sb, "\tGuaranteed Performance: %03ju\n", (data >> 8) & 0xff); 154ca9fb12aSCy Schubert sbuf_printf(sb, "\tEfficient Performance: %03ju\n", (data >> 16) & 0xff); 155ca9fb12aSCy Schubert sbuf_printf(sb, "\tLowest Performance: %03ju\n", (data >> 24) & 0xff); 1564577cf37SConrad Meyer 1574577cf37SConrad Meyer rdmsr_safe(MSR_IA32_HWP_REQUEST, &data); 15807a65f9dSConrad Meyer data2 = 0; 15907a65f9dSConrad Meyer if (sc->hwp_pkg_ctrl && (data & IA32_HWP_REQUEST_PACKAGE_CONTROL)) 1604577cf37SConrad Meyer rdmsr_safe(MSR_IA32_HWP_REQUEST_PKG, &data2); 1614577cf37SConrad Meyer 1624577cf37SConrad Meyer sbuf_putc(sb, '\n'); 1634577cf37SConrad Meyer 1644577cf37SConrad Meyer #define pkg_print(x, name, offset) do { \ 1654577cf37SConrad Meyer if (!sc->hwp_pkg_ctrl || (data & x) != 0) \ 16607a65f9dSConrad Meyer sbuf_printf(sb, "\t%s: %03u\n", name, \ 16707a65f9dSConrad Meyer (unsigned)(data >> offset) & 0xff); \ 1684577cf37SConrad Meyer else \ 16907a65f9dSConrad Meyer sbuf_printf(sb, "\t%s: %03u\n", name, \ 17007a65f9dSConrad Meyer (unsigned)(data2 >> offset) & 0xff); \ 1714577cf37SConrad Meyer } while (0) 1724577cf37SConrad Meyer 1734577cf37SConrad Meyer pkg_print(IA32_HWP_REQUEST_EPP_VALID, 1744577cf37SConrad Meyer "Requested Efficiency Performance Preference", 24); 1754577cf37SConrad Meyer pkg_print(IA32_HWP_REQUEST_DESIRED_VALID, 1764577cf37SConrad Meyer "Requested Desired Performance", 16); 1774577cf37SConrad Meyer pkg_print(IA32_HWP_REQUEST_MAXIMUM_VALID, 1784577cf37SConrad Meyer "Requested Maximum Performance", 8); 1794577cf37SConrad Meyer pkg_print(IA32_HWP_REQUEST_MINIMUM_VALID, 1804577cf37SConrad Meyer "Requested Minimum Performance", 0); 1814577cf37SConrad Meyer #undef pkg_print 1824577cf37SConrad Meyer 1834577cf37SConrad Meyer sbuf_putc(sb, '\n'); 1844577cf37SConrad Meyer 1854577cf37SConrad Meyer out: 1864577cf37SConrad Meyer thread_lock(curthread); 1874577cf37SConrad Meyer sched_unbind(curthread); 1884577cf37SConrad Meyer thread_unlock(curthread); 1894577cf37SConrad Meyer 1904577cf37SConrad Meyer ret = sbuf_finish(sb); 1914577cf37SConrad Meyer if (ret == 0) 1924577cf37SConrad Meyer ret = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb)); 1934577cf37SConrad Meyer sbuf_delete(sb); 1944577cf37SConrad Meyer 1954577cf37SConrad Meyer return (ret); 1964577cf37SConrad Meyer } 1974577cf37SConrad Meyer 1984577cf37SConrad Meyer static inline int 1994577cf37SConrad Meyer percent_to_raw(int x) 2004577cf37SConrad Meyer { 2014577cf37SConrad Meyer 2024577cf37SConrad Meyer MPASS(x <= 100 && x >= 0); 2034577cf37SConrad Meyer return (0xff * x / 100); 2044577cf37SConrad Meyer } 2054577cf37SConrad Meyer 2064577cf37SConrad Meyer /* 2074577cf37SConrad Meyer * Given x * 10 in [0, 1000], round to the integer nearest x. 2084577cf37SConrad Meyer * 2094577cf37SConrad Meyer * This allows round-tripping nice human readable numbers through this 2104577cf37SConrad Meyer * interface. Otherwise, user-provided percentages such as 25, 50, 75 get 2114577cf37SConrad Meyer * rounded down to 24, 49, and 74, which is a bit ugly. 2124577cf37SConrad Meyer */ 2134577cf37SConrad Meyer static inline int 2144577cf37SConrad Meyer round10(int xtimes10) 2154577cf37SConrad Meyer { 2164577cf37SConrad Meyer return ((xtimes10 + 5) / 10); 2174577cf37SConrad Meyer } 2184577cf37SConrad Meyer 2194577cf37SConrad Meyer static inline int 2204577cf37SConrad Meyer raw_to_percent(int x) 2214577cf37SConrad Meyer { 2224577cf37SConrad Meyer MPASS(x <= 0xff && x >= 0); 2234577cf37SConrad Meyer return (round10(x * 1000 / 0xff)); 2244577cf37SConrad Meyer } 2254577cf37SConrad Meyer 226556a1a0bSConrad Meyer /* Range of MSR_IA32_ENERGY_PERF_BIAS is more limited: 0-0xf. */ 227556a1a0bSConrad Meyer static inline int 228556a1a0bSConrad Meyer percent_to_raw_perf_bias(int x) 229556a1a0bSConrad Meyer { 230556a1a0bSConrad Meyer /* 231556a1a0bSConrad Meyer * Round up so that raw values present as nice round human numbers and 232556a1a0bSConrad Meyer * also round-trip to the same raw value. 233556a1a0bSConrad Meyer */ 234556a1a0bSConrad Meyer MPASS(x <= 100 && x >= 0); 235556a1a0bSConrad Meyer return (((0xf * x) + 50) / 100); 236556a1a0bSConrad Meyer } 237556a1a0bSConrad Meyer 238556a1a0bSConrad Meyer static inline int 239556a1a0bSConrad Meyer raw_to_percent_perf_bias(int x) 240556a1a0bSConrad Meyer { 241556a1a0bSConrad Meyer /* Rounding to nice human numbers despite a step interval of 6.67%. */ 242556a1a0bSConrad Meyer MPASS(x <= 0xf && x >= 0); 243556a1a0bSConrad Meyer return (((x * 20) / 0xf) * 5); 244556a1a0bSConrad Meyer } 245556a1a0bSConrad Meyer 2464577cf37SConrad Meyer static int 2474577cf37SConrad Meyer sysctl_epp_select(SYSCTL_HANDLER_ARGS) 2484577cf37SConrad Meyer { 249b80d476cSConrad Meyer struct hwp_softc *sc; 2504577cf37SConrad Meyer device_t dev; 2514577cf37SConrad Meyer struct pcpu *pc; 252e656fa70SConrad Meyer uint64_t epb; 2534577cf37SConrad Meyer uint32_t val; 2544577cf37SConrad Meyer int ret; 2554577cf37SConrad Meyer 2564577cf37SConrad Meyer dev = oidp->oid_arg1; 257b80d476cSConrad Meyer sc = device_get_softc(dev); 258556a1a0bSConrad Meyer if (!sc->hwp_pref_ctrl && !sc->hwp_perf_bias) 259b80d476cSConrad Meyer return (ENODEV); 260b80d476cSConrad Meyer 2614577cf37SConrad Meyer pc = cpu_get_pcpu(dev); 2624577cf37SConrad Meyer if (pc == NULL) 2634577cf37SConrad Meyer return (ENXIO); 2644577cf37SConrad Meyer 2654577cf37SConrad Meyer thread_lock(curthread); 2664577cf37SConrad Meyer sched_bind(curthread, pc->pc_cpuid); 2674577cf37SConrad Meyer thread_unlock(curthread); 2684577cf37SConrad Meyer 269556a1a0bSConrad Meyer if (sc->hwp_pref_ctrl) { 270e656fa70SConrad Meyer val = (sc->req & IA32_HWP_REQUEST_ENERGY_PERFORMANCE_PREFERENCE) >> 24; 2714577cf37SConrad Meyer val = raw_to_percent(val); 272556a1a0bSConrad Meyer } else { 273556a1a0bSConrad Meyer /* 274556a1a0bSConrad Meyer * If cpuid indicates EPP is not supported, the HWP controller 275556a1a0bSConrad Meyer * uses MSR_IA32_ENERGY_PERF_BIAS instead (Intel SDM §14.4.4). 276556a1a0bSConrad Meyer * This register is per-core (but not HT). 277556a1a0bSConrad Meyer */ 278e656fa70SConrad Meyer if (!sc->hwp_perf_bias_cached) { 279e656fa70SConrad Meyer ret = rdmsr_safe(MSR_IA32_ENERGY_PERF_BIAS, &epb); 280556a1a0bSConrad Meyer if (ret) 281556a1a0bSConrad Meyer goto out; 282e656fa70SConrad Meyer sc->hwp_energy_perf_bias = epb; 283e656fa70SConrad Meyer sc->hwp_perf_bias_cached = true; 284e656fa70SConrad Meyer } 285e656fa70SConrad Meyer val = sc->hwp_energy_perf_bias & 286e656fa70SConrad Meyer IA32_ENERGY_PERF_BIAS_POLICY_HINT_MASK; 287556a1a0bSConrad Meyer val = raw_to_percent_perf_bias(val); 288556a1a0bSConrad Meyer } 2894577cf37SConrad Meyer 2904577cf37SConrad Meyer MPASS(val >= 0 && val <= 100); 2914577cf37SConrad Meyer 2924577cf37SConrad Meyer ret = sysctl_handle_int(oidp, &val, 0, req); 2934577cf37SConrad Meyer if (ret || req->newptr == NULL) 2944577cf37SConrad Meyer goto out; 2954577cf37SConrad Meyer 2964577cf37SConrad Meyer if (val > 100) { 2974577cf37SConrad Meyer ret = EINVAL; 2984577cf37SConrad Meyer goto out; 2994577cf37SConrad Meyer } 3004577cf37SConrad Meyer 301556a1a0bSConrad Meyer if (sc->hwp_pref_ctrl) { 3024577cf37SConrad Meyer val = percent_to_raw(val); 3034577cf37SConrad Meyer 304e656fa70SConrad Meyer sc->req = 305e656fa70SConrad Meyer ((sc->req & ~IA32_HWP_REQUEST_ENERGY_PERFORMANCE_PREFERENCE) 306e656fa70SConrad Meyer | (val << 24u)); 3074577cf37SConrad Meyer 308cd4e43b2SConrad Meyer if (sc->hwp_pkg_ctrl_en) 309e656fa70SConrad Meyer ret = wrmsr_safe(MSR_IA32_HWP_REQUEST_PKG, sc->req); 310cd4e43b2SConrad Meyer else 311e656fa70SConrad Meyer ret = wrmsr_safe(MSR_IA32_HWP_REQUEST, sc->req); 312556a1a0bSConrad Meyer } else { 313e656fa70SConrad Meyer val = percent_to_raw_perf_bias(val); 314e656fa70SConrad Meyer MPASS((val & ~IA32_ENERGY_PERF_BIAS_POLICY_HINT_MASK) == 0); 315e656fa70SConrad Meyer 316e656fa70SConrad Meyer sc->hwp_energy_perf_bias = 317e656fa70SConrad Meyer ((sc->hwp_energy_perf_bias & 318e656fa70SConrad Meyer ~IA32_ENERGY_PERF_BIAS_POLICY_HINT_MASK) | val); 319e656fa70SConrad Meyer ret = wrmsr_safe(MSR_IA32_ENERGY_PERF_BIAS, 320e656fa70SConrad Meyer sc->hwp_energy_perf_bias); 321556a1a0bSConrad Meyer } 3224577cf37SConrad Meyer 3234577cf37SConrad Meyer out: 3244577cf37SConrad Meyer thread_lock(curthread); 3254577cf37SConrad Meyer sched_unbind(curthread); 3264577cf37SConrad Meyer thread_unlock(curthread); 3274577cf37SConrad Meyer 3284577cf37SConrad Meyer return (ret); 3294577cf37SConrad Meyer } 3304577cf37SConrad Meyer 3314577cf37SConrad Meyer void 3324577cf37SConrad Meyer intel_hwpstate_identify(driver_t *driver, device_t parent) 3334577cf37SConrad Meyer { 3344577cf37SConrad Meyer if (device_find_child(parent, "hwpstate_intel", -1) != NULL) 3354577cf37SConrad Meyer return; 3364577cf37SConrad Meyer 3374577cf37SConrad Meyer if (cpu_vendor_id != CPU_VENDOR_INTEL) 3384577cf37SConrad Meyer return; 3394577cf37SConrad Meyer 3404577cf37SConrad Meyer if (resource_disabled("hwpstate_intel", 0)) 3414577cf37SConrad Meyer return; 3424577cf37SConrad Meyer 3434577cf37SConrad Meyer /* 3444577cf37SConrad Meyer * Intel SDM 14.4.1 (HWP Programming Interfaces): 3454577cf37SConrad Meyer * Availability of HWP baseline resource and capability, 3464577cf37SConrad Meyer * CPUID.06H:EAX[bit 7]: If this bit is set, HWP provides several new 3474577cf37SConrad Meyer * architectural MSRs: IA32_PM_ENABLE, IA32_HWP_CAPABILITIES, 3484577cf37SConrad Meyer * IA32_HWP_REQUEST, IA32_HWP_STATUS. 3494577cf37SConrad Meyer */ 350f591c3c8SConrad Meyer if ((cpu_power_eax & CPUTPM1_HWP) == 0) 3514577cf37SConrad Meyer return; 3524577cf37SConrad Meyer 353d3a8f98aSAlexander Motin if (BUS_ADD_CHILD(parent, 10, "hwpstate_intel", device_get_unit(parent)) 354d3a8f98aSAlexander Motin == NULL) 355d3a8f98aSAlexander Motin device_printf(parent, "hwpstate_intel: add child failed\n"); 3564577cf37SConrad Meyer } 3574577cf37SConrad Meyer 3584577cf37SConrad Meyer static int 3594577cf37SConrad Meyer intel_hwpstate_probe(device_t dev) 3604577cf37SConrad Meyer { 3614577cf37SConrad Meyer 3624577cf37SConrad Meyer device_set_desc(dev, "Intel Speed Shift"); 3634577cf37SConrad Meyer return (BUS_PROBE_NOWILDCARD); 3644577cf37SConrad Meyer } 3654577cf37SConrad Meyer 3664577cf37SConrad Meyer static int 3674577cf37SConrad Meyer set_autonomous_hwp(struct hwp_softc *sc) 3684577cf37SConrad Meyer { 3694577cf37SConrad Meyer struct pcpu *pc; 3704577cf37SConrad Meyer device_t dev; 3714577cf37SConrad Meyer uint64_t caps; 3724577cf37SConrad Meyer int ret; 3734577cf37SConrad Meyer 3744577cf37SConrad Meyer dev = sc->dev; 3754577cf37SConrad Meyer 3764577cf37SConrad Meyer pc = cpu_get_pcpu(dev); 3774577cf37SConrad Meyer if (pc == NULL) 3784577cf37SConrad Meyer return (ENXIO); 3794577cf37SConrad Meyer 3804577cf37SConrad Meyer thread_lock(curthread); 3814577cf37SConrad Meyer sched_bind(curthread, pc->pc_cpuid); 3824577cf37SConrad Meyer thread_unlock(curthread); 3834577cf37SConrad Meyer 3844577cf37SConrad Meyer /* XXX: Many MSRs aren't readable until feature is enabled */ 3854577cf37SConrad Meyer ret = wrmsr_safe(MSR_IA32_PM_ENABLE, 1); 3864577cf37SConrad Meyer if (ret) { 3875e3574c8SConrad Meyer /* 3885e3574c8SConrad Meyer * This is actually a package-level MSR, and only the first 3895e3574c8SConrad Meyer * write is not ignored. So it is harmless to enable it across 3905e3574c8SConrad Meyer * all devices, and this allows us not to care especially in 3915e3574c8SConrad Meyer * which order cores (and packages) are probed. This error 3925e3574c8SConrad Meyer * condition should not happen given we gate on the HWP CPUID 3935e3574c8SConrad Meyer * feature flag, if the Intel SDM is correct. 3945e3574c8SConrad Meyer */ 3954577cf37SConrad Meyer device_printf(dev, "Failed to enable HWP for cpu%d (%d)\n", 3964577cf37SConrad Meyer pc->pc_cpuid, ret); 3974577cf37SConrad Meyer goto out; 3984577cf37SConrad Meyer } 3994577cf37SConrad Meyer 4004577cf37SConrad Meyer ret = rdmsr_safe(MSR_IA32_HWP_REQUEST, &sc->req); 401351896d3SConrad Meyer if (ret) { 402351896d3SConrad Meyer device_printf(dev, 403351896d3SConrad Meyer "Failed to read HWP request MSR for cpu%d (%d)\n", 404351896d3SConrad Meyer pc->pc_cpuid, ret); 405351896d3SConrad Meyer goto out; 406351896d3SConrad Meyer } 4074577cf37SConrad Meyer 4084577cf37SConrad Meyer ret = rdmsr_safe(MSR_IA32_HWP_CAPABILITIES, &caps); 409351896d3SConrad Meyer if (ret) { 410351896d3SConrad Meyer device_printf(dev, 411351896d3SConrad Meyer "Failed to read HWP capabilities MSR for cpu%d (%d)\n", 412351896d3SConrad Meyer pc->pc_cpuid, ret); 413351896d3SConrad Meyer goto out; 414351896d3SConrad Meyer } 4154577cf37SConrad Meyer 4165e3574c8SConrad Meyer /* 4175e3574c8SConrad Meyer * High and low are static; "guaranteed" is dynamic; and efficient is 4185e3574c8SConrad Meyer * also dynamic. 4195e3574c8SConrad Meyer */ 4204577cf37SConrad Meyer sc->high = IA32_HWP_CAPABILITIES_HIGHEST_PERFORMANCE(caps); 4214577cf37SConrad Meyer sc->guaranteed = IA32_HWP_CAPABILITIES_GUARANTEED_PERFORMANCE(caps); 4224577cf37SConrad Meyer sc->efficient = IA32_HWP_CAPABILITIES_EFFICIENT_PERFORMANCE(caps); 4234577cf37SConrad Meyer sc->low = IA32_HWP_CAPABILITIES_LOWEST_PERFORMANCE(caps); 4244577cf37SConrad Meyer 4254577cf37SConrad Meyer /* hardware autonomous selection determines the performance target */ 4264577cf37SConrad Meyer sc->req &= ~IA32_HWP_DESIRED_PERFORMANCE; 4274577cf37SConrad Meyer 4284577cf37SConrad Meyer /* enable HW dynamic selection of window size */ 4294577cf37SConrad Meyer sc->req &= ~IA32_HWP_ACTIVITY_WINDOW; 4304577cf37SConrad Meyer 4314577cf37SConrad Meyer /* IA32_HWP_REQUEST.Minimum_Performance = IA32_HWP_CAPABILITIES.Lowest_Performance */ 4324577cf37SConrad Meyer sc->req &= ~IA32_HWP_MINIMUM_PERFORMANCE; 4334577cf37SConrad Meyer sc->req |= sc->low; 4344577cf37SConrad Meyer 4354577cf37SConrad Meyer /* IA32_HWP_REQUEST.Maximum_Performance = IA32_HWP_CAPABILITIES.Highest_Performance. */ 4364577cf37SConrad Meyer sc->req &= ~IA32_HWP_REQUEST_MAXIMUM_PERFORMANCE; 4374577cf37SConrad Meyer sc->req |= sc->high << 8; 4384577cf37SConrad Meyer 439cd4e43b2SConrad Meyer /* If supported, request package-level control for this CPU. */ 440cd4e43b2SConrad Meyer if (sc->hwp_pkg_ctrl_en) 441cd4e43b2SConrad Meyer ret = wrmsr_safe(MSR_IA32_HWP_REQUEST, sc->req | 442cd4e43b2SConrad Meyer IA32_HWP_REQUEST_PACKAGE_CONTROL); 443cd4e43b2SConrad Meyer else 4444577cf37SConrad Meyer ret = wrmsr_safe(MSR_IA32_HWP_REQUEST, sc->req); 4454577cf37SConrad Meyer if (ret) { 4464577cf37SConrad Meyer device_printf(dev, 447cd4e43b2SConrad Meyer "Failed to setup%s autonomous HWP for cpu%d\n", 448cd4e43b2SConrad Meyer sc->hwp_pkg_ctrl_en ? " PKG" : "", pc->pc_cpuid); 449cd4e43b2SConrad Meyer goto out; 450cd4e43b2SConrad Meyer } 451cd4e43b2SConrad Meyer 452cd4e43b2SConrad Meyer /* If supported, write the PKG-wide control MSR. */ 453cd4e43b2SConrad Meyer if (sc->hwp_pkg_ctrl_en) { 454cd4e43b2SConrad Meyer /* 455cd4e43b2SConrad Meyer * "The structure of the IA32_HWP_REQUEST_PKG MSR 456cd4e43b2SConrad Meyer * (package-level) is identical to the IA32_HWP_REQUEST MSR 457cd4e43b2SConrad Meyer * with the exception of the Package Control field, which does 458cd4e43b2SConrad Meyer * not exist." (Intel SDM §14.4.4) 459cd4e43b2SConrad Meyer */ 460cd4e43b2SConrad Meyer ret = wrmsr_safe(MSR_IA32_HWP_REQUEST_PKG, sc->req); 46141062d68SYuri Pankov if (ret) { 462cd4e43b2SConrad Meyer device_printf(dev, 463cd4e43b2SConrad Meyer "Failed to set autonomous HWP for package\n"); 4644577cf37SConrad Meyer } 46541062d68SYuri Pankov } 4664577cf37SConrad Meyer 4674577cf37SConrad Meyer out: 4684577cf37SConrad Meyer thread_lock(curthread); 4694577cf37SConrad Meyer sched_unbind(curthread); 4704577cf37SConrad Meyer thread_unlock(curthread); 4714577cf37SConrad Meyer 4724577cf37SConrad Meyer return (ret); 4734577cf37SConrad Meyer } 4744577cf37SConrad Meyer 4754577cf37SConrad Meyer static int 4764577cf37SConrad Meyer intel_hwpstate_attach(device_t dev) 4774577cf37SConrad Meyer { 4784577cf37SConrad Meyer struct hwp_softc *sc; 4794577cf37SConrad Meyer int ret; 4804577cf37SConrad Meyer 4814577cf37SConrad Meyer sc = device_get_softc(dev); 4824577cf37SConrad Meyer sc->dev = dev; 4834577cf37SConrad Meyer 484556a1a0bSConrad Meyer /* eax */ 485f591c3c8SConrad Meyer if (cpu_power_eax & CPUTPM1_HWP_NOTIFICATION) 4864577cf37SConrad Meyer sc->hwp_notifications = true; 487f591c3c8SConrad Meyer if (cpu_power_eax & CPUTPM1_HWP_ACTIVITY_WINDOW) 4884577cf37SConrad Meyer sc->hwp_activity_window = true; 489f591c3c8SConrad Meyer if (cpu_power_eax & CPUTPM1_HWP_PERF_PREF) 4904577cf37SConrad Meyer sc->hwp_pref_ctrl = true; 491f591c3c8SConrad Meyer if (cpu_power_eax & CPUTPM1_HWP_PKG) 4924577cf37SConrad Meyer sc->hwp_pkg_ctrl = true; 4934577cf37SConrad Meyer 494cd4e43b2SConrad Meyer /* Allow administrators to disable pkg-level control. */ 495cd4e43b2SConrad Meyer sc->hwp_pkg_ctrl_en = (sc->hwp_pkg_ctrl && hwpstate_pkg_ctrl_enable); 496cd4e43b2SConrad Meyer 497556a1a0bSConrad Meyer /* ecx */ 498556a1a0bSConrad Meyer if (cpu_power_ecx & CPUID_PERF_BIAS) 499556a1a0bSConrad Meyer sc->hwp_perf_bias = true; 500556a1a0bSConrad Meyer 5014577cf37SConrad Meyer ret = set_autonomous_hwp(sc); 5024577cf37SConrad Meyer if (ret) 5034577cf37SConrad Meyer return (ret); 5044577cf37SConrad Meyer 5054577cf37SConrad Meyer SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 5064577cf37SConrad Meyer SYSCTL_STATIC_CHILDREN(_debug), OID_AUTO, device_get_nameunit(dev), 5071d6fb900SAlexander Motin CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE, 5084577cf37SConrad Meyer sc, 0, intel_hwp_dump_sysctl_handler, "A", ""); 5094577cf37SConrad Meyer 5104577cf37SConrad Meyer SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 5114577cf37SConrad Meyer SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, 5121d6fb900SAlexander Motin "epp", CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, dev, 0, 5134577cf37SConrad Meyer sysctl_epp_select, "I", 5144577cf37SConrad Meyer "Efficiency/Performance Preference " 5154577cf37SConrad Meyer "(range from 0, most performant, through 100, most efficient)"); 5164577cf37SConrad Meyer 5174577cf37SConrad Meyer return (cpufreq_register(dev)); 5184577cf37SConrad Meyer } 5194577cf37SConrad Meyer 5204577cf37SConrad Meyer static int 5214577cf37SConrad Meyer intel_hwpstate_detach(device_t dev) 5224577cf37SConrad Meyer { 5234577cf37SConrad Meyer 5244577cf37SConrad Meyer return (cpufreq_unregister(dev)); 5254577cf37SConrad Meyer } 5264577cf37SConrad Meyer 5274577cf37SConrad Meyer static int 5284577cf37SConrad Meyer intel_hwpstate_get(device_t dev, struct cf_setting *set) 5294577cf37SConrad Meyer { 5304577cf37SConrad Meyer struct pcpu *pc; 5314577cf37SConrad Meyer uint64_t rate; 5324577cf37SConrad Meyer int ret; 5334577cf37SConrad Meyer 5344577cf37SConrad Meyer if (set == NULL) 5354577cf37SConrad Meyer return (EINVAL); 5364577cf37SConrad Meyer 5374577cf37SConrad Meyer pc = cpu_get_pcpu(dev); 5384577cf37SConrad Meyer if (pc == NULL) 5394577cf37SConrad Meyer return (ENXIO); 5404577cf37SConrad Meyer 5414577cf37SConrad Meyer memset(set, CPUFREQ_VAL_UNKNOWN, sizeof(*set)); 5424577cf37SConrad Meyer set->dev = dev; 5434577cf37SConrad Meyer 5444577cf37SConrad Meyer ret = cpu_est_clockrate(pc->pc_cpuid, &rate); 5454577cf37SConrad Meyer if (ret == 0) 5464577cf37SConrad Meyer set->freq = rate / 1000000; 5474577cf37SConrad Meyer 5484577cf37SConrad Meyer set->volts = CPUFREQ_VAL_UNKNOWN; 5494577cf37SConrad Meyer set->power = CPUFREQ_VAL_UNKNOWN; 5504577cf37SConrad Meyer set->lat = CPUFREQ_VAL_UNKNOWN; 5514577cf37SConrad Meyer 5524577cf37SConrad Meyer return (0); 5534577cf37SConrad Meyer } 5544577cf37SConrad Meyer 5554577cf37SConrad Meyer static int 5564577cf37SConrad Meyer intel_hwpstate_type(device_t dev, int *type) 5574577cf37SConrad Meyer { 5584577cf37SConrad Meyer if (type == NULL) 5594577cf37SConrad Meyer return (EINVAL); 5604577cf37SConrad Meyer *type = CPUFREQ_TYPE_ABSOLUTE | CPUFREQ_FLAG_INFO_ONLY | CPUFREQ_FLAG_UNCACHED; 5614577cf37SConrad Meyer 5624577cf37SConrad Meyer return (0); 5634577cf37SConrad Meyer } 5644577cf37SConrad Meyer 5654577cf37SConrad Meyer static int 5664577cf37SConrad Meyer intel_hwpstate_suspend(device_t dev) 5674577cf37SConrad Meyer { 5684577cf37SConrad Meyer return (0); 5694577cf37SConrad Meyer } 5704577cf37SConrad Meyer 5714577cf37SConrad Meyer /* 5724577cf37SConrad Meyer * Redo a subset of set_autonomous_hwp on resume; untested. Without this, 5734577cf37SConrad Meyer * testers observed that on resume MSR_IA32_HWP_REQUEST was bogus. 5744577cf37SConrad Meyer */ 5754577cf37SConrad Meyer static int 5764577cf37SConrad Meyer intel_hwpstate_resume(device_t dev) 5774577cf37SConrad Meyer { 5784577cf37SConrad Meyer struct hwp_softc *sc; 5794577cf37SConrad Meyer struct pcpu *pc; 5804577cf37SConrad Meyer int ret; 5814577cf37SConrad Meyer 5824577cf37SConrad Meyer sc = device_get_softc(dev); 5834577cf37SConrad Meyer 5844577cf37SConrad Meyer pc = cpu_get_pcpu(dev); 5854577cf37SConrad Meyer if (pc == NULL) 5864577cf37SConrad Meyer return (ENXIO); 5874577cf37SConrad Meyer 5884577cf37SConrad Meyer thread_lock(curthread); 5894577cf37SConrad Meyer sched_bind(curthread, pc->pc_cpuid); 5904577cf37SConrad Meyer thread_unlock(curthread); 5914577cf37SConrad Meyer 5924577cf37SConrad Meyer ret = wrmsr_safe(MSR_IA32_PM_ENABLE, 1); 5934577cf37SConrad Meyer if (ret) { 5944577cf37SConrad Meyer device_printf(dev, 5954577cf37SConrad Meyer "Failed to enable HWP for cpu%d after suspend (%d)\n", 5964577cf37SConrad Meyer pc->pc_cpuid, ret); 5974577cf37SConrad Meyer goto out; 5984577cf37SConrad Meyer } 5994577cf37SConrad Meyer 600cd4e43b2SConrad Meyer if (sc->hwp_pkg_ctrl_en) 601cd4e43b2SConrad Meyer ret = wrmsr_safe(MSR_IA32_HWP_REQUEST, sc->req | 602cd4e43b2SConrad Meyer IA32_HWP_REQUEST_PACKAGE_CONTROL); 603cd4e43b2SConrad Meyer else 6044577cf37SConrad Meyer ret = wrmsr_safe(MSR_IA32_HWP_REQUEST, sc->req); 6054577cf37SConrad Meyer if (ret) { 6064577cf37SConrad Meyer device_printf(dev, 607cd4e43b2SConrad Meyer "Failed to set%s autonomous HWP for cpu%d after suspend\n", 608cd4e43b2SConrad Meyer sc->hwp_pkg_ctrl_en ? " PKG" : "", pc->pc_cpuid); 609cd4e43b2SConrad Meyer goto out; 610cd4e43b2SConrad Meyer } 611cd4e43b2SConrad Meyer if (sc->hwp_pkg_ctrl_en) { 612cd4e43b2SConrad Meyer ret = wrmsr_safe(MSR_IA32_HWP_REQUEST_PKG, sc->req); 613e656fa70SConrad Meyer if (ret) { 614cd4e43b2SConrad Meyer device_printf(dev, 615c93eb470SConrad Meyer "Failed to set autonomous HWP for package after " 616c93eb470SConrad Meyer "suspend\n"); 617e656fa70SConrad Meyer goto out; 618e656fa70SConrad Meyer } 619e656fa70SConrad Meyer } 620e656fa70SConrad Meyer if (!sc->hwp_pref_ctrl && sc->hwp_perf_bias_cached) { 621e656fa70SConrad Meyer ret = wrmsr_safe(MSR_IA32_ENERGY_PERF_BIAS, 622e656fa70SConrad Meyer sc->hwp_energy_perf_bias); 623e656fa70SConrad Meyer if (ret) { 624e656fa70SConrad Meyer device_printf(dev, 625e656fa70SConrad Meyer "Failed to set energy perf bias for cpu%d after " 626e656fa70SConrad Meyer "suspend\n", pc->pc_cpuid); 627e656fa70SConrad Meyer } 6284577cf37SConrad Meyer } 6294577cf37SConrad Meyer 6304577cf37SConrad Meyer out: 6314577cf37SConrad Meyer thread_lock(curthread); 6324577cf37SConrad Meyer sched_unbind(curthread); 6334577cf37SConrad Meyer thread_unlock(curthread); 6344577cf37SConrad Meyer 6354577cf37SConrad Meyer return (ret); 6364577cf37SConrad Meyer } 637