1 /* 2 * Copyright (c) 2020 iXsystems, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #include <sys/types.h> 30 #include <sys/cdefs.h> 31 #include <sys/proc.h> 32 #include <sys/systm.h> 33 34 #include <machine/pcb.h> 35 #include <x86/x86_var.h> 36 #include <x86/specialreg.h> 37 38 #define kfpu_init() (0) 39 #define kfpu_fini() do {} while (0) 40 #define kfpu_allowed() 1 41 #define kfpu_initialize(tsk) do {} while (0) 42 43 44 #define kfpu_begin() { \ 45 if (__predict_false(!is_fpu_kern_thread(0))) \ 46 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);\ 47 } 48 49 #ifndef PCB_FPUNOSAVE 50 #define PCB_FPUNOSAVE PCB_NPXNOSAVE 51 #endif 52 53 #define kfpu_end() { \ 54 if (__predict_false(curpcb->pcb_flags & PCB_FPUNOSAVE)) \ 55 fpu_kern_leave(curthread, NULL); \ 56 } 57 58 /* 59 * Check if OS supports AVX and AVX2 by checking XCR0 60 * Only call this function if CPUID indicates that AVX feature is 61 * supported by the CPU, otherwise it might be an illegal instruction. 62 */ 63 static inline uint64_t 64 xgetbv(uint32_t index) 65 { 66 uint32_t eax, edx; 67 /* xgetbv - instruction byte code */ 68 __asm__ __volatile__(".byte 0x0f; .byte 0x01; .byte 0xd0" 69 : "=a" (eax), "=d" (edx) 70 : "c" (index)); 71 72 return ((((uint64_t)edx)<<32) | (uint64_t)eax); 73 } 74 75 76 /* 77 * Detect register set support 78 */ 79 static inline boolean_t 80 __simd_state_enabled(const uint64_t state) 81 { 82 boolean_t has_osxsave; 83 uint64_t xcr0; 84 85 has_osxsave = (cpu_feature2 & CPUID2_OSXSAVE) != 0; 86 87 if (!has_osxsave) 88 return (B_FALSE); 89 90 xcr0 = xgetbv(0); 91 return ((xcr0 & state) == state); 92 } 93 94 #define _XSTATE_SSE_AVX (0x2 | 0x4) 95 #define _XSTATE_AVX512 (0xE0 | _XSTATE_SSE_AVX) 96 97 #define __ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX) 98 #define __zmm_enabled() __simd_state_enabled(_XSTATE_AVX512) 99 100 101 /* 102 * Check if SSE instruction set is available 103 */ 104 static inline boolean_t 105 zfs_sse_available(void) 106 { 107 return ((cpu_feature & CPUID_SSE) != 0); 108 } 109 110 /* 111 * Check if SSE2 instruction set is available 112 */ 113 static inline boolean_t 114 zfs_sse2_available(void) 115 { 116 return ((cpu_feature & CPUID_SSE2) != 0); 117 } 118 119 /* 120 * Check if SSE3 instruction set is available 121 */ 122 static inline boolean_t 123 zfs_sse3_available(void) 124 { 125 return ((cpu_feature2 & CPUID2_SSE3) != 0); 126 } 127 128 /* 129 * Check if SSSE3 instruction set is available 130 */ 131 static inline boolean_t 132 zfs_ssse3_available(void) 133 { 134 return ((cpu_feature2 & CPUID2_SSSE3) != 0); 135 } 136 137 /* 138 * Check if SSE4.1 instruction set is available 139 */ 140 static inline boolean_t 141 zfs_sse4_1_available(void) 142 { 143 return ((cpu_feature2 & CPUID2_SSE41) != 0); 144 } 145 146 /* 147 * Check if SSE4.2 instruction set is available 148 */ 149 static inline boolean_t 150 zfs_sse4_2_available(void) 151 { 152 return ((cpu_feature2 & CPUID2_SSE42) != 0); 153 } 154 155 /* 156 * Check if AVX instruction set is available 157 */ 158 static inline boolean_t 159 zfs_avx_available(void) 160 { 161 boolean_t has_avx; 162 163 has_avx = (cpu_feature2 & CPUID2_AVX) != 0; 164 165 return (has_avx && __ymm_enabled()); 166 } 167 168 /* 169 * Check if AVX2 instruction set is available 170 */ 171 static inline boolean_t 172 zfs_avx2_available(void) 173 { 174 boolean_t has_avx2; 175 176 has_avx2 = (cpu_stdext_feature & CPUID_STDEXT_AVX2) != 0; 177 178 return (has_avx2 && __ymm_enabled()); 179 } 180 181 /* 182 * Check if SHA_NI instruction set is available 183 */ 184 static inline boolean_t 185 zfs_shani_available(void) 186 { 187 boolean_t has_shani; 188 189 has_shani = (cpu_stdext_feature & CPUID_STDEXT_SHA) != 0; 190 191 return (has_shani && __ymm_enabled()); 192 } 193 194 /* 195 * AVX-512 family of instruction sets: 196 * 197 * AVX512F Foundation 198 * AVX512CD Conflict Detection Instructions 199 * AVX512ER Exponential and Reciprocal Instructions 200 * AVX512PF Prefetch Instructions 201 * 202 * AVX512BW Byte and Word Instructions 203 * AVX512DQ Double-word and Quadword Instructions 204 * AVX512VL Vector Length Extensions 205 * 206 * AVX512IFMA Integer Fused Multiply Add (Not supported by kernel 4.4) 207 * AVX512VBMI Vector Byte Manipulation Instructions 208 */ 209 210 211 /* Check if AVX512F instruction set is available */ 212 static inline boolean_t 213 zfs_avx512f_available(void) 214 { 215 boolean_t has_avx512; 216 217 has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0; 218 219 return (has_avx512 && __zmm_enabled()); 220 } 221 222 /* Check if AVX512CD instruction set is available */ 223 static inline boolean_t 224 zfs_avx512cd_available(void) 225 { 226 boolean_t has_avx512; 227 228 has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && 229 (cpu_stdext_feature & CPUID_STDEXT_AVX512CD) != 0; 230 231 return (has_avx512 && __zmm_enabled()); 232 } 233 234 /* Check if AVX512ER instruction set is available */ 235 static inline boolean_t 236 zfs_avx512er_available(void) 237 { 238 boolean_t has_avx512; 239 240 has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && 241 (cpu_stdext_feature & CPUID_STDEXT_AVX512CD) != 0; 242 243 return (has_avx512 && __zmm_enabled()); 244 } 245 246 /* Check if AVX512PF instruction set is available */ 247 static inline boolean_t 248 zfs_avx512pf_available(void) 249 { 250 boolean_t has_avx512; 251 252 has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && 253 (cpu_stdext_feature & CPUID_STDEXT_AVX512PF) != 0; 254 255 return (has_avx512 && __zmm_enabled()); 256 } 257 258 /* Check if AVX512BW instruction set is available */ 259 static inline boolean_t 260 zfs_avx512bw_available(void) 261 { 262 boolean_t has_avx512 = B_FALSE; 263 264 has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512BW) != 0; 265 266 return (has_avx512 && __zmm_enabled()); 267 } 268 269 /* Check if AVX512DQ instruction set is available */ 270 static inline boolean_t 271 zfs_avx512dq_available(void) 272 { 273 boolean_t has_avx512; 274 275 has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && 276 (cpu_stdext_feature & CPUID_STDEXT_AVX512DQ) != 0; 277 278 return (has_avx512 && __zmm_enabled()); 279 } 280 281 /* Check if AVX512VL instruction set is available */ 282 static inline boolean_t 283 zfs_avx512vl_available(void) 284 { 285 boolean_t has_avx512; 286 287 has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && 288 (cpu_stdext_feature & CPUID_STDEXT_AVX512VL) != 0; 289 290 return (has_avx512 && __zmm_enabled()); 291 } 292 293 /* Check if AVX512IFMA instruction set is available */ 294 static inline boolean_t 295 zfs_avx512ifma_available(void) 296 { 297 boolean_t has_avx512; 298 299 has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && 300 (cpu_stdext_feature & CPUID_STDEXT_AVX512IFMA) != 0; 301 302 return (has_avx512 && __zmm_enabled()); 303 } 304 305 /* Check if AVX512VBMI instruction set is available */ 306 static inline boolean_t 307 zfs_avx512vbmi_available(void) 308 { 309 boolean_t has_avx512; 310 311 has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && 312 (cpu_stdext_feature & CPUID_STDEXT_BMI1) != 0; 313 314 return (has_avx512 && __zmm_enabled()); 315 } 316