1 /* 2 * Copyright (c) 2020 iXsystems, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #include <sys/types.h> 30 #include <sys/cdefs.h> 31 #include <sys/proc.h> 32 #include <sys/systm.h> 33 34 #include <machine/pcb.h> 35 #include <x86/x86_var.h> 36 #include <x86/specialreg.h> 37 38 #define kfpu_init() (0) 39 #define kfpu_fini() do {} while (0) 40 #define kfpu_allowed() 1 41 #define kfpu_initialize(tsk) do {} while (0) 42 43 #define kfpu_begin() { \ 44 if (__predict_false(!is_fpu_kern_thread(0))) \ 45 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);\ 46 } 47 48 #define kfpu_end() { \ 49 if (__predict_false(curpcb->pcb_flags & PCB_FPUNOSAVE)) \ 50 fpu_kern_leave(curthread, NULL); \ 51 } 52 53 /* 54 * Check if OS supports AVX and AVX2 by checking XCR0 55 * Only call this function if CPUID indicates that AVX feature is 56 * supported by the CPU, otherwise it might be an illegal instruction. 57 */ 58 static inline uint64_t 59 xgetbv(uint32_t index) 60 { 61 uint32_t eax, edx; 62 /* xgetbv - instruction byte code */ 63 __asm__ __volatile__(".byte 0x0f; .byte 0x01; .byte 0xd0" 64 : "=a" (eax), "=d" (edx) 65 : "c" (index)); 66 67 return ((((uint64_t)edx)<<32) | (uint64_t)eax); 68 } 69 70 71 /* 72 * Detect register set support 73 */ 74 static inline boolean_t 75 __simd_state_enabled(const uint64_t state) 76 { 77 boolean_t has_osxsave; 78 uint64_t xcr0; 79 80 has_osxsave = !!(cpu_feature2 & CPUID2_OSXSAVE); 81 82 if (!has_osxsave) 83 return (B_FALSE); 84 85 xcr0 = xgetbv(0); 86 return ((xcr0 & state) == state); 87 } 88 89 #define _XSTATE_SSE_AVX (0x2 | 0x4) 90 #define _XSTATE_AVX512 (0xE0 | _XSTATE_SSE_AVX) 91 92 #define __ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX) 93 #define __zmm_enabled() __simd_state_enabled(_XSTATE_AVX512) 94 95 96 /* 97 * Check if SSE instruction set is available 98 */ 99 static inline boolean_t 100 zfs_sse_available(void) 101 { 102 return (!!(cpu_feature & CPUID_SSE)); 103 } 104 105 /* 106 * Check if SSE2 instruction set is available 107 */ 108 static inline boolean_t 109 zfs_sse2_available(void) 110 { 111 return (!!(cpu_feature & CPUID_SSE2)); 112 } 113 114 /* 115 * Check if SSE3 instruction set is available 116 */ 117 static inline boolean_t 118 zfs_sse3_available(void) 119 { 120 return (!!(cpu_feature2 & CPUID2_SSE3)); 121 } 122 123 /* 124 * Check if SSSE3 instruction set is available 125 */ 126 static inline boolean_t 127 zfs_ssse3_available(void) 128 { 129 return (!!(cpu_feature2 & CPUID2_SSSE3)); 130 } 131 132 /* 133 * Check if SSE4.1 instruction set is available 134 */ 135 static inline boolean_t 136 zfs_sse4_1_available(void) 137 { 138 return (!!(cpu_feature2 & CPUID2_SSE41)); 139 } 140 141 /* 142 * Check if SSE4.2 instruction set is available 143 */ 144 static inline boolean_t 145 zfs_sse4_2_available(void) 146 { 147 return (!!(cpu_feature2 & CPUID2_SSE42)); 148 } 149 150 /* 151 * Check if AVX instruction set is available 152 */ 153 static inline boolean_t 154 zfs_avx_available(void) 155 { 156 boolean_t has_avx; 157 158 has_avx = !!(cpu_feature2 & CPUID2_AVX); 159 160 return (has_avx && __ymm_enabled()); 161 } 162 163 /* 164 * Check if AVX2 instruction set is available 165 */ 166 static inline boolean_t 167 zfs_avx2_available(void) 168 { 169 boolean_t has_avx2; 170 171 has_avx2 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX2); 172 173 return (has_avx2 && __ymm_enabled()); 174 } 175 176 /* 177 * AVX-512 family of instruction sets: 178 * 179 * AVX512F Foundation 180 * AVX512CD Conflict Detection Instructions 181 * AVX512ER Exponential and Reciprocal Instructions 182 * AVX512PF Prefetch Instructions 183 * 184 * AVX512BW Byte and Word Instructions 185 * AVX512DQ Double-word and Quadword Instructions 186 * AVX512VL Vector Length Extensions 187 * 188 * AVX512IFMA Integer Fused Multiply Add (Not supported by kernel 4.4) 189 * AVX512VBMI Vector Byte Manipulation Instructions 190 */ 191 192 193 /* Check if AVX512F instruction set is available */ 194 static inline boolean_t 195 zfs_avx512f_available(void) 196 { 197 boolean_t has_avx512; 198 199 has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F); 200 201 return (has_avx512 && __zmm_enabled()); 202 } 203 204 /* Check if AVX512CD instruction set is available */ 205 static inline boolean_t 206 zfs_avx512cd_available(void) 207 { 208 boolean_t has_avx512; 209 210 has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && 211 !!(cpu_stdext_feature & CPUID_STDEXT_AVX512CD); 212 213 return (has_avx512 && __zmm_enabled()); 214 } 215 216 /* Check if AVX512ER instruction set is available */ 217 static inline boolean_t 218 zfs_avx512er_available(void) 219 { 220 boolean_t has_avx512; 221 222 has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && 223 !!(cpu_stdext_feature & CPUID_STDEXT_AVX512CD); 224 225 return (has_avx512 && __zmm_enabled()); 226 } 227 228 /* Check if AVX512PF instruction set is available */ 229 static inline boolean_t 230 zfs_avx512pf_available(void) 231 { 232 boolean_t has_avx512; 233 234 has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && 235 !!(cpu_stdext_feature & CPUID_STDEXT_AVX512PF); 236 237 return (has_avx512 && __zmm_enabled()); 238 } 239 240 /* Check if AVX512BW instruction set is available */ 241 static inline boolean_t 242 zfs_avx512bw_available(void) 243 { 244 boolean_t has_avx512 = B_FALSE; 245 246 has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512BW); 247 248 return (has_avx512 && __zmm_enabled()); 249 } 250 251 /* Check if AVX512DQ instruction set is available */ 252 static inline boolean_t 253 zfs_avx512dq_available(void) 254 { 255 boolean_t has_avx512; 256 257 has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && 258 !!(cpu_stdext_feature & CPUID_STDEXT_AVX512DQ); 259 260 return (has_avx512 && __zmm_enabled()); 261 } 262 263 /* Check if AVX512VL instruction set is available */ 264 static inline boolean_t 265 zfs_avx512vl_available(void) 266 { 267 boolean_t has_avx512; 268 269 has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && 270 !!(cpu_stdext_feature & CPUID_STDEXT_AVX512VL); 271 272 return (has_avx512 && __zmm_enabled()); 273 } 274 275 /* Check if AVX512IFMA instruction set is available */ 276 static inline boolean_t 277 zfs_avx512ifma_available(void) 278 { 279 boolean_t has_avx512; 280 281 has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && 282 !!(cpu_stdext_feature & CPUID_STDEXT_AVX512IFMA); 283 284 return (has_avx512 && __zmm_enabled()); 285 } 286 287 /* Check if AVX512VBMI instruction set is available */ 288 static inline boolean_t 289 zfs_avx512vbmi_available(void) 290 { 291 boolean_t has_avx512; 292 293 has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && 294 !!(cpu_stdext_feature & CPUID_STDEXT_BMI1); 295 296 return (has_avx512 && __zmm_enabled()); 297 } 298