1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (C) 2016 Gvozden Neskovic <neskovic@compeng.uni-frankfurt.de>. 23 */ 24 25 /* 26 * USER API: 27 * 28 * Kernel fpu methods: 29 * kfpu_allowed() 30 * kfpu_begin() 31 * kfpu_end() 32 * kfpu_init() 33 * kfpu_fini() 34 * 35 * SIMD support: 36 * 37 * Following functions should be called to determine whether CPU feature 38 * is supported. All functions are usable in kernel and user space. 39 * If a SIMD algorithm is using more than one instruction set 40 * all relevant feature test functions should be called. 41 * 42 * Supported features: 43 * zfs_sse_available() 44 * zfs_sse2_available() 45 * zfs_sse3_available() 46 * zfs_ssse3_available() 47 * zfs_sse4_1_available() 48 * zfs_sse4_2_available() 49 * 50 * zfs_avx_available() 51 * zfs_avx2_available() 52 * 53 * zfs_bmi1_available() 54 * zfs_bmi2_available() 55 * 56 * zfs_shani_available() 57 * 58 * zfs_avx512f_available() 59 * zfs_avx512cd_available() 60 * zfs_avx512er_available() 61 * zfs_avx512pf_available() 62 * zfs_avx512bw_available() 63 * zfs_avx512dq_available() 64 * zfs_avx512vl_available() 65 * zfs_avx512ifma_available() 66 * zfs_avx512vbmi_available() 67 * 68 * NOTE(AVX-512VL): If using AVX-512 instructions with 128Bit registers 69 * also add zfs_avx512vl_available() to feature check. 70 */ 71 72 #ifndef _LINUX_SIMD_X86_H 73 #define _LINUX_SIMD_X86_H 74 75 /* only for __x86 */ 76 #if defined(__x86) 77 78 #include <sys/types.h> 79 #include <asm/cpufeature.h> 80 81 /* 82 * Disable the WARN_ON_FPU() macro to prevent additional dependencies 83 * when providing the kfpu_* functions. Relevant warnings are included 84 * as appropriate and are unconditionally enabled. 85 */ 86 #if defined(CONFIG_X86_DEBUG_FPU) && !defined(KERNEL_EXPORTS_X86_FPU) 87 #undef CONFIG_X86_DEBUG_FPU 88 #endif 89 90 /* 91 * The following cases are for kernels which export either the 92 * kernel_fpu_* or __kernel_fpu_* functions. 93 */ 94 #if defined(KERNEL_EXPORTS_X86_FPU) 95 96 #if defined(HAVE_KERNEL_FPU_API_HEADER) 97 #include <asm/fpu/api.h> 98 #if defined(HAVE_KERNEL_FPU_INTERNAL_HEADER) 99 #include <asm/fpu/internal.h> 100 #endif 101 #else 102 #include <asm/i387.h> 103 #endif 104 105 #define kfpu_allowed() 1 106 #define kfpu_init() 0 107 #define kfpu_fini() ((void) 0) 108 109 #if defined(HAVE_UNDERSCORE_KERNEL_FPU) 110 #define kfpu_begin() \ 111 { \ 112 preempt_disable(); \ 113 __kernel_fpu_begin(); \ 114 } 115 #define kfpu_end() \ 116 { \ 117 __kernel_fpu_end(); \ 118 preempt_enable(); \ 119 } 120 121 #elif defined(HAVE_KERNEL_FPU) 122 #define kfpu_begin() kernel_fpu_begin() 123 #define kfpu_end() kernel_fpu_end() 124 125 #else 126 /* 127 * This case is unreachable. When KERNEL_EXPORTS_X86_FPU is defined then 128 * either HAVE_UNDERSCORE_KERNEL_FPU or HAVE_KERNEL_FPU must be defined. 129 */ 130 #error "Unreachable kernel configuration" 131 #endif 132 133 #else /* defined(KERNEL_EXPORTS_X86_FPU) */ 134 135 /* 136 * When the kernel_fpu_* symbols are unavailable then provide our own 137 * versions which allow the FPU to be safely used. 138 */ 139 #if defined(HAVE_KERNEL_FPU_INTERNAL) 140 141 /* 142 * For kernels not exporting *kfpu_{begin,end} we have to use inline assembly 143 * with the XSAVE{,OPT,S} instructions, so we need the toolchain to support at 144 * least XSAVE. 145 */ 146 #if !defined(HAVE_XSAVE) 147 #error "Toolchain needs to support the XSAVE assembler instruction" 148 #endif 149 150 #ifndef XFEATURE_MASK_XTILE 151 /* 152 * For kernels where this doesn't exist yet, we still don't want to break 153 * by save/restoring this broken nonsense. 154 * See issue #14989 or Intel errata SPR4 for why 155 */ 156 #define XFEATURE_MASK_XTILE 0x60000 157 #endif 158 159 #include <linux/mm.h> 160 #include <linux/slab.h> 161 162 extern uint8_t **zfs_kfpu_fpregs; 163 164 /* 165 * Return the size in bytes required by the XSAVE instruction for an 166 * XSAVE area containing all the user state components supported by this CPU. 167 * See: Intel 64 and IA-32 Architectures Software Developer’s Manual. 168 * Dec. 2021. Vol. 2A p. 3-222. 169 */ 170 static inline uint32_t 171 get_xsave_area_size(void) 172 { 173 if (!boot_cpu_has(X86_FEATURE_OSXSAVE)) { 174 return (0); 175 } 176 /* 177 * Call CPUID with leaf 13 and subleaf 0. The size is in ecx. 178 * We don't need to check for cpuid_max here, since if this CPU has 179 * OSXSAVE set, it has leaf 13 (0x0D) as well. 180 */ 181 uint32_t eax, ebx, ecx, edx; 182 183 eax = 13U; 184 ecx = 0U; 185 __asm__ __volatile__("cpuid" 186 : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) 187 : "a" (eax), "c" (ecx)); 188 189 return (ecx); 190 } 191 192 /* 193 * Return the allocation order of the maximum buffer size required to save the 194 * FPU state on this architecture. The value returned is the same as Linux' 195 * get_order() function would return (i.e. 2^order = nr. of pages required). 196 * Currently this will always return 0 since the save area is below 4k even for 197 * a full fledged AVX-512 implementation. 198 */ 199 static inline int 200 get_fpuregs_save_area_order(void) 201 { 202 size_t area_size = (size_t)get_xsave_area_size(); 203 204 /* 205 * If we are dealing with a CPU not supporting XSAVE, 206 * get_xsave_area_size() will return 0. Thus the maximum memory 207 * required is the FXSAVE area size which is 512 bytes. See: Intel 64 208 * and IA-32 Architectures Software Developer’s Manual. Dec. 2021. 209 * Vol. 2A p. 3-451. 210 */ 211 if (area_size == 0) { 212 area_size = 512; 213 } 214 return (get_order(area_size)); 215 } 216 217 /* 218 * Initialize per-cpu variables to store FPU state. 219 */ 220 static inline void 221 kfpu_fini(void) 222 { 223 int cpu; 224 int order = get_fpuregs_save_area_order(); 225 226 for_each_possible_cpu(cpu) { 227 if (zfs_kfpu_fpregs[cpu] != NULL) { 228 free_pages((unsigned long)zfs_kfpu_fpregs[cpu], order); 229 } 230 } 231 232 kfree(zfs_kfpu_fpregs); 233 } 234 235 static inline int 236 kfpu_init(void) 237 { 238 zfs_kfpu_fpregs = kzalloc(num_possible_cpus() * sizeof (uint8_t *), 239 GFP_KERNEL); 240 241 if (zfs_kfpu_fpregs == NULL) 242 return (-ENOMEM); 243 244 /* 245 * The fxsave and xsave operations require 16-/64-byte alignment of 246 * the target memory. Since kmalloc() provides no alignment 247 * guarantee instead use alloc_pages_node(). 248 */ 249 int cpu; 250 int order = get_fpuregs_save_area_order(); 251 252 for_each_possible_cpu(cpu) { 253 struct page *page = alloc_pages_node(cpu_to_node(cpu), 254 GFP_KERNEL | __GFP_ZERO, order); 255 if (page == NULL) { 256 kfpu_fini(); 257 return (-ENOMEM); 258 } 259 260 zfs_kfpu_fpregs[cpu] = page_address(page); 261 } 262 263 return (0); 264 } 265 266 #define kfpu_allowed() 1 267 268 /* 269 * FPU save and restore instructions. 270 */ 271 #define __asm __asm__ __volatile__ 272 #define kfpu_fxsave(addr) __asm("fxsave %0" : "=m" (*(addr))) 273 #define kfpu_fxsaveq(addr) __asm("fxsaveq %0" : "=m" (*(addr))) 274 #define kfpu_fnsave(addr) __asm("fnsave %0; fwait" : "=m" (*(addr))) 275 #define kfpu_fxrstor(addr) __asm("fxrstor %0" : : "m" (*(addr))) 276 #define kfpu_fxrstorq(addr) __asm("fxrstorq %0" : : "m" (*(addr))) 277 #define kfpu_frstor(addr) __asm("frstor %0" : : "m" (*(addr))) 278 #define kfpu_fxsr_clean(rval) __asm("fnclex; emms; fildl %P[addr]" \ 279 : : [addr] "m" (rval)); 280 281 #define kfpu_do_xsave(instruction, addr, mask) \ 282 { \ 283 uint32_t low, hi; \ 284 \ 285 low = mask; \ 286 hi = (uint64_t)(mask) >> 32; \ 287 __asm(instruction " %[dst]\n\t" \ 288 : \ 289 : [dst] "m" (*(addr)), "a" (low), "d" (hi) \ 290 : "memory"); \ 291 } 292 293 static inline void 294 kfpu_save_fxsr(uint8_t *addr) 295 { 296 if (IS_ENABLED(CONFIG_X86_32)) 297 kfpu_fxsave(addr); 298 else 299 kfpu_fxsaveq(addr); 300 } 301 302 static inline void 303 kfpu_save_fsave(uint8_t *addr) 304 { 305 kfpu_fnsave(addr); 306 } 307 308 static inline void 309 kfpu_begin(void) 310 { 311 /* 312 * Preemption and interrupts must be disabled for the critical 313 * region where the FPU state is being modified. 314 */ 315 preempt_disable(); 316 local_irq_disable(); 317 318 /* 319 * The current FPU registers need to be preserved by kfpu_begin() 320 * and restored by kfpu_end(). They are stored in a dedicated 321 * per-cpu variable, not in the task struct, this allows any user 322 * FPU state to be correctly preserved and restored. 323 */ 324 uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()]; 325 #if defined(HAVE_XSAVES) 326 if (static_cpu_has(X86_FEATURE_XSAVES)) { 327 kfpu_do_xsave("xsaves", state, ~XFEATURE_MASK_XTILE); 328 return; 329 } 330 #endif 331 #if defined(HAVE_XSAVEOPT) 332 if (static_cpu_has(X86_FEATURE_XSAVEOPT)) { 333 kfpu_do_xsave("xsaveopt", state, ~XFEATURE_MASK_XTILE); 334 return; 335 } 336 #endif 337 if (static_cpu_has(X86_FEATURE_XSAVE)) { 338 kfpu_do_xsave("xsave", state, ~XFEATURE_MASK_XTILE); 339 } else if (static_cpu_has(X86_FEATURE_FXSR)) { 340 kfpu_save_fxsr(state); 341 } else { 342 kfpu_save_fsave(state); 343 } 344 } 345 346 #define kfpu_do_xrstor(instruction, addr, mask) \ 347 { \ 348 uint32_t low, hi; \ 349 \ 350 low = mask; \ 351 hi = (uint64_t)(mask) >> 32; \ 352 __asm(instruction " %[src]" \ 353 : \ 354 : [src] "m" (*(addr)), "a" (low), "d" (hi) \ 355 : "memory"); \ 356 } 357 358 static inline void 359 kfpu_restore_fxsr(uint8_t *addr) 360 { 361 /* 362 * On AuthenticAMD K7 and K8 processors the fxrstor instruction only 363 * restores the _x87 FOP, FIP, and FDP registers when an exception 364 * is pending. Clean the _x87 state to force the restore. 365 */ 366 if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK))) 367 kfpu_fxsr_clean(addr); 368 369 if (IS_ENABLED(CONFIG_X86_32)) { 370 kfpu_fxrstor(addr); 371 } else { 372 kfpu_fxrstorq(addr); 373 } 374 } 375 376 static inline void 377 kfpu_restore_fsave(uint8_t *addr) 378 { 379 kfpu_frstor(addr); 380 } 381 382 static inline void 383 kfpu_end(void) 384 { 385 uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()]; 386 #if defined(HAVE_XSAVES) 387 if (static_cpu_has(X86_FEATURE_XSAVES)) { 388 kfpu_do_xrstor("xrstors", state, ~XFEATURE_MASK_XTILE); 389 goto out; 390 } 391 #endif 392 if (static_cpu_has(X86_FEATURE_XSAVE)) { 393 kfpu_do_xrstor("xrstor", state, ~XFEATURE_MASK_XTILE); 394 } else if (static_cpu_has(X86_FEATURE_FXSR)) { 395 kfpu_restore_fxsr(state); 396 } else { 397 kfpu_restore_fsave(state); 398 } 399 out: 400 local_irq_enable(); 401 preempt_enable(); 402 403 } 404 405 #else 406 407 #error "Exactly one of KERNEL_EXPORTS_X86_FPU or HAVE_KERNEL_FPU_INTERNAL" \ 408 " must be defined" 409 410 #endif /* defined(HAVE_KERNEL_FPU_INTERNAL */ 411 #endif /* defined(KERNEL_EXPORTS_X86_FPU) */ 412 413 /* 414 * Linux kernel provides an interface for CPU feature testing. 415 */ 416 417 /* 418 * Detect register set support 419 */ 420 421 /* 422 * Check if OS supports AVX and AVX2 by checking XCR0 423 * Only call this function if CPUID indicates that AVX feature is 424 * supported by the CPU, otherwise it might be an illegal instruction. 425 */ 426 static inline uint64_t 427 zfs_xgetbv(uint32_t index) 428 { 429 uint32_t eax, edx; 430 /* xgetbv - instruction byte code */ 431 __asm__ __volatile__(".byte 0x0f; .byte 0x01; .byte 0xd0" 432 : "=a" (eax), "=d" (edx) 433 : "c" (index)); 434 435 return ((((uint64_t)edx)<<32) | (uint64_t)eax); 436 } 437 438 439 static inline boolean_t 440 __simd_state_enabled(const uint64_t state) 441 { 442 boolean_t has_osxsave; 443 uint64_t xcr0; 444 445 #if defined(X86_FEATURE_OSXSAVE) 446 has_osxsave = !!boot_cpu_has(X86_FEATURE_OSXSAVE); 447 #else 448 has_osxsave = B_FALSE; 449 #endif 450 if (!has_osxsave) 451 return (B_FALSE); 452 453 xcr0 = zfs_xgetbv(0); 454 return ((xcr0 & state) == state); 455 } 456 457 #define _XSTATE_SSE_AVX (0x2 | 0x4) 458 #define _XSTATE_AVX512 (0xE0 | _XSTATE_SSE_AVX) 459 460 #define __ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX) 461 #define __zmm_enabled() __simd_state_enabled(_XSTATE_AVX512) 462 463 /* 464 * Check if SSE instruction set is available 465 */ 466 static inline boolean_t 467 zfs_sse_available(void) 468 { 469 return (!!boot_cpu_has(X86_FEATURE_XMM)); 470 } 471 472 /* 473 * Check if SSE2 instruction set is available 474 */ 475 static inline boolean_t 476 zfs_sse2_available(void) 477 { 478 return (!!boot_cpu_has(X86_FEATURE_XMM2)); 479 } 480 481 /* 482 * Check if SSE3 instruction set is available 483 */ 484 static inline boolean_t 485 zfs_sse3_available(void) 486 { 487 return (!!boot_cpu_has(X86_FEATURE_XMM3)); 488 } 489 490 /* 491 * Check if SSSE3 instruction set is available 492 */ 493 static inline boolean_t 494 zfs_ssse3_available(void) 495 { 496 return (!!boot_cpu_has(X86_FEATURE_SSSE3)); 497 } 498 499 /* 500 * Check if SSE4.1 instruction set is available 501 */ 502 static inline boolean_t 503 zfs_sse4_1_available(void) 504 { 505 return (!!boot_cpu_has(X86_FEATURE_XMM4_1)); 506 } 507 508 /* 509 * Check if SSE4.2 instruction set is available 510 */ 511 static inline boolean_t 512 zfs_sse4_2_available(void) 513 { 514 return (!!boot_cpu_has(X86_FEATURE_XMM4_2)); 515 } 516 517 /* 518 * Check if AVX instruction set is available 519 */ 520 static inline boolean_t 521 zfs_avx_available(void) 522 { 523 return (boot_cpu_has(X86_FEATURE_AVX) && __ymm_enabled()); 524 } 525 526 /* 527 * Check if AVX2 instruction set is available 528 */ 529 static inline boolean_t 530 zfs_avx2_available(void) 531 { 532 return (boot_cpu_has(X86_FEATURE_AVX2) && __ymm_enabled()); 533 } 534 535 /* 536 * Check if BMI1 instruction set is available 537 */ 538 static inline boolean_t 539 zfs_bmi1_available(void) 540 { 541 #if defined(X86_FEATURE_BMI1) 542 return (!!boot_cpu_has(X86_FEATURE_BMI1)); 543 #else 544 return (B_FALSE); 545 #endif 546 } 547 548 /* 549 * Check if BMI2 instruction set is available 550 */ 551 static inline boolean_t 552 zfs_bmi2_available(void) 553 { 554 #if defined(X86_FEATURE_BMI2) 555 return (!!boot_cpu_has(X86_FEATURE_BMI2)); 556 #else 557 return (B_FALSE); 558 #endif 559 } 560 561 /* 562 * Check if AES instruction set is available 563 */ 564 static inline boolean_t 565 zfs_aes_available(void) 566 { 567 #if defined(X86_FEATURE_AES) 568 return (!!boot_cpu_has(X86_FEATURE_AES)); 569 #else 570 return (B_FALSE); 571 #endif 572 } 573 574 /* 575 * Check if PCLMULQDQ instruction set is available 576 */ 577 static inline boolean_t 578 zfs_pclmulqdq_available(void) 579 { 580 #if defined(X86_FEATURE_PCLMULQDQ) 581 return (!!boot_cpu_has(X86_FEATURE_PCLMULQDQ)); 582 #else 583 return (B_FALSE); 584 #endif 585 } 586 587 /* 588 * Check if MOVBE instruction is available 589 */ 590 static inline boolean_t 591 zfs_movbe_available(void) 592 { 593 #if defined(X86_FEATURE_MOVBE) 594 return (!!boot_cpu_has(X86_FEATURE_MOVBE)); 595 #else 596 return (B_FALSE); 597 #endif 598 } 599 600 /* 601 * Check if SHA_NI instruction set is available 602 */ 603 static inline boolean_t 604 zfs_shani_available(void) 605 { 606 #if defined(X86_FEATURE_SHA_NI) 607 return (!!boot_cpu_has(X86_FEATURE_SHA_NI)); 608 #else 609 return (B_FALSE); 610 #endif 611 } 612 613 /* 614 * AVX-512 family of instruction sets: 615 * 616 * AVX512F Foundation 617 * AVX512CD Conflict Detection Instructions 618 * AVX512ER Exponential and Reciprocal Instructions 619 * AVX512PF Prefetch Instructions 620 * 621 * AVX512BW Byte and Word Instructions 622 * AVX512DQ Double-word and Quadword Instructions 623 * AVX512VL Vector Length Extensions 624 * 625 * AVX512IFMA Integer Fused Multiply Add (Not supported by kernel 4.4) 626 * AVX512VBMI Vector Byte Manipulation Instructions 627 */ 628 629 /* 630 * Check if AVX512F instruction set is available 631 */ 632 static inline boolean_t 633 zfs_avx512f_available(void) 634 { 635 boolean_t has_avx512 = B_FALSE; 636 637 #if defined(X86_FEATURE_AVX512F) 638 has_avx512 = !!boot_cpu_has(X86_FEATURE_AVX512F); 639 #endif 640 return (has_avx512 && __zmm_enabled()); 641 } 642 643 /* 644 * Check if AVX512CD instruction set is available 645 */ 646 static inline boolean_t 647 zfs_avx512cd_available(void) 648 { 649 boolean_t has_avx512 = B_FALSE; 650 651 #if defined(X86_FEATURE_AVX512CD) 652 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && 653 boot_cpu_has(X86_FEATURE_AVX512CD); 654 #endif 655 return (has_avx512 && __zmm_enabled()); 656 } 657 658 /* 659 * Check if AVX512ER instruction set is available 660 */ 661 static inline boolean_t 662 zfs_avx512er_available(void) 663 { 664 boolean_t has_avx512 = B_FALSE; 665 666 #if defined(X86_FEATURE_AVX512ER) 667 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && 668 boot_cpu_has(X86_FEATURE_AVX512ER); 669 #endif 670 return (has_avx512 && __zmm_enabled()); 671 } 672 673 /* 674 * Check if AVX512PF instruction set is available 675 */ 676 static inline boolean_t 677 zfs_avx512pf_available(void) 678 { 679 boolean_t has_avx512 = B_FALSE; 680 681 #if defined(X86_FEATURE_AVX512PF) 682 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && 683 boot_cpu_has(X86_FEATURE_AVX512PF); 684 #endif 685 return (has_avx512 && __zmm_enabled()); 686 } 687 688 /* 689 * Check if AVX512BW instruction set is available 690 */ 691 static inline boolean_t 692 zfs_avx512bw_available(void) 693 { 694 boolean_t has_avx512 = B_FALSE; 695 696 #if defined(X86_FEATURE_AVX512BW) 697 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && 698 boot_cpu_has(X86_FEATURE_AVX512BW); 699 #endif 700 701 return (has_avx512 && __zmm_enabled()); 702 } 703 704 /* 705 * Check if AVX512DQ instruction set is available 706 */ 707 static inline boolean_t 708 zfs_avx512dq_available(void) 709 { 710 boolean_t has_avx512 = B_FALSE; 711 712 #if defined(X86_FEATURE_AVX512DQ) 713 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && 714 boot_cpu_has(X86_FEATURE_AVX512DQ); 715 #endif 716 return (has_avx512 && __zmm_enabled()); 717 } 718 719 /* 720 * Check if AVX512VL instruction set is available 721 */ 722 static inline boolean_t 723 zfs_avx512vl_available(void) 724 { 725 boolean_t has_avx512 = B_FALSE; 726 727 #if defined(X86_FEATURE_AVX512VL) 728 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && 729 boot_cpu_has(X86_FEATURE_AVX512VL); 730 #endif 731 return (has_avx512 && __zmm_enabled()); 732 } 733 734 /* 735 * Check if AVX512IFMA instruction set is available 736 */ 737 static inline boolean_t 738 zfs_avx512ifma_available(void) 739 { 740 boolean_t has_avx512 = B_FALSE; 741 742 #if defined(X86_FEATURE_AVX512IFMA) 743 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && 744 boot_cpu_has(X86_FEATURE_AVX512IFMA); 745 #endif 746 return (has_avx512 && __zmm_enabled()); 747 } 748 749 /* 750 * Check if AVX512VBMI instruction set is available 751 */ 752 static inline boolean_t 753 zfs_avx512vbmi_available(void) 754 { 755 boolean_t has_avx512 = B_FALSE; 756 757 #if defined(X86_FEATURE_AVX512VBMI) 758 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && 759 boot_cpu_has(X86_FEATURE_AVX512VBMI); 760 #endif 761 return (has_avx512 && __zmm_enabled()); 762 } 763 764 #endif /* defined(__x86) */ 765 766 #endif /* _LINUX_SIMD_X86_H */ 767