1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (C) 2016 Gvozden Neskovic <neskovic@compeng.uni-frankfurt.de>.
23 */
24
25 /*
26 * USER API:
27 *
28 * Kernel fpu methods:
29 * kfpu_allowed()
30 * kfpu_begin()
31 * kfpu_end()
32 * kfpu_init()
33 * kfpu_fini()
34 *
35 * SIMD support:
36 *
37 * Following functions should be called to determine whether CPU feature
38 * is supported. All functions are usable in kernel and user space.
39 * If a SIMD algorithm is using more than one instruction set
40 * all relevant feature test functions should be called.
41 *
42 * Supported features:
43 * zfs_sse_available()
44 * zfs_sse2_available()
45 * zfs_sse3_available()
46 * zfs_ssse3_available()
47 * zfs_sse4_1_available()
48 * zfs_sse4_2_available()
49 *
50 * zfs_avx_available()
51 * zfs_avx2_available()
52 *
53 * zfs_bmi1_available()
54 * zfs_bmi2_available()
55 *
56 * zfs_shani_available()
57 *
58 * zfs_avx512f_available()
59 * zfs_avx512cd_available()
60 * zfs_avx512er_available()
61 * zfs_avx512pf_available()
62 * zfs_avx512bw_available()
63 * zfs_avx512dq_available()
64 * zfs_avx512vl_available()
65 * zfs_avx512ifma_available()
66 * zfs_avx512vbmi_available()
67 *
68 * NOTE(AVX-512VL): If using AVX-512 instructions with 128Bit registers
69 * also add zfs_avx512vl_available() to feature check.
70 */
71
72 #ifndef _LINUX_SIMD_X86_H
73 #define _LINUX_SIMD_X86_H
74
75 /* only for __x86 */
76 #if defined(__x86)
77
78 #include <sys/types.h>
79 #include <asm/cpufeature.h>
80
81 /*
82 * Disable the WARN_ON_FPU() macro to prevent additional dependencies
83 * when providing the kfpu_* functions. Relevant warnings are included
84 * as appropriate and are unconditionally enabled.
85 */
86 #if defined(CONFIG_X86_DEBUG_FPU) && !defined(KERNEL_EXPORTS_X86_FPU)
87 #undef CONFIG_X86_DEBUG_FPU
88 #endif
89
90 /*
91 * The following cases are for kernels which export either the
92 * kernel_fpu_* or __kernel_fpu_* functions.
93 */
94 #if defined(KERNEL_EXPORTS_X86_FPU)
95
96 #if defined(HAVE_KERNEL_FPU_API_HEADER)
97 #include <asm/fpu/api.h>
98 #if defined(HAVE_KERNEL_FPU_INTERNAL_HEADER)
99 #include <asm/fpu/internal.h>
100 #endif
101 #else
102 #include <asm/i387.h>
103 #endif
104
105 #define kfpu_allowed() 1
106 #define kfpu_init() 0
107 #define kfpu_fini() ((void) 0)
108
109 #if defined(HAVE_UNDERSCORE_KERNEL_FPU)
110 #define kfpu_begin() \
111 { \
112 preempt_disable(); \
113 __kernel_fpu_begin(); \
114 }
115 #define kfpu_end() \
116 { \
117 __kernel_fpu_end(); \
118 preempt_enable(); \
119 }
120
121 #elif defined(HAVE_KERNEL_FPU)
122 #define kfpu_begin() kernel_fpu_begin()
123 #define kfpu_end() kernel_fpu_end()
124
125 #else
126 /*
127 * This case is unreachable. When KERNEL_EXPORTS_X86_FPU is defined then
128 * either HAVE_UNDERSCORE_KERNEL_FPU or HAVE_KERNEL_FPU must be defined.
129 */
130 #error "Unreachable kernel configuration"
131 #endif
132
133 #else /* defined(KERNEL_EXPORTS_X86_FPU) */
134
135 /*
136 * When the kernel_fpu_* symbols are unavailable then provide our own
137 * versions which allow the FPU to be safely used.
138 */
139 #if defined(HAVE_KERNEL_FPU_INTERNAL)
140
141 /*
142 * For kernels not exporting *kfpu_{begin,end} we have to use inline assembly
143 * with the XSAVE{,OPT,S} instructions, so we need the toolchain to support at
144 * least XSAVE.
145 */
146 #if !defined(HAVE_XSAVE)
147 #error "Toolchain needs to support the XSAVE assembler instruction"
148 #endif
149
150 #ifndef XFEATURE_MASK_XTILE
151 /*
152 * For kernels where this doesn't exist yet, we still don't want to break
153 * by save/restoring this broken nonsense.
154 * See issue #14989 or Intel errata SPR4 for why
155 */
156 #define XFEATURE_MASK_XTILE 0x60000
157 #endif
158
159 #include <linux/mm.h>
160 #include <linux/slab.h>
161
162 extern uint8_t **zfs_kfpu_fpregs;
163
164 /*
165 * Return the size in bytes required by the XSAVE instruction for an
166 * XSAVE area containing all the user state components supported by this CPU.
167 * See: Intel 64 and IA-32 Architectures Software Developer’s Manual.
168 * Dec. 2021. Vol. 2A p. 3-222.
169 */
170 static inline uint32_t
get_xsave_area_size(void)171 get_xsave_area_size(void)
172 {
173 if (!boot_cpu_has(X86_FEATURE_OSXSAVE)) {
174 return (0);
175 }
176 /*
177 * Call CPUID with leaf 13 and subleaf 0. The size is in ecx.
178 * We don't need to check for cpuid_max here, since if this CPU has
179 * OSXSAVE set, it has leaf 13 (0x0D) as well.
180 */
181 uint32_t eax, ebx, ecx, edx;
182
183 eax = 13U;
184 ecx = 0U;
185 __asm__ __volatile__("cpuid"
186 : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
187 : "a" (eax), "c" (ecx));
188
189 return (ecx);
190 }
191
192 /*
193 * Return the allocation order of the maximum buffer size required to save the
194 * FPU state on this architecture. The value returned is the same as Linux'
195 * get_order() function would return (i.e. 2^order = nr. of pages required).
196 * Currently this will always return 0 since the save area is below 4k even for
197 * a full fledged AVX-512 implementation.
198 */
199 static inline int
get_fpuregs_save_area_order(void)200 get_fpuregs_save_area_order(void)
201 {
202 size_t area_size = (size_t)get_xsave_area_size();
203
204 /*
205 * If we are dealing with a CPU not supporting XSAVE,
206 * get_xsave_area_size() will return 0. Thus the maximum memory
207 * required is the FXSAVE area size which is 512 bytes. See: Intel 64
208 * and IA-32 Architectures Software Developer’s Manual. Dec. 2021.
209 * Vol. 2A p. 3-451.
210 */
211 if (area_size == 0) {
212 area_size = 512;
213 }
214 return (get_order(area_size));
215 }
216
217 /*
218 * Initialize per-cpu variables to store FPU state.
219 */
220 static inline void
kfpu_fini(void)221 kfpu_fini(void)
222 {
223 int cpu;
224 int order = get_fpuregs_save_area_order();
225
226 for_each_possible_cpu(cpu) {
227 if (zfs_kfpu_fpregs[cpu] != NULL) {
228 free_pages((unsigned long)zfs_kfpu_fpregs[cpu], order);
229 }
230 }
231
232 kfree(zfs_kfpu_fpregs);
233 }
234
235 static inline int
kfpu_init(void)236 kfpu_init(void)
237 {
238 zfs_kfpu_fpregs = kzalloc(num_possible_cpus() * sizeof (uint8_t *),
239 GFP_KERNEL);
240
241 if (zfs_kfpu_fpregs == NULL)
242 return (-ENOMEM);
243
244 /*
245 * The fxsave and xsave operations require 16-/64-byte alignment of
246 * the target memory. Since kmalloc() provides no alignment
247 * guarantee instead use alloc_pages_node().
248 */
249 int cpu;
250 int order = get_fpuregs_save_area_order();
251
252 for_each_possible_cpu(cpu) {
253 struct page *page = alloc_pages_node(cpu_to_node(cpu),
254 GFP_KERNEL | __GFP_ZERO, order);
255 if (page == NULL) {
256 kfpu_fini();
257 return (-ENOMEM);
258 }
259
260 zfs_kfpu_fpregs[cpu] = page_address(page);
261 }
262
263 return (0);
264 }
265
266 #define kfpu_allowed() 1
267
268 /*
269 * FPU save and restore instructions.
270 */
271 #define __asm __asm__ __volatile__
272 #define kfpu_fxsave(addr) __asm("fxsave %0" : "=m" (*(addr)))
273 #define kfpu_fxsaveq(addr) __asm("fxsaveq %0" : "=m" (*(addr)))
274 #define kfpu_fnsave(addr) __asm("fnsave %0; fwait" : "=m" (*(addr)))
275 #define kfpu_fxrstor(addr) __asm("fxrstor %0" : : "m" (*(addr)))
276 #define kfpu_fxrstorq(addr) __asm("fxrstorq %0" : : "m" (*(addr)))
277 #define kfpu_frstor(addr) __asm("frstor %0" : : "m" (*(addr)))
278 #define kfpu_fxsr_clean(rval) __asm("fnclex; emms; fildl %P[addr]" \
279 : : [addr] "m" (rval));
280
281 #define kfpu_do_xsave(instruction, addr, mask) \
282 { \
283 uint32_t low, hi; \
284 \
285 low = mask; \
286 hi = (uint64_t)(mask) >> 32; \
287 __asm(instruction " %[dst]\n\t" \
288 : \
289 : [dst] "m" (*(addr)), "a" (low), "d" (hi) \
290 : "memory"); \
291 }
292
293 static inline void
kfpu_save_fxsr(uint8_t * addr)294 kfpu_save_fxsr(uint8_t *addr)
295 {
296 if (IS_ENABLED(CONFIG_X86_32))
297 kfpu_fxsave(addr);
298 else
299 kfpu_fxsaveq(addr);
300 }
301
302 static inline void
kfpu_save_fsave(uint8_t * addr)303 kfpu_save_fsave(uint8_t *addr)
304 {
305 kfpu_fnsave(addr);
306 }
307
308 static inline void
kfpu_begin(void)309 kfpu_begin(void)
310 {
311 /*
312 * Preemption and interrupts must be disabled for the critical
313 * region where the FPU state is being modified.
314 */
315 preempt_disable();
316 local_irq_disable();
317
318 /*
319 * The current FPU registers need to be preserved by kfpu_begin()
320 * and restored by kfpu_end(). They are stored in a dedicated
321 * per-cpu variable, not in the task struct, this allows any user
322 * FPU state to be correctly preserved and restored.
323 */
324 uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()];
325 #if defined(HAVE_XSAVES)
326 if (static_cpu_has(X86_FEATURE_XSAVES)) {
327 kfpu_do_xsave("xsaves", state, ~XFEATURE_MASK_XTILE);
328 return;
329 }
330 #endif
331 #if defined(HAVE_XSAVEOPT)
332 if (static_cpu_has(X86_FEATURE_XSAVEOPT)) {
333 kfpu_do_xsave("xsaveopt", state, ~XFEATURE_MASK_XTILE);
334 return;
335 }
336 #endif
337 if (static_cpu_has(X86_FEATURE_XSAVE)) {
338 kfpu_do_xsave("xsave", state, ~XFEATURE_MASK_XTILE);
339 } else if (static_cpu_has(X86_FEATURE_FXSR)) {
340 kfpu_save_fxsr(state);
341 } else {
342 kfpu_save_fsave(state);
343 }
344 }
345
346 #define kfpu_do_xrstor(instruction, addr, mask) \
347 { \
348 uint32_t low, hi; \
349 \
350 low = mask; \
351 hi = (uint64_t)(mask) >> 32; \
352 __asm(instruction " %[src]" \
353 : \
354 : [src] "m" (*(addr)), "a" (low), "d" (hi) \
355 : "memory"); \
356 }
357
358 static inline void
kfpu_restore_fxsr(uint8_t * addr)359 kfpu_restore_fxsr(uint8_t *addr)
360 {
361 /*
362 * On AuthenticAMD K7 and K8 processors the fxrstor instruction only
363 * restores the _x87 FOP, FIP, and FDP registers when an exception
364 * is pending. Clean the _x87 state to force the restore.
365 */
366 if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK)))
367 kfpu_fxsr_clean(addr);
368
369 if (IS_ENABLED(CONFIG_X86_32)) {
370 kfpu_fxrstor(addr);
371 } else {
372 kfpu_fxrstorq(addr);
373 }
374 }
375
376 static inline void
kfpu_restore_fsave(uint8_t * addr)377 kfpu_restore_fsave(uint8_t *addr)
378 {
379 kfpu_frstor(addr);
380 }
381
382 static inline void
kfpu_end(void)383 kfpu_end(void)
384 {
385 uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()];
386 #if defined(HAVE_XSAVES)
387 if (static_cpu_has(X86_FEATURE_XSAVES)) {
388 kfpu_do_xrstor("xrstors", state, ~XFEATURE_MASK_XTILE);
389 goto out;
390 }
391 #endif
392 if (static_cpu_has(X86_FEATURE_XSAVE)) {
393 kfpu_do_xrstor("xrstor", state, ~XFEATURE_MASK_XTILE);
394 } else if (static_cpu_has(X86_FEATURE_FXSR)) {
395 kfpu_restore_fxsr(state);
396 } else {
397 kfpu_restore_fsave(state);
398 }
399 out:
400 local_irq_enable();
401 preempt_enable();
402
403 }
404
405 #else
406
407 #error "Exactly one of KERNEL_EXPORTS_X86_FPU or HAVE_KERNEL_FPU_INTERNAL" \
408 " must be defined"
409
410 #endif /* defined(HAVE_KERNEL_FPU_INTERNAL */
411 #endif /* defined(KERNEL_EXPORTS_X86_FPU) */
412
413 /*
414 * Linux kernel provides an interface for CPU feature testing.
415 */
416
417 /*
418 * Detect register set support
419 */
420
421 /*
422 * Check if OS supports AVX and AVX2 by checking XCR0
423 * Only call this function if CPUID indicates that AVX feature is
424 * supported by the CPU, otherwise it might be an illegal instruction.
425 */
426 static inline uint64_t
zfs_xgetbv(uint32_t index)427 zfs_xgetbv(uint32_t index)
428 {
429 uint32_t eax, edx;
430 /* xgetbv - instruction byte code */
431 __asm__ __volatile__(".byte 0x0f; .byte 0x01; .byte 0xd0"
432 : "=a" (eax), "=d" (edx)
433 : "c" (index));
434
435 return ((((uint64_t)edx)<<32) | (uint64_t)eax);
436 }
437
438
439 static inline boolean_t
__simd_state_enabled(const uint64_t state)440 __simd_state_enabled(const uint64_t state)
441 {
442 boolean_t has_osxsave;
443 uint64_t xcr0;
444
445 #if defined(X86_FEATURE_OSXSAVE)
446 has_osxsave = !!boot_cpu_has(X86_FEATURE_OSXSAVE);
447 #else
448 has_osxsave = B_FALSE;
449 #endif
450 if (!has_osxsave)
451 return (B_FALSE);
452
453 xcr0 = zfs_xgetbv(0);
454 return ((xcr0 & state) == state);
455 }
456
457 #define _XSTATE_SSE_AVX (0x2 | 0x4)
458 #define _XSTATE_AVX512 (0xE0 | _XSTATE_SSE_AVX)
459
460 #define __ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX)
461 #define __zmm_enabled() __simd_state_enabled(_XSTATE_AVX512)
462
463 /*
464 * Check if SSE instruction set is available
465 */
466 static inline boolean_t
zfs_sse_available(void)467 zfs_sse_available(void)
468 {
469 return (!!boot_cpu_has(X86_FEATURE_XMM));
470 }
471
472 /*
473 * Check if SSE2 instruction set is available
474 */
475 static inline boolean_t
zfs_sse2_available(void)476 zfs_sse2_available(void)
477 {
478 return (!!boot_cpu_has(X86_FEATURE_XMM2));
479 }
480
481 /*
482 * Check if SSE3 instruction set is available
483 */
484 static inline boolean_t
zfs_sse3_available(void)485 zfs_sse3_available(void)
486 {
487 return (!!boot_cpu_has(X86_FEATURE_XMM3));
488 }
489
490 /*
491 * Check if SSSE3 instruction set is available
492 */
493 static inline boolean_t
zfs_ssse3_available(void)494 zfs_ssse3_available(void)
495 {
496 return (!!boot_cpu_has(X86_FEATURE_SSSE3));
497 }
498
499 /*
500 * Check if SSE4.1 instruction set is available
501 */
502 static inline boolean_t
zfs_sse4_1_available(void)503 zfs_sse4_1_available(void)
504 {
505 return (!!boot_cpu_has(X86_FEATURE_XMM4_1));
506 }
507
508 /*
509 * Check if SSE4.2 instruction set is available
510 */
511 static inline boolean_t
zfs_sse4_2_available(void)512 zfs_sse4_2_available(void)
513 {
514 return (!!boot_cpu_has(X86_FEATURE_XMM4_2));
515 }
516
517 /*
518 * Check if AVX instruction set is available
519 */
520 static inline boolean_t
zfs_avx_available(void)521 zfs_avx_available(void)
522 {
523 return (boot_cpu_has(X86_FEATURE_AVX) && __ymm_enabled());
524 }
525
526 /*
527 * Check if AVX2 instruction set is available
528 */
529 static inline boolean_t
zfs_avx2_available(void)530 zfs_avx2_available(void)
531 {
532 return (boot_cpu_has(X86_FEATURE_AVX2) && __ymm_enabled());
533 }
534
535 /*
536 * Check if BMI1 instruction set is available
537 */
538 static inline boolean_t
zfs_bmi1_available(void)539 zfs_bmi1_available(void)
540 {
541 #if defined(X86_FEATURE_BMI1)
542 return (!!boot_cpu_has(X86_FEATURE_BMI1));
543 #else
544 return (B_FALSE);
545 #endif
546 }
547
548 /*
549 * Check if BMI2 instruction set is available
550 */
551 static inline boolean_t
zfs_bmi2_available(void)552 zfs_bmi2_available(void)
553 {
554 #if defined(X86_FEATURE_BMI2)
555 return (!!boot_cpu_has(X86_FEATURE_BMI2));
556 #else
557 return (B_FALSE);
558 #endif
559 }
560
561 /*
562 * Check if AES instruction set is available
563 */
564 static inline boolean_t
zfs_aes_available(void)565 zfs_aes_available(void)
566 {
567 #if defined(X86_FEATURE_AES)
568 return (!!boot_cpu_has(X86_FEATURE_AES));
569 #else
570 return (B_FALSE);
571 #endif
572 }
573
574 /*
575 * Check if PCLMULQDQ instruction set is available
576 */
577 static inline boolean_t
zfs_pclmulqdq_available(void)578 zfs_pclmulqdq_available(void)
579 {
580 #if defined(X86_FEATURE_PCLMULQDQ)
581 return (!!boot_cpu_has(X86_FEATURE_PCLMULQDQ));
582 #else
583 return (B_FALSE);
584 #endif
585 }
586
587 /*
588 * Check if MOVBE instruction is available
589 */
590 static inline boolean_t
zfs_movbe_available(void)591 zfs_movbe_available(void)
592 {
593 #if defined(X86_FEATURE_MOVBE)
594 return (!!boot_cpu_has(X86_FEATURE_MOVBE));
595 #else
596 return (B_FALSE);
597 #endif
598 }
599
600 /*
601 * Check if SHA_NI instruction set is available
602 */
603 static inline boolean_t
zfs_shani_available(void)604 zfs_shani_available(void)
605 {
606 #if defined(X86_FEATURE_SHA_NI)
607 return (!!boot_cpu_has(X86_FEATURE_SHA_NI));
608 #else
609 return (B_FALSE);
610 #endif
611 }
612
613 /*
614 * AVX-512 family of instruction sets:
615 *
616 * AVX512F Foundation
617 * AVX512CD Conflict Detection Instructions
618 * AVX512ER Exponential and Reciprocal Instructions
619 * AVX512PF Prefetch Instructions
620 *
621 * AVX512BW Byte and Word Instructions
622 * AVX512DQ Double-word and Quadword Instructions
623 * AVX512VL Vector Length Extensions
624 *
625 * AVX512IFMA Integer Fused Multiply Add (Not supported by kernel 4.4)
626 * AVX512VBMI Vector Byte Manipulation Instructions
627 */
628
629 /*
630 * Check if AVX512F instruction set is available
631 */
632 static inline boolean_t
zfs_avx512f_available(void)633 zfs_avx512f_available(void)
634 {
635 boolean_t has_avx512 = B_FALSE;
636
637 #if defined(X86_FEATURE_AVX512F)
638 has_avx512 = !!boot_cpu_has(X86_FEATURE_AVX512F);
639 #endif
640 return (has_avx512 && __zmm_enabled());
641 }
642
643 /*
644 * Check if AVX512CD instruction set is available
645 */
646 static inline boolean_t
zfs_avx512cd_available(void)647 zfs_avx512cd_available(void)
648 {
649 boolean_t has_avx512 = B_FALSE;
650
651 #if defined(X86_FEATURE_AVX512CD)
652 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
653 boot_cpu_has(X86_FEATURE_AVX512CD);
654 #endif
655 return (has_avx512 && __zmm_enabled());
656 }
657
658 /*
659 * Check if AVX512ER instruction set is available
660 */
661 static inline boolean_t
zfs_avx512er_available(void)662 zfs_avx512er_available(void)
663 {
664 boolean_t has_avx512 = B_FALSE;
665
666 #if defined(X86_FEATURE_AVX512ER)
667 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
668 boot_cpu_has(X86_FEATURE_AVX512ER);
669 #endif
670 return (has_avx512 && __zmm_enabled());
671 }
672
673 /*
674 * Check if AVX512PF instruction set is available
675 */
676 static inline boolean_t
zfs_avx512pf_available(void)677 zfs_avx512pf_available(void)
678 {
679 boolean_t has_avx512 = B_FALSE;
680
681 #if defined(X86_FEATURE_AVX512PF)
682 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
683 boot_cpu_has(X86_FEATURE_AVX512PF);
684 #endif
685 return (has_avx512 && __zmm_enabled());
686 }
687
688 /*
689 * Check if AVX512BW instruction set is available
690 */
691 static inline boolean_t
zfs_avx512bw_available(void)692 zfs_avx512bw_available(void)
693 {
694 boolean_t has_avx512 = B_FALSE;
695
696 #if defined(X86_FEATURE_AVX512BW)
697 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
698 boot_cpu_has(X86_FEATURE_AVX512BW);
699 #endif
700
701 return (has_avx512 && __zmm_enabled());
702 }
703
704 /*
705 * Check if AVX512DQ instruction set is available
706 */
707 static inline boolean_t
zfs_avx512dq_available(void)708 zfs_avx512dq_available(void)
709 {
710 boolean_t has_avx512 = B_FALSE;
711
712 #if defined(X86_FEATURE_AVX512DQ)
713 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
714 boot_cpu_has(X86_FEATURE_AVX512DQ);
715 #endif
716 return (has_avx512 && __zmm_enabled());
717 }
718
719 /*
720 * Check if AVX512VL instruction set is available
721 */
722 static inline boolean_t
zfs_avx512vl_available(void)723 zfs_avx512vl_available(void)
724 {
725 boolean_t has_avx512 = B_FALSE;
726
727 #if defined(X86_FEATURE_AVX512VL)
728 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
729 boot_cpu_has(X86_FEATURE_AVX512VL);
730 #endif
731 return (has_avx512 && __zmm_enabled());
732 }
733
734 /*
735 * Check if AVX512IFMA instruction set is available
736 */
737 static inline boolean_t
zfs_avx512ifma_available(void)738 zfs_avx512ifma_available(void)
739 {
740 boolean_t has_avx512 = B_FALSE;
741
742 #if defined(X86_FEATURE_AVX512IFMA)
743 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
744 boot_cpu_has(X86_FEATURE_AVX512IFMA);
745 #endif
746 return (has_avx512 && __zmm_enabled());
747 }
748
749 /*
750 * Check if AVX512VBMI instruction set is available
751 */
752 static inline boolean_t
zfs_avx512vbmi_available(void)753 zfs_avx512vbmi_available(void)
754 {
755 boolean_t has_avx512 = B_FALSE;
756
757 #if defined(X86_FEATURE_AVX512VBMI)
758 has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
759 boot_cpu_has(X86_FEATURE_AVX512VBMI);
760 #endif
761 return (has_avx512 && __zmm_enabled());
762 }
763
764 #endif /* defined(__x86) */
765
766 #endif /* _LINUX_SIMD_X86_H */
767