xref: /illumos-gate/usr/src/uts/common/fs/zfs/sys/simd.h (revision 4c827c36)
1e86372a0SGvozden Neskovic /*
2e86372a0SGvozden Neskovic  * This file and its contents are supplied under the terms of the
3e86372a0SGvozden Neskovic  * Common Development and Distribution License ("CDDL"), version 1.0.
4e86372a0SGvozden Neskovic  * You may only use this file in accordance with the terms of version
5e86372a0SGvozden Neskovic  * 1.0 of the CDDL.
6e86372a0SGvozden Neskovic  *
7e86372a0SGvozden Neskovic  * A full copy of the text of the CDDL should have accompanied this
8e86372a0SGvozden Neskovic  * source.  A copy of the CDDL is also available via the Internet at
9e86372a0SGvozden Neskovic  * http://www.illumos.org/license/CDDL.
10e86372a0SGvozden Neskovic  */
11e86372a0SGvozden Neskovic 
12e86372a0SGvozden Neskovic /*
13e86372a0SGvozden Neskovic  * Copyright 2020 Joyent, Inc.
14*4c827c36SAndy Fiddaman  * Copyright 2024 Oxide Computer Company
15e86372a0SGvozden Neskovic  */
16e86372a0SGvozden Neskovic 
17e86372a0SGvozden Neskovic #ifndef _SIMD_H
18e86372a0SGvozden Neskovic #define	_SIMD_H
19e86372a0SGvozden Neskovic 
20e86372a0SGvozden Neskovic #if defined(__amd64__) || defined(__i386__)
21e86372a0SGvozden Neskovic 
22e86372a0SGvozden Neskovic #define	kfpu_initialize(tsk)	do {} while (0)
23e86372a0SGvozden Neskovic #define	kfpu_init()		(0)
24e86372a0SGvozden Neskovic #define	kfpu_fini()		do {} while (0)
25e86372a0SGvozden Neskovic 
26f91a4547SGvozden Neskovic #ifdef _KERNEL
27f91a4547SGvozden Neskovic #include <sys/x86_archext.h>
28f91a4547SGvozden Neskovic #include <sys/archsystm.h>
29*4c827c36SAndy Fiddaman #include <sys/systm.h>
30f91a4547SGvozden Neskovic #include <sys/kfpu.h>
31f91a4547SGvozden Neskovic #include <sys/proc.h>
32f91a4547SGvozden Neskovic #include <sys/disp.h>
33f91a4547SGvozden Neskovic #include <sys/cpuvar.h>
34f91a4547SGvozden Neskovic 
35fd440315SJoshua M. Clulow static inline int
kfpu_allowed(void)36fd440315SJoshua M. Clulow kfpu_allowed(void)
37fd440315SJoshua M. Clulow {
38fd440315SJoshua M. Clulow 	extern int zfs_fpu_enabled;
39fd440315SJoshua M. Clulow 
40*4c827c36SAndy Fiddaman 	/*
41*4c827c36SAndy Fiddaman 	 * When panicking, play it safe and avoid kfpu use. This gives the best
42*4c827c36SAndy Fiddaman 	 * chance of being able to dump successfully, particularly if the panic
43*4c827c36SAndy Fiddaman 	 * occured around an FPU context switch.
44*4c827c36SAndy Fiddaman 	 */
45*4c827c36SAndy Fiddaman 	if (panicstr != NULL)
46*4c827c36SAndy Fiddaman 		return (0);
47*4c827c36SAndy Fiddaman 
48fd440315SJoshua M. Clulow 	return (zfs_fpu_enabled != 0 ? 1 : 0);
49fd440315SJoshua M. Clulow }
50fd440315SJoshua M. Clulow 
51f91a4547SGvozden Neskovic static inline void
kfpu_begin(void)52f91a4547SGvozden Neskovic kfpu_begin(void)
53f91a4547SGvozden Neskovic {
54f91a4547SGvozden Neskovic 	if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) {
55f91a4547SGvozden Neskovic 		kernel_fpu_begin(NULL, KFPU_USE_LWP);
56f91a4547SGvozden Neskovic 	} else {
57f91a4547SGvozden Neskovic 		kpreempt_disable();
58f91a4547SGvozden Neskovic 		kernel_fpu_begin(NULL, KFPU_NO_STATE);
59f91a4547SGvozden Neskovic 	}
60f91a4547SGvozden Neskovic }
61f91a4547SGvozden Neskovic 
62f91a4547SGvozden Neskovic static inline void
kfpu_end(void)63f91a4547SGvozden Neskovic kfpu_end(void)
64f91a4547SGvozden Neskovic {
65f91a4547SGvozden Neskovic 	if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) {
66f91a4547SGvozden Neskovic 		kernel_fpu_end(NULL, KFPU_USE_LWP);
67f91a4547SGvozden Neskovic 	} else {
68f91a4547SGvozden Neskovic 		kernel_fpu_end(NULL, KFPU_NO_STATE);
69f91a4547SGvozden Neskovic 		kpreempt_enable();
70f91a4547SGvozden Neskovic 	}
71f91a4547SGvozden Neskovic }
72f91a4547SGvozden Neskovic 
73f91a4547SGvozden Neskovic /*
74f91a4547SGvozden Neskovic  * Check if various vector instruction sets are available.
75f91a4547SGvozden Neskovic  */
76f91a4547SGvozden Neskovic 
77f91a4547SGvozden Neskovic static inline boolean_t
zfs_sse_available(void)78f91a4547SGvozden Neskovic zfs_sse_available(void)
79f91a4547SGvozden Neskovic {
80f91a4547SGvozden Neskovic 	return (is_x86_feature(x86_featureset, X86FSET_SSE));
81f91a4547SGvozden Neskovic }
82f91a4547SGvozden Neskovic 
83f91a4547SGvozden Neskovic static inline boolean_t
zfs_sse2_available(void)84f91a4547SGvozden Neskovic zfs_sse2_available(void)
85f91a4547SGvozden Neskovic {
86f91a4547SGvozden Neskovic 	return (is_x86_feature(x86_featureset, X86FSET_SSE2));
87f91a4547SGvozden Neskovic }
88f91a4547SGvozden Neskovic 
89f91a4547SGvozden Neskovic static inline boolean_t
zfs_sse3_available(void)90f91a4547SGvozden Neskovic zfs_sse3_available(void)
91f91a4547SGvozden Neskovic {
92f91a4547SGvozden Neskovic 	return (is_x86_feature(x86_featureset, X86FSET_SSE3));
93f91a4547SGvozden Neskovic }
94f91a4547SGvozden Neskovic 
95f91a4547SGvozden Neskovic static inline boolean_t
zfs_ssse3_available(void)96f91a4547SGvozden Neskovic zfs_ssse3_available(void)
97f91a4547SGvozden Neskovic {
98f91a4547SGvozden Neskovic 	return (is_x86_feature(x86_featureset, X86FSET_SSSE3));
99f91a4547SGvozden Neskovic }
100f91a4547SGvozden Neskovic 
101f91a4547SGvozden Neskovic static inline boolean_t
zfs_avx_available(void)102f91a4547SGvozden Neskovic zfs_avx_available(void)
103f91a4547SGvozden Neskovic {
104f91a4547SGvozden Neskovic 	return (is_x86_feature(x86_featureset, X86FSET_AVX));
105f91a4547SGvozden Neskovic }
106f91a4547SGvozden Neskovic 
107f91a4547SGvozden Neskovic static inline boolean_t
zfs_avx2_available(void)108f91a4547SGvozden Neskovic zfs_avx2_available(void)
109f91a4547SGvozden Neskovic {
110f91a4547SGvozden Neskovic 	return (is_x86_feature(x86_featureset, X86FSET_AVX2));
111f91a4547SGvozden Neskovic }
112f91a4547SGvozden Neskovic 
1130886dcadSAndy Fiddaman static inline boolean_t
zfs_avx512f_available(void)1140886dcadSAndy Fiddaman zfs_avx512f_available(void)
1150886dcadSAndy Fiddaman {
1160886dcadSAndy Fiddaman 	return (is_x86_feature(x86_featureset, X86FSET_AVX512F));
1170886dcadSAndy Fiddaman }
1180886dcadSAndy Fiddaman 
1190886dcadSAndy Fiddaman static inline boolean_t
zfs_avx512bw_available(void)1200886dcadSAndy Fiddaman zfs_avx512bw_available(void)
1210886dcadSAndy Fiddaman {
1220886dcadSAndy Fiddaman 	return (is_x86_feature(x86_featureset, X86FSET_AVX512BW));
1230886dcadSAndy Fiddaman }
1240886dcadSAndy Fiddaman 
125f91a4547SGvozden Neskovic #else	/* ! _KERNEL */
126f91a4547SGvozden Neskovic 
127f91a4547SGvozden Neskovic #include <sys/auxv.h>
128f91a4547SGvozden Neskovic #include <sys/auxv_386.h>
129f91a4547SGvozden Neskovic 
130fd440315SJoshua M. Clulow #define	kfpu_allowed()		1
131f91a4547SGvozden Neskovic #define	kfpu_begin()		do {} while (0)
132f91a4547SGvozden Neskovic #define	kfpu_end()		do {} while (0)
133f91a4547SGvozden Neskovic 
134f91a4547SGvozden Neskovic /*
135f91a4547SGvozden Neskovic  * User-level check if various vector instruction sets are available.
136f91a4547SGvozden Neskovic  */
137f91a4547SGvozden Neskovic 
138f91a4547SGvozden Neskovic static inline boolean_t
zfs_sse_available(void)139f91a4547SGvozden Neskovic zfs_sse_available(void)
140f91a4547SGvozden Neskovic {
141f91a4547SGvozden Neskovic 	uint32_t u = 0;
142f91a4547SGvozden Neskovic 
143f91a4547SGvozden Neskovic 	(void) getisax(&u, 1);
144f91a4547SGvozden Neskovic 	return ((u & AV_386_SSE) != 0);
145f91a4547SGvozden Neskovic }
146f91a4547SGvozden Neskovic 
147f91a4547SGvozden Neskovic static inline boolean_t
zfs_sse2_available(void)148f91a4547SGvozden Neskovic zfs_sse2_available(void)
149f91a4547SGvozden Neskovic {
150f91a4547SGvozden Neskovic 	uint32_t u = 0;
151f91a4547SGvozden Neskovic 
152f91a4547SGvozden Neskovic 	(void) getisax(&u, 1);
153f91a4547SGvozden Neskovic 	return ((u & AV_386_SSE2) != 0);
154f91a4547SGvozden Neskovic }
155f91a4547SGvozden Neskovic 
156f91a4547SGvozden Neskovic static inline boolean_t
zfs_sse3_available(void)157f91a4547SGvozden Neskovic zfs_sse3_available(void)
158f91a4547SGvozden Neskovic {
159f91a4547SGvozden Neskovic 	uint32_t u = 0;
160f91a4547SGvozden Neskovic 
161f91a4547SGvozden Neskovic 	(void) getisax(&u, 1);
162f91a4547SGvozden Neskovic 	return ((u & AV_386_SSE3) != 0);
163f91a4547SGvozden Neskovic }
164f91a4547SGvozden Neskovic 
165f91a4547SGvozden Neskovic static inline boolean_t
zfs_ssse3_available(void)166f91a4547SGvozden Neskovic zfs_ssse3_available(void)
167f91a4547SGvozden Neskovic {
168f91a4547SGvozden Neskovic 	uint32_t u = 0;
169f91a4547SGvozden Neskovic 
170f91a4547SGvozden Neskovic 	(void) getisax(&u, 1);
171f91a4547SGvozden Neskovic 	return ((u & AV_386_SSSE3) != 0);
172f91a4547SGvozden Neskovic }
173f91a4547SGvozden Neskovic 
174f91a4547SGvozden Neskovic static inline boolean_t
zfs_avx_available(void)175f91a4547SGvozden Neskovic zfs_avx_available(void)
176f91a4547SGvozden Neskovic {
177f91a4547SGvozden Neskovic 	uint_t u = 0;
178f91a4547SGvozden Neskovic 
179f91a4547SGvozden Neskovic 	(void) getisax(&u, 1);
180f91a4547SGvozden Neskovic 	return ((u & AV_386_AVX) != 0);
181f91a4547SGvozden Neskovic }
182f91a4547SGvozden Neskovic 
183f91a4547SGvozden Neskovic static inline boolean_t
zfs_avx2_available(void)184f91a4547SGvozden Neskovic zfs_avx2_available(void)
185f91a4547SGvozden Neskovic {
186f91a4547SGvozden Neskovic 	uint32_t u[2] = { 0 };
187f91a4547SGvozden Neskovic 
188f91a4547SGvozden Neskovic 	(void) getisax((uint32_t *)&u, 2);
189f91a4547SGvozden Neskovic 	return ((u[1] & AV_386_2_AVX2) != 0);
190f91a4547SGvozden Neskovic }
191f91a4547SGvozden Neskovic 
1920886dcadSAndy Fiddaman static inline boolean_t
zfs_avx512f_available(void)1930886dcadSAndy Fiddaman zfs_avx512f_available(void)
1940886dcadSAndy Fiddaman {
1950886dcadSAndy Fiddaman 	uint32_t u[2] = { 0 };
1960886dcadSAndy Fiddaman 
1970886dcadSAndy Fiddaman 	(void) getisax((uint32_t *)&u, 2);
1980886dcadSAndy Fiddaman 	return ((u[1] & AV_386_2_AVX512F) != 0);
1990886dcadSAndy Fiddaman }
2000886dcadSAndy Fiddaman 
2010886dcadSAndy Fiddaman static inline boolean_t
zfs_avx512bw_available(void)2020886dcadSAndy Fiddaman zfs_avx512bw_available(void)
2030886dcadSAndy Fiddaman {
2040886dcadSAndy Fiddaman 	uint32_t u[2] = { 0 };
2050886dcadSAndy Fiddaman 
2060886dcadSAndy Fiddaman 	(void) getisax((uint32_t *)&u, 2);
2070886dcadSAndy Fiddaman 	return ((u[1] & AV_386_2_AVX512BW) != 0);
2080886dcadSAndy Fiddaman }
2090886dcadSAndy Fiddaman 
210f91a4547SGvozden Neskovic #endif	/* _KERNEL */
211f91a4547SGvozden Neskovic 
212f91a4547SGvozden Neskovic 
213e86372a0SGvozden Neskovic #else
214e86372a0SGvozden Neskovic 
215e86372a0SGvozden Neskovic /* Non-x86 CPUs currently always disallow kernel FPU support */
216e86372a0SGvozden Neskovic #define	kfpu_allowed()		0
217e86372a0SGvozden Neskovic #define	kfpu_initialize(tsk)	do {} while (0)
218e86372a0SGvozden Neskovic #define	kfpu_begin()		do {} while (0)
219e86372a0SGvozden Neskovic #define	kfpu_end()		do {} while (0)
220e86372a0SGvozden Neskovic #define	kfpu_init()		(0)
221e86372a0SGvozden Neskovic #define	kfpu_fini()		do {} while (0)
222e86372a0SGvozden Neskovic #endif
223e86372a0SGvozden Neskovic 
224e86372a0SGvozden Neskovic #endif /* _SIMD_H */
225