1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2020 Joyent, Inc.
14 * Copyright 2024 Oxide Computer Company
15 */
16
17 #ifndef _SIMD_H
18 #define _SIMD_H
19
20 #if defined(__amd64__) || defined(__i386__)
21
22 #define kfpu_initialize(tsk) do {} while (0)
23 #define kfpu_init() (0)
24 #define kfpu_fini() do {} while (0)
25
26 #ifdef _KERNEL
27 #include <sys/x86_archext.h>
28 #include <sys/archsystm.h>
29 #include <sys/systm.h>
30 #include <sys/kfpu.h>
31 #include <sys/proc.h>
32 #include <sys/disp.h>
33 #include <sys/cpuvar.h>
34
35 static inline int
kfpu_allowed(void)36 kfpu_allowed(void)
37 {
38 extern int zfs_fpu_enabled;
39
40 /*
41 * When panicking, play it safe and avoid kfpu use. This gives the best
42 * chance of being able to dump successfully, particularly if the panic
43 * occured around an FPU context switch.
44 */
45 if (panicstr != NULL)
46 return (0);
47
48 return (zfs_fpu_enabled != 0 ? 1 : 0);
49 }
50
51 static inline void
kfpu_begin(void)52 kfpu_begin(void)
53 {
54 if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) {
55 kernel_fpu_begin(NULL, KFPU_USE_LWP);
56 } else {
57 kpreempt_disable();
58 kernel_fpu_begin(NULL, KFPU_NO_STATE);
59 }
60 }
61
62 static inline void
kfpu_end(void)63 kfpu_end(void)
64 {
65 if (curthread->t_lwp != NULL && (curthread->t_procp->p_flag & SSYS)) {
66 kernel_fpu_end(NULL, KFPU_USE_LWP);
67 } else {
68 kernel_fpu_end(NULL, KFPU_NO_STATE);
69 kpreempt_enable();
70 }
71 }
72
73 /*
74 * Check if various vector instruction sets are available.
75 */
76
77 static inline boolean_t
zfs_sse_available(void)78 zfs_sse_available(void)
79 {
80 return (is_x86_feature(x86_featureset, X86FSET_SSE));
81 }
82
83 static inline boolean_t
zfs_sse2_available(void)84 zfs_sse2_available(void)
85 {
86 return (is_x86_feature(x86_featureset, X86FSET_SSE2));
87 }
88
89 static inline boolean_t
zfs_sse3_available(void)90 zfs_sse3_available(void)
91 {
92 return (is_x86_feature(x86_featureset, X86FSET_SSE3));
93 }
94
95 static inline boolean_t
zfs_ssse3_available(void)96 zfs_ssse3_available(void)
97 {
98 return (is_x86_feature(x86_featureset, X86FSET_SSSE3));
99 }
100
101 static inline boolean_t
zfs_avx_available(void)102 zfs_avx_available(void)
103 {
104 return (is_x86_feature(x86_featureset, X86FSET_AVX));
105 }
106
107 static inline boolean_t
zfs_avx2_available(void)108 zfs_avx2_available(void)
109 {
110 return (is_x86_feature(x86_featureset, X86FSET_AVX2));
111 }
112
113 static inline boolean_t
zfs_avx512f_available(void)114 zfs_avx512f_available(void)
115 {
116 return (is_x86_feature(x86_featureset, X86FSET_AVX512F));
117 }
118
119 static inline boolean_t
zfs_avx512bw_available(void)120 zfs_avx512bw_available(void)
121 {
122 return (is_x86_feature(x86_featureset, X86FSET_AVX512BW));
123 }
124
125 #else /* ! _KERNEL */
126
127 #include <sys/auxv.h>
128 #include <sys/auxv_386.h>
129
130 #define kfpu_allowed() 1
131 #define kfpu_begin() do {} while (0)
132 #define kfpu_end() do {} while (0)
133
134 /*
135 * User-level check if various vector instruction sets are available.
136 */
137
138 static inline boolean_t
zfs_sse_available(void)139 zfs_sse_available(void)
140 {
141 uint32_t u = 0;
142
143 (void) getisax(&u, 1);
144 return ((u & AV_386_SSE) != 0);
145 }
146
147 static inline boolean_t
zfs_sse2_available(void)148 zfs_sse2_available(void)
149 {
150 uint32_t u = 0;
151
152 (void) getisax(&u, 1);
153 return ((u & AV_386_SSE2) != 0);
154 }
155
156 static inline boolean_t
zfs_sse3_available(void)157 zfs_sse3_available(void)
158 {
159 uint32_t u = 0;
160
161 (void) getisax(&u, 1);
162 return ((u & AV_386_SSE3) != 0);
163 }
164
165 static inline boolean_t
zfs_ssse3_available(void)166 zfs_ssse3_available(void)
167 {
168 uint32_t u = 0;
169
170 (void) getisax(&u, 1);
171 return ((u & AV_386_SSSE3) != 0);
172 }
173
174 static inline boolean_t
zfs_avx_available(void)175 zfs_avx_available(void)
176 {
177 uint_t u = 0;
178
179 (void) getisax(&u, 1);
180 return ((u & AV_386_AVX) != 0);
181 }
182
183 static inline boolean_t
zfs_avx2_available(void)184 zfs_avx2_available(void)
185 {
186 uint32_t u[2] = { 0 };
187
188 (void) getisax((uint32_t *)&u, 2);
189 return ((u[1] & AV_386_2_AVX2) != 0);
190 }
191
192 static inline boolean_t
zfs_avx512f_available(void)193 zfs_avx512f_available(void)
194 {
195 uint32_t u[2] = { 0 };
196
197 (void) getisax((uint32_t *)&u, 2);
198 return ((u[1] & AV_386_2_AVX512F) != 0);
199 }
200
201 static inline boolean_t
zfs_avx512bw_available(void)202 zfs_avx512bw_available(void)
203 {
204 uint32_t u[2] = { 0 };
205
206 (void) getisax((uint32_t *)&u, 2);
207 return ((u[1] & AV_386_2_AVX512BW) != 0);
208 }
209
210 #endif /* _KERNEL */
211
212
213 #else
214
215 /* Non-x86 CPUs currently always disallow kernel FPU support */
216 #define kfpu_allowed() 0
217 #define kfpu_initialize(tsk) do {} while (0)
218 #define kfpu_begin() do {} while (0)
219 #define kfpu_end() do {} while (0)
220 #define kfpu_init() (0)
221 #define kfpu_fini() do {} while (0)
222 #endif
223
224 #endif /* _SIMD_H */
225