1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
23 */
24 #include <sys/isa_defs.h>
25
26 #if defined(__x86_64) && defined(HAVE_AVX2)
27
28 #include <sys/types.h>
29 #include <sys/simd.h>
30
31 #ifdef __linux__
32 #define __asm __asm__ __volatile__
33 #endif
34
35 #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N
36 #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1)
37
38 #define VR0_(REG, ...) "ymm"#REG
39 #define VR1_(_1, REG, ...) "ymm"#REG
40 #define VR2_(_1, _2, REG, ...) "ymm"#REG
41 #define VR3_(_1, _2, _3, REG, ...) "ymm"#REG
42 #define VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG
43 #define VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG
44 #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG
45 #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG
46
47 #define VR0(r...) VR0_(r)
48 #define VR1(r...) VR1_(r)
49 #define VR2(r...) VR2_(r, 1)
50 #define VR3(r...) VR3_(r, 1, 2)
51 #define VR4(r...) VR4_(r, 1, 2)
52 #define VR5(r...) VR5_(r, 1, 2, 3)
53 #define VR6(r...) VR6_(r, 1, 2, 3, 4)
54 #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5)
55
56 #define R_01(REG1, REG2, ...) REG1, REG2
57 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3
58 #define R_23(REG...) _R_23(REG, 1, 2, 3)
59
60 #define ZFS_ASM_BUG() ASSERT(0)
61
62 extern const uint8_t gf_clmul_mod_lt[4*256][16];
63
64 #define ELEM_SIZE 32
65
66 typedef struct v {
67 uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE)));
68 } v_t;
69
70
71 #define XOR_ACC(src, r...) \
72 { \
73 switch (REG_CNT(r)) { \
74 case 4: \
75 __asm( \
76 "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \
77 "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \
78 "vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \
79 "vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \
80 : : [SRC] "r" (src)); \
81 break; \
82 case 2: \
83 __asm( \
84 "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \
85 "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \
86 : : [SRC] "r" (src)); \
87 break; \
88 default: \
89 ZFS_ASM_BUG(); \
90 } \
91 }
92
93 #define XOR(r...) \
94 { \
95 switch (REG_CNT(r)) { \
96 case 8: \
97 __asm( \
98 "vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \
99 "vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \
100 "vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \
101 "vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r)); \
102 break; \
103 case 4: \
104 __asm( \
105 "vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \
106 "vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r)); \
107 break; \
108 default: \
109 ZFS_ASM_BUG(); \
110 } \
111 }
112
113 #define ZERO(r...) XOR(r, r)
114
115 #define COPY(r...) \
116 { \
117 switch (REG_CNT(r)) { \
118 case 8: \
119 __asm( \
120 "vmovdqa %" VR0(r) ", %" VR4(r) "\n" \
121 "vmovdqa %" VR1(r) ", %" VR5(r) "\n" \
122 "vmovdqa %" VR2(r) ", %" VR6(r) "\n" \
123 "vmovdqa %" VR3(r) ", %" VR7(r)); \
124 break; \
125 case 4: \
126 __asm( \
127 "vmovdqa %" VR0(r) ", %" VR2(r) "\n" \
128 "vmovdqa %" VR1(r) ", %" VR3(r)); \
129 break; \
130 default: \
131 ZFS_ASM_BUG(); \
132 } \
133 }
134
135 #define LOAD(src, r...) \
136 { \
137 switch (REG_CNT(r)) { \
138 case 4: \
139 __asm( \
140 "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \
141 "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \
142 "vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n" \
143 "vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n" \
144 : : [SRC] "r" (src)); \
145 break; \
146 case 2: \
147 __asm( \
148 "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \
149 "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \
150 : : [SRC] "r" (src)); \
151 break; \
152 default: \
153 ZFS_ASM_BUG(); \
154 } \
155 }
156
157 #define STORE(dst, r...) \
158 { \
159 switch (REG_CNT(r)) { \
160 case 4: \
161 __asm( \
162 "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \
163 "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \
164 "vmovdqa %%" VR2(r) ", 0x40(%[DST])\n" \
165 "vmovdqa %%" VR3(r) ", 0x60(%[DST])\n" \
166 : : [DST] "r" (dst)); \
167 break; \
168 case 2: \
169 __asm( \
170 "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \
171 "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \
172 : : [DST] "r" (dst)); \
173 break; \
174 default: \
175 ZFS_ASM_BUG(); \
176 } \
177 }
178
179 #define FLUSH() \
180 { \
181 __asm("vzeroupper"); \
182 }
183
184 #define MUL2_SETUP() \
185 { \
186 __asm("vmovq %0, %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d)); \
187 __asm("vpbroadcastq %xmm14, %ymm14"); \
188 __asm("vpxor %ymm15, %ymm15 ,%ymm15"); \
189 }
190
191 #define _MUL2(r...) \
192 { \
193 switch (REG_CNT(r)) { \
194 case 2: \
195 __asm( \
196 "vpcmpgtb %" VR0(r)", %ymm15, %ymm12\n" \
197 "vpcmpgtb %" VR1(r)", %ymm15, %ymm13\n" \
198 "vpaddb %" VR0(r)", %" VR0(r)", %" VR0(r) "\n" \
199 "vpaddb %" VR1(r)", %" VR1(r)", %" VR1(r) "\n" \
200 "vpand %ymm14, %ymm12, %ymm12\n" \
201 "vpand %ymm14, %ymm13, %ymm13\n" \
202 "vpxor %ymm12, %" VR0(r)", %" VR0(r) "\n" \
203 "vpxor %ymm13, %" VR1(r)", %" VR1(r)); \
204 break; \
205 default: \
206 ZFS_ASM_BUG(); \
207 } \
208 }
209
210 #define MUL2(r...) \
211 { \
212 switch (REG_CNT(r)) { \
213 case 4: \
214 _MUL2(R_01(r)); \
215 _MUL2(R_23(r)); \
216 break; \
217 case 2: \
218 _MUL2(r); \
219 break; \
220 default: \
221 ZFS_ASM_BUG(); \
222 } \
223 }
224
225 #define MUL4(r...) \
226 { \
227 MUL2(r); \
228 MUL2(r); \
229 }
230
231 #define _0f "ymm15"
232 #define _as "ymm14"
233 #define _bs "ymm13"
234 #define _ltmod "ymm12"
235 #define _ltmul "ymm11"
236 #define _ta "ymm10"
237 #define _tb "ymm15"
238
239 static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F;
240
241 #define _MULx2(c, r...) \
242 { \
243 switch (REG_CNT(r)) { \
244 case 2: \
245 __asm( \
246 "vpbroadcastb (%[mask]), %%" _0f "\n" \
247 /* upper bits */ \
248 "vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n" \
249 "vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n" \
250 \
251 "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n" \
252 "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n" \
253 "vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \
254 "vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \
255 "vpand %%" _0f ", %%" _as ", %%" _as "\n" \
256 "vpand %%" _0f ", %%" _bs ", %%" _bs "\n" \
257 \
258 "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n" \
259 "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n" \
260 "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n" \
261 "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n" \
262 /* lower bits */ \
263 "vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n" \
264 "vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n" \
265 \
266 "vpxor %%" _ta ", %%" _as ", %%" _as "\n" \
267 "vpxor %%" _tb ", %%" _bs ", %%" _bs "\n" \
268 \
269 "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n" \
270 "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n" \
271 "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\
272 "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\
273 \
274 "vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \
275 "vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \
276 "vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \
277 "vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \
278 : : [mask] "r" (&_mul_mask), \
279 [lt] "r" (gf_clmul_mod_lt[4*(c)])); \
280 break; \
281 default: \
282 ZFS_ASM_BUG(); \
283 } \
284 }
285
286 #define MUL(c, r...) \
287 { \
288 switch (REG_CNT(r)) { \
289 case 4: \
290 _MULx2(c, R_01(r)); \
291 _MULx2(c, R_23(r)); \
292 break; \
293 case 2: \
294 _MULx2(c, R_01(r)); \
295 break; \
296 default: \
297 ZFS_ASM_BUG(); \
298 } \
299 }
300
301 #define raidz_math_begin() kfpu_begin()
302 #define raidz_math_end() \
303 { \
304 FLUSH(); \
305 kfpu_end(); \
306 }
307
308
309 #define SYN_STRIDE 4
310
311 #define ZERO_STRIDE 4
312 #define ZERO_DEFINE() {}
313 #define ZERO_D 0, 1, 2, 3
314
315 #define COPY_STRIDE 4
316 #define COPY_DEFINE() {}
317 #define COPY_D 0, 1, 2, 3
318
319 #define ADD_STRIDE 4
320 #define ADD_DEFINE() {}
321 #define ADD_D 0, 1, 2, 3
322
323 #define MUL_STRIDE 4
324 #define MUL_DEFINE() {}
325 #define MUL_D 0, 1, 2, 3
326
327 #define GEN_P_STRIDE 4
328 #define GEN_P_DEFINE() {}
329 #define GEN_P_P 0, 1, 2, 3
330
331 #define GEN_PQ_STRIDE 4
332 #define GEN_PQ_DEFINE() {}
333 #define GEN_PQ_D 0, 1, 2, 3
334 #define GEN_PQ_C 4, 5, 6, 7
335
336 #define GEN_PQR_STRIDE 4
337 #define GEN_PQR_DEFINE() {}
338 #define GEN_PQR_D 0, 1, 2, 3
339 #define GEN_PQR_C 4, 5, 6, 7
340
341 #define SYN_Q_DEFINE() {}
342 #define SYN_Q_D 0, 1, 2, 3
343 #define SYN_Q_X 4, 5, 6, 7
344
345 #define SYN_R_DEFINE() {}
346 #define SYN_R_D 0, 1, 2, 3
347 #define SYN_R_X 4, 5, 6, 7
348
349 #define SYN_PQ_DEFINE() {}
350 #define SYN_PQ_D 0, 1, 2, 3
351 #define SYN_PQ_X 4, 5, 6, 7
352
353 #define REC_PQ_STRIDE 2
354 #define REC_PQ_DEFINE() {}
355 #define REC_PQ_X 0, 1
356 #define REC_PQ_Y 2, 3
357 #define REC_PQ_T 4, 5
358
359 #define SYN_PR_DEFINE() {}
360 #define SYN_PR_D 0, 1, 2, 3
361 #define SYN_PR_X 4, 5, 6, 7
362
363 #define REC_PR_STRIDE 2
364 #define REC_PR_DEFINE() {}
365 #define REC_PR_X 0, 1
366 #define REC_PR_Y 2, 3
367 #define REC_PR_T 4, 5
368
369 #define SYN_QR_DEFINE() {}
370 #define SYN_QR_D 0, 1, 2, 3
371 #define SYN_QR_X 4, 5, 6, 7
372
373 #define REC_QR_STRIDE 2
374 #define REC_QR_DEFINE() {}
375 #define REC_QR_X 0, 1
376 #define REC_QR_Y 2, 3
377 #define REC_QR_T 4, 5
378
379 #define SYN_PQR_DEFINE() {}
380 #define SYN_PQR_D 0, 1, 2, 3
381 #define SYN_PQR_X 4, 5, 6, 7
382
383 #define REC_PQR_STRIDE 2
384 #define REC_PQR_DEFINE() {}
385 #define REC_PQR_X 0, 1
386 #define REC_PQR_Y 2, 3
387 #define REC_PQR_Z 4, 5
388 #define REC_PQR_XS 6, 7
389 #define REC_PQR_YS 8, 9
390
391
392 #include <sys/vdev_raidz_impl.h>
393 #include "vdev_raidz_math_impl.h"
394
395 DEFINE_GEN_METHODS(avx2);
396 DEFINE_REC_METHODS(avx2);
397
398 static boolean_t
raidz_will_avx2_work(void)399 raidz_will_avx2_work(void)
400 {
401 return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available());
402 }
403
404 const raidz_impl_ops_t vdev_raidz_avx2_impl = {
405 .init = NULL,
406 .fini = NULL,
407 .gen = RAIDZ_GEN_METHODS(avx2),
408 .rec = RAIDZ_REC_METHODS(avx2),
409 .is_supported = &raidz_will_avx2_work,
410 .name = "avx2"
411 };
412
413 #endif /* defined(__x86_64) && defined(HAVE_AVX2) */
414