1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
23 */
24
25 #include <sys/simd.h>
26 #include <sys/zfs_context.h>
27 #include <sys/types.h>
28 #include <sys/zio.h>
29 #include <sys/debug.h>
30 #include <sys/zfs_debug.h>
31 #include <sys/vdev_raidz.h>
32 #include <sys/vdev_raidz_impl.h>
33
34 /* Opaque implementation with NULL methods to represent original methods */
35 static const raidz_impl_ops_t vdev_raidz_original_impl = {
36 .name = "original",
37 .is_supported = raidz_will_scalar_work,
38 };
39
40 /* RAIDZ parity op that contain the fastest methods */
41 static raidz_impl_ops_t vdev_raidz_fastest_impl = {
42 .name = "fastest"
43 };
44
45 /* All compiled in implementations */
46 static const raidz_impl_ops_t *const raidz_all_maths[] = {
47 &vdev_raidz_original_impl,
48 &vdev_raidz_scalar_impl,
49 #if defined(__x86_64) && defined(HAVE_SSE2) /* only x86_64 for now */
50 &vdev_raidz_sse2_impl,
51 #endif
52 #if defined(__x86_64) && defined(HAVE_SSSE3) /* only x86_64 for now */
53 &vdev_raidz_ssse3_impl,
54 #endif
55 #if defined(__x86_64) && defined(HAVE_AVX2) /* only x86_64 for now */
56 &vdev_raidz_avx2_impl,
57 #endif
58 #if defined(__x86_64) && defined(HAVE_AVX512F) /* only x86_64 for now */
59 &vdev_raidz_avx512f_impl,
60 #endif
61 #if defined(__x86_64) && defined(HAVE_AVX512BW) /* only x86_64 for now */
62 &vdev_raidz_avx512bw_impl,
63 #endif
64 #if defined(__aarch64__) && !defined(__FreeBSD__)
65 &vdev_raidz_aarch64_neon_impl,
66 &vdev_raidz_aarch64_neonx2_impl,
67 #endif
68 #if defined(__powerpc__) && defined(__altivec__)
69 &vdev_raidz_powerpc_altivec_impl,
70 #endif
71 };
72
73 /* Indicate that benchmark has been completed */
74 static boolean_t raidz_math_initialized = B_FALSE;
75
76 /* Select raidz implementation */
77 #define IMPL_FASTEST (UINT32_MAX)
78 #define IMPL_CYCLE (UINT32_MAX - 1)
79 #define IMPL_ORIGINAL (0)
80 #define IMPL_SCALAR (1)
81
82 #define RAIDZ_IMPL_READ(i) (*(volatile uint32_t *) &(i))
83
84 static uint32_t zfs_vdev_raidz_impl = IMPL_SCALAR;
85 static uint32_t user_sel_impl = IMPL_FASTEST;
86
87 /* Hold all supported implementations */
88 static size_t raidz_supp_impl_cnt = 0;
89 static raidz_impl_ops_t *raidz_supp_impl[ARRAY_SIZE(raidz_all_maths)];
90
91 #if defined(_KERNEL)
92 /*
93 * kstats values for supported implementations
94 * Values represent per disk throughput of 8 disk+parity raidz vdev [B/s]
95 */
96 static raidz_impl_kstat_t raidz_impl_kstats[ARRAY_SIZE(raidz_all_maths) + 1];
97
98 /* kstat for benchmarked implementations */
99 static kstat_t *raidz_math_kstat = NULL;
100 #endif
101
102 /*
103 * Returns the RAIDZ operations for raidz_map() parity calculations. When
104 * a SIMD implementation is not allowed in the current context, then fallback
105 * to the fastest generic implementation.
106 */
107 const raidz_impl_ops_t *
vdev_raidz_math_get_ops(void)108 vdev_raidz_math_get_ops(void)
109 {
110 if (!kfpu_allowed())
111 return (&vdev_raidz_scalar_impl);
112
113 raidz_impl_ops_t *ops = NULL;
114 const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl);
115
116 switch (impl) {
117 case IMPL_FASTEST:
118 ASSERT(raidz_math_initialized);
119 ops = &vdev_raidz_fastest_impl;
120 break;
121 case IMPL_CYCLE:
122 /* Cycle through all supported implementations */
123 ASSERT(raidz_math_initialized);
124 ASSERT3U(raidz_supp_impl_cnt, >, 0);
125 static size_t cycle_impl_idx = 0;
126 size_t idx = (++cycle_impl_idx) % raidz_supp_impl_cnt;
127 ops = raidz_supp_impl[idx];
128 break;
129 case IMPL_ORIGINAL:
130 ops = (raidz_impl_ops_t *)&vdev_raidz_original_impl;
131 break;
132 case IMPL_SCALAR:
133 ops = (raidz_impl_ops_t *)&vdev_raidz_scalar_impl;
134 break;
135 default:
136 ASSERT3U(impl, <, raidz_supp_impl_cnt);
137 ASSERT3U(raidz_supp_impl_cnt, >, 0);
138 if (impl < ARRAY_SIZE(raidz_all_maths))
139 ops = raidz_supp_impl[impl];
140 break;
141 }
142
143 ASSERT3P(ops, !=, NULL);
144
145 return (ops);
146 }
147
148 /*
149 * Select parity generation method for raidz_map
150 */
151 int
vdev_raidz_math_generate(raidz_map_t * rm,raidz_row_t * rr)152 vdev_raidz_math_generate(raidz_map_t *rm, raidz_row_t *rr)
153 {
154 raidz_gen_f gen_parity = NULL;
155
156 switch (raidz_parity(rm)) {
157 case 1:
158 gen_parity = rm->rm_ops->gen[RAIDZ_GEN_P];
159 break;
160 case 2:
161 gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQ];
162 break;
163 case 3:
164 gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQR];
165 break;
166 default:
167 gen_parity = NULL;
168 cmn_err(CE_PANIC, "invalid RAID-Z configuration %llu",
169 (u_longlong_t)raidz_parity(rm));
170 break;
171 }
172
173 /* if method is NULL execute the original implementation */
174 if (gen_parity == NULL)
175 return (RAIDZ_ORIGINAL_IMPL);
176
177 gen_parity(rr);
178
179 return (0);
180 }
181
182 static raidz_rec_f
reconstruct_fun_p_sel(raidz_map_t * rm,const int * parity_valid,const int nbaddata)183 reconstruct_fun_p_sel(raidz_map_t *rm, const int *parity_valid,
184 const int nbaddata)
185 {
186 if (nbaddata == 1 && parity_valid[CODE_P]) {
187 return (rm->rm_ops->rec[RAIDZ_REC_P]);
188 }
189 return ((raidz_rec_f) NULL);
190 }
191
192 static raidz_rec_f
reconstruct_fun_pq_sel(raidz_map_t * rm,const int * parity_valid,const int nbaddata)193 reconstruct_fun_pq_sel(raidz_map_t *rm, const int *parity_valid,
194 const int nbaddata)
195 {
196 if (nbaddata == 1) {
197 if (parity_valid[CODE_P]) {
198 return (rm->rm_ops->rec[RAIDZ_REC_P]);
199 } else if (parity_valid[CODE_Q]) {
200 return (rm->rm_ops->rec[RAIDZ_REC_Q]);
201 }
202 } else if (nbaddata == 2 &&
203 parity_valid[CODE_P] && parity_valid[CODE_Q]) {
204 return (rm->rm_ops->rec[RAIDZ_REC_PQ]);
205 }
206 return ((raidz_rec_f) NULL);
207 }
208
209 static raidz_rec_f
reconstruct_fun_pqr_sel(raidz_map_t * rm,const int * parity_valid,const int nbaddata)210 reconstruct_fun_pqr_sel(raidz_map_t *rm, const int *parity_valid,
211 const int nbaddata)
212 {
213 if (nbaddata == 1) {
214 if (parity_valid[CODE_P]) {
215 return (rm->rm_ops->rec[RAIDZ_REC_P]);
216 } else if (parity_valid[CODE_Q]) {
217 return (rm->rm_ops->rec[RAIDZ_REC_Q]);
218 } else if (parity_valid[CODE_R]) {
219 return (rm->rm_ops->rec[RAIDZ_REC_R]);
220 }
221 } else if (nbaddata == 2) {
222 if (parity_valid[CODE_P] && parity_valid[CODE_Q]) {
223 return (rm->rm_ops->rec[RAIDZ_REC_PQ]);
224 } else if (parity_valid[CODE_P] && parity_valid[CODE_R]) {
225 return (rm->rm_ops->rec[RAIDZ_REC_PR]);
226 } else if (parity_valid[CODE_Q] && parity_valid[CODE_R]) {
227 return (rm->rm_ops->rec[RAIDZ_REC_QR]);
228 }
229 } else if (nbaddata == 3 &&
230 parity_valid[CODE_P] && parity_valid[CODE_Q] &&
231 parity_valid[CODE_R]) {
232 return (rm->rm_ops->rec[RAIDZ_REC_PQR]);
233 }
234 return ((raidz_rec_f) NULL);
235 }
236
237 /*
238 * Select data reconstruction method for raidz_map
239 * @parity_valid - Parity validity flag
240 * @dt - Failed data index array
241 * @nbaddata - Number of failed data columns
242 */
243 int
vdev_raidz_math_reconstruct(raidz_map_t * rm,raidz_row_t * rr,const int * parity_valid,const int * dt,const int nbaddata)244 vdev_raidz_math_reconstruct(raidz_map_t *rm, raidz_row_t *rr,
245 const int *parity_valid, const int *dt, const int nbaddata)
246 {
247 raidz_rec_f rec_fn = NULL;
248
249 switch (raidz_parity(rm)) {
250 case PARITY_P:
251 rec_fn = reconstruct_fun_p_sel(rm, parity_valid, nbaddata);
252 break;
253 case PARITY_PQ:
254 rec_fn = reconstruct_fun_pq_sel(rm, parity_valid, nbaddata);
255 break;
256 case PARITY_PQR:
257 rec_fn = reconstruct_fun_pqr_sel(rm, parity_valid, nbaddata);
258 break;
259 default:
260 cmn_err(CE_PANIC, "invalid RAID-Z configuration %llu",
261 (u_longlong_t)raidz_parity(rm));
262 break;
263 }
264
265 if (rec_fn == NULL)
266 return (RAIDZ_ORIGINAL_IMPL);
267 else
268 return (rec_fn(rr, dt));
269 }
270
271 const char *const raidz_gen_name[] = {
272 "gen_p", "gen_pq", "gen_pqr"
273 };
274 const char *const raidz_rec_name[] = {
275 "rec_p", "rec_q", "rec_r",
276 "rec_pq", "rec_pr", "rec_qr", "rec_pqr"
277 };
278
279 #if defined(_KERNEL)
280
281 #define RAIDZ_KSTAT_LINE_LEN (17 + 10*12 + 1)
282
283 static int
raidz_math_kstat_headers(char * buf,size_t size)284 raidz_math_kstat_headers(char *buf, size_t size)
285 {
286 ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN);
287
288 ssize_t off = kmem_scnprintf(buf, size, "%-17s", "implementation");
289
290 for (int i = 0; i < ARRAY_SIZE(raidz_gen_name); i++)
291 off += kmem_scnprintf(buf + off, size - off, "%-16s",
292 raidz_gen_name[i]);
293
294 for (int i = 0; i < ARRAY_SIZE(raidz_rec_name); i++)
295 off += kmem_scnprintf(buf + off, size - off, "%-16s",
296 raidz_rec_name[i]);
297
298 (void) kmem_scnprintf(buf + off, size - off, "\n");
299
300 return (0);
301 }
302
303 static int
raidz_math_kstat_data(char * buf,size_t size,void * data)304 raidz_math_kstat_data(char *buf, size_t size, void *data)
305 {
306 raidz_impl_kstat_t *fstat = &raidz_impl_kstats[raidz_supp_impl_cnt];
307 raidz_impl_kstat_t *cstat = (raidz_impl_kstat_t *)data;
308 ssize_t off = 0;
309 int i;
310
311 ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN);
312
313 if (cstat == fstat) {
314 off += kmem_scnprintf(buf + off, size - off, "%-17s",
315 "fastest");
316
317 for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++) {
318 int id = fstat->gen[i];
319 off += kmem_scnprintf(buf + off, size - off, "%-16s",
320 raidz_supp_impl[id]->name);
321 }
322 for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++) {
323 int id = fstat->rec[i];
324 off += kmem_scnprintf(buf + off, size - off, "%-16s",
325 raidz_supp_impl[id]->name);
326 }
327 } else {
328 ptrdiff_t id = cstat - raidz_impl_kstats;
329
330 off += kmem_scnprintf(buf + off, size - off, "%-17s",
331 raidz_supp_impl[id]->name);
332
333 for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++)
334 off += kmem_scnprintf(buf + off, size - off, "%-16llu",
335 (u_longlong_t)cstat->gen[i]);
336
337 for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++)
338 off += kmem_scnprintf(buf + off, size - off, "%-16llu",
339 (u_longlong_t)cstat->rec[i]);
340 }
341
342 (void) kmem_scnprintf(buf + off, size - off, "\n");
343
344 return (0);
345 }
346
347 static void *
raidz_math_kstat_addr(kstat_t * ksp,loff_t n)348 raidz_math_kstat_addr(kstat_t *ksp, loff_t n)
349 {
350 if (n <= raidz_supp_impl_cnt)
351 ksp->ks_private = (void *) (raidz_impl_kstats + n);
352 else
353 ksp->ks_private = NULL;
354
355 return (ksp->ks_private);
356 }
357
358 #define BENCH_D_COLS (8ULL)
359 #define BENCH_COLS (BENCH_D_COLS + PARITY_PQR)
360 #define BENCH_ZIO_SIZE (1ULL << SPA_OLD_MAXBLOCKSHIFT) /* 128 kiB */
361 #define BENCH_NS MSEC2NSEC(1) /* 1ms */
362
363 typedef void (*benchmark_fn)(raidz_map_t *rm, const int fn);
364
365 static void
benchmark_gen_impl(raidz_map_t * rm,const int fn)366 benchmark_gen_impl(raidz_map_t *rm, const int fn)
367 {
368 (void) fn;
369 vdev_raidz_generate_parity(rm);
370 }
371
372 static void
benchmark_rec_impl(raidz_map_t * rm,const int fn)373 benchmark_rec_impl(raidz_map_t *rm, const int fn)
374 {
375 static const int rec_tgt[7][3] = {
376 {1, 2, 3}, /* rec_p: bad QR & D[0] */
377 {0, 2, 3}, /* rec_q: bad PR & D[0] */
378 {0, 1, 3}, /* rec_r: bad PQ & D[0] */
379 {2, 3, 4}, /* rec_pq: bad R & D[0][1] */
380 {1, 3, 4}, /* rec_pr: bad Q & D[0][1] */
381 {0, 3, 4}, /* rec_qr: bad P & D[0][1] */
382 {3, 4, 5} /* rec_pqr: bad & D[0][1][2] */
383 };
384
385 vdev_raidz_reconstruct(rm, rec_tgt[fn], 3);
386 }
387
388 /*
389 * Benchmarking of all supported implementations (raidz_supp_impl_cnt)
390 * is performed by setting the rm_ops pointer and calling the top level
391 * generate/reconstruct methods of bench_rm.
392 */
393 static void
benchmark_raidz_impl(raidz_map_t * bench_rm,const int fn,benchmark_fn bench_fn)394 benchmark_raidz_impl(raidz_map_t *bench_rm, const int fn, benchmark_fn bench_fn)
395 {
396 uint64_t run_cnt, speed, best_speed = 0;
397 hrtime_t t_start, t_diff;
398 raidz_impl_ops_t *curr_impl;
399 raidz_impl_kstat_t *fstat = &raidz_impl_kstats[raidz_supp_impl_cnt];
400 int impl, i;
401
402 for (impl = 0; impl < raidz_supp_impl_cnt; impl++) {
403 /* set an implementation to benchmark */
404 curr_impl = raidz_supp_impl[impl];
405 bench_rm->rm_ops = curr_impl;
406
407 run_cnt = 0;
408 t_start = gethrtime();
409
410 do {
411 for (i = 0; i < 5; i++, run_cnt++)
412 bench_fn(bench_rm, fn);
413
414 t_diff = gethrtime() - t_start;
415 } while (t_diff < BENCH_NS);
416
417 speed = run_cnt * BENCH_ZIO_SIZE * NANOSEC;
418 speed /= (t_diff * BENCH_COLS);
419
420 if (bench_fn == benchmark_gen_impl)
421 raidz_impl_kstats[impl].gen[fn] = speed;
422 else
423 raidz_impl_kstats[impl].rec[fn] = speed;
424
425 /* Update fastest implementation method */
426 if (speed > best_speed) {
427 best_speed = speed;
428
429 if (bench_fn == benchmark_gen_impl) {
430 fstat->gen[fn] = impl;
431 vdev_raidz_fastest_impl.gen[fn] =
432 curr_impl->gen[fn];
433 } else {
434 fstat->rec[fn] = impl;
435 vdev_raidz_fastest_impl.rec[fn] =
436 curr_impl->rec[fn];
437 }
438 }
439 }
440 }
441 #endif
442
443 /*
444 * Initialize and benchmark all supported implementations.
445 */
446 static void
benchmark_raidz(void)447 benchmark_raidz(void)
448 {
449 raidz_impl_ops_t *curr_impl;
450 int i, c;
451
452 /* Move supported impl into raidz_supp_impl */
453 for (i = 0, c = 0; i < ARRAY_SIZE(raidz_all_maths); i++) {
454 curr_impl = (raidz_impl_ops_t *)raidz_all_maths[i];
455
456 if (curr_impl->init)
457 curr_impl->init();
458
459 if (curr_impl->is_supported())
460 raidz_supp_impl[c++] = (raidz_impl_ops_t *)curr_impl;
461 }
462 membar_producer(); /* complete raidz_supp_impl[] init */
463 raidz_supp_impl_cnt = c; /* number of supported impl */
464
465 #if defined(_KERNEL)
466 abd_t *pabd;
467 zio_t *bench_zio = NULL;
468 raidz_map_t *bench_rm = NULL;
469 uint64_t bench_parity;
470
471 /* Fake a zio and run the benchmark on a warmed up buffer */
472 bench_zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP);
473 bench_zio->io_offset = 0;
474 bench_zio->io_size = BENCH_ZIO_SIZE; /* only data columns */
475 bench_zio->io_abd = abd_alloc_linear(BENCH_ZIO_SIZE, B_TRUE);
476 memset(abd_to_buf(bench_zio->io_abd), 0xAA, BENCH_ZIO_SIZE);
477
478 /* Benchmark parity generation methods */
479 for (int fn = 0; fn < RAIDZ_GEN_NUM; fn++) {
480 bench_parity = fn + 1;
481 /* New raidz_map is needed for each generate_p/q/r */
482 bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT,
483 BENCH_D_COLS + bench_parity, bench_parity);
484
485 benchmark_raidz_impl(bench_rm, fn, benchmark_gen_impl);
486
487 vdev_raidz_map_free(bench_rm);
488 }
489
490 /* Benchmark data reconstruction methods */
491 bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT,
492 BENCH_COLS, PARITY_PQR);
493
494 /* Ensure that fake parity blocks are initialized */
495 for (c = 0; c < bench_rm->rm_row[0]->rr_firstdatacol; c++) {
496 pabd = bench_rm->rm_row[0]->rr_col[c].rc_abd;
497 memset(abd_to_buf(pabd), 0xAA, abd_get_size(pabd));
498 }
499
500 for (int fn = 0; fn < RAIDZ_REC_NUM; fn++)
501 benchmark_raidz_impl(bench_rm, fn, benchmark_rec_impl);
502
503 vdev_raidz_map_free(bench_rm);
504
505 /* cleanup the bench zio */
506 abd_free(bench_zio->io_abd);
507 kmem_free(bench_zio, sizeof (zio_t));
508 #else
509 /*
510 * Skip the benchmark in user space to avoid impacting libzpool
511 * consumers (zdb, zhack, zinject, ztest). The last implementation
512 * is assumed to be the fastest and used by default.
513 */
514 memcpy(&vdev_raidz_fastest_impl,
515 raidz_supp_impl[raidz_supp_impl_cnt - 1],
516 sizeof (vdev_raidz_fastest_impl));
517 strcpy(vdev_raidz_fastest_impl.name, "fastest");
518 #endif /* _KERNEL */
519 }
520
521 void
vdev_raidz_math_init(void)522 vdev_raidz_math_init(void)
523 {
524 /* Determine the fastest available implementation. */
525 benchmark_raidz();
526
527 #if defined(_KERNEL)
528 /* Install kstats for all implementations */
529 raidz_math_kstat = kstat_create("zfs", 0, "vdev_raidz_bench", "misc",
530 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);
531 if (raidz_math_kstat != NULL) {
532 raidz_math_kstat->ks_data = NULL;
533 raidz_math_kstat->ks_ndata = UINT32_MAX;
534 kstat_set_raw_ops(raidz_math_kstat,
535 raidz_math_kstat_headers,
536 raidz_math_kstat_data,
537 raidz_math_kstat_addr);
538 kstat_install(raidz_math_kstat);
539 }
540 #endif
541
542 /* Finish initialization */
543 atomic_swap_32(&zfs_vdev_raidz_impl, user_sel_impl);
544 raidz_math_initialized = B_TRUE;
545 }
546
547 void
vdev_raidz_math_fini(void)548 vdev_raidz_math_fini(void)
549 {
550 raidz_impl_ops_t const *curr_impl;
551
552 #if defined(_KERNEL)
553 if (raidz_math_kstat != NULL) {
554 kstat_delete(raidz_math_kstat);
555 raidz_math_kstat = NULL;
556 }
557 #endif
558
559 for (int i = 0; i < ARRAY_SIZE(raidz_all_maths); i++) {
560 curr_impl = raidz_all_maths[i];
561 if (curr_impl->fini)
562 curr_impl->fini();
563 }
564 }
565
566 static const struct {
567 const char *name;
568 uint32_t sel;
569 } math_impl_opts[] = {
570 { "cycle", IMPL_CYCLE },
571 { "fastest", IMPL_FASTEST },
572 { "original", IMPL_ORIGINAL },
573 { "scalar", IMPL_SCALAR }
574 };
575
576 /*
577 * Function sets desired raidz implementation.
578 *
579 * If we are called before init(), user preference will be saved in
580 * user_sel_impl, and applied in later init() call. This occurs when module
581 * parameter is specified on module load. Otherwise, directly update
582 * zfs_vdev_raidz_impl.
583 *
584 * @val Name of raidz implementation to use
585 * @param Unused.
586 */
587 int
vdev_raidz_impl_set(const char * val)588 vdev_raidz_impl_set(const char *val)
589 {
590 int err = -EINVAL;
591 char req_name[RAIDZ_IMPL_NAME_MAX];
592 uint32_t impl = RAIDZ_IMPL_READ(user_sel_impl);
593 size_t i;
594
595 /* sanitize input */
596 i = strnlen(val, RAIDZ_IMPL_NAME_MAX);
597 if (i == 0 || i == RAIDZ_IMPL_NAME_MAX)
598 return (err);
599
600 strlcpy(req_name, val, RAIDZ_IMPL_NAME_MAX);
601 while (i > 0 && !!isspace(req_name[i-1]))
602 i--;
603 req_name[i] = '\0';
604
605 /* Check mandatory options */
606 for (i = 0; i < ARRAY_SIZE(math_impl_opts); i++) {
607 if (strcmp(req_name, math_impl_opts[i].name) == 0) {
608 impl = math_impl_opts[i].sel;
609 err = 0;
610 break;
611 }
612 }
613
614 /* check all supported impl if init() was already called */
615 if (err != 0 && raidz_math_initialized) {
616 /* check all supported implementations */
617 for (i = 0; i < raidz_supp_impl_cnt; i++) {
618 if (strcmp(req_name, raidz_supp_impl[i]->name) == 0) {
619 impl = i;
620 err = 0;
621 break;
622 }
623 }
624 }
625
626 if (err == 0) {
627 if (raidz_math_initialized)
628 atomic_swap_32(&zfs_vdev_raidz_impl, impl);
629 else
630 atomic_swap_32(&user_sel_impl, impl);
631 }
632
633 return (err);
634 }
635
636 #if defined(_KERNEL) && defined(__linux__)
637
638 static int
zfs_vdev_raidz_impl_set(const char * val,zfs_kernel_param_t * kp)639 zfs_vdev_raidz_impl_set(const char *val, zfs_kernel_param_t *kp)
640 {
641 return (vdev_raidz_impl_set(val));
642 }
643
644 static int
zfs_vdev_raidz_impl_get(char * buffer,zfs_kernel_param_t * kp)645 zfs_vdev_raidz_impl_get(char *buffer, zfs_kernel_param_t *kp)
646 {
647 int i, cnt = 0;
648 char *fmt;
649 const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl);
650
651 ASSERT(raidz_math_initialized);
652
653 /* list mandatory options */
654 for (i = 0; i < ARRAY_SIZE(math_impl_opts) - 2; i++) {
655 fmt = (impl == math_impl_opts[i].sel) ? "[%s] " : "%s ";
656 cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt,
657 math_impl_opts[i].name);
658 }
659
660 /* list all supported implementations */
661 for (i = 0; i < raidz_supp_impl_cnt; i++) {
662 fmt = (i == impl) ? "[%s] " : "%s ";
663 cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt,
664 raidz_supp_impl[i]->name);
665 }
666
667 return (cnt);
668 }
669
670 module_param_call(zfs_vdev_raidz_impl, zfs_vdev_raidz_impl_set,
671 zfs_vdev_raidz_impl_get, NULL, 0644);
672 MODULE_PARM_DESC(zfs_vdev_raidz_impl, "Select raidz implementation.");
673 #endif
674