1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 * Copyright (C) 2016 Gvozden Nešković. All rights reserved. 25 */ 26 /* 27 * Copyright 2013 Saso Kiselkov. All rights reserved. 28 */ 29 30 /* 31 * Copyright (c) 2016 by Delphix. All rights reserved. 32 */ 33 34 /* 35 * Fletcher Checksums 36 * ------------------ 37 * 38 * ZFS's 2nd and 4th order Fletcher checksums are defined by the following 39 * recurrence relations: 40 * 41 * a = a + f 42 * i i-1 i-1 43 * 44 * b = b + a 45 * i i-1 i 46 * 47 * c = c + b (fletcher-4 only) 48 * i i-1 i 49 * 50 * d = d + c (fletcher-4 only) 51 * i i-1 i 52 * 53 * Where 54 * a_0 = b_0 = c_0 = d_0 = 0 55 * and 56 * f_0 .. f_(n-1) are the input data. 57 * 58 * Using standard techniques, these translate into the following series: 59 * 60 * __n_ __n_ 61 * \ | \ | 62 * a = > f b = > i * f 63 * n /___| n - i n /___| n - i 64 * i = 1 i = 1 65 * 66 * 67 * __n_ __n_ 68 * \ | i*(i+1) \ | i*(i+1)*(i+2) 69 * c = > ------- f d = > ------------- f 70 * n /___| 2 n - i n /___| 6 n - i 71 * i = 1 i = 1 72 * 73 * For fletcher-2, the f_is are 64-bit, and [ab]_i are 64-bit accumulators. 74 * Since the additions are done mod (2^64), errors in the high bits may not 75 * be noticed. For this reason, fletcher-2 is deprecated. 76 * 77 * For fletcher-4, the f_is are 32-bit, and [abcd]_i are 64-bit accumulators. 78 * A conservative estimate of how big the buffer can get before we overflow 79 * can be estimated using f_i = 0xffffffff for all i: 80 * 81 * % bc 82 * f=2^32-1;d=0; for (i = 1; d<2^64; i++) { d += f*i*(i+1)*(i+2)/6 }; (i-1)*4 83 * 2264 84 * quit 85 * % 86 * 87 * So blocks of up to 2k will not overflow. Our largest block size is 88 * 128k, which has 32k 4-byte words, so we can compute the largest possible 89 * accumulators, then divide by 2^64 to figure the max amount of overflow: 90 * 91 * % bc 92 * a=b=c=d=0; f=2^32-1; for (i=1; i<=32*1024; i++) { a+=f; b+=a; c+=b; d+=c } 93 * a/2^64;b/2^64;c/2^64;d/2^64 94 * 0 95 * 0 96 * 1365 97 * 11186858 98 * quit 99 * % 100 * 101 * So a and b cannot overflow. To make sure each bit of input has some 102 * effect on the contents of c and d, we can look at what the factors of 103 * the coefficients in the equations for c_n and d_n are. The number of 2s 104 * in the factors determines the lowest set bit in the multiplier. Running 105 * through the cases for n*(n+1)/2 reveals that the highest power of 2 is 106 * 2^14, and for n*(n+1)*(n+2)/6 it is 2^15. So while some data may overflow 107 * the 64-bit accumulators, every bit of every f_i effects every accumulator, 108 * even for 128k blocks. 109 * 110 * If we wanted to make a stronger version of fletcher4 (fletcher4c?), 111 * we could do our calculations mod (2^32 - 1) by adding in the carries 112 * periodically, and store the number of carries in the top 32-bits. 113 * 114 * -------------------- 115 * Checksum Performance 116 * -------------------- 117 * 118 * There are two interesting components to checksum performance: cached and 119 * uncached performance. With cached data, fletcher-2 is about four times 120 * faster than fletcher-4. With uncached data, the performance difference is 121 * negligible, since the cost of a cache fill dominates the processing time. 122 * Even though fletcher-4 is slower than fletcher-2, it is still a pretty 123 * efficient pass over the data. 124 * 125 * In normal operation, the data which is being checksummed is in a buffer 126 * which has been filled either by: 127 * 128 * 1. a compression step, which will be mostly cached, or 129 * 2. a memcpy() or copyin(), which will be uncached 130 * (because the copy is cache-bypassing). 131 * 132 * For both cached and uncached data, both fletcher checksums are much faster 133 * than sha-256, and slower than 'off', which doesn't touch the data at all. 134 */ 135 136 #include <sys/types.h> 137 #include <sys/sysmacros.h> 138 #include <sys/byteorder.h> 139 #include <sys/spa.h> 140 #include <sys/simd.h> 141 #include <sys/zio_checksum.h> 142 #include <sys/zfs_context.h> 143 #include <zfs_fletcher.h> 144 145 #define FLETCHER_MIN_SIMD_SIZE 64 146 147 static void fletcher_4_scalar_init(fletcher_4_ctx_t *ctx); 148 static void fletcher_4_scalar_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp); 149 static void fletcher_4_scalar_native(fletcher_4_ctx_t *ctx, 150 const void *buf, uint64_t size); 151 static void fletcher_4_scalar_byteswap(fletcher_4_ctx_t *ctx, 152 const void *buf, uint64_t size); 153 static boolean_t fletcher_4_scalar_valid(void); 154 155 static const fletcher_4_ops_t fletcher_4_scalar_ops = { 156 .init_native = fletcher_4_scalar_init, 157 .fini_native = fletcher_4_scalar_fini, 158 .compute_native = fletcher_4_scalar_native, 159 .init_byteswap = fletcher_4_scalar_init, 160 .fini_byteswap = fletcher_4_scalar_fini, 161 .compute_byteswap = fletcher_4_scalar_byteswap, 162 .valid = fletcher_4_scalar_valid, 163 .name = "scalar" 164 }; 165 166 static fletcher_4_ops_t fletcher_4_fastest_impl = { 167 .name = "fastest", 168 .valid = fletcher_4_scalar_valid 169 }; 170 171 static const fletcher_4_ops_t *fletcher_4_impls[] = { 172 &fletcher_4_scalar_ops, 173 &fletcher_4_superscalar_ops, 174 &fletcher_4_superscalar4_ops, 175 #if defined(HAVE_SSE2) 176 &fletcher_4_sse2_ops, 177 #endif 178 #if defined(HAVE_SSE2) && defined(HAVE_SSSE3) 179 &fletcher_4_ssse3_ops, 180 #endif 181 #if defined(HAVE_AVX) && defined(HAVE_AVX2) 182 &fletcher_4_avx2_ops, 183 #endif 184 #if defined(__x86_64) && defined(HAVE_AVX512F) 185 &fletcher_4_avx512f_ops, 186 #endif 187 #if defined(__x86_64) && defined(HAVE_AVX512BW) 188 &fletcher_4_avx512bw_ops, 189 #endif 190 #if defined(__aarch64__) && !defined(__FreeBSD__) 191 &fletcher_4_aarch64_neon_ops, 192 #endif 193 }; 194 195 /* Hold all supported implementations */ 196 static uint32_t fletcher_4_supp_impls_cnt = 0; 197 static fletcher_4_ops_t *fletcher_4_supp_impls[ARRAY_SIZE(fletcher_4_impls)]; 198 199 /* Select fletcher4 implementation */ 200 #define IMPL_FASTEST (UINT32_MAX) 201 #define IMPL_CYCLE (UINT32_MAX - 1) 202 #define IMPL_SCALAR (0) 203 204 static uint32_t fletcher_4_impl_chosen = IMPL_FASTEST; 205 206 #define IMPL_READ(i) (*(volatile uint32_t *) &(i)) 207 208 static struct fletcher_4_impl_selector { 209 const char *fis_name; 210 uint32_t fis_sel; 211 } fletcher_4_impl_selectors[] = { 212 { "cycle", IMPL_CYCLE }, 213 { "fastest", IMPL_FASTEST }, 214 { "scalar", IMPL_SCALAR } 215 }; 216 217 #if defined(_KERNEL) 218 static kstat_t *fletcher_4_kstat; 219 220 static struct fletcher_4_kstat { 221 uint64_t native; 222 uint64_t byteswap; 223 } fletcher_4_stat_data[ARRAY_SIZE(fletcher_4_impls) + 1]; 224 #endif 225 226 /* Indicate that benchmark has been completed */ 227 static boolean_t fletcher_4_initialized = B_FALSE; 228 229 void 230 fletcher_init(zio_cksum_t *zcp) 231 { 232 ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0); 233 } 234 235 int 236 fletcher_2_incremental_native(void *buf, size_t size, void *data) 237 { 238 zio_cksum_t *zcp = data; 239 240 const uint64_t *ip = buf; 241 const uint64_t *ipend = ip + (size / sizeof (uint64_t)); 242 uint64_t a0, b0, a1, b1; 243 244 a0 = zcp->zc_word[0]; 245 a1 = zcp->zc_word[1]; 246 b0 = zcp->zc_word[2]; 247 b1 = zcp->zc_word[3]; 248 249 for (; ip < ipend; ip += 2) { 250 a0 += ip[0]; 251 a1 += ip[1]; 252 b0 += a0; 253 b1 += a1; 254 } 255 256 ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1); 257 return (0); 258 } 259 260 void 261 fletcher_2_native(const void *buf, uint64_t size, 262 const void *ctx_template, zio_cksum_t *zcp) 263 { 264 (void) ctx_template; 265 fletcher_init(zcp); 266 (void) fletcher_2_incremental_native((void *) buf, size, zcp); 267 } 268 269 int 270 fletcher_2_incremental_byteswap(void *buf, size_t size, void *data) 271 { 272 zio_cksum_t *zcp = data; 273 274 const uint64_t *ip = buf; 275 const uint64_t *ipend = ip + (size / sizeof (uint64_t)); 276 uint64_t a0, b0, a1, b1; 277 278 a0 = zcp->zc_word[0]; 279 a1 = zcp->zc_word[1]; 280 b0 = zcp->zc_word[2]; 281 b1 = zcp->zc_word[3]; 282 283 for (; ip < ipend; ip += 2) { 284 a0 += BSWAP_64(ip[0]); 285 a1 += BSWAP_64(ip[1]); 286 b0 += a0; 287 b1 += a1; 288 } 289 290 ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1); 291 return (0); 292 } 293 294 void 295 fletcher_2_byteswap(const void *buf, uint64_t size, 296 const void *ctx_template, zio_cksum_t *zcp) 297 { 298 (void) ctx_template; 299 fletcher_init(zcp); 300 (void) fletcher_2_incremental_byteswap((void *) buf, size, zcp); 301 } 302 303 ZFS_NO_SANITIZE_UNDEFINED 304 static void 305 fletcher_4_scalar_init(fletcher_4_ctx_t *ctx) 306 { 307 ZIO_SET_CHECKSUM(&ctx->scalar, 0, 0, 0, 0); 308 } 309 310 ZFS_NO_SANITIZE_UNDEFINED 311 static void 312 fletcher_4_scalar_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp) 313 { 314 memcpy(zcp, &ctx->scalar, sizeof (zio_cksum_t)); 315 } 316 317 ZFS_NO_SANITIZE_UNDEFINED 318 static void 319 fletcher_4_scalar_native(fletcher_4_ctx_t *ctx, const void *buf, 320 uint64_t size) 321 { 322 const uint32_t *ip = buf; 323 const uint32_t *ipend = ip + (size / sizeof (uint32_t)); 324 uint64_t a, b, c, d; 325 326 a = ctx->scalar.zc_word[0]; 327 b = ctx->scalar.zc_word[1]; 328 c = ctx->scalar.zc_word[2]; 329 d = ctx->scalar.zc_word[3]; 330 331 for (; ip < ipend; ip++) { 332 a += ip[0]; 333 b += a; 334 c += b; 335 d += c; 336 } 337 338 ZIO_SET_CHECKSUM(&ctx->scalar, a, b, c, d); 339 } 340 341 ZFS_NO_SANITIZE_UNDEFINED 342 static void 343 fletcher_4_scalar_byteswap(fletcher_4_ctx_t *ctx, const void *buf, 344 uint64_t size) 345 { 346 const uint32_t *ip = buf; 347 const uint32_t *ipend = ip + (size / sizeof (uint32_t)); 348 uint64_t a, b, c, d; 349 350 a = ctx->scalar.zc_word[0]; 351 b = ctx->scalar.zc_word[1]; 352 c = ctx->scalar.zc_word[2]; 353 d = ctx->scalar.zc_word[3]; 354 355 for (; ip < ipend; ip++) { 356 a += BSWAP_32(ip[0]); 357 b += a; 358 c += b; 359 d += c; 360 } 361 362 ZIO_SET_CHECKSUM(&ctx->scalar, a, b, c, d); 363 } 364 365 static boolean_t 366 fletcher_4_scalar_valid(void) 367 { 368 return (B_TRUE); 369 } 370 371 int 372 fletcher_4_impl_set(const char *val) 373 { 374 int err = -EINVAL; 375 uint32_t impl = IMPL_READ(fletcher_4_impl_chosen); 376 size_t i, val_len; 377 378 val_len = strlen(val); 379 while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */ 380 val_len--; 381 382 /* check mandatory implementations */ 383 for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) { 384 const char *name = fletcher_4_impl_selectors[i].fis_name; 385 386 if (val_len == strlen(name) && 387 strncmp(val, name, val_len) == 0) { 388 impl = fletcher_4_impl_selectors[i].fis_sel; 389 err = 0; 390 break; 391 } 392 } 393 394 if (err != 0 && fletcher_4_initialized) { 395 /* check all supported implementations */ 396 for (i = 0; i < fletcher_4_supp_impls_cnt; i++) { 397 const char *name = fletcher_4_supp_impls[i]->name; 398 399 if (val_len == strlen(name) && 400 strncmp(val, name, val_len) == 0) { 401 impl = i; 402 err = 0; 403 break; 404 } 405 } 406 } 407 408 if (err == 0) { 409 atomic_swap_32(&fletcher_4_impl_chosen, impl); 410 membar_producer(); 411 } 412 413 return (err); 414 } 415 416 /* 417 * Returns the Fletcher 4 operations for checksums. When a SIMD 418 * implementation is not allowed in the current context, then fallback 419 * to the fastest generic implementation. 420 */ 421 static inline const fletcher_4_ops_t * 422 fletcher_4_impl_get(void) 423 { 424 if (!kfpu_allowed()) 425 return (&fletcher_4_superscalar4_ops); 426 427 const fletcher_4_ops_t *ops = NULL; 428 uint32_t impl = IMPL_READ(fletcher_4_impl_chosen); 429 430 switch (impl) { 431 case IMPL_FASTEST: 432 ASSERT(fletcher_4_initialized); 433 ops = &fletcher_4_fastest_impl; 434 break; 435 case IMPL_CYCLE: 436 /* Cycle through supported implementations */ 437 ASSERT(fletcher_4_initialized); 438 ASSERT3U(fletcher_4_supp_impls_cnt, >, 0); 439 static uint32_t cycle_count = 0; 440 uint32_t idx = (++cycle_count) % fletcher_4_supp_impls_cnt; 441 ops = fletcher_4_supp_impls[idx]; 442 break; 443 default: 444 ASSERT3U(fletcher_4_supp_impls_cnt, >, 0); 445 ASSERT3U(impl, <, fletcher_4_supp_impls_cnt); 446 ops = fletcher_4_supp_impls[impl]; 447 break; 448 } 449 450 ASSERT3P(ops, !=, NULL); 451 452 return (ops); 453 } 454 455 static inline void 456 fletcher_4_native_impl(const void *buf, uint64_t size, zio_cksum_t *zcp) 457 { 458 fletcher_4_ctx_t ctx; 459 const fletcher_4_ops_t *ops = fletcher_4_impl_get(); 460 461 ops->init_native(&ctx); 462 ops->compute_native(&ctx, buf, size); 463 ops->fini_native(&ctx, zcp); 464 } 465 466 void 467 fletcher_4_native(const void *buf, uint64_t size, 468 const void *ctx_template, zio_cksum_t *zcp) 469 { 470 (void) ctx_template; 471 const uint64_t p2size = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE); 472 473 ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t))); 474 475 if (size == 0 || p2size == 0) { 476 ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0); 477 478 if (size > 0) 479 fletcher_4_scalar_native((fletcher_4_ctx_t *)zcp, 480 buf, size); 481 } else { 482 fletcher_4_native_impl(buf, p2size, zcp); 483 484 if (p2size < size) 485 fletcher_4_scalar_native((fletcher_4_ctx_t *)zcp, 486 (char *)buf + p2size, size - p2size); 487 } 488 } 489 490 void 491 fletcher_4_native_varsize(const void *buf, uint64_t size, zio_cksum_t *zcp) 492 { 493 ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0); 494 fletcher_4_scalar_native((fletcher_4_ctx_t *)zcp, buf, size); 495 } 496 497 static inline void 498 fletcher_4_byteswap_impl(const void *buf, uint64_t size, zio_cksum_t *zcp) 499 { 500 fletcher_4_ctx_t ctx; 501 const fletcher_4_ops_t *ops = fletcher_4_impl_get(); 502 503 ops->init_byteswap(&ctx); 504 ops->compute_byteswap(&ctx, buf, size); 505 ops->fini_byteswap(&ctx, zcp); 506 } 507 508 void 509 fletcher_4_byteswap(const void *buf, uint64_t size, 510 const void *ctx_template, zio_cksum_t *zcp) 511 { 512 (void) ctx_template; 513 const uint64_t p2size = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE); 514 515 ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t))); 516 517 if (size == 0 || p2size == 0) { 518 ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0); 519 520 if (size > 0) 521 fletcher_4_scalar_byteswap((fletcher_4_ctx_t *)zcp, 522 buf, size); 523 } else { 524 fletcher_4_byteswap_impl(buf, p2size, zcp); 525 526 if (p2size < size) 527 fletcher_4_scalar_byteswap((fletcher_4_ctx_t *)zcp, 528 (char *)buf + p2size, size - p2size); 529 } 530 } 531 532 /* Incremental Fletcher 4 */ 533 534 #define ZFS_FLETCHER_4_INC_MAX_SIZE (8ULL << 20) 535 536 static inline void 537 fletcher_4_incremental_combine(zio_cksum_t *zcp, const uint64_t size, 538 const zio_cksum_t *nzcp) 539 { 540 const uint64_t c1 = size / sizeof (uint32_t); 541 const uint64_t c2 = c1 * (c1 + 1) / 2; 542 const uint64_t c3 = c2 * (c1 + 2) / 3; 543 544 /* 545 * Value of 'c3' overflows on buffer sizes close to 16MiB. For that 546 * reason we split incremental fletcher4 computation of large buffers 547 * to steps of (ZFS_FLETCHER_4_INC_MAX_SIZE) size. 548 */ 549 ASSERT3U(size, <=, ZFS_FLETCHER_4_INC_MAX_SIZE); 550 551 zcp->zc_word[3] += nzcp->zc_word[3] + c1 * zcp->zc_word[2] + 552 c2 * zcp->zc_word[1] + c3 * zcp->zc_word[0]; 553 zcp->zc_word[2] += nzcp->zc_word[2] + c1 * zcp->zc_word[1] + 554 c2 * zcp->zc_word[0]; 555 zcp->zc_word[1] += nzcp->zc_word[1] + c1 * zcp->zc_word[0]; 556 zcp->zc_word[0] += nzcp->zc_word[0]; 557 } 558 559 static inline void 560 fletcher_4_incremental_impl(boolean_t native, const void *buf, uint64_t size, 561 zio_cksum_t *zcp) 562 { 563 while (size > 0) { 564 zio_cksum_t nzc; 565 uint64_t len = MIN(size, ZFS_FLETCHER_4_INC_MAX_SIZE); 566 567 if (native) 568 fletcher_4_native(buf, len, NULL, &nzc); 569 else 570 fletcher_4_byteswap(buf, len, NULL, &nzc); 571 572 fletcher_4_incremental_combine(zcp, len, &nzc); 573 574 size -= len; 575 buf += len; 576 } 577 } 578 579 int 580 fletcher_4_incremental_native(void *buf, size_t size, void *data) 581 { 582 zio_cksum_t *zcp = data; 583 /* Use scalar impl to directly update cksum of small blocks */ 584 if (size < SPA_MINBLOCKSIZE) 585 fletcher_4_scalar_native((fletcher_4_ctx_t *)zcp, buf, size); 586 else 587 fletcher_4_incremental_impl(B_TRUE, buf, size, zcp); 588 return (0); 589 } 590 591 int 592 fletcher_4_incremental_byteswap(void *buf, size_t size, void *data) 593 { 594 zio_cksum_t *zcp = data; 595 /* Use scalar impl to directly update cksum of small blocks */ 596 if (size < SPA_MINBLOCKSIZE) 597 fletcher_4_scalar_byteswap((fletcher_4_ctx_t *)zcp, buf, size); 598 else 599 fletcher_4_incremental_impl(B_FALSE, buf, size, zcp); 600 return (0); 601 } 602 603 #if defined(_KERNEL) 604 /* 605 * Fletcher 4 kstats 606 */ 607 static int 608 fletcher_4_kstat_headers(char *buf, size_t size) 609 { 610 ssize_t off = 0; 611 612 off += snprintf(buf + off, size, "%-17s", "implementation"); 613 off += snprintf(buf + off, size - off, "%-15s", "native"); 614 (void) snprintf(buf + off, size - off, "%-15s\n", "byteswap"); 615 616 return (0); 617 } 618 619 static int 620 fletcher_4_kstat_data(char *buf, size_t size, void *data) 621 { 622 struct fletcher_4_kstat *fastest_stat = 623 &fletcher_4_stat_data[fletcher_4_supp_impls_cnt]; 624 struct fletcher_4_kstat *curr_stat = (struct fletcher_4_kstat *)data; 625 ssize_t off = 0; 626 627 if (curr_stat == fastest_stat) { 628 off += snprintf(buf + off, size - off, "%-17s", "fastest"); 629 off += snprintf(buf + off, size - off, "%-15s", 630 fletcher_4_supp_impls[fastest_stat->native]->name); 631 off += snprintf(buf + off, size - off, "%-15s\n", 632 fletcher_4_supp_impls[fastest_stat->byteswap]->name); 633 } else { 634 ptrdiff_t id = curr_stat - fletcher_4_stat_data; 635 636 off += snprintf(buf + off, size - off, "%-17s", 637 fletcher_4_supp_impls[id]->name); 638 off += snprintf(buf + off, size - off, "%-15llu", 639 (u_longlong_t)curr_stat->native); 640 off += snprintf(buf + off, size - off, "%-15llu\n", 641 (u_longlong_t)curr_stat->byteswap); 642 } 643 644 return (0); 645 } 646 647 static void * 648 fletcher_4_kstat_addr(kstat_t *ksp, loff_t n) 649 { 650 if (n <= fletcher_4_supp_impls_cnt) 651 ksp->ks_private = (void *) (fletcher_4_stat_data + n); 652 else 653 ksp->ks_private = NULL; 654 655 return (ksp->ks_private); 656 } 657 #endif 658 659 #define FLETCHER_4_FASTEST_FN_COPY(type, src) \ 660 { \ 661 fletcher_4_fastest_impl.init_ ## type = src->init_ ## type; \ 662 fletcher_4_fastest_impl.fini_ ## type = src->fini_ ## type; \ 663 fletcher_4_fastest_impl.compute_ ## type = src->compute_ ## type; \ 664 } 665 666 #define FLETCHER_4_BENCH_NS (MSEC2NSEC(1)) /* 1ms */ 667 668 typedef void fletcher_checksum_func_t(const void *, uint64_t, const void *, 669 zio_cksum_t *); 670 671 #if defined(_KERNEL) 672 static void 673 fletcher_4_benchmark_impl(boolean_t native, char *data, uint64_t data_size) 674 { 675 676 struct fletcher_4_kstat *fastest_stat = 677 &fletcher_4_stat_data[fletcher_4_supp_impls_cnt]; 678 hrtime_t start; 679 uint64_t run_bw, run_time_ns, best_run = 0; 680 zio_cksum_t zc; 681 uint32_t i, l, sel_save = IMPL_READ(fletcher_4_impl_chosen); 682 683 fletcher_checksum_func_t *fletcher_4_test = native ? 684 fletcher_4_native : fletcher_4_byteswap; 685 686 for (i = 0; i < fletcher_4_supp_impls_cnt; i++) { 687 struct fletcher_4_kstat *stat = &fletcher_4_stat_data[i]; 688 uint64_t run_count = 0; 689 690 /* temporary set an implementation */ 691 fletcher_4_impl_chosen = i; 692 693 kpreempt_disable(); 694 start = gethrtime(); 695 do { 696 for (l = 0; l < 32; l++, run_count++) 697 fletcher_4_test(data, data_size, NULL, &zc); 698 699 run_time_ns = gethrtime() - start; 700 } while (run_time_ns < FLETCHER_4_BENCH_NS); 701 kpreempt_enable(); 702 703 run_bw = data_size * run_count * NANOSEC; 704 run_bw /= run_time_ns; /* B/s */ 705 706 if (native) 707 stat->native = run_bw; 708 else 709 stat->byteswap = run_bw; 710 711 if (run_bw > best_run) { 712 best_run = run_bw; 713 714 if (native) { 715 fastest_stat->native = i; 716 FLETCHER_4_FASTEST_FN_COPY(native, 717 fletcher_4_supp_impls[i]); 718 } else { 719 fastest_stat->byteswap = i; 720 FLETCHER_4_FASTEST_FN_COPY(byteswap, 721 fletcher_4_supp_impls[i]); 722 } 723 } 724 } 725 726 /* restore original selection */ 727 atomic_swap_32(&fletcher_4_impl_chosen, sel_save); 728 } 729 #endif /* _KERNEL */ 730 731 /* 732 * Initialize and benchmark all supported implementations. 733 */ 734 static void 735 fletcher_4_benchmark(void) 736 { 737 fletcher_4_ops_t *curr_impl; 738 int i, c; 739 740 /* Move supported implementations into fletcher_4_supp_impls */ 741 for (i = 0, c = 0; i < ARRAY_SIZE(fletcher_4_impls); i++) { 742 curr_impl = (fletcher_4_ops_t *)fletcher_4_impls[i]; 743 744 if (curr_impl->valid && curr_impl->valid()) 745 fletcher_4_supp_impls[c++] = curr_impl; 746 } 747 membar_producer(); /* complete fletcher_4_supp_impls[] init */ 748 fletcher_4_supp_impls_cnt = c; /* number of supported impl */ 749 750 #if defined(_KERNEL) 751 static const size_t data_size = 1 << SPA_OLD_MAXBLOCKSHIFT; /* 128kiB */ 752 char *databuf = vmem_alloc(data_size, KM_SLEEP); 753 754 for (i = 0; i < data_size / sizeof (uint64_t); i++) 755 ((uint64_t *)databuf)[i] = (uintptr_t)(databuf+i); /* warm-up */ 756 757 fletcher_4_benchmark_impl(B_FALSE, databuf, data_size); 758 fletcher_4_benchmark_impl(B_TRUE, databuf, data_size); 759 760 vmem_free(databuf, data_size); 761 #else 762 /* 763 * Skip the benchmark in user space to avoid impacting libzpool 764 * consumers (zdb, zhack, zinject, ztest). The last implementation 765 * is assumed to be the fastest and used by default. 766 */ 767 memcpy(&fletcher_4_fastest_impl, 768 fletcher_4_supp_impls[fletcher_4_supp_impls_cnt - 1], 769 sizeof (fletcher_4_fastest_impl)); 770 fletcher_4_fastest_impl.name = "fastest"; 771 membar_producer(); 772 #endif /* _KERNEL */ 773 } 774 775 void 776 fletcher_4_init(void) 777 { 778 /* Determine the fastest available implementation. */ 779 fletcher_4_benchmark(); 780 781 #if defined(_KERNEL) 782 /* Install kstats for all implementations */ 783 fletcher_4_kstat = kstat_create("zfs", 0, "fletcher_4_bench", "misc", 784 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); 785 if (fletcher_4_kstat != NULL) { 786 fletcher_4_kstat->ks_data = NULL; 787 fletcher_4_kstat->ks_ndata = UINT32_MAX; 788 kstat_set_raw_ops(fletcher_4_kstat, 789 fletcher_4_kstat_headers, 790 fletcher_4_kstat_data, 791 fletcher_4_kstat_addr); 792 kstat_install(fletcher_4_kstat); 793 } 794 #endif 795 796 /* Finish initialization */ 797 fletcher_4_initialized = B_TRUE; 798 } 799 800 void 801 fletcher_4_fini(void) 802 { 803 #if defined(_KERNEL) 804 if (fletcher_4_kstat != NULL) { 805 kstat_delete(fletcher_4_kstat); 806 fletcher_4_kstat = NULL; 807 } 808 #endif 809 } 810 811 /* ABD adapters */ 812 813 static void 814 abd_fletcher_4_init(zio_abd_checksum_data_t *cdp) 815 { 816 const fletcher_4_ops_t *ops = fletcher_4_impl_get(); 817 cdp->acd_private = (void *) ops; 818 819 if (cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE) 820 ops->init_native(cdp->acd_ctx); 821 else 822 ops->init_byteswap(cdp->acd_ctx); 823 } 824 825 static void 826 abd_fletcher_4_fini(zio_abd_checksum_data_t *cdp) 827 { 828 fletcher_4_ops_t *ops = (fletcher_4_ops_t *)cdp->acd_private; 829 830 ASSERT(ops); 831 832 if (cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE) 833 ops->fini_native(cdp->acd_ctx, cdp->acd_zcp); 834 else 835 ops->fini_byteswap(cdp->acd_ctx, cdp->acd_zcp); 836 } 837 838 static void 839 abd_fletcher_4_simd2scalar(boolean_t native, void *data, size_t size, 840 zio_abd_checksum_data_t *cdp) 841 { 842 zio_cksum_t *zcp = cdp->acd_zcp; 843 844 ASSERT3U(size, <, FLETCHER_MIN_SIMD_SIZE); 845 846 abd_fletcher_4_fini(cdp); 847 cdp->acd_private = (void *)&fletcher_4_scalar_ops; 848 849 if (native) 850 fletcher_4_incremental_native(data, size, zcp); 851 else 852 fletcher_4_incremental_byteswap(data, size, zcp); 853 } 854 855 static int 856 abd_fletcher_4_iter(void *data, size_t size, void *private) 857 { 858 zio_abd_checksum_data_t *cdp = (zio_abd_checksum_data_t *)private; 859 fletcher_4_ctx_t *ctx = cdp->acd_ctx; 860 fletcher_4_ops_t *ops = (fletcher_4_ops_t *)cdp->acd_private; 861 boolean_t native = cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE; 862 uint64_t asize = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE); 863 864 ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t))); 865 866 if (asize > 0) { 867 if (native) 868 ops->compute_native(ctx, data, asize); 869 else 870 ops->compute_byteswap(ctx, data, asize); 871 872 size -= asize; 873 data = (char *)data + asize; 874 } 875 876 if (size > 0) { 877 ASSERT3U(size, <, FLETCHER_MIN_SIMD_SIZE); 878 /* At this point we have to switch to scalar impl */ 879 abd_fletcher_4_simd2scalar(native, data, size, cdp); 880 } 881 882 return (0); 883 } 884 885 zio_abd_checksum_func_t fletcher_4_abd_ops = { 886 .acf_init = abd_fletcher_4_init, 887 .acf_fini = abd_fletcher_4_fini, 888 .acf_iter = abd_fletcher_4_iter 889 }; 890 891 #if defined(_KERNEL) 892 893 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ") 894 895 #if defined(__linux__) 896 897 static int 898 fletcher_4_param_get(char *buffer, zfs_kernel_param_t *unused) 899 { 900 const uint32_t impl = IMPL_READ(fletcher_4_impl_chosen); 901 char *fmt; 902 int cnt = 0; 903 904 /* list fastest */ 905 fmt = IMPL_FMT(impl, IMPL_FASTEST); 906 cnt += sprintf(buffer + cnt, fmt, "fastest"); 907 908 /* list all supported implementations */ 909 for (uint32_t i = 0; i < fletcher_4_supp_impls_cnt; ++i) { 910 fmt = IMPL_FMT(impl, i); 911 cnt += sprintf(buffer + cnt, fmt, 912 fletcher_4_supp_impls[i]->name); 913 } 914 915 return (cnt); 916 } 917 918 static int 919 fletcher_4_param_set(const char *val, zfs_kernel_param_t *unused) 920 { 921 return (fletcher_4_impl_set(val)); 922 } 923 924 #else 925 926 #include <sys/sbuf.h> 927 928 static int 929 fletcher_4_param(ZFS_MODULE_PARAM_ARGS) 930 { 931 int err; 932 933 if (req->newptr == NULL) { 934 const uint32_t impl = IMPL_READ(fletcher_4_impl_chosen); 935 const int init_buflen = 64; 936 const char *fmt; 937 struct sbuf *s; 938 939 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req); 940 941 /* list fastest */ 942 fmt = IMPL_FMT(impl, IMPL_FASTEST); 943 (void) sbuf_printf(s, fmt, "fastest"); 944 945 /* list all supported implementations */ 946 for (uint32_t i = 0; i < fletcher_4_supp_impls_cnt; ++i) { 947 fmt = IMPL_FMT(impl, i); 948 (void) sbuf_printf(s, fmt, 949 fletcher_4_supp_impls[i]->name); 950 } 951 952 err = sbuf_finish(s); 953 sbuf_delete(s); 954 955 return (err); 956 } 957 958 char buf[16]; 959 960 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 961 if (err) 962 return (err); 963 return (-fletcher_4_impl_set(buf)); 964 } 965 966 #endif 967 968 #undef IMPL_FMT 969 970 /* 971 * Choose a fletcher 4 implementation in ZFS. 972 * Users can choose "cycle" to exercise all implementations, but this is 973 * for testing purpose therefore it can only be set in user space. 974 */ 975 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, fletcher_4_impl, 976 fletcher_4_param_set, fletcher_4_param_get, ZMOD_RW, 977 "Select fletcher 4 implementation."); 978 979 EXPORT_SYMBOL(fletcher_init); 980 EXPORT_SYMBOL(fletcher_2_incremental_native); 981 EXPORT_SYMBOL(fletcher_2_incremental_byteswap); 982 EXPORT_SYMBOL(fletcher_4_init); 983 EXPORT_SYMBOL(fletcher_4_fini); 984 EXPORT_SYMBOL(fletcher_2_native); 985 EXPORT_SYMBOL(fletcher_2_byteswap); 986 EXPORT_SYMBOL(fletcher_4_native); 987 EXPORT_SYMBOL(fletcher_4_native_varsize); 988 EXPORT_SYMBOL(fletcher_4_byteswap); 989 EXPORT_SYMBOL(fletcher_4_incremental_native); 990 EXPORT_SYMBOL(fletcher_4_incremental_byteswap); 991 EXPORT_SYMBOL(fletcher_4_abd_ops); 992 #endif 993