1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2012, 2018 by Delphix. All rights reserved. 24 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 25 */ 26 27 #include <assert.h> 28 #include <fcntl.h> 29 #include <libgen.h> 30 #include <poll.h> 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <string.h> 34 #include <limits.h> 35 #include <libzutil.h> 36 #include <sys/crypto/icp.h> 37 #include <sys/processor.h> 38 #include <sys/rrwlock.h> 39 #include <sys/spa.h> 40 #include <sys/stat.h> 41 #include <sys/systeminfo.h> 42 #include <sys/time.h> 43 #include <sys/utsname.h> 44 #include <sys/zfs_context.h> 45 #include <sys/zfs_onexit.h> 46 #include <sys/zfs_vfsops.h> 47 #include <sys/zstd/zstd.h> 48 #include <sys/zvol.h> 49 #include <zfs_fletcher.h> 50 #include <zlib.h> 51 52 /* 53 * Emulation of kernel services in userland. 54 */ 55 56 uint64_t physmem; 57 uint32_t hostid; 58 struct utsname hw_utsname; 59 60 /* If set, all blocks read will be copied to the specified directory. */ 61 char *vn_dumpdir = NULL; 62 63 /* this only exists to have its address taken */ 64 struct proc p0; 65 66 /* 67 * ========================================================================= 68 * threads 69 * ========================================================================= 70 * 71 * TS_STACK_MIN is dictated by the minimum allowed pthread stack size. While 72 * TS_STACK_MAX is somewhat arbitrary, it was selected to be large enough for 73 * the expected stack depth while small enough to avoid exhausting address 74 * space with high thread counts. 75 */ 76 #define TS_STACK_MIN MAX(PTHREAD_STACK_MIN, 32768) 77 #define TS_STACK_MAX (256 * 1024) 78 79 struct zk_thread_wrapper { 80 void (*func)(void *); 81 void *arg; 82 }; 83 84 static void * 85 zk_thread_wrapper(void *arg) 86 { 87 struct zk_thread_wrapper ztw; 88 memcpy(&ztw, arg, sizeof (ztw)); 89 free(arg); 90 ztw.func(ztw.arg); 91 return (NULL); 92 } 93 94 kthread_t * 95 zk_thread_create(void (*func)(void *), void *arg, size_t stksize, int state) 96 { 97 pthread_attr_t attr; 98 pthread_t tid; 99 char *stkstr; 100 struct zk_thread_wrapper *ztw; 101 int detachstate = PTHREAD_CREATE_DETACHED; 102 103 VERIFY0(pthread_attr_init(&attr)); 104 105 if (state & TS_JOINABLE) 106 detachstate = PTHREAD_CREATE_JOINABLE; 107 108 VERIFY0(pthread_attr_setdetachstate(&attr, detachstate)); 109 110 /* 111 * We allow the default stack size in user space to be specified by 112 * setting the ZFS_STACK_SIZE environment variable. This allows us 113 * the convenience of observing and debugging stack overruns in 114 * user space. Explicitly specified stack sizes will be honored. 115 * The usage of ZFS_STACK_SIZE is discussed further in the 116 * ENVIRONMENT VARIABLES sections of the ztest(1) man page. 117 */ 118 if (stksize == 0) { 119 stkstr = getenv("ZFS_STACK_SIZE"); 120 121 if (stkstr == NULL) 122 stksize = TS_STACK_MAX; 123 else 124 stksize = MAX(atoi(stkstr), TS_STACK_MIN); 125 } 126 127 VERIFY3S(stksize, >, 0); 128 stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE); 129 130 /* 131 * If this ever fails, it may be because the stack size is not a 132 * multiple of system page size. 133 */ 134 VERIFY0(pthread_attr_setstacksize(&attr, stksize)); 135 VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE)); 136 137 VERIFY(ztw = malloc(sizeof (*ztw))); 138 ztw->func = func; 139 ztw->arg = arg; 140 VERIFY0(pthread_create(&tid, &attr, zk_thread_wrapper, ztw)); 141 VERIFY0(pthread_attr_destroy(&attr)); 142 143 return ((void *)(uintptr_t)tid); 144 } 145 146 /* 147 * ========================================================================= 148 * kstats 149 * ========================================================================= 150 */ 151 kstat_t * 152 kstat_create(const char *module, int instance, const char *name, 153 const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag) 154 { 155 (void) module, (void) instance, (void) name, (void) class, (void) type, 156 (void) ndata, (void) ks_flag; 157 return (NULL); 158 } 159 160 void 161 kstat_install(kstat_t *ksp) 162 { 163 (void) ksp; 164 } 165 166 void 167 kstat_delete(kstat_t *ksp) 168 { 169 (void) ksp; 170 } 171 172 void 173 kstat_set_raw_ops(kstat_t *ksp, 174 int (*headers)(char *buf, size_t size), 175 int (*data)(char *buf, size_t size, void *data), 176 void *(*addr)(kstat_t *ksp, loff_t index)) 177 { 178 (void) ksp, (void) headers, (void) data, (void) addr; 179 } 180 181 /* 182 * ========================================================================= 183 * mutexes 184 * ========================================================================= 185 */ 186 187 void 188 mutex_init(kmutex_t *mp, char *name, int type, void *cookie) 189 { 190 (void) name, (void) type, (void) cookie; 191 VERIFY0(pthread_mutex_init(&mp->m_lock, NULL)); 192 memset(&mp->m_owner, 0, sizeof (pthread_t)); 193 } 194 195 void 196 mutex_destroy(kmutex_t *mp) 197 { 198 VERIFY0(pthread_mutex_destroy(&mp->m_lock)); 199 } 200 201 void 202 mutex_enter(kmutex_t *mp) 203 { 204 VERIFY0(pthread_mutex_lock(&mp->m_lock)); 205 mp->m_owner = pthread_self(); 206 } 207 208 int 209 mutex_tryenter(kmutex_t *mp) 210 { 211 int error = pthread_mutex_trylock(&mp->m_lock); 212 if (error == 0) { 213 mp->m_owner = pthread_self(); 214 return (1); 215 } else { 216 VERIFY3S(error, ==, EBUSY); 217 return (0); 218 } 219 } 220 221 void 222 mutex_exit(kmutex_t *mp) 223 { 224 memset(&mp->m_owner, 0, sizeof (pthread_t)); 225 VERIFY0(pthread_mutex_unlock(&mp->m_lock)); 226 } 227 228 /* 229 * ========================================================================= 230 * rwlocks 231 * ========================================================================= 232 */ 233 234 void 235 rw_init(krwlock_t *rwlp, char *name, int type, void *arg) 236 { 237 (void) name, (void) type, (void) arg; 238 VERIFY0(pthread_rwlock_init(&rwlp->rw_lock, NULL)); 239 rwlp->rw_readers = 0; 240 rwlp->rw_owner = 0; 241 } 242 243 void 244 rw_destroy(krwlock_t *rwlp) 245 { 246 VERIFY0(pthread_rwlock_destroy(&rwlp->rw_lock)); 247 } 248 249 void 250 rw_enter(krwlock_t *rwlp, krw_t rw) 251 { 252 if (rw == RW_READER) { 253 VERIFY0(pthread_rwlock_rdlock(&rwlp->rw_lock)); 254 atomic_inc_uint(&rwlp->rw_readers); 255 } else { 256 VERIFY0(pthread_rwlock_wrlock(&rwlp->rw_lock)); 257 rwlp->rw_owner = pthread_self(); 258 } 259 } 260 261 void 262 rw_exit(krwlock_t *rwlp) 263 { 264 if (RW_READ_HELD(rwlp)) 265 atomic_dec_uint(&rwlp->rw_readers); 266 else 267 rwlp->rw_owner = 0; 268 269 VERIFY0(pthread_rwlock_unlock(&rwlp->rw_lock)); 270 } 271 272 int 273 rw_tryenter(krwlock_t *rwlp, krw_t rw) 274 { 275 int error; 276 277 if (rw == RW_READER) 278 error = pthread_rwlock_tryrdlock(&rwlp->rw_lock); 279 else 280 error = pthread_rwlock_trywrlock(&rwlp->rw_lock); 281 282 if (error == 0) { 283 if (rw == RW_READER) 284 atomic_inc_uint(&rwlp->rw_readers); 285 else 286 rwlp->rw_owner = pthread_self(); 287 288 return (1); 289 } 290 291 VERIFY3S(error, ==, EBUSY); 292 293 return (0); 294 } 295 296 uint32_t 297 zone_get_hostid(void *zonep) 298 { 299 /* 300 * We're emulating the system's hostid in userland. 301 */ 302 (void) zonep; 303 return (hostid); 304 } 305 306 int 307 rw_tryupgrade(krwlock_t *rwlp) 308 { 309 (void) rwlp; 310 return (0); 311 } 312 313 /* 314 * ========================================================================= 315 * condition variables 316 * ========================================================================= 317 */ 318 319 void 320 cv_init(kcondvar_t *cv, char *name, int type, void *arg) 321 { 322 (void) name, (void) type, (void) arg; 323 VERIFY0(pthread_cond_init(cv, NULL)); 324 } 325 326 void 327 cv_destroy(kcondvar_t *cv) 328 { 329 VERIFY0(pthread_cond_destroy(cv)); 330 } 331 332 void 333 cv_wait(kcondvar_t *cv, kmutex_t *mp) 334 { 335 memset(&mp->m_owner, 0, sizeof (pthread_t)); 336 VERIFY0(pthread_cond_wait(cv, &mp->m_lock)); 337 mp->m_owner = pthread_self(); 338 } 339 340 int 341 cv_wait_sig(kcondvar_t *cv, kmutex_t *mp) 342 { 343 cv_wait(cv, mp); 344 return (1); 345 } 346 347 int 348 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime) 349 { 350 int error; 351 struct timeval tv; 352 struct timespec ts; 353 clock_t delta; 354 355 delta = abstime - ddi_get_lbolt(); 356 if (delta <= 0) 357 return (-1); 358 359 VERIFY(gettimeofday(&tv, NULL) == 0); 360 361 ts.tv_sec = tv.tv_sec + delta / hz; 362 ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % hz) * (NANOSEC / hz); 363 if (ts.tv_nsec >= NANOSEC) { 364 ts.tv_sec++; 365 ts.tv_nsec -= NANOSEC; 366 } 367 368 memset(&mp->m_owner, 0, sizeof (pthread_t)); 369 error = pthread_cond_timedwait(cv, &mp->m_lock, &ts); 370 mp->m_owner = pthread_self(); 371 372 if (error == ETIMEDOUT) 373 return (-1); 374 375 VERIFY0(error); 376 377 return (1); 378 } 379 380 int 381 cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res, 382 int flag) 383 { 384 (void) res; 385 int error; 386 struct timeval tv; 387 struct timespec ts; 388 hrtime_t delta; 389 390 ASSERT(flag == 0 || flag == CALLOUT_FLAG_ABSOLUTE); 391 392 delta = tim; 393 if (flag & CALLOUT_FLAG_ABSOLUTE) 394 delta -= gethrtime(); 395 396 if (delta <= 0) 397 return (-1); 398 399 VERIFY0(gettimeofday(&tv, NULL)); 400 401 ts.tv_sec = tv.tv_sec + delta / NANOSEC; 402 ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % NANOSEC); 403 if (ts.tv_nsec >= NANOSEC) { 404 ts.tv_sec++; 405 ts.tv_nsec -= NANOSEC; 406 } 407 408 memset(&mp->m_owner, 0, sizeof (pthread_t)); 409 error = pthread_cond_timedwait(cv, &mp->m_lock, &ts); 410 mp->m_owner = pthread_self(); 411 412 if (error == ETIMEDOUT) 413 return (-1); 414 415 VERIFY0(error); 416 417 return (1); 418 } 419 420 void 421 cv_signal(kcondvar_t *cv) 422 { 423 VERIFY0(pthread_cond_signal(cv)); 424 } 425 426 void 427 cv_broadcast(kcondvar_t *cv) 428 { 429 VERIFY0(pthread_cond_broadcast(cv)); 430 } 431 432 /* 433 * ========================================================================= 434 * procfs list 435 * ========================================================================= 436 */ 437 438 void 439 seq_printf(struct seq_file *m, const char *fmt, ...) 440 { 441 (void) m, (void) fmt; 442 } 443 444 void 445 procfs_list_install(const char *module, 446 const char *submodule, 447 const char *name, 448 mode_t mode, 449 procfs_list_t *procfs_list, 450 int (*show)(struct seq_file *f, void *p), 451 int (*show_header)(struct seq_file *f), 452 int (*clear)(procfs_list_t *procfs_list), 453 size_t procfs_list_node_off) 454 { 455 (void) module, (void) submodule, (void) name, (void) mode, (void) show, 456 (void) show_header, (void) clear; 457 mutex_init(&procfs_list->pl_lock, NULL, MUTEX_DEFAULT, NULL); 458 list_create(&procfs_list->pl_list, 459 procfs_list_node_off + sizeof (procfs_list_node_t), 460 procfs_list_node_off + offsetof(procfs_list_node_t, pln_link)); 461 procfs_list->pl_next_id = 1; 462 procfs_list->pl_node_offset = procfs_list_node_off; 463 } 464 465 void 466 procfs_list_uninstall(procfs_list_t *procfs_list) 467 { 468 (void) procfs_list; 469 } 470 471 void 472 procfs_list_destroy(procfs_list_t *procfs_list) 473 { 474 ASSERT(list_is_empty(&procfs_list->pl_list)); 475 list_destroy(&procfs_list->pl_list); 476 mutex_destroy(&procfs_list->pl_lock); 477 } 478 479 #define NODE_ID(procfs_list, obj) \ 480 (((procfs_list_node_t *)(((char *)obj) + \ 481 (procfs_list)->pl_node_offset))->pln_id) 482 483 void 484 procfs_list_add(procfs_list_t *procfs_list, void *p) 485 { 486 ASSERT(MUTEX_HELD(&procfs_list->pl_lock)); 487 NODE_ID(procfs_list, p) = procfs_list->pl_next_id++; 488 list_insert_tail(&procfs_list->pl_list, p); 489 } 490 491 /* 492 * ========================================================================= 493 * vnode operations 494 * ========================================================================= 495 */ 496 497 /* 498 * ========================================================================= 499 * Figure out which debugging statements to print 500 * ========================================================================= 501 */ 502 503 static char *dprintf_string; 504 static int dprintf_print_all; 505 506 int 507 dprintf_find_string(const char *string) 508 { 509 char *tmp_str = dprintf_string; 510 int len = strlen(string); 511 512 /* 513 * Find out if this is a string we want to print. 514 * String format: file1.c,function_name1,file2.c,file3.c 515 */ 516 517 while (tmp_str != NULL) { 518 if (strncmp(tmp_str, string, len) == 0 && 519 (tmp_str[len] == ',' || tmp_str[len] == '\0')) 520 return (1); 521 tmp_str = strchr(tmp_str, ','); 522 if (tmp_str != NULL) 523 tmp_str++; /* Get rid of , */ 524 } 525 return (0); 526 } 527 528 void 529 dprintf_setup(int *argc, char **argv) 530 { 531 int i, j; 532 533 /* 534 * Debugging can be specified two ways: by setting the 535 * environment variable ZFS_DEBUG, or by including a 536 * "debug=..." argument on the command line. The command 537 * line setting overrides the environment variable. 538 */ 539 540 for (i = 1; i < *argc; i++) { 541 int len = strlen("debug="); 542 /* First look for a command line argument */ 543 if (strncmp("debug=", argv[i], len) == 0) { 544 dprintf_string = argv[i] + len; 545 /* Remove from args */ 546 for (j = i; j < *argc; j++) 547 argv[j] = argv[j+1]; 548 argv[j] = NULL; 549 (*argc)--; 550 } 551 } 552 553 if (dprintf_string == NULL) { 554 /* Look for ZFS_DEBUG environment variable */ 555 dprintf_string = getenv("ZFS_DEBUG"); 556 } 557 558 /* 559 * Are we just turning on all debugging? 560 */ 561 if (dprintf_find_string("on")) 562 dprintf_print_all = 1; 563 564 if (dprintf_string != NULL) 565 zfs_flags |= ZFS_DEBUG_DPRINTF; 566 } 567 568 /* 569 * ========================================================================= 570 * debug printfs 571 * ========================================================================= 572 */ 573 void 574 __dprintf(boolean_t dprint, const char *file, const char *func, 575 int line, const char *fmt, ...) 576 { 577 /* Get rid of annoying "../common/" prefix to filename. */ 578 const char *newfile = zfs_basename(file); 579 580 va_list adx; 581 if (dprint) { 582 /* dprintf messages are printed immediately */ 583 584 if (!dprintf_print_all && 585 !dprintf_find_string(newfile) && 586 !dprintf_find_string(func)) 587 return; 588 589 /* Print out just the function name if requested */ 590 flockfile(stdout); 591 if (dprintf_find_string("pid")) 592 (void) printf("%d ", getpid()); 593 if (dprintf_find_string("tid")) 594 (void) printf("%ju ", 595 (uintmax_t)(uintptr_t)pthread_self()); 596 if (dprintf_find_string("cpu")) 597 (void) printf("%u ", getcpuid()); 598 if (dprintf_find_string("time")) 599 (void) printf("%llu ", gethrtime()); 600 if (dprintf_find_string("long")) 601 (void) printf("%s, line %d: ", newfile, line); 602 (void) printf("dprintf: %s: ", func); 603 va_start(adx, fmt); 604 (void) vprintf(fmt, adx); 605 va_end(adx); 606 funlockfile(stdout); 607 } else { 608 /* zfs_dbgmsg is logged for dumping later */ 609 size_t size; 610 char *buf; 611 int i; 612 613 size = 1024; 614 buf = umem_alloc(size, UMEM_NOFAIL); 615 i = snprintf(buf, size, "%s:%d:%s(): ", newfile, line, func); 616 617 if (i < size) { 618 va_start(adx, fmt); 619 (void) vsnprintf(buf + i, size - i, fmt, adx); 620 va_end(adx); 621 } 622 623 __zfs_dbgmsg(buf); 624 625 umem_free(buf, size); 626 } 627 } 628 629 /* 630 * ========================================================================= 631 * cmn_err() and panic() 632 * ========================================================================= 633 */ 634 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" }; 635 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" }; 636 637 __attribute__((noreturn)) void 638 vpanic(const char *fmt, va_list adx) 639 { 640 (void) fprintf(stderr, "error: "); 641 (void) vfprintf(stderr, fmt, adx); 642 (void) fprintf(stderr, "\n"); 643 644 abort(); /* think of it as a "user-level crash dump" */ 645 } 646 647 __attribute__((noreturn)) void 648 panic(const char *fmt, ...) 649 { 650 va_list adx; 651 652 va_start(adx, fmt); 653 vpanic(fmt, adx); 654 va_end(adx); 655 } 656 657 void 658 vcmn_err(int ce, const char *fmt, va_list adx) 659 { 660 if (ce == CE_PANIC) 661 vpanic(fmt, adx); 662 if (ce != CE_NOTE) { /* suppress noise in userland stress testing */ 663 (void) fprintf(stderr, "%s", ce_prefix[ce]); 664 (void) vfprintf(stderr, fmt, adx); 665 (void) fprintf(stderr, "%s", ce_suffix[ce]); 666 } 667 } 668 669 void 670 cmn_err(int ce, const char *fmt, ...) 671 { 672 va_list adx; 673 674 va_start(adx, fmt); 675 vcmn_err(ce, fmt, adx); 676 va_end(adx); 677 } 678 679 /* 680 * ========================================================================= 681 * misc routines 682 * ========================================================================= 683 */ 684 685 void 686 delay(clock_t ticks) 687 { 688 (void) poll(0, 0, ticks * (1000 / hz)); 689 } 690 691 /* 692 * Find highest one bit set. 693 * Returns bit number + 1 of highest bit that is set, otherwise returns 0. 694 * The __builtin_clzll() function is supported by both GCC and Clang. 695 */ 696 int 697 highbit64(uint64_t i) 698 { 699 if (i == 0) 700 return (0); 701 702 return (NBBY * sizeof (uint64_t) - __builtin_clzll(i)); 703 } 704 705 /* 706 * Find lowest one bit set. 707 * Returns bit number + 1 of lowest bit that is set, otherwise returns 0. 708 * The __builtin_ffsll() function is supported by both GCC and Clang. 709 */ 710 int 711 lowbit64(uint64_t i) 712 { 713 if (i == 0) 714 return (0); 715 716 return (__builtin_ffsll(i)); 717 } 718 719 const char *random_path = "/dev/random"; 720 const char *urandom_path = "/dev/urandom"; 721 static int random_fd = -1, urandom_fd = -1; 722 723 void 724 random_init(void) 725 { 726 VERIFY((random_fd = open(random_path, O_RDONLY | O_CLOEXEC)) != -1); 727 VERIFY((urandom_fd = open(urandom_path, O_RDONLY | O_CLOEXEC)) != -1); 728 } 729 730 void 731 random_fini(void) 732 { 733 close(random_fd); 734 close(urandom_fd); 735 736 random_fd = -1; 737 urandom_fd = -1; 738 } 739 740 static int 741 random_get_bytes_common(uint8_t *ptr, size_t len, int fd) 742 { 743 size_t resid = len; 744 ssize_t bytes; 745 746 ASSERT(fd != -1); 747 748 while (resid != 0) { 749 bytes = read(fd, ptr, resid); 750 ASSERT3S(bytes, >=, 0); 751 ptr += bytes; 752 resid -= bytes; 753 } 754 755 return (0); 756 } 757 758 int 759 random_get_bytes(uint8_t *ptr, size_t len) 760 { 761 return (random_get_bytes_common(ptr, len, random_fd)); 762 } 763 764 int 765 random_get_pseudo_bytes(uint8_t *ptr, size_t len) 766 { 767 return (random_get_bytes_common(ptr, len, urandom_fd)); 768 } 769 770 int 771 ddi_strtoull(const char *str, char **nptr, int base, u_longlong_t *result) 772 { 773 errno = 0; 774 *result = strtoull(str, nptr, base); 775 if (*result == 0) 776 return (errno); 777 return (0); 778 } 779 780 utsname_t * 781 utsname(void) 782 { 783 return (&hw_utsname); 784 } 785 786 /* 787 * ========================================================================= 788 * kernel emulation setup & teardown 789 * ========================================================================= 790 */ 791 static int 792 umem_out_of_memory(void) 793 { 794 char errmsg[] = "out of memory -- generating core dump\n"; 795 796 (void) fprintf(stderr, "%s", errmsg); 797 abort(); 798 return (0); 799 } 800 801 void 802 kernel_init(int mode) 803 { 804 extern uint_t rrw_tsd_key; 805 806 umem_nofail_callback(umem_out_of_memory); 807 808 physmem = sysconf(_SC_PHYS_PAGES); 809 810 dprintf("physmem = %llu pages (%.2f GB)\n", (u_longlong_t)physmem, 811 (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30)); 812 813 hostid = (mode & SPA_MODE_WRITE) ? get_system_hostid() : 0; 814 815 random_init(); 816 817 VERIFY0(uname(&hw_utsname)); 818 819 system_taskq_init(); 820 icp_init(); 821 822 zstd_init(); 823 824 spa_init((spa_mode_t)mode); 825 826 fletcher_4_init(); 827 828 tsd_create(&rrw_tsd_key, rrw_tsd_destroy); 829 } 830 831 void 832 kernel_fini(void) 833 { 834 fletcher_4_fini(); 835 spa_fini(); 836 837 zstd_fini(); 838 839 icp_fini(); 840 system_taskq_fini(); 841 842 random_fini(); 843 } 844 845 uid_t 846 crgetuid(cred_t *cr) 847 { 848 (void) cr; 849 return (0); 850 } 851 852 uid_t 853 crgetruid(cred_t *cr) 854 { 855 (void) cr; 856 return (0); 857 } 858 859 gid_t 860 crgetgid(cred_t *cr) 861 { 862 (void) cr; 863 return (0); 864 } 865 866 int 867 crgetngroups(cred_t *cr) 868 { 869 (void) cr; 870 return (0); 871 } 872 873 gid_t * 874 crgetgroups(cred_t *cr) 875 { 876 (void) cr; 877 return (NULL); 878 } 879 880 int 881 zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr) 882 { 883 (void) name, (void) cr; 884 return (0); 885 } 886 887 int 888 zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr) 889 { 890 (void) from, (void) to, (void) cr; 891 return (0); 892 } 893 894 int 895 zfs_secpolicy_destroy_perms(const char *name, cred_t *cr) 896 { 897 (void) name, (void) cr; 898 return (0); 899 } 900 901 int 902 secpolicy_zfs(const cred_t *cr) 903 { 904 (void) cr; 905 return (0); 906 } 907 908 int 909 secpolicy_zfs_proc(const cred_t *cr, proc_t *proc) 910 { 911 (void) cr, (void) proc; 912 return (0); 913 } 914 915 ksiddomain_t * 916 ksid_lookupdomain(const char *dom) 917 { 918 ksiddomain_t *kd; 919 920 kd = umem_zalloc(sizeof (ksiddomain_t), UMEM_NOFAIL); 921 kd->kd_name = spa_strdup(dom); 922 return (kd); 923 } 924 925 void 926 ksiddomain_rele(ksiddomain_t *ksid) 927 { 928 spa_strfree(ksid->kd_name); 929 umem_free(ksid, sizeof (ksiddomain_t)); 930 } 931 932 char * 933 kmem_vasprintf(const char *fmt, va_list adx) 934 { 935 char *buf = NULL; 936 va_list adx_copy; 937 938 va_copy(adx_copy, adx); 939 VERIFY(vasprintf(&buf, fmt, adx_copy) != -1); 940 va_end(adx_copy); 941 942 return (buf); 943 } 944 945 char * 946 kmem_asprintf(const char *fmt, ...) 947 { 948 char *buf = NULL; 949 va_list adx; 950 951 va_start(adx, fmt); 952 VERIFY(vasprintf(&buf, fmt, adx) != -1); 953 va_end(adx); 954 955 return (buf); 956 } 957 958 /* 959 * kmem_scnprintf() will return the number of characters that it would have 960 * printed whenever it is limited by value of the size variable, rather than 961 * the number of characters that it did print. This can cause misbehavior on 962 * subsequent uses of the return value, so we define a safe version that will 963 * return the number of characters actually printed, minus the NULL format 964 * character. Subsequent use of this by the safe string functions is safe 965 * whether it is snprintf(), strlcat() or strlcpy(). 966 */ 967 int 968 kmem_scnprintf(char *restrict str, size_t size, const char *restrict fmt, ...) 969 { 970 int n; 971 va_list ap; 972 973 /* Make the 0 case a no-op so that we do not return -1 */ 974 if (size == 0) 975 return (0); 976 977 va_start(ap, fmt); 978 n = vsnprintf(str, size, fmt, ap); 979 va_end(ap); 980 981 if (n >= size) 982 n = size - 1; 983 984 return (n); 985 } 986 987 zfs_file_t * 988 zfs_onexit_fd_hold(int fd, minor_t *minorp) 989 { 990 (void) fd; 991 *minorp = 0; 992 return (NULL); 993 } 994 995 void 996 zfs_onexit_fd_rele(zfs_file_t *fp) 997 { 998 (void) fp; 999 } 1000 1001 int 1002 zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data, 1003 uintptr_t *action_handle) 1004 { 1005 (void) minor, (void) func, (void) data, (void) action_handle; 1006 return (0); 1007 } 1008 1009 fstrans_cookie_t 1010 spl_fstrans_mark(void) 1011 { 1012 return ((fstrans_cookie_t)0); 1013 } 1014 1015 void 1016 spl_fstrans_unmark(fstrans_cookie_t cookie) 1017 { 1018 (void) cookie; 1019 } 1020 1021 int 1022 __spl_pf_fstrans_check(void) 1023 { 1024 return (0); 1025 } 1026 1027 int 1028 kmem_cache_reap_active(void) 1029 { 1030 return (0); 1031 } 1032 1033 void 1034 zvol_create_minor(const char *name) 1035 { 1036 (void) name; 1037 } 1038 1039 void 1040 zvol_create_minors_recursive(const char *name) 1041 { 1042 (void) name; 1043 } 1044 1045 void 1046 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async) 1047 { 1048 (void) spa, (void) name, (void) async; 1049 } 1050 1051 void 1052 zvol_rename_minors(spa_t *spa, const char *oldname, const char *newname, 1053 boolean_t async) 1054 { 1055 (void) spa, (void) oldname, (void) newname, (void) async; 1056 } 1057 1058 /* 1059 * Open file 1060 * 1061 * path - fully qualified path to file 1062 * flags - file attributes O_READ / O_WRITE / O_EXCL 1063 * fpp - pointer to return file pointer 1064 * 1065 * Returns 0 on success underlying error on failure. 1066 */ 1067 int 1068 zfs_file_open(const char *path, int flags, int mode, zfs_file_t **fpp) 1069 { 1070 int fd = -1; 1071 int dump_fd = -1; 1072 int err; 1073 int old_umask = 0; 1074 zfs_file_t *fp; 1075 struct stat64 st; 1076 1077 if (!(flags & O_CREAT) && stat64(path, &st) == -1) 1078 return (errno); 1079 1080 if (!(flags & O_CREAT) && S_ISBLK(st.st_mode)) 1081 flags |= O_DIRECT; 1082 1083 if (flags & O_CREAT) 1084 old_umask = umask(0); 1085 1086 fd = open64(path, flags, mode); 1087 if (fd == -1) 1088 return (errno); 1089 1090 if (flags & O_CREAT) 1091 (void) umask(old_umask); 1092 1093 if (vn_dumpdir != NULL) { 1094 char *dumppath = umem_zalloc(MAXPATHLEN, UMEM_NOFAIL); 1095 const char *inpath = zfs_basename(path); 1096 1097 (void) snprintf(dumppath, MAXPATHLEN, 1098 "%s/%s", vn_dumpdir, inpath); 1099 dump_fd = open64(dumppath, O_CREAT | O_WRONLY, 0666); 1100 umem_free(dumppath, MAXPATHLEN); 1101 if (dump_fd == -1) { 1102 err = errno; 1103 close(fd); 1104 return (err); 1105 } 1106 } else { 1107 dump_fd = -1; 1108 } 1109 1110 (void) fcntl(fd, F_SETFD, FD_CLOEXEC); 1111 1112 fp = umem_zalloc(sizeof (zfs_file_t), UMEM_NOFAIL); 1113 fp->f_fd = fd; 1114 fp->f_dump_fd = dump_fd; 1115 *fpp = fp; 1116 1117 return (0); 1118 } 1119 1120 void 1121 zfs_file_close(zfs_file_t *fp) 1122 { 1123 close(fp->f_fd); 1124 if (fp->f_dump_fd != -1) 1125 close(fp->f_dump_fd); 1126 1127 umem_free(fp, sizeof (zfs_file_t)); 1128 } 1129 1130 /* 1131 * Stateful write - use os internal file pointer to determine where to 1132 * write and update on successful completion. 1133 * 1134 * fp - pointer to file (pipe, socket, etc) to write to 1135 * buf - buffer to write 1136 * count - # of bytes to write 1137 * resid - pointer to count of unwritten bytes (if short write) 1138 * 1139 * Returns 0 on success errno on failure. 1140 */ 1141 int 1142 zfs_file_write(zfs_file_t *fp, const void *buf, size_t count, ssize_t *resid) 1143 { 1144 ssize_t rc; 1145 1146 rc = write(fp->f_fd, buf, count); 1147 if (rc < 0) 1148 return (errno); 1149 1150 if (resid) { 1151 *resid = count - rc; 1152 } else if (rc != count) { 1153 return (EIO); 1154 } 1155 1156 return (0); 1157 } 1158 1159 /* 1160 * Stateless write - os internal file pointer is not updated. 1161 * 1162 * fp - pointer to file (pipe, socket, etc) to write to 1163 * buf - buffer to write 1164 * count - # of bytes to write 1165 * off - file offset to write to (only valid for seekable types) 1166 * resid - pointer to count of unwritten bytes 1167 * 1168 * Returns 0 on success errno on failure. 1169 */ 1170 int 1171 zfs_file_pwrite(zfs_file_t *fp, const void *buf, 1172 size_t count, loff_t pos, ssize_t *resid) 1173 { 1174 ssize_t rc, split, done; 1175 int sectors; 1176 1177 /* 1178 * To simulate partial disk writes, we split writes into two 1179 * system calls so that the process can be killed in between. 1180 * This is used by ztest to simulate realistic failure modes. 1181 */ 1182 sectors = count >> SPA_MINBLOCKSHIFT; 1183 split = (sectors > 0 ? rand() % sectors : 0) << SPA_MINBLOCKSHIFT; 1184 rc = pwrite64(fp->f_fd, buf, split, pos); 1185 if (rc != -1) { 1186 done = rc; 1187 rc = pwrite64(fp->f_fd, (char *)buf + split, 1188 count - split, pos + split); 1189 } 1190 #ifdef __linux__ 1191 if (rc == -1 && errno == EINVAL) { 1192 /* 1193 * Under Linux, this most likely means an alignment issue 1194 * (memory or disk) due to O_DIRECT, so we abort() in order 1195 * to catch the offender. 1196 */ 1197 abort(); 1198 } 1199 #endif 1200 1201 if (rc < 0) 1202 return (errno); 1203 1204 done += rc; 1205 1206 if (resid) { 1207 *resid = count - done; 1208 } else if (done != count) { 1209 return (EIO); 1210 } 1211 1212 return (0); 1213 } 1214 1215 /* 1216 * Stateful read - use os internal file pointer to determine where to 1217 * read and update on successful completion. 1218 * 1219 * fp - pointer to file (pipe, socket, etc) to read from 1220 * buf - buffer to write 1221 * count - # of bytes to read 1222 * resid - pointer to count of unread bytes (if short read) 1223 * 1224 * Returns 0 on success errno on failure. 1225 */ 1226 int 1227 zfs_file_read(zfs_file_t *fp, void *buf, size_t count, ssize_t *resid) 1228 { 1229 int rc; 1230 1231 rc = read(fp->f_fd, buf, count); 1232 if (rc < 0) 1233 return (errno); 1234 1235 if (resid) { 1236 *resid = count - rc; 1237 } else if (rc != count) { 1238 return (EIO); 1239 } 1240 1241 return (0); 1242 } 1243 1244 /* 1245 * Stateless read - os internal file pointer is not updated. 1246 * 1247 * fp - pointer to file (pipe, socket, etc) to read from 1248 * buf - buffer to write 1249 * count - # of bytes to write 1250 * off - file offset to read from (only valid for seekable types) 1251 * resid - pointer to count of unwritten bytes (if short write) 1252 * 1253 * Returns 0 on success errno on failure. 1254 */ 1255 int 1256 zfs_file_pread(zfs_file_t *fp, void *buf, size_t count, loff_t off, 1257 ssize_t *resid) 1258 { 1259 ssize_t rc; 1260 1261 rc = pread64(fp->f_fd, buf, count, off); 1262 if (rc < 0) { 1263 #ifdef __linux__ 1264 /* 1265 * Under Linux, this most likely means an alignment issue 1266 * (memory or disk) due to O_DIRECT, so we abort() in order to 1267 * catch the offender. 1268 */ 1269 if (errno == EINVAL) 1270 abort(); 1271 #endif 1272 return (errno); 1273 } 1274 1275 if (fp->f_dump_fd != -1) { 1276 int status; 1277 1278 status = pwrite64(fp->f_dump_fd, buf, rc, off); 1279 ASSERT(status != -1); 1280 } 1281 1282 if (resid) { 1283 *resid = count - rc; 1284 } else if (rc != count) { 1285 return (EIO); 1286 } 1287 1288 return (0); 1289 } 1290 1291 /* 1292 * lseek - set / get file pointer 1293 * 1294 * fp - pointer to file (pipe, socket, etc) to read from 1295 * offp - value to seek to, returns current value plus passed offset 1296 * whence - see man pages for standard lseek whence values 1297 * 1298 * Returns 0 on success errno on failure (ESPIPE for non seekable types) 1299 */ 1300 int 1301 zfs_file_seek(zfs_file_t *fp, loff_t *offp, int whence) 1302 { 1303 loff_t rc; 1304 1305 rc = lseek(fp->f_fd, *offp, whence); 1306 if (rc < 0) 1307 return (errno); 1308 1309 *offp = rc; 1310 1311 return (0); 1312 } 1313 1314 /* 1315 * Get file attributes 1316 * 1317 * filp - file pointer 1318 * zfattr - pointer to file attr structure 1319 * 1320 * Currently only used for fetching size and file mode 1321 * 1322 * Returns 0 on success or error code of underlying getattr call on failure. 1323 */ 1324 int 1325 zfs_file_getattr(zfs_file_t *fp, zfs_file_attr_t *zfattr) 1326 { 1327 struct stat64 st; 1328 1329 if (fstat64_blk(fp->f_fd, &st) == -1) 1330 return (errno); 1331 1332 zfattr->zfa_size = st.st_size; 1333 zfattr->zfa_mode = st.st_mode; 1334 1335 return (0); 1336 } 1337 1338 /* 1339 * Sync file to disk 1340 * 1341 * filp - file pointer 1342 * flags - O_SYNC and or O_DSYNC 1343 * 1344 * Returns 0 on success or error code of underlying sync call on failure. 1345 */ 1346 int 1347 zfs_file_fsync(zfs_file_t *fp, int flags) 1348 { 1349 (void) flags; 1350 1351 if (fsync(fp->f_fd) < 0) 1352 return (errno); 1353 1354 return (0); 1355 } 1356 1357 /* 1358 * fallocate - allocate or free space on disk 1359 * 1360 * fp - file pointer 1361 * mode (non-standard options for hole punching etc) 1362 * offset - offset to start allocating or freeing from 1363 * len - length to free / allocate 1364 * 1365 * OPTIONAL 1366 */ 1367 int 1368 zfs_file_fallocate(zfs_file_t *fp, int mode, loff_t offset, loff_t len) 1369 { 1370 #ifdef __linux__ 1371 return (fallocate(fp->f_fd, mode, offset, len)); 1372 #else 1373 (void) fp, (void) mode, (void) offset, (void) len; 1374 return (EOPNOTSUPP); 1375 #endif 1376 } 1377 1378 /* 1379 * Request current file pointer offset 1380 * 1381 * fp - pointer to file 1382 * 1383 * Returns current file offset. 1384 */ 1385 loff_t 1386 zfs_file_off(zfs_file_t *fp) 1387 { 1388 return (lseek(fp->f_fd, SEEK_CUR, 0)); 1389 } 1390 1391 /* 1392 * unlink file 1393 * 1394 * path - fully qualified file path 1395 * 1396 * Returns 0 on success. 1397 * 1398 * OPTIONAL 1399 */ 1400 int 1401 zfs_file_unlink(const char *path) 1402 { 1403 return (remove(path)); 1404 } 1405 1406 /* 1407 * Get reference to file pointer 1408 * 1409 * fd - input file descriptor 1410 * 1411 * Returns pointer to file struct or NULL. 1412 * Unsupported in user space. 1413 */ 1414 zfs_file_t * 1415 zfs_file_get(int fd) 1416 { 1417 (void) fd; 1418 abort(); 1419 return (NULL); 1420 } 1421 /* 1422 * Drop reference to file pointer 1423 * 1424 * fp - pointer to file struct 1425 * 1426 * Unsupported in user space. 1427 */ 1428 void 1429 zfs_file_put(zfs_file_t *fp) 1430 { 1431 abort(); 1432 (void) fp; 1433 } 1434 1435 void 1436 zfsvfs_update_fromname(const char *oldname, const char *newname) 1437 { 1438 (void) oldname, (void) newname; 1439 } 1440 1441 void 1442 spa_import_os(spa_t *spa) 1443 { 1444 (void) spa; 1445 } 1446 1447 void 1448 spa_export_os(spa_t *spa) 1449 { 1450 (void) spa; 1451 } 1452 1453 void 1454 spa_activate_os(spa_t *spa) 1455 { 1456 (void) spa; 1457 } 1458 1459 void 1460 spa_deactivate_os(spa_t *spa) 1461 { 1462 (void) spa; 1463 } 1464