1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2012, 2018 by Delphix. All rights reserved. 24 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 25 */ 26 27 #include <assert.h> 28 #include <fcntl.h> 29 #include <libgen.h> 30 #include <poll.h> 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <string.h> 34 #include <libzutil.h> 35 #include <sys/crypto/icp.h> 36 #include <sys/processor.h> 37 #include <sys/rrwlock.h> 38 #include <sys/spa.h> 39 #include <sys/stat.h> 40 #include <sys/systeminfo.h> 41 #include <sys/time.h> 42 #include <sys/utsname.h> 43 #include <sys/zfs_context.h> 44 #include <sys/zfs_onexit.h> 45 #include <sys/zfs_vfsops.h> 46 #include <sys/zstd/zstd.h> 47 #include <sys/zvol.h> 48 #include <zfs_fletcher.h> 49 #include <zlib.h> 50 51 /* 52 * Emulation of kernel services in userland. 53 */ 54 55 uint64_t physmem; 56 uint32_t hostid; 57 struct utsname hw_utsname; 58 59 /* If set, all blocks read will be copied to the specified directory. */ 60 char *vn_dumpdir = NULL; 61 62 /* this only exists to have its address taken */ 63 struct proc p0; 64 65 /* 66 * ========================================================================= 67 * threads 68 * ========================================================================= 69 * 70 * TS_STACK_MIN is dictated by the minimum allowed pthread stack size. While 71 * TS_STACK_MAX is somewhat arbitrary, it was selected to be large enough for 72 * the expected stack depth while small enough to avoid exhausting address 73 * space with high thread counts. 74 */ 75 #define TS_STACK_MIN MAX(PTHREAD_STACK_MIN, 32768) 76 #define TS_STACK_MAX (256 * 1024) 77 78 struct zk_thread_wrapper { 79 void (*func)(void *); 80 void *arg; 81 }; 82 83 static void * 84 zk_thread_wrapper(void *arg) 85 { 86 struct zk_thread_wrapper ztw; 87 memcpy(&ztw, arg, sizeof (ztw)); 88 free(arg); 89 ztw.func(ztw.arg); 90 return (NULL); 91 } 92 93 kthread_t * 94 zk_thread_create(void (*func)(void *), void *arg, size_t stksize, int state) 95 { 96 pthread_attr_t attr; 97 pthread_t tid; 98 char *stkstr; 99 struct zk_thread_wrapper *ztw; 100 int detachstate = PTHREAD_CREATE_DETACHED; 101 102 VERIFY0(pthread_attr_init(&attr)); 103 104 if (state & TS_JOINABLE) 105 detachstate = PTHREAD_CREATE_JOINABLE; 106 107 VERIFY0(pthread_attr_setdetachstate(&attr, detachstate)); 108 109 /* 110 * We allow the default stack size in user space to be specified by 111 * setting the ZFS_STACK_SIZE environment variable. This allows us 112 * the convenience of observing and debugging stack overruns in 113 * user space. Explicitly specified stack sizes will be honored. 114 * The usage of ZFS_STACK_SIZE is discussed further in the 115 * ENVIRONMENT VARIABLES sections of the ztest(1) man page. 116 */ 117 if (stksize == 0) { 118 stkstr = getenv("ZFS_STACK_SIZE"); 119 120 if (stkstr == NULL) 121 stksize = TS_STACK_MAX; 122 else 123 stksize = MAX(atoi(stkstr), TS_STACK_MIN); 124 } 125 126 VERIFY3S(stksize, >, 0); 127 stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE); 128 129 /* 130 * If this ever fails, it may be because the stack size is not a 131 * multiple of system page size. 132 */ 133 VERIFY0(pthread_attr_setstacksize(&attr, stksize)); 134 VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE)); 135 136 VERIFY(ztw = malloc(sizeof (*ztw))); 137 ztw->func = func; 138 ztw->arg = arg; 139 VERIFY0(pthread_create(&tid, &attr, zk_thread_wrapper, ztw)); 140 VERIFY0(pthread_attr_destroy(&attr)); 141 142 return ((void *)(uintptr_t)tid); 143 } 144 145 /* 146 * ========================================================================= 147 * kstats 148 * ========================================================================= 149 */ 150 kstat_t * 151 kstat_create(const char *module, int instance, const char *name, 152 const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag) 153 { 154 (void) module, (void) instance, (void) name, (void) class, (void) type, 155 (void) ndata, (void) ks_flag; 156 return (NULL); 157 } 158 159 void 160 kstat_install(kstat_t *ksp) 161 { 162 (void) ksp; 163 } 164 165 void 166 kstat_delete(kstat_t *ksp) 167 { 168 (void) ksp; 169 } 170 171 void 172 kstat_set_raw_ops(kstat_t *ksp, 173 int (*headers)(char *buf, size_t size), 174 int (*data)(char *buf, size_t size, void *data), 175 void *(*addr)(kstat_t *ksp, loff_t index)) 176 { 177 (void) ksp, (void) headers, (void) data, (void) addr; 178 } 179 180 /* 181 * ========================================================================= 182 * mutexes 183 * ========================================================================= 184 */ 185 186 void 187 mutex_init(kmutex_t *mp, char *name, int type, void *cookie) 188 { 189 (void) name, (void) type, (void) cookie; 190 VERIFY0(pthread_mutex_init(&mp->m_lock, NULL)); 191 memset(&mp->m_owner, 0, sizeof (pthread_t)); 192 } 193 194 void 195 mutex_destroy(kmutex_t *mp) 196 { 197 VERIFY0(pthread_mutex_destroy(&mp->m_lock)); 198 } 199 200 void 201 mutex_enter(kmutex_t *mp) 202 { 203 VERIFY0(pthread_mutex_lock(&mp->m_lock)); 204 mp->m_owner = pthread_self(); 205 } 206 207 int 208 mutex_tryenter(kmutex_t *mp) 209 { 210 int error = pthread_mutex_trylock(&mp->m_lock); 211 if (error == 0) { 212 mp->m_owner = pthread_self(); 213 return (1); 214 } else { 215 VERIFY3S(error, ==, EBUSY); 216 return (0); 217 } 218 } 219 220 void 221 mutex_exit(kmutex_t *mp) 222 { 223 memset(&mp->m_owner, 0, sizeof (pthread_t)); 224 VERIFY0(pthread_mutex_unlock(&mp->m_lock)); 225 } 226 227 /* 228 * ========================================================================= 229 * rwlocks 230 * ========================================================================= 231 */ 232 233 void 234 rw_init(krwlock_t *rwlp, char *name, int type, void *arg) 235 { 236 (void) name, (void) type, (void) arg; 237 VERIFY0(pthread_rwlock_init(&rwlp->rw_lock, NULL)); 238 rwlp->rw_readers = 0; 239 rwlp->rw_owner = 0; 240 } 241 242 void 243 rw_destroy(krwlock_t *rwlp) 244 { 245 VERIFY0(pthread_rwlock_destroy(&rwlp->rw_lock)); 246 } 247 248 void 249 rw_enter(krwlock_t *rwlp, krw_t rw) 250 { 251 if (rw == RW_READER) { 252 VERIFY0(pthread_rwlock_rdlock(&rwlp->rw_lock)); 253 atomic_inc_uint(&rwlp->rw_readers); 254 } else { 255 VERIFY0(pthread_rwlock_wrlock(&rwlp->rw_lock)); 256 rwlp->rw_owner = pthread_self(); 257 } 258 } 259 260 void 261 rw_exit(krwlock_t *rwlp) 262 { 263 if (RW_READ_HELD(rwlp)) 264 atomic_dec_uint(&rwlp->rw_readers); 265 else 266 rwlp->rw_owner = 0; 267 268 VERIFY0(pthread_rwlock_unlock(&rwlp->rw_lock)); 269 } 270 271 int 272 rw_tryenter(krwlock_t *rwlp, krw_t rw) 273 { 274 int error; 275 276 if (rw == RW_READER) 277 error = pthread_rwlock_tryrdlock(&rwlp->rw_lock); 278 else 279 error = pthread_rwlock_trywrlock(&rwlp->rw_lock); 280 281 if (error == 0) { 282 if (rw == RW_READER) 283 atomic_inc_uint(&rwlp->rw_readers); 284 else 285 rwlp->rw_owner = pthread_self(); 286 287 return (1); 288 } 289 290 VERIFY3S(error, ==, EBUSY); 291 292 return (0); 293 } 294 295 uint32_t 296 zone_get_hostid(void *zonep) 297 { 298 /* 299 * We're emulating the system's hostid in userland. 300 */ 301 (void) zonep; 302 return (hostid); 303 } 304 305 int 306 rw_tryupgrade(krwlock_t *rwlp) 307 { 308 (void) rwlp; 309 return (0); 310 } 311 312 /* 313 * ========================================================================= 314 * condition variables 315 * ========================================================================= 316 */ 317 318 void 319 cv_init(kcondvar_t *cv, char *name, int type, void *arg) 320 { 321 (void) name, (void) type, (void) arg; 322 VERIFY0(pthread_cond_init(cv, NULL)); 323 } 324 325 void 326 cv_destroy(kcondvar_t *cv) 327 { 328 VERIFY0(pthread_cond_destroy(cv)); 329 } 330 331 void 332 cv_wait(kcondvar_t *cv, kmutex_t *mp) 333 { 334 memset(&mp->m_owner, 0, sizeof (pthread_t)); 335 VERIFY0(pthread_cond_wait(cv, &mp->m_lock)); 336 mp->m_owner = pthread_self(); 337 } 338 339 int 340 cv_wait_sig(kcondvar_t *cv, kmutex_t *mp) 341 { 342 cv_wait(cv, mp); 343 return (1); 344 } 345 346 int 347 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime) 348 { 349 int error; 350 struct timeval tv; 351 struct timespec ts; 352 clock_t delta; 353 354 delta = abstime - ddi_get_lbolt(); 355 if (delta <= 0) 356 return (-1); 357 358 VERIFY(gettimeofday(&tv, NULL) == 0); 359 360 ts.tv_sec = tv.tv_sec + delta / hz; 361 ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % hz) * (NANOSEC / hz); 362 if (ts.tv_nsec >= NANOSEC) { 363 ts.tv_sec++; 364 ts.tv_nsec -= NANOSEC; 365 } 366 367 memset(&mp->m_owner, 0, sizeof (pthread_t)); 368 error = pthread_cond_timedwait(cv, &mp->m_lock, &ts); 369 mp->m_owner = pthread_self(); 370 371 if (error == ETIMEDOUT) 372 return (-1); 373 374 VERIFY0(error); 375 376 return (1); 377 } 378 379 int 380 cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res, 381 int flag) 382 { 383 (void) res; 384 int error; 385 struct timeval tv; 386 struct timespec ts; 387 hrtime_t delta; 388 389 ASSERT(flag == 0 || flag == CALLOUT_FLAG_ABSOLUTE); 390 391 delta = tim; 392 if (flag & CALLOUT_FLAG_ABSOLUTE) 393 delta -= gethrtime(); 394 395 if (delta <= 0) 396 return (-1); 397 398 VERIFY0(gettimeofday(&tv, NULL)); 399 400 ts.tv_sec = tv.tv_sec + delta / NANOSEC; 401 ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % NANOSEC); 402 if (ts.tv_nsec >= NANOSEC) { 403 ts.tv_sec++; 404 ts.tv_nsec -= NANOSEC; 405 } 406 407 memset(&mp->m_owner, 0, sizeof (pthread_t)); 408 error = pthread_cond_timedwait(cv, &mp->m_lock, &ts); 409 mp->m_owner = pthread_self(); 410 411 if (error == ETIMEDOUT) 412 return (-1); 413 414 VERIFY0(error); 415 416 return (1); 417 } 418 419 void 420 cv_signal(kcondvar_t *cv) 421 { 422 VERIFY0(pthread_cond_signal(cv)); 423 } 424 425 void 426 cv_broadcast(kcondvar_t *cv) 427 { 428 VERIFY0(pthread_cond_broadcast(cv)); 429 } 430 431 /* 432 * ========================================================================= 433 * procfs list 434 * ========================================================================= 435 */ 436 437 void 438 seq_printf(struct seq_file *m, const char *fmt, ...) 439 { 440 (void) m, (void) fmt; 441 } 442 443 void 444 procfs_list_install(const char *module, 445 const char *submodule, 446 const char *name, 447 mode_t mode, 448 procfs_list_t *procfs_list, 449 int (*show)(struct seq_file *f, void *p), 450 int (*show_header)(struct seq_file *f), 451 int (*clear)(procfs_list_t *procfs_list), 452 size_t procfs_list_node_off) 453 { 454 (void) module, (void) submodule, (void) name, (void) mode, (void) show, 455 (void) show_header, (void) clear; 456 mutex_init(&procfs_list->pl_lock, NULL, MUTEX_DEFAULT, NULL); 457 list_create(&procfs_list->pl_list, 458 procfs_list_node_off + sizeof (procfs_list_node_t), 459 procfs_list_node_off + offsetof(procfs_list_node_t, pln_link)); 460 procfs_list->pl_next_id = 1; 461 procfs_list->pl_node_offset = procfs_list_node_off; 462 } 463 464 void 465 procfs_list_uninstall(procfs_list_t *procfs_list) 466 { 467 (void) procfs_list; 468 } 469 470 void 471 procfs_list_destroy(procfs_list_t *procfs_list) 472 { 473 ASSERT(list_is_empty(&procfs_list->pl_list)); 474 list_destroy(&procfs_list->pl_list); 475 mutex_destroy(&procfs_list->pl_lock); 476 } 477 478 #define NODE_ID(procfs_list, obj) \ 479 (((procfs_list_node_t *)(((char *)obj) + \ 480 (procfs_list)->pl_node_offset))->pln_id) 481 482 void 483 procfs_list_add(procfs_list_t *procfs_list, void *p) 484 { 485 ASSERT(MUTEX_HELD(&procfs_list->pl_lock)); 486 NODE_ID(procfs_list, p) = procfs_list->pl_next_id++; 487 list_insert_tail(&procfs_list->pl_list, p); 488 } 489 490 /* 491 * ========================================================================= 492 * vnode operations 493 * ========================================================================= 494 */ 495 496 /* 497 * ========================================================================= 498 * Figure out which debugging statements to print 499 * ========================================================================= 500 */ 501 502 static char *dprintf_string; 503 static int dprintf_print_all; 504 505 int 506 dprintf_find_string(const char *string) 507 { 508 char *tmp_str = dprintf_string; 509 int len = strlen(string); 510 511 /* 512 * Find out if this is a string we want to print. 513 * String format: file1.c,function_name1,file2.c,file3.c 514 */ 515 516 while (tmp_str != NULL) { 517 if (strncmp(tmp_str, string, len) == 0 && 518 (tmp_str[len] == ',' || tmp_str[len] == '\0')) 519 return (1); 520 tmp_str = strchr(tmp_str, ','); 521 if (tmp_str != NULL) 522 tmp_str++; /* Get rid of , */ 523 } 524 return (0); 525 } 526 527 void 528 dprintf_setup(int *argc, char **argv) 529 { 530 int i, j; 531 532 /* 533 * Debugging can be specified two ways: by setting the 534 * environment variable ZFS_DEBUG, or by including a 535 * "debug=..." argument on the command line. The command 536 * line setting overrides the environment variable. 537 */ 538 539 for (i = 1; i < *argc; i++) { 540 int len = strlen("debug="); 541 /* First look for a command line argument */ 542 if (strncmp("debug=", argv[i], len) == 0) { 543 dprintf_string = argv[i] + len; 544 /* Remove from args */ 545 for (j = i; j < *argc; j++) 546 argv[j] = argv[j+1]; 547 argv[j] = NULL; 548 (*argc)--; 549 } 550 } 551 552 if (dprintf_string == NULL) { 553 /* Look for ZFS_DEBUG environment variable */ 554 dprintf_string = getenv("ZFS_DEBUG"); 555 } 556 557 /* 558 * Are we just turning on all debugging? 559 */ 560 if (dprintf_find_string("on")) 561 dprintf_print_all = 1; 562 563 if (dprintf_string != NULL) 564 zfs_flags |= ZFS_DEBUG_DPRINTF; 565 } 566 567 /* 568 * ========================================================================= 569 * debug printfs 570 * ========================================================================= 571 */ 572 void 573 __dprintf(boolean_t dprint, const char *file, const char *func, 574 int line, const char *fmt, ...) 575 { 576 /* Get rid of annoying "../common/" prefix to filename. */ 577 const char *newfile = zfs_basename(file); 578 579 va_list adx; 580 if (dprint) { 581 /* dprintf messages are printed immediately */ 582 583 if (!dprintf_print_all && 584 !dprintf_find_string(newfile) && 585 !dprintf_find_string(func)) 586 return; 587 588 /* Print out just the function name if requested */ 589 flockfile(stdout); 590 if (dprintf_find_string("pid")) 591 (void) printf("%d ", getpid()); 592 if (dprintf_find_string("tid")) 593 (void) printf("%ju ", 594 (uintmax_t)(uintptr_t)pthread_self()); 595 if (dprintf_find_string("cpu")) 596 (void) printf("%u ", getcpuid()); 597 if (dprintf_find_string("time")) 598 (void) printf("%llu ", gethrtime()); 599 if (dprintf_find_string("long")) 600 (void) printf("%s, line %d: ", newfile, line); 601 (void) printf("dprintf: %s: ", func); 602 va_start(adx, fmt); 603 (void) vprintf(fmt, adx); 604 va_end(adx); 605 funlockfile(stdout); 606 } else { 607 /* zfs_dbgmsg is logged for dumping later */ 608 size_t size; 609 char *buf; 610 int i; 611 612 size = 1024; 613 buf = umem_alloc(size, UMEM_NOFAIL); 614 i = snprintf(buf, size, "%s:%d:%s(): ", newfile, line, func); 615 616 if (i < size) { 617 va_start(adx, fmt); 618 (void) vsnprintf(buf + i, size - i, fmt, adx); 619 va_end(adx); 620 } 621 622 __zfs_dbgmsg(buf); 623 624 umem_free(buf, size); 625 } 626 } 627 628 /* 629 * ========================================================================= 630 * cmn_err() and panic() 631 * ========================================================================= 632 */ 633 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" }; 634 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" }; 635 636 __attribute__((noreturn)) void 637 vpanic(const char *fmt, va_list adx) 638 { 639 (void) fprintf(stderr, "error: "); 640 (void) vfprintf(stderr, fmt, adx); 641 (void) fprintf(stderr, "\n"); 642 643 abort(); /* think of it as a "user-level crash dump" */ 644 } 645 646 __attribute__((noreturn)) void 647 panic(const char *fmt, ...) 648 { 649 va_list adx; 650 651 va_start(adx, fmt); 652 vpanic(fmt, adx); 653 va_end(adx); 654 } 655 656 void 657 vcmn_err(int ce, const char *fmt, va_list adx) 658 { 659 if (ce == CE_PANIC) 660 vpanic(fmt, adx); 661 if (ce != CE_NOTE) { /* suppress noise in userland stress testing */ 662 (void) fprintf(stderr, "%s", ce_prefix[ce]); 663 (void) vfprintf(stderr, fmt, adx); 664 (void) fprintf(stderr, "%s", ce_suffix[ce]); 665 } 666 } 667 668 void 669 cmn_err(int ce, const char *fmt, ...) 670 { 671 va_list adx; 672 673 va_start(adx, fmt); 674 vcmn_err(ce, fmt, adx); 675 va_end(adx); 676 } 677 678 /* 679 * ========================================================================= 680 * misc routines 681 * ========================================================================= 682 */ 683 684 void 685 delay(clock_t ticks) 686 { 687 (void) poll(0, 0, ticks * (1000 / hz)); 688 } 689 690 /* 691 * Find highest one bit set. 692 * Returns bit number + 1 of highest bit that is set, otherwise returns 0. 693 * The __builtin_clzll() function is supported by both GCC and Clang. 694 */ 695 int 696 highbit64(uint64_t i) 697 { 698 if (i == 0) 699 return (0); 700 701 return (NBBY * sizeof (uint64_t) - __builtin_clzll(i)); 702 } 703 704 /* 705 * Find lowest one bit set. 706 * Returns bit number + 1 of lowest bit that is set, otherwise returns 0. 707 * The __builtin_ffsll() function is supported by both GCC and Clang. 708 */ 709 int 710 lowbit64(uint64_t i) 711 { 712 if (i == 0) 713 return (0); 714 715 return (__builtin_ffsll(i)); 716 } 717 718 const char *random_path = "/dev/random"; 719 const char *urandom_path = "/dev/urandom"; 720 static int random_fd = -1, urandom_fd = -1; 721 722 void 723 random_init(void) 724 { 725 VERIFY((random_fd = open(random_path, O_RDONLY | O_CLOEXEC)) != -1); 726 VERIFY((urandom_fd = open(urandom_path, O_RDONLY | O_CLOEXEC)) != -1); 727 } 728 729 void 730 random_fini(void) 731 { 732 close(random_fd); 733 close(urandom_fd); 734 735 random_fd = -1; 736 urandom_fd = -1; 737 } 738 739 static int 740 random_get_bytes_common(uint8_t *ptr, size_t len, int fd) 741 { 742 size_t resid = len; 743 ssize_t bytes; 744 745 ASSERT(fd != -1); 746 747 while (resid != 0) { 748 bytes = read(fd, ptr, resid); 749 ASSERT3S(bytes, >=, 0); 750 ptr += bytes; 751 resid -= bytes; 752 } 753 754 return (0); 755 } 756 757 int 758 random_get_bytes(uint8_t *ptr, size_t len) 759 { 760 return (random_get_bytes_common(ptr, len, random_fd)); 761 } 762 763 int 764 random_get_pseudo_bytes(uint8_t *ptr, size_t len) 765 { 766 return (random_get_bytes_common(ptr, len, urandom_fd)); 767 } 768 769 int 770 ddi_strtoull(const char *str, char **nptr, int base, u_longlong_t *result) 771 { 772 (void) nptr; 773 char *end; 774 775 *result = strtoull(str, &end, base); 776 if (*result == 0) 777 return (errno); 778 return (0); 779 } 780 781 utsname_t * 782 utsname(void) 783 { 784 return (&hw_utsname); 785 } 786 787 /* 788 * ========================================================================= 789 * kernel emulation setup & teardown 790 * ========================================================================= 791 */ 792 static int 793 umem_out_of_memory(void) 794 { 795 char errmsg[] = "out of memory -- generating core dump\n"; 796 797 (void) fprintf(stderr, "%s", errmsg); 798 abort(); 799 return (0); 800 } 801 802 void 803 kernel_init(int mode) 804 { 805 extern uint_t rrw_tsd_key; 806 807 umem_nofail_callback(umem_out_of_memory); 808 809 physmem = sysconf(_SC_PHYS_PAGES); 810 811 dprintf("physmem = %llu pages (%.2f GB)\n", (u_longlong_t)physmem, 812 (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30)); 813 814 hostid = (mode & SPA_MODE_WRITE) ? get_system_hostid() : 0; 815 816 random_init(); 817 818 VERIFY0(uname(&hw_utsname)); 819 820 system_taskq_init(); 821 icp_init(); 822 823 zstd_init(); 824 825 spa_init((spa_mode_t)mode); 826 827 fletcher_4_init(); 828 829 tsd_create(&rrw_tsd_key, rrw_tsd_destroy); 830 } 831 832 void 833 kernel_fini(void) 834 { 835 fletcher_4_fini(); 836 spa_fini(); 837 838 zstd_fini(); 839 840 icp_fini(); 841 system_taskq_fini(); 842 843 random_fini(); 844 } 845 846 uid_t 847 crgetuid(cred_t *cr) 848 { 849 (void) cr; 850 return (0); 851 } 852 853 uid_t 854 crgetruid(cred_t *cr) 855 { 856 (void) cr; 857 return (0); 858 } 859 860 gid_t 861 crgetgid(cred_t *cr) 862 { 863 (void) cr; 864 return (0); 865 } 866 867 int 868 crgetngroups(cred_t *cr) 869 { 870 (void) cr; 871 return (0); 872 } 873 874 gid_t * 875 crgetgroups(cred_t *cr) 876 { 877 (void) cr; 878 return (NULL); 879 } 880 881 int 882 zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr) 883 { 884 (void) name, (void) cr; 885 return (0); 886 } 887 888 int 889 zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr) 890 { 891 (void) from, (void) to, (void) cr; 892 return (0); 893 } 894 895 int 896 zfs_secpolicy_destroy_perms(const char *name, cred_t *cr) 897 { 898 (void) name, (void) cr; 899 return (0); 900 } 901 902 int 903 secpolicy_zfs(const cred_t *cr) 904 { 905 (void) cr; 906 return (0); 907 } 908 909 int 910 secpolicy_zfs_proc(const cred_t *cr, proc_t *proc) 911 { 912 (void) cr, (void) proc; 913 return (0); 914 } 915 916 ksiddomain_t * 917 ksid_lookupdomain(const char *dom) 918 { 919 ksiddomain_t *kd; 920 921 kd = umem_zalloc(sizeof (ksiddomain_t), UMEM_NOFAIL); 922 kd->kd_name = spa_strdup(dom); 923 return (kd); 924 } 925 926 void 927 ksiddomain_rele(ksiddomain_t *ksid) 928 { 929 spa_strfree(ksid->kd_name); 930 umem_free(ksid, sizeof (ksiddomain_t)); 931 } 932 933 char * 934 kmem_vasprintf(const char *fmt, va_list adx) 935 { 936 char *buf = NULL; 937 va_list adx_copy; 938 939 va_copy(adx_copy, adx); 940 VERIFY(vasprintf(&buf, fmt, adx_copy) != -1); 941 va_end(adx_copy); 942 943 return (buf); 944 } 945 946 char * 947 kmem_asprintf(const char *fmt, ...) 948 { 949 char *buf = NULL; 950 va_list adx; 951 952 va_start(adx, fmt); 953 VERIFY(vasprintf(&buf, fmt, adx) != -1); 954 va_end(adx); 955 956 return (buf); 957 } 958 959 /* 960 * kmem_scnprintf() will return the number of characters that it would have 961 * printed whenever it is limited by value of the size variable, rather than 962 * the number of characters that it did print. This can cause misbehavior on 963 * subsequent uses of the return value, so we define a safe version that will 964 * return the number of characters actually printed, minus the NULL format 965 * character. Subsequent use of this by the safe string functions is safe 966 * whether it is snprintf(), strlcat() or strlcpy(). 967 */ 968 int 969 kmem_scnprintf(char *restrict str, size_t size, const char *restrict fmt, ...) 970 { 971 int n; 972 va_list ap; 973 974 /* Make the 0 case a no-op so that we do not return -1 */ 975 if (size == 0) 976 return (0); 977 978 va_start(ap, fmt); 979 n = vsnprintf(str, size, fmt, ap); 980 va_end(ap); 981 982 if (n >= size) 983 n = size - 1; 984 985 return (n); 986 } 987 988 zfs_file_t * 989 zfs_onexit_fd_hold(int fd, minor_t *minorp) 990 { 991 (void) fd; 992 *minorp = 0; 993 return (NULL); 994 } 995 996 void 997 zfs_onexit_fd_rele(zfs_file_t *fp) 998 { 999 (void) fp; 1000 } 1001 1002 int 1003 zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data, 1004 uintptr_t *action_handle) 1005 { 1006 (void) minor, (void) func, (void) data, (void) action_handle; 1007 return (0); 1008 } 1009 1010 fstrans_cookie_t 1011 spl_fstrans_mark(void) 1012 { 1013 return ((fstrans_cookie_t)0); 1014 } 1015 1016 void 1017 spl_fstrans_unmark(fstrans_cookie_t cookie) 1018 { 1019 (void) cookie; 1020 } 1021 1022 int 1023 __spl_pf_fstrans_check(void) 1024 { 1025 return (0); 1026 } 1027 1028 int 1029 kmem_cache_reap_active(void) 1030 { 1031 return (0); 1032 } 1033 1034 void 1035 zvol_create_minor(const char *name) 1036 { 1037 (void) name; 1038 } 1039 1040 void 1041 zvol_create_minors_recursive(const char *name) 1042 { 1043 (void) name; 1044 } 1045 1046 void 1047 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async) 1048 { 1049 (void) spa, (void) name, (void) async; 1050 } 1051 1052 void 1053 zvol_rename_minors(spa_t *spa, const char *oldname, const char *newname, 1054 boolean_t async) 1055 { 1056 (void) spa, (void) oldname, (void) newname, (void) async; 1057 } 1058 1059 /* 1060 * Open file 1061 * 1062 * path - fully qualified path to file 1063 * flags - file attributes O_READ / O_WRITE / O_EXCL 1064 * fpp - pointer to return file pointer 1065 * 1066 * Returns 0 on success underlying error on failure. 1067 */ 1068 int 1069 zfs_file_open(const char *path, int flags, int mode, zfs_file_t **fpp) 1070 { 1071 int fd = -1; 1072 int dump_fd = -1; 1073 int err; 1074 int old_umask = 0; 1075 zfs_file_t *fp; 1076 struct stat64 st; 1077 1078 if (!(flags & O_CREAT) && stat64(path, &st) == -1) 1079 return (errno); 1080 1081 if (!(flags & O_CREAT) && S_ISBLK(st.st_mode)) 1082 flags |= O_DIRECT; 1083 1084 if (flags & O_CREAT) 1085 old_umask = umask(0); 1086 1087 fd = open64(path, flags, mode); 1088 if (fd == -1) 1089 return (errno); 1090 1091 if (flags & O_CREAT) 1092 (void) umask(old_umask); 1093 1094 if (vn_dumpdir != NULL) { 1095 char *dumppath = umem_zalloc(MAXPATHLEN, UMEM_NOFAIL); 1096 const char *inpath = zfs_basename(path); 1097 1098 (void) snprintf(dumppath, MAXPATHLEN, 1099 "%s/%s", vn_dumpdir, inpath); 1100 dump_fd = open64(dumppath, O_CREAT | O_WRONLY, 0666); 1101 umem_free(dumppath, MAXPATHLEN); 1102 if (dump_fd == -1) { 1103 err = errno; 1104 close(fd); 1105 return (err); 1106 } 1107 } else { 1108 dump_fd = -1; 1109 } 1110 1111 (void) fcntl(fd, F_SETFD, FD_CLOEXEC); 1112 1113 fp = umem_zalloc(sizeof (zfs_file_t), UMEM_NOFAIL); 1114 fp->f_fd = fd; 1115 fp->f_dump_fd = dump_fd; 1116 *fpp = fp; 1117 1118 return (0); 1119 } 1120 1121 void 1122 zfs_file_close(zfs_file_t *fp) 1123 { 1124 close(fp->f_fd); 1125 if (fp->f_dump_fd != -1) 1126 close(fp->f_dump_fd); 1127 1128 umem_free(fp, sizeof (zfs_file_t)); 1129 } 1130 1131 /* 1132 * Stateful write - use os internal file pointer to determine where to 1133 * write and update on successful completion. 1134 * 1135 * fp - pointer to file (pipe, socket, etc) to write to 1136 * buf - buffer to write 1137 * count - # of bytes to write 1138 * resid - pointer to count of unwritten bytes (if short write) 1139 * 1140 * Returns 0 on success errno on failure. 1141 */ 1142 int 1143 zfs_file_write(zfs_file_t *fp, const void *buf, size_t count, ssize_t *resid) 1144 { 1145 ssize_t rc; 1146 1147 rc = write(fp->f_fd, buf, count); 1148 if (rc < 0) 1149 return (errno); 1150 1151 if (resid) { 1152 *resid = count - rc; 1153 } else if (rc != count) { 1154 return (EIO); 1155 } 1156 1157 return (0); 1158 } 1159 1160 /* 1161 * Stateless write - os internal file pointer is not updated. 1162 * 1163 * fp - pointer to file (pipe, socket, etc) to write to 1164 * buf - buffer to write 1165 * count - # of bytes to write 1166 * off - file offset to write to (only valid for seekable types) 1167 * resid - pointer to count of unwritten bytes 1168 * 1169 * Returns 0 on success errno on failure. 1170 */ 1171 int 1172 zfs_file_pwrite(zfs_file_t *fp, const void *buf, 1173 size_t count, loff_t pos, ssize_t *resid) 1174 { 1175 ssize_t rc, split, done; 1176 int sectors; 1177 1178 /* 1179 * To simulate partial disk writes, we split writes into two 1180 * system calls so that the process can be killed in between. 1181 * This is used by ztest to simulate realistic failure modes. 1182 */ 1183 sectors = count >> SPA_MINBLOCKSHIFT; 1184 split = (sectors > 0 ? rand() % sectors : 0) << SPA_MINBLOCKSHIFT; 1185 rc = pwrite64(fp->f_fd, buf, split, pos); 1186 if (rc != -1) { 1187 done = rc; 1188 rc = pwrite64(fp->f_fd, (char *)buf + split, 1189 count - split, pos + split); 1190 } 1191 #ifdef __linux__ 1192 if (rc == -1 && errno == EINVAL) { 1193 /* 1194 * Under Linux, this most likely means an alignment issue 1195 * (memory or disk) due to O_DIRECT, so we abort() in order 1196 * to catch the offender. 1197 */ 1198 abort(); 1199 } 1200 #endif 1201 1202 if (rc < 0) 1203 return (errno); 1204 1205 done += rc; 1206 1207 if (resid) { 1208 *resid = count - done; 1209 } else if (done != count) { 1210 return (EIO); 1211 } 1212 1213 return (0); 1214 } 1215 1216 /* 1217 * Stateful read - use os internal file pointer to determine where to 1218 * read and update on successful completion. 1219 * 1220 * fp - pointer to file (pipe, socket, etc) to read from 1221 * buf - buffer to write 1222 * count - # of bytes to read 1223 * resid - pointer to count of unread bytes (if short read) 1224 * 1225 * Returns 0 on success errno on failure. 1226 */ 1227 int 1228 zfs_file_read(zfs_file_t *fp, void *buf, size_t count, ssize_t *resid) 1229 { 1230 int rc; 1231 1232 rc = read(fp->f_fd, buf, count); 1233 if (rc < 0) 1234 return (errno); 1235 1236 if (resid) { 1237 *resid = count - rc; 1238 } else if (rc != count) { 1239 return (EIO); 1240 } 1241 1242 return (0); 1243 } 1244 1245 /* 1246 * Stateless read - os internal file pointer is not updated. 1247 * 1248 * fp - pointer to file (pipe, socket, etc) to read from 1249 * buf - buffer to write 1250 * count - # of bytes to write 1251 * off - file offset to read from (only valid for seekable types) 1252 * resid - pointer to count of unwritten bytes (if short write) 1253 * 1254 * Returns 0 on success errno on failure. 1255 */ 1256 int 1257 zfs_file_pread(zfs_file_t *fp, void *buf, size_t count, loff_t off, 1258 ssize_t *resid) 1259 { 1260 ssize_t rc; 1261 1262 rc = pread64(fp->f_fd, buf, count, off); 1263 if (rc < 0) { 1264 #ifdef __linux__ 1265 /* 1266 * Under Linux, this most likely means an alignment issue 1267 * (memory or disk) due to O_DIRECT, so we abort() in order to 1268 * catch the offender. 1269 */ 1270 if (errno == EINVAL) 1271 abort(); 1272 #endif 1273 return (errno); 1274 } 1275 1276 if (fp->f_dump_fd != -1) { 1277 int status; 1278 1279 status = pwrite64(fp->f_dump_fd, buf, rc, off); 1280 ASSERT(status != -1); 1281 } 1282 1283 if (resid) { 1284 *resid = count - rc; 1285 } else if (rc != count) { 1286 return (EIO); 1287 } 1288 1289 return (0); 1290 } 1291 1292 /* 1293 * lseek - set / get file pointer 1294 * 1295 * fp - pointer to file (pipe, socket, etc) to read from 1296 * offp - value to seek to, returns current value plus passed offset 1297 * whence - see man pages for standard lseek whence values 1298 * 1299 * Returns 0 on success errno on failure (ESPIPE for non seekable types) 1300 */ 1301 int 1302 zfs_file_seek(zfs_file_t *fp, loff_t *offp, int whence) 1303 { 1304 loff_t rc; 1305 1306 rc = lseek(fp->f_fd, *offp, whence); 1307 if (rc < 0) 1308 return (errno); 1309 1310 *offp = rc; 1311 1312 return (0); 1313 } 1314 1315 /* 1316 * Get file attributes 1317 * 1318 * filp - file pointer 1319 * zfattr - pointer to file attr structure 1320 * 1321 * Currently only used for fetching size and file mode 1322 * 1323 * Returns 0 on success or error code of underlying getattr call on failure. 1324 */ 1325 int 1326 zfs_file_getattr(zfs_file_t *fp, zfs_file_attr_t *zfattr) 1327 { 1328 struct stat64 st; 1329 1330 if (fstat64_blk(fp->f_fd, &st) == -1) 1331 return (errno); 1332 1333 zfattr->zfa_size = st.st_size; 1334 zfattr->zfa_mode = st.st_mode; 1335 1336 return (0); 1337 } 1338 1339 /* 1340 * Sync file to disk 1341 * 1342 * filp - file pointer 1343 * flags - O_SYNC and or O_DSYNC 1344 * 1345 * Returns 0 on success or error code of underlying sync call on failure. 1346 */ 1347 int 1348 zfs_file_fsync(zfs_file_t *fp, int flags) 1349 { 1350 (void) flags; 1351 1352 if (fsync(fp->f_fd) < 0) 1353 return (errno); 1354 1355 return (0); 1356 } 1357 1358 /* 1359 * fallocate - allocate or free space on disk 1360 * 1361 * fp - file pointer 1362 * mode (non-standard options for hole punching etc) 1363 * offset - offset to start allocating or freeing from 1364 * len - length to free / allocate 1365 * 1366 * OPTIONAL 1367 */ 1368 int 1369 zfs_file_fallocate(zfs_file_t *fp, int mode, loff_t offset, loff_t len) 1370 { 1371 #ifdef __linux__ 1372 return (fallocate(fp->f_fd, mode, offset, len)); 1373 #else 1374 (void) fp, (void) mode, (void) offset, (void) len; 1375 return (EOPNOTSUPP); 1376 #endif 1377 } 1378 1379 /* 1380 * Request current file pointer offset 1381 * 1382 * fp - pointer to file 1383 * 1384 * Returns current file offset. 1385 */ 1386 loff_t 1387 zfs_file_off(zfs_file_t *fp) 1388 { 1389 return (lseek(fp->f_fd, SEEK_CUR, 0)); 1390 } 1391 1392 /* 1393 * unlink file 1394 * 1395 * path - fully qualified file path 1396 * 1397 * Returns 0 on success. 1398 * 1399 * OPTIONAL 1400 */ 1401 int 1402 zfs_file_unlink(const char *path) 1403 { 1404 return (remove(path)); 1405 } 1406 1407 /* 1408 * Get reference to file pointer 1409 * 1410 * fd - input file descriptor 1411 * 1412 * Returns pointer to file struct or NULL. 1413 * Unsupported in user space. 1414 */ 1415 zfs_file_t * 1416 zfs_file_get(int fd) 1417 { 1418 (void) fd; 1419 abort(); 1420 return (NULL); 1421 } 1422 /* 1423 * Drop reference to file pointer 1424 * 1425 * fp - pointer to file struct 1426 * 1427 * Unsupported in user space. 1428 */ 1429 void 1430 zfs_file_put(zfs_file_t *fp) 1431 { 1432 abort(); 1433 (void) fp; 1434 } 1435 1436 void 1437 zfsvfs_update_fromname(const char *oldname, const char *newname) 1438 { 1439 (void) oldname, (void) newname; 1440 } 1441 1442 void 1443 spa_import_os(spa_t *spa) 1444 { 1445 (void) spa; 1446 } 1447 1448 void 1449 spa_export_os(spa_t *spa) 1450 { 1451 (void) spa; 1452 } 1453 1454 void 1455 spa_activate_os(spa_t *spa) 1456 { 1457 (void) spa; 1458 } 1459 1460 void 1461 spa_deactivate_os(spa_t *spa) 1462 { 1463 (void) spa; 1464 } 1465