18a272653SPeter Holm#!/bin/sh 28a272653SPeter Holm 38a272653SPeter Holm# 48a272653SPeter Holm# Copyright (c) 2016 EMC Corp. 58a272653SPeter Holm# All rights reserved. 68a272653SPeter Holm# 78a272653SPeter Holm# Redistribution and use in source and binary forms, with or without 88a272653SPeter Holm# modification, are permitted provided that the following conditions 98a272653SPeter Holm# are met: 108a272653SPeter Holm# 1. Redistributions of source code must retain the above copyright 118a272653SPeter Holm# notice, this list of conditions and the following disclaimer. 128a272653SPeter Holm# 2. Redistributions in binary form must reproduce the above copyright 138a272653SPeter Holm# notice, this list of conditions and the following disclaimer in the 148a272653SPeter Holm# documentation and/or other materials provided with the distribution. 158a272653SPeter Holm# 168a272653SPeter Holm# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 178a272653SPeter Holm# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 188a272653SPeter Holm# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 198a272653SPeter Holm# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 208a272653SPeter Holm# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 218a272653SPeter Holm# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 228a272653SPeter Holm# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 238a272653SPeter Holm# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 248a272653SPeter Holm# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 258a272653SPeter Holm# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 268a272653SPeter Holm# SUCH DAMAGE. 278a272653SPeter Holm# 288a272653SPeter Holm 298a272653SPeter Holm# uma_zalloc_arc() fail point test scenario. 308a272653SPeter Holm# Test scenario by Ryan Libby <rlibby@gmail.com>. 318a272653SPeter Holm 328a272653SPeter Holm# "panic: backing_object 0xfffff8016dd74420 was somehow re-referenced during 338a272653SPeter Holm# collapse!" seen. 348a272653SPeter Holm# https://people.freebsd.org/~pho/stress/log/uma_zalloc_arg.txt 358a272653SPeter Holm 368a272653SPeter Holm# Hang seen: 378a272653SPeter Holm# https://people.freebsd.org/~pho/stress/log/kostik869.txt 388a272653SPeter Holm 398a272653SPeter Holm[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1 408a272653SPeter Holm 418a272653SPeter Holm. ../default.cfg 428a272653SPeter Holm 438a272653SPeter Holmsysctl debug.mnowait_failure.zalloc_whitelist > /dev/null 2>&1 || exit 0 448a272653SPeter Holm 458a272653SPeter Holms1=`sysctl -n debug.mnowait_failure.zalloc_whitelist` 468a272653SPeter Holms2=`sysctl -n debug.fail_point.uma_zalloc_arg` 478a272653SPeter Holmcleanup() { 488a272653SPeter Holm sysctl debug.mnowait_failure.zalloc_whitelist="$s1" > /dev/null 498a272653SPeter Holm sysctl debug.fail_point.uma_zalloc_arg="$s2" > /dev/null 508a272653SPeter Holm} 518a272653SPeter Holmtrap "cleanup" EXIT INT 528a272653SPeter Holmsysctl debug.mnowait_failure.zalloc_whitelist='RADIX NODE' 538a272653SPeter Holmsysctl debug.fail_point.uma_zalloc_arg='1%return' 548a272653SPeter Holm 558a272653SPeter Holmstart=`date '+%s'` 568a272653SPeter Holmwhile [ $((`date '+%s'` - start)) -lt 300 ]; do 578a272653SPeter Holm sh -c "echo | cat | cat > /dev/null" 588a272653SPeter Holmdone 598a272653SPeter Holmexit 0 608a272653SPeter Holm 618a272653SPeter HolmThe patch from 628a272653SPeter Holmhttps://github.com/freebsd/freebsd/compare/master...rlibby:mnowait-dbg 638a272653SPeter Holm 648a272653SPeter Holmdiff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c 658a272653SPeter Holmindex 01aff78..9d557a1 100644 668a272653SPeter Holm--- a/sys/kern/kern_malloc.c 678a272653SPeter Holm+++ b/sys/kern/kern_malloc.c 688a272653SPeter Holm@@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$"); 698a272653SPeter Holm 708a272653SPeter Holm #include <sys/param.h> 718a272653SPeter Holm #include <sys/systm.h> 728a272653SPeter Holm+#include <sys/fail.h> 738a272653SPeter Holm #include <sys/kdb.h> 748a272653SPeter Holm #include <sys/kernel.h> 758a272653SPeter Holm #include <sys/lock.h> 768a272653SPeter Holm@@ -472,6 +473,19 @@ malloc(unsigned long size, struct malloc_type *mtp, int flags) 778a272653SPeter Holm } 788a272653SPeter Holm } 798a272653SPeter Holm #endif 808a272653SPeter Holm+ if (flags & M_NOWAIT) { 818a272653SPeter Holm+ KFAIL_POINT_CODE(DEBUG_FP, malloc, { 828a272653SPeter Holm+ if (uma_dbg_nowait_fail_enabled_malloc( 838a272653SPeter Holm+ mtp->ks_shortdesc)) { 848a272653SPeter Holm+ /* XXX record call stack */ 858a272653SPeter Holm+#ifdef MALLOC_MAKE_FAILURES 868a272653SPeter Holm+ atomic_add_int(&malloc_failure_count, 1); 878a272653SPeter Holm+ t_malloc_fail = time_uptime; 888a272653SPeter Holm+#endif 898a272653SPeter Holm+ return (NULL); 908a272653SPeter Holm+ } 918a272653SPeter Holm+ }); 928a272653SPeter Holm+ } 938a272653SPeter Holm if (flags & M_WAITOK) 948a272653SPeter Holm KASSERT(curthread->td_intr_nesting_level == 0, 958a272653SPeter Holm ("malloc(M_WAITOK) in interrupt context")); 968a272653SPeter Holmdiff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c 978a272653SPeter Holmindex 1f57dff..dfa18e6 100644 988a272653SPeter Holm--- a/sys/vm/uma_core.c 998a272653SPeter Holm+++ b/sys/vm/uma_core.c 1008a272653SPeter Holm@@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$"); 1018a272653SPeter Holm #include <sys/param.h> 1028a272653SPeter Holm #include <sys/systm.h> 1038a272653SPeter Holm #include <sys/bitset.h> 1048a272653SPeter Holm+#include <sys/fail.h> 1058a272653SPeter Holm #include <sys/kernel.h> 1068a272653SPeter Holm #include <sys/types.h> 1078a272653SPeter Holm #include <sys/queue.h> 1088a272653SPeter Holm@@ -2148,6 +2149,23 @@ uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) 1098a272653SPeter Holm if (flags & M_WAITOK) { 1108a272653SPeter Holm WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 1118a272653SPeter Holm "uma_zalloc_arg: zone \"%s\"", zone->uz_name); 1128a272653SPeter Holm+ } else { 1138a272653SPeter Holm+ KFAIL_POINT_CODE(DEBUG_FP, uma_zalloc_arg, { 1148a272653SPeter Holm+ /* 1158a272653SPeter Holm+ * XXX hack. Setting the fail point to 0 (default) 1168a272653SPeter Holm+ * causes it to ignore malloc zones, nonzero causes it 1178a272653SPeter Holm+ * to inject failures for malloc zones regardless of 1188a272653SPeter Holm+ * the malloc black/white lists. 1198a272653SPeter Holm+ */ 1208a272653SPeter Holm+ if (((zone->uz_flags & UMA_ZONE_MALLOC) == 0 || 1218a272653SPeter Holm+ RETURN_VALUE != 0) && 1228a272653SPeter Holm+ uma_dbg_nowait_fail_enabled_zalloc( 1238a272653SPeter Holm+ zone->uz_name)) { 1248a272653SPeter Holm+ /* XXX record call stack */ 1258a272653SPeter Holm+ atomic_add_long(&zone->uz_fails, 1); 1268a272653SPeter Holm+ return NULL; 1278a272653SPeter Holm+ } 1288a272653SPeter Holm+ }); 1298a272653SPeter Holm } 1308a272653SPeter Holm 1318a272653SPeter Holm KASSERT(curthread->td_critnest == 0, 1328a272653SPeter Holmdiff --git a/sys/vm/uma_dbg.c b/sys/vm/uma_dbg.c 1338a272653SPeter Holmindex 3fbd29b..bed3130 100644 1348a272653SPeter Holm--- a/sys/vm/uma_dbg.c 1358a272653SPeter Holm+++ b/sys/vm/uma_dbg.c 1368a272653SPeter Holm@@ -42,6 +42,8 @@ __FBSDID("$FreeBSD$"); 1378a272653SPeter Holm #include <sys/lock.h> 1388a272653SPeter Holm #include <sys/mutex.h> 1398a272653SPeter Holm #include <sys/malloc.h> 1408a272653SPeter Holm+#include <sys/rwlock.h> 1418a272653SPeter Holm+#include <sys/sysctl.h> 1428a272653SPeter Holm 1438a272653SPeter Holm #include <vm/vm.h> 1448a272653SPeter Holm #include <vm/vm_object.h> 1458a272653SPeter Holm@@ -292,4 +294,143 @@ uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item) 1468a272653SPeter Holm BIT_CLR_ATOMIC(SLAB_SETSIZE, freei, &slab->us_debugfree); 1478a272653SPeter Holm } 1488a272653SPeter Holm 1498a272653SPeter Holm+/* XXX explain */ 1508a272653SPeter Holm+struct rwlock g_uma_dbg_nowait_lock; 1518a272653SPeter Holm+RW_SYSINIT(uma_dbg_nowait, &g_uma_dbg_nowait_lock, "uma dbg nowait"); 1528a272653SPeter Holm+ 1538a272653SPeter Holm+#define NOWAIT_FAIL_LIST_BUFSIZE 4096 1548a272653SPeter Holm+char malloc_fail_blacklist[NOWAIT_FAIL_LIST_BUFSIZE] = "kobj"; 1558a272653SPeter Holm+char malloc_fail_whitelist[NOWAIT_FAIL_LIST_BUFSIZE] = ""; 1568a272653SPeter Holm+char zalloc_fail_blacklist[NOWAIT_FAIL_LIST_BUFSIZE] = 1578a272653SPeter Holm+ "BUF TRIE,ata_request,sackhole"; 1588a272653SPeter Holm+char zalloc_fail_whitelist[NOWAIT_FAIL_LIST_BUFSIZE] = ""; 1598a272653SPeter Holm+ 1608a272653SPeter Holm+static bool 1618a272653SPeter Holm+str_in_list(const char *list, char delim, const char *str) 1628a272653SPeter Holm+{ 1638a272653SPeter Holm+ const char *b, *e; 1648a272653SPeter Holm+ size_t blen, slen; 1658a272653SPeter Holm+ 1668a272653SPeter Holm+ b = list; 1678a272653SPeter Holm+ slen = strlen(str); 1688a272653SPeter Holm+ for (;;) { 1698a272653SPeter Holm+ e = strchr(b, delim); 1708a272653SPeter Holm+ blen = e == NULL ? strlen(b) : e - b; 1718a272653SPeter Holm+ if (blen == slen && strncmp(b, str, slen) == 0) 1728a272653SPeter Holm+ return true; 1738a272653SPeter Holm+ if (e == NULL) 1748a272653SPeter Holm+ break; 1758a272653SPeter Holm+ b = e + 1; 1768a272653SPeter Holm+ } 1778a272653SPeter Holm+ return false; 1788a272653SPeter Holm+} 1798a272653SPeter Holm+ 1808a272653SPeter Holm+static bool 1818a272653SPeter Holm+uma_dbg_nowait_fail_enabled_internal(const char *blacklist, 1828a272653SPeter Holm+ const char *whitelist, const char *name) 1838a272653SPeter Holm+{ 1848a272653SPeter Holm+ bool fail; 1858a272653SPeter Holm+ 1868a272653SPeter Holm+ /* Protect ourselves from the sysctl handlers. */ 1878a272653SPeter Holm+ rw_rlock(&g_uma_dbg_nowait_lock); 1888a272653SPeter Holm+ if (whitelist[0] == '\0') 1898a272653SPeter Holm+ fail = !str_in_list(blacklist, ',', name); 1908a272653SPeter Holm+ else 1918a272653SPeter Holm+ fail = str_in_list(whitelist, ',', name); 1928a272653SPeter Holm+ rw_runlock(&g_uma_dbg_nowait_lock); 1938a272653SPeter Holm+ 1948a272653SPeter Holm+ return fail; 1958a272653SPeter Holm+} 1968a272653SPeter Holm+ 1978a272653SPeter Holm+bool 1988a272653SPeter Holm+uma_dbg_nowait_fail_enabled_malloc(const char *name) 1998a272653SPeter Holm+{ 2008a272653SPeter Holm+ return uma_dbg_nowait_fail_enabled_internal(malloc_fail_blacklist, 2018a272653SPeter Holm+ malloc_fail_whitelist, name); 2028a272653SPeter Holm+} 2038a272653SPeter Holm+ 2048a272653SPeter Holm+bool 2058a272653SPeter Holm+uma_dbg_nowait_fail_enabled_zalloc(const char *name) 2068a272653SPeter Holm+{ 2078a272653SPeter Holm+ return uma_dbg_nowait_fail_enabled_internal(zalloc_fail_blacklist, 2088a272653SPeter Holm+ zalloc_fail_whitelist, name); 2098a272653SPeter Holm+} 2108a272653SPeter Holm+ 2118a272653SPeter Holm+/* 2128a272653SPeter Holm+ * XXX provide SYSCTL_STRING_LOCKED / sysctl_string_locked_handler? 2138a272653SPeter Holm+ * This is basically just a different sysctl_string_handler. This one wraps 2148a272653SPeter Holm+ * the string manipulation in a lock and in a way that will not cause a sleep 2158a272653SPeter Holm+ * under that lock. 2168a272653SPeter Holm+ */ 2178a272653SPeter Holm+static int 2188a272653SPeter Holm+sysctl_debug_mnowait_failure_list(SYSCTL_HANDLER_ARGS) 2198a272653SPeter Holm+{ 2208a272653SPeter Holm+ char *newbuf = NULL; 2218a272653SPeter Holm+ int error, newlen; 2228a272653SPeter Holm+ bool have_lock = false; 2238a272653SPeter Holm+ 2248a272653SPeter Holm+ if (req->newptr != NULL) { 2258a272653SPeter Holm+ newlen = req->newlen - req->newidx; 2268a272653SPeter Holm+ if (newlen >= arg2) { 2278a272653SPeter Holm+ error = EINVAL; 2288a272653SPeter Holm+ goto out; 2298a272653SPeter Holm+ } 2308a272653SPeter Holm+ newbuf = malloc(newlen, M_TEMP, M_WAITOK); 2318a272653SPeter Holm+ error = SYSCTL_IN(req, newbuf, newlen); 2328a272653SPeter Holm+ if (error != 0) 2338a272653SPeter Holm+ goto out; 2348a272653SPeter Holm+ } 2358a272653SPeter Holm+ 2368a272653SPeter Holm+ error = sysctl_wire_old_buffer(req, arg2); 2378a272653SPeter Holm+ if (error != 0) 2388a272653SPeter Holm+ goto out; 2398a272653SPeter Holm+ 2408a272653SPeter Holm+ rw_wlock(&g_uma_dbg_nowait_lock); 2418a272653SPeter Holm+ have_lock = true; 2428a272653SPeter Holm+ 2438a272653SPeter Holm+ error = SYSCTL_OUT(req, arg1, strnlen(arg1, arg2 - 1) + 1); 2448a272653SPeter Holm+ if (error != 0) 2458a272653SPeter Holm+ goto out; 2468a272653SPeter Holm+ 2478a272653SPeter Holm+ if (newbuf == NULL) 2488a272653SPeter Holm+ goto out; 2498a272653SPeter Holm+ 2508a272653SPeter Holm+ bcopy(newbuf, arg1, newlen); 2518a272653SPeter Holm+ ((char *)arg1)[newlen] = '\0'; 2528a272653SPeter Holm+ out: 2538a272653SPeter Holm+ if (have_lock) 2548a272653SPeter Holm+ rw_wunlock(&g_uma_dbg_nowait_lock); 2558a272653SPeter Holm+ free(newbuf, M_TEMP); 2568a272653SPeter Holm+ return error; 2578a272653SPeter Holm+} 2588a272653SPeter Holm+ 2598a272653SPeter Holm+SYSCTL_NODE(_debug, OID_AUTO, mnowait_failure, CTLFLAG_RW, 0, 2608a272653SPeter Holm+ "Control of M_NOWAIT memory allocation failure injection."); 2618a272653SPeter Holm+ 2628a272653SPeter Holm+SYSCTL_PROC(_debug_mnowait_failure, OID_AUTO, malloc_blacklist, 2638a272653SPeter Holm+ CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, malloc_fail_blacklist, 2648a272653SPeter Holm+ sizeof(malloc_fail_blacklist), sysctl_debug_mnowait_failure_list, "A", 2658a272653SPeter Holm+ "With debug.fail_point.malloc and with an empty whitelist, CSV list of " 2668a272653SPeter Holm+ "zones which remain unaffected."); 2678a272653SPeter Holm+ 2688a272653SPeter Holm+SYSCTL_PROC(_debug_mnowait_failure, OID_AUTO, malloc_whitelist, 2698a272653SPeter Holm+ CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, malloc_fail_whitelist, 2708a272653SPeter Holm+ sizeof(malloc_fail_whitelist), sysctl_debug_mnowait_failure_list, "A", 2718a272653SPeter Holm+ "With debug.fail_point.malloc, CSV list of zones exclusively affected. " 2728a272653SPeter Holm+ "With an empty whitelist, all zones but those on the blacklist" 2738a272653SPeter Holm+ "are affected."); 2748a272653SPeter Holm+ 2758a272653SPeter Holm+SYSCTL_PROC(_debug_mnowait_failure, OID_AUTO, zalloc_blacklist, 2768a272653SPeter Holm+ CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, zalloc_fail_blacklist, 2778a272653SPeter Holm+ sizeof(zalloc_fail_blacklist), sysctl_debug_mnowait_failure_list, "A", 2788a272653SPeter Holm+ "With debug.fail_point.uma_zalloc_arg and with an empty whitelist, CSV " 2798a272653SPeter Holm+ "list of zones which remain unaffected."); 2808a272653SPeter Holm+ 2818a272653SPeter Holm+SYSCTL_PROC(_debug_mnowait_failure, OID_AUTO, zalloc_whitelist, 2828a272653SPeter Holm+ CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, zalloc_fail_whitelist, 2838a272653SPeter Holm+ sizeof(zalloc_fail_whitelist), sysctl_debug_mnowait_failure_list, "A", 2848a272653SPeter Holm+ "With debug.fail_point.uma_zalloc_arg, CSV list of zones exclusively " 2858a272653SPeter Holm+ "affected. With an empty whitelist, all zones but those on the blacklist" 2868a272653SPeter Holm+ "are affected."); 2878a272653SPeter Holm+ 2888a272653SPeter Holm #endif /* INVARIANTS */ 2898a272653SPeter Holmdiff --git a/sys/vm/uma_int.h b/sys/vm/uma_int.h 2908a272653SPeter Holmindex ad2a405..284747f 100644 2918a272653SPeter Holm--- a/sys/vm/uma_int.h 2928a272653SPeter Holm+++ b/sys/vm/uma_int.h 2938a272653SPeter Holm@@ -427,6 +427,9 @@ vsetslab(vm_offset_t va, uma_slab_t slab) 2948a272653SPeter Holm void *uma_small_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *pflag, 2958a272653SPeter Holm int wait); 2968a272653SPeter Holm void uma_small_free(void *mem, vm_size_t size, uint8_t flags); 2978a272653SPeter Holm+ 2988a272653SPeter Holm+bool uma_dbg_nowait_fail_enabled_malloc(const char *name); 2998a272653SPeter Holm+bool uma_dbg_nowait_fail_enabled_zalloc(const char *name); 3008a272653SPeter Holm #endif /* _KERNEL */ 3018a272653SPeter Holm 3028a272653SPeter Holm #endif /* VM_UMA_INT_H */ 303