1eda14cbcSMatt Macy /*
2eda14cbcSMatt Macy * CDDL HEADER START
3eda14cbcSMatt Macy *
4eda14cbcSMatt Macy * The contents of this file are subject to the terms of the
5eda14cbcSMatt Macy * Common Development and Distribution License (the "License").
6eda14cbcSMatt Macy * You may not use this file except in compliance with the License.
7eda14cbcSMatt Macy *
8eda14cbcSMatt Macy * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9eda14cbcSMatt Macy * or https://opensource.org/licenses/CDDL-1.0.
10eda14cbcSMatt Macy * See the License for the specific language governing permissions
11eda14cbcSMatt Macy * and limitations under the License.
12eda14cbcSMatt Macy *
13eda14cbcSMatt Macy * When distributing Covered Code, include this CDDL HEADER in each
14eda14cbcSMatt Macy * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15eda14cbcSMatt Macy * If applicable, add the following below this CDDL HEADER, with the
16eda14cbcSMatt Macy * fields enclosed by brackets "[]" replaced with your own identifying
17eda14cbcSMatt Macy * information: Portions Copyright [yyyy] [name of copyright owner]
18eda14cbcSMatt Macy *
19eda14cbcSMatt Macy * CDDL HEADER END
20eda14cbcSMatt Macy */
21eda14cbcSMatt Macy /*
22eda14cbcSMatt Macy * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23eda14cbcSMatt Macy * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
24eda14cbcSMatt Macy * Copyright (c) 2017, Intel Corporation.
25eda14cbcSMatt Macy * Copyright (c) 2024, Klara Inc.
26eda14cbcSMatt Macy */
27eda14cbcSMatt Macy
28eda14cbcSMatt Macy /*
29eda14cbcSMatt Macy * ZFS fault injection
30eda14cbcSMatt Macy *
31eda14cbcSMatt Macy * To handle fault injection, we keep track of a series of zinject_record_t
32eda14cbcSMatt Macy * structures which describe which logical block(s) should be injected with a
33eda14cbcSMatt Macy * fault. These are kept in a global list. Each record corresponds to a given
34eda14cbcSMatt Macy * spa_t and maintains a special hold on the spa_t so that it cannot be deleted
35eda14cbcSMatt Macy * or exported while the injection record exists.
36eda14cbcSMatt Macy *
37eda14cbcSMatt Macy * Device level injection is done using the 'zi_guid' field. If this is set, it
38eda14cbcSMatt Macy * means that the error is destined for a particular device, not a piece of
39eda14cbcSMatt Macy * data.
40eda14cbcSMatt Macy *
41eda14cbcSMatt Macy * This is a rather poor data structure and algorithm, but we don't expect more
42eda14cbcSMatt Macy * than a few faults at any one time, so it should be sufficient for our needs.
43eda14cbcSMatt Macy */
44eda14cbcSMatt Macy
45eda14cbcSMatt Macy #include <sys/arc.h>
46eda14cbcSMatt Macy #include <sys/zio.h>
47eda14cbcSMatt Macy #include <sys/zfs_ioctl.h>
48eda14cbcSMatt Macy #include <sys/vdev_impl.h>
49eda14cbcSMatt Macy #include <sys/dmu_objset.h>
50eda14cbcSMatt Macy #include <sys/dsl_dataset.h>
51eda14cbcSMatt Macy #include <sys/fs/zfs.h>
52eda14cbcSMatt Macy
53eda14cbcSMatt Macy uint32_t zio_injection_enabled = 0;
54eda14cbcSMatt Macy
55eda14cbcSMatt Macy /*
56eda14cbcSMatt Macy * Data describing each zinject handler registered on the system, and
57eda14cbcSMatt Macy * contains the list node linking the handler in the global zinject
58eda14cbcSMatt Macy * handler list.
59eda14cbcSMatt Macy */
60eda14cbcSMatt Macy typedef struct inject_handler {
61eda14cbcSMatt Macy int zi_id;
62eda14cbcSMatt Macy spa_t *zi_spa;
63eda14cbcSMatt Macy char *zi_spa_name; /* ZINJECT_DELAY_IMPORT only */
64eda14cbcSMatt Macy zinject_record_t zi_record;
65eda14cbcSMatt Macy uint64_t *zi_lanes;
66eda14cbcSMatt Macy int zi_next_lane;
67eda14cbcSMatt Macy list_node_t zi_link;
68eda14cbcSMatt Macy } inject_handler_t;
69eda14cbcSMatt Macy
70eda14cbcSMatt Macy /*
71eda14cbcSMatt Macy * List of all zinject handlers registered on the system, protected by
72eda14cbcSMatt Macy * the inject_lock defined below.
73eda14cbcSMatt Macy */
74eda14cbcSMatt Macy static list_t inject_handlers;
75eda14cbcSMatt Macy
76eda14cbcSMatt Macy /*
77eda14cbcSMatt Macy * This protects insertion into, and traversal of, the inject handler
78eda14cbcSMatt Macy * list defined above; as well as the inject_delay_count. Any time a
79eda14cbcSMatt Macy * handler is inserted or removed from the list, this lock should be
80eda14cbcSMatt Macy * taken as a RW_WRITER; and any time traversal is done over the list
81eda14cbcSMatt Macy * (without modification to it) this lock should be taken as a RW_READER.
82eda14cbcSMatt Macy */
83eda14cbcSMatt Macy static krwlock_t inject_lock;
84eda14cbcSMatt Macy
85eda14cbcSMatt Macy /*
86eda14cbcSMatt Macy * This holds the number of zinject delay handlers that have been
87eda14cbcSMatt Macy * registered on the system. It is protected by the inject_lock defined
88eda14cbcSMatt Macy * above. Thus modifications to this count must be a RW_WRITER of the
89eda14cbcSMatt Macy * inject_lock, and reads of this count must be (at least) a RW_READER
90eda14cbcSMatt Macy * of the lock.
91eda14cbcSMatt Macy */
92eda14cbcSMatt Macy static int inject_delay_count = 0;
93eda14cbcSMatt Macy
94eda14cbcSMatt Macy /*
95eda14cbcSMatt Macy * This lock is used only in zio_handle_io_delay(), refer to the comment
96eda14cbcSMatt Macy * in that function for more details.
97eda14cbcSMatt Macy */
98eda14cbcSMatt Macy static kmutex_t inject_delay_mtx;
99eda14cbcSMatt Macy
100eda14cbcSMatt Macy /*
101eda14cbcSMatt Macy * Used to assign unique identifying numbers to each new zinject handler.
102eda14cbcSMatt Macy */
103eda14cbcSMatt Macy static int inject_next_id = 1;
104eda14cbcSMatt Macy
105eda14cbcSMatt Macy /*
106eda14cbcSMatt Macy * Test if the requested frequency was triggered
107eda14cbcSMatt Macy */
108eda14cbcSMatt Macy static boolean_t
freq_triggered(uint32_t frequency)109eda14cbcSMatt Macy freq_triggered(uint32_t frequency)
110eda14cbcSMatt Macy {
111eda14cbcSMatt Macy /*
112eda14cbcSMatt Macy * zero implies always (100%)
113eda14cbcSMatt Macy */
114eda14cbcSMatt Macy if (frequency == 0)
115eda14cbcSMatt Macy return (B_TRUE);
116eda14cbcSMatt Macy
117eda14cbcSMatt Macy /*
118eda14cbcSMatt Macy * Note: we still handle legacy (unscaled) frequency values
119eda14cbcSMatt Macy */
120eda14cbcSMatt Macy uint32_t maximum = (frequency <= 100) ? 100 : ZI_PERCENTAGE_MAX;
121eda14cbcSMatt Macy
122eda14cbcSMatt Macy return (random_in_range(maximum) < frequency);
123eda14cbcSMatt Macy }
124eda14cbcSMatt Macy
125eda14cbcSMatt Macy /*
126eda14cbcSMatt Macy * Returns true if the given record matches the I/O in progress.
127eda14cbcSMatt Macy */
128eda14cbcSMatt Macy static boolean_t
zio_match_handler(const zbookmark_phys_t * zb,uint64_t type,int dva,zinject_record_t * record,int error)129eda14cbcSMatt Macy zio_match_handler(const zbookmark_phys_t *zb, uint64_t type, int dva,
130eda14cbcSMatt Macy zinject_record_t *record, int error)
131eda14cbcSMatt Macy {
132eda14cbcSMatt Macy /*
133eda14cbcSMatt Macy * Check for a match against the MOS, which is based on type
134eda14cbcSMatt Macy */
135eda14cbcSMatt Macy if (zb->zb_objset == DMU_META_OBJSET &&
136eda14cbcSMatt Macy record->zi_objset == DMU_META_OBJSET &&
137eda14cbcSMatt Macy record->zi_object == DMU_META_DNODE_OBJECT) {
138eda14cbcSMatt Macy if (record->zi_type == DMU_OT_NONE ||
139eda14cbcSMatt Macy type == record->zi_type)
140eda14cbcSMatt Macy return (freq_triggered(record->zi_freq));
141eda14cbcSMatt Macy else
142eda14cbcSMatt Macy return (B_FALSE);
143eda14cbcSMatt Macy }
144eda14cbcSMatt Macy
145eda14cbcSMatt Macy /*
146eda14cbcSMatt Macy * Check for an exact match.
147eda14cbcSMatt Macy */
148eda14cbcSMatt Macy if (zb->zb_objset == record->zi_objset &&
149eda14cbcSMatt Macy zb->zb_object == record->zi_object &&
150eda14cbcSMatt Macy zb->zb_level == record->zi_level &&
151eda14cbcSMatt Macy zb->zb_blkid >= record->zi_start &&
152eda14cbcSMatt Macy zb->zb_blkid <= record->zi_end &&
153eda14cbcSMatt Macy (record->zi_dvas == 0 ||
154eda14cbcSMatt Macy (dva != ZI_NO_DVA && (record->zi_dvas & (1ULL << dva)))) &&
155eda14cbcSMatt Macy error == record->zi_error) {
156eda14cbcSMatt Macy return (freq_triggered(record->zi_freq));
157eda14cbcSMatt Macy }
158eda14cbcSMatt Macy
159eda14cbcSMatt Macy return (B_FALSE);
160eda14cbcSMatt Macy }
161eda14cbcSMatt Macy
162eda14cbcSMatt Macy /*
163eda14cbcSMatt Macy * Panic the system when a config change happens in the function
164eda14cbcSMatt Macy * specified by tag.
165eda14cbcSMatt Macy */
166eda14cbcSMatt Macy void
zio_handle_panic_injection(spa_t * spa,const char * tag,uint64_t type)167eda14cbcSMatt Macy zio_handle_panic_injection(spa_t *spa, const char *tag, uint64_t type)
168eda14cbcSMatt Macy {
169eda14cbcSMatt Macy inject_handler_t *handler;
170eda14cbcSMatt Macy
171eda14cbcSMatt Macy rw_enter(&inject_lock, RW_READER);
172eda14cbcSMatt Macy
173eda14cbcSMatt Macy for (handler = list_head(&inject_handlers); handler != NULL;
174eda14cbcSMatt Macy handler = list_next(&inject_handlers, handler)) {
175eda14cbcSMatt Macy
176eda14cbcSMatt Macy if (spa != handler->zi_spa)
177eda14cbcSMatt Macy continue;
178eda14cbcSMatt Macy
179eda14cbcSMatt Macy if (handler->zi_record.zi_type == type &&
180eda14cbcSMatt Macy strcmp(tag, handler->zi_record.zi_func) == 0)
181eda14cbcSMatt Macy panic("Panic requested in function %s\n", tag);
182eda14cbcSMatt Macy }
183eda14cbcSMatt Macy
184eda14cbcSMatt Macy rw_exit(&inject_lock);
185eda14cbcSMatt Macy }
186eda14cbcSMatt Macy
187eda14cbcSMatt Macy /*
188eda14cbcSMatt Macy * Inject a decryption failure. Decryption failures can occur in
189eda14cbcSMatt Macy * both the ARC and the ZIO layers.
190eda14cbcSMatt Macy */
191eda14cbcSMatt Macy int
zio_handle_decrypt_injection(spa_t * spa,const zbookmark_phys_t * zb,uint64_t type,int error)192eda14cbcSMatt Macy zio_handle_decrypt_injection(spa_t *spa, const zbookmark_phys_t *zb,
193eda14cbcSMatt Macy uint64_t type, int error)
194eda14cbcSMatt Macy {
195eda14cbcSMatt Macy int ret = 0;
196eda14cbcSMatt Macy inject_handler_t *handler;
197eda14cbcSMatt Macy
198eda14cbcSMatt Macy rw_enter(&inject_lock, RW_READER);
199eda14cbcSMatt Macy
200eda14cbcSMatt Macy for (handler = list_head(&inject_handlers); handler != NULL;
201eda14cbcSMatt Macy handler = list_next(&inject_handlers, handler)) {
202eda14cbcSMatt Macy
203eda14cbcSMatt Macy if (spa != handler->zi_spa ||
204eda14cbcSMatt Macy handler->zi_record.zi_cmd != ZINJECT_DECRYPT_FAULT)
205eda14cbcSMatt Macy continue;
206eda14cbcSMatt Macy
207eda14cbcSMatt Macy if (zio_match_handler(zb, type, ZI_NO_DVA,
208eda14cbcSMatt Macy &handler->zi_record, error)) {
209eda14cbcSMatt Macy ret = error;
210eda14cbcSMatt Macy break;
211eda14cbcSMatt Macy }
212eda14cbcSMatt Macy }
213eda14cbcSMatt Macy
214eda14cbcSMatt Macy rw_exit(&inject_lock);
215eda14cbcSMatt Macy return (ret);
216eda14cbcSMatt Macy }
217eda14cbcSMatt Macy
218eda14cbcSMatt Macy /*
219eda14cbcSMatt Macy * If this is a physical I/O for a vdev child determine which DVA it is
220eda14cbcSMatt Macy * for. We iterate backwards through the DVAs matching on the offset so
221eda14cbcSMatt Macy * that we end up with ZI_NO_DVA (-1) if we don't find a match.
222eda14cbcSMatt Macy */
223eda14cbcSMatt Macy static int
zio_match_dva(zio_t * zio)224eda14cbcSMatt Macy zio_match_dva(zio_t *zio)
225eda14cbcSMatt Macy {
226eda14cbcSMatt Macy int i = ZI_NO_DVA;
227eda14cbcSMatt Macy
228eda14cbcSMatt Macy if (zio->io_bp != NULL && zio->io_vd != NULL &&
229eda14cbcSMatt Macy zio->io_child_type == ZIO_CHILD_VDEV) {
230eda14cbcSMatt Macy for (i = BP_GET_NDVAS(zio->io_bp) - 1; i >= 0; i--) {
231eda14cbcSMatt Macy dva_t *dva = &zio->io_bp->blk_dva[i];
232eda14cbcSMatt Macy uint64_t off = DVA_GET_OFFSET(dva);
233eda14cbcSMatt Macy vdev_t *vd = vdev_lookup_top(zio->io_spa,
234eda14cbcSMatt Macy DVA_GET_VDEV(dva));
235eda14cbcSMatt Macy
236eda14cbcSMatt Macy /* Compensate for vdev label added to leaves */
237eda14cbcSMatt Macy if (zio->io_vd->vdev_ops->vdev_op_leaf)
238eda14cbcSMatt Macy off += VDEV_LABEL_START_SIZE;
239eda14cbcSMatt Macy
240eda14cbcSMatt Macy if (zio->io_vd == vd && zio->io_offset == off)
241eda14cbcSMatt Macy break;
242eda14cbcSMatt Macy }
243eda14cbcSMatt Macy }
244eda14cbcSMatt Macy
245eda14cbcSMatt Macy return (i);
246eda14cbcSMatt Macy }
247eda14cbcSMatt Macy
248eda14cbcSMatt Macy
249eda14cbcSMatt Macy /*
250eda14cbcSMatt Macy * Determine if the I/O in question should return failure. Returns the errno
251eda14cbcSMatt Macy * to be returned to the caller.
252eda14cbcSMatt Macy */
253eda14cbcSMatt Macy int
zio_handle_fault_injection(zio_t * zio,int error)254eda14cbcSMatt Macy zio_handle_fault_injection(zio_t *zio, int error)
255eda14cbcSMatt Macy {
256eda14cbcSMatt Macy int ret = 0;
257eda14cbcSMatt Macy inject_handler_t *handler;
258eda14cbcSMatt Macy
259eda14cbcSMatt Macy /*
260eda14cbcSMatt Macy * Ignore I/O not associated with any logical data.
261eda14cbcSMatt Macy */
262eda14cbcSMatt Macy if (zio->io_logical == NULL)
263eda14cbcSMatt Macy return (0);
264eda14cbcSMatt Macy
265eda14cbcSMatt Macy /*
266eda14cbcSMatt Macy * Currently, we only support fault injection on reads.
267eda14cbcSMatt Macy */
2687877fdebSMatt Macy if (zio->io_type != ZIO_TYPE_READ)
2697877fdebSMatt Macy return (0);
2707877fdebSMatt Macy
2717877fdebSMatt Macy /*
2727877fdebSMatt Macy * A rebuild I/O has no checksum to verify.
2737877fdebSMatt Macy */
274eda14cbcSMatt Macy if (zio->io_priority == ZIO_PRIORITY_REBUILD && error == ECKSUM)
275eda14cbcSMatt Macy return (0);
276eda14cbcSMatt Macy
277eda14cbcSMatt Macy rw_enter(&inject_lock, RW_READER);
278eda14cbcSMatt Macy
279eda14cbcSMatt Macy for (handler = list_head(&inject_handlers); handler != NULL;
280eda14cbcSMatt Macy handler = list_next(&inject_handlers, handler)) {
281eda14cbcSMatt Macy if (zio->io_spa != handler->zi_spa ||
282eda14cbcSMatt Macy handler->zi_record.zi_cmd != ZINJECT_DATA_FAULT)
283eda14cbcSMatt Macy continue;
284eda14cbcSMatt Macy
285eda14cbcSMatt Macy /* If this handler matches, return the specified error */
286eda14cbcSMatt Macy if (zio_match_handler(&zio->io_logical->io_bookmark,
287eda14cbcSMatt Macy zio->io_bp ? BP_GET_TYPE(zio->io_bp) : DMU_OT_NONE,
288eda14cbcSMatt Macy zio_match_dva(zio), &handler->zi_record, error)) {
289eda14cbcSMatt Macy ret = error;
290eda14cbcSMatt Macy break;
291eda14cbcSMatt Macy }
292eda14cbcSMatt Macy }
293eda14cbcSMatt Macy
294eda14cbcSMatt Macy rw_exit(&inject_lock);
295eda14cbcSMatt Macy
296eda14cbcSMatt Macy return (ret);
297eda14cbcSMatt Macy }
298eda14cbcSMatt Macy
299eda14cbcSMatt Macy /*
300eda14cbcSMatt Macy * Determine if the zio is part of a label update and has an injection
301eda14cbcSMatt Macy * handler associated with that portion of the label. Currently, we
302eda14cbcSMatt Macy * allow error injection in either the nvlist or the uberblock region of
303eda14cbcSMatt Macy * of the vdev label.
304eda14cbcSMatt Macy */
305eda14cbcSMatt Macy int
zio_handle_label_injection(zio_t * zio,int error)306eda14cbcSMatt Macy zio_handle_label_injection(zio_t *zio, int error)
307eda14cbcSMatt Macy {
308eda14cbcSMatt Macy inject_handler_t *handler;
309eda14cbcSMatt Macy vdev_t *vd = zio->io_vd;
310eda14cbcSMatt Macy uint64_t offset = zio->io_offset;
311eda14cbcSMatt Macy int label;
312eda14cbcSMatt Macy int ret = 0;
313eda14cbcSMatt Macy
314eda14cbcSMatt Macy if (offset >= VDEV_LABEL_START_SIZE &&
315eda14cbcSMatt Macy offset < vd->vdev_psize - VDEV_LABEL_END_SIZE)
316eda14cbcSMatt Macy return (0);
317eda14cbcSMatt Macy
318eda14cbcSMatt Macy rw_enter(&inject_lock, RW_READER);
319eda14cbcSMatt Macy
320eda14cbcSMatt Macy for (handler = list_head(&inject_handlers); handler != NULL;
321eda14cbcSMatt Macy handler = list_next(&inject_handlers, handler)) {
322eda14cbcSMatt Macy uint64_t start = handler->zi_record.zi_start;
323eda14cbcSMatt Macy uint64_t end = handler->zi_record.zi_end;
324eda14cbcSMatt Macy
325eda14cbcSMatt Macy if (handler->zi_record.zi_cmd != ZINJECT_LABEL_FAULT)
326eda14cbcSMatt Macy continue;
327eda14cbcSMatt Macy
328eda14cbcSMatt Macy /*
329eda14cbcSMatt Macy * The injection region is the relative offsets within a
330eda14cbcSMatt Macy * vdev label. We must determine the label which is being
331eda14cbcSMatt Macy * updated and adjust our region accordingly.
332eda14cbcSMatt Macy */
333eda14cbcSMatt Macy label = vdev_label_number(vd->vdev_psize, offset);
334eda14cbcSMatt Macy start = vdev_label_offset(vd->vdev_psize, label, start);
335eda14cbcSMatt Macy end = vdev_label_offset(vd->vdev_psize, label, end);
336eda14cbcSMatt Macy
337eda14cbcSMatt Macy if (zio->io_vd->vdev_guid == handler->zi_record.zi_guid &&
338eda14cbcSMatt Macy (offset >= start && offset <= end)) {
339eda14cbcSMatt Macy ret = error;
340eda14cbcSMatt Macy break;
341eda14cbcSMatt Macy }
342eda14cbcSMatt Macy }
343eda14cbcSMatt Macy rw_exit(&inject_lock);
344eda14cbcSMatt Macy return (ret);
345eda14cbcSMatt Macy }
346eda14cbcSMatt Macy
347eda14cbcSMatt Macy static int
zio_inject_bitflip_cb(void * data,size_t len,void * private)348eda14cbcSMatt Macy zio_inject_bitflip_cb(void *data, size_t len, void *private)
349eda14cbcSMatt Macy {
350eda14cbcSMatt Macy zio_t *zio = private;
351eda14cbcSMatt Macy uint8_t *buffer = data;
352eda14cbcSMatt Macy uint_t byte = random_in_range(len);
353eda14cbcSMatt Macy
354eda14cbcSMatt Macy ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ);
355eda14cbcSMatt Macy
356eda14cbcSMatt Macy /* flip a single random bit in an abd data buffer */
357eda14cbcSMatt Macy buffer[byte] ^= 1 << random_in_range(8);
358eda14cbcSMatt Macy
359eda14cbcSMatt Macy return (1); /* stop after first flip */
360eda14cbcSMatt Macy }
361eda14cbcSMatt Macy
362eda14cbcSMatt Macy static int
zio_handle_device_injection_impl(vdev_t * vd,zio_t * zio,int err1,int err2)363eda14cbcSMatt Macy zio_handle_device_injection_impl(vdev_t *vd, zio_t *zio, int err1, int err2)
364eda14cbcSMatt Macy {
365eda14cbcSMatt Macy inject_handler_t *handler;
366eda14cbcSMatt Macy int ret = 0;
367eda14cbcSMatt Macy
368eda14cbcSMatt Macy /*
369eda14cbcSMatt Macy * We skip over faults in the labels unless it's during device open
370eda14cbcSMatt Macy * (i.e. zio == NULL) or a device flush (offset is meaningless)
371eda14cbcSMatt Macy */
372eda14cbcSMatt Macy if (zio != NULL && zio->io_type != ZIO_TYPE_FLUSH) {
373eda14cbcSMatt Macy uint64_t offset = zio->io_offset;
374eda14cbcSMatt Macy
375eda14cbcSMatt Macy if (offset < VDEV_LABEL_START_SIZE ||
376eda14cbcSMatt Macy offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE)
377eda14cbcSMatt Macy return (0);
378eda14cbcSMatt Macy }
379eda14cbcSMatt Macy
380eda14cbcSMatt Macy rw_enter(&inject_lock, RW_READER);
381eda14cbcSMatt Macy
382eda14cbcSMatt Macy for (handler = list_head(&inject_handlers); handler != NULL;
383eda14cbcSMatt Macy handler = list_next(&inject_handlers, handler)) {
384eda14cbcSMatt Macy
385eda14cbcSMatt Macy if (handler->zi_record.zi_cmd != ZINJECT_DEVICE_FAULT)
386eda14cbcSMatt Macy continue;
387eda14cbcSMatt Macy
388eda14cbcSMatt Macy if (vd->vdev_guid == handler->zi_record.zi_guid) {
389eda14cbcSMatt Macy if (handler->zi_record.zi_failfast &&
390eda14cbcSMatt Macy (zio == NULL || (zio->io_flags &
391eda14cbcSMatt Macy (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD)))) {
392eda14cbcSMatt Macy continue;
393eda14cbcSMatt Macy }
394eda14cbcSMatt Macy
395eda14cbcSMatt Macy /* Handle type specific I/O failures */
396eda14cbcSMatt Macy if (zio != NULL &&
397eda14cbcSMatt Macy handler->zi_record.zi_iotype != ZIO_TYPES &&
398eda14cbcSMatt Macy handler->zi_record.zi_iotype != zio->io_type)
399eda14cbcSMatt Macy continue;
400eda14cbcSMatt Macy
401eda14cbcSMatt Macy if (handler->zi_record.zi_error == err1 ||
402eda14cbcSMatt Macy handler->zi_record.zi_error == err2) {
403eda14cbcSMatt Macy /*
404eda14cbcSMatt Macy * limit error injection if requested
405eda14cbcSMatt Macy */
406eda14cbcSMatt Macy if (!freq_triggered(handler->zi_record.zi_freq))
407eda14cbcSMatt Macy continue;
408eda14cbcSMatt Macy
409eda14cbcSMatt Macy /*
410eda14cbcSMatt Macy * For a failed open, pretend like the device
411eda14cbcSMatt Macy * has gone away.
412eda14cbcSMatt Macy */
413eda14cbcSMatt Macy if (err1 == ENXIO)
414eda14cbcSMatt Macy vd->vdev_stat.vs_aux =
415eda14cbcSMatt Macy VDEV_AUX_OPEN_FAILED;
416eda14cbcSMatt Macy
417eda14cbcSMatt Macy /*
418eda14cbcSMatt Macy * Treat these errors as if they had been
419eda14cbcSMatt Macy * retried so that all the appropriate stats
420eda14cbcSMatt Macy * and FMA events are generated.
421eda14cbcSMatt Macy */
422eda14cbcSMatt Macy if (!handler->zi_record.zi_failfast &&
423eda14cbcSMatt Macy zio != NULL)
424eda14cbcSMatt Macy zio->io_flags |= ZIO_FLAG_IO_RETRY;
425eda14cbcSMatt Macy
426eda14cbcSMatt Macy /*
427eda14cbcSMatt Macy * EILSEQ means flip a bit after a read
428eda14cbcSMatt Macy */
429eda14cbcSMatt Macy if (handler->zi_record.zi_error == EILSEQ) {
430eda14cbcSMatt Macy if (zio == NULL)
431eda14cbcSMatt Macy break;
432eda14cbcSMatt Macy
433eda14cbcSMatt Macy /* locate buffer data and flip a bit */
434eda14cbcSMatt Macy (void) abd_iterate_func(zio->io_abd, 0,
435eda14cbcSMatt Macy zio->io_size, zio_inject_bitflip_cb,
436eda14cbcSMatt Macy zio);
437eda14cbcSMatt Macy break;
438eda14cbcSMatt Macy }
439eda14cbcSMatt Macy
440eda14cbcSMatt Macy ret = handler->zi_record.zi_error;
441eda14cbcSMatt Macy break;
442eda14cbcSMatt Macy }
443eda14cbcSMatt Macy if (handler->zi_record.zi_error == ENXIO) {
444eda14cbcSMatt Macy ret = SET_ERROR(EIO);
445eda14cbcSMatt Macy break;
446eda14cbcSMatt Macy }
447eda14cbcSMatt Macy }
448eda14cbcSMatt Macy }
449eda14cbcSMatt Macy
450eda14cbcSMatt Macy rw_exit(&inject_lock);
451eda14cbcSMatt Macy
452eda14cbcSMatt Macy return (ret);
453eda14cbcSMatt Macy }
454eda14cbcSMatt Macy
455eda14cbcSMatt Macy int
zio_handle_device_injection(vdev_t * vd,zio_t * zio,int error)456eda14cbcSMatt Macy zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error)
457eda14cbcSMatt Macy {
458eda14cbcSMatt Macy return (zio_handle_device_injection_impl(vd, zio, error, INT_MAX));
459eda14cbcSMatt Macy }
460eda14cbcSMatt Macy
461eda14cbcSMatt Macy int
zio_handle_device_injections(vdev_t * vd,zio_t * zio,int err1,int err2)462eda14cbcSMatt Macy zio_handle_device_injections(vdev_t *vd, zio_t *zio, int err1, int err2)
463eda14cbcSMatt Macy {
464eda14cbcSMatt Macy return (zio_handle_device_injection_impl(vd, zio, err1, err2));
465eda14cbcSMatt Macy }
466eda14cbcSMatt Macy
467eda14cbcSMatt Macy /*
468eda14cbcSMatt Macy * Simulate hardware that ignores cache flushes. For requested number
469eda14cbcSMatt Macy * of seconds nix the actual writing to disk.
470eda14cbcSMatt Macy */
471eda14cbcSMatt Macy void
zio_handle_ignored_writes(zio_t * zio)472eda14cbcSMatt Macy zio_handle_ignored_writes(zio_t *zio)
473eda14cbcSMatt Macy {
474eda14cbcSMatt Macy inject_handler_t *handler;
475eda14cbcSMatt Macy
476eda14cbcSMatt Macy rw_enter(&inject_lock, RW_READER);
477eda14cbcSMatt Macy
478eda14cbcSMatt Macy for (handler = list_head(&inject_handlers); handler != NULL;
479eda14cbcSMatt Macy handler = list_next(&inject_handlers, handler)) {
480eda14cbcSMatt Macy
481eda14cbcSMatt Macy /* Ignore errors not destined for this pool */
482eda14cbcSMatt Macy if (zio->io_spa != handler->zi_spa ||
483eda14cbcSMatt Macy handler->zi_record.zi_cmd != ZINJECT_IGNORED_WRITES)
484eda14cbcSMatt Macy continue;
485eda14cbcSMatt Macy
486eda14cbcSMatt Macy /*
487eda14cbcSMatt Macy * Positive duration implies # of seconds, negative
488eda14cbcSMatt Macy * a number of txgs
489eda14cbcSMatt Macy */
490eda14cbcSMatt Macy if (handler->zi_record.zi_timer == 0) {
491eda14cbcSMatt Macy if (handler->zi_record.zi_duration > 0)
492eda14cbcSMatt Macy handler->zi_record.zi_timer = ddi_get_lbolt64();
493eda14cbcSMatt Macy else
494eda14cbcSMatt Macy handler->zi_record.zi_timer = zio->io_txg;
495eda14cbcSMatt Macy }
496eda14cbcSMatt Macy
497eda14cbcSMatt Macy /* Have a "problem" writing 60% of the time */
498eda14cbcSMatt Macy if (random_in_range(100) < 60)
499eda14cbcSMatt Macy zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES;
500eda14cbcSMatt Macy break;
501eda14cbcSMatt Macy }
502eda14cbcSMatt Macy
503eda14cbcSMatt Macy rw_exit(&inject_lock);
504eda14cbcSMatt Macy }
505eda14cbcSMatt Macy
506eda14cbcSMatt Macy void
spa_handle_ignored_writes(spa_t * spa)507eda14cbcSMatt Macy spa_handle_ignored_writes(spa_t *spa)
508eda14cbcSMatt Macy {
509eda14cbcSMatt Macy inject_handler_t *handler;
510eda14cbcSMatt Macy
511eda14cbcSMatt Macy if (zio_injection_enabled == 0)
512eda14cbcSMatt Macy return;
513eda14cbcSMatt Macy
514eda14cbcSMatt Macy rw_enter(&inject_lock, RW_READER);
515eda14cbcSMatt Macy
516eda14cbcSMatt Macy for (handler = list_head(&inject_handlers); handler != NULL;
517eda14cbcSMatt Macy handler = list_next(&inject_handlers, handler)) {
518eda14cbcSMatt Macy
519eda14cbcSMatt Macy if (spa != handler->zi_spa ||
520eda14cbcSMatt Macy handler->zi_record.zi_cmd != ZINJECT_IGNORED_WRITES)
521eda14cbcSMatt Macy continue;
522eda14cbcSMatt Macy
523eda14cbcSMatt Macy if (handler->zi_record.zi_duration > 0) {
524eda14cbcSMatt Macy VERIFY(handler->zi_record.zi_timer == 0 ||
525eda14cbcSMatt Macy ddi_time_after64(
526eda14cbcSMatt Macy (int64_t)handler->zi_record.zi_timer +
527eda14cbcSMatt Macy handler->zi_record.zi_duration * hz,
528eda14cbcSMatt Macy ddi_get_lbolt64()));
529eda14cbcSMatt Macy } else {
530eda14cbcSMatt Macy /* duration is negative so the subtraction here adds */
531eda14cbcSMatt Macy VERIFY(handler->zi_record.zi_timer == 0 ||
532eda14cbcSMatt Macy handler->zi_record.zi_timer -
533eda14cbcSMatt Macy handler->zi_record.zi_duration >=
534eda14cbcSMatt Macy spa_syncing_txg(spa));
535eda14cbcSMatt Macy }
536eda14cbcSMatt Macy }
537eda14cbcSMatt Macy
538eda14cbcSMatt Macy rw_exit(&inject_lock);
539eda14cbcSMatt Macy }
540eda14cbcSMatt Macy
541eda14cbcSMatt Macy hrtime_t
zio_handle_io_delay(zio_t * zio)542eda14cbcSMatt Macy zio_handle_io_delay(zio_t *zio)
543eda14cbcSMatt Macy {
544eda14cbcSMatt Macy vdev_t *vd = zio->io_vd;
545eda14cbcSMatt Macy inject_handler_t *min_handler = NULL;
546eda14cbcSMatt Macy hrtime_t min_target = 0;
547eda14cbcSMatt Macy
548eda14cbcSMatt Macy rw_enter(&inject_lock, RW_READER);
549eda14cbcSMatt Macy
550eda14cbcSMatt Macy /*
551eda14cbcSMatt Macy * inject_delay_count is a subset of zio_injection_enabled that
552eda14cbcSMatt Macy * is only incremented for delay handlers. These checks are
553eda14cbcSMatt Macy * mainly added to remind the reader why we're not explicitly
554eda14cbcSMatt Macy * checking zio_injection_enabled like the other functions.
555eda14cbcSMatt Macy */
556eda14cbcSMatt Macy IMPLY(inject_delay_count > 0, zio_injection_enabled > 0);
557eda14cbcSMatt Macy IMPLY(zio_injection_enabled == 0, inject_delay_count == 0);
558eda14cbcSMatt Macy
559eda14cbcSMatt Macy /*
560eda14cbcSMatt Macy * If there aren't any inject delay handlers registered, then we
561eda14cbcSMatt Macy * can short circuit and simply return 0 here. A value of zero
562eda14cbcSMatt Macy * informs zio_delay_interrupt() that this request should not be
563eda14cbcSMatt Macy * delayed. This short circuit keeps us from acquiring the
564eda14cbcSMatt Macy * inject_delay_mutex unnecessarily.
565eda14cbcSMatt Macy */
566eda14cbcSMatt Macy if (inject_delay_count == 0) {
567eda14cbcSMatt Macy rw_exit(&inject_lock);
568eda14cbcSMatt Macy return (0);
569eda14cbcSMatt Macy }
570eda14cbcSMatt Macy
571eda14cbcSMatt Macy /*
572eda14cbcSMatt Macy * Each inject handler has a number of "lanes" associated with
573eda14cbcSMatt Macy * it. Each lane is able to handle requests independently of one
574eda14cbcSMatt Macy * another, and at a latency defined by the inject handler
575eda14cbcSMatt Macy * record's zi_timer field. Thus if a handler in configured with
576eda14cbcSMatt Macy * a single lane with a 10ms latency, it will delay requests
577eda14cbcSMatt Macy * such that only a single request is completed every 10ms. So,
578eda14cbcSMatt Macy * if more than one request is attempted per each 10ms interval,
579eda14cbcSMatt Macy * the average latency of the requests will be greater than
580eda14cbcSMatt Macy * 10ms; but if only a single request is submitted each 10ms
581eda14cbcSMatt Macy * interval the average latency will be 10ms.
582eda14cbcSMatt Macy *
583eda14cbcSMatt Macy * We need to acquire this mutex to prevent multiple concurrent
584eda14cbcSMatt Macy * threads being assigned to the same lane of a given inject
585eda14cbcSMatt Macy * handler. The mutex allows us to perform the following two
586eda14cbcSMatt Macy * operations atomically:
587eda14cbcSMatt Macy *
588eda14cbcSMatt Macy * 1. determine the minimum handler and minimum target
589eda14cbcSMatt Macy * value of all the possible handlers
590eda14cbcSMatt Macy * 2. update that minimum handler's lane array
591eda14cbcSMatt Macy *
592eda14cbcSMatt Macy * Without atomicity, two (or more) threads could pick the same
593eda14cbcSMatt Macy * lane in step (1), and then conflict with each other in step
594eda14cbcSMatt Macy * (2). This could allow a single lane handler to process
595eda14cbcSMatt Macy * multiple requests simultaneously, which shouldn't be possible.
596eda14cbcSMatt Macy */
597eda14cbcSMatt Macy mutex_enter(&inject_delay_mtx);
598eda14cbcSMatt Macy
599eda14cbcSMatt Macy for (inject_handler_t *handler = list_head(&inject_handlers);
600eda14cbcSMatt Macy handler != NULL; handler = list_next(&inject_handlers, handler)) {
601eda14cbcSMatt Macy if (handler->zi_record.zi_cmd != ZINJECT_DELAY_IO)
602eda14cbcSMatt Macy continue;
603eda14cbcSMatt Macy
604eda14cbcSMatt Macy if (!freq_triggered(handler->zi_record.zi_freq))
605eda14cbcSMatt Macy continue;
606eda14cbcSMatt Macy
607eda14cbcSMatt Macy if (vd->vdev_guid != handler->zi_record.zi_guid)
608eda14cbcSMatt Macy continue;
609eda14cbcSMatt Macy
610eda14cbcSMatt Macy /* also match on I/O type (e.g., -T read) */
611eda14cbcSMatt Macy if (handler->zi_record.zi_iotype != ZIO_TYPES &&
612eda14cbcSMatt Macy handler->zi_record.zi_iotype != zio->io_type) {
613eda14cbcSMatt Macy continue;
614eda14cbcSMatt Macy }
615eda14cbcSMatt Macy
616eda14cbcSMatt Macy /*
617eda14cbcSMatt Macy * Defensive; should never happen as the array allocation
618eda14cbcSMatt Macy * occurs prior to inserting this handler on the list.
619eda14cbcSMatt Macy */
620eda14cbcSMatt Macy ASSERT3P(handler->zi_lanes, !=, NULL);
621eda14cbcSMatt Macy
622eda14cbcSMatt Macy /*
623eda14cbcSMatt Macy * This should never happen, the zinject command should
624eda14cbcSMatt Macy * prevent a user from setting an IO delay with zero lanes.
625eda14cbcSMatt Macy */
626eda14cbcSMatt Macy ASSERT3U(handler->zi_record.zi_nlanes, !=, 0);
627eda14cbcSMatt Macy
628eda14cbcSMatt Macy ASSERT3U(handler->zi_record.zi_nlanes, >,
629eda14cbcSMatt Macy handler->zi_next_lane);
630eda14cbcSMatt Macy
631eda14cbcSMatt Macy /*
632eda14cbcSMatt Macy * We want to issue this IO to the lane that will become
633eda14cbcSMatt Macy * idle the soonest, so we compare the soonest this
634eda14cbcSMatt Macy * specific handler can complete the IO with all other
635eda14cbcSMatt Macy * handlers, to find the lowest value of all possible
636eda14cbcSMatt Macy * lanes. We then use this lane to submit the request.
637eda14cbcSMatt Macy *
638eda14cbcSMatt Macy * Since each handler has a constant value for its
639eda14cbcSMatt Macy * delay, we can just use the "next" lane for that
640eda14cbcSMatt Macy * handler; as it will always be the lane with the
641eda14cbcSMatt Macy * lowest value for that particular handler (i.e. the
642eda14cbcSMatt Macy * lane that will become idle the soonest). This saves a
643eda14cbcSMatt Macy * scan of each handler's lanes array.
644eda14cbcSMatt Macy *
645eda14cbcSMatt Macy * There's two cases to consider when determining when
646eda14cbcSMatt Macy * this specific IO request should complete. If this
647eda14cbcSMatt Macy * lane is idle, we want to "submit" the request now so
648eda14cbcSMatt Macy * it will complete after zi_timer milliseconds. Thus,
649eda14cbcSMatt Macy * we set the target to now + zi_timer.
650eda14cbcSMatt Macy *
651eda14cbcSMatt Macy * If the lane is busy, we want this request to complete
652eda14cbcSMatt Macy * zi_timer milliseconds after the lane becomes idle.
653eda14cbcSMatt Macy * Since the 'zi_lanes' array holds the time at which
654eda14cbcSMatt Macy * each lane will become idle, we use that value to
655eda14cbcSMatt Macy * determine when this request should complete.
656eda14cbcSMatt Macy */
657eda14cbcSMatt Macy hrtime_t idle = handler->zi_record.zi_timer + gethrtime();
658eda14cbcSMatt Macy hrtime_t busy = handler->zi_record.zi_timer +
659eda14cbcSMatt Macy handler->zi_lanes[handler->zi_next_lane];
660eda14cbcSMatt Macy hrtime_t target = MAX(idle, busy);
661eda14cbcSMatt Macy
662eda14cbcSMatt Macy if (min_handler == NULL) {
663eda14cbcSMatt Macy min_handler = handler;
664eda14cbcSMatt Macy min_target = target;
665eda14cbcSMatt Macy continue;
666eda14cbcSMatt Macy }
667eda14cbcSMatt Macy
668eda14cbcSMatt Macy ASSERT3P(min_handler, !=, NULL);
669eda14cbcSMatt Macy ASSERT3U(min_target, !=, 0);
670eda14cbcSMatt Macy
671eda14cbcSMatt Macy /*
672eda14cbcSMatt Macy * We don't yet increment the "next lane" variable since
673eda14cbcSMatt Macy * we still might find a lower value lane in another
674eda14cbcSMatt Macy * handler during any remaining iterations. Once we're
675eda14cbcSMatt Macy * sure we've selected the absolute minimum, we'll claim
676eda14cbcSMatt Macy * the lane and increment the handler's "next lane"
677eda14cbcSMatt Macy * field below.
678eda14cbcSMatt Macy */
679eda14cbcSMatt Macy
680eda14cbcSMatt Macy if (target < min_target) {
681eda14cbcSMatt Macy min_handler = handler;
682eda14cbcSMatt Macy min_target = target;
683eda14cbcSMatt Macy }
684eda14cbcSMatt Macy }
685eda14cbcSMatt Macy
686eda14cbcSMatt Macy /*
687eda14cbcSMatt Macy * 'min_handler' will be NULL if no IO delays are registered for
688eda14cbcSMatt Macy * this vdev, otherwise it will point to the handler containing
689eda14cbcSMatt Macy * the lane that will become idle the soonest.
690eda14cbcSMatt Macy */
691eda14cbcSMatt Macy if (min_handler != NULL) {
692eda14cbcSMatt Macy ASSERT3U(min_target, !=, 0);
693eda14cbcSMatt Macy min_handler->zi_lanes[min_handler->zi_next_lane] = min_target;
694eda14cbcSMatt Macy
695eda14cbcSMatt Macy /*
696eda14cbcSMatt Macy * If we've used all possible lanes for this handler,
697eda14cbcSMatt Macy * loop back and start using the first lane again;
698eda14cbcSMatt Macy * otherwise, just increment the lane index.
699eda14cbcSMatt Macy */
700eda14cbcSMatt Macy min_handler->zi_next_lane = (min_handler->zi_next_lane + 1) %
701eda14cbcSMatt Macy min_handler->zi_record.zi_nlanes;
702eda14cbcSMatt Macy }
703eda14cbcSMatt Macy
704eda14cbcSMatt Macy mutex_exit(&inject_delay_mtx);
705eda14cbcSMatt Macy rw_exit(&inject_lock);
706eda14cbcSMatt Macy
707eda14cbcSMatt Macy return (min_target);
708eda14cbcSMatt Macy }
709eda14cbcSMatt Macy
710eda14cbcSMatt Macy static void
zio_handle_pool_delay(spa_t * spa,hrtime_t elapsed,zinject_type_t command)711eda14cbcSMatt Macy zio_handle_pool_delay(spa_t *spa, hrtime_t elapsed, zinject_type_t command)
712eda14cbcSMatt Macy {
713eda14cbcSMatt Macy inject_handler_t *handler;
714eda14cbcSMatt Macy hrtime_t delay = 0;
715eda14cbcSMatt Macy int id = 0;
716eda14cbcSMatt Macy
717eda14cbcSMatt Macy rw_enter(&inject_lock, RW_READER);
718eda14cbcSMatt Macy
719eda14cbcSMatt Macy for (handler = list_head(&inject_handlers);
720eda14cbcSMatt Macy handler != NULL && handler->zi_record.zi_cmd == command;
721eda14cbcSMatt Macy handler = list_next(&inject_handlers, handler)) {
722eda14cbcSMatt Macy ASSERT3P(handler->zi_spa_name, !=, NULL);
723eda14cbcSMatt Macy if (strcmp(spa_name(spa), handler->zi_spa_name) == 0) {
724eda14cbcSMatt Macy uint64_t pause =
725eda14cbcSMatt Macy SEC2NSEC(handler->zi_record.zi_duration);
726eda14cbcSMatt Macy if (pause > elapsed) {
727eda14cbcSMatt Macy delay = pause - elapsed;
728eda14cbcSMatt Macy }
729eda14cbcSMatt Macy id = handler->zi_id;
730eda14cbcSMatt Macy break;
731eda14cbcSMatt Macy }
732eda14cbcSMatt Macy }
733eda14cbcSMatt Macy
734eda14cbcSMatt Macy rw_exit(&inject_lock);
735eda14cbcSMatt Macy
736eda14cbcSMatt Macy if (delay) {
737eda14cbcSMatt Macy if (command == ZINJECT_DELAY_IMPORT) {
738eda14cbcSMatt Macy spa_import_progress_set_notes(spa, "injecting %llu "
739eda14cbcSMatt Macy "sec delay", (u_longlong_t)NSEC2SEC(delay));
740eda14cbcSMatt Macy }
741eda14cbcSMatt Macy zfs_sleep_until(gethrtime() + delay);
742eda14cbcSMatt Macy }
743eda14cbcSMatt Macy if (id) {
744eda14cbcSMatt Macy /* all done with this one-shot handler */
745eda14cbcSMatt Macy zio_clear_fault(id);
746eda14cbcSMatt Macy }
747eda14cbcSMatt Macy }
748eda14cbcSMatt Macy
749eda14cbcSMatt Macy /*
750eda14cbcSMatt Macy * For testing, inject a delay during an import
751eda14cbcSMatt Macy */
752eda14cbcSMatt Macy void
zio_handle_import_delay(spa_t * spa,hrtime_t elapsed)753eda14cbcSMatt Macy zio_handle_import_delay(spa_t *spa, hrtime_t elapsed)
754eda14cbcSMatt Macy {
755eda14cbcSMatt Macy zio_handle_pool_delay(spa, elapsed, ZINJECT_DELAY_IMPORT);
756eda14cbcSMatt Macy }
757eda14cbcSMatt Macy
758eda14cbcSMatt Macy /*
759eda14cbcSMatt Macy * For testing, inject a delay during an export
760eda14cbcSMatt Macy */
761eda14cbcSMatt Macy void
zio_handle_export_delay(spa_t * spa,hrtime_t elapsed)762eda14cbcSMatt Macy zio_handle_export_delay(spa_t *spa, hrtime_t elapsed)
763eda14cbcSMatt Macy {
764eda14cbcSMatt Macy zio_handle_pool_delay(spa, elapsed, ZINJECT_DELAY_EXPORT);
765eda14cbcSMatt Macy }
766eda14cbcSMatt Macy
767eda14cbcSMatt Macy static int
zio_calculate_range(const char * pool,zinject_record_t * record)768eda14cbcSMatt Macy zio_calculate_range(const char *pool, zinject_record_t *record)
769eda14cbcSMatt Macy {
770eda14cbcSMatt Macy dsl_pool_t *dp;
771eda14cbcSMatt Macy dsl_dataset_t *ds;
772eda14cbcSMatt Macy objset_t *os = NULL;
773eda14cbcSMatt Macy dnode_t *dn = NULL;
774eda14cbcSMatt Macy int error;
775eda14cbcSMatt Macy
776eda14cbcSMatt Macy /*
777eda14cbcSMatt Macy * Obtain the dnode for object using pool, objset, and object
778eda14cbcSMatt Macy */
779eda14cbcSMatt Macy error = dsl_pool_hold(pool, FTAG, &dp);
780eda14cbcSMatt Macy if (error)
781eda14cbcSMatt Macy return (error);
782eda14cbcSMatt Macy
783eda14cbcSMatt Macy error = dsl_dataset_hold_obj(dp, record->zi_objset, FTAG, &ds);
784eda14cbcSMatt Macy dsl_pool_rele(dp, FTAG);
785eda14cbcSMatt Macy if (error)
786eda14cbcSMatt Macy return (error);
787eda14cbcSMatt Macy
788eda14cbcSMatt Macy error = dmu_objset_from_ds(ds, &os);
789eda14cbcSMatt Macy dsl_dataset_rele(ds, FTAG);
790eda14cbcSMatt Macy if (error)
791eda14cbcSMatt Macy return (error);
792eda14cbcSMatt Macy
793eda14cbcSMatt Macy error = dnode_hold(os, record->zi_object, FTAG, &dn);
794eda14cbcSMatt Macy if (error)
795eda14cbcSMatt Macy return (error);
796eda14cbcSMatt Macy
797eda14cbcSMatt Macy /*
798eda14cbcSMatt Macy * Translate the range into block IDs
799eda14cbcSMatt Macy */
800eda14cbcSMatt Macy if (record->zi_start != 0 || record->zi_end != -1ULL) {
801eda14cbcSMatt Macy record->zi_start >>= dn->dn_datablkshift;
802eda14cbcSMatt Macy record->zi_end >>= dn->dn_datablkshift;
803eda14cbcSMatt Macy }
804eda14cbcSMatt Macy if (record->zi_level > 0) {
805eda14cbcSMatt Macy if (record->zi_level >= dn->dn_nlevels) {
806eda14cbcSMatt Macy dnode_rele(dn, FTAG);
807eda14cbcSMatt Macy return (SET_ERROR(EDOM));
808eda14cbcSMatt Macy }
809eda14cbcSMatt Macy
810eda14cbcSMatt Macy if (record->zi_start != 0 || record->zi_end != 0) {
811eda14cbcSMatt Macy int shift = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
812eda14cbcSMatt Macy
813eda14cbcSMatt Macy for (int level = record->zi_level; level > 0; level--) {
814eda14cbcSMatt Macy record->zi_start >>= shift;
815eda14cbcSMatt Macy record->zi_end >>= shift;
816eda14cbcSMatt Macy }
817eda14cbcSMatt Macy }
818eda14cbcSMatt Macy }
819eda14cbcSMatt Macy
820eda14cbcSMatt Macy dnode_rele(dn, FTAG);
821eda14cbcSMatt Macy return (0);
822eda14cbcSMatt Macy }
823eda14cbcSMatt Macy
824eda14cbcSMatt Macy static boolean_t
zio_pool_handler_exists(const char * name,zinject_type_t command)825eda14cbcSMatt Macy zio_pool_handler_exists(const char *name, zinject_type_t command)
826eda14cbcSMatt Macy {
827eda14cbcSMatt Macy boolean_t exists = B_FALSE;
828eda14cbcSMatt Macy
829eda14cbcSMatt Macy rw_enter(&inject_lock, RW_READER);
830eda14cbcSMatt Macy for (inject_handler_t *handler = list_head(&inject_handlers);
831eda14cbcSMatt Macy handler != NULL; handler = list_next(&inject_handlers, handler)) {
832eda14cbcSMatt Macy if (command != handler->zi_record.zi_cmd)
833eda14cbcSMatt Macy continue;
834eda14cbcSMatt Macy
835eda14cbcSMatt Macy const char *pool = (handler->zi_spa_name != NULL) ?
836eda14cbcSMatt Macy handler->zi_spa_name : spa_name(handler->zi_spa);
837eda14cbcSMatt Macy if (strcmp(name, pool) == 0) {
838eda14cbcSMatt Macy exists = B_TRUE;
839eda14cbcSMatt Macy break;
840eda14cbcSMatt Macy }
841eda14cbcSMatt Macy }
842eda14cbcSMatt Macy rw_exit(&inject_lock);
843eda14cbcSMatt Macy
844eda14cbcSMatt Macy return (exists);
845eda14cbcSMatt Macy }
846eda14cbcSMatt Macy /*
847eda14cbcSMatt Macy * Create a new handler for the given record. We add it to the list, adding
848eda14cbcSMatt Macy * a reference to the spa_t in the process. We increment zio_injection_enabled,
849eda14cbcSMatt Macy * which is the switch to trigger all fault injection.
850eda14cbcSMatt Macy */
851eda14cbcSMatt Macy int
zio_inject_fault(char * name,int flags,int * id,zinject_record_t * record)852eda14cbcSMatt Macy zio_inject_fault(char *name, int flags, int *id, zinject_record_t *record)
853eda14cbcSMatt Macy {
854eda14cbcSMatt Macy inject_handler_t *handler;
855eda14cbcSMatt Macy int error;
856eda14cbcSMatt Macy spa_t *spa;
857eda14cbcSMatt Macy
858eda14cbcSMatt Macy /*
859eda14cbcSMatt Macy * If this is pool-wide metadata, make sure we unload the corresponding
860eda14cbcSMatt Macy * spa_t, so that the next attempt to load it will trigger the fault.
861eda14cbcSMatt Macy * We call spa_reset() to unload the pool appropriately.
862eda14cbcSMatt Macy */
863eda14cbcSMatt Macy if (flags & ZINJECT_UNLOAD_SPA)
864eda14cbcSMatt Macy if ((error = spa_reset(name)) != 0)
865eda14cbcSMatt Macy return (error);
866eda14cbcSMatt Macy
867eda14cbcSMatt Macy if (record->zi_cmd == ZINJECT_DELAY_IO) {
868eda14cbcSMatt Macy /*
869eda14cbcSMatt Macy * A value of zero for the number of lanes or for the
870eda14cbcSMatt Macy * delay time doesn't make sense.
871eda14cbcSMatt Macy */
872eda14cbcSMatt Macy if (record->zi_timer == 0 || record->zi_nlanes == 0)
873eda14cbcSMatt Macy return (SET_ERROR(EINVAL));
874eda14cbcSMatt Macy
875eda14cbcSMatt Macy /*
876eda14cbcSMatt Macy * The number of lanes is directly mapped to the size of
877eda14cbcSMatt Macy * an array used by the handler. Thus, to ensure the
878eda14cbcSMatt Macy * user doesn't trigger an allocation that's "too large"
879eda14cbcSMatt Macy * we cap the number of lanes here.
880eda14cbcSMatt Macy */
881eda14cbcSMatt Macy if (record->zi_nlanes >= UINT16_MAX)
882eda14cbcSMatt Macy return (SET_ERROR(EINVAL));
883eda14cbcSMatt Macy }
884eda14cbcSMatt Macy
885eda14cbcSMatt Macy /*
886eda14cbcSMatt Macy * If the supplied range was in bytes -- calculate the actual blkid
887eda14cbcSMatt Macy */
888eda14cbcSMatt Macy if (flags & ZINJECT_CALC_RANGE) {
889eda14cbcSMatt Macy error = zio_calculate_range(name, record);
890eda14cbcSMatt Macy if (error != 0)
891eda14cbcSMatt Macy return (error);
892eda14cbcSMatt Macy }
893eda14cbcSMatt Macy
894eda14cbcSMatt Macy if (!(flags & ZINJECT_NULL)) {
895eda14cbcSMatt Macy /*
896eda14cbcSMatt Macy * Pool delays for import or export don't take an
897eda14cbcSMatt Macy * injection reference on the spa. Instead they
898eda14cbcSMatt Macy * rely on matching by name.
899eda14cbcSMatt Macy */
900eda14cbcSMatt Macy if (record->zi_cmd == ZINJECT_DELAY_IMPORT ||
901eda14cbcSMatt Macy record->zi_cmd == ZINJECT_DELAY_EXPORT) {
902eda14cbcSMatt Macy if (record->zi_duration <= 0)
903eda14cbcSMatt Macy return (SET_ERROR(EINVAL));
904eda14cbcSMatt Macy /*
905eda14cbcSMatt Macy * Only one import | export delay handler per pool.
906eda14cbcSMatt Macy */
907eda14cbcSMatt Macy if (zio_pool_handler_exists(name, record->zi_cmd))
908eda14cbcSMatt Macy return (SET_ERROR(EEXIST));
909eda14cbcSMatt Macy
910eda14cbcSMatt Macy mutex_enter(&spa_namespace_lock);
911eda14cbcSMatt Macy boolean_t has_spa = spa_lookup(name) != NULL;
912eda14cbcSMatt Macy mutex_exit(&spa_namespace_lock);
913eda14cbcSMatt Macy
914eda14cbcSMatt Macy if (record->zi_cmd == ZINJECT_DELAY_IMPORT && has_spa)
915eda14cbcSMatt Macy return (SET_ERROR(EEXIST));
916eda14cbcSMatt Macy if (record->zi_cmd == ZINJECT_DELAY_EXPORT && !has_spa)
917eda14cbcSMatt Macy return (SET_ERROR(ENOENT));
918eda14cbcSMatt Macy spa = NULL;
919eda14cbcSMatt Macy } else {
920eda14cbcSMatt Macy /*
921eda14cbcSMatt Macy * spa_inject_ref() will add an injection reference,
922eda14cbcSMatt Macy * which will prevent the pool from being removed
923eda14cbcSMatt Macy * from the namespace while still allowing it to be
924eda14cbcSMatt Macy * unloaded.
925eda14cbcSMatt Macy */
926eda14cbcSMatt Macy if ((spa = spa_inject_addref(name)) == NULL)
927eda14cbcSMatt Macy return (SET_ERROR(ENOENT));
928eda14cbcSMatt Macy }
929eda14cbcSMatt Macy
930eda14cbcSMatt Macy handler = kmem_alloc(sizeof (inject_handler_t), KM_SLEEP);
931eda14cbcSMatt Macy handler->zi_spa = spa; /* note: can be NULL */
932eda14cbcSMatt Macy handler->zi_record = *record;
933eda14cbcSMatt Macy
934eda14cbcSMatt Macy if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
935eda14cbcSMatt Macy handler->zi_lanes = kmem_zalloc(
936eda14cbcSMatt Macy sizeof (*handler->zi_lanes) *
937eda14cbcSMatt Macy handler->zi_record.zi_nlanes, KM_SLEEP);
938eda14cbcSMatt Macy handler->zi_next_lane = 0;
939eda14cbcSMatt Macy } else {
940eda14cbcSMatt Macy handler->zi_lanes = NULL;
941eda14cbcSMatt Macy handler->zi_next_lane = 0;
942eda14cbcSMatt Macy }
943eda14cbcSMatt Macy
944eda14cbcSMatt Macy if (handler->zi_spa == NULL)
945eda14cbcSMatt Macy handler->zi_spa_name = spa_strdup(name);
946eda14cbcSMatt Macy else
947eda14cbcSMatt Macy handler->zi_spa_name = NULL;
948eda14cbcSMatt Macy
949eda14cbcSMatt Macy rw_enter(&inject_lock, RW_WRITER);
950eda14cbcSMatt Macy
951eda14cbcSMatt Macy /*
952eda14cbcSMatt Macy * We can't move this increment into the conditional
953eda14cbcSMatt Macy * above because we need to hold the RW_WRITER lock of
954eda14cbcSMatt Macy * inject_lock, and we don't want to hold that while
955eda14cbcSMatt Macy * allocating the handler's zi_lanes array.
956eda14cbcSMatt Macy */
957eda14cbcSMatt Macy if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
958eda14cbcSMatt Macy ASSERT3S(inject_delay_count, >=, 0);
959eda14cbcSMatt Macy inject_delay_count++;
960eda14cbcSMatt Macy ASSERT3S(inject_delay_count, >, 0);
961eda14cbcSMatt Macy }
962eda14cbcSMatt Macy
963eda14cbcSMatt Macy *id = handler->zi_id = inject_next_id++;
964eda14cbcSMatt Macy list_insert_tail(&inject_handlers, handler);
965eda14cbcSMatt Macy atomic_inc_32(&zio_injection_enabled);
966eda14cbcSMatt Macy
967eda14cbcSMatt Macy rw_exit(&inject_lock);
968eda14cbcSMatt Macy }
969eda14cbcSMatt Macy
970eda14cbcSMatt Macy /*
971eda14cbcSMatt Macy * Flush the ARC, so that any attempts to read this data will end up
972eda14cbcSMatt Macy * going to the ZIO layer. Note that this is a little overkill, but
973 * we don't have the necessary ARC interfaces to do anything else, and
974 * fault injection isn't a performance critical path.
975 */
976 if (flags & ZINJECT_FLUSH_ARC)
977 /*
978 * We must use FALSE to ensure arc_flush returns, since
979 * we're not preventing concurrent ARC insertions.
980 */
981 arc_flush(NULL, FALSE);
982
983 return (0);
984 }
985
986 /*
987 * Returns the next record with an ID greater than that supplied to the
988 * function. Used to iterate over all handlers in the system.
989 */
990 int
zio_inject_list_next(int * id,char * name,size_t buflen,zinject_record_t * record)991 zio_inject_list_next(int *id, char *name, size_t buflen,
992 zinject_record_t *record)
993 {
994 inject_handler_t *handler;
995 int ret;
996
997 mutex_enter(&spa_namespace_lock);
998 rw_enter(&inject_lock, RW_READER);
999
1000 for (handler = list_head(&inject_handlers); handler != NULL;
1001 handler = list_next(&inject_handlers, handler))
1002 if (handler->zi_id > *id)
1003 break;
1004
1005 if (handler) {
1006 *record = handler->zi_record;
1007 *id = handler->zi_id;
1008 ASSERT(handler->zi_spa || handler->zi_spa_name);
1009 if (handler->zi_spa != NULL)
1010 (void) strlcpy(name, spa_name(handler->zi_spa), buflen);
1011 else
1012 (void) strlcpy(name, handler->zi_spa_name, buflen);
1013 ret = 0;
1014 } else {
1015 ret = SET_ERROR(ENOENT);
1016 }
1017
1018 rw_exit(&inject_lock);
1019 mutex_exit(&spa_namespace_lock);
1020
1021 return (ret);
1022 }
1023
1024 /*
1025 * Clear the fault handler with the given identifier, or return ENOENT if none
1026 * exists.
1027 */
1028 int
zio_clear_fault(int id)1029 zio_clear_fault(int id)
1030 {
1031 inject_handler_t *handler;
1032
1033 rw_enter(&inject_lock, RW_WRITER);
1034
1035 for (handler = list_head(&inject_handlers); handler != NULL;
1036 handler = list_next(&inject_handlers, handler))
1037 if (handler->zi_id == id)
1038 break;
1039
1040 if (handler == NULL) {
1041 rw_exit(&inject_lock);
1042 return (SET_ERROR(ENOENT));
1043 }
1044
1045 if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
1046 ASSERT3S(inject_delay_count, >, 0);
1047 inject_delay_count--;
1048 ASSERT3S(inject_delay_count, >=, 0);
1049 }
1050
1051 list_remove(&inject_handlers, handler);
1052 rw_exit(&inject_lock);
1053
1054 if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
1055 ASSERT3P(handler->zi_lanes, !=, NULL);
1056 kmem_free(handler->zi_lanes, sizeof (*handler->zi_lanes) *
1057 handler->zi_record.zi_nlanes);
1058 } else {
1059 ASSERT3P(handler->zi_lanes, ==, NULL);
1060 }
1061
1062 if (handler->zi_spa_name != NULL)
1063 spa_strfree(handler->zi_spa_name);
1064
1065 if (handler->zi_spa != NULL)
1066 spa_inject_delref(handler->zi_spa);
1067 kmem_free(handler, sizeof (inject_handler_t));
1068 atomic_dec_32(&zio_injection_enabled);
1069
1070 return (0);
1071 }
1072
1073 void
zio_inject_init(void)1074 zio_inject_init(void)
1075 {
1076 rw_init(&inject_lock, NULL, RW_DEFAULT, NULL);
1077 mutex_init(&inject_delay_mtx, NULL, MUTEX_DEFAULT, NULL);
1078 list_create(&inject_handlers, sizeof (inject_handler_t),
1079 offsetof(inject_handler_t, zi_link));
1080 }
1081
1082 void
zio_inject_fini(void)1083 zio_inject_fini(void)
1084 {
1085 list_destroy(&inject_handlers);
1086 mutex_destroy(&inject_delay_mtx);
1087 rw_destroy(&inject_lock);
1088 }
1089
1090 #if defined(_KERNEL)
1091 EXPORT_SYMBOL(zio_injection_enabled);
1092 EXPORT_SYMBOL(zio_inject_fault);
1093 EXPORT_SYMBOL(zio_inject_list_next);
1094 EXPORT_SYMBOL(zio_clear_fault);
1095 EXPORT_SYMBOL(zio_handle_fault_injection);
1096 EXPORT_SYMBOL(zio_handle_device_injection);
1097 EXPORT_SYMBOL(zio_handle_label_injection);
1098 #endif
1099