1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 #include <assert.h>
26 #include <stddef.h>
27 #include <strings.h>
28 #include <libuutil.h>
29 #include <libzfs.h>
30 #include <fm/fmd_api.h>
31 #include <fm/libtopo.h>
32 #include <sys/types.h>
33 #include <sys/time.h>
34 #include <sys/fs/zfs.h>
35 #include <sys/fm/protocol.h>
36 #include <sys/fm/fs/zfs.h>
37 
38 /*
39  * Our serd engines are named 'zfs_<pool_guid>_<vdev_guid>_{checksum,io}'.  This
40  * #define reserves enough space for two 64-bit hex values plus the length of
41  * the longest string.
42  */
43 #define	MAX_SERDLEN	(16 * 2 + sizeof ("zfs___checksum"))
44 
45 /*
46  * On-disk case structure.  This must maintain backwards compatibility with
47  * previous versions of the DE.  By default, any members appended to the end
48  * will be filled with zeros if they don't exist in a previous version.
49  */
50 typedef struct zfs_case_data {
51 	uint64_t	zc_version;
52 	uint64_t	zc_ena;
53 	uint64_t	zc_pool_guid;
54 	uint64_t	zc_vdev_guid;
55 	int		zc_has_timer;		/* defunct */
56 	int		zc_pool_state;
57 	char		zc_serd_checksum[MAX_SERDLEN];
58 	char		zc_serd_io[MAX_SERDLEN];
59 	int		zc_has_remove_timer;
60 } zfs_case_data_t;
61 
62 /*
63  * Time-of-day
64  */
65 typedef struct er_timeval {
66 	uint64_t	ertv_sec;
67 	uint64_t	ertv_nsec;
68 } er_timeval_t;
69 
70 /*
71  * In-core case structure.
72  */
73 typedef struct zfs_case {
74 	boolean_t	zc_present;
75 	uint32_t	zc_version;
76 	zfs_case_data_t	zc_data;
77 	fmd_case_t	*zc_case;
78 	uu_list_node_t	zc_node;
79 	id_t		zc_remove_timer;
80 	char		*zc_fru;
81 	er_timeval_t	zc_when;
82 } zfs_case_t;
83 
84 #define	CASE_DATA			"data"
85 #define	CASE_FRU			"fru"
86 #define	CASE_DATA_VERSION_INITIAL	1
87 #define	CASE_DATA_VERSION_SERD		2
88 
89 typedef struct zfs_de_stats {
90 	fmd_stat_t	old_drops;
91 	fmd_stat_t	dev_drops;
92 	fmd_stat_t	vdev_drops;
93 	fmd_stat_t	import_drops;
94 	fmd_stat_t	resource_drops;
95 } zfs_de_stats_t;
96 
97 zfs_de_stats_t zfs_stats = {
98 	{ "old_drops", FMD_TYPE_UINT64, "ereports dropped (from before load)" },
99 	{ "dev_drops", FMD_TYPE_UINT64, "ereports dropped (dev during open)"},
100 	{ "vdev_drops", FMD_TYPE_UINT64, "ereports dropped (weird vdev types)"},
101 	{ "import_drops", FMD_TYPE_UINT64, "ereports dropped (during import)" },
102 	{ "resource_drops", FMD_TYPE_UINT64, "resource related ereports" }
103 };
104 
105 static hrtime_t zfs_remove_timeout;
106 
107 uu_list_pool_t *zfs_case_pool;
108 uu_list_t *zfs_cases;
109 
110 #define	ZFS_MAKE_RSRC(type)	\
111     FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type
112 #define	ZFS_MAKE_EREPORT(type)	\
113     FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type
114 
115 /*
116  * Write out the persistent representation of an active case.
117  */
118 static void
119 zfs_case_serialize(fmd_hdl_t *hdl, zfs_case_t *zcp)
120 {
121 	/*
122 	 * Always update cases to the latest version, even if they were the
123 	 * previous version when unserialized.
124 	 */
125 	zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD;
126 	fmd_buf_write(hdl, zcp->zc_case, CASE_DATA, &zcp->zc_data,
127 	    sizeof (zcp->zc_data));
128 
129 	if (zcp->zc_fru != NULL)
130 		fmd_buf_write(hdl, zcp->zc_case, CASE_FRU, zcp->zc_fru,
131 		    strlen(zcp->zc_fru));
132 }
133 
134 /*
135  * Read back the persistent representation of an active case.
136  */
137 static zfs_case_t *
138 zfs_case_unserialize(fmd_hdl_t *hdl, fmd_case_t *cp)
139 {
140 	zfs_case_t *zcp;
141 	size_t frulen;
142 
143 	zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP);
144 	zcp->zc_case = cp;
145 
146 	fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data,
147 	    sizeof (zcp->zc_data));
148 
149 	if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) {
150 		fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
151 		return (NULL);
152 	}
153 
154 	if ((frulen = fmd_buf_size(hdl, zcp->zc_case, CASE_FRU)) > 0) {
155 		zcp->zc_fru = fmd_hdl_alloc(hdl, frulen + 1, FMD_SLEEP);
156 		fmd_buf_read(hdl, zcp->zc_case, CASE_FRU, zcp->zc_fru,
157 		    frulen);
158 		zcp->zc_fru[frulen] = '\0';
159 	}
160 
161 	/*
162 	 * fmd_buf_read() will have already zeroed out the remainder of the
163 	 * buffer, so we don't have to do anything special if the version
164 	 * doesn't include the SERD engine name.
165 	 */
166 
167 	if (zcp->zc_data.zc_has_remove_timer)
168 		zcp->zc_remove_timer = fmd_timer_install(hdl, zcp,
169 		    NULL, zfs_remove_timeout);
170 
171 	(void) uu_list_insert_before(zfs_cases, NULL, zcp);
172 
173 	fmd_case_setspecific(hdl, cp, zcp);
174 
175 	return (zcp);
176 }
177 
178 /*
179  * Iterate over any active cases.  If any cases are associated with a pool or
180  * vdev which is no longer present on the system, close the associated case.
181  */
182 static void
183 zfs_mark_vdev(uint64_t pool_guid, nvlist_t *vd, er_timeval_t *loaded)
184 {
185 	uint64_t vdev_guid;
186 	uint_t c, children;
187 	nvlist_t **child;
188 	zfs_case_t *zcp;
189 	int ret;
190 
191 	ret = nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid);
192 	assert(ret == 0);
193 
194 	/*
195 	 * Mark any cases associated with this (pool, vdev) pair.
196 	 */
197 	for (zcp = uu_list_first(zfs_cases); zcp != NULL;
198 	    zcp = uu_list_next(zfs_cases, zcp)) {
199 		if (zcp->zc_data.zc_pool_guid == pool_guid &&
200 		    zcp->zc_data.zc_vdev_guid == vdev_guid) {
201 			zcp->zc_present = B_TRUE;
202 			zcp->zc_when = *loaded;
203 		}
204 	}
205 
206 	/*
207 	 * Iterate over all children.
208 	 */
209 	if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child,
210 	    &children) == 0) {
211 		for (c = 0; c < children; c++)
212 			zfs_mark_vdev(pool_guid, child[c], loaded);
213 	}
214 
215 	if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child,
216 	    &children) == 0) {
217 		for (c = 0; c < children; c++)
218 			zfs_mark_vdev(pool_guid, child[c], loaded);
219 	}
220 
221 	if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child,
222 	    &children) == 0) {
223 		for (c = 0; c < children; c++)
224 			zfs_mark_vdev(pool_guid, child[c], loaded);
225 	}
226 }
227 
228 /*ARGSUSED*/
229 static int
230 zfs_mark_pool(zpool_handle_t *zhp, void *unused)
231 {
232 	zfs_case_t *zcp;
233 	uint64_t pool_guid;
234 	uint64_t *tod;
235 	er_timeval_t loaded = { 0 };
236 	nvlist_t *config, *vd;
237 	uint_t nelem = 0;
238 	int ret;
239 
240 	pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
241 	/*
242 	 * Mark any cases associated with just this pool.
243 	 */
244 	for (zcp = uu_list_first(zfs_cases); zcp != NULL;
245 	    zcp = uu_list_next(zfs_cases, zcp)) {
246 		if (zcp->zc_data.zc_pool_guid == pool_guid &&
247 		    zcp->zc_data.zc_vdev_guid == 0)
248 			zcp->zc_present = B_TRUE;
249 	}
250 
251 	if ((config = zpool_get_config(zhp, NULL)) == NULL) {
252 		zpool_close(zhp);
253 		return (-1);
254 	}
255 
256 	(void) nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
257 	    &tod, &nelem);
258 	if (nelem == 2) {
259 		loaded.ertv_sec = tod[0];
260 		loaded.ertv_nsec = tod[1];
261 		for (zcp = uu_list_first(zfs_cases); zcp != NULL;
262 		    zcp = uu_list_next(zfs_cases, zcp)) {
263 			if (zcp->zc_data.zc_pool_guid == pool_guid &&
264 			    zcp->zc_data.zc_vdev_guid == 0) {
265 				zcp->zc_when = loaded;
266 			}
267 		}
268 	}
269 
270 	ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd);
271 	assert(ret == 0);
272 
273 	zfs_mark_vdev(pool_guid, vd, &loaded);
274 
275 	zpool_close(zhp);
276 
277 	return (0);
278 }
279 
280 struct load_time_arg {
281 	uint64_t lt_guid;
282 	er_timeval_t *lt_time;
283 	boolean_t lt_found;
284 };
285 
286 static int
287 zpool_find_load_time(zpool_handle_t *zhp, void *arg)
288 {
289 	struct load_time_arg *lta = arg;
290 	uint64_t pool_guid;
291 	uint64_t *tod;
292 	nvlist_t *config;
293 	uint_t nelem;
294 
295 	if (lta->lt_found)
296 		return (0);
297 
298 	pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
299 	if (pool_guid != lta->lt_guid)
300 		return (0);
301 
302 	if ((config = zpool_get_config(zhp, NULL)) == NULL) {
303 		zpool_close(zhp);
304 		return (-1);
305 	}
306 
307 	if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
308 	    &tod, &nelem) == 0 && nelem == 2) {
309 		lta->lt_found = B_TRUE;
310 		lta->lt_time->ertv_sec = tod[0];
311 		lta->lt_time->ertv_nsec = tod[1];
312 	}
313 
314 	return (0);
315 }
316 
317 static void
318 zfs_purge_cases(fmd_hdl_t *hdl)
319 {
320 	zfs_case_t *zcp;
321 	uu_list_walk_t *walk;
322 	libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
323 
324 	/*
325 	 * There is no way to open a pool by GUID, or lookup a vdev by GUID.  No
326 	 * matter what we do, we're going to have to stomach a O(vdevs * cases)
327 	 * algorithm.  In reality, both quantities are likely so small that
328 	 * neither will matter. Given that iterating over pools is more
329 	 * expensive than iterating over the in-memory case list, we opt for a
330 	 * 'present' flag in each case that starts off cleared.  We then iterate
331 	 * over all pools, marking those that are still present, and removing
332 	 * those that aren't found.
333 	 *
334 	 * Note that we could also construct an FMRI and rely on
335 	 * fmd_nvl_fmri_present(), but this would end up doing the same search.
336 	 */
337 
338 	/*
339 	 * Mark the cases an not present.
340 	 */
341 	for (zcp = uu_list_first(zfs_cases); zcp != NULL;
342 	    zcp = uu_list_next(zfs_cases, zcp))
343 		zcp->zc_present = B_FALSE;
344 
345 	/*
346 	 * Iterate over all pools and mark the pools and vdevs found.  If this
347 	 * fails (most probably because we're out of memory), then don't close
348 	 * any of the cases and we cannot be sure they are accurate.
349 	 */
350 	if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0)
351 		return;
352 
353 	/*
354 	 * Remove those cases which were not found.
355 	 */
356 	walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
357 	while ((zcp = uu_list_walk_next(walk)) != NULL) {
358 		if (!zcp->zc_present)
359 			fmd_case_close(hdl, zcp->zc_case);
360 	}
361 	uu_list_walk_end(walk);
362 }
363 
364 /*
365  * Construct the name of a serd engine given the pool/vdev GUID and type (io or
366  * checksum).
367  */
368 static void
369 zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid,
370     const char *type)
371 {
372 	(void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s", pool_guid,
373 	    vdev_guid, type);
374 }
375 
376 /*
377  * Solve a given ZFS case.  This first checks to make sure the diagnosis is
378  * still valid, as well as cleaning up any pending timer associated with the
379  * case.
380  */
381 static void
382 zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname,
383     boolean_t checkunusable)
384 {
385 	libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
386 	nvlist_t *detector, *fault;
387 	boolean_t serialize;
388 	nvlist_t *fmri, *fru;
389 	topo_hdl_t *thp;
390 	int err;
391 
392 	/*
393 	 * Construct the detector from the case data.  The detector is in the
394 	 * ZFS scheme, and is either the pool or the vdev, depending on whether
395 	 * this is a vdev or pool fault.
396 	 */
397 	detector = fmd_nvl_alloc(hdl, FMD_SLEEP);
398 
399 	(void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
400 	(void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
401 	(void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
402 	    zcp->zc_data.zc_pool_guid);
403 	if (zcp->zc_data.zc_vdev_guid != 0) {
404 		(void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
405 		    zcp->zc_data.zc_vdev_guid);
406 	}
407 
408 	/*
409 	 * We also want to make sure that the detector (pool or vdev) properly
410 	 * reflects the diagnosed state, when the fault corresponds to internal
411 	 * ZFS state (i.e. not checksum or I/O error-induced).  Otherwise, a
412 	 * device which was unavailable early in boot (because the driver/file
413 	 * wasn't available) and is now healthy will be mis-diagnosed.
414 	 */
415 	if (!fmd_nvl_fmri_present(hdl, detector) ||
416 	    (checkunusable && !fmd_nvl_fmri_unusable(hdl, detector))) {
417 		fmd_case_close(hdl, zcp->zc_case);
418 		nvlist_free(detector);
419 		return;
420 	}
421 
422 
423 	fru = NULL;
424 	if (zcp->zc_fru != NULL &&
425 	    (thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION)) != NULL) {
426 		/*
427 		 * If the vdev had an associated FRU, then get the FRU nvlist
428 		 * from the topo handle and use that in the suspect list.  We
429 		 * explicitly lookup the FRU because the fmri reported from the
430 		 * kernel may not have up to date details about the disk itself
431 		 * (serial, part, etc).
432 		 */
433 		if (topo_fmri_str2nvl(thp, zcp->zc_fru, &fmri, &err) == 0) {
434 			/*
435 			 * If the disk is part of the system chassis, but the
436 			 * FRU indicates a different chassis ID than our
437 			 * current system, then ignore the error.  This
438 			 * indicates that the device was part of another
439 			 * cluster head, and for obvious reasons cannot be
440 			 * imported on this system.
441 			 */
442 			if (libzfs_fru_notself(zhdl, zcp->zc_fru)) {
443 				fmd_case_close(hdl, zcp->zc_case);
444 				nvlist_free(fmri);
445 				fmd_hdl_topo_rele(hdl, thp);
446 				nvlist_free(detector);
447 				return;
448 			}
449 
450 			/*
451 			 * If the device is no longer present on the system, or
452 			 * topo_fmri_fru() fails for other reasons, then fall
453 			 * back to the fmri specified in the vdev.
454 			 */
455 			if (topo_fmri_fru(thp, fmri, &fru, &err) != 0)
456 				fru = fmd_nvl_dup(hdl, fmri, FMD_SLEEP);
457 			nvlist_free(fmri);
458 		}
459 
460 		fmd_hdl_topo_rele(hdl, thp);
461 	}
462 
463 	fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
464 	    fru, detector);
465 	fmd_case_add_suspect(hdl, zcp->zc_case, fault);
466 
467 	nvlist_free(fru);
468 
469 	fmd_case_solve(hdl, zcp->zc_case);
470 
471 	serialize = B_FALSE;
472 	if (zcp->zc_data.zc_has_remove_timer) {
473 		fmd_timer_remove(hdl, zcp->zc_remove_timer);
474 		zcp->zc_data.zc_has_remove_timer = 0;
475 		serialize = B_TRUE;
476 	}
477 	if (serialize)
478 		zfs_case_serialize(hdl, zcp);
479 
480 	nvlist_free(detector);
481 }
482 
483 /*
484  * This #define and function access a private interface of the FMA
485  * framework.  Ereports include a time-of-day upper bound.
486  * We want to look at that so we can compare it to when pools get
487  * loaded.
488  */
489 #define	FMD_EVN_TOD	"__tod"
490 
491 static boolean_t
492 timeval_earlier(er_timeval_t *a, er_timeval_t *b)
493 {
494 	return (a->ertv_sec < b->ertv_sec ||
495 	    (a->ertv_sec == b->ertv_sec && a->ertv_nsec < b->ertv_nsec));
496 }
497 
498 /*ARGSUSED*/
499 static void
500 zfs_ereport_when(fmd_hdl_t *hdl, nvlist_t *nvl, er_timeval_t *when)
501 {
502 	uint64_t *tod;
503 	uint_t	nelem;
504 
505 	if (nvlist_lookup_uint64_array(nvl, FMD_EVN_TOD, &tod, &nelem) == 0 &&
506 	    nelem == 2) {
507 		when->ertv_sec = tod[0];
508 		when->ertv_nsec = tod[1];
509 	} else {
510 		when->ertv_sec = when->ertv_nsec = UINT64_MAX;
511 	}
512 }
513 
514 /*
515  * Main fmd entry point.
516  */
517 /*ARGSUSED*/
518 static void
519 zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
520 {
521 	zfs_case_t *zcp, *dcp;
522 	int32_t pool_state;
523 	uint64_t ena, pool_guid, vdev_guid;
524 	er_timeval_t pool_load;
525 	er_timeval_t er_when;
526 	nvlist_t *detector;
527 	boolean_t pool_found = B_FALSE;
528 	boolean_t isresource;
529 	char *fru, *type;
530 
531 	/*
532 	 * We subscribe to notifications for vdev or pool removal.  In these
533 	 * cases, there may be cases that no longer apply.  Purge any cases
534 	 * that no longer apply.
535 	 */
536 	if (fmd_nvl_class_match(hdl, nvl, "resource.sysevent.EC_zfs.*")) {
537 		zfs_purge_cases(hdl);
538 		zfs_stats.resource_drops.fmds_value.ui64++;
539 		return;
540 	}
541 
542 	isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*");
543 
544 	if (isresource) {
545 		/*
546 		 * For resources, we don't have a normal payload.
547 		 */
548 		if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
549 		    &vdev_guid) != 0)
550 			pool_state = SPA_LOAD_OPEN;
551 		else
552 			pool_state = SPA_LOAD_NONE;
553 		detector = NULL;
554 	} else {
555 		(void) nvlist_lookup_nvlist(nvl,
556 		    FM_EREPORT_DETECTOR, &detector);
557 		(void) nvlist_lookup_int32(nvl,
558 		    FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state);
559 	}
560 
561 	/*
562 	 * We also ignore all ereports generated during an import of a pool,
563 	 * since the only possible fault (.pool) would result in import failure,
564 	 * and hence no persistent fault.  Some day we may want to do something
565 	 * with these ereports, so we continue generating them internally.
566 	 */
567 	if (pool_state == SPA_LOAD_IMPORT) {
568 		zfs_stats.import_drops.fmds_value.ui64++;
569 		return;
570 	}
571 
572 	/*
573 	 * Device I/O errors are ignored during pool open.
574 	 */
575 	if (pool_state == SPA_LOAD_OPEN &&
576 	    (fmd_nvl_class_match(hdl, nvl,
577 	    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
578 	    fmd_nvl_class_match(hdl, nvl,
579 	    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
580 	    fmd_nvl_class_match(hdl, nvl,
581 	    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) {
582 		zfs_stats.dev_drops.fmds_value.ui64++;
583 		return;
584 	}
585 
586 	/*
587 	 * We ignore ereports for anything except disks and files.
588 	 */
589 	if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
590 	    &type) == 0) {
591 		if (strcmp(type, VDEV_TYPE_DISK) != 0 &&
592 		    strcmp(type, VDEV_TYPE_FILE) != 0) {
593 			zfs_stats.vdev_drops.fmds_value.ui64++;
594 			return;
595 		}
596 	}
597 
598 	/*
599 	 * Determine if this ereport corresponds to an open case.  Previous
600 	 * incarnations of this DE used the ENA to chain events together as
601 	 * part of the same case.  The problem with this is that we rely on
602 	 * global uniqueness of cases based on (pool_guid, vdev_guid) pair when
603 	 * generating SERD engines.  Instead, we have a case for each vdev or
604 	 * pool, regardless of the ENA.
605 	 */
606 	(void) nvlist_lookup_uint64(nvl,
607 	    FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid);
608 	if (nvlist_lookup_uint64(nvl,
609 	    FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
610 		vdev_guid = 0;
611 	if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0)
612 		ena = 0;
613 
614 	zfs_ereport_when(hdl, nvl, &er_when);
615 
616 	for (zcp = uu_list_first(zfs_cases); zcp != NULL;
617 	    zcp = uu_list_next(zfs_cases, zcp)) {
618 		if (zcp->zc_data.zc_pool_guid == pool_guid) {
619 			pool_found = B_TRUE;
620 			pool_load = zcp->zc_when;
621 		}
622 		if (zcp->zc_data.zc_vdev_guid == vdev_guid)
623 			break;
624 	}
625 
626 	if (pool_found) {
627 		fmd_hdl_debug(hdl, "pool %llx, "
628 		    "ereport time %lld.%lld, pool load time = %lld.%lld\n",
629 		    pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
630 		    pool_load.ertv_sec, pool_load.ertv_nsec);
631 	}
632 
633 	/*
634 	 * Avoid falsely accusing a pool of being faulty.  Do so by
635 	 * not replaying ereports that were generated prior to the
636 	 * current import.  If the failure that generated them was
637 	 * transient because the device was actually removed but we
638 	 * didn't receive the normal asynchronous notification, we
639 	 * don't want to mark it as faulted and potentially panic. If
640 	 * there is still a problem we'd expect not to be able to
641 	 * import the pool, or that new ereports will be generated
642 	 * once the pool is used.
643 	 */
644 	if (pool_found && timeval_earlier(&er_when, &pool_load)) {
645 		zfs_stats.old_drops.fmds_value.ui64++;
646 		return;
647 	}
648 
649 	if (!pool_found) {
650 		/*
651 		 * Haven't yet seen this pool, but same situation
652 		 * may apply.
653 		 */
654 		libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
655 		struct load_time_arg la;
656 
657 		la.lt_guid = pool_guid;
658 		la.lt_time = &pool_load;
659 		la.lt_found = B_FALSE;
660 
661 		if (zhdl != NULL &&
662 		    zpool_iter(zhdl, zpool_find_load_time, &la) == 0 &&
663 		    la.lt_found == B_TRUE) {
664 			pool_found = B_TRUE;
665 			fmd_hdl_debug(hdl, "pool %llx, "
666 			    "ereport time %lld.%lld, "
667 			    "pool load time = %lld.%lld\n",
668 			    pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
669 			    pool_load.ertv_sec, pool_load.ertv_nsec);
670 			if (timeval_earlier(&er_when, &pool_load)) {
671 				zfs_stats.old_drops.fmds_value.ui64++;
672 				return;
673 			}
674 		}
675 	}
676 
677 	if (zcp == NULL) {
678 		fmd_case_t *cs;
679 		zfs_case_data_t data = { 0 };
680 
681 		/*
682 		 * If this is one of our 'fake' resource ereports, and there is
683 		 * no case open, simply discard it.
684 		 */
685 		if (isresource) {
686 			zfs_stats.resource_drops.fmds_value.ui64++;
687 			return;
688 		}
689 
690 		/*
691 		 * Open a new case.
692 		 */
693 		cs = fmd_case_open(hdl, NULL);
694 
695 		/*
696 		 * Initialize the case buffer.  To commonize code, we actually
697 		 * create the buffer with existing data, and then call
698 		 * zfs_case_unserialize() to instantiate the in-core structure.
699 		 */
700 		fmd_buf_create(hdl, cs, CASE_DATA,
701 		    sizeof (zfs_case_data_t));
702 
703 		data.zc_version = CASE_DATA_VERSION_SERD;
704 		data.zc_ena = ena;
705 		data.zc_pool_guid = pool_guid;
706 		data.zc_vdev_guid = vdev_guid;
707 		data.zc_pool_state = (int)pool_state;
708 
709 		fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data));
710 
711 		zcp = zfs_case_unserialize(hdl, cs);
712 		assert(zcp != NULL);
713 		if (pool_found)
714 			zcp->zc_when = pool_load;
715 	}
716 
717 
718 	/*
719 	 * If this is an ereport for a case with an associated vdev FRU, make
720 	 * sure it is accurate and up to date.
721 	 */
722 	if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_FRU,
723 	    &fru) == 0) {
724 		topo_hdl_t *thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION);
725 		if (zcp->zc_fru == NULL ||
726 		    !topo_fmri_strcmp(thp, zcp->zc_fru, fru)) {
727 			if (zcp->zc_fru != NULL) {
728 				fmd_hdl_strfree(hdl, zcp->zc_fru);
729 				fmd_buf_destroy(hdl, zcp->zc_case, CASE_FRU);
730 			}
731 			zcp->zc_fru = fmd_hdl_strdup(hdl, fru, FMD_SLEEP);
732 			zfs_case_serialize(hdl, zcp);
733 		}
734 		fmd_hdl_topo_rele(hdl, thp);
735 	}
736 
737 	if (isresource) {
738 		if (fmd_nvl_class_match(hdl, nvl,
739 		    ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) {
740 			/*
741 			 * The 'resource.fs.zfs.autoreplace' event indicates
742 			 * that the pool was loaded with the 'autoreplace'
743 			 * property set.  In this case, any pending device
744 			 * failures should be ignored, as the asynchronous
745 			 * autoreplace handling will take care of them.
746 			 */
747 			fmd_case_close(hdl, zcp->zc_case);
748 		} else if (fmd_nvl_class_match(hdl, nvl,
749 		    ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) {
750 			/*
751 			 * The 'resource.fs.zfs.removed' event indicates that
752 			 * device removal was detected, and the device was
753 			 * closed asynchronously.  If this is the case, we
754 			 * assume that any recent I/O errors were due to the
755 			 * device removal, not any fault of the device itself.
756 			 * We reset the SERD engine, and cancel any pending
757 			 * timers.
758 			 */
759 			if (zcp->zc_data.zc_has_remove_timer) {
760 				fmd_timer_remove(hdl, zcp->zc_remove_timer);
761 				zcp->zc_data.zc_has_remove_timer = 0;
762 				zfs_case_serialize(hdl, zcp);
763 			}
764 			if (zcp->zc_data.zc_serd_io[0] != '\0')
765 				fmd_serd_reset(hdl,
766 				    zcp->zc_data.zc_serd_io);
767 			if (zcp->zc_data.zc_serd_checksum[0] != '\0')
768 				fmd_serd_reset(hdl,
769 				    zcp->zc_data.zc_serd_checksum);
770 		}
771 		zfs_stats.resource_drops.fmds_value.ui64++;
772 		return;
773 	}
774 
775 	/*
776 	 * Associate the ereport with this case.
777 	 */
778 	fmd_case_add_ereport(hdl, zcp->zc_case, ep);
779 
780 	/*
781 	 * Don't do anything else if this case is already solved.
782 	 */
783 	if (fmd_case_solved(hdl, zcp->zc_case))
784 		return;
785 
786 	/*
787 	 * Determine if we should solve the case and generate a fault.  We solve
788 	 * a case if:
789 	 *
790 	 * 	a. A pool failed to open (ereport.fs.zfs.pool)
791 	 * 	b. A device failed to open (ereport.fs.zfs.pool) while a pool
792 	 *	   was up and running.
793 	 *
794 	 * We may see a series of ereports associated with a pool open, all
795 	 * chained together by the same ENA.  If the pool open succeeds, then
796 	 * we'll see no further ereports.  To detect when a pool open has
797 	 * succeeded, we associate a timer with the event.  When it expires, we
798 	 * close the case.
799 	 */
800 	if (fmd_nvl_class_match(hdl, nvl,
801 	    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) {
802 		/*
803 		 * Pool level fault.  Before solving the case, go through and
804 		 * close any open device cases that may be pending.
805 		 */
806 		for (dcp = uu_list_first(zfs_cases); dcp != NULL;
807 		    dcp = uu_list_next(zfs_cases, dcp)) {
808 			if (dcp->zc_data.zc_pool_guid ==
809 			    zcp->zc_data.zc_pool_guid &&
810 			    dcp->zc_data.zc_vdev_guid != 0)
811 				fmd_case_close(hdl, dcp->zc_case);
812 		}
813 
814 		zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool", B_TRUE);
815 	} else if (fmd_nvl_class_match(hdl, nvl,
816 	    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) {
817 		/*
818 		 * Pool level fault for reading the intent logs.
819 		 */
820 		zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay", B_TRUE);
821 	} else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) {
822 		/*
823 		 * Device fault.
824 		 */
825 		zfs_case_solve(hdl, zcp, "fault.fs.zfs.device",  B_TRUE);
826 	} else if (fmd_nvl_class_match(hdl, nvl,
827 	    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
828 	    fmd_nvl_class_match(hdl, nvl,
829 	    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
830 	    fmd_nvl_class_match(hdl, nvl,
831 	    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) ||
832 	    fmd_nvl_class_match(hdl, nvl,
833 	    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
834 		char *failmode = NULL;
835 		boolean_t checkremove = B_FALSE;
836 
837 		/*
838 		 * If this is a checksum or I/O error, then toss it into the
839 		 * appropriate SERD engine and check to see if it has fired.
840 		 * Ideally, we want to do something more sophisticated,
841 		 * (persistent errors for a single data block, etc).  For now,
842 		 * a single SERD engine is sufficient.
843 		 */
844 		if (fmd_nvl_class_match(hdl, nvl,
845 		    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) {
846 			if (zcp->zc_data.zc_serd_io[0] == '\0') {
847 				zfs_serd_name(zcp->zc_data.zc_serd_io,
848 				    pool_guid, vdev_guid, "io");
849 				fmd_serd_create(hdl, zcp->zc_data.zc_serd_io,
850 				    fmd_prop_get_int32(hdl, "io_N"),
851 				    fmd_prop_get_int64(hdl, "io_T"));
852 				zfs_case_serialize(hdl, zcp);
853 			}
854 			if (fmd_serd_record(hdl, zcp->zc_data.zc_serd_io, ep))
855 				checkremove = B_TRUE;
856 		} else if (fmd_nvl_class_match(hdl, nvl,
857 		    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) {
858 			if (zcp->zc_data.zc_serd_checksum[0] == '\0') {
859 				zfs_serd_name(zcp->zc_data.zc_serd_checksum,
860 				    pool_guid, vdev_guid, "checksum");
861 				fmd_serd_create(hdl,
862 				    zcp->zc_data.zc_serd_checksum,
863 				    fmd_prop_get_int32(hdl, "checksum_N"),
864 				    fmd_prop_get_int64(hdl, "checksum_T"));
865 				zfs_case_serialize(hdl, zcp);
866 			}
867 			if (fmd_serd_record(hdl,
868 			    zcp->zc_data.zc_serd_checksum, ep)) {
869 				zfs_case_solve(hdl, zcp,
870 				    "fault.fs.zfs.vdev.checksum", B_FALSE);
871 			}
872 		} else if (fmd_nvl_class_match(hdl, nvl,
873 		    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) &&
874 		    (nvlist_lookup_string(nvl,
875 		    FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) &&
876 		    failmode != NULL) {
877 			if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE,
878 			    strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) {
879 				zfs_case_solve(hdl, zcp,
880 				    "fault.fs.zfs.io_failure_continue",
881 				    B_FALSE);
882 			} else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT,
883 			    strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) {
884 				zfs_case_solve(hdl, zcp,
885 				    "fault.fs.zfs.io_failure_wait", B_FALSE);
886 			}
887 		} else if (fmd_nvl_class_match(hdl, nvl,
888 		    ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
889 			checkremove = B_TRUE;
890 		}
891 
892 		/*
893 		 * Because I/O errors may be due to device removal, we postpone
894 		 * any diagnosis until we're sure that we aren't about to
895 		 * receive a 'resource.fs.zfs.removed' event.
896 		 */
897 		if (checkremove) {
898 			if (zcp->zc_data.zc_has_remove_timer)
899 				fmd_timer_remove(hdl, zcp->zc_remove_timer);
900 			zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL,
901 			    zfs_remove_timeout);
902 			if (!zcp->zc_data.zc_has_remove_timer) {
903 				zcp->zc_data.zc_has_remove_timer = 1;
904 				zfs_case_serialize(hdl, zcp);
905 			}
906 		}
907 	}
908 }
909 
910 /*
911  * The timeout is fired when we diagnosed an I/O error, and it was not due to
912  * device removal (which would cause the timeout to be cancelled).
913  */
914 /* ARGSUSED */
915 static void
916 zfs_fm_timeout(fmd_hdl_t *hdl, id_t id, void *data)
917 {
918 	zfs_case_t *zcp = data;
919 
920 	if (id == zcp->zc_remove_timer)
921 		zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io", B_FALSE);
922 }
923 
924 static void
925 zfs_fm_close(fmd_hdl_t *hdl, fmd_case_t *cs)
926 {
927 	zfs_case_t *zcp = fmd_case_getspecific(hdl, cs);
928 
929 	if (zcp->zc_data.zc_serd_checksum[0] != '\0')
930 		fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum);
931 	if (zcp->zc_data.zc_serd_io[0] != '\0')
932 		fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io);
933 	if (zcp->zc_data.zc_has_remove_timer)
934 		fmd_timer_remove(hdl, zcp->zc_remove_timer);
935 	uu_list_remove(zfs_cases, zcp);
936 	fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
937 }
938 
939 /*
940  * We use the fmd gc entry point to look for old cases that no longer apply.
941  * This allows us to keep our set of case data small in a long running system.
942  */
943 static void
944 zfs_fm_gc(fmd_hdl_t *hdl)
945 {
946 	zfs_purge_cases(hdl);
947 }
948 
949 static const fmd_hdl_ops_t fmd_ops = {
950 	zfs_fm_recv,	/* fmdo_recv */
951 	zfs_fm_timeout,	/* fmdo_timeout */
952 	zfs_fm_close,	/* fmdo_close */
953 	NULL,		/* fmdo_stats */
954 	zfs_fm_gc,	/* fmdo_gc */
955 };
956 
957 static const fmd_prop_t fmd_props[] = {
958 	{ "checksum_N", FMD_TYPE_UINT32, "10" },
959 	{ "checksum_T", FMD_TYPE_TIME, "10min" },
960 	{ "io_N", FMD_TYPE_UINT32, "10" },
961 	{ "io_T", FMD_TYPE_TIME, "10min" },
962 	{ "remove_timeout", FMD_TYPE_TIME, "15sec" },
963 	{ NULL, 0, NULL }
964 };
965 
966 static const fmd_hdl_info_t fmd_info = {
967 	"ZFS Diagnosis Engine", "1.0", &fmd_ops, fmd_props
968 };
969 
970 void
971 _fmd_init(fmd_hdl_t *hdl)
972 {
973 	fmd_case_t *cp;
974 	libzfs_handle_t *zhdl;
975 
976 	if ((zhdl = libzfs_init()) == NULL)
977 		return;
978 
979 	if ((zfs_case_pool = uu_list_pool_create("zfs_case_pool",
980 	    sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node),
981 	    NULL, 0)) == NULL) {
982 		libzfs_fini(zhdl);
983 		return;
984 	}
985 
986 	if ((zfs_cases = uu_list_create(zfs_case_pool, NULL, 0)) == NULL) {
987 		uu_list_pool_destroy(zfs_case_pool);
988 		libzfs_fini(zhdl);
989 		return;
990 	}
991 
992 	if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
993 		uu_list_destroy(zfs_cases);
994 		uu_list_pool_destroy(zfs_case_pool);
995 		libzfs_fini(zhdl);
996 		return;
997 	}
998 
999 	fmd_hdl_setspecific(hdl, zhdl);
1000 
1001 	(void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (zfs_stats) /
1002 	    sizeof (fmd_stat_t), (fmd_stat_t *)&zfs_stats);
1003 
1004 	/*
1005 	 * Iterate over all active cases and unserialize the associated buffers,
1006 	 * adding them to our list of open cases.
1007 	 */
1008 	for (cp = fmd_case_next(hdl, NULL);
1009 	    cp != NULL; cp = fmd_case_next(hdl, cp))
1010 		(void) zfs_case_unserialize(hdl, cp);
1011 
1012 	/*
1013 	 * Clear out any old cases that are no longer valid.
1014 	 */
1015 	zfs_purge_cases(hdl);
1016 
1017 	zfs_remove_timeout = fmd_prop_get_int64(hdl, "remove_timeout");
1018 }
1019 
1020 void
1021 _fmd_fini(fmd_hdl_t *hdl)
1022 {
1023 	zfs_case_t *zcp;
1024 	uu_list_walk_t *walk;
1025 	libzfs_handle_t *zhdl;
1026 
1027 	/*
1028 	 * Remove all active cases.
1029 	 */
1030 	walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
1031 	while ((zcp = uu_list_walk_next(walk)) != NULL) {
1032 		uu_list_remove(zfs_cases, zcp);
1033 		fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1034 	}
1035 	uu_list_walk_end(walk);
1036 
1037 	uu_list_destroy(zfs_cases);
1038 	uu_list_pool_destroy(zfs_case_pool);
1039 
1040 	zhdl = fmd_hdl_getspecific(hdl);
1041 	libzfs_fini(zhdl);
1042 }
1043