1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <assert.h> 27 #include <stddef.h> 28 #include <strings.h> 29 #include <libuutil.h> 30 #include <libzfs.h> 31 #include <fm/fmd_api.h> 32 #include <sys/fs/zfs.h> 33 #include <sys/fm/protocol.h> 34 #include <sys/fm/fs/zfs.h> 35 36 /* 37 * Our serd engines are named 'zfs_<pool_guid>_<vdev_guid>_{checksum,io}'. This 38 * #define reserves enough space for two 64-bit hex values plus the length of 39 * the longest string. 40 */ 41 #define MAX_SERDLEN (16 * 2 + sizeof ("zfs___checksum")) 42 43 /* 44 * On-disk case structure. This must maintain backwards compatibility with 45 * previous versions of the DE. By default, any members appended to the end 46 * will be filled with zeros if they don't exist in a previous version. 47 */ 48 typedef struct zfs_case_data { 49 uint64_t zc_version; 50 uint64_t zc_ena; 51 uint64_t zc_pool_guid; 52 uint64_t zc_vdev_guid; 53 int zc_has_timer; /* defunct */ 54 int zc_pool_state; 55 char zc_serd_checksum[MAX_SERDLEN]; 56 char zc_serd_io[MAX_SERDLEN]; 57 int zc_has_remove_timer; 58 } zfs_case_data_t; 59 60 /* 61 * In-core case structure. 62 */ 63 typedef struct zfs_case { 64 boolean_t zc_present; 65 uint32_t zc_version; 66 zfs_case_data_t zc_data; 67 fmd_case_t *zc_case; 68 uu_list_node_t zc_node; 69 id_t zc_remove_timer; 70 } zfs_case_t; 71 72 #define CASE_DATA "data" 73 #define CASE_DATA_VERSION_INITIAL 1 74 #define CASE_DATA_VERSION_SERD 2 75 76 static hrtime_t zfs_remove_timeout; 77 78 uu_list_pool_t *zfs_case_pool; 79 uu_list_t *zfs_cases; 80 81 #define ZFS_MAKE_RSRC(type) \ 82 FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type 83 #define ZFS_MAKE_EREPORT(type) \ 84 FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type 85 86 /* 87 * Write out the persistent representation of an active case. 88 */ 89 static void 90 zfs_case_serialize(fmd_hdl_t *hdl, zfs_case_t *zcp) 91 { 92 /* 93 * Always update cases to the latest version, even if they were the 94 * previous version when unserialized. 95 */ 96 zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD; 97 fmd_buf_write(hdl, zcp->zc_case, CASE_DATA, &zcp->zc_data, 98 sizeof (zcp->zc_data)); 99 } 100 101 /* 102 * Read back the persistent representation of an active case. 103 */ 104 static zfs_case_t * 105 zfs_case_unserialize(fmd_hdl_t *hdl, fmd_case_t *cp) 106 { 107 zfs_case_t *zcp; 108 109 zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP); 110 zcp->zc_case = cp; 111 112 fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data, 113 sizeof (zcp->zc_data)); 114 115 if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) { 116 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t)); 117 return (NULL); 118 } 119 120 /* 121 * fmd_buf_read() will have already zeroed out the remainder of the 122 * buffer, so we don't have to do anything special if the version 123 * doesn't include the SERD engine name. 124 */ 125 126 if (zcp->zc_data.zc_has_remove_timer) 127 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, 128 NULL, zfs_remove_timeout); 129 130 (void) uu_list_insert_before(zfs_cases, NULL, zcp); 131 132 fmd_case_setspecific(hdl, cp, zcp); 133 134 return (zcp); 135 } 136 137 /* 138 * Iterate over any active cases. If any cases are associated with a pool or 139 * vdev which is no longer present on the system, close the associated case. 140 */ 141 static void 142 zfs_mark_vdev(uint64_t pool_guid, nvlist_t *vd) 143 { 144 uint64_t vdev_guid; 145 uint_t c, children; 146 nvlist_t **child; 147 zfs_case_t *zcp; 148 int ret; 149 150 ret = nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid); 151 assert(ret == 0); 152 153 /* 154 * Mark any cases associated with this (pool, vdev) pair. 155 */ 156 for (zcp = uu_list_first(zfs_cases); zcp != NULL; 157 zcp = uu_list_next(zfs_cases, zcp)) { 158 if (zcp->zc_data.zc_pool_guid == pool_guid && 159 zcp->zc_data.zc_vdev_guid == vdev_guid) 160 zcp->zc_present = B_TRUE; 161 } 162 163 /* 164 * Iterate over all children. 165 */ 166 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child, 167 &children) == 0) { 168 for (c = 0; c < children; c++) 169 zfs_mark_vdev(pool_guid, child[c]); 170 } 171 172 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child, 173 &children) == 0) { 174 for (c = 0; c < children; c++) 175 zfs_mark_vdev(pool_guid, child[c]); 176 } 177 178 if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child, 179 &children) == 0) { 180 for (c = 0; c < children; c++) 181 zfs_mark_vdev(pool_guid, child[c]); 182 } 183 } 184 185 /*ARGSUSED*/ 186 static int 187 zfs_mark_pool(zpool_handle_t *zhp, void *unused) 188 { 189 zfs_case_t *zcp; 190 uint64_t pool_guid; 191 nvlist_t *config, *vd; 192 int ret; 193 194 pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL); 195 /* 196 * Mark any cases associated with just this pool. 197 */ 198 for (zcp = uu_list_first(zfs_cases); zcp != NULL; 199 zcp = uu_list_next(zfs_cases, zcp)) { 200 if (zcp->zc_data.zc_pool_guid == pool_guid && 201 zcp->zc_data.zc_vdev_guid == 0) 202 zcp->zc_present = B_TRUE; 203 } 204 205 if ((config = zpool_get_config(zhp, NULL)) == NULL) { 206 zpool_close(zhp); 207 return (-1); 208 } 209 210 ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd); 211 assert(ret == 0); 212 213 zfs_mark_vdev(pool_guid, vd); 214 215 zpool_close(zhp); 216 217 return (0); 218 } 219 220 static void 221 zfs_purge_cases(fmd_hdl_t *hdl) 222 { 223 zfs_case_t *zcp; 224 uu_list_walk_t *walk; 225 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl); 226 227 /* 228 * There is no way to open a pool by GUID, or lookup a vdev by GUID. No 229 * matter what we do, we're going to have to stomach a O(vdevs * cases) 230 * algorithm. In reality, both quantities are likely so small that 231 * neither will matter. Given that iterating over pools is more 232 * expensive than iterating over the in-memory case list, we opt for a 233 * 'present' flag in each case that starts off cleared. We then iterate 234 * over all pools, marking those that are still present, and removing 235 * those that aren't found. 236 * 237 * Note that we could also construct an FMRI and rely on 238 * fmd_nvl_fmri_present(), but this would end up doing the same search. 239 */ 240 241 /* 242 * Mark the cases an not present. 243 */ 244 for (zcp = uu_list_first(zfs_cases); zcp != NULL; 245 zcp = uu_list_next(zfs_cases, zcp)) 246 zcp->zc_present = B_FALSE; 247 248 /* 249 * Iterate over all pools and mark the pools and vdevs found. If this 250 * fails (most probably because we're out of memory), then don't close 251 * any of the cases and we cannot be sure they are accurate. 252 */ 253 if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0) 254 return; 255 256 /* 257 * Remove those cases which were not found. 258 */ 259 walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST); 260 while ((zcp = uu_list_walk_next(walk)) != NULL) { 261 if (!zcp->zc_present) 262 fmd_case_close(hdl, zcp->zc_case); 263 } 264 uu_list_walk_end(walk); 265 } 266 267 /* 268 * Construct the name of a serd engine given the pool/vdev GUID and type (io or 269 * checksum). 270 */ 271 static void 272 zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid, 273 const char *type) 274 { 275 (void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s", pool_guid, 276 vdev_guid, type); 277 } 278 279 /* 280 * Solve a given ZFS case. This first checks to make sure the diagnosis is 281 * still valid, as well as cleaning up any pending timer associated with the 282 * case. 283 */ 284 static void 285 zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname, 286 boolean_t checkunusable) 287 { 288 nvlist_t *detector, *fault; 289 boolean_t serialize; 290 291 /* 292 * Construct the detector from the case data. The detector is in the 293 * ZFS scheme, and is either the pool or the vdev, depending on whether 294 * this is a vdev or pool fault. 295 */ 296 if (nvlist_alloc(&detector, NV_UNIQUE_NAME, 0) != 0) 297 return; 298 299 if (nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0) != 0 || 300 nvlist_add_string(detector, FM_FMRI_SCHEME, 301 FM_FMRI_SCHEME_ZFS) != 0 || 302 nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL, 303 zcp->zc_data.zc_pool_guid) != 0 || 304 (zcp->zc_data.zc_vdev_guid != 0 && 305 nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV, 306 zcp->zc_data.zc_vdev_guid) != 0)) { 307 nvlist_free(detector); 308 return; 309 } 310 311 /* 312 * We also want to make sure that the detector (pool or vdev) properly 313 * reflects the diagnosed state, when the fault corresponds to internal 314 * ZFS state (i.e. not checksum or I/O error-induced). Otherwise, a 315 * device which was unavailable early in boot (because the driver/file 316 * wasn't available) and is now healthy will be mis-diagnosed. 317 */ 318 if (!fmd_nvl_fmri_present(hdl, detector) || 319 (checkunusable && !fmd_nvl_fmri_unusable(hdl, detector))) { 320 fmd_case_close(hdl, zcp->zc_case); 321 nvlist_free(detector); 322 return; 323 } 324 325 fault = fmd_nvl_create_fault(hdl, faultname, 100, detector, NULL, 326 detector); 327 fmd_case_add_suspect(hdl, zcp->zc_case, fault); 328 fmd_case_solve(hdl, zcp->zc_case); 329 330 serialize = B_FALSE; 331 if (zcp->zc_data.zc_has_remove_timer) { 332 fmd_timer_remove(hdl, zcp->zc_remove_timer); 333 zcp->zc_data.zc_has_remove_timer = 0; 334 serialize = B_TRUE; 335 } 336 if (serialize) 337 zfs_case_serialize(hdl, zcp); 338 339 nvlist_free(detector); 340 } 341 342 /* 343 * Main fmd entry point. 344 */ 345 /*ARGSUSED*/ 346 static void 347 zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class) 348 { 349 zfs_case_t *zcp, *dcp; 350 int32_t pool_state; 351 uint64_t ena, pool_guid, vdev_guid; 352 nvlist_t *detector; 353 boolean_t isresource; 354 char *type; 355 356 isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*"); 357 358 if (isresource) { 359 /* 360 * For resources, we don't have a normal payload. 361 */ 362 if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, 363 &vdev_guid) != 0) 364 pool_state = SPA_LOAD_OPEN; 365 else 366 pool_state = SPA_LOAD_NONE; 367 detector = NULL; 368 } else { 369 (void) nvlist_lookup_nvlist(nvl, 370 FM_EREPORT_DETECTOR, &detector); 371 (void) nvlist_lookup_int32(nvl, 372 FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state); 373 } 374 375 /* 376 * We also ignore all ereports generated during an import of a pool, 377 * since the only possible fault (.pool) would result in import failure, 378 * and hence no persistent fault. Some day we may want to do something 379 * with these ereports, so we continue generating them internally. 380 */ 381 if (pool_state == SPA_LOAD_IMPORT) 382 return; 383 384 /* 385 * Device I/O errors are ignored during pool open. 386 */ 387 if (pool_state == SPA_LOAD_OPEN && 388 (fmd_nvl_class_match(hdl, nvl, 389 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) || 390 fmd_nvl_class_match(hdl, nvl, 391 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) || 392 fmd_nvl_class_match(hdl, nvl, 393 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) 394 return; 395 396 /* 397 * We ignore ereports for anything except disks and files. 398 */ 399 if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE, 400 &type) == 0) { 401 if (strcmp(type, VDEV_TYPE_DISK) != 0 && 402 strcmp(type, VDEV_TYPE_FILE) != 0) 403 return; 404 } 405 406 /* 407 * Determine if this ereport corresponds to an open case. Previous 408 * incarnations of this DE used the ENA to chain events together as 409 * part of the same case. The problem with this is that we rely on 410 * global uniqueness of cases based on (pool_guid, vdev_guid) pair when 411 * generating SERD engines. Instead, we have a case for each vdev or 412 * pool, regardless of the ENA. 413 */ 414 (void) nvlist_lookup_uint64(nvl, 415 FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid); 416 if (nvlist_lookup_uint64(nvl, 417 FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0) 418 vdev_guid = 0; 419 if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0) 420 ena = 0; 421 422 for (zcp = uu_list_first(zfs_cases); zcp != NULL; 423 zcp = uu_list_next(zfs_cases, zcp)) { 424 if (zcp->zc_data.zc_pool_guid == pool_guid && 425 zcp->zc_data.zc_vdev_guid == vdev_guid) 426 break; 427 } 428 429 if (zcp == NULL) { 430 fmd_case_t *cs; 431 zfs_case_data_t data = { 0 }; 432 433 /* 434 * If this is one of our 'fake' resource ereports, and there is 435 * no case open, simply discard it. 436 */ 437 if (isresource) 438 return; 439 440 /* 441 * Open a new case. 442 */ 443 cs = fmd_case_open(hdl, NULL); 444 445 /* 446 * Initialize the case buffer. To commonize code, we actually 447 * create the buffer with existing data, and then call 448 * zfs_case_unserialize() to instantiate the in-core structure. 449 */ 450 fmd_buf_create(hdl, cs, CASE_DATA, 451 sizeof (zfs_case_data_t)); 452 453 data.zc_version = CASE_DATA_VERSION_SERD; 454 data.zc_ena = ena; 455 data.zc_pool_guid = pool_guid; 456 data.zc_vdev_guid = vdev_guid; 457 data.zc_pool_state = (int)pool_state; 458 459 fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data)); 460 461 zcp = zfs_case_unserialize(hdl, cs); 462 assert(zcp != NULL); 463 } 464 465 if (isresource) { 466 if (fmd_nvl_class_match(hdl, nvl, 467 ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) { 468 /* 469 * The 'resource.fs.zfs.autoreplace' event indicates 470 * that the pool was loaded with the 'autoreplace' 471 * property set. In this case, any pending device 472 * failures should be ignored, as the asynchronous 473 * autoreplace handling will take care of them. 474 */ 475 fmd_case_close(hdl, zcp->zc_case); 476 } else if (fmd_nvl_class_match(hdl, nvl, 477 ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) { 478 /* 479 * The 'resource.fs.zfs.removed' event indicates that 480 * device removal was detected, and the device was 481 * closed asynchronously. If this is the case, we 482 * assume that any recent I/O errors were due to the 483 * device removal, not any fault of the device itself. 484 * We reset the SERD engine, and cancel any pending 485 * timers. 486 */ 487 if (zcp->zc_data.zc_has_remove_timer) { 488 fmd_timer_remove(hdl, zcp->zc_remove_timer); 489 zcp->zc_data.zc_has_remove_timer = 0; 490 zfs_case_serialize(hdl, zcp); 491 } 492 if (zcp->zc_data.zc_serd_io[0] != '\0') 493 fmd_serd_reset(hdl, 494 zcp->zc_data.zc_serd_io); 495 if (zcp->zc_data.zc_serd_checksum[0] != '\0') 496 fmd_serd_reset(hdl, 497 zcp->zc_data.zc_serd_checksum); 498 } 499 return; 500 } 501 502 /* 503 * Associate the ereport with this case. 504 */ 505 fmd_case_add_ereport(hdl, zcp->zc_case, ep); 506 507 /* 508 * Don't do anything else if this case is already solved. 509 */ 510 if (fmd_case_solved(hdl, zcp->zc_case)) 511 return; 512 513 /* 514 * Determine if we should solve the case and generate a fault. We solve 515 * a case if: 516 * 517 * a. A pool failed to open (ereport.fs.zfs.pool) 518 * b. A device failed to open (ereport.fs.zfs.pool) while a pool 519 * was up and running. 520 * 521 * We may see a series of ereports associated with a pool open, all 522 * chained together by the same ENA. If the pool open succeeds, then 523 * we'll see no further ereports. To detect when a pool open has 524 * succeeded, we associate a timer with the event. When it expires, we 525 * close the case. 526 */ 527 if (fmd_nvl_class_match(hdl, nvl, 528 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) { 529 /* 530 * Pool level fault. Before solving the case, go through and 531 * close any open device cases that may be pending. 532 */ 533 for (dcp = uu_list_first(zfs_cases); dcp != NULL; 534 dcp = uu_list_next(zfs_cases, dcp)) { 535 if (dcp->zc_data.zc_pool_guid == 536 zcp->zc_data.zc_pool_guid && 537 dcp->zc_data.zc_vdev_guid != 0) 538 fmd_case_close(hdl, dcp->zc_case); 539 } 540 541 zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool", B_TRUE); 542 } else if (fmd_nvl_class_match(hdl, nvl, 543 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) { 544 /* 545 * Pool level fault for reading the intent logs. 546 */ 547 zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay", B_TRUE); 548 } else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) { 549 /* 550 * Device fault. 551 */ 552 zfs_case_solve(hdl, zcp, "fault.fs.zfs.device", B_TRUE); 553 } else if (fmd_nvl_class_match(hdl, nvl, 554 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) || 555 fmd_nvl_class_match(hdl, nvl, 556 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) || 557 fmd_nvl_class_match(hdl, nvl, 558 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) || 559 fmd_nvl_class_match(hdl, nvl, 560 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) { 561 char *failmode = NULL; 562 boolean_t checkremove = B_FALSE; 563 564 /* 565 * If this is a checksum or I/O error, then toss it into the 566 * appropriate SERD engine and check to see if it has fired. 567 * Ideally, we want to do something more sophisticated, 568 * (persistent errors for a single data block, etc). For now, 569 * a single SERD engine is sufficient. 570 */ 571 if (fmd_nvl_class_match(hdl, nvl, 572 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) { 573 if (zcp->zc_data.zc_serd_io[0] == '\0') { 574 zfs_serd_name(zcp->zc_data.zc_serd_io, 575 pool_guid, vdev_guid, "io"); 576 fmd_serd_create(hdl, zcp->zc_data.zc_serd_io, 577 fmd_prop_get_int32(hdl, "io_N"), 578 fmd_prop_get_int64(hdl, "io_T")); 579 zfs_case_serialize(hdl, zcp); 580 } 581 if (fmd_serd_record(hdl, zcp->zc_data.zc_serd_io, ep)) 582 checkremove = B_TRUE; 583 } else if (fmd_nvl_class_match(hdl, nvl, 584 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) { 585 if (zcp->zc_data.zc_serd_checksum[0] == '\0') { 586 zfs_serd_name(zcp->zc_data.zc_serd_checksum, 587 pool_guid, vdev_guid, "checksum"); 588 fmd_serd_create(hdl, 589 zcp->zc_data.zc_serd_checksum, 590 fmd_prop_get_int32(hdl, "checksum_N"), 591 fmd_prop_get_int64(hdl, "checksum_T")); 592 zfs_case_serialize(hdl, zcp); 593 } 594 if (fmd_serd_record(hdl, 595 zcp->zc_data.zc_serd_checksum, ep)) { 596 zfs_case_solve(hdl, zcp, 597 "fault.fs.zfs.vdev.checksum", B_FALSE); 598 } 599 } else if (fmd_nvl_class_match(hdl, nvl, 600 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) && 601 (nvlist_lookup_string(nvl, 602 FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) && 603 failmode != NULL) { 604 if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE, 605 strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) { 606 zfs_case_solve(hdl, zcp, 607 "fault.fs.zfs.io_failure_continue", 608 B_FALSE); 609 } else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT, 610 strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) { 611 zfs_case_solve(hdl, zcp, 612 "fault.fs.zfs.io_failure_wait", B_FALSE); 613 } 614 } else if (fmd_nvl_class_match(hdl, nvl, 615 ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) { 616 checkremove = B_TRUE; 617 } 618 619 /* 620 * Because I/O errors may be due to device removal, we postpone 621 * any diagnosis until we're sure that we aren't about to 622 * receive a 'resource.fs.zfs.removed' event. 623 */ 624 if (checkremove) { 625 if (zcp->zc_data.zc_has_remove_timer) 626 fmd_timer_remove(hdl, zcp->zc_remove_timer); 627 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL, 628 zfs_remove_timeout); 629 if (!zcp->zc_data.zc_has_remove_timer) { 630 zcp->zc_data.zc_has_remove_timer = 1; 631 zfs_case_serialize(hdl, zcp); 632 } 633 } 634 } 635 } 636 637 /* 638 * The timeout is fired when we diagnosed an I/O error, and it was not due to 639 * device removal (which would cause the timeout to be cancelled). 640 */ 641 /* ARGSUSED */ 642 static void 643 zfs_fm_timeout(fmd_hdl_t *hdl, id_t id, void *data) 644 { 645 zfs_case_t *zcp = data; 646 647 if (id == zcp->zc_remove_timer) 648 zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io", B_FALSE); 649 } 650 651 static void 652 zfs_fm_close(fmd_hdl_t *hdl, fmd_case_t *cs) 653 { 654 zfs_case_t *zcp = fmd_case_getspecific(hdl, cs); 655 656 if (zcp->zc_data.zc_serd_checksum[0] != '\0') 657 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum); 658 if (zcp->zc_data.zc_serd_io[0] != '\0') 659 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io); 660 if (zcp->zc_data.zc_has_remove_timer) 661 fmd_timer_remove(hdl, zcp->zc_remove_timer); 662 uu_list_remove(zfs_cases, zcp); 663 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t)); 664 } 665 666 /* 667 * We use the fmd gc entry point to look for old cases that no longer apply. 668 * This allows us to keep our set of case data small in a long running system. 669 */ 670 static void 671 zfs_fm_gc(fmd_hdl_t *hdl) 672 { 673 zfs_purge_cases(hdl); 674 } 675 676 static const fmd_hdl_ops_t fmd_ops = { 677 zfs_fm_recv, /* fmdo_recv */ 678 zfs_fm_timeout, /* fmdo_timeout */ 679 zfs_fm_close, /* fmdo_close */ 680 NULL, /* fmdo_stats */ 681 zfs_fm_gc, /* fmdo_gc */ 682 }; 683 684 static const fmd_prop_t fmd_props[] = { 685 { "checksum_N", FMD_TYPE_UINT32, "10" }, 686 { "checksum_T", FMD_TYPE_TIME, "10min" }, 687 { "io_N", FMD_TYPE_UINT32, "10" }, 688 { "io_T", FMD_TYPE_TIME, "10min" }, 689 { "remove_timeout", FMD_TYPE_TIME, "15sec" }, 690 { NULL, 0, NULL } 691 }; 692 693 static const fmd_hdl_info_t fmd_info = { 694 "ZFS Diagnosis Engine", "1.0", &fmd_ops, fmd_props 695 }; 696 697 void 698 _fmd_init(fmd_hdl_t *hdl) 699 { 700 fmd_case_t *cp; 701 libzfs_handle_t *zhdl; 702 703 if ((zhdl = libzfs_init()) == NULL) 704 return; 705 706 if ((zfs_case_pool = uu_list_pool_create("zfs_case_pool", 707 sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node), 708 NULL, 0)) == NULL) { 709 libzfs_fini(zhdl); 710 return; 711 } 712 713 if ((zfs_cases = uu_list_create(zfs_case_pool, NULL, 0)) == NULL) { 714 uu_list_pool_destroy(zfs_case_pool); 715 libzfs_fini(zhdl); 716 return; 717 } 718 719 if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) { 720 uu_list_destroy(zfs_cases); 721 uu_list_pool_destroy(zfs_case_pool); 722 libzfs_fini(zhdl); 723 return; 724 } 725 726 fmd_hdl_setspecific(hdl, zhdl); 727 728 /* 729 * Iterate over all active cases and unserialize the associated buffers, 730 * adding them to our list of open cases. 731 */ 732 for (cp = fmd_case_next(hdl, NULL); 733 cp != NULL; cp = fmd_case_next(hdl, cp)) 734 (void) zfs_case_unserialize(hdl, cp); 735 736 /* 737 * Clear out any old cases that are no longer valid. 738 */ 739 zfs_purge_cases(hdl); 740 741 zfs_remove_timeout = fmd_prop_get_int64(hdl, "remove_timeout"); 742 } 743 744 void 745 _fmd_fini(fmd_hdl_t *hdl) 746 { 747 zfs_case_t *zcp; 748 uu_list_walk_t *walk; 749 libzfs_handle_t *zhdl; 750 751 /* 752 * Remove all active cases. 753 */ 754 walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST); 755 while ((zcp = uu_list_walk_next(walk)) != NULL) { 756 uu_list_remove(zfs_cases, zcp); 757 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t)); 758 } 759 uu_list_walk_end(walk); 760 761 uu_list_destroy(zfs_cases); 762 uu_list_pool_destroy(zfs_case_pool); 763 764 zhdl = fmd_hdl_getspecific(hdl); 765 libzfs_fini(zhdl); 766 } 767