1 /*-
2  * Copyright (c) 2012, 2013, 2014 Spectra Logic Corporation
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions, and the following disclaimer,
10  *    without modification.
11  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
12  *    substantially similar to the "NO WARRANTY" disclaimer below
13  *    ("Disclaimer") and any redistribution must be conditioned upon
14  *    including a substantially similar Disclaimer requirement for further
15  *    binary redistribution.
16  *
17  * NO WARRANTY
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
27  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28  * POSSIBILITY OF SUCH DAMAGES.
29  *
30  * Authors: Alan Somers         (Spectra Logic Corporation)
31  */
32 #include <sys/cdefs.h>
33 #include <sys/byteorder.h>
34 
35 #include <stdarg.h>
36 #include <syslog.h>
37 
38 #include <libnvpair.h>
39 #include <libzfs.h>
40 
41 #include <list>
42 #include <map>
43 #include <sstream>
44 #include <string>
45 
46 #include <gmock/gmock.h>
47 #include <gtest/gtest.h>
48 
49 #include <devdctl/guid.h>
50 #include <devdctl/event.h>
51 #include <devdctl/event_factory.h>
52 #include <devdctl/exception.h>
53 #include <devdctl/consumer.h>
54 
55 #include <zfsd/callout.h>
56 #include <zfsd/vdev_iterator.h>
57 #include <zfsd/zfsd_event.h>
58 #include <zfsd/case_file.h>
59 #include <zfsd/vdev.h>
60 #include <zfsd/zfsd.h>
61 #include <zfsd/zfsd_exception.h>
62 #include <zfsd/zpool_list.h>
63 
64 #include "libmocks.h"
65 /*================================== Macros ==================================*/
66 #define	NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x))
67 
68 /*============================ Namespace Control =============================*/
69 using std::string;
70 using std::stringstream;
71 
72 using DevdCtl::Event;
73 using DevdCtl::EventFactory;
74 using DevdCtl::EventList;
75 using DevdCtl::Guid;
76 using DevdCtl::NVPairMap;
77 
78 /* redefine zpool_handle here because libzfs_impl.h is not includable */
79 struct zpool_handle
80 {
81         libzfs_handle_t *zpool_hdl;
82         zpool_handle_t *zpool_next;
83         char zpool_name[ZFS_MAX_DATASET_NAME_LEN];
84         int zpool_state;
85         size_t zpool_config_size;
86         nvlist_t *zpool_config;
87         nvlist_t *zpool_old_config;
88         nvlist_t *zpool_props;
89         diskaddr_t zpool_start_block;
90 };
91 
92 class MockZfsEvent : public ZfsEvent
93 {
94 public:
95 	MockZfsEvent(Event::Type, NVPairMap&, const string&);
96 	virtual ~MockZfsEvent() {}
97 
98 	static BuildMethod MockZfsEventBuilder;
99 
100 	MOCK_CONST_METHOD0(ProcessPoolEvent, void());
101 
102 	static EventFactory::Record s_buildRecords[];
103 };
104 
105 EventFactory::Record MockZfsEvent::s_buildRecords[] =
106 {
107         { Event::NOTIFY, "ZFS", &MockZfsEvent::MockZfsEventBuilder }
108 };
109 
110 MockZfsEvent::MockZfsEvent(Event::Type type, NVPairMap& map,
111 			   const string& str)
112  : ZfsEvent(type, map, str)
113 {
114 }
115 
116 Event *
117 MockZfsEvent::MockZfsEventBuilder(Event::Type type,
118 				  NVPairMap &nvpairs,
119 			  	  const string &eventString)
120 {
121 	return (new MockZfsEvent(type, nvpairs, eventString));
122 }
123 
124 /*
125  * A dummy Vdev class used for testing other classes
126  */
127 class MockVdev : public Vdev
128 {
129 public:
130 	MockVdev(nvlist_t *vdevConfig);
131 	virtual ~MockVdev() {}
132 
133 	MOCK_CONST_METHOD0(GUID, Guid());
134 	MOCK_CONST_METHOD0(PoolGUID, Guid());
135 	MOCK_CONST_METHOD0(State, vdev_state());
136 	MOCK_CONST_METHOD0(PhysicalPath, string());
137 	MOCK_CONST_METHOD2(Name, string(zpool_handle_t * zhp, bool verbose));
138 };
139 
140 MockVdev::MockVdev(nvlist_t *vdevConfig)
141  : Vdev(vdevConfig)
142 {
143 }
144 
145 /*
146  * A CaseFile class with side effects removed, for testing
147  */
148 class TestableCaseFile : public CaseFile
149 {
150 public:
151 	static TestableCaseFile &Create(Vdev &vdev);
152 	TestableCaseFile(Vdev &vdev);
153 	virtual ~TestableCaseFile() {}
154 
155 	MOCK_METHOD0(Close, void());
156 	MOCK_METHOD1(RegisterCallout, void(const Event &event));
157 	MOCK_METHOD0(RefreshVdevState, bool());
158 	MOCK_METHOD1(ReEvaluate, bool(const ZfsEvent &event));
159 
160 	bool RealReEvaluate(const ZfsEvent &event)
161 	{
162 		return (CaseFile::ReEvaluate(event));
163 	}
164 
165 	/*
166 	 * This splices the event lists, a procedure that would normally be done
167 	 * by OnGracePeriodEnded, but we don't necessarily call that in the
168 	 * unit tests
169 	 */
170 	void SpliceEvents();
171 
172 	/*
173 	 * Used by some of our expectations.  CaseFile does not publicize this
174 	 */
175 	static int getActiveCases()
176 	{
177 		return (s_activeCases.size());
178 	}
179 };
180 
181 TestableCaseFile::TestableCaseFile(Vdev &vdev)
182  : CaseFile(vdev)
183 {
184 }
185 
186 TestableCaseFile &
187 TestableCaseFile::Create(Vdev &vdev)
188 {
189 	TestableCaseFile *newCase;
190 	newCase = new TestableCaseFile(vdev);
191 	return (*newCase);
192 }
193 
194 void
195 TestableCaseFile::SpliceEvents()
196 {
197 	m_events.splice(m_events.begin(), m_tentativeEvents);
198 }
199 
200 
201 /*
202  * Test class ZfsdException
203  */
204 class ZfsdExceptionTest : public ::testing::Test
205 {
206 protected:
207 	virtual void SetUp()
208 	{
209 		ASSERT_EQ(0, nvlist_alloc(&poolConfig, NV_UNIQUE_NAME, 0));
210 		ASSERT_EQ(0, nvlist_add_string(poolConfig,
211 				ZPOOL_CONFIG_POOL_NAME, "unit_test_pool"));
212 		ASSERT_EQ(0, nvlist_add_uint64(poolConfig,
213 				ZPOOL_CONFIG_POOL_GUID, 0x1234));
214 
215 		ASSERT_EQ(0, nvlist_alloc(&vdevConfig, NV_UNIQUE_NAME, 0));
216 		ASSERT_EQ(0, nvlist_add_uint64(vdevConfig,
217 				ZPOOL_CONFIG_GUID, 0x5678));
218 		bzero(&poolHandle, sizeof(poolHandle));
219 		poolHandle.zpool_config = poolConfig;
220 	}
221 
222 	virtual void TearDown()
223 	{
224 		nvlist_free(poolConfig);
225 		nvlist_free(vdevConfig);
226 	}
227 
228 	nvlist_t	*poolConfig;
229 	nvlist_t	*vdevConfig;
230 	zpool_handle_t   poolHandle;
231 };
232 
233 TEST_F(ZfsdExceptionTest, StringConstructorNull)
234 {
235 	ZfsdException ze("");
236 	EXPECT_STREQ("", ze.GetString().c_str());
237 }
238 
239 TEST_F(ZfsdExceptionTest, StringConstructorFormatted)
240 {
241 	ZfsdException ze(" %d %s", 55, "hello world");
242 	EXPECT_STREQ(" 55 hello world", ze.GetString().c_str());
243 }
244 
245 TEST_F(ZfsdExceptionTest, LogSimple)
246 {
247 	ZfsdException ze("unit test w/o vdev or pool");
248 	ze.Log();
249 	EXPECT_EQ(LOG_ERR, syslog_last_priority);
250 	EXPECT_STREQ("unit test w/o vdev or pool\n", syslog_last_message);
251 }
252 
253 TEST_F(ZfsdExceptionTest, Pool)
254 {
255 	const char msg[] = "Exception with pool name";
256 	char expected[4096];
257 	sprintf(expected, "Pool unit_test_pool: %s\n", msg);
258 	ZfsdException ze(poolConfig, msg);
259 	ze.Log();
260 	EXPECT_STREQ(expected, syslog_last_message);
261 }
262 
263 TEST_F(ZfsdExceptionTest, PoolHandle)
264 {
265 	const char msg[] = "Exception with pool handle";
266 	char expected[4096];
267 	sprintf(expected, "Pool unit_test_pool: %s\n", msg);
268 	ZfsdException ze(&poolHandle, msg);
269 	ze.Log();
270 	EXPECT_STREQ(expected, syslog_last_message);
271 }
272 
273 /*
274  * Test class Vdev
275  */
276 class VdevTest : public ::testing::Test
277 {
278 protected:
279 	virtual void SetUp()
280 	{
281 		ASSERT_EQ(0, nvlist_alloc(&m_poolConfig, NV_UNIQUE_NAME, 0));
282 		ASSERT_EQ(0, nvlist_add_uint64(m_poolConfig,
283 					       ZPOOL_CONFIG_POOL_GUID,
284 					       0x1234));
285 
286 		ASSERT_EQ(0, nvlist_alloc(&m_vdevConfig, NV_UNIQUE_NAME, 0));
287 		ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig, ZPOOL_CONFIG_GUID,
288 					       0x5678));
289 	}
290 
291 	virtual void TearDown()
292 	{
293 		nvlist_free(m_poolConfig);
294 		nvlist_free(m_vdevConfig);
295 	}
296 
297 	nvlist_t	*m_poolConfig;
298 	nvlist_t	*m_vdevConfig;
299 };
300 
301 
302 TEST_F(VdevTest, StateFromConfig)
303 {
304 	vdev_stat_t vs;
305 
306 	vs.vs_state = VDEV_STATE_OFFLINE;
307 
308 	ASSERT_EQ(0, nvlist_add_uint64_array(m_vdevConfig,
309 					     ZPOOL_CONFIG_VDEV_STATS,
310 					     (uint64_t*)&vs,
311 					     sizeof(vs) / sizeof(uint64_t)));
312 
313 	Vdev vdev(m_poolConfig, m_vdevConfig);
314 
315 	EXPECT_EQ(VDEV_STATE_OFFLINE, vdev.State());
316 }
317 
318 TEST_F(VdevTest, StateFaulted)
319 {
320 	ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig, ZPOOL_CONFIG_FAULTED, 1));
321 
322 	Vdev vdev(m_poolConfig, m_vdevConfig);
323 
324 	EXPECT_EQ(VDEV_STATE_FAULTED, vdev.State());
325 }
326 
327 /*
328  * Test that we can construct a Vdev from the label information that is stored
329  * on an available spare drive
330  */
331 TEST_F(VdevTest, ConstructAvailSpare)
332 {
333 	nvlist_t	*labelConfig;
334 
335 	ASSERT_EQ(0, nvlist_alloc(&labelConfig, NV_UNIQUE_NAME, 0));
336 	ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_GUID,
337 				       1948339428197961030));
338 	ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_POOL_STATE,
339 				       POOL_STATE_SPARE));
340 
341 	EXPECT_NO_THROW(Vdev vdev(labelConfig));
342 
343 	nvlist_free(labelConfig);
344 }
345 
346 /* Available spares will always show the HEALTHY state */
347 TEST_F(VdevTest, AvailSpareState) {
348 	nvlist_t	*labelConfig;
349 
350 	ASSERT_EQ(0, nvlist_alloc(&labelConfig, NV_UNIQUE_NAME, 0));
351 	ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_GUID,
352 				       1948339428197961030));
353 	ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_POOL_STATE,
354 				       POOL_STATE_SPARE));
355 
356 	Vdev vdev(labelConfig);
357 	EXPECT_EQ(VDEV_STATE_HEALTHY, vdev.State());
358 
359 	nvlist_free(labelConfig);
360 }
361 
362 /* Test the Vdev::IsSpare method */
363 TEST_F(VdevTest, IsSpare) {
364 	Vdev notSpare(m_poolConfig, m_vdevConfig);
365 	EXPECT_EQ(false, notSpare.IsSpare());
366 
367 	ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig, ZPOOL_CONFIG_IS_SPARE, 1));
368 	Vdev isSpare(m_poolConfig, m_vdevConfig);
369 	EXPECT_EQ(true, isSpare.IsSpare());
370 }
371 
372 /*
373  * Test class ZFSEvent
374  */
375 class ZfsEventTest : public ::testing::Test
376 {
377 protected:
378 	virtual void SetUp()
379 	{
380 		m_eventFactory = new EventFactory();
381 		m_eventFactory->UpdateRegistry(MockZfsEvent::s_buildRecords,
382 		    NUM_ELEMENTS(MockZfsEvent::s_buildRecords));
383 
384 		m_event = NULL;
385 	}
386 
387 	virtual void TearDown()
388 	{
389 		delete m_eventFactory;
390 		delete m_event;
391 	}
392 
393 	EventFactory	*m_eventFactory;
394 	Event		*m_event;
395 };
396 
397 TEST_F(ZfsEventTest, ProcessPoolEventGetsCalled)
398 {
399 	string evString("!system=ZFS "
400 			"subsystem=ZFS "
401 			"type=sysevent.fs.zfs.vdev_remove "
402 			"pool_name=foo "
403 			"pool_guid=9756779504028057996 "
404 			"vdev_guid=1631193447431603339 "
405 			"vdev_path=/dev/da1 "
406 			"timestamp=1348871594");
407 	m_event = Event::CreateEvent(*m_eventFactory, evString);
408 	MockZfsEvent *mock_event = static_cast<MockZfsEvent*>(m_event);
409 
410 	EXPECT_CALL(*mock_event, ProcessPoolEvent()).Times(1);
411 	mock_event->Process();
412 }
413 
414 /*
415  * Test class CaseFile
416  */
417 
418 class CaseFileTest : public ::testing::Test
419 {
420 protected:
421 	virtual void SetUp()
422 	{
423 		m_eventFactory = new EventFactory();
424 		m_eventFactory->UpdateRegistry(MockZfsEvent::s_buildRecords,
425 		    NUM_ELEMENTS(MockZfsEvent::s_buildRecords));
426 
427 		m_event = NULL;
428 
429 		nvlist_alloc(&m_vdevConfig, NV_UNIQUE_NAME, 0);
430 		ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig,
431 					       ZPOOL_CONFIG_GUID, 0xbeef));
432 		m_vdev = new MockVdev(m_vdevConfig);
433 		ON_CALL(*m_vdev, GUID())
434 		    .WillByDefault(::testing::Return(Guid(123)));
435 		ON_CALL(*m_vdev, Name(::testing::_, ::testing::_))
436 		    .WillByDefault(::testing::Return(string("/dev/da999")));
437 		ON_CALL(*m_vdev, PoolGUID())
438 		    .WillByDefault(::testing::Return(Guid(456)));
439 		ON_CALL(*m_vdev, State())
440 		    .WillByDefault(::testing::Return(VDEV_STATE_HEALTHY));
441 		m_caseFile = &TestableCaseFile::Create(*m_vdev);
442 		ON_CALL(*m_caseFile, ReEvaluate(::testing::_))
443 		    .WillByDefault(::testing::Invoke(m_caseFile, &TestableCaseFile::RealReEvaluate));
444 		return;
445 	}
446 
447 	virtual void TearDown()
448 	{
449 		delete m_caseFile;
450 		nvlist_free(m_vdevConfig);
451 		delete m_vdev;
452 		delete m_event;
453 		delete m_eventFactory;
454 	}
455 
456 	nvlist_t		*m_vdevConfig;
457 	MockVdev		*m_vdev;
458 	TestableCaseFile 	*m_caseFile;
459 	Event			*m_event;
460 	EventFactory		*m_eventFactory;
461 };
462 
463 /*
464  * A Vdev with no events should not be degraded or faulted
465  */
466 TEST_F(CaseFileTest, HealthyVdev)
467 {
468 	EXPECT_FALSE(m_caseFile->ShouldDegrade());
469 	EXPECT_FALSE(m_caseFile->ShouldFault());
470 }
471 
472 /*
473  * A Vdev with only one event should not be degraded or faulted
474  * For performance reasons, RefreshVdevState should not be called.
475  */
476 TEST_F(CaseFileTest, HealthyishVdev)
477 {
478 	string evString("!system=ZFS "
479 			"class=ereport.fs.zfs.io "
480 			"ena=12091638756982918145 "
481 			"parent_guid=13237004955564865395 "
482 			"parent_type=raidz "
483 			"pool=testpool.4415 "
484 			"pool_context=0 "
485 			"pool_failmode=wait "
486 			"pool_guid=456 "
487 			"subsystem=ZFS "
488 			"timestamp=1348867914 "
489 			"type=ereport.fs.zfs.io "
490 			"vdev_guid=123 "
491 			"vdev_path=/dev/da400 "
492 			"vdev_type=disk "
493 			"zio_blkid=622 "
494 			"zio_err=1 "
495 			"zio_level=-2 "
496 			"zio_object=0 "
497 			"zio_objset=37 "
498 			"zio_offset=25598976 "
499 			"zio_size=1024");
500 	m_event = Event::CreateEvent(*m_eventFactory, evString);
501 	ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
502 
503 	EXPECT_CALL(*m_caseFile, RefreshVdevState())
504 	    .Times(::testing::Exactly(0));
505 	EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
506 	EXPECT_FALSE(m_caseFile->ShouldDegrade());
507 	EXPECT_FALSE(m_caseFile->ShouldFault());
508 }
509 
510 /* The case file should be closed when its pool is destroyed */
511 TEST_F(CaseFileTest, PoolDestroy)
512 {
513 	string evString("!system=ZFS "
514 			"pool_name=testpool.4415 "
515 			"pool_guid=456 "
516 			"subsystem=ZFS "
517 			"timestamp=1348867914 "
518 			"type=sysevent.fs.zfs.pool_destroy ");
519 	m_event = Event::CreateEvent(*m_eventFactory, evString);
520 	ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
521 	EXPECT_CALL(*m_caseFile, Close());
522 	EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
523 }
524 
525 /*
526  * A Vdev with a very large number of Delay errors should fault
527  * For performance reasons, RefreshVdevState should be called at most once
528  */
529 TEST_F(CaseFileTest, VeryManyDelayErrors)
530 {
531 	EXPECT_CALL(*m_caseFile, RefreshVdevState())
532 	    .Times(::testing::AtMost(1))
533 	    .WillRepeatedly(::testing::Return(true));
534 
535 	for(int i=0; i<100; i++) {
536 		stringstream evStringStream;
537 		evStringStream <<
538 			"!system=ZFS "
539 			"class=ereport.fs.zfs.delay "
540 			"ena=12091638756982918145 "
541 			"parent_guid=13237004955564865395 "
542 			"parent_type=raidz "
543 			"pool=testpool.4415 "
544 			"pool_context=0 "
545 			"pool_failmode=wait "
546 			"pool_guid=456 "
547 			"pool_state= 0"
548 			"subsystem=ZFS "
549 			"time=";
550 		evStringStream << i << "0000000000000000 ";
551 		evStringStream << "timestamp=" << i << " ";
552 		evStringStream <<
553 			"type=ereport.fs.zfs.delay "
554 			"vdev_ashift=12 "
555 			"vdev_cksum_errors=0 "
556 			"vdev_complete_ts=948336226469 "
557 			"vdev_delays=77 "
558 			"vdev_delta_ts=123998485899 "
559 			"vdev_guid=123 "
560 			"vdev_path=/dev/da400 "
561 			"vdev_read_errors=0 "
562 			"vdev_spare_guids= "
563 			"vdev_type=disk "
564 			"vdev_write_errors=0 "
565 			"zio_blkid=622 "
566 			"zio_delay=31000041101 "
567 			"zio_delta=123998485899 "
568 			"zio_err=0 "
569 			"zio_flags=1572992 "
570 			"zio_level=-2 "
571 			"zio_object=0 "
572 			"zio_objset=37 "
573 			"zio_offset=25598976 "
574 			"zio_pipeline=48234496 "
575 			"zio_priority=3 "
576 			"zio_size=1024"
577 			"zio_stage=33554432 "
578 			"zio_timestamp=824337740570 ";
579 		Event *event(Event::CreateEvent(*m_eventFactory,
580 						evStringStream.str()));
581 		ZfsEvent *zfs_event = static_cast<ZfsEvent*>(event);
582 		EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
583 		delete event;
584 	}
585 
586 	m_caseFile->SpliceEvents();
587 	EXPECT_FALSE(m_caseFile->ShouldDegrade());
588 	EXPECT_TRUE(m_caseFile->ShouldFault());
589 }
590 
591 /*
592  * A Vdev with a very large number of IO errors should fault
593  * For performance reasons, RefreshVdevState should be called at most once
594  */
595 TEST_F(CaseFileTest, VeryManyIOErrors)
596 {
597 	EXPECT_CALL(*m_caseFile, RefreshVdevState())
598 	    .Times(::testing::AtMost(1))
599 	    .WillRepeatedly(::testing::Return(true));
600 
601 	for(int i=0; i<100; i++) {
602 		stringstream evStringStream;
603 		evStringStream <<
604 			"!system=ZFS "
605 			"class=ereport.fs.zfs.io "
606 			"ena=12091638756982918145 "
607 			"parent_guid=13237004955564865395 "
608 			"parent_type=raidz "
609 			"pool=testpool.4415 "
610 			"pool_context=0 "
611 			"pool_failmode=wait "
612 			"pool_guid=456 "
613 			"subsystem=ZFS "
614 			"timestamp=";
615 		evStringStream << i << " ";
616 		evStringStream <<
617 			"type=ereport.fs.zfs.io "
618 			"vdev_guid=123 "
619 			"vdev_path=/dev/da400 "
620 			"vdev_type=disk "
621 			"zio_blkid=622 "
622 			"zio_err=1 "
623 			"zio_level=-2 "
624 			"zio_object=0 "
625 			"zio_objset=37 "
626 			"zio_offset=25598976 "
627 			"zio_size=1024";
628 		Event *event(Event::CreateEvent(*m_eventFactory,
629 						evStringStream.str()));
630 		ZfsEvent *zfs_event = static_cast<ZfsEvent*>(event);
631 		EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
632 		delete event;
633 	}
634 
635 	m_caseFile->SpliceEvents();
636 	EXPECT_FALSE(m_caseFile->ShouldDegrade());
637 	EXPECT_TRUE(m_caseFile->ShouldFault());
638 }
639 
640 /*
641  * A Vdev with a very large number of checksum errors should degrade
642  * For performance reasons, RefreshVdevState should be called at most once
643  */
644 TEST_F(CaseFileTest, VeryManyChecksumErrors)
645 {
646 	EXPECT_CALL(*m_caseFile, RefreshVdevState())
647 	    .Times(::testing::AtMost(1))
648 	    .WillRepeatedly(::testing::Return(true));
649 
650 	for(int i=0; i<100; i++) {
651 		stringstream evStringStream;
652 		evStringStream <<
653 			"!system=ZFS "
654 			"bad_cleared_bits=03000000000000803f50b00000000000 "
655 			"bad_range_clears=0000000e "
656 			"bad_range_sets=00000000 "
657 			"bad_ranges=0000000000000010 "
658 			"bad_ranges_min_gap=8 "
659 			"bad_set_bits=00000000000000000000000000000000 "
660 			"class=ereport.fs.zfs.checksum "
661 			"ena=12272856582652437505 "
662 			"parent_guid=5838204195352909894 "
663 			"parent_type=raidz pool=testpool.7640 "
664 			"pool_context=0 "
665 			"pool_failmode=wait "
666 			"pool_guid=456 "
667 			"subsystem=ZFS timestamp=";
668 		evStringStream << i << " ";
669 		evStringStream <<
670 			"type=ereport.fs.zfs.checksum "
671 			"vdev_guid=123 "
672 			"vdev_path=/mnt/tmp/file1.7702 "
673 			"vdev_type=file "
674 			"zio_blkid=0 "
675 			"zio_err=0 "
676 			"zio_level=0 "
677 			"zio_object=3 "
678 			"zio_objset=0 "
679 			"zio_offset=16896 "
680 			"zio_size=512";
681 		Event *event(Event::CreateEvent(*m_eventFactory,
682 						evStringStream.str()));
683 		ZfsEvent *zfs_event = static_cast<ZfsEvent*>(event);
684 		EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
685 		delete event;
686 	}
687 
688 	m_caseFile->SpliceEvents();
689 	EXPECT_TRUE(m_caseFile->ShouldDegrade());
690 	EXPECT_FALSE(m_caseFile->ShouldFault());
691 }
692 
693 /*
694  * Test CaseFile::ReEvaluateByGuid
695  */
696 class ReEvaluateByGuidTest : public ::testing::Test
697 {
698 protected:
699 	virtual void SetUp()
700 	{
701 		m_eventFactory = new EventFactory();
702 		m_eventFactory->UpdateRegistry(MockZfsEvent::s_buildRecords,
703 		    NUM_ELEMENTS(MockZfsEvent::s_buildRecords));
704 		m_event = Event::CreateEvent(*m_eventFactory, s_evString);
705 		nvlist_alloc(&m_vdevConfig, NV_UNIQUE_NAME, 0);
706 		ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig,
707 					       ZPOOL_CONFIG_GUID, 0xbeef));
708 		m_vdev456 = new ::testing::NiceMock<MockVdev>(m_vdevConfig);
709 		m_vdev789 = new ::testing::NiceMock<MockVdev>(m_vdevConfig);
710 		ON_CALL(*m_vdev456, GUID())
711 		    .WillByDefault(::testing::Return(Guid(123)));
712 		ON_CALL(*m_vdev456, PoolGUID())
713 		    .WillByDefault(::testing::Return(Guid(456)));
714 		ON_CALL(*m_vdev456, State())
715 		    .WillByDefault(::testing::Return(VDEV_STATE_HEALTHY));
716 		ON_CALL(*m_vdev789, GUID())
717 		    .WillByDefault(::testing::Return(Guid(123)));
718 		ON_CALL(*m_vdev789, PoolGUID())
719 		    .WillByDefault(::testing::Return(Guid(789)));
720 		ON_CALL(*m_vdev789, State())
721 		    .WillByDefault(::testing::Return(VDEV_STATE_HEALTHY));
722 		m_caseFile456 = NULL;
723 		m_caseFile789 = NULL;
724 		return;
725 	}
726 
727 	virtual void TearDown()
728 	{
729 		delete m_caseFile456;
730 		delete m_caseFile789;
731 		nvlist_free(m_vdevConfig);
732 		delete m_vdev456;
733 		delete m_vdev789;
734 		delete m_event;
735 		delete m_eventFactory;
736 	}
737 
738 	static string			 s_evString;
739 	nvlist_t			*m_vdevConfig;
740 	::testing::NiceMock<MockVdev>	*m_vdev456;
741 	::testing::NiceMock<MockVdev>	*m_vdev789;
742 	TestableCaseFile 		*m_caseFile456;
743 	TestableCaseFile 		*m_caseFile789;
744 	Event				*m_event;
745 	EventFactory			*m_eventFactory;
746 };
747 
748 string ReEvaluateByGuidTest::s_evString(
749 	"!system=ZFS "
750 	"pool_guid=16271873792808333580 "
751 	"pool_name=foo "
752 	"subsystem=ZFS "
753 	"timestamp=1360620391 "
754 	"type=sysevent.fs.zfs.config_sync");
755 
756 
757 /*
758  * Test the ReEvaluateByGuid method on an empty list of casefiles.
759  * We must create one event, even though it never gets used, because it will
760  * be passed by reference to ReEvaluateByGuid
761  */
762 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_empty)
763 {
764 	ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
765 
766 	EXPECT_EQ(0, TestableCaseFile::getActiveCases());
767 	CaseFile::ReEvaluateByGuid(Guid(456), *zfs_event);
768 	EXPECT_EQ(0, TestableCaseFile::getActiveCases());
769 }
770 
771 /*
772  * Test the ReEvaluateByGuid method on a list of CaseFiles that contains only
773  * one CaseFile, which doesn't match the criteria
774  */
775 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_oneFalse)
776 {
777 	m_caseFile456 = &TestableCaseFile::Create(*m_vdev456);
778 	ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
779 
780 	EXPECT_EQ(1, TestableCaseFile::getActiveCases());
781 	EXPECT_CALL(*m_caseFile456, ReEvaluate(::testing::_))
782 	    .Times(::testing::Exactly(0));
783 	CaseFile::ReEvaluateByGuid(Guid(789), *zfs_event);
784 	EXPECT_EQ(1, TestableCaseFile::getActiveCases());
785 }
786 
787 /*
788  * Test the ReEvaluateByGuid method on a list of CaseFiles that contains only
789  * one CaseFile, which does match the criteria
790  */
791 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_oneTrue)
792 {
793 	m_caseFile456 = &TestableCaseFile::Create(*m_vdev456);
794 	ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
795 
796 	EXPECT_EQ(1, TestableCaseFile::getActiveCases());
797 	EXPECT_CALL(*m_caseFile456, ReEvaluate(::testing::_))
798 	    .Times(::testing::Exactly(1))
799 	    .WillRepeatedly(::testing::Return(false));
800 	CaseFile::ReEvaluateByGuid(Guid(456), *zfs_event);
801 	EXPECT_EQ(1, TestableCaseFile::getActiveCases());
802 }
803 
804 /*
805  * Test the ReEvaluateByGuid method on a long list of CaseFiles that contains a
806  * few cases which meet the criteria
807  */
808 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_five)
809 {
810 	TestableCaseFile *CaseFile1 = &TestableCaseFile::Create(*m_vdev456);
811 	TestableCaseFile *CaseFile2 = &TestableCaseFile::Create(*m_vdev789);
812 	TestableCaseFile *CaseFile3 = &TestableCaseFile::Create(*m_vdev456);
813 	TestableCaseFile *CaseFile4 = &TestableCaseFile::Create(*m_vdev789);
814 	TestableCaseFile *CaseFile5 = &TestableCaseFile::Create(*m_vdev789);
815 	ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
816 
817 	EXPECT_EQ(5, TestableCaseFile::getActiveCases());
818 	EXPECT_CALL(*CaseFile1, ReEvaluate(::testing::_))
819 	    .Times(::testing::Exactly(1))
820 	    .WillRepeatedly(::testing::Return(false));
821 	EXPECT_CALL(*CaseFile3, ReEvaluate(::testing::_))
822 	    .Times(::testing::Exactly(1))
823 	    .WillRepeatedly(::testing::Return(false));
824 	EXPECT_CALL(*CaseFile2, ReEvaluate(::testing::_))
825 	    .Times(::testing::Exactly(0));
826 	EXPECT_CALL(*CaseFile4, ReEvaluate(::testing::_))
827 	    .Times(::testing::Exactly(0));
828 	EXPECT_CALL(*CaseFile5, ReEvaluate(::testing::_))
829 	    .Times(::testing::Exactly(0));
830 	CaseFile::ReEvaluateByGuid(Guid(456), *zfs_event);
831 	EXPECT_EQ(5, TestableCaseFile::getActiveCases());
832 	delete CaseFile1;
833 	delete CaseFile2;
834 	delete CaseFile3;
835 	delete CaseFile4;
836 	delete CaseFile5;
837 }
838 
839 /*
840  * Test VdevIterator
841  */
842 class VdevIteratorTest : public ::testing::Test
843 {
844 };
845 
846 bool VdevIteratorTestCB(Vdev &vdev, void *cbArg) {
847 	return (false);
848 }
849 
850 /*
851  * VdevIterator::Next should not crash when run on a pool that has a previously
852  * removed vdev.  Regression for
853  * https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=273663
854  */
855 TEST_F(VdevIteratorTest, VdevRemoval)
856 {
857 	nvlist_t* poolConfig, *rootVdev;
858 
859 	ASSERT_EQ(0, nvlist_alloc(&rootVdev, NV_UNIQUE_NAME, 0));
860 	ASSERT_EQ(0, nvlist_add_uint64(rootVdev, ZPOOL_CONFIG_GUID, 0x5678));
861 	/*
862 	 * Note: pools with previously-removed top-level VDEVs will contain a
863 	 * TLV in their labels that has 0 children.
864 	 */
865 	ASSERT_EQ(0, nvlist_add_nvlist_array(rootVdev, ZPOOL_CONFIG_CHILDREN,
866 				NULL, 0));
867 	ASSERT_EQ(0, nvlist_alloc(&poolConfig, NV_UNIQUE_NAME, 0));
868 	ASSERT_EQ(0, nvlist_add_uint64(poolConfig,
869 			ZPOOL_CONFIG_POOL_GUID, 0x1234));
870 	ASSERT_EQ(0, nvlist_add_nvlist(poolConfig, ZPOOL_CONFIG_VDEV_TREE,
871 				rootVdev));
872 
873 	VdevIterator(poolConfig).Each(VdevIteratorTestCB, NULL);
874 }
875