1 /*-
2 * Copyright (c) 2012, 2013, 2014 Spectra Logic Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions, and the following disclaimer,
10 * without modification.
11 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
12 * substantially similar to the "NO WARRANTY" disclaimer below
13 * ("Disclaimer") and any redistribution must be conditioned upon
14 * including a substantially similar Disclaimer requirement for further
15 * binary redistribution.
16 *
17 * NO WARRANTY
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
27 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGES.
29 *
30 * Authors: Alan Somers (Spectra Logic Corporation)
31 */
32 #include <sys/cdefs.h>
33 #include <sys/byteorder.h>
34
35 #include <stdarg.h>
36 #include <syslog.h>
37
38 #include <libnvpair.h>
39 #include <libzfs.h>
40
41 #include <list>
42 #include <map>
43 #include <sstream>
44 #include <string>
45
46 #include <gmock/gmock.h>
47 #include <gtest/gtest.h>
48
49 #include <devdctl/guid.h>
50 #include <devdctl/event.h>
51 #include <devdctl/event_factory.h>
52 #include <devdctl/exception.h>
53 #include <devdctl/consumer.h>
54
55 #include <zfsd/callout.h>
56 #include <zfsd/vdev_iterator.h>
57 #include <zfsd/zfsd_event.h>
58 #include <zfsd/case_file.h>
59 #include <zfsd/vdev.h>
60 #include <zfsd/zfsd.h>
61 #include <zfsd/zfsd_exception.h>
62 #include <zfsd/zpool_list.h>
63
64 #include "libmocks.h"
65 /*================================== Macros ==================================*/
66 #define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x))
67
68 /*============================ Namespace Control =============================*/
69 using std::string;
70 using std::stringstream;
71
72 using DevdCtl::Event;
73 using DevdCtl::EventFactory;
74 using DevdCtl::EventList;
75 using DevdCtl::Guid;
76 using DevdCtl::NVPairMap;
77
78 /* redefine zpool_handle here because libzfs_impl.h is not includable */
79 struct zpool_handle
80 {
81 libzfs_handle_t *zpool_hdl;
82 zpool_handle_t *zpool_next;
83 char zpool_name[ZFS_MAX_DATASET_NAME_LEN];
84 int zpool_state;
85 size_t zpool_config_size;
86 nvlist_t *zpool_config;
87 nvlist_t *zpool_old_config;
88 nvlist_t *zpool_props;
89 diskaddr_t zpool_start_block;
90 };
91
92 class MockZfsEvent : public ZfsEvent
93 {
94 public:
95 MockZfsEvent(Event::Type, NVPairMap&, const string&);
~MockZfsEvent()96 virtual ~MockZfsEvent() {}
97
98 static BuildMethod MockZfsEventBuilder;
99
100 MOCK_CONST_METHOD0(ProcessPoolEvent, void());
101
102 static EventFactory::Record s_buildRecords[];
103 };
104
105 EventFactory::Record MockZfsEvent::s_buildRecords[] =
106 {
107 { Event::NOTIFY, "ZFS", &MockZfsEvent::MockZfsEventBuilder }
108 };
109
MockZfsEvent(Event::Type type,NVPairMap & map,const string & str)110 MockZfsEvent::MockZfsEvent(Event::Type type, NVPairMap& map,
111 const string& str)
112 : ZfsEvent(type, map, str)
113 {
114 }
115
116 Event *
MockZfsEventBuilder(Event::Type type,NVPairMap & nvpairs,const string & eventString)117 MockZfsEvent::MockZfsEventBuilder(Event::Type type,
118 NVPairMap &nvpairs,
119 const string &eventString)
120 {
121 return (new MockZfsEvent(type, nvpairs, eventString));
122 }
123
124 /*
125 * A dummy Vdev class used for testing other classes
126 */
127 class MockVdev : public Vdev
128 {
129 public:
130 MockVdev(nvlist_t *vdevConfig);
~MockVdev()131 virtual ~MockVdev() {}
132
133 MOCK_CONST_METHOD0(GUID, Guid());
134 MOCK_CONST_METHOD0(PoolGUID, Guid());
135 MOCK_CONST_METHOD0(State, vdev_state());
136 MOCK_CONST_METHOD0(PhysicalPath, string());
137 MOCK_CONST_METHOD2(Name, string(zpool_handle_t * zhp, bool verbose));
138 };
139
MockVdev(nvlist_t * vdevConfig)140 MockVdev::MockVdev(nvlist_t *vdevConfig)
141 : Vdev(vdevConfig)
142 {
143 }
144
145 /*
146 * A CaseFile class with side effects removed, for testing
147 */
148 class TestableCaseFile : public CaseFile
149 {
150 public:
151 static TestableCaseFile &Create(Vdev &vdev);
152 TestableCaseFile(Vdev &vdev);
~TestableCaseFile()153 virtual ~TestableCaseFile() {}
154
155 MOCK_METHOD0(Close, void());
156 MOCK_METHOD1(RegisterCallout, void(const Event &event));
157 MOCK_METHOD0(RefreshVdevState, bool());
158 MOCK_METHOD1(ReEvaluate, bool(const ZfsEvent &event));
159
RealReEvaluate(const ZfsEvent & event)160 bool RealReEvaluate(const ZfsEvent &event)
161 {
162 return (CaseFile::ReEvaluate(event));
163 }
164
165 /*
166 * This splices the event lists, a procedure that would normally be done
167 * by OnGracePeriodEnded, but we don't necessarily call that in the
168 * unit tests
169 */
170 void SpliceEvents();
171
172 /*
173 * Used by some of our expectations. CaseFile does not publicize this
174 */
getActiveCases()175 static int getActiveCases()
176 {
177 return (s_activeCases.size());
178 }
179 };
180
TestableCaseFile(Vdev & vdev)181 TestableCaseFile::TestableCaseFile(Vdev &vdev)
182 : CaseFile(vdev)
183 {
184 }
185
186 TestableCaseFile &
Create(Vdev & vdev)187 TestableCaseFile::Create(Vdev &vdev)
188 {
189 TestableCaseFile *newCase;
190 newCase = new TestableCaseFile(vdev);
191 return (*newCase);
192 }
193
194 void
SpliceEvents()195 TestableCaseFile::SpliceEvents()
196 {
197 m_events.splice(m_events.begin(), m_tentativeEvents);
198 }
199
200
201 /*
202 * Test class ZfsdException
203 */
204 class ZfsdExceptionTest : public ::testing::Test
205 {
206 protected:
SetUp()207 virtual void SetUp()
208 {
209 ASSERT_EQ(0, nvlist_alloc(&poolConfig, NV_UNIQUE_NAME, 0));
210 ASSERT_EQ(0, nvlist_add_string(poolConfig,
211 ZPOOL_CONFIG_POOL_NAME, "unit_test_pool"));
212 ASSERT_EQ(0, nvlist_add_uint64(poolConfig,
213 ZPOOL_CONFIG_POOL_GUID, 0x1234));
214
215 ASSERT_EQ(0, nvlist_alloc(&vdevConfig, NV_UNIQUE_NAME, 0));
216 ASSERT_EQ(0, nvlist_add_uint64(vdevConfig,
217 ZPOOL_CONFIG_GUID, 0x5678));
218 bzero(&poolHandle, sizeof(poolHandle));
219 poolHandle.zpool_config = poolConfig;
220 }
221
TearDown()222 virtual void TearDown()
223 {
224 nvlist_free(poolConfig);
225 nvlist_free(vdevConfig);
226 }
227
228 nvlist_t *poolConfig;
229 nvlist_t *vdevConfig;
230 zpool_handle_t poolHandle;
231 };
232
TEST_F(ZfsdExceptionTest,StringConstructorNull)233 TEST_F(ZfsdExceptionTest, StringConstructorNull)
234 {
235 ZfsdException ze("");
236 EXPECT_STREQ("", ze.GetString().c_str());
237 }
238
TEST_F(ZfsdExceptionTest,StringConstructorFormatted)239 TEST_F(ZfsdExceptionTest, StringConstructorFormatted)
240 {
241 ZfsdException ze(" %d %s", 55, "hello world");
242 EXPECT_STREQ(" 55 hello world", ze.GetString().c_str());
243 }
244
TEST_F(ZfsdExceptionTest,LogSimple)245 TEST_F(ZfsdExceptionTest, LogSimple)
246 {
247 ZfsdException ze("unit test w/o vdev or pool");
248 ze.Log();
249 EXPECT_EQ(LOG_ERR, syslog_last_priority);
250 EXPECT_STREQ("unit test w/o vdev or pool\n", syslog_last_message);
251 }
252
TEST_F(ZfsdExceptionTest,Pool)253 TEST_F(ZfsdExceptionTest, Pool)
254 {
255 const char msg[] = "Exception with pool name";
256 char expected[4096];
257 sprintf(expected, "Pool unit_test_pool: %s\n", msg);
258 ZfsdException ze(poolConfig, msg);
259 ze.Log();
260 EXPECT_STREQ(expected, syslog_last_message);
261 }
262
TEST_F(ZfsdExceptionTest,PoolHandle)263 TEST_F(ZfsdExceptionTest, PoolHandle)
264 {
265 const char msg[] = "Exception with pool handle";
266 char expected[4096];
267 sprintf(expected, "Pool unit_test_pool: %s\n", msg);
268 ZfsdException ze(&poolHandle, msg);
269 ze.Log();
270 EXPECT_STREQ(expected, syslog_last_message);
271 }
272
273 /*
274 * Test class Vdev
275 */
276 class VdevTest : public ::testing::Test
277 {
278 protected:
SetUp()279 virtual void SetUp()
280 {
281 ASSERT_EQ(0, nvlist_alloc(&m_poolConfig, NV_UNIQUE_NAME, 0));
282 ASSERT_EQ(0, nvlist_add_uint64(m_poolConfig,
283 ZPOOL_CONFIG_POOL_GUID,
284 0x1234));
285
286 ASSERT_EQ(0, nvlist_alloc(&m_vdevConfig, NV_UNIQUE_NAME, 0));
287 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig, ZPOOL_CONFIG_GUID,
288 0x5678));
289 }
290
TearDown()291 virtual void TearDown()
292 {
293 nvlist_free(m_poolConfig);
294 nvlist_free(m_vdevConfig);
295 }
296
297 nvlist_t *m_poolConfig;
298 nvlist_t *m_vdevConfig;
299 };
300
301
TEST_F(VdevTest,StateFromConfig)302 TEST_F(VdevTest, StateFromConfig)
303 {
304 vdev_stat_t vs;
305
306 vs.vs_state = VDEV_STATE_OFFLINE;
307
308 ASSERT_EQ(0, nvlist_add_uint64_array(m_vdevConfig,
309 ZPOOL_CONFIG_VDEV_STATS,
310 (uint64_t*)&vs,
311 sizeof(vs) / sizeof(uint64_t)));
312
313 Vdev vdev(m_poolConfig, m_vdevConfig);
314
315 EXPECT_EQ(VDEV_STATE_OFFLINE, vdev.State());
316 }
317
TEST_F(VdevTest,StateFaulted)318 TEST_F(VdevTest, StateFaulted)
319 {
320 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig, ZPOOL_CONFIG_FAULTED, 1));
321
322 Vdev vdev(m_poolConfig, m_vdevConfig);
323
324 EXPECT_EQ(VDEV_STATE_FAULTED, vdev.State());
325 }
326
327 /*
328 * Test that we can construct a Vdev from the label information that is stored
329 * on an available spare drive
330 */
TEST_F(VdevTest,ConstructAvailSpare)331 TEST_F(VdevTest, ConstructAvailSpare)
332 {
333 nvlist_t *labelConfig;
334
335 ASSERT_EQ(0, nvlist_alloc(&labelConfig, NV_UNIQUE_NAME, 0));
336 ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_GUID,
337 1948339428197961030));
338 ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_POOL_STATE,
339 POOL_STATE_SPARE));
340
341 EXPECT_NO_THROW(Vdev vdev(labelConfig));
342
343 nvlist_free(labelConfig);
344 }
345
346 /* Available spares will always show the HEALTHY state */
TEST_F(VdevTest,AvailSpareState)347 TEST_F(VdevTest, AvailSpareState) {
348 nvlist_t *labelConfig;
349
350 ASSERT_EQ(0, nvlist_alloc(&labelConfig, NV_UNIQUE_NAME, 0));
351 ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_GUID,
352 1948339428197961030));
353 ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_POOL_STATE,
354 POOL_STATE_SPARE));
355
356 Vdev vdev(labelConfig);
357 EXPECT_EQ(VDEV_STATE_HEALTHY, vdev.State());
358
359 nvlist_free(labelConfig);
360 }
361
362 /* Test the Vdev::IsSpare method */
TEST_F(VdevTest,IsSpare)363 TEST_F(VdevTest, IsSpare) {
364 Vdev notSpare(m_poolConfig, m_vdevConfig);
365 EXPECT_EQ(false, notSpare.IsSpare());
366
367 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig, ZPOOL_CONFIG_IS_SPARE, 1));
368 Vdev isSpare(m_poolConfig, m_vdevConfig);
369 EXPECT_EQ(true, isSpare.IsSpare());
370 }
371
372 /*
373 * Test class ZFSEvent
374 */
375 class ZfsEventTest : public ::testing::Test
376 {
377 protected:
SetUp()378 virtual void SetUp()
379 {
380 m_eventFactory = new EventFactory();
381 m_eventFactory->UpdateRegistry(MockZfsEvent::s_buildRecords,
382 NUM_ELEMENTS(MockZfsEvent::s_buildRecords));
383
384 m_event = NULL;
385 }
386
TearDown()387 virtual void TearDown()
388 {
389 delete m_eventFactory;
390 delete m_event;
391 }
392
393 EventFactory *m_eventFactory;
394 Event *m_event;
395 };
396
TEST_F(ZfsEventTest,ProcessPoolEventGetsCalled)397 TEST_F(ZfsEventTest, ProcessPoolEventGetsCalled)
398 {
399 string evString("!system=ZFS "
400 "subsystem=ZFS "
401 "type=sysevent.fs.zfs.vdev_remove "
402 "pool_name=foo "
403 "pool_guid=9756779504028057996 "
404 "vdev_guid=1631193447431603339 "
405 "vdev_path=/dev/da1 "
406 "timestamp=1348871594");
407 m_event = Event::CreateEvent(*m_eventFactory, evString);
408 MockZfsEvent *mock_event = static_cast<MockZfsEvent*>(m_event);
409
410 EXPECT_CALL(*mock_event, ProcessPoolEvent()).Times(1);
411 mock_event->Process();
412 }
413
414 /*
415 * Test class CaseFile
416 */
417
418 class CaseFileTest : public ::testing::Test
419 {
420 protected:
SetUp()421 virtual void SetUp()
422 {
423 m_eventFactory = new EventFactory();
424 m_eventFactory->UpdateRegistry(MockZfsEvent::s_buildRecords,
425 NUM_ELEMENTS(MockZfsEvent::s_buildRecords));
426
427 m_event = NULL;
428
429 nvlist_alloc(&m_vdevConfig, NV_UNIQUE_NAME, 0);
430 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig,
431 ZPOOL_CONFIG_GUID, 0xbeef));
432 m_vdev = new MockVdev(m_vdevConfig);
433 ON_CALL(*m_vdev, GUID())
434 .WillByDefault(::testing::Return(Guid(123)));
435 ON_CALL(*m_vdev, Name(::testing::_, ::testing::_))
436 .WillByDefault(::testing::Return(string("/dev/da999")));
437 ON_CALL(*m_vdev, PoolGUID())
438 .WillByDefault(::testing::Return(Guid(456)));
439 ON_CALL(*m_vdev, State())
440 .WillByDefault(::testing::Return(VDEV_STATE_HEALTHY));
441 m_caseFile = &TestableCaseFile::Create(*m_vdev);
442 ON_CALL(*m_caseFile, ReEvaluate(::testing::_))
443 .WillByDefault(::testing::Invoke(m_caseFile, &TestableCaseFile::RealReEvaluate));
444 return;
445 }
446
TearDown()447 virtual void TearDown()
448 {
449 delete m_caseFile;
450 nvlist_free(m_vdevConfig);
451 delete m_vdev;
452 delete m_event;
453 delete m_eventFactory;
454 }
455
456 nvlist_t *m_vdevConfig;
457 MockVdev *m_vdev;
458 TestableCaseFile *m_caseFile;
459 Event *m_event;
460 EventFactory *m_eventFactory;
461 };
462
463 /*
464 * A Vdev with no events should not be degraded or faulted
465 */
TEST_F(CaseFileTest,HealthyVdev)466 TEST_F(CaseFileTest, HealthyVdev)
467 {
468 EXPECT_FALSE(m_caseFile->ShouldDegrade());
469 EXPECT_FALSE(m_caseFile->ShouldFault());
470 }
471
472 /*
473 * A Vdev with only one event should not be degraded or faulted
474 * For performance reasons, RefreshVdevState should not be called.
475 */
TEST_F(CaseFileTest,HealthyishVdev)476 TEST_F(CaseFileTest, HealthyishVdev)
477 {
478 string evString("!system=ZFS "
479 "class=ereport.fs.zfs.io "
480 "ena=12091638756982918145 "
481 "parent_guid=13237004955564865395 "
482 "parent_type=raidz "
483 "pool=testpool.4415 "
484 "pool_context=0 "
485 "pool_failmode=wait "
486 "pool_guid=456 "
487 "subsystem=ZFS "
488 "timestamp=1348867914 "
489 "type=ereport.fs.zfs.io "
490 "vdev_guid=123 "
491 "vdev_path=/dev/da400 "
492 "vdev_type=disk "
493 "zio_blkid=622 "
494 "zio_err=1 "
495 "zio_level=-2 "
496 "zio_object=0 "
497 "zio_objset=37 "
498 "zio_offset=25598976 "
499 "zio_size=1024");
500 m_event = Event::CreateEvent(*m_eventFactory, evString);
501 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
502
503 EXPECT_CALL(*m_caseFile, RefreshVdevState())
504 .Times(::testing::Exactly(0));
505 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
506 EXPECT_FALSE(m_caseFile->ShouldDegrade());
507 EXPECT_FALSE(m_caseFile->ShouldFault());
508 }
509
510 /* The case file should be closed when its pool is destroyed */
TEST_F(CaseFileTest,PoolDestroy)511 TEST_F(CaseFileTest, PoolDestroy)
512 {
513 string evString("!system=ZFS "
514 "pool_name=testpool.4415 "
515 "pool_guid=456 "
516 "subsystem=ZFS "
517 "timestamp=1348867914 "
518 "type=sysevent.fs.zfs.pool_destroy ");
519 m_event = Event::CreateEvent(*m_eventFactory, evString);
520 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
521 EXPECT_CALL(*m_caseFile, Close());
522 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
523 }
524
525 /*
526 * A Vdev with a very large number of Delay errors should fault
527 * For performance reasons, RefreshVdevState should be called at most once
528 */
TEST_F(CaseFileTest,VeryManyDelayErrors)529 TEST_F(CaseFileTest, VeryManyDelayErrors)
530 {
531 EXPECT_CALL(*m_caseFile, RefreshVdevState())
532 .Times(::testing::AtMost(1))
533 .WillRepeatedly(::testing::Return(true));
534
535 for(int i=0; i<100; i++) {
536 stringstream evStringStream;
537 evStringStream <<
538 "!system=ZFS "
539 "class=ereport.fs.zfs.delay "
540 "ena=12091638756982918145 "
541 "parent_guid=13237004955564865395 "
542 "parent_type=raidz "
543 "pool=testpool.4415 "
544 "pool_context=0 "
545 "pool_failmode=wait "
546 "pool_guid=456 "
547 "pool_state= 0"
548 "subsystem=ZFS "
549 "time=";
550 evStringStream << i << "0000000000000000 ";
551 evStringStream << "timestamp=" << i << " ";
552 evStringStream <<
553 "type=ereport.fs.zfs.delay "
554 "vdev_ashift=12 "
555 "vdev_cksum_errors=0 "
556 "vdev_complete_ts=948336226469 "
557 "vdev_delays=77 "
558 "vdev_delta_ts=123998485899 "
559 "vdev_guid=123 "
560 "vdev_path=/dev/da400 "
561 "vdev_read_errors=0 "
562 "vdev_spare_guids= "
563 "vdev_type=disk "
564 "vdev_write_errors=0 "
565 "zio_blkid=622 "
566 "zio_delay=31000041101 "
567 "zio_delta=123998485899 "
568 "zio_err=0 "
569 "zio_flags=1572992 "
570 "zio_level=-2 "
571 "zio_object=0 "
572 "zio_objset=37 "
573 "zio_offset=25598976 "
574 "zio_pipeline=48234496 "
575 "zio_priority=3 "
576 "zio_size=1024"
577 "zio_stage=33554432 "
578 "zio_timestamp=824337740570 ";
579 Event *event(Event::CreateEvent(*m_eventFactory,
580 evStringStream.str()));
581 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(event);
582 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
583 delete event;
584 }
585
586 m_caseFile->SpliceEvents();
587 EXPECT_FALSE(m_caseFile->ShouldDegrade());
588 EXPECT_TRUE(m_caseFile->ShouldFault());
589 }
590
591 /*
592 * A Vdev with a very large number of IO errors should fault
593 * For performance reasons, RefreshVdevState should be called at most once
594 */
TEST_F(CaseFileTest,VeryManyIOErrors)595 TEST_F(CaseFileTest, VeryManyIOErrors)
596 {
597 EXPECT_CALL(*m_caseFile, RefreshVdevState())
598 .Times(::testing::AtMost(1))
599 .WillRepeatedly(::testing::Return(true));
600
601 for(int i=0; i<100; i++) {
602 stringstream evStringStream;
603 evStringStream <<
604 "!system=ZFS "
605 "class=ereport.fs.zfs.io "
606 "ena=12091638756982918145 "
607 "parent_guid=13237004955564865395 "
608 "parent_type=raidz "
609 "pool=testpool.4415 "
610 "pool_context=0 "
611 "pool_failmode=wait "
612 "pool_guid=456 "
613 "subsystem=ZFS "
614 "timestamp=";
615 evStringStream << i << " ";
616 evStringStream <<
617 "type=ereport.fs.zfs.io "
618 "vdev_guid=123 "
619 "vdev_path=/dev/da400 "
620 "vdev_type=disk "
621 "zio_blkid=622 "
622 "zio_err=1 "
623 "zio_level=-2 "
624 "zio_object=0 "
625 "zio_objset=37 "
626 "zio_offset=25598976 "
627 "zio_size=1024";
628 Event *event(Event::CreateEvent(*m_eventFactory,
629 evStringStream.str()));
630 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(event);
631 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
632 delete event;
633 }
634
635 m_caseFile->SpliceEvents();
636 EXPECT_FALSE(m_caseFile->ShouldDegrade());
637 EXPECT_TRUE(m_caseFile->ShouldFault());
638 }
639
640 /*
641 * A Vdev with a very large number of checksum errors should degrade
642 * For performance reasons, RefreshVdevState should be called at most once
643 */
TEST_F(CaseFileTest,VeryManyChecksumErrors)644 TEST_F(CaseFileTest, VeryManyChecksumErrors)
645 {
646 EXPECT_CALL(*m_caseFile, RefreshVdevState())
647 .Times(::testing::AtMost(1))
648 .WillRepeatedly(::testing::Return(true));
649
650 for(int i=0; i<100; i++) {
651 stringstream evStringStream;
652 evStringStream <<
653 "!system=ZFS "
654 "bad_cleared_bits=03000000000000803f50b00000000000 "
655 "bad_range_clears=0000000e "
656 "bad_range_sets=00000000 "
657 "bad_ranges=0000000000000010 "
658 "bad_ranges_min_gap=8 "
659 "bad_set_bits=00000000000000000000000000000000 "
660 "class=ereport.fs.zfs.checksum "
661 "ena=12272856582652437505 "
662 "parent_guid=5838204195352909894 "
663 "parent_type=raidz pool=testpool.7640 "
664 "pool_context=0 "
665 "pool_failmode=wait "
666 "pool_guid=456 "
667 "subsystem=ZFS timestamp=";
668 evStringStream << i << " ";
669 evStringStream <<
670 "type=ereport.fs.zfs.checksum "
671 "vdev_guid=123 "
672 "vdev_path=/mnt/tmp/file1.7702 "
673 "vdev_type=file "
674 "zio_blkid=0 "
675 "zio_err=0 "
676 "zio_level=0 "
677 "zio_object=3 "
678 "zio_objset=0 "
679 "zio_offset=16896 "
680 "zio_size=512";
681 Event *event(Event::CreateEvent(*m_eventFactory,
682 evStringStream.str()));
683 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(event);
684 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
685 delete event;
686 }
687
688 m_caseFile->SpliceEvents();
689 EXPECT_TRUE(m_caseFile->ShouldDegrade());
690 EXPECT_FALSE(m_caseFile->ShouldFault());
691 }
692
693 /*
694 * Test CaseFile::ReEvaluateByGuid
695 */
696 class ReEvaluateByGuidTest : public ::testing::Test
697 {
698 protected:
SetUp()699 virtual void SetUp()
700 {
701 m_eventFactory = new EventFactory();
702 m_eventFactory->UpdateRegistry(MockZfsEvent::s_buildRecords,
703 NUM_ELEMENTS(MockZfsEvent::s_buildRecords));
704 m_event = Event::CreateEvent(*m_eventFactory, s_evString);
705 nvlist_alloc(&m_vdevConfig, NV_UNIQUE_NAME, 0);
706 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig,
707 ZPOOL_CONFIG_GUID, 0xbeef));
708 m_vdev456 = new ::testing::NiceMock<MockVdev>(m_vdevConfig);
709 m_vdev789 = new ::testing::NiceMock<MockVdev>(m_vdevConfig);
710 ON_CALL(*m_vdev456, GUID())
711 .WillByDefault(::testing::Return(Guid(123)));
712 ON_CALL(*m_vdev456, PoolGUID())
713 .WillByDefault(::testing::Return(Guid(456)));
714 ON_CALL(*m_vdev456, State())
715 .WillByDefault(::testing::Return(VDEV_STATE_HEALTHY));
716 ON_CALL(*m_vdev789, GUID())
717 .WillByDefault(::testing::Return(Guid(123)));
718 ON_CALL(*m_vdev789, PoolGUID())
719 .WillByDefault(::testing::Return(Guid(789)));
720 ON_CALL(*m_vdev789, State())
721 .WillByDefault(::testing::Return(VDEV_STATE_HEALTHY));
722 m_caseFile456 = NULL;
723 m_caseFile789 = NULL;
724 return;
725 }
726
TearDown()727 virtual void TearDown()
728 {
729 delete m_caseFile456;
730 delete m_caseFile789;
731 nvlist_free(m_vdevConfig);
732 delete m_vdev456;
733 delete m_vdev789;
734 delete m_event;
735 delete m_eventFactory;
736 }
737
738 static string s_evString;
739 nvlist_t *m_vdevConfig;
740 ::testing::NiceMock<MockVdev> *m_vdev456;
741 ::testing::NiceMock<MockVdev> *m_vdev789;
742 TestableCaseFile *m_caseFile456;
743 TestableCaseFile *m_caseFile789;
744 Event *m_event;
745 EventFactory *m_eventFactory;
746 };
747
748 string ReEvaluateByGuidTest::s_evString(
749 "!system=ZFS "
750 "pool_guid=16271873792808333580 "
751 "pool_name=foo "
752 "subsystem=ZFS "
753 "timestamp=1360620391 "
754 "type=sysevent.fs.zfs.config_sync");
755
756
757 /*
758 * Test the ReEvaluateByGuid method on an empty list of casefiles.
759 * We must create one event, even though it never gets used, because it will
760 * be passed by reference to ReEvaluateByGuid
761 */
TEST_F(ReEvaluateByGuidTest,ReEvaluateByGuid_empty)762 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_empty)
763 {
764 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
765
766 EXPECT_EQ(0, TestableCaseFile::getActiveCases());
767 CaseFile::ReEvaluateByGuid(Guid(456), *zfs_event);
768 EXPECT_EQ(0, TestableCaseFile::getActiveCases());
769 }
770
771 /*
772 * Test the ReEvaluateByGuid method on a list of CaseFiles that contains only
773 * one CaseFile, which doesn't match the criteria
774 */
TEST_F(ReEvaluateByGuidTest,ReEvaluateByGuid_oneFalse)775 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_oneFalse)
776 {
777 m_caseFile456 = &TestableCaseFile::Create(*m_vdev456);
778 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
779
780 EXPECT_EQ(1, TestableCaseFile::getActiveCases());
781 EXPECT_CALL(*m_caseFile456, ReEvaluate(::testing::_))
782 .Times(::testing::Exactly(0));
783 CaseFile::ReEvaluateByGuid(Guid(789), *zfs_event);
784 EXPECT_EQ(1, TestableCaseFile::getActiveCases());
785 }
786
787 /*
788 * Test the ReEvaluateByGuid method on a list of CaseFiles that contains only
789 * one CaseFile, which does match the criteria
790 */
TEST_F(ReEvaluateByGuidTest,ReEvaluateByGuid_oneTrue)791 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_oneTrue)
792 {
793 m_caseFile456 = &TestableCaseFile::Create(*m_vdev456);
794 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
795
796 EXPECT_EQ(1, TestableCaseFile::getActiveCases());
797 EXPECT_CALL(*m_caseFile456, ReEvaluate(::testing::_))
798 .Times(::testing::Exactly(1))
799 .WillRepeatedly(::testing::Return(false));
800 CaseFile::ReEvaluateByGuid(Guid(456), *zfs_event);
801 EXPECT_EQ(1, TestableCaseFile::getActiveCases());
802 }
803
804 /*
805 * Test the ReEvaluateByGuid method on a long list of CaseFiles that contains a
806 * few cases which meet the criteria
807 */
TEST_F(ReEvaluateByGuidTest,ReEvaluateByGuid_five)808 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_five)
809 {
810 TestableCaseFile *CaseFile1 = &TestableCaseFile::Create(*m_vdev456);
811 TestableCaseFile *CaseFile2 = &TestableCaseFile::Create(*m_vdev789);
812 TestableCaseFile *CaseFile3 = &TestableCaseFile::Create(*m_vdev456);
813 TestableCaseFile *CaseFile4 = &TestableCaseFile::Create(*m_vdev789);
814 TestableCaseFile *CaseFile5 = &TestableCaseFile::Create(*m_vdev789);
815 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
816
817 EXPECT_EQ(5, TestableCaseFile::getActiveCases());
818 EXPECT_CALL(*CaseFile1, ReEvaluate(::testing::_))
819 .Times(::testing::Exactly(1))
820 .WillRepeatedly(::testing::Return(false));
821 EXPECT_CALL(*CaseFile3, ReEvaluate(::testing::_))
822 .Times(::testing::Exactly(1))
823 .WillRepeatedly(::testing::Return(false));
824 EXPECT_CALL(*CaseFile2, ReEvaluate(::testing::_))
825 .Times(::testing::Exactly(0));
826 EXPECT_CALL(*CaseFile4, ReEvaluate(::testing::_))
827 .Times(::testing::Exactly(0));
828 EXPECT_CALL(*CaseFile5, ReEvaluate(::testing::_))
829 .Times(::testing::Exactly(0));
830 CaseFile::ReEvaluateByGuid(Guid(456), *zfs_event);
831 EXPECT_EQ(5, TestableCaseFile::getActiveCases());
832 delete CaseFile1;
833 delete CaseFile2;
834 delete CaseFile3;
835 delete CaseFile4;
836 delete CaseFile5;
837 }
838
839 /*
840 * Test VdevIterator
841 */
842 class VdevIteratorTest : public ::testing::Test
843 {
844 };
845
VdevIteratorTestCB(Vdev & vdev,void * cbArg)846 bool VdevIteratorTestCB(Vdev &vdev, void *cbArg) {
847 return (false);
848 }
849
850 /*
851 * VdevIterator::Next should not crash when run on a pool that has a previously
852 * removed vdev. Regression for
853 * https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=273663
854 */
TEST_F(VdevIteratorTest,VdevRemoval)855 TEST_F(VdevIteratorTest, VdevRemoval)
856 {
857 nvlist_t* poolConfig, *rootVdev;
858
859 ASSERT_EQ(0, nvlist_alloc(&rootVdev, NV_UNIQUE_NAME, 0));
860 ASSERT_EQ(0, nvlist_add_uint64(rootVdev, ZPOOL_CONFIG_GUID, 0x5678));
861 /*
862 * Note: pools with previously-removed top-level VDEVs will contain a
863 * TLV in their labels that has 0 children.
864 */
865 ASSERT_EQ(0, nvlist_add_nvlist_array(rootVdev, ZPOOL_CONFIG_CHILDREN,
866 NULL, 0));
867 ASSERT_EQ(0, nvlist_alloc(&poolConfig, NV_UNIQUE_NAME, 0));
868 ASSERT_EQ(0, nvlist_add_uint64(poolConfig,
869 ZPOOL_CONFIG_POOL_GUID, 0x1234));
870 ASSERT_EQ(0, nvlist_add_nvlist(poolConfig, ZPOOL_CONFIG_VDEV_TREE,
871 rootVdev));
872
873 VdevIterator(poolConfig).Each(VdevIteratorTestCB, NULL);
874 }
875