1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9 #include "rocksdb/stats_history.h"
10
11 #include <limits>
12 #include <string>
13 #include <unordered_map>
14
15 #include "db/column_family.h"
16 #include "db/db_impl/db_impl.h"
17 #include "db/db_test_util.h"
18 #include "db/periodic_work_scheduler.h"
19 #include "monitoring/persistent_stats_history.h"
20 #include "options/options_helper.h"
21 #include "port/stack_trace.h"
22 #include "rocksdb/cache.h"
23 #include "rocksdb/convenience.h"
24 #include "rocksdb/rate_limiter.h"
25 #include "test_util/mock_time_env.h"
26 #include "test_util/sync_point.h"
27 #include "test_util/testutil.h"
28 #include "util/random.h"
29
30 namespace ROCKSDB_NAMESPACE {
31
32 #ifndef ROCKSDB_LITE
33 class StatsHistoryTest : public DBTestBase {
34 public:
StatsHistoryTest()35 StatsHistoryTest() : DBTestBase("stats_history_test", /*env_do_fsync=*/true) {
36 mock_clock_ = std::make_shared<MockSystemClock>(env_->GetSystemClock());
37 mock_env_.reset(new CompositeEnvWrapper(env_, mock_clock_));
38 }
39
40 protected:
41 std::shared_ptr<MockSystemClock> mock_clock_;
42 std::unique_ptr<Env> mock_env_;
43
SetUp()44 void SetUp() override {
45 mock_clock_->InstallTimedWaitFixCallback();
46 SyncPoint::GetInstance()->SetCallBack(
47 "DBImpl::StartPeriodicWorkScheduler:Init", [&](void* arg) {
48 auto* periodic_work_scheduler_ptr =
49 reinterpret_cast<PeriodicWorkScheduler**>(arg);
50 *periodic_work_scheduler_ptr =
51 PeriodicWorkTestScheduler::Default(mock_clock_);
52 });
53 }
54 };
55
TEST_F(StatsHistoryTest,RunStatsDumpPeriodSec)56 TEST_F(StatsHistoryTest, RunStatsDumpPeriodSec) {
57 constexpr int kPeriodSec = 5;
58 Options options;
59 options.create_if_missing = true;
60 options.stats_dump_period_sec = kPeriodSec;
61 options.env = mock_env_.get();
62 int counter = 0;
63 SyncPoint::GetInstance()->SetCallBack("DBImpl::DumpStats:1",
64 [&](void* /*arg*/) { counter++; });
65 Reopen(options);
66 ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_dump_period_sec);
67
68 // Wait for the first stats persist to finish, as the initial delay could be
69 // different.
70 dbfull()->TEST_WaitForStatsDumpRun(
71 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
72
73 dbfull()->TEST_WaitForStatsDumpRun(
74 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
75 ASSERT_GE(counter, 1);
76
77 // Test cancel job through SetOptions
78 ASSERT_OK(dbfull()->SetDBOptions({{"stats_dump_period_sec", "0"}}));
79 int old_val = counter;
80 for (int i = 1; i < 20; ++i) {
81 mock_clock_->MockSleepForSeconds(kPeriodSec);
82 }
83 ASSERT_EQ(counter, old_val);
84 Close();
85 }
86
87 // Test persistent stats background thread scheduling and cancelling
TEST_F(StatsHistoryTest,StatsPersistScheduling)88 TEST_F(StatsHistoryTest, StatsPersistScheduling) {
89 constexpr int kPeriodSec = 5;
90 Options options;
91 options.create_if_missing = true;
92 options.stats_persist_period_sec = kPeriodSec;
93 options.env = mock_env_.get();
94 int counter = 0;
95 SyncPoint::GetInstance()->SetCallBack("DBImpl::PersistStats:Entry",
96 [&](void* /*arg*/) { counter++; });
97 Reopen(options);
98 ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_persist_period_sec);
99
100 // Wait for the first stats persist to finish, as the initial delay could be
101 // different.
102 dbfull()->TEST_WaitForStatsDumpRun(
103 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
104
105 dbfull()->TEST_WaitForStatsDumpRun(
106 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
107 ASSERT_GE(counter, 1);
108
109 // Test cancel job through SetOptions
110 ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}}));
111 int old_val = counter;
112 dbfull()->TEST_WaitForStatsDumpRun(
113 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec * 2); });
114 ASSERT_EQ(counter, old_val);
115
116 Close();
117 }
118
119 // Test enabling persistent stats for the first time
TEST_F(StatsHistoryTest,PersistentStatsFreshInstall)120 TEST_F(StatsHistoryTest, PersistentStatsFreshInstall) {
121 constexpr unsigned int kPeriodSec = 5;
122 Options options;
123 options.create_if_missing = true;
124 options.stats_persist_period_sec = 0;
125 options.env = mock_env_.get();
126 int counter = 0;
127 SyncPoint::GetInstance()->SetCallBack("DBImpl::PersistStats:Entry",
128 [&](void* /*arg*/) { counter++; });
129 Reopen(options);
130 ASSERT_OK(dbfull()->SetDBOptions(
131 {{"stats_persist_period_sec", std::to_string(kPeriodSec)}}));
132 ASSERT_EQ(kPeriodSec, dbfull()->GetDBOptions().stats_persist_period_sec);
133
134 dbfull()->TEST_WaitForStatsDumpRun(
135 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
136 ASSERT_GE(counter, 1);
137 Close();
138 }
139
140 // TODO(Zhongyi): Move persistent stats related tests to a separate file
TEST_F(StatsHistoryTest,GetStatsHistoryInMemory)141 TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) {
142 constexpr int kPeriodSec = 5;
143 Options options;
144 options.create_if_missing = true;
145 options.stats_persist_period_sec = kPeriodSec;
146 options.statistics = CreateDBStatistics();
147 options.env = mock_env_.get();
148 CreateColumnFamilies({"pikachu"}, options);
149 ASSERT_OK(Put("foo", "bar"));
150 ReopenWithColumnFamilies({"default", "pikachu"}, options);
151
152 // make sure the first stats persist to finish
153 dbfull()->TEST_WaitForStatsDumpRun(
154 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
155
156 // Wait for stats persist to finish
157 dbfull()->TEST_WaitForStatsDumpRun(
158 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
159
160 std::unique_ptr<StatsHistoryIterator> stats_iter;
161 ASSERT_OK(
162 db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
163 ASSERT_TRUE(stats_iter != nullptr);
164 // disabled stats snapshots
165 ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "0"}}));
166 size_t stats_count = 0;
167 for (; stats_iter->Valid(); stats_iter->Next()) {
168 auto stats_map = stats_iter->GetStatsMap();
169 ASSERT_EQ(stats_iter->GetStatsTime(), mock_clock_->NowSeconds());
170 stats_count += stats_map.size();
171 }
172 ASSERT_GT(stats_count, 0);
173 // Wait a bit and verify no more stats are found
174 for (int i = 0; i < 10; ++i) {
175 dbfull()->TEST_WaitForStatsDumpRun(
176 [&] { mock_clock_->MockSleepForSeconds(1); });
177 }
178 ASSERT_OK(db_->GetStatsHistory(0, mock_clock_->NowSeconds(), &stats_iter));
179 ASSERT_TRUE(stats_iter != nullptr);
180 size_t stats_count_new = 0;
181 for (; stats_iter->Valid(); stats_iter->Next()) {
182 stats_count_new += stats_iter->GetStatsMap().size();
183 }
184 ASSERT_EQ(stats_count_new, stats_count);
185 Close();
186 }
187
TEST_F(StatsHistoryTest,InMemoryStatsHistoryPurging)188 TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
189 constexpr int kPeriodSec = 1;
190 Options options;
191 options.create_if_missing = true;
192 options.statistics = CreateDBStatistics();
193 options.stats_persist_period_sec = kPeriodSec;
194 options.env = mock_env_.get();
195
196 CreateColumnFamilies({"pikachu"}, options);
197 ASSERT_OK(Put("foo", "bar"));
198 ReopenWithColumnFamilies({"default", "pikachu"}, options);
199 // some random operation to populate statistics
200 ASSERT_OK(Delete("foo"));
201 ASSERT_OK(Put("sol", "sol"));
202 ASSERT_OK(Put("epic", "epic"));
203 ASSERT_OK(Put("ltd", "ltd"));
204 ASSERT_EQ("sol", Get("sol"));
205 ASSERT_EQ("epic", Get("epic"));
206 ASSERT_EQ("ltd", Get("ltd"));
207 Iterator* iterator = db_->NewIterator(ReadOptions());
208 for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
209 ASSERT_TRUE(iterator->key() == iterator->value());
210 }
211 delete iterator;
212 ASSERT_OK(Flush());
213 ASSERT_OK(Delete("sol"));
214 ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
215
216 // second round of ops
217 ASSERT_OK(Put("saigon", "saigon"));
218 ASSERT_OK(Put("noodle talk", "noodle talk"));
219 ASSERT_OK(Put("ping bistro", "ping bistro"));
220 iterator = db_->NewIterator(ReadOptions());
221 for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
222 ASSERT_TRUE(iterator->key() == iterator->value());
223 }
224 delete iterator;
225 ASSERT_OK(Flush());
226 ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
227
228 const int kIterations = 10;
229 for (int i = 0; i < kIterations; ++i) {
230 dbfull()->TEST_WaitForStatsDumpRun(
231 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
232 }
233
234 std::unique_ptr<StatsHistoryIterator> stats_iter;
235 ASSERT_OK(
236 db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
237 ASSERT_TRUE(stats_iter != nullptr);
238 size_t stats_count = 0;
239 int slice_count = 0;
240 for (; stats_iter->Valid(); stats_iter->Next()) {
241 slice_count++;
242 auto stats_map = stats_iter->GetStatsMap();
243 stats_count += stats_map.size();
244 }
245 size_t stats_history_size = dbfull()->TEST_EstimateInMemoryStatsHistorySize();
246 ASSERT_GE(slice_count, kIterations - 1);
247 ASSERT_GE(stats_history_size, 15000);
248 // capping memory cost at 15000 bytes since one slice is around 10000~15000
249 ASSERT_OK(dbfull()->SetDBOptions({{"stats_history_buffer_size", "15000"}}));
250 ASSERT_EQ(15000, dbfull()->GetDBOptions().stats_history_buffer_size);
251
252 // Wait for stats persist to finish
253 for (int i = 0; i < kIterations; ++i) {
254 dbfull()->TEST_WaitForStatsDumpRun(
255 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
256 }
257
258 ASSERT_OK(
259 db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
260 ASSERT_TRUE(stats_iter != nullptr);
261 size_t stats_count_reopen = 0;
262 slice_count = 0;
263 for (; stats_iter->Valid(); stats_iter->Next()) {
264 slice_count++;
265 auto stats_map = stats_iter->GetStatsMap();
266 stats_count_reopen += stats_map.size();
267 }
268 size_t stats_history_size_reopen =
269 dbfull()->TEST_EstimateInMemoryStatsHistorySize();
270 // only one slice can fit under the new stats_history_buffer_size
271 ASSERT_LT(slice_count, 2);
272 ASSERT_TRUE(stats_history_size_reopen < 15000 &&
273 stats_history_size_reopen > 0);
274 ASSERT_TRUE(stats_count_reopen < stats_count && stats_count_reopen > 0);
275 Close();
276 // TODO: may also want to verify stats timestamp to make sure we are purging
277 // the correct stats snapshot
278 }
279
countkeys(Iterator * iter)280 int countkeys(Iterator* iter) {
281 int count = 0;
282 for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
283 count++;
284 }
285 return count;
286 }
287
TEST_F(StatsHistoryTest,GetStatsHistoryFromDisk)288 TEST_F(StatsHistoryTest, GetStatsHistoryFromDisk) {
289 constexpr int kPeriodSec = 5;
290 Options options;
291 options.create_if_missing = true;
292 options.stats_persist_period_sec = kPeriodSec;
293 options.statistics = CreateDBStatistics();
294 options.persist_stats_to_disk = true;
295 options.env = mock_env_.get();
296 CreateColumnFamilies({"pikachu"}, options);
297 ASSERT_OK(Put("foo", "bar"));
298 ReopenWithColumnFamilies({"default", "pikachu"}, options);
299 ASSERT_EQ(Get("foo"), "bar");
300
301 // Wait for the first stats persist to finish, as the initial delay could be
302 // different.
303 dbfull()->TEST_WaitForStatsDumpRun(
304 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
305
306 // Wait for stats persist to finish
307 dbfull()->TEST_WaitForStatsDumpRun(
308 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
309
310 auto iter =
311 db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
312 int key_count1 = countkeys(iter);
313 delete iter;
314
315 dbfull()->TEST_WaitForStatsDumpRun(
316 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
317 iter =
318 db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
319 int key_count2 = countkeys(iter);
320 delete iter;
321
322 dbfull()->TEST_WaitForStatsDumpRun(
323 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
324 iter =
325 db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
326 int key_count3 = countkeys(iter);
327 delete iter;
328 ASSERT_GE(key_count2, key_count1);
329 ASSERT_GE(key_count3, key_count2);
330 ASSERT_EQ(key_count3 - key_count2, key_count2 - key_count1);
331 std::unique_ptr<StatsHistoryIterator> stats_iter;
332 ASSERT_OK(
333 db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
334 ASSERT_TRUE(stats_iter != nullptr);
335 size_t stats_count = 0;
336 int slice_count = 0;
337 int non_zero_count = 0;
338 for (int i = 2; stats_iter->Valid(); stats_iter->Next(), i++) {
339 slice_count++;
340 auto stats_map = stats_iter->GetStatsMap();
341 ASSERT_EQ(stats_iter->GetStatsTime(), kPeriodSec * i - 1);
342 for (auto& stat : stats_map) {
343 if (stat.second != 0) {
344 non_zero_count++;
345 }
346 }
347 stats_count += stats_map.size();
348 }
349 ASSERT_EQ(slice_count, 3);
350 // 2 extra keys for format version
351 ASSERT_EQ(stats_count, key_count3 - 2);
352 // verify reopen will not cause data loss
353 ReopenWithColumnFamilies({"default", "pikachu"}, options);
354 ASSERT_OK(
355 db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
356 ASSERT_TRUE(stats_iter != nullptr);
357 size_t stats_count_reopen = 0;
358 int slice_count_reopen = 0;
359 int non_zero_count_recover = 0;
360 for (; stats_iter->Valid(); stats_iter->Next()) {
361 slice_count_reopen++;
362 auto stats_map = stats_iter->GetStatsMap();
363 for (auto& stat : stats_map) {
364 if (stat.second != 0) {
365 non_zero_count_recover++;
366 }
367 }
368 stats_count_reopen += stats_map.size();
369 }
370
371 ASSERT_EQ(non_zero_count, non_zero_count_recover);
372 ASSERT_EQ(slice_count, slice_count_reopen);
373 ASSERT_EQ(stats_count, stats_count_reopen);
374 Close();
375 }
376
377 // Test persisted stats matches the value found in options.statistics and
378 // the stats value retains after DB reopen
TEST_F(StatsHistoryTest,PersitentStatsVerifyValue)379 TEST_F(StatsHistoryTest, PersitentStatsVerifyValue) {
380 constexpr int kPeriodSec = 5;
381 Options options;
382 options.create_if_missing = true;
383 options.stats_persist_period_sec = kPeriodSec;
384 options.statistics = CreateDBStatistics();
385 options.persist_stats_to_disk = true;
386 std::map<std::string, uint64_t> stats_map_before;
387 ASSERT_TRUE(options.statistics->getTickerMap(&stats_map_before));
388 options.env = mock_env_.get();
389 CreateColumnFamilies({"pikachu"}, options);
390 ASSERT_OK(Put("foo", "bar"));
391 ReopenWithColumnFamilies({"default", "pikachu"}, options);
392 ASSERT_EQ(Get("foo"), "bar");
393
394 // Wait for the first stats persist to finish, as the initial delay could be
395 // different.
396 dbfull()->TEST_WaitForStatsDumpRun(
397 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
398
399 // Wait for stats persist to finish
400 dbfull()->TEST_WaitForStatsDumpRun(
401 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
402 auto iter =
403 db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
404 countkeys(iter);
405 delete iter;
406
407 dbfull()->TEST_WaitForStatsDumpRun(
408 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
409 iter =
410 db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
411 countkeys(iter);
412 delete iter;
413
414 dbfull()->TEST_WaitForStatsDumpRun(
415 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
416 iter =
417 db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
418 countkeys(iter);
419 delete iter;
420
421 dbfull()->TEST_WaitForStatsDumpRun(
422 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
423
424 std::map<std::string, uint64_t> stats_map_after;
425 ASSERT_TRUE(options.statistics->getTickerMap(&stats_map_after));
426 std::unique_ptr<StatsHistoryIterator> stats_iter;
427 ASSERT_OK(
428 db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
429 ASSERT_TRUE(stats_iter != nullptr);
430 std::string sample = "rocksdb.num.iterator.deleted";
431 uint64_t recovered_value = 0;
432 for (int i = 2; stats_iter->Valid(); stats_iter->Next(), ++i) {
433 auto stats_map = stats_iter->GetStatsMap();
434 ASSERT_EQ(stats_iter->GetStatsTime(), kPeriodSec * i - 1);
435 for (const auto& stat : stats_map) {
436 if (sample.compare(stat.first) == 0) {
437 recovered_value += stat.second;
438 }
439 }
440 }
441 ASSERT_EQ(recovered_value, stats_map_after[sample]);
442
443 // test stats value retains after recovery
444 ReopenWithColumnFamilies({"default", "pikachu"}, options);
445 ASSERT_OK(
446 db_->GetStatsHistory(0, mock_clock_->NowSeconds() + 1, &stats_iter));
447 ASSERT_TRUE(stats_iter != nullptr);
448 uint64_t new_recovered_value = 0;
449 for (int i = 2; stats_iter->Valid(); stats_iter->Next(), i++) {
450 auto stats_map = stats_iter->GetStatsMap();
451 ASSERT_EQ(stats_iter->GetStatsTime(), kPeriodSec * i - 1);
452 for (const auto& stat : stats_map) {
453 if (sample.compare(stat.first) == 0) {
454 new_recovered_value += stat.second;
455 }
456 }
457 }
458 ASSERT_EQ(recovered_value, new_recovered_value);
459
460 // TODO(Zhongyi): also add test to read raw values from disk and verify
461 // correctness
462 Close();
463 }
464
465 // TODO(Zhongyi): add test for different format versions
466
TEST_F(StatsHistoryTest,PersistentStatsCreateColumnFamilies)467 TEST_F(StatsHistoryTest, PersistentStatsCreateColumnFamilies) {
468 constexpr int kPeriodSec = 5;
469 Options options;
470 options.create_if_missing = true;
471 options.stats_persist_period_sec = kPeriodSec;
472 options.statistics = CreateDBStatistics();
473 options.persist_stats_to_disk = true;
474 options.env = mock_env_.get();
475 ASSERT_OK(TryReopen(options));
476 CreateColumnFamilies({"one", "two", "three"}, options);
477 ASSERT_OK(Put(1, "foo", "bar"));
478 ReopenWithColumnFamilies({"default", "one", "two", "three"}, options);
479 ASSERT_EQ(Get(2, "foo"), "bar");
480 CreateColumnFamilies({"four"}, options);
481 ReopenWithColumnFamilies({"default", "one", "two", "three", "four"}, options);
482 ASSERT_EQ(Get(2, "foo"), "bar");
483
484 // make sure the first stats persist to finish
485 dbfull()->TEST_WaitForStatsDumpRun(
486 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
487
488 dbfull()->TEST_WaitForStatsDumpRun(
489 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
490 auto iter =
491 db_->NewIterator(ReadOptions(), dbfull()->PersistentStatsColumnFamily());
492 int key_count = countkeys(iter);
493 delete iter;
494 ASSERT_GE(key_count, 0);
495 uint64_t num_write_wal = 0;
496 std::string sample = "rocksdb.write.wal";
497 std::unique_ptr<StatsHistoryIterator> stats_iter;
498 ASSERT_OK(db_->GetStatsHistory(0, mock_clock_->NowSeconds(), &stats_iter));
499 ASSERT_TRUE(stats_iter != nullptr);
500 for (; stats_iter->Valid(); stats_iter->Next()) {
501 auto stats_map = stats_iter->GetStatsMap();
502 for (const auto& stat : stats_map) {
503 if (sample.compare(stat.first) == 0) {
504 num_write_wal += stat.second;
505 }
506 }
507 }
508 stats_iter.reset();
509 ASSERT_EQ(num_write_wal, 2);
510
511 options.persist_stats_to_disk = false;
512 ReopenWithColumnFamilies({"default", "one", "two", "three", "four"}, options);
513 int cf_count = 0;
514 for (auto cfd : *dbfull()->versions_->GetColumnFamilySet()) {
515 (void)cfd;
516 cf_count++;
517 }
518 // persistent stats cf will be implicitly opened even if
519 // persist_stats_to_disk is false
520 ASSERT_EQ(cf_count, 6);
521 ASSERT_EQ(Get(2, "foo"), "bar");
522
523 // attempt to create column family using same name, should fail
524 ColumnFamilyOptions cf_opts(options);
525 ColumnFamilyHandle* handle;
526 ASSERT_NOK(db_->CreateColumnFamily(cf_opts, kPersistentStatsColumnFamilyName,
527 &handle));
528
529 options.persist_stats_to_disk = true;
530 ReopenWithColumnFamilies({"default", "one", "two", "three", "four"}, options);
531 ASSERT_NOK(db_->CreateColumnFamily(cf_opts, kPersistentStatsColumnFamilyName,
532 &handle));
533 // verify stats is not affected by prior failed CF creation
534 ASSERT_OK(db_->GetStatsHistory(0, mock_clock_->NowSeconds(), &stats_iter));
535 ASSERT_TRUE(stats_iter != nullptr);
536 num_write_wal = 0;
537 for (; stats_iter->Valid(); stats_iter->Next()) {
538 auto stats_map = stats_iter->GetStatsMap();
539 for (const auto& stat : stats_map) {
540 if (sample.compare(stat.first) == 0) {
541 num_write_wal += stat.second;
542 }
543 }
544 }
545 ASSERT_EQ(num_write_wal, 2);
546
547 Close();
548 Destroy(options);
549 }
550
TEST_F(StatsHistoryTest,PersistentStatsReadOnly)551 TEST_F(StatsHistoryTest, PersistentStatsReadOnly) {
552 ASSERT_OK(Put("bar", "v2"));
553 Close();
554
555 auto options = CurrentOptions();
556 options.stats_persist_period_sec = 5;
557 options.persist_stats_to_disk = true;
558 assert(options.env == env_);
559 ASSERT_OK(ReadOnlyReopen(options));
560 ASSERT_EQ("v2", Get("bar"));
561 Close();
562
563 // Reopen and flush memtable.
564 ASSERT_OK(TryReopen(options));
565 ASSERT_OK(Flush());
566 Close();
567 // Now check keys in read only mode.
568 ASSERT_OK(ReadOnlyReopen(options));
569 }
570
TEST_F(StatsHistoryTest,ForceManualFlushStatsCF)571 TEST_F(StatsHistoryTest, ForceManualFlushStatsCF) {
572 constexpr int kPeriodSec = 5;
573 Options options;
574 options.create_if_missing = true;
575 options.write_buffer_size = 1024 * 1024 * 10; // 10 Mb
576 options.stats_persist_period_sec = kPeriodSec;
577 options.statistics = CreateDBStatistics();
578 options.persist_stats_to_disk = true;
579 options.env = mock_env_.get();
580 CreateColumnFamilies({"pikachu"}, options);
581 ReopenWithColumnFamilies({"default", "pikachu"}, options);
582
583 // Wait for the first stats persist to finish, as the initial delay could be
584 // different.
585 dbfull()->TEST_WaitForStatsDumpRun(
586 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec - 1); });
587
588 ColumnFamilyData* cfd_default =
589 static_cast<ColumnFamilyHandleImpl*>(dbfull()->DefaultColumnFamily())
590 ->cfd();
591 ColumnFamilyData* cfd_stats = static_cast<ColumnFamilyHandleImpl*>(
592 dbfull()->PersistentStatsColumnFamily())
593 ->cfd();
594 ColumnFamilyData* cfd_test =
595 static_cast<ColumnFamilyHandleImpl*>(handles_[1])->cfd();
596
597 ASSERT_OK(Put("foo", "v0"));
598 ASSERT_OK(Put("bar", "v0"));
599 ASSERT_EQ("v0", Get("bar"));
600 ASSERT_EQ("v0", Get("foo"));
601 ASSERT_OK(Put(1, "Eevee", "v0"));
602 ASSERT_EQ("v0", Get(1, "Eevee"));
603
604 dbfull()->TEST_WaitForStatsDumpRun(
605 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
606 // writing to all three cf, flush default cf
607 // LogNumbers: default: 14, stats: 4, pikachu: 4
608 ASSERT_OK(Flush());
609 ASSERT_EQ(cfd_stats->GetLogNumber(), cfd_test->GetLogNumber());
610 ASSERT_LT(cfd_stats->GetLogNumber(), cfd_default->GetLogNumber());
611
612 ASSERT_OK(Put("foo1", "v1"));
613 ASSERT_OK(Put("bar1", "v1"));
614 ASSERT_EQ("v1", Get("bar1"));
615 ASSERT_EQ("v1", Get("foo1"));
616 ASSERT_OK(Put(1, "Vaporeon", "v1"));
617 ASSERT_EQ("v1", Get(1, "Vaporeon"));
618 // writing to default and test cf, flush test cf
619 // LogNumbers: default: 14, stats: 16, pikachu: 16
620 ASSERT_OK(Flush(1));
621 ASSERT_EQ(cfd_stats->GetLogNumber(), cfd_test->GetLogNumber());
622 ASSERT_GT(cfd_stats->GetLogNumber(), cfd_default->GetLogNumber());
623
624 ASSERT_OK(Put("foo2", "v2"));
625 ASSERT_OK(Put("bar2", "v2"));
626 ASSERT_EQ("v2", Get("bar2"));
627 ASSERT_EQ("v2", Get("foo2"));
628
629 dbfull()->TEST_WaitForStatsDumpRun(
630 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
631 // writing to default and stats cf, flushing default cf
632 // LogNumbers: default: 19, stats: 19, pikachu: 19
633 ASSERT_OK(Flush());
634 ASSERT_EQ(cfd_stats->GetLogNumber(), cfd_test->GetLogNumber());
635 ASSERT_EQ(cfd_stats->GetLogNumber(), cfd_default->GetLogNumber());
636
637 ASSERT_OK(Put("foo3", "v3"));
638 ASSERT_OK(Put("bar3", "v3"));
639 ASSERT_EQ("v3", Get("bar3"));
640 ASSERT_EQ("v3", Get("foo3"));
641 ASSERT_OK(Put(1, "Jolteon", "v3"));
642 ASSERT_EQ("v3", Get(1, "Jolteon"));
643
644 dbfull()->TEST_WaitForStatsDumpRun(
645 [&] { mock_clock_->MockSleepForSeconds(kPeriodSec); });
646 // writing to all three cf, flushing test cf
647 // LogNumbers: default: 19, stats: 19, pikachu: 22
648 ASSERT_OK(Flush(1));
649 ASSERT_LT(cfd_stats->GetLogNumber(), cfd_test->GetLogNumber());
650 ASSERT_EQ(cfd_stats->GetLogNumber(), cfd_default->GetLogNumber());
651 Close();
652 }
653
654 #endif // !ROCKSDB_LITE
655 } // namespace ROCKSDB_NAMESPACE
656
main(int argc,char ** argv)657 int main(int argc, char** argv) {
658 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
659 ::testing::InitGoogleTest(&argc, argv);
660 return RUN_ALL_TESTS();
661 }
662