1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6
7 /**
8 * An persistent map : key -> (list of strings), using rocksdb merge.
9 * This file is a test-harness / use-case for the StringAppendOperator.
10 *
11 * @author Deon Nicholas (dnicholas@fb.com)
12 * Copyright 2013 Facebook, Inc.
13 */
14
15 #include "utilities/merge_operators/string_append/stringappend.h"
16
17 #include <iostream>
18 #include <map>
19 #include <tuple>
20
21 #include "port/stack_trace.h"
22 #include "rocksdb/db.h"
23 #include "rocksdb/merge_operator.h"
24 #include "rocksdb/utilities/db_ttl.h"
25 #include "test_util/testharness.h"
26 #include "util/random.h"
27 #include "utilities/merge_operators.h"
28 #include "utilities/merge_operators/string_append/stringappend2.h"
29
30
31 namespace ROCKSDB_NAMESPACE {
32
33 // Path to the database on file system
34 const std::string kDbName = test::PerThreadDBPath("stringappend_test");
35
36 namespace {
37 // OpenDb opens a (possibly new) rocksdb database with a StringAppendOperator
OpenNormalDb(char delim_char)38 std::shared_ptr<DB> OpenNormalDb(char delim_char) {
39 DB* db;
40 Options options;
41 options.create_if_missing = true;
42 options.merge_operator.reset(new StringAppendOperator(delim_char));
43 EXPECT_OK(DB::Open(options, kDbName, &db));
44 return std::shared_ptr<DB>(db);
45 }
46
47 #ifndef ROCKSDB_LITE // TtlDb is not supported in Lite
48 // Open a TtlDB with a non-associative StringAppendTESTOperator
OpenTtlDb(char delim_char)49 std::shared_ptr<DB> OpenTtlDb(char delim_char) {
50 DBWithTTL* db;
51 Options options;
52 options.create_if_missing = true;
53 options.merge_operator.reset(new StringAppendTESTOperator(delim_char));
54 EXPECT_OK(DBWithTTL::Open(options, kDbName, &db, 123456));
55 return std::shared_ptr<DB>(db);
56 }
57 #endif // !ROCKSDB_LITE
58 } // namespace
59
60 /// StringLists represents a set of string-lists, each with a key-index.
61 /// Supports Append(list, string) and Get(list)
62 class StringLists {
63 public:
64
65 //Constructor: specifies the rocksdb db
66 /* implicit */
StringLists(std::shared_ptr<DB> db)67 StringLists(std::shared_ptr<DB> db)
68 : db_(db),
69 merge_option_(),
70 get_option_() {
71 assert(db);
72 }
73
74 // Append string val onto the list defined by key; return true on success
Append(const std::string & key,const std::string & val)75 bool Append(const std::string& key, const std::string& val){
76 Slice valSlice(val.data(), val.size());
77 auto s = db_->Merge(merge_option_, key, valSlice);
78
79 if (s.ok()) {
80 return true;
81 } else {
82 std::cerr << "ERROR " << s.ToString() << std::endl;
83 return false;
84 }
85 }
86
87 // Returns the list of strings associated with key (or "" if does not exist)
Get(const std::string & key,std::string * const result)88 bool Get(const std::string& key, std::string* const result){
89 assert(result != nullptr); // we should have a place to store the result
90 auto s = db_->Get(get_option_, key, result);
91
92 if (s.ok()) {
93 return true;
94 }
95
96 // Either key does not exist, or there is some error.
97 *result = ""; // Always return empty string (just for convention)
98
99 //NotFound is okay; just return empty (similar to std::map)
100 //But network or db errors, etc, should fail the test (or at least yell)
101 if (!s.IsNotFound()) {
102 std::cerr << "ERROR " << s.ToString() << std::endl;
103 }
104
105 // Always return false if s.ok() was not true
106 return false;
107 }
108
109
110 private:
111 std::shared_ptr<DB> db_;
112 WriteOptions merge_option_;
113 ReadOptions get_option_;
114
115 };
116
117
118 // The class for unit-testing
119 class StringAppendOperatorTest : public testing::Test,
120 public ::testing::WithParamInterface<bool> {
121 public:
StringAppendOperatorTest()122 StringAppendOperatorTest() {
123 EXPECT_OK(
124 DestroyDB(kDbName, Options())); // Start each test with a fresh DB
125 }
126
SetUp()127 void SetUp() override {
128 #ifndef ROCKSDB_LITE // TtlDb is not supported in Lite
129 bool if_use_ttl = GetParam();
130 if (if_use_ttl) {
131 fprintf(stderr, "Running tests with ttl db and generic operator.\n");
132 StringAppendOperatorTest::SetOpenDbFunction(&OpenTtlDb);
133 return;
134 }
135 #endif // !ROCKSDB_LITE
136 fprintf(stderr, "Running tests with regular db and operator.\n");
137 StringAppendOperatorTest::SetOpenDbFunction(&OpenNormalDb);
138 }
139
140 typedef std::shared_ptr<DB> (* OpenFuncPtr)(char);
141
142 // Allows user to open databases with different configurations.
143 // e.g.: Can open a DB or a TtlDB, etc.
SetOpenDbFunction(OpenFuncPtr func)144 static void SetOpenDbFunction(OpenFuncPtr func) {
145 OpenDb = func;
146 }
147
148 protected:
149 static OpenFuncPtr OpenDb;
150 };
151 StringAppendOperatorTest::OpenFuncPtr StringAppendOperatorTest::OpenDb = nullptr;
152
153 // THE TEST CASES BEGIN HERE
154
TEST_P(StringAppendOperatorTest,IteratorTest)155 TEST_P(StringAppendOperatorTest, IteratorTest) {
156 auto db_ = OpenDb(',');
157 StringLists slists(db_);
158
159 slists.Append("k1", "v1");
160 slists.Append("k1", "v2");
161 slists.Append("k1", "v3");
162
163 slists.Append("k2", "a1");
164 slists.Append("k2", "a2");
165 slists.Append("k2", "a3");
166
167 std::string res;
168 std::unique_ptr<ROCKSDB_NAMESPACE::Iterator> it(
169 db_->NewIterator(ReadOptions()));
170 std::string k1("k1");
171 std::string k2("k2");
172 bool first = true;
173 for (it->Seek(k1); it->Valid(); it->Next()) {
174 res = it->value().ToString();
175 if (first) {
176 ASSERT_EQ(res, "v1,v2,v3");
177 first = false;
178 } else {
179 ASSERT_EQ(res, "a1,a2,a3");
180 }
181 }
182 slists.Append("k2", "a4");
183 slists.Append("k1", "v4");
184
185 // Snapshot should still be the same. Should ignore a4 and v4.
186 first = true;
187 for (it->Seek(k1); it->Valid(); it->Next()) {
188 res = it->value().ToString();
189 if (first) {
190 ASSERT_EQ(res, "v1,v2,v3");
191 first = false;
192 } else {
193 ASSERT_EQ(res, "a1,a2,a3");
194 }
195 }
196
197
198 // Should release the snapshot and be aware of the new stuff now
199 it.reset(db_->NewIterator(ReadOptions()));
200 first = true;
201 for (it->Seek(k1); it->Valid(); it->Next()) {
202 res = it->value().ToString();
203 if (first) {
204 ASSERT_EQ(res, "v1,v2,v3,v4");
205 first = false;
206 } else {
207 ASSERT_EQ(res, "a1,a2,a3,a4");
208 }
209 }
210
211 // start from k2 this time.
212 for (it->Seek(k2); it->Valid(); it->Next()) {
213 res = it->value().ToString();
214 if (first) {
215 ASSERT_EQ(res, "v1,v2,v3,v4");
216 first = false;
217 } else {
218 ASSERT_EQ(res, "a1,a2,a3,a4");
219 }
220 }
221
222 slists.Append("k3", "g1");
223
224 it.reset(db_->NewIterator(ReadOptions()));
225 first = true;
226 std::string k3("k3");
227 for(it->Seek(k2); it->Valid(); it->Next()) {
228 res = it->value().ToString();
229 if (first) {
230 ASSERT_EQ(res, "a1,a2,a3,a4");
231 first = false;
232 } else {
233 ASSERT_EQ(res, "g1");
234 }
235 }
236 for(it->Seek(k3); it->Valid(); it->Next()) {
237 res = it->value().ToString();
238 if (first) {
239 // should not be hit
240 ASSERT_EQ(res, "a1,a2,a3,a4");
241 first = false;
242 } else {
243 ASSERT_EQ(res, "g1");
244 }
245 }
246 }
247
TEST_P(StringAppendOperatorTest,SimpleTest)248 TEST_P(StringAppendOperatorTest, SimpleTest) {
249 auto db = OpenDb(',');
250 StringLists slists(db);
251
252 slists.Append("k1", "v1");
253 slists.Append("k1", "v2");
254 slists.Append("k1", "v3");
255
256 std::string res;
257 ASSERT_TRUE(slists.Get("k1", &res));
258 ASSERT_EQ(res, "v1,v2,v3");
259 }
260
TEST_P(StringAppendOperatorTest,SimpleDelimiterTest)261 TEST_P(StringAppendOperatorTest, SimpleDelimiterTest) {
262 auto db = OpenDb('|');
263 StringLists slists(db);
264
265 slists.Append("k1", "v1");
266 slists.Append("k1", "v2");
267 slists.Append("k1", "v3");
268
269 std::string res;
270 ASSERT_TRUE(slists.Get("k1", &res));
271 ASSERT_EQ(res, "v1|v2|v3");
272 }
273
TEST_P(StringAppendOperatorTest,OneValueNoDelimiterTest)274 TEST_P(StringAppendOperatorTest, OneValueNoDelimiterTest) {
275 auto db = OpenDb('!');
276 StringLists slists(db);
277
278 slists.Append("random_key", "single_val");
279
280 std::string res;
281 ASSERT_TRUE(slists.Get("random_key", &res));
282 ASSERT_EQ(res, "single_val");
283 }
284
TEST_P(StringAppendOperatorTest,VariousKeys)285 TEST_P(StringAppendOperatorTest, VariousKeys) {
286 auto db = OpenDb('\n');
287 StringLists slists(db);
288
289 slists.Append("c", "asdasd");
290 slists.Append("a", "x");
291 slists.Append("b", "y");
292 slists.Append("a", "t");
293 slists.Append("a", "r");
294 slists.Append("b", "2");
295 slists.Append("c", "asdasd");
296
297 std::string a, b, c;
298 bool sa, sb, sc;
299 sa = slists.Get("a", &a);
300 sb = slists.Get("b", &b);
301 sc = slists.Get("c", &c);
302
303 ASSERT_TRUE(sa && sb && sc); // All three keys should have been found
304
305 ASSERT_EQ(a, "x\nt\nr");
306 ASSERT_EQ(b, "y\n2");
307 ASSERT_EQ(c, "asdasd\nasdasd");
308 }
309
310 // Generate semi random keys/words from a small distribution.
TEST_P(StringAppendOperatorTest,RandomMixGetAppend)311 TEST_P(StringAppendOperatorTest, RandomMixGetAppend) {
312 auto db = OpenDb(' ');
313 StringLists slists(db);
314
315 // Generate a list of random keys and values
316 const int kWordCount = 15;
317 std::string words[] = {"sdasd", "triejf", "fnjsdfn", "dfjisdfsf", "342839",
318 "dsuha", "mabuais", "sadajsid", "jf9834hf", "2d9j89",
319 "dj9823jd", "a", "dk02ed2dh", "$(jd4h984$(*", "mabz"};
320 const int kKeyCount = 6;
321 std::string keys[] = {"dhaiusdhu", "denidw", "daisda", "keykey", "muki",
322 "shzassdianmd"};
323
324 // Will store a local copy of all data in order to verify correctness
325 std::map<std::string, std::string> parallel_copy;
326
327 // Generate a bunch of random queries (Append and Get)!
328 enum query_t { APPEND_OP, GET_OP, NUM_OPS };
329 Random randomGen(1337); //deterministic seed; always get same results!
330
331 const int kNumQueries = 30;
332 for (int q=0; q<kNumQueries; ++q) {
333 // Generate a random query (Append or Get) and random parameters
334 query_t query = (query_t)randomGen.Uniform((int)NUM_OPS);
335 std::string key = keys[randomGen.Uniform((int)kKeyCount)];
336 std::string word = words[randomGen.Uniform((int)kWordCount)];
337
338 // Apply the query and any checks.
339 if (query == APPEND_OP) {
340
341 // Apply the rocksdb test-harness Append defined above
342 slists.Append(key, word); //apply the rocksdb append
343
344 // Apply the similar "Append" to the parallel copy
345 if (parallel_copy[key].size() > 0) {
346 parallel_copy[key] += " " + word;
347 } else {
348 parallel_copy[key] = word;
349 }
350
351 } else if (query == GET_OP) {
352 // Assumes that a non-existent key just returns <empty>
353 std::string res;
354 slists.Get(key, &res);
355 ASSERT_EQ(res, parallel_copy[key]);
356 }
357
358 }
359 }
360
TEST_P(StringAppendOperatorTest,BIGRandomMixGetAppend)361 TEST_P(StringAppendOperatorTest, BIGRandomMixGetAppend) {
362 auto db = OpenDb(' ');
363 StringLists slists(db);
364
365 // Generate a list of random keys and values
366 const int kWordCount = 15;
367 std::string words[] = {"sdasd", "triejf", "fnjsdfn", "dfjisdfsf", "342839",
368 "dsuha", "mabuais", "sadajsid", "jf9834hf", "2d9j89",
369 "dj9823jd", "a", "dk02ed2dh", "$(jd4h984$(*", "mabz"};
370 const int kKeyCount = 6;
371 std::string keys[] = {"dhaiusdhu", "denidw", "daisda", "keykey", "muki",
372 "shzassdianmd"};
373
374 // Will store a local copy of all data in order to verify correctness
375 std::map<std::string, std::string> parallel_copy;
376
377 // Generate a bunch of random queries (Append and Get)!
378 enum query_t { APPEND_OP, GET_OP, NUM_OPS };
379 Random randomGen(9138204); // deterministic seed
380
381 const int kNumQueries = 1000;
382 for (int q=0; q<kNumQueries; ++q) {
383 // Generate a random query (Append or Get) and random parameters
384 query_t query = (query_t)randomGen.Uniform((int)NUM_OPS);
385 std::string key = keys[randomGen.Uniform((int)kKeyCount)];
386 std::string word = words[randomGen.Uniform((int)kWordCount)];
387
388 //Apply the query and any checks.
389 if (query == APPEND_OP) {
390
391 // Apply the rocksdb test-harness Append defined above
392 slists.Append(key, word); //apply the rocksdb append
393
394 // Apply the similar "Append" to the parallel copy
395 if (parallel_copy[key].size() > 0) {
396 parallel_copy[key] += " " + word;
397 } else {
398 parallel_copy[key] = word;
399 }
400
401 } else if (query == GET_OP) {
402 // Assumes that a non-existent key just returns <empty>
403 std::string res;
404 slists.Get(key, &res);
405 ASSERT_EQ(res, parallel_copy[key]);
406 }
407
408 }
409 }
410
TEST_P(StringAppendOperatorTest,PersistentVariousKeys)411 TEST_P(StringAppendOperatorTest, PersistentVariousKeys) {
412 // Perform the following operations in limited scope
413 {
414 auto db = OpenDb('\n');
415 StringLists slists(db);
416
417 slists.Append("c", "asdasd");
418 slists.Append("a", "x");
419 slists.Append("b", "y");
420 slists.Append("a", "t");
421 slists.Append("a", "r");
422 slists.Append("b", "2");
423 slists.Append("c", "asdasd");
424
425 std::string a, b, c;
426 ASSERT_TRUE(slists.Get("a", &a));
427 ASSERT_TRUE(slists.Get("b", &b));
428 ASSERT_TRUE(slists.Get("c", &c));
429
430 ASSERT_EQ(a, "x\nt\nr");
431 ASSERT_EQ(b, "y\n2");
432 ASSERT_EQ(c, "asdasd\nasdasd");
433 }
434
435 // Reopen the database (the previous changes should persist / be remembered)
436 {
437 auto db = OpenDb('\n');
438 StringLists slists(db);
439
440 slists.Append("c", "bbnagnagsx");
441 slists.Append("a", "sa");
442 slists.Append("b", "df");
443 slists.Append("a", "gh");
444 slists.Append("a", "jk");
445 slists.Append("b", "l;");
446 slists.Append("c", "rogosh");
447
448 // The previous changes should be on disk (L0)
449 // The most recent changes should be in memory (MemTable)
450 // Hence, this will test both Get() paths.
451 std::string a, b, c;
452 ASSERT_TRUE(slists.Get("a", &a));
453 ASSERT_TRUE(slists.Get("b", &b));
454 ASSERT_TRUE(slists.Get("c", &c));
455
456 ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
457 ASSERT_EQ(b, "y\n2\ndf\nl;");
458 ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
459 }
460
461 // Reopen the database (the previous changes should persist / be remembered)
462 {
463 auto db = OpenDb('\n');
464 StringLists slists(db);
465
466 // All changes should be on disk. This will test VersionSet Get()
467 std::string a, b, c;
468 ASSERT_TRUE(slists.Get("a", &a));
469 ASSERT_TRUE(slists.Get("b", &b));
470 ASSERT_TRUE(slists.Get("c", &c));
471
472 ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
473 ASSERT_EQ(b, "y\n2\ndf\nl;");
474 ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
475 }
476 }
477
TEST_P(StringAppendOperatorTest,PersistentFlushAndCompaction)478 TEST_P(StringAppendOperatorTest, PersistentFlushAndCompaction) {
479 // Perform the following operations in limited scope
480 {
481 auto db = OpenDb('\n');
482 StringLists slists(db);
483 std::string a, b, c;
484
485 // Append, Flush, Get
486 slists.Append("c", "asdasd");
487 ASSERT_OK(db->Flush(ROCKSDB_NAMESPACE::FlushOptions()));
488 ASSERT_TRUE(slists.Get("c", &c));
489 ASSERT_EQ(c, "asdasd");
490
491 // Append, Flush, Append, Get
492 slists.Append("a", "x");
493 slists.Append("b", "y");
494 ASSERT_OK(db->Flush(ROCKSDB_NAMESPACE::FlushOptions()));
495 slists.Append("a", "t");
496 slists.Append("a", "r");
497 slists.Append("b", "2");
498
499 ASSERT_TRUE(slists.Get("a", &a));
500 ASSERT_EQ(a, "x\nt\nr");
501
502 ASSERT_TRUE(slists.Get("b", &b));
503 ASSERT_EQ(b, "y\n2");
504
505 // Append, Get
506 ASSERT_TRUE(slists.Append("c", "asdasd"));
507 ASSERT_TRUE(slists.Append("b", "monkey"));
508
509 ASSERT_TRUE(slists.Get("a", &a));
510 ASSERT_TRUE(slists.Get("b", &b));
511 ASSERT_TRUE(slists.Get("c", &c));
512
513 ASSERT_EQ(a, "x\nt\nr");
514 ASSERT_EQ(b, "y\n2\nmonkey");
515 ASSERT_EQ(c, "asdasd\nasdasd");
516 }
517
518 // Reopen the database (the previous changes should persist / be remembered)
519 {
520 auto db = OpenDb('\n');
521 StringLists slists(db);
522 std::string a, b, c;
523
524 // Get (Quick check for persistence of previous database)
525 ASSERT_TRUE(slists.Get("a", &a));
526 ASSERT_EQ(a, "x\nt\nr");
527
528 //Append, Compact, Get
529 slists.Append("c", "bbnagnagsx");
530 slists.Append("a", "sa");
531 slists.Append("b", "df");
532 ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr));
533 ASSERT_TRUE(slists.Get("a", &a));
534 ASSERT_TRUE(slists.Get("b", &b));
535 ASSERT_TRUE(slists.Get("c", &c));
536 ASSERT_EQ(a, "x\nt\nr\nsa");
537 ASSERT_EQ(b, "y\n2\nmonkey\ndf");
538 ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx");
539
540 // Append, Get
541 slists.Append("a", "gh");
542 slists.Append("a", "jk");
543 slists.Append("b", "l;");
544 slists.Append("c", "rogosh");
545 ASSERT_TRUE(slists.Get("a", &a));
546 ASSERT_TRUE(slists.Get("b", &b));
547 ASSERT_TRUE(slists.Get("c", &c));
548 ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
549 ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;");
550 ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
551
552 // Compact, Get
553 ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr));
554 ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
555 ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;");
556 ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
557
558 // Append, Flush, Compact, Get
559 slists.Append("b", "afcg");
560 ASSERT_OK(db->Flush(ROCKSDB_NAMESPACE::FlushOptions()));
561 ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr));
562 ASSERT_TRUE(slists.Get("b", &b));
563 ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;\nafcg");
564 }
565 }
566
TEST_P(StringAppendOperatorTest,SimpleTestNullDelimiter)567 TEST_P(StringAppendOperatorTest, SimpleTestNullDelimiter) {
568 auto db = OpenDb('\0');
569 StringLists slists(db);
570
571 slists.Append("k1", "v1");
572 slists.Append("k1", "v2");
573 slists.Append("k1", "v3");
574
575 std::string res;
576 ASSERT_TRUE(slists.Get("k1", &res));
577
578 // Construct the desired string. Default constructor doesn't like '\0' chars.
579 std::string checker("v1,v2,v3"); // Verify that the string is right size.
580 checker[2] = '\0'; // Use null delimiter instead of comma.
581 checker[5] = '\0';
582 ASSERT_EQ(checker.size(), 8); // Verify it is still the correct size
583
584 // Check that the rocksdb result string matches the desired string
585 ASSERT_EQ(res.size(), checker.size());
586 ASSERT_EQ(res, checker);
587 }
588
589 INSTANTIATE_TEST_CASE_P(StringAppendOperatorTest, StringAppendOperatorTest,
590 testing::Bool());
591
592 } // namespace ROCKSDB_NAMESPACE
593
main(int argc,char ** argv)594 int main(int argc, char** argv) {
595 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
596 ::testing::InitGoogleTest(&argc, argv);
597 return RUN_ALL_TESTS();
598 }
599