1 // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
2 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file. See the AUTHORS file for names of contributors.
5 
6 #ifndef ROCKSDB_LITE
7 
8 #include <map>
9 #include <memory>
10 #include "rocksdb/compaction_filter.h"
11 #include "rocksdb/utilities/db_ttl.h"
12 #include "test_util/testharness.h"
13 #include "util/string_util.h"
14 #ifndef OS_WIN
15 #include <unistd.h>
16 #endif
17 
18 namespace ROCKSDB_NAMESPACE {
19 
20 namespace {
21 
22 typedef std::map<std::string, std::string> KVMap;
23 
24 enum BatchOperation { OP_PUT = 0, OP_DELETE = 1 };
25 }
26 
27 class SpecialTimeEnv : public EnvWrapper {
28  public:
SpecialTimeEnv(Env * base)29   explicit SpecialTimeEnv(Env* base) : EnvWrapper(base) {
30     base->GetCurrentTime(&current_time_);
31   }
32 
Sleep(int64_t sleep_time)33   void Sleep(int64_t sleep_time) { current_time_ += sleep_time; }
GetCurrentTime(int64_t * current_time)34   Status GetCurrentTime(int64_t* current_time) override {
35     *current_time = current_time_;
36     return Status::OK();
37   }
38 
39  private:
40   int64_t current_time_ = 0;
41 };
42 
43 class TtlTest : public testing::Test {
44  public:
TtlTest()45   TtlTest() {
46     env_.reset(new SpecialTimeEnv(Env::Default()));
47     dbname_ = test::PerThreadDBPath("db_ttl");
48     options_.create_if_missing = true;
49     options_.env = env_.get();
50     // ensure that compaction is kicked in to always strip timestamp from kvs
51     options_.max_compaction_bytes = 1;
52     // compaction should take place always from level0 for determinism
53     db_ttl_ = nullptr;
54     DestroyDB(dbname_, Options());
55   }
56 
~TtlTest()57   ~TtlTest() override {
58     CloseTtl();
59     DestroyDB(dbname_, Options());
60   }
61 
62   // Open database with TTL support when TTL not provided with db_ttl_ pointer
OpenTtl()63   void OpenTtl() {
64     ASSERT_TRUE(db_ttl_ ==
65                 nullptr);  //  db should be closed before opening again
66     ASSERT_OK(DBWithTTL::Open(options_, dbname_, &db_ttl_));
67   }
68 
69   // Open database with TTL support when TTL provided with db_ttl_ pointer
OpenTtl(int32_t ttl)70   void OpenTtl(int32_t ttl) {
71     ASSERT_TRUE(db_ttl_ == nullptr);
72     ASSERT_OK(DBWithTTL::Open(options_, dbname_, &db_ttl_, ttl));
73   }
74 
75   // Open with TestFilter compaction filter
OpenTtlWithTestCompaction(int32_t ttl)76   void OpenTtlWithTestCompaction(int32_t ttl) {
77     options_.compaction_filter_factory =
78       std::shared_ptr<CompactionFilterFactory>(
79           new TestFilterFactory(kSampleSize_, kNewValue_));
80     OpenTtl(ttl);
81   }
82 
83   // Open database with TTL support in read_only mode
OpenReadOnlyTtl(int32_t ttl)84   void OpenReadOnlyTtl(int32_t ttl) {
85     ASSERT_TRUE(db_ttl_ == nullptr);
86     ASSERT_OK(DBWithTTL::Open(options_, dbname_, &db_ttl_, ttl, true));
87   }
88 
89   // Call db_ttl_->Close() before delete db_ttl_
CloseTtl()90   void CloseTtl() { CloseTtlHelper(true); }
91 
92   // No db_ttl_->Close() before delete db_ttl_
CloseTtlNoDBClose()93   void CloseTtlNoDBClose() { CloseTtlHelper(false); }
94 
CloseTtlHelper(bool close_db)95   void CloseTtlHelper(bool close_db) {
96     if (db_ttl_ != nullptr) {
97       if (close_db) {
98         db_ttl_->Close();
99       }
100       delete db_ttl_;
101       db_ttl_ = nullptr;
102     }
103   }
104 
105   // Populates and returns a kv-map
MakeKVMap(int64_t num_entries)106   void MakeKVMap(int64_t num_entries) {
107     kvmap_.clear();
108     int digits = 1;
109     for (int64_t dummy = num_entries; dummy /= 10; ++digits) {
110     }
111     int digits_in_i = 1;
112     for (int64_t i = 0; i < num_entries; i++) {
113       std::string key = "key";
114       std::string value = "value";
115       if (i % 10 == 0) {
116         digits_in_i++;
117       }
118       for(int j = digits_in_i; j < digits; j++) {
119         key.append("0");
120         value.append("0");
121       }
122       AppendNumberTo(&key, i);
123       AppendNumberTo(&value, i);
124       kvmap_[key] = value;
125     }
126     ASSERT_EQ(static_cast<int64_t>(kvmap_.size()),
127               num_entries);  // check all insertions done
128   }
129 
130   // Makes a write-batch with key-vals from kvmap_ and 'Write''s it
MakePutWriteBatch(const BatchOperation * batch_ops,int64_t num_ops)131   void MakePutWriteBatch(const BatchOperation* batch_ops, int64_t num_ops) {
132     ASSERT_LE(num_ops, static_cast<int64_t>(kvmap_.size()));
133     static WriteOptions wopts;
134     static FlushOptions flush_opts;
135     WriteBatch batch;
136     kv_it_ = kvmap_.begin();
137     for (int64_t i = 0; i < num_ops && kv_it_ != kvmap_.end(); i++, ++kv_it_) {
138       switch (batch_ops[i]) {
139         case OP_PUT:
140           batch.Put(kv_it_->first, kv_it_->second);
141           break;
142         case OP_DELETE:
143           batch.Delete(kv_it_->first);
144           break;
145         default:
146           FAIL();
147       }
148     }
149     db_ttl_->Write(wopts, &batch);
150     db_ttl_->Flush(flush_opts);
151   }
152 
153   // Puts num_entries starting from start_pos_map from kvmap_ into the database
PutValues(int64_t start_pos_map,int64_t num_entries,bool flush=true,ColumnFamilyHandle * cf=nullptr)154   void PutValues(int64_t start_pos_map, int64_t num_entries, bool flush = true,
155                  ColumnFamilyHandle* cf = nullptr) {
156     ASSERT_TRUE(db_ttl_);
157     ASSERT_LE(start_pos_map + num_entries, static_cast<int64_t>(kvmap_.size()));
158     static WriteOptions wopts;
159     static FlushOptions flush_opts;
160     kv_it_ = kvmap_.begin();
161     advance(kv_it_, start_pos_map);
162     for (int64_t i = 0; kv_it_ != kvmap_.end() && i < num_entries;
163          i++, ++kv_it_) {
164       ASSERT_OK(cf == nullptr
165                     ? db_ttl_->Put(wopts, kv_it_->first, kv_it_->second)
166                     : db_ttl_->Put(wopts, cf, kv_it_->first, kv_it_->second));
167     }
168     // Put a mock kv at the end because CompactionFilter doesn't delete last key
169     ASSERT_OK(cf == nullptr ? db_ttl_->Put(wopts, "keymock", "valuemock")
170                             : db_ttl_->Put(wopts, cf, "keymock", "valuemock"));
171     if (flush) {
172       if (cf == nullptr) {
173         db_ttl_->Flush(flush_opts);
174       } else {
175         db_ttl_->Flush(flush_opts, cf);
176       }
177     }
178   }
179 
180   // Runs a manual compaction
ManualCompact(ColumnFamilyHandle * cf=nullptr)181   void ManualCompact(ColumnFamilyHandle* cf = nullptr) {
182     if (cf == nullptr) {
183       db_ttl_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
184     } else {
185       db_ttl_->CompactRange(CompactRangeOptions(), cf, nullptr, nullptr);
186     }
187   }
188 
189   // checks the whole kvmap_ to return correct values using KeyMayExist
SimpleKeyMayExistCheck()190   void SimpleKeyMayExistCheck() {
191     static ReadOptions ropts;
192     bool value_found;
193     std::string val;
194     for(auto &kv : kvmap_) {
195       bool ret = db_ttl_->KeyMayExist(ropts, kv.first, &val, &value_found);
196       if (ret == false || value_found == false) {
197         fprintf(stderr, "KeyMayExist could not find key=%s in the database but"
198                         " should have\n", kv.first.c_str());
199         FAIL();
200       } else if (val.compare(kv.second) != 0) {
201         fprintf(stderr, " value for key=%s present in database is %s but"
202                         " should be %s\n", kv.first.c_str(), val.c_str(),
203                         kv.second.c_str());
204         FAIL();
205       }
206     }
207   }
208 
209   // checks the whole kvmap_ to return correct values using MultiGet
SimpleMultiGetTest()210   void SimpleMultiGetTest() {
211     static ReadOptions ropts;
212     std::vector<Slice> keys;
213     std::vector<std::string> values;
214 
215     for (auto& kv : kvmap_) {
216       keys.emplace_back(kv.first);
217     }
218 
219     auto statuses = db_ttl_->MultiGet(ropts, keys, &values);
220     size_t i = 0;
221     for (auto& kv : kvmap_) {
222       ASSERT_OK(statuses[i]);
223       ASSERT_EQ(values[i], kv.second);
224       ++i;
225     }
226   }
227 
228   // Sleeps for slp_tim then runs a manual compaction
229   // Checks span starting from st_pos from kvmap_ in the db and
230   // Gets should return true if check is true and false otherwise
231   // Also checks that value that we got is the same as inserted; and =kNewValue
232   //   if test_compaction_change is true
SleepCompactCheck(int slp_tim,int64_t st_pos,int64_t span,bool check=true,bool test_compaction_change=false,ColumnFamilyHandle * cf=nullptr)233   void SleepCompactCheck(int slp_tim, int64_t st_pos, int64_t span,
234                          bool check = true, bool test_compaction_change = false,
235                          ColumnFamilyHandle* cf = nullptr) {
236     ASSERT_TRUE(db_ttl_);
237 
238     env_->Sleep(slp_tim);
239     ManualCompact(cf);
240     static ReadOptions ropts;
241     kv_it_ = kvmap_.begin();
242     advance(kv_it_, st_pos);
243     std::string v;
244     for (int64_t i = 0; kv_it_ != kvmap_.end() && i < span; i++, ++kv_it_) {
245       Status s = (cf == nullptr) ? db_ttl_->Get(ropts, kv_it_->first, &v)
246                                  : db_ttl_->Get(ropts, cf, kv_it_->first, &v);
247       if (s.ok() != check) {
248         fprintf(stderr, "key=%s ", kv_it_->first.c_str());
249         if (!s.ok()) {
250           fprintf(stderr, "is absent from db but was expected to be present\n");
251         } else {
252           fprintf(stderr, "is present in db but was expected to be absent\n");
253         }
254         FAIL();
255       } else if (s.ok()) {
256           if (test_compaction_change && v.compare(kNewValue_) != 0) {
257             fprintf(stderr, " value for key=%s present in database is %s but "
258                             " should be %s\n", kv_it_->first.c_str(), v.c_str(),
259                             kNewValue_.c_str());
260             FAIL();
261           } else if (!test_compaction_change && v.compare(kv_it_->second) !=0) {
262             fprintf(stderr, " value for key=%s present in database is %s but "
263                             " should be %s\n", kv_it_->first.c_str(), v.c_str(),
264                             kv_it_->second.c_str());
265             FAIL();
266           }
267       }
268     }
269   }
270 
271   // Similar as SleepCompactCheck but uses TtlIterator to read from db
SleepCompactCheckIter(int slp,int st_pos,int64_t span,bool check=true)272   void SleepCompactCheckIter(int slp, int st_pos, int64_t span,
273                              bool check = true) {
274     ASSERT_TRUE(db_ttl_);
275     env_->Sleep(slp);
276     ManualCompact();
277     static ReadOptions ropts;
278     Iterator *dbiter = db_ttl_->NewIterator(ropts);
279     kv_it_ = kvmap_.begin();
280     advance(kv_it_, st_pos);
281 
282     dbiter->Seek(kv_it_->first);
283     if (!check) {
284       if (dbiter->Valid()) {
285         ASSERT_NE(dbiter->value().compare(kv_it_->second), 0);
286       }
287     } else {  // dbiter should have found out kvmap_[st_pos]
288       for (int64_t i = st_pos; kv_it_ != kvmap_.end() && i < st_pos + span;
289            i++, ++kv_it_) {
290         ASSERT_TRUE(dbiter->Valid());
291         ASSERT_EQ(dbiter->value().compare(kv_it_->second), 0);
292         dbiter->Next();
293       }
294     }
295     delete dbiter;
296   }
297 
298   // Set ttl on open db
SetTtl(int32_t ttl,ColumnFamilyHandle * cf=nullptr)299   void SetTtl(int32_t ttl, ColumnFamilyHandle* cf = nullptr) {
300     ASSERT_TRUE(db_ttl_);
301     cf == nullptr ? db_ttl_->SetTtl(ttl) : db_ttl_->SetTtl(cf, ttl);
302   }
303 
304   class TestFilter : public CompactionFilter {
305    public:
TestFilter(const int64_t kSampleSize,const std::string & kNewValue)306     TestFilter(const int64_t kSampleSize, const std::string& kNewValue)
307       : kSampleSize_(kSampleSize),
308         kNewValue_(kNewValue) {
309     }
310 
311     // Works on keys of the form "key<number>"
312     // Drops key if number at the end of key is in [0, kSampleSize_/3),
313     // Keeps key if it is in [kSampleSize_/3, 2*kSampleSize_/3),
314     // Change value if it is in [2*kSampleSize_/3, kSampleSize_)
315     // Eg. kSampleSize_=6. Drop:key0-1...Keep:key2-3...Change:key4-5...
Filter(int,const Slice & key,const Slice &,std::string * new_value,bool * value_changed) const316     bool Filter(int /*level*/, const Slice& key, const Slice& /*value*/,
317                 std::string* new_value, bool* value_changed) const override {
318       assert(new_value != nullptr);
319 
320       std::string search_str = "0123456789";
321       std::string key_string = key.ToString();
322       size_t pos = key_string.find_first_of(search_str);
323       int num_key_end;
324       if (pos != std::string::npos) {
325         auto key_substr = key_string.substr(pos, key.size() - pos);
326 #ifndef CYGWIN
327         num_key_end = std::stoi(key_substr);
328 #else
329         num_key_end = std::strtol(key_substr.c_str(), 0, 10);
330 #endif
331 
332       } else {
333         return false; // Keep keys not matching the format "key<NUMBER>"
334       }
335 
336       int64_t partition = kSampleSize_ / 3;
337       if (num_key_end < partition) {
338         return true;
339       } else if (num_key_end < partition * 2) {
340         return false;
341       } else {
342         *new_value = kNewValue_;
343         *value_changed = true;
344         return false;
345       }
346     }
347 
Name() const348     const char* Name() const override { return "TestFilter"; }
349 
350    private:
351     const int64_t kSampleSize_;
352     const std::string kNewValue_;
353   };
354 
355   class TestFilterFactory : public CompactionFilterFactory {
356     public:
TestFilterFactory(const int64_t kSampleSize,const std::string & kNewValue)357       TestFilterFactory(const int64_t kSampleSize, const std::string& kNewValue)
358         : kSampleSize_(kSampleSize),
359           kNewValue_(kNewValue) {
360       }
361 
CreateCompactionFilter(const CompactionFilter::Context &)362       std::unique_ptr<CompactionFilter> CreateCompactionFilter(
363           const CompactionFilter::Context& /*context*/) override {
364         return std::unique_ptr<CompactionFilter>(
365             new TestFilter(kSampleSize_, kNewValue_));
366       }
367 
Name() const368       const char* Name() const override { return "TestFilterFactory"; }
369 
370      private:
371       const int64_t kSampleSize_;
372       const std::string kNewValue_;
373   };
374 
375 
376   // Choose carefully so that Put, Gets & Compaction complete in 1 second buffer
377   static const int64_t kSampleSize_ = 100;
378   std::string dbname_;
379   DBWithTTL* db_ttl_;
380   std::unique_ptr<SpecialTimeEnv> env_;
381 
382  private:
383   Options options_;
384   KVMap kvmap_;
385   KVMap::iterator kv_it_;
386   const std::string kNewValue_ = "new_value";
387   std::unique_ptr<CompactionFilter> test_comp_filter_;
388 }; // class TtlTest
389 
390 // If TTL is non positive or not provided, the behaviour is TTL = infinity
391 // This test opens the db 3 times with such default behavior and inserts a
392 // bunch of kvs each time. All kvs should accumulate in the db till the end
393 // Partitions the sample-size provided into 3 sets over boundary1 and boundary2
TEST_F(TtlTest,NoEffect)394 TEST_F(TtlTest, NoEffect) {
395   MakeKVMap(kSampleSize_);
396   int64_t boundary1 = kSampleSize_ / 3;
397   int64_t boundary2 = 2 * boundary1;
398 
399   OpenTtl();
400   PutValues(0, boundary1);                       //T=0: Set1 never deleted
401   SleepCompactCheck(1, 0, boundary1);            //T=1: Set1 still there
402   CloseTtl();
403 
404   OpenTtl(0);
405   PutValues(boundary1, boundary2 - boundary1);   //T=1: Set2 never deleted
406   SleepCompactCheck(1, 0, boundary2);            //T=2: Sets1 & 2 still there
407   CloseTtl();
408 
409   OpenTtl(-1);
410   PutValues(boundary2, kSampleSize_ - boundary2); //T=3: Set3 never deleted
411   SleepCompactCheck(1, 0, kSampleSize_, true);    //T=4: Sets 1,2,3 still there
412   CloseTtl();
413 }
414 
415 // Rerun the NoEffect test with a different version of CloseTtl
416 // function, where db is directly deleted without close.
TEST_F(TtlTest,DestructWithoutClose)417 TEST_F(TtlTest, DestructWithoutClose) {
418   MakeKVMap(kSampleSize_);
419   int64_t boundary1 = kSampleSize_ / 3;
420   int64_t boundary2 = 2 * boundary1;
421 
422   OpenTtl();
423   PutValues(0, boundary1);             // T=0: Set1 never deleted
424   SleepCompactCheck(1, 0, boundary1);  // T=1: Set1 still there
425   CloseTtlNoDBClose();
426 
427   OpenTtl(0);
428   PutValues(boundary1, boundary2 - boundary1);  // T=1: Set2 never deleted
429   SleepCompactCheck(1, 0, boundary2);           // T=2: Sets1 & 2 still there
430   CloseTtlNoDBClose();
431 
432   OpenTtl(-1);
433   PutValues(boundary2, kSampleSize_ - boundary2);  // T=3: Set3 never deleted
434   SleepCompactCheck(1, 0, kSampleSize_, true);  // T=4: Sets 1,2,3 still there
435   CloseTtlNoDBClose();
436 }
437 
438 // Puts a set of values and checks its presence using Get during ttl
TEST_F(TtlTest,PresentDuringTTL)439 TEST_F(TtlTest, PresentDuringTTL) {
440   MakeKVMap(kSampleSize_);
441 
442   OpenTtl(2);                                 // T=0:Open the db with ttl = 2
443   PutValues(0, kSampleSize_);                  // T=0:Insert Set1. Delete at t=2
444   SleepCompactCheck(1, 0, kSampleSize_, true); // T=1:Set1 should still be there
445   CloseTtl();
446 }
447 
448 // Puts a set of values and checks its absence using Get after ttl
TEST_F(TtlTest,AbsentAfterTTL)449 TEST_F(TtlTest, AbsentAfterTTL) {
450   MakeKVMap(kSampleSize_);
451 
452   OpenTtl(1);                                  // T=0:Open the db with ttl = 2
453   PutValues(0, kSampleSize_);                  // T=0:Insert Set1. Delete at t=2
454   SleepCompactCheck(2, 0, kSampleSize_, false); // T=2:Set1 should not be there
455   CloseTtl();
456 }
457 
458 // Resets the timestamp of a set of kvs by updating them and checks that they
459 // are not deleted according to the old timestamp
TEST_F(TtlTest,ResetTimestamp)460 TEST_F(TtlTest, ResetTimestamp) {
461   MakeKVMap(kSampleSize_);
462 
463   OpenTtl(3);
464   PutValues(0, kSampleSize_);            // T=0: Insert Set1. Delete at t=3
465   env_->Sleep(2);                        // T=2
466   PutValues(0, kSampleSize_);            // T=2: Insert Set1. Delete at t=5
467   SleepCompactCheck(2, 0, kSampleSize_); // T=4: Set1 should still be there
468   CloseTtl();
469 }
470 
471 // Similar to PresentDuringTTL but uses Iterator
TEST_F(TtlTest,IterPresentDuringTTL)472 TEST_F(TtlTest, IterPresentDuringTTL) {
473   MakeKVMap(kSampleSize_);
474 
475   OpenTtl(2);
476   PutValues(0, kSampleSize_);                 // T=0: Insert. Delete at t=2
477   SleepCompactCheckIter(1, 0, kSampleSize_);  // T=1: Set should be there
478   CloseTtl();
479 }
480 
481 // Similar to AbsentAfterTTL but uses Iterator
TEST_F(TtlTest,IterAbsentAfterTTL)482 TEST_F(TtlTest, IterAbsentAfterTTL) {
483   MakeKVMap(kSampleSize_);
484 
485   OpenTtl(1);
486   PutValues(0, kSampleSize_);                      // T=0: Insert. Delete at t=1
487   SleepCompactCheckIter(2, 0, kSampleSize_, false); // T=2: Should not be there
488   CloseTtl();
489 }
490 
491 // Checks presence while opening the same db more than once with the same ttl
492 // Note: The second open will open the same db
TEST_F(TtlTest,MultiOpenSamePresent)493 TEST_F(TtlTest, MultiOpenSamePresent) {
494   MakeKVMap(kSampleSize_);
495 
496   OpenTtl(2);
497   PutValues(0, kSampleSize_);                   // T=0: Insert. Delete at t=2
498   CloseTtl();
499 
500   OpenTtl(2);                                  // T=0. Delete at t=2
501   SleepCompactCheck(1, 0, kSampleSize_);        // T=1: Set should be there
502   CloseTtl();
503 }
504 
505 // Checks absence while opening the same db more than once with the same ttl
506 // Note: The second open will open the same db
TEST_F(TtlTest,MultiOpenSameAbsent)507 TEST_F(TtlTest, MultiOpenSameAbsent) {
508   MakeKVMap(kSampleSize_);
509 
510   OpenTtl(1);
511   PutValues(0, kSampleSize_);                   // T=0: Insert. Delete at t=1
512   CloseTtl();
513 
514   OpenTtl(1);                                  // T=0.Delete at t=1
515   SleepCompactCheck(2, 0, kSampleSize_, false); // T=2: Set should not be there
516   CloseTtl();
517 }
518 
519 // Checks presence while opening the same db more than once with bigger ttl
TEST_F(TtlTest,MultiOpenDifferent)520 TEST_F(TtlTest, MultiOpenDifferent) {
521   MakeKVMap(kSampleSize_);
522 
523   OpenTtl(1);
524   PutValues(0, kSampleSize_);            // T=0: Insert. Delete at t=1
525   CloseTtl();
526 
527   OpenTtl(3);                           // T=0: Set deleted at t=3
528   SleepCompactCheck(2, 0, kSampleSize_); // T=2: Set should be there
529   CloseTtl();
530 }
531 
532 // Checks presence during ttl in read_only mode
TEST_F(TtlTest,ReadOnlyPresentForever)533 TEST_F(TtlTest, ReadOnlyPresentForever) {
534   MakeKVMap(kSampleSize_);
535 
536   OpenTtl(1);                                 // T=0:Open the db normally
537   PutValues(0, kSampleSize_);                  // T=0:Insert Set1. Delete at t=1
538   CloseTtl();
539 
540   OpenReadOnlyTtl(1);
541   SleepCompactCheck(2, 0, kSampleSize_);       // T=2:Set1 should still be there
542   CloseTtl();
543 }
544 
545 // Checks whether WriteBatch works well with TTL
546 // Puts all kvs in kvmap_ in a batch and writes first, then deletes first half
TEST_F(TtlTest,WriteBatchTest)547 TEST_F(TtlTest, WriteBatchTest) {
548   MakeKVMap(kSampleSize_);
549   BatchOperation batch_ops[kSampleSize_];
550   for (int i = 0; i < kSampleSize_; i++) {
551     batch_ops[i] = OP_PUT;
552   }
553 
554   OpenTtl(2);
555   MakePutWriteBatch(batch_ops, kSampleSize_);
556   for (int i = 0; i < kSampleSize_ / 2; i++) {
557     batch_ops[i] = OP_DELETE;
558   }
559   MakePutWriteBatch(batch_ops, kSampleSize_ / 2);
560   SleepCompactCheck(0, 0, kSampleSize_ / 2, false);
561   SleepCompactCheck(0, kSampleSize_ / 2, kSampleSize_ - kSampleSize_ / 2);
562   CloseTtl();
563 }
564 
565 // Checks user's compaction filter for correctness with TTL logic
TEST_F(TtlTest,CompactionFilter)566 TEST_F(TtlTest, CompactionFilter) {
567   MakeKVMap(kSampleSize_);
568 
569   OpenTtlWithTestCompaction(1);
570   PutValues(0, kSampleSize_);                  // T=0:Insert Set1. Delete at t=1
571   // T=2: TTL logic takes precedence over TestFilter:-Set1 should not be there
572   SleepCompactCheck(2, 0, kSampleSize_, false);
573   CloseTtl();
574 
575   OpenTtlWithTestCompaction(3);
576   PutValues(0, kSampleSize_);                   // T=0:Insert Set1.
577   int64_t partition = kSampleSize_ / 3;
578   SleepCompactCheck(1, 0, partition, false);                  // Part dropped
579   SleepCompactCheck(0, partition, partition);                 // Part kept
580   SleepCompactCheck(0, 2 * partition, partition, true, true); // Part changed
581   CloseTtl();
582 }
583 
584 // Insert some key-values which KeyMayExist should be able to get and check that
585 // values returned are fine
TEST_F(TtlTest,KeyMayExist)586 TEST_F(TtlTest, KeyMayExist) {
587   MakeKVMap(kSampleSize_);
588 
589   OpenTtl();
590   PutValues(0, kSampleSize_, false);
591 
592   SimpleKeyMayExistCheck();
593 
594   CloseTtl();
595 }
596 
TEST_F(TtlTest,MultiGetTest)597 TEST_F(TtlTest, MultiGetTest) {
598   MakeKVMap(kSampleSize_);
599 
600   OpenTtl();
601   PutValues(0, kSampleSize_, false);
602 
603   SimpleMultiGetTest();
604 
605   CloseTtl();
606 }
607 
TEST_F(TtlTest,ColumnFamiliesTest)608 TEST_F(TtlTest, ColumnFamiliesTest) {
609   DB* db;
610   Options options;
611   options.create_if_missing = true;
612   options.env = env_.get();
613 
614   DB::Open(options, dbname_, &db);
615   ColumnFamilyHandle* handle;
616   ASSERT_OK(db->CreateColumnFamily(ColumnFamilyOptions(options),
617                                    "ttl_column_family", &handle));
618 
619   delete handle;
620   delete db;
621 
622   std::vector<ColumnFamilyDescriptor> column_families;
623   column_families.push_back(ColumnFamilyDescriptor(
624       kDefaultColumnFamilyName, ColumnFamilyOptions(options)));
625   column_families.push_back(ColumnFamilyDescriptor(
626       "ttl_column_family", ColumnFamilyOptions(options)));
627 
628   std::vector<ColumnFamilyHandle*> handles;
629 
630   ASSERT_OK(DBWithTTL::Open(DBOptions(options), dbname_, column_families,
631                             &handles, &db_ttl_, {3, 5}, false));
632   ASSERT_EQ(handles.size(), 2U);
633   ColumnFamilyHandle* new_handle;
634   ASSERT_OK(db_ttl_->CreateColumnFamilyWithTtl(options, "ttl_column_family_2",
635                                                &new_handle, 2));
636   handles.push_back(new_handle);
637 
638   MakeKVMap(kSampleSize_);
639   PutValues(0, kSampleSize_, false, handles[0]);
640   PutValues(0, kSampleSize_, false, handles[1]);
641   PutValues(0, kSampleSize_, false, handles[2]);
642 
643   // everything should be there after 1 second
644   SleepCompactCheck(1, 0, kSampleSize_, true, false, handles[0]);
645   SleepCompactCheck(0, 0, kSampleSize_, true, false, handles[1]);
646   SleepCompactCheck(0, 0, kSampleSize_, true, false, handles[2]);
647 
648   // only column family 1 should be alive after 4 seconds
649   SleepCompactCheck(3, 0, kSampleSize_, false, false, handles[0]);
650   SleepCompactCheck(0, 0, kSampleSize_, true, false, handles[1]);
651   SleepCompactCheck(0, 0, kSampleSize_, false, false, handles[2]);
652 
653   // nothing should be there after 6 seconds
654   SleepCompactCheck(2, 0, kSampleSize_, false, false, handles[0]);
655   SleepCompactCheck(0, 0, kSampleSize_, false, false, handles[1]);
656   SleepCompactCheck(0, 0, kSampleSize_, false, false, handles[2]);
657 
658   for (auto h : handles) {
659     delete h;
660   }
661   delete db_ttl_;
662   db_ttl_ = nullptr;
663 }
664 
665 // Puts a set of values and checks its absence using Get after ttl
TEST_F(TtlTest,ChangeTtlOnOpenDb)666 TEST_F(TtlTest, ChangeTtlOnOpenDb) {
667   MakeKVMap(kSampleSize_);
668 
669   OpenTtl(1);                                  // T=0:Open the db with ttl = 2
670   SetTtl(3);
671   // @lint-ignore TXT2 T25377293 Grandfathered in
672   PutValues(0, kSampleSize_);		       // T=0:Insert Set1. Delete at t=2
673   SleepCompactCheck(2, 0, kSampleSize_, true); // T=2:Set1 should be there
674   CloseTtl();
675 }
676 
677 }  // namespace ROCKSDB_NAMESPACE
678 
679 // A black-box test for the ttl wrapper around rocksdb
main(int argc,char ** argv)680 int main(int argc, char** argv) {
681   ::testing::InitGoogleTest(&argc, argv);
682   return RUN_ALL_TESTS();
683 }
684 
685 #else
686 #include <stdio.h>
687 
main(int,char **)688 int main(int /*argc*/, char** /*argv*/) {
689   fprintf(stderr, "SKIPPED as DBWithTTL is not supported in ROCKSDB_LITE\n");
690   return 0;
691 }
692 
693 #endif  // !ROCKSDB_LITE
694