1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
2 //  This source code is licensed under both the GPLv2 (found in the
3 //  COPYING file in the root directory) and Apache 2.0 License
4 //  (found in the LICENSE.Apache file in the root directory).
5 
6 #include <atomic>
7 #include <cinttypes>
8 #include <thread>
9 #include <vector>
10 
11 #include "file/delete_scheduler.h"
12 #include "file/sst_file_manager_impl.h"
13 #include "rocksdb/env.h"
14 #include "rocksdb/options.h"
15 #include "test_util/sync_point.h"
16 #include "test_util/testharness.h"
17 #include "test_util/testutil.h"
18 #include "util/string_util.h"
19 
20 #ifndef ROCKSDB_LITE
21 
22 namespace ROCKSDB_NAMESPACE {
23 
24 class DeleteSchedulerTest : public testing::Test {
25  public:
DeleteSchedulerTest()26   DeleteSchedulerTest() : env_(Env::Default()) {
27     const int kNumDataDirs = 3;
28     dummy_files_dirs_.reserve(kNumDataDirs);
29     for (size_t i = 0; i < kNumDataDirs; ++i) {
30       dummy_files_dirs_.emplace_back(
31           test::PerThreadDBPath(env_, "delete_scheduler_dummy_data_dir") +
32           ToString(i));
33       DestroyAndCreateDir(dummy_files_dirs_.back());
34     }
35   }
36 
~DeleteSchedulerTest()37   ~DeleteSchedulerTest() override {
38     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
39     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({});
40     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
41     for (const auto& dummy_files_dir : dummy_files_dirs_) {
42       test::DestroyDir(env_, dummy_files_dir);
43     }
44   }
45 
DestroyAndCreateDir(const std::string & dir)46   void DestroyAndCreateDir(const std::string& dir) {
47     ASSERT_OK(test::DestroyDir(env_, dir));
48     EXPECT_OK(env_->CreateDir(dir));
49   }
50 
CountNormalFiles(size_t dummy_files_dirs_idx=0)51   int CountNormalFiles(size_t dummy_files_dirs_idx = 0) {
52     std::vector<std::string> files_in_dir;
53     EXPECT_OK(env_->GetChildren(dummy_files_dirs_[dummy_files_dirs_idx],
54                                 &files_in_dir));
55 
56     int normal_cnt = 0;
57     for (auto& f : files_in_dir) {
58       if (!DeleteScheduler::IsTrashFile(f) && f != "." && f != "..") {
59         normal_cnt++;
60       }
61     }
62     return normal_cnt;
63   }
64 
CountTrashFiles(size_t dummy_files_dirs_idx=0)65   int CountTrashFiles(size_t dummy_files_dirs_idx = 0) {
66     std::vector<std::string> files_in_dir;
67     EXPECT_OK(env_->GetChildren(dummy_files_dirs_[dummy_files_dirs_idx],
68                                 &files_in_dir));
69 
70     int trash_cnt = 0;
71     for (auto& f : files_in_dir) {
72       if (DeleteScheduler::IsTrashFile(f)) {
73         trash_cnt++;
74       }
75     }
76     return trash_cnt;
77   }
78 
NewDummyFile(const std::string & file_name,uint64_t size=1024,size_t dummy_files_dirs_idx=0)79   std::string NewDummyFile(const std::string& file_name, uint64_t size = 1024,
80                            size_t dummy_files_dirs_idx = 0) {
81     std::string file_path =
82         dummy_files_dirs_[dummy_files_dirs_idx] + "/" + file_name;
83     std::unique_ptr<WritableFile> f;
84     env_->NewWritableFile(file_path, &f, EnvOptions());
85     std::string data(size, 'A');
86     EXPECT_OK(f->Append(data));
87     EXPECT_OK(f->Close());
88     sst_file_mgr_->OnAddFile(file_path, false);
89     return file_path;
90   }
91 
NewDeleteScheduler()92   void NewDeleteScheduler() {
93     // Tests in this file are for DeleteScheduler component and dont create any
94     // DBs, so we need to set max_trash_db_ratio to 100% (instead of default
95     // 25%)
96     std::shared_ptr<FileSystem>
97                 fs(std::make_shared<LegacyFileSystemWrapper>(env_));
98     sst_file_mgr_.reset(
99         new SstFileManagerImpl(env_, fs, nullptr, rate_bytes_per_sec_,
100                                /* max_trash_db_ratio= */ 1.1, 128 * 1024));
101     delete_scheduler_ = sst_file_mgr_->delete_scheduler();
102   }
103 
104   Env* env_;
105   std::vector<std::string> dummy_files_dirs_;
106   int64_t rate_bytes_per_sec_;
107   DeleteScheduler* delete_scheduler_;
108   std::unique_ptr<SstFileManagerImpl> sst_file_mgr_;
109 };
110 
111 // Test the basic functionality of DeleteScheduler (Rate Limiting).
112 // 1- Create 100 dummy files
113 // 2- Delete the 100 dummy files using DeleteScheduler
114 // --- Hold DeleteScheduler::BackgroundEmptyTrash ---
115 // 3- Wait for DeleteScheduler to delete all files in trash
116 // 4- Verify that BackgroundEmptyTrash used to correct penlties for the files
117 // 5- Make sure that all created files were completely deleted
TEST_F(DeleteSchedulerTest,BasicRateLimiting)118 TEST_F(DeleteSchedulerTest, BasicRateLimiting) {
119   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
120       {"DeleteSchedulerTest::BasicRateLimiting:1",
121        "DeleteScheduler::BackgroundEmptyTrash"},
122   });
123 
124   std::vector<uint64_t> penalties;
125   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
126       "DeleteScheduler::BackgroundEmptyTrash:Wait",
127       [&](void* arg) { penalties.push_back(*(static_cast<uint64_t*>(arg))); });
128   int dir_synced = 0;
129   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
130       "DeleteScheduler::DeleteTrashFile::AfterSyncDir", [&](void* arg) {
131         dir_synced++;
132         std::string* dir = reinterpret_cast<std::string*>(arg);
133         EXPECT_EQ(dummy_files_dirs_[0], *dir);
134       });
135 
136   int num_files = 100;  // 100 files
137   uint64_t file_size = 1024;  // every file is 1 kb
138   std::vector<uint64_t> delete_kbs_per_sec = {512, 200, 100, 50, 25};
139 
140   for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) {
141     penalties.clear();
142     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace();
143     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
144 
145     DestroyAndCreateDir(dummy_files_dirs_[0]);
146     rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024;
147     NewDeleteScheduler();
148 
149     dir_synced = 0;
150     // Create 100 dummy files, every file is 1 Kb
151     std::vector<std::string> generated_files;
152     for (int i = 0; i < num_files; i++) {
153       std::string file_name = "file" + ToString(i) + ".data";
154       generated_files.push_back(NewDummyFile(file_name, file_size));
155     }
156 
157     // Delete dummy files and measure time spent to empty trash
158     for (int i = 0; i < num_files; i++) {
159       ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[i],
160                                               dummy_files_dirs_[0]));
161     }
162     ASSERT_EQ(CountNormalFiles(), 0);
163 
164     uint64_t delete_start_time = env_->NowMicros();
165     TEST_SYNC_POINT("DeleteSchedulerTest::BasicRateLimiting:1");
166     delete_scheduler_->WaitForEmptyTrash();
167     uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time;
168 
169     auto bg_errors = delete_scheduler_->GetBackgroundErrors();
170     ASSERT_EQ(bg_errors.size(), 0);
171 
172     uint64_t total_files_size = 0;
173     uint64_t expected_penlty = 0;
174     ASSERT_EQ(penalties.size(), num_files);
175     for (int i = 0; i < num_files; i++) {
176       total_files_size += file_size;
177       expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec_);
178       ASSERT_EQ(expected_penlty, penalties[i]);
179     }
180     ASSERT_GT(time_spent_deleting, expected_penlty * 0.9);
181 
182     ASSERT_EQ(num_files, dir_synced);
183 
184     ASSERT_EQ(CountTrashFiles(), 0);
185     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
186   }
187 }
188 
TEST_F(DeleteSchedulerTest,MultiDirectoryDeletionsScheduled)189 TEST_F(DeleteSchedulerTest, MultiDirectoryDeletionsScheduled) {
190   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
191       {"DeleteSchedulerTest::MultiDbPathDeletionsScheduled:1",
192        "DeleteScheduler::BackgroundEmptyTrash"},
193   });
194   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
195   rate_bytes_per_sec_ = 1 << 20;  // 1MB
196   NewDeleteScheduler();
197 
198   // Generate dummy files in multiple directories
199   const size_t kNumFiles = dummy_files_dirs_.size();
200   const size_t kFileSize = 1 << 10;  // 1KB
201   std::vector<std::string> generated_files;
202   for (size_t i = 0; i < kNumFiles; i++) {
203     generated_files.push_back(NewDummyFile("file", kFileSize, i));
204     ASSERT_EQ(1, CountNormalFiles(i));
205   }
206 
207   // Mark dummy files as trash
208   for (size_t i = 0; i < kNumFiles; i++) {
209     ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[i], ""));
210     ASSERT_EQ(0, CountNormalFiles(i));
211     ASSERT_EQ(1, CountTrashFiles(i));
212   }
213   TEST_SYNC_POINT("DeleteSchedulerTest::MultiDbPathDeletionsScheduled:1");
214   delete_scheduler_->WaitForEmptyTrash();
215 
216   // Verify dummy files eventually got deleted
217   for (size_t i = 0; i < kNumFiles; i++) {
218     ASSERT_EQ(0, CountNormalFiles(i));
219     ASSERT_EQ(0, CountTrashFiles(i));
220   }
221 
222   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
223 }
224 
225 // Same as the BasicRateLimiting test but delete files in multiple threads.
226 // 1- Create 100 dummy files
227 // 2- Delete the 100 dummy files using DeleteScheduler using 10 threads
228 // --- Hold DeleteScheduler::BackgroundEmptyTrash ---
229 // 3- Wait for DeleteScheduler to delete all files in queue
230 // 4- Verify that BackgroundEmptyTrash used to correct penlties for the files
231 // 5- Make sure that all created files were completely deleted
TEST_F(DeleteSchedulerTest,RateLimitingMultiThreaded)232 TEST_F(DeleteSchedulerTest, RateLimitingMultiThreaded) {
233   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
234       {"DeleteSchedulerTest::RateLimitingMultiThreaded:1",
235        "DeleteScheduler::BackgroundEmptyTrash"},
236   });
237 
238   std::vector<uint64_t> penalties;
239   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
240       "DeleteScheduler::BackgroundEmptyTrash:Wait",
241       [&](void* arg) { penalties.push_back(*(static_cast<uint64_t*>(arg))); });
242 
243   int thread_cnt = 10;
244   int num_files = 10;  // 10 files per thread
245   uint64_t file_size = 1024;  // every file is 1 kb
246 
247   std::vector<uint64_t> delete_kbs_per_sec = {512, 200, 100, 50, 25};
248   for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) {
249     penalties.clear();
250     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace();
251     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
252 
253     DestroyAndCreateDir(dummy_files_dirs_[0]);
254     rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024;
255     NewDeleteScheduler();
256 
257     // Create 100 dummy files, every file is 1 Kb
258     std::vector<std::string> generated_files;
259     for (int i = 0; i < num_files * thread_cnt; i++) {
260       std::string file_name = "file" + ToString(i) + ".data";
261       generated_files.push_back(NewDummyFile(file_name, file_size));
262     }
263 
264     // Delete dummy files using 10 threads and measure time spent to empty trash
265     std::atomic<int> thread_num(0);
266     std::vector<port::Thread> threads;
267     std::function<void()> delete_thread = [&]() {
268       int idx = thread_num.fetch_add(1);
269       int range_start = idx * num_files;
270       int range_end = range_start + num_files;
271       for (int j = range_start; j < range_end; j++) {
272         ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[j], ""));
273       }
274     };
275 
276     for (int i = 0; i < thread_cnt; i++) {
277       threads.emplace_back(delete_thread);
278     }
279 
280     for (size_t i = 0; i < threads.size(); i++) {
281       threads[i].join();
282     }
283 
284     uint64_t delete_start_time = env_->NowMicros();
285     TEST_SYNC_POINT("DeleteSchedulerTest::RateLimitingMultiThreaded:1");
286     delete_scheduler_->WaitForEmptyTrash();
287     uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time;
288 
289     auto bg_errors = delete_scheduler_->GetBackgroundErrors();
290     ASSERT_EQ(bg_errors.size(), 0);
291 
292     uint64_t total_files_size = 0;
293     uint64_t expected_penlty = 0;
294     ASSERT_EQ(penalties.size(), num_files * thread_cnt);
295     for (int i = 0; i < num_files * thread_cnt; i++) {
296       total_files_size += file_size;
297       expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec_);
298       ASSERT_EQ(expected_penlty, penalties[i]);
299     }
300     ASSERT_GT(time_spent_deleting, expected_penlty * 0.9);
301 
302     ASSERT_EQ(CountNormalFiles(), 0);
303     ASSERT_EQ(CountTrashFiles(), 0);
304     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
305   }
306 }
307 
308 // Disable rate limiting by setting rate_bytes_per_sec_ to 0 and make sure
309 // that when DeleteScheduler delete a file it delete it immediately and dont
310 // move it to trash
TEST_F(DeleteSchedulerTest,DisableRateLimiting)311 TEST_F(DeleteSchedulerTest, DisableRateLimiting) {
312   int bg_delete_file = 0;
313   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
314       "DeleteScheduler::DeleteTrashFile:DeleteFile",
315       [&](void* /*arg*/) { bg_delete_file++; });
316 
317   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
318 
319   rate_bytes_per_sec_ = 0;
320   NewDeleteScheduler();
321 
322   for (int i = 0; i < 10; i++) {
323     // Every file we delete will be deleted immediately
324     std::string dummy_file = NewDummyFile("dummy.data");
325     ASSERT_OK(delete_scheduler_->DeleteFile(dummy_file, ""));
326     ASSERT_TRUE(env_->FileExists(dummy_file).IsNotFound());
327     ASSERT_EQ(CountNormalFiles(), 0);
328     ASSERT_EQ(CountTrashFiles(), 0);
329   }
330 
331   ASSERT_EQ(bg_delete_file, 0);
332 
333   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
334 }
335 
336 // Testing that moving files to trash with the same name is not a problem
337 // 1- Create 10 files with the same name "conflict.data"
338 // 2- Delete the 10 files using DeleteScheduler
339 // 3- Make sure that trash directory contain 10 files ("conflict.data" x 10)
340 // --- Hold DeleteScheduler::BackgroundEmptyTrash ---
341 // 4- Make sure that files are deleted from trash
TEST_F(DeleteSchedulerTest,ConflictNames)342 TEST_F(DeleteSchedulerTest, ConflictNames) {
343   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
344       {"DeleteSchedulerTest::ConflictNames:1",
345        "DeleteScheduler::BackgroundEmptyTrash"},
346   });
347   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
348 
349   rate_bytes_per_sec_ = 1024 * 1024;  // 1 Mb/sec
350   NewDeleteScheduler();
351 
352   // Create "conflict.data" and move it to trash 10 times
353   for (int i = 0; i < 10; i++) {
354     std::string dummy_file = NewDummyFile("conflict.data");
355     ASSERT_OK(delete_scheduler_->DeleteFile(dummy_file, ""));
356   }
357   ASSERT_EQ(CountNormalFiles(), 0);
358   // 10 files ("conflict.data" x 10) in trash
359   ASSERT_EQ(CountTrashFiles(), 10);
360 
361   // Hold BackgroundEmptyTrash
362   TEST_SYNC_POINT("DeleteSchedulerTest::ConflictNames:1");
363   delete_scheduler_->WaitForEmptyTrash();
364   ASSERT_EQ(CountTrashFiles(), 0);
365 
366   auto bg_errors = delete_scheduler_->GetBackgroundErrors();
367   ASSERT_EQ(bg_errors.size(), 0);
368 
369   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
370 }
371 
372 // 1- Create 10 dummy files
373 // 2- Delete the 10 files using DeleteScheduler (move them to trsah)
374 // 3- Delete the 10 files directly (using env_->DeleteFile)
375 // --- Hold DeleteScheduler::BackgroundEmptyTrash ---
376 // 4- Make sure that DeleteScheduler failed to delete the 10 files and
377 //    reported 10 background errors
TEST_F(DeleteSchedulerTest,BackgroundError)378 TEST_F(DeleteSchedulerTest, BackgroundError) {
379   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
380       {"DeleteSchedulerTest::BackgroundError:1",
381        "DeleteScheduler::BackgroundEmptyTrash"},
382   });
383   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
384 
385   rate_bytes_per_sec_ = 1024 * 1024;  // 1 Mb/sec
386   NewDeleteScheduler();
387 
388   // Generate 10 dummy files and move them to trash
389   for (int i = 0; i < 10; i++) {
390     std::string file_name = "data_" + ToString(i) + ".data";
391     ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name), ""));
392   }
393   ASSERT_EQ(CountNormalFiles(), 0);
394   ASSERT_EQ(CountTrashFiles(), 10);
395 
396   // Delete 10 files from trash, this will cause background errors in
397   // BackgroundEmptyTrash since we already deleted the files it was
398   // goind to delete
399   for (int i = 0; i < 10; i++) {
400     std::string file_name = "data_" + ToString(i) + ".data.trash";
401     ASSERT_OK(env_->DeleteFile(dummy_files_dirs_[0] + "/" + file_name));
402   }
403 
404   // Hold BackgroundEmptyTrash
405   TEST_SYNC_POINT("DeleteSchedulerTest::BackgroundError:1");
406   delete_scheduler_->WaitForEmptyTrash();
407   auto bg_errors = delete_scheduler_->GetBackgroundErrors();
408   ASSERT_EQ(bg_errors.size(), 10);
409 
410   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
411 }
412 
413 // 1- Create 10 dummy files
414 // 2- Delete 10 dummy files using DeleteScheduler
415 // 3- Wait for DeleteScheduler to delete all files in queue
416 // 4- Make sure all files in trash directory were deleted
417 // 5- Repeat previous steps 5 times
TEST_F(DeleteSchedulerTest,StartBGEmptyTrashMultipleTimes)418 TEST_F(DeleteSchedulerTest, StartBGEmptyTrashMultipleTimes) {
419   int bg_delete_file = 0;
420   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
421       "DeleteScheduler::DeleteTrashFile:DeleteFile",
422       [&](void* /*arg*/) { bg_delete_file++; });
423   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
424 
425   rate_bytes_per_sec_ = 1024 * 1024;  // 1 MB / sec
426   NewDeleteScheduler();
427 
428   // Move files to trash, wait for empty trash, start again
429   for (int run = 1; run <= 5; run++) {
430     // Generate 10 dummy files and move them to trash
431     for (int i = 0; i < 10; i++) {
432       std::string file_name = "data_" + ToString(i) + ".data";
433       ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name), ""));
434     }
435     ASSERT_EQ(CountNormalFiles(), 0);
436     delete_scheduler_->WaitForEmptyTrash();
437     ASSERT_EQ(bg_delete_file, 10 * run);
438     ASSERT_EQ(CountTrashFiles(), 0);
439 
440     auto bg_errors = delete_scheduler_->GetBackgroundErrors();
441     ASSERT_EQ(bg_errors.size(), 0);
442   }
443 
444   ASSERT_EQ(bg_delete_file, 50);
445   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
446 }
447 
TEST_F(DeleteSchedulerTest,DeletePartialFile)448 TEST_F(DeleteSchedulerTest, DeletePartialFile) {
449   int bg_delete_file = 0;
450   int bg_fsync = 0;
451   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
452       "DeleteScheduler::DeleteTrashFile:DeleteFile",
453       [&](void*) { bg_delete_file++; });
454   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
455       "DeleteScheduler::DeleteTrashFile:Fsync", [&](void*) { bg_fsync++; });
456   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
457 
458   rate_bytes_per_sec_ = 1024 * 1024;  // 1 MB / sec
459   NewDeleteScheduler();
460 
461   // Should delete in 4 batch
462   ASSERT_OK(
463       delete_scheduler_->DeleteFile(NewDummyFile("data_1", 500 * 1024), ""));
464   ASSERT_OK(
465       delete_scheduler_->DeleteFile(NewDummyFile("data_2", 100 * 1024), ""));
466   // Should delete in 2 batch
467   ASSERT_OK(
468       delete_scheduler_->DeleteFile(NewDummyFile("data_2", 200 * 1024), ""));
469 
470   delete_scheduler_->WaitForEmptyTrash();
471 
472   auto bg_errors = delete_scheduler_->GetBackgroundErrors();
473   ASSERT_EQ(bg_errors.size(), 0);
474   ASSERT_EQ(7, bg_delete_file);
475   ASSERT_EQ(4, bg_fsync);
476   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
477 }
478 
479 #ifdef OS_LINUX
TEST_F(DeleteSchedulerTest,NoPartialDeleteWithLink)480 TEST_F(DeleteSchedulerTest, NoPartialDeleteWithLink) {
481   int bg_delete_file = 0;
482   int bg_fsync = 0;
483   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
484       "DeleteScheduler::DeleteTrashFile:DeleteFile",
485       [&](void*) { bg_delete_file++; });
486   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
487       "DeleteScheduler::DeleteTrashFile:Fsync", [&](void*) { bg_fsync++; });
488   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
489 
490   rate_bytes_per_sec_ = 1024 * 1024;  // 1 MB / sec
491   NewDeleteScheduler();
492 
493   std::string file1 = NewDummyFile("data_1", 500 * 1024);
494   std::string file2 = NewDummyFile("data_2", 100 * 1024);
495 
496   ASSERT_OK(env_->LinkFile(file1, dummy_files_dirs_[0] + "/data_1b"));
497   ASSERT_OK(env_->LinkFile(file2, dummy_files_dirs_[0] + "/data_2b"));
498 
499   // Should delete in 4 batch if there is no hardlink
500   ASSERT_OK(delete_scheduler_->DeleteFile(file1, ""));
501   ASSERT_OK(delete_scheduler_->DeleteFile(file2, ""));
502 
503   delete_scheduler_->WaitForEmptyTrash();
504 
505   auto bg_errors = delete_scheduler_->GetBackgroundErrors();
506   ASSERT_EQ(bg_errors.size(), 0);
507   ASSERT_EQ(2, bg_delete_file);
508   ASSERT_EQ(0, bg_fsync);
509   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
510 }
511 #endif
512 
513 // 1- Create a DeleteScheduler with very slow rate limit (1 Byte / sec)
514 // 2- Delete 100 files using DeleteScheduler
515 // 3- Delete the DeleteScheduler (call the destructor while queue is not empty)
516 // 4- Make sure that not all files were deleted from trash and that
517 //    DeleteScheduler background thread did not delete all files
TEST_F(DeleteSchedulerTest,DestructorWithNonEmptyQueue)518 TEST_F(DeleteSchedulerTest, DestructorWithNonEmptyQueue) {
519   int bg_delete_file = 0;
520   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
521       "DeleteScheduler::DeleteTrashFile:DeleteFile",
522       [&](void* /*arg*/) { bg_delete_file++; });
523   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
524 
525   rate_bytes_per_sec_ = 1;  // 1 Byte / sec
526   NewDeleteScheduler();
527 
528   for (int i = 0; i < 100; i++) {
529     std::string file_name = "data_" + ToString(i) + ".data";
530     ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name), ""));
531   }
532 
533   // Deleting 100 files will need >28 hours to delete
534   // we will delete the DeleteScheduler while delete queue is not empty
535   sst_file_mgr_.reset();
536 
537   ASSERT_LT(bg_delete_file, 100);
538   ASSERT_GT(CountTrashFiles(), 0);
539 
540   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
541 }
542 
TEST_F(DeleteSchedulerTest,DISABLED_DynamicRateLimiting1)543 TEST_F(DeleteSchedulerTest, DISABLED_DynamicRateLimiting1) {
544   std::vector<uint64_t> penalties;
545   int bg_delete_file = 0;
546   int fg_delete_file = 0;
547   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
548       "DeleteScheduler::DeleteTrashFile:DeleteFile",
549       [&](void* /*arg*/) { bg_delete_file++; });
550   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
551       "DeleteScheduler::DeleteFile", [&](void* /*arg*/) { fg_delete_file++; });
552   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
553       "DeleteScheduler::BackgroundEmptyTrash:Wait",
554       [&](void* arg) { penalties.push_back(*(static_cast<int*>(arg))); });
555 
556   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
557       {"DeleteSchedulerTest::DynamicRateLimiting1:1",
558        "DeleteScheduler::BackgroundEmptyTrash"},
559   });
560   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
561 
562   rate_bytes_per_sec_ = 0;  // Disable rate limiting initially
563   NewDeleteScheduler();
564 
565 
566   int num_files = 10;  // 10 files
567   uint64_t file_size = 1024;  // every file is 1 kb
568 
569   std::vector<int64_t> delete_kbs_per_sec = {512, 200, 0, 100, 50, -2, 25};
570   for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) {
571     penalties.clear();
572     bg_delete_file = 0;
573     fg_delete_file = 0;
574     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace();
575     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
576 
577     DestroyAndCreateDir(dummy_files_dirs_[0]);
578     rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024;
579     delete_scheduler_->SetRateBytesPerSecond(rate_bytes_per_sec_);
580 
581     // Create 100 dummy files, every file is 1 Kb
582     std::vector<std::string> generated_files;
583     for (int i = 0; i < num_files; i++) {
584       std::string file_name = "file" + ToString(i) + ".data";
585       generated_files.push_back(NewDummyFile(file_name, file_size));
586     }
587 
588     // Delete dummy files and measure time spent to empty trash
589     for (int i = 0; i < num_files; i++) {
590       ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[i], ""));
591     }
592     ASSERT_EQ(CountNormalFiles(), 0);
593 
594     if (rate_bytes_per_sec_ > 0) {
595       uint64_t delete_start_time = env_->NowMicros();
596       TEST_SYNC_POINT("DeleteSchedulerTest::DynamicRateLimiting1:1");
597       delete_scheduler_->WaitForEmptyTrash();
598       uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time;
599 
600       auto bg_errors = delete_scheduler_->GetBackgroundErrors();
601       ASSERT_EQ(bg_errors.size(), 0);
602 
603       uint64_t total_files_size = 0;
604       uint64_t expected_penlty = 0;
605       ASSERT_EQ(penalties.size(), num_files);
606       for (int i = 0; i < num_files; i++) {
607         total_files_size += file_size;
608         expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec_);
609         ASSERT_EQ(expected_penlty, penalties[i]);
610       }
611       ASSERT_GT(time_spent_deleting, expected_penlty * 0.9);
612       ASSERT_EQ(bg_delete_file, num_files);
613       ASSERT_EQ(fg_delete_file, 0);
614     } else {
615       ASSERT_EQ(penalties.size(), 0);
616       ASSERT_EQ(bg_delete_file, 0);
617       ASSERT_EQ(fg_delete_file, num_files);
618     }
619 
620     ASSERT_EQ(CountTrashFiles(), 0);
621     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
622   }
623 }
624 
TEST_F(DeleteSchedulerTest,ImmediateDeleteOn25PercDBSize)625 TEST_F(DeleteSchedulerTest, ImmediateDeleteOn25PercDBSize) {
626   int bg_delete_file = 0;
627   int fg_delete_file = 0;
628   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
629       "DeleteScheduler::DeleteTrashFile:DeleteFile",
630       [&](void* /*arg*/) { bg_delete_file++; });
631   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
632       "DeleteScheduler::DeleteFile", [&](void* /*arg*/) { fg_delete_file++; });
633 
634   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
635 
636   int num_files = 100;  // 100 files
637   uint64_t file_size = 1024 * 10; // 100 KB as a file size
638   rate_bytes_per_sec_ = 1;  // 1 byte per sec (very slow trash delete)
639 
640   NewDeleteScheduler();
641   delete_scheduler_->SetMaxTrashDBRatio(0.25);
642 
643   std::vector<std::string> generated_files;
644   for (int i = 0; i < num_files; i++) {
645     std::string file_name = "file" + ToString(i) + ".data";
646     generated_files.push_back(NewDummyFile(file_name, file_size));
647   }
648 
649   for (std::string& file_name : generated_files) {
650     delete_scheduler_->DeleteFile(file_name, "");
651   }
652 
653   // When we end up with 26 files in trash we will start
654   // deleting new files immediately
655   ASSERT_EQ(fg_delete_file, 74);
656 
657   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
658 }
659 
TEST_F(DeleteSchedulerTest,IsTrashCheck)660 TEST_F(DeleteSchedulerTest, IsTrashCheck) {
661   // Trash files
662   ASSERT_TRUE(DeleteScheduler::IsTrashFile("x.trash"));
663   ASSERT_TRUE(DeleteScheduler::IsTrashFile(".trash"));
664   ASSERT_TRUE(DeleteScheduler::IsTrashFile("abc.sst.trash"));
665   ASSERT_TRUE(DeleteScheduler::IsTrashFile("/a/b/c/abc..sst.trash"));
666   ASSERT_TRUE(DeleteScheduler::IsTrashFile("log.trash"));
667   ASSERT_TRUE(DeleteScheduler::IsTrashFile("^^^^^.log.trash"));
668   ASSERT_TRUE(DeleteScheduler::IsTrashFile("abc.t.trash"));
669 
670   // Not trash files
671   ASSERT_FALSE(DeleteScheduler::IsTrashFile("abc.sst"));
672   ASSERT_FALSE(DeleteScheduler::IsTrashFile("abc.txt"));
673   ASSERT_FALSE(DeleteScheduler::IsTrashFile("/a/b/c/abc.sst"));
674   ASSERT_FALSE(DeleteScheduler::IsTrashFile("/a/b/c/abc.sstrash"));
675   ASSERT_FALSE(DeleteScheduler::IsTrashFile("^^^^^.trashh"));
676   ASSERT_FALSE(DeleteScheduler::IsTrashFile("abc.ttrash"));
677   ASSERT_FALSE(DeleteScheduler::IsTrashFile(".ttrash"));
678   ASSERT_FALSE(DeleteScheduler::IsTrashFile("abc.trashx"));
679 }
680 
681 }  // namespace ROCKSDB_NAMESPACE
682 
main(int argc,char ** argv)683 int main(int argc, char** argv) {
684   ::testing::InitGoogleTest(&argc, argv);
685   return RUN_ALL_TESTS();
686 }
687 
688 #else
main(int,char **)689 int main(int /*argc*/, char** /*argv*/) {
690   printf("DeleteScheduler is not supported in ROCKSDB_LITE\n");
691   return 0;
692 }
693 #endif  // ROCKSDB_LITE
694