1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9 //
10 // The test uses an array to compare against values written to the database.
11 // Keys written to the array are in 1:1 correspondence to the actual values in
12 // the database according to the formula in the function GenerateValue.
13
14 // Space is reserved in the array from 0 to FLAGS_max_key and values are
15 // randomly written/deleted/read from those positions. During verification we
16 // compare all the positions in the array. To shorten/elongate the running
17 // time, you could change the settings: FLAGS_max_key, FLAGS_ops_per_thread,
18 // (sometimes also FLAGS_threads).
19 //
20 // NOTE that if FLAGS_test_batches_snapshots is set, the test will have
21 // different behavior. See comment of the flag for details.
22
23 #ifdef GFLAGS
24 #include "db_stress_tool/db_stress_common.h"
25 #include "db_stress_tool/db_stress_driver.h"
26
27 namespace ROCKSDB_NAMESPACE {
28 namespace {
29 static std::shared_ptr<ROCKSDB_NAMESPACE::Env> env_guard;
30 static std::shared_ptr<ROCKSDB_NAMESPACE::DbStressEnvWrapper> env_wrapper_guard;
31 } // namespace
32
33 KeyGenContext key_gen_ctx;
34
db_stress_tool(int argc,char ** argv)35 int db_stress_tool(int argc, char** argv) {
36 SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +
37 " [OPTIONS]...");
38 ParseCommandLineFlags(&argc, &argv, true);
39
40 SanitizeDoubleParam(&FLAGS_bloom_bits);
41 SanitizeDoubleParam(&FLAGS_memtable_prefix_bloom_size_ratio);
42 SanitizeDoubleParam(&FLAGS_max_bytes_for_level_multiplier);
43
44 if (FLAGS_statistics) {
45 dbstats = ROCKSDB_NAMESPACE::CreateDBStatistics();
46 if (FLAGS_test_secondary) {
47 dbstats_secondaries = ROCKSDB_NAMESPACE::CreateDBStatistics();
48 }
49 }
50 compression_type_e = StringToCompressionType(FLAGS_compression_type.c_str());
51 bottommost_compression_type_e =
52 StringToCompressionType(FLAGS_bottommost_compression_type.c_str());
53 checksum_type_e = StringToChecksumType(FLAGS_checksum_type.c_str());
54
55 Env* raw_env;
56
57 if (!FLAGS_hdfs.empty()) {
58 if (!FLAGS_env_uri.empty()) {
59 fprintf(stderr, "Cannot specify both --hdfs and --env_uri.\n");
60 exit(1);
61 }
62 raw_env = new ROCKSDB_NAMESPACE::HdfsEnv(FLAGS_hdfs);
63 } else if (!FLAGS_env_uri.empty()) {
64 Status s = Env::LoadEnv(FLAGS_env_uri, &raw_env, &env_guard);
65 if (raw_env == nullptr) {
66 fprintf(stderr, "No Env registered for URI: %s\n", FLAGS_env_uri.c_str());
67 exit(1);
68 }
69 } else {
70 raw_env = Env::Default();
71 }
72 env_wrapper_guard = std::make_shared<DbStressEnvWrapper>(raw_env);
73 db_stress_env = env_wrapper_guard.get();
74
75 FLAGS_rep_factory = StringToRepFactory(FLAGS_memtablerep.c_str());
76
77 // The number of background threads should be at least as much the
78 // max number of concurrent compactions.
79 db_stress_env->SetBackgroundThreads(FLAGS_max_background_compactions,
80 ROCKSDB_NAMESPACE::Env::Priority::LOW);
81 db_stress_env->SetBackgroundThreads(FLAGS_num_bottom_pri_threads,
82 ROCKSDB_NAMESPACE::Env::Priority::BOTTOM);
83 if (FLAGS_prefixpercent > 0 && FLAGS_prefix_size < 0) {
84 fprintf(stderr,
85 "Error: prefixpercent is non-zero while prefix_size is "
86 "not positive!\n");
87 exit(1);
88 }
89 if (FLAGS_test_batches_snapshots && FLAGS_prefix_size <= 0) {
90 fprintf(stderr,
91 "Error: please specify prefix_size for "
92 "test_batches_snapshots test!\n");
93 exit(1);
94 }
95 if (FLAGS_memtable_prefix_bloom_size_ratio > 0.0 && FLAGS_prefix_size < 0) {
96 fprintf(stderr,
97 "Error: please specify positive prefix_size in order to use "
98 "memtable_prefix_bloom_size_ratio\n");
99 exit(1);
100 }
101 if ((FLAGS_readpercent + FLAGS_prefixpercent + FLAGS_writepercent +
102 FLAGS_delpercent + FLAGS_delrangepercent + FLAGS_iterpercent) != 100) {
103 fprintf(stderr,
104 "Error: Read+Prefix+Write+Delete+DeleteRange+Iterate percents != "
105 "100!\n");
106 exit(1);
107 }
108 if (FLAGS_disable_wal == 1 && FLAGS_reopen > 0) {
109 fprintf(stderr, "Error: Db cannot reopen safely with disable_wal set!\n");
110 exit(1);
111 }
112 if ((unsigned)FLAGS_reopen >= FLAGS_ops_per_thread) {
113 fprintf(stderr,
114 "Error: #DB-reopens should be < ops_per_thread\n"
115 "Provided reopens = %d and ops_per_thread = %lu\n",
116 FLAGS_reopen, (unsigned long)FLAGS_ops_per_thread);
117 exit(1);
118 }
119 if (FLAGS_test_batches_snapshots && FLAGS_delrangepercent > 0) {
120 fprintf(stderr,
121 "Error: nonzero delrangepercent unsupported in "
122 "test_batches_snapshots mode\n");
123 exit(1);
124 }
125 if (FLAGS_active_width > FLAGS_max_key) {
126 fprintf(stderr, "Error: active_width can be at most max_key\n");
127 exit(1);
128 } else if (FLAGS_active_width == 0) {
129 FLAGS_active_width = FLAGS_max_key;
130 }
131 if (FLAGS_value_size_mult * kRandomValueMaxFactor > kValueMaxLen) {
132 fprintf(stderr, "Error: value_size_mult can be at most %d\n",
133 kValueMaxLen / kRandomValueMaxFactor);
134 exit(1);
135 }
136 if (FLAGS_use_merge && FLAGS_nooverwritepercent == 100) {
137 fprintf(
138 stderr,
139 "Error: nooverwritepercent must not be 100 when using merge operands");
140 exit(1);
141 }
142 if (FLAGS_ingest_external_file_one_in > 0 && FLAGS_nooverwritepercent > 0) {
143 fprintf(stderr,
144 "Error: nooverwritepercent must be 0 when using file ingestion\n");
145 exit(1);
146 }
147 if (FLAGS_clear_column_family_one_in > 0 && FLAGS_backup_one_in > 0) {
148 fprintf(stderr,
149 "Error: clear_column_family_one_in must be 0 when using backup\n");
150 exit(1);
151 }
152 if (FLAGS_test_cf_consistency && FLAGS_disable_wal) {
153 FLAGS_atomic_flush = true;
154 }
155
156 if (FLAGS_read_only) {
157 if (FLAGS_writepercent != 0 || FLAGS_delpercent != 0 ||
158 FLAGS_delrangepercent != 0) {
159 fprintf(stderr, "Error: updates are not supported in read only mode\n");
160 exit(1);
161 } else if (FLAGS_checkpoint_one_in > 0 &&
162 FLAGS_clear_column_family_one_in > 0) {
163 fprintf(stdout,
164 "Warn: checkpoint won't be validated since column families may "
165 "be dropped.\n");
166 }
167 }
168
169 // Choose a location for the test database if none given with --db=<path>
170 if (FLAGS_db.empty()) {
171 std::string default_db_path;
172 db_stress_env->GetTestDirectory(&default_db_path);
173 default_db_path += "/dbstress";
174 FLAGS_db = default_db_path;
175 }
176
177 if ((FLAGS_test_secondary || FLAGS_continuous_verification_interval > 0) &&
178 FLAGS_secondaries_base.empty()) {
179 std::string default_secondaries_path;
180 db_stress_env->GetTestDirectory(&default_secondaries_path);
181 default_secondaries_path += "/dbstress_secondaries";
182 ROCKSDB_NAMESPACE::Status s =
183 db_stress_env->CreateDirIfMissing(default_secondaries_path);
184 if (!s.ok()) {
185 fprintf(stderr, "Failed to create directory %s: %s\n",
186 default_secondaries_path.c_str(), s.ToString().c_str());
187 exit(1);
188 }
189 FLAGS_secondaries_base = default_secondaries_path;
190 }
191
192 if (!FLAGS_test_secondary && FLAGS_secondary_catch_up_one_in > 0) {
193 fprintf(
194 stderr,
195 "Must set -test_secondary=true if secondary_catch_up_one_in > 0.\n");
196 exit(1);
197 }
198
199 rocksdb_kill_odds = FLAGS_kill_random_test;
200 rocksdb_kill_prefix_blacklist = SplitString(FLAGS_kill_prefix_blacklist);
201
202 unsigned int levels = FLAGS_max_key_len;
203 std::vector<std::string> weights;
204 uint64_t scale_factor = FLAGS_key_window_scale_factor;
205 key_gen_ctx.window = scale_factor * 100;
206 if (!FLAGS_key_len_percent_dist.empty()) {
207 weights = SplitString(FLAGS_key_len_percent_dist);
208 if (weights.size() != levels) {
209 fprintf(stderr,
210 "Number of weights in key_len_dist should be equal to"
211 " max_key_len");
212 exit(1);
213 }
214
215 uint64_t total_weight = 0;
216 for (std::string& weight : weights) {
217 uint64_t val = std::stoull(weight);
218 key_gen_ctx.weights.emplace_back(val * scale_factor);
219 total_weight += val;
220 }
221 if (total_weight != 100) {
222 fprintf(stderr, "Sum of all weights in key_len_dist should be 100");
223 exit(1);
224 }
225 } else {
226 uint64_t keys_per_level = key_gen_ctx.window / levels;
227 for (unsigned int level = 0; level < levels - 1; ++level) {
228 key_gen_ctx.weights.emplace_back(keys_per_level);
229 }
230 key_gen_ctx.weights.emplace_back(key_gen_ctx.window -
231 keys_per_level * (levels - 1));
232 }
233
234 std::unique_ptr<ROCKSDB_NAMESPACE::StressTest> stress;
235 if (FLAGS_test_cf_consistency) {
236 stress.reset(CreateCfConsistencyStressTest());
237 } else if (FLAGS_test_batches_snapshots) {
238 stress.reset(CreateBatchedOpsStressTest());
239 } else {
240 stress.reset(CreateNonBatchedOpsStressTest());
241 }
242 // Initialize the Zipfian pre-calculated array
243 InitializeHotKeyGenerator(FLAGS_hot_key_alpha);
244 if (RunStressTest(stress.get())) {
245 return 0;
246 } else {
247 return 1;
248 }
249 }
250
251 } // namespace ROCKSDB_NAMESPACE
252 #endif // GFLAGS
253