1 // Author: Bruce Allen
2 // Created: 2/25/2013
3 //
4 // The software provided here is released by the Naval Postgraduate
5 // School, an agency of the U.S. Department of Navy. The software
6 // bears no warranty, either expressed or implied. NPS does not assume
7 // legal liability nor responsibility for a User's use of the software
8 // or the results of such use.
9 //
10 // Please note that within the United States, copyright protection,
11 // under Section 105 of the United States Code, Title 17, is not
12 // available for any work of the United States Government and/or for
13 // any works created by United States Government employees. User
14 // acknowledges that this software contains work which was created by
15 // NPS government employees and is therefore in the public domain and
16 // not subject to copyright.
17 //
18 // Released into the public domain on February 25, 2013 by Bruce Allen.
19
20 /**
21 * \file
22 * Test the LMDB data managers
23 */
24
25 #include <config.h>
26 #include <iostream>
27 #include <iomanip>
28 #include <cstdio>
29 #include "unit_test.h"
30 #include "lmdb_hash_data_manager.hpp"
31 #include "lmdb_hash_manager.hpp"
32 #include "lmdb_source_data_manager.hpp"
33 #include "lmdb_source_id_manager.hpp"
34 #include "lmdb_source_name_manager.hpp"
35 #include "lmdb_helper.h"
36 #include "lmdb_changes.hpp"
37 #include "source_id_sub_counts.hpp"
38 #include "../src_libhashdb/hashdb.hpp"
39 #include "directory_helper.hpp"
40
41 typedef std::pair<std::string, std::string> source_name_t;
42 typedef std::set<source_name_t> source_names_t;
43
44 static const std::string hashdb_dir = "temp_dir_lmdb_managers_test.hdb";
45 static const std::string binary_00(hashdb::hex_to_bin(
46 "00000000000000000000000000000000"));
47 static const std::string binary_01(hashdb::hex_to_bin(
48 "00000000000000000000000000000001"));
49 static const std::string binary_2(hashdb::hex_to_bin(
50 "00000000000000000000000000000002"));
51 static const std::string binary_10(hashdb::hex_to_bin(
52 "10000000000000000000000000000000"));
53 static const std::string binary_11(hashdb::hex_to_bin(
54 "10000000000000000000000000000001"));
55 static const std::string binary_12(hashdb::hex_to_bin(
56 "10000000000000000000000000000002"));
57 static const std::string binary_13(hashdb::hex_to_bin(
58 "10000000000000000000000000000003"));
59 static const std::string binary_14(hashdb::hex_to_bin(
60 "10000000000000000000000000000004"));
61 static const std::string binary_15(hashdb::hex_to_bin(
62 "10000000000000000000000000000005"));
63 static const std::string binary_26(hashdb::hex_to_bin(
64 "20000000000000000000000000000006"));
65
make_new_hashdb_dir(std::string p_hashdb_dir)66 void make_new_hashdb_dir(std::string p_hashdb_dir) {
67 // remove any previous hashdb_dir
68 rm_hashdb_dir(p_hashdb_dir);
69
70 // create the hashdb directory
71 require_no_dir(p_hashdb_dir);
72 create_new_dir(p_hashdb_dir);
73 }
74
75 // ************************************************************
76 // lmdb_hash_manager
77 // ************************************************************
lmdb_hash_manager_create()78 void lmdb_hash_manager_create() {
79 // create new manager
80 make_new_hashdb_dir(hashdb_dir);
81 hashdb::lmdb_hash_manager_t manager(hashdb_dir, hashdb::RW_NEW);
82 }
83
84 // run after create
lmdb_hash_manager_write()85 void lmdb_hash_manager_write() {
86 hashdb::lmdb_hash_manager_t manager(hashdb_dir, hashdb::RW_MODIFY);
87 hashdb::lmdb_changes_t changes;
88
89 // find when empty
90 TEST_EQ(manager.find(binary_00), 0);
91
92 // add
93 manager.insert(binary_00, 1, changes);
94 TEST_EQ(changes.hash_inserted, 1);
95 TEST_EQ(changes.hash_count_changed, 0);
96 TEST_EQ(changes.hash_count_not_changed, 0);
97 TEST_EQ(manager.find(binary_00), 1);
98
99 // re-add same
100 manager.insert(binary_00, 1, changes);
101 TEST_EQ(changes.hash_inserted, 1);
102 TEST_EQ(changes.hash_count_changed, 0);
103 TEST_EQ(changes.hash_count_not_changed, 1);
104 TEST_EQ(manager.find(binary_00), 1);
105
106 // change count
107 manager.insert(binary_00, 2, changes);
108 TEST_EQ(changes.hash_inserted, 1);
109 TEST_EQ(changes.hash_count_changed, 1);
110 TEST_EQ(changes.hash_count_not_changed, 1);
111 TEST_EQ(manager.find(binary_00), 2);
112
113 // check that similar prefix, different suffix is equivalent
114 TEST_EQ(manager.find(binary_01), 2);
115
116 // check that different prefix, same suffix is different
117 TEST_EQ(manager.find(binary_10), 0);
118
119 // add another
120 manager.insert(binary_26, 1, changes);
121 TEST_EQ(manager.find(binary_26), 1);
122
123 // size
124 TEST_EQ(manager.size(), 2);
125 }
126
127 // run after read
lmdb_hash_manager_read()128 void lmdb_hash_manager_read() {
129 hashdb::lmdb_hash_manager_t manager(hashdb_dir, hashdb::READ_ONLY);
130
131 // find
132 TEST_EQ(manager.find(binary_00), 2);
133 TEST_EQ(manager.find(binary_01), 2);
134 TEST_EQ(manager.find(binary_12), 0);
135 TEST_EQ(manager.find(binary_26), 1);
136 // size
137 TEST_EQ(manager.size(), 2);
138 }
139
140 // test corner-case values for count
lmdb_hash_manager_count()141 void lmdb_hash_manager_count() {
142 make_new_hashdb_dir(hashdb_dir);
143 hashdb::lmdb_hash_manager_t manager(hashdb_dir, hashdb::RW_NEW);
144 hashdb::lmdb_changes_t changes;
145 manager.insert(binary_00, 1494, changes);
146 TEST_EQ(manager.find(binary_00), 1370);
147 manager.insert(binary_00, 1495, changes);
148 TEST_EQ(manager.find(binary_00), 1495);
149 }
150
151 // ************************************************************
152 // lmdb_source_id_manager
153 // ************************************************************
lmdb_source_id_manager()154 void lmdb_source_id_manager() {
155 // resources
156 hashdb::lmdb_changes_t changes;
157 bool did_find;
158 bool did_insert;
159 uint64_t source_id;
160 std::string file_binary_hash;
161
162 // create new manager
163 make_new_hashdb_dir(hashdb_dir);
164 hashdb::lmdb_source_id_manager_t manager(hashdb_dir, hashdb::RW_NEW);
165
166 // iterator when empty
167 file_binary_hash = manager.first_source();
168 TEST_EQ(file_binary_hash, "");
169
170 // search when empty
171 did_find = manager.find(binary_00, source_id);
172 TEST_EQ(did_find, false);
173 TEST_EQ(source_id, 0)
174
175 // add items
176 did_insert = manager.insert(binary_00, changes, source_id);
177 TEST_EQ(did_insert, true);
178 TEST_EQ(source_id, 1);
179 TEST_EQ(changes.source_id_inserted, 1);
180 TEST_EQ(changes.source_id_already_present, 0);
181
182 did_insert = manager.insert(binary_00, changes, source_id);
183 TEST_EQ(did_insert, false);
184 TEST_EQ(source_id, 1);
185 TEST_EQ(changes.source_id_inserted, 1);
186 TEST_EQ(changes.source_id_already_present, 1);
187
188 did_find = manager.find(binary_00, source_id);
189 TEST_EQ(did_find, true);
190 TEST_EQ(source_id, 1)
191
192 // iterator
193 did_insert = manager.insert(binary_2, changes, source_id);
194 TEST_EQ(source_id, 2);
195 did_insert = manager.insert(binary_01, changes, source_id);
196 TEST_EQ(source_id, 3);
197 file_binary_hash = manager.first_source();
198 TEST_EQ(file_binary_hash, binary_00);
199 file_binary_hash = manager.next_source(file_binary_hash);
200 TEST_EQ(file_binary_hash, binary_01);
201 file_binary_hash = manager.next_source(file_binary_hash);
202 TEST_EQ(file_binary_hash, binary_2);
203 file_binary_hash = manager.next_source(file_binary_hash);
204 TEST_EQ(file_binary_hash, "");
205
206 // allow empty request
207 file_binary_hash = manager.next_source(file_binary_hash);
208 TEST_EQ(file_binary_hash, "")
209
210 // allow invalid request
211 file_binary_hash = manager.next_source(binary_26);
212 TEST_EQ(file_binary_hash, "")
213
214 }
215
216 // ************************************************************
217 // lmdb_source_data_manager
218 // ************************************************************
lmdb_source_data_manager()219 void lmdb_source_data_manager() {
220 hashdb::lmdb_changes_t changes;
221
222 // variables
223 std::string file_binary_hash;
224 uint64_t filesize;
225 std::string file_type;
226 uint64_t zero_count;
227 uint64_t nonprobative_count;
228 bool found;
229
230 // create new manager
231 make_new_hashdb_dir(hashdb_dir);
232 hashdb::lmdb_source_data_manager_t manager(hashdb_dir, hashdb::RW_NEW);
233
234 // no source ID
235 found =
236 manager.find(1, file_binary_hash, filesize, file_type, zero_count, nonprobative_count);
237 TEST_EQ(found, false);
238
239 // insert
240 manager.insert(1, "fbh", 2, "ft", 1, 3, changes);
241 TEST_EQ(changes.source_data_inserted, 1);
242 TEST_EQ(changes.source_data_changed, 0);
243 TEST_EQ(changes.source_data_same, 0);
244 found =
245 manager.find(1, file_binary_hash, filesize, file_type, zero_count, nonprobative_count);
246 TEST_EQ(found, true);
247 TEST_EQ(file_binary_hash, "fbh");
248 TEST_EQ(filesize, 2);
249 TEST_EQ(file_type, "ft");
250 TEST_EQ(zero_count, 1);
251 TEST_EQ(nonprobative_count, 3);
252
253 // insert same
254 manager.insert(1, "fbh", 2, "ft", 1, 3, changes);
255 TEST_EQ(changes.source_data_inserted, 1);
256 TEST_EQ(changes.source_data_changed, 0);
257 TEST_EQ(changes.source_data_same, 1);
258 found =
259 manager.find(1, file_binary_hash, filesize, file_type, zero_count, nonprobative_count);
260 TEST_EQ(found, true);
261 TEST_EQ(file_binary_hash, "fbh");
262 TEST_EQ(filesize, 2);
263 TEST_EQ(file_type, "ft");
264 TEST_EQ(zero_count, 1);
265 TEST_EQ(nonprobative_count, 3);
266
267 // change
268 manager.insert(1, "fbh2", 22, "ft2", 31, 32, changes);
269 TEST_EQ(changes.source_data_inserted, 1);
270 TEST_EQ(changes.source_data_changed, 1);
271 TEST_EQ(changes.source_data_same, 1);
272 manager.find(1, file_binary_hash, filesize, file_type, zero_count, nonprobative_count);
273 TEST_EQ(file_binary_hash, "fbh2");
274 TEST_EQ(filesize, 22);
275 TEST_EQ(file_type, "ft2");
276 TEST_EQ(zero_count, 31);
277 TEST_EQ(nonprobative_count, 32);
278
279 // insert second
280 manager.insert(0, "", 0, "", 0, 0, changes);
281 manager.find(0, file_binary_hash, filesize, file_type, zero_count, nonprobative_count);
282 TEST_EQ(file_binary_hash, "");
283 TEST_EQ(filesize, 0);
284 TEST_EQ(file_type, "");
285 TEST_EQ(zero_count, 0);
286 TEST_EQ(nonprobative_count, 0);
287
288 // make sure 1 is still in place
289 manager.find(1, file_binary_hash, filesize, file_type, zero_count, nonprobative_count);
290 TEST_EQ(file_binary_hash, "fbh2");
291 TEST_EQ(filesize, 22);
292 TEST_EQ(file_type, "ft2");
293 TEST_EQ(zero_count, 31);
294 TEST_EQ(nonprobative_count, 32);
295
296 // size
297 TEST_EQ(manager.size(), 2);
298 }
299
300 // ************************************************************
301 // lmdb_source_name_manager
302 // ************************************************************
lmdb_source_name_manager()303 void lmdb_source_name_manager() {
304
305 // variables
306 source_names_t source_names;
307 hashdb::lmdb_changes_t changes;
308 bool found;
309
310 // create new manager
311 make_new_hashdb_dir(hashdb_dir);
312 hashdb::lmdb_source_name_manager_t manager(hashdb_dir, hashdb::RW_NEW);
313
314 // no source ID when DB is empty
315 found = manager.find(1, source_names);
316 TEST_EQ(found, false);
317
318 // insert first element
319 manager.insert(1, "rn", "fn", changes);
320 TEST_EQ(changes.source_name_inserted, 1);
321 manager.insert(1, "rn", "fn", changes);
322 TEST_EQ(changes.source_name_already_present, 1);
323 manager.insert(1, "rn2", "fn2", changes);
324 TEST_EQ(changes.source_name_inserted, 2);
325 manager.insert(1, "rn1", "fn1", changes);
326 TEST_EQ(changes.source_name_inserted, 3);
327
328 // insert second element
329 manager.insert(2, "rn11", "fn11", changes);
330 TEST_EQ(changes.source_name_inserted, 4);
331
332 // find first element
333 found = manager.find(1, source_names);
334 TEST_EQ(found, true);
335 source_names_t::const_iterator it = source_names.begin();
336 TEST_EQ(source_names.size(), 3);
337 TEST_EQ(it->first, "rn");
338 TEST_EQ(it->second, "fn");
339 ++it;
340 TEST_EQ(it->first, "rn1");
341 TEST_EQ(it->second, "fn1");
342 ++it;
343 TEST_EQ(it->first, "rn2");
344 TEST_EQ(it->second, "fn2");
345
346 // find second element
347 found = manager.find(2, source_names);
348 TEST_EQ(found, true);
349 it = source_names.begin();
350 TEST_EQ(source_names.size(), 1);
351 TEST_EQ(it->first, "rn11");
352 TEST_EQ(it->second, "fn11");
353
354 // no source ID when DB is not empty
355 found = manager.find(3, source_names);
356 TEST_EQ(found, false);
357 TEST_EQ(source_names.size(), 0);
358
359 // size
360 TEST_EQ(manager.size(), 4);
361 }
362
363 // ************************************************************
364 // main
365 // ************************************************************
main(int argc,char * argv[])366 int main(int argc, char* argv[]) {
367
368 // lmdb_hash_manager
369 lmdb_hash_manager_create();
370 lmdb_hash_manager_write();
371 lmdb_hash_manager_read();
372 lmdb_hash_manager_count();
373
374 // source ID manager
375 lmdb_source_id_manager();
376
377 // source data manager
378 lmdb_source_data_manager();
379
380 // source name manager
381 lmdb_source_name_manager();
382
383 // done
384 std::cout << "lmdb_other_managers_test Done.\n";
385 return 0;
386 }
387
388