1 /*
2 Copyright 2021 Northern.tech AS
3
4 This file is part of CFEngine 3 - written and maintained by Northern.tech AS.
5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; version 3.
9
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
18
19 To the extent this program is licensed as part of the Enterprise
20 versions of CFEngine, the applicable Commercial Open Source License
21 (COSL) may apply to this file if you as a licensee so wish it. See
22 included file COSL.txt.
23 */
24
25 #include <platform.h>
26 #include <tokyo_check.h>
27
28 #ifdef TCDB
29
30 #include <array_map_priv.h>
31 #include <hash_map_priv.h>
32 #include <map.h>
33 #include <string_lib.h>
34 #include <logging.h>
35 #include <cf3.defs.h>
36
37 /*
38 * The idea behind the following code comes from : copiousfreetime@github
39 */
40
41 /*
42 * Fields we will need from a TC record
43 */
44 typedef struct TokyoCabinetRecord
45 {
46 uint64_t offset;
47 uint64_t length;
48
49 uint64_t left;
50 uint64_t right;
51
52 uint32_t key_size;
53 uint32_t rec_size;
54 uint16_t pad_size;
55
56 uint8_t magic;
57 uint8_t hash;
58 } TokyoCabinetRecord;
59
60 /* meta information from the Hash Database
61 * used to coordinate the other operations
62 */
63 typedef struct DBMeta
64 {
65 uint64_t bucket_count; /* Number of hash buckets */
66 uint64_t bucket_offset; /* Start of the bucket list */
67
68 uint64_t record_count; /* Number of records */
69 uint64_t record_offset; /* First record offset in file */
70
71 short alignment_pow; /* power of 2 for calculating offsets */
72 short bytes_per; /* 4 or 8 */
73 char dbpath[PATH_MAX + 1]; /* full pathname to the database file */
74
75 int fd;
76
77 StringMap *offset_map;
78 StringMap *record_map;
79 } DBMeta;
80
DBMetaNewDirect(const char * dbfilename)81 static DBMeta *DBMetaNewDirect(const char *dbfilename)
82 {
83 char hbuf[256];
84 DBMeta *dbmeta = xcalloc(1, sizeof(*dbmeta));
85
86 char *p = realpath(dbfilename, dbmeta->dbpath);
87 if (p == NULL ||
88 -1 == (dbmeta->fd = open(dbmeta->dbpath, O_RDONLY)))
89 {
90 Log(LOG_LEVEL_ERR, "Failure opening file '%s' (%s: %s)",
91 (p == NULL) ? dbfilename : dbmeta->dbpath,
92 (p == NULL) ? "realpath" : "open",
93 GetErrorStr());
94
95 free(dbmeta);
96 return NULL;
97 }
98
99 if (256 != read(dbmeta->fd, hbuf, 256))
100 {
101 Log(LOG_LEVEL_ERR, "Failure reading from database '%s'. (read: %s)",
102 dbmeta->dbpath, GetErrorStr());
103 close(dbmeta->fd);
104 if (dbmeta)
105 {
106 free(dbmeta);
107 }
108 return NULL;
109 }
110
111 memcpy(&(dbmeta->bucket_count), hbuf + 40, sizeof(uint64_t));
112 dbmeta->bucket_offset = 256;
113 uint8_t opts;
114 memcpy(&opts, hbuf + 36, sizeof(uint8_t));
115 dbmeta->bytes_per =
116 (opts & (1 << 0)) ? sizeof(uint64_t) : sizeof(uint32_t);
117
118 memcpy(&(dbmeta->record_count), hbuf + 48, sizeof(uint64_t));
119 memcpy(&(dbmeta->record_offset), hbuf + 64, sizeof(uint64_t));
120 memcpy(&(dbmeta->alignment_pow), hbuf + 34, sizeof(uint8_t));
121 dbmeta->offset_map = StringMapNew();
122 dbmeta->record_map = StringMapNew();
123
124 Log(LOG_LEVEL_VERBOSE, "Database : %s", dbmeta->dbpath);
125 Log(LOG_LEVEL_VERBOSE, " number of buckets : %" PRIu64,
126 dbmeta->bucket_count);
127 Log(LOG_LEVEL_VERBOSE, " offset of buckets : %" PRIu64,
128 dbmeta->bucket_offset);
129 Log(LOG_LEVEL_VERBOSE, " bytes per pointer : %hd",
130 dbmeta->bytes_per);
131 Log(LOG_LEVEL_VERBOSE, " alignment power : %hd",
132 dbmeta->alignment_pow);
133 Log(LOG_LEVEL_VERBOSE, " number of records : %" PRIu64,
134 dbmeta->record_count);
135 Log(LOG_LEVEL_VERBOSE, " offset of records : %" PRIu64,
136 dbmeta->record_offset);
137
138 return dbmeta;
139 }
140
DBMetaFree(DBMeta * dbmeta)141 static void DBMetaFree(DBMeta * dbmeta)
142 {
143 StringMapDestroy(dbmeta->offset_map);
144 StringMapDestroy(dbmeta->record_map);
145
146 close(dbmeta->fd);
147
148 if (dbmeta)
149 {
150 free(dbmeta);
151 }
152 }
153
AddOffsetToMapUnlessExists(StringMap ** tree,uint64_t offset,int64_t bucket_index)154 static int AddOffsetToMapUnlessExists(StringMap ** tree, uint64_t offset,
155 int64_t bucket_index)
156 {
157 char *tmp;
158 xasprintf(&tmp, "%" PRIu64, offset);
159 char *val;
160 if (StringMapHasKey(*tree, tmp) == false)
161 {
162 xasprintf(&val, "%" PRIu64, bucket_index);
163 StringMapInsert(*tree, tmp, val);
164 }
165 else
166 {
167 Log(LOG_LEVEL_ERR,
168 "Duplicate offset for value %" PRIu64 " at index %" PRId64 ", other value %" PRIu64 ", other index '%s'",
169 offset, bucket_index,
170 offset, (char *) StringMapGet(*tree, tmp));
171 free(tmp);
172 }
173 return 0;
174 }
175
DBMetaPopulateOffsetMap(DBMeta * dbmeta)176 static int DBMetaPopulateOffsetMap(DBMeta * dbmeta)
177 {
178 uint64_t i;
179
180 if (lseek(dbmeta->fd, dbmeta->bucket_offset, SEEK_SET) == -1)
181 {
182 Log(LOG_LEVEL_ERR,
183 "Error traversing bucket section to find record offsets '%s'",
184 strerror(errno));
185 return 1;
186 }
187
188 for (i = 0; i < dbmeta->bucket_count; i++)
189 {
190 uint64_t offset = 0LL;
191 int b = read(dbmeta->fd, &offset, dbmeta->bytes_per);
192
193 if (b != dbmeta->bytes_per)
194 {
195 Log(LOG_LEVEL_ERR, "Read the wrong number of bytes (%d)", b);
196 return 2;
197 }
198
199 /* if the value is > 0 then we have a number so do something with it */
200 if (offset > 0)
201 {
202 offset = offset << dbmeta->alignment_pow;
203 if (AddOffsetToMapUnlessExists(&(dbmeta->offset_map), offset, i))
204 {
205 return 3;
206 }
207 }
208 }
209
210 Log(LOG_LEVEL_VERBOSE, "Found %zu buckets with offsets",
211 StringMapSize(dbmeta->offset_map));
212 return 0;
213 }
214
215 typedef enum
216 { // enumeration for magic data
217 MAGIC_DATA_BLOCK = 0xc8, // for data block
218 MAGIC_FREE_BLOCK = 0xb0 // for free block
219 } TypeOfBlock;
220
TCReadVaryInt(int fd,uint32_t * result)221 static int TCReadVaryInt(int fd, uint32_t * result)
222 {
223 uint64_t num = 0;
224 unsigned int base = 1;
225 unsigned int i = 0;
226 int read_bytes = 0;
227 char c;
228
229 while (true)
230 {
231 read_bytes += read(fd, &c, 1);
232 if (c >= 0)
233 {
234 num += (c * base);
235 break;
236 }
237 num += (base * (c + 1) * -1);
238 base <<= 7;
239 i += 1;
240 }
241
242 *result = num;
243
244 return read_bytes;
245 }
246
247
DBMetaReadOneRecord(DBMeta * dbmeta,TokyoCabinetRecord * rec)248 static bool DBMetaReadOneRecord(DBMeta * dbmeta, TokyoCabinetRecord * rec)
249 {
250 if (lseek(dbmeta->fd, rec->offset, SEEK_SET) == -1)
251 {
252 Log(LOG_LEVEL_ERR, "Error traversing record section to find records : ");
253 }
254
255 while (true)
256 {
257 // get the location of the current read
258 rec->offset = lseek(dbmeta->fd, 0, SEEK_CUR);
259 if (rec->offset == (off_t) - 1)
260 {
261 Log(LOG_LEVEL_ERR,
262 "Error traversing record section to find records");
263 }
264
265 if (1 != read(dbmeta->fd, &(rec->magic), 1))
266 {
267 Log(LOG_LEVEL_ERR, "Failure reading 1 byte, (read: %s)",
268 GetErrorStr());
269 return false;
270 }
271
272 if (MAGIC_DATA_BLOCK == rec->magic)
273 {
274 Log(LOG_LEVEL_VERBOSE, "off=%" PRIu64 "[c8]", rec->offset);
275 int length = 1;
276
277 length += read(dbmeta->fd, &(rec->hash), 1);
278 length += read(dbmeta->fd, &(rec->left), dbmeta->bytes_per);
279 rec->left = rec->left << dbmeta->alignment_pow;
280
281 length += read(dbmeta->fd, &(rec->right), dbmeta->bytes_per);
282 rec->right = rec->right << dbmeta->alignment_pow;
283
284 length += read(dbmeta->fd, &(rec->pad_size), 2);
285 length += rec->pad_size;
286
287 length += TCReadVaryInt(dbmeta->fd, &(rec->key_size));
288 length += TCReadVaryInt(dbmeta->fd, &(rec->rec_size));
289
290 rec->length = length + rec->key_size + rec->rec_size;
291 return true;
292
293 }
294 else if (MAGIC_FREE_BLOCK == rec->magic)
295 {
296 Log(LOG_LEVEL_VERBOSE, "off=%" PRIu64 "[b0]", rec->offset);
297 uint32_t length;
298 rec->length = 1;
299 rec->length += read(dbmeta->fd, &length, sizeof(length));
300 rec->length += length;
301 return true;
302
303 }
304 else
305 {
306 Log(LOG_LEVEL_VERBOSE, "Read a non-magic byte (skip it)");
307 }
308 }
309 Log(LOG_LEVEL_ERR, "Read loop reached here");
310 return false;
311 }
312
DBMetaPopulateRecordMap(DBMeta * dbmeta)313 static int DBMetaPopulateRecordMap(DBMeta * dbmeta)
314 {
315 off_t offset;
316 uint64_t data_blocks = 0;
317 uint64_t free_blocks = 0;
318 struct stat st;
319
320 offset = dbmeta->record_offset;
321 if (fstat(dbmeta->fd, &st) == -1)
322 {
323 Log(LOG_LEVEL_ERR, "Error getting file stats. (fstat: %s)", GetErrorStr());
324 return 1;
325 }
326
327 while (offset < st.st_size)
328 {
329
330 TokyoCabinetRecord new_rec;
331 memset(&new_rec, 0, sizeof(TokyoCabinetRecord));
332 new_rec.offset = offset;
333
334 // read a variable-length record
335 if (!DBMetaReadOneRecord(dbmeta, &new_rec))
336 {
337 Log(LOG_LEVEL_ERR, "Unable to fetch a new record from DB file");
338 return 2;
339 }
340 else
341 {
342 offset = new_rec.offset + new_rec.length;
343 }
344
345 // if it is a data record then:
346 // for the record, its left and right do:
347 // look up that record in the offset tree
348 // 1) remove it if it exists
349 // 2) add it to the record_tree if it doesn't
350
351 if (MAGIC_DATA_BLOCK == new_rec.magic)
352 {
353
354 if (new_rec.offset > 0)
355 {
356
357 char *key;
358 xasprintf(&key, "%" PRIu64, new_rec.offset);
359 if (StringMapHasKey(dbmeta->offset_map, key) == true)
360 {
361 if (key)
362 {
363 free(key);
364 }
365 }
366 else
367 {
368 StringMapInsert(dbmeta->record_map, key, xstrdup("0"));
369 }
370 }
371 else
372 {
373 Log(LOG_LEVEL_ERR,
374 "new_rec.offset cannot be <= 0 ???");
375 }
376
377 if (new_rec.left > 0)
378 {
379 Log(LOG_LEVEL_VERBOSE, "handle left %" PRIu64, new_rec.left);
380 if (AddOffsetToMapUnlessExists
381 (&(dbmeta->offset_map), new_rec.left, -1))
382 {
383 return 4;
384 }
385 }
386
387 if (new_rec.right > 0)
388 {
389 Log(LOG_LEVEL_VERBOSE, "handle right %" PRIu64, new_rec.right);
390 if (AddOffsetToMapUnlessExists
391 (&(dbmeta->offset_map), new_rec.right, -1))
392 {
393 return 4;
394 }
395 }
396
397 data_blocks++;
398 }
399 else if (MAGIC_FREE_BLOCK == new_rec.magic)
400 {
401 // if it is a fragment record, then skip it
402 free_blocks++;
403 }
404 else
405 {
406 Log(LOG_LEVEL_ERR, "NO record found at offset %" PRIu64,
407 new_rec.offset);
408 }
409 }
410
411 // if we are not at the end of the file, output the current file offset
412 // with an appropriate message and return
413 Log(LOG_LEVEL_VERBOSE, "Found %" PRIu64 " data records and "
414 "%" PRIu64 " free block records", data_blocks, free_blocks);
415
416 return 0;
417 }
418
DBMetaGetResults(DBMeta * dbmeta)419 static int DBMetaGetResults(DBMeta * dbmeta)
420 {
421 uint64_t buckets_no_record = StringMapSize(dbmeta->offset_map);
422 uint64_t records_no_bucket = StringMapSize(dbmeta->record_map);
423 int ret = 0;
424
425 Log(LOG_LEVEL_VERBOSE,
426 "Found %" PRIu64 " offsets listed in buckets that do not have records",
427 buckets_no_record);
428 Log(LOG_LEVEL_VERBOSE,
429 "Found %" PRIu64 " records in data that do not have an offset pointing to them",
430 records_no_bucket);
431
432 if (buckets_no_record > 0)
433 {
434 ret += 1;
435 }
436
437 if (records_no_bucket > 0)
438 {
439 ret += 2;
440 }
441 return ret;
442 }
443
CheckTokyoDBCoherence(const char * path)444 int CheckTokyoDBCoherence(const char *path)
445 {
446 int ret = 0;
447 DBMeta *dbmeta;
448
449 dbmeta = DBMetaNewDirect(path);
450 if (dbmeta == NULL)
451 {
452 return 1;
453 }
454
455 Log(LOG_LEVEL_VERBOSE, "Populating with bucket section offsets");
456 ret = DBMetaPopulateOffsetMap(dbmeta);
457 if (ret)
458 {
459 goto clean;
460 }
461
462 Log(LOG_LEVEL_VERBOSE, "Populating with record section offsets");
463 ret = DBMetaPopulateRecordMap(dbmeta);
464 if (ret)
465 {
466 goto clean;
467 }
468
469 ret = DBMetaGetResults(dbmeta);
470
471 clean:
472 if (dbmeta)
473 {
474 DBMetaFree(dbmeta);
475 }
476
477 return ret;
478 }
479
480 #else
481
482 #include <compiler.h> // ARG_UNUSED
483
CheckTokyoDBCoherence(ARG_UNUSED const char * path)484 int CheckTokyoDBCoherence(ARG_UNUSED const char *path)
485 {
486 return 0;
487 }
488
489 #endif
490