1 /**
2 * gkhash.c -- default hash table functions
3 * ______ ___
4 * / ____/___ / | _____________ __________
5 * / / __/ __ \/ /| |/ ___/ ___/ _ \/ ___/ ___/
6 * / /_/ / /_/ / ___ / /__/ /__/ __(__ |__ )
7 * \____/\____/_/ |_\___/\___/\___/____/____/
8 *
9 * The MIT License (MIT)
10 * Copyright (c) 2009-2020 Gerardo Orellana <hello @ goaccess.io>
11 *
12 * Permission is hereby granted, free of charge, to any person obtaining a copy
13 * of this software and associated documentation files (the "Software"), to deal
14 * in the Software without restriction, including without limitation the rights
15 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
16 * copies of the Software, and to permit persons to whom the Software is
17 * furnished to do so, subject to the following conditions:
18 *
19 * The above copyright notice and this permission notice shall be included in all
20 * copies or substantial portions of the Software.
21 *
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
23 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
25 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
27 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
28 * SOFTWARE.
29 */
30
31 #if HAVE_CONFIG_H
32 #include <config.h>
33 #endif
34
35 #include <errno.h>
36 #include <stdio.h>
37 #include <stdlib.h>
38 #include <string.h>
39 #include <sys/stat.h>
40 #include <sys/types.h>
41 #include <unistd.h>
42
43 #include "gkhash.h"
44
45 #include "error.h"
46 #include "sort.h"
47 #include "tpl.h"
48 #include "util.h"
49 #include "xmalloc.h"
50
51 /* *INDENT-OFF* */
52 /* Hash tables used across the whole app */
53 static khash_t (igkh) * ht_dates = NULL;
54 static khash_t (si32) * ht_seqs = NULL;
55 static khash_t (ss32) * ht_hostnames = NULL;
56 static khash_t (si32) * ht_cnt_overall = NULL;
57 static khash_t (iglp) * ht_last_parse = NULL;
58 static khash_t (ss32) * ht_json_logfmt = NULL;
59
60 static GKHashModule *cache_storage = NULL;
61
62 static uint32_t *persisted_dates = NULL;
63 static uint32_t persisted_dates_len = 0;
64 /* *INDENT-ON* */
65
66 /* Allocate memory for a new store container GKHashStorage instance.
67 *
68 * On success, the newly allocated GKHashStorage is returned . */
69 static GKHashStorage *
new_gkhstorage(void)70 new_gkhstorage (void) {
71 GKHashStorage *storage = xcalloc (1, sizeof (GKHashStorage));
72 return storage;
73 }
74
75 /* Allocate memory for a new module GKHashModule instance.
76 *
77 * On success, the newly allocated GKHashStorage is returned . */
78 static GKHashModule *
new_gkhmodule(uint32_t size)79 new_gkhmodule (uint32_t size) {
80 GKHashModule *storage = xcalloc (size, sizeof (GKHashModule));
81 return storage;
82 }
83
84 /* Allocate memory for a new global GKHashGlobal instance.
85 *
86 * On success, the newly allocated GKHashGlobal is returned . */
87 static GKHashGlobal *
new_gkhglobal(void)88 new_gkhglobal (void) {
89 GKHashGlobal *storage = xcalloc (1, sizeof (GKHashGlobal));
90 return storage;
91 }
92
93 /* Get the module string value given a metric enum value.
94 *
95 * On error, NULL is returned.
96 * On success, the string module value is returned. */
97 static char *
get_mtr_type_str(GSMetricType type)98 get_mtr_type_str (GSMetricType type) {
99 GEnum enum_metric_types[] = {
100 {"II32", MTRC_TYPE_II32},
101 {"IS32", MTRC_TYPE_IS32},
102 {"IU64", MTRC_TYPE_IU64},
103 {"SI32", MTRC_TYPE_SI32},
104 {"SS32", MTRC_TYPE_SS32},
105 {"IGSL", MTRC_TYPE_IGSL},
106 {"SU64", MTRC_TYPE_SU64},
107 {"IGKH", MTRC_TYPE_IGKH},
108 {"U648", MTRC_TYPE_U648},
109 };
110 return enum2str (enum_metric_types, ARRAY_SIZE (enum_metric_types), type);
111 }
112
113 /* Initialize a new uint32_t key - uint32_t value hash table */
114 static void *
new_ii32_ht(void)115 new_ii32_ht (void) {
116 khash_t (ii32) * h = kh_init (ii32);
117 return h;
118 }
119
120 /* Initialize a new uint32_t key - GLastParse value hash table */
121 static void *
new_iglp_ht(void)122 new_iglp_ht (void) {
123 khash_t (iglp) * h = kh_init (iglp);
124 return h;
125 }
126
127 /* Initialize a new uint32_t key - GKHashStorage value hash table */
128 static void *
new_igkh_ht(void)129 new_igkh_ht (void) {
130 khash_t (igkh) * h = kh_init (igkh);
131 return h;
132 }
133
134 /* Initialize a new uint32_t key - string value hash table */
135 static void *
new_is32_ht(void)136 new_is32_ht (void) {
137 khash_t (is32) * h = kh_init (is32);
138 return h;
139 }
140
141 /* Initialize a new uint32_t key - uint64_t value hash table */
142 static void *
new_iu64_ht(void)143 new_iu64_ht (void) {
144 khash_t (iu64) * h = kh_init (iu64);
145 return h;
146 }
147
148 /* Initialize a new uint64_t key - uint8_t value hash table */
149 static void *
new_u648_ht(void)150 new_u648_ht (void) {
151 khash_t (u648) * h = kh_init (u648);
152 return h;
153 }
154
155 /* Initialize a new string key - uint32_t value hash table */
156 static void *
new_si32_ht(void)157 new_si32_ht (void) {
158 khash_t (si32) * h = kh_init (si32);
159 return h;
160 }
161
162 /* Initialize a new string key - string value hash table */
163 static void *
new_ss32_ht(void)164 new_ss32_ht (void) {
165 khash_t (ss32) * h = kh_init (ss32);
166 return h;
167 }
168
169 /* Initialize a new uint32_t key - GSLList value hash table */
170 static void *
new_igsl_ht(void)171 new_igsl_ht (void) {
172 khash_t (igsl) * h = kh_init (igsl);
173 return h;
174 }
175
176 /* Initialize a new string key - uint64_t value hash table */
177 static void *
new_su64_ht(void)178 new_su64_ht (void) {
179 khash_t (su64) * h = kh_init (su64);
180 return h;
181 }
182
183 /* Destroys both the hash structure and the keys for a
184 * string key - uint32_t value hash */
185 static void
des_si32_free(void * h,uint8_t free_data)186 des_si32_free (void *h, uint8_t free_data) {
187 khint_t k;
188 khash_t (si32) * hash = h;
189 if (!hash)
190 return;
191
192 if (!free_data)
193 goto des;
194
195 for (k = 0; k < kh_end (hash); ++k) {
196 if (kh_exist (hash, k)) {
197 free ((char *) kh_key (hash, k));
198 }
199 }
200
201 des:
202 kh_destroy (si32, hash);
203 }
204
205 /* Deletes an entry from the hash table and optionally the keys for a string
206 * key - uint32_t value hash */
207 static void
del_si32_free(void * h,uint8_t free_data)208 del_si32_free (void *h, uint8_t free_data) {
209 khint_t k;
210 khash_t (si32) * hash = h;
211 if (!hash)
212 return;
213
214 for (k = 0; k < kh_end (hash); ++k) {
215 if (kh_exist (hash, k)) {
216 if (free_data)
217 free ((char *) kh_key (hash, k));
218 kh_del (si32, hash, k);
219 }
220 }
221 }
222
223 /* Destroys both the hash structure and its string values */
224 static void
des_is32_free(void * h,uint8_t free_data)225 des_is32_free (void *h, uint8_t free_data) {
226 khint_t k;
227 khash_t (is32) * hash = h;
228 if (!hash)
229 return;
230
231 if (!free_data)
232 goto des;
233
234 for (k = 0; k < kh_end (hash); ++k) {
235 if (kh_exist (hash, k)) {
236 free ((char *) kh_value (hash, k));
237 }
238 }
239 des:
240 kh_destroy (is32, hash);
241 }
242
243 /* Deletes both the hash entry and its string values */
244 static void
del_is32_free(void * h,uint8_t free_data)245 del_is32_free (void *h, uint8_t free_data) {
246 khint_t k;
247 khash_t (is32) * hash = h;
248 if (!hash)
249 return;
250
251 for (k = 0; k < kh_end (hash); ++k) {
252 if (kh_exist (hash, k)) {
253 if (free_data)
254 free ((char *) kh_value (hash, k));
255 kh_del (is32, hash, k);
256 }
257 }
258 }
259
260 /* Destroys both the hash structure and its string
261 * keys and string values */
262 static void
des_ss32_free(void * h,uint8_t free_data)263 des_ss32_free (void *h, uint8_t free_data) {
264 khint_t k;
265 khash_t (ss32) * hash = h;
266 if (!hash)
267 return;
268
269 if (!free_data)
270 goto des;
271
272 for (k = 0; k < kh_end (hash); ++k) {
273 if (kh_exist (hash, k)) {
274 free ((char *) kh_key (hash, k));
275 free ((char *) kh_value (hash, k));
276 }
277 }
278
279 des:
280 kh_destroy (ss32, hash);
281 }
282
283 /* Destroys the hash structure */
284 static void
des_ii32(void * h,GO_UNUSED uint8_t free_data)285 des_ii32 (void *h, GO_UNUSED uint8_t free_data) {
286 khash_t (ii32) * hash = h;
287 if (!hash)
288 return;
289 kh_destroy (ii32, hash);
290 }
291
292 /* Deletes all entries from the hash table */
293 static void
del_ii32(void * h,GO_UNUSED uint8_t free_data)294 del_ii32 (void *h, GO_UNUSED uint8_t free_data) {
295 khint_t k;
296 khash_t (ii32) * hash = h;
297 if (!hash)
298 return;
299
300 for (k = 0; k < kh_end (hash); ++k) {
301 if (kh_exist (hash, k)) {
302 kh_del (ii32, hash, k);
303 }
304 }
305 }
306
307 /* Destroys the hash structure */
308 static void
des_u648(void * h,GO_UNUSED uint8_t free_data)309 des_u648 (void *h, GO_UNUSED uint8_t free_data) {
310 khash_t (u648) * hash = h;
311 if (!hash)
312 return;
313 kh_destroy (u648, hash);
314 }
315
316 /* Destroys the hash structure */
317 static void
des_iglp(void * h,GO_UNUSED uint8_t free_data)318 des_iglp (void *h, GO_UNUSED uint8_t free_data) {
319 khash_t (iglp) * hash = h;
320 if (!hash)
321 return;
322 kh_destroy (iglp, hash);
323 }
324
325 /* Deletes all entries from the hash table */
326 static void
del_u648(void * h,GO_UNUSED uint8_t free_data)327 del_u648 (void *h, GO_UNUSED uint8_t free_data) {
328 khint_t k;
329 khash_t (u648) * hash = h;
330 if (!hash)
331 return;
332
333 for (k = 0; k < kh_end (hash); ++k) {
334 if (kh_exist (hash, k)) {
335 kh_del (u648, hash, k);
336 }
337 }
338 }
339
340 /* Destroys both the hash structure and its GSLList
341 * values */
342 static void
des_igsl_free(void * h,uint8_t free_data)343 des_igsl_free (void *h, uint8_t free_data) {
344 khash_t (igsl) * hash = h;
345 khint_t k;
346 void *list = NULL;
347 if (!hash)
348 return;
349
350 if (!free_data)
351 goto des;
352
353 for (k = 0; k < kh_end (hash); ++k) {
354 if (kh_exist (hash, k) && (list = kh_value (hash, k))) {
355 list_remove_nodes (list);
356 }
357 }
358 des:
359 kh_destroy (igsl, hash);
360 }
361
362 /* Deletes all entries from the hash table and optionally frees its GSLList */
363 static void
del_igsl_free(void * h,uint8_t free_data)364 del_igsl_free (void *h, uint8_t free_data) {
365 khint_t k;
366 khash_t (igsl) * hash = h;
367 void *list = NULL;
368 if (!hash)
369 return;
370
371 for (k = 0; k < kh_end (hash); ++k) {
372 if (!kh_exist (hash, k))
373 continue;
374
375 if (free_data) {
376 list = kh_value (hash, k);
377 list_remove_nodes (list);
378 }
379 kh_del (igsl, hash, k);
380 }
381 }
382
383 /* Destroys both the hash structure and the keys for a
384 * string key - uint64_t value hash */
385 static void
des_su64_free(void * h,uint8_t free_data)386 des_su64_free (void *h, uint8_t free_data) {
387 khash_t (su64) * hash = h;
388 khint_t k;
389 if (!hash)
390 return;
391
392 if (!free_data)
393 goto des;
394
395 for (k = 0; k < kh_end (hash); ++k) {
396 if (kh_exist (hash, k)) {
397 free ((char *) kh_key (hash, k));
398 }
399 }
400
401 des:
402 kh_destroy (su64, hash);
403 }
404
405 /* Deletes all entries from the hash table and optionally frees its string key */
406 static void
del_su64_free(void * h,uint8_t free_data)407 del_su64_free (void *h, uint8_t free_data) {
408 khint_t k;
409 khash_t (su64) * hash = h;
410 if (!hash)
411 return;
412
413 for (k = 0; k < kh_end (hash); ++k) {
414 if (kh_exist (hash, k)) {
415 if (free_data)
416 free ((char *) kh_key (hash, k));
417 kh_del (su64, hash, k);
418 }
419 }
420 }
421
422 /* Destroys the hash structure */
423 static void
des_iu64(void * h,GO_UNUSED uint8_t free_data)424 des_iu64 (void *h, GO_UNUSED uint8_t free_data) {
425 khash_t (iu64) * hash = h;
426 if (!hash)
427 return;
428 kh_destroy (iu64, hash);
429 }
430
431 /* Deletes all entries from the hash table */
432 static void
del_iu64(void * h,GO_UNUSED uint8_t free_data)433 del_iu64 (void *h, GO_UNUSED uint8_t free_data) {
434 khint_t k;
435 khash_t (iu64) * hash = h;
436 if (!hash)
437 return;
438
439 for (k = 0; k < kh_end (hash); ++k) {
440 if (kh_exist (hash, k)) {
441 kh_del (iu64, hash, k);
442 }
443 }
444 }
445
446 /* *INDENT-OFF* */
447 static const GKHashMetric global_metrics[] = {
448 { MTRC_UNIQUE_KEYS , MTRC_TYPE_SI32 , new_si32_ht , des_si32_free , del_si32_free , NULL , "SI32_UNIQUE_KEYS.db" } ,
449 { MTRC_AGENT_KEYS , MTRC_TYPE_SI32 , new_si32_ht , des_si32_free , del_si32_free , NULL , "SI32_AGENT_KEYS.db" } ,
450 { MTRC_AGENT_VALS , MTRC_TYPE_IS32 , new_is32_ht , des_is32_free , del_is32_free , NULL , "IS32_AGENT_VALS.db" } ,
451 { MTRC_CNT_VALID , MTRC_TYPE_II32 , new_ii32_ht , des_ii32 , del_ii32 , NULL , "II32_CNT_VALID.db" } ,
452 { MTRC_CNT_BW , MTRC_TYPE_IU64 , new_iu64_ht , des_iu64 , del_iu64 , NULL , "IU64_CNT_BW.db" } ,
453 };
454
455 static GKHashMetric module_metrics[] = {
456 { MTRC_KEYMAP , MTRC_TYPE_SI32 , new_si32_ht , des_si32_free , del_si32_free , NULL , NULL} ,
457 { MTRC_ROOTMAP , MTRC_TYPE_IS32 , new_is32_ht , des_is32_free , del_is32_free , NULL , NULL} ,
458 { MTRC_DATAMAP , MTRC_TYPE_IS32 , new_is32_ht , des_is32_free , del_is32_free , NULL , NULL} ,
459 { MTRC_UNIQMAP , MTRC_TYPE_U648 , new_u648_ht , des_u648 , del_u648 , NULL , NULL} ,
460 { MTRC_ROOT , MTRC_TYPE_II32 , new_ii32_ht , des_ii32 , del_ii32 , NULL , NULL} ,
461 { MTRC_HITS , MTRC_TYPE_II32 , new_ii32_ht , des_ii32 , del_ii32 , NULL , NULL} ,
462 { MTRC_VISITORS , MTRC_TYPE_II32 , new_ii32_ht , des_ii32 , del_ii32 , NULL , NULL} ,
463 { MTRC_BW , MTRC_TYPE_IU64 , new_iu64_ht , des_iu64 , del_iu64 , NULL , NULL} ,
464 { MTRC_CUMTS , MTRC_TYPE_IU64 , new_iu64_ht , des_iu64 , del_iu64 , NULL , NULL} ,
465 { MTRC_MAXTS , MTRC_TYPE_IU64 , new_iu64_ht , des_iu64 , del_iu64 , NULL , NULL} ,
466 { MTRC_METHODS , MTRC_TYPE_IS32 , new_is32_ht , des_is32_free , del_is32_free , NULL , NULL} ,
467 { MTRC_PROTOCOLS , MTRC_TYPE_IS32 , new_is32_ht , des_is32_free , del_is32_free , NULL , NULL} ,
468 { MTRC_AGENTS , MTRC_TYPE_IGSL , new_igsl_ht , des_igsl_free , del_igsl_free , NULL , NULL} ,
469 { MTRC_METADATA , MTRC_TYPE_SU64 , new_su64_ht , des_su64_free , del_su64_free , NULL , NULL} ,
470 };
471 /* *INDENT-ON* */
472
473 /* Initialize module metrics and mallocs its hash structure */
474 static void
init_tables(GModule module,GKHashModule * storage)475 init_tables (GModule module, GKHashModule * storage) {
476 int n = 0, i;
477
478 n = ARRAY_SIZE (module_metrics);
479 for (i = 0; i < n; i++) {
480 storage[module].metrics[i] = module_metrics[i];
481 storage[module].metrics[i].hash = module_metrics[i].alloc ();
482 }
483 }
484
485 /* Destroys malloc'd global metrics */
486 static void
free_global_metrics(GKHashGlobal * ghash)487 free_global_metrics (GKHashGlobal * ghash) {
488 int i, n = 0;
489 GKHashMetric mtrc;
490
491 if (!ghash)
492 return;
493
494 n = ARRAY_SIZE (global_metrics);
495 for (i = 0; i < n; i++) {
496 mtrc = ghash->metrics[i];
497 mtrc.des (mtrc.hash, 1);
498 }
499 }
500
501 /* Destroys malloc'd mdule metrics */
502 static void
free_module_metrics(GKHashModule * mhash,GModule module,uint8_t free_data)503 free_module_metrics (GKHashModule * mhash, GModule module, uint8_t free_data) {
504 int i, n = 0;
505 GKHashMetric mtrc;
506
507 if (!mhash)
508 return;
509
510 n = ARRAY_SIZE (module_metrics);
511 for (i = 0; i < n; i++) {
512 mtrc = mhash[module].metrics[i];
513 mtrc.des (mtrc.hash, free_data);
514 }
515 }
516
517 /* For each module metric, deletes all entries from the hash table */
518 static void
del_module_metrics(GKHashModule * mhash,GModule module,uint8_t free_data)519 del_module_metrics (GKHashModule * mhash, GModule module, uint8_t free_data) {
520 int i, n = 0;
521 GKHashMetric mtrc;
522
523 n = ARRAY_SIZE (module_metrics);
524 for (i = 0; i < n; i++) {
525 mtrc = mhash[module].metrics[i];
526 mtrc.del (mtrc.hash, free_data);
527 }
528 }
529
530 /* Destroys all hash tables and possibly all the malloc'd data within */
531 static void
free_stores(GKHashStorage * store)532 free_stores (GKHashStorage * store) {
533 GModule module;
534 size_t idx = 0;
535
536 free_global_metrics (store->ghash);
537 FOREACH_MODULE (idx, module_list) {
538 module = module_list[idx];
539 free_module_metrics (store->mhash, module, 1);
540 }
541
542 free (store->ghash);
543 free (store->mhash);
544 free (store);
545 }
546
547 /* Given a key (date), get the relevant store
548 *
549 * On error or not found, NULL is returned.
550 * On success, a pointer to that store is returned. */
551 static void *
get_store(uint32_t key)552 get_store (uint32_t key) {
553 GKHashStorage *store = NULL;
554 khint_t k;
555
556 khash_t (igkh) * hash = ht_dates;
557
558 k = kh_get (igkh, hash, key);
559 /* key not found, return NULL */
560 if (k == kh_end (hash))
561 return NULL;
562
563 store = kh_val (hash, k);
564 return store;
565 }
566
567 /* Given a store, a module and the metric, get the hash table
568 *
569 * On error or not found, NULL is returned.
570 * On success, a pointer to that hash table is returned. */
571 static void *
get_hash_from_store(GKHashStorage * store,int module,GSMetric metric)572 get_hash_from_store (GKHashStorage * store, int module, GSMetric metric) {
573 int mtrc = 0, cnt = 0;
574 if (!store)
575 return NULL;
576
577 if (module == -1) {
578 mtrc = metric - MTRC_METADATA - 1;
579 cnt = MTRC_CNT_BW - MTRC_UNIQUE_KEYS + 1;
580 if (mtrc >= cnt) {
581 LOG_DEBUG (("Out of bounds when attempting to get hash %d\n", metric));
582 return NULL;
583 }
584 }
585
586 /* ###NOTE: BE CAREFUL here, to avoid the almost unnecessary loop, we simply
587 * use the index from the enum to make it O(1). The metrics array has to be
588 * created in the same order as the GSMetric enum */
589 if (module < 0)
590 return store->ghash->metrics[mtrc].hash;
591 return store->mhash[module].metrics[metric].hash;
592 }
593
594 /* Given a module a key (date) and the metric, get the hash table
595 *
596 * On error or not found, NULL is returned.
597 * On success, a pointer to that hash table is returned. */
598 static void *
get_hash(int module,uint64_t key,GSMetric metric)599 get_hash (int module, uint64_t key, GSMetric metric) {
600 GKHashStorage *store = NULL;
601
602 if ((store = get_store (key)) == NULL)
603 return NULL;
604 return get_hash_from_store (store, module, metric);
605 }
606
607 /* Given a module and a metric, get the cache hash table
608 *
609 * On success, a pointer to that hash table is returned. */
610 static void *
get_hash_from_cache(GModule module,GSMetric metric)611 get_hash_from_cache (GModule module, GSMetric metric) {
612 return cache_storage[module].metrics[metric].hash;
613 }
614
615 /* Initialize a global hash structure.
616 *
617 * On success, a pointer to that hash structure is returned. */
618 static GKHashGlobal *
init_gkhashglobal(void)619 init_gkhashglobal (void) {
620 GKHashGlobal *storage = NULL;
621
622 int n = 0, i;
623
624 storage = new_gkhglobal ();
625 n = ARRAY_SIZE (global_metrics);
626 for (i = 0; i < n; i++) {
627 storage->metrics[i] = global_metrics[i];
628 storage->metrics[i].hash = global_metrics[i].alloc ();
629 }
630
631 return storage;
632 }
633
634 /* Initialize a module hash structure.
635 *
636 * On success, a pointer to that hash structure is returned. */
637 static GKHashModule *
init_gkhashmodule(void)638 init_gkhashmodule (void) {
639 GKHashModule *storage = NULL;
640 GModule module;
641 size_t idx = 0;
642
643 storage = new_gkhmodule (TOTAL_MODULES);
644 FOREACH_MODULE (idx, module_list) {
645 module = module_list[idx];
646
647 storage[module].module = module;
648 init_tables (module, storage);
649 }
650
651 return storage;
652 }
653
654 /* Insert an uint32_t key (date) and a GKHashStorage payload
655 *
656 * On error, -1 is returned.
657 * On key found, 1 is returned.
658 * On success 0 is returned */
659 static int
ins_igkh(khash_t (igkh)* hash,uint32_t key)660 ins_igkh (khash_t (igkh) * hash, uint32_t key) {
661 GKHashStorage *store = NULL;
662 khint_t k;
663 int ret;
664
665 if (!hash)
666 return -1;
667
668 k = kh_put (igkh, hash, key, &ret);
669 /* operation failed */
670 if (ret == -1)
671 return -1;
672 /* the key is present in the hash table */
673 if (ret == 0)
674 return 1;
675
676 store = new_gkhstorage ();
677 store->mhash = init_gkhashmodule ();
678 store->ghash = init_gkhashglobal ();
679
680 kh_val (hash, k) = store;
681
682 return 0;
683 }
684
685 /* Insert an uint32_t key and an GLastParse value
686 * Note: If the key exists, its value is replaced by the given value.
687 *
688 * On error, -1 is returned.
689 * On success 0 is returned */
690 static int
ins_iglp(khash_t (iglp)* hash,uint32_t key,GLastParse lp)691 ins_iglp (khash_t (iglp) * hash, uint32_t key, GLastParse lp) {
692 khint_t k;
693 int ret;
694
695 if (!hash)
696 return -1;
697
698 k = kh_put (iglp, hash, key, &ret);
699 if (ret == -1)
700 return -1;
701
702 kh_val (hash, k) = lp;
703
704 return 0;
705 }
706
707 /* Insert a string key and the corresponding uint32_t value.
708 * Note: If the key exists, the value is not replaced.
709 *
710 * On error, or if key exists, -1 is returned.
711 * On success 0 is returned */
712 static int
ins_si32(khash_t (si32)* hash,const char * key,uint32_t value)713 ins_si32 (khash_t (si32) * hash, const char *key, uint32_t value) {
714 khint_t k;
715 int ret;
716 char *dupkey = NULL;
717
718 if (!hash)
719 return -1;
720
721 dupkey = xstrdup (key);
722 k = kh_put (si32, hash, dupkey, &ret);
723 /* operation failed, or key exists */
724 if (ret == -1 || ret == 0) {
725 free (dupkey);
726 return -1;
727 }
728
729 kh_val (hash, k) = value;
730
731 return 0;
732 }
733
734 /* Increment a string key and with the corresponding incremental uint32_t value.
735 * Note: If the key exists, the value is not replaced.
736 *
737 * On error, 0 is returned.
738 * On success or if the key exists, the value is returned */
739 static uint32_t
ins_si32_inc(khash_t (si32)* hash,const char * key,uint32_t (* cb)(const char *),const char * seqk)740 ins_si32_inc (khash_t (si32) * hash, const char *key, uint32_t (*cb) (const char *),
741 const char *seqk) {
742 khint_t k;
743 int ret;
744 uint32_t value = 0;
745
746 if (!hash)
747 return 0;
748
749 k = kh_put (si32, hash, key, &ret);
750 /* operation failed, or key exists */
751 if (ret == -1 || ret == 0)
752 return 0;
753
754 if ((value = cb (seqk)) == 0)
755 return 0;
756 kh_val (hash, k) = value;
757
758 return value;
759 }
760
761 /* Insert an uint32_t key and the corresponding string value.
762 * Note: If the key exists, the value is not replaced.
763 *
764 * On error, or if key exists, -1 is returned.
765 * On success 0 is returned */
766 static int
ins_is32(khash_t (is32)* hash,uint32_t key,char * value)767 ins_is32 (khash_t (is32) * hash, uint32_t key, char *value) {
768 khint_t k;
769 int ret;
770
771 if (!hash)
772 return -1;
773
774 k = kh_put (is32, hash, key, &ret);
775 if (ret == -1 || ret == 0)
776 return -1;
777
778 kh_val (hash, k) = value;
779
780 return 0;
781 }
782
783 /* Insert a string key and the corresponding string value.
784 * Note: If the key exists, the value is not replaced.
785 *
786 * On error, -1 is returned.
787 * If key exists, 1 is returned.
788 * On success 0 is returned */
789 static int
ins_ss32(khash_t (ss32)* hash,const char * key,const char * value)790 ins_ss32 (khash_t (ss32) * hash, const char *key, const char *value) {
791 khint_t k;
792 int ret;
793 char *dupkey = NULL;
794
795 if (!hash)
796 return -1;
797
798 dupkey = xstrdup (key);
799 k = kh_put (ss32, hash, dupkey, &ret);
800 /* operation failed */
801 if (ret == -1) {
802 free (dupkey);
803 return -1;
804 }
805 /* key exists */
806 if (ret == 0) {
807 free (dupkey);
808 return 1;
809 }
810
811 kh_val (hash, k) = xstrdup (value);
812
813 return 0;
814 }
815
816 /* Insert an uint32_t key and an uint32_t value
817 * Note: If the key exists, its value is replaced by the given value.
818 *
819 * On error, -1 is returned.
820 * On success 0 is returned */
821 static int
ins_ii32(khash_t (ii32)* hash,uint32_t key,uint32_t value)822 ins_ii32 (khash_t (ii32) * hash, uint32_t key, uint32_t value) {
823 khint_t k;
824 int ret;
825
826 if (!hash)
827 return -1;
828
829 k = kh_put (ii32, hash, key, &ret);
830 if (ret == -1)
831 return -1;
832
833 kh_val (hash, k) = value;
834
835 return 0;
836 }
837
838 /* Insert a uint32_t key and a uint64_t value
839 * Note: If the key exists, its value is replaced by the given value.
840 *
841 * On error, -1 is returned.
842 * On success 0 is returned */
843 static int
ins_iu64(khash_t (iu64)* hash,uint32_t key,uint64_t value)844 ins_iu64 (khash_t (iu64) * hash, uint32_t key, uint64_t value) {
845 khint_t k;
846 int ret;
847
848 if (!hash)
849 return -1;
850
851 k = kh_put (iu64, hash, key, &ret);
852 if (ret == -1)
853 return -1;
854
855 kh_val (hash, k) = value;
856
857 return 0;
858 }
859
860 /* Insert a string key and a uint64_t value
861 * Note: If the key exists, the value is not replaced.
862 *
863 * On error or key exists, -1 is returned.
864 * On success 0 is returned */
865 static int
ins_su64(khash_t (su64)* hash,const char * key,uint64_t value)866 ins_su64 (khash_t (su64) * hash, const char *key, uint64_t value) {
867 khint_t k;
868 int ret;
869 char *dupkey = NULL;
870
871 if (!hash)
872 return -1;
873
874 dupkey = xstrdup (key);
875 k = kh_put (su64, hash, dupkey, &ret);
876 /* operation failed, or key exists */
877 if (ret == -1 || ret == 0) {
878 free (dupkey);
879 return -1;
880 }
881
882 kh_val (hash, k) = value;
883
884 return 0;
885 }
886
887 /* Insert a uint64_t key and a uint8_t value
888 *
889 * On error or key exists, -1 is returned.
890 * On key exists, 1 is returned.
891 * On success 0 is returned */
892 static int
ins_u648(khash_t (u648)* hash,uint64_t key,uint8_t value)893 ins_u648 (khash_t (u648) * hash, uint64_t key, uint8_t value) {
894 khint_t k;
895 int ret;
896
897 if (!hash)
898 return -1;
899
900 k = kh_put (u648, hash, key, &ret);
901 if (ret == -1)
902 return -1;
903 if (ret == 0)
904 return 1;
905
906 kh_val (hash, k) = value;
907
908 return 0;
909 }
910
911 /* Increase an uint32_t value given an uint32_t key.
912 * Note: If the key exists, its value is increased by the given inc.
913 *
914 * On error, 0 is returned.
915 * On success the inserted value is returned */
916 static uint32_t
inc_ii32(khash_t (ii32)* hash,uint32_t key,uint32_t inc)917 inc_ii32 (khash_t (ii32) * hash, uint32_t key, uint32_t inc) {
918 khint_t k;
919 int ret;
920 uint32_t value = inc;
921
922 if (!hash)
923 return 0;
924
925 k = kh_get (ii32, hash, key);
926 /* key found, increment current value by the given `inc` */
927 if (k != kh_end (hash))
928 value = kh_val (hash, k) + inc;
929
930 k = kh_put (ii32, hash, key, &ret);
931 if (ret == -1)
932 return 0;
933
934 kh_val (hash, k) = value;
935
936 return value;
937 }
938
939 /* Increase a uint64_t value given a string key.
940 *
941 * On error, -1 is returned.
942 * On success 0 is returned */
943 static int
inc_su64(khash_t (su64)* hash,const char * key,uint64_t inc)944 inc_su64 (khash_t (su64) * hash, const char *key, uint64_t inc) {
945 khint_t k;
946 int ret;
947 uint64_t value = inc;
948 char *dupkey = NULL;
949
950 if (!hash)
951 return -1;
952
953 k = kh_get (su64, hash, key);
954 /* key not found, set new value to the given `inc` */
955 if (k == kh_end (hash)) {
956 dupkey = xstrdup (key);
957 k = kh_put (su64, hash, dupkey, &ret);
958 /* operation failed */
959 if (ret == -1)
960 return -1;
961 } else {
962 value = kh_val (hash, k) + inc;
963 }
964
965 kh_val (hash, k) = value;
966
967 return 0;
968 }
969
970 /* Increase a uint64_t value given a uint32_t key.
971 * Note: If the key exists, its value is increased by the given inc.
972 *
973 * On error, -1 is returned.
974 * On success 0 is returned */
975 static int
inc_iu64(khash_t (iu64)* hash,uint32_t key,uint64_t inc)976 inc_iu64 (khash_t (iu64) * hash, uint32_t key, uint64_t inc) {
977 khint_t k;
978 int ret;
979 uint64_t value = inc;
980
981 if (!hash)
982 return -1;
983
984 k = kh_get (iu64, hash, key);
985 /* key found, increment current value by the given `inc` */
986 if (k != kh_end (hash))
987 value = (uint64_t) kh_val (hash, k) + inc;
988
989 k = kh_put (iu64, hash, key, &ret);
990 if (ret == -1)
991 return -1;
992
993 kh_val (hash, k) = value;
994
995 return 0;
996 }
997
998 /* Increase a uint32_t value given a string key.
999 *
1000 * On error, 0 is returned.
1001 * On success the increased value is returned */
1002 static uint32_t
inc_si32(khash_t (si32)* hash,const char * key,uint32_t inc)1003 inc_si32 (khash_t (si32) * hash, const char *key, uint32_t inc) {
1004 khint_t k;
1005 int ret;
1006 uint32_t value = inc;
1007
1008 if (!hash)
1009 return 0;
1010
1011 k = kh_get (si32, hash, key);
1012 /* key not found, set new value to the given `inc` */
1013 if (k == kh_end (hash)) {
1014 k = kh_put (si32, hash, key, &ret);
1015 /* operation failed */
1016 if (ret == -1)
1017 return 0;
1018 } else {
1019 value = kh_val (hash, k) + inc;
1020 }
1021
1022 kh_val (hash, k) = value;
1023
1024 return value;
1025 }
1026
1027 /* Insert a string key and auto increment int value.
1028 *
1029 * On error, 0 is returned.
1030 * On key found, the stored value is returned
1031 * On success the value of the key inserted is returned */
1032 static uint32_t
ins_si32_ai(khash_t (si32)* hash,const char * key)1033 ins_si32_ai (khash_t (si32) * hash, const char *key) {
1034 int size = 0, value = 0;
1035 int ret;
1036 khint_t k;
1037
1038 if (!hash)
1039 return 0;
1040
1041 size = kh_size (hash);
1042 /* the auto increment value starts at SIZE (hash table) + 1 */
1043 value = size > 0 ? size + 1 : 1;
1044
1045 k = kh_put (si32, hash, key, &ret);
1046 /* operation failed */
1047 if (ret == -1)
1048 return 0;
1049 /* key exists */
1050 if (ret == 0)
1051 return kh_val (hash, k);
1052
1053 kh_val (hash, k) = value;
1054
1055 return value;
1056 }
1057
1058 /* Compare if the given needle is in the haystack
1059 *
1060 * if equal, 1 is returned, else 0 */
1061 static int
find_int_key_in_list(void * data,void * needle)1062 find_int_key_in_list (void *data, void *needle) {
1063 return (*(uint32_t *) data) == (*(uint32_t *) needle) ? 1 : 0;
1064 }
1065
1066 /* Insert an int key and the corresponding GSLList (Single linked-list) value.
1067 * Note: If the key exists within the list, the value is not appended.
1068 *
1069 * On error, -1 is returned.
1070 * On success or if key is found, 0 is returned */
1071 static int
ins_igsl(khash_t (igsl)* hash,uint32_t key,uint32_t value)1072 ins_igsl (khash_t (igsl) * hash, uint32_t key, uint32_t value) {
1073 khint_t k;
1074 GSLList *list;
1075 int ret;
1076
1077 if (!hash)
1078 return -1;
1079
1080 k = kh_get (igsl, hash, key);
1081 /* key found, check if key exists within the list */
1082 if (k != kh_end (hash) && (list = kh_val (hash, k))) {
1083 if (list_find (list, find_int_key_in_list, &value))
1084 return 0;
1085 list = list_insert_prepend (list, i322ptr (value));
1086 } else {
1087 list = list_create (i322ptr (value));
1088 }
1089
1090 k = kh_put (igsl, hash, key, &ret);
1091 if (ret == -1) {
1092 list_remove_nodes (list);
1093 return -1;
1094 }
1095
1096 kh_val (hash, k) = list;
1097
1098 return 0;
1099 }
1100
1101 /* Determine the path for the given database file.
1102 *
1103 * On error, a fatal error is thrown.
1104 * On success, the databases path string is returned. */
1105 static char *
set_db_path(const char * fn)1106 set_db_path (const char *fn) {
1107 struct stat info;
1108 char *rpath = NULL, *path = NULL;
1109 const char *dbpath = NULL;
1110
1111 if (!conf.db_path)
1112 dbpath = DB_PATH;
1113 else
1114 dbpath = conf.db_path;
1115
1116 rpath = realpath (dbpath, NULL);
1117 if (rpath == NULL)
1118 FATAL ("Unable to open the specified config file. %s", strerror (errno));
1119
1120 /* sanity check: Is db_path accessible and a directory? */
1121 if (stat (rpath, &info) != 0)
1122 FATAL ("Unable to access database path: %s", strerror (errno));
1123 else if (!(info.st_mode & S_IFDIR))
1124 FATAL ("Database path is not a directory.");
1125
1126 path = xmalloc (snprintf (NULL, 0, "%s/%s", rpath, fn) + 1);
1127 sprintf (path, "%s/%s", rpath, fn);
1128 free (rpath);
1129
1130 return path;
1131 }
1132
1133 /* Get the database filename given a module and a metric.
1134 *
1135 * On error, a fatal error is triggered.
1136 * On success, the filename is returned */
1137 static char *
get_filename(GModule module,GKHashMetric mtrc)1138 get_filename (GModule module, GKHashMetric mtrc) {
1139 char *mtrstr = NULL, *modstr = NULL, *type = NULL, *fn = NULL;
1140
1141 if (!(mtrstr = get_mtr_str (mtrc.metric)))
1142 FATAL ("Unable to allocate metric name.");
1143 if (!(modstr = get_module_str (module)))
1144 FATAL ("Unable to allocate module name.");
1145 if (!(type = get_mtr_type_str (mtrc.type)))
1146 FATAL ("Unable to allocate module name.");
1147
1148 fn = xmalloc (snprintf (NULL, 0, "%s_%s_%s.db", type, modstr, mtrstr) + 1);
1149 sprintf (fn, "%s_%s_%s.db", type, modstr, mtrstr);
1150
1151 free (mtrstr);
1152 free (type);
1153 free (modstr);
1154
1155 return fn;
1156 }
1157
1158 /* Dump to disk the database file and frees its memory */
1159 static void
close_tpl(tpl_node * tn,const char * fn)1160 close_tpl (tpl_node * tn, const char *fn) {
1161 tpl_dump (tn, TPL_FILE, fn);
1162 tpl_free (tn);
1163 }
1164
1165 /* Given a database filename, restore a string key, uint32_t value back to the
1166 * storage */
1167 static void
restore_global_si32(khash_t (si32)* hash,const char * fn)1168 restore_global_si32 (khash_t (si32) * hash, const char *fn) {
1169 tpl_node *tn;
1170 char *key = NULL;
1171 char fmt[] = "A(su)";
1172 uint32_t val;
1173
1174 tn = tpl_map (fmt, &key, &val);
1175 tpl_load (tn, TPL_FILE, fn);
1176 while (tpl_unpack (tn, 1) > 0) {
1177 ins_si32 (hash, key, val);
1178 free (key);
1179 }
1180 tpl_free (tn);
1181 }
1182
1183 /* Given a hash and a filename, persist to disk a string key, uint32_t value */
1184 static void
persist_global_si32(khash_t (si32)* hash,const char * fn)1185 persist_global_si32 (khash_t (si32) * hash, const char *fn) {
1186 tpl_node *tn;
1187 khint_t k;
1188 const char *key = NULL;
1189 char fmt[] = "A(su)";
1190 uint32_t val;
1191
1192 if (!hash || kh_size (hash) == 0)
1193 return;
1194
1195 tn = tpl_map (fmt, &key, &val);
1196 for (k = 0; k < kh_end (hash); ++k) {
1197 if (!kh_exist (hash, k) || (!(key = kh_key (hash, k))))
1198 continue;
1199 val = kh_value (hash, k);
1200 tpl_pack (tn, 1);
1201 }
1202
1203 tpl_dump (tn, TPL_FILE, fn);
1204 tpl_free (tn);
1205 }
1206
1207 /* Given a database filename, restore a uint32_t key, GLastParse value back to
1208 * the storage */
1209 static void
restore_global_iglp(khash_t (iglp)* hash,const char * fn)1210 restore_global_iglp (khash_t (iglp) * hash, const char *fn) {
1211 tpl_node *tn;
1212 uint32_t key;
1213 GLastParse val = { 0 };
1214 char fmt[] = "A(uS(uIUvc#))";
1215
1216 tn = tpl_map (fmt, &key, &val, READ_BYTES);
1217 tpl_load (tn, TPL_FILE, fn);
1218 while (tpl_unpack (tn, 1) > 0) {
1219 ins_iglp (hash, key, val);
1220 }
1221 tpl_free (tn);
1222 }
1223
1224 /* Given a hash and a filename, persist to disk a uint32_t key, uint32_t value */
1225 static void
persist_global_iglp(khash_t (iglp)* hash,const char * fn)1226 persist_global_iglp (khash_t (iglp) * hash, const char *fn) {
1227 tpl_node *tn;
1228 khint_t k;
1229 uint32_t key;
1230 GLastParse val = { 0 };
1231 char fmt[] = "A(uS(uIUvc#))";
1232
1233 if (!hash || kh_size (hash) == 0)
1234 return;
1235
1236 tn = tpl_map (fmt, &key, &val, READ_BYTES);
1237 for (k = 0; k < kh_end (hash); ++k) {
1238 if (!kh_exist (hash, k))
1239 continue;
1240 key = kh_key (hash, k);
1241 val = kh_value (hash, k);
1242 tpl_pack (tn, 1);
1243 }
1244
1245 tpl_dump (tn, TPL_FILE, fn);
1246 tpl_free (tn);
1247 }
1248
1249 /* Check if the given date can be inserted based on how many dates we need to
1250 * keep conf.keep_last.
1251 *
1252 * Returns -1 if it fails to insert the date.
1253 * Returns 1 if the date exists.
1254 * Returns 2 if the date shouldn't be inserted.
1255 * On success or if the date is inserted 0 is returned */
1256 static int
insert_restored_date(uint32_t date)1257 insert_restored_date (uint32_t date) {
1258 uint32_t i, len = 0;
1259
1260 /* no keep last, simply insert the restored date to our storage */
1261 if (!conf.keep_last || persisted_dates_len < conf.keep_last)
1262 return ht_insert_date (date);
1263
1264 len = MIN (persisted_dates_len, conf.keep_last);
1265 for (i = 0; i < len; ++i)
1266 if (persisted_dates[i] == date)
1267 return ht_insert_date (date);
1268 return 2;
1269 }
1270
1271 /* Given a database filename, restore a string key, uint32_t value back to
1272 * the storage */
1273 static int
restore_si32(GSMetric metric,const char * path,int module)1274 restore_si32 (GSMetric metric, const char *path, int module) {
1275 khash_t (si32) * hash = NULL;
1276 tpl_node *tn;
1277 char fmt[] = "A(iA(su))";
1278 int date = 0, ret = 0;
1279 char *key = NULL;
1280 uint32_t val = 0;
1281
1282 if (!(tn = tpl_map (fmt, &date, &key, &val)))
1283 return 1;
1284
1285 tpl_load (tn, TPL_FILE, path);
1286 while (tpl_unpack (tn, 1) > 0) {
1287 if ((ret = insert_restored_date (date)) == 2)
1288 continue;
1289 if (ret == -1 || !(hash = get_hash (module, date, metric)))
1290 break;
1291
1292 while (tpl_unpack (tn, 2) > 0) {
1293 ins_si32 (hash, key, val);
1294 free (key);
1295 }
1296 }
1297 tpl_free (tn);
1298
1299 return 0;
1300 }
1301
1302 /* Given a hash and a filename, persist to disk a string key, uint32_t value */
1303 static int
persist_si32(GSMetric metric,const char * path,int module)1304 persist_si32 (GSMetric metric, const char *path, int module) {
1305 khash_t (si32) * hash = NULL;
1306 tpl_node *tn = NULL;
1307 int date = 0;
1308 char fmt[] = "A(iA(su))";
1309 uint32_t val = 0;
1310 const char *key = NULL;
1311
1312 if (!ht_dates || !(tn = tpl_map (fmt, &date, &key, &val)))
1313 return 1;
1314
1315 /* *INDENT-OFF* */
1316 HT_FOREACH_KEY (ht_dates, date, {
1317 if (!(hash = get_hash (module, date, metric)))
1318 return -1;
1319 kh_foreach (hash, key, val, { tpl_pack (tn, 2); });
1320 tpl_pack (tn, 1);
1321 });
1322 /* *INDENT-ON* */
1323 close_tpl (tn, path);
1324
1325 return 0;
1326 }
1327
1328 /* Given a database filename, restore a uint32_t key, string value back to
1329 * the storage */
1330 static int
restore_is32(GSMetric metric,const char * path,int module)1331 restore_is32 (GSMetric metric, const char *path, int module) {
1332 khash_t (is32) * hash = NULL;
1333 tpl_node *tn;
1334 char fmt[] = "A(iA(us))";
1335 int date = 0, ret = 0;
1336 uint32_t key = 0;
1337 char *val = NULL, *dupval = NULL;
1338
1339 if (!(tn = tpl_map (fmt, &date, &key, &val)))
1340 return 1;
1341
1342 tpl_load (tn, TPL_FILE, path);
1343 while (tpl_unpack (tn, 1) > 0) {
1344 if ((ret = insert_restored_date (date)) == 2)
1345 continue;
1346 if (ret == -1 || !(hash = get_hash (module, date, metric)))
1347 break;
1348
1349 while (tpl_unpack (tn, 2) > 0) {
1350 dupval = xstrdup (val);
1351 if (ins_is32 (hash, key, dupval) != 0)
1352 free (dupval);
1353 free (val);
1354 }
1355 }
1356 tpl_free (tn);
1357
1358 return 0;
1359 }
1360
1361 /* Given a hash and a filename, persist to disk a uint32_t key, string value */
1362 static int
persist_is32(GSMetric metric,const char * path,int module)1363 persist_is32 (GSMetric metric, const char *path, int module) {
1364 khash_t (is32) * hash = NULL;
1365 tpl_node *tn = NULL;
1366 int date = 0;
1367 char fmt[] = "A(iA(us))";
1368 char *val = NULL;
1369 uint32_t key = 0;
1370
1371 if (!ht_dates || !(tn = tpl_map (fmt, &date, &key, &val)))
1372 return 1;
1373
1374 /* *INDENT-OFF* */
1375 HT_FOREACH_KEY (ht_dates, date, {
1376 if (!(hash = get_hash (module, date, metric)))
1377 return -1;
1378 kh_foreach (hash, key, val, { tpl_pack (tn, 2); });
1379 tpl_pack (tn, 1);
1380 });
1381 /* *INDENT-ON* */
1382 close_tpl (tn, path);
1383
1384 return 0;
1385 }
1386
1387 /* Given a database filename, restore a uint32_t key, uint32_t value back to
1388 * the storage */
1389 static int
restore_ii32(GSMetric metric,const char * path,int module)1390 restore_ii32 (GSMetric metric, const char *path, int module) {
1391 khash_t (ii32) * hash = NULL;
1392 tpl_node *tn;
1393 char fmt[] = "A(iA(uu))";
1394 int date = 0, ret = 0;
1395 uint32_t key = 0, val = 0;
1396
1397 if (!(tn = tpl_map (fmt, &date, &key, &val)))
1398 return 1;
1399
1400 tpl_load (tn, TPL_FILE, path);
1401 while (tpl_unpack (tn, 1) > 0) {
1402 if ((ret = insert_restored_date (date)) == 2)
1403 continue;
1404 if (ret == -1 || !(hash = get_hash (module, date, metric)))
1405 break;
1406
1407 while (tpl_unpack (tn, 2) > 0) {
1408 ins_ii32 (hash, key, val);
1409 }
1410 }
1411 tpl_free (tn);
1412
1413 return 0;
1414 }
1415
1416 /* Given a hash and a filename, persist to disk a uint32_t key, uint32_t value */
1417 static int
persist_ii32(GSMetric metric,const char * path,int module)1418 persist_ii32 (GSMetric metric, const char *path, int module) {
1419 khash_t (ii32) * hash = NULL;
1420 tpl_node *tn = NULL;
1421 int date = 0;
1422 char fmt[] = "A(iA(uu))";
1423 uint32_t key = 0, val = 0;
1424
1425 if (!ht_dates || !(tn = tpl_map (fmt, &date, &key, &val)))
1426 return 1;
1427
1428 /* *INDENT-OFF* */
1429 HT_FOREACH_KEY (ht_dates, date, {
1430 if (!(hash = get_hash (module, date, metric)))
1431 return -1;
1432 kh_foreach (hash, key, val, { tpl_pack (tn, 2); });
1433 tpl_pack (tn, 1);
1434 });
1435 /* *INDENT-ON* */
1436 close_tpl (tn, path);
1437
1438 return 0;
1439 }
1440
1441 /* Given a database filename, restore a uint64_t key, uint8_t value back to
1442 * the storage */
1443 static int
restore_u648(GSMetric metric,const char * path,int module)1444 restore_u648 (GSMetric metric, const char *path, int module) {
1445 khash_t (u648) * hash = NULL;
1446 tpl_node *tn;
1447 char fmt[] = "A(iA(Uv))";
1448 int date = 0, ret = 0;
1449 uint64_t key;
1450 uint16_t val = 0;
1451
1452 if (!(tn = tpl_map (fmt, &date, &key, &val)))
1453 return 1;
1454
1455 tpl_load (tn, TPL_FILE, path);
1456 while (tpl_unpack (tn, 1) > 0) {
1457 if ((ret = insert_restored_date (date)) == 2)
1458 continue;
1459 if (ret == -1 || !(hash = get_hash (module, date, metric)))
1460 break;
1461
1462 while (tpl_unpack (tn, 2) > 0) {
1463 ins_u648 (hash, key, val);
1464 }
1465 }
1466 tpl_free (tn);
1467
1468 return 0;
1469 }
1470
1471 /* Given a hash and a filename, persist to disk a uint64_t key, uint8_t value */
1472 static int
persist_u648(GSMetric metric,const char * path,int module)1473 persist_u648 (GSMetric metric, const char *path, int module) {
1474 khash_t (u648) * hash = NULL;
1475 tpl_node *tn = NULL;
1476 int date = 0;
1477 char fmt[] = "A(iA(Uv))";
1478 uint64_t key;
1479 uint16_t val = 0;
1480
1481 if (!ht_dates || !(tn = tpl_map (fmt, &date, &key, &val)))
1482 return 1;
1483
1484 /* *INDENT-OFF* */
1485 HT_FOREACH_KEY (ht_dates, date, {
1486 if (!(hash = get_hash (module, date, metric)))
1487 return -1;
1488 kh_foreach (hash, key, val, { tpl_pack (tn, 2); });
1489 tpl_pack (tn, 1);
1490 });
1491 /* *INDENT-ON* */
1492 close_tpl (tn, path);
1493
1494 return 0;
1495 }
1496
1497 /* Given a database filename, restore a uint32_t key, uint64_t value back to
1498 * the storage */
1499 static int
restore_iu64(GSMetric metric,const char * path,int module)1500 restore_iu64 (GSMetric metric, const char *path, int module) {
1501 khash_t (iu64) * hash = NULL;
1502 tpl_node *tn;
1503 char fmt[] = "A(iA(uU))";
1504 int date = 0, ret = 0;
1505 uint32_t key;
1506 uint64_t val;
1507
1508 if (!(tn = tpl_map (fmt, &date, &key, &val)))
1509 return 1;
1510
1511 tpl_load (tn, TPL_FILE, path);
1512 while (tpl_unpack (tn, 1) > 0) {
1513 if ((ret = insert_restored_date (date)) == 2)
1514 continue;
1515 if (ret == -1 || !(hash = get_hash (module, date, metric)))
1516 break;
1517
1518 while (tpl_unpack (tn, 2) > 0) {
1519 ins_iu64 (hash, key, val);
1520 }
1521 }
1522 tpl_free (tn);
1523
1524 return 0;
1525 }
1526
1527 /* Given a hash and a filename, persist to disk a uint32_t key, uint64_t value */
1528 static int
persist_iu64(GSMetric metric,const char * path,int module)1529 persist_iu64 (GSMetric metric, const char *path, int module) {
1530 khash_t (iu64) * hash = NULL;
1531 tpl_node *tn = NULL;
1532 int date = 0;
1533 char fmt[] = "A(iA(uU))";
1534 uint32_t key;
1535 uint64_t val;
1536
1537 if (!ht_dates || !(tn = tpl_map (fmt, &date, &key, &val)))
1538 return 1;
1539
1540 /* *INDENT-OFF* */
1541 HT_FOREACH_KEY (ht_dates, date, {
1542 if (!(hash = get_hash (module, date, metric)))
1543 return -1;
1544 kh_foreach (hash, key, val, { tpl_pack (tn, 2); });
1545 tpl_pack (tn, 1);
1546 });
1547 /* *INDENT-ON* */
1548 close_tpl (tn, path);
1549
1550 return 0;
1551 }
1552
1553 /* Given a database filename, restore a string key, uint64_t value back to
1554 * the storage */
1555 static int
restore_su64(GSMetric metric,const char * path,int module)1556 restore_su64 (GSMetric metric, const char *path, int module) {
1557 khash_t (su64) * hash = NULL;
1558 tpl_node *tn;
1559 char fmt[] = "A(iA(sU))";
1560 int date = 0, ret = 0;
1561 char *key = NULL;
1562 uint64_t val;
1563
1564 if (!(tn = tpl_map (fmt, &date, &key, &val)))
1565 return 1;
1566
1567 tpl_load (tn, TPL_FILE, path);
1568 while (tpl_unpack (tn, 1) > 0) {
1569 if ((ret = insert_restored_date (date)) == 2)
1570 continue;
1571 if (ret == -1 || !(hash = get_hash (module, date, metric)))
1572 break;
1573
1574 while (tpl_unpack (tn, 2) > 0) {
1575 ins_su64 (hash, key, val);
1576 free (key);
1577 }
1578 }
1579 tpl_free (tn);
1580
1581 return 0;
1582 }
1583
1584 /* Given a hash and a filename, persist to disk a string key, uint64_t value */
1585 static int
persist_su64(GSMetric metric,const char * path,int module)1586 persist_su64 (GSMetric metric, const char *path, int module) {
1587 khash_t (su64) * hash = NULL;
1588 tpl_node *tn = NULL;
1589 int date = 0;
1590 char fmt[] = "A(iA(sU))";
1591 const char *key = NULL;
1592 uint64_t val;
1593
1594 if (!ht_dates || !(tn = tpl_map (fmt, &date, &key, &val)))
1595 return 1;
1596
1597 /* *INDENT-OFF* */
1598 HT_FOREACH_KEY (ht_dates, date, {
1599 if (!(hash = get_hash (module, date, metric)))
1600 return -1;
1601 kh_foreach (hash, key, val, { tpl_pack (tn, 2); });
1602 tpl_pack (tn, 1);
1603 });
1604 /* *INDENT-ON* */
1605 close_tpl (tn, path);
1606
1607 return 0;
1608 }
1609
1610 /* Given a database filename, restore a uint32_t key, GSLList value back to the
1611 * storage */
1612 static int
restore_igsl(GSMetric metric,const char * path,int module)1613 restore_igsl (GSMetric metric, const char *path, int module) {
1614 khash_t (igsl) * hash = NULL;
1615 tpl_node *tn;
1616 char fmt[] = "A(iA(uu))";
1617 int date = 0, ret = 0;
1618 uint32_t key, val;
1619
1620 if (!(tn = tpl_map (fmt, &date, &key, &val)))
1621 return 1;
1622
1623 tpl_load (tn, TPL_FILE, path);
1624 while (tpl_unpack (tn, 1) > 0) {
1625 if ((ret = insert_restored_date (date)) == 2)
1626 continue;
1627 if (ret == -1 || !(hash = get_hash (module, date, metric)))
1628 break;
1629
1630 while (tpl_unpack (tn, 2) > 0) {
1631 ins_igsl (hash, key, val);
1632 }
1633 }
1634 tpl_free (tn);
1635
1636 return 0;
1637 }
1638
1639 /* Given a hash and a filename, persist to disk a uint32_t key, GSLList value */
1640 static int
persist_igsl(GSMetric metric,const char * path,int module)1641 persist_igsl (GSMetric metric, const char *path, int module) {
1642 khash_t (igsl) * hash = NULL;
1643 GSLList *node;
1644 tpl_node *tn = NULL;
1645 int date = 0;
1646 char fmt[] = "A(iA(uu))";
1647 uint32_t key, val;
1648
1649 if (!ht_dates || !(tn = tpl_map (fmt, &date, &key, &val)))
1650 return 1;
1651
1652 /* *INDENT-OFF* */
1653 HT_FOREACH_KEY (ht_dates, date, {
1654 if (!(hash = get_hash (module, date, metric)))
1655 return -1;
1656 kh_foreach (hash, key, node, {
1657 while (node) {
1658 val = (*(uint32_t *) node->data);
1659 node = node->next;
1660 }
1661 tpl_pack (tn, 2);
1662 });
1663 tpl_pack (tn, 1);
1664 });
1665 /* *INDENT-ON* */
1666 close_tpl (tn, path);
1667
1668 return 0;
1669 }
1670
1671 /* Given a filename, ensure we have a valid return path
1672 *
1673 * On error, NULL is returned.
1674 * On success, the valid path is returned */
1675 static char *
check_restore_path(const char * fn)1676 check_restore_path (const char *fn) {
1677 char *path = set_db_path (fn);
1678 if (access (path, F_OK) != -1)
1679 return path;
1680
1681 LOG_DEBUG (("DB file %s doesn't exist. %s\n", path, strerror (errno)));
1682 free (path);
1683 return NULL;
1684 }
1685
1686 /* Entry function to restore hash data by type */
1687 static void
restore_by_type(GKHashMetric mtrc,const char * fn,int module)1688 restore_by_type (GKHashMetric mtrc, const char *fn, int module) {
1689 char *path = NULL;
1690
1691 if (!(path = check_restore_path (fn)))
1692 goto clean;
1693
1694 switch (mtrc.type) {
1695 case MTRC_TYPE_SI32:
1696 restore_si32 (mtrc.metric, path, module);
1697 break;
1698 case MTRC_TYPE_IS32:
1699 restore_is32 (mtrc.metric, path, module);
1700 break;
1701 case MTRC_TYPE_II32:
1702 restore_ii32 (mtrc.metric, path, module);
1703 break;
1704 case MTRC_TYPE_U648:
1705 restore_u648 (mtrc.metric, path, module);
1706 break;
1707 case MTRC_TYPE_IU64:
1708 restore_iu64 (mtrc.metric, path, module);
1709 break;
1710 case MTRC_TYPE_SU64:
1711 restore_su64 (mtrc.metric, path, module);
1712 break;
1713 case MTRC_TYPE_IGSL:
1714 restore_igsl (mtrc.metric, path, module);
1715 break;
1716 default:
1717 break;
1718 }
1719 clean:
1720 free (path);
1721 }
1722
1723 /* Entry function to restore hash data by metric type */
1724 static void
restore_metric_type(GModule module,GKHashMetric mtrc)1725 restore_metric_type (GModule module, GKHashMetric mtrc) {
1726 char *fn = NULL;
1727
1728 fn = get_filename (module, mtrc);
1729 restore_by_type (mtrc, fn, module);
1730 free (fn);
1731 }
1732
1733 /* Given all the dates that we have processed, persist to disk a copy of them. */
1734 static void
persist_dates(void)1735 persist_dates (void) {
1736 tpl_node *tn;
1737 char *path = NULL;
1738 uint32_t *dates = NULL, len = 0, i, date = 0;
1739 char fmt[] = "A(u)";
1740
1741 if (!(path = set_db_path ("I32_DATES.db")))
1742 return;
1743
1744 dates = get_sorted_dates (&len);
1745
1746 tn = tpl_map (fmt, &date);
1747 for (i = 0; i < len; ++i) {
1748 date = dates[i];
1749 tpl_pack (tn, 1);
1750 }
1751 tpl_dump (tn, TPL_FILE, path);
1752
1753 tpl_free (tn);
1754 free (path);
1755 free (dates);
1756 }
1757
1758 /* Restore all the processed dates from our last dataset */
1759 static void
restore_dates(void)1760 restore_dates (void) {
1761 tpl_node *tn;
1762 char *path = NULL;
1763 uint32_t date, idx = 0;
1764 char fmt[] = "A(u)";
1765 int len;
1766
1767 if (!(path = check_restore_path ("I32_DATES.db")))
1768 return;
1769
1770 tn = tpl_map (fmt, &date);
1771 tpl_load (tn, TPL_FILE, path);
1772
1773 len = tpl_Alen (tn, 1);
1774 if (len < 0)
1775 return;
1776 persisted_dates_len = len;
1777 persisted_dates = xcalloc (persisted_dates_len, sizeof (uint32_t));
1778 while (tpl_unpack (tn, 1) > 0)
1779 persisted_dates[idx++] = date;
1780
1781 qsort (persisted_dates, idx, sizeof (uint32_t), cmp_ui32_desc);
1782 tpl_free (tn);
1783 free (path);
1784 }
1785
1786 /* Entry function to restore a global hashes */
1787 static void
restore_global(void)1788 restore_global (void) {
1789 char *path = NULL;
1790
1791 restore_dates ();
1792 if ((path = check_restore_path ("SI32_CNT_OVERALL.db"))) {
1793 restore_global_si32 (ht_cnt_overall, path);
1794 free (path);
1795 }
1796 if ((path = check_restore_path ("SI32_SEQS.db"))) {
1797 restore_global_si32 (ht_seqs, path);
1798 free (path);
1799 }
1800 if ((path = check_restore_path ("IGLP_LAST_PARSE.db"))) {
1801 restore_global_iglp (ht_last_parse, path);
1802 free (path);
1803 }
1804 }
1805
1806 /* Entry function to restore hashes */
1807 static void
restore_data(void)1808 restore_data (void) {
1809 GModule module;
1810 int i, n = 0;
1811 size_t idx = 0;
1812
1813 restore_global ();
1814
1815 n = ARRAY_SIZE (global_metrics);
1816 for (i = 0; i < n; ++i)
1817 restore_by_type (global_metrics[i], global_metrics[i].filename, -1);
1818
1819 n = ARRAY_SIZE (module_metrics);
1820 FOREACH_MODULE (idx, module_list) {
1821 module = module_list[idx];
1822 for (i = 0; i < n; ++i) {
1823 restore_metric_type (module, module_metrics[i]);
1824 }
1825 }
1826 }
1827
1828 static void
persist_by_type(GKHashMetric mtrc,const char * fn,int module)1829 persist_by_type (GKHashMetric mtrc, const char *fn, int module) {
1830 char *path = NULL;
1831 path = set_db_path (fn);
1832
1833 switch (mtrc.type) {
1834 case MTRC_TYPE_SI32:
1835 persist_si32 (mtrc.metric, path, module);
1836 break;
1837 case MTRC_TYPE_IS32:
1838 persist_is32 (mtrc.metric, path, module);
1839 break;
1840 case MTRC_TYPE_II32:
1841 persist_ii32 (mtrc.metric, path, module);
1842 break;
1843 case MTRC_TYPE_U648:
1844 persist_u648 (mtrc.metric, path, module);
1845 break;
1846 case MTRC_TYPE_IU64:
1847 persist_iu64 (mtrc.metric, path, module);
1848 break;
1849 case MTRC_TYPE_SU64:
1850 persist_su64 (mtrc.metric, path, module);
1851 break;
1852 case MTRC_TYPE_IGSL:
1853 persist_igsl (mtrc.metric, path, module);
1854 break;
1855 default:
1856 break;
1857 }
1858 free (path);
1859 }
1860
1861 static void
persist_metric_type(GModule module,GKHashMetric mtrc)1862 persist_metric_type (GModule module, GKHashMetric mtrc) {
1863 char *fn = NULL;
1864 fn = get_filename (module, mtrc);
1865 persist_by_type (mtrc, fn, module);
1866 free (fn);
1867 }
1868
1869 static void
persist_global(void)1870 persist_global (void) {
1871 char *path = NULL;
1872
1873 persist_dates ();
1874 if ((path = set_db_path ("SI32_CNT_OVERALL.db"))) {
1875 persist_global_si32 (ht_cnt_overall, path);
1876 free (path);
1877 }
1878 if ((path = set_db_path ("SI32_SEQS.db"))) {
1879 persist_global_si32 (ht_seqs, path);
1880 free (path);
1881 }
1882 if ((path = set_db_path ("IGLP_LAST_PARSE.db"))) {
1883 persist_global_iglp (ht_last_parse, path);
1884 free (path);
1885 }
1886 free (persisted_dates);
1887 }
1888
1889 static void
persist_data(void)1890 persist_data (void) {
1891 GModule module;
1892 int i, n = 0;
1893 size_t idx = 0;
1894
1895 persist_global ();
1896
1897 n = ARRAY_SIZE (global_metrics);
1898 for (i = 0; i < n; ++i)
1899 persist_by_type (global_metrics[i], global_metrics[i].filename, -1);
1900
1901 n = ARRAY_SIZE (module_metrics);
1902 FOREACH_MODULE (idx, module_list) {
1903 module = module_list[idx];
1904 for (i = 0; i < n; ++i) {
1905 persist_metric_type (module, module_metrics[i]);
1906 }
1907 }
1908 }
1909
1910 /* Get the uint32_t value of a given string key.
1911 *
1912 * On error, 0 is returned.
1913 * On success the uint32_t value for the given key is returned */
1914 static uint32_t
get_si32(khash_t (si32)* hash,const char * key)1915 get_si32 (khash_t (si32) * hash, const char *key) {
1916 khint_t k;
1917
1918 if (!hash)
1919 return 0;
1920
1921 k = kh_get (si32, hash, key);
1922 /* key found, return current value */
1923 if (k != kh_end (hash))
1924 return kh_val (hash, k);
1925
1926 return 0;
1927 }
1928
1929 /* Get the string value of a given uint32_t key.
1930 *
1931 * On error, NULL is returned.
1932 * On success the string value for the given key is returned */
1933 static char *
get_is32(khash_t (is32)* hash,uint32_t key)1934 get_is32 (khash_t (is32) * hash, uint32_t key) {
1935 khint_t k;
1936 char *value = NULL;
1937
1938 if (!hash)
1939 return NULL;
1940
1941 k = kh_get (is32, hash, key);
1942 /* key found, return current value */
1943 if (k != kh_end (hash) && (value = kh_val (hash, k)))
1944 return xstrdup (value);
1945
1946 return NULL;
1947 }
1948
1949 /* Get the string value of a given string key.
1950 *
1951 * On error, NULL is returned.
1952 * On success the string value for the given key is returned */
1953 static char *
get_ss32(khash_t (ss32)* hash,const char * key)1954 get_ss32 (khash_t (ss32) * hash, const char *key) {
1955 khint_t k;
1956 char *value = NULL;
1957
1958 if (!hash)
1959 return NULL;
1960
1961 k = kh_get (ss32, hash, key);
1962 /* key found, return current value */
1963 if (k != kh_end (hash) && (value = kh_val (hash, k)))
1964 return xstrdup (value);
1965
1966 return NULL;
1967 }
1968
1969 /* Get the uint32_t value of a given uint32_t key.
1970 *
1971 * If key is not found, 0 is returned.
1972 * On error, -1 is returned.
1973 * On success the uint32_t value for the given key is returned */
1974 static uint32_t
get_ii32(khash_t (ii32)* hash,uint32_t key)1975 get_ii32 (khash_t (ii32) * hash, uint32_t key) {
1976 khint_t k;
1977 uint32_t value = 0;
1978
1979 if (!hash)
1980 return 0;
1981
1982 k = kh_get (ii32, hash, key);
1983 /* key found, return current value */
1984 if (k != kh_end (hash) && (value = kh_val (hash, k)))
1985 return value;
1986
1987 return 0;
1988 }
1989
1990 /* Get the uint64_t value of a given uint32_t key.
1991 *
1992 * On error, or if key is not found, 0 is returned.
1993 * On success the uint64_t value for the given key is returned */
1994 static uint64_t
get_iu64(khash_t (iu64)* hash,uint32_t key)1995 get_iu64 (khash_t (iu64) * hash, uint32_t key) {
1996 khint_t k;
1997 uint64_t value = 0;
1998
1999 if (!hash)
2000 return 0;
2001
2002 k = kh_get (iu64, hash, key);
2003 /* key found, return current value */
2004 if (k != kh_end (hash) && (value = kh_val (hash, k)))
2005 return value;
2006
2007 return 0;
2008 }
2009
2010 /* Get the uint64_t value of a given string key.
2011 *
2012 * On error, or if key is not found, 0 is returned.
2013 * On success the uint64_t value for the given key is returned */
2014 static uint64_t
get_su64(khash_t (su64)* hash,const char * key)2015 get_su64 (khash_t (su64) * hash, const char *key) {
2016 khint_t k;
2017 uint64_t val = 0;
2018
2019 if (!hash)
2020 return 0;
2021
2022 k = kh_get (su64, hash, key);
2023 /* key found, return current value */
2024 if (k != kh_end (hash) && (val = kh_val (hash, k)))
2025 return val;
2026
2027 return 0;
2028 }
2029
2030 /* Get the GLastParse value of a given uint32_t key.
2031 *
2032 * If key is not found, {0} is returned.
2033 * On error, -1 is returned.
2034 * On success the GLastParse value for the given key is returned */
2035 static GLastParse
get_iglp(khash_t (iglp)* hash,uint32_t key)2036 get_iglp (khash_t (iglp) * hash, uint32_t key) {
2037 khint_t k;
2038 GLastParse lp = { 0 };
2039
2040 if (!hash)
2041 return lp;
2042
2043 k = kh_get (iglp, hash, key);
2044 /* key found, return current value */
2045 if (k != kh_end (hash)) {
2046 lp = kh_val (hash, k);
2047 return lp;
2048 }
2049
2050 return lp;
2051 }
2052
2053 GSLList *
ht_get_keymap_list_from_key(GModule module,const char * key)2054 ht_get_keymap_list_from_key (GModule module, const char *key) {
2055 GSLList *list = NULL;
2056 khiter_t kv;
2057 khint_t k;
2058 khash_t (si32) * hash = NULL;
2059
2060 if (!ht_dates)
2061 return NULL;
2062
2063 for (k = kh_begin (ht_dates); k != kh_end (ht_dates); ++k) {
2064 if (!kh_exist (ht_dates, k))
2065 continue;
2066 if (!(hash = get_hash (module, kh_key (ht_dates, k), MTRC_KEYMAP)))
2067 continue;
2068 if ((kv = kh_get (si32, hash, key)) == kh_end (hash))
2069 continue;
2070 list = list_insert_prepend (list, i322ptr (kh_val (hash, kv)));
2071 }
2072
2073 return list;
2074 }
2075
2076 /* Iterate over all the key/value pairs for the given hash structure
2077 * and set the maximum and minimum values found on an integer key and
2078 * integer value.
2079 *
2080 * Note: This are expensive calls since it has to iterate over all
2081 * key-value pairs
2082 *
2083 * If the hash structure is empty, no values are set.
2084 * On success the minimum and maximum values are set. */
2085 static void
get_ii32_min_max(khash_t (ii32)* hash,uint32_t * min,uint32_t * max)2086 get_ii32_min_max (khash_t (ii32) * hash, uint32_t * min, uint32_t * max) {
2087 khint_t k;
2088 uint32_t curvalue = 0;
2089 int i;
2090
2091 for (i = 0, k = kh_begin (hash); k != kh_end (hash); ++k) {
2092 if (!kh_exist (hash, k))
2093 continue;
2094
2095 curvalue = kh_value (hash, k);
2096 if (i++ == 0)
2097 *min = curvalue;
2098 if (curvalue > *max)
2099 *max = curvalue;
2100 if (curvalue < *min)
2101 *min = curvalue;
2102 }
2103 }
2104
2105 /* Iterate over all the key/value pairs for the given hash structure
2106 * and set the maximum and minimum values found on an integer key and
2107 * a uint64_t value.
2108 *
2109 * Note: This are expensive calls since it has to iterate over all
2110 * key-value pairs
2111 *
2112 * If the hash structure is empty, no values are set.
2113 * On success the minimum and maximum values are set. */
2114 static void
get_iu64_min_max(khash_t (iu64)* hash,uint64_t * min,uint64_t * max)2115 get_iu64_min_max (khash_t (iu64) * hash, uint64_t * min, uint64_t * max) {
2116 khint_t k;
2117 uint64_t curvalue = 0;
2118 int i;
2119
2120 for (i = 0, k = kh_begin (hash); k != kh_end (hash); ++k) {
2121 if (!kh_exist (hash, k))
2122 continue;
2123
2124 curvalue = kh_value (hash, k);
2125 if (i++ == 0)
2126 *min = curvalue;
2127 if (curvalue > *max)
2128 *max = curvalue;
2129 if (curvalue < *min)
2130 *min = curvalue;
2131 }
2132 }
2133
2134 /* Get the number of elements in a dates hash.
2135 *
2136 * Return 0 if the operation fails, else number of elements. */
2137 uint32_t
ht_get_size_dates(void)2138 ht_get_size_dates (void) {
2139 khash_t (igkh) * hash = ht_dates;
2140
2141 if (!hash)
2142 return 0;
2143
2144 return kh_size (hash);
2145 }
2146
2147 uint32_t
ht_get_excluded_ips(void)2148 ht_get_excluded_ips (void) {
2149 khash_t (si32) * hash = ht_cnt_overall;
2150
2151 if (!hash)
2152 return 0;
2153
2154 return get_si32 (hash, "excluded_ip");
2155 }
2156
2157 uint32_t
ht_get_invalid(void)2158 ht_get_invalid (void) {
2159 khash_t (si32) * hash = ht_cnt_overall;
2160
2161 if (!hash)
2162 return 0;
2163
2164 return get_si32 (hash, "failed_requests");
2165 }
2166
2167 uint32_t
ht_get_processed(void)2168 ht_get_processed (void) {
2169 khash_t (si32) * hash = ht_cnt_overall;
2170
2171 if (!hash)
2172 return 0;
2173
2174 return get_si32 (hash, "total_requests");
2175 }
2176
2177 uint32_t
ht_get_processing_time(void)2178 ht_get_processing_time (void) {
2179 khash_t (si32) * hash = ht_cnt_overall;
2180
2181 if (!hash)
2182 return 0;
2183
2184 return get_si32 (hash, "processing_time");
2185 }
2186
2187 uint32_t
ht_sum_valid(void)2188 ht_sum_valid (void) {
2189 khash_t (ii32) * hash = NULL;
2190 uint32_t k = 0;
2191 uint32_t sum = 0;
2192
2193 if (!ht_dates)
2194 return 0;
2195
2196 /* *INDENT-OFF* */
2197 HT_SUM_VAL (ht_dates, k, {
2198 if ((hash = get_hash (-1, k, MTRC_CNT_VALID)))
2199 sum += get_ii32 (hash, 1);
2200 });
2201 /* *INDENT-ON* */
2202
2203 return sum;
2204 }
2205
2206 uint64_t
ht_sum_bw(void)2207 ht_sum_bw (void) {
2208 khash_t (iu64) * hash = NULL;
2209 uint32_t k = 0;
2210 uint64_t sum = 0;
2211
2212 if (!ht_dates)
2213 return 0;
2214
2215 /* *INDENT-OFF* */
2216 HT_SUM_VAL (ht_dates, k, {
2217 if ((hash = get_hash (-1, k, MTRC_CNT_BW)))
2218 sum += get_iu64 (hash, 1);
2219 });
2220 /* *INDENT-ON* */
2221
2222 return sum;
2223 }
2224
2225 int
ht_insert_last_parse(uint32_t key,GLastParse lp)2226 ht_insert_last_parse (uint32_t key, GLastParse lp) {
2227 khash_t (iglp) * hash = ht_last_parse;
2228
2229 if (!hash)
2230 return 0;
2231
2232 return ins_iglp (hash, key, lp);
2233 }
2234
2235 int
ht_insert_date(uint32_t key)2236 ht_insert_date (uint32_t key) {
2237 khash_t (igkh) * hash = ht_dates;
2238
2239 if (!hash)
2240 return -1;
2241
2242 return ins_igkh (hash, key);
2243 }
2244
2245 uint32_t
ht_inc_cnt_overall(const char * key,uint32_t val)2246 ht_inc_cnt_overall (const char *key, uint32_t val) {
2247 khash_t (si32) * hash = ht_cnt_overall;
2248
2249 if (!hash)
2250 return 0;
2251
2252 if (get_si32 (hash, key) != 0)
2253 return inc_si32 (hash, key, val);
2254 return inc_si32 (hash, xstrdup (key), val);
2255 }
2256
2257 uint32_t
ht_inc_cnt_valid(uint32_t date,uint32_t inc)2258 ht_inc_cnt_valid (uint32_t date, uint32_t inc) {
2259 khash_t (ii32) * hash = get_hash (-1, date, MTRC_CNT_VALID);
2260
2261 if (!hash)
2262 return 0;
2263
2264 return inc_ii32 (hash, 1, inc);
2265 }
2266
2267 int
ht_inc_cnt_bw(uint32_t date,uint64_t inc)2268 ht_inc_cnt_bw (uint32_t date, uint64_t inc) {
2269 khash_t (iu64) * hash = get_hash (-1, date, MTRC_CNT_BW);
2270
2271 if (!hash)
2272 return 0;
2273
2274 return inc_iu64 (hash, 1, inc);
2275 }
2276
2277 /* Increases the unique key counter from a uint32_t key.
2278 *
2279 * On error, 0 is returned.
2280 * On success the inserted key is returned */
2281 static uint32_t
ht_ins_seq(const char * key)2282 ht_ins_seq (const char *key) {
2283 khash_t (si32) * hash = ht_seqs;
2284
2285 if (!hash)
2286 return 0;
2287
2288 if (get_si32 (hash, key) != 0)
2289 return inc_si32 (hash, key, 1);
2290 return inc_si32 (hash, xstrdup (key), 1);
2291 }
2292
2293
2294 /* Encode a data key and a unique visitor's key to a new uint64_t key
2295 *
2296 * ###NOTE: THIS LIMITS THE MAX VALUE OF A DATA TABLE TO uint32_t
2297 * WILL NEED TO CHANGE THIS IF WE GO OVER uint32_t
2298 */
2299 static uint64_t
u64encode(uint32_t x,uint32_t y)2300 u64encode (uint32_t x, uint32_t y) {
2301 return x > y ? (uint32_t) y | ((uint64_t) x << 32) : (uint32_t) x | ((uint64_t) y << 32);
2302 }
2303
2304 /* Decode a uint64_t number into the original two uint32_t */
2305 void
u64decode(uint64_t n,uint32_t * x,uint32_t * y)2306 u64decode (uint64_t n, uint32_t * x, uint32_t * y) {
2307 *x = (uint64_t) n >> 32;
2308 *y = (uint64_t) n & 0xFFFFFFFF;
2309 }
2310
2311 /* Insert a unique visitor key string (IP/DATE/UA), mapped to an auto
2312 * incremented value.
2313 *
2314 * If the given key exists, its value is returned.
2315 * On error, 0 is returned.
2316 * On success the value of the key inserted is returned */
2317 uint32_t
ht_insert_unique_key(uint32_t date,const char * key)2318 ht_insert_unique_key (uint32_t date, const char *key) {
2319 uint32_t val = 0;
2320 char *dupkey = NULL;
2321 khash_t (si32) * hash = get_hash (-1, date, MTRC_UNIQUE_KEYS);
2322
2323 if (!hash)
2324 return 0;
2325
2326 if ((val = get_si32 (hash, key)) != 0)
2327 return val;
2328
2329 dupkey = xstrdup (key);
2330 if ((val = ins_si32_inc (hash, dupkey, ht_ins_seq, "ht_unique_keys")) == 0)
2331 free (dupkey);
2332 return val;
2333 }
2334
2335 /* Insert a user agent key string, mapped to an auto incremented value.
2336 *
2337 * If the given key exists, its value is returned.
2338 * On error, 0 is returned.
2339 * On success the value of the key inserted is returned */
2340 uint32_t
ht_insert_agent_key(uint32_t date,const char * key)2341 ht_insert_agent_key (uint32_t date, const char *key) {
2342 char *dupkey = NULL;
2343 uint32_t val = 0;
2344 khash_t (si32) * hash = get_hash (-1, date, MTRC_AGENT_KEYS);
2345
2346 if (!hash)
2347 return 0;
2348
2349 if ((val = get_si32 (hash, key)) != 0)
2350 return val;
2351
2352 dupkey = xstrdup (key);
2353 if ((val = ins_si32_inc (hash, dupkey, ht_ins_seq, "ht_agent_keys")) == 0)
2354 free (dupkey);
2355 return val;
2356 }
2357
2358 /* Insert a user agent uint32_t key, mapped to a user agent string value.
2359 *
2360 * On error, -1 is returned.
2361 * On success 0 is returned */
2362 int
ht_insert_agent_value(uint32_t date,uint32_t key,const char * value)2363 ht_insert_agent_value (uint32_t date, uint32_t key, const char *value) {
2364 khash_t (is32) * hash = get_hash (-1, date, MTRC_AGENT_VALS);
2365 char *dupval = NULL;
2366 int ret = 0;
2367
2368 if (!hash)
2369 return -1;
2370
2371 dupval = xstrdup (value);
2372 if ((ret = ins_is32 (hash, key, dupval)) != 0)
2373 free (dupval);
2374 return ret;
2375 }
2376
2377 /* Insert a keymap string key.
2378 *
2379 * If the given key exists, its value is returned.
2380 * On error, 0 is returned.
2381 * On success the value of the key inserted is returned */
2382 uint32_t
ht_insert_keymap(GModule module,uint32_t date,const char * key,uint32_t * ckey)2383 ht_insert_keymap (GModule module, uint32_t date, const char *key, uint32_t * ckey) {
2384 khash_t (si32) * hash = get_hash (module, date, MTRC_KEYMAP);
2385 khash_t (si32) * cache = get_hash_from_cache (module, MTRC_KEYMAP);
2386
2387 uint32_t val = 0;
2388 char *modstr = NULL, *dupkey = NULL;
2389
2390 if (!hash)
2391 return 0;
2392
2393 if ((val = get_si32 (hash, key)) != 0) {
2394 *ckey = get_si32 (cache, key);
2395 return val;
2396 }
2397
2398 modstr = get_module_str (module);
2399 dupkey = xstrdup (key);
2400
2401 if ((val = ins_si32_inc (hash, dupkey, ht_ins_seq, modstr)) == 0) {
2402 free (dupkey);
2403 free (modstr);
2404 return val;
2405 }
2406 *ckey = ins_si32_ai (cache, dupkey);
2407 free (modstr);
2408
2409 return val;
2410 }
2411
2412 /* Insert a uniqmap string key.
2413 *
2414 * If the given key exists, 0 is returned.
2415 * On error, 0 is returned.
2416 * On success the value of the key inserted is returned */
2417 int
ht_insert_uniqmap(GModule module,uint32_t date,uint32_t key,uint32_t value)2418 ht_insert_uniqmap (GModule module, uint32_t date, uint32_t key, uint32_t value) {
2419 khash_t (u648) * hash = get_hash (module, date, MTRC_UNIQMAP);
2420 uint64_t k = 0;
2421
2422 if (!hash)
2423 return 0;
2424
2425 k = u64encode (key, value);
2426 return ins_u648 (hash, k, 1) == 0 ? 1 : 0;
2427 }
2428
2429 /* Insert a datamap uint32_t key and string value.
2430 *
2431 * On error, -1 is returned.
2432 * On success 0 is returned */
2433 int
ht_insert_datamap(GModule module,uint32_t date,uint32_t key,const char * value,uint32_t ckey)2434 ht_insert_datamap (GModule module, uint32_t date, uint32_t key, const char *value,
2435 uint32_t ckey) {
2436 khash_t (is32) * hash = get_hash (module, date, MTRC_DATAMAP);
2437 khash_t (is32) * cache = get_hash_from_cache (module, MTRC_DATAMAP);
2438 char *dupval = NULL;
2439 int ret = 0;
2440
2441 if (!hash)
2442 return -1;
2443
2444 dupval = xstrdup (value);
2445 if ((ret = ins_is32 (hash, key, dupval)) == 0)
2446 ins_is32 (cache, ckey, dupval);
2447 else
2448 free (dupval);
2449
2450 return ret;
2451 }
2452
2453 /* Insert a rootmap uint32_t key from the keymap store mapped to its string
2454 * value.
2455 *
2456 * On error, -1 is returned.
2457 * On success 0 is returned */
2458 int
ht_insert_rootmap(GModule module,uint32_t date,uint32_t key,const char * value,uint32_t ckey)2459 ht_insert_rootmap (GModule module, uint32_t date, uint32_t key, const char *value,
2460 uint32_t ckey) {
2461 khash_t (is32) * hash = get_hash (module, date, MTRC_ROOTMAP);
2462 khash_t (is32) * cache = get_hash_from_cache (module, MTRC_ROOTMAP);
2463 char *dupval = NULL;
2464 int ret = 0;
2465
2466 if (!hash)
2467 return -1;
2468
2469 dupval = xstrdup (value);
2470 if ((ret = ins_is32 (hash, key, dupval)) == 0)
2471 ins_is32 (cache, ckey, dupval);
2472 else
2473 free (dupval);
2474
2475 return ret;
2476 }
2477
2478 /* Insert a data uint32_t key mapped to the corresponding uint32_t root key.
2479 *
2480 * On error, -1 is returned.
2481 * On success 0 is returned */
2482 int
ht_insert_root(GModule module,uint32_t date,uint32_t key,uint32_t value,uint32_t dkey,uint32_t rkey)2483 ht_insert_root (GModule module, uint32_t date, uint32_t key, uint32_t value, uint32_t dkey,
2484 uint32_t rkey) {
2485 khash_t (ii32) * hash = get_hash (module, date, MTRC_ROOT);
2486 khash_t (ii32) * cache = get_hash_from_cache (module, MTRC_ROOT);
2487
2488 if (!hash)
2489 return -1;
2490
2491 ins_ii32 (cache, dkey, rkey);
2492 return ins_ii32 (hash, key, value);
2493 }
2494
2495 /* Increases hits counter from a uint32_t key.
2496 *
2497 * On error, 0 is returned.
2498 * On success the inserted value is returned */
2499 uint32_t
ht_insert_hits(GModule module,uint32_t date,uint32_t key,uint32_t inc,uint32_t ckey)2500 ht_insert_hits (GModule module, uint32_t date, uint32_t key, uint32_t inc, uint32_t ckey) {
2501 khash_t (ii32) * hash = get_hash (module, date, MTRC_HITS);
2502 khash_t (ii32) * cache = get_hash_from_cache (module, MTRC_HITS);
2503
2504 if (!hash)
2505 return 0;
2506
2507 inc_ii32 (cache, ckey, inc);
2508 return inc_ii32 (hash, key, inc);
2509 }
2510
2511 /* Increases visitors counter from a uint32_t key.
2512 *
2513 * On error, 0 is returned.
2514 * On success the inserted value is returned */
2515 uint32_t
ht_insert_visitor(GModule module,uint32_t date,uint32_t key,uint32_t inc,uint32_t ckey)2516 ht_insert_visitor (GModule module, uint32_t date, uint32_t key, uint32_t inc, uint32_t ckey) {
2517 khash_t (ii32) * hash = get_hash (module, date, MTRC_VISITORS);
2518 khash_t (ii32) * cache = get_hash_from_cache (module, MTRC_VISITORS);
2519
2520 if (!hash)
2521 return 0;
2522
2523 inc_ii32 (cache, ckey, inc);
2524 return inc_ii32 (hash, key, inc);
2525 }
2526
2527 /* Increases bandwidth counter from a uint32_t key.
2528 *
2529 * On error, -1 is returned.
2530 * On success 0 is returned */
2531 int
ht_insert_bw(GModule module,uint32_t date,uint32_t key,uint64_t inc,uint32_t ckey)2532 ht_insert_bw (GModule module, uint32_t date, uint32_t key, uint64_t inc, uint32_t ckey) {
2533 khash_t (iu64) * hash = get_hash (module, date, MTRC_BW);
2534 khash_t (iu64) * cache = get_hash_from_cache (module, MTRC_BW);
2535
2536 if (!hash)
2537 return -1;
2538
2539 inc_iu64 (cache, ckey, inc);
2540 return inc_iu64 (hash, key, inc);
2541 }
2542
2543 /* Increases cumulative time served counter from a uint32_t key.
2544 *
2545 * On error, -1 is returned.
2546 * On success 0 is returned */
2547 int
ht_insert_cumts(GModule module,uint32_t date,uint32_t key,uint64_t inc,uint32_t ckey)2548 ht_insert_cumts (GModule module, uint32_t date, uint32_t key, uint64_t inc, uint32_t ckey) {
2549 khash_t (iu64) * hash = get_hash (module, date, MTRC_CUMTS);
2550 khash_t (iu64) * cache = get_hash_from_cache (module, MTRC_CUMTS);
2551
2552 if (!hash)
2553 return -1;
2554
2555 inc_iu64 (cache, ckey, inc);
2556 return inc_iu64 (hash, key, inc);
2557 }
2558
2559 /* Insert the maximum time served counter from a uint32_t key.
2560 * Note: it compares the current value with the given value.
2561 *
2562 * On error, -1 is returned.
2563 * On success 0 is returned */
2564 int
ht_insert_maxts(GModule module,uint32_t date,uint32_t key,uint64_t value,uint32_t ckey)2565 ht_insert_maxts (GModule module, uint32_t date, uint32_t key, uint64_t value, uint32_t ckey) {
2566 khash_t (iu64) * hash = get_hash (module, date, MTRC_MAXTS);
2567 khash_t (iu64) * cache = get_hash_from_cache (module, MTRC_MAXTS);
2568
2569 if (!hash)
2570 return -1;
2571
2572 if (get_iu64 (cache, ckey) < value)
2573 ins_iu64 (cache, ckey, value);
2574 if (get_iu64 (hash, key) < value)
2575 ins_iu64 (hash, key, value);
2576
2577 return 0;
2578 }
2579
2580 /* Insert a method given an uint32_t key and string value.
2581 *
2582 * On error, or if key exists, -1 is returned.
2583 * On success 0 is returned */
2584 int
ht_insert_method(GModule module,uint32_t date,uint32_t key,const char * value,uint32_t ckey)2585 ht_insert_method (GModule module, uint32_t date, uint32_t key, const char *value,
2586 uint32_t ckey) {
2587 khash_t (is32) * hash = get_hash (module, date, MTRC_METHODS);
2588 khash_t (is32) * cache = get_hash_from_cache (module, MTRC_METHODS);
2589 char *dupval = NULL;
2590 int ret = 0;
2591
2592 if (!hash)
2593 return -1;
2594
2595 dupval = xstrdup (value);
2596 if ((ret = ins_is32 (hash, key, dupval)) == 0)
2597 ins_is32 (cache, ckey, dupval);
2598 else
2599 free (dupval);
2600
2601 return ret;
2602 }
2603
2604 /* Insert a protocol given an uint32_t key and string value.
2605 *
2606 * On error, or if key exists, -1 is returned.
2607 * On success 0 is returned */
2608 int
ht_insert_protocol(GModule module,uint32_t date,uint32_t key,const char * value,uint32_t ckey)2609 ht_insert_protocol (GModule module, uint32_t date, uint32_t key, const char *value,
2610 uint32_t ckey) {
2611 khash_t (is32) * hash = get_hash (module, date, MTRC_PROTOCOLS);
2612 khash_t (is32) * cache = get_hash_from_cache (module, MTRC_PROTOCOLS);
2613 char *dupval = NULL;
2614 int ret = 0;
2615
2616 if (!hash)
2617 return -1;
2618
2619 dupval = xstrdup (value);
2620 if ((ret = ins_is32 (hash, key, dupval)) == 0)
2621 ins_is32 (cache, ckey, dupval);
2622 else
2623 free (dupval);
2624
2625 return ret;
2626 }
2627
2628 /* Insert an agent for a hostname given an uint32_t key and uint32_t value.
2629 *
2630 * On error, -1 is returned.
2631 * On success 0 is returned */
2632 int
ht_insert_agent(GModule module,uint32_t date,uint32_t key,uint32_t value)2633 ht_insert_agent (GModule module, uint32_t date, uint32_t key, uint32_t value) {
2634 khash_t (igsl) * hash = get_hash (module, date, MTRC_AGENTS);
2635
2636 if (!hash)
2637 return -1;
2638
2639 return ins_igsl (hash, key, value);
2640 }
2641
2642 /* Insert meta data counters from a string key.
2643 *
2644 * On error, -1 is returned.
2645 * On success 0 is returned */
2646 int
ht_insert_meta_data(GModule module,uint32_t date,const char * key,uint64_t value)2647 ht_insert_meta_data (GModule module, uint32_t date, const char *key, uint64_t value) {
2648 khash_t (su64) * hash = get_hash (module, date, MTRC_METADATA);
2649
2650 if (!hash)
2651 return -1;
2652
2653 return inc_su64 (hash, key, value);
2654 }
2655
2656 /* Insert an IP hostname mapped to the corresponding hostname.
2657 *
2658 * On error, or if key exists, -1 is returned.
2659 * On success 0 is returned */
2660 int
ht_insert_hostname(const char * ip,const char * host)2661 ht_insert_hostname (const char *ip, const char *host) {
2662 khash_t (ss32) * hash = ht_hostnames;
2663
2664 if (!hash)
2665 return -1;
2666
2667 return ins_ss32 (hash, ip, host);
2668 }
2669
2670 /* Insert a JSON log format specification such as request.method => %m.
2671 *
2672 * On error -1 is returned.
2673 * On success or if key exists, 0 is returned */
2674 int
ht_insert_json_logfmt(GO_UNUSED void * userdata,char * key,char * spec)2675 ht_insert_json_logfmt (GO_UNUSED void *userdata, char *key, char *spec) {
2676 int ret;
2677 char *dupkey = NULL;
2678 khash_t (ss32) * hash = ht_json_logfmt;
2679 khint_t k;
2680
2681 if (!hash)
2682 return -1;
2683
2684 k = kh_get (ss32, hash, key);
2685 /* key found, free it then to insert */
2686 if (k != kh_end (hash))
2687 free (kh_val (hash, k));
2688 else {
2689 dupkey = xstrdup (key);
2690 k = kh_put (ss32, hash, dupkey, &ret);
2691 /* operation failed */
2692 if (ret == -1) {
2693 free (dupkey);
2694 return -1;
2695 }
2696 }
2697 kh_val (hash, k) = xstrdup (spec);
2698
2699 return 0;
2700 }
2701
2702 GLastParse
ht_get_last_parse(uint32_t key)2703 ht_get_last_parse (uint32_t key) {
2704 khash_t (iglp) * hash = ht_last_parse;
2705 return get_iglp (hash, key);
2706 }
2707
2708 /* Get the number of elements in a datamap.
2709 *
2710 * Return -1 if the operation fails, else number of elements. */
2711 uint32_t
ht_get_size_datamap(GModule module)2712 ht_get_size_datamap (GModule module) {
2713 khash_t (is32) * cache = get_hash_from_cache (module, MTRC_DATAMAP);
2714
2715 if (!cache)
2716 return 0;
2717
2718 return kh_size (cache);
2719 }
2720
2721 /* Get the number of elements in a uniqmap.
2722 *
2723 * On error, 0 is returned.
2724 * On success the number of elements in MTRC_UNIQMAP is returned */
2725 uint32_t
ht_get_size_uniqmap(GModule module)2726 ht_get_size_uniqmap (GModule module) {
2727 khash_t (u648) * hash = NULL;
2728 uint32_t k = 0;
2729 uint32_t sum = 0;
2730
2731 if (!ht_dates)
2732 return 0;
2733
2734 /* *INDENT-OFF* */
2735 HT_SUM_VAL (ht_dates, k, {
2736 if ((hash = get_hash (module, k, MTRC_UNIQMAP)))
2737 sum += kh_size (hash);
2738 });
2739 /* *INDENT-ON* */
2740
2741 return sum;
2742 }
2743
2744 /* Get the string data value of a given uint32_t key.
2745 *
2746 * On error, NULL is returned.
2747 * On success the string value for the given key is returned */
2748 char *
ht_get_datamap(GModule module,uint32_t key)2749 ht_get_datamap (GModule module, uint32_t key) {
2750 khash_t (is32) * cache = get_hash_from_cache (module, MTRC_DATAMAP);
2751
2752 if (!cache)
2753 return NULL;
2754
2755 return get_is32 (cache, key);
2756 }
2757
2758 /* Get the string root from MTRC_ROOTMAP given an uint32_t data key.
2759 *
2760 * On error, NULL is returned.
2761 * On success the string value for the given key is returned */
2762 char *
ht_get_root(GModule module,uint32_t key)2763 ht_get_root (GModule module, uint32_t key) {
2764 int root_key = 0;
2765 khash_t (ii32) * hashroot = get_hash_from_cache (module, MTRC_ROOT);
2766 khash_t (is32) * hashrootmap = get_hash_from_cache (module, MTRC_ROOTMAP);
2767
2768 if (!hashroot || !hashrootmap)
2769 return NULL;
2770
2771 /* not found */
2772 if ((root_key = get_ii32 (hashroot, key)) == 0)
2773 return NULL;
2774
2775 return get_is32 (hashrootmap, root_key);
2776 }
2777
2778
2779 /* Get the int visitors value from MTRC_VISITORS given an int key.
2780 *
2781 * If key is not found, 0 is returned.
2782 * On error, -1 is returned.
2783 * On success the int value for the given key is returned */
2784 uint32_t
ht_get_hits(GModule module,int key)2785 ht_get_hits (GModule module, int key) {
2786 khash_t (ii32) * cache = get_hash_from_cache (module, MTRC_HITS);
2787
2788 if (!cache)
2789 return 0;
2790
2791 return get_ii32 (cache, key);
2792 }
2793
2794 /* Get the uint32_t visitors value from MTRC_VISITORS given an uint32_t key.
2795 *
2796 * If key is not found, 0 is returned.
2797 * On error, -1 is returned.
2798 * On success the uint32_t value for the given key is returned */
2799 uint32_t
ht_get_visitors(GModule module,uint32_t key)2800 ht_get_visitors (GModule module, uint32_t key) {
2801 khash_t (ii32) * cache = get_hash_from_cache (module, MTRC_VISITORS);
2802
2803 if (!cache)
2804 return 0;
2805
2806 return get_ii32 (cache, key);
2807 }
2808
2809 /* Get the uint64_t value from MTRC_BW given an uint32_t key.
2810 *
2811 * On error, or if key is not found, 0 is returned.
2812 * On success the uint64_t value for the given key is returned */
2813 uint64_t
ht_get_bw(GModule module,uint32_t key)2814 ht_get_bw (GModule module, uint32_t key) {
2815 khash_t (iu64) * cache = get_hash_from_cache (module, MTRC_BW);
2816
2817 if (!cache)
2818 return 0;
2819
2820 return get_iu64 (cache, key);
2821 }
2822
2823 /* Get the uint64_t value from MTRC_CUMTS given an uint32_t key.
2824 *
2825 * On error, or if key is not found, 0 is returned.
2826 * On success the uint64_t value for the given key is returned */
2827 uint64_t
ht_get_cumts(GModule module,uint32_t key)2828 ht_get_cumts (GModule module, uint32_t key) {
2829 khash_t (iu64) * cache = get_hash_from_cache (module, MTRC_CUMTS);
2830
2831 if (!cache)
2832 return 0;
2833
2834 return get_iu64 (cache, key);
2835 }
2836
2837 /* Get the uint64_t value from MTRC_MAXTS given an uint32_t key.
2838 *
2839 * On error, or if key is not found, 0 is returned.
2840 * On success the uint64_t value for the given key is returned */
2841 uint64_t
ht_get_maxts(GModule module,uint32_t key)2842 ht_get_maxts (GModule module, uint32_t key) {
2843 khash_t (iu64) * cache = get_hash_from_cache (module, MTRC_MAXTS);
2844
2845 if (!cache)
2846 return 0;
2847
2848 return get_iu64 (cache, key);
2849 }
2850
2851 /* Get the string value from MTRC_METHODS given an uint32_t key.
2852 *
2853 * On error, NULL is returned.
2854 * On success the string value for the given key is returned */
2855 char *
ht_get_method(GModule module,uint32_t key)2856 ht_get_method (GModule module, uint32_t key) {
2857 khash_t (is32) * cache = get_hash_from_cache (module, MTRC_METHODS);
2858
2859 if (!cache)
2860 return NULL;
2861
2862 return get_is32 (cache, key);
2863 }
2864
2865 /* Get the string value from MTRC_PROTOCOLS given an uint32_t key.
2866 *
2867 * On error, NULL is returned.
2868 * On success the string value for the given key is returned */
2869 char *
ht_get_protocol(GModule module,uint32_t key)2870 ht_get_protocol (GModule module, uint32_t key) {
2871 khash_t (is32) * cache = get_hash_from_cache (module, MTRC_PROTOCOLS);
2872
2873 if (!cache)
2874 return NULL;
2875
2876 return get_is32 (cache, key);
2877 }
2878
2879 /* Get the string value from ht_hostnames given a string key (IP).
2880 *
2881 * On error, NULL is returned.
2882 * On success the string value for the given key is returned */
2883 char *
ht_get_hostname(const char * host)2884 ht_get_hostname (const char *host) {
2885 khash_t (ss32) * hash = ht_hostnames;
2886
2887 if (!hash)
2888 return NULL;
2889
2890 return get_ss32 (hash, host);
2891 }
2892
2893 /* Get the string value from ht_json_logfmt given a JSON specifier key.
2894 *
2895 * On error, NULL is returned.
2896 * On success the string value for the given key is returned */
2897 char *
ht_get_json_logfmt(const char * key)2898 ht_get_json_logfmt (const char *key) {
2899 khash_t (ss32) * hash = ht_json_logfmt;
2900
2901 if (!hash)
2902 return NULL;
2903
2904 return get_ss32 (hash, key);
2905 }
2906
2907 /* Get the string value from ht_agent_vals (user agent) given an uint32_t key.
2908 *
2909 * On error, NULL is returned.
2910 * On success the string value for the given key is returned */
2911 char *
ht_get_host_agent_val(uint32_t key)2912 ht_get_host_agent_val (uint32_t key) {
2913 khash_t (is32) * hash = NULL;
2914 char *data = NULL;
2915 uint32_t k = 0;
2916
2917 if (!ht_dates)
2918 return NULL;
2919
2920 /* *INDENT-OFF* */
2921 HT_FIRST_VAL (ht_dates, k, {
2922 if ((hash = get_hash (-1, k, MTRC_AGENT_VALS)))
2923 if ((data = get_is32 (hash, key)))
2924 return data;
2925 });
2926 /* *INDENT-ON* */
2927
2928 return NULL;
2929 }
2930
2931 /* Get the list value from MTRC_AGENTS given an uint32_t key.
2932 *
2933 * On error, or if key is not found, NULL is returned.
2934 * On success the GSLList value for the given key is returned */
2935 GSLList *
ht_get_host_agent_list(GModule module,uint32_t key)2936 ht_get_host_agent_list (GModule module, uint32_t key) {
2937 GSLList *res = NULL, *list = NULL;
2938 khiter_t kv;
2939 khint_t k;
2940 khash_t (igsl) * hash = NULL;
2941 void *data = NULL;
2942
2943 if (!ht_dates)
2944 return NULL;
2945
2946 for (k = kh_begin (ht_dates); k != kh_end (ht_dates); ++k) {
2947 if (!kh_exist (ht_dates, k))
2948 continue;
2949 if (!(hash = get_hash (module, kh_key (ht_dates, k), MTRC_AGENTS)))
2950 continue;
2951 if ((kv = kh_get (igsl, hash, key)) == kh_end (hash))
2952 continue;
2953
2954 list = kh_val (hash, kv);
2955 /* *INDENT-OFF* */
2956 GSLIST_FOREACH (list, data, {
2957 res = list_insert_prepend (res, i322ptr ((*(uint32_t *) data)));
2958 });
2959 /* *INDENT-ON* */
2960 }
2961
2962 return res;
2963 }
2964
2965 uint32_t
ht_get_keymap(GModule module,const char * key)2966 ht_get_keymap (GModule module, const char *key) {
2967 khash_t (si32) * cache = get_hash_from_cache (module, MTRC_KEYMAP);
2968
2969 if (!cache)
2970 return 0;
2971
2972 return get_si32 (cache, key);
2973 }
2974
2975 /* Get the meta data uint64_t from MTRC_METADATA given a string key.
2976 *
2977 * On error, or if key is not found, 0 is returned.
2978 * On success the uint64_t value for the given key is returned */
2979 uint64_t
ht_get_meta_data(GModule module,const char * key)2980 ht_get_meta_data (GModule module, const char *key) {
2981 khash_t (su64) * hash = NULL;
2982 uint32_t k = 0;
2983 uint64_t sum = 0;
2984
2985 /* *INDENT-OFF* */
2986 HT_SUM_VAL (ht_dates, k, {
2987 if ((hash = get_hash (module, k, MTRC_METADATA)))
2988 sum += get_su64 (hash, key);
2989 });
2990 /* *INDENT-ON* */
2991
2992 return sum;
2993 }
2994
2995 /* Set the maximum and minimum values found on an integer key and
2996 * integer value found on the MTRC_VISITORS hash structure.
2997 *
2998 * If the hash structure is empty, no values are set.
2999 * On success the minimum and maximum values are set. */
3000 void
ht_get_hits_min_max(GModule module,uint32_t * min,uint32_t * max)3001 ht_get_hits_min_max (GModule module, uint32_t * min, uint32_t * max) {
3002 khash_t (ii32) * cache = get_hash_from_cache (module, MTRC_HITS);
3003
3004 if (!cache)
3005 return;
3006
3007 get_ii32_min_max (cache, min, max);
3008 }
3009
3010 /* Set the maximum and minimum values found on an integer key and
3011 * integer value found on the MTRC_VISITORS hash structure.
3012 *
3013 * If the hash structure is empty, no values are set.
3014 * On success the minimum and maximum values are set. */
3015 void
ht_get_visitors_min_max(GModule module,uint32_t * min,uint32_t * max)3016 ht_get_visitors_min_max (GModule module, uint32_t * min, uint32_t * max) {
3017 khash_t (ii32) * cache = get_hash_from_cache (module, MTRC_VISITORS);
3018
3019 if (!cache)
3020 return;
3021
3022 get_ii32_min_max (cache, min, max);
3023 }
3024
3025 /* Set the maximum and minimum values found on an integer key and
3026 * a uint64_t value found on the MTRC_BW hash structure.
3027 *
3028 * If the hash structure is empty, no values are set.
3029 * On success the minimum and maximum values are set. */
3030 void
ht_get_bw_min_max(GModule module,uint64_t * min,uint64_t * max)3031 ht_get_bw_min_max (GModule module, uint64_t * min, uint64_t * max) {
3032 khash_t (iu64) * cache = get_hash_from_cache (module, MTRC_BW);
3033
3034 if (!cache)
3035 return;
3036
3037 get_iu64_min_max (cache, min, max);
3038 }
3039
3040 /* Set the maximum and minimum values found on an integer key and
3041 * a uint64_t value found on the MTRC_CUMTS hash structure.
3042 *
3043 * If the hash structure is empty, no values are set.
3044 * On success the minimum and maximum values are set. */
3045 void
ht_get_cumts_min_max(GModule module,uint64_t * min,uint64_t * max)3046 ht_get_cumts_min_max (GModule module, uint64_t * min, uint64_t * max) {
3047 khash_t (iu64) * cache = get_hash_from_cache (module, MTRC_CUMTS);
3048
3049 if (!cache)
3050 return;
3051
3052 get_iu64_min_max (cache, min, max);
3053 }
3054
3055 /* Set the maximum and minimum values found on an integer key and
3056 * a uint64_t value found on the MTRC_MAXTS hash structure.
3057 *
3058 * If the hash structure is empty, no values are set.
3059 * On success the minimum and maximum values are set. */
3060 void
ht_get_maxts_min_max(GModule module,uint64_t * min,uint64_t * max)3061 ht_get_maxts_min_max (GModule module, uint64_t * min, uint64_t * max) {
3062 khash_t (iu64) * cache = get_hash_from_cache (module, MTRC_MAXTS);
3063
3064 if (!cache)
3065 return;
3066
3067 get_iu64_min_max (cache, min, max);
3068 }
3069
3070 uint32_t *
get_sorted_dates(uint32_t * len)3071 get_sorted_dates (uint32_t * len) {
3072 khiter_t key;
3073 uint32_t *dates = NULL;
3074 int i = 0;
3075 uint32_t size = 0;
3076
3077 khash_t (igkh) * hash = ht_dates;
3078 if (!hash)
3079 return NULL;
3080
3081 size = kh_size (hash);
3082 dates = xcalloc (size, sizeof (uint32_t));
3083 for (key = kh_begin (hash); key != kh_end (hash); ++key)
3084 if (kh_exist (hash, key))
3085 dates[i++] = kh_key (hash, key);
3086 qsort (dates, i, sizeof (uint32_t), cmp_ui32_asc);
3087 *len = i;
3088
3089 return dates;
3090 }
3091
3092 void
destroy_date_stores(int date)3093 destroy_date_stores (int date) {
3094 khash_t (igkh) * hash = ht_dates;
3095 khiter_t k;
3096
3097 k = kh_get (igkh, hash, date);
3098 free_stores (kh_value (hash, k));
3099 kh_del (igkh, hash, k);
3100 }
3101
3102 int
invalidate_date(int date)3103 invalidate_date (int date) {
3104 khash_t (igkh) * hash = ht_dates;
3105 GModule module;
3106 size_t idx = 0;
3107
3108 if (!hash)
3109 return -1;
3110
3111 FOREACH_MODULE (idx, module_list) {
3112 module = module_list[idx];
3113 del_module_metrics (cache_storage, module, 0);
3114 }
3115
3116 destroy_date_stores (date);
3117
3118 return 0;
3119 }
3120
3121 static uint32_t
ins_cache_map(GModule module,GSMetric metric,const char * key)3122 ins_cache_map (GModule module, GSMetric metric, const char *key) {
3123 khash_t (si32) * cache = get_hash_from_cache (module, metric);
3124
3125 if (!cache)
3126 return 0;
3127 return ins_si32_ai (cache, key);
3128 }
3129
3130 static int
ins_cache_is32(GKHashStorage * store,GModule module,GSMetric metric,uint32_t key,uint32_t ckey)3131 ins_cache_is32 (GKHashStorage * store, GModule module, GSMetric metric, uint32_t key,
3132 uint32_t ckey) {
3133 khash_t (is32) * hash = get_hash_from_store (store, module, metric);
3134 khash_t (is32) * cache = get_hash_from_cache (module, metric);
3135 khint_t k;
3136
3137 if ((k = kh_get (is32, hash, key)) == kh_end (hash))
3138 return -1;
3139 return ins_is32 (cache, ckey, kh_val (hash, k));
3140 }
3141
3142 static int
inc_cache_ii32(GKHashStorage * store,GModule module,GSMetric metric,uint32_t key,uint32_t ckey)3143 inc_cache_ii32 (GKHashStorage * store, GModule module, GSMetric metric, uint32_t key,
3144 uint32_t ckey) {
3145 khash_t (ii32) * hash = get_hash_from_store (store, module, metric);
3146 khash_t (ii32) * cache = get_hash_from_cache (module, metric);
3147 khint_t k;
3148
3149 if ((k = kh_get (ii32, hash, key)) == kh_end (hash))
3150 return -1;
3151 return inc_ii32 (cache, ckey, kh_val (hash, k));
3152 }
3153
3154 static int
max_cache_iu64(GKHashStorage * store,GModule module,GSMetric metric,uint32_t key,uint32_t ckey)3155 max_cache_iu64 (GKHashStorage * store, GModule module, GSMetric metric, uint32_t key,
3156 uint32_t ckey) {
3157 khash_t (iu64) * hash = get_hash_from_store (store, module, metric);
3158 khash_t (iu64) * cache = get_hash_from_cache (module, metric);
3159 khint_t k;
3160
3161 if ((k = kh_get (iu64, hash, key)) == kh_end (hash))
3162 return -1;
3163
3164 if (get_iu64 (cache, ckey) < kh_val (hash, k))
3165 return ins_iu64 (cache, ckey, kh_val (hash, k));
3166 return -1;
3167 }
3168
3169 static int
inc_cache_iu64(GKHashStorage * store,GModule module,GSMetric metric,uint32_t key,uint32_t ckey)3170 inc_cache_iu64 (GKHashStorage * store, GModule module, GSMetric metric, uint32_t key,
3171 uint32_t ckey) {
3172 khash_t (iu64) * hash = get_hash_from_store (store, module, metric);
3173 khash_t (iu64) * cache = get_hash_from_cache (module, metric);
3174 khint_t k;
3175
3176 if ((k = kh_get (iu64, hash, key)) == kh_end (hash))
3177 return -1;
3178 return inc_iu64 (cache, ckey, kh_val (hash, k));
3179 }
3180
3181 static int
ins_raw_num_data(GModule module,uint32_t date)3182 ins_raw_num_data (GModule module, uint32_t date) {
3183 khiter_t k, kr;
3184 GKHashStorage *store = get_store (date);
3185 uint32_t ckey = 0, rkey = 0, nrkey = 0;
3186 char *val = NULL;
3187
3188 khash_t (si32) * kmap = get_hash_from_store (store, module, MTRC_KEYMAP);
3189 khash_t (ii32) * root = get_hash_from_store (store, module, MTRC_ROOT);
3190 khash_t (is32) * rmap = get_hash_from_store (store, module, MTRC_ROOTMAP);
3191 khash_t (ii32) * cache = get_hash_from_cache (module, MTRC_ROOT);
3192
3193 if (!kmap)
3194 return -1;
3195
3196 for (k = kh_begin (kmap); k != kh_end (kmap); ++k) {
3197 if (!kh_exist (kmap, k))
3198 continue;
3199 if ((ckey = ins_cache_map (module, MTRC_KEYMAP, kh_key (kmap, k))) == 0)
3200 continue;
3201
3202 if ((rkey = get_ii32 (root, kh_val (kmap, k)))) {
3203 kr = kh_get (is32, rmap, rkey);
3204 if (kr != kh_end (rmap) && (val = kh_val (rmap, kr))) {
3205 nrkey = ins_cache_map (module, MTRC_KEYMAP, val);
3206 ins_cache_is32 (store, module, MTRC_ROOTMAP, rkey, nrkey);
3207 ins_ii32 (cache, ckey, nrkey);
3208 }
3209 }
3210
3211 ins_cache_is32 (store, module, MTRC_DATAMAP, kh_val (kmap, k), ckey);
3212 inc_cache_ii32 (store, module, MTRC_HITS, kh_val (kmap, k), ckey);
3213 inc_cache_ii32 (store, module, MTRC_VISITORS, kh_val (kmap, k), ckey);
3214 inc_cache_iu64 (store, module, MTRC_BW, kh_val (kmap, k), ckey);
3215 inc_cache_iu64 (store, module, MTRC_CUMTS, kh_val (kmap, k), ckey);
3216 max_cache_iu64 (store, module, MTRC_MAXTS, kh_val (kmap, k), ckey);
3217 ins_cache_is32 (store, module, MTRC_METHODS, kh_val (kmap, k), ckey);
3218 ins_cache_is32 (store, module, MTRC_PROTOCOLS, kh_val (kmap, k), ckey);
3219 }
3220
3221 return 0;
3222 }
3223
3224 static int
set_raw_num_data_date(GModule module)3225 set_raw_num_data_date (GModule module) {
3226 khiter_t k;
3227 khash_t (igkh) * hash = ht_dates;
3228
3229 if (!hash)
3230 return -1;
3231
3232 /* iterate over the stored dates */
3233 for (k = kh_begin (hash); k != kh_end (hash); ++k) {
3234 if (kh_exist (hash, k))
3235 ins_raw_num_data (module, kh_key (hash, k));
3236 }
3237
3238 return 0;
3239 }
3240
3241 int
rebuild_rawdata_cache(void)3242 rebuild_rawdata_cache (void) {
3243 GModule module;
3244 size_t idx = 0;
3245
3246 FOREACH_MODULE (idx, module_list) {
3247 module = module_list[idx];
3248 set_raw_num_data_date (module);
3249 }
3250
3251 return 2;
3252 }
3253
3254 /* A wrapper to initialize a raw data structure.
3255 *
3256 * On success a GRawData structure is returned. */
3257 static GRawData *
init_new_raw_data(GModule module,uint32_t ht_size)3258 init_new_raw_data (GModule module, uint32_t ht_size) {
3259 GRawData *raw_data;
3260
3261 raw_data = new_grawdata ();
3262 raw_data->idx = 0;
3263 raw_data->module = module;
3264 raw_data->size = ht_size;
3265 raw_data->items = new_grawdata_item (ht_size);
3266
3267 return raw_data;
3268 }
3269
3270 static GRawData *
get_u32_raw_data(GModule module)3271 get_u32_raw_data (GModule module) {
3272 khash_t (ii32) * hash = get_hash_from_cache (module, MTRC_HITS);
3273 GRawData *raw_data;
3274 khiter_t key;
3275 uint32_t ht_size = 0;
3276
3277 if (!hash)
3278 return NULL;
3279
3280 ht_size = kh_size (hash);
3281 raw_data = init_new_raw_data (module, ht_size);
3282 raw_data->type = U32;
3283
3284 for (key = kh_begin (hash); key != kh_end (hash); ++key) {
3285 if (!kh_exist (hash, key))
3286 continue;
3287 raw_data->items[raw_data->idx].nkey = kh_key (hash, key);
3288 raw_data->items[raw_data->idx].hits = kh_val (hash, key);
3289 raw_data->idx++;
3290 }
3291
3292 return raw_data;
3293 }
3294
3295 /* Store the key/value pairs from a hash table into raw_data and sorts
3296 * the hits (numeric) value.
3297 *
3298 * On error, NULL is returned.
3299 * On success the GRawData sorted is returned */
3300 static GRawData *
get_str_raw_data(GModule module)3301 get_str_raw_data (GModule module) {
3302 khash_t (is32) * hash = get_hash_from_cache (module, MTRC_DATAMAP);
3303 GRawData *raw_data;
3304 khiter_t key;
3305 uint32_t ht_size = 0;
3306
3307 if (!hash)
3308 return NULL;
3309
3310 ht_size = kh_size (hash);
3311 raw_data = init_new_raw_data (module, ht_size);
3312 raw_data->type = STR;
3313
3314 for (key = kh_begin (hash); key != kh_end (hash); ++key) {
3315 if (!kh_exist (hash, key))
3316 continue;
3317 raw_data->items[raw_data->idx].nkey = kh_key (hash, key);
3318 raw_data->items[raw_data->idx].data = kh_val (hash, key);
3319 raw_data->idx++;
3320 }
3321
3322 return raw_data;
3323 }
3324
3325 /* Entry point to load the raw data from the data store into our
3326 * GRawData structure.
3327 *
3328 * On error, NULL is returned.
3329 * On success the GRawData sorted is returned */
3330 GRawData *
parse_raw_data(GModule module)3331 parse_raw_data (GModule module) {
3332 GRawData *raw_data = NULL;
3333
3334 #ifdef _DEBUG
3335 clock_t begin = clock ();
3336 double taken;
3337 char *modstr = NULL;
3338 LOG_DEBUG (("== parse_raw_data ==\n"));
3339 #endif
3340
3341 switch (module) {
3342 case VISITORS:
3343 raw_data = get_str_raw_data (module);
3344 if (raw_data)
3345 sort_raw_str_data (raw_data, raw_data->idx);
3346 break;
3347 default:
3348 raw_data = get_u32_raw_data (module);
3349 if (raw_data)
3350 sort_raw_num_data (raw_data, raw_data->idx);
3351 }
3352
3353 #ifdef _DEBUG
3354 modstr = get_module_str (module);
3355 taken = (double) (clock () - begin) / CLOCKS_PER_SEC;
3356 LOG_DEBUG (("== %-30s%f\n\n", modstr, taken));
3357 free (modstr);
3358 #endif
3359
3360 return raw_data;
3361 }
3362
3363 void
init_pre_storage(void)3364 init_pre_storage (void) {
3365 ht_json_logfmt = (khash_t (ss32) *) new_ss32_ht ();
3366 }
3367
3368 /* Initialize hash tables */
3369 void
init_storage(void)3370 init_storage (void) {
3371 /* *INDENT-OFF* */
3372 ht_hostnames = (khash_t (ss32) *) new_ss32_ht ();
3373 ht_dates = (khash_t (igkh) *) new_igkh_ht ();
3374 ht_seqs = (khash_t (si32) *) new_si32_ht ();
3375 ht_cnt_overall = (khash_t (si32) *) new_si32_ht ();
3376 ht_last_parse = (khash_t (iglp) *) new_iglp_ht ();
3377 /* *INDENT-ON* */
3378
3379 cache_storage = init_gkhashmodule ();
3380
3381 if (conf.restore)
3382 restore_data ();
3383 }
3384
3385 /* Destroys the hash structure */
3386 static void
des_igkh(void * h)3387 des_igkh (void *h) {
3388 khint_t k;
3389 khash_t (igkh) * hash = h;
3390
3391 if (!hash)
3392 return;
3393
3394 for (k = kh_begin (hash); k != kh_end (hash); ++k) {
3395 if (!kh_exist (hash, k))
3396 continue;
3397 free_stores (kh_value (hash, k));
3398 }
3399 kh_destroy (igkh, hash);
3400 }
3401
3402 /* Destroys the hash structure and its content */
3403 void
free_storage(void)3404 free_storage (void) {
3405 GModule module;
3406 size_t idx = 0;
3407
3408 if (conf.persist)
3409 persist_data ();
3410
3411 des_igkh (ht_dates);
3412 des_si32_free (ht_seqs, 1);
3413 des_ss32_free (ht_hostnames, 1);
3414 des_ss32_free (ht_json_logfmt, 1);
3415
3416 FOREACH_MODULE (idx, module_list) {
3417 module = module_list[idx];
3418 free_module_metrics (cache_storage, module, 0);
3419 }
3420 free (cache_storage);
3421
3422 des_si32_free (ht_cnt_overall, 1);
3423 des_iglp (ht_last_parse, 1);
3424 }
3425