1 /* Copyright (C) 2007-2012 Open Information Security Foundation
2 *
3 * You can copy, redistribute or modify this Program under the terms of
4 * the GNU General Public License version 2 as published by the Free
5 * Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 * You should have received a copy of the GNU General Public License
13 * version 2 along with this program; if not, write to the Free Software
14 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15 * 02110-1301, USA.
16 */
17
18 /**
19 * \file
20 *
21 * \author Victor Julien <victor@inliniac.net>
22 *
23 * Information about hosts.
24 */
25
26 #include "suricata-common.h"
27 #include "conf.h"
28
29 #include "util-debug.h"
30 #include "host.h"
31 #include "host-storage.h"
32 #include "host-bit.h"
33
34 #include "util-random.h"
35 #include "util-misc.h"
36 #include "util-byte.h"
37
38 #include "host-queue.h"
39
40 #include "detect-tag.h"
41 #include "detect-engine-tag.h"
42 #include "detect-engine-threshold.h"
43
44 #include "util-hash-lookup3.h"
45
46 static Host *HostGetUsedHost(void);
47
48 /** host hash table */
49 HostHashRow *host_hash;
50 /** queue with spare hosts */
51 static HostQueue host_spare_q;
52 HostConfig host_config;
53
54 SC_ATOMIC_DECLARE(uint64_t,host_memuse);
55 SC_ATOMIC_DECLARE(uint32_t,host_counter);
56 SC_ATOMIC_DECLARE(uint32_t,host_prune_idx);
57
58 /** size of the host object. Maybe updated in HostInitConfig to include
59 * the storage APIs additions. */
60 static uint16_t g_host_size = sizeof(Host);
61
62 /**
63 * \brief Update memcap value
64 *
65 * \param size new memcap value
66 */
HostSetMemcap(uint64_t size)67 int HostSetMemcap(uint64_t size)
68 {
69 if ((uint64_t)SC_ATOMIC_GET(host_memuse) < size) {
70 SC_ATOMIC_SET(host_config.memcap, size);
71 return 1;
72 }
73
74 return 0;
75 }
76
77 /**
78 * \brief Return memcap value
79 *
80 * \retval memcap value
81 */
HostGetMemcap(void)82 uint64_t HostGetMemcap(void)
83 {
84 uint64_t memcapcopy = SC_ATOMIC_GET(host_config.memcap);
85 return memcapcopy;
86 }
87
88 /**
89 * \brief Return memuse value
90 *
91 * \retval memuse value
92 */
HostGetMemuse(void)93 uint64_t HostGetMemuse(void)
94 {
95 uint64_t memuse = SC_ATOMIC_GET(host_memuse);
96 return memuse;
97 }
98
HostSpareQueueGetSize(void)99 uint32_t HostSpareQueueGetSize(void)
100 {
101 return HostQueueLen(&host_spare_q);
102 }
103
HostMoveToSpare(Host * h)104 void HostMoveToSpare(Host *h)
105 {
106 HostEnqueue(&host_spare_q, h);
107 (void) SC_ATOMIC_SUB(host_counter, 1);
108 }
109
HostAlloc(void)110 Host *HostAlloc(void)
111 {
112 if (!(HOST_CHECK_MEMCAP(g_host_size))) {
113 return NULL;
114 }
115 (void) SC_ATOMIC_ADD(host_memuse, g_host_size);
116
117 Host *h = SCMalloc(g_host_size);
118 if (unlikely(h == NULL))
119 goto error;
120
121 memset(h, 0x00, g_host_size);
122
123 SCMutexInit(&h->m, NULL);
124 SC_ATOMIC_INIT(h->use_cnt);
125 return h;
126
127 error:
128 return NULL;
129 }
130
HostFree(Host * h)131 void HostFree(Host *h)
132 {
133 if (h != NULL) {
134 HostClearMemory(h);
135 SCMutexDestroy(&h->m);
136 SCFree(h);
137 (void) SC_ATOMIC_SUB(host_memuse, g_host_size);
138 }
139 }
140
HostNew(Address * a)141 static Host *HostNew(Address *a)
142 {
143 Host *h = HostAlloc();
144 if (h == NULL)
145 goto error;
146
147 /* copy address */
148 COPY_ADDRESS(a, &h->a);
149
150 return h;
151
152 error:
153 return NULL;
154 }
155
HostClearMemory(Host * h)156 void HostClearMemory(Host *h)
157 {
158 if (h->iprep != NULL) {
159 SCFree(h->iprep);
160 h->iprep = NULL;
161 }
162
163 if (HostStorageSize() > 0)
164 HostFreeStorage(h);
165 }
166
167 #define HOST_DEFAULT_HASHSIZE 4096
168 #define HOST_DEFAULT_MEMCAP 16777216
169 #define HOST_DEFAULT_PREALLOC 1000
170
171 /** \brief initialize the configuration
172 * \warning Not thread safe */
HostInitConfig(char quiet)173 void HostInitConfig(char quiet)
174 {
175 SCLogDebug("initializing host engine...");
176 if (HostStorageSize() > 0)
177 g_host_size = sizeof(Host) + HostStorageSize();
178
179 memset(&host_config, 0, sizeof(host_config));
180 //SC_ATOMIC_INIT(flow_flags);
181 SC_ATOMIC_INIT(host_counter);
182 SC_ATOMIC_INIT(host_memuse);
183 SC_ATOMIC_INIT(host_prune_idx);
184 SC_ATOMIC_INIT(host_config.memcap);
185 HostQueueInit(&host_spare_q);
186
187 /* set defaults */
188 host_config.hash_rand = (uint32_t)RandomGet();
189 host_config.hash_size = HOST_DEFAULT_HASHSIZE;
190 host_config.prealloc = HOST_DEFAULT_PREALLOC;
191 SC_ATOMIC_SET(host_config.memcap, HOST_DEFAULT_MEMCAP);
192
193 /* Check if we have memcap and hash_size defined at config */
194 const char *conf_val;
195 uint32_t configval = 0;
196
197 /** set config values for memcap, prealloc and hash_size */
198 if ((ConfGetValue("host.memcap", &conf_val)) == 1)
199 {
200 uint64_t host_memcap = 0;
201 if (ParseSizeStringU64(conf_val, &host_memcap) < 0) {
202 SCLogError(SC_ERR_SIZE_PARSE, "Error parsing host.memcap "
203 "from conf file - %s. Killing engine",
204 conf_val);
205 exit(EXIT_FAILURE);
206 } else {
207 SC_ATOMIC_SET(host_config.memcap, host_memcap);
208 }
209 }
210 if ((ConfGetValue("host.hash-size", &conf_val)) == 1)
211 {
212 if (StringParseUint32(&configval, 10, strlen(conf_val),
213 conf_val) > 0) {
214 host_config.hash_size = configval;
215 }
216 }
217
218 if ((ConfGetValue("host.prealloc", &conf_val)) == 1)
219 {
220 if (StringParseUint32(&configval, 10, strlen(conf_val),
221 conf_val) > 0) {
222 host_config.prealloc = configval;
223 } else {
224 WarnInvalidConfEntry("host.prealloc", "%"PRIu32, host_config.prealloc);
225 }
226 }
227 SCLogDebug("Host config from suricata.yaml: memcap: %"PRIu64", hash-size: "
228 "%"PRIu32", prealloc: %"PRIu32, SC_ATOMIC_GET(host_config.memcap),
229 host_config.hash_size, host_config.prealloc);
230
231 /* alloc hash memory */
232 uint64_t hash_size = host_config.hash_size * sizeof(HostHashRow);
233 if (!(HOST_CHECK_MEMCAP(hash_size))) {
234 SCLogError(SC_ERR_HOST_INIT, "allocating host hash failed: "
235 "max host memcap is smaller than projected hash size. "
236 "Memcap: %"PRIu64", Hash table size %"PRIu64". Calculate "
237 "total hash size by multiplying \"host.hash-size\" with %"PRIuMAX", "
238 "which is the hash bucket size.", SC_ATOMIC_GET(host_config.memcap), hash_size,
239 (uintmax_t)sizeof(HostHashRow));
240 exit(EXIT_FAILURE);
241 }
242 host_hash = SCMallocAligned(host_config.hash_size * sizeof(HostHashRow), CLS);
243 if (unlikely(host_hash == NULL)) {
244 FatalError(SC_ERR_FATAL,
245 "Fatal error encountered in HostInitConfig. Exiting...");
246 }
247 memset(host_hash, 0, host_config.hash_size * sizeof(HostHashRow));
248
249 uint32_t i = 0;
250 for (i = 0; i < host_config.hash_size; i++) {
251 HRLOCK_INIT(&host_hash[i]);
252 }
253 (void) SC_ATOMIC_ADD(host_memuse, (host_config.hash_size * sizeof(HostHashRow)));
254
255 if (quiet == FALSE) {
256 SCLogConfig("allocated %"PRIu64" bytes of memory for the host hash... "
257 "%" PRIu32 " buckets of size %" PRIuMAX "",
258 SC_ATOMIC_GET(host_memuse), host_config.hash_size,
259 (uintmax_t)sizeof(HostHashRow));
260 }
261
262 /* pre allocate hosts */
263 for (i = 0; i < host_config.prealloc; i++) {
264 if (!(HOST_CHECK_MEMCAP(g_host_size))) {
265 SCLogError(SC_ERR_HOST_INIT, "preallocating hosts failed: "
266 "max host memcap reached. Memcap %"PRIu64", "
267 "Memuse %"PRIu64".", SC_ATOMIC_GET(host_config.memcap),
268 ((uint64_t)SC_ATOMIC_GET(host_memuse) + g_host_size));
269 exit(EXIT_FAILURE);
270 }
271
272 Host *h = HostAlloc();
273 if (h == NULL) {
274 SCLogError(SC_ERR_HOST_INIT, "preallocating host failed: %s", strerror(errno));
275 exit(EXIT_FAILURE);
276 }
277 HostEnqueue(&host_spare_q,h);
278 }
279
280 if (quiet == FALSE) {
281 SCLogConfig("preallocated %" PRIu32 " hosts of size %" PRIu16 "",
282 host_spare_q.len, g_host_size);
283 SCLogConfig("host memory usage: %"PRIu64" bytes, maximum: %"PRIu64,
284 SC_ATOMIC_GET(host_memuse), SC_ATOMIC_GET(host_config.memcap));
285 }
286
287 return;
288 }
289
290 /** \brief print some host stats
291 * \warning Not thread safe */
HostPrintStats(void)292 void HostPrintStats (void)
293 {
294 #ifdef HOSTBITS_STATS
295 SCLogPerf("hostbits added: %" PRIu32 ", removed: %" PRIu32 ", max memory usage: %" PRIu32 "",
296 hostbits_added, hostbits_removed, hostbits_memuse_max);
297 #endif /* HOSTBITS_STATS */
298 SCLogPerf("host memory usage: %"PRIu64" bytes, maximum: %"PRIu64,
299 SC_ATOMIC_GET(host_memuse), SC_ATOMIC_GET(host_config.memcap));
300 return;
301 }
302
303 /** \brief shutdown the flow engine
304 * \warning Not thread safe */
HostShutdown(void)305 void HostShutdown(void)
306 {
307 Host *h;
308 uint32_t u;
309
310 HostPrintStats();
311
312 /* free spare queue */
313 while((h = HostDequeue(&host_spare_q))) {
314 BUG_ON(SC_ATOMIC_GET(h->use_cnt) > 0);
315 HostFree(h);
316 }
317
318 /* clear and free the hash */
319 if (host_hash != NULL) {
320 for (u = 0; u < host_config.hash_size; u++) {
321 h = host_hash[u].head;
322 while (h) {
323 Host *n = h->hnext;
324 HostFree(h);
325 h = n;
326 }
327
328 HRLOCK_DESTROY(&host_hash[u]);
329 }
330 SCFreeAligned(host_hash);
331 host_hash = NULL;
332 }
333 (void) SC_ATOMIC_SUB(host_memuse, host_config.hash_size * sizeof(HostHashRow));
334 HostQueueDestroy(&host_spare_q);
335 return;
336 }
337
338 /** \brief Cleanup the host engine
339 *
340 * Cleanup the host engine from tag and threshold.
341 *
342 */
HostCleanup(void)343 void HostCleanup(void)
344 {
345 Host *h;
346 uint32_t u;
347
348 if (host_hash != NULL) {
349 for (u = 0; u < host_config.hash_size; u++) {
350 h = host_hash[u].head;
351 HostHashRow *hb = &host_hash[u];
352 HRLOCK_LOCK(hb);
353 while (h) {
354 if ((SC_ATOMIC_GET(h->use_cnt) > 0) && (h->iprep != NULL)) {
355 /* iprep is attached to host only clear local storage */
356 HostFreeStorage(h);
357 h = h->hnext;
358 } else {
359 Host *n = h->hnext;
360 /* remove from the hash */
361 if (h->hprev != NULL)
362 h->hprev->hnext = h->hnext;
363 if (h->hnext != NULL)
364 h->hnext->hprev = h->hprev;
365 if (hb->head == h)
366 hb->head = h->hnext;
367 if (hb->tail == h)
368 hb->tail = h->hprev;
369 h->hnext = NULL;
370 h->hprev = NULL;
371 HostClearMemory(h);
372 HostMoveToSpare(h);
373 h = n;
374 }
375 }
376 HRLOCK_UNLOCK(hb);
377 }
378 }
379
380 return;
381 }
382
383 /* calculate the hash key for this packet
384 *
385 * we're using:
386 * hash_rand -- set at init time
387 * source address
388 */
HostGetKey(Address * a)389 static inline uint32_t HostGetKey(Address *a)
390 {
391 uint32_t key;
392
393 if (a->family == AF_INET) {
394 uint32_t hash = hashword(&a->addr_data32[0], 1, host_config.hash_rand);
395 key = hash % host_config.hash_size;
396 } else if (a->family == AF_INET6) {
397 uint32_t hash = hashword(a->addr_data32, 4, host_config.hash_rand);
398 key = hash % host_config.hash_size;
399 } else
400 key = 0;
401
402 return key;
403 }
404
HostCompare(Host * h,Address * a)405 static inline int HostCompare(Host *h, Address *a)
406 {
407 if (h->a.family == a->family) {
408 switch (a->family) {
409 case AF_INET:
410 return (h->a.addr_data32[0] == a->addr_data32[0]);
411 case AF_INET6:
412 return CMP_ADDR(&h->a, a);
413 }
414 }
415 return 0;
416 }
417
418 /**
419 * \brief Get a new host
420 *
421 * Get a new host. We're checking memcap first and will try to make room
422 * if the memcap is reached.
423 *
424 * \retval h *LOCKED* host on succes, NULL on error.
425 */
HostGetNew(Address * a)426 static Host *HostGetNew(Address *a)
427 {
428 Host *h = NULL;
429
430 /* get a host from the spare queue */
431 h = HostDequeue(&host_spare_q);
432 if (h == NULL) {
433 /* If we reached the max memcap, we get a used host */
434 if (!(HOST_CHECK_MEMCAP(g_host_size))) {
435 /* declare state of emergency */
436 //if (!(SC_ATOMIC_GET(host_flags) & HOST_EMERGENCY)) {
437 // SC_ATOMIC_OR(host_flags, HOST_EMERGENCY);
438
439 /* under high load, waking up the flow mgr each time leads
440 * to high cpu usage. Flows are not timed out much faster if
441 * we check a 1000 times a second. */
442 // FlowWakeupFlowManagerThread();
443 //}
444
445 h = HostGetUsedHost();
446 if (h == NULL) {
447 return NULL;
448 }
449
450 /* freed a host, but it's unlocked */
451 } else {
452 /* now see if we can alloc a new host */
453 h = HostNew(a);
454 if (h == NULL) {
455 return NULL;
456 }
457
458 /* host is initialized but *unlocked* */
459 }
460 } else {
461 /* host has been recycled before it went into the spare queue */
462
463 /* host is initialized (recylced) but *unlocked* */
464 }
465
466 (void) SC_ATOMIC_ADD(host_counter, 1);
467 SCMutexLock(&h->m);
468 return h;
469 }
470
HostInit(Host * h,Address * a)471 static void HostInit(Host *h, Address *a)
472 {
473 COPY_ADDRESS(a, &h->a);
474 (void) HostIncrUsecnt(h);
475 }
476
HostRelease(Host * h)477 void HostRelease(Host *h)
478 {
479 (void) HostDecrUsecnt(h);
480 SCMutexUnlock(&h->m);
481 }
482
HostLock(Host * h)483 void HostLock(Host *h)
484 {
485 SCMutexLock(&h->m);
486 }
487
HostUnlock(Host * h)488 void HostUnlock(Host *h)
489 {
490 SCMutexUnlock(&h->m);
491 }
492
493
494 /* HostGetHostFromHash
495 *
496 * Hash retrieval function for hosts. Looks up the hash bucket containing the
497 * host pointer. Then compares the packet with the found host to see if it is
498 * the host we need. If it isn't, walk the list until the right host is found.
499 *
500 * returns a *LOCKED* host or NULL
501 */
HostGetHostFromHash(Address * a)502 Host *HostGetHostFromHash (Address *a)
503 {
504 Host *h = NULL;
505
506 /* get the key to our bucket */
507 uint32_t key = HostGetKey(a);
508 /* get our hash bucket and lock it */
509 HostHashRow *hb = &host_hash[key];
510 HRLOCK_LOCK(hb);
511
512 /* see if the bucket already has a host */
513 if (hb->head == NULL) {
514 h = HostGetNew(a);
515 if (h == NULL) {
516 HRLOCK_UNLOCK(hb);
517 return NULL;
518 }
519
520 /* host is locked */
521 hb->head = h;
522 hb->tail = h;
523
524 /* got one, now lock, initialize and return */
525 HostInit(h,a);
526
527 HRLOCK_UNLOCK(hb);
528 return h;
529 }
530
531 /* ok, we have a host in the bucket. Let's find out if it is our host */
532 h = hb->head;
533
534 /* see if this is the host we are looking for */
535 if (HostCompare(h, a) == 0) {
536 Host *ph = NULL; /* previous host */
537
538 while (h) {
539 ph = h;
540 h = h->hnext;
541
542 if (h == NULL) {
543 h = ph->hnext = HostGetNew(a);
544 if (h == NULL) {
545 HRLOCK_UNLOCK(hb);
546 return NULL;
547 }
548 hb->tail = h;
549
550 /* host is locked */
551
552 h->hprev = ph;
553
554 /* initialize and return */
555 HostInit(h,a);
556
557 HRLOCK_UNLOCK(hb);
558 return h;
559 }
560
561 if (HostCompare(h, a) != 0) {
562 /* we found our host, lets put it on top of the
563 * hash list -- this rewards active hosts */
564 if (h->hnext) {
565 h->hnext->hprev = h->hprev;
566 }
567 if (h->hprev) {
568 h->hprev->hnext = h->hnext;
569 }
570 if (h == hb->tail) {
571 hb->tail = h->hprev;
572 }
573
574 h->hnext = hb->head;
575 h->hprev = NULL;
576 hb->head->hprev = h;
577 hb->head = h;
578
579 /* found our host, lock & return */
580 SCMutexLock(&h->m);
581 (void) HostIncrUsecnt(h);
582 HRLOCK_UNLOCK(hb);
583 return h;
584 }
585 }
586 }
587
588 /* lock & return */
589 SCMutexLock(&h->m);
590 (void) HostIncrUsecnt(h);
591 HRLOCK_UNLOCK(hb);
592 return h;
593 }
594
595 /** \brief look up a host in the hash
596 *
597 * \param a address to look up
598 *
599 * \retval h *LOCKED* host or NULL
600 */
HostLookupHostFromHash(Address * a)601 Host *HostLookupHostFromHash (Address *a)
602 {
603 Host *h = NULL;
604
605 /* get the key to our bucket */
606 uint32_t key = HostGetKey(a);
607 /* get our hash bucket and lock it */
608 HostHashRow *hb = &host_hash[key];
609 HRLOCK_LOCK(hb);
610
611 /* see if the bucket already has a host */
612 if (hb->head == NULL) {
613 HRLOCK_UNLOCK(hb);
614 return h;
615 }
616
617 /* ok, we have a host in the bucket. Let's find out if it is our host */
618 h = hb->head;
619
620 /* see if this is the host we are looking for */
621 if (HostCompare(h, a) == 0) {
622 while (h) {
623 h = h->hnext;
624
625 if (h == NULL) {
626 HRLOCK_UNLOCK(hb);
627 return h;
628 }
629
630 if (HostCompare(h, a) != 0) {
631 /* we found our host, lets put it on top of the
632 * hash list -- this rewards active hosts */
633 if (h->hnext) {
634 h->hnext->hprev = h->hprev;
635 }
636 if (h->hprev) {
637 h->hprev->hnext = h->hnext;
638 }
639 if (h == hb->tail) {
640 hb->tail = h->hprev;
641 }
642
643 h->hnext = hb->head;
644 h->hprev = NULL;
645 hb->head->hprev = h;
646 hb->head = h;
647
648 /* found our host, lock & return */
649 SCMutexLock(&h->m);
650 (void) HostIncrUsecnt(h);
651 HRLOCK_UNLOCK(hb);
652 return h;
653 }
654 }
655 }
656
657 /* lock & return */
658 SCMutexLock(&h->m);
659 (void) HostIncrUsecnt(h);
660 HRLOCK_UNLOCK(hb);
661 return h;
662 }
663
664 /** \internal
665 * \brief Get a host from the hash directly.
666 *
667 * Called in conditions where the spare queue is empty and memcap is reached.
668 *
669 * Walks the hash until a host can be freed. "host_prune_idx" atomic int makes
670 * sure we don't start at the top each time since that would clear the top of
671 * the hash leading to longer and longer search times under high pressure (observed).
672 *
673 * \retval h host or NULL
674 */
HostGetUsedHost(void)675 static Host *HostGetUsedHost(void)
676 {
677 uint32_t idx = SC_ATOMIC_GET(host_prune_idx) % host_config.hash_size;
678 uint32_t cnt = host_config.hash_size;
679
680 while (cnt--) {
681 if (++idx >= host_config.hash_size)
682 idx = 0;
683
684 HostHashRow *hb = &host_hash[idx];
685
686 if (HRLOCK_TRYLOCK(hb) != 0)
687 continue;
688
689 Host *h = hb->tail;
690 if (h == NULL) {
691 HRLOCK_UNLOCK(hb);
692 continue;
693 }
694
695 if (SCMutexTrylock(&h->m) != 0) {
696 HRLOCK_UNLOCK(hb);
697 continue;
698 }
699
700 /** never prune a host that is used by a packets
701 * we are currently processing in one of the threads */
702 if (SC_ATOMIC_GET(h->use_cnt) > 0) {
703 HRLOCK_UNLOCK(hb);
704 SCMutexUnlock(&h->m);
705 continue;
706 }
707
708 /* remove from the hash */
709 if (h->hprev != NULL)
710 h->hprev->hnext = h->hnext;
711 if (h->hnext != NULL)
712 h->hnext->hprev = h->hprev;
713 if (hb->head == h)
714 hb->head = h->hnext;
715 if (hb->tail == h)
716 hb->tail = h->hprev;
717
718 h->hnext = NULL;
719 h->hprev = NULL;
720 HRLOCK_UNLOCK(hb);
721
722 HostClearMemory (h);
723
724 SCMutexUnlock(&h->m);
725
726 (void) SC_ATOMIC_ADD(host_prune_idx, (host_config.hash_size - cnt));
727 return h;
728 }
729
730 return NULL;
731 }
732
HostRegisterUnittests(void)733 void HostRegisterUnittests(void)
734 {
735 RegisterHostStorageTests();
736 }
737
738