1 /* Copyright (C) 2015 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  */
23 
24 #include "suricata-common.h"
25 #include "suricata.h"
26 #include "conf.h"
27 #include "debug.h"
28 #include "detect.h"
29 #include "detect-parse.h"
30 
31 #include "runmodes.h"
32 #include "threads.h"
33 #include "threadvars.h"
34 #include "tm-threads.h"
35 #include "queue.h"
36 #include "util-signal.h"
37 
38 #include "detect-engine-loader.h"
39 #include "detect-engine-analyzer.h"
40 #include "detect-engine-mpm.h"
41 #include "detect-engine-sigorder.h"
42 
43 #include "util-detect.h"
44 #include "util-threshold-config.h"
45 
46 #ifdef HAVE_GLOB_H
47 #include <glob.h>
48 #endif
49 
50 extern int rule_reload;
51 extern int engine_analysis;
52 static int fp_engine_analysis_set = 0;
53 int rule_engine_analysis_set = 0;
54 
55 /**
56  *  \brief Create the path if default-rule-path was specified
57  *  \param sig_file The name of the file
58  *  \retval str Pointer to the string path + sig_file
59  */
DetectLoadCompleteSigPath(const DetectEngineCtx * de_ctx,const char * sig_file)60 char *DetectLoadCompleteSigPath(const DetectEngineCtx *de_ctx, const char *sig_file)
61 {
62     const char *defaultpath = NULL;
63     char *path = NULL;
64     char varname[128];
65 
66     if (sig_file == NULL) {
67         SCLogError(SC_ERR_INVALID_ARGUMENTS,"invalid sig_file argument - NULL");
68         return NULL;
69     }
70 
71     if (strlen(de_ctx->config_prefix) > 0) {
72         snprintf(varname, sizeof(varname), "%s.default-rule-path",
73                 de_ctx->config_prefix);
74     } else {
75         snprintf(varname, sizeof(varname), "default-rule-path");
76     }
77 
78     /* Path not specified */
79     if (PathIsRelative(sig_file)) {
80         if (ConfGet(varname, &defaultpath) == 1) {
81             SCLogDebug("Default path: %s", defaultpath);
82             size_t path_len = sizeof(char) * (strlen(defaultpath) +
83                           strlen(sig_file) + 2);
84             path = SCMalloc(path_len);
85             if (unlikely(path == NULL))
86                 return NULL;
87             strlcpy(path, defaultpath, path_len);
88 #if defined OS_WIN32 || defined __CYGWIN__
89             if (path[strlen(path) - 1] != '\\')
90                 strlcat(path, "\\\\", path_len);
91 #else
92             if (path[strlen(path) - 1] != '/')
93                 strlcat(path, "/", path_len);
94 #endif
95             strlcat(path, sig_file, path_len);
96        } else {
97             path = SCStrdup(sig_file);
98             if (unlikely(path == NULL))
99                 return NULL;
100         }
101     } else {
102         path = SCStrdup(sig_file);
103         if (unlikely(path == NULL))
104             return NULL;
105     }
106     return path;
107 }
108 
109 /**
110  *  \brief Load a file with signatures
111  *  \param de_ctx Pointer to the detection engine context
112  *  \param sig_file Filename to load signatures from
113  *  \param goodsigs_tot Will store number of valid signatures in the file
114  *  \param badsigs_tot Will store number of invalid signatures in the file
115  *  \retval 0 on success, -1 on error
116  */
DetectLoadSigFile(DetectEngineCtx * de_ctx,char * sig_file,int * goodsigs,int * badsigs)117 static int DetectLoadSigFile(DetectEngineCtx *de_ctx, char *sig_file,
118         int *goodsigs, int *badsigs)
119 {
120     Signature *sig = NULL;
121     int good = 0, bad = 0;
122     char line[DETECT_MAX_RULE_SIZE] = "";
123     size_t offset = 0;
124     int lineno = 0, multiline = 0;
125 
126     (*goodsigs) = 0;
127     (*badsigs) = 0;
128 
129     FILE *fp = fopen(sig_file, "r");
130     if (fp == NULL) {
131         SCLogError(SC_ERR_OPENING_RULE_FILE, "opening rule file %s:"
132                    " %s.", sig_file, strerror(errno));
133         return -1;
134     }
135 
136     while(fgets(line + offset, (int)sizeof(line) - offset, fp) != NULL) {
137         lineno++;
138         size_t len = strlen(line);
139 
140         /* ignore comments and empty lines */
141         if (line[0] == '\n' || line [0] == '\r' || line[0] == ' ' || line[0] == '#' || line[0] == '\t')
142             continue;
143 
144         /* Check for multiline rules. */
145         while (len > 0 && isspace((unsigned char)line[--len]));
146         if (line[len] == '\\') {
147             multiline++;
148             offset = len;
149             if (offset < sizeof(line) - 1) {
150                 /* We have room for more. */
151                 continue;
152             }
153             /* No more room in line buffer, continue, rule will fail
154              * to parse. */
155         }
156 
157         /* Check if we have a trailing newline, and remove it */
158         len = strlen(line);
159         if (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) {
160             line[len - 1] = '\0';
161         }
162 
163         /* Reset offset. */
164         offset = 0;
165 
166         de_ctx->rule_file = sig_file;
167         de_ctx->rule_line = lineno - multiline;
168 
169         sig = DetectEngineAppendSig(de_ctx, line);
170         if (sig != NULL) {
171             if (rule_engine_analysis_set || fp_engine_analysis_set) {
172                 RetrieveFPForSig(de_ctx, sig);
173                 if (fp_engine_analysis_set) {
174                     EngineAnalysisFP(de_ctx, sig, line);
175                 }
176                 if (rule_engine_analysis_set) {
177                     EngineAnalysisRules(de_ctx, sig, line);
178                 }
179             }
180             SCLogDebug("signature %"PRIu32" loaded", sig->id);
181             good++;
182         } else {
183             if (!de_ctx->sigerror_silent) {
184                 SCLogError(SC_ERR_INVALID_SIGNATURE, "error parsing signature \"%s\" from "
185                         "file %s at line %"PRId32"", line, sig_file, lineno - multiline);
186 
187                 if (!SigStringAppend(&de_ctx->sig_stat, sig_file, line, de_ctx->sigerror, (lineno - multiline))) {
188                     SCLogError(SC_ERR_MEM_ALLOC, "Error adding sig \"%s\" from "
189                             "file %s at line %"PRId32"", line, sig_file, lineno - multiline);
190                 }
191                 if (de_ctx->sigerror) {
192                     de_ctx->sigerror = NULL;
193                 }
194             }
195             if (rule_engine_analysis_set) {
196                 EngineAnalysisRulesFailure(line, sig_file, lineno - multiline);
197             }
198             if (!de_ctx->sigerror_ok) {
199                 bad++;
200             }
201         }
202         multiline = 0;
203     }
204     fclose(fp);
205 
206     *goodsigs = good;
207     *badsigs = bad;
208     return 0;
209 }
210 
211 /**
212  *  \brief Expands wildcards and reads signatures from each matching file
213  *  \param de_ctx Pointer to the detection engine context
214  *  \param sig_file Filename (or pattern) holding signatures
215  *  \retval -1 on error
216  */
ProcessSigFiles(DetectEngineCtx * de_ctx,char * pattern,SigFileLoaderStat * st,int * good_sigs,int * bad_sigs)217 static int ProcessSigFiles(DetectEngineCtx *de_ctx, char *pattern,
218         SigFileLoaderStat *st, int *good_sigs, int *bad_sigs)
219 {
220     int r = 0;
221 
222     if (pattern == NULL) {
223         SCLogError(SC_ERR_INVALID_ARGUMENT, "opening rule file null");
224         return -1;
225     }
226 
227 #ifdef HAVE_GLOB_H
228     glob_t files;
229     r = glob(pattern, 0, NULL, &files);
230 
231     if (r == GLOB_NOMATCH) {
232         SCLogWarning(SC_ERR_NO_RULES, "No rule files match the pattern %s", pattern);
233         ++(st->bad_files);
234         ++(st->total_files);
235         return -1;
236     } else if (r != 0) {
237         SCLogError(SC_ERR_OPENING_RULE_FILE, "error expanding template %s: %s",
238                  pattern, strerror(errno));
239         return -1;
240     }
241 
242     for (size_t i = 0; i < (size_t)files.gl_pathc; i++) {
243         char *fname = files.gl_pathv[i];
244         if (strcmp("/dev/null", fname) == 0)
245             continue;
246 #else
247         char *fname = pattern;
248         if (strcmp("/dev/null", fname) == 0)
249             return 0;
250 #endif
251         SCLogConfig("Loading rule file: %s", fname);
252         r = DetectLoadSigFile(de_ctx, fname, good_sigs, bad_sigs);
253         if (r < 0) {
254             ++(st->bad_files);
255         }
256 
257         ++(st->total_files);
258 
259         st->good_sigs_total += *good_sigs;
260         st->bad_sigs_total += *bad_sigs;
261 
262 #ifdef HAVE_GLOB_H
263     }
264     globfree(&files);
265 #endif
266     return r;
267 }
268 
269 /**
270  *  \brief Load signatures
271  *  \param de_ctx Pointer to the detection engine context
272  *  \param sig_file Filename (or pattern) holding signatures
273  *  \param sig_file_exclusive File passed in 'sig_file' should be loaded exclusively.
274  *  \retval -1 on error
275  */
SigLoadSignatures(DetectEngineCtx * de_ctx,char * sig_file,int sig_file_exclusive)276 int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file, int sig_file_exclusive)
277 {
278     SCEnter();
279 
280     ConfNode *rule_files;
281     ConfNode *file = NULL;
282     SigFileLoaderStat *sig_stat = &de_ctx->sig_stat;
283     int ret = 0;
284     char *sfile = NULL;
285     char varname[128] = "rule-files";
286     int good_sigs = 0;
287     int bad_sigs = 0;
288 
289     if (strlen(de_ctx->config_prefix) > 0) {
290         snprintf(varname, sizeof(varname), "%s.rule-files",
291                 de_ctx->config_prefix);
292     }
293 
294     if (RunmodeGetCurrent() == RUNMODE_ENGINE_ANALYSIS) {
295         fp_engine_analysis_set = SetupFPAnalyzer();
296         rule_engine_analysis_set = SetupRuleAnalyzer();
297     }
298 
299     /* ok, let's load signature files from the general config */
300     if (!(sig_file != NULL && sig_file_exclusive == TRUE)) {
301         rule_files = ConfGetNode(varname);
302         if (rule_files != NULL) {
303             if (!ConfNodeIsSequence(rule_files)) {
304                 SCLogWarning(SC_ERR_INVALID_ARGUMENT,
305                     "Invalid rule-files configuration section: "
306                     "expected a list of filenames.");
307             }
308             else {
309                 TAILQ_FOREACH(file, &rule_files->head, next) {
310                     sfile = DetectLoadCompleteSigPath(de_ctx, file->val);
311                     good_sigs = bad_sigs = 0;
312                     ret = ProcessSigFiles(de_ctx, sfile, sig_stat, &good_sigs, &bad_sigs);
313                     SCFree(sfile);
314 
315                     if (de_ctx->failure_fatal && ret != 0) {
316                         /* Some rules failed to load, just exit as
317                          * errors would have already been logged. */
318                         exit(EXIT_FAILURE);
319                     }
320 
321                     if (good_sigs == 0) {
322                         SCLogConfig("No rules loaded from %s.", file->val);
323                     }
324                 }
325             }
326         }
327     }
328 
329     /* If a Signature file is specified from commandline, parse it too */
330     if (sig_file != NULL) {
331         ret = ProcessSigFiles(de_ctx, sig_file, sig_stat, &good_sigs, &bad_sigs);
332 
333         if (ret != 0) {
334             if (de_ctx->failure_fatal == 1) {
335                 exit(EXIT_FAILURE);
336             }
337         }
338 
339         if (good_sigs == 0) {
340             SCLogConfig("No rules loaded from %s", sig_file);
341         }
342     }
343 
344     /* now we should have signatures to work with */
345     if (sig_stat->good_sigs_total <= 0) {
346         if (sig_stat->total_files > 0) {
347            SCLogWarning(SC_ERR_NO_RULES_LOADED, "%d rule files specified, but no rules were loaded!", sig_stat->total_files);
348         } else {
349             SCLogInfo("No signatures supplied.");
350             goto end;
351         }
352     } else {
353         /* we report the total of files and rules successfully loaded and failed */
354         SCLogInfo("%" PRId32 " rule files processed. %" PRId32 " rules successfully loaded, %" PRId32 " rules failed",
355             sig_stat->total_files, sig_stat->good_sigs_total, sig_stat->bad_sigs_total);
356     }
357 
358     if ((sig_stat->bad_sigs_total || sig_stat->bad_files) && de_ctx->failure_fatal) {
359         ret = -1;
360         goto end;
361     }
362 
363     SCSigRegisterSignatureOrderingFuncs(de_ctx);
364     SCSigOrderSignatures(de_ctx);
365     SCSigSignatureOrderingModuleCleanup(de_ctx);
366 
367     SCThresholdConfInitContext(de_ctx);
368 
369     /* Setup the signature group lookup structure and pattern matchers */
370     if (SigGroupBuild(de_ctx) < 0)
371         goto end;
372 
373     ret = 0;
374 
375  end:
376     gettimeofday(&de_ctx->last_reload, NULL);
377     if (RunmodeGetCurrent() == RUNMODE_ENGINE_ANALYSIS) {
378         if (rule_engine_analysis_set) {
379             CleanupRuleAnalyzer();
380         }
381         if (fp_engine_analysis_set) {
382             CleanupFPAnalyzer();
383         }
384     }
385 
386     DetectParseDupSigHashFree(de_ctx);
387     SCReturnInt(ret);
388 }
389 
390 #define NLOADERS 4
391 static DetectLoaderControl *loaders = NULL;
392 static int cur_loader = 0;
393 void TmThreadWakeupDetectLoaderThreads(void);
394 static int num_loaders = NLOADERS;
395 
396 /** \param loader -1 for auto select
397  *  \retval loader_id or negative in case of error */
DetectLoaderQueueTask(int loader_id,LoaderFunc Func,void * func_ctx)398 int DetectLoaderQueueTask(int loader_id, LoaderFunc Func, void *func_ctx)
399 {
400     if (loader_id == -1) {
401         loader_id = cur_loader;
402         cur_loader++;
403         if (cur_loader >= num_loaders)
404             cur_loader = 0;
405     }
406     if (loader_id >= num_loaders || loader_id < 0) {
407         return -ERANGE;
408     }
409 
410     DetectLoaderControl *loader = &loaders[loader_id];
411 
412     DetectLoaderTask *t = SCCalloc(1, sizeof(*t));
413     if (t == NULL)
414         return -ENOMEM;
415 
416     t->Func = Func;
417     t->ctx = func_ctx;
418 
419     SCMutexLock(&loader->m);
420     TAILQ_INSERT_TAIL(&loader->task_list, t, next);
421     SCMutexUnlock(&loader->m);
422 
423     TmThreadWakeupDetectLoaderThreads();
424 
425     SCLogDebug("%d %p %p", loader_id, Func, func_ctx);
426     return loader_id;
427 }
428 
429 /** \brief wait for loader tasks to complete
430  *  \retval result 0 for ok, -1 for errors */
DetectLoadersSync(void)431 int DetectLoadersSync(void)
432 {
433     SCLogDebug("waiting");
434     int errors = 0;
435     int i;
436     for (i = 0; i < num_loaders; i++) {
437         int done = 0;
438         DetectLoaderControl *loader = &loaders[i];
439         while (!done) {
440             SCMutexLock(&loader->m);
441             if (TAILQ_EMPTY(&loader->task_list)) {
442                 done = 1;
443             }
444             SCMutexUnlock(&loader->m);
445         }
446         SCMutexLock(&loader->m);
447         if (loader->result != 0) {
448             errors++;
449             loader->result = 0;
450         }
451         SCMutexUnlock(&loader->m);
452 
453     }
454     if (errors) {
455         SCLogError(SC_ERR_INITIALIZATION, "%d loaders reported errors", errors);
456         return -1;
457     }
458     SCLogDebug("done");
459     return 0;
460 }
461 
DetectLoaderInit(DetectLoaderControl * loader)462 static void DetectLoaderInit(DetectLoaderControl *loader)
463 {
464     memset(loader, 0x00, sizeof(*loader));
465     SCMutexInit(&loader->m, NULL);
466     TAILQ_INIT(&loader->task_list);
467 }
468 
DetectLoadersInit(void)469 void DetectLoadersInit(void)
470 {
471     intmax_t setting = NLOADERS;
472     (void)ConfGetInt("multi-detect.loaders", &setting);
473 
474     if (setting < 1 || setting > 1024) {
475         SCLogError(SC_ERR_INVALID_ARGUMENTS,
476                 "invalid multi-detect.loaders setting %"PRIdMAX, setting);
477         exit(EXIT_FAILURE);
478     }
479     num_loaders = (int32_t)setting;
480 
481     SCLogInfo("using %d detect loader threads", num_loaders);
482 
483     BUG_ON(loaders != NULL);
484     loaders = SCCalloc(num_loaders, sizeof(DetectLoaderControl));
485     BUG_ON(loaders == NULL);
486 
487     int i;
488     for (i = 0; i < num_loaders; i++) {
489         DetectLoaderInit(&loaders[i]);
490     }
491 }
492 
493 /**
494  * \brief Unpauses all threads present in tv_root
495  */
TmThreadWakeupDetectLoaderThreads()496 void TmThreadWakeupDetectLoaderThreads()
497 {
498     ThreadVars *tv = NULL;
499     int i = 0;
500 
501     SCMutexLock(&tv_root_lock);
502     for (i = 0; i < TVT_MAX; i++) {
503         tv = tv_root[i];
504         while (tv != NULL) {
505             if (strncmp(tv->name,"DL#",3) == 0) {
506                 BUG_ON(tv->ctrl_cond == NULL);
507                 pthread_cond_broadcast(tv->ctrl_cond);
508             }
509             tv = tv->next;
510         }
511     }
512     SCMutexUnlock(&tv_root_lock);
513 
514     return;
515 }
516 
517 /**
518  * \brief Unpauses all threads present in tv_root
519  */
TmThreadContinueDetectLoaderThreads()520 void TmThreadContinueDetectLoaderThreads()
521 {
522     ThreadVars *tv = NULL;
523     int i = 0;
524 
525     SCMutexLock(&tv_root_lock);
526     for (i = 0; i < TVT_MAX; i++) {
527         tv = tv_root[i];
528         while (tv != NULL) {
529             if (strncmp(tv->name,"DL#",3) == 0)
530                 TmThreadContinue(tv);
531 
532             tv = tv->next;
533         }
534     }
535     SCMutexUnlock(&tv_root_lock);
536 
537     return;
538 }
539 
540 
541 SC_ATOMIC_DECLARE(int, detect_loader_cnt);
542 
543 typedef struct DetectLoaderThreadData_ {
544     uint32_t instance;
545 } DetectLoaderThreadData;
546 
DetectLoaderThreadInit(ThreadVars * t,const void * initdata,void ** data)547 static TmEcode DetectLoaderThreadInit(ThreadVars *t, const void *initdata, void **data)
548 {
549     DetectLoaderThreadData *ftd = SCCalloc(1, sizeof(DetectLoaderThreadData));
550     if (ftd == NULL)
551         return TM_ECODE_FAILED;
552 
553     ftd->instance = SC_ATOMIC_ADD(detect_loader_cnt, 1); /* id's start at 0 */
554     SCLogDebug("detect loader instance %u", ftd->instance);
555 
556     /* pass thread data back to caller */
557     *data = ftd;
558 
559     return TM_ECODE_OK;
560 }
561 
DetectLoaderThreadDeinit(ThreadVars * t,void * data)562 static TmEcode DetectLoaderThreadDeinit(ThreadVars *t, void *data)
563 {
564     SCFree(data);
565     return TM_ECODE_OK;
566 }
567 
568 
DetectLoader(ThreadVars * th_v,void * thread_data)569 static TmEcode DetectLoader(ThreadVars *th_v, void *thread_data)
570 {
571     DetectLoaderThreadData *ftd = (DetectLoaderThreadData *)thread_data;
572     BUG_ON(ftd == NULL);
573 
574     SCLogDebug("loader thread started");
575     while (1)
576     {
577         if (TmThreadsCheckFlag(th_v, THV_PAUSE)) {
578             TmThreadsSetFlag(th_v, THV_PAUSED);
579             TmThreadTestThreadUnPaused(th_v);
580             TmThreadsUnsetFlag(th_v, THV_PAUSED);
581         }
582 
583         /* see if we have tasks */
584 
585         DetectLoaderControl *loader = &loaders[ftd->instance];
586         SCMutexLock(&loader->m);
587 
588         DetectLoaderTask *task = NULL, *tmptask = NULL;
589         TAILQ_FOREACH_SAFE(task, &loader->task_list, next, tmptask) {
590             int r = task->Func(task->ctx, ftd->instance);
591             loader->result |= r;
592             TAILQ_REMOVE(&loader->task_list, task, next);
593             SCFree(task->ctx);
594             SCFree(task);
595         }
596 
597         SCMutexUnlock(&loader->m);
598 
599         if (TmThreadsCheckFlag(th_v, THV_KILL)) {
600             break;
601         }
602 
603         /* just wait until someone wakes us up */
604         SCCtrlMutexLock(th_v->ctrl_mutex);
605         SCCtrlCondWait(th_v->ctrl_cond, th_v->ctrl_mutex);
606         SCCtrlMutexUnlock(th_v->ctrl_mutex);
607 
608         SCLogDebug("woke up...");
609     }
610 
611     return TM_ECODE_OK;
612 }
613 
614 /** \brief spawn the detect loader manager thread */
DetectLoaderThreadSpawn(void)615 void DetectLoaderThreadSpawn(void)
616 {
617     int i;
618     for (i = 0; i < num_loaders; i++) {
619         ThreadVars *tv_loader = NULL;
620 
621         char name[TM_THREAD_NAME_MAX];
622         snprintf(name, sizeof(name), "%s#%02d", thread_name_detect_loader, i+1);
623 
624         tv_loader = TmThreadCreateCmdThreadByName(name,
625                 "DetectLoader", 1);
626         BUG_ON(tv_loader == NULL);
627 
628         if (tv_loader == NULL) {
629             printf("ERROR: TmThreadsCreate failed\n");
630             exit(1);
631         }
632         if (TmThreadSpawn(tv_loader) != TM_ECODE_OK) {
633             printf("ERROR: TmThreadSpawn failed\n");
634             exit(1);
635         }
636     }
637     return;
638 }
639 
TmModuleDetectLoaderRegister(void)640 void TmModuleDetectLoaderRegister (void)
641 {
642     tmm_modules[TMM_DETECTLOADER].name = "DetectLoader";
643     tmm_modules[TMM_DETECTLOADER].ThreadInit = DetectLoaderThreadInit;
644     tmm_modules[TMM_DETECTLOADER].ThreadDeinit = DetectLoaderThreadDeinit;
645     tmm_modules[TMM_DETECTLOADER].Management = DetectLoader;
646     tmm_modules[TMM_DETECTLOADER].cap_flags = 0;
647     tmm_modules[TMM_DETECTLOADER].flags = TM_FLAG_MANAGEMENT_TM;
648     SCLogDebug("%s registered", tmm_modules[TMM_DETECTLOADER].name);
649 
650     SC_ATOMIC_INIT(detect_loader_cnt);
651 }
652