1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2 *
3 * You can copy, redistribute or modify this Program under the terms of
4 * the GNU General Public License version 2 as published by the Free
5 * Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 * You should have received a copy of the GNU General Public License
13 * version 2 along with this program; if not, write to the Free Software
14 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15 * 02110-1301, USA.
16 */
17
18 #include "suricata-common.h"
19 #include "detect.h"
20 #include "detect-engine.h"
21 #include "detect-parse.h"
22
23 #include "detect-engine-address.h"
24 #include "detect-engine-analyzer.h"
25 #include "detect-engine-iponly.h"
26 #include "detect-engine-mpm.h"
27 #include "detect-engine-siggroup.h"
28 #include "detect-engine-port.h"
29 #include "detect-engine-prefilter.h"
30 #include "detect-engine-proto.h"
31 #include "detect-engine-threshold.h"
32
33 #include "detect-dsize.h"
34 #include "detect-tcp-flags.h"
35 #include "detect-flow.h"
36 #include "detect-flowbits.h"
37
38 #include "util-profiling.h"
39
SigCleanSignatures(DetectEngineCtx * de_ctx)40 void SigCleanSignatures(DetectEngineCtx *de_ctx)
41 {
42 if (de_ctx == NULL)
43 return;
44
45 for (Signature *s = de_ctx->sig_list; s != NULL;) {
46 Signature *ns = s->next;
47 SigFree(de_ctx, s);
48 s = ns;
49 }
50 de_ctx->sig_list = NULL;
51
52 DetectEngineResetMaxSigId(de_ctx);
53 de_ctx->sig_list = NULL;
54 }
55
56 /** \brief Find a specific signature by sid and gid
57 * \param de_ctx detection engine ctx
58 * \param sid the signature id
59 * \param gid the signature group id
60 *
61 * \retval s sig found
62 * \retval NULL sig not found
63 */
SigFindSignatureBySidGid(DetectEngineCtx * de_ctx,uint32_t sid,uint32_t gid)64 Signature *SigFindSignatureBySidGid(DetectEngineCtx *de_ctx, uint32_t sid, uint32_t gid)
65 {
66 if (de_ctx == NULL)
67 return NULL;
68
69 for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
70 if (s->id == sid && s->gid == gid)
71 return s;
72 }
73
74 return NULL;
75 }
76
77 /**
78 * \brief Check if a signature contains the filestore keyword.
79 *
80 * \param s signature
81 *
82 * \retval 0 no
83 * \retval 1 yes
84 */
SignatureIsFilestoring(const Signature * s)85 int SignatureIsFilestoring(const Signature *s)
86 {
87 if (s == NULL)
88 return 0;
89
90 if (s->flags & SIG_FLAG_FILESTORE)
91 return 1;
92
93 return 0;
94 }
95
96 /**
97 * \brief Check if a signature contains the filemagic keyword.
98 *
99 * \param s signature
100 *
101 * \retval 0 no
102 * \retval 1 yes
103 */
SignatureIsFilemagicInspecting(const Signature * s)104 int SignatureIsFilemagicInspecting(const Signature *s)
105 {
106 if (s == NULL)
107 return 0;
108
109 if (s->file_flags & FILE_SIG_NEED_MAGIC)
110 return 1;
111
112 return 0;
113 }
114
115 /**
116 * \brief Check if a signature contains the filemd5 keyword.
117 *
118 * \param s signature
119 *
120 * \retval 0 no
121 * \retval 1 yes
122 */
SignatureIsFileMd5Inspecting(const Signature * s)123 int SignatureIsFileMd5Inspecting(const Signature *s)
124 {
125 if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_MD5))
126 return 1;
127
128 return 0;
129 }
130
131 /**
132 * \brief Check if a signature contains the filesha1 keyword.
133 *
134 * \param s signature
135 *
136 * \retval 0 no
137 * \retval 1 yes
138 */
SignatureIsFileSha1Inspecting(const Signature * s)139 int SignatureIsFileSha1Inspecting(const Signature *s)
140 {
141 if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA1))
142 return 1;
143
144 return 0;
145 }
146
147 /**
148 * \brief Check if a signature contains the filesha256 keyword.
149 *
150 * \param s signature
151 *
152 * \retval 0 no
153 * \retval 1 yes
154 */
SignatureIsFileSha256Inspecting(const Signature * s)155 int SignatureIsFileSha256Inspecting(const Signature *s)
156 {
157 if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA256))
158 return 1;
159
160 return 0;
161 }
162
163 /**
164 * \brief Check if a signature contains the filesize keyword.
165 *
166 * \param s signature
167 *
168 * \retval 0 no
169 * \retval 1 yes
170 */
SignatureIsFilesizeInspecting(const Signature * s)171 int SignatureIsFilesizeInspecting(const Signature *s)
172 {
173 if (s == NULL)
174 return 0;
175
176 if (s->file_flags & FILE_SIG_NEED_SIZE)
177 return 1;
178
179 return 0;
180 }
181
182 /** \brief Test is a initialized signature is IP only
183 * \param de_ctx detection engine ctx
184 * \param s the signature
185 * \retval 1 sig is ip only
186 * \retval 0 sig is not ip only
187 */
SignatureIsIPOnly(DetectEngineCtx * de_ctx,const Signature * s)188 int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s)
189 {
190 if (s->alproto != ALPROTO_UNKNOWN)
191 return 0;
192
193 if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
194 return 0;
195
196 /* if flow dir is set we can't process it in ip-only */
197 if (!(((s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) == 0) ||
198 (s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) ==
199 (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)))
200 return 0;
201
202 /* for now assume that all registered buffer types are incompatible */
203 const int nlists = s->init_data->smlists_array_size;
204 for (int i = 0; i < nlists; i++) {
205 if (s->init_data->smlists[i] == NULL)
206 continue;
207 if (!(DetectBufferTypeGetNameById(de_ctx, i)))
208 continue;
209
210 SCReturnInt(0);
211 }
212
213 /* TMATCH list can be ignored, it contains TAGs and
214 * tags are compatible to IP-only. */
215
216 /* if any of the addresses uses negation, we don't support
217 * it in ip-only */
218 if (s->init_data->src_contains_negation)
219 return 0;
220 if (s->init_data->dst_contains_negation)
221 return 0;
222
223 SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH];
224 if (sm == NULL)
225 goto iponly;
226
227 for ( ; sm != NULL; sm = sm->next) {
228 if ( !(sigmatch_table[sm->type].flags & SIGMATCH_IPONLY_COMPAT))
229 return 0;
230 /* we have enabled flowbits to be compatible with ip only sigs, as long
231 * as the sig only has a "set" flowbits */
232 if (sm->type == DETECT_FLOWBITS &&
233 (((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) {
234 return 0;
235 }
236 }
237
238 iponly:
239 if (!(de_ctx->flags & DE_QUIET)) {
240 SCLogDebug("IP-ONLY (%" PRIu32 "): source %s, dest %s", s->id,
241 s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET",
242 s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET");
243 }
244 return 1;
245 }
246
247 /** \internal
248 * \brief Test is a initialized signature is inspecting protocol detection only
249 * \param de_ctx detection engine ctx
250 * \param s the signature
251 * \retval 1 sig is dp only
252 * \retval 0 sig is not dp only
253 */
SignatureIsPDOnly(const DetectEngineCtx * de_ctx,const Signature * s)254 static int SignatureIsPDOnly(const DetectEngineCtx *de_ctx, const Signature *s)
255 {
256 if (s->alproto != ALPROTO_UNKNOWN)
257 return 0;
258
259 if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
260 return 0;
261
262 /* for now assume that all registered buffer types are incompatible */
263 const int nlists = s->init_data->smlists_array_size;
264 for (int i = 0; i < nlists; i++) {
265 if (s->init_data->smlists[i] == NULL)
266 continue;
267 if (!(DetectBufferTypeGetNameById(de_ctx, i)))
268 continue;
269
270 SCReturnInt(0);
271 }
272
273 /* TMATCH list can be ignored, it contains TAGs and
274 * tags are compatible to DP-only. */
275
276 /* match list matches may be compatible to DP only. We follow the same
277 * logic as IP-only so we can use that flag */
278
279 SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH];
280 if (sm == NULL)
281 return 0;
282
283 int pd = 0;
284 for ( ; sm != NULL; sm = sm->next) {
285 if (sm->type == DETECT_AL_APP_LAYER_PROTOCOL) {
286 pd = 1;
287 } else {
288 /* flowbits are supported for dp only sigs, as long
289 * as the sig only has a "set" flowbits */
290 if (sm->type == DETECT_FLOWBITS) {
291 if ((((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) {
292 SCLogDebug("%u: not PD-only: flowbit settings other than 'set'", s->id);
293 return 0;
294 }
295 } else if (sm->type == DETECT_FLOW) {
296 if (((DetectFlowData *)sm->ctx)->flags & ~(DETECT_FLOW_FLAG_TOSERVER|DETECT_FLOW_FLAG_TOCLIENT)) {
297 SCLogDebug("%u: not PD-only: flow settings other than toserver/toclient", s->id);
298 return 0;
299 }
300 } else if ( !(sigmatch_table[sm->type].flags & SIGMATCH_IPONLY_COMPAT)) {
301 SCLogDebug("%u: not PD-only: %s not PD/IP-only compat", s->id, sigmatch_table[sm->type].name);
302 return 0;
303 }
304 }
305 }
306
307 if (pd) {
308 SCLogDebug("PD-ONLY (%" PRIu32 ")", s->id);
309 }
310 return pd;
311 }
312
313 /**
314 * \internal
315 * \brief Check if the initialized signature is inspecting the packet payload
316 * \param de_ctx detection engine ctx
317 * \param s the signature
318 * \retval 1 sig is inspecting the payload
319 * \retval 0 sig is not inspecting the payload
320 */
SignatureIsInspectingPayload(DetectEngineCtx * de_ctx,const Signature * s)321 static int SignatureIsInspectingPayload(DetectEngineCtx *de_ctx, const Signature *s)
322 {
323
324 if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) {
325 return 1;
326 }
327 return 0;
328 }
329
330 /**
331 * \internal
332 * \brief check if a signature is decoder event matching only
333 * \param de_ctx detection engine
334 * \param s the signature to test
335 * \retval 0 not a DEOnly sig
336 * \retval 1 DEOnly sig
337 */
SignatureIsDEOnly(DetectEngineCtx * de_ctx,const Signature * s)338 static int SignatureIsDEOnly(DetectEngineCtx *de_ctx, const Signature *s)
339 {
340 if (s->alproto != ALPROTO_UNKNOWN) {
341 SCReturnInt(0);
342 }
343
344 if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
345 {
346 SCReturnInt(0);
347 }
348
349 /* for now assume that all registered buffer types are incompatible */
350 const int nlists = s->init_data->smlists_array_size;
351 for (int i = 0; i < nlists; i++) {
352 if (s->init_data->smlists[i] == NULL)
353 continue;
354 if (!(DetectBufferTypeGetNameById(de_ctx, i)))
355 continue;
356
357 SCReturnInt(0);
358 }
359
360 /* check for conflicting keywords */
361 SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH];
362 for ( ;sm != NULL; sm = sm->next) {
363 if ( !(sigmatch_table[sm->type].flags & SIGMATCH_DEONLY_COMPAT))
364 SCReturnInt(0);
365 }
366
367 /* need at least one decode event keyword to be considered decode event. */
368 sm = s->init_data->smlists[DETECT_SM_LIST_MATCH];
369 for ( ;sm != NULL; sm = sm->next) {
370 if (sm->type == DETECT_DECODE_EVENT)
371 goto deonly;
372 if (sm->type == DETECT_ENGINE_EVENT)
373 goto deonly;
374 if (sm->type == DETECT_STREAM_EVENT)
375 goto deonly;
376 }
377
378 SCReturnInt(0);
379
380 deonly:
381 if (!(de_ctx->flags & DE_QUIET)) {
382 SCLogDebug("DE-ONLY (%" PRIu32 "): source %s, dest %s", s->id,
383 s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET",
384 s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET");
385 }
386
387 SCReturnInt(1);
388 }
389
390 #define MASK_TCP_INITDEINIT_FLAGS (TH_SYN|TH_RST|TH_FIN)
391 #define MASK_TCP_UNUSUAL_FLAGS (TH_URG|TH_ECN|TH_CWR)
392
393 /* Create mask for this packet + it's flow if it has one
394 */
395 void
PacketCreateMask(Packet * p,SignatureMask * mask,AppProto alproto,bool app_decoder_events)396 PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto,
397 bool app_decoder_events)
398 {
399 if (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && p->payload_len > 0) {
400 SCLogDebug("packet has payload");
401 (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
402 } else if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
403 SCLogDebug("stream data available");
404 (*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
405 } else {
406 SCLogDebug("packet has no payload");
407 (*mask) |= SIG_MASK_REQUIRE_NO_PAYLOAD;
408 }
409
410 if (p->events.cnt > 0 || app_decoder_events != 0 || p->app_layer_events != NULL) {
411 SCLogDebug("packet/flow has events set");
412 (*mask) |= SIG_MASK_REQUIRE_ENGINE_EVENT;
413 }
414
415 if (!(PKT_IS_PSEUDOPKT(p)) && PKT_IS_TCP(p)) {
416 if ((p->tcph->th_flags & MASK_TCP_INITDEINIT_FLAGS) != 0) {
417 (*mask) |= SIG_MASK_REQUIRE_FLAGS_INITDEINIT;
418 }
419 if ((p->tcph->th_flags & MASK_TCP_UNUSUAL_FLAGS) != 0) {
420 (*mask) |= SIG_MASK_REQUIRE_FLAGS_UNUSUAL;
421 }
422 }
423
424 if (p->flags & PKT_HAS_FLOW) {
425 SCLogDebug("packet has flow");
426 (*mask) |= SIG_MASK_REQUIRE_FLOW;
427 }
428
429 if (alproto == ALPROTO_SMB || alproto == ALPROTO_DCERPC) {
430 SCLogDebug("packet will be inspected for DCERPC");
431 (*mask) |= SIG_MASK_REQUIRE_DCERPC;
432 }
433 }
434
435 static int g_dce_generic_list_id = -1;
436 static int g_dce_stub_data_buffer_id = -1;
437
SignatureNeedsDCERPCMask(const Signature * s)438 static bool SignatureNeedsDCERPCMask(const Signature *s)
439 {
440 if (g_dce_generic_list_id == -1) {
441 g_dce_generic_list_id = DetectBufferTypeGetByName("dce_generic");
442 SCLogDebug("g_dce_generic_list_id %d", g_dce_generic_list_id);
443 }
444 if (g_dce_stub_data_buffer_id == -1) {
445 g_dce_stub_data_buffer_id = DetectBufferTypeGetByName("dce_stub_data");
446 SCLogDebug("g_dce_stub_data_buffer_id %d", g_dce_stub_data_buffer_id);
447 }
448
449 if (g_dce_generic_list_id >= 0 &&
450 s->init_data->smlists[g_dce_generic_list_id] != NULL)
451 {
452 return true;
453 }
454 if (g_dce_stub_data_buffer_id >= 0 &&
455 s->init_data->smlists[g_dce_stub_data_buffer_id] != NULL)
456 {
457 return true;
458 }
459 return false;
460 }
461
SignatureCreateMask(Signature * s)462 static int SignatureCreateMask(Signature *s)
463 {
464 SCEnter();
465
466 if (SignatureNeedsDCERPCMask(s)) {
467 s->mask |= SIG_MASK_REQUIRE_DCERPC;
468 SCLogDebug("sig requires DCERPC");
469 }
470
471 if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) {
472 s->mask |= SIG_MASK_REQUIRE_PAYLOAD;
473 SCLogDebug("sig requires payload");
474 }
475
476 SigMatch *sm;
477 for (sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) {
478 switch(sm->type) {
479 case DETECT_FLOWBITS:
480 {
481 /* figure out what flowbit action */
482 DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx;
483 if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) {
484 /* not a mask flag, but still set it here */
485 s->flags |= SIG_FLAG_REQUIRE_FLOWVAR;
486
487 SCLogDebug("SIG_FLAG_REQUIRE_FLOWVAR set as sig has "
488 "flowbit isset option.");
489 }
490
491 /* flow is required for any flowbit manipulation */
492 s->mask |= SIG_MASK_REQUIRE_FLOW;
493 SCLogDebug("sig requires flow to be able to manipulate "
494 "flowbit(s)");
495 break;
496 }
497 case DETECT_FLOWINT:
498 /* flow is required for any flowint manipulation */
499 s->mask |= SIG_MASK_REQUIRE_FLOW;
500 SCLogDebug("sig requires flow to be able to manipulate "
501 "flowint(s)");
502 break;
503 case DETECT_FLAGS:
504 {
505 DetectFlagsData *fl = (DetectFlagsData *)sm->ctx;
506
507 if (fl->flags & TH_SYN) {
508 s->mask |= SIG_MASK_REQUIRE_FLAGS_INITDEINIT;
509 SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
510 }
511 if (fl->flags & TH_RST) {
512 s->mask |= SIG_MASK_REQUIRE_FLAGS_INITDEINIT;
513 SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
514 }
515 if (fl->flags & TH_FIN) {
516 s->mask |= SIG_MASK_REQUIRE_FLAGS_INITDEINIT;
517 SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT");
518 }
519 if (fl->flags & TH_URG) {
520 s->mask |= SIG_MASK_REQUIRE_FLAGS_UNUSUAL;
521 SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
522 }
523 if (fl->flags & TH_ECN) {
524 s->mask |= SIG_MASK_REQUIRE_FLAGS_UNUSUAL;
525 SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
526 }
527 if (fl->flags & TH_CWR) {
528 s->mask |= SIG_MASK_REQUIRE_FLAGS_UNUSUAL;
529 SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL");
530 }
531 break;
532 }
533 case DETECT_DSIZE:
534 {
535 DetectDsizeData *ds = (DetectDsizeData *)sm->ctx;
536 switch (ds->mode) {
537 case DETECTDSIZE_LT:
538 /* LT will include 0, so no payload.
539 * if GT is used in the same rule the
540 * flag will be set anyway. */
541 break;
542 case DETECTDSIZE_RA:
543 case DETECTDSIZE_GT:
544 s->mask |= SIG_MASK_REQUIRE_PAYLOAD;
545 SCLogDebug("sig requires payload");
546 break;
547 case DETECTDSIZE_EQ:
548 if (ds->dsize > 0) {
549 s->mask |= SIG_MASK_REQUIRE_PAYLOAD;
550 SCLogDebug("sig requires payload");
551 } else if (ds->dsize == 0) {
552 s->mask |= SIG_MASK_REQUIRE_NO_PAYLOAD;
553 SCLogDebug("sig requires no payload");
554 }
555 break;
556 }
557 break;
558 }
559 case DETECT_AL_APP_LAYER_EVENT:
560 s->mask |= SIG_MASK_REQUIRE_ENGINE_EVENT;
561 break;
562 case DETECT_ENGINE_EVENT:
563 s->mask |= SIG_MASK_REQUIRE_ENGINE_EVENT;
564 break;
565 }
566 }
567
568 if (s->init_data->init_flags & SIG_FLAG_INIT_FLOW) {
569 s->mask |= SIG_MASK_REQUIRE_FLOW;
570 SCLogDebug("sig requires flow");
571 }
572
573 if (s->flags & SIG_FLAG_APPLAYER) {
574 s->mask |= SIG_MASK_REQUIRE_FLOW;
575 SCLogDebug("sig requires flow");
576 }
577
578 SCLogDebug("mask %02X", s->mask);
579 SCReturnInt(0);
580 }
581
SigInitStandardMpmFactoryContexts(DetectEngineCtx * de_ctx)582 static void SigInitStandardMpmFactoryContexts(DetectEngineCtx *de_ctx)
583 {
584 DetectMpmInitializeBuiltinMpms(de_ctx);
585 }
586
587 /** \brief Pure-PCRE or bytetest rule */
RuleInspectsPayloadHasNoMpm(const Signature * s)588 static int RuleInspectsPayloadHasNoMpm(const Signature *s)
589 {
590 if (s->init_data->mpm_sm == NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
591 return 1;
592 return 0;
593 }
594
RuleGetMpmPatternSize(const Signature * s)595 static int RuleGetMpmPatternSize(const Signature *s)
596 {
597 if (s->init_data->mpm_sm == NULL)
598 return -1;
599 int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
600 if (mpm_list < 0)
601 return -1;
602 const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
603 if (cd == NULL)
604 return -1;
605 return (int)cd->content_len;
606 }
607
RuleMpmIsNegated(const Signature * s)608 static int RuleMpmIsNegated(const Signature *s)
609 {
610 if (s->init_data->mpm_sm == NULL)
611 return 0;
612 int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
613 if (mpm_list < 0)
614 return 0;
615 const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
616 if (cd == NULL)
617 return 0;
618 return (cd->flags & DETECT_CONTENT_NEGATED);
619 }
620
RulesGroupPrintSghStats(const SigGroupHead * sgh,const int add_rules,const int add_mpm_stats)621 static json_t *RulesGroupPrintSghStats(const SigGroupHead *sgh,
622 const int add_rules, const int add_mpm_stats)
623 {
624 uint32_t mpm_cnt = 0;
625 uint32_t nonmpm_cnt = 0;
626 uint32_t negmpm_cnt = 0;
627 uint32_t any5_cnt = 0;
628 uint32_t payload_no_mpm_cnt = 0;
629 uint32_t syn_cnt = 0;
630
631 uint32_t mpms_min = 0;
632 uint32_t mpms_max = 0;
633
634 int max_buffer_type_id = DetectBufferTypeMaxId() + 1;
635
636 struct {
637 uint32_t total;
638 uint32_t cnt;
639 uint32_t min;
640 uint32_t max;
641 } mpm_stats[max_buffer_type_id];
642 memset(mpm_stats, 0x00, sizeof(mpm_stats));
643
644 uint32_t alstats[ALPROTO_MAX] = {0};
645 uint32_t mpm_sizes[max_buffer_type_id][256];
646 memset(mpm_sizes, 0, sizeof(mpm_sizes));
647 uint32_t alproto_mpm_bufs[ALPROTO_MAX][max_buffer_type_id];
648 memset(alproto_mpm_bufs, 0, sizeof(alproto_mpm_bufs));
649
650 json_t *js = json_object();
651 if (unlikely(js == NULL))
652 return NULL;
653
654 json_object_set_new(js, "id", json_integer(sgh->id));
655
656 json_t *js_array = json_array();
657
658 const Signature *s;
659 uint32_t x;
660 for (x = 0; x < sgh->sig_cnt; x++) {
661 s = sgh->match_array[x];
662 if (s == NULL)
663 continue;
664
665 int any = 0;
666 if (s->proto.flags & DETECT_PROTO_ANY) {
667 any++;
668 }
669 if (s->flags & SIG_FLAG_DST_ANY) {
670 any++;
671 }
672 if (s->flags & SIG_FLAG_SRC_ANY) {
673 any++;
674 }
675 if (s->flags & SIG_FLAG_DP_ANY) {
676 any++;
677 }
678 if (s->flags & SIG_FLAG_SP_ANY) {
679 any++;
680 }
681 if (any == 5) {
682 any5_cnt++;
683 }
684
685 if (s->init_data->mpm_sm == NULL) {
686 nonmpm_cnt++;
687
688 if (s->sm_arrays[DETECT_SM_LIST_MATCH] != NULL) {
689 SCLogDebug("SGH %p Non-MPM inspecting only packets. Rule %u", sgh, s->id);
690 }
691
692 DetectPort *sp = s->sp;
693 DetectPort *dp = s->dp;
694
695 if (s->flags & SIG_FLAG_TOSERVER && (dp->port == 0 && dp->port2 == 65535)) {
696 SCLogDebug("SGH %p Non-MPM toserver and to 'any'. Rule %u", sgh, s->id);
697 }
698 if (s->flags & SIG_FLAG_TOCLIENT && (sp->port == 0 && sp->port2 == 65535)) {
699 SCLogDebug("SGH %p Non-MPM toclient and to 'any'. Rule %u", sgh, s->id);
700 }
701
702 if (DetectFlagsSignatureNeedsSynPackets(s)) {
703 syn_cnt++;
704 }
705
706 } else {
707 int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
708 BUG_ON(mpm_list < 0);
709 const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx;
710 uint32_t size = cd->content_len < 256 ? cd->content_len : 255;
711
712 mpm_sizes[mpm_list][size]++;
713 if (s->alproto != ALPROTO_UNKNOWN) {
714 alproto_mpm_bufs[s->alproto][mpm_list]++;
715 }
716
717 if (mpm_list == DETECT_SM_LIST_PMATCH) {
718 if (size == 1) {
719 DetectPort *sp = s->sp;
720 DetectPort *dp = s->dp;
721 if (s->flags & SIG_FLAG_TOSERVER) {
722 if (dp->port == 0 && dp->port2 == 65535) {
723 SCLogDebug("SGH %p toserver 1byte fast_pattern to ANY. Rule %u", sgh, s->id);
724 } else {
725 SCLogDebug("SGH %p toserver 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, dp->port, dp->port2, s->id);
726 }
727 }
728 if (s->flags & SIG_FLAG_TOCLIENT) {
729 if (sp->port == 0 && sp->port2 == 65535) {
730 SCLogDebug("SGH %p toclient 1byte fast_pattern to ANY. Rule %u", sgh, s->id);
731 } else {
732 SCLogDebug("SGH %p toclient 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, sp->port, sp->port2, s->id);
733 }
734 }
735 }
736 }
737
738 uint32_t w = PatternStrength(cd->content, cd->content_len);
739 if (mpms_min == 0)
740 mpms_min = w;
741 if (w < mpms_min)
742 mpms_min = w;
743 if (w > mpms_max)
744 mpms_max = w;
745
746 mpm_stats[mpm_list].total += w;
747 mpm_stats[mpm_list].cnt++;
748 if (mpm_stats[mpm_list].min == 0 || w < mpm_stats[mpm_list].min)
749 mpm_stats[mpm_list].min = w;
750 if (w > mpm_stats[mpm_list].max)
751 mpm_stats[mpm_list].max = w;
752
753 mpm_cnt++;
754
755 if (w < 10) {
756 SCLogDebug("SGH %p Weak MPM Pattern on %s. Rule %u", sgh, DetectListToString(mpm_list), s->id);
757 }
758 if (w < 10 && any == 5) {
759 SCLogDebug("SGH %p Weak MPM Pattern on %s, rule is 5xAny. Rule %u", sgh, DetectListToString(mpm_list), s->id);
760 }
761
762 if (cd->flags & DETECT_CONTENT_NEGATED) {
763 SCLogDebug("SGH %p MPM Pattern on %s, is negated. Rule %u", sgh, DetectListToString(mpm_list), s->id);
764 negmpm_cnt++;
765 }
766 }
767
768 if (RuleInspectsPayloadHasNoMpm(s)) {
769 SCLogDebug("SGH %p No MPM. Payload inspecting. Rule %u", sgh, s->id);
770 payload_no_mpm_cnt++;
771 }
772
773 if (s->alproto != ALPROTO_UNKNOWN) {
774 alstats[s->alproto]++;
775 }
776
777 if (add_rules) {
778 json_t *js_sig = json_object();
779 if (unlikely(js == NULL))
780 continue;
781 json_object_set_new(js_sig, "sig_id", json_integer(s->id));
782 json_array_append_new(js_array, js_sig);
783 }
784 }
785
786 json_object_set_new(js, "rules", js_array);
787
788 json_t *stats = json_object();
789 json_object_set_new(stats, "total", json_integer(sgh->sig_cnt));
790
791 json_t *types = json_object();
792 json_object_set_new(types, "mpm", json_integer(mpm_cnt));
793 json_object_set_new(types, "non_mpm", json_integer(nonmpm_cnt));
794 json_object_set_new(types, "negated_mpm", json_integer(negmpm_cnt));
795 json_object_set_new(types, "payload_but_no_mpm", json_integer(payload_no_mpm_cnt));
796 json_object_set_new(types, "syn", json_integer(syn_cnt));
797 json_object_set_new(types, "any5", json_integer(any5_cnt));
798 json_object_set_new(stats, "types", types);
799
800 int i;
801 for (i = 0; i < ALPROTO_MAX; i++) {
802 if (alstats[i] > 0) {
803 json_t *app = json_object();
804 json_object_set_new(app, "total", json_integer(alstats[i]));
805
806 for (int y = 0; y < max_buffer_type_id; y++) {
807 if (alproto_mpm_bufs[i][y] == 0)
808 continue;
809 json_object_set_new(
810 app, DetectListToHumanString(y), json_integer(alproto_mpm_bufs[i][y]));
811 }
812
813 json_object_set_new(stats, AppProtoToString(i), app);
814 }
815 }
816
817 if (add_mpm_stats) {
818 json_t *mpm_js = json_object();
819
820 for (i = 0; i < max_buffer_type_id; i++) {
821 if (mpm_stats[i].cnt > 0) {
822
823 json_t *mpm_sizes_array = json_array();
824 for (int y = 0; y < 256; y++) {
825 if (mpm_sizes[i][y] == 0)
826 continue;
827
828 json_t *e = json_object();
829 json_object_set_new(e, "size", json_integer(y));
830 json_object_set_new(e, "count", json_integer(mpm_sizes[i][y]));
831 json_array_append_new(mpm_sizes_array, e);
832 }
833
834 json_t *buf = json_object();
835 json_object_set_new(buf, "total", json_integer(mpm_stats[i].cnt));
836 json_object_set_new(buf, "avg_strength", json_integer(mpm_stats[i].total / mpm_stats[i].cnt));
837 json_object_set_new(buf, "min_strength", json_integer(mpm_stats[i].min));
838 json_object_set_new(buf, "max_strength", json_integer(mpm_stats[i].max));
839
840 json_object_set_new(buf, "sizes", mpm_sizes_array);
841
842 json_object_set_new(mpm_js, DetectListToHumanString(i), buf);
843 }
844 }
845
846 json_object_set_new(stats, "mpm", mpm_js);
847 }
848 json_object_set_new(js, "stats", stats);
849
850 if (sgh->init)
851 json_object_set_new(js, "whitelist", json_integer(sgh->init->whitelist));
852
853 return js;
854 }
855
RulesDumpGrouping(const DetectEngineCtx * de_ctx,const int add_rules,const int add_mpm_stats)856 static void RulesDumpGrouping(const DetectEngineCtx *de_ctx,
857 const int add_rules, const int add_mpm_stats)
858 {
859 json_t *js = json_object();
860 if (unlikely(js == NULL))
861 return;
862
863 int p;
864 for (p = 0; p < 256; p++) {
865 if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
866 const char *name = (p == IPPROTO_TCP) ? "tcp" : "udp";
867
868 json_t *tcp = json_object();
869
870 json_t *ts_array = json_array();
871 DetectPort *list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[1].tcp :
872 de_ctx->flow_gh[1].udp;
873 while (list != NULL) {
874 json_t *port = json_object();
875 json_object_set_new(port, "port", json_integer(list->port));
876 json_object_set_new(port, "port2", json_integer(list->port2));
877
878 json_t *tcp_ts = RulesGroupPrintSghStats(list->sh,
879 add_rules, add_mpm_stats);
880 json_object_set_new(port, "rulegroup", tcp_ts);
881 json_array_append_new(ts_array, port);
882
883 list = list->next;
884 }
885 json_object_set_new(tcp, "toserver", ts_array);
886
887 json_t *tc_array = json_array();
888 list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[0].tcp :
889 de_ctx->flow_gh[0].udp;
890 while (list != NULL) {
891 json_t *port = json_object();
892 json_object_set_new(port, "port", json_integer(list->port));
893 json_object_set_new(port, "port2", json_integer(list->port2));
894
895 json_t *tcp_tc = RulesGroupPrintSghStats(list->sh,
896 add_rules, add_mpm_stats);
897 json_object_set_new(port, "rulegroup", tcp_tc);
898 json_array_append_new(tc_array, port);
899
900 list = list->next;
901 }
902 json_object_set_new(tcp, "toclient", tc_array);
903
904 json_object_set_new(js, name, tcp);
905 }
906
907 }
908
909 const char *filename = "rule_group.json";
910 const char *log_dir = ConfigGetLogDirectory();
911 char log_path[PATH_MAX] = "";
912
913 snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename);
914
915 FILE *fp = fopen(log_path, "w");
916 if (fp == NULL) {
917 return;
918 }
919
920 char *js_s = json_dumps(js,
921 JSON_PRESERVE_ORDER|JSON_ESCAPE_SLASH);
922 if (unlikely(js_s == NULL)) {
923 fclose(fp);
924 return;
925 }
926
927 json_object_clear(js);
928 json_decref(js);
929
930 fprintf(fp, "%s\n", js_s);
931 free(js_s);
932 fclose(fp);
933 return;
934 }
935
RulesGroupByProto(DetectEngineCtx * de_ctx)936 static int RulesGroupByProto(DetectEngineCtx *de_ctx)
937 {
938 Signature *s = de_ctx->sig_list;
939
940 uint32_t max_idx = 0;
941 SigGroupHead *sgh_ts[256] = {NULL};
942 SigGroupHead *sgh_tc[256] = {NULL};
943
944 for ( ; s != NULL; s = s->next) {
945 if (s->flags & SIG_FLAG_IPONLY)
946 continue;
947
948 int p;
949 for (p = 0; p < 256; p++) {
950 if (p == IPPROTO_TCP || p == IPPROTO_UDP) {
951 continue;
952 }
953 if (!(s->proto.proto[p / 8] & (1<<(p % 8)) || (s->proto.flags & DETECT_PROTO_ANY))) {
954 continue;
955 }
956
957 if (s->flags & SIG_FLAG_TOCLIENT) {
958 SigGroupHeadAppendSig(de_ctx, &sgh_tc[p], s);
959 max_idx = s->num;
960 }
961 if (s->flags & SIG_FLAG_TOSERVER) {
962 SigGroupHeadAppendSig(de_ctx, &sgh_ts[p], s);
963 max_idx = s->num;
964 }
965 }
966 }
967 SCLogDebug("max_idx %u", max_idx);
968
969 /* lets look at deduplicating this list */
970 SigGroupHeadHashFree(de_ctx);
971 SigGroupHeadHashInit(de_ctx);
972
973 uint32_t cnt = 0;
974 uint32_t own = 0;
975 uint32_t ref = 0;
976 int p;
977 for (p = 0; p < 256; p++) {
978 if (p == IPPROTO_TCP || p == IPPROTO_UDP)
979 continue;
980 if (sgh_ts[p] == NULL)
981 continue;
982
983 cnt++;
984
985 SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_ts[p]);
986 if (lookup_sgh == NULL) {
987 SCLogDebug("proto group %d sgh %p is the original", p, sgh_ts[p]);
988
989 SigGroupHeadSetSigCnt(sgh_ts[p], max_idx);
990 SigGroupHeadBuildMatchArray(de_ctx, sgh_ts[p], max_idx);
991
992 SigGroupHeadHashAdd(de_ctx, sgh_ts[p]);
993 SigGroupHeadStore(de_ctx, sgh_ts[p]);
994
995 de_ctx->gh_unique++;
996 own++;
997 } else {
998 SCLogDebug("proto group %d sgh %p is a copy", p, sgh_ts[p]);
999
1000 SigGroupHeadFree(de_ctx, sgh_ts[p]);
1001 sgh_ts[p] = lookup_sgh;
1002
1003 de_ctx->gh_reuse++;
1004 ref++;
1005 }
1006 }
1007 SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies",
1008 "toserver", cnt, own, ref);
1009
1010 cnt = 0;
1011 own = 0;
1012 ref = 0;
1013 for (p = 0; p < 256; p++) {
1014 if (p == IPPROTO_TCP || p == IPPROTO_UDP)
1015 continue;
1016 if (sgh_tc[p] == NULL)
1017 continue;
1018
1019 cnt++;
1020
1021 SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_tc[p]);
1022 if (lookup_sgh == NULL) {
1023 SCLogDebug("proto group %d sgh %p is the original", p, sgh_tc[p]);
1024
1025 SigGroupHeadSetSigCnt(sgh_tc[p], max_idx);
1026 SigGroupHeadBuildMatchArray(de_ctx, sgh_tc[p], max_idx);
1027
1028 SigGroupHeadHashAdd(de_ctx, sgh_tc[p]);
1029 SigGroupHeadStore(de_ctx, sgh_tc[p]);
1030
1031 de_ctx->gh_unique++;
1032 own++;
1033
1034 } else {
1035 SCLogDebug("proto group %d sgh %p is a copy", p, sgh_tc[p]);
1036
1037 SigGroupHeadFree(de_ctx, sgh_tc[p]);
1038 sgh_tc[p] = lookup_sgh;
1039
1040 de_ctx->gh_reuse++;
1041 ref++;
1042 }
1043 }
1044 SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies",
1045 "toclient", cnt, own, ref);
1046
1047 for (p = 0; p < 256; p++) {
1048 if (p == IPPROTO_TCP || p == IPPROTO_UDP)
1049 continue;
1050
1051 de_ctx->flow_gh[0].sgh[p] = sgh_tc[p];
1052 de_ctx->flow_gh[1].sgh[p] = sgh_ts[p];
1053 }
1054
1055 return 0;
1056 }
1057
PortIsWhitelisted(const DetectEngineCtx * de_ctx,const DetectPort * a,int ipproto)1058 static int PortIsWhitelisted(const DetectEngineCtx *de_ctx,
1059 const DetectPort *a, int ipproto)
1060 {
1061 DetectPort *w = de_ctx->tcp_whitelist;
1062 if (ipproto == IPPROTO_UDP)
1063 w = de_ctx->udp_whitelist;
1064
1065 while (w) {
1066 if (a->port >= w->port && a->port2 <= w->port) {
1067 SCLogDebug("port group %u:%u whitelisted -> %d", a->port, a->port2, w->port);
1068 return 1;
1069 }
1070 w = w->next;
1071 }
1072
1073 return 0;
1074 }
1075
RuleSetWhitelist(Signature * s)1076 static int RuleSetWhitelist(Signature *s)
1077 {
1078 DetectPort *p = NULL;
1079 if (s->flags & SIG_FLAG_TOSERVER)
1080 p = s->dp;
1081 else if (s->flags & SIG_FLAG_TOCLIENT)
1082 p = s->sp;
1083 else
1084 return 0;
1085
1086 /* for sigs that don't use 'any' as port, see if we want to
1087 * whitelist poor sigs */
1088 int wl = 0;
1089 if (!(p->port == 0 && p->port2 == 65535)) {
1090 /* pure pcre, bytetest, etc rules */
1091 if (RuleInspectsPayloadHasNoMpm(s)) {
1092 SCLogDebug("Rule %u MPM has 1 byte fast_pattern. Whitelisting SGH's.", s->id);
1093 wl = 99;
1094
1095 } else if (RuleMpmIsNegated(s)) {
1096 SCLogDebug("Rule %u MPM is negated. Whitelisting SGH's.", s->id);
1097 wl = 77;
1098
1099 /* one byte pattern in packet/stream payloads */
1100 } else if (s->init_data->mpm_sm != NULL &&
1101 SigMatchListSMBelongsTo(s, s->init_data->mpm_sm) == DETECT_SM_LIST_PMATCH &&
1102 RuleGetMpmPatternSize(s) == 1)
1103 {
1104 SCLogDebug("Rule %u No MPM. Payload inspecting. Whitelisting SGH's.", s->id);
1105 wl = 55;
1106
1107 } else if (DetectFlagsSignatureNeedsSynPackets(s) &&
1108 DetectFlagsSignatureNeedsSynOnlyPackets(s))
1109 {
1110 SCLogDebug("Rule %u Needs SYN, so inspected often. Whitelisting SGH's.", s->id);
1111 wl = 33;
1112 }
1113 }
1114
1115 s->init_data->whitelist = wl;
1116 return wl;
1117 }
1118
1119 int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx);
1120 int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b);
1121
RulesGroupByPorts(DetectEngineCtx * de_ctx,int ipproto,uint32_t direction)1122 static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, int ipproto, uint32_t direction) {
1123 /* step 1: create a hash of 'DetectPort' objects based on all the
1124 * rules. Each object will have a SGH with the sigs added
1125 * that belong to the SGH. */
1126 DetectPortHashInit(de_ctx);
1127
1128 uint32_t max_idx = 0;
1129 const Signature *s = de_ctx->sig_list;
1130 DetectPort *list = NULL;
1131 while (s) {
1132 /* IP Only rules are handled separately */
1133 if (s->flags & SIG_FLAG_IPONLY)
1134 goto next;
1135 if (!(s->proto.proto[ipproto / 8] & (1<<(ipproto % 8)) || (s->proto.flags & DETECT_PROTO_ANY)))
1136 goto next;
1137 if (direction == SIG_FLAG_TOSERVER) {
1138 if (!(s->flags & SIG_FLAG_TOSERVER))
1139 goto next;
1140 } else if (direction == SIG_FLAG_TOCLIENT) {
1141 if (!(s->flags & SIG_FLAG_TOCLIENT))
1142 goto next;
1143 }
1144
1145 DetectPort *p = NULL;
1146 if (direction == SIG_FLAG_TOSERVER)
1147 p = s->dp;
1148 else if (direction == SIG_FLAG_TOCLIENT)
1149 p = s->sp;
1150 else
1151 BUG_ON(1);
1152
1153 /* see if we want to exclude directionless sigs that really care only for
1154 * to_server syn scans/floods */
1155 if ((direction == SIG_FLAG_TOCLIENT) &&
1156 DetectFlagsSignatureNeedsSynPackets(s) &&
1157 DetectFlagsSignatureNeedsSynOnlyPackets(s) &&
1158 ((s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) == (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) &&
1159 (!(s->dp->port == 0 && s->dp->port2 == 65535)))
1160 {
1161 SCLogWarning(SC_WARN_POOR_RULE, "rule %u: SYN-only to port(s) %u:%u "
1162 "w/o direction specified, disabling for toclient direction",
1163 s->id, s->dp->port, s->dp->port2);
1164 goto next;
1165 }
1166
1167 int wl = s->init_data->whitelist;
1168 while (p) {
1169 int pwl = PortIsWhitelisted(de_ctx, p, ipproto) ? 111 : 0;
1170 pwl = MAX(wl,pwl);
1171
1172 DetectPort *lookup = DetectPortHashLookup(de_ctx, p);
1173 if (lookup) {
1174 SigGroupHeadAppendSig(de_ctx, &lookup->sh, s);
1175 lookup->sh->init->whitelist = MAX(lookup->sh->init->whitelist, pwl);
1176 } else {
1177 DetectPort *tmp2 = DetectPortCopySingle(de_ctx, p);
1178 BUG_ON(tmp2 == NULL);
1179 SigGroupHeadAppendSig(de_ctx, &tmp2->sh, s);
1180 tmp2->sh->init->whitelist = pwl;
1181 DetectPortHashAdd(de_ctx, tmp2);
1182 }
1183
1184 p = p->next;
1185 }
1186 max_idx = s->num;
1187 next:
1188 s = s->next;
1189 }
1190
1191 /* step 2: create a list of DetectPort objects */
1192 HashListTableBucket *htb = NULL;
1193 for (htb = HashListTableGetListHead(de_ctx->dport_hash_table);
1194 htb != NULL;
1195 htb = HashListTableGetListNext(htb))
1196 {
1197 DetectPort *p = HashListTableGetListData(htb);
1198 DetectPort *tmp = DetectPortCopySingle(de_ctx, p);
1199 BUG_ON(tmp == NULL);
1200 int r = DetectPortInsert(de_ctx, &list , tmp);
1201 BUG_ON(r == -1);
1202 }
1203 DetectPortHashFree(de_ctx);
1204 de_ctx->dport_hash_table = NULL;
1205
1206 SCLogDebug("rules analyzed");
1207
1208 /* step 3: group the list and shrink it if necessary */
1209 DetectPort *newlist = NULL;
1210 uint16_t groupmax = (direction == SIG_FLAG_TOCLIENT) ? de_ctx->max_uniq_toclient_groups :
1211 de_ctx->max_uniq_toserver_groups;
1212 CreateGroupedPortList(de_ctx, list, &newlist, groupmax, CreateGroupedPortListCmpCnt, max_idx);
1213 list = newlist;
1214
1215 /* step 4: deduplicate the SGH's */
1216 SigGroupHeadHashFree(de_ctx);
1217 SigGroupHeadHashInit(de_ctx);
1218
1219 uint32_t cnt = 0;
1220 uint32_t own = 0;
1221 uint32_t ref = 0;
1222 DetectPort *iter;
1223 for (iter = list ; iter != NULL; iter = iter->next) {
1224 BUG_ON (iter->sh == NULL);
1225 cnt++;
1226
1227 SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, iter->sh);
1228 if (lookup_sgh == NULL) {
1229 SCLogDebug("port group %p sgh %p is the original", iter, iter->sh);
1230
1231 SigGroupHeadSetSigCnt(iter->sh, max_idx);
1232 SigGroupHeadBuildMatchArray(de_ctx, iter->sh, max_idx);
1233 SigGroupHeadSetProtoAndDirection(iter->sh, ipproto, direction);
1234 SigGroupHeadHashAdd(de_ctx, iter->sh);
1235 SigGroupHeadStore(de_ctx, iter->sh);
1236 iter->flags |= PORT_SIGGROUPHEAD_COPY;
1237 de_ctx->gh_unique++;
1238 own++;
1239 } else {
1240 SCLogDebug("port group %p sgh %p is a copy", iter, iter->sh);
1241
1242 SigGroupHeadFree(de_ctx, iter->sh);
1243 iter->sh = lookup_sgh;
1244 iter->flags |= PORT_SIGGROUPHEAD_COPY;
1245
1246 de_ctx->gh_reuse++;
1247 ref++;
1248 }
1249 }
1250 #if 0
1251 for (iter = list ; iter != NULL; iter = iter->next) {
1252 SCLogInfo("PORT %u-%u %p (sgh=%s, whitelisted=%s/%d)",
1253 iter->port, iter->port2, iter->sh,
1254 iter->flags & PORT_SIGGROUPHEAD_COPY ? "ref" : "own",
1255 iter->sh->init->whitelist ? "true" : "false",
1256 iter->sh->init->whitelist);
1257 }
1258 #endif
1259 SCLogPerf("%s %s: %u port groups, %u unique SGH's, %u copies",
1260 ipproto == 6 ? "TCP" : "UDP",
1261 direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
1262 cnt, own, ref);
1263 return list;
1264 }
1265
SignatureSetType(DetectEngineCtx * de_ctx,Signature * s)1266 void SignatureSetType(DetectEngineCtx *de_ctx, Signature *s)
1267 {
1268 /* see if the sig is dp only */
1269 if (SignatureIsPDOnly(de_ctx, s) == 1) {
1270 s->flags |= SIG_FLAG_PDONLY;
1271
1272 /* see if the sig is ip only */
1273 } else if (SignatureIsIPOnly(de_ctx, s) == 1) {
1274 s->flags |= SIG_FLAG_IPONLY;
1275
1276 } else if (SignatureIsDEOnly(de_ctx, s) == 1) {
1277 s->init_data->init_flags |= SIG_FLAG_INIT_DEONLY;
1278 }
1279 }
1280
1281 /**
1282 * \brief Preprocess signature, classify ip-only, etc, build sig array
1283 *
1284 * \param de_ctx Pointer to the Detection Engine Context
1285 *
1286 * \retval 0 on success
1287 * \retval -1 on failure
1288 */
SigAddressPrepareStage1(DetectEngineCtx * de_ctx)1289 int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
1290 {
1291 uint32_t cnt_iponly = 0;
1292 uint32_t cnt_payload = 0;
1293 uint32_t cnt_applayer = 0;
1294 uint32_t cnt_deonly = 0;
1295
1296 if (!(de_ctx->flags & DE_QUIET)) {
1297 SCLogDebug("building signature grouping structure, stage 1: "
1298 "preprocessing rules...");
1299 }
1300
1301 de_ctx->sig_array_len = DetectEngineGetMaxSigId(de_ctx);
1302 de_ctx->sig_array_size = (de_ctx->sig_array_len * sizeof(Signature *));
1303 de_ctx->sig_array = (Signature **)SCMalloc(de_ctx->sig_array_size);
1304 if (de_ctx->sig_array == NULL)
1305 goto error;
1306 memset(de_ctx->sig_array,0,de_ctx->sig_array_size);
1307
1308 SCLogDebug("signature lookup array: %" PRIu32 " sigs, %" PRIu32 " bytes",
1309 de_ctx->sig_array_len, de_ctx->sig_array_size);
1310
1311 /* now for every rule add the source group */
1312 for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
1313 de_ctx->sig_array[s->num] = s;
1314
1315 SCLogDebug("Signature %" PRIu32 ", internal id %" PRIu32 ", ptrs %p %p ", s->id, s->num, s, de_ctx->sig_array[s->num]);
1316
1317 if (s->flags & SIG_FLAG_PDONLY) {
1318 SCLogDebug("Signature %"PRIu32" is considered \"PD only\"", s->id);
1319 } else if (s->flags & SIG_FLAG_IPONLY) {
1320 SCLogDebug("Signature %"PRIu32" is considered \"IP only\"", s->id);
1321 cnt_iponly++;
1322 } else if (SignatureIsInspectingPayload(de_ctx, s) == 1) {
1323 SCLogDebug("Signature %"PRIu32" is considered \"Payload inspecting\"", s->id);
1324 cnt_payload++;
1325 } else if (s->init_data->init_flags & SIG_FLAG_INIT_DEONLY) {
1326 SCLogDebug("Signature %"PRIu32" is considered \"Decoder Event only\"", s->id);
1327 cnt_deonly++;
1328 } else if (s->flags & SIG_FLAG_APPLAYER) {
1329 SCLogDebug("Signature %"PRIu32" is considered \"Applayer inspecting\"", s->id);
1330 cnt_applayer++;
1331 }
1332
1333 #ifdef DEBUG
1334 if (SCLogDebugEnabled()) {
1335 uint16_t colen = 0;
1336 char copresent = 0;
1337 SigMatch *sm;
1338 DetectContentData *co;
1339 for (sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; sm = sm->next) {
1340 if (sm->type != DETECT_CONTENT)
1341 continue;
1342
1343 copresent = 1;
1344 co = (DetectContentData *)sm->ctx;
1345 if (co->content_len > colen)
1346 colen = co->content_len;
1347 }
1348
1349 if (copresent && colen == 1) {
1350 SCLogDebug("signature %8u content maxlen 1", s->id);
1351 for (int proto = 0; proto < 256; proto++) {
1352 if (s->proto.proto[(proto/8)] & (1<<(proto%8)))
1353 SCLogDebug("=> proto %" PRId32 "", proto);
1354 }
1355 }
1356 }
1357 #endif /* DEBUG */
1358
1359 if (RuleMpmIsNegated(s)) {
1360 s->flags |= SIG_FLAG_MPM_NEG;
1361 }
1362
1363 SignatureCreateMask(s);
1364 DetectContentPropagateLimits(s);
1365 SigParseApplyDsizeToContent(s);
1366
1367 RuleSetWhitelist(s);
1368
1369 /* if keyword engines are enabled in the config, handle them here */
1370 if (de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO &&
1371 !(s->flags & SIG_FLAG_PREFILTER))
1372 {
1373 int prefilter_list = DETECT_TBLSIZE;
1374
1375 /* get the keyword supporting prefilter with the lowest type */
1376 for (int i = 0; i < (int)s->init_data->smlists_array_size; i++) {
1377 SigMatch *sm = s->init_data->smlists[i];
1378 while (sm != NULL) {
1379 if (sigmatch_table[sm->type].SupportsPrefilter != NULL) {
1380 if (sigmatch_table[sm->type].SupportsPrefilter(s) == TRUE) {
1381 prefilter_list = MIN(prefilter_list, sm->type);
1382 }
1383 }
1384 sm = sm->next;
1385 }
1386 }
1387
1388 /* apply that keyword as prefilter */
1389 if (prefilter_list != DETECT_TBLSIZE) {
1390 for (int i = 0; i < (int)s->init_data->smlists_array_size; i++) {
1391 SigMatch *sm = s->init_data->smlists[i];
1392 while (sm != NULL) {
1393 if (sm->type == prefilter_list) {
1394 s->init_data->prefilter_sm = sm;
1395 s->flags |= SIG_FLAG_PREFILTER;
1396 SCLogConfig("sid %u: prefilter is on \"%s\"", s->id, sigmatch_table[sm->type].name);
1397 break;
1398 }
1399 sm = sm->next;
1400 }
1401 }
1402 }
1403 }
1404
1405 /* run buffer type callbacks if any */
1406 for (int x = 0; x < (int)s->init_data->smlists_array_size; x++) {
1407 if (s->init_data->smlists[x])
1408 DetectBufferRunSetupCallback(de_ctx, x, s);
1409 }
1410
1411 de_ctx->sig_cnt++;
1412 }
1413
1414 if (!(de_ctx->flags & DE_QUIET)) {
1415 SCLogInfo("%" PRIu32 " signatures processed. %" PRIu32 " are IP-only "
1416 "rules, %" PRIu32 " are inspecting packet payload, %"PRIu32
1417 " inspect application layer, %"PRIu32" are decoder event only",
1418 de_ctx->sig_cnt, cnt_iponly, cnt_payload, cnt_applayer,
1419 cnt_deonly);
1420
1421 SCLogConfig("building signature grouping structure, stage 1: "
1422 "preprocessing rules... complete");
1423 }
1424
1425 if (DetectFlowbitsAnalyze(de_ctx) != 0)
1426 goto error;
1427
1428 return 0;
1429
1430 error:
1431 return -1;
1432 }
1433
PortGroupWhitelist(const DetectPort * a)1434 static int PortGroupWhitelist(const DetectPort *a)
1435 {
1436 return a->sh->init->whitelist;
1437 }
1438
CreateGroupedPortListCmpCnt(DetectPort * a,DetectPort * b)1439 int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b)
1440 {
1441 if (PortGroupWhitelist(a) && !PortGroupWhitelist(b)) {
1442 SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1443 a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1444 b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1445 return 1;
1446 } else if (!PortGroupWhitelist(a) && PortGroupWhitelist(b)) {
1447 SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)",
1448 a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1449 b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1450 return 0;
1451 } else if (PortGroupWhitelist(a) > PortGroupWhitelist(b)) {
1452 SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1453 a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1454 b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1455 return 1;
1456 } else if (PortGroupWhitelist(a) == PortGroupWhitelist(b)) {
1457 if (a->sh->sig_cnt > b->sh->sig_cnt) {
1458 SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)",
1459 a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1460 b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1461 return 1;
1462 }
1463 }
1464
1465 SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)",
1466 a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a),
1467 b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b));
1468 return 0;
1469 }
1470
1471 /** \internal
1472 * \brief Create a list of DetectPort objects sorted based on CompareFunc's
1473 * logic.
1474 *
1475 * List can limit the number of groups. In this case an extra "join" group
1476 * is created that contains the sigs belonging to that. It's *appended* to
1477 * the list, meaning that if the list is walked linearly it's found last.
1478 * The joingr is meant to be a catch all.
1479 *
1480 */
CreateGroupedPortList(DetectEngineCtx * de_ctx,DetectPort * port_list,DetectPort ** newhead,uint32_t unique_groups,int (* CompareFunc)(DetectPort *,DetectPort *),uint32_t max_idx)1481 int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx)
1482 {
1483 DetectPort *tmplist = NULL, *joingr = NULL;
1484 char insert = 0;
1485 uint32_t groups = 0;
1486 DetectPort *list;
1487
1488 /* insert the addresses into the tmplist, where it will
1489 * be sorted descending on 'cnt' and on wehther a group
1490 * is whitelisted. */
1491
1492 DetectPort *oldhead = port_list;
1493 while (oldhead) {
1494 /* take the top of the list */
1495 list = oldhead;
1496 oldhead = oldhead->next;
1497 list->next = NULL;
1498
1499 groups++;
1500
1501 SigGroupHeadSetSigCnt(list->sh, max_idx);
1502
1503 /* insert it */
1504 DetectPort *tmpgr = tmplist, *prevtmpgr = NULL;
1505 if (tmplist == NULL) {
1506 /* empty list, set head */
1507 tmplist = list;
1508 } else {
1509 /* look for the place to insert */
1510 for ( ; tmpgr != NULL && !insert; tmpgr = tmpgr->next) {
1511 if (CompareFunc(list, tmpgr) == 1) {
1512 if (tmpgr == tmplist) {
1513 list->next = tmplist;
1514 tmplist = list;
1515 SCLogDebug("new list top: %u:%u", tmplist->port, tmplist->port2);
1516 } else {
1517 list->next = prevtmpgr->next;
1518 prevtmpgr->next = list;
1519 }
1520 insert = 1;
1521 break;
1522 }
1523 prevtmpgr = tmpgr;
1524 }
1525 if (insert == 0) {
1526 list->next = NULL;
1527 prevtmpgr->next = list;
1528 }
1529 insert = 0;
1530 }
1531 }
1532
1533 uint32_t left = unique_groups;
1534 if (left == 0)
1535 left = groups;
1536
1537 /* create another list: take the port groups from above
1538 * and add them to the 2nd list until we have met our
1539 * count. The rest is added to the 'join' group. */
1540 DetectPort *tmplist2 = NULL, *tmplist2_tail = NULL;
1541 DetectPort *gr, *next_gr;
1542 for (gr = tmplist; gr != NULL; ) {
1543 next_gr = gr->next;
1544
1545 SCLogDebug("temp list gr %p %u:%u", gr, gr->port, gr->port2);
1546 DetectPortPrint(gr);
1547
1548 /* if we've set up all the unique groups, add the rest to the
1549 * catch-all joingr */
1550 if (left == 0) {
1551 if (joingr == NULL) {
1552 DetectPortParse(de_ctx, &joingr, "0:65535");
1553 if (joingr == NULL) {
1554 goto error;
1555 }
1556 SCLogDebug("joingr => %u-%u", joingr->port, joingr->port2);
1557 joingr->next = NULL;
1558 }
1559 SigGroupHeadCopySigs(de_ctx,gr->sh,&joingr->sh);
1560
1561 /* when a group's sigs are added to the joingr, we can free it */
1562 gr->next = NULL;
1563 DetectPortFree(de_ctx, gr);
1564 gr = NULL;
1565
1566 /* append */
1567 } else {
1568 gr->next = NULL;
1569
1570 if (tmplist2 == NULL) {
1571 tmplist2 = gr;
1572 tmplist2_tail = gr;
1573 } else {
1574 tmplist2_tail->next = gr;
1575 tmplist2_tail = gr;
1576 }
1577 }
1578
1579 if (left > 0)
1580 left--;
1581
1582 gr = next_gr;
1583 }
1584
1585 /* if present, append the joingr that covers the rest */
1586 if (joingr != NULL) {
1587 SCLogDebug("appending joingr %p %u:%u", joingr, joingr->port, joingr->port2);
1588
1589 if (tmplist2 == NULL) {
1590 tmplist2 = joingr;
1591 //tmplist2_tail = joingr;
1592 } else {
1593 tmplist2_tail->next = joingr;
1594 //tmplist2_tail = joingr;
1595 }
1596 } else {
1597 SCLogDebug("no joingr");
1598 }
1599
1600 /* pass back our new list to the caller */
1601 *newhead = tmplist2;
1602 DetectPortPrintList(*newhead);
1603
1604 return 0;
1605 error:
1606 return -1;
1607 }
1608
1609 /**
1610 * \internal
1611 * \brief add a decoder event signature to the detection engine ctx
1612 */
DetectEngineAddDecoderEventSig(DetectEngineCtx * de_ctx,Signature * s)1613 static void DetectEngineAddDecoderEventSig(DetectEngineCtx *de_ctx, Signature *s)
1614 {
1615 SCLogDebug("adding signature %"PRIu32" to the decoder event sgh", s->id);
1616 SigGroupHeadAppendSig(de_ctx, &de_ctx->decoder_event_sgh, s);
1617 }
1618
1619 /**
1620 * \brief Fill the global src group head, with the sigs included
1621 *
1622 * \param de_ctx Pointer to the Detection Engine Context whose Signatures have
1623 * to be processed
1624 *
1625 * \retval 0 On success
1626 * \retval -1 On failure
1627 */
SigAddressPrepareStage2(DetectEngineCtx * de_ctx)1628 int SigAddressPrepareStage2(DetectEngineCtx *de_ctx)
1629 {
1630 uint32_t sigs = 0;
1631
1632 SCLogDebug("building signature grouping structure, stage 2: "
1633 "building source address lists...");
1634
1635 IPOnlyInit(de_ctx, &de_ctx->io_ctx);
1636
1637 de_ctx->flow_gh[1].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOSERVER);
1638 de_ctx->flow_gh[0].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOCLIENT);
1639 de_ctx->flow_gh[1].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOSERVER);
1640 de_ctx->flow_gh[0].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOCLIENT);
1641
1642 /* Setup the other IP Protocols (so not TCP/UDP) */
1643 RulesGroupByProto(de_ctx);
1644
1645 /* now for every rule add the source group to our temp lists */
1646 for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
1647 SCLogDebug("s->id %"PRIu32, s->id);
1648 if (s->flags & SIG_FLAG_IPONLY) {
1649 IPOnlyAddSignature(de_ctx, &de_ctx->io_ctx, s);
1650 }
1651
1652 if (s->init_data->init_flags & SIG_FLAG_INIT_DEONLY) {
1653 DetectEngineAddDecoderEventSig(de_ctx, s);
1654 }
1655
1656 sigs++;
1657 }
1658
1659 IPOnlyPrepare(de_ctx);
1660 IPOnlyPrint(de_ctx, &de_ctx->io_ctx);
1661
1662 return 0;
1663 }
1664
DetectEngineBuildDecoderEventSgh(DetectEngineCtx * de_ctx)1665 static void DetectEngineBuildDecoderEventSgh(DetectEngineCtx *de_ctx)
1666 {
1667 if (de_ctx->decoder_event_sgh == NULL)
1668 return;
1669
1670 uint32_t max_idx = DetectEngineGetMaxSigId(de_ctx);
1671 SigGroupHeadSetSigCnt(de_ctx->decoder_event_sgh, max_idx);
1672 SigGroupHeadBuildMatchArray(de_ctx, de_ctx->decoder_event_sgh, max_idx);
1673 }
1674
SigAddressPrepareStage3(DetectEngineCtx * de_ctx)1675 int SigAddressPrepareStage3(DetectEngineCtx *de_ctx)
1676 {
1677 /* prepare the decoder event sgh */
1678 DetectEngineBuildDecoderEventSgh(de_ctx);
1679 return 0;
1680 }
1681
SigAddressCleanupStage1(DetectEngineCtx * de_ctx)1682 int SigAddressCleanupStage1(DetectEngineCtx *de_ctx)
1683 {
1684 BUG_ON(de_ctx == NULL);
1685
1686 if (!(de_ctx->flags & DE_QUIET)) {
1687 SCLogDebug("cleaning up signature grouping structure...");
1688 }
1689 if (de_ctx->decoder_event_sgh)
1690 SigGroupHeadFree(de_ctx, de_ctx->decoder_event_sgh);
1691 de_ctx->decoder_event_sgh = NULL;
1692
1693 for (int f = 0; f < FLOW_STATES; f++) {
1694 for (int p = 0; p < 256; p++) {
1695 de_ctx->flow_gh[f].sgh[p] = NULL;
1696 }
1697
1698 /* free lookup lists */
1699 DetectPortCleanupList(de_ctx, de_ctx->flow_gh[f].tcp);
1700 de_ctx->flow_gh[f].tcp = NULL;
1701 DetectPortCleanupList(de_ctx, de_ctx->flow_gh[f].udp);
1702 de_ctx->flow_gh[f].udp = NULL;
1703 }
1704
1705 for (uint32_t idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
1706 SigGroupHead *sgh = de_ctx->sgh_array[idx];
1707 if (sgh == NULL)
1708 continue;
1709
1710 SCLogDebug("sgh %p", sgh);
1711 SigGroupHeadFree(de_ctx, sgh);
1712 }
1713 SCFree(de_ctx->sgh_array);
1714 de_ctx->sgh_array = NULL;
1715 de_ctx->sgh_array_cnt = 0;
1716 de_ctx->sgh_array_size = 0;
1717
1718 IPOnlyDeinit(de_ctx, &de_ctx->io_ctx);
1719
1720 if (!(de_ctx->flags & DE_QUIET)) {
1721 SCLogInfo("cleaning up signature grouping structure... complete");
1722 }
1723 return 0;
1724 }
1725
1726 #if 0
1727 static void DbgPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1728 {
1729 if (sgh == NULL) {
1730 printf("\n");
1731 return;
1732 }
1733
1734 uint32_t sig;
1735 for (sig = 0; sig < sgh->sig_cnt; sig++) {
1736 printf("%" PRIu32 " ", sgh->match_array[sig]->id);
1737 }
1738 printf("\n");
1739 }
1740
1741 static void DbgPrintSigs2(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
1742 {
1743 if (sgh == NULL || sgh->init == NULL) {
1744 printf("\n");
1745 return;
1746 }
1747
1748 uint32_t sig;
1749 for (sig = 0; sig < DetectEngineGetMaxSigId(de_ctx); sig++) {
1750 if (sgh->init->sig_array[(sig/8)] & (1<<(sig%8))) {
1751 printf("%" PRIu32 " ", de_ctx->sig_array[sig]->id);
1752 }
1753 }
1754 printf("\n");
1755 }
1756 #endif
1757
1758 /** \brief finalize preparing sgh's */
SigAddressPrepareStage4(DetectEngineCtx * de_ctx)1759 int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
1760 {
1761 SCEnter();
1762
1763 //SCLogInfo("sgh's %"PRIu32, de_ctx->sgh_array_cnt);
1764
1765 uint32_t cnt = 0;
1766 for (uint32_t idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
1767 SigGroupHead *sgh = de_ctx->sgh_array[idx];
1768 if (sgh == NULL)
1769 continue;
1770
1771 SCLogDebug("sgh %p", sgh);
1772
1773 SigGroupHeadSetFilemagicFlag(de_ctx, sgh);
1774 SigGroupHeadSetFileHashFlag(de_ctx, sgh);
1775 SigGroupHeadSetFilesizeFlag(de_ctx, sgh);
1776 SigGroupHeadSetFilestoreCount(de_ctx, sgh);
1777 SCLogDebug("filestore count %u", sgh->filestore_cnt);
1778
1779 PrefilterSetupRuleGroup(de_ctx, sgh);
1780
1781 SigGroupHeadBuildNonPrefilterArray(de_ctx, sgh);
1782
1783 SigGroupHeadInitDataFree(sgh->init);
1784 sgh->init = NULL;
1785
1786 sgh->id = idx;
1787 cnt++;
1788 }
1789 SCLogPerf("Unique rule groups: %u", cnt);
1790
1791 MpmStoreReportStats(de_ctx);
1792
1793 if (de_ctx->decoder_event_sgh != NULL) {
1794 /* no need to set filestore count here as that would make a
1795 * signature not decode event only. */
1796 }
1797
1798 /* cleanup the hashes now since we won't need them
1799 * after the initialization phase. */
1800 SigGroupHeadHashFree(de_ctx);
1801
1802 int dump_grouping = 0;
1803 (void)ConfGetBool("detect.profiling.grouping.dump-to-disk", &dump_grouping);
1804
1805 if (dump_grouping) {
1806 int add_rules = 0;
1807 (void)ConfGetBool("detect.profiling.grouping.include-rules", &add_rules);
1808 int add_mpm_stats = 0;
1809 (void)ConfGetBool("detect.profiling.grouping.include-mpm-stats", &add_mpm_stats);
1810
1811 RulesDumpGrouping(de_ctx, add_rules, add_mpm_stats);
1812 }
1813
1814 #ifdef PROFILING
1815 SCProfilingSghInitCounters(de_ctx);
1816 #endif
1817 SCReturnInt(0);
1818 }
1819
1820 extern int rule_engine_analysis_set;
1821 /** \internal
1822 * \brief perform final per signature setup tasks
1823 *
1824 * - Create SigMatchData arrays from the init only SigMatch lists
1825 * - Setup per signature inspect engines
1826 * - remove signature init data.
1827 */
SigMatchPrepare(DetectEngineCtx * de_ctx)1828 static int SigMatchPrepare(DetectEngineCtx *de_ctx)
1829 {
1830 SCEnter();
1831
1832 Signature *s = de_ctx->sig_list;
1833 for (; s != NULL; s = s->next) {
1834 /* set up inspect engines */
1835 DetectEngineAppInspectionEngine2Signature(de_ctx, s);
1836
1837 /* built-ins */
1838 int type;
1839 for (type = 0; type < DETECT_SM_LIST_MAX; type++) {
1840 /* skip PMATCH if it is used in a stream 'app engine' instead */
1841 if (type == DETECT_SM_LIST_PMATCH && (s->init_data->init_flags & SIG_FLAG_INIT_STATE_MATCH))
1842 continue;
1843 SigMatch *sm = s->init_data->smlists[type];
1844 s->sm_arrays[type] = SigMatchList2DataArray(sm);
1845 }
1846 /* set up the pkt inspection engines */
1847 DetectEnginePktInspectionSetup(s);
1848
1849 if (rule_engine_analysis_set) {
1850 EngineAnalysisRules2(de_ctx, s);
1851 }
1852 /* free lists. Ctx' are xferred to sm_arrays so won't get freed */
1853 uint32_t i;
1854 for (i = 0; i < s->init_data->smlists_array_size; i++) {
1855 SigMatch *sm = s->init_data->smlists[i];
1856 while (sm != NULL) {
1857 SigMatch *nsm = sm->next;
1858 SigMatchFree(de_ctx, sm);
1859 sm = nsm;
1860 }
1861 }
1862 SCFree(s->init_data->smlists);
1863 SCFree(s->init_data->smlists_tail);
1864 for (i = 0; i < (uint32_t)s->init_data->transforms.cnt; i++) {
1865 if (s->init_data->transforms.transforms[i].options) {
1866 SCFree(s->init_data->transforms.transforms[i].options);
1867 s->init_data->transforms.transforms[i].options = NULL;
1868 }
1869 }
1870 SCFree(s->init_data);
1871 s->init_data = NULL;
1872 }
1873
1874
1875 SCReturnInt(0);
1876 }
1877
1878 /**
1879 * \brief Convert the signature list into the runtime match structure.
1880 *
1881 * \param de_ctx Pointer to the Detection Engine Context whose Signatures have
1882 * to be processed
1883 *
1884 * \retval 0 On Success.
1885 * \retval -1 On failure.
1886 */
SigGroupBuild(DetectEngineCtx * de_ctx)1887 int SigGroupBuild(DetectEngineCtx *de_ctx)
1888 {
1889 Signature *s = de_ctx->sig_list;
1890
1891 /* Assign the unique order id of signatures after sorting,
1892 * so the IP Only engine process them in order too. Also
1893 * reset the old signums and assign new signums. We would
1894 * have experienced Sig reordering by now, hence the new
1895 * signums. */
1896 de_ctx->signum = 0;
1897 while (s != NULL) {
1898 s->num = de_ctx->signum++;
1899
1900 s = s->next;
1901 }
1902
1903 if (DetectSetFastPatternAndItsId(de_ctx) < 0)
1904 return -1;
1905
1906 SigInitStandardMpmFactoryContexts(de_ctx);
1907
1908 if (SigAddressPrepareStage1(de_ctx) != 0) {
1909 FatalError(SC_ERR_FATAL, "initializing the detection engine failed");
1910 }
1911
1912 if (SigAddressPrepareStage2(de_ctx) != 0) {
1913 FatalError(SC_ERR_FATAL, "initializing the detection engine failed");
1914 }
1915
1916 if (SigAddressPrepareStage3(de_ctx) != 0) {
1917 FatalError(SC_ERR_FATAL, "initializing the detection engine failed");
1918 }
1919 if (SigAddressPrepareStage4(de_ctx) != 0) {
1920 FatalError(SC_ERR_FATAL, "initializing the detection engine failed");
1921 }
1922
1923 int r = DetectMpmPrepareBuiltinMpms(de_ctx);
1924 r |= DetectMpmPrepareAppMpms(de_ctx);
1925 r |= DetectMpmPreparePktMpms(de_ctx);
1926 if (r != 0) {
1927 FatalError(SC_ERR_FATAL, "initializing the detection engine failed");
1928 }
1929
1930 if (SigMatchPrepare(de_ctx) != 0) {
1931 FatalError(SC_ERR_FATAL, "initializing the detection engine failed");
1932 }
1933
1934 #ifdef PROFILING
1935 SCProfilingKeywordInitCounters(de_ctx);
1936 SCProfilingPrefilterInitCounters(de_ctx);
1937 de_ctx->profile_match_logging_threshold = UINT_MAX; // disabled
1938
1939 intmax_t v = 0;
1940 if (ConfGetInt("detect.profiling.inspect-logging-threshold", &v) == 1)
1941 de_ctx->profile_match_logging_threshold = (uint32_t)v;
1942
1943 SCProfilingRuleInitCounters(de_ctx);
1944 #endif
1945
1946 ThresholdHashAllocate(de_ctx);
1947
1948 if (!DetectEngineMultiTenantEnabled()) {
1949 VarNameStoreActivateStaging();
1950 }
1951 return 0;
1952 }
1953
SigGroupCleanup(DetectEngineCtx * de_ctx)1954 int SigGroupCleanup (DetectEngineCtx *de_ctx)
1955 {
1956 SigAddressCleanupStage1(de_ctx);
1957
1958 return 0;
1959 }
1960