1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * Implements the filestore keyword
24  */
25 
26 #include "suricata-common.h"
27 #include "threads.h"
28 #include "debug.h"
29 #include "decode.h"
30 
31 #include "detect.h"
32 #include "detect-parse.h"
33 
34 #include "detect-engine.h"
35 #include "detect-engine-mpm.h"
36 #include "detect-engine-state.h"
37 
38 #include "feature.h"
39 
40 #include "flow.h"
41 #include "flow-var.h"
42 #include "flow-util.h"
43 
44 #include "util-debug.h"
45 #include "util-spm-bm.h"
46 #include "util-unittest.h"
47 #include "util-unittest-helper.h"
48 
49 #include "app-layer.h"
50 #include "app-layer-parser.h"
51 #include "app-layer-htp.h"
52 
53 #include "stream-tcp.h"
54 
55 #include "detect-filestore.h"
56 
57 /**
58  * \brief Regex for parsing our flow options
59  */
60 #define PARSE_REGEX  "^\\s*([A-z_]+)\\s*(?:,\\s*([A-z_]+))?\\s*(?:,\\s*([A-z_]+))?\\s*$"
61 
62 static DetectParseRegex parse_regex;
63 
64 static int DetectFilestoreMatch (DetectEngineThreadCtx *,
65         Flow *, uint8_t, File *, const Signature *, const SigMatchCtx *);
66 static int DetectFilestorePostMatch(DetectEngineThreadCtx *det_ctx,
67         Packet *p, const Signature *s, const SigMatchCtx *ctx);
68 static int DetectFilestoreSetup (DetectEngineCtx *, Signature *, const char *);
69 static void DetectFilestoreFree(DetectEngineCtx *, void *);
70 #ifdef UNITTESTS
71 static void DetectFilestoreRegisterTests(void);
72 #endif
73 static int g_file_match_list_id = 0;
74 
75 /**
76  * \brief Registration function for keyword: filestore
77  */
DetectFilestoreRegister(void)78 void DetectFilestoreRegister(void)
79 {
80     sigmatch_table[DETECT_FILESTORE].name = "filestore";
81     sigmatch_table[DETECT_FILESTORE].desc = "stores files to disk if the rule matched";
82     sigmatch_table[DETECT_FILESTORE].url = "/rules/file-keywords.html#filestore";
83     sigmatch_table[DETECT_FILESTORE].FileMatch = DetectFilestoreMatch;
84     sigmatch_table[DETECT_FILESTORE].Setup = DetectFilestoreSetup;
85     sigmatch_table[DETECT_FILESTORE].Free  = DetectFilestoreFree;
86 #ifdef UNITTESTS
87     sigmatch_table[DETECT_FILESTORE].RegisterTests = DetectFilestoreRegisterTests;
88 #endif
89     sigmatch_table[DETECT_FILESTORE].flags = SIGMATCH_OPTIONAL_OPT;
90 
91     sigmatch_table[DETECT_FILESTORE_POSTMATCH].name = "__filestore__postmatch__";
92     sigmatch_table[DETECT_FILESTORE_POSTMATCH].Match = DetectFilestorePostMatch;
93     sigmatch_table[DETECT_FILESTORE_POSTMATCH].Free  = DetectFilestoreFree;
94 
95     DetectSetupParseRegexes(PARSE_REGEX, &parse_regex);
96 
97     g_file_match_list_id = DetectBufferTypeRegister("files");
98 }
99 
100 /**
101  *  \brief apply the post match filestore with options
102  */
FilestorePostMatchWithOptions(Packet * p,Flow * f,const DetectFilestoreData * filestore,FileContainer * fc,uint32_t file_id,uint64_t tx_id)103 static int FilestorePostMatchWithOptions(Packet *p, Flow *f, const DetectFilestoreData *filestore,
104         FileContainer *fc, uint32_t file_id, uint64_t tx_id)
105 {
106     if (filestore == NULL) {
107         SCReturnInt(0);
108     }
109 
110     int this_file = 0;
111     int this_tx = 0;
112     int this_flow = 0;
113     int rule_dir = 0;
114     int toserver_dir = 0;
115     int toclient_dir = 0;
116 
117     switch (filestore->direction) {
118         case FILESTORE_DIR_DEFAULT:
119             rule_dir = 1;
120             break;
121         case FILESTORE_DIR_BOTH:
122             toserver_dir = 1;
123             toclient_dir = 1;
124             break;
125         case FILESTORE_DIR_TOSERVER:
126             toserver_dir = 1;
127             break;
128         case FILESTORE_DIR_TOCLIENT:
129             toclient_dir = 1;
130             break;
131     }
132 
133     switch (filestore->scope) {
134         case FILESTORE_SCOPE_DEFAULT:
135             if (rule_dir) {
136                 this_file = 1;
137             } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && toclient_dir) {
138                 this_file = 1;
139             } else if ((p->flowflags & FLOW_PKT_TOSERVER) && toserver_dir) {
140                 this_file = 1;
141             }
142             break;
143         case FILESTORE_SCOPE_TX:
144             this_tx = 1;
145             break;
146         case FILESTORE_SCOPE_SSN:
147             this_flow = 1;
148             break;
149     }
150 
151     if (this_file)  {
152         FileStoreFileById(fc, file_id);
153     } else if (this_tx) {
154         /* flag tx all files will be stored */
155         if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) {
156             HtpState *htp_state = f->alstate;
157             if (toserver_dir) {
158                 htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TS;
159                 FileStoreAllFilesForTx(htp_state->files_ts, tx_id);
160             }
161             if (toclient_dir) {
162                 htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TC;
163                 FileStoreAllFilesForTx(htp_state->files_tc, tx_id);
164             }
165             htp_state->store_tx_id = tx_id;
166         }
167     } else if (this_flow) {
168         /* flag flow all files will be stored */
169         if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) {
170             HtpState *htp_state = f->alstate;
171             if (toserver_dir) {
172                 htp_state->flags |= HTP_FLAG_STORE_FILES_TS;
173                 FileStoreAllFiles(htp_state->files_ts);
174             }
175             if (toclient_dir) {
176                 htp_state->flags |= HTP_FLAG_STORE_FILES_TC;
177                 FileStoreAllFiles(htp_state->files_tc);
178             }
179         }
180     } else {
181         FileStoreFileById(fc, file_id);
182     }
183 
184     SCReturnInt(0);
185 }
186 
187 /**
188  *  \brief post-match function for filestore
189  *
190  *  \param t thread local vars
191  *  \param det_ctx pattern matcher thread local data
192  *  \param p packet
193  *
194  *  The match function for filestore records store candidates in the det_ctx.
195  *  When we are sure all parts of the signature matched, we run this function
196  *  to finalize the filestore.
197  */
DetectFilestorePostMatch(DetectEngineThreadCtx * det_ctx,Packet * p,const Signature * s,const SigMatchCtx * ctx)198 static int DetectFilestorePostMatch(DetectEngineThreadCtx *det_ctx,
199         Packet *p, const Signature *s, const SigMatchCtx *ctx)
200 {
201     uint8_t flags = 0;
202 
203     SCEnter();
204 
205     if (det_ctx->filestore_cnt == 0) {
206         SCReturnInt(0);
207     }
208 
209     if ((s->filestore_ctx == NULL && !(s->flags & SIG_FLAG_FILESTORE)) || p->flow == NULL) {
210 #ifndef DEBUG
211         SCReturnInt(0);
212 #else
213         BUG_ON(1);
214 #endif
215     }
216 
217     if (p->proto == IPPROTO_TCP && p->flow->protoctx != NULL) {
218         /* set filestore depth for stream reassembling */
219         TcpSession *ssn = (TcpSession *)p->flow->protoctx;
220         TcpSessionSetReassemblyDepth(ssn, FileReassemblyDepth());
221     }
222     if (p->flowflags & FLOW_PKT_TOCLIENT)
223         flags |= STREAM_TOCLIENT;
224     else
225         flags |= STREAM_TOSERVER;
226 
227     for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
228         AppLayerParserSetStreamDepthFlag(p->flow->proto, p->flow->alproto,
229                                          FlowGetAppState(p->flow),
230                                          det_ctx->filestore[u].tx_id,
231                                          flags);
232     }
233 
234     FileContainer *ffc = AppLayerParserGetFiles(p->flow, flags);
235 
236     /* filestore for single files only */
237     if (s->filestore_ctx == NULL) {
238         for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
239             FileStoreFileById(ffc, det_ctx->filestore[u].file_id);
240         }
241     } else {
242         for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
243             FilestorePostMatchWithOptions(p, p->flow, s->filestore_ctx, ffc,
244                     det_ctx->filestore[u].file_id, det_ctx->filestore[u].tx_id);
245         }
246     }
247 
248     SCReturnInt(0);
249 }
250 
251 /**
252  * \brief match the specified filestore
253  *
254  * \param t thread local vars
255  * \param det_ctx pattern matcher thread local data
256  * \param f *LOCKED* flow
257  * \param flags direction flags
258  * \param file file being inspected
259  * \param s signature being inspected
260  * \param m sigmatch that we will cast into DetectFilestoreData
261  *
262  * \retval 0 no match
263  * \retval 1 match
264  *
265  * \todo when we start supporting more protocols, the logic in this function
266  *       needs to be put behind a api.
267  */
DetectFilestoreMatch(DetectEngineThreadCtx * det_ctx,Flow * f,uint8_t flags,File * file,const Signature * s,const SigMatchCtx * m)268 static int DetectFilestoreMatch (DetectEngineThreadCtx *det_ctx, Flow *f,
269         uint8_t flags, File *file, const Signature *s, const SigMatchCtx *m)
270 {
271     uint32_t file_id = 0;
272 
273     SCEnter();
274 
275     if (det_ctx->filestore_cnt >= DETECT_FILESTORE_MAX) {
276         SCReturnInt(1);
277     }
278 
279     /* file can be NULL when a rule with filestore scope > file
280      * matches. */
281     if (file != NULL) {
282         file_id = file->file_track_id;
283         if (file->sid != NULL && s->id > 0) {
284             if (file->sid_cnt >= file->sid_max) {
285                 void *p = SCRealloc(file->sid, sizeof(uint32_t) * (file->sid_max + 8));
286                 if (p == NULL) {
287                     SCFree(file->sid);
288                     file->sid = NULL;
289                     file->sid_cnt = 0;
290                     file->sid_max = 0;
291                     goto continue_after_realloc_fail;
292                 } else {
293                     file->sid = p;
294                     file->sid_max += 8;
295                 }
296             }
297             file->sid[file->sid_cnt] = s->id;
298             file->sid_cnt++;
299         }
300     }
301 
302 continue_after_realloc_fail:
303 
304     det_ctx->filestore[det_ctx->filestore_cnt].file_id = file_id;
305     det_ctx->filestore[det_ctx->filestore_cnt].tx_id = det_ctx->tx_id;
306 
307     SCLogDebug("%u, file %u, tx %"PRIu64, det_ctx->filestore_cnt,
308         det_ctx->filestore[det_ctx->filestore_cnt].file_id,
309         det_ctx->filestore[det_ctx->filestore_cnt].tx_id);
310 
311     det_ctx->filestore_cnt++;
312     SCReturnInt(1);
313 }
314 
315 /**
316  * \brief this function is used to parse filestore options
317  * \brief into the current signature
318  *
319  * \param de_ctx pointer to the Detection Engine Context
320  * \param s pointer to the Current Signature
321  * \param str pointer to the user provided "filestore" option
322  *
323  * \retval 0 on Success
324  * \retval -1 on Failure
325  */
DetectFilestoreSetup(DetectEngineCtx * de_ctx,Signature * s,const char * str)326 static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, const char *str)
327 {
328     SCEnter();
329 
330     static bool warn_not_configured = false;
331     static uint32_t de_version = 0;
332 
333     /* Check on first-time loads (includes following a reload) */
334     if (!warn_not_configured || (de_ctx->version != de_version)) {
335         if (de_version != de_ctx->version) {
336             SCLogDebug("reload-detected; re-checking feature presence; DE version now %"PRIu32,
337                        de_ctx->version);
338         }
339         if (!RequiresFeature(FEATURE_OUTPUT_FILESTORE)) {
340             SCLogWarning(SC_WARN_ALERT_CONFIG, "One or more rule(s) depends on the "
341                          "file-store output log which is not enabled. "
342                          "Enable the output \"file-store\".");
343         }
344         warn_not_configured = true;
345         de_version = de_ctx->version;
346     }
347 
348     DetectFilestoreData *fd = NULL;
349     SigMatch *sm = NULL;
350     char *args[3] = {NULL,NULL,NULL};
351     int ret = 0, res = 0;
352     int ov[MAX_SUBSTRINGS];
353 
354     /* filestore and bypass keywords can't work together */
355     if (s->flags & SIG_FLAG_BYPASS) {
356         SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS,
357                    "filestore can't work with bypass keyword");
358         return -1;
359     }
360 
361     sm = SigMatchAlloc();
362     if (sm == NULL)
363         goto error;
364 
365     sm->type = DETECT_FILESTORE;
366 
367     if (str != NULL && strlen(str) > 0) {
368         char str_0[32];
369         char str_1[32];
370         char str_2[32];
371         SCLogDebug("str %s", str);
372 
373         ret = DetectParsePcreExec(&parse_regex, str, 0, 0, ov, MAX_SUBSTRINGS);
374         if (ret < 1 || ret > 4) {
375             SCLogError(SC_ERR_PCRE_MATCH, "parse error, ret %" PRId32 ", string %s", ret, str);
376             goto error;
377         }
378 
379         if (ret > 1) {
380             res = pcre_copy_substring((char *)str, ov, MAX_SUBSTRINGS, 1, str_0, sizeof(str_0));
381             if (res < 0) {
382                 SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed");
383                 goto error;
384             }
385             args[0] = (char *)str_0;
386 
387             if (ret > 2) {
388                 res = pcre_copy_substring((char *)str, ov, MAX_SUBSTRINGS, 2, str_1, sizeof(str_1));
389                 if (res < 0) {
390                     SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed");
391                     goto error;
392                 }
393                 args[1] = (char *)str_1;
394             }
395             if (ret > 3) {
396                 res = pcre_copy_substring((char *)str, ov, MAX_SUBSTRINGS, 3, str_2, sizeof(str_2));
397                 if (res < 0) {
398                     SCLogError(SC_ERR_PCRE_COPY_SUBSTRING, "pcre_copy_substring failed");
399                     goto error;
400                 }
401                 args[2] = (char *)str_2;
402             }
403         }
404 
405         fd = SCMalloc(sizeof(DetectFilestoreData));
406         if (unlikely(fd == NULL))
407             goto error;
408         memset(fd, 0x00, sizeof(DetectFilestoreData));
409 
410         if (args[0] != NULL) {
411             SCLogDebug("first arg %s", args[0]);
412 
413             if (strcasecmp(args[0], "request") == 0 ||
414                     strcasecmp(args[0], "to_server") == 0)
415             {
416                 fd->direction = FILESTORE_DIR_TOSERVER;
417                 fd->scope = FILESTORE_SCOPE_TX;
418             }
419             else if (strcasecmp(args[0], "response") == 0 ||
420                     strcasecmp(args[0], "to_client") == 0)
421             {
422                 fd->direction = FILESTORE_DIR_TOCLIENT;
423                 fd->scope = FILESTORE_SCOPE_TX;
424             }
425             else if (strcasecmp(args[0], "both") == 0)
426             {
427                 fd->direction = FILESTORE_DIR_BOTH;
428                 fd->scope = FILESTORE_SCOPE_TX;
429             }
430         } else {
431             fd->direction = FILESTORE_DIR_DEFAULT;
432         }
433 
434         if (args[1] != NULL) {
435             SCLogDebug("second arg %s", args[1]);
436 
437             if (strcasecmp(args[1], "file") == 0)
438             {
439                 fd->scope = FILESTORE_SCOPE_DEFAULT;
440             } else if (strcasecmp(args[1], "tx") == 0)
441             {
442                 fd->scope = FILESTORE_SCOPE_TX;
443             } else if (strcasecmp(args[1], "ssn") == 0 ||
444                        strcasecmp(args[1], "flow") == 0)
445             {
446                 fd->scope = FILESTORE_SCOPE_SSN;
447             }
448         } else {
449             if (fd->scope == 0)
450                 fd->scope = FILESTORE_SCOPE_DEFAULT;
451         }
452 
453         sm->ctx = (SigMatchCtx*)fd;
454     } else {
455         sm->ctx = (SigMatchCtx*)NULL;
456     }
457 
458     if (s->alproto == ALPROTO_HTTP) {
459         AppLayerHtpNeedFileInspection();
460     }
461 
462     SigMatchAppendSMToList(s, sm, g_file_match_list_id);
463     s->filestore_ctx = (const DetectFilestoreData *)sm->ctx;
464 
465     sm = SigMatchAlloc();
466     if (unlikely(sm == NULL))
467         goto error;
468     sm->type = DETECT_FILESTORE_POSTMATCH;
469     sm->ctx = NULL;
470     SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_POSTMATCH);
471 
472 
473     s->flags |= SIG_FLAG_FILESTORE;
474     return 0;
475 
476 error:
477     if (sm != NULL)
478         SCFree(sm);
479     return -1;
480 }
481 
DetectFilestoreFree(DetectEngineCtx * de_ctx,void * ptr)482 static void DetectFilestoreFree(DetectEngineCtx *de_ctx, void *ptr)
483 {
484     if (ptr != NULL) {
485         SCFree(ptr);
486     }
487 }
488 
489 #ifdef UNITTESTS
490 /*
491  * The purpose of this test is to confirm that
492  * filestore and bypass keywords can't
493  * can't work together
494  */
DetectFilestoreTest01(void)495 static int DetectFilestoreTest01(void)
496 {
497     DetectEngineCtx *de_ctx = NULL;
498     int result = 1;
499 
500     de_ctx = DetectEngineCtxInit();
501     FAIL_IF(de_ctx == NULL);
502 
503     de_ctx->flags |= DE_QUIET;
504 
505     de_ctx->sig_list = SigInit(de_ctx,"alert http any any -> any any "
506                                "(bypass; filestore; "
507                                "content:\"message\"; http_host; "
508                                "sid:1;)");
509     FAIL_IF_NOT_NULL(de_ctx->sig_list);
510 
511     DetectEngineCtxFree(de_ctx);
512 
513     return result;
514 }
515 
DetectFilestoreRegisterTests(void)516 void DetectFilestoreRegisterTests(void)
517 {
518     UtRegisterTest("DetectFilestoreTest01", DetectFilestoreTest01);
519 }
520 #endif /* UNITTESTS */
521