1 /* Copyright (C) 2007-2021 Open Information Security Foundation
2 *
3 * You can copy, redistribute or modify this Program under the terms of
4 * the GNU General Public License version 2 as published by the Free
5 * Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 * You should have received a copy of the GNU General Public License
13 * version 2 along with this program; if not, write to the Free Software
14 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15 * 02110-1301, USA.
16 */
17
18 /**
19 * \file
20 *
21 * \author Victor Julien <victor@inliniac.net>
22 *
23 */
24
25 #include "suricata-common.h"
26 #include "threads.h"
27 #include "debug.h"
28 #include "decode.h"
29
30 #include "detect.h"
31 #include "detect-parse.h"
32
33 #include "detect-engine.h"
34 #include "detect-engine-mpm.h"
35 #include "detect-engine-state.h"
36 #include "detect-engine-prefilter.h"
37 #include "detect-engine-content-inspection.h"
38 #include "detect-file-data.h"
39
40 #include "app-layer-parser.h"
41 #include "app-layer-htp.h"
42 #include "app-layer-smtp.h"
43
44 #include "flow.h"
45 #include "flow-var.h"
46 #include "flow-util.h"
47
48 #include "util-debug.h"
49 #include "util-spm-bm.h"
50 #include "util-unittest.h"
51 #include "util-unittest-helper.h"
52 #include "util-file-decompression.h"
53
54 static int DetectFiledataSetup (DetectEngineCtx *, Signature *, const char *);
55 #ifdef UNITTESTS
56 static void DetectFiledataRegisterTests(void);
57 #endif
58 static void DetectFiledataSetupCallback(const DetectEngineCtx *de_ctx,
59 Signature *s);
60 static int g_file_data_buffer_id = 0;
61
62 static inline HtpBody *GetResponseBody(htp_tx_t *tx);
63
64 /* HTTP */
65 static int PrefilterMpmHTTPFiledataRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
66 MpmCtx *mpm_ctx, const DetectBufferMpmRegistery *mpm_reg, int list_id);
67
68 /* file API */
69 static int DetectEngineInspectFiledata(
70 DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
71 const DetectEngineAppInspectionEngine *engine,
72 const Signature *s,
73 Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
74 int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx,
75 SigGroupHead *sgh, MpmCtx *mpm_ctx,
76 const DetectBufferMpmRegistery *mpm_reg, int list_id);
77
78 static int DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx,
79 DetectEngineThreadCtx *det_ctx, const DetectEngineAppInspectionEngine *engine,
80 const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
81
82 /**
83 * \brief Registration function for keyword: file_data
84 */
DetectFiledataRegister(void)85 void DetectFiledataRegister(void)
86 {
87 sigmatch_table[DETECT_FILE_DATA].name = "file.data";
88 sigmatch_table[DETECT_FILE_DATA].alias = "file_data";
89 sigmatch_table[DETECT_FILE_DATA].desc = "make content keywords match on file data";
90 sigmatch_table[DETECT_FILE_DATA].url = "/rules/http-keywords.html#file-data";
91 sigmatch_table[DETECT_FILE_DATA].Setup = DetectFiledataSetup;
92 #ifdef UNITTESTS
93 sigmatch_table[DETECT_FILE_DATA].RegisterTests = DetectFiledataRegisterTests;
94 #endif
95 sigmatch_table[DETECT_FILE_DATA].flags = SIGMATCH_NOOPT;
96
97 DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOSERVER, 2,
98 PrefilterMpmFiledataRegister, NULL,
99 ALPROTO_SMTP, 0);
100 DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOCLIENT, 2, PrefilterMpmHTTPFiledataRegister,
101 NULL, ALPROTO_HTTP, HTP_RESPONSE_BODY);
102 DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOSERVER, 2,
103 PrefilterMpmFiledataRegister, NULL,
104 ALPROTO_SMB, 0);
105 DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOCLIENT, 2,
106 PrefilterMpmFiledataRegister, NULL,
107 ALPROTO_SMB, 0);
108 DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOSERVER, 2,
109 PrefilterMpmFiledataRegister, NULL,
110 ALPROTO_HTTP2, HTTP2StateDataClient);
111 DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOCLIENT, 2,
112 PrefilterMpmFiledataRegister, NULL,
113 ALPROTO_HTTP2, HTTP2StateDataServer);
114 DetectAppLayerInspectEngineRegister2("file_data", ALPROTO_HTTP, SIG_FLAG_TOCLIENT,
115 HTP_RESPONSE_BODY, DetectEngineInspectBufferHttpBody, NULL);
116 DetectAppLayerInspectEngineRegister2("file_data",
117 ALPROTO_SMTP, SIG_FLAG_TOSERVER, 0,
118 DetectEngineInspectFiledata, NULL);
119 DetectBufferTypeRegisterSetupCallback("file_data",
120 DetectFiledataSetupCallback);
121 DetectAppLayerInspectEngineRegister2("file_data",
122 ALPROTO_SMB, SIG_FLAG_TOSERVER, 0,
123 DetectEngineInspectFiledata, NULL);
124 DetectAppLayerInspectEngineRegister2("file_data",
125 ALPROTO_SMB, SIG_FLAG_TOCLIENT, 0,
126 DetectEngineInspectFiledata, NULL);
127 DetectAppLayerInspectEngineRegister2("file_data",
128 ALPROTO_HTTP2, SIG_FLAG_TOSERVER, HTTP2StateDataClient,
129 DetectEngineInspectFiledata, NULL);
130 DetectAppLayerInspectEngineRegister2("file_data",
131 ALPROTO_HTTP2, SIG_FLAG_TOCLIENT, HTTP2StateDataServer,
132 DetectEngineInspectFiledata, NULL);
133
134 DetectBufferTypeSetDescriptionByName("file_data",
135 "http response body, smb files or smtp attachments data");
136
137 g_file_data_buffer_id = DetectBufferTypeGetByName("file_data");
138 }
139
140 #define FILEDATA_CONTENT_LIMIT 100000
141 #define FILEDATA_CONTENT_INSPECT_MIN_SIZE 32768
142 #define FILEDATA_CONTENT_INSPECT_WINDOW 4096
143
SetupDetectEngineConfig(DetectEngineCtx * de_ctx)144 static void SetupDetectEngineConfig(DetectEngineCtx *de_ctx) {
145 if (de_ctx->filedata_config_initialized)
146 return;
147
148 /* initialize default */
149 for (int i = 0; i < (int)ALPROTO_MAX; i++) {
150 de_ctx->filedata_config[i].content_limit = FILEDATA_CONTENT_LIMIT;
151 de_ctx->filedata_config[i].content_inspect_min_size = FILEDATA_CONTENT_INSPECT_MIN_SIZE;
152 de_ctx->filedata_config[i].content_inspect_window = FILEDATA_CONTENT_INSPECT_WINDOW;
153 }
154
155 /* add protocol specific settings here */
156
157 /* SMTP */
158 de_ctx->filedata_config[ALPROTO_SMTP].content_limit = smtp_config.content_limit;
159 de_ctx->filedata_config[ALPROTO_SMTP].content_inspect_min_size = smtp_config.content_inspect_min_size;
160 de_ctx->filedata_config[ALPROTO_SMTP].content_inspect_window = smtp_config.content_inspect_window;
161
162 de_ctx->filedata_config_initialized = true;
163 }
164
165 /**
166 * \brief this function is used to parse filedata options
167 * \brief into the current signature
168 *
169 * \param de_ctx pointer to the Detection Engine Context
170 * \param s pointer to the Current Signature
171 * \param str pointer to the user provided "filestore" option
172 *
173 * \retval 0 on Success
174 * \retval -1 on Failure
175 */
DetectFiledataSetup(DetectEngineCtx * de_ctx,Signature * s,const char * str)176 static int DetectFiledataSetup (DetectEngineCtx *de_ctx, Signature *s, const char *str)
177 {
178 SCEnter();
179
180 if (!DetectProtoContainsProto(&s->proto, IPPROTO_TCP) ||
181 (s->alproto != ALPROTO_UNKNOWN && s->alproto != ALPROTO_HTTP &&
182 s->alproto != ALPROTO_SMTP && s->alproto != ALPROTO_SMB &&
183 s->alproto != ALPROTO_HTTP2)) {
184 SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rule contains conflicting keywords.");
185 return -1;
186 }
187
188 if (s->alproto == ALPROTO_HTTP && (s->init_data->init_flags & SIG_FLAG_INIT_FLOW) &&
189 (s->flags & SIG_FLAG_TOSERVER) && !(s->flags & SIG_FLAG_TOCLIENT)) {
190 SCLogError(SC_ERR_INVALID_SIGNATURE, "Can't use file_data with "
191 "flow:to_server or flow:from_client with http.");
192 return -1;
193 }
194
195 if (s->alproto == ALPROTO_SMTP && (s->init_data->init_flags & SIG_FLAG_INIT_FLOW) &&
196 !(s->flags & SIG_FLAG_TOSERVER) && (s->flags & SIG_FLAG_TOCLIENT)) {
197 SCLogError(SC_ERR_INVALID_SIGNATURE, "Can't use file_data with "
198 "flow:to_client or flow:from_server with smtp.");
199 return -1;
200 }
201
202 if (DetectBufferSetActiveList(s, DetectBufferTypeGetByName("file_data")) < 0)
203 return -1;
204
205 s->init_data->init_flags |= SIG_FLAG_INIT_FILEDATA;
206 SetupDetectEngineConfig(de_ctx);
207 return 0;
208 }
209
DetectFiledataSetupCallback(const DetectEngineCtx * de_ctx,Signature * s)210 static void DetectFiledataSetupCallback(const DetectEngineCtx *de_ctx,
211 Signature *s)
212 {
213 if (s->alproto == ALPROTO_HTTP || s->alproto == ALPROTO_UNKNOWN) {
214 AppLayerHtpEnableResponseBodyCallback();
215 }
216
217
218 /* server body needs to be inspected in sync with stream if possible */
219 s->init_data->init_flags |= SIG_FLAG_INIT_NEED_FLUSH;
220
221 SCLogDebug("callback invoked by %u", s->id);
222 }
223
224 /* common */
225
226 typedef struct PrefilterMpmFiledata {
227 int list_id;
228 int base_list_id;
229 const MpmCtx *mpm_ctx;
230 const DetectEngineTransforms *transforms;
231 } PrefilterMpmFiledata;
232
PrefilterMpmFiledataFree(void * ptr)233 static void PrefilterMpmFiledataFree(void *ptr)
234 {
235 SCFree(ptr);
236 }
237
238 /* HTTP based detection */
239
GetResponseBody(htp_tx_t * tx)240 static inline HtpBody *GetResponseBody(htp_tx_t *tx)
241 {
242 HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
243 if (htud == NULL) {
244 SCLogDebug("no htud");
245 return NULL;
246 }
247
248 return &htud->response_body;
249 }
250
HttpServerBodyXformsGetDataCallback(DetectEngineThreadCtx * det_ctx,const DetectEngineTransforms * transforms,const int list_id,InspectionBuffer * base_buffer)251 static inline InspectionBuffer *HttpServerBodyXformsGetDataCallback(DetectEngineThreadCtx *det_ctx,
252 const DetectEngineTransforms *transforms, const int list_id, InspectionBuffer *base_buffer)
253 {
254 InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
255 if (buffer->inspect != NULL)
256 return buffer;
257
258 InspectionBufferSetup(det_ctx, list_id, buffer, base_buffer->inspect, base_buffer->inspect_len);
259 buffer->inspect_offset = base_buffer->inspect_offset;
260 InspectionBufferApplyTransforms(buffer, transforms);
261 SCLogDebug("xformed buffer %p size %u", buffer, buffer->inspect_len);
262 SCReturnPtr(buffer, "InspectionBuffer");
263 }
264
HttpServerBodyGetDataCallback(DetectEngineThreadCtx * det_ctx,const DetectEngineTransforms * transforms,Flow * f,const uint8_t flow_flags,void * txv,const int list_id,const int base_id)265 static InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
266 const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv,
267 const int list_id, const int base_id)
268 {
269 SCEnter();
270
271 InspectionBuffer *buffer = InspectionBufferGet(det_ctx, base_id);
272 if (base_id != list_id && buffer->inspect != NULL)
273 return HttpServerBodyXformsGetDataCallback(det_ctx, transforms, list_id, buffer);
274 else if (buffer->inspect != NULL)
275 return buffer;
276
277 htp_tx_t *tx = txv;
278 HtpState *htp_state = f->alstate;
279 const uint8_t flags = flow_flags;
280
281 HtpBody *body = GetResponseBody(tx);
282 if (body == NULL) {
283 return NULL;
284 }
285
286 /* no new data */
287 if (body->body_inspected == body->content_len_so_far) {
288 SCLogDebug("no new data");
289 return NULL;
290 }
291
292 HtpBodyChunk *cur = body->first;
293 if (cur == NULL) {
294 SCLogDebug("No http chunks to inspect for this transaction");
295 return NULL;
296 }
297
298 SCLogDebug("response.body_limit %u response_body.content_len_so_far %"PRIu64
299 ", response.inspect_min_size %"PRIu32", EOF %s, progress > body? %s",
300 htp_state->cfg->response.body_limit,
301 body->content_len_so_far,
302 htp_state->cfg->response.inspect_min_size,
303 flags & STREAM_EOF ? "true" : "false",
304 (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_BODY) ? "true" : "false");
305
306 if (!htp_state->cfg->http_body_inline) {
307 /* inspect the body if the transfer is complete or we have hit
308 * our body size limit */
309 if ((htp_state->cfg->response.body_limit == 0 ||
310 body->content_len_so_far < htp_state->cfg->response.body_limit) &&
311 body->content_len_so_far < htp_state->cfg->response.inspect_min_size &&
312 !(AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_BODY) &&
313 !(flags & STREAM_EOF)) {
314 SCLogDebug("we still haven't seen the entire response body. "
315 "Let's defer body inspection till we see the "
316 "entire body.");
317 return NULL;
318 }
319 }
320
321 /* get the inspect buffer
322 *
323 * make sure that we have at least the configured inspect_win size.
324 * If we have more, take at least 1/4 of the inspect win size before
325 * the new data.
326 */
327 uint64_t offset = 0;
328 if (body->body_inspected > htp_state->cfg->response.inspect_min_size) {
329 BUG_ON(body->content_len_so_far < body->body_inspected);
330 uint64_t inspect_win = body->content_len_so_far - body->body_inspected;
331 SCLogDebug("inspect_win %"PRIu64, inspect_win);
332 if (inspect_win < htp_state->cfg->response.inspect_window) {
333 uint64_t inspect_short = htp_state->cfg->response.inspect_window - inspect_win;
334 if (body->body_inspected < inspect_short)
335 offset = 0;
336 else
337 offset = body->body_inspected - inspect_short;
338 } else {
339 offset = body->body_inspected - (htp_state->cfg->response.inspect_window / 4);
340 }
341 }
342
343 const uint8_t *data;
344 uint32_t data_len;
345
346 StreamingBufferGetDataAtOffset(body->sb,
347 &data, &data_len, offset);
348 InspectionBufferSetup(det_ctx, base_id, buffer, data, data_len);
349 buffer->inspect_offset = offset;
350 body->body_inspected = body->content_len_so_far;
351 SCLogDebug("body->body_inspected now: %" PRIu64, body->body_inspected);
352
353 /* built-in 'transformation' */
354 if (htp_state->cfg->swf_decompression_enabled) {
355 int swf_file_type = FileIsSwfFile(data, data_len);
356 if (swf_file_type == FILE_SWF_ZLIB_COMPRESSION ||
357 swf_file_type == FILE_SWF_LZMA_COMPRESSION)
358 {
359 (void)FileSwfDecompression(data, data_len,
360 det_ctx,
361 buffer,
362 htp_state->cfg->swf_compression_type,
363 htp_state->cfg->swf_decompress_depth,
364 htp_state->cfg->swf_compress_depth);
365 }
366 }
367
368 if (base_id != list_id) {
369 buffer = HttpServerBodyXformsGetDataCallback(det_ctx, transforms, list_id, buffer);
370 }
371 SCReturnPtr(buffer, "InspectionBuffer");
372 }
373
DetectEngineInspectBufferHttpBody(DetectEngineCtx * de_ctx,DetectEngineThreadCtx * det_ctx,const DetectEngineAppInspectionEngine * engine,const Signature * s,Flow * f,uint8_t flags,void * alstate,void * txv,uint64_t tx_id)374 static int DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx,
375 DetectEngineThreadCtx *det_ctx, const DetectEngineAppInspectionEngine *engine,
376 const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
377 {
378 bool eof =
379 (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > engine->progress);
380 const InspectionBuffer *buffer = HttpServerBodyGetDataCallback(
381 det_ctx, engine->v2.transforms, f, flags, txv, engine->sm_list, engine->sm_list_base);
382 if (buffer == NULL || buffer->inspect == NULL) {
383 return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH : DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
384 }
385
386 const uint32_t data_len = buffer->inspect_len;
387 const uint8_t *data = buffer->inspect;
388 const uint64_t offset = buffer->inspect_offset;
389
390 uint8_t ci_flags = eof ? DETECT_CI_FLAGS_END : 0;
391 ci_flags |= (offset == 0 ? DETECT_CI_FLAGS_START : 0);
392 ci_flags |= buffer->flags;
393
394 det_ctx->discontinue_matching = 0;
395 det_ctx->buffer_offset = 0;
396 det_ctx->inspection_recursion_counter = 0;
397
398 /* Inspect all the uricontents fetched on each
399 * transaction at the app layer */
400 int r = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, NULL, f, (uint8_t *)data,
401 data_len, offset, ci_flags, DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE);
402 if (r == 1) {
403 return DETECT_ENGINE_INSPECT_SIG_MATCH;
404 }
405
406 if (flags & STREAM_TOSERVER) {
407 if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, txv, flags) >
408 HTP_REQUEST_BODY)
409 return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
410 } else {
411 if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, txv, flags) >
412 HTP_RESPONSE_BODY)
413 return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
414 }
415 return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
416 }
417
418 /** \brief Filedata Filedata Mpm prefilter callback
419 *
420 * \param det_ctx detection engine thread ctx
421 * \param pectx inspection context
422 * \param p packet to inspect
423 * \param f flow to inspect
424 * \param txv tx to inspect
425 * \param idx transaction id
426 * \param flags STREAM_* flags including direction
427 */
PrefilterTxHTTPFiledata(DetectEngineThreadCtx * det_ctx,const void * pectx,Packet * p,Flow * f,void * txv,const uint64_t idx,const uint8_t flags)428 static void PrefilterTxHTTPFiledata(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p,
429 Flow *f, void *txv, const uint64_t idx, const uint8_t flags)
430 {
431 SCEnter();
432
433 const PrefilterMpmFiledata *ctx = (const PrefilterMpmFiledata *)pectx;
434 const MpmCtx *mpm_ctx = ctx->mpm_ctx;
435 const int list_id = ctx->list_id;
436
437 InspectionBuffer *buffer = HttpServerBodyGetDataCallback(
438 det_ctx, ctx->transforms, f, flags, txv, list_id, ctx->base_list_id);
439 if (buffer == NULL)
440 return;
441
442 if (buffer->inspect_len >= mpm_ctx->minlen) {
443 (void)mpm_table[mpm_ctx->mpm_type].Search(
444 mpm_ctx, &det_ctx->mtcu, &det_ctx->pmq, buffer->inspect, buffer->inspect_len);
445 }
446 }
447
PrefilterMpmHTTPFiledataRegister(DetectEngineCtx * de_ctx,SigGroupHead * sgh,MpmCtx * mpm_ctx,const DetectBufferMpmRegistery * mpm_reg,int list_id)448 static int PrefilterMpmHTTPFiledataRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
449 MpmCtx *mpm_ctx, const DetectBufferMpmRegistery *mpm_reg, int list_id)
450 {
451 PrefilterMpmFiledata *pectx = SCCalloc(1, sizeof(*pectx));
452 if (pectx == NULL)
453 return -1;
454 pectx->list_id = list_id;
455 pectx->base_list_id = mpm_reg->sm_list_base;
456 SCLogDebug("list_id %d base_list_id %d", list_id, pectx->base_list_id);
457 pectx->mpm_ctx = mpm_ctx;
458 pectx->transforms = &mpm_reg->transforms;
459
460 return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxHTTPFiledata, mpm_reg->app_v2.alproto,
461 mpm_reg->app_v2.tx_min_progress, pectx, PrefilterMpmFiledataFree, mpm_reg->pname);
462 }
463
464 /* file API based inspection */
465
FiledataWithXformsGetDataCallback(DetectEngineThreadCtx * det_ctx,const DetectEngineTransforms * transforms,const int list_id,int local_file_id,InspectionBuffer * base_buffer,const bool first)466 static inline InspectionBuffer *FiledataWithXformsGetDataCallback(DetectEngineThreadCtx *det_ctx,
467 const DetectEngineTransforms *transforms, const int list_id, int local_file_id,
468 InspectionBuffer *base_buffer, const bool first)
469 {
470 InspectionBuffer *buffer = InspectionBufferMultipleForListGet(det_ctx, list_id, local_file_id);
471 if (buffer == NULL) {
472 SCLogDebug("list_id: %d: no buffer", list_id);
473 return NULL;
474 }
475 if (!first && buffer->inspect != NULL) {
476 SCLogDebug("list_id: %d: returning %p", list_id, buffer);
477 return buffer;
478 }
479
480 InspectionBufferSetupMulti(buffer, transforms, base_buffer->inspect, base_buffer->inspect_len);
481 buffer->inspect_offset = base_buffer->inspect_offset;
482 SCLogDebug("xformed buffer %p size %u", buffer, buffer->inspect_len);
483 SCReturnPtr(buffer, "InspectionBuffer");
484 }
485
FiledataGetDataCallback(DetectEngineThreadCtx * det_ctx,const DetectEngineTransforms * transforms,Flow * f,uint8_t flow_flags,File * cur_file,const int list_id,const int base_id,int local_file_id,bool first)486 static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx,
487 const DetectEngineTransforms *transforms, Flow *f, uint8_t flow_flags, File *cur_file,
488 const int list_id, const int base_id, int local_file_id, bool first)
489 {
490 SCEnter();
491 SCLogDebug(
492 "starting: list_id %d base_id %d first %s", list_id, base_id, first ? "true" : "false");
493
494 InspectionBuffer *buffer = InspectionBufferMultipleForListGet(det_ctx, base_id, local_file_id);
495 SCLogDebug("base: buffer %p", buffer);
496 if (buffer == NULL)
497 return NULL;
498 if (base_id != list_id && buffer->inspect != NULL) {
499 SCLogDebug("handle xform %s", (list_id != base_id) ? "true" : "false");
500 return FiledataWithXformsGetDataCallback(
501 det_ctx, transforms, list_id, local_file_id, buffer, first);
502 }
503 if (!first && buffer->inspect != NULL) {
504 SCLogDebug("base_id: %d, not first: use %p", base_id, buffer);
505 return buffer;
506 }
507
508 const uint64_t file_size = FileDataSize(cur_file);
509 const DetectEngineCtx *de_ctx = det_ctx->de_ctx;
510 const uint32_t content_limit = de_ctx->filedata_config[f->alproto].content_limit;
511 const uint32_t content_inspect_min_size = de_ctx->filedata_config[f->alproto].content_inspect_min_size;
512 // TODO this is unused, is that right?
513 //const uint32_t content_inspect_window = de_ctx->filedata_config[f->alproto].content_inspect_window;
514
515 SCLogDebug("[list %d] first: %d, content_limit %u, content_inspect_min_size %u", list_id,
516 first ? 1 : 0, content_limit, content_inspect_min_size);
517
518 SCLogDebug("[list %d] file %p size %" PRIu64 ", state %d", list_id, cur_file, file_size,
519 cur_file->state);
520
521 /* no new data */
522 if (cur_file->content_inspected == file_size) {
523 SCLogDebug("no new data");
524 return NULL;
525 }
526
527 if (file_size == 0) {
528 SCLogDebug("no data to inspect for this transaction");
529 return NULL;
530 }
531
532 if ((content_limit == 0 || file_size < content_limit) &&
533 file_size < content_inspect_min_size &&
534 !(flow_flags & STREAM_EOF) && !(cur_file->state > FILE_STATE_OPENED)) {
535 SCLogDebug("we still haven't seen the entire content. "
536 "Let's defer content inspection till we see the "
537 "entire content.");
538 return NULL;
539 }
540
541 const uint8_t *data;
542 uint32_t data_len;
543
544 StreamingBufferGetDataAtOffset(cur_file->sb,
545 &data, &data_len,
546 cur_file->content_inspected);
547 InspectionBufferSetupMulti(buffer, NULL, data, data_len);
548 SCLogDebug("[list %d] [before] buffer offset %" PRIu64 "; buffer len %" PRIu32
549 "; data_len %" PRIu32 "; file_size %" PRIu64,
550 list_id, buffer->inspect_offset, buffer->inspect_len, data_len, file_size);
551 buffer->inspect_offset = cur_file->content_inspected;
552
553 /* update inspected tracker */
554 cur_file->content_inspected = buffer->inspect_len + buffer->inspect_offset;
555 SCLogDebug("content inspected: %" PRIu64, cur_file->content_inspected);
556
557 /* get buffer for the list id if it is different from the base id */
558 if (list_id != base_id) {
559 SCLogDebug("regular %d has been set up: now handle xforms id %d", base_id, list_id);
560 InspectionBuffer *tbuffer = FiledataWithXformsGetDataCallback(
561 det_ctx, transforms, list_id, local_file_id, buffer, first);
562 SCReturnPtr(tbuffer, "InspectionBuffer");
563 } else {
564 SCLogDebug("regular buffer %p size %u", buffer, buffer->inspect_len);
565 SCReturnPtr(buffer, "InspectionBuffer");
566 }
567 }
568
DetectEngineInspectFiledata(DetectEngineCtx * de_ctx,DetectEngineThreadCtx * det_ctx,const DetectEngineAppInspectionEngine * engine,const Signature * s,Flow * f,uint8_t flags,void * alstate,void * txv,uint64_t tx_id)569 static int DetectEngineInspectFiledata(
570 DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
571 const DetectEngineAppInspectionEngine *engine,
572 const Signature *s,
573 Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
574 {
575 int r = 0;
576 int match = 0;
577
578 const DetectEngineTransforms *transforms = NULL;
579 if (!engine->mpm) {
580 transforms = engine->v2.transforms;
581 }
582
583 FileContainer *ffc = AppLayerParserGetFiles(f, flags);
584 if (ffc == NULL) {
585 return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
586 }
587
588 int local_file_id = 0;
589 File *file = ffc->head;
590 for (; file != NULL; file = file->next) {
591 if (file->txid != tx_id)
592 continue;
593
594 InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, transforms, f, flags, file,
595 engine->sm_list, engine->sm_list_base, local_file_id, false);
596 if (buffer == NULL)
597 continue;
598
599 bool eof = (file->state == FILE_STATE_CLOSED);
600 uint8_t ciflags = eof ? DETECT_CI_FLAGS_END : 0;
601 if (buffer->inspect_offset == 0)
602 ciflags |= DETECT_CI_FLAGS_START;
603
604 det_ctx->buffer_offset = 0;
605 det_ctx->discontinue_matching = 0;
606 det_ctx->inspection_recursion_counter = 0;
607 match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd,
608 NULL, f,
609 (uint8_t *)buffer->inspect,
610 buffer->inspect_len,
611 buffer->inspect_offset, ciflags,
612 DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE);
613 if (match == 1) {
614 r = 1;
615 break;
616 }
617 local_file_id++;
618 }
619
620 if (r == 1)
621 return DETECT_ENGINE_INSPECT_SIG_MATCH;
622 else
623 return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
624 }
625
626 /** \brief Filedata Filedata Mpm prefilter callback
627 *
628 * \param det_ctx detection engine thread ctx
629 * \param pectx inspection context
630 * \param p packet to inspect
631 * \param f flow to inspect
632 * \param txv tx to inspect
633 * \param idx transaction id
634 * \param flags STREAM_* flags including direction
635 */
PrefilterTxFiledata(DetectEngineThreadCtx * det_ctx,const void * pectx,Packet * p,Flow * f,void * txv,const uint64_t idx,const uint8_t flags)636 static void PrefilterTxFiledata(DetectEngineThreadCtx *det_ctx,
637 const void *pectx,
638 Packet *p, Flow *f, void *txv,
639 const uint64_t idx, const uint8_t flags)
640 {
641 SCEnter();
642
643 const PrefilterMpmFiledata *ctx = (const PrefilterMpmFiledata *)pectx;
644 const MpmCtx *mpm_ctx = ctx->mpm_ctx;
645 const int list_id = ctx->list_id;
646
647 FileContainer *ffc = AppLayerParserGetFiles(f, flags);
648 int local_file_id = 0;
649 if (ffc != NULL) {
650 File *file = ffc->head;
651 for (; file != NULL; file = file->next) {
652 if (file->txid != idx)
653 continue;
654
655 InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, ctx->transforms, f, flags,
656 file, list_id, ctx->base_list_id, local_file_id, true);
657 if (buffer == NULL)
658 continue;
659
660 if (buffer->inspect_len >= mpm_ctx->minlen) {
661 (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
662 &det_ctx->mtcu, &det_ctx->pmq,
663 buffer->inspect, buffer->inspect_len);
664 }
665 local_file_id++;
666 }
667 }
668 }
669
PrefilterMpmFiledataRegister(DetectEngineCtx * de_ctx,SigGroupHead * sgh,MpmCtx * mpm_ctx,const DetectBufferMpmRegistery * mpm_reg,int list_id)670 int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx,
671 SigGroupHead *sgh, MpmCtx *mpm_ctx,
672 const DetectBufferMpmRegistery *mpm_reg, int list_id)
673 {
674 PrefilterMpmFiledata *pectx = SCCalloc(1, sizeof(*pectx));
675 if (pectx == NULL)
676 return -1;
677 pectx->list_id = list_id;
678 pectx->base_list_id = mpm_reg->sm_list_base;
679 pectx->mpm_ctx = mpm_ctx;
680 pectx->transforms = &mpm_reg->transforms;
681
682 return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxFiledata,
683 mpm_reg->app_v2.alproto, mpm_reg->app_v2.tx_min_progress,
684 pectx, PrefilterMpmFiledataFree, mpm_reg->pname);
685 }
686
687 #ifdef UNITTESTS
688 #include "tests/detect-file-data.c"
689 #endif
690