1 /* Copyright (C) 2007-2021 Open Information Security Foundation
2 *
3 * You can copy, redistribute or modify this Program under the terms of
4 * the GNU General Public License version 2 as published by the Free
5 * Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 * You should have received a copy of the GNU General Public License
13 * version 2 along with this program; if not, write to the Free Software
14 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15 * 02110-1301, USA.
16 */
17
18 /**
19 * \file
20 *
21 * \author Victor Julien <victor@inliniac.net>
22 *
23 * AppLayer Filedata Logger Output registration functions
24 */
25
26 #include "suricata-common.h"
27 #include "tm-modules.h"
28 #include "output.h"
29 #include "output-filedata.h"
30 #include "app-layer.h"
31 #include "app-layer-parser.h"
32 #include "detect-filemagic.h"
33 #include "conf.h"
34 #include "util-profiling.h"
35 #include "util-validate.h"
36 #include "util-magic.h"
37
38 bool g_filedata_logger_enabled = false;
39
40 typedef struct OutputLoggerThreadStore_ {
41 void *thread_data;
42 struct OutputLoggerThreadStore_ *next;
43 } OutputLoggerThreadStore;
44
45 /** per thread data for this module, contains a list of per thread
46 * data for the packet loggers. */
47 typedef struct OutputLoggerThreadData_ {
48 OutputLoggerThreadStore *store;
49 #ifdef HAVE_MAGIC
50 magic_t magic_ctx;
51 #endif
52 } OutputLoggerThreadData;
53
54 /* logger instance, a module + a output ctx,
55 * it's perfectly valid that have multiple instances of the same
56 * log module (e.g. http.log) with different output ctx'. */
57 typedef struct OutputFiledataLogger_ {
58 FiledataLogger LogFunc;
59 OutputCtx *output_ctx;
60 struct OutputFiledataLogger_ *next;
61 const char *name;
62 LoggerId logger_id;
63 ThreadInitFunc ThreadInit;
64 ThreadDeinitFunc ThreadDeinit;
65 ThreadExitPrintStatsFunc ThreadExitPrintStats;
66 } OutputFiledataLogger;
67
68 static OutputFiledataLogger *list = NULL;
69 static char g_waldo[PATH_MAX] = "";
70 static SCMutex g_waldo_mutex = SCMUTEX_INITIALIZER;
71 static int g_waldo_init = 0;
72 static int g_waldo_deinit = 0;
73
OutputRegisterFiledataLogger(LoggerId id,const char * name,FiledataLogger LogFunc,OutputCtx * output_ctx,ThreadInitFunc ThreadInit,ThreadDeinitFunc ThreadDeinit,ThreadExitPrintStatsFunc ThreadExitPrintStats)74 int OutputRegisterFiledataLogger(LoggerId id, const char *name,
75 FiledataLogger LogFunc, OutputCtx *output_ctx, ThreadInitFunc ThreadInit,
76 ThreadDeinitFunc ThreadDeinit,
77 ThreadExitPrintStatsFunc ThreadExitPrintStats)
78 {
79 OutputFiledataLogger *op = SCMalloc(sizeof(*op));
80 if (op == NULL)
81 return -1;
82 memset(op, 0x00, sizeof(*op));
83
84 op->LogFunc = LogFunc;
85 op->output_ctx = output_ctx;
86 op->name = name;
87 op->logger_id = id;
88 op->ThreadInit = ThreadInit;
89 op->ThreadDeinit = ThreadDeinit;
90 op->ThreadExitPrintStats = ThreadExitPrintStats;
91
92 if (list == NULL)
93 list = op;
94 else {
95 OutputFiledataLogger *t = list;
96 while (t->next)
97 t = t->next;
98 t->next = op;
99 }
100
101 SCLogDebug("OutputRegisterFiledataLogger happy");
102 g_filedata_logger_enabled = true;
103 return 0;
104 }
105
106 SC_ATOMIC_DECLARE(unsigned int, g_file_store_id);
107
CallLoggers(ThreadVars * tv,OutputLoggerThreadStore * store_list,Packet * p,File * ff,const uint8_t * data,uint32_t data_len,uint8_t flags,uint8_t dir)108 static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
109 Packet *p, File *ff,
110 const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t dir)
111 {
112 OutputFiledataLogger *logger = list;
113 OutputLoggerThreadStore *store = store_list;
114 int file_logged = 0;
115
116 while (logger && store) {
117 DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
118
119 SCLogDebug("logger %p", logger);
120 PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
121 logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, data, data_len, flags, dir);
122 PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
123
124 file_logged = 1;
125
126 logger = logger->next;
127 store = store->next;
128
129 DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
130 DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
131 }
132 return file_logged;
133 }
134
CloseFile(const Packet * p,Flow * f,File * file)135 static void CloseFile(const Packet *p, Flow *f, File *file)
136 {
137 void *txv = AppLayerParserGetTx(p->proto, f->alproto, f->alstate, file->txid);
138 if (txv) {
139 AppLayerTxData *txd = AppLayerParserGetTxData(p->proto, f->alproto, txv);
140 if (txd)
141 txd->files_stored++;
142 }
143 file->flags |= FILE_STORED;
144 }
145
OutputFiledataLogFfc(ThreadVars * tv,OutputLoggerThreadData * td,Packet * p,FileContainer * ffc,const uint8_t call_flags,const bool file_close,const bool file_trunc,const uint8_t dir)146 static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadData *td,
147 Packet *p, FileContainer *ffc, const uint8_t call_flags,
148 const bool file_close, const bool file_trunc, const uint8_t dir)
149 {
150 if (ffc != NULL) {
151 OutputLoggerThreadStore *store = td->store;
152 File *ff;
153 for (ff = ffc->head; ff != NULL; ff = ff->next) {
154 uint8_t file_flags = call_flags;
155 #ifdef HAVE_MAGIC
156 if (FileForceMagic() && ff->magic == NULL) {
157 FilemagicThreadLookup(&td->magic_ctx, ff);
158 }
159 #endif
160 SCLogDebug("ff %p", ff);
161 if (ff->flags & FILE_STORED) {
162 SCLogDebug("stored flag set");
163 continue;
164 }
165
166 if (!(ff->flags & FILE_STORE)) {
167 SCLogDebug("ff FILE_STORE not set");
168 continue;
169 }
170
171 /* if we have no data chunks left to log, we should still
172 * close the logger(s) */
173 if (FileDataSize(ff) == ff->content_stored &&
174 (file_trunc || file_close)) {
175 if (ff->state < FILE_STATE_CLOSED) {
176 FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
177 }
178 CallLoggers(tv, store, p, ff, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir);
179 CloseFile(p, p->flow, ff);
180 continue;
181 }
182
183 /* store */
184
185 /* if file_store_id == 0, this is the first store of this file */
186 if (ff->file_store_id == 0) {
187 /* new file */
188 ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1);
189 file_flags |= OUTPUT_FILEDATA_FLAG_OPEN;
190 } else {
191 /* existing file */
192 }
193
194 /* if file needs to be closed or truncated, inform
195 * loggers */
196 if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
197 FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
198 }
199
200 /* tell the logger we're closing up */
201 if (ff->state >= FILE_STATE_CLOSED)
202 file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
203
204 /* do the actual logging */
205 const uint8_t *data = NULL;
206 uint32_t data_len = 0;
207
208 StreamingBufferGetDataAtOffset(ff->sb,
209 &data, &data_len,
210 ff->content_stored);
211
212 const int file_logged = CallLoggers(tv, store, p, ff, data, data_len, file_flags, dir);
213 if (file_logged) {
214 ff->content_stored += data_len;
215
216 /* all done */
217 if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
218 CloseFile(p, p->flow, ff);
219 }
220 }
221 }
222 }
223 }
224
OutputFiledataLog(ThreadVars * tv,Packet * p,void * thread_data)225 static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data)
226 {
227 DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
228
229 if (list == NULL) {
230 /* No child loggers. */
231 return TM_ECODE_OK;
232 }
233
234 OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
235
236 /* no flow, no files */
237 Flow * const f = p->flow;
238 if (f == NULL || f->alstate == NULL) {
239 SCReturnInt(TM_ECODE_OK);
240 }
241
242 const bool file_trunc = StreamTcpReassembleDepthReached(p);
243 if (p->flowflags & FLOW_PKT_TOSERVER) {
244 const bool file_close_ts = p->flags & PKT_PSEUDO_STREAM_END;
245 FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER);
246 SCLogDebug("ffc_ts %p", ffc_ts);
247 OutputFiledataLogFfc(tv, op_thread_data, p, ffc_ts, STREAM_TOSERVER, file_close_ts,
248 file_trunc, STREAM_TOSERVER);
249 } else if (p->flowflags & FLOW_PKT_TOCLIENT) {
250 const bool file_close_tc = p->flags & PKT_PSEUDO_STREAM_END;
251 FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT);
252 SCLogDebug("ffc_tc %p", ffc_tc);
253 OutputFiledataLogFfc(tv, op_thread_data, p, ffc_tc, STREAM_TOCLIENT, file_close_tc,
254 file_trunc, STREAM_TOCLIENT);
255 }
256
257 return TM_ECODE_OK;
258 }
259
260 /**
261 * \internal
262 *
263 * \brief Open the waldo file (if available) and load the file_id
264 *
265 * \param path full path for the waldo file
266 */
LogFiledataLogLoadWaldo(const char * path)267 static void LogFiledataLogLoadWaldo(const char *path)
268 {
269 char line[16] = "";
270 unsigned int id = 0;
271
272 FILE *fp = fopen(path, "r");
273 if (fp == NULL) {
274 SCLogInfo("couldn't open waldo: %s", strerror(errno));
275 SCReturn;
276 }
277
278 if (fgets(line, (int)sizeof(line), fp) != NULL) {
279 if (sscanf(line, "%10u", &id) == 1) {
280 SCLogInfo("id %u", id);
281 SC_ATOMIC_SET(g_file_store_id, id);
282 }
283 }
284 fclose(fp);
285 }
286
287 /**
288 * \internal
289 *
290 * \brief Store the waldo file based on the file_id
291 *
292 * \param path full path for the waldo file
293 */
LogFiledataLogStoreWaldo(const char * path)294 static void LogFiledataLogStoreWaldo(const char *path)
295 {
296 char line[16] = "";
297
298 if (SC_ATOMIC_GET(g_file_store_id) == 1) {
299 SCReturn;
300 }
301
302 FILE *fp = fopen(path, "w");
303 if (fp == NULL) {
304 SCLogInfo("couldn't open waldo: %s", strerror(errno));
305 SCReturn;
306 }
307
308 snprintf(line, sizeof(line), "%u\n", SC_ATOMIC_GET(g_file_store_id));
309 if (fwrite(line, strlen(line), 1, fp) != 1) {
310 SCLogError(SC_ERR_FWRITE, "fwrite failed: %s", strerror(errno));
311 }
312 fclose(fp);
313 }
314
315 /** \brief thread init for the tx logger
316 * This will run the thread init functions for the individual registered
317 * loggers */
OutputFiledataLogThreadInit(ThreadVars * tv,const void * initdata,void ** data)318 static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
319 {
320 OutputLoggerThreadData *td = SCMalloc(sizeof(*td));
321 if (td == NULL)
322 return TM_ECODE_FAILED;
323 memset(td, 0x00, sizeof(*td));
324
325 *data = (void *)td;
326
327 #ifdef HAVE_MAGIC
328 td->magic_ctx = MagicInitContext();
329 if (td->magic_ctx == NULL) {
330 SCFree(td);
331 return TM_ECODE_FAILED;
332 }
333 #endif
334
335 SCLogDebug("OutputFiledataLogThreadInit happy (*data %p)", *data);
336
337 OutputFiledataLogger *logger = list;
338 while (logger) {
339 if (logger->ThreadInit) {
340 void *retptr = NULL;
341 if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
342 OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
343 /* todo */ BUG_ON(ts == NULL);
344 memset(ts, 0x00, sizeof(*ts));
345
346 /* store thread handle */
347 ts->thread_data = retptr;
348
349 if (td->store == NULL) {
350 td->store = ts;
351 } else {
352 OutputLoggerThreadStore *tmp = td->store;
353 while (tmp->next != NULL)
354 tmp = tmp->next;
355 tmp->next = ts;
356 }
357
358 SCLogDebug("%s is now set up", logger->name);
359 }
360 }
361
362 logger = logger->next;
363 }
364
365 SCMutexLock(&g_waldo_mutex);
366 if (g_waldo_init == 0) {
367 ConfNode *node = ConfGetNode("file-store-waldo");
368 if (node == NULL) {
369 ConfNode *outputs = ConfGetNode("outputs");
370 if (outputs) {
371 ConfNode *output;
372 TAILQ_FOREACH(output, &outputs->head, next) {
373 /* we only care about file and file-store */
374 if (!(strcmp(output->val, "file") == 0 || strcmp(output->val, "file-store") == 0))
375 continue;
376
377 ConfNode *file = ConfNodeLookupChild(output, output->val);
378 BUG_ON(file == NULL);
379 if (file == NULL) {
380 SCLogDebug("file-store failed, lets try 'file'");
381 file = ConfNodeLookupChild(outputs, "file");
382 if (file == NULL)
383 SCLogDebug("file failed as well, giving up");
384 }
385
386 if (file != NULL) {
387 node = ConfNodeLookupChild(file, "waldo");
388 if (node == NULL)
389 SCLogDebug("no waldo node");
390 }
391 }
392 }
393 }
394 if (node != NULL) {
395 const char *s_default_log_dir = NULL;
396 s_default_log_dir = ConfigGetLogDirectory();
397
398 const char *waldo = node->val;
399 SCLogDebug("loading waldo %s", waldo);
400 if (waldo != NULL && strlen(waldo) > 0) {
401 if (PathIsAbsolute(waldo)) {
402 snprintf(g_waldo, sizeof(g_waldo), "%s", waldo);
403 } else {
404 snprintf(g_waldo, sizeof(g_waldo), "%s/%s", s_default_log_dir, waldo);
405 }
406
407 SCLogDebug("loading waldo file %s", g_waldo);
408 LogFiledataLogLoadWaldo(g_waldo);
409 }
410 }
411 g_waldo_init = 1;
412 }
413 SCMutexUnlock(&g_waldo_mutex);
414 return TM_ECODE_OK;
415 }
416
OutputFiledataLogThreadDeinit(ThreadVars * tv,void * thread_data)417 static TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, void *thread_data)
418 {
419 OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
420 OutputLoggerThreadStore *store = op_thread_data->store;
421 OutputFiledataLogger *logger = list;
422
423 while (logger && store) {
424 if (logger->ThreadDeinit) {
425 logger->ThreadDeinit(tv, store->thread_data);
426 }
427
428 OutputLoggerThreadStore *next_store = store->next;
429 SCFree(store);
430 store = next_store;
431 logger = logger->next;
432 }
433
434 SCMutexLock(&g_waldo_mutex);
435 if (g_waldo_deinit == 0) {
436 if (strlen(g_waldo) > 0) {
437 SCLogDebug("we have a waldo at %s", g_waldo);
438 LogFiledataLogStoreWaldo(g_waldo);
439 }
440 g_waldo_deinit = 1;
441 }
442 SCMutexUnlock(&g_waldo_mutex);
443
444 #ifdef HAVE_MAGIC
445 MagicDeinitContext(op_thread_data->magic_ctx);
446 #endif
447
448 SCFree(op_thread_data);
449 return TM_ECODE_OK;
450 }
451
OutputFiledataLogExitPrintStats(ThreadVars * tv,void * thread_data)452 static void OutputFiledataLogExitPrintStats(ThreadVars *tv, void *thread_data)
453 {
454 OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
455 OutputLoggerThreadStore *store = op_thread_data->store;
456 OutputFiledataLogger *logger = list;
457
458 while (logger && store) {
459 if (logger->ThreadExitPrintStats) {
460 logger->ThreadExitPrintStats(tv, store->thread_data);
461 }
462
463 logger = logger->next;
464 store = store->next;
465 }
466 }
467
OutputFiledataLoggerGetActiveCount(void)468 static uint32_t OutputFiledataLoggerGetActiveCount(void)
469 {
470 uint32_t cnt = 0;
471 for (OutputFiledataLogger *p = list; p != NULL; p = p->next) {
472 cnt++;
473 }
474 return cnt;
475 }
476
OutputFiledataLoggerRegister(void)477 void OutputFiledataLoggerRegister(void)
478 {
479 OutputRegisterRootLogger(OutputFiledataLogThreadInit,
480 OutputFiledataLogThreadDeinit, OutputFiledataLogExitPrintStats,
481 OutputFiledataLog, OutputFiledataLoggerGetActiveCount);
482 SC_ATOMIC_INIT(g_file_store_id);
483 SC_ATOMIC_SET(g_file_store_id, 1);
484 }
485
OutputFiledataShutdown(void)486 void OutputFiledataShutdown(void)
487 {
488 OutputFiledataLogger *logger = list;
489 while (logger) {
490 OutputFiledataLogger *next_logger = logger->next;
491 SCFree(logger);
492 logger = next_logger;
493 }
494
495 list = NULL;
496 }
497