1 /* Copyright (C) 2007-2021 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  * AppLayer Filedata Logger Output registration functions
24  */
25 
26 #include "suricata-common.h"
27 #include "tm-modules.h"
28 #include "output.h"
29 #include "output-filedata.h"
30 #include "app-layer.h"
31 #include "app-layer-parser.h"
32 #include "detect-filemagic.h"
33 #include "conf.h"
34 #include "util-profiling.h"
35 #include "util-validate.h"
36 #include "util-magic.h"
37 
38 bool g_filedata_logger_enabled = false;
39 
40 typedef struct OutputLoggerThreadStore_ {
41     void *thread_data;
42     struct OutputLoggerThreadStore_ *next;
43 } OutputLoggerThreadStore;
44 
45 /** per thread data for this module, contains a list of per thread
46  *  data for the packet loggers. */
47 typedef struct OutputLoggerThreadData_ {
48     OutputLoggerThreadStore *store;
49 #ifdef HAVE_MAGIC
50     magic_t magic_ctx;
51 #endif
52 } OutputLoggerThreadData;
53 
54 /* logger instance, a module + a output ctx,
55  * it's perfectly valid that have multiple instances of the same
56  * log module (e.g. http.log) with different output ctx'. */
57 typedef struct OutputFiledataLogger_ {
58     FiledataLogger LogFunc;
59     OutputCtx *output_ctx;
60     struct OutputFiledataLogger_ *next;
61     const char *name;
62     LoggerId logger_id;
63     ThreadInitFunc ThreadInit;
64     ThreadDeinitFunc ThreadDeinit;
65     ThreadExitPrintStatsFunc ThreadExitPrintStats;
66 } OutputFiledataLogger;
67 
68 static OutputFiledataLogger *list = NULL;
69 static char g_waldo[PATH_MAX] = "";
70 static SCMutex g_waldo_mutex = SCMUTEX_INITIALIZER;
71 static int g_waldo_init = 0;
72 static int g_waldo_deinit = 0;
73 
OutputRegisterFiledataLogger(LoggerId id,const char * name,FiledataLogger LogFunc,OutputCtx * output_ctx,ThreadInitFunc ThreadInit,ThreadDeinitFunc ThreadDeinit,ThreadExitPrintStatsFunc ThreadExitPrintStats)74 int OutputRegisterFiledataLogger(LoggerId id, const char *name,
75     FiledataLogger LogFunc, OutputCtx *output_ctx, ThreadInitFunc ThreadInit,
76     ThreadDeinitFunc ThreadDeinit,
77     ThreadExitPrintStatsFunc ThreadExitPrintStats)
78 {
79     OutputFiledataLogger *op = SCMalloc(sizeof(*op));
80     if (op == NULL)
81         return -1;
82     memset(op, 0x00, sizeof(*op));
83 
84     op->LogFunc = LogFunc;
85     op->output_ctx = output_ctx;
86     op->name = name;
87     op->logger_id = id;
88     op->ThreadInit = ThreadInit;
89     op->ThreadDeinit = ThreadDeinit;
90     op->ThreadExitPrintStats = ThreadExitPrintStats;
91 
92     if (list == NULL)
93         list = op;
94     else {
95         OutputFiledataLogger *t = list;
96         while (t->next)
97             t = t->next;
98         t->next = op;
99     }
100 
101     SCLogDebug("OutputRegisterFiledataLogger happy");
102     g_filedata_logger_enabled = true;
103     return 0;
104 }
105 
106 SC_ATOMIC_DECLARE(unsigned int, g_file_store_id);
107 
CallLoggers(ThreadVars * tv,OutputLoggerThreadStore * store_list,Packet * p,File * ff,const uint8_t * data,uint32_t data_len,uint8_t flags,uint8_t dir)108 static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
109         Packet *p, File *ff,
110         const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t dir)
111 {
112     OutputFiledataLogger *logger = list;
113     OutputLoggerThreadStore *store = store_list;
114     int file_logged = 0;
115 
116     while (logger && store) {
117         DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
118 
119         SCLogDebug("logger %p", logger);
120         PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
121         logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, data, data_len, flags, dir);
122         PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
123 
124         file_logged = 1;
125 
126         logger = logger->next;
127         store = store->next;
128 
129         DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
130         DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
131     }
132     return file_logged;
133 }
134 
CloseFile(const Packet * p,Flow * f,File * file)135 static void CloseFile(const Packet *p, Flow *f, File *file)
136 {
137     void *txv = AppLayerParserGetTx(p->proto, f->alproto, f->alstate, file->txid);
138     if (txv) {
139         AppLayerTxData *txd = AppLayerParserGetTxData(p->proto, f->alproto, txv);
140         if (txd)
141             txd->files_stored++;
142     }
143     file->flags |= FILE_STORED;
144 }
145 
OutputFiledataLogFfc(ThreadVars * tv,OutputLoggerThreadData * td,Packet * p,FileContainer * ffc,const uint8_t call_flags,const bool file_close,const bool file_trunc,const uint8_t dir)146 static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadData *td,
147         Packet *p, FileContainer *ffc, const uint8_t call_flags,
148         const bool file_close, const bool file_trunc, const uint8_t dir)
149 {
150     if (ffc != NULL) {
151         OutputLoggerThreadStore *store = td->store;
152         File *ff;
153         for (ff = ffc->head; ff != NULL; ff = ff->next) {
154             uint8_t file_flags = call_flags;
155 #ifdef HAVE_MAGIC
156             if (FileForceMagic() && ff->magic == NULL) {
157                 FilemagicThreadLookup(&td->magic_ctx, ff);
158             }
159 #endif
160             SCLogDebug("ff %p", ff);
161             if (ff->flags & FILE_STORED) {
162                 SCLogDebug("stored flag set");
163                 continue;
164             }
165 
166             if (!(ff->flags & FILE_STORE)) {
167                 SCLogDebug("ff FILE_STORE not set");
168                 continue;
169             }
170 
171             /* if we have no data chunks left to log, we should still
172              * close the logger(s) */
173             if (FileDataSize(ff) == ff->content_stored &&
174                 (file_trunc || file_close)) {
175                 if (ff->state < FILE_STATE_CLOSED) {
176                     FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
177                 }
178                 CallLoggers(tv, store, p, ff, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir);
179                 CloseFile(p, p->flow, ff);
180                 continue;
181             }
182 
183             /* store */
184 
185             /* if file_store_id == 0, this is the first store of this file */
186             if (ff->file_store_id == 0) {
187                 /* new file */
188                 ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1);
189                 file_flags |= OUTPUT_FILEDATA_FLAG_OPEN;
190             } else {
191                 /* existing file */
192             }
193 
194             /* if file needs to be closed or truncated, inform
195              * loggers */
196             if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
197                 FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
198             }
199 
200             /* tell the logger we're closing up */
201             if (ff->state >= FILE_STATE_CLOSED)
202                 file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
203 
204             /* do the actual logging */
205             const uint8_t *data = NULL;
206             uint32_t data_len = 0;
207 
208             StreamingBufferGetDataAtOffset(ff->sb,
209                     &data, &data_len,
210                     ff->content_stored);
211 
212             const int file_logged = CallLoggers(tv, store, p, ff, data, data_len, file_flags, dir);
213             if (file_logged) {
214                 ff->content_stored += data_len;
215 
216                 /* all done */
217                 if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
218                     CloseFile(p, p->flow, ff);
219                 }
220             }
221         }
222     }
223 }
224 
OutputFiledataLog(ThreadVars * tv,Packet * p,void * thread_data)225 static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data)
226 {
227     DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
228 
229     if (list == NULL) {
230         /* No child loggers. */
231         return TM_ECODE_OK;
232     }
233 
234     OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
235 
236     /* no flow, no files */
237     Flow * const f = p->flow;
238     if (f == NULL || f->alstate == NULL) {
239         SCReturnInt(TM_ECODE_OK);
240     }
241 
242     const bool file_trunc = StreamTcpReassembleDepthReached(p);
243     if (p->flowflags & FLOW_PKT_TOSERVER) {
244         const bool file_close_ts = p->flags & PKT_PSEUDO_STREAM_END;
245         FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER);
246         SCLogDebug("ffc_ts %p", ffc_ts);
247         OutputFiledataLogFfc(tv, op_thread_data, p, ffc_ts, STREAM_TOSERVER, file_close_ts,
248                 file_trunc, STREAM_TOSERVER);
249     } else if (p->flowflags & FLOW_PKT_TOCLIENT) {
250         const bool file_close_tc = p->flags & PKT_PSEUDO_STREAM_END;
251         FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT);
252         SCLogDebug("ffc_tc %p", ffc_tc);
253         OutputFiledataLogFfc(tv, op_thread_data, p, ffc_tc, STREAM_TOCLIENT, file_close_tc,
254                 file_trunc, STREAM_TOCLIENT);
255     }
256 
257     return TM_ECODE_OK;
258 }
259 
260 /**
261  *  \internal
262  *
263  *  \brief Open the waldo file (if available) and load the file_id
264  *
265  *  \param path full path for the waldo file
266  */
LogFiledataLogLoadWaldo(const char * path)267 static void LogFiledataLogLoadWaldo(const char *path)
268 {
269     char line[16] = "";
270     unsigned int id = 0;
271 
272     FILE *fp = fopen(path, "r");
273     if (fp == NULL) {
274         SCLogInfo("couldn't open waldo: %s", strerror(errno));
275         SCReturn;
276     }
277 
278     if (fgets(line, (int)sizeof(line), fp) != NULL) {
279         if (sscanf(line, "%10u", &id) == 1) {
280             SCLogInfo("id %u", id);
281             SC_ATOMIC_SET(g_file_store_id, id);
282         }
283     }
284     fclose(fp);
285 }
286 
287 /**
288  *  \internal
289  *
290  *  \brief Store the waldo file based on the file_id
291  *
292  *  \param path full path for the waldo file
293  */
LogFiledataLogStoreWaldo(const char * path)294 static void LogFiledataLogStoreWaldo(const char *path)
295 {
296     char line[16] = "";
297 
298     if (SC_ATOMIC_GET(g_file_store_id) == 1) {
299         SCReturn;
300     }
301 
302     FILE *fp = fopen(path, "w");
303     if (fp == NULL) {
304         SCLogInfo("couldn't open waldo: %s", strerror(errno));
305         SCReturn;
306     }
307 
308     snprintf(line, sizeof(line), "%u\n", SC_ATOMIC_GET(g_file_store_id));
309     if (fwrite(line, strlen(line), 1, fp) != 1) {
310         SCLogError(SC_ERR_FWRITE, "fwrite failed: %s", strerror(errno));
311     }
312     fclose(fp);
313 }
314 
315 /** \brief thread init for the tx logger
316  *  This will run the thread init functions for the individual registered
317  *  loggers */
OutputFiledataLogThreadInit(ThreadVars * tv,const void * initdata,void ** data)318 static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
319 {
320     OutputLoggerThreadData *td = SCMalloc(sizeof(*td));
321     if (td == NULL)
322         return TM_ECODE_FAILED;
323     memset(td, 0x00, sizeof(*td));
324 
325     *data = (void *)td;
326 
327 #ifdef HAVE_MAGIC
328     td->magic_ctx = MagicInitContext();
329     if (td->magic_ctx == NULL) {
330         SCFree(td);
331         return TM_ECODE_FAILED;
332     }
333 #endif
334 
335     SCLogDebug("OutputFiledataLogThreadInit happy (*data %p)", *data);
336 
337     OutputFiledataLogger *logger = list;
338     while (logger) {
339         if (logger->ThreadInit) {
340             void *retptr = NULL;
341             if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
342                 OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
343 /* todo */      BUG_ON(ts == NULL);
344                 memset(ts, 0x00, sizeof(*ts));
345 
346                 /* store thread handle */
347                 ts->thread_data = retptr;
348 
349                 if (td->store == NULL) {
350                     td->store = ts;
351                 } else {
352                     OutputLoggerThreadStore *tmp = td->store;
353                     while (tmp->next != NULL)
354                         tmp = tmp->next;
355                     tmp->next = ts;
356                 }
357 
358                 SCLogDebug("%s is now set up", logger->name);
359             }
360         }
361 
362         logger = logger->next;
363     }
364 
365     SCMutexLock(&g_waldo_mutex);
366     if (g_waldo_init == 0) {
367         ConfNode *node = ConfGetNode("file-store-waldo");
368         if (node == NULL) {
369             ConfNode *outputs = ConfGetNode("outputs");
370             if (outputs) {
371                 ConfNode *output;
372                 TAILQ_FOREACH(output, &outputs->head, next) {
373                     /* we only care about file and file-store */
374                     if (!(strcmp(output->val, "file") == 0 || strcmp(output->val, "file-store") == 0))
375                         continue;
376 
377                     ConfNode *file = ConfNodeLookupChild(output, output->val);
378                     BUG_ON(file == NULL);
379                     if (file == NULL) {
380                         SCLogDebug("file-store failed, lets try 'file'");
381                         file = ConfNodeLookupChild(outputs, "file");
382                         if (file == NULL)
383                             SCLogDebug("file failed as well, giving up");
384                     }
385 
386                     if (file != NULL) {
387                         node = ConfNodeLookupChild(file, "waldo");
388                         if (node == NULL)
389                             SCLogDebug("no waldo node");
390                     }
391                 }
392             }
393         }
394         if (node != NULL) {
395             const char *s_default_log_dir = NULL;
396             s_default_log_dir = ConfigGetLogDirectory();
397 
398             const char *waldo = node->val;
399             SCLogDebug("loading waldo %s", waldo);
400             if (waldo != NULL && strlen(waldo) > 0) {
401                 if (PathIsAbsolute(waldo)) {
402                     snprintf(g_waldo, sizeof(g_waldo), "%s", waldo);
403                 } else {
404                     snprintf(g_waldo, sizeof(g_waldo), "%s/%s", s_default_log_dir, waldo);
405                 }
406 
407                 SCLogDebug("loading waldo file %s", g_waldo);
408                 LogFiledataLogLoadWaldo(g_waldo);
409             }
410         }
411         g_waldo_init = 1;
412     }
413     SCMutexUnlock(&g_waldo_mutex);
414     return TM_ECODE_OK;
415 }
416 
OutputFiledataLogThreadDeinit(ThreadVars * tv,void * thread_data)417 static TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, void *thread_data)
418 {
419     OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
420     OutputLoggerThreadStore *store = op_thread_data->store;
421     OutputFiledataLogger *logger = list;
422 
423     while (logger && store) {
424         if (logger->ThreadDeinit) {
425             logger->ThreadDeinit(tv, store->thread_data);
426         }
427 
428         OutputLoggerThreadStore *next_store = store->next;
429         SCFree(store);
430         store = next_store;
431         logger = logger->next;
432     }
433 
434     SCMutexLock(&g_waldo_mutex);
435     if (g_waldo_deinit == 0) {
436         if (strlen(g_waldo) > 0) {
437             SCLogDebug("we have a waldo at %s", g_waldo);
438             LogFiledataLogStoreWaldo(g_waldo);
439         }
440         g_waldo_deinit = 1;
441     }
442     SCMutexUnlock(&g_waldo_mutex);
443 
444 #ifdef HAVE_MAGIC
445     MagicDeinitContext(op_thread_data->magic_ctx);
446 #endif
447 
448     SCFree(op_thread_data);
449     return TM_ECODE_OK;
450 }
451 
OutputFiledataLogExitPrintStats(ThreadVars * tv,void * thread_data)452 static void OutputFiledataLogExitPrintStats(ThreadVars *tv, void *thread_data)
453 {
454     OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
455     OutputLoggerThreadStore *store = op_thread_data->store;
456     OutputFiledataLogger *logger = list;
457 
458     while (logger && store) {
459         if (logger->ThreadExitPrintStats) {
460             logger->ThreadExitPrintStats(tv, store->thread_data);
461         }
462 
463         logger = logger->next;
464         store = store->next;
465     }
466 }
467 
OutputFiledataLoggerGetActiveCount(void)468 static uint32_t OutputFiledataLoggerGetActiveCount(void)
469 {
470     uint32_t cnt = 0;
471     for (OutputFiledataLogger *p = list; p != NULL; p = p->next) {
472         cnt++;
473     }
474     return cnt;
475 }
476 
OutputFiledataLoggerRegister(void)477 void OutputFiledataLoggerRegister(void)
478 {
479     OutputRegisterRootLogger(OutputFiledataLogThreadInit,
480         OutputFiledataLogThreadDeinit, OutputFiledataLogExitPrintStats,
481         OutputFiledataLog, OutputFiledataLoggerGetActiveCount);
482     SC_ATOMIC_INIT(g_file_store_id);
483     SC_ATOMIC_SET(g_file_store_id, 1);
484 }
485 
OutputFiledataShutdown(void)486 void OutputFiledataShutdown(void)
487 {
488     OutputFiledataLogger *logger = list;
489     while (logger) {
490         OutputFiledataLogger *next_logger = logger->next;
491         SCFree(logger);
492         logger = next_logger;
493     }
494 
495     list = NULL;
496 }
497