1 /* Copyright (C) 2019-2020 Open Information Security Foundation
2 *
3 * You can copy, redistribute or modify this Program under the terms of
4 * the GNU General Public License version 2 as published by the Free
5 * Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 * You should have received a copy of the GNU General Public License
13 * version 2 along with this program; if not, write to the Free Software
14 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15 * 02110-1301, USA.
16 */
17
18 /**
19 * \file
20 *
21 * \author Jeff Lucovsky <jeff@lucovsky.org>
22 *
23 * Logs anomalies in JSON format.
24 *
25 */
26
27 #include "suricata-common.h"
28 #include "debug.h"
29 #include "detect.h"
30 #include "flow.h"
31 #include "conf.h"
32 #include "app-layer.h"
33 #include "app-layer-events.h"
34 #include "app-layer-parser.h"
35
36 #include "threads.h"
37 #include "tm-threads.h"
38 #include "threadvars.h"
39 #include "util-debug.h"
40
41 #include "util-misc.h"
42
43 #include "detect-parse.h"
44 #include "detect-engine.h"
45 #include "util-logopenfile.h"
46
47 #include "output.h"
48 #include "output-json.h"
49 #include "output-json-anomaly.h"
50
51 #include "util-byte.h"
52 #include "util-enum.h"
53 #include "util-privs.h"
54 #include "util-print.h"
55 #include "util-proto-name.h"
56 #include "util-optimize.h"
57 #include "util-buffer.h"
58 #include "util-crypt.h"
59 #include "util-validate.h"
60
61 #define MODULE_NAME "JsonAnomalyLog"
62
63 #define ANOMALY_EVENT_TYPE "anomaly"
64 #define LOG_JSON_DECODE_TYPE BIT_U16(0)
65 #define LOG_JSON_STREAM_TYPE BIT_U16(1)
66 #define LOG_JSON_APPLAYER_TYPE BIT_U16(2)
67 #define LOG_JSON_PACKETHDR BIT_U16(3)
68
69 #define LOG_JSON_PACKET_TYPE (LOG_JSON_DECODE_TYPE | LOG_JSON_STREAM_TYPE)
70 #define ANOMALY_DEFAULTS LOG_JSON_APPLAYER_TYPE
71
72 #define TX_ID_UNUSED UINT64_MAX
73
74 typedef struct AnomalyJsonOutputCtx_ {
75 LogFileCtx* file_ctx;
76 uint16_t flags;
77 OutputJsonCommonSettings cfg;
78 } AnomalyJsonOutputCtx;
79
80 typedef struct JsonAnomalyLogThread_ {
81 /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
82 LogFileCtx* file_ctx;
83 MemBuffer *json_buffer;
84 AnomalyJsonOutputCtx* json_output_ctx;
85 } JsonAnomalyLogThread;
86
87 /*
88 * Restrict the anomaly logger count due to decoder state maintenance issues
89 */
90
91 #define MAX_ANOMALY_LOGGERS 1
92 static int anomaly_loggers = 0;
OutputAnomalyLoggerEnable(void)93 static bool OutputAnomalyLoggerEnable(void)
94 {
95 if (anomaly_loggers < MAX_ANOMALY_LOGGERS) {
96 anomaly_loggers++;
97 return true;
98 }
99 return false;
100 }
101
OutputAnomalyLoggerDisable(void)102 static void OutputAnomalyLoggerDisable(void)
103 {
104 if (anomaly_loggers)
105 anomaly_loggers--;
106 }
107
AnomalyDecodeEventJson(ThreadVars * tv,JsonAnomalyLogThread * aft,const Packet * p)108 static int AnomalyDecodeEventJson(ThreadVars *tv, JsonAnomalyLogThread *aft,
109 const Packet *p)
110 {
111 const bool is_ip_pkt = PKT_IS_IPV4(p) || PKT_IS_IPV6(p);
112 const uint16_t log_type = aft->json_output_ctx->flags;
113 const bool log_stream = log_type & LOG_JSON_STREAM_TYPE;
114 const bool log_decode = log_type & LOG_JSON_DECODE_TYPE;
115
116 for (int i = 0; i < p->events.cnt; i++) {
117 uint8_t event_code = p->events.events[i];
118 bool is_decode = EVENT_IS_DECODER_PACKET_ERROR(event_code);
119 if (is_decode && !log_decode)
120 continue;
121 if (!is_decode && !log_stream)
122 continue;
123
124 MemBufferReset(aft->json_buffer);
125
126 JsonBuilder *js = CreateEveHeader(p, LOG_DIR_PACKET, ANOMALY_EVENT_TYPE, NULL);
127 if (unlikely(js == NULL)) {
128 return TM_ECODE_OK;
129 }
130
131 if (is_ip_pkt) {
132 EveAddCommonOptions(&aft->json_output_ctx->cfg, p, p->flow, js);
133 }
134
135 jb_open_object(js, ANOMALY_EVENT_TYPE);
136
137 if (event_code < DECODE_EVENT_MAX) {
138 const char *event = DEvents[event_code].event_name;
139 if (EVENT_IS_DECODER_PACKET_ERROR(event_code)) {
140 JB_SET_STRING(js, "type", "decode");
141 } else {
142 JB_SET_STRING(js, "type", "stream");
143 }
144 jb_set_string(js, "event", event);
145 } else {
146 JB_SET_STRING(js, "type", "unknown");
147 jb_set_uint(js, "code", event_code);
148 }
149
150 /* Close anomaly object. */
151 jb_close(js);
152
153 if (aft->json_output_ctx->flags & LOG_JSON_PACKETHDR) {
154 EvePacket(p, js, GET_PKT_LEN(p) < 32 ? GET_PKT_LEN(p) : 32);
155 }
156
157 OutputJsonBuilderBuffer(js, aft->file_ctx, &aft->json_buffer);
158 jb_free(js);
159 }
160
161 return TM_ECODE_OK;
162 }
163
AnomalyAppLayerDecoderEventJson(JsonAnomalyLogThread * aft,const Packet * p,AppLayerDecoderEvents * decoder_events,bool is_pktlayer,const char * layer,uint64_t tx_id)164 static int AnomalyAppLayerDecoderEventJson(JsonAnomalyLogThread *aft,
165 const Packet *p, AppLayerDecoderEvents *decoder_events,
166 bool is_pktlayer, const char *layer, uint64_t tx_id)
167 {
168 const char *alprotoname = AppLayerGetProtoName(p->flow->alproto);
169
170 SCLogDebug("decoder_events %p event_count %d (last logged %d) %s",
171 decoder_events, decoder_events->cnt,
172 decoder_events->event_last_logged,
173 tx_id != TX_ID_UNUSED ? "tx" : "no-tx");
174
175 for (int i = decoder_events->event_last_logged; i < decoder_events->cnt; i++) {
176 MemBufferReset(aft->json_buffer);
177
178 JsonBuilder *js;
179 if (tx_id != TX_ID_UNUSED) {
180 js = CreateEveHeaderWithTxId(p, LOG_DIR_PACKET,
181 ANOMALY_EVENT_TYPE, NULL, tx_id);
182 } else {
183 js = CreateEveHeader(p, LOG_DIR_PACKET, ANOMALY_EVENT_TYPE, NULL);
184 }
185 if (unlikely(js == NULL)) {
186 return TM_ECODE_OK;
187 }
188
189 EveAddCommonOptions(&aft->json_output_ctx->cfg, p, p->flow, js);
190
191 jb_open_object(js, ANOMALY_EVENT_TYPE);
192
193 jb_set_string(js, "app_proto", alprotoname);
194
195 const char *event_name = NULL;
196 uint8_t event_code = decoder_events->events[i];
197 AppLayerEventType event_type;
198 int r;
199 if (is_pktlayer) {
200 r = AppLayerGetEventInfoById(event_code, &event_name, &event_type);
201 } else {
202 r = AppLayerParserGetEventInfoById(p->flow->proto, p->flow->alproto,
203 event_code, &event_name, &event_type);
204 }
205 if (r == 0) {
206 JB_SET_STRING(js, "type", "applayer");
207 jb_set_string(js, "event", event_name);
208 } else {
209 JB_SET_STRING(js, "type", "unknown");
210 jb_set_uint(js, "code", event_code);
211 }
212
213 jb_set_string(js, "layer", layer);
214
215 /* anomaly */
216 jb_close(js);
217 OutputJsonBuilderBuffer(js, aft->file_ctx, &aft->json_buffer);
218 jb_free(js);
219
220 /* Current implementation assumes a single owner for this value */
221 decoder_events->event_last_logged++;
222 }
223
224 return TM_ECODE_OK;
225 }
226
JsonAnomalyTxLogger(ThreadVars * tv,void * thread_data,const Packet * p,Flow * f,void * state,void * tx,uint64_t tx_id)227 static int JsonAnomalyTxLogger(ThreadVars *tv, void *thread_data, const Packet *p,
228 Flow *f, void *state, void *tx, uint64_t tx_id)
229 {
230 JsonAnomalyLogThread *aft = thread_data;
231 if (!(aft->json_output_ctx->flags & LOG_JSON_APPLAYER_TYPE)) {
232 return TM_ECODE_OK;
233 }
234
235 AppLayerDecoderEvents *decoder_events;
236 decoder_events = AppLayerParserGetEventsByTx(f->proto, f->alproto, tx);
237 if (decoder_events && decoder_events->event_last_logged < decoder_events->cnt) {
238 SCLogDebug("state %p, tx: %p, tx_id: %"PRIu64, state, tx, tx_id);
239 AnomalyAppLayerDecoderEventJson(aft, p, decoder_events, false,
240 "proto_parser", tx_id);
241 }
242 return TM_ECODE_OK;
243 }
244
AnomalyHasParserEvents(const Packet * p)245 static inline bool AnomalyHasParserEvents(const Packet *p)
246 {
247 return (p->flow && p->flow->alparser &&
248 AppLayerParserHasDecoderEvents(p->flow->alparser));
249 }
250
AnomalyHasPacketAppLayerEvents(const Packet * p)251 static inline bool AnomalyHasPacketAppLayerEvents(const Packet *p)
252 {
253 return p->app_layer_events && p->app_layer_events->cnt;
254 }
255
AnomalyJson(ThreadVars * tv,JsonAnomalyLogThread * aft,const Packet * p)256 static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet *p)
257 {
258 int rc = TM_ECODE_OK;
259
260 /* decode or stream */
261 if (aft->json_output_ctx->flags & LOG_JSON_PACKET_TYPE) {
262 if (p->events.cnt) {
263 rc = AnomalyDecodeEventJson(tv, aft, p);
264 }
265 }
266
267 /* applayer */
268 if (aft->json_output_ctx->flags & LOG_JSON_APPLAYER_TYPE) {
269 /* app layer proto detect events */
270 if (rc == TM_ECODE_OK && AnomalyHasPacketAppLayerEvents(p)) {
271 rc = AnomalyAppLayerDecoderEventJson(aft, p, p->app_layer_events,
272 true, "proto_detect", TX_ID_UNUSED);
273 }
274
275 /* parser state events */
276 if (rc == TM_ECODE_OK && AnomalyHasParserEvents(p)) {
277 SCLogDebug("Checking for anomaly events; alproto %d", p->flow->alproto);
278 AppLayerDecoderEvents *parser_events = AppLayerParserGetDecoderEvents(p->flow->alparser);
279 if (parser_events && (parser_events->event_last_logged < parser_events->cnt)) {
280 rc = AnomalyAppLayerDecoderEventJson(aft, p, parser_events,
281 false, "parser", TX_ID_UNUSED);
282 }
283 }
284 }
285
286 return rc;
287 }
288
JsonAnomalyLogger(ThreadVars * tv,void * thread_data,const Packet * p)289 static int JsonAnomalyLogger(ThreadVars *tv, void *thread_data, const Packet *p)
290 {
291 JsonAnomalyLogThread *aft = thread_data;
292 return AnomalyJson(tv, aft, p);
293 }
294
JsonAnomalyLogCondition(ThreadVars * tv,const Packet * p)295 static int JsonAnomalyLogCondition(ThreadVars *tv, const Packet *p)
296 {
297 return p->events.cnt > 0 ||
298 (p->app_layer_events && p->app_layer_events->cnt > 0) ||
299 AnomalyHasParserEvents(p);
300 }
301
JsonAnomalyLogThreadInit(ThreadVars * t,const void * initdata,void ** data)302 static TmEcode JsonAnomalyLogThreadInit(ThreadVars *t, const void *initdata, void **data)
303 {
304 JsonAnomalyLogThread *aft = SCCalloc(1, sizeof(JsonAnomalyLogThread));
305 if (unlikely(aft == NULL)) {
306 return TM_ECODE_FAILED;
307 }
308
309 if (initdata == NULL) {
310 SCLogDebug("Error getting context for EveLogAnomaly. \"initdata\" argument NULL");
311 goto error_exit;
312 }
313
314 aft->json_buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
315 if (aft->json_buffer == NULL) {
316 goto error_exit;
317 }
318
319 /** Use the Output Context (file pointer and mutex) */
320 AnomalyJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data;
321
322 aft->file_ctx = LogFileEnsureExists(json_output_ctx->file_ctx, t->id);
323 if (!aft->file_ctx) {
324 goto error_exit;
325 }
326 aft->json_output_ctx = json_output_ctx;
327
328 *data = (void *)aft;
329 return TM_ECODE_OK;
330
331 error_exit:
332 if (aft->json_buffer != NULL) {
333 MemBufferFree(aft->json_buffer);
334 }
335 SCFree(aft);
336 return TM_ECODE_FAILED;
337 }
338
JsonAnomalyLogThreadDeinit(ThreadVars * t,void * data)339 static TmEcode JsonAnomalyLogThreadDeinit(ThreadVars *t, void *data)
340 {
341 JsonAnomalyLogThread *aft = (JsonAnomalyLogThread *)data;
342 if (aft == NULL) {
343 return TM_ECODE_OK;
344 }
345
346 MemBufferFree(aft->json_buffer);
347
348 /* clear memory */
349 memset(aft, 0, sizeof(JsonAnomalyLogThread));
350
351 SCFree(aft);
352 return TM_ECODE_OK;
353 }
354
JsonAnomalyLogDeInitCtxSubHelper(OutputCtx * output_ctx)355 static void JsonAnomalyLogDeInitCtxSubHelper(OutputCtx *output_ctx)
356 {
357 SCLogDebug("cleaning up sub output_ctx %p", output_ctx);
358
359 AnomalyJsonOutputCtx *json_output_ctx = (AnomalyJsonOutputCtx *) output_ctx->data;
360
361 if (json_output_ctx != NULL) {
362 SCFree(json_output_ctx);
363 }
364 SCFree(output_ctx);
365 }
366
JsonAnomalyLogDeInitCtxSub(OutputCtx * output_ctx)367 static void JsonAnomalyLogDeInitCtxSub(OutputCtx *output_ctx)
368 {
369 OutputAnomalyLoggerDisable();
370
371 JsonAnomalyLogDeInitCtxSubHelper(output_ctx);
372 }
373
374 #define DEFAULT_LOG_FILENAME "anomaly.json"
SetFlag(const ConfNode * conf,const char * name,uint16_t flag,uint16_t * out_flags)375 static void SetFlag(const ConfNode *conf, const char *name, uint16_t flag, uint16_t *out_flags)
376 {
377 DEBUG_VALIDATE_BUG_ON(conf == NULL);
378 const char *setting = ConfNodeLookupChildValue(conf, name);
379 if (setting != NULL) {
380 if (ConfValIsTrue(setting)) {
381 *out_flags |= flag;
382 } else {
383 *out_flags &= ~flag;
384 }
385 }
386 }
387
JsonAnomalyLogConf(AnomalyJsonOutputCtx * json_output_ctx,ConfNode * conf)388 static void JsonAnomalyLogConf(AnomalyJsonOutputCtx *json_output_ctx,
389 ConfNode *conf)
390 {
391 static bool warn_no_flags = false;
392 static bool warn_no_packet = false;
393 uint16_t flags = ANOMALY_DEFAULTS;
394 if (conf != NULL) {
395 /* Check for metadata to enable/disable. */
396 ConfNode *typeconf = ConfNodeLookupChild(conf, "types");
397 if (typeconf != NULL) {
398 SetFlag(typeconf, "applayer", LOG_JSON_APPLAYER_TYPE, &flags);
399 SetFlag(typeconf, "stream", LOG_JSON_STREAM_TYPE, &flags);
400 SetFlag(typeconf, "decode", LOG_JSON_DECODE_TYPE, &flags);
401 }
402 SetFlag(conf, "packethdr", LOG_JSON_PACKETHDR, &flags);
403 }
404 if (((flags & (LOG_JSON_DECODE_TYPE | LOG_JSON_PACKETHDR)) == LOG_JSON_PACKETHDR) && !warn_no_packet) {
405 SCLogWarning(SC_WARN_ANOMALY_CONFIG, "Anomaly logging configured to include packet headers, however decode "
406 "type logging has not been selected. Packet headers will not be logged.");
407 warn_no_packet = true;
408 flags &= ~LOG_JSON_PACKETHDR;
409 }
410
411 if (flags == 0 && !warn_no_flags) {
412 SCLogWarning(SC_WARN_ANOMALY_CONFIG, "Anomaly logging has been configured; however, no logging types "
413 "have been selected. Select one or more logging types.");
414 warn_no_flags = true;
415 }
416 json_output_ctx->flags |= flags;
417 }
418
JsonAnomalyLogInitCtxHelper(ConfNode * conf,OutputCtx * parent_ctx)419 static OutputInitResult JsonAnomalyLogInitCtxHelper(ConfNode *conf, OutputCtx *parent_ctx)
420 {
421 OutputInitResult result = { NULL, false };
422 OutputJsonCtx *ajt = parent_ctx->data;
423 AnomalyJsonOutputCtx *json_output_ctx = NULL;
424
425 OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
426 if (unlikely(output_ctx == NULL))
427 return result;
428
429 json_output_ctx = SCCalloc(1, sizeof(AnomalyJsonOutputCtx));
430 if (unlikely(json_output_ctx == NULL)) {
431 goto error;
432 }
433
434 json_output_ctx->file_ctx = ajt->file_ctx;
435 JsonAnomalyLogConf(json_output_ctx, conf);
436 json_output_ctx->cfg = ajt->cfg;
437
438 output_ctx->data = json_output_ctx;
439 output_ctx->DeInit = JsonAnomalyLogDeInitCtxSubHelper;
440
441 result.ctx = output_ctx;
442 result.ok = true;
443 return result;
444
445 error:
446 SCFree(output_ctx);
447
448 return result;
449 }
450
451 /**
452 * \brief Create a new LogFileCtx for "fast" output style.
453 * \param conf The configuration node for this output.
454 * \return A LogFileCtx pointer on success, NULL on failure.
455 */
JsonAnomalyLogInitCtxSub(ConfNode * conf,OutputCtx * parent_ctx)456 static OutputInitResult JsonAnomalyLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
457 {
458
459 if (!OutputAnomalyLoggerEnable()) {
460 OutputInitResult result = { NULL, false };
461 SCLogError(SC_ERR_CONF_YAML_ERROR, "only one 'anomaly' logger "
462 "can be enabled");
463 return result;
464 }
465
466 OutputInitResult result = JsonAnomalyLogInitCtxHelper(conf, parent_ctx);
467 if (result.ok) {
468 result.ctx->DeInit = JsonAnomalyLogDeInitCtxSub;
469 }
470
471 return result;
472 }
473
JsonAnomalyLogRegister(void)474 void JsonAnomalyLogRegister (void)
475 {
476 OutputRegisterPacketSubModule(LOGGER_JSON_ANOMALY, "eve-log", MODULE_NAME,
477 "eve-log.anomaly", JsonAnomalyLogInitCtxSub, JsonAnomalyLogger,
478 JsonAnomalyLogCondition, JsonAnomalyLogThreadInit, JsonAnomalyLogThreadDeinit,
479 NULL);
480
481 OutputRegisterTxSubModule(LOGGER_JSON_ANOMALY, "eve-log", MODULE_NAME,
482 "eve-log.anomaly", JsonAnomalyLogInitCtxHelper, ALPROTO_UNKNOWN,
483 JsonAnomalyTxLogger, JsonAnomalyLogThreadInit,
484 JsonAnomalyLogThreadDeinit, NULL);
485 }
486