1 /* Copyright (C) 2007-2016 Open Information Security Foundation
2 *
3 * You can copy, redistribute or modify this Program under the terms of
4 * the GNU General Public License version 2 as published by the Free
5 * Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 * You should have received a copy of the GNU General Public License
13 * version 2 along with this program; if not, write to the Free Software
14 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15 * 02110-1301, USA.
16 */
17
18 /**
19 * \file
20 *
21 * \author Danny Browning <danny.browning@protectwise.com>
22 *
23 * Helper methods for directory based packet acquisition
24 */
25
26 #include "source-pcap-file-directory-helper.h"
27 #include "runmode-unix-socket.h"
28 #include "util-mem.h"
29 #include "source-pcap-file.h"
30
31 static void GetTime(struct timespec *tm);
32 static void CopyTime(struct timespec *from, struct timespec *to);
33 static int CompareTimes(struct timespec *left, struct timespec *right);
34 static TmEcode PcapRunStatus(PcapFileDirectoryVars *);
35 static TmEcode PcapDirectoryFailure(PcapFileDirectoryVars *ptv);
36 static TmEcode PcapDirectoryDone(PcapFileDirectoryVars *ptv);
37 static int PcapDirectoryGetModifiedTime(char const * file, struct timespec * out);
38 static TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv,
39 PendingFile *file_to_add);
40 static TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *ptv,
41 struct timespec * older_than);
42 static TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv,
43 struct timespec *older_than);
44
GetTime(struct timespec * tm)45 void GetTime(struct timespec *tm)
46 {
47 struct timeval now;
48 if(gettimeofday(&now, NULL) == 0) {
49 tm->tv_sec = now.tv_sec;
50 tm->tv_nsec = now.tv_usec * 1000L;
51 }
52 }
53
CopyTime(struct timespec * from,struct timespec * to)54 void CopyTime(struct timespec *from, struct timespec *to)
55 {
56 to->tv_sec = from->tv_sec;
57 to->tv_nsec = from->tv_nsec;
58 }
59
CompareTimes(struct timespec * left,struct timespec * right)60 int CompareTimes(struct timespec *left, struct timespec *right)
61 {
62 if (left->tv_sec < right->tv_sec) {
63 return -1;
64 } else if (left->tv_sec > right->tv_sec) {
65 return 1;
66 } else {
67 if (left->tv_nsec < right->tv_nsec) {
68 return -1;
69 } else if (left->tv_nsec > right->tv_nsec) {
70 return 1;
71 } else {
72 return 0;
73 }
74 }
75 }
76
77 /**
78 * Pcap Folder Utilities
79 */
PcapRunStatus(PcapFileDirectoryVars * ptv)80 TmEcode PcapRunStatus(PcapFileDirectoryVars *ptv)
81 {
82 if (RunModeUnixSocketIsActive()) {
83 TmEcode done = UnixSocketPcapFile(TM_ECODE_OK, &ptv->shared->last_processed);
84 if ( (suricata_ctl_flags & SURICATA_STOP) || done != TM_ECODE_OK) {
85 SCReturnInt(TM_ECODE_DONE);
86 }
87 } else {
88 if (suricata_ctl_flags & SURICATA_STOP) {
89 SCReturnInt(TM_ECODE_DONE);
90 }
91 }
92 SCReturnInt(TM_ECODE_OK);
93 }
94
CleanupPendingFile(PendingFile * pending)95 void CleanupPendingFile(PendingFile *pending) {
96 if (pending != NULL) {
97 if (pending->filename != NULL) {
98 SCFree(pending->filename);
99 }
100 SCFree(pending);
101 }
102 }
103
CleanupPcapFileDirectoryVars(PcapFileDirectoryVars * ptv)104 void CleanupPcapFileDirectoryVars(PcapFileDirectoryVars *ptv)
105 {
106 if (ptv != NULL) {
107 if (ptv->current_file != NULL) {
108 CleanupPcapFileFileVars(ptv->current_file);
109 ptv->current_file = NULL;
110 }
111 if (ptv->directory != NULL) {
112 closedir(ptv->directory);
113 ptv->directory = NULL;
114 }
115 if (ptv->filename != NULL) {
116 SCFree(ptv->filename);
117 }
118 ptv->shared = NULL;
119 PendingFile *current_file = NULL;
120 while (!TAILQ_EMPTY(&ptv->directory_content)) {
121 current_file = TAILQ_FIRST(&ptv->directory_content);
122 TAILQ_REMOVE(&ptv->directory_content, current_file, next);
123 CleanupPendingFile(current_file);
124 }
125 SCFree(ptv);
126 }
127 }
128
PcapDirectoryFailure(PcapFileDirectoryVars * ptv)129 TmEcode PcapDirectoryFailure(PcapFileDirectoryVars *ptv)
130 {
131 TmEcode status = TM_ECODE_FAILED;
132
133 if (unlikely(ptv == NULL)) {
134 SCLogError(SC_ERR_INVALID_ARGUMENT, "Directory vars was null");
135 SCReturnInt(TM_ECODE_FAILED);
136 }
137 if (unlikely(ptv->shared == NULL)) {
138 SCLogError(SC_ERR_INVALID_ARGUMENT, "Directory shared vars was null");
139 SCReturnInt(TM_ECODE_FAILED);
140 }
141
142 if (RunModeUnixSocketIsActive()) {
143 status = UnixSocketPcapFile(status, &ptv->shared->last_processed);
144 }
145
146 SCReturnInt(status);
147 }
148
PcapDirectoryDone(PcapFileDirectoryVars * ptv)149 TmEcode PcapDirectoryDone(PcapFileDirectoryVars *ptv)
150 {
151 TmEcode status = TM_ECODE_DONE;
152
153 if (unlikely(ptv == NULL)) {
154 SCLogError(SC_ERR_INVALID_ARGUMENT, "Directory vars was null");
155 SCReturnInt(TM_ECODE_FAILED);
156 }
157 if (unlikely(ptv->shared == NULL)) {
158 SCLogError(SC_ERR_INVALID_ARGUMENT, "Directory shared vars was null");
159 SCReturnInt(TM_ECODE_FAILED);
160 }
161
162 if (RunModeUnixSocketIsActive()) {
163 status = UnixSocketPcapFile(status, &ptv->shared->last_processed);
164 }
165
166 SCReturnInt(status);
167 }
168
PcapDetermineDirectoryOrFile(char * filename,DIR ** directory)169 TmEcode PcapDetermineDirectoryOrFile(char *filename, DIR **directory)
170 {
171 DIR *temp_dir = NULL;
172 TmEcode return_code = TM_ECODE_FAILED;
173
174 temp_dir = opendir(filename);
175
176 if (temp_dir == NULL) {//if null, our filename may just be a normal file
177 switch (errno) {
178 case EACCES:
179 SCLogError(SC_ERR_FOPEN, "%s: Permission denied", filename);
180 break;
181
182 case EBADF:
183 SCLogError(SC_ERR_FOPEN,
184 "%s: Not a valid file descriptor opened for reading",
185 filename);
186 break;
187
188 case EMFILE:
189 SCLogError(SC_ERR_FOPEN,
190 "%s: Per process open file descriptor limit reached",
191 filename);
192 break;
193
194 case ENFILE:
195 SCLogError(SC_ERR_FOPEN,
196 "%s: System wide open file descriptor limit reached",
197 filename);
198 break;
199
200 case ENOENT:
201 SCLogError(SC_ERR_FOPEN,
202 "%s: Does not exist, or name is an empty string",
203 filename);
204 break;
205 case ENOMEM:
206 SCLogError(SC_ERR_FOPEN,
207 "%s: Insufficient memory to complete the operation",
208 filename);
209 break;
210
211 case ENOTDIR: //no error checking the directory, just is a plain file
212 SCLogDebug("%s: plain file, not a directory", filename);
213 return_code = TM_ECODE_OK;
214 break;
215
216 default:
217 SCLogError(SC_ERR_FOPEN, "%s: %" PRId32, filename, errno);
218 }
219 } else {
220 //no error, filename references a directory
221 *directory = temp_dir;
222 return_code = TM_ECODE_OK;
223 }
224
225 return return_code;
226 }
227
PcapDirectoryGetModifiedTime(char const * file,struct timespec * out)228 int PcapDirectoryGetModifiedTime(char const *file, struct timespec *out)
229 {
230 #ifdef OS_WIN32
231 struct _stat buf;
232 #else
233 struct stat buf;
234 #endif /* OS_WIN32 */
235 int ret;
236
237 if (file == NULL)
238 return -1;
239
240 #ifdef OS_WIN32
241 if((ret = _stat(file, &buf)) != 0)
242 return ret;
243 #else
244 if ((ret = stat(file, &buf)) != 0)
245 return ret;
246 #endif
247
248 #ifdef OS_DARWIN
249 out->tv_sec = buf.st_mtimespec.tv_sec;
250 out->tv_nsec = buf.st_mtimespec.tv_nsec;
251 #elif OS_WIN32
252 out->tv_sec = buf.st_mtime;
253 #else
254 out->tv_sec = buf.st_mtim.tv_sec;
255 out->tv_nsec = buf.st_mtim.tv_nsec;
256 #endif
257
258 return ret;
259 }
260
PcapDirectoryInsertFile(PcapFileDirectoryVars * pv,PendingFile * file_to_add)261 TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv,
262 PendingFile *file_to_add
263 ) {
264 PendingFile *file_to_compare = NULL;
265 PendingFile *next_file_to_compare = NULL;
266
267 if (unlikely(pv == NULL)) {
268 SCLogError(SC_ERR_INVALID_ARGUMENT, "No directory vars passed");
269 SCReturnInt(TM_ECODE_FAILED);
270 }
271
272 if (unlikely(file_to_add == NULL)) {
273 SCLogError(SC_ERR_INVALID_ARGUMENT, "File passed was null");
274 SCReturnInt(TM_ECODE_FAILED);
275 }
276
277 if (unlikely(file_to_add->filename == NULL)) {
278 SCLogError(SC_ERR_INVALID_ARGUMENT, "File was passed with null filename");
279 SCReturnInt(TM_ECODE_FAILED);
280 }
281
282 SCLogDebug("Inserting %s into directory buffer", file_to_add->filename);
283
284 if (TAILQ_EMPTY(&pv->directory_content)) {
285 TAILQ_INSERT_TAIL(&pv->directory_content, file_to_add, next);
286 } else {
287 file_to_compare = TAILQ_FIRST(&pv->directory_content);
288 while(file_to_compare != NULL) {
289 if (CompareTimes(&file_to_add->modified_time, &file_to_compare->modified_time) < 0) {
290 TAILQ_INSERT_BEFORE(file_to_compare, file_to_add, next);
291 file_to_compare = NULL;
292 } else {
293 next_file_to_compare = TAILQ_NEXT(file_to_compare, next);
294 if (next_file_to_compare == NULL) {
295 TAILQ_INSERT_AFTER(&pv->directory_content, file_to_compare,
296 file_to_add, next);
297 }
298 file_to_compare = next_file_to_compare;
299 }
300 }
301 }
302
303 SCReturnInt(TM_ECODE_OK);
304 }
305
PcapDirectoryPopulateBuffer(PcapFileDirectoryVars * pv,struct timespec * older_than)306 TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *pv,
307 struct timespec *older_than
308 ) {
309 if (unlikely(pv == NULL)) {
310 SCLogError(SC_ERR_INVALID_ARGUMENT, "No directory vars passed");
311 SCReturnInt(TM_ECODE_FAILED);
312 }
313 if (unlikely(pv->filename == NULL)) {
314 SCLogError(SC_ERR_INVALID_ARGUMENT, "No directory filename was passed");
315 SCReturnInt(TM_ECODE_FAILED);
316 }
317 struct dirent * dir = NULL;
318 PendingFile *file_to_add = NULL;
319
320 while ((dir = readdir(pv->directory)) != NULL) {
321 #ifndef OS_WIN32
322 if (dir->d_type != DT_REG) {
323 continue;
324 }
325 #endif
326 if (strcmp(dir->d_name, ".") == 0 ||
327 strcmp(dir->d_name, "..") == 0) {
328 continue;
329 }
330
331 char pathbuff[PATH_MAX] = {0};
332
333 int written = 0;
334
335 written = snprintf(pathbuff, PATH_MAX, "%s/%s", pv->filename, dir->d_name);
336
337 if (written <= 0 || written >= PATH_MAX) {
338 SCLogError(SC_ERR_SPRINTF, "Could not write path");
339
340 SCReturnInt(TM_ECODE_FAILED);
341 } else {
342 struct timespec temp_time;
343 memset(&temp_time, 0, sizeof(struct timespec));
344
345 if (PcapDirectoryGetModifiedTime(pathbuff, &temp_time) == 0) {
346 SCLogDebug("%" PRIuMAX " < %" PRIuMAX "(%s) < %" PRIuMAX ")",
347 (uintmax_t)SCTimespecAsEpochMillis(&pv->shared->last_processed),
348 (uintmax_t)SCTimespecAsEpochMillis(&temp_time),
349 pathbuff,
350 (uintmax_t)SCTimespecAsEpochMillis(older_than));
351
352 // Skip files outside of our time range
353 if (CompareTimes(&temp_time, &pv->shared->last_processed) <= 0) {
354 SCLogDebug("Skipping old file %s", pathbuff);
355 continue;
356 }
357 else if (CompareTimes(&temp_time, older_than) >= 0) {
358 SCLogDebug("Skipping new file %s", pathbuff);
359 continue;
360 }
361 } else {
362 SCLogDebug("Unable to get modified time on %s, skipping", pathbuff);
363 continue;
364 }
365
366 file_to_add = SCCalloc(1, sizeof(PendingFile));
367 if (unlikely(file_to_add == NULL)) {
368 SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate pending file");
369
370 SCReturnInt(TM_ECODE_FAILED);
371 }
372
373 file_to_add->filename = SCStrdup(pathbuff);
374 if (unlikely(file_to_add->filename == NULL)) {
375 SCLogError(SC_ERR_MEM_ALLOC, "Failed to copy filename");
376 CleanupPendingFile(file_to_add);
377
378 SCReturnInt(TM_ECODE_FAILED);
379 }
380
381 memset(&file_to_add->modified_time, 0, sizeof(struct timespec));
382 CopyTime(&temp_time, &file_to_add->modified_time);
383
384 SCLogInfo("Found \"%s\" at %" PRIuMAX, file_to_add->filename,
385 (uintmax_t)SCTimespecAsEpochMillis(&file_to_add->modified_time));
386
387 if (PcapDirectoryInsertFile(pv, file_to_add) == TM_ECODE_FAILED) {
388 SCLogError(SC_ERR_INVALID_ARGUMENT, "Failed to add file");
389 CleanupPendingFile(file_to_add);
390
391 SCReturnInt(TM_ECODE_FAILED);
392 }
393 }
394 }
395
396 SCReturnInt(TM_ECODE_OK);
397 }
398
399
PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars * pv,struct timespec * older_than)400 TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv,
401 struct timespec *older_than)
402 {
403 if (PcapDirectoryPopulateBuffer(pv, older_than) == TM_ECODE_FAILED) {
404 SCLogError(SC_ERR_INVALID_ARGUMENT, "Failed to populate directory buffer");
405 SCReturnInt(TM_ECODE_FAILED);
406 }
407
408 TmEcode status = TM_ECODE_OK;
409
410 if (TAILQ_EMPTY(&pv->directory_content)) {
411 SCLogDebug("Directory %s has no files to process", pv->filename);
412 GetTime(older_than);
413 older_than->tv_sec = older_than->tv_sec - pv->delay;
414 rewinddir(pv->directory);
415 status = TM_ECODE_OK;
416 } else {
417 PendingFile *current_file = NULL;
418
419 struct timespec last_time_seen;
420 memset(&last_time_seen, 0, sizeof(struct timespec));
421
422 while (status == TM_ECODE_OK && !TAILQ_EMPTY(&pv->directory_content)) {
423 current_file = TAILQ_FIRST(&pv->directory_content);
424 TAILQ_REMOVE(&pv->directory_content, current_file, next);
425
426 if (unlikely(current_file == NULL)) {
427 SCLogWarning(SC_ERR_PCAP_DISPATCH, "Current file was null");
428 } else if (unlikely(current_file->filename == NULL)) {
429 SCLogWarning(SC_ERR_PCAP_DISPATCH, "Current file filename was null");
430 } else {
431 SCLogDebug("Processing file %s", current_file->filename);
432
433 PcapFileFileVars *pftv = SCMalloc(sizeof(PcapFileFileVars));
434 if (unlikely(pftv == NULL)) {
435 SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate PcapFileFileVars");
436 SCReturnInt(TM_ECODE_FAILED);
437 }
438 memset(pftv, 0, sizeof(PcapFileFileVars));
439
440 pftv->filename = SCStrdup(current_file->filename);
441 if (unlikely(pftv->filename == NULL)) {
442 SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate filename");
443 CleanupPcapFileFileVars(pftv);
444 SCReturnInt(TM_ECODE_FAILED);
445 }
446 pftv->shared = pv->shared;
447
448 if (InitPcapFile(pftv) == TM_ECODE_FAILED) {
449 SCLogWarning(SC_ERR_PCAP_DISPATCH,
450 "Failed to init pcap file %s, skipping",
451 current_file->filename);
452 CleanupPendingFile(current_file);
453 CleanupPcapFileFileVars(pftv);
454 status = TM_ECODE_OK;
455 } else {
456 pv->current_file = pftv;
457
458 status = PcapFileDispatch(pftv);
459
460 CleanupPcapFileFileVars(pftv);
461
462 if (status == TM_ECODE_FAILED) {
463 CleanupPendingFile(current_file);
464 SCReturnInt(status);
465 }
466
467 SCLogInfo("Processed file %s, processed up to %" PRIuMAX,
468 current_file->filename,
469 (uintmax_t)SCTimespecAsEpochMillis(¤t_file->modified_time));
470
471 if(CompareTimes(¤t_file->modified_time, &last_time_seen) > 0) {
472 CopyTime(¤t_file->modified_time, &last_time_seen);
473 }
474
475 CleanupPendingFile(current_file);
476 pv->current_file = NULL;
477
478 status = PcapRunStatus(pv);
479 }
480 }
481 }
482
483 if(CompareTimes(&last_time_seen, &pv->shared->last_processed) > 0) {
484 SCLogInfo("Updating processed to %" PRIuMAX,
485 (uintmax_t)SCTimespecAsEpochMillis(&last_time_seen));
486 CopyTime(&last_time_seen, &pv->shared->last_processed);
487 status = PcapRunStatus(pv);
488 }
489 }
490 GetTime(older_than);
491 older_than->tv_sec = older_than->tv_sec - pv->delay;
492
493 SCReturnInt(status);
494 }
495
PcapDirectoryDispatch(PcapFileDirectoryVars * ptv)496 TmEcode PcapDirectoryDispatch(PcapFileDirectoryVars *ptv)
497 {
498 SCEnter();
499
500 DIR *directory_check = NULL;
501
502 struct timespec older_than;
503 memset(&older_than, 0, sizeof(struct timespec));
504 older_than.tv_sec = LONG_MAX;
505 uint32_t poll_seconds = (uint32_t)localtime(&ptv->poll_interval)->tm_sec;
506
507 if (ptv->should_loop) {
508 GetTime(&older_than);
509 older_than.tv_sec = older_than.tv_sec - ptv->delay;
510 }
511 TmEcode status = TM_ECODE_OK;
512
513 while (status == TM_ECODE_OK) {
514 //loop while directory is ok
515 SCLogInfo("Processing pcaps directory %s, files must be newer than %" PRIuMAX " and older than %" PRIuMAX,
516 ptv->filename, (uintmax_t)SCTimespecAsEpochMillis(&ptv->shared->last_processed),
517 (uintmax_t)SCTimespecAsEpochMillis(&older_than));
518 status = PcapDirectoryDispatchForTimeRange(ptv, &older_than);
519 if (ptv->should_loop && status == TM_ECODE_OK) {
520 sleep(poll_seconds);
521 //update our status based on suricata control flags or unix command socket
522 status = PcapRunStatus(ptv);
523 if (status == TM_ECODE_OK) {
524 SCLogDebug("Checking if directory %s still exists", ptv->filename);
525 //check directory
526 if (PcapDetermineDirectoryOrFile(ptv->filename,
527 &directory_check) == TM_ECODE_FAILED) {
528 SCLogInfo("Directory %s no longer exists, stopping",
529 ptv->filename);
530 status = TM_ECODE_DONE;
531 } else if(directory_check != NULL) {
532 closedir(directory_check);
533 directory_check = NULL;
534 }
535 }
536 } else if (status == TM_ECODE_OK) { //not looping, mark done
537 SCLogDebug("Not looping, stopping directory mode");
538 status = TM_ECODE_DONE;
539 }
540 }
541
542 StatsSyncCountersIfSignalled(ptv->shared->tv);
543
544 if (status == TM_ECODE_FAILED) {
545 SCLogError(SC_ERR_PCAP_DISPATCH, "Directory %s run mode failed", ptv->filename);
546 status = PcapDirectoryFailure(ptv);
547 } else {
548 SCLogInfo("Directory run mode complete");
549 status = PcapDirectoryDone(ptv);
550 }
551
552 SCReturnInt(status);
553 }
554
555 /* eof */
556