1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3 * common.cpp:
4 * bulk_extractor backend stuff, used for both standalone executable and bulk_extractor.
5 */
6
7 #include "config.h"
8 #include <sys/types.h>
9 #include <sys/stat.h>
10 #include <dirent.h>
11 #include <algorithm>
12
13 #ifdef HAVE_ERR_H
14 #include <err.h>
15 #endif
16
17 #ifdef HAVE_DLFCN_H
18 #include <dlfcn.h>
19 #endif
20
21 #include "bulk_extractor_i.h"
22 #include "aftimer.h"
23 #include "../dfxml/src/hash_t.h"
24
25
26 uint32_t scanner_def::max_depth = 7; // max recursion depth
27 uint32_t scanner_def::max_ngram = 10; // max recursion depth
28 static int debug; // local debug variable
29 static uint32_t max_depth_seen=0;
30 static cppmutex max_depth_seenM;
31 bool be13::plugin::dup_data_alerts = false; // by default, is disabled
32 uint64_t be13::plugin::dup_data_encountered = 0; // amount that was not processed
33
34 class scanner_command {
35 public:
36 enum command_t {DISABLE_ALL=0,ENABLE_ALL,DISABLE,ENABLE};
scanner_command(const scanner_command & sc)37 scanner_command(const scanner_command &sc):command(sc.command),name(sc.name){};
scanner_command(scanner_command::command_t c,const std::string & n)38 scanner_command(scanner_command::command_t c,const std::string &n):command(c),name(n){};
39 command_t command;
40 std::string name;
41 };
42 static std::vector<scanner_command> scanner_commands;
43 bool scanner_commands_processed = false;
44
45 /****************************************************************
46 *** misc support
47 ****************************************************************/
48
49
50 /****************************************************************
51 *** SCANNER PLUG-IN SYSTEM
52 ****************************************************************/
53
54 /* scanner_params */
55
56 scanner_params::PrintOptions scanner_params::no_options;
57
58 /* vector object for keeping track of packet callbacks */
59
60 class packet_plugin_info {
61 public:
packet_plugin_info(void * user_,packet_callback_t * callback_)62 packet_plugin_info(void *user_,packet_callback_t *callback_):user(user_),callback(callback_){};
63 void *user;
64 packet_callback_t *callback;
65 };
66
67 typedef std::vector<packet_plugin_info> packet_plugin_info_vector_t;
68 packet_plugin_info_vector_t packet_handlers; // pcap callback handlers
69
70
71 /* plugin */
72
73 /**
74 * the vector of current scanners
75 */
76
77 be13::plugin::scanner_vector be13::plugin::current_scanners;
78
set_scanner_debug(int adebug)79 void be13::plugin::set_scanner_debug(int adebug)
80 {
81 debug = adebug;
82 }
83
84
85 /**
86 * return true a scanner is enabled
87 */
88
89 /* enable or disable a specific scanner.
90 * enable = 0 - disable that scanner.
91 * enable = 1 - enable that scanner
92 * 'all' is a special scanner that enables all scanners.
93 */
94
set_scanner_enabled(const std::string & name,bool enable)95 void be13::plugin::set_scanner_enabled(const std::string &name,bool enable)
96 {
97 for(scanner_vector::iterator it = current_scanners.begin();it!=current_scanners.end();it++){
98 if(name=="all" && (((*it)->info.flags & scanner_info::SCANNER_NO_ALL)==0)){
99 (*it)->enabled = enable;
100 }
101 if((*it)->info.name==name){
102 (*it)->enabled = enable;
103 return;
104 }
105 }
106 if(name=="all") return;
107 std::cerr << "Invalid scanner name '" << name << "'\n";
108 exit(1);
109 }
110
set_scanner_enabled_all(bool enable)111 void be13::plugin::set_scanner_enabled_all(bool enable)
112 {
113 for(scanner_vector::const_iterator it = current_scanners.begin();it!=current_scanners.end();it++){
114 (*it)->enabled = enable;
115 }
116 }
117
118 /** Name of feature files that should be histogramed.
119 * The histogram should be done in the plug-in
120 */
121
122 /****************************************************************
123 *** scanner plugin loading
124 ****************************************************************/
125
126 /**
127 * plugin system phase 0: Load a scanner.
128 *
129 * As part of scanner loading:
130 * - pass configuration to the scanner
131 * - feature files that the scanner requires
132 * - Histograms that the scanner makes (see feature_recorder_set)
133 * This is called before scanners are enabled or disabled, so the pcap handlers
134 * need to be set afterwards
135 */
load_scanner(scanner_t scanner,const scanner_info::scanner_config & sc)136 void be13::plugin::load_scanner(scanner_t scanner,const scanner_info::scanner_config &sc)
137 {
138 /* If scanner is already loaded, return */
139 for(scanner_vector::const_iterator it = current_scanners.begin();it!=current_scanners.end();it++){
140 if((*it)->scanner==scanner) return;
141 }
142
143 /* Use an empty sbuf and an empty feature recorder set as the parameters for the sp below.
144 * we use static values so that the sbuf is not constantly being created and destroyed.
145 */
146 static const sbuf_t sbuf;
147 static feature_recorder_set fs(feature_recorder_set::SET_DISABLED,feature_recorder_set::null_hasher,
148 feature_recorder_set::NO_INPUT,
149 feature_recorder_set::NO_OUTDIR); // dummy
150
151 //
152 // Each scanner's params are stored in a scanner_def object that
153 // is created here and retained for the duration of the run.
154 // The scanner_def includes its own scanner_info structure.
155 // We pre-load the structure with the configuration for this scanner
156 // and the global debug variable
157 //
158 // currently every scanner gets the same config. In the future, we might
159 // want to give different scanners different variables.
160 //
161
162 scanner_params sp(scanner_params::PHASE_STARTUP,sbuf,fs); //
163 scanner_def *sd = new scanner_def();
164 sd->scanner = scanner;
165 sd->info.config = ≻
166
167 sp.info = &sd->info;
168
169 // Make an empty recursion control block and call the scanner's
170 // initialization function.
171 recursion_control_block rcb(0,"");
172 (*scanner)(sp,rcb); // phase 0
173
174 sd->enabled = !(sd->info.flags & scanner_info::SCANNER_DISABLED);
175 current_scanners.push_back(sd);
176 }
177
load_scanner_file(std::string fn,const scanner_info::scanner_config & sc)178 void be13::plugin::load_scanner_file(std::string fn,const scanner_info::scanner_config &sc)
179 {
180 /* Figure out the function name */
181 size_t extloc = fn.rfind('.');
182 if(extloc==std::string::npos){
183 fprintf(stderr,"Cannot find '.' in %s",fn.c_str());
184 exit(1);
185 }
186 std::string func_name = fn.substr(0,extloc);
187 size_t slashloc = func_name.rfind('/');
188 if(slashloc!=std::string::npos) func_name = func_name.substr(slashloc+1);
189 slashloc = func_name.rfind('\\');
190 if(slashloc!=std::string::npos) func_name = func_name.substr(slashloc+1);
191
192 if(debug) std::cout << "Loading: " << fn << " (" << func_name << ")\n";
193 scanner_t *scanner = 0;
194 #if defined(HAVE_DLOPEN)
195 void *lib=dlopen(fn.c_str(), RTLD_LAZY);
196
197 if(lib==0){
198 fprintf(stderr,"dlopen: %s\n",dlerror());
199 exit(1);
200 }
201
202 /* Resolve the symbol */
203 scanner = (scanner_t *)dlsym(lib, func_name.c_str());
204
205 if(scanner==0){
206 fprintf(stderr,"dlsym: %s\n",dlerror());
207 exit(1);
208 }
209 #elif defined(HAVE_LOADLIBRARY)
210 /* Use Win32 LoadLibrary function */
211 /* See http://msdn.microsoft.com/en-us/library/ms686944(v=vs.85).aspx */
212 HINSTANCE hinstLib = LoadLibrary(TEXT(fn.c_str()));
213 if(hinstLib==0){
214 fprintf(stderr,"LoadLibrary(%s) failed",fn.c_str());
215 exit(1);
216 }
217 scanner = (scanner_t *)GetProcAddress(hinstLib,func_name.c_str());
218 if(scanner==0){
219 fprintf(stderr,"GetProcAddress(%s) failed",func_name.c_str());
220 exit(1);
221 }
222 #else
223 std::cout << " ERROR: Support for loadable libraries not enabled\n";
224 return;
225 #endif
226 load_scanner(*scanner,sc);
227 }
228
load_scanners(scanner_t * const * scanners,const scanner_info::scanner_config & sc)229 void be13::plugin::load_scanners(scanner_t * const *scanners,const scanner_info::scanner_config &sc)
230 {
231 for(int i=0;scanners[i];i++){
232 load_scanner(scanners[i],sc);
233 }
234 }
235
load_scanner_directory(const std::string & dirname,const scanner_info::scanner_config & sc)236 void be13::plugin::load_scanner_directory(const std::string &dirname,const scanner_info::scanner_config &sc )
237 {
238 DIR *dirp = opendir(dirname.c_str());
239 if(dirp==0){
240 fprintf(stderr,"Cannot open directory %s:",dirname.c_str());
241 exit(1);
242 }
243 struct dirent *dp;
244 while ((dp = readdir(dirp)) != NULL){
245 std::string fname = dp->d_name;
246 if(fname.substr(0,5)=="scan_" || fname.substr(0,5)=="SCAN_"){
247 size_t extloc = fname.rfind('.');
248 if(extloc==std::string::npos) continue; // no '.'
249 std::string ext = fname.substr(extloc+1);
250 #ifdef WIN32
251 if(ext!="DLL") continue; // not a DLL
252 #else
253 if(ext!="so") continue; // not a shared library
254 #endif
255 load_scanner_file(dirname+"/"+fname,sc );
256 }
257 }
258 }
259
load_scanner_directories(const std::vector<std::string> & dirnames,const scanner_info::scanner_config & sc)260 void be13::plugin::load_scanner_directories(const std::vector<std::string> &dirnames,
261 const scanner_info::scanner_config &sc)
262 {
263 for(std::vector<std::string>::const_iterator it = dirnames.begin();it!=dirnames.end();it++){
264 load_scanner_directory(*it,sc);
265 }
266 }
267
load_scanner_packet_handlers()268 void be13::plugin::load_scanner_packet_handlers()
269 {
270 for(scanner_vector::const_iterator it = current_scanners.begin(); it!=current_scanners.end(); it++){
271 if((*it)->enabled){
272 const scanner_def *sd = (*it);
273 if(sd->info.packet_cb){
274 packet_handlers.push_back(packet_plugin_info(sd->info.packet_user,sd->info.packet_cb));
275 }
276 }
277 }
278 }
279
280 // send every enabled scanner the phase message
message_enabled_scanners(scanner_params::phase_t phase,feature_recorder_set & fs)281 void be13::plugin::message_enabled_scanners(scanner_params::phase_t phase,feature_recorder_set &fs)
282 {
283 /* make an empty sbuf and feature recorder set */
284 const sbuf_t sbuf;
285 scanner_params sp(phase,sbuf,fs);
286 for(scanner_vector::iterator it = current_scanners.begin(); it!=current_scanners.end(); it++){
287 if((*it)->enabled){
288 recursion_control_block rcb(0,""); // dummy rcb
289 ((*it)->scanner)(sp,rcb);
290 }
291 }
292 }
293
find_scanner(const std::string & search_name)294 scanner_t *be13::plugin::find_scanner(const std::string &search_name)
295 {
296 for(scanner_vector::const_iterator it = current_scanners.begin();it!=current_scanners.end();it++){
297 if(search_name == (*it)->info.name){
298 return (*it)->scanner;
299 }
300 }
301 return 0;
302 }
303
304 // put the enabled scanners into the vector
get_enabled_scanners(std::vector<std::string> & svector)305 void be13::plugin::get_enabled_scanners(std::vector<std::string> &svector)
306 {
307 for(scanner_vector::const_iterator it=current_scanners.begin();it!=current_scanners.end();it++){
308 if((*it)->enabled){
309 svector.push_back((*it)->info.name);
310 }
311 }
312 }
313
find_scanner_enabled()314 bool be13::plugin::find_scanner_enabled()
315 {
316 for(scanner_vector::const_iterator it = current_scanners.begin(); it!=current_scanners.end(); it++){
317 if( ((*it)->info.flags & scanner_info::SCANNER_FIND_SCANNER)
318 && ((*it)->enabled)){
319 return true;
320 }
321 }
322 return false;
323 }
324
325
add_enabled_scanner_histograms_to_feature_recorder_set(feature_recorder_set & fs)326 void be13::plugin::add_enabled_scanner_histograms_to_feature_recorder_set(feature_recorder_set &fs)
327 {
328 for(scanner_vector::const_iterator it = current_scanners.begin(); it!=current_scanners.end(); it++){
329 if((*it)->enabled){
330 const scanner_def *sd = (*it);
331 for(histogram_defs_t::const_iterator i2 = sd->info.histogram_defs.begin();
332 i2 != sd->info.histogram_defs.end(); i2++){
333 fs.add_histogram((*i2));
334 }
335 }
336 }
337 }
338
scanners_init(feature_recorder_set & fs)339 void be13::plugin::scanners_init(feature_recorder_set &fs)
340 {
341 assert(scanner_commands_processed==true);
342 message_enabled_scanners(scanner_params::PHASE_INIT,fs); // tell all enabled scanners to init
343 }
344
345
346
347 /****************************************************************
348 *** Scanner Commands (which one is enabled or disabled)
349 ****************************************************************/
350
scanners_disable_all()351 void be13::plugin::scanners_disable_all()
352 {
353 assert(scanner_commands_processed==false);
354 scanner_commands.push_back(scanner_command(scanner_command::DISABLE_ALL,std::string("")));
355 }
356
scanners_enable_all()357 void be13::plugin::scanners_enable_all()
358 {
359 assert(scanner_commands_processed==false);
360 scanner_commands.push_back(scanner_command(scanner_command::ENABLE_ALL,std::string("")));
361 }
362
scanners_enable(const std::string & name)363 void be13::plugin::scanners_enable(const std::string &name)
364 {
365 assert(scanner_commands_processed==false);
366 scanner_commands.push_back(scanner_command(scanner_command::ENABLE,name));
367 }
368
scanners_disable(const std::string & name)369 void be13::plugin::scanners_disable(const std::string &name)
370 {
371 assert(scanner_commands_processed==false);
372 scanner_commands.push_back(scanner_command(scanner_command::DISABLE,name));
373 }
374
scanners_process_enable_disable_commands()375 void be13::plugin::scanners_process_enable_disable_commands()
376 {
377 for(std::vector<scanner_command>::const_iterator it=scanner_commands.begin();
378 it!=scanner_commands.end();it++){
379 switch((*it).command){
380 case scanner_command::ENABLE_ALL: set_scanner_enabled_all(true);break;
381 case scanner_command::DISABLE_ALL: set_scanner_enabled_all(false); break;
382 case scanner_command::ENABLE: set_scanner_enabled((*it).name,true);break;
383 case scanner_command::DISABLE: set_scanner_enabled((*it).name,false);break;
384 }
385 }
386 load_scanner_packet_handlers(); // can't do until enable/disable commands are run
387 scanner_commands_processed = true;
388 }
389
390
391 /****************************************************************
392 *** PHASE_SHUTDOWN (formerly phase 2): shut down the scanners
393 ****************************************************************/
394
phase_shutdown(feature_recorder_set & fs,std::stringstream * sxml)395 void be13::plugin::phase_shutdown(feature_recorder_set &fs,std::stringstream *sxml)
396 {
397 assert(scanner_commands_processed==true);
398 for(scanner_vector::iterator it = current_scanners.begin();it!=current_scanners.end();it++){
399 if((*it)->enabled){
400 const sbuf_t sbuf; // empty sbuf
401 scanner_params sp(scanner_params::PHASE_SHUTDOWN,sbuf,fs,sxml);
402 recursion_control_block rcb(0,""); // empty rcb
403 (*(*it)->scanner)(sp,rcb);
404 }
405 }
406 }
407
408 /************************************
409 *** HELP and option processing ***
410 ************************************/
411
412 /* Get the config and build the help strings at the same time! */
413
414 std::stringstream scanner_info::helpstream;
get_config(const scanner_info::config_t & c,const std::string & n,std::string * val,const std::string & help)415 void scanner_info::get_config(const scanner_info::config_t &c,
416 const std::string &n,std::string *val,const std::string &help)
417 {
418 /* Check to see if we are being called as part of a help operation */
419 helpstream << " -S " << n << "=" << *val << " " << help << " (" << name << ")\n";
420 scanner_info::config_t::const_iterator it = c.find(n);
421 if(it!=c.end() && val){
422 *val = it->second;
423 }
424 }
425
get_config(const std::string & n,std::string * val,const std::string & help)426 void scanner_info::get_config(const std::string &n,std::string *val,const std::string &help)
427 {
428 scanner_info::get_config(config->namevals,n,val,help);
429 }
430
431 #define GET_CONFIG(T) void scanner_info::get_config(const std::string &n,T *val,const std::string &help) {\
432 std::stringstream ss;\
433 ss << *val;\
434 std::string v(ss.str());\
435 get_config(n,&v,help);\
436 ss.str(v);\
437 ss >> *val;\
438 }
439
440 GET_CONFIG(uint64_t)
GET_CONFIG(int32_t)441 GET_CONFIG(int32_t) // both int32_t and uint32_t
442 GET_CONFIG(uint32_t)
443 GET_CONFIG(uint16_t)
444 #ifdef HAVE_GET_CONFIG_SIZE_T
445 GET_CONFIG(size_t)
446 #endif
447
448
449 /* uint8_t needs cast to uint32_t for <<
450 * Otherwise it is interpreted as a character.
451 */
452 void scanner_info::get_config(const std::string &n,uint8_t *val_,const std::string &help)
453 {
454 uint32_t val = *val_;
455 std::stringstream ss;
456 ss << val;
457 std::string v(ss.str());
458 get_config(n,&v,help);
459 ss.str(v);
460 ss >> val;
461 *val_ = (uint8_t)val;
462 }
463
464 /* bool needs special processing for YES/NO/TRUE/FALSE */
get_config(const std::string & n,bool * val,const std::string & help)465 void scanner_info::get_config(const std::string &n,bool *val,const std::string &help)
466 {
467 std::stringstream ss;
468 ss << ((*val) ? "YES" : "NO");
469 std::string v(ss.str());
470 get_config(n,&v,help);
471 switch(v.at(0)){
472 case 'Y':case 'y':case 'T':case 't':case '1':
473 *val = true;
474 break;
475 default:
476 *val = false;
477 }
478 }
479
480
481 /**
482 * Print a list of scanners.
483 * We need to load them to do this, so they are loaded with empty config
484 * Note that scanners can only be loaded once, so this exits.
485 */
info_scanners(bool detailed_info,bool detailed_settings,scanner_t * const * scanners_builtin,const char enable_opt,const char disable_opt)486 void be13::plugin::info_scanners(bool detailed_info,
487 bool detailed_settings,
488 scanner_t * const *scanners_builtin,
489 const char enable_opt,const char disable_opt)
490 {
491 const scanner_info::scanner_config empty_config;
492
493 load_scanners(scanners_builtin,empty_config);
494 std::cout << "\n";
495 std::vector<std::string> enabled_wordlist;
496 std::vector<std::string> disabled_wordlist;
497 for(scanner_vector::const_iterator it = current_scanners.begin();it!=current_scanners.end();it++){
498 if(detailed_info){
499 if ((*it)->info.name.size()) std::cout << "Scanner Name: " << (*it)->info.name << "\n";
500 std::cout << "flags: " << scanner_info::flag_to_string((*it)->info.flags) << "\n";
501 std::cout << "Scanner Interface version: " << (*it)->info.si_version << "\n";
502 if ((*it)->info.author.size()) std::cout << "Author: " << (*it)->info.author << "\n";
503 if ((*it)->info.description.size()) std::cout << "Description: " << (*it)->info.description << "\n";
504 if ((*it)->info.url.size()) std::cout << "URL: " << (*it)->info.url << "\n";
505 if ((*it)->info.scanner_version.size()) std::cout << "Scanner Version: " << (*it)->info.scanner_version << "\n";
506 std::cout << "Feature Names: ";
507 for(std::set<std::string>::const_iterator i2 = (*it)->info.feature_names.begin();
508 i2 != (*it)->info.feature_names.end();
509 i2++){
510 std::cout << *i2 << " ";
511 }
512 std::cout << "\n\n";
513 }
514 if((*it)->info.flags & scanner_info::SCANNER_NO_USAGE) continue;
515 if((*it)->info.flags & scanner_info::SCANNER_DISABLED){
516 disabled_wordlist.push_back((*it)->info.name);
517 } else {
518 enabled_wordlist.push_back((*it)->info.name);
519 }
520 }
521 if(detailed_settings){
522 std::cout << "Settable Options (and their defaults): \n";
523 std::cout << scanner_info::helpstr();
524 }
525 sort(disabled_wordlist.begin(),disabled_wordlist.end());
526 sort(enabled_wordlist.begin(),enabled_wordlist.end());
527 std::cout << "\n";
528 std::cout << "These scanners disabled by default; enable with -" << enable_opt << ":\n";
529 for(std::vector<std::string>::const_iterator it = disabled_wordlist.begin();
530 it!=disabled_wordlist.end();it++){
531 std::cout << " -" << enable_opt << " " << *it << " - enable scanner " << *it << "\n";
532 }
533 std::cout << "\n";
534 std::cout << "These scanners enabled by default; disable with -" << disable_opt << ":\n";
535 for(std::vector<std::string>::const_iterator it = enabled_wordlist.begin();it!=enabled_wordlist.end();it++){
536 std::cout << " -" << disable_opt << " " << *it << " - disable scanner " << *it << "\n";
537 }
538 }
539
540 /**
541 * upperstr - Turns an ASCII string into upper case (should be UTF-8)
542 */
543
upperstr(const std::string & str)544 static std::string upperstr(const std::string &str)
545 {
546 std::string ret;
547 for(std::string::const_iterator i=str.begin();i!=str.end();i++){
548 ret.push_back(toupper(*i));
549 }
550 return ret;
551 }
552
553 /* Determine if the sbuf consists of a repeating ngram */
find_ngram_size(const sbuf_t & sbuf)554 static size_t find_ngram_size(const sbuf_t &sbuf)
555 {
556 for(size_t ngram_size = 1; ngram_size < scanner_def::max_ngram; ngram_size++){
557 bool ngram_match = true;
558 for(size_t i=ngram_size;i<sbuf.pagesize && ngram_match;i++){
559 if(sbuf[i%ngram_size]!=sbuf[i]) ngram_match = false;
560 }
561 if(ngram_match) return ngram_size;
562 }
563 return 0; // no ngram size
564 }
565
get_max_depth_seen()566 uint32_t be13::plugin::get_max_depth_seen()
567 {
568 cppmutex::lock lock(max_depth_seenM);
569 return max_depth_seen;
570 }
571
572 /** process_sbuf is the main workhorse. It is calls each scanner on each page.
573 * @param sp - the scanner params, including the sbuf to process
574 * It is also the recursive entry point for sub-analysis.
575 */
576
process_sbuf(const class scanner_params & sp)577 void be13::plugin::process_sbuf(const class scanner_params &sp)
578 {
579 const pos0_t &pos0 = sp.sbuf.pos0;
580 class feature_recorder_set &fs = sp.fs;
581
582 fs.heartbeat(); // note that we are alive
583
584 {
585 /* note the maximum depth that we've seen */
586 cppmutex::lock lock(max_depth_seenM);
587 if(sp.depth > max_depth_seen) max_depth_seen = sp.depth;
588 }
589
590 /* If we are too deep, error out */
591 if(sp.depth >= scanner_def::max_depth){
592 feature_recorder *fr = fs.get_alert_recorder();
593 if(fr) fr->write(pos0,"process_extract: MAX DEPTH REACHED","");
594 return;
595 }
596
597 /* Determine if we have seen this buffer before */
598 bool seen_before = fs.check_previously_processed(sp.sbuf.buf,sp.sbuf.bufsize);
599 if(seen_before){
600 md5_t md5 = md5_generator::hash_buf(sp.sbuf.buf,sp.sbuf.bufsize);
601 feature_recorder *alert_recorder = fs.get_alert_recorder();
602 std::stringstream ss;
603 ss << "<buflen>" << sp.sbuf.bufsize << "</buflen>";
604 if(alert_recorder && dup_data_alerts) alert_recorder->write(sp.sbuf.pos0,"DUP SBUF "+md5.hexdigest(),ss.str());
605 #ifdef HAVE__SYNC_ADD_AND_FETCH
606 __sync_add_and_fetch(&dup_data_encountered,sp.sbuf.bufsize);
607 #endif
608 }
609
610 /* Determine if the sbuf consists of a repeating ngram. If so,
611 * it's only passed to the parsers that want ngrams. (By default,
612 * such sbufs are booring.)
613 */
614
615 size_t ngram_size = find_ngram_size(sp.sbuf);
616
617 /****************************************************************
618 *** CALL EACH OF THE SCANNERS ON THE SBUF
619 ****************************************************************/
620
621 if(debug & DEBUG_DUMP_DATA){
622 sp.sbuf.hex_dump(std::cerr);
623 }
624
625 for(scanner_vector::iterator it = current_scanners.begin();it!=current_scanners.end();it++){
626 // Look for reasons not to run a scanner
627 if((*it)->enabled==false) continue; // not enabled
628
629 if(((*it)->info.flags & scanner_info::SCANNER_WANTS_NGRAMS)==0){
630 /* If the scanner does not want ngrams, don't run it if we have ngrams or duplicate data */
631 if(ngram_size > 0) continue;
632 if(seen_before) continue;
633 }
634
635 if(sp.depth > 0 && ((*it)->info.flags & scanner_info::SCANNER_DEPTH_0)){
636 // depth >0 and this scanner only run at depth 0
637 continue;
638 }
639
640 const std::string &name = (*it)->info.name;
641
642 try {
643
644 /* Compute the effective path for stats */
645 bool inname=false;
646 std::string epath;
647 for(std::string::const_iterator cc=sp.sbuf.pos0.path.begin();cc!=sp.sbuf.pos0.path.end();cc++){
648 if(isupper(*cc)) inname=true;
649 if(inname) epath.push_back(toupper(*cc));
650 if(*cc=='-') inname=false;
651 }
652 if(epath.size()>0) epath.push_back('-');
653 for(std::string::const_iterator cc=name.begin();cc!=name.end();cc++){
654 epath.push_back(toupper(*cc));
655 }
656
657
658 /* Create a RCB that will recursively call process_sbuf() */
659 recursion_control_block rcb(process_sbuf,upperstr(name));
660
661 /* Call the scanner.*/
662 {
663 aftimer t;
664 if(debug & DEBUG_PRINT_STEPS){
665 std::cerr << "sbuf.pos0=" << sp.sbuf.pos0 << " calling scanner " << name << "\n";
666 }
667 t.start();
668 ((*it)->scanner)(sp,rcb);
669 t.stop();
670 if(debug & DEBUG_PRINT_STEPS){
671 std::cerr << "sbuf.pos0=" << sp.sbuf.pos0 << " scanner "
672 << name << " t=" << t.elapsed_seconds() << "\n";
673 }
674 sp.fs.add_stats(epath,t.elapsed_seconds());
675 }
676
677 }
678 catch (const std::exception &e ) {
679 std::stringstream ss;
680 ss << "std::exception Scanner: " << name
681 << " Exception: " << e.what()
682 << " sbuf.pos0: " << sp.sbuf.pos0 << " bufsize=" << sp.sbuf.bufsize << "\n";
683 std::cerr << ss.str();
684 feature_recorder *alert_recorder = fs.get_alert_recorder();
685 if(alert_recorder) alert_recorder->write(sp.sbuf.pos0,"scanner="+name,
686 std::string("<exception>")+e.what()+"</exception>");
687 }
688 catch (...) {
689 std::stringstream ss;
690 ss << "std::exception Scanner: " << name
691 << " Unknown Exception "
692 << " sbuf.pos0: " << sp.sbuf.pos0 << " bufsize=" << sp.sbuf.bufsize << "\n";
693 std::cerr << ss.str();
694 feature_recorder *alert_recorder = fs.get_alert_recorder();
695 if(alert_recorder) alert_recorder->write(sp.sbuf.pos0,"scanner="+name,"<unknown_exception/>");
696 }
697 }
698 fs.flush_all();
699 }
700
701
702
703 /**
704 * Process a pcap packet.
705 * Designed to be very efficient because we have so many packets.
706 */
process_packet(const be13::packet_info & pi)707 void be13::plugin::process_packet(const be13::packet_info &pi)
708 {
709 for(packet_plugin_info_vector_t::iterator it = packet_handlers.begin(); it != packet_handlers.end(); it++){
710 (*(*it).callback)((*it).user,pi);
711 }
712 }
713
714
get_scanner_feature_file_names(feature_file_names_t & feature_file_names)715 void be13::plugin::get_scanner_feature_file_names(feature_file_names_t &feature_file_names)
716 {
717 for(scanner_vector::const_iterator it=current_scanners.begin();it!=current_scanners.end();it++){
718 if((*it)->enabled){
719 for(std::set<std::string>::const_iterator fi=(*it)->info.feature_names.begin();
720 fi!=(*it)->info.feature_names.end();
721 fi++){
722 feature_file_names.insert(*fi);
723 }
724 }
725 }
726 }
727
728