1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  * common.cpp:
4  * bulk_extractor backend stuff, used for both standalone executable and bulk_extractor.
5  */
6 
7 #include "config.h"
8 #include <sys/types.h>
9 #include <sys/stat.h>
10 #include <dirent.h>
11 #include <algorithm>
12 
13 #ifdef HAVE_ERR_H
14 #include <err.h>
15 #endif
16 
17 #ifdef HAVE_DLFCN_H
18 #include <dlfcn.h>
19 #endif
20 
21 #include "bulk_extractor_i.h"
22 #include "aftimer.h"
23 #include "../dfxml/src/hash_t.h"
24 
25 
26 uint32_t scanner_def::max_depth = 7;            // max recursion depth
27 uint32_t scanner_def::max_ngram = 10;            // max recursion depth
28 static int debug;                               // local debug variable
29 static uint32_t max_depth_seen=0;
30 static cppmutex max_depth_seenM;
31 bool be13::plugin::dup_data_alerts = false; // by default, is disabled
32 uint64_t be13::plugin::dup_data_encountered = 0; // amount that was not processed
33 
34 class scanner_command {
35 public:
36     enum command_t {DISABLE_ALL=0,ENABLE_ALL,DISABLE,ENABLE};
scanner_command(const scanner_command & sc)37     scanner_command(const scanner_command &sc):command(sc.command),name(sc.name){};
scanner_command(scanner_command::command_t c,const std::string & n)38     scanner_command(scanner_command::command_t c,const std::string &n):command(c),name(n){};
39     command_t command;
40     std::string name;
41 };
42 static std::vector<scanner_command> scanner_commands;
43 bool scanner_commands_processed = false;
44 
45 /****************************************************************
46  *** misc support
47  ****************************************************************/
48 
49 
50 /****************************************************************
51  *** SCANNER PLUG-IN SYSTEM
52  ****************************************************************/
53 
54 /* scanner_params */
55 
56 scanner_params::PrintOptions scanner_params::no_options;
57 
58 /* vector object for keeping track of packet callbacks */
59 
60 class packet_plugin_info {
61 public:
packet_plugin_info(void * user_,packet_callback_t * callback_)62     packet_plugin_info(void *user_,packet_callback_t *callback_):user(user_),callback(callback_){};
63     void *user;
64     packet_callback_t *callback;
65 };
66 
67 typedef std::vector<packet_plugin_info> packet_plugin_info_vector_t;
68 packet_plugin_info_vector_t  packet_handlers;   // pcap callback handlers
69 
70 
71 /* plugin */
72 
73 /**
74  * the vector of current scanners
75  */
76 
77 be13::plugin::scanner_vector be13::plugin::current_scanners;
78 
set_scanner_debug(int adebug)79 void be13::plugin::set_scanner_debug(int adebug)
80 {
81     debug = adebug;
82 }
83 
84 
85 /**
86  * return true a scanner is enabled
87  */
88 
89 /* enable or disable a specific scanner.
90  * enable = 0  - disable that scanner.
91  * enable = 1  - enable that scanner
92  * 'all' is a special scanner that enables all scanners.
93  */
94 
set_scanner_enabled(const std::string & name,bool enable)95 void be13::plugin::set_scanner_enabled(const std::string &name,bool enable)
96 {
97     for(scanner_vector::iterator it = current_scanners.begin();it!=current_scanners.end();it++){
98         if(name=="all" && (((*it)->info.flags & scanner_info::SCANNER_NO_ALL)==0)){
99             (*it)->enabled = enable;
100         }
101         if((*it)->info.name==name){
102             (*it)->enabled = enable;
103             return;
104         }
105     }
106     if(name=="all") return;
107     std::cerr << "Invalid scanner name '" << name << "'\n";
108     exit(1);
109 }
110 
set_scanner_enabled_all(bool enable)111 void be13::plugin::set_scanner_enabled_all(bool enable)
112 {
113     for(scanner_vector::const_iterator it = current_scanners.begin();it!=current_scanners.end();it++){
114         (*it)->enabled = enable;
115     }
116 }
117 
118 /** Name of feature files that should be histogramed.
119  * The histogram should be done in the plug-in
120  */
121 
122 /****************************************************************
123  *** scanner plugin loading
124  ****************************************************************/
125 
126 /**
127  * plugin system phase 0: Load a scanner.
128  *
129  * As part of scanner loading:
130  * - pass configuration to the scanner
131  * - feature files that the scanner requires
132  * - Histograms that the scanner makes (see feature_recorder_set)
133  * This is called before scanners are enabled or disabled, so the pcap handlers
134  * need to be set afterwards
135  */
load_scanner(scanner_t scanner,const scanner_info::scanner_config & sc)136 void be13::plugin::load_scanner(scanner_t scanner,const scanner_info::scanner_config &sc)
137 {
138     /* If scanner is already loaded, return */
139     for(scanner_vector::const_iterator it = current_scanners.begin();it!=current_scanners.end();it++){
140         if((*it)->scanner==scanner) return;
141     }
142 
143     /* Use an empty sbuf and an empty feature recorder set as the parameters for the sp below.
144      * we use static values so that the sbuf is not constantly being created and destroyed.
145      */
146     static const sbuf_t sbuf;
147     static feature_recorder_set fs(feature_recorder_set::SET_DISABLED,feature_recorder_set::null_hasher,
148                                    feature_recorder_set::NO_INPUT,
149                                    feature_recorder_set::NO_OUTDIR); // dummy
150 
151     //
152     // Each scanner's params are stored in a scanner_def object that
153     // is created here and retained for the duration of the run.
154     // The scanner_def includes its own scanner_info structure.
155     // We pre-load the structure with the configuration for this scanner
156     // and the global debug variable
157     //
158     // currently every scanner gets the same config. In the future, we might
159     // want to give different scanners different variables.
160     //
161 
162     scanner_params sp(scanner_params::PHASE_STARTUP,sbuf,fs); //
163     scanner_def *sd = new scanner_def();
164     sd->scanner = scanner;
165     sd->info.config = &sc;
166 
167     sp.info  = &sd->info;
168 
169     // Make an empty recursion control block and call the scanner's
170     // initialization function.
171     recursion_control_block rcb(0,"");
172     (*scanner)(sp,rcb);                  // phase 0
173 
174     sd->enabled      = !(sd->info.flags & scanner_info::SCANNER_DISABLED);
175     current_scanners.push_back(sd);
176 }
177 
load_scanner_file(std::string fn,const scanner_info::scanner_config & sc)178 void be13::plugin::load_scanner_file(std::string fn,const scanner_info::scanner_config &sc)
179 {
180     /* Figure out the function name */
181     size_t extloc = fn.rfind('.');
182     if(extloc==std::string::npos){
183         fprintf(stderr,"Cannot find '.' in %s",fn.c_str());
184         exit(1);
185     }
186     std::string func_name = fn.substr(0,extloc);
187     size_t slashloc = func_name.rfind('/');
188     if(slashloc!=std::string::npos) func_name = func_name.substr(slashloc+1);
189     slashloc = func_name.rfind('\\');
190     if(slashloc!=std::string::npos) func_name = func_name.substr(slashloc+1);
191 
192     if(debug) std::cout << "Loading: " << fn << " (" << func_name << ")\n";
193     scanner_t *scanner = 0;
194 #if defined(HAVE_DLOPEN)
195     void *lib=dlopen(fn.c_str(), RTLD_LAZY);
196 
197     if(lib==0){
198         fprintf(stderr,"dlopen: %s\n",dlerror());
199         exit(1);
200     }
201 
202     /* Resolve the symbol */
203     scanner = (scanner_t *)dlsym(lib, func_name.c_str());
204 
205     if(scanner==0){
206         fprintf(stderr,"dlsym: %s\n",dlerror());
207         exit(1);
208     }
209 #elif defined(HAVE_LOADLIBRARY)
210     /* Use Win32 LoadLibrary function */
211     /* See http://msdn.microsoft.com/en-us/library/ms686944(v=vs.85).aspx */
212     HINSTANCE hinstLib = LoadLibrary(TEXT(fn.c_str()));
213     if(hinstLib==0){
214         fprintf(stderr,"LoadLibrary(%s) failed",fn.c_str());
215         exit(1);
216     }
217     scanner = (scanner_t *)GetProcAddress(hinstLib,func_name.c_str());
218     if(scanner==0){
219         fprintf(stderr,"GetProcAddress(%s) failed",func_name.c_str());
220         exit(1);
221     }
222 #else
223     std::cout << "  ERROR: Support for loadable libraries not enabled\n";
224     return;
225 #endif
226     load_scanner(*scanner,sc);
227 }
228 
load_scanners(scanner_t * const * scanners,const scanner_info::scanner_config & sc)229 void be13::plugin::load_scanners(scanner_t * const *scanners,const scanner_info::scanner_config &sc)
230 {
231     for(int i=0;scanners[i];i++){
232         load_scanner(scanners[i],sc);
233     }
234 }
235 
load_scanner_directory(const std::string & dirname,const scanner_info::scanner_config & sc)236 void be13::plugin::load_scanner_directory(const std::string &dirname,const scanner_info::scanner_config &sc )
237 {
238     DIR *dirp = opendir(dirname.c_str());
239     if(dirp==0){
240         fprintf(stderr,"Cannot open directory %s:",dirname.c_str());
241         exit(1);
242     }
243     struct dirent *dp;
244     while ((dp = readdir(dirp)) != NULL){
245         std::string fname = dp->d_name;
246         if(fname.substr(0,5)=="scan_" || fname.substr(0,5)=="SCAN_"){
247             size_t extloc = fname.rfind('.');
248             if(extloc==std::string::npos) continue; // no '.'
249             std::string ext = fname.substr(extloc+1);
250 #ifdef WIN32
251             if(ext!="DLL") continue;    // not a DLL
252 #else
253             if(ext!="so") continue;     // not a shared library
254 #endif
255             load_scanner_file(dirname+"/"+fname,sc );
256         }
257     }
258 }
259 
load_scanner_directories(const std::vector<std::string> & dirnames,const scanner_info::scanner_config & sc)260 void be13::plugin::load_scanner_directories(const std::vector<std::string> &dirnames,
261                                             const scanner_info::scanner_config &sc)
262 {
263     for(std::vector<std::string>::const_iterator it = dirnames.begin();it!=dirnames.end();it++){
264         load_scanner_directory(*it,sc);
265     }
266 }
267 
load_scanner_packet_handlers()268 void be13::plugin::load_scanner_packet_handlers()
269 {
270     for(scanner_vector::const_iterator it = current_scanners.begin(); it!=current_scanners.end(); it++){
271         if((*it)->enabled){
272             const scanner_def *sd = (*it);
273             if(sd->info.packet_cb){
274                 packet_handlers.push_back(packet_plugin_info(sd->info.packet_user,sd->info.packet_cb));
275             }
276         }
277     }
278 }
279 
280 // send every enabled scanner the phase message
message_enabled_scanners(scanner_params::phase_t phase,feature_recorder_set & fs)281 void be13::plugin::message_enabled_scanners(scanner_params::phase_t phase,feature_recorder_set &fs)
282 {
283     /* make an empty sbuf and feature recorder set */
284     const sbuf_t sbuf;
285     scanner_params sp(phase,sbuf,fs);
286     for(scanner_vector::iterator it = current_scanners.begin(); it!=current_scanners.end(); it++){
287         if((*it)->enabled){
288             recursion_control_block rcb(0,""); // dummy rcb
289             ((*it)->scanner)(sp,rcb);
290         }
291     }
292 }
293 
find_scanner(const std::string & search_name)294 scanner_t *be13::plugin::find_scanner(const std::string &search_name)
295 {
296     for(scanner_vector::const_iterator it = current_scanners.begin();it!=current_scanners.end();it++){
297 	if(search_name == (*it)->info.name){
298             return (*it)->scanner;
299         }
300     }
301     return 0;
302 }
303 
304 // put the enabled scanners into the vector
get_enabled_scanners(std::vector<std::string> & svector)305 void be13::plugin::get_enabled_scanners(std::vector<std::string> &svector)
306 {
307     for(scanner_vector::const_iterator it=current_scanners.begin();it!=current_scanners.end();it++){
308 	if((*it)->enabled){
309             svector.push_back((*it)->info.name);
310 	}
311     }
312 }
313 
find_scanner_enabled()314 bool be13::plugin::find_scanner_enabled()
315 {
316     for(scanner_vector::const_iterator it = current_scanners.begin(); it!=current_scanners.end(); it++){
317         if( ((*it)->info.flags & scanner_info::SCANNER_FIND_SCANNER)
318             && ((*it)->enabled)){
319             return true;
320         }
321     }
322     return false;
323 }
324 
325 
add_enabled_scanner_histograms_to_feature_recorder_set(feature_recorder_set & fs)326 void be13::plugin::add_enabled_scanner_histograms_to_feature_recorder_set(feature_recorder_set &fs)
327 {
328     for(scanner_vector::const_iterator it = current_scanners.begin(); it!=current_scanners.end(); it++){
329         if((*it)->enabled){
330             const scanner_def *sd = (*it);
331             for(histogram_defs_t::const_iterator i2 = sd->info.histogram_defs.begin();
332                 i2 != sd->info.histogram_defs.end(); i2++){
333                 fs.add_histogram((*i2));
334             }
335         }
336     }
337 }
338 
scanners_init(feature_recorder_set & fs)339 void be13::plugin::scanners_init(feature_recorder_set &fs)
340 {
341     assert(scanner_commands_processed==true);
342     message_enabled_scanners(scanner_params::PHASE_INIT,fs); // tell all enabled scanners to init
343 }
344 
345 
346 
347 /****************************************************************
348  *** Scanner Commands (which one is enabled or disabled)
349  ****************************************************************/
350 
scanners_disable_all()351 void be13::plugin::scanners_disable_all()
352 {
353     assert(scanner_commands_processed==false);
354     scanner_commands.push_back(scanner_command(scanner_command::DISABLE_ALL,std::string("")));
355 }
356 
scanners_enable_all()357 void be13::plugin::scanners_enable_all()
358 {
359     assert(scanner_commands_processed==false);
360     scanner_commands.push_back(scanner_command(scanner_command::ENABLE_ALL,std::string("")));
361 }
362 
scanners_enable(const std::string & name)363 void be13::plugin::scanners_enable(const std::string &name)
364 {
365     assert(scanner_commands_processed==false);
366     scanner_commands.push_back(scanner_command(scanner_command::ENABLE,name));
367 }
368 
scanners_disable(const std::string & name)369 void be13::plugin::scanners_disable(const std::string &name)
370 {
371     assert(scanner_commands_processed==false);
372     scanner_commands.push_back(scanner_command(scanner_command::DISABLE,name));
373 }
374 
scanners_process_enable_disable_commands()375 void be13::plugin::scanners_process_enable_disable_commands()
376 {
377     for(std::vector<scanner_command>::const_iterator it=scanner_commands.begin();
378         it!=scanner_commands.end();it++){
379         switch((*it).command){
380         case scanner_command::ENABLE_ALL:  set_scanner_enabled_all(true);break;
381         case scanner_command::DISABLE_ALL: set_scanner_enabled_all(false); break;
382         case scanner_command::ENABLE:      set_scanner_enabled((*it).name,true);break;
383         case scanner_command::DISABLE:     set_scanner_enabled((*it).name,false);break;
384         }
385     }
386     load_scanner_packet_handlers();     // can't do until enable/disable commands are run
387     scanner_commands_processed = true;
388 }
389 
390 
391 /****************************************************************
392  *** PHASE_SHUTDOWN (formerly phase 2): shut down the scanners
393  ****************************************************************/
394 
phase_shutdown(feature_recorder_set & fs,std::stringstream * sxml)395 void be13::plugin::phase_shutdown(feature_recorder_set &fs,std::stringstream *sxml)
396 {
397     assert(scanner_commands_processed==true);
398     for(scanner_vector::iterator it = current_scanners.begin();it!=current_scanners.end();it++){
399         if((*it)->enabled){
400             const sbuf_t sbuf; // empty sbuf
401             scanner_params sp(scanner_params::PHASE_SHUTDOWN,sbuf,fs,sxml);
402             recursion_control_block rcb(0,"");        // empty rcb
403             (*(*it)->scanner)(sp,rcb);
404         }
405     }
406 }
407 
408 /************************************
409  *** HELP and  option processing  ***
410  ************************************/
411 
412 /* Get the config and build the help strings at the same time! */
413 
414 std::stringstream scanner_info::helpstream;
get_config(const scanner_info::config_t & c,const std::string & n,std::string * val,const std::string & help)415 void scanner_info::get_config(const scanner_info::config_t &c,
416                               const std::string &n,std::string *val,const std::string &help)
417 {
418     /* Check to see if we are being called as part of a help operation */
419     helpstream << "   -S " << n << "=" << *val << "    " << help << " (" << name << ")\n";
420     scanner_info::config_t::const_iterator it = c.find(n);
421     if(it!=c.end() && val){
422         *val = it->second;
423     }
424 }
425 
get_config(const std::string & n,std::string * val,const std::string & help)426 void scanner_info::get_config(const std::string &n,std::string *val,const std::string &help)
427 {
428     scanner_info::get_config(config->namevals,n,val,help);
429 }
430 
431 #define GET_CONFIG(T) void scanner_info::get_config(const std::string &n,T *val,const std::string &help) {\
432         std::stringstream ss;\
433         ss << *val;\
434         std::string v(ss.str());\
435         get_config(n,&v,help);\
436         ss.str(v);\
437         ss >> *val;\
438     }
439 
440 GET_CONFIG(uint64_t)
GET_CONFIG(int32_t)441 GET_CONFIG(int32_t)                     // both int32_t and uint32_t
442 GET_CONFIG(uint32_t)
443 GET_CONFIG(uint16_t)
444 #ifdef HAVE_GET_CONFIG_SIZE_T
445 GET_CONFIG(size_t)
446 #endif
447 
448 
449 /* uint8_t needs cast to uint32_t for <<
450  * Otherwise it is interpreted as a character.
451  */
452 void scanner_info::get_config(const std::string &n,uint8_t *val_,const std::string &help)
453 {
454     uint32_t val = *val_;
455     std::stringstream ss;
456     ss << val;
457     std::string v(ss.str());
458     get_config(n,&v,help);
459     ss.str(v);
460     ss >> val;
461     *val_ = (uint8_t)val;
462 }
463 
464 /* bool needs special processing for YES/NO/TRUE/FALSE */
get_config(const std::string & n,bool * val,const std::string & help)465 void scanner_info::get_config(const std::string &n,bool *val,const std::string &help)
466 {
467     std::stringstream ss;
468     ss << ((*val) ? "YES" : "NO");
469     std::string v(ss.str());
470     get_config(n,&v,help);
471     switch(v.at(0)){
472     case 'Y':case 'y':case 'T':case 't':case '1':
473         *val = true;
474         break;
475     default:
476         *val = false;
477     }
478 }
479 
480 
481 /**
482  * Print a list of scanners.
483  * We need to load them to do this, so they are loaded with empty config
484  * Note that scanners can only be loaded once, so this exits.
485  */
info_scanners(bool detailed_info,bool detailed_settings,scanner_t * const * scanners_builtin,const char enable_opt,const char disable_opt)486 void be13::plugin::info_scanners(bool detailed_info,
487                                  bool detailed_settings,
488                                  scanner_t * const *scanners_builtin,
489                                  const char enable_opt,const char disable_opt)
490 {
491     const scanner_info::scanner_config empty_config;
492 
493     load_scanners(scanners_builtin,empty_config);
494     std::cout << "\n";
495     std::vector<std::string> enabled_wordlist;
496     std::vector<std::string> disabled_wordlist;
497     for(scanner_vector::const_iterator it = current_scanners.begin();it!=current_scanners.end();it++){
498         if(detailed_info){
499             if ((*it)->info.name.size()) std::cout << "Scanner Name: " << (*it)->info.name << "\n";
500             std::cout << "flags:  " << scanner_info::flag_to_string((*it)->info.flags) << "\n";
501             std::cout << "Scanner Interface version: " << (*it)->info.si_version << "\n";
502             if ((*it)->info.author.size()) std::cout << "Author: " << (*it)->info.author << "\n";
503             if ((*it)->info.description.size()) std::cout << "Description: " << (*it)->info.description << "\n";
504             if ((*it)->info.url.size()) std::cout << "URL: " << (*it)->info.url << "\n";
505             if ((*it)->info.scanner_version.size()) std::cout << "Scanner Version: " << (*it)->info.scanner_version << "\n";
506             std::cout << "Feature Names: ";
507             for(std::set<std::string>::const_iterator i2 = (*it)->info.feature_names.begin();
508                 i2 != (*it)->info.feature_names.end();
509                 i2++){
510                 std::cout << *i2 << " ";
511             }
512             std::cout << "\n\n";
513         }
514         if((*it)->info.flags & scanner_info::SCANNER_NO_USAGE) continue;
515         if((*it)->info.flags & scanner_info::SCANNER_DISABLED){
516             disabled_wordlist.push_back((*it)->info.name);
517         } else {
518             enabled_wordlist.push_back((*it)->info.name);
519         }
520     }
521     if(detailed_settings){
522         std::cout << "Settable Options (and their defaults): \n";
523         std::cout << scanner_info::helpstr();
524     }
525     sort(disabled_wordlist.begin(),disabled_wordlist.end());
526     sort(enabled_wordlist.begin(),enabled_wordlist.end());
527     std::cout << "\n";
528     std::cout << "These scanners disabled by default; enable with -" << enable_opt << ":\n";
529     for(std::vector<std::string>::const_iterator it = disabled_wordlist.begin();
530         it!=disabled_wordlist.end();it++){
531         std::cout << "   -" << enable_opt << " " <<  *it << " - enable scanner " << *it << "\n";
532     }
533     std::cout << "\n";
534     std::cout << "These scanners enabled by default; disable with -" << disable_opt << ":\n";
535     for(std::vector<std::string>::const_iterator it = enabled_wordlist.begin();it!=enabled_wordlist.end();it++){
536         std::cout << "   -" << disable_opt << " " <<  *it << " - disable scanner " << *it << "\n";
537     }
538 }
539 
540 /**
541  * upperstr - Turns an ASCII string into upper case (should be UTF-8)
542  */
543 
upperstr(const std::string & str)544 static std::string upperstr(const std::string &str)
545 {
546     std::string ret;
547     for(std::string::const_iterator i=str.begin();i!=str.end();i++){
548         ret.push_back(toupper(*i));
549     }
550     return ret;
551 }
552 
553 /* Determine if the sbuf consists of a repeating ngram */
find_ngram_size(const sbuf_t & sbuf)554 static size_t find_ngram_size(const sbuf_t &sbuf)
555 {
556     for(size_t ngram_size = 1; ngram_size < scanner_def::max_ngram; ngram_size++){
557 	bool ngram_match = true;
558 	for(size_t i=ngram_size;i<sbuf.pagesize && ngram_match;i++){
559 	    if(sbuf[i%ngram_size]!=sbuf[i]) ngram_match = false;
560 	}
561 	if(ngram_match) return ngram_size;
562     }
563     return 0;                           // no ngram size
564 }
565 
get_max_depth_seen()566 uint32_t be13::plugin::get_max_depth_seen()
567 {
568     cppmutex::lock lock(max_depth_seenM);
569     return max_depth_seen;
570 }
571 
572 /** process_sbuf is the main workhorse. It is calls each scanner on each page.
573  * @param sp    - the scanner params, including the sbuf to process
574  * It is also the recursive entry point for sub-analysis.
575  */
576 
process_sbuf(const class scanner_params & sp)577 void be13::plugin::process_sbuf(const class scanner_params &sp)
578 {
579     const pos0_t &pos0 = sp.sbuf.pos0;
580     class feature_recorder_set &fs = sp.fs;
581 
582     fs.heartbeat();                     // note that we are alive
583 
584     {
585         /* note the maximum depth that we've seen */
586         cppmutex::lock lock(max_depth_seenM);
587         if(sp.depth > max_depth_seen) max_depth_seen = sp.depth;
588     }
589 
590     /* If we are too deep, error out */
591     if(sp.depth >= scanner_def::max_depth){
592         feature_recorder *fr = fs.get_alert_recorder();
593         if(fr) fr->write(pos0,"process_extract: MAX DEPTH REACHED","");
594         return;
595     }
596 
597     /* Determine if we have seen this buffer before */
598     bool seen_before = fs.check_previously_processed(sp.sbuf.buf,sp.sbuf.bufsize);
599     if(seen_before){
600         md5_t md5 = md5_generator::hash_buf(sp.sbuf.buf,sp.sbuf.bufsize);
601         feature_recorder *alert_recorder = fs.get_alert_recorder();
602         std::stringstream ss;
603         ss << "<buflen>" << sp.sbuf.bufsize  << "</buflen>";
604         if(alert_recorder && dup_data_alerts) alert_recorder->write(sp.sbuf.pos0,"DUP SBUF "+md5.hexdigest(),ss.str());
605 #ifdef HAVE__SYNC_ADD_AND_FETCH
606         __sync_add_and_fetch(&dup_data_encountered,sp.sbuf.bufsize);
607 #endif
608     }
609 
610     /* Determine if the sbuf consists of a repeating ngram. If so,
611      * it's only passed to the parsers that want ngrams. (By default,
612      * such sbufs are booring.)
613      */
614 
615     size_t ngram_size = find_ngram_size(sp.sbuf);
616 
617     /****************************************************************
618      *** CALL EACH OF THE SCANNERS ON THE SBUF
619      ****************************************************************/
620 
621     if(debug & DEBUG_DUMP_DATA){
622         sp.sbuf.hex_dump(std::cerr);
623     }
624 
625     for(scanner_vector::iterator it = current_scanners.begin();it!=current_scanners.end();it++){
626         // Look for reasons not to run a scanner
627         if((*it)->enabled==false) continue; // not enabled
628 
629         if(((*it)->info.flags & scanner_info::SCANNER_WANTS_NGRAMS)==0){
630             /* If the scanner does not want ngrams, don't run it if we have ngrams or duplicate data */
631             if(ngram_size > 0) continue;
632             if(seen_before)    continue;
633         }
634 
635         if(sp.depth > 0 && ((*it)->info.flags & scanner_info::SCANNER_DEPTH_0)){
636             // depth >0 and this scanner only run at depth 0
637             continue;
638         }
639 
640         const std::string &name = (*it)->info.name;
641 
642         try {
643 
644             /* Compute the effective path for stats */
645             bool inname=false;
646             std::string epath;
647             for(std::string::const_iterator cc=sp.sbuf.pos0.path.begin();cc!=sp.sbuf.pos0.path.end();cc++){
648                 if(isupper(*cc)) inname=true;
649                 if(inname) epath.push_back(toupper(*cc));
650                 if(*cc=='-') inname=false;
651             }
652             if(epath.size()>0) epath.push_back('-');
653             for(std::string::const_iterator cc=name.begin();cc!=name.end();cc++){
654                 epath.push_back(toupper(*cc));
655             }
656 
657 
658             /* Create a RCB that will recursively call process_sbuf() */
659             recursion_control_block rcb(process_sbuf,upperstr(name));
660 
661             /* Call the scanner.*/
662             {
663                 aftimer t;
664                 if(debug & DEBUG_PRINT_STEPS){
665                     std::cerr << "sbuf.pos0=" << sp.sbuf.pos0 << " calling scanner " << name << "\n";
666                 }
667                 t.start();
668                 ((*it)->scanner)(sp,rcb);
669                 t.stop();
670                 if(debug & DEBUG_PRINT_STEPS){
671                     std::cerr << "sbuf.pos0=" << sp.sbuf.pos0 << " scanner "
672                          << name << " t=" << t.elapsed_seconds() << "\n";
673                 }
674                 sp.fs.add_stats(epath,t.elapsed_seconds());
675             }
676 
677         }
678         catch (const std::exception &e ) {
679             std::stringstream ss;
680             ss << "std::exception Scanner: " << name
681                << " Exception: " << e.what()
682                << " sbuf.pos0: " << sp.sbuf.pos0 << " bufsize=" << sp.sbuf.bufsize << "\n";
683             std::cerr << ss.str();
684             feature_recorder *alert_recorder = fs.get_alert_recorder();
685             if(alert_recorder) alert_recorder->write(sp.sbuf.pos0,"scanner="+name,
686                                                      std::string("<exception>")+e.what()+"</exception>");
687         }
688         catch (...) {
689             std::stringstream ss;
690             ss << "std::exception Scanner: " << name
691                << " Unknown Exception "
692                << " sbuf.pos0: " << sp.sbuf.pos0 << " bufsize=" << sp.sbuf.bufsize << "\n";
693             std::cerr << ss.str();
694             feature_recorder *alert_recorder = fs.get_alert_recorder();
695             if(alert_recorder) alert_recorder->write(sp.sbuf.pos0,"scanner="+name,"<unknown_exception/>");
696         }
697     }
698     fs.flush_all();
699 }
700 
701 
702 
703 /**
704  * Process a pcap packet.
705  * Designed to be very efficient because we have so many packets.
706  */
process_packet(const be13::packet_info & pi)707 void be13::plugin::process_packet(const be13::packet_info &pi)
708 {
709     for(packet_plugin_info_vector_t::iterator it = packet_handlers.begin(); it != packet_handlers.end(); it++){
710         (*(*it).callback)((*it).user,pi);
711     }
712 }
713 
714 
get_scanner_feature_file_names(feature_file_names_t & feature_file_names)715 void be13::plugin::get_scanner_feature_file_names(feature_file_names_t &feature_file_names)
716 {
717     for(scanner_vector::const_iterator it=current_scanners.begin();it!=current_scanners.end();it++){
718         if((*it)->enabled){
719             for(std::set<std::string>::const_iterator fi=(*it)->info.feature_names.begin();
720                 fi!=(*it)->info.feature_names.end();
721                 fi++){
722                 feature_file_names.insert(*fi);
723             }
724         }
725     }
726 }
727 
728