1#!/usr/bin/env python 2# 3# Naxsi learning utility 4# 5 6# Builtins 7import glob, fcntl, termios 8import sys 9import socket 10import time 11import os 12import tempfile 13import subprocess 14import json 15import logging 16from collections import defaultdict 17from optparse import OptionParser, OptionGroup 18 19# Third party 20import elasticsearch 21from nxapi.nxtransform import * 22from nxapi.nxparse import * 23 24# Default values 25F_SETPIPE_SZ = 1031 # Linux 2.6.35+ 26F_GETPIPE_SZ = 1032 # Linux 2.6.35+ 27 28# Initialize logging 29logging.basicConfig(stream=sys.stdout, level=logging.INFO, 30 format=None, 31 datefmt=None) 32 33def open_fifo(fifo): 34 try: 35 os.mkfifo(fifo) 36 except OSError: 37 logging.warning("Fifo ["+fifo+"] already exists (non fatal).") 38 except Exception, e: 39 logging.error("Unable to create fifo ["+fifo+"]") 40 try: 41 logging.debug("Opening fifo ... will return when data is available.") 42 fifo_fd = open(fifo, 'r') 43 fcntl.fcntl(fifo_fd, F_SETPIPE_SZ, 1000000) 44 logging.debug("Pipe (modified) size : "+str(fcntl.fcntl(fifo_fd, F_GETPIPE_SZ))) 45 except Exception, e: 46 logging.error("Unable to create fifo, error: "+str(e)) 47 return None 48 return fifo_fd 49 50def macquire(line): 51 z = parser.parse_raw_line(line) 52 # add data str and country 53 if z is not None: 54 for event in z['events']: 55 event['date'] = z['date'] 56 try: 57 event['coords'] = geoloc.ip2ll(event['ip']) 58 event['country'] = geoloc.ip2cc(event['ip']) 59 except NameError: 60 pass 61 logging.debug(z) 62 injector.insert(z) 63 else: 64 pass 65 66opt = OptionParser() 67# group : config 68p = OptionGroup(opt, "Configuration options") 69p.add_option('-c', '--config', dest="cfg_path", default="/usr/local/etc/nxapi.json", help="Path to nxapi.json (config).") 70p.add_option('--colors', dest="colors", action="store_false", default="true", help="Disable output colorz.") 71# p.add_option('-q', '--quiet', dest="quiet_flag", action="store_true", help="Be quiet.") 72# p.add_option('-v', '--verbose', dest="verb_flag", action="store_true", help="Be verbose.") 73opt.add_option_group(p) 74# group : in option 75p = OptionGroup(opt, "Input options (log acquisition)") 76p.add_option('--files', dest="files_in", help="Path to log files to parse.") 77p.add_option('--fifo', dest="fifo_in", help="Path to a FIFO to be created & read from. [infinite]") 78p.add_option('--stdin', dest="stdin", action="store_true", help="Read from stdin.") 79p.add_option('--no-timeout', dest="infinite_flag", action="store_true", help="Disable timeout on read operations (stdin/fifo).") 80p.add_option('--syslog', dest="syslog_in", action="store_true", help="Listen on tcp port for syslog logging.") 81opt.add_option_group(p) 82# group : filtering 83p = OptionGroup(opt, "Filtering options (for whitelist generation)") 84p.add_option('-s', '--server', dest="server", help="FQDN to which we should restrict operations.") 85p.add_option('--filter', dest="filter", action="append", help="This option specify a filter for each type of filter, filter are merge with existing templates/filters. (--filter 'uri /foobar')") 86opt.add_option_group(p) 87# group : tagging 88p = OptionGroup(opt, "Tagging options (tag existing events in database)") 89p.add_option('-w', '--whitelist-path', dest="wl_file", help="A path to whitelist file, will find matching events in DB.") 90p.add_option('-i', '--ip-path', dest="ips", help="A path to IP list file, will find matching events in DB.") 91p.add_option('--tag', dest="tag", action="store_true", help="Actually tag matching items in DB.") 92opt.add_option_group(p) 93# group : whitelist generation 94p = OptionGroup(opt, "Whitelist Generation") 95p.add_option('-f', '--full-auto', dest="full_auto", action="store_true", help="Attempt fully automatic whitelist generation process.") 96p.add_option('-t', '--template', dest="template", help="Path to template to apply.") 97p.add_option('--slack', dest="slack", action="store_false", help="Enables less strict mode.") 98p.add_option('--type', dest="type_wl", action="store_true", help="Generate whitelists based on param type") 99opt.add_option_group(p) 100# group : statistics 101p = OptionGroup(opt, "Statistics Generation") 102p.add_option('-x', '--stats', dest="stats", action="store_true", help="Generate statistics about current's db content.") 103opt.add_option_group(p) 104# group : interactive generation 105p = OptionGroup(opt, "Interactive Whitelists Generation") 106p.add_option('-g', '--interactive-generation', dest="int_gen", action="store_true", help="Use your favorite text editor for whitelist generation.") 107opt.add_option_group(p) 108 109(options, args) = opt.parse_args() 110 111 112try: 113 cfg = NxConfig(options.cfg_path) 114except ValueError: 115 sys.exit(-1) 116 117if options.server is not None: 118 cfg.cfg["global_filters"]["server"] = options.server 119 120# https://github.com/nbs-system/naxsi/issues/231 121mutally_exclusive = ['stats', 'full_auto', 'template', 'wl_file', 'ips', 'files_in', 'fifo_in', 'syslog_in'] 122count=0 123for x in mutally_exclusive: 124 if options.ensure_value(x, None) is not None: 125 count += 1 126if count > 1: 127 logging.critical("Mutually exclusive options are present (ie. import and stats), aborting.") 128 sys.exit(-1) 129 130 131cfg.cfg["output"]["colors"] = "false" if options.int_gen else str(options.colors).lower() 132cfg.cfg["naxsi"]["strict"] = str(options.slack).lower() 133 134def get_filter(arg_filter): 135 x = {} 136 to_parse = [] 137 kwlist = ['server', 'uri', 'zone', 'var_name', 'ip', 'id', 'content', 'country', 'date', 138 '?server', '?uri', '?var_name', '?content'] 139 try: 140 for argstr in arg_filter: 141 argstr = ' '.join(argstr.split()) 142 to_parse += argstr.split(' ') 143 if [a for a in kwlist if a in to_parse]: 144 for kw in to_parse: 145 if kw in kwlist: 146 x[kw] = to_parse[to_parse.index(kw)+1] 147 else: 148 raise 149 except: 150 logging.critical('option --filter must have at least one option') 151 sys.exit(-1) 152 return x 153 154if options.filter is not None: 155 cfg.cfg["global_filters"].update(get_filter(options.filter)) 156 157try: 158 use_ssl = bool(cfg.cfg["elastic"]["use_ssl"]) 159except KeyError: 160 use_ssl = False 161 162es = elasticsearch.Elasticsearch(cfg.cfg["elastic"]["host"], use_ssl=use_ssl) 163# Get ES version from the client and avail it at cfg 164es_version = es.info()['version'].get('number', None) 165if es_version is not None: 166 cfg.cfg["elastic"]["version"] = es_version.split(".")[0] 167if cfg.cfg["elastic"].get("version", None) is None: 168 logging.critical("Failed to get version from ES, Specify version ['1'/'2'/'5'] in [elasticsearch] section") 169 sys.exit(-1) 170 171translate = NxTranslate(es, cfg) 172 173 174 175if options.type_wl is True: 176 translate.wl_on_type() 177 sys.exit(0) 178 179# whitelist generation options 180if options.full_auto is True: 181 translate.load_cr_file(translate.cfg["naxsi"]["rules_path"]) 182 results = translate.full_auto() 183 if results: 184 for result in results: 185 logging.info("{0}".format(result)) 186 else: 187 logging.critical("No hits for this filter.") 188 sys.exit(1) 189 sys.exit(0) 190 191if options.template is not None: 192 scoring = NxRating(cfg.cfg, es, translate) 193 194 tpls = translate.expand_tpl_path(options.template) 195 gstats = {} 196 if len(tpls) <= 0: 197 logging.error("No template matching") 198 sys.exit(1) 199 # prepare statistics for global scope 200 scoring.refresh_scope('global', translate.tpl2esq(cfg.cfg["global_filters"])) 201 for tpl_f in tpls: 202 scoring.refresh_scope('rule', {}) 203 scoring.refresh_scope('template', {}) 204 205 logging.debug(translate.grn.format("Loading template '"+tpl_f+"'")) 206 tpl = translate.load_tpl_file(tpl_f) 207 # prepare statistics for filter scope 208 scoring.refresh_scope('template', translate.tpl2esq(tpl)) 209 logging.debug("Hits of template : "+str(scoring.get('template', 'total'))) 210 211 whitelists = translate.gen_wl(tpl, rule={}) 212 logging.debug(str(len(whitelists))+" whitelists ...") 213 for genrule in whitelists: 214 #pprint.pprint(genrule) 215 scoring.refresh_scope('rule', genrule['rule']) 216 scores = scoring.check_rule_score(tpl) 217 if (len(scores['success']) > len(scores['warnings']) and scores['deny'] == False) or cfg.cfg["naxsi"]["strict"] == "false": 218 logging.debug(translate.fancy_display(genrule, scores, tpl)) 219 logging.debug(translate.grn.format(translate.tpl2wl(genrule['rule'], tpl)).encode('utf-8')) 220 sys.exit(0) 221 222# tagging options 223 224if options.wl_file is not None and options.server is None: 225 logging.critical(translate.red.format("Cannot tag events in database without a server name !")) 226 sys.exit(2) 227 228if options.wl_file is not None: 229 wl_files = [] 230 wl_files.extend(glob.glob(options.wl_file)) 231 count = 0 232 for wlf in wl_files: 233 logging.debug(translate.grn.format("Loading template '"+wlf+"'")) 234 try: 235 wlfd = open(wlf, "r") 236 except: 237 logging.critical(translate.red.format("Unable to open wl file '"+wlf+"'")) 238 sys.exit(-1) 239 for wl in wlfd: 240 [res, esq] = translate.wl2esq(wl) 241 if res is True: 242 count = 0 243 while True: 244 x = translate.tag_events(esq, "Whitelisted", tag=options.tag) 245 count += x 246 if x == 0: 247 break 248 249 logging.debug(translate.grn.format(str(count)) + " items tagged ...") 250 count = 0 251 sys.exit(0) 252 253if options.ips is not None: 254 ip_files = [] 255 ip_files.extend(glob.glob(options.ips)) 256 tpl = {} 257 count = 0 258# esq = translate.tpl2esq(cfg.cfg["global_filters"]) 259 260 for wlf in ip_files: 261 try: 262 wlfd = open(wlf, "r") 263 except: 264 logging.critical("Unable to open ip file '"+wlf+"'") 265 sys.exit(-1) 266 for wl in wlfd: 267 logging.debug("=>"+wl) 268 tpl["ip"] = wl.strip('\n') 269 esq = translate.tpl2esq(tpl) 270 pprint.pprint(esq) 271 pprint.pprint(tpl) 272 count += translate.tag_events(esq, "BadIPS", tag=options.tag) 273 logging.debug(translate.grn.format(str(count)) + " items to be tagged ...") 274 count = 0 275 sys.exit(0) 276 277# statistics 278if options.stats is True: 279 logging.info(translate.red.format("# Whitelist(ing) ratio :")) 280 for e in translate.fetch_top(cfg.cfg["global_filters"], "whitelisted", limit=2): 281 try: 282 list_e = e.split() 283 logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3])) 284 except: 285 logging.warning("--malformed--") 286 logging.info(translate.red.format("# Top servers :")) 287 for e in translate.fetch_top(cfg.cfg["global_filters"], "server", limit=10): 288 try: 289 list_e = e.split() 290 logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3])) 291 except: 292 logging.warning("--malformed--") 293 logging.info(translate.red.format("# Top URI(s) :")) 294 for e in translate.fetch_top(cfg.cfg["global_filters"], "uri", limit=10): 295 try: 296 list_e = e.split() 297 logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3])) 298 except: 299 logging.warning("--malformed--") 300 logging.info(translate.red.format("# Top Zone(s) :")) 301 for e in translate.fetch_top(cfg.cfg["global_filters"], "zone", limit=10): 302 try: 303 list_e = e.split() 304 logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3])) 305 except: 306 logging.warning("--malformed--") 307 logging.info(translate.red.format("# Top Peer(s) :")) 308 for e in translate.fetch_top(cfg.cfg["global_filters"], "ip", limit=10): 309 try: 310 list_e = e.split() 311 logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3])) 312 except: 313 logging.warning("--malformed--") 314 logging.info(translate.red.format("# Top Country(ies) :")) 315 for e in translate.fetch_top(cfg.cfg["global_filters"], "country", limit=10): 316 try: 317 list_e = e.split() 318 logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3])) 319 except: 320 logging.warning("--malformed--") 321 sys.exit(0) 322 323 324def write_generated_wl(filename, results): 325 326 with open('/tmp/{0}'.format(filename), 'w') as wl_file: 327 for result in results: 328 for key, items in result.iteritems(): 329 if items: 330 logging.debug("{} {}".format(key, items)) 331 if key == 'genrule': 332 wl_file.write("# {}\n{}\n".format(key, items)) 333 else: 334 wl_file.write("# {} {}\n".format(key, items)) 335 wl_file.flush() 336 337def ask_user_for_server_selection(editor, welcome_sentences, selection): 338 with tempfile.NamedTemporaryFile(suffix='.tmp') as temporary_file: 339 top_selection = translate.fetch_top(cfg.cfg["global_filters"], 340 selection, 341 limit=10 342 ) 343 temporary_file.write(welcome_sentences) 344 for line in top_selection: 345 temporary_file.write('{0}\n'.format(line)) 346 temporary_file.flush() 347 subprocess.call([editor, temporary_file.name]) 348 temporary_file.seek(len(welcome_sentences)) 349 ret = [] 350 for line in temporary_file: 351 if not line.startswith('#'): 352 ret.append(line.strip().split()[0]) 353 return ret 354 355def ask_user_for_selection(editor, welcome_sentences, selection, servers): 356 regex_message = "# as in the --filter option you can add ? for regex\n" 357 ret = {} 358 for server in servers: 359 server_reminder = "server: {0}\n\n".format(server) 360 ret[server] = [] 361 with tempfile.NamedTemporaryFile(suffix='.tmp') as temporary_file: 362 temporary_file.write(welcome_sentences + regex_message + server_reminder) 363 cfg.cfg["global_filters"]["server"] = server 364 top_selection = translate.fetch_top(cfg.cfg["global_filters"], 365 selection, 366 limit=10 367 ) 368 for line in top_selection: 369 temporary_file.write('{0} {1}\n'.format(selection, line)) 370 temporary_file.flush() 371 subprocess.call([editor, temporary_file.name]) 372 temporary_file.seek(len(welcome_sentences) + len(server_reminder) + len(regex_message)) 373 for line in temporary_file: 374 if not line.startswith('#'): 375 res = line.strip().split() 376 ret[server].append((res[0], res[1])) 377 return ret 378 379def generate_wl(selection_dict): 380 for key, items in selection_dict.iteritems(): 381 if not items: 382 return False 383 global_filters_context = cfg.cfg["global_filters"] 384 global_filters_context["server"] = key 385 for idx, (selection, item) in enumerate(items): 386 global_filters_context[selection] = item 387 translate.cfg["global_filters"] = global_filters_context 388 logging.debug('generating wl with filters {0}'.format(global_filters_context)) 389 wl_dict_list = [] 390 res = translate.full_auto(wl_dict_list) 391 del global_filters_context[selection] 392 write_generated_wl( 393 "server_{0}_{1}.wl".format( 394 key, 395 idx if (selection == "uri") else "zone_{0}".format(item), 396 ), 397 wl_dict_list 398 ) 399 400if options.int_gen is True: 401 editor = os.environ.get('EDITOR', 'vi') 402 403 welcome_sentences = '{0}\n{1}\n'.format( 404 '# all deleted line or starting with a # will be ignore', 405 '# if you want to use slack option you have to specify it on the command line options' 406 ) 407 408 servers = ask_user_for_server_selection(editor, welcome_sentences, "server") 409 410 uris = ask_user_for_selection(editor, welcome_sentences, "uri", servers) 411 zones = ask_user_for_selection(editor, welcome_sentences, "zone", servers) 412 413 if uris: 414 generate_wl(uris) 415 if zones: 416 generate_wl(zones) 417 # in case the user let uri and zone files empty generate wl for all 418 # selected server(s) 419 if not uris and not zones: 420 for server in servers: 421 translate.cfg["global_filters"]["server"] = server 422 logging.debug('generating with filters: {0}'.format(translate.cfg["global_filters"])) 423 res = translate.full_auto() 424 writing_generated_wl("server_{0}.wl".format(server), res) 425 426 sys.exit(0) 427 428# input options, only setup injector if one input option is present 429if options.files_in is not None or options.fifo_in is not None or options.stdin is not None or options.syslog_in is not None: 430 if options.fifo_in is not None or options.syslog_in is not None: 431 injector = ESInject(es, cfg.cfg, auto_commit_limit=1) 432 else: 433 injector = ESInject(es, cfg.cfg) 434 parser = NxParser() 435 offset = time.timezone if (time.localtime().tm_isdst == 0) else time.altzone 436 offset = offset / 60 / 60 * -1 437 if offset < 0: 438 offset = str(-offset) 439 else: 440 offset = str(offset) 441 offset = offset.zfill(2) 442 parser.out_date_format = "%Y-%m-%dT%H:%M:%S+"+offset #ES-friendly 443 try: 444 geoloc = NxGeoLoc(cfg.cfg) 445 except: 446 logging.critical("Unable to get GeoIP") 447 448if options.files_in is not None: 449 reader = NxReader(macquire, lglob=[options.files_in]) 450 reader.read_files() 451 injector.stop() 452 sys.exit(0) 453 454if options.fifo_in is not None: 455 fd = open_fifo(options.fifo_in) 456 if options.infinite_flag is True: 457 reader = NxReader(macquire, fd=fd, stdin_timeout=None) 458 else: 459 reader = NxReader(macquire, fd=fd) 460 while True: 461 logging.debug("start-",) 462 if reader.read_files() == False: 463 break 464 logging.debug("stop") 465 logging.debug('End of fifo input...') 466 injector.stop() 467 sys.exit(0) 468 469if options.syslog_in is not None: 470 sysloghost = cfg.cfg["syslogd"]["host"] 471 syslogport = cfg.cfg["syslogd"]["port"] 472 while 1: 473 reader = NxReader(macquire, syslog=True, syslogport=syslogport, sysloghost=sysloghost) 474 reader.read_files() 475 injector.stop() 476 sys.exit(0) 477 478if options.stdin is True: 479 if options.infinite_flag: 480 reader = NxReader(macquire, lglob=[], fd=sys.stdin, stdin_timeout=None) 481 else: 482 reader = NxReader(macquire, lglob=[], fd=sys.stdin) 483 while True: 484 logging.debug("start-",) 485 if reader.read_files() == False: 486 break 487 logging.debug("stop") 488 logging.debug('End of stdin input...') 489 injector.stop() 490 sys.exit(0) 491 492opt.print_help() 493sys.exit(0) 494