1#!/usr/bin/env python
2#
3# Naxsi learning utility
4#
5
6# Builtins
7import glob, fcntl, termios
8import sys
9import socket
10import time
11import os
12import tempfile
13import subprocess
14import json
15import logging
16from collections import defaultdict
17from optparse import OptionParser, OptionGroup
18
19# Third party
20import elasticsearch
21from nxapi.nxtransform import *
22from nxapi.nxparse import *
23
24# Default values
25F_SETPIPE_SZ = 1031  # Linux 2.6.35+
26F_GETPIPE_SZ = 1032  # Linux 2.6.35+
27
28# Initialize logging
29logging.basicConfig(stream=sys.stdout, level=logging.INFO,
30                    format=None,
31                    datefmt=None)
32
33def open_fifo(fifo):
34    try:
35        os.mkfifo(fifo)
36    except OSError:
37        logging.warning("Fifo ["+fifo+"] already exists (non fatal).")
38    except Exception, e:
39        logging.error("Unable to create fifo ["+fifo+"]")
40    try:
41        logging.debug("Opening fifo ... will return when data is available.")
42        fifo_fd = open(fifo, 'r')
43        fcntl.fcntl(fifo_fd, F_SETPIPE_SZ, 1000000)
44        logging.debug("Pipe (modified) size : "+str(fcntl.fcntl(fifo_fd, F_GETPIPE_SZ)))
45    except Exception, e:
46        logging.error("Unable to create fifo, error: "+str(e))
47        return None
48    return fifo_fd
49
50def macquire(line):
51    z = parser.parse_raw_line(line)
52    # add data str and country
53    if z is not None:
54        for event in z['events']:
55            event['date'] = z['date']
56            try:
57                event['coords'] = geoloc.ip2ll(event['ip'])
58                event['country'] = geoloc.ip2cc(event['ip'])
59            except NameError:
60                pass
61        logging.debug(z)
62        injector.insert(z)
63    else:
64        pass
65
66opt = OptionParser()
67# group : config
68p = OptionGroup(opt, "Configuration options")
69p.add_option('-c', '--config', dest="cfg_path", default="/usr/local/etc/nxapi.json", help="Path to nxapi.json (config).")
70p.add_option('--colors', dest="colors", action="store_false", default="true", help="Disable output colorz.")
71# p.add_option('-q', '--quiet', dest="quiet_flag", action="store_true", help="Be quiet.")
72# p.add_option('-v', '--verbose', dest="verb_flag", action="store_true", help="Be verbose.")
73opt.add_option_group(p)
74# group : in option
75p = OptionGroup(opt, "Input options (log acquisition)")
76p.add_option('--files', dest="files_in", help="Path to log files to parse.")
77p.add_option('--fifo', dest="fifo_in", help="Path to a FIFO to be created & read from. [infinite]")
78p.add_option('--stdin', dest="stdin", action="store_true", help="Read from stdin.")
79p.add_option('--no-timeout', dest="infinite_flag", action="store_true", help="Disable timeout on read operations (stdin/fifo).")
80p.add_option('--syslog', dest="syslog_in", action="store_true", help="Listen on tcp port for syslog logging.")
81opt.add_option_group(p)
82# group : filtering
83p = OptionGroup(opt, "Filtering options (for whitelist generation)")
84p.add_option('-s', '--server', dest="server", help="FQDN to which we should restrict operations.")
85p.add_option('--filter', dest="filter", action="append", help="This option specify a filter for each type of filter, filter are merge with existing templates/filters. (--filter 'uri /foobar')")
86opt.add_option_group(p)
87# group : tagging
88p = OptionGroup(opt, "Tagging options (tag existing events in database)")
89p.add_option('-w', '--whitelist-path', dest="wl_file", help="A path to whitelist file, will find matching events in DB.")
90p.add_option('-i', '--ip-path', dest="ips", help="A path to IP list file, will find matching events in DB.")
91p.add_option('--tag', dest="tag", action="store_true", help="Actually tag matching items in DB.")
92opt.add_option_group(p)
93# group : whitelist generation
94p = OptionGroup(opt, "Whitelist Generation")
95p.add_option('-f', '--full-auto', dest="full_auto", action="store_true", help="Attempt fully automatic whitelist generation process.")
96p.add_option('-t', '--template', dest="template", help="Path to template to apply.")
97p.add_option('--slack', dest="slack", action="store_false", help="Enables less strict mode.")
98p.add_option('--type', dest="type_wl", action="store_true", help="Generate whitelists based on param type")
99opt.add_option_group(p)
100# group : statistics
101p = OptionGroup(opt, "Statistics Generation")
102p.add_option('-x', '--stats', dest="stats", action="store_true", help="Generate statistics about current's db content.")
103opt.add_option_group(p)
104# group : interactive generation
105p = OptionGroup(opt, "Interactive Whitelists Generation")
106p.add_option('-g', '--interactive-generation', dest="int_gen", action="store_true", help="Use your favorite text editor for whitelist generation.")
107opt.add_option_group(p)
108
109(options, args) = opt.parse_args()
110
111
112try:
113    cfg = NxConfig(options.cfg_path)
114except ValueError:
115    sys.exit(-1)
116
117if options.server is not None:
118    cfg.cfg["global_filters"]["server"] = options.server
119
120# https://github.com/nbs-system/naxsi/issues/231
121mutally_exclusive = ['stats', 'full_auto', 'template', 'wl_file', 'ips', 'files_in', 'fifo_in', 'syslog_in']
122count=0
123for x in mutally_exclusive:
124    if options.ensure_value(x, None) is not None:
125        count += 1
126if count > 1:
127    logging.critical("Mutually exclusive options are present (ie. import and stats), aborting.")
128    sys.exit(-1)
129
130
131cfg.cfg["output"]["colors"] = "false" if options.int_gen else str(options.colors).lower()
132cfg.cfg["naxsi"]["strict"] = str(options.slack).lower()
133
134def get_filter(arg_filter):
135    x = {}
136    to_parse = []
137    kwlist = ['server', 'uri', 'zone', 'var_name', 'ip', 'id', 'content', 'country', 'date',
138              '?server', '?uri', '?var_name', '?content']
139    try:
140        for argstr in arg_filter:
141            argstr = ' '.join(argstr.split())
142            to_parse += argstr.split(' ')
143        if [a for a in kwlist if a in to_parse]:
144            for kw in to_parse:
145                if kw in kwlist:
146                    x[kw] = to_parse[to_parse.index(kw)+1]
147        else:
148            raise
149    except:
150        logging.critical('option --filter must have at least one option')
151        sys.exit(-1)
152    return x
153
154if options.filter is not None:
155    cfg.cfg["global_filters"].update(get_filter(options.filter))
156
157try:
158    use_ssl = bool(cfg.cfg["elastic"]["use_ssl"])
159except KeyError:
160    use_ssl = False
161
162es = elasticsearch.Elasticsearch(cfg.cfg["elastic"]["host"], use_ssl=use_ssl)
163# Get ES version from the client and avail it at cfg
164es_version =  es.info()['version'].get('number', None)
165if es_version is not None:
166    cfg.cfg["elastic"]["version"] = es_version.split(".")[0]
167if cfg.cfg["elastic"].get("version", None) is None:
168    logging.critical("Failed to get version from ES, Specify version ['1'/'2'/'5'] in [elasticsearch] section")
169    sys.exit(-1)
170
171translate = NxTranslate(es, cfg)
172
173
174
175if options.type_wl is True:
176    translate.wl_on_type()
177    sys.exit(0)
178
179# whitelist generation options
180if options.full_auto is True:
181    translate.load_cr_file(translate.cfg["naxsi"]["rules_path"])
182    results = translate.full_auto()
183    if results:
184        for result in results:
185            logging.info("{0}".format(result))
186    else:
187        logging.critical("No hits for this filter.")
188        sys.exit(1)
189    sys.exit(0)
190
191if options.template is not None:
192    scoring = NxRating(cfg.cfg, es, translate)
193
194    tpls = translate.expand_tpl_path(options.template)
195    gstats = {}
196    if len(tpls) <= 0:
197        logging.error("No template matching")
198        sys.exit(1)
199    # prepare statistics for global scope
200    scoring.refresh_scope('global', translate.tpl2esq(cfg.cfg["global_filters"]))
201    for tpl_f in tpls:
202        scoring.refresh_scope('rule', {})
203        scoring.refresh_scope('template', {})
204
205        logging.debug(translate.grn.format("Loading template '"+tpl_f+"'"))
206        tpl = translate.load_tpl_file(tpl_f)
207        # prepare statistics for filter scope
208        scoring.refresh_scope('template', translate.tpl2esq(tpl))
209        logging.debug("Hits of template : "+str(scoring.get('template', 'total')))
210
211        whitelists = translate.gen_wl(tpl, rule={})
212        logging.debug(str(len(whitelists))+" whitelists ...")
213        for genrule in whitelists:
214            #pprint.pprint(genrule)
215            scoring.refresh_scope('rule', genrule['rule'])
216            scores = scoring.check_rule_score(tpl)
217            if (len(scores['success']) > len(scores['warnings']) and scores['deny'] == False) or cfg.cfg["naxsi"]["strict"] == "false":
218                logging.debug(translate.fancy_display(genrule, scores, tpl))
219                logging.debug(translate.grn.format(translate.tpl2wl(genrule['rule'], tpl)).encode('utf-8'))
220    sys.exit(0)
221
222# tagging options
223
224if options.wl_file is not None and options.server is None:
225    logging.critical(translate.red.format("Cannot tag events in database without a server name !"))
226    sys.exit(2)
227
228if options.wl_file is not None:
229    wl_files = []
230    wl_files.extend(glob.glob(options.wl_file))
231    count = 0
232    for wlf in wl_files:
233        logging.debug(translate.grn.format("Loading template '"+wlf+"'"))
234        try:
235            wlfd = open(wlf, "r")
236        except:
237            logging.critical(translate.red.format("Unable to open wl file '"+wlf+"'"))
238            sys.exit(-1)
239        for wl in wlfd:
240            [res, esq] = translate.wl2esq(wl)
241            if res is True:
242                count = 0
243                while True:
244                    x = translate.tag_events(esq, "Whitelisted", tag=options.tag)
245                    count += x
246                    if x == 0:
247                        break
248
249        logging.debug(translate.grn.format(str(count)) + " items tagged ...")
250        count = 0
251    sys.exit(0)
252
253if options.ips is not None:
254    ip_files = []
255    ip_files.extend(glob.glob(options.ips))
256    tpl = {}
257    count = 0
258#    esq = translate.tpl2esq(cfg.cfg["global_filters"])
259
260    for wlf in ip_files:
261        try:
262            wlfd = open(wlf, "r")
263        except:
264            logging.critical("Unable to open ip file '"+wlf+"'")
265            sys.exit(-1)
266        for wl in wlfd:
267            logging.debug("=>"+wl)
268            tpl["ip"] = wl.strip('\n')
269            esq = translate.tpl2esq(tpl)
270            pprint.pprint(esq)
271            pprint.pprint(tpl)
272            count += translate.tag_events(esq, "BadIPS", tag=options.tag)
273        logging.debug(translate.grn.format(str(count)) + " items to be tagged ...")
274        count = 0
275    sys.exit(0)
276
277# statistics
278if options.stats is True:
279    logging.info(translate.red.format("# Whitelist(ing) ratio :"))
280    for e in translate.fetch_top(cfg.cfg["global_filters"], "whitelisted", limit=2):
281        try:
282            list_e = e.split()
283            logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3]))
284        except:
285            logging.warning("--malformed--")
286    logging.info(translate.red.format("# Top servers :"))
287    for e in translate.fetch_top(cfg.cfg["global_filters"], "server", limit=10):
288        try:
289            list_e = e.split()
290            logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3]))
291        except:
292            logging.warning("--malformed--")
293    logging.info(translate.red.format("# Top URI(s) :"))
294    for e in translate.fetch_top(cfg.cfg["global_filters"], "uri", limit=10):
295        try:
296            list_e = e.split()
297            logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3]))
298        except:
299            logging.warning("--malformed--")
300    logging.info(translate.red.format("# Top Zone(s) :"))
301    for e in translate.fetch_top(cfg.cfg["global_filters"], "zone", limit=10):
302        try:
303            list_e = e.split()
304            logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3]))
305        except:
306            logging.warning("--malformed--")
307    logging.info(translate.red.format("# Top Peer(s) :"))
308    for e in translate.fetch_top(cfg.cfg["global_filters"], "ip", limit=10):
309        try:
310            list_e = e.split()
311            logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3]))
312        except:
313            logging.warning("--malformed--")
314    logging.info(translate.red.format("# Top Country(ies) :"))
315    for e in translate.fetch_top(cfg.cfg["global_filters"], "country", limit=10):
316        try:
317            list_e = e.split()
318            logging.info('# {0} {1} {2}{3}'.format(translate.grn.format(list_e[0]), list_e[1], list_e[2], list_e[3]))
319        except:
320            logging.warning("--malformed--")
321    sys.exit(0)
322
323
324def write_generated_wl(filename, results):
325
326    with open('/tmp/{0}'.format(filename), 'w') as wl_file:
327        for result in results:
328            for key, items in result.iteritems():
329                if items:
330                    logging.debug("{} {}".format(key, items))
331                    if key == 'genrule':
332                        wl_file.write("# {}\n{}\n".format(key, items))
333                    else:
334                        wl_file.write("# {} {}\n".format(key, items))
335        wl_file.flush()
336
337def ask_user_for_server_selection(editor, welcome_sentences, selection):
338    with tempfile.NamedTemporaryFile(suffix='.tmp') as temporary_file:
339        top_selection = translate.fetch_top(cfg.cfg["global_filters"],
340                            selection,
341                            limit=10
342                        )
343        temporary_file.write(welcome_sentences)
344        for line in top_selection:
345            temporary_file.write('{0}\n'.format(line))
346        temporary_file.flush()
347        subprocess.call([editor, temporary_file.name])
348        temporary_file.seek(len(welcome_sentences))
349        ret = []
350        for line in temporary_file:
351            if not line.startswith('#'):
352                ret.append(line.strip().split()[0])
353    return ret
354
355def ask_user_for_selection(editor, welcome_sentences, selection, servers):
356    regex_message = "# as in the --filter option you can add ? for regex\n"
357    ret = {}
358    for server in servers:
359        server_reminder = "server: {0}\n\n".format(server)
360        ret[server] = []
361        with tempfile.NamedTemporaryFile(suffix='.tmp') as temporary_file:
362            temporary_file.write(welcome_sentences + regex_message + server_reminder)
363            cfg.cfg["global_filters"]["server"] = server
364            top_selection = translate.fetch_top(cfg.cfg["global_filters"],
365                                selection,
366                                limit=10
367                            )
368            for line in top_selection:
369                temporary_file.write('{0} {1}\n'.format(selection, line))
370            temporary_file.flush()
371            subprocess.call([editor, temporary_file.name])
372            temporary_file.seek(len(welcome_sentences) + len(server_reminder) + len(regex_message))
373            for line in temporary_file:
374                if not line.startswith('#'):
375                    res = line.strip().split()
376                    ret[server].append((res[0], res[1]))
377    return ret
378
379def generate_wl(selection_dict):
380    for key, items in selection_dict.iteritems():
381        if not items:
382            return False
383        global_filters_context = cfg.cfg["global_filters"]
384        global_filters_context["server"] = key
385        for idx, (selection, item) in enumerate(items):
386           global_filters_context[selection] = item
387           translate.cfg["global_filters"] = global_filters_context
388           logging.debug('generating wl with filters {0}'.format(global_filters_context))
389           wl_dict_list = []
390           res = translate.full_auto(wl_dict_list)
391           del global_filters_context[selection]
392           write_generated_wl(
393               "server_{0}_{1}.wl".format(
394                                    key,
395                                    idx if (selection == "uri") else "zone_{0}".format(item),
396                                ),
397               wl_dict_list
398           )
399
400if options.int_gen is True:
401    editor = os.environ.get('EDITOR', 'vi')
402
403    welcome_sentences = '{0}\n{1}\n'.format(
404        '# all deleted line or starting with a # will be ignore',
405        '# if you want to use slack option you have to specify it on the command line options'
406    )
407
408    servers = ask_user_for_server_selection(editor, welcome_sentences, "server")
409
410    uris = ask_user_for_selection(editor, welcome_sentences, "uri", servers)
411    zones = ask_user_for_selection(editor, welcome_sentences, "zone", servers)
412
413    if uris:
414        generate_wl(uris)
415    if zones:
416        generate_wl(zones)
417    # in case the user let uri and zone files empty generate wl for all
418    # selected server(s)
419    if not uris and not zones:
420        for server in servers:
421            translate.cfg["global_filters"]["server"] = server
422            logging.debug('generating with filters: {0}'.format(translate.cfg["global_filters"]))
423            res = translate.full_auto()
424            writing_generated_wl("server_{0}.wl".format(server), res)
425
426    sys.exit(0)
427
428# input options, only setup injector if one input option is present
429if options.files_in is not None or options.fifo_in is not None or options.stdin is not None or options.syslog_in is not None:
430    if options.fifo_in is not None or options.syslog_in is not None:
431        injector = ESInject(es, cfg.cfg, auto_commit_limit=1)
432    else:
433        injector = ESInject(es, cfg.cfg)
434    parser = NxParser()
435    offset = time.timezone if (time.localtime().tm_isdst == 0) else time.altzone
436    offset = offset / 60 / 60 * -1
437    if offset < 0:
438        offset = str(-offset)
439    else:
440        offset = str(offset)
441    offset = offset.zfill(2)
442    parser.out_date_format = "%Y-%m-%dT%H:%M:%S+"+offset #ES-friendly
443    try:
444        geoloc = NxGeoLoc(cfg.cfg)
445    except:
446        logging.critical("Unable to get GeoIP")
447
448if options.files_in is not None:
449    reader = NxReader(macquire, lglob=[options.files_in])
450    reader.read_files()
451    injector.stop()
452    sys.exit(0)
453
454if options.fifo_in is not None:
455    fd = open_fifo(options.fifo_in)
456    if options.infinite_flag is True:
457        reader = NxReader(macquire, fd=fd, stdin_timeout=None)
458    else:
459        reader = NxReader(macquire, fd=fd)
460    while True:
461        logging.debug("start-",)
462        if reader.read_files() == False:
463            break
464        logging.debug("stop")
465    logging.debug('End of fifo input...')
466    injector.stop()
467    sys.exit(0)
468
469if options.syslog_in is not None:
470    sysloghost = cfg.cfg["syslogd"]["host"]
471    syslogport = cfg.cfg["syslogd"]["port"]
472    while 1:
473      reader = NxReader(macquire, syslog=True, syslogport=syslogport, sysloghost=sysloghost)
474      reader.read_files()
475    injector.stop()
476    sys.exit(0)
477
478if options.stdin is True:
479    if options.infinite_flag:
480        reader = NxReader(macquire, lglob=[], fd=sys.stdin, stdin_timeout=None)
481    else:
482        reader = NxReader(macquire, lglob=[], fd=sys.stdin)
483    while True:
484        logging.debug("start-",)
485        if reader.read_files() == False:
486            break
487        logging.debug("stop")
488    logging.debug('End of stdin input...')
489    injector.stop()
490    sys.exit(0)
491
492opt.print_help()
493sys.exit(0)
494