1# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- 2# vi: set ft=python sts=4 ts=4 sw=4 noet : 3# 4# This file is part of Fail2Ban. 5# 6# Fail2Ban is free software; you can redistribute it and/or modify 7# it under the terms of the GNU General Public License as published by 8# the Free Software Foundation; either version 2 of the License, or 9# (at your option) any later version. 10# 11# Fail2Ban is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License 17# along with Fail2Ban; if not, write to the Free Software 18# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19""" 20Fail2Ban reads log file that contains password failure report 21and bans the corresponding IP addresses using firewall rules. 22 23This tools can test regular expressions for "fail2ban". 24""" 25 26__author__ = "Fail2Ban Developers" 27__copyright__ = """Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors 28Copyright of modifications held by their respective authors. 29Licensed under the GNU General Public License v2 (GPL). 30 31Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>. 32Many contributions by Yaroslav O. Halchenko, Steven Hiscocks, Sergey G. Brester (sebres).""" 33 34__license__ = "GPL" 35 36import getopt 37import logging 38import os 39import shlex 40import sys 41import time 42import time 43import urllib.request, urllib.parse, urllib.error 44from optparse import OptionParser, Option 45 46from configparser import NoOptionError, NoSectionError, MissingSectionHeaderError 47 48try: # pragma: no cover 49 from ..server.filtersystemd import FilterSystemd 50except ImportError: 51 FilterSystemd = None 52 53from ..version import version, normVersion 54from .filterreader import FilterReader 55from ..server.filter import Filter, FileContainer 56from ..server.failregex import Regex, RegexException 57 58from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, \ 59 extractOptions, PREFER_ENC 60# Gets the instance of the logger. 61logSys = getLogger("fail2ban") 62 63def debuggexURL(sample, regex, multiline=False, useDns="yes"): 64 args = { 65 're': Regex._resolveHostTag(regex, useDns=useDns), 66 'str': sample, 67 'flavor': 'python' 68 } 69 if multiline: args['flags'] = 'm' 70 return 'https://www.debuggex.com/?' + urllib.parse.urlencode(args) 71 72def output(args): # pragma: no cover (overriden in test-cases) 73 print(args) 74 75def shortstr(s, l=53): 76 """Return shortened string 77 """ 78 if len(s) > l: 79 return s[:l-3] + '...' 80 return s 81 82def pprint_list(l, header=None): 83 if not len(l): 84 return 85 if header: 86 s = "|- %s\n" % header 87 else: 88 s = '' 89 output( s + "| " + "\n| ".join(l) + '\n`-' ) 90 91def journal_lines_gen(flt, myjournal): # pragma: no cover 92 while True: 93 try: 94 entry = myjournal.get_next() 95 except OSError: 96 continue 97 if not entry: 98 break 99 yield flt.formatJournalEntry(entry) 100 101def dumpNormVersion(*args): 102 output(normVersion()) 103 sys.exit(0) 104 105usage = lambda: "%s [OPTIONS] <LOG> <REGEX> [IGNOREREGEX]" % sys.argv[0] 106 107class _f2bOptParser(OptionParser): 108 def format_help(self, *args, **kwargs): 109 """ Overwritten format helper with full ussage.""" 110 self.usage = '' 111 return "Usage: " + usage() + "\n" + __doc__ + """ 112LOG: 113 string a string representing a log line 114 filename path to a log file (/var/log/auth.log) 115 systemd-journal search systemd journal (systemd-python required), 116 optionally with backend parameters, see `man jail.conf` 117 for usage and examples (systemd-journal[journalflags=1]). 118 119REGEX: 120 string a string representing a 'failregex' 121 filter name of filter, optionally with options (sshd[mode=aggressive]) 122 filename path to a filter file (filter.d/sshd.conf) 123 124IGNOREREGEX: 125 string a string representing an 'ignoreregex' 126 filename path to a filter file (filter.d/sshd.conf) 127\n""" + OptionParser.format_help(self, *args, **kwargs) + """\n 128Report bugs to https://github.com/fail2ban/fail2ban/issues\n 129""" + __copyright__ + "\n" 130 131 132def get_opt_parser(): 133 # use module docstring for help output 134 p = _f2bOptParser( 135 usage=usage(), 136 version="%prog " + version) 137 138 p.add_options([ 139 Option("-c", "--config", default='/usr/local/etc/fail2ban', 140 help="set alternate config directory"), 141 Option("-d", "--datepattern", 142 help="set custom pattern used to match date/times"), 143 Option("--timezone", "--TZ", action='store', default=None, 144 help="set time-zone used by convert time format"), 145 Option("-e", "--encoding", default=PREFER_ENC, 146 help="File encoding. Default: system locale"), 147 Option("-r", "--raw", action='store_true', default=False, 148 help="Raw hosts, don't resolve dns"), 149 Option("--usedns", action='store', default=None, 150 help="DNS specified replacement of tags <HOST> in regexp " 151 "('yes' - matches all form of hosts, 'no' - IP addresses only)"), 152 Option("-L", "--maxlines", type=int, default=0, 153 help="maxlines for multi-line regex."), 154 Option("-m", "--journalmatch", 155 help="journalctl style matches overriding filter file. " 156 "\"systemd-journal\" only"), 157 Option('-l', "--log-level", 158 dest="log_level", 159 default='critical', 160 help="Log level for the Fail2Ban logger to use"), 161 Option('-V', action="callback", callback=dumpNormVersion, 162 help="get version in machine-readable short format"), 163 Option('-v', '--verbose', action="count", dest="verbose", 164 default=0, 165 help="Increase verbosity"), 166 Option("--verbosity", action="store", dest="verbose", type=int, 167 help="Set numerical level of verbosity (0..4)"), 168 Option("--verbose-date", "--VD", action='store_true', 169 help="Verbose date patterns/regex in output"), 170 Option("-D", "--debuggex", action='store_true', 171 help="Produce debuggex.com urls for debugging there"), 172 Option("--no-check-all", action="store_false", dest="checkAllRegex", default=True, 173 help="Disable check for all regex's"), 174 Option("-o", "--out", action="store", dest="out", default=None, 175 help="Set token to print failure information only (row, id, ip, msg, host, ip4, ip6, dns, matches, ...)"), 176 Option("--print-no-missed", action='store_true', 177 help="Do not print any missed lines"), 178 Option("--print-no-ignored", action='store_true', 179 help="Do not print any ignored lines"), 180 Option("--print-all-matched", action='store_true', 181 help="Print all matched lines"), 182 Option("--print-all-missed", action='store_true', 183 help="Print all missed lines, no matter how many"), 184 Option("--print-all-ignored", action='store_true', 185 help="Print all ignored lines, no matter how many"), 186 Option("-t", "--log-traceback", action='store_true', 187 help="Enrich log-messages with compressed tracebacks"), 188 Option("--full-traceback", action='store_true', 189 help="Either to make the tracebacks full, not compressed (as by default)"), 190 ]) 191 192 return p 193 194 195class RegexStat(object): 196 197 def __init__(self, failregex): 198 self._stats = 0 199 self._failregex = failregex 200 self._ipList = list() 201 202 def __str__(self): 203 return "%s(%r) %d failed: %s" \ 204 % (self.__class__, self._failregex, self._stats, self._ipList) 205 206 def inc(self): 207 self._stats += 1 208 209 def getStats(self): 210 return self._stats 211 212 def getFailRegex(self): 213 return self._failregex 214 215 def appendIP(self, value): 216 self._ipList.append(value) 217 218 def getIPList(self): 219 return self._ipList 220 221 222class LineStats(object): 223 """Just a convenience container for stats 224 """ 225 def __init__(self, opts): 226 self.tested = self.matched = 0 227 self.matched_lines = [] 228 self.missed = 0 229 self.missed_lines = [] 230 self.ignored = 0 231 self.ignored_lines = [] 232 if opts.debuggex: 233 self.matched_lines_timeextracted = [] 234 self.missed_lines_timeextracted = [] 235 self.ignored_lines_timeextracted = [] 236 237 def __str__(self): 238 return "%(tested)d lines, %(ignored)d ignored, %(matched)d matched, %(missed)d missed" % self 239 240 # just for convenient str 241 def __getitem__(self, key): 242 return getattr(self, key) if hasattr(self, key) else '' 243 244 245class Fail2banRegex(object): 246 247 def __init__(self, opts): 248 # set local protected members from given options: 249 self.__dict__.update(dict(('_'+o,v) for o,v in opts.__dict__.items())) 250 self._opts = opts 251 self._maxlines_set = False # so we allow to override maxlines in cmdline 252 self._datepattern_set = False 253 self._journalmatch = None 254 255 self.share_config=dict() 256 self._filter = Filter(None) 257 self._prefREMatched = 0 258 self._prefREGroups = list() 259 self._ignoreregex = list() 260 self._failregex = list() 261 self._time_elapsed = None 262 self._line_stats = LineStats(opts) 263 264 if opts.maxlines: 265 self.setMaxLines(opts.maxlines) 266 else: 267 self._maxlines = 20 268 if opts.journalmatch is not None: 269 self.setJournalMatch(shlex.split(opts.journalmatch)) 270 if opts.timezone: 271 self._filter.setLogTimeZone(opts.timezone) 272 if opts.datepattern: 273 self.setDatePattern(opts.datepattern) 274 if opts.usedns: 275 self._filter.setUseDns(opts.usedns) 276 self._filter.returnRawHost = opts.raw 277 self._filter.checkFindTime = False 278 self._filter.checkAllRegex = opts.checkAllRegex and not opts.out 279 # ignore pending (without ID/IP), added to matches if it hits later (if ID/IP can be retreved) 280 self._filter.ignorePending = opts.out 281 # callback to increment ignored RE's by index (during process): 282 self._filter.onIgnoreRegex = self._onIgnoreRegex 283 self._backend = 'auto' 284 285 def output(self, line): 286 if not self._opts.out: output(line) 287 288 def decode_line(self, line): 289 return FileContainer.decode_line('<LOG>', self._encoding, line) 290 291 def encode_line(self, line): 292 return line.encode(self._encoding, 'ignore') 293 294 def setDatePattern(self, pattern): 295 if not self._datepattern_set: 296 self._filter.setDatePattern(pattern) 297 self._datepattern_set = True 298 if pattern is not None: 299 self.output( "Use datepattern : %s : %s" % ( 300 pattern, self._filter.getDatePattern()[1], ) ) 301 302 def setMaxLines(self, v): 303 if not self._maxlines_set: 304 self._filter.setMaxLines(int(v)) 305 self._maxlines_set = True 306 self.output( "Use maxlines : %d" % self._filter.getMaxLines() ) 307 308 def setJournalMatch(self, v): 309 self._journalmatch = v 310 311 def _dumpRealOptions(self, reader, fltOpt): 312 realopts = {} 313 combopts = reader.getCombined() 314 # output all options that are specified in filter-argument as well as some special (mostly interested): 315 for k in ['logtype', 'datepattern'] + list(fltOpt.keys()): 316 # combined options win, but they contain only a sub-set in filter expected keys, 317 # so get the rest from definition section: 318 try: 319 realopts[k] = combopts[k] if k in combopts else reader.get('Definition', k) 320 except NoOptionError: # pragma: no cover 321 pass 322 self.output("Real filter options : %r" % realopts) 323 324 def readRegex(self, value, regextype): 325 assert(regextype in ('fail', 'ignore')) 326 regex = regextype + 'regex' 327 # try to check - we've case filter?[options...]?: 328 basedir = self._opts.config 329 fltFile = None 330 fltOpt = {} 331 if regextype == 'fail': 332 fltName, fltOpt = extractOptions(value) 333 if fltName is not None: 334 if "." in fltName[~5:]: 335 tryNames = (fltName,) 336 else: 337 tryNames = (fltName, fltName + '.conf', fltName + '.local') 338 for fltFile in tryNames: 339 if not "/" in fltFile: 340 if os.path.basename(basedir) == 'filter.d': 341 fltFile = os.path.join(basedir, fltFile) 342 else: 343 fltFile = os.path.join(basedir, 'filter.d', fltFile) 344 else: 345 basedir = os.path.dirname(fltFile) 346 if os.path.isfile(fltFile): 347 break 348 fltFile = None 349 # if it is filter file: 350 if fltFile is not None: 351 if (basedir == self._opts.config 352 or os.path.basename(basedir) == 'filter.d' 353 or ("." not in fltName[~5:] and "/" not in fltName) 354 ): 355 ## within filter.d folder - use standard loading algorithm to load filter completely (with .local etc.): 356 if os.path.basename(basedir) == 'filter.d': 357 basedir = os.path.dirname(basedir) 358 fltName = os.path.splitext(os.path.basename(fltName))[0] 359 self.output( "Use %11s filter file : %s, basedir: %s" % (regex, fltName, basedir) ) 360 else: 361 ## foreign file - readexplicit this file and includes if possible: 362 self.output( "Use %11s file : %s" % (regex, fltName) ) 363 basedir = None 364 if not os.path.isabs(fltName): # avoid join with "filter.d" inside FilterReader 365 fltName = os.path.abspath(fltName) 366 if fltOpt: 367 self.output( "Use filter options : %r" % fltOpt ) 368 reader = FilterReader(fltName, 'fail2ban-regex-jail', fltOpt, share_config=self.share_config, basedir=basedir) 369 ret = None 370 try: 371 if basedir is not None: 372 ret = reader.read() 373 else: 374 ## foreign file - readexplicit this file and includes if possible: 375 reader.setBaseDir(None) 376 ret = reader.readexplicit() 377 except Exception as e: 378 output("Wrong config file: %s" % (str(e),)) 379 if self._verbose: raise(e) 380 if not ret: 381 output( "ERROR: failed to load filter %s" % value ) 382 return False 383 # set backend-related options (logtype): 384 reader.applyAutoOptions(self._backend) 385 # get, interpolate and convert options: 386 reader.getOptions(None) 387 # show real options if expected: 388 if self._verbose > 1 or logSys.getEffectiveLevel()<=logging.DEBUG: 389 self._dumpRealOptions(reader, fltOpt) 390 # to stream: 391 readercommands = reader.convert() 392 393 regex_values = {} 394 for opt in readercommands: 395 if opt[0] == 'multi-set': 396 optval = opt[3] 397 elif opt[0] == 'set': 398 optval = opt[3:] 399 else: # pragma: no cover 400 continue 401 try: 402 if opt[2] == "prefregex": 403 for optval in optval: 404 self._filter.prefRegex = optval 405 elif opt[2] == "addfailregex": 406 stor = regex_values.get('fail') 407 if not stor: stor = regex_values['fail'] = list() 408 for optval in optval: 409 stor.append(RegexStat(optval)) 410 #self._filter.addFailRegex(optval) 411 elif opt[2] == "addignoreregex": 412 stor = regex_values.get('ignore') 413 if not stor: stor = regex_values['ignore'] = list() 414 for optval in optval: 415 stor.append(RegexStat(optval)) 416 #self._filter.addIgnoreRegex(optval) 417 elif opt[2] == "maxlines": 418 for optval in optval: 419 self.setMaxLines(optval) 420 elif opt[2] == "datepattern": 421 for optval in optval: 422 self.setDatePattern(optval) 423 elif opt[2] == "addjournalmatch": # pragma: no cover 424 if self._opts.journalmatch is None: 425 self.setJournalMatch(optval) 426 except ValueError as e: # pragma: no cover 427 output( "ERROR: Invalid value for %s (%r) " \ 428 "read from %s: %s" % (opt[2], optval, value, e) ) 429 return False 430 431 else: 432 self.output( "Use %11s line : %s" % (regex, shortstr(value)) ) 433 regex_values = {regextype: [RegexStat(value)]} 434 435 for regextype, regex_values in regex_values.items(): 436 regex = regextype + 'regex' 437 setattr(self, "_" + regex, regex_values) 438 for regex in regex_values: 439 getattr( 440 self._filter, 441 'add%sRegex' % regextype.title())(regex.getFailRegex()) 442 return True 443 444 def _onIgnoreRegex(self, idx, ignoreRegex): 445 self._lineIgnored = True 446 self._ignoreregex[idx].inc() 447 448 def testRegex(self, line, date=None): 449 orgLineBuffer = self._filter._Filter__lineBuffer 450 # duplicate line buffer (list can be changed inplace during processLine): 451 if self._filter.getMaxLines() > 1: 452 orgLineBuffer = orgLineBuffer[:] 453 fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines() 454 is_ignored = self._lineIgnored = False 455 try: 456 found = self._filter.processLine(line, date) 457 lines = [] 458 ret = [] 459 for match in found: 460 if not self._opts.out: 461 # Append True/False flag depending if line was matched by 462 # more than one regex 463 match.append(len(ret)>1) 464 regex = self._failregex[match[0]] 465 regex.inc() 466 regex.appendIP(match) 467 if not match[3].get('nofail'): 468 ret.append(match) 469 else: 470 is_ignored = True 471 if self._opts.out: # (formated) output - don't need stats: 472 return None, ret, None 473 # prefregex stats: 474 if self._filter.prefRegex: 475 pre = self._filter.prefRegex 476 if pre.hasMatched(): 477 self._prefREMatched += 1 478 if self._verbose: 479 if len(self._prefREGroups) < self._maxlines: 480 self._prefREGroups.append(pre.getGroups()) 481 else: 482 if len(self._prefREGroups) == self._maxlines: 483 self._prefREGroups.append('...') 484 except RegexException as e: # pragma: no cover 485 output( 'ERROR: %s' % e ) 486 return None, 0, None 487 if self._filter.getMaxLines() > 1: 488 for bufLine in orgLineBuffer[int(fullBuffer):]: 489 if bufLine not in self._filter._Filter__lineBuffer: 490 try: 491 self._line_stats.missed_lines.pop( 492 self._line_stats.missed_lines.index("".join(bufLine))) 493 if self._debuggex: 494 self._line_stats.missed_lines_timeextracted.pop( 495 self._line_stats.missed_lines_timeextracted.index( 496 "".join(bufLine[::2]))) 497 except ValueError: 498 pass 499 # if buffering - add also another lines from match: 500 if self._print_all_matched: 501 if not self._debuggex: 502 self._line_stats.matched_lines.append("".join(bufLine)) 503 else: 504 lines.append(bufLine[0] + bufLine[2]) 505 self._line_stats.matched += 1 506 self._line_stats.missed -= 1 507 if lines: # pre-lines parsed in multiline mode (buffering) 508 lines.append(self._filter.processedLine()) 509 line = "\n".join(lines) 510 return line, ret, (is_ignored or self._lineIgnored) 511 512 def _prepaireOutput(self): 513 """Prepares output- and fetch-function corresponding given '--out' option (format)""" 514 ofmt = self._opts.out 515 if ofmt in ('id', 'ip'): 516 def _out(ret): 517 for r in ret: 518 output(r[1]) 519 elif ofmt == 'msg': 520 def _out(ret): 521 for r in ret: 522 for r in r[3].get('matches'): 523 if not isinstance(r, str): 524 r = ''.join(r for r in r) 525 output(r) 526 elif ofmt == 'row': 527 def _out(ret): 528 for r in ret: 529 output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].items() if k != 'matches'))) 530 elif '<' not in ofmt: 531 def _out(ret): 532 for r in ret: 533 output(r[3].get(ofmt)) 534 else: # extended format with tags substitution: 535 from ..server.actions import Actions, CommandAction, BanTicket 536 def _escOut(t, v): 537 # use safe escape (avoid inject on pseudo tag "\x00msg\x00"): 538 if t not in ('msg',): 539 return v.replace('\x00', '\\x00') 540 return v 541 def _out(ret): 542 rows = [] 543 wrap = {'NL':0} 544 for r in ret: 545 ticket = BanTicket(r[1], time=r[2], data=r[3]) 546 aInfo = Actions.ActionInfo(ticket) 547 # if msg tag is used - output if single line (otherwise let it as is to wrap multilines later): 548 def _get_msg(self): 549 if not wrap['NL'] and len(r[3].get('matches', [])) <= 1: 550 return self['matches'] 551 else: # pseudo tag for future replacement: 552 wrap['NL'] = 1 553 return "\x00msg\x00" 554 aInfo['msg'] = _get_msg 555 # not recursive interpolation (use safe escape): 556 v = CommandAction.replaceDynamicTags(ofmt, aInfo, escapeVal=_escOut) 557 if wrap['NL']: # contains multiline tags (msg): 558 rows.append((r, v)) 559 continue 560 output(v) 561 # wrap multiline tag (msg) interpolations to single line: 562 for r, v in rows: 563 for r in r[3].get('matches'): 564 if not isinstance(r, str): 565 r = ''.join(r for r in r) 566 r = v.replace("\x00msg\x00", r) 567 output(r) 568 return _out 569 570 571 def process(self, test_lines): 572 t0 = time.time() 573 if self._opts.out: # get out function 574 out = self._prepaireOutput() 575 for line in test_lines: 576 if isinstance(line, tuple): 577 line_datetimestripped, ret, is_ignored = self.testRegex(line[0], line[1]) 578 line = "".join(line[0]) 579 else: 580 line = line.rstrip('\r\n') 581 if line.startswith('#') or not line: 582 # skip comment and empty lines 583 continue 584 line_datetimestripped, ret, is_ignored = self.testRegex(line) 585 586 if self._opts.out: # (formated) output: 587 if len(ret) > 0 and not is_ignored: out(ret) 588 continue 589 590 if is_ignored: 591 self._line_stats.ignored += 1 592 if not self._print_no_ignored and (self._print_all_ignored or self._line_stats.ignored <= self._maxlines + 1): 593 self._line_stats.ignored_lines.append(line) 594 if self._debuggex: 595 self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped) 596 elif len(ret) > 0: 597 self._line_stats.matched += 1 598 if self._print_all_matched: 599 self._line_stats.matched_lines.append(line) 600 if self._debuggex: 601 self._line_stats.matched_lines_timeextracted.append(line_datetimestripped) 602 else: 603 self._line_stats.missed += 1 604 if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1): 605 self._line_stats.missed_lines.append(line) 606 if self._debuggex: 607 self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) 608 self._line_stats.tested += 1 609 610 self._time_elapsed = time.time() - t0 611 612 def printLines(self, ltype): 613 lstats = self._line_stats 614 assert(lstats.missed == lstats.tested - (lstats.matched + lstats.ignored)) 615 lines = lstats[ltype] 616 l = lstats[ltype + '_lines'] 617 multiline = self._filter.getMaxLines() > 1 618 if lines: 619 header = "%s line(s):" % (ltype.capitalize(),) 620 if self._debuggex: 621 if ltype == 'missed' or ltype == 'matched': 622 regexlist = self._failregex 623 else: 624 regexlist = self._ignoreregex 625 l = lstats[ltype + '_lines_timeextracted'] 626 if lines < self._maxlines or getattr(self, '_print_all_' + ltype): 627 ans = [[]] 628 for arg in [l, regexlist]: 629 ans = [ x + [y] for x in ans for y in arg ] 630 b = [a[0] + ' | ' + a[1].getFailRegex() + ' | ' + 631 debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), 632 multiline, self._opts.usedns) for a in ans] 633 pprint_list([x.rstrip() for x in b], header) 634 else: 635 output( "%s too many to print. Use --print-all-%s " \ 636 "to print all %d lines" % (header, ltype, lines) ) 637 elif lines < self._maxlines or getattr(self, '_print_all_' + ltype): 638 pprint_list([x.rstrip() for x in l], header) 639 else: 640 output( "%s too many to print. Use --print-all-%s " \ 641 "to print all %d lines" % (header, ltype, lines) ) 642 643 def printStats(self): 644 if self._opts.out: return True 645 output( "" ) 646 output( "Results" ) 647 output( "=======" ) 648 649 def print_failregexes(title, failregexes): 650 # Print title 651 total, out = 0, [] 652 for cnt, failregex in enumerate(failregexes): 653 match = failregex.getStats() 654 total += match 655 if (match or self._verbose): 656 out.append("%2d) [%d] %s" % (cnt+1, match, failregex.getFailRegex())) 657 658 if self._verbose and len(failregex.getIPList()): 659 for ip in failregex.getIPList(): 660 timeTuple = time.localtime(ip[2]) 661 timeString = time.strftime("%a %b %d %H:%M:%S %Y", timeTuple) 662 out.append( 663 " %s %s%s" % ( 664 ip[1], 665 timeString, 666 ip[-1] and " (multiple regex matched)" or "")) 667 668 output( "\n%s: %d total" % (title, total) ) 669 pprint_list(out, " #) [# of hits] regular expression") 670 return total 671 672 # Print prefregex: 673 if self._filter.prefRegex: 674 #self._filter.prefRegex.hasMatched() 675 pre = self._filter.prefRegex 676 out = [pre.getRegex()] 677 if self._verbose: 678 for grp in self._prefREGroups: 679 out.append(" %s" % (grp,)) 680 output( "\n%s: %d total" % ("Prefregex", self._prefREMatched) ) 681 pprint_list(out) 682 683 # Print regex's: 684 total = print_failregexes("Failregex", self._failregex) 685 _ = print_failregexes("Ignoreregex", self._ignoreregex) 686 687 688 if self._filter.dateDetector is not None: 689 output( "\nDate template hits:" ) 690 out = [] 691 for template in self._filter.dateDetector.templates: 692 if self._verbose or template.hits: 693 out.append("[%d] %s" % (template.hits, template.name)) 694 if self._verbose_date: 695 out.append(" # weight: %.3f (%.3f), pattern: %s" % ( 696 template.weight, template.template.weight, 697 getattr(template, 'pattern', ''),)) 698 out.append(" # regex: %s" % (getattr(template, 'regex', ''),)) 699 pprint_list(out, "[# of hits] date format") 700 701 output( "\nLines: %s" % self._line_stats, ) 702 if self._time_elapsed is not None: 703 output( "[processed in %.2f sec]" % self._time_elapsed, ) 704 output( "" ) 705 706 if self._print_all_matched: 707 self.printLines('matched') 708 if not self._print_no_ignored: 709 self.printLines('ignored') 710 if not self._print_no_missed: 711 self.printLines('missed') 712 713 return True 714 715 def file_lines_gen(self, hdlr): 716 for line in hdlr: 717 yield self.decode_line(line) 718 719 def start(self, args): 720 721 cmd_log, cmd_regex = args[:2] 722 723 if cmd_log.startswith("systemd-journal"): # pragma: no cover 724 self._backend = 'systemd' 725 726 try: 727 if not self.readRegex(cmd_regex, 'fail'): # pragma: no cover 728 return False 729 if len(args) == 3 and not self.readRegex(args[2], 'ignore'): # pragma: no cover 730 return False 731 except RegexException as e: 732 output( 'ERROR: %s' % e ) 733 return False 734 735 if os.path.isfile(cmd_log): 736 try: 737 hdlr = open(cmd_log, 'rb') 738 self.output( "Use log file : %s" % cmd_log ) 739 self.output( "Use encoding : %s" % self._encoding ) 740 test_lines = self.file_lines_gen(hdlr) 741 except IOError as e: # pragma: no cover 742 output( e ) 743 return False 744 elif cmd_log.startswith("systemd-journal"): # pragma: no cover 745 if not FilterSystemd: 746 output( "Error: systemd library not found. Exiting..." ) 747 return False 748 self.output( "Use systemd journal" ) 749 self.output( "Use encoding : %s" % self._encoding ) 750 backend, beArgs = extractOptions(cmd_log) 751 flt = FilterSystemd(None, **beArgs) 752 flt.setLogEncoding(self._encoding) 753 myjournal = flt.getJournalReader() 754 journalmatch = self._journalmatch 755 self.setDatePattern(None) 756 if journalmatch: 757 flt.addJournalMatch(journalmatch) 758 self.output( "Use journal match : %s" % " ".join(journalmatch) ) 759 test_lines = journal_lines_gen(flt, myjournal) 760 else: 761 # if single line parsing (without buffering) 762 if self._filter.getMaxLines() <= 1 and '\n' not in cmd_log: 763 self.output( "Use single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) ) 764 test_lines = [ cmd_log ] 765 else: # multi line parsing (with and without buffering) 766 test_lines = cmd_log.split("\n") 767 self.output( "Use multi line : %s line(s)" % len(test_lines) ) 768 for i, l in enumerate(test_lines): 769 if i >= 5: 770 self.output( "| ..." ); break 771 self.output( "| %2.2s: %s" % (i+1, shortstr(l)) ) 772 self.output( "`-" ) 773 774 self.output( "" ) 775 776 self.process(test_lines) 777 778 if not self.printStats(): 779 return False 780 781 return True 782 783 784def exec_command_line(*args): 785 logging.exitOnIOError = True 786 parser = get_opt_parser() 787 (opts, args) = parser.parse_args(*args) 788 errors = [] 789 if opts.print_no_missed and opts.print_all_missed: # pragma: no cover 790 errors.append("ERROR: --print-no-missed and --print-all-missed are mutually exclusive.") 791 if opts.print_no_ignored and opts.print_all_ignored: # pragma: no cover 792 errors.append("ERROR: --print-no-ignored and --print-all-ignored are mutually exclusive.") 793 794 # We need 2 or 3 parameters 795 if not len(args) in (2, 3): 796 errors.append("ERROR: provide both <LOG> and <REGEX>.") 797 if errors: 798 parser.print_help() 799 sys.stderr.write("\n" + "\n".join(errors) + "\n") 800 sys.exit(255) 801 802 if not opts.out: 803 output( "" ) 804 output( "Running tests" ) 805 output( "=============" ) 806 output( "" ) 807 808 # Log level (default critical): 809 opts.log_level = str2LogLevel(opts.log_level) 810 logSys.setLevel(opts.log_level) 811 812 # Add the default logging handler 813 stdout = logging.StreamHandler(sys.stdout) 814 815 fmt = '%(levelname)-1.1s: %(message)s' if opts.verbose <= 1 else ' %(message)s' 816 817 if opts.log_traceback: 818 Formatter = FormatterWithTraceBack 819 fmt = (opts.full_traceback and ' %(tb)s' or ' %(tbc)s') + fmt 820 else: 821 Formatter = logging.Formatter 822 823 # Custom log format for the verbose tests runs 824 stdout.setFormatter(Formatter(getVerbosityFormat(opts.verbose, fmt))) 825 logSys.addHandler(stdout) 826 827 try: 828 fail2banRegex = Fail2banRegex(opts) 829 except Exception as e: 830 if opts.verbose or logSys.getEffectiveLevel()<=logging.DEBUG: 831 logSys.critical(e, exc_info=True) 832 else: 833 output( 'ERROR: %s' % e ) 834 sys.exit(255) 835 836 if not fail2banRegex.start(args): 837 sys.exit(255) 838