1# -*- coding: UTF-8 -*- 2# Copyright 2009-2018 Oli Schacher 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16""" 17Antiphish / Forging Plugins (DKIM / SPF / SRS etc) 18 19EXPERIMENTAL plugins 20 21TODO: SRS 22 23requires: dkimpy (not pydkim!!) 24requires: pyspf 25requires: pydns (or alternatively dnspython if only dkim is used) 26""" 27 28from fuglu.shared import ScannerPlugin, apply_template, DUNNO, FileList, string_to_actioncode, get_default_cache 29from fuglu.extensions.sql import get_session, SQL_EXTENSION_ENABLED 30from fuglu.extensions.dnsquery import HAVE_PYDNS, HAVE_DNSPYTHON 31import logging 32import os 33import re 34 35DKIMPY_AVAILABLE = False 36PYSPF_AVAILABLE = False 37IPADDRESS_AVAILABLE = False 38IPADDR_AVAILABLE = False 39 40try: 41 import ipaddress 42 IPADDRESS_AVAILABLE = True 43except ImportError: 44 pass 45 46try: 47 import ipaddr 48 IPADDR_AVAILABLE = True 49except ImportError: 50 pass 51 52try: 53 import pkg_resources 54 pkg_resources.get_distribution("dkimpy") 55 from dkim import DKIM, sign, Simple, Relaxed, DKIMException 56 57 if not (HAVE_PYDNS or HAVE_DNSPYTHON): 58 raise Exception("no supported dns library available") 59 60 DKIMPY_AVAILABLE = True 61except Exception: 62 pass 63 64 65try: 66 if not HAVE_PYDNS: 67 raise Exception("pydns not available") 68 if not (IPADDR_AVAILABLE or IPADDRESS_AVAILABLE): 69 raise Exception("ipaddress/ipaddr not available") 70 import spf 71 PYSPF_AVAILABLE = True 72except Exception as e: 73 print(e) 74 pass 75 76 77def extract_from_domain(suspect, get_address_part=True): 78 msgrep = suspect.get_message_rep() 79 from_headers = msgrep.get_all("From", []) 80 if len(from_headers) != 1: 81 return None 82 83 from_header = from_headers[0] 84 parts = from_header.rsplit(None, 1) 85 check_part = parts[-1] 86 if len(parts) == 2 and not get_address_part: 87 check_part = parts[0] 88 elif not get_address_part: 89 return None # no display part found 90 91 domain_match = re.search("(?<=@)[\w.-]+", check_part) 92 if domain_match is None: 93 return None 94 domain = domain_match.group() 95 return domain 96 97 98class DKIMVerifyPlugin(ScannerPlugin): 99 100 """**EXPERIMENTAL** 101This plugin checks the DKIM signature of the message and sets tags... 102DKIMVerify.sigvalid : True if there was a valid DKIM signature, False if there was an invalid DKIM signature 103the tag is not set if there was no dkim header at all 104 105DKIMVerify.skipreason: set if the verification has been skipped 106 107The plugin does not take any action based on the DKIM test result since a failed DKIM validation by itself 108should not cause a message to be treated any differently. Other plugins might use the DKIM result 109in combination with other factors to take action (for example a "DMARC" plugin could use this information) 110 111It is currently recommended to leave both header and body canonicalization as 'relaxed'. Using 'simple' can cause the signature to fail. 112 """ 113 114 def __init__(self, config, section=None): 115 ScannerPlugin.__init__(self, config, section) 116 self.requiredvars = { 117 118 } 119 self.logger = self._logger() 120 121 def __str__(self): 122 return "DKIM Verify" 123 124 def examine(self, suspect): 125 if not DKIMPY_AVAILABLE: 126 suspect.debug("dkimpy not available, can not check") 127 suspect.set_tag( 128 'DKIMVerify.skipreason', 'dkimpy library not available') 129 return DUNNO 130 131 source = suspect.get_original_source() 132 if "dkim-signature: " not in suspect.get_headers().lower(): 133 suspect.set_tag('DKIMVerify.skipreason', 'not dkim signed') 134 suspect.debug("No dkim signature header found") 135 return DUNNO 136 d = DKIM(source, logger=suspect.get_tag('debugfile')) 137 138 try: 139 valid = d.verify() 140 except DKIMException as de: 141 self.logger.warning("%s: DKIM validation failed: %s" % 142 (suspect.id, str(de))) 143 valid = False 144 145 suspect.set_tag("DKIMVerify.sigvalid", valid) 146 return DUNNO 147 148 def lint(self): 149 if not DKIMPY_AVAILABLE: 150 print("Missing dependency: dkimpy https://launchpad.net/dkimpy") 151 print("(also requires either dnspython or pydns)") 152 return False 153 154 return self.check_config() 155 156# test: 157# plugdummy.py -p ... domainauth.DKIMSignPlugin -s <sender> -o canonicalizeheaders:relaxed -o canonicalizebody:simple -o signbodylength:False 158# cat /tmp/fuglu_dummy_message_out.eml | swaks -f <sender> -s <server> 159# -au <username> -ap <password> -4 -p 587 -tls -d - -t 160# <someuser>@gmail.com 161 162 163class DKIMSignPlugin(ScannerPlugin): 164 165 """**EXPERIMENTAL** 166Add DKIM Signature to outgoing mails 167 168Setting up your keys: 169 170:: 171 172 mkdir -p /etc/fuglu/dkim 173 domain=example.com 174 openssl genrsa -out /etc/fuglu/dkim/${domain}.key 1024 175 openssl rsa -in /etc/fuglu/dkim/${domain}.key -out /etc/fuglu/dkim/${domain}.pub -pubout -outform PEM 176 # print out the DNS record: 177 echo -n "default._domainkey TXT \\"v=DKIM1; k=rsa; p=" ; cat /etc/fuglu/dkim/${domain}.pub | grep -v 'PUBLIC KEY' | tr -d '\\n' ; echo ";\\"" 178 179 180If fuglu handles both incoming and outgoing mails you should make sure that this plugin is skipped for incoming mails 181 182 183known issues: 184 185 - setting canonicalizeheaders = simple will cause invalid signature. 186 - signbodylength causes a crash in dkimlib "TypeError: sequence item 1: expected string, int found" 187 188 """ 189 190 def __init__(self, config, section=None): 191 ScannerPlugin.__init__(self, config, section) 192 self.requiredvars = { 193 'privatekeyfile': { 194 'description': "Location of the private key file. supports standard template variables plus additional ${header_from_domain} which extracts the domain name from the From: -Header", 195 'default': "/etc/fuglu/dkim/${header_from_domain}.key", 196 }, 197 198 'canonicalizeheaders': { 199 'description': "Type of header canonicalization (simple or relaxed)", 200 'default': "relaxed", 201 }, 202 203 'canonicalizebody': { 204 'description': "Type of body canonicalization (simple or relaxed)", 205 'default': "relaxed", 206 }, 207 208 'selector': { 209 'description': 'selector to use when signing, supports templates', 210 'default': 'default', 211 }, 212 213 'signheaders': { 214 'description': 'comma separated list of headers to sign. empty string=sign all headers', 215 'default': 'From,Reply-To,Subject,Date,To,CC,Resent-Date,Resent-From,Resent-To,Resent-CC,In-Reply-To,References,List-Id,List-Help,List-Unsubscribe,List-Subscribe,List-Post,List-Owner,List-Archive', 216 }, 217 218 'signbodylength': { 219 'description': 'include l= tag in dkim header', 220 'default': 'False', 221 }, 222 } 223 224 def __str__(self): 225 return "DKIM Sign" 226 227 def examine(self, suspect): 228 if not DKIMPY_AVAILABLE: 229 suspect.debug("dkimpy not available, can not check") 230 self._logger().error( 231 "DKIM signing skipped - missing dkimpy library") 232 return DUNNO 233 234 message = suspect.get_source() 235 domain = extract_from_domain(suspect) 236 addvalues = dict(header_from_domain=domain) 237 selector = apply_template( 238 self.config.get(self.section, 'selector'), suspect, addvalues) 239 240 if domain is None: 241 self._logger().error( 242 "%s: Failed to extract From-header domain for DKIM signing" % suspect.id) 243 return DUNNO 244 245 privkeyfile = apply_template( 246 self.config.get(self.section, 'privatekeyfile'), suspect, addvalues) 247 if not os.path.isfile(privkeyfile): 248 self._logger().error("%s: DKIM signing failed for domain %s, private key not found: %s" % 249 (suspect.id, domain, privkeyfile)) 250 return DUNNO 251 privkeycontent = open(privkeyfile, 'r').read() 252 253 canH = Simple 254 canB = Simple 255 256 if self.config.get(self.section, 'canonicalizeheaders').lower() == 'relaxed': 257 canH = Relaxed 258 if self.config.get(self.section, 'canonicalizebody').lower() == 'relaxed': 259 canB = Relaxed 260 canon = (canH, canB) 261 headerconfig = self.config.get(self.section, 'signheaders') 262 if headerconfig is None or headerconfig.strip() == '': 263 inc_headers = None 264 else: 265 inc_headers = headerconfig.strip().split(',') 266 267 blength = self.config.getboolean(self.section, 'signbodylength') 268 269 dkimhdr = sign(message, selector, domain, privkeycontent, canonicalize=canon, 270 include_headers=inc_headers, length=blength, logger=suspect.get_tag('debugfile')) 271 if dkimhdr.startswith('DKIM-Signature: '): 272 dkimhdr = dkimhdr[16:] 273 274 suspect.addheader('DKIM-Signature', dkimhdr, immediate=True) 275 276 def lint(self): 277 if not DKIMPY_AVAILABLE: 278 print("Missing dependency: dkimpy https://launchpad.net/dkimpy") 279 print("(also requires either dnspython or pydns)") 280 return False 281 282 # if privkey is a filename (no placeholders) check if it exists 283 privkeytemplate = self.config.get(self.section, 'privatekeyfile') 284 if '{' not in privkeytemplate and not os.path.exists(privkeytemplate): 285 print("Private key file %s not found" % privkeytemplate) 286 return False 287 288 return self.check_config() 289 290 291class SPFPlugin(ScannerPlugin): 292 293 """**EXPERIMENTAL** 294This plugin checks the SPF status and sets tag 'SPF.status' to one of the official states 'pass', 'fail', 'neutral', 295'softfail, 'permerror', 'temperror' or 'skipped' if the SPF check could not be peformed. 296Tag 'SPF.explanation' contains a human readable explanation of the result 297 298The plugin does not take any action based on the SPF test result since. Other plugins might use the SPF result 299in combination with other factors to take action (for example a "DMARC" plugin could use this information) 300 """ 301 302 def __init__(self, config, section=None): 303 ScannerPlugin.__init__(self, config, section) 304 self.requiredvars = { 305 306 } 307 self.logger = self._logger() 308 309 def __str__(self): 310 return "SPF Check" 311 312 def examine(self, suspect): 313 if not PYSPF_AVAILABLE: 314 suspect.debug("pyspf not available, can not check") 315 self._logger().warning( 316 "%s: SPF Check skipped, pyspf unavailable" % (suspect.id)) 317 suspect.set_tag('SPF.status', 'skipped') 318 suspect.set_tag("SPF.explanation", 'missing dependency') 319 return DUNNO 320 321 clientinfo = suspect.get_client_info(self.config) 322 if clientinfo is None: 323 suspect.debug("client info not available for SPF check") 324 self._logger().warning( 325 "%s: SPF Check skipped, could not get client info" % (suspect.id)) 326 suspect.set_tag('SPF.status', 'skipped') 327 suspect.set_tag( 328 "SPF.explanation", 'could not extract client information') 329 return DUNNO 330 331 helo, ip, revdns = clientinfo 332 result, explanation = spf.check2(ip, suspect.from_address, helo) 333 suspect.set_tag("SPF.status", result) 334 suspect.set_tag("SPF.explanation", explanation) 335 suspect.debug("SPF status: %s (%s)" % (result, explanation)) 336 return DUNNO 337 338 def lint(self): 339 if not PYSPF_AVAILABLE: 340 print("Missing dependency: pyspf") 341 print("(also requires pydns and ipaddress or ipaddr)") 342 return False 343 344 return self.check_config() 345 346 347class DomainAuthPlugin(ScannerPlugin): 348 349 """**EXPERIMENTAL** 350This plugin checks the header from domain against a list of domains which must be authenticated by DKIM and/or SPF. 351This is somewhat similar to DMARC but instead of asking the sender domain for a DMARC policy record this plugin allows you to force authentication on the recipient side. 352 353This plugin depends on tags written by SPFPlugin and DKIMVerifyPlugin, so they must run beforehand. 354 """ 355 356 def __init__(self, config, section=None): 357 ScannerPlugin.__init__(self, config, section) 358 self.requiredvars = { 359 'domainsfile': { 360 'description': "File containing a list of domains (one per line) which must be DKIM and/or SPF authenticated", 361 'default': "/etc/fuglu/auth_required_domains.txt", 362 }, 363 'failaction': { 364 'default': 'DUNNO', 365 'description': "action if the message doesn't pass authentication (DUNNO, REJECT)", 366 }, 367 368 'rejectmessage': { 369 'default': 'sender domain ${header_from_domain} must pass DKIM and/or SPF authentication', 370 'description': "reject message template if running in pre-queue mode", 371 }, 372 } 373 self.logger = self._logger() 374 self.filelist = FileList( 375 filename=None, strip=True, skip_empty=True, skip_comments=True, lowercase=True) 376 377 def examine(self, suspect): 378 self.filelist.filename = self.config.get(self.section, 'domainsfile') 379 checkdomains = self.filelist.get_list() 380 381 envelope_sender_domain = suspect.from_domain.lower() 382 header_from_domain = extract_from_domain(suspect) 383 if header_from_domain is None: 384 return 385 386 if header_from_domain not in checkdomains: 387 return 388 389 # TODO: do we need a tag from dkim to check if the verified dkim domain 390 # actually matches the header from domain? 391 dkimresult = suspect.get_tag('DKIMVerify.sigvalid', False) 392 if dkimresult == True: 393 return DUNNO 394 395 # DKIM failed, check SPF if envelope senderdomain belongs to header 396 # from domain 397 spfresult = suspect.get_tag('SPF.status', 'unknown') 398 if (envelope_sender_domain == header_from_domain or envelope_sender_domain.endswith('.%s' % header_from_domain)) and spfresult == 'pass': 399 return DUNNO 400 401 failaction = self.config.get(self.section, 'failaction') 402 actioncode = string_to_actioncode(failaction, self.config) 403 404 values = dict( 405 header_from_domain=header_from_domain) 406 message = apply_template( 407 self.config.get(self.section, 'rejectmessage'), suspect, values) 408 return actioncode, message 409 410 def flag_as_spam(self, suspect): 411 suspect.tags['spam']['domainauth'] = True 412 413 def __str__(self): 414 return "DomainAuth" 415 416 def lint(self): 417 allok = self.check_config() and self.lint_file() 418 return allok 419 420 def lint_file(self): 421 filename = self.config.get(self.section, 'domainsfile') 422 if not os.path.exists(filename): 423 print("domains file %s not found" % (filename)) 424 return False 425 return True 426 427 428class SpearPhishPlugin(ScannerPlugin): 429 """Mark spear phishing mails as virus 430 431 The spearphish plugin checks if the sender domain in the "From"-Header matches the envelope recipient Domain ("Mail 432 from my own domain") but the message uses a different envelope sender domain. This blocks many spearphish attempts. 433 434 Note that this plugin can cause blocks of legitimate mail , for example if the recipient domain is using a third party service 435 to send newsletters in their name. Such services often set the customers domain in the from headers but use their own domains in the envelope for 436 bounce processing. Use the 'Plugin Skipper' or any other form of whitelisting in such cases. 437 """ 438 439 def __init__(self, section=None): 440 ScannerPlugin.__init__(self, section) 441 self.logger = self._logger() 442 self.filelist = FileList(strip=True, skip_empty=True, skip_comments=True, lowercase=True, 443 additional_filters=None, minimum_time_between_reloads=30) 444 445 self.requiredvars = { 446 'domainsfile': { 447 'default': '/usr/local/etc/fuglu/spearphish-domains', 448 'description': 'Filename where we load spearphish domains from. One domain per line. If this setting is empty, the check will be applied to all domains.', 449 }, 450 'virusenginename': { 451 'default': 'Fuglu SpearPhishing Protection', 452 'description': 'Name of this plugins av engine', 453 }, 454 'virusname': { 455 'default': 'TRAIT.SPEARPHISH', 456 'description': 'Name to use as virus signature', 457 }, 458 'virusaction': { 459 'default': 'DEFAULTVIRUSACTION', 460 'description': "action if spear phishing attempt is detected (DUNNO, REJECT, DELETE)", 461 }, 462 'rejectmessage': { 463 'default': 'threat detected: ${virusname}', 464 'description': "reject message template if running in pre-queue mode and virusaction=REJECT", 465 }, 466 'dbconnection':{ 467 'default':"mysql://root@localhost/spfcheck?charset=utf8", 468 'description':'SQLAlchemy Connection string. Leave empty to disable SQL lookups', 469 }, 470 'domain_sql_query':{ 471 'default':"SELECT check_spearphish from domain where domain_name=:domain", 472 'description':'get from sql database :domain will be replaced with the actual domain name. must return boolean field check_spearphish', 473 }, 474 'check_display_part': { 475 'default': 'False', 476 'description': "set to True to also check display part of From header (else email part only)", 477 }, 478 } 479 480 481 def get_domain_setting(self, domain, dbconnection, sqlquery, cache, cachename, default_value=None, logger=None): 482 if logger is None: 483 logger = logging.getLogger() 484 485 cachekey = '%s-%s' % (cachename, domain) 486 cached = cache.get_cache(cachekey) 487 if cached is not None: 488 logger.debug("got cached setting for %s" % domain) 489 return cached 490 491 settings = default_value 492 493 try: 494 session = get_session(dbconnection) 495 496 # get domain settings 497 dom = session.execute(sqlquery, {'domain': domain}).fetchall() 498 499 if not dom and not dom[0] and len(dom[0]) == 0: 500 logger.warning( 501 "Can not load domain setting - domain %s not found. Using default settings." % domain) 502 else: 503 settings = dom[0][0] 504 505 session.close() 506 507 except Exception as e: 508 logger.error("Exception while loading setting for %s : %s" % (domain, str(e))) 509 510 cache.put_cache(cachekey, settings) 511 logger.debug("refreshed setting for %s" % domain) 512 return settings 513 514 515 def should_we_check_this_domain(self,suspect): 516 domainsfile = self.config.get(self.section, 'domainsfile') 517 if domainsfile.strip()=='': # empty config -> check all domains 518 return True 519 520 if not os.path.exists(domainsfile): 521 return False 522 523 self.filelist.filename = domainsfile 524 envelope_recipient_domain = suspect.to_domain.lower() 525 checkdomains = self.filelist.get_list() 526 if envelope_recipient_domain in checkdomains: 527 return True 528 529 dbconnection = self.config.get(self.section, 'dbconnection').strip() 530 sqlquery = self.config.get(self.section,'domain_sql_query') 531 do_check = False 532 if dbconnection != '': 533 cache = get_default_cache() 534 cachename = self.section 535 do_check = self.get_domain_setting(suspect.to_domain, dbconnection, sqlquery, cache, cachename, False, self.logger) 536 return do_check 537 538 539 def examine(self, suspect): 540 if not self.should_we_check_this_domain(suspect): 541 return DUNNO 542 envelope_recipient_domain = suspect.to_domain.lower() 543 envelope_sender_domain = suspect.from_domain.lower() 544 if envelope_sender_domain == envelope_recipient_domain: 545 return DUNNO # we only check the message if the env_sender_domain differs. If it's the same it will be caught by other means (like SPF) 546 547 header_from_domains = [] 548 header_from_domain = extract_from_domain(suspect) 549 if header_from_domain is None: 550 self.logger.warn("%s: Could not extract header from domain for spearphish check" % suspect.id) 551 return DUNNO 552 else: 553 header_from_domains.append(header_from_domain) 554 self.logger.debug('%s: checking domain %s (source: From header address part)' % (suspect.id, header_from_domain)) 555 556 if self.config.getboolean(self.section, 'check_display_part'): 557 display_from_domain = extract_from_domain(suspect, False) 558 if display_from_domain is not None and display_from_domain not in header_from_domains: 559 header_from_domains.append(display_from_domain) 560 self.logger.debug('%s: checking domain %s (source: From header display part)' % (suspect.id, display_from_domain)) 561 562 actioncode = DUNNO 563 message = None 564 565 for header_from_domain in header_from_domains: 566 if header_from_domain == envelope_recipient_domain: 567 virusname = self.config.get(self.section, 'virusname') 568 virusaction = self.config.get(self.section, 'virusaction') 569 actioncode = string_to_actioncode(virusaction, self.config) 570 571 logmsg = '%s: spear phish pattern detected, env_rcpt_domain=%s env_sender_domain=%s header_from_domain=%s' % \ 572 (suspect.id, envelope_recipient_domain, envelope_sender_domain, header_from_domain) 573 self.logger.info(logmsg) 574 self.flag_as_phish(suspect, virusname) 575 576 message = apply_template(self.config.get(self.section, 'rejectmessage'), suspect, {'virusname': virusname}) 577 break 578 579 return actioncode, message 580 581 582 def flag_as_phish(self, suspect, virusname): 583 suspect.tags['%s.virus' % self.config.get(self.section, 'virusenginename')] = {'message content': virusname} 584 suspect.tags['virus'][self.config.get(self.section, 'virusenginename')] = True 585 586 587 def __str__(self): 588 return "Spearphish Check" 589 590 591 def lint(self): 592 allok = self.check_config() and self._lint_file() and self._lint_sql() 593 return allok 594 595 596 def _lint_file(self): 597 filename = self.config.get(self.section, 'domainsfile') 598 if not os.path.exists(filename): 599 print("Spearphish domains file %s not found" % filename) 600 return False 601 return True 602 603 604 def _lint_sql(self): 605 lint_ok = True 606 sqlquery = self.config.get(self.section, 'domain_sql_query') 607 dbconnection = self.config.get(self.section, 'dbconnection').strip() 608 if not SQL_EXTENSION_ENABLED and dbconnection != '': 609 print('SQLAlchemy not available, cannot use SQL backend') 610 lint_ok = False 611 elif dbconnection == '': 612 print('No DB connection defined. Disabling SQL backend') 613 else: 614 if not sqlquery.lower().startswith('select '): 615 lint_ok = False 616 print('SQL statement must be a SELECT query') 617 if lint_ok: 618 try: 619 conn = get_session(dbconnection) 620 conn.execute(sqlquery, {'domain': 'example.com'}) 621 except Exception as e: 622 lint_ok = False 623 print(str(e)) 624 return lint_ok 625 626 627