1# -*- coding: UTF-8 -*-
2#   Copyright 2009-2018 Oli Schacher
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16"""
17Antiphish / Forging Plugins (DKIM / SPF / SRS etc)
18
19EXPERIMENTAL plugins
20
21TODO: SRS
22
23requires: dkimpy (not pydkim!!)
24requires: pyspf
25requires: pydns (or alternatively dnspython if only dkim is used)
26"""
27
28from fuglu.shared import ScannerPlugin, apply_template, DUNNO, FileList, string_to_actioncode, get_default_cache
29from fuglu.extensions.sql import get_session, SQL_EXTENSION_ENABLED
30from fuglu.extensions.dnsquery import HAVE_PYDNS, HAVE_DNSPYTHON
31import logging
32import os
33import re
34
35DKIMPY_AVAILABLE = False
36PYSPF_AVAILABLE = False
37IPADDRESS_AVAILABLE = False
38IPADDR_AVAILABLE = False
39
40try:
41    import ipaddress
42    IPADDRESS_AVAILABLE = True
43except ImportError:
44    pass
45
46try:
47    import ipaddr
48    IPADDR_AVAILABLE = True
49except ImportError:
50    pass
51
52try:
53    import pkg_resources
54    pkg_resources.get_distribution("dkimpy")
55    from dkim import DKIM, sign, Simple, Relaxed, DKIMException
56
57    if not (HAVE_PYDNS or HAVE_DNSPYTHON):
58        raise Exception("no supported dns library available")
59
60    DKIMPY_AVAILABLE = True
61except Exception:
62    pass
63
64
65try:
66    if not HAVE_PYDNS:
67        raise Exception("pydns not available")
68    if not (IPADDR_AVAILABLE or IPADDRESS_AVAILABLE):
69        raise Exception("ipaddress/ipaddr not available")
70    import spf
71    PYSPF_AVAILABLE = True
72except Exception as e:
73    print(e)
74    pass
75
76
77def extract_from_domain(suspect, get_address_part=True):
78    msgrep = suspect.get_message_rep()
79    from_headers = msgrep.get_all("From", [])
80    if len(from_headers) != 1:
81        return None
82
83    from_header = from_headers[0]
84    parts = from_header.rsplit(None, 1)
85    check_part = parts[-1]
86    if len(parts) == 2 and not get_address_part:
87        check_part = parts[0]
88    elif not get_address_part:
89        return None # no display part found
90
91    domain_match = re.search("(?<=@)[\w.-]+", check_part)
92    if domain_match is None:
93        return None
94    domain = domain_match.group()
95    return domain
96
97
98class DKIMVerifyPlugin(ScannerPlugin):
99
100    """**EXPERIMENTAL**
101This plugin checks the DKIM signature of the message and sets tags...
102DKIMVerify.sigvalid : True if there was a valid DKIM signature, False if there was an invalid DKIM signature
103the tag is not set if there was no dkim header at all
104
105DKIMVerify.skipreason: set if the verification has been skipped
106
107The plugin does not take any action based on the DKIM test result since a failed DKIM validation by itself
108should not cause a message to be treated any differently. Other plugins might use the DKIM result
109in combination with other factors to take action (for example a "DMARC" plugin could use this information)
110
111It is currently recommended to leave both header and body canonicalization as 'relaxed'. Using 'simple' can cause the signature to fail.
112    """
113
114    def __init__(self, config, section=None):
115        ScannerPlugin.__init__(self, config, section)
116        self.requiredvars = {
117
118        }
119        self.logger = self._logger()
120
121    def __str__(self):
122        return "DKIM Verify"
123
124    def examine(self, suspect):
125        if not DKIMPY_AVAILABLE:
126            suspect.debug("dkimpy not available, can not check")
127            suspect.set_tag(
128                'DKIMVerify.skipreason', 'dkimpy library not available')
129            return DUNNO
130
131        source = suspect.get_original_source()
132        if "dkim-signature: " not in suspect.get_headers().lower():
133            suspect.set_tag('DKIMVerify.skipreason', 'not dkim signed')
134            suspect.debug("No dkim signature header found")
135            return DUNNO
136        d = DKIM(source, logger=suspect.get_tag('debugfile'))
137
138        try:
139            valid = d.verify()
140        except DKIMException as de:
141            self.logger.warning("%s: DKIM validation failed: %s" %
142                                (suspect.id, str(de)))
143            valid = False
144
145        suspect.set_tag("DKIMVerify.sigvalid", valid)
146        return DUNNO
147
148    def lint(self):
149        if not DKIMPY_AVAILABLE:
150            print("Missing dependency: dkimpy https://launchpad.net/dkimpy")
151            print("(also requires either dnspython or pydns)")
152            return False
153
154        return self.check_config()
155
156# test:
157# plugdummy.py -p ...  domainauth.DKIMSignPlugin -s <sender> -o canonicalizeheaders:relaxed -o canonicalizebody:simple -o signbodylength:False
158# cat /tmp/fuglu_dummy_message_out.eml | swaks -f <sender>  -s <server>
159# -au <username> -ap <password> -4 -p 587 -tls -d -  -t
160# <someuser>@gmail.com
161
162
163class DKIMSignPlugin(ScannerPlugin):
164
165    """**EXPERIMENTAL**
166Add DKIM Signature to outgoing mails
167
168Setting up your keys:
169
170::
171
172    mkdir -p /etc/fuglu/dkim
173    domain=example.com
174    openssl genrsa -out /etc/fuglu/dkim/${domain}.key 1024
175    openssl rsa -in /etc/fuglu/dkim/${domain}.key -out /etc/fuglu/dkim/${domain}.pub -pubout -outform PEM
176    # print out the DNS record:
177    echo -n "default._domainkey TXT  \\"v=DKIM1; k=rsa; p=" ; cat /etc/fuglu/dkim/${domain}.pub | grep -v 'PUBLIC KEY' | tr -d '\\n' ; echo ";\\""
178
179
180If fuglu handles both incoming and outgoing mails you should make sure that this plugin is skipped for incoming mails
181
182
183known issues:
184
185 - setting canonicalizeheaders = simple will cause invalid signature.
186 - signbodylength causes a crash in dkimlib "TypeError: sequence item 1: expected string, int found"
187
188    """
189
190    def __init__(self, config, section=None):
191        ScannerPlugin.__init__(self, config, section)
192        self.requiredvars = {
193            'privatekeyfile': {
194                'description': "Location of the private key file. supports standard template variables plus additional ${header_from_domain} which extracts the domain name from the From: -Header",
195                'default': "/etc/fuglu/dkim/${header_from_domain}.key",
196            },
197
198            'canonicalizeheaders': {
199                'description': "Type of header canonicalization (simple or relaxed)",
200                'default': "relaxed",
201            },
202
203            'canonicalizebody': {
204                'description': "Type of body canonicalization (simple or relaxed)",
205                'default': "relaxed",
206            },
207
208            'selector': {
209                'description': 'selector to use when signing, supports templates',
210                'default': 'default',
211            },
212
213            'signheaders': {
214                'description': 'comma separated list of headers to sign. empty string=sign all headers',
215                'default': 'From,Reply-To,Subject,Date,To,CC,Resent-Date,Resent-From,Resent-To,Resent-CC,In-Reply-To,References,List-Id,List-Help,List-Unsubscribe,List-Subscribe,List-Post,List-Owner,List-Archive',
216            },
217
218            'signbodylength': {
219                'description': 'include l= tag in dkim header',
220                'default': 'False',
221            },
222        }
223
224    def __str__(self):
225        return "DKIM Sign"
226
227    def examine(self, suspect):
228        if not DKIMPY_AVAILABLE:
229            suspect.debug("dkimpy not available, can not check")
230            self._logger().error(
231                "DKIM signing skipped - missing dkimpy library")
232            return DUNNO
233
234        message = suspect.get_source()
235        domain = extract_from_domain(suspect)
236        addvalues = dict(header_from_domain=domain)
237        selector = apply_template(
238            self.config.get(self.section, 'selector'), suspect, addvalues)
239
240        if domain is None:
241            self._logger().error(
242                "%s: Failed to extract From-header domain for DKIM signing" % suspect.id)
243            return DUNNO
244
245        privkeyfile = apply_template(
246            self.config.get(self.section, 'privatekeyfile'), suspect, addvalues)
247        if not os.path.isfile(privkeyfile):
248            self._logger().error("%s: DKIM signing failed for domain %s, private key not found: %s" %
249                                 (suspect.id, domain, privkeyfile))
250            return DUNNO
251        privkeycontent = open(privkeyfile, 'r').read()
252
253        canH = Simple
254        canB = Simple
255
256        if self.config.get(self.section, 'canonicalizeheaders').lower() == 'relaxed':
257            canH = Relaxed
258        if self.config.get(self.section, 'canonicalizebody').lower() == 'relaxed':
259            canB = Relaxed
260        canon = (canH, canB)
261        headerconfig = self.config.get(self.section, 'signheaders')
262        if headerconfig is None or headerconfig.strip() == '':
263            inc_headers = None
264        else:
265            inc_headers = headerconfig.strip().split(',')
266
267        blength = self.config.getboolean(self.section, 'signbodylength')
268
269        dkimhdr = sign(message, selector, domain, privkeycontent, canonicalize=canon,
270                       include_headers=inc_headers, length=blength, logger=suspect.get_tag('debugfile'))
271        if dkimhdr.startswith('DKIM-Signature: '):
272            dkimhdr = dkimhdr[16:]
273
274        suspect.addheader('DKIM-Signature', dkimhdr, immediate=True)
275
276    def lint(self):
277        if not DKIMPY_AVAILABLE:
278            print("Missing dependency: dkimpy https://launchpad.net/dkimpy")
279            print("(also requires either dnspython or pydns)")
280            return False
281
282        # if privkey is a filename (no placeholders) check if it exists
283        privkeytemplate = self.config.get(self.section, 'privatekeyfile')
284        if '{' not in privkeytemplate and not os.path.exists(privkeytemplate):
285            print("Private key file %s not found" % privkeytemplate)
286            return False
287
288        return self.check_config()
289
290
291class SPFPlugin(ScannerPlugin):
292
293    """**EXPERIMENTAL**
294This plugin checks the SPF status and sets tag 'SPF.status' to one of the official states 'pass', 'fail', 'neutral',
295'softfail, 'permerror', 'temperror' or 'skipped' if the SPF check could not be peformed.
296Tag 'SPF.explanation' contains a human readable explanation of the result
297
298The plugin does not take any action based on the SPF test result since. Other plugins might use the SPF result
299in combination with other factors to take action (for example a "DMARC" plugin could use this information)
300    """
301
302    def __init__(self, config, section=None):
303        ScannerPlugin.__init__(self, config, section)
304        self.requiredvars = {
305
306        }
307        self.logger = self._logger()
308
309    def __str__(self):
310        return "SPF Check"
311
312    def examine(self, suspect):
313        if not PYSPF_AVAILABLE:
314            suspect.debug("pyspf not available, can not check")
315            self._logger().warning(
316                "%s: SPF Check skipped, pyspf unavailable" % (suspect.id))
317            suspect.set_tag('SPF.status', 'skipped')
318            suspect.set_tag("SPF.explanation", 'missing dependency')
319            return DUNNO
320
321        clientinfo = suspect.get_client_info(self.config)
322        if clientinfo is None:
323            suspect.debug("client info not available for SPF check")
324            self._logger().warning(
325                "%s: SPF Check skipped, could not get client info" % (suspect.id))
326            suspect.set_tag('SPF.status', 'skipped')
327            suspect.set_tag(
328                "SPF.explanation", 'could not extract client information')
329            return DUNNO
330
331        helo, ip, revdns = clientinfo
332        result, explanation = spf.check2(ip, suspect.from_address, helo)
333        suspect.set_tag("SPF.status", result)
334        suspect.set_tag("SPF.explanation", explanation)
335        suspect.debug("SPF status: %s (%s)" % (result, explanation))
336        return DUNNO
337
338    def lint(self):
339        if not PYSPF_AVAILABLE:
340            print("Missing dependency: pyspf")
341            print("(also requires pydns and ipaddress or ipaddr)")
342            return False
343
344        return self.check_config()
345
346
347class DomainAuthPlugin(ScannerPlugin):
348
349    """**EXPERIMENTAL**
350This plugin checks the header from domain against a list of domains which must be authenticated by DKIM and/or SPF.
351This is somewhat similar to DMARC but instead of asking the sender domain for a DMARC policy record this plugin allows you to force authentication on the recipient side.
352
353This plugin depends on tags written by SPFPlugin and DKIMVerifyPlugin, so they must run beforehand.
354    """
355
356    def __init__(self, config, section=None):
357        ScannerPlugin.__init__(self, config, section)
358        self.requiredvars = {
359            'domainsfile': {
360                'description': "File containing a list of domains (one per line) which must be DKIM and/or SPF authenticated",
361                'default': "/etc/fuglu/auth_required_domains.txt",
362            },
363            'failaction': {
364                'default': 'DUNNO',
365                'description': "action if the message doesn't pass authentication (DUNNO, REJECT)",
366            },
367
368            'rejectmessage': {
369                'default': 'sender domain ${header_from_domain} must pass DKIM and/or SPF authentication',
370                'description': "reject message template if running in pre-queue mode",
371            },
372        }
373        self.logger = self._logger()
374        self.filelist = FileList(
375            filename=None, strip=True, skip_empty=True, skip_comments=True, lowercase=True)
376
377    def examine(self, suspect):
378        self.filelist.filename = self.config.get(self.section, 'domainsfile')
379        checkdomains = self.filelist.get_list()
380
381        envelope_sender_domain = suspect.from_domain.lower()
382        header_from_domain = extract_from_domain(suspect)
383        if header_from_domain is None:
384            return
385
386        if header_from_domain not in checkdomains:
387            return
388
389        # TODO: do we need a tag from dkim to check if the verified dkim domain
390        # actually matches the header from domain?
391        dkimresult = suspect.get_tag('DKIMVerify.sigvalid', False)
392        if dkimresult == True:
393            return DUNNO
394
395        # DKIM failed, check SPF if envelope senderdomain belongs to header
396        # from domain
397        spfresult = suspect.get_tag('SPF.status', 'unknown')
398        if (envelope_sender_domain == header_from_domain or envelope_sender_domain.endswith('.%s' % header_from_domain)) and spfresult == 'pass':
399            return DUNNO
400
401        failaction = self.config.get(self.section, 'failaction')
402        actioncode = string_to_actioncode(failaction, self.config)
403
404        values = dict(
405            header_from_domain=header_from_domain)
406        message = apply_template(
407            self.config.get(self.section, 'rejectmessage'), suspect, values)
408        return actioncode, message
409
410    def flag_as_spam(self, suspect):
411        suspect.tags['spam']['domainauth'] = True
412
413    def __str__(self):
414        return "DomainAuth"
415
416    def lint(self):
417        allok = self.check_config() and self.lint_file()
418        return allok
419
420    def lint_file(self):
421        filename = self.config.get(self.section, 'domainsfile')
422        if not os.path.exists(filename):
423            print("domains file %s not found" % (filename))
424            return False
425        return True
426
427
428class SpearPhishPlugin(ScannerPlugin):
429    """Mark spear phishing mails as virus
430
431    The spearphish plugin checks if the sender domain in the "From"-Header matches the envelope recipient Domain ("Mail
432    from my own domain") but the message uses a different envelope sender domain. This blocks many spearphish attempts.
433
434    Note that this plugin can cause blocks of legitimate mail , for example if the recipient domain is using a third party service
435    to send newsletters in their name. Such services often set the customers domain in the from headers but use their own domains in the envelope for
436    bounce processing. Use the 'Plugin Skipper' or any other form of whitelisting in such cases.
437    """
438
439    def __init__(self, section=None):
440        ScannerPlugin.__init__(self, section)
441        self.logger = self._logger()
442        self.filelist = FileList(strip=True, skip_empty=True, skip_comments=True, lowercase=True,
443                                 additional_filters=None, minimum_time_between_reloads=30)
444
445        self.requiredvars = {
446            'domainsfile': {
447                'default': '/usr/local/etc/fuglu/spearphish-domains',
448                'description': 'Filename where we load spearphish domains from. One domain per line. If this setting is empty, the check will be applied to all domains.',
449            },
450            'virusenginename': {
451                'default': 'Fuglu SpearPhishing Protection',
452                'description': 'Name of this plugins av engine',
453            },
454            'virusname': {
455                'default': 'TRAIT.SPEARPHISH',
456                'description': 'Name to use as virus signature',
457            },
458            'virusaction': {
459                'default': 'DEFAULTVIRUSACTION',
460                'description': "action if spear phishing attempt is detected (DUNNO, REJECT, DELETE)",
461            },
462            'rejectmessage': {
463                'default': 'threat detected: ${virusname}',
464                'description': "reject message template if running in pre-queue mode and virusaction=REJECT",
465            },
466            'dbconnection':{
467                'default':"mysql://root@localhost/spfcheck?charset=utf8",
468                'description':'SQLAlchemy Connection string. Leave empty to disable SQL lookups',
469            },
470            'domain_sql_query':{
471                'default':"SELECT check_spearphish from domain where domain_name=:domain",
472                'description':'get from sql database :domain will be replaced with the actual domain name. must return boolean field check_spearphish',
473            },
474            'check_display_part': {
475                'default': 'False',
476                'description': "set to True to also check display part of From header (else email part only)",
477            },
478        }
479
480
481    def get_domain_setting(self, domain, dbconnection, sqlquery, cache, cachename, default_value=None, logger=None):
482        if logger is None:
483            logger = logging.getLogger()
484
485        cachekey = '%s-%s' % (cachename, domain)
486        cached = cache.get_cache(cachekey)
487        if cached is not None:
488            logger.debug("got cached setting for %s" % domain)
489            return cached
490
491        settings = default_value
492
493        try:
494            session = get_session(dbconnection)
495
496            # get domain settings
497            dom = session.execute(sqlquery, {'domain': domain}).fetchall()
498
499            if not dom and not dom[0] and len(dom[0]) == 0:
500                logger.warning(
501                    "Can not load domain setting - domain %s not found. Using default settings." % domain)
502            else:
503                settings = dom[0][0]
504
505            session.close()
506
507        except Exception as e:
508            logger.error("Exception while loading setting for %s : %s" % (domain, str(e)))
509
510        cache.put_cache(cachekey, settings)
511        logger.debug("refreshed setting for %s" % domain)
512        return settings
513
514
515    def should_we_check_this_domain(self,suspect):
516        domainsfile = self.config.get(self.section, 'domainsfile')
517        if domainsfile.strip()=='': # empty config -> check all domains
518            return True
519
520        if not os.path.exists(domainsfile):
521            return False
522
523        self.filelist.filename = domainsfile
524        envelope_recipient_domain = suspect.to_domain.lower()
525        checkdomains = self.filelist.get_list()
526        if envelope_recipient_domain in checkdomains:
527            return True
528
529        dbconnection = self.config.get(self.section, 'dbconnection').strip()
530        sqlquery = self.config.get(self.section,'domain_sql_query')
531        do_check = False
532        if dbconnection != '':
533            cache = get_default_cache()
534            cachename = self.section
535            do_check = self.get_domain_setting(suspect.to_domain, dbconnection, sqlquery, cache, cachename, False, self.logger)
536        return do_check
537
538
539    def examine(self, suspect):
540        if not self.should_we_check_this_domain(suspect):
541            return DUNNO
542        envelope_recipient_domain = suspect.to_domain.lower()
543        envelope_sender_domain = suspect.from_domain.lower()
544        if envelope_sender_domain == envelope_recipient_domain:
545            return DUNNO  # we only check the message if the env_sender_domain differs. If it's the same it will be caught by other means (like SPF)
546
547        header_from_domains = []
548        header_from_domain = extract_from_domain(suspect)
549        if header_from_domain is None:
550            self.logger.warn("%s: Could not extract header from domain for spearphish check" % suspect.id)
551            return DUNNO
552        else:
553            header_from_domains.append(header_from_domain)
554            self.logger.debug('%s: checking domain %s (source: From header address part)' % (suspect.id, header_from_domain))
555
556        if self.config.getboolean(self.section, 'check_display_part'):
557            display_from_domain = extract_from_domain(suspect, False)
558            if display_from_domain is not None and display_from_domain not in header_from_domains:
559                header_from_domains.append(display_from_domain)
560                self.logger.debug('%s: checking domain %s (source: From header display part)' % (suspect.id, display_from_domain))
561
562        actioncode = DUNNO
563        message = None
564
565        for header_from_domain in header_from_domains:
566            if header_from_domain == envelope_recipient_domain:
567                virusname = self.config.get(self.section, 'virusname')
568                virusaction = self.config.get(self.section, 'virusaction')
569                actioncode = string_to_actioncode(virusaction, self.config)
570
571                logmsg = '%s: spear phish pattern detected, env_rcpt_domain=%s env_sender_domain=%s header_from_domain=%s' % \
572                         (suspect.id, envelope_recipient_domain, envelope_sender_domain, header_from_domain)
573                self.logger.info(logmsg)
574                self.flag_as_phish(suspect, virusname)
575
576                message = apply_template(self.config.get(self.section, 'rejectmessage'), suspect, {'virusname': virusname})
577                break
578
579        return actioncode, message
580
581
582    def flag_as_phish(self, suspect, virusname):
583        suspect.tags['%s.virus' % self.config.get(self.section, 'virusenginename')] = {'message content': virusname}
584        suspect.tags['virus'][self.config.get(self.section, 'virusenginename')] = True
585
586
587    def __str__(self):
588        return "Spearphish Check"
589
590
591    def lint(self):
592        allok = self.check_config() and self._lint_file() and self._lint_sql()
593        return allok
594
595
596    def _lint_file(self):
597        filename = self.config.get(self.section, 'domainsfile')
598        if not os.path.exists(filename):
599            print("Spearphish domains file %s not found" % filename)
600            return False
601        return True
602
603
604    def _lint_sql(self):
605        lint_ok = True
606        sqlquery = self.config.get(self.section, 'domain_sql_query')
607        dbconnection = self.config.get(self.section, 'dbconnection').strip()
608        if not SQL_EXTENSION_ENABLED and dbconnection != '':
609            print('SQLAlchemy not available, cannot use SQL backend')
610            lint_ok = False
611        elif dbconnection == '':
612            print('No DB connection defined. Disabling SQL backend')
613        else:
614            if not sqlquery.lower().startswith('select '):
615                lint_ok = False
616                print('SQL statement must be a SELECT query')
617            if lint_ok:
618                try:
619                    conn = get_session(dbconnection)
620                    conn.execute(sqlquery, {'domain': 'example.com'})
621                except Exception as e:
622                    lint_ok = False
623                    print(str(e))
624        return lint_ok
625
626
627