1# Copyright (C) 1998-2018 by the Free Software Foundation, Inc.
2#
3# This program is free software; you can redistribute it and/or
4# modify it under the terms of the GNU General Public License
5# as published by the Free Software Foundation; either version 2
6# of the License, or (at your option) any later version.
7#
8# This program is distributed in the hope that it will be useful,
9# but WITHOUT ANY WARRANTY; without even the implied warranty of
10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11# GNU General Public License for more details.
12#
13# You should have received a copy of the GNU General Public License
14# along with this program; if not, write to the Free Software
15# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16
17"""Mixin class for MailList which handles administrative requests.
18
19Two types of admin requests are currently supported: adding members to a
20closed or semi-closed list, and moderated posts.
21
22Pending subscriptions which are requiring a user's confirmation are handled
23elsewhere.
24"""
25
26import os
27import time
28import errno
29import cPickle
30import marshal
31from cStringIO import StringIO
32
33import email
34from email.MIMEMessage import MIMEMessage
35from email.Generator import Generator
36from email.Utils import getaddresses
37
38from Mailman import mm_cfg
39from Mailman import Utils
40from Mailman import Message
41from Mailman import Errors
42from Mailman.UserDesc import UserDesc
43from Mailman.Queue.sbcache import get_switchboard
44from Mailman.Logging.Syslog import syslog
45from Mailman import i18n
46
47_ = i18n._
48def D_(s):
49    return s
50
51# Request types requiring admin approval
52IGN = 0
53HELDMSG = 1
54SUBSCRIPTION = 2
55UNSUBSCRIPTION = 3
56
57# Return status from __handlepost()
58DEFER = 0
59REMOVE = 1
60LOST = 2
61
62DASH = '-'
63NL = '\n'
64
65try:
66    True, False
67except NameError:
68    True = 1
69    False = 0
70
71
72
73class ListAdmin:
74    def InitVars(self):
75        # non-configurable data
76        self.next_request_id = 1
77
78    def InitTempVars(self):
79        self.__db = None
80        self.__filename = os.path.join(self.fullpath(), 'request.pck')
81
82    def __opendb(self):
83        if self.__db is None:
84            assert self.Locked()
85            try:
86                fp = open(self.__filename)
87                try:
88                    self.__db = cPickle.load(fp)
89                finally:
90                    fp.close()
91            except IOError, e:
92                if e.errno <> errno.ENOENT: raise
93                self.__db = {}
94                # put version number in new database
95                self.__db['version'] = IGN, mm_cfg.REQUESTS_FILE_SCHEMA_VERSION
96
97    def __closedb(self):
98        if self.__db is not None:
99            assert self.Locked()
100            # Save the version number
101            self.__db['version'] = IGN, mm_cfg.REQUESTS_FILE_SCHEMA_VERSION
102            # Now save a temp file and do the tmpfile->real file dance.  BAW:
103            # should we be as paranoid as for the config.pck file?  Should we
104            # use pickle?
105            tmpfile = self.__filename + '.tmp'
106            omask = os.umask(007)
107            try:
108                fp = open(tmpfile, 'w')
109                try:
110                    cPickle.dump(self.__db, fp, 1)
111                    fp.flush()
112                    os.fsync(fp.fileno())
113                finally:
114                    fp.close()
115            finally:
116                os.umask(omask)
117            self.__db = None
118            # Do the dance
119            os.rename(tmpfile, self.__filename)
120
121    def __nextid(self):
122        assert self.Locked()
123        while True:
124            next = self.next_request_id
125            self.next_request_id += 1
126            if not self.__db.has_key(next):
127                break
128        return next
129
130    def SaveRequestsDb(self):
131        self.__closedb()
132
133    def NumRequestsPending(self):
134        self.__opendb()
135        # Subtract one for the version pseudo-entry
136        return len(self.__db) - 1
137
138    def __getmsgids(self, rtype):
139        self.__opendb()
140        ids = [k for k, (op, data) in self.__db.items() if op == rtype]
141        ids.sort()
142        return ids
143
144    def GetHeldMessageIds(self):
145        return self.__getmsgids(HELDMSG)
146
147    def GetSubscriptionIds(self):
148        return self.__getmsgids(SUBSCRIPTION)
149
150    def GetUnsubscriptionIds(self):
151        return self.__getmsgids(UNSUBSCRIPTION)
152
153    def GetRecord(self, id):
154        self.__opendb()
155        type, data = self.__db[id]
156        return data
157
158    def GetRecordType(self, id):
159        self.__opendb()
160        type, data = self.__db[id]
161        return type
162
163    def HandleRequest(self, id, value, comment=None, preserve=None,
164                      forward=None, addr=None):
165        self.__opendb()
166        rtype, data = self.__db[id]
167        if rtype == HELDMSG:
168            status = self.__handlepost(data, value, comment, preserve,
169                                       forward, addr)
170        elif rtype == UNSUBSCRIPTION:
171            status = self.__handleunsubscription(data, value, comment)
172        else:
173            assert rtype == SUBSCRIPTION
174            status = self.__handlesubscription(data, value, comment)
175        if status <> DEFER:
176            # BAW: Held message ids are linked to Pending cookies, allowing
177            # the user to cancel their post before the moderator has approved
178            # it.  We should probably remove the cookie associated with this
179            # id, but we have no way currently of correlating them. :(
180            del self.__db[id]
181
182    def HoldMessage(self, msg, reason, msgdata={}):
183        # Make a copy of msgdata so that subsequent changes won't corrupt the
184        # request database.  TBD: remove the `filebase' key since this will
185        # not be relevant when the message is resurrected.
186        msgdata = msgdata.copy()
187        # assure that the database is open for writing
188        self.__opendb()
189        # get the next unique id
190        id = self.__nextid()
191        # get the message sender
192        sender = msg.get_sender()
193        # calculate the file name for the message text and write it to disk
194        if mm_cfg.HOLD_MESSAGES_AS_PICKLES:
195            ext = 'pck'
196        else:
197            ext = 'txt'
198        filename = 'heldmsg-%s-%d.%s' % (self.internal_name(), id, ext)
199        omask = os.umask(007)
200        try:
201            fp = open(os.path.join(mm_cfg.DATA_DIR, filename), 'w')
202            try:
203                if mm_cfg.HOLD_MESSAGES_AS_PICKLES:
204                    cPickle.dump(msg, fp, 1)
205                else:
206                    g = Generator(fp)
207                    g.flatten(msg, 1)
208                fp.flush()
209                os.fsync(fp.fileno())
210            finally:
211                fp.close()
212        finally:
213            os.umask(omask)
214        # save the information to the request database.  for held message
215        # entries, each record in the database will be of the following
216        # format:
217        #
218        # the time the message was received
219        # the sender of the message
220        # the message's subject
221        # a string description of the problem
222        # name of the file in $PREFIX/data containing the msg text
223        # an additional dictionary of message metadata
224        #
225        msgsubject = msg.get('subject', _('(no subject)'))
226        if not sender:
227            sender = _('<missing>')
228        data = time.time(), sender, msgsubject, reason, filename, msgdata
229        self.__db[id] = (HELDMSG, data)
230        return id
231
232    def __handlepost(self, record, value, comment, preserve, forward, addr):
233        # For backwards compatibility with pre 2.0beta3
234        ptime, sender, subject, reason, filename, msgdata = record
235        path = os.path.join(mm_cfg.DATA_DIR, filename)
236        # Handle message preservation
237        if preserve:
238            parts = os.path.split(path)[1].split(DASH)
239            parts[0] = 'spam'
240            spamfile = DASH.join(parts)
241            # Preserve the message as plain text, not as a pickle
242            try:
243                fp = open(path)
244            except IOError, e:
245                if e.errno <> errno.ENOENT: raise
246                return LOST
247            try:
248                if path.endswith('.pck'):
249                    msg = cPickle.load(fp)
250                else:
251                    assert path.endswith('.txt'), '%s not .pck or .txt' % path
252                    msg = fp.read()
253            finally:
254                fp.close()
255            # Save the plain text to a .msg file, not a .pck file
256            outpath = os.path.join(mm_cfg.SPAM_DIR, spamfile)
257            head, ext = os.path.splitext(outpath)
258            outpath = head + '.msg'
259            outfp = open(outpath, 'w')
260            try:
261                if path.endswith('.pck'):
262                    g = Generator(outfp)
263                    g.flatten(msg, 1)
264                else:
265                    outfp.write(msg)
266            finally:
267                outfp.close()
268        # Now handle updates to the database
269        rejection = None
270        fp = None
271        msg = None
272        status = REMOVE
273        if value == mm_cfg.DEFER:
274            # Defer
275            status = DEFER
276        elif value == mm_cfg.APPROVE:
277            # Approved.
278            try:
279                msg = readMessage(path)
280            except IOError, e:
281                if e.errno <> errno.ENOENT: raise
282                return LOST
283            msg = readMessage(path)
284            msgdata['approved'] = 1
285            # adminapproved is used by the Emergency handler
286            msgdata['adminapproved'] = 1
287            # Calculate a new filebase for the approved message, otherwise
288            # delivery errors will cause duplicates.
289            try:
290                del msgdata['filebase']
291            except KeyError:
292                pass
293            # Queue the file for delivery by qrunner.  Trying to deliver the
294            # message directly here can lead to a huge delay in web
295            # turnaround.  Log the moderation and add a header.
296            msg['X-Mailman-Approved-At'] = email.Utils.formatdate(localtime=1)
297            syslog('vette', '%s: held message approved, message-id: %s',
298                   self.internal_name(),
299                   msg.get('message-id', 'n/a'))
300            # Stick the message back in the incoming queue for further
301            # processing.
302            inq = get_switchboard(mm_cfg.INQUEUE_DIR)
303            inq.enqueue(msg, _metadata=msgdata)
304        elif value == mm_cfg.REJECT:
305            # Rejected
306            rejection = 'Refused'
307            lang = self.getMemberLanguage(sender)
308            subject = Utils.oneline(subject, Utils.GetCharSet(lang))
309            self.__refuse(_('Posting of your message titled "%(subject)s"'),
310                          sender, comment or _('[No reason given]'),
311                          lang=lang)
312        else:
313            assert value == mm_cfg.DISCARD
314            # Discarded
315            rejection = 'Discarded'
316        # Forward the message
317        if forward and addr:
318            # If we've approved the message, we need to be sure to craft a
319            # completely unique second message for the forwarding operation,
320            # since we don't want to share any state or information with the
321            # normal delivery.
322            try:
323                copy = readMessage(path)
324            except IOError, e:
325                if e.errno <> errno.ENOENT: raise
326                raise Errors.LostHeldMessage(path)
327            # It's possible the addr is a comma separated list of addresses.
328            addrs = getaddresses([addr])
329            if len(addrs) == 1:
330                realname, addr = addrs[0]
331                # If the address getting the forwarded message is a member of
332                # the list, we want the headers of the outer message to be
333                # encoded in their language.  Otherwise it'll be the preferred
334                # language of the mailing list.
335                lang = self.getMemberLanguage(addr)
336            else:
337                # Throw away the realnames
338                addr = [a for realname, a in addrs]
339                # Which member language do we attempt to use?  We could use
340                # the first match or the first address, but in the face of
341                # ambiguity, let's just use the list's preferred language
342                lang = self.preferred_language
343            otrans = i18n.get_translation()
344            i18n.set_language(lang)
345            try:
346                fmsg = Message.UserNotification(
347                    addr, self.GetBouncesEmail(),
348                    _('Forward of moderated message'),
349                    lang=lang)
350            finally:
351                i18n.set_translation(otrans)
352            fmsg.set_type('message/rfc822')
353            fmsg.attach(copy)
354            fmsg.send(self)
355        # Log the rejection
356        if rejection:
357            note = '''%(listname)s: %(rejection)s posting:
358\tFrom: %(sender)s
359\tSubject: %(subject)s''' % {
360                'listname' : self.internal_name(),
361                'rejection': rejection,
362                'sender'   : str(sender).replace('%', '%%'),
363                'subject'  : str(subject).replace('%', '%%'),
364                }
365            if comment:
366                note += '\n\tReason: ' + comment.replace('%', '%%')
367            syslog('vette', note)
368        # Always unlink the file containing the message text.  It's not
369        # necessary anymore, regardless of the disposition of the message.
370        if status <> DEFER:
371            try:
372                os.unlink(path)
373            except OSError, e:
374                if e.errno <> errno.ENOENT: raise
375                # We lost the message text file.  Clean up our housekeeping
376                # and inform of this status.
377                return LOST
378        return status
379
380    def HoldSubscription(self, addr, fullname, password, digest, lang):
381        # Assure that the database is open for writing
382        self.__opendb()
383        # Get the next unique id
384        id = self.__nextid()
385        # Save the information to the request database. for held subscription
386        # entries, each record in the database will be one of the following
387        # format:
388        #
389        # the time the subscription request was received
390        # the subscriber's address
391        # the subscriber's selected password (TBD: is this safe???)
392        # the digest flag
393        # the user's preferred language
394        data = time.time(), addr, fullname, password, digest, lang
395        self.__db[id] = (SUBSCRIPTION, data)
396        #
397        # TBD: this really shouldn't go here but I'm not sure where else is
398        # appropriate.
399        syslog('vette', '%s: held subscription request from %s',
400               self.internal_name(), addr)
401        # Possibly notify the administrator in default list language
402        if self.admin_immed_notify:
403            i18n.set_language(self.preferred_language)
404            realname = self.real_name
405            subject = _(
406                'New subscription request to list %(realname)s from %(addr)s')
407            text = Utils.maketext(
408                'subauth.txt',
409                {'username'   : addr,
410                 'listname'   : self.internal_name(),
411                 'hostname'   : self.host_name,
412                 'admindb_url': self.GetScriptURL('admindb', absolute=1),
413                 }, mlist=self)
414            # This message should appear to come from the <list>-owner so as
415            # to avoid any useless bounce processing.
416            owneraddr = self.GetOwnerEmail()
417            msg = Message.UserNotification(owneraddr, owneraddr, subject, text,
418                                           self.preferred_language)
419            msg.send(self, **{'tomoderators': 1})
420            # Restore the user's preferred language.
421            i18n.set_language(lang)
422
423    def __handlesubscription(self, record, value, comment):
424        global _
425        stime, addr, fullname, password, digest, lang = record
426        if value == mm_cfg.DEFER:
427            return DEFER
428        elif value == mm_cfg.DISCARD:
429            syslog('vette', '%s: discarded subscription request from %s',
430                   self.internal_name(), addr)
431        elif value == mm_cfg.REJECT:
432            self.__refuse(_('Subscription request'), addr,
433                          comment or _('[No reason given]'),
434                          lang=lang)
435            syslog('vette', """%s: rejected subscription request from %s
436\tReason: %s""", self.internal_name(), addr, comment or '[No reason given]')
437        else:
438            # subscribe
439            assert value == mm_cfg.SUBSCRIBE
440            try:
441                _ = D_
442                whence = _('via admin approval')
443                _ = i18n._
444                userdesc = UserDesc(addr, fullname, password, digest, lang)
445                self.ApprovedAddMember(userdesc, whence=whence)
446            except Errors.MMAlreadyAMember:
447                # User has already been subscribed, after sending the request
448                pass
449            # TBD: disgusting hack: ApprovedAddMember() can end up closing
450            # the request database.
451            self.__opendb()
452        return REMOVE
453
454    def HoldUnsubscription(self, addr):
455        # Assure the database is open for writing
456        self.__opendb()
457        # Get the next unique id
458        id = self.__nextid()
459        # All we need to do is save the unsubscribing address
460        self.__db[id] = (UNSUBSCRIPTION, addr)
461        syslog('vette', '%s: held unsubscription request from %s',
462               self.internal_name(), addr)
463        # Possibly notify the administrator of the hold
464        if self.admin_immed_notify:
465            realname = self.real_name
466            subject = _(
467                'New unsubscription request from %(realname)s by %(addr)s')
468            text = Utils.maketext(
469                'unsubauth.txt',
470                {'username'   : addr,
471                 'listname'   : self.internal_name(),
472                 'hostname'   : self.host_name,
473                 'admindb_url': self.GetScriptURL('admindb', absolute=1),
474                 }, mlist=self)
475            # This message should appear to come from the <list>-owner so as
476            # to avoid any useless bounce processing.
477            owneraddr = self.GetOwnerEmail()
478            msg = Message.UserNotification(owneraddr, owneraddr, subject, text,
479                                           self.preferred_language)
480            msg.send(self, **{'tomoderators': 1})
481
482    def __handleunsubscription(self, record, value, comment):
483        addr = record
484        if value == mm_cfg.DEFER:
485            return DEFER
486        elif value == mm_cfg.DISCARD:
487            syslog('vette', '%s: discarded unsubscription request from %s',
488                   self.internal_name(), addr)
489        elif value == mm_cfg.REJECT:
490            self.__refuse(_('Unsubscription request'), addr, comment)
491            syslog('vette', """%s: rejected unsubscription request from %s
492\tReason: %s""", self.internal_name(), addr, comment or '[No reason given]')
493        else:
494            assert value == mm_cfg.UNSUBSCRIBE
495            try:
496                self.ApprovedDeleteMember(addr)
497            except Errors.NotAMemberError:
498                # User has already been unsubscribed
499                pass
500        return REMOVE
501
502    def __refuse(self, request, recip, comment, origmsg=None, lang=None):
503        # As this message is going to the requestor, try to set the language
504        # to his/her language choice, if they are a member.  Otherwise use the
505        # list's preferred language.
506        realname = self.real_name
507        if lang is None:
508            lang = self.getMemberLanguage(recip)
509        text = Utils.maketext(
510            'refuse.txt',
511            {'listname' : realname,
512             'request'  : request,
513             'reason'   : comment,
514             'adminaddr': self.GetOwnerEmail(),
515            }, lang=lang, mlist=self)
516        otrans = i18n.get_translation()
517        i18n.set_language(lang)
518        try:
519            # add in original message, but not wrap/filled
520            if origmsg:
521                text = NL.join(
522                    [text,
523                     '---------- ' + _('Original Message') + ' ----------',
524                     str(origmsg)
525                     ])
526            subject = _('Request to mailing list %(realname)s rejected')
527        finally:
528            i18n.set_translation(otrans)
529        msg = Message.UserNotification(recip, self.GetOwnerEmail(),
530                                       subject, text, lang)
531        msg.send(self)
532
533    def _UpdateRecords(self):
534        # Subscription records have changed since MM2.0.x.  In that family,
535        # the records were of length 4, containing the request time, the
536        # address, the password, and the digest flag.  In MM2.1a2, they grew
537        # an additional language parameter at the end.  In MM2.1a4, they grew
538        # a fullname slot after the address.  This semi-public method is used
539        # by the update script to coerce all subscription records to the
540        # latest MM2.1 format.
541        #
542        # Held message records have historically either 5 or 6 items too.
543        # These always include the requests time, the sender, subject, default
544        # rejection reason, and message text.  When of length 6, it also
545        # includes the message metadata dictionary on the end of the tuple.
546        #
547        # In Mailman 2.1.5 we converted these files to pickles.
548        filename = os.path.join(self.fullpath(), 'request.db')
549        try:
550            fp = open(filename)
551            try:
552                self.__db = marshal.load(fp)
553            finally:
554                fp.close()
555            os.unlink(filename)
556        except IOError, e:
557            if e.errno <> errno.ENOENT: raise
558            filename = os.path.join(self.fullpath(), 'request.pck')
559            try:
560                fp = open(filename)
561                try:
562                    self.__db = cPickle.load(fp)
563                finally:
564                    fp.close()
565            except IOError, e:
566                if e.errno <> errno.ENOENT: raise
567                self.__db = {}
568        for id, x in self.__db.items():
569            # A bug in versions 2.1.1 through 2.1.11 could have resulted in
570            # just info being stored instead of (op, info)
571            if len(x) == 2:
572                op, info = x
573            elif len(x) == 6:
574                # This is the buggy info. Check for digest flag.
575                if x[4] in (0, 1):
576                    op = SUBSCRIPTION
577                else:
578                    op = HELDMSG
579                self.__db[id] = op, x
580                continue
581            else:
582                assert False, 'Unknown record format in %s' % self.__filename
583            if op == SUBSCRIPTION:
584                if len(info) == 4:
585                    # pre-2.1a2 compatibility
586                    when, addr, passwd, digest = info
587                    fullname = ''
588                    lang = self.preferred_language
589                elif len(info) == 5:
590                    # pre-2.1a4 compatibility
591                    when, addr, passwd, digest, lang = info
592                    fullname = ''
593                else:
594                    assert len(info) == 6, 'Unknown subscription record layout'
595                    continue
596                # Here's the new layout
597                self.__db[id] = op, (when, addr, fullname, passwd,
598                                     digest, lang)
599            elif op == HELDMSG:
600                if len(info) == 5:
601                    when, sender, subject, reason, text = info
602                    msgdata = {}
603                else:
604                    assert len(info) == 6, 'Unknown held msg record layout'
605                    continue
606                # Here's the new layout
607                self.__db[id] = op, (when, sender, subject, reason,
608                                     text, msgdata)
609        # All done
610        self.__closedb()
611
612
613
614def readMessage(path):
615    # For backwards compatibility, we must be able to read either a flat text
616    # file or a pickle.
617    ext = os.path.splitext(path)[1]
618    fp = open(path)
619    try:
620        if ext == '.txt':
621            msg = email.message_from_file(fp, Message.Message)
622        else:
623            assert ext == '.pck'
624            msg = cPickle.load(fp)
625    finally:
626        fp.close()
627    return msg
628