1# Copyright (C) 1998-2018 by the Free Software Foundation, Inc. 2# 3# This program is free software; you can redistribute it and/or 4# modify it under the terms of the GNU General Public License 5# as published by the Free Software Foundation; either version 2 6# of the License, or (at your option) any later version. 7# 8# This program is distributed in the hope that it will be useful, 9# but WITHOUT ANY WARRANTY; without even the implied warranty of 10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11# GNU General Public License for more details. 12# 13# You should have received a copy of the GNU General Public License 14# along with this program; if not, write to the Free Software 15# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 16 17"""Mixin class for MailList which handles administrative requests. 18 19Two types of admin requests are currently supported: adding members to a 20closed or semi-closed list, and moderated posts. 21 22Pending subscriptions which are requiring a user's confirmation are handled 23elsewhere. 24""" 25 26import os 27import time 28import errno 29import cPickle 30import marshal 31from cStringIO import StringIO 32 33import email 34from email.MIMEMessage import MIMEMessage 35from email.Generator import Generator 36from email.Utils import getaddresses 37 38from Mailman import mm_cfg 39from Mailman import Utils 40from Mailman import Message 41from Mailman import Errors 42from Mailman.UserDesc import UserDesc 43from Mailman.Queue.sbcache import get_switchboard 44from Mailman.Logging.Syslog import syslog 45from Mailman import i18n 46 47_ = i18n._ 48def D_(s): 49 return s 50 51# Request types requiring admin approval 52IGN = 0 53HELDMSG = 1 54SUBSCRIPTION = 2 55UNSUBSCRIPTION = 3 56 57# Return status from __handlepost() 58DEFER = 0 59REMOVE = 1 60LOST = 2 61 62DASH = '-' 63NL = '\n' 64 65try: 66 True, False 67except NameError: 68 True = 1 69 False = 0 70 71 72 73class ListAdmin: 74 def InitVars(self): 75 # non-configurable data 76 self.next_request_id = 1 77 78 def InitTempVars(self): 79 self.__db = None 80 self.__filename = os.path.join(self.fullpath(), 'request.pck') 81 82 def __opendb(self): 83 if self.__db is None: 84 assert self.Locked() 85 try: 86 fp = open(self.__filename) 87 try: 88 self.__db = cPickle.load(fp) 89 finally: 90 fp.close() 91 except IOError, e: 92 if e.errno <> errno.ENOENT: raise 93 self.__db = {} 94 # put version number in new database 95 self.__db['version'] = IGN, mm_cfg.REQUESTS_FILE_SCHEMA_VERSION 96 97 def __closedb(self): 98 if self.__db is not None: 99 assert self.Locked() 100 # Save the version number 101 self.__db['version'] = IGN, mm_cfg.REQUESTS_FILE_SCHEMA_VERSION 102 # Now save a temp file and do the tmpfile->real file dance. BAW: 103 # should we be as paranoid as for the config.pck file? Should we 104 # use pickle? 105 tmpfile = self.__filename + '.tmp' 106 omask = os.umask(007) 107 try: 108 fp = open(tmpfile, 'w') 109 try: 110 cPickle.dump(self.__db, fp, 1) 111 fp.flush() 112 os.fsync(fp.fileno()) 113 finally: 114 fp.close() 115 finally: 116 os.umask(omask) 117 self.__db = None 118 # Do the dance 119 os.rename(tmpfile, self.__filename) 120 121 def __nextid(self): 122 assert self.Locked() 123 while True: 124 next = self.next_request_id 125 self.next_request_id += 1 126 if not self.__db.has_key(next): 127 break 128 return next 129 130 def SaveRequestsDb(self): 131 self.__closedb() 132 133 def NumRequestsPending(self): 134 self.__opendb() 135 # Subtract one for the version pseudo-entry 136 return len(self.__db) - 1 137 138 def __getmsgids(self, rtype): 139 self.__opendb() 140 ids = [k for k, (op, data) in self.__db.items() if op == rtype] 141 ids.sort() 142 return ids 143 144 def GetHeldMessageIds(self): 145 return self.__getmsgids(HELDMSG) 146 147 def GetSubscriptionIds(self): 148 return self.__getmsgids(SUBSCRIPTION) 149 150 def GetUnsubscriptionIds(self): 151 return self.__getmsgids(UNSUBSCRIPTION) 152 153 def GetRecord(self, id): 154 self.__opendb() 155 type, data = self.__db[id] 156 return data 157 158 def GetRecordType(self, id): 159 self.__opendb() 160 type, data = self.__db[id] 161 return type 162 163 def HandleRequest(self, id, value, comment=None, preserve=None, 164 forward=None, addr=None): 165 self.__opendb() 166 rtype, data = self.__db[id] 167 if rtype == HELDMSG: 168 status = self.__handlepost(data, value, comment, preserve, 169 forward, addr) 170 elif rtype == UNSUBSCRIPTION: 171 status = self.__handleunsubscription(data, value, comment) 172 else: 173 assert rtype == SUBSCRIPTION 174 status = self.__handlesubscription(data, value, comment) 175 if status <> DEFER: 176 # BAW: Held message ids are linked to Pending cookies, allowing 177 # the user to cancel their post before the moderator has approved 178 # it. We should probably remove the cookie associated with this 179 # id, but we have no way currently of correlating them. :( 180 del self.__db[id] 181 182 def HoldMessage(self, msg, reason, msgdata={}): 183 # Make a copy of msgdata so that subsequent changes won't corrupt the 184 # request database. TBD: remove the `filebase' key since this will 185 # not be relevant when the message is resurrected. 186 msgdata = msgdata.copy() 187 # assure that the database is open for writing 188 self.__opendb() 189 # get the next unique id 190 id = self.__nextid() 191 # get the message sender 192 sender = msg.get_sender() 193 # calculate the file name for the message text and write it to disk 194 if mm_cfg.HOLD_MESSAGES_AS_PICKLES: 195 ext = 'pck' 196 else: 197 ext = 'txt' 198 filename = 'heldmsg-%s-%d.%s' % (self.internal_name(), id, ext) 199 omask = os.umask(007) 200 try: 201 fp = open(os.path.join(mm_cfg.DATA_DIR, filename), 'w') 202 try: 203 if mm_cfg.HOLD_MESSAGES_AS_PICKLES: 204 cPickle.dump(msg, fp, 1) 205 else: 206 g = Generator(fp) 207 g.flatten(msg, 1) 208 fp.flush() 209 os.fsync(fp.fileno()) 210 finally: 211 fp.close() 212 finally: 213 os.umask(omask) 214 # save the information to the request database. for held message 215 # entries, each record in the database will be of the following 216 # format: 217 # 218 # the time the message was received 219 # the sender of the message 220 # the message's subject 221 # a string description of the problem 222 # name of the file in $PREFIX/data containing the msg text 223 # an additional dictionary of message metadata 224 # 225 msgsubject = msg.get('subject', _('(no subject)')) 226 if not sender: 227 sender = _('<missing>') 228 data = time.time(), sender, msgsubject, reason, filename, msgdata 229 self.__db[id] = (HELDMSG, data) 230 return id 231 232 def __handlepost(self, record, value, comment, preserve, forward, addr): 233 # For backwards compatibility with pre 2.0beta3 234 ptime, sender, subject, reason, filename, msgdata = record 235 path = os.path.join(mm_cfg.DATA_DIR, filename) 236 # Handle message preservation 237 if preserve: 238 parts = os.path.split(path)[1].split(DASH) 239 parts[0] = 'spam' 240 spamfile = DASH.join(parts) 241 # Preserve the message as plain text, not as a pickle 242 try: 243 fp = open(path) 244 except IOError, e: 245 if e.errno <> errno.ENOENT: raise 246 return LOST 247 try: 248 if path.endswith('.pck'): 249 msg = cPickle.load(fp) 250 else: 251 assert path.endswith('.txt'), '%s not .pck or .txt' % path 252 msg = fp.read() 253 finally: 254 fp.close() 255 # Save the plain text to a .msg file, not a .pck file 256 outpath = os.path.join(mm_cfg.SPAM_DIR, spamfile) 257 head, ext = os.path.splitext(outpath) 258 outpath = head + '.msg' 259 outfp = open(outpath, 'w') 260 try: 261 if path.endswith('.pck'): 262 g = Generator(outfp) 263 g.flatten(msg, 1) 264 else: 265 outfp.write(msg) 266 finally: 267 outfp.close() 268 # Now handle updates to the database 269 rejection = None 270 fp = None 271 msg = None 272 status = REMOVE 273 if value == mm_cfg.DEFER: 274 # Defer 275 status = DEFER 276 elif value == mm_cfg.APPROVE: 277 # Approved. 278 try: 279 msg = readMessage(path) 280 except IOError, e: 281 if e.errno <> errno.ENOENT: raise 282 return LOST 283 msg = readMessage(path) 284 msgdata['approved'] = 1 285 # adminapproved is used by the Emergency handler 286 msgdata['adminapproved'] = 1 287 # Calculate a new filebase for the approved message, otherwise 288 # delivery errors will cause duplicates. 289 try: 290 del msgdata['filebase'] 291 except KeyError: 292 pass 293 # Queue the file for delivery by qrunner. Trying to deliver the 294 # message directly here can lead to a huge delay in web 295 # turnaround. Log the moderation and add a header. 296 msg['X-Mailman-Approved-At'] = email.Utils.formatdate(localtime=1) 297 syslog('vette', '%s: held message approved, message-id: %s', 298 self.internal_name(), 299 msg.get('message-id', 'n/a')) 300 # Stick the message back in the incoming queue for further 301 # processing. 302 inq = get_switchboard(mm_cfg.INQUEUE_DIR) 303 inq.enqueue(msg, _metadata=msgdata) 304 elif value == mm_cfg.REJECT: 305 # Rejected 306 rejection = 'Refused' 307 lang = self.getMemberLanguage(sender) 308 subject = Utils.oneline(subject, Utils.GetCharSet(lang)) 309 self.__refuse(_('Posting of your message titled "%(subject)s"'), 310 sender, comment or _('[No reason given]'), 311 lang=lang) 312 else: 313 assert value == mm_cfg.DISCARD 314 # Discarded 315 rejection = 'Discarded' 316 # Forward the message 317 if forward and addr: 318 # If we've approved the message, we need to be sure to craft a 319 # completely unique second message for the forwarding operation, 320 # since we don't want to share any state or information with the 321 # normal delivery. 322 try: 323 copy = readMessage(path) 324 except IOError, e: 325 if e.errno <> errno.ENOENT: raise 326 raise Errors.LostHeldMessage(path) 327 # It's possible the addr is a comma separated list of addresses. 328 addrs = getaddresses([addr]) 329 if len(addrs) == 1: 330 realname, addr = addrs[0] 331 # If the address getting the forwarded message is a member of 332 # the list, we want the headers of the outer message to be 333 # encoded in their language. Otherwise it'll be the preferred 334 # language of the mailing list. 335 lang = self.getMemberLanguage(addr) 336 else: 337 # Throw away the realnames 338 addr = [a for realname, a in addrs] 339 # Which member language do we attempt to use? We could use 340 # the first match or the first address, but in the face of 341 # ambiguity, let's just use the list's preferred language 342 lang = self.preferred_language 343 otrans = i18n.get_translation() 344 i18n.set_language(lang) 345 try: 346 fmsg = Message.UserNotification( 347 addr, self.GetBouncesEmail(), 348 _('Forward of moderated message'), 349 lang=lang) 350 finally: 351 i18n.set_translation(otrans) 352 fmsg.set_type('message/rfc822') 353 fmsg.attach(copy) 354 fmsg.send(self) 355 # Log the rejection 356 if rejection: 357 note = '''%(listname)s: %(rejection)s posting: 358\tFrom: %(sender)s 359\tSubject: %(subject)s''' % { 360 'listname' : self.internal_name(), 361 'rejection': rejection, 362 'sender' : str(sender).replace('%', '%%'), 363 'subject' : str(subject).replace('%', '%%'), 364 } 365 if comment: 366 note += '\n\tReason: ' + comment.replace('%', '%%') 367 syslog('vette', note) 368 # Always unlink the file containing the message text. It's not 369 # necessary anymore, regardless of the disposition of the message. 370 if status <> DEFER: 371 try: 372 os.unlink(path) 373 except OSError, e: 374 if e.errno <> errno.ENOENT: raise 375 # We lost the message text file. Clean up our housekeeping 376 # and inform of this status. 377 return LOST 378 return status 379 380 def HoldSubscription(self, addr, fullname, password, digest, lang): 381 # Assure that the database is open for writing 382 self.__opendb() 383 # Get the next unique id 384 id = self.__nextid() 385 # Save the information to the request database. for held subscription 386 # entries, each record in the database will be one of the following 387 # format: 388 # 389 # the time the subscription request was received 390 # the subscriber's address 391 # the subscriber's selected password (TBD: is this safe???) 392 # the digest flag 393 # the user's preferred language 394 data = time.time(), addr, fullname, password, digest, lang 395 self.__db[id] = (SUBSCRIPTION, data) 396 # 397 # TBD: this really shouldn't go here but I'm not sure where else is 398 # appropriate. 399 syslog('vette', '%s: held subscription request from %s', 400 self.internal_name(), addr) 401 # Possibly notify the administrator in default list language 402 if self.admin_immed_notify: 403 i18n.set_language(self.preferred_language) 404 realname = self.real_name 405 subject = _( 406 'New subscription request to list %(realname)s from %(addr)s') 407 text = Utils.maketext( 408 'subauth.txt', 409 {'username' : addr, 410 'listname' : self.internal_name(), 411 'hostname' : self.host_name, 412 'admindb_url': self.GetScriptURL('admindb', absolute=1), 413 }, mlist=self) 414 # This message should appear to come from the <list>-owner so as 415 # to avoid any useless bounce processing. 416 owneraddr = self.GetOwnerEmail() 417 msg = Message.UserNotification(owneraddr, owneraddr, subject, text, 418 self.preferred_language) 419 msg.send(self, **{'tomoderators': 1}) 420 # Restore the user's preferred language. 421 i18n.set_language(lang) 422 423 def __handlesubscription(self, record, value, comment): 424 global _ 425 stime, addr, fullname, password, digest, lang = record 426 if value == mm_cfg.DEFER: 427 return DEFER 428 elif value == mm_cfg.DISCARD: 429 syslog('vette', '%s: discarded subscription request from %s', 430 self.internal_name(), addr) 431 elif value == mm_cfg.REJECT: 432 self.__refuse(_('Subscription request'), addr, 433 comment or _('[No reason given]'), 434 lang=lang) 435 syslog('vette', """%s: rejected subscription request from %s 436\tReason: %s""", self.internal_name(), addr, comment or '[No reason given]') 437 else: 438 # subscribe 439 assert value == mm_cfg.SUBSCRIBE 440 try: 441 _ = D_ 442 whence = _('via admin approval') 443 _ = i18n._ 444 userdesc = UserDesc(addr, fullname, password, digest, lang) 445 self.ApprovedAddMember(userdesc, whence=whence) 446 except Errors.MMAlreadyAMember: 447 # User has already been subscribed, after sending the request 448 pass 449 # TBD: disgusting hack: ApprovedAddMember() can end up closing 450 # the request database. 451 self.__opendb() 452 return REMOVE 453 454 def HoldUnsubscription(self, addr): 455 # Assure the database is open for writing 456 self.__opendb() 457 # Get the next unique id 458 id = self.__nextid() 459 # All we need to do is save the unsubscribing address 460 self.__db[id] = (UNSUBSCRIPTION, addr) 461 syslog('vette', '%s: held unsubscription request from %s', 462 self.internal_name(), addr) 463 # Possibly notify the administrator of the hold 464 if self.admin_immed_notify: 465 realname = self.real_name 466 subject = _( 467 'New unsubscription request from %(realname)s by %(addr)s') 468 text = Utils.maketext( 469 'unsubauth.txt', 470 {'username' : addr, 471 'listname' : self.internal_name(), 472 'hostname' : self.host_name, 473 'admindb_url': self.GetScriptURL('admindb', absolute=1), 474 }, mlist=self) 475 # This message should appear to come from the <list>-owner so as 476 # to avoid any useless bounce processing. 477 owneraddr = self.GetOwnerEmail() 478 msg = Message.UserNotification(owneraddr, owneraddr, subject, text, 479 self.preferred_language) 480 msg.send(self, **{'tomoderators': 1}) 481 482 def __handleunsubscription(self, record, value, comment): 483 addr = record 484 if value == mm_cfg.DEFER: 485 return DEFER 486 elif value == mm_cfg.DISCARD: 487 syslog('vette', '%s: discarded unsubscription request from %s', 488 self.internal_name(), addr) 489 elif value == mm_cfg.REJECT: 490 self.__refuse(_('Unsubscription request'), addr, comment) 491 syslog('vette', """%s: rejected unsubscription request from %s 492\tReason: %s""", self.internal_name(), addr, comment or '[No reason given]') 493 else: 494 assert value == mm_cfg.UNSUBSCRIBE 495 try: 496 self.ApprovedDeleteMember(addr) 497 except Errors.NotAMemberError: 498 # User has already been unsubscribed 499 pass 500 return REMOVE 501 502 def __refuse(self, request, recip, comment, origmsg=None, lang=None): 503 # As this message is going to the requestor, try to set the language 504 # to his/her language choice, if they are a member. Otherwise use the 505 # list's preferred language. 506 realname = self.real_name 507 if lang is None: 508 lang = self.getMemberLanguage(recip) 509 text = Utils.maketext( 510 'refuse.txt', 511 {'listname' : realname, 512 'request' : request, 513 'reason' : comment, 514 'adminaddr': self.GetOwnerEmail(), 515 }, lang=lang, mlist=self) 516 otrans = i18n.get_translation() 517 i18n.set_language(lang) 518 try: 519 # add in original message, but not wrap/filled 520 if origmsg: 521 text = NL.join( 522 [text, 523 '---------- ' + _('Original Message') + ' ----------', 524 str(origmsg) 525 ]) 526 subject = _('Request to mailing list %(realname)s rejected') 527 finally: 528 i18n.set_translation(otrans) 529 msg = Message.UserNotification(recip, self.GetOwnerEmail(), 530 subject, text, lang) 531 msg.send(self) 532 533 def _UpdateRecords(self): 534 # Subscription records have changed since MM2.0.x. In that family, 535 # the records were of length 4, containing the request time, the 536 # address, the password, and the digest flag. In MM2.1a2, they grew 537 # an additional language parameter at the end. In MM2.1a4, they grew 538 # a fullname slot after the address. This semi-public method is used 539 # by the update script to coerce all subscription records to the 540 # latest MM2.1 format. 541 # 542 # Held message records have historically either 5 or 6 items too. 543 # These always include the requests time, the sender, subject, default 544 # rejection reason, and message text. When of length 6, it also 545 # includes the message metadata dictionary on the end of the tuple. 546 # 547 # In Mailman 2.1.5 we converted these files to pickles. 548 filename = os.path.join(self.fullpath(), 'request.db') 549 try: 550 fp = open(filename) 551 try: 552 self.__db = marshal.load(fp) 553 finally: 554 fp.close() 555 os.unlink(filename) 556 except IOError, e: 557 if e.errno <> errno.ENOENT: raise 558 filename = os.path.join(self.fullpath(), 'request.pck') 559 try: 560 fp = open(filename) 561 try: 562 self.__db = cPickle.load(fp) 563 finally: 564 fp.close() 565 except IOError, e: 566 if e.errno <> errno.ENOENT: raise 567 self.__db = {} 568 for id, x in self.__db.items(): 569 # A bug in versions 2.1.1 through 2.1.11 could have resulted in 570 # just info being stored instead of (op, info) 571 if len(x) == 2: 572 op, info = x 573 elif len(x) == 6: 574 # This is the buggy info. Check for digest flag. 575 if x[4] in (0, 1): 576 op = SUBSCRIPTION 577 else: 578 op = HELDMSG 579 self.__db[id] = op, x 580 continue 581 else: 582 assert False, 'Unknown record format in %s' % self.__filename 583 if op == SUBSCRIPTION: 584 if len(info) == 4: 585 # pre-2.1a2 compatibility 586 when, addr, passwd, digest = info 587 fullname = '' 588 lang = self.preferred_language 589 elif len(info) == 5: 590 # pre-2.1a4 compatibility 591 when, addr, passwd, digest, lang = info 592 fullname = '' 593 else: 594 assert len(info) == 6, 'Unknown subscription record layout' 595 continue 596 # Here's the new layout 597 self.__db[id] = op, (when, addr, fullname, passwd, 598 digest, lang) 599 elif op == HELDMSG: 600 if len(info) == 5: 601 when, sender, subject, reason, text = info 602 msgdata = {} 603 else: 604 assert len(info) == 6, 'Unknown held msg record layout' 605 continue 606 # Here's the new layout 607 self.__db[id] = op, (when, sender, subject, reason, 608 text, msgdata) 609 # All done 610 self.__closedb() 611 612 613 614def readMessage(path): 615 # For backwards compatibility, we must be able to read either a flat text 616 # file or a pickle. 617 ext = os.path.splitext(path)[1] 618 fp = open(path) 619 try: 620 if ext == '.txt': 621 msg = email.message_from_file(fp, Message.Message) 622 else: 623 assert ext == '.pck' 624 msg = cPickle.load(fp) 625 finally: 626 fp.close() 627 return msg 628