1"""Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes.""" 2 3# Notes for authors of new mailbox subclasses: 4# 5# Remember to fsync() changes to disk before closing a modified file 6# or returning from a flush() method. See functions _sync_flush() and 7# _sync_close(). 8 9import os 10import time 11import calendar 12import socket 13import errno 14import copy 15import warnings 16import email 17import email.message 18import email.generator 19import io 20import contextlib 21try: 22 import fcntl 23except ImportError: 24 fcntl = None 25 26__all__ = ['Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF', 27 'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage', 28 'BabylMessage', 'MMDFMessage', 'Error', 'NoSuchMailboxError', 29 'NotEmptyError', 'ExternalClashError', 'FormatError'] 30 31linesep = os.linesep.encode('ascii') 32 33class Mailbox: 34 """A group of messages in a particular place.""" 35 36 def __init__(self, path, factory=None, create=True): 37 """Initialize a Mailbox instance.""" 38 self._path = os.path.abspath(os.path.expanduser(path)) 39 self._factory = factory 40 41 def add(self, message): 42 """Add message and return assigned key.""" 43 raise NotImplementedError('Method must be implemented by subclass') 44 45 def remove(self, key): 46 """Remove the keyed message; raise KeyError if it doesn't exist.""" 47 raise NotImplementedError('Method must be implemented by subclass') 48 49 def __delitem__(self, key): 50 self.remove(key) 51 52 def discard(self, key): 53 """If the keyed message exists, remove it.""" 54 try: 55 self.remove(key) 56 except KeyError: 57 pass 58 59 def __setitem__(self, key, message): 60 """Replace the keyed message; raise KeyError if it doesn't exist.""" 61 raise NotImplementedError('Method must be implemented by subclass') 62 63 def get(self, key, default=None): 64 """Return the keyed message, or default if it doesn't exist.""" 65 try: 66 return self.__getitem__(key) 67 except KeyError: 68 return default 69 70 def __getitem__(self, key): 71 """Return the keyed message; raise KeyError if it doesn't exist.""" 72 if not self._factory: 73 return self.get_message(key) 74 else: 75 with contextlib.closing(self.get_file(key)) as file: 76 return self._factory(file) 77 78 def get_message(self, key): 79 """Return a Message representation or raise a KeyError.""" 80 raise NotImplementedError('Method must be implemented by subclass') 81 82 def get_string(self, key): 83 """Return a string representation or raise a KeyError. 84 85 Uses email.message.Message to create a 7bit clean string 86 representation of the message.""" 87 return email.message_from_bytes(self.get_bytes(key)).as_string() 88 89 def get_bytes(self, key): 90 """Return a byte string representation or raise a KeyError.""" 91 raise NotImplementedError('Method must be implemented by subclass') 92 93 def get_file(self, key): 94 """Return a file-like representation or raise a KeyError.""" 95 raise NotImplementedError('Method must be implemented by subclass') 96 97 def iterkeys(self): 98 """Return an iterator over keys.""" 99 raise NotImplementedError('Method must be implemented by subclass') 100 101 def keys(self): 102 """Return a list of keys.""" 103 return list(self.iterkeys()) 104 105 def itervalues(self): 106 """Return an iterator over all messages.""" 107 for key in self.iterkeys(): 108 try: 109 value = self[key] 110 except KeyError: 111 continue 112 yield value 113 114 def __iter__(self): 115 return self.itervalues() 116 117 def values(self): 118 """Return a list of messages. Memory intensive.""" 119 return list(self.itervalues()) 120 121 def iteritems(self): 122 """Return an iterator over (key, message) tuples.""" 123 for key in self.iterkeys(): 124 try: 125 value = self[key] 126 except KeyError: 127 continue 128 yield (key, value) 129 130 def items(self): 131 """Return a list of (key, message) tuples. Memory intensive.""" 132 return list(self.iteritems()) 133 134 def __contains__(self, key): 135 """Return True if the keyed message exists, False otherwise.""" 136 raise NotImplementedError('Method must be implemented by subclass') 137 138 def __len__(self): 139 """Return a count of messages in the mailbox.""" 140 raise NotImplementedError('Method must be implemented by subclass') 141 142 def clear(self): 143 """Delete all messages.""" 144 for key in self.keys(): 145 self.discard(key) 146 147 def pop(self, key, default=None): 148 """Delete the keyed message and return it, or default.""" 149 try: 150 result = self[key] 151 except KeyError: 152 return default 153 self.discard(key) 154 return result 155 156 def popitem(self): 157 """Delete an arbitrary (key, message) pair and return it.""" 158 for key in self.iterkeys(): 159 return (key, self.pop(key)) # This is only run once. 160 else: 161 raise KeyError('No messages in mailbox') 162 163 def update(self, arg=None): 164 """Change the messages that correspond to certain keys.""" 165 if hasattr(arg, 'iteritems'): 166 source = arg.iteritems() 167 elif hasattr(arg, 'items'): 168 source = arg.items() 169 else: 170 source = arg 171 bad_key = False 172 for key, message in source: 173 try: 174 self[key] = message 175 except KeyError: 176 bad_key = True 177 if bad_key: 178 raise KeyError('No message with key(s)') 179 180 def flush(self): 181 """Write any pending changes to the disk.""" 182 raise NotImplementedError('Method must be implemented by subclass') 183 184 def lock(self): 185 """Lock the mailbox.""" 186 raise NotImplementedError('Method must be implemented by subclass') 187 188 def unlock(self): 189 """Unlock the mailbox if it is locked.""" 190 raise NotImplementedError('Method must be implemented by subclass') 191 192 def close(self): 193 """Flush and close the mailbox.""" 194 raise NotImplementedError('Method must be implemented by subclass') 195 196 def _string_to_bytes(self, message): 197 # If a message is not 7bit clean, we refuse to handle it since it 198 # likely came from reading invalid messages in text mode, and that way 199 # lies mojibake. 200 try: 201 return message.encode('ascii') 202 except UnicodeError: 203 raise ValueError("String input must be ASCII-only; " 204 "use bytes or a Message instead") 205 206 # Whether each message must end in a newline 207 _append_newline = False 208 209 def _dump_message(self, message, target, mangle_from_=False): 210 # This assumes the target file is open in binary mode. 211 """Dump message contents to target file.""" 212 if isinstance(message, email.message.Message): 213 buffer = io.BytesIO() 214 gen = email.generator.BytesGenerator(buffer, mangle_from_, 0) 215 gen.flatten(message) 216 buffer.seek(0) 217 data = buffer.read() 218 data = data.replace(b'\n', linesep) 219 target.write(data) 220 if self._append_newline and not data.endswith(linesep): 221 # Make sure the message ends with a newline 222 target.write(linesep) 223 elif isinstance(message, (str, bytes, io.StringIO)): 224 if isinstance(message, io.StringIO): 225 warnings.warn("Use of StringIO input is deprecated, " 226 "use BytesIO instead", DeprecationWarning, 3) 227 message = message.getvalue() 228 if isinstance(message, str): 229 message = self._string_to_bytes(message) 230 if mangle_from_: 231 message = message.replace(b'\nFrom ', b'\n>From ') 232 message = message.replace(b'\n', linesep) 233 target.write(message) 234 if self._append_newline and not message.endswith(linesep): 235 # Make sure the message ends with a newline 236 target.write(linesep) 237 elif hasattr(message, 'read'): 238 if hasattr(message, 'buffer'): 239 warnings.warn("Use of text mode files is deprecated, " 240 "use a binary mode file instead", DeprecationWarning, 3) 241 message = message.buffer 242 lastline = None 243 while True: 244 line = message.readline() 245 # Universal newline support. 246 if line.endswith(b'\r\n'): 247 line = line[:-2] + b'\n' 248 elif line.endswith(b'\r'): 249 line = line[:-1] + b'\n' 250 if not line: 251 break 252 if mangle_from_ and line.startswith(b'From '): 253 line = b'>From ' + line[5:] 254 line = line.replace(b'\n', linesep) 255 target.write(line) 256 lastline = line 257 if self._append_newline and lastline and not lastline.endswith(linesep): 258 # Make sure the message ends with a newline 259 target.write(linesep) 260 else: 261 raise TypeError('Invalid message type: %s' % type(message)) 262 263 264class Maildir(Mailbox): 265 """A qmail-style Maildir mailbox.""" 266 267 colon = ':' 268 269 def __init__(self, dirname, factory=None, create=True): 270 """Initialize a Maildir instance.""" 271 Mailbox.__init__(self, dirname, factory, create) 272 self._paths = { 273 'tmp': os.path.join(self._path, 'tmp'), 274 'new': os.path.join(self._path, 'new'), 275 'cur': os.path.join(self._path, 'cur'), 276 } 277 if not os.path.exists(self._path): 278 if create: 279 os.mkdir(self._path, 0o700) 280 for path in self._paths.values(): 281 os.mkdir(path, 0o700) 282 else: 283 raise NoSuchMailboxError(self._path) 284 self._toc = {} 285 self._toc_mtimes = {'cur': 0, 'new': 0} 286 self._last_read = 0 # Records last time we read cur/new 287 self._skewfactor = 0.1 # Adjust if os/fs clocks are skewing 288 289 def add(self, message): 290 """Add message and return assigned key.""" 291 tmp_file = self._create_tmp() 292 try: 293 self._dump_message(message, tmp_file) 294 except BaseException: 295 tmp_file.close() 296 os.remove(tmp_file.name) 297 raise 298 _sync_close(tmp_file) 299 if isinstance(message, MaildirMessage): 300 subdir = message.get_subdir() 301 suffix = self.colon + message.get_info() 302 if suffix == self.colon: 303 suffix = '' 304 else: 305 subdir = 'new' 306 suffix = '' 307 uniq = os.path.basename(tmp_file.name).split(self.colon)[0] 308 dest = os.path.join(self._path, subdir, uniq + suffix) 309 if isinstance(message, MaildirMessage): 310 os.utime(tmp_file.name, 311 (os.path.getatime(tmp_file.name), message.get_date())) 312 # No file modification should be done after the file is moved to its 313 # final position in order to prevent race conditions with changes 314 # from other programs 315 try: 316 try: 317 os.link(tmp_file.name, dest) 318 except (AttributeError, PermissionError): 319 os.rename(tmp_file.name, dest) 320 else: 321 os.remove(tmp_file.name) 322 except OSError as e: 323 os.remove(tmp_file.name) 324 if e.errno == errno.EEXIST: 325 raise ExternalClashError('Name clash with existing message: %s' 326 % dest) 327 else: 328 raise 329 return uniq 330 331 def remove(self, key): 332 """Remove the keyed message; raise KeyError if it doesn't exist.""" 333 os.remove(os.path.join(self._path, self._lookup(key))) 334 335 def discard(self, key): 336 """If the keyed message exists, remove it.""" 337 # This overrides an inapplicable implementation in the superclass. 338 try: 339 self.remove(key) 340 except (KeyError, FileNotFoundError): 341 pass 342 343 def __setitem__(self, key, message): 344 """Replace the keyed message; raise KeyError if it doesn't exist.""" 345 old_subpath = self._lookup(key) 346 temp_key = self.add(message) 347 temp_subpath = self._lookup(temp_key) 348 if isinstance(message, MaildirMessage): 349 # temp's subdir and suffix were specified by message. 350 dominant_subpath = temp_subpath 351 else: 352 # temp's subdir and suffix were defaults from add(). 353 dominant_subpath = old_subpath 354 subdir = os.path.dirname(dominant_subpath) 355 if self.colon in dominant_subpath: 356 suffix = self.colon + dominant_subpath.split(self.colon)[-1] 357 else: 358 suffix = '' 359 self.discard(key) 360 tmp_path = os.path.join(self._path, temp_subpath) 361 new_path = os.path.join(self._path, subdir, key + suffix) 362 if isinstance(message, MaildirMessage): 363 os.utime(tmp_path, 364 (os.path.getatime(tmp_path), message.get_date())) 365 # No file modification should be done after the file is moved to its 366 # final position in order to prevent race conditions with changes 367 # from other programs 368 os.rename(tmp_path, new_path) 369 370 def get_message(self, key): 371 """Return a Message representation or raise a KeyError.""" 372 subpath = self._lookup(key) 373 with open(os.path.join(self._path, subpath), 'rb') as f: 374 if self._factory: 375 msg = self._factory(f) 376 else: 377 msg = MaildirMessage(f) 378 subdir, name = os.path.split(subpath) 379 msg.set_subdir(subdir) 380 if self.colon in name: 381 msg.set_info(name.split(self.colon)[-1]) 382 msg.set_date(os.path.getmtime(os.path.join(self._path, subpath))) 383 return msg 384 385 def get_bytes(self, key): 386 """Return a bytes representation or raise a KeyError.""" 387 with open(os.path.join(self._path, self._lookup(key)), 'rb') as f: 388 return f.read().replace(linesep, b'\n') 389 390 def get_file(self, key): 391 """Return a file-like representation or raise a KeyError.""" 392 f = open(os.path.join(self._path, self._lookup(key)), 'rb') 393 return _ProxyFile(f) 394 395 def iterkeys(self): 396 """Return an iterator over keys.""" 397 self._refresh() 398 for key in self._toc: 399 try: 400 self._lookup(key) 401 except KeyError: 402 continue 403 yield key 404 405 def __contains__(self, key): 406 """Return True if the keyed message exists, False otherwise.""" 407 self._refresh() 408 return key in self._toc 409 410 def __len__(self): 411 """Return a count of messages in the mailbox.""" 412 self._refresh() 413 return len(self._toc) 414 415 def flush(self): 416 """Write any pending changes to disk.""" 417 # Maildir changes are always written immediately, so there's nothing 418 # to do. 419 pass 420 421 def lock(self): 422 """Lock the mailbox.""" 423 return 424 425 def unlock(self): 426 """Unlock the mailbox if it is locked.""" 427 return 428 429 def close(self): 430 """Flush and close the mailbox.""" 431 return 432 433 def list_folders(self): 434 """Return a list of folder names.""" 435 result = [] 436 for entry in os.listdir(self._path): 437 if len(entry) > 1 and entry[0] == '.' and \ 438 os.path.isdir(os.path.join(self._path, entry)): 439 result.append(entry[1:]) 440 return result 441 442 def get_folder(self, folder): 443 """Return a Maildir instance for the named folder.""" 444 return Maildir(os.path.join(self._path, '.' + folder), 445 factory=self._factory, 446 create=False) 447 448 def add_folder(self, folder): 449 """Create a folder and return a Maildir instance representing it.""" 450 path = os.path.join(self._path, '.' + folder) 451 result = Maildir(path, factory=self._factory) 452 maildirfolder_path = os.path.join(path, 'maildirfolder') 453 if not os.path.exists(maildirfolder_path): 454 os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY, 455 0o666)) 456 return result 457 458 def remove_folder(self, folder): 459 """Delete the named folder, which must be empty.""" 460 path = os.path.join(self._path, '.' + folder) 461 for entry in os.listdir(os.path.join(path, 'new')) + \ 462 os.listdir(os.path.join(path, 'cur')): 463 if len(entry) < 1 or entry[0] != '.': 464 raise NotEmptyError('Folder contains message(s): %s' % folder) 465 for entry in os.listdir(path): 466 if entry != 'new' and entry != 'cur' and entry != 'tmp' and \ 467 os.path.isdir(os.path.join(path, entry)): 468 raise NotEmptyError("Folder contains subdirectory '%s': %s" % 469 (folder, entry)) 470 for root, dirs, files in os.walk(path, topdown=False): 471 for entry in files: 472 os.remove(os.path.join(root, entry)) 473 for entry in dirs: 474 os.rmdir(os.path.join(root, entry)) 475 os.rmdir(path) 476 477 def clean(self): 478 """Delete old files in "tmp".""" 479 now = time.time() 480 for entry in os.listdir(os.path.join(self._path, 'tmp')): 481 path = os.path.join(self._path, 'tmp', entry) 482 if now - os.path.getatime(path) > 129600: # 60 * 60 * 36 483 os.remove(path) 484 485 _count = 1 # This is used to generate unique file names. 486 487 def _create_tmp(self): 488 """Create a file in the tmp subdirectory and open and return it.""" 489 now = time.time() 490 hostname = socket.gethostname() 491 if '/' in hostname: 492 hostname = hostname.replace('/', r'\057') 493 if ':' in hostname: 494 hostname = hostname.replace(':', r'\072') 495 uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(), 496 Maildir._count, hostname) 497 path = os.path.join(self._path, 'tmp', uniq) 498 try: 499 os.stat(path) 500 except FileNotFoundError: 501 Maildir._count += 1 502 try: 503 return _create_carefully(path) 504 except FileExistsError: 505 pass 506 507 # Fall through to here if stat succeeded or open raised EEXIST. 508 raise ExternalClashError('Name clash prevented file creation: %s' % 509 path) 510 511 def _refresh(self): 512 """Update table of contents mapping.""" 513 # If it has been less than two seconds since the last _refresh() call, 514 # we have to unconditionally re-read the mailbox just in case it has 515 # been modified, because os.path.mtime() has a 2 sec resolution in the 516 # most common worst case (FAT) and a 1 sec resolution typically. This 517 # results in a few unnecessary re-reads when _refresh() is called 518 # multiple times in that interval, but once the clock ticks over, we 519 # will only re-read as needed. Because the filesystem might be being 520 # served by an independent system with its own clock, we record and 521 # compare with the mtimes from the filesystem. Because the other 522 # system's clock might be skewing relative to our clock, we add an 523 # extra delta to our wait. The default is one tenth second, but is an 524 # instance variable and so can be adjusted if dealing with a 525 # particularly skewed or irregular system. 526 if time.time() - self._last_read > 2 + self._skewfactor: 527 refresh = False 528 for subdir in self._toc_mtimes: 529 mtime = os.path.getmtime(self._paths[subdir]) 530 if mtime > self._toc_mtimes[subdir]: 531 refresh = True 532 self._toc_mtimes[subdir] = mtime 533 if not refresh: 534 return 535 # Refresh toc 536 self._toc = {} 537 for subdir in self._toc_mtimes: 538 path = self._paths[subdir] 539 for entry in os.listdir(path): 540 p = os.path.join(path, entry) 541 if os.path.isdir(p): 542 continue 543 uniq = entry.split(self.colon)[0] 544 self._toc[uniq] = os.path.join(subdir, entry) 545 self._last_read = time.time() 546 547 def _lookup(self, key): 548 """Use TOC to return subpath for given key, or raise a KeyError.""" 549 try: 550 if os.path.exists(os.path.join(self._path, self._toc[key])): 551 return self._toc[key] 552 except KeyError: 553 pass 554 self._refresh() 555 try: 556 return self._toc[key] 557 except KeyError: 558 raise KeyError('No message with key: %s' % key) from None 559 560 # This method is for backward compatibility only. 561 def next(self): 562 """Return the next message in a one-time iteration.""" 563 if not hasattr(self, '_onetime_keys'): 564 self._onetime_keys = self.iterkeys() 565 while True: 566 try: 567 return self[next(self._onetime_keys)] 568 except StopIteration: 569 return None 570 except KeyError: 571 continue 572 573 574class _singlefileMailbox(Mailbox): 575 """A single-file mailbox.""" 576 577 def __init__(self, path, factory=None, create=True): 578 """Initialize a single-file mailbox.""" 579 Mailbox.__init__(self, path, factory, create) 580 try: 581 f = open(self._path, 'rb+') 582 except OSError as e: 583 if e.errno == errno.ENOENT: 584 if create: 585 f = open(self._path, 'wb+') 586 else: 587 raise NoSuchMailboxError(self._path) 588 elif e.errno in (errno.EACCES, errno.EROFS): 589 f = open(self._path, 'rb') 590 else: 591 raise 592 self._file = f 593 self._toc = None 594 self._next_key = 0 595 self._pending = False # No changes require rewriting the file. 596 self._pending_sync = False # No need to sync the file 597 self._locked = False 598 self._file_length = None # Used to record mailbox size 599 600 def add(self, message): 601 """Add message and return assigned key.""" 602 self._lookup() 603 self._toc[self._next_key] = self._append_message(message) 604 self._next_key += 1 605 # _append_message appends the message to the mailbox file. We 606 # don't need a full rewrite + rename, sync is enough. 607 self._pending_sync = True 608 return self._next_key - 1 609 610 def remove(self, key): 611 """Remove the keyed message; raise KeyError if it doesn't exist.""" 612 self._lookup(key) 613 del self._toc[key] 614 self._pending = True 615 616 def __setitem__(self, key, message): 617 """Replace the keyed message; raise KeyError if it doesn't exist.""" 618 self._lookup(key) 619 self._toc[key] = self._append_message(message) 620 self._pending = True 621 622 def iterkeys(self): 623 """Return an iterator over keys.""" 624 self._lookup() 625 yield from self._toc.keys() 626 627 def __contains__(self, key): 628 """Return True if the keyed message exists, False otherwise.""" 629 self._lookup() 630 return key in self._toc 631 632 def __len__(self): 633 """Return a count of messages in the mailbox.""" 634 self._lookup() 635 return len(self._toc) 636 637 def lock(self): 638 """Lock the mailbox.""" 639 if not self._locked: 640 _lock_file(self._file) 641 self._locked = True 642 643 def unlock(self): 644 """Unlock the mailbox if it is locked.""" 645 if self._locked: 646 _unlock_file(self._file) 647 self._locked = False 648 649 def flush(self): 650 """Write any pending changes to disk.""" 651 if not self._pending: 652 if self._pending_sync: 653 # Messages have only been added, so syncing the file 654 # is enough. 655 _sync_flush(self._file) 656 self._pending_sync = False 657 return 658 659 # In order to be writing anything out at all, self._toc must 660 # already have been generated (and presumably has been modified 661 # by adding or deleting an item). 662 assert self._toc is not None 663 664 # Check length of self._file; if it's changed, some other process 665 # has modified the mailbox since we scanned it. 666 self._file.seek(0, 2) 667 cur_len = self._file.tell() 668 if cur_len != self._file_length: 669 raise ExternalClashError('Size of mailbox file changed ' 670 '(expected %i, found %i)' % 671 (self._file_length, cur_len)) 672 673 new_file = _create_temporary(self._path) 674 try: 675 new_toc = {} 676 self._pre_mailbox_hook(new_file) 677 for key in sorted(self._toc.keys()): 678 start, stop = self._toc[key] 679 self._file.seek(start) 680 self._pre_message_hook(new_file) 681 new_start = new_file.tell() 682 while True: 683 buffer = self._file.read(min(4096, 684 stop - self._file.tell())) 685 if not buffer: 686 break 687 new_file.write(buffer) 688 new_toc[key] = (new_start, new_file.tell()) 689 self._post_message_hook(new_file) 690 self._file_length = new_file.tell() 691 except: 692 new_file.close() 693 os.remove(new_file.name) 694 raise 695 _sync_close(new_file) 696 # self._file is about to get replaced, so no need to sync. 697 self._file.close() 698 # Make sure the new file's mode is the same as the old file's 699 mode = os.stat(self._path).st_mode 700 os.chmod(new_file.name, mode) 701 try: 702 os.rename(new_file.name, self._path) 703 except FileExistsError: 704 os.remove(self._path) 705 os.rename(new_file.name, self._path) 706 self._file = open(self._path, 'rb+') 707 self._toc = new_toc 708 self._pending = False 709 self._pending_sync = False 710 if self._locked: 711 _lock_file(self._file, dotlock=False) 712 713 def _pre_mailbox_hook(self, f): 714 """Called before writing the mailbox to file f.""" 715 return 716 717 def _pre_message_hook(self, f): 718 """Called before writing each message to file f.""" 719 return 720 721 def _post_message_hook(self, f): 722 """Called after writing each message to file f.""" 723 return 724 725 def close(self): 726 """Flush and close the mailbox.""" 727 try: 728 self.flush() 729 finally: 730 try: 731 if self._locked: 732 self.unlock() 733 finally: 734 self._file.close() # Sync has been done by self.flush() above. 735 736 def _lookup(self, key=None): 737 """Return (start, stop) or raise KeyError.""" 738 if self._toc is None: 739 self._generate_toc() 740 if key is not None: 741 try: 742 return self._toc[key] 743 except KeyError: 744 raise KeyError('No message with key: %s' % key) from None 745 746 def _append_message(self, message): 747 """Append message to mailbox and return (start, stop) offsets.""" 748 self._file.seek(0, 2) 749 before = self._file.tell() 750 if len(self._toc) == 0 and not self._pending: 751 # This is the first message, and the _pre_mailbox_hook 752 # hasn't yet been called. If self._pending is True, 753 # messages have been removed, so _pre_mailbox_hook must 754 # have been called already. 755 self._pre_mailbox_hook(self._file) 756 try: 757 self._pre_message_hook(self._file) 758 offsets = self._install_message(message) 759 self._post_message_hook(self._file) 760 except BaseException: 761 self._file.truncate(before) 762 raise 763 self._file.flush() 764 self._file_length = self._file.tell() # Record current length of mailbox 765 return offsets 766 767 768 769class _mboxMMDF(_singlefileMailbox): 770 """An mbox or MMDF mailbox.""" 771 772 _mangle_from_ = True 773 774 def get_message(self, key): 775 """Return a Message representation or raise a KeyError.""" 776 start, stop = self._lookup(key) 777 self._file.seek(start) 778 from_line = self._file.readline().replace(linesep, b'') 779 string = self._file.read(stop - self._file.tell()) 780 msg = self._message_factory(string.replace(linesep, b'\n')) 781 msg.set_from(from_line[5:].decode('ascii')) 782 return msg 783 784 def get_string(self, key, from_=False): 785 """Return a string representation or raise a KeyError.""" 786 return email.message_from_bytes( 787 self.get_bytes(key, from_)).as_string(unixfrom=from_) 788 789 def get_bytes(self, key, from_=False): 790 """Return a string representation or raise a KeyError.""" 791 start, stop = self._lookup(key) 792 self._file.seek(start) 793 if not from_: 794 self._file.readline() 795 string = self._file.read(stop - self._file.tell()) 796 return string.replace(linesep, b'\n') 797 798 def get_file(self, key, from_=False): 799 """Return a file-like representation or raise a KeyError.""" 800 start, stop = self._lookup(key) 801 self._file.seek(start) 802 if not from_: 803 self._file.readline() 804 return _PartialFile(self._file, self._file.tell(), stop) 805 806 def _install_message(self, message): 807 """Format a message and blindly write to self._file.""" 808 from_line = None 809 if isinstance(message, str): 810 message = self._string_to_bytes(message) 811 if isinstance(message, bytes) and message.startswith(b'From '): 812 newline = message.find(b'\n') 813 if newline != -1: 814 from_line = message[:newline] 815 message = message[newline + 1:] 816 else: 817 from_line = message 818 message = b'' 819 elif isinstance(message, _mboxMMDFMessage): 820 author = message.get_from().encode('ascii') 821 from_line = b'From ' + author 822 elif isinstance(message, email.message.Message): 823 from_line = message.get_unixfrom() # May be None. 824 if from_line is not None: 825 from_line = from_line.encode('ascii') 826 if from_line is None: 827 from_line = b'From MAILER-DAEMON ' + time.asctime(time.gmtime()).encode() 828 start = self._file.tell() 829 self._file.write(from_line + linesep) 830 self._dump_message(message, self._file, self._mangle_from_) 831 stop = self._file.tell() 832 return (start, stop) 833 834 835class mbox(_mboxMMDF): 836 """A classic mbox mailbox.""" 837 838 _mangle_from_ = True 839 840 # All messages must end in a newline character, and 841 # _post_message_hooks outputs an empty line between messages. 842 _append_newline = True 843 844 def __init__(self, path, factory=None, create=True): 845 """Initialize an mbox mailbox.""" 846 self._message_factory = mboxMessage 847 _mboxMMDF.__init__(self, path, factory, create) 848 849 def _post_message_hook(self, f): 850 """Called after writing each message to file f.""" 851 f.write(linesep) 852 853 def _generate_toc(self): 854 """Generate key-to-(start, stop) table of contents.""" 855 starts, stops = [], [] 856 last_was_empty = False 857 self._file.seek(0) 858 while True: 859 line_pos = self._file.tell() 860 line = self._file.readline() 861 if line.startswith(b'From '): 862 if len(stops) < len(starts): 863 if last_was_empty: 864 stops.append(line_pos - len(linesep)) 865 else: 866 # The last line before the "From " line wasn't 867 # blank, but we consider it a start of a 868 # message anyway. 869 stops.append(line_pos) 870 starts.append(line_pos) 871 last_was_empty = False 872 elif not line: 873 if last_was_empty: 874 stops.append(line_pos - len(linesep)) 875 else: 876 stops.append(line_pos) 877 break 878 elif line == linesep: 879 last_was_empty = True 880 else: 881 last_was_empty = False 882 self._toc = dict(enumerate(zip(starts, stops))) 883 self._next_key = len(self._toc) 884 self._file_length = self._file.tell() 885 886 887class MMDF(_mboxMMDF): 888 """An MMDF mailbox.""" 889 890 def __init__(self, path, factory=None, create=True): 891 """Initialize an MMDF mailbox.""" 892 self._message_factory = MMDFMessage 893 _mboxMMDF.__init__(self, path, factory, create) 894 895 def _pre_message_hook(self, f): 896 """Called before writing each message to file f.""" 897 f.write(b'\001\001\001\001' + linesep) 898 899 def _post_message_hook(self, f): 900 """Called after writing each message to file f.""" 901 f.write(linesep + b'\001\001\001\001' + linesep) 902 903 def _generate_toc(self): 904 """Generate key-to-(start, stop) table of contents.""" 905 starts, stops = [], [] 906 self._file.seek(0) 907 next_pos = 0 908 while True: 909 line_pos = next_pos 910 line = self._file.readline() 911 next_pos = self._file.tell() 912 if line.startswith(b'\001\001\001\001' + linesep): 913 starts.append(next_pos) 914 while True: 915 line_pos = next_pos 916 line = self._file.readline() 917 next_pos = self._file.tell() 918 if line == b'\001\001\001\001' + linesep: 919 stops.append(line_pos - len(linesep)) 920 break 921 elif not line: 922 stops.append(line_pos) 923 break 924 elif not line: 925 break 926 self._toc = dict(enumerate(zip(starts, stops))) 927 self._next_key = len(self._toc) 928 self._file.seek(0, 2) 929 self._file_length = self._file.tell() 930 931 932class MH(Mailbox): 933 """An MH mailbox.""" 934 935 def __init__(self, path, factory=None, create=True): 936 """Initialize an MH instance.""" 937 Mailbox.__init__(self, path, factory, create) 938 if not os.path.exists(self._path): 939 if create: 940 os.mkdir(self._path, 0o700) 941 os.close(os.open(os.path.join(self._path, '.mh_sequences'), 942 os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)) 943 else: 944 raise NoSuchMailboxError(self._path) 945 self._locked = False 946 947 def add(self, message): 948 """Add message and return assigned key.""" 949 keys = self.keys() 950 if len(keys) == 0: 951 new_key = 1 952 else: 953 new_key = max(keys) + 1 954 new_path = os.path.join(self._path, str(new_key)) 955 f = _create_carefully(new_path) 956 closed = False 957 try: 958 if self._locked: 959 _lock_file(f) 960 try: 961 try: 962 self._dump_message(message, f) 963 except BaseException: 964 # Unlock and close so it can be deleted on Windows 965 if self._locked: 966 _unlock_file(f) 967 _sync_close(f) 968 closed = True 969 os.remove(new_path) 970 raise 971 if isinstance(message, MHMessage): 972 self._dump_sequences(message, new_key) 973 finally: 974 if self._locked: 975 _unlock_file(f) 976 finally: 977 if not closed: 978 _sync_close(f) 979 return new_key 980 981 def remove(self, key): 982 """Remove the keyed message; raise KeyError if it doesn't exist.""" 983 path = os.path.join(self._path, str(key)) 984 try: 985 f = open(path, 'rb+') 986 except OSError as e: 987 if e.errno == errno.ENOENT: 988 raise KeyError('No message with key: %s' % key) 989 else: 990 raise 991 else: 992 f.close() 993 os.remove(path) 994 995 def __setitem__(self, key, message): 996 """Replace the keyed message; raise KeyError if it doesn't exist.""" 997 path = os.path.join(self._path, str(key)) 998 try: 999 f = open(path, 'rb+') 1000 except OSError as e: 1001 if e.errno == errno.ENOENT: 1002 raise KeyError('No message with key: %s' % key) 1003 else: 1004 raise 1005 try: 1006 if self._locked: 1007 _lock_file(f) 1008 try: 1009 os.close(os.open(path, os.O_WRONLY | os.O_TRUNC)) 1010 self._dump_message(message, f) 1011 if isinstance(message, MHMessage): 1012 self._dump_sequences(message, key) 1013 finally: 1014 if self._locked: 1015 _unlock_file(f) 1016 finally: 1017 _sync_close(f) 1018 1019 def get_message(self, key): 1020 """Return a Message representation or raise a KeyError.""" 1021 try: 1022 if self._locked: 1023 f = open(os.path.join(self._path, str(key)), 'rb+') 1024 else: 1025 f = open(os.path.join(self._path, str(key)), 'rb') 1026 except OSError as e: 1027 if e.errno == errno.ENOENT: 1028 raise KeyError('No message with key: %s' % key) 1029 else: 1030 raise 1031 with f: 1032 if self._locked: 1033 _lock_file(f) 1034 try: 1035 msg = MHMessage(f) 1036 finally: 1037 if self._locked: 1038 _unlock_file(f) 1039 for name, key_list in self.get_sequences().items(): 1040 if key in key_list: 1041 msg.add_sequence(name) 1042 return msg 1043 1044 def get_bytes(self, key): 1045 """Return a bytes representation or raise a KeyError.""" 1046 try: 1047 if self._locked: 1048 f = open(os.path.join(self._path, str(key)), 'rb+') 1049 else: 1050 f = open(os.path.join(self._path, str(key)), 'rb') 1051 except OSError as e: 1052 if e.errno == errno.ENOENT: 1053 raise KeyError('No message with key: %s' % key) 1054 else: 1055 raise 1056 with f: 1057 if self._locked: 1058 _lock_file(f) 1059 try: 1060 return f.read().replace(linesep, b'\n') 1061 finally: 1062 if self._locked: 1063 _unlock_file(f) 1064 1065 def get_file(self, key): 1066 """Return a file-like representation or raise a KeyError.""" 1067 try: 1068 f = open(os.path.join(self._path, str(key)), 'rb') 1069 except OSError as e: 1070 if e.errno == errno.ENOENT: 1071 raise KeyError('No message with key: %s' % key) 1072 else: 1073 raise 1074 return _ProxyFile(f) 1075 1076 def iterkeys(self): 1077 """Return an iterator over keys.""" 1078 return iter(sorted(int(entry) for entry in os.listdir(self._path) 1079 if entry.isdigit())) 1080 1081 def __contains__(self, key): 1082 """Return True if the keyed message exists, False otherwise.""" 1083 return os.path.exists(os.path.join(self._path, str(key))) 1084 1085 def __len__(self): 1086 """Return a count of messages in the mailbox.""" 1087 return len(list(self.iterkeys())) 1088 1089 def lock(self): 1090 """Lock the mailbox.""" 1091 if not self._locked: 1092 self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+') 1093 _lock_file(self._file) 1094 self._locked = True 1095 1096 def unlock(self): 1097 """Unlock the mailbox if it is locked.""" 1098 if self._locked: 1099 _unlock_file(self._file) 1100 _sync_close(self._file) 1101 del self._file 1102 self._locked = False 1103 1104 def flush(self): 1105 """Write any pending changes to the disk.""" 1106 return 1107 1108 def close(self): 1109 """Flush and close the mailbox.""" 1110 if self._locked: 1111 self.unlock() 1112 1113 def list_folders(self): 1114 """Return a list of folder names.""" 1115 result = [] 1116 for entry in os.listdir(self._path): 1117 if os.path.isdir(os.path.join(self._path, entry)): 1118 result.append(entry) 1119 return result 1120 1121 def get_folder(self, folder): 1122 """Return an MH instance for the named folder.""" 1123 return MH(os.path.join(self._path, folder), 1124 factory=self._factory, create=False) 1125 1126 def add_folder(self, folder): 1127 """Create a folder and return an MH instance representing it.""" 1128 return MH(os.path.join(self._path, folder), 1129 factory=self._factory) 1130 1131 def remove_folder(self, folder): 1132 """Delete the named folder, which must be empty.""" 1133 path = os.path.join(self._path, folder) 1134 entries = os.listdir(path) 1135 if entries == ['.mh_sequences']: 1136 os.remove(os.path.join(path, '.mh_sequences')) 1137 elif entries == []: 1138 pass 1139 else: 1140 raise NotEmptyError('Folder not empty: %s' % self._path) 1141 os.rmdir(path) 1142 1143 def get_sequences(self): 1144 """Return a name-to-key-list dictionary to define each sequence.""" 1145 results = {} 1146 with open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') as f: 1147 all_keys = set(self.keys()) 1148 for line in f: 1149 try: 1150 name, contents = line.split(':') 1151 keys = set() 1152 for spec in contents.split(): 1153 if spec.isdigit(): 1154 keys.add(int(spec)) 1155 else: 1156 start, stop = (int(x) for x in spec.split('-')) 1157 keys.update(range(start, stop + 1)) 1158 results[name] = [key for key in sorted(keys) \ 1159 if key in all_keys] 1160 if len(results[name]) == 0: 1161 del results[name] 1162 except ValueError: 1163 raise FormatError('Invalid sequence specification: %s' % 1164 line.rstrip()) 1165 return results 1166 1167 def set_sequences(self, sequences): 1168 """Set sequences using the given name-to-key-list dictionary.""" 1169 f = open(os.path.join(self._path, '.mh_sequences'), 'r+', encoding='ASCII') 1170 try: 1171 os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC)) 1172 for name, keys in sequences.items(): 1173 if len(keys) == 0: 1174 continue 1175 f.write(name + ':') 1176 prev = None 1177 completing = False 1178 for key in sorted(set(keys)): 1179 if key - 1 == prev: 1180 if not completing: 1181 completing = True 1182 f.write('-') 1183 elif completing: 1184 completing = False 1185 f.write('%s %s' % (prev, key)) 1186 else: 1187 f.write(' %s' % key) 1188 prev = key 1189 if completing: 1190 f.write(str(prev) + '\n') 1191 else: 1192 f.write('\n') 1193 finally: 1194 _sync_close(f) 1195 1196 def pack(self): 1197 """Re-name messages to eliminate numbering gaps. Invalidates keys.""" 1198 sequences = self.get_sequences() 1199 prev = 0 1200 changes = [] 1201 for key in self.iterkeys(): 1202 if key - 1 != prev: 1203 changes.append((key, prev + 1)) 1204 try: 1205 os.link(os.path.join(self._path, str(key)), 1206 os.path.join(self._path, str(prev + 1))) 1207 except (AttributeError, PermissionError): 1208 os.rename(os.path.join(self._path, str(key)), 1209 os.path.join(self._path, str(prev + 1))) 1210 else: 1211 os.unlink(os.path.join(self._path, str(key))) 1212 prev += 1 1213 self._next_key = prev + 1 1214 if len(changes) == 0: 1215 return 1216 for name, key_list in sequences.items(): 1217 for old, new in changes: 1218 if old in key_list: 1219 key_list[key_list.index(old)] = new 1220 self.set_sequences(sequences) 1221 1222 def _dump_sequences(self, message, key): 1223 """Inspect a new MHMessage and update sequences appropriately.""" 1224 pending_sequences = message.get_sequences() 1225 all_sequences = self.get_sequences() 1226 for name, key_list in all_sequences.items(): 1227 if name in pending_sequences: 1228 key_list.append(key) 1229 elif key in key_list: 1230 del key_list[key_list.index(key)] 1231 for sequence in pending_sequences: 1232 if sequence not in all_sequences: 1233 all_sequences[sequence] = [key] 1234 self.set_sequences(all_sequences) 1235 1236 1237class Babyl(_singlefileMailbox): 1238 """An Rmail-style Babyl mailbox.""" 1239 1240 _special_labels = frozenset({'unseen', 'deleted', 'filed', 'answered', 1241 'forwarded', 'edited', 'resent'}) 1242 1243 def __init__(self, path, factory=None, create=True): 1244 """Initialize a Babyl mailbox.""" 1245 _singlefileMailbox.__init__(self, path, factory, create) 1246 self._labels = {} 1247 1248 def add(self, message): 1249 """Add message and return assigned key.""" 1250 key = _singlefileMailbox.add(self, message) 1251 if isinstance(message, BabylMessage): 1252 self._labels[key] = message.get_labels() 1253 return key 1254 1255 def remove(self, key): 1256 """Remove the keyed message; raise KeyError if it doesn't exist.""" 1257 _singlefileMailbox.remove(self, key) 1258 if key in self._labels: 1259 del self._labels[key] 1260 1261 def __setitem__(self, key, message): 1262 """Replace the keyed message; raise KeyError if it doesn't exist.""" 1263 _singlefileMailbox.__setitem__(self, key, message) 1264 if isinstance(message, BabylMessage): 1265 self._labels[key] = message.get_labels() 1266 1267 def get_message(self, key): 1268 """Return a Message representation or raise a KeyError.""" 1269 start, stop = self._lookup(key) 1270 self._file.seek(start) 1271 self._file.readline() # Skip b'1,' line specifying labels. 1272 original_headers = io.BytesIO() 1273 while True: 1274 line = self._file.readline() 1275 if line == b'*** EOOH ***' + linesep or not line: 1276 break 1277 original_headers.write(line.replace(linesep, b'\n')) 1278 visible_headers = io.BytesIO() 1279 while True: 1280 line = self._file.readline() 1281 if line == linesep or not line: 1282 break 1283 visible_headers.write(line.replace(linesep, b'\n')) 1284 # Read up to the stop, or to the end 1285 n = stop - self._file.tell() 1286 assert n >= 0 1287 body = self._file.read(n) 1288 body = body.replace(linesep, b'\n') 1289 msg = BabylMessage(original_headers.getvalue() + body) 1290 msg.set_visible(visible_headers.getvalue()) 1291 if key in self._labels: 1292 msg.set_labels(self._labels[key]) 1293 return msg 1294 1295 def get_bytes(self, key): 1296 """Return a string representation or raise a KeyError.""" 1297 start, stop = self._lookup(key) 1298 self._file.seek(start) 1299 self._file.readline() # Skip b'1,' line specifying labels. 1300 original_headers = io.BytesIO() 1301 while True: 1302 line = self._file.readline() 1303 if line == b'*** EOOH ***' + linesep or not line: 1304 break 1305 original_headers.write(line.replace(linesep, b'\n')) 1306 while True: 1307 line = self._file.readline() 1308 if line == linesep or not line: 1309 break 1310 headers = original_headers.getvalue() 1311 n = stop - self._file.tell() 1312 assert n >= 0 1313 data = self._file.read(n) 1314 data = data.replace(linesep, b'\n') 1315 return headers + data 1316 1317 def get_file(self, key): 1318 """Return a file-like representation or raise a KeyError.""" 1319 return io.BytesIO(self.get_bytes(key).replace(b'\n', linesep)) 1320 1321 def get_labels(self): 1322 """Return a list of user-defined labels in the mailbox.""" 1323 self._lookup() 1324 labels = set() 1325 for label_list in self._labels.values(): 1326 labels.update(label_list) 1327 labels.difference_update(self._special_labels) 1328 return list(labels) 1329 1330 def _generate_toc(self): 1331 """Generate key-to-(start, stop) table of contents.""" 1332 starts, stops = [], [] 1333 self._file.seek(0) 1334 next_pos = 0 1335 label_lists = [] 1336 while True: 1337 line_pos = next_pos 1338 line = self._file.readline() 1339 next_pos = self._file.tell() 1340 if line == b'\037\014' + linesep: 1341 if len(stops) < len(starts): 1342 stops.append(line_pos - len(linesep)) 1343 starts.append(next_pos) 1344 labels = [label.strip() for label 1345 in self._file.readline()[1:].split(b',') 1346 if label.strip()] 1347 label_lists.append(labels) 1348 elif line == b'\037' or line == b'\037' + linesep: 1349 if len(stops) < len(starts): 1350 stops.append(line_pos - len(linesep)) 1351 elif not line: 1352 stops.append(line_pos - len(linesep)) 1353 break 1354 self._toc = dict(enumerate(zip(starts, stops))) 1355 self._labels = dict(enumerate(label_lists)) 1356 self._next_key = len(self._toc) 1357 self._file.seek(0, 2) 1358 self._file_length = self._file.tell() 1359 1360 def _pre_mailbox_hook(self, f): 1361 """Called before writing the mailbox to file f.""" 1362 babyl = b'BABYL OPTIONS:' + linesep 1363 babyl += b'Version: 5' + linesep 1364 labels = self.get_labels() 1365 labels = (label.encode() for label in labels) 1366 babyl += b'Labels:' + b','.join(labels) + linesep 1367 babyl += b'\037' 1368 f.write(babyl) 1369 1370 def _pre_message_hook(self, f): 1371 """Called before writing each message to file f.""" 1372 f.write(b'\014' + linesep) 1373 1374 def _post_message_hook(self, f): 1375 """Called after writing each message to file f.""" 1376 f.write(linesep + b'\037') 1377 1378 def _install_message(self, message): 1379 """Write message contents and return (start, stop).""" 1380 start = self._file.tell() 1381 if isinstance(message, BabylMessage): 1382 special_labels = [] 1383 labels = [] 1384 for label in message.get_labels(): 1385 if label in self._special_labels: 1386 special_labels.append(label) 1387 else: 1388 labels.append(label) 1389 self._file.write(b'1') 1390 for label in special_labels: 1391 self._file.write(b', ' + label.encode()) 1392 self._file.write(b',,') 1393 for label in labels: 1394 self._file.write(b' ' + label.encode() + b',') 1395 self._file.write(linesep) 1396 else: 1397 self._file.write(b'1,,' + linesep) 1398 if isinstance(message, email.message.Message): 1399 orig_buffer = io.BytesIO() 1400 orig_generator = email.generator.BytesGenerator(orig_buffer, False, 0) 1401 orig_generator.flatten(message) 1402 orig_buffer.seek(0) 1403 while True: 1404 line = orig_buffer.readline() 1405 self._file.write(line.replace(b'\n', linesep)) 1406 if line == b'\n' or not line: 1407 break 1408 self._file.write(b'*** EOOH ***' + linesep) 1409 if isinstance(message, BabylMessage): 1410 vis_buffer = io.BytesIO() 1411 vis_generator = email.generator.BytesGenerator(vis_buffer, False, 0) 1412 vis_generator.flatten(message.get_visible()) 1413 while True: 1414 line = vis_buffer.readline() 1415 self._file.write(line.replace(b'\n', linesep)) 1416 if line == b'\n' or not line: 1417 break 1418 else: 1419 orig_buffer.seek(0) 1420 while True: 1421 line = orig_buffer.readline() 1422 self._file.write(line.replace(b'\n', linesep)) 1423 if line == b'\n' or not line: 1424 break 1425 while True: 1426 buffer = orig_buffer.read(4096) # Buffer size is arbitrary. 1427 if not buffer: 1428 break 1429 self._file.write(buffer.replace(b'\n', linesep)) 1430 elif isinstance(message, (bytes, str, io.StringIO)): 1431 if isinstance(message, io.StringIO): 1432 warnings.warn("Use of StringIO input is deprecated, " 1433 "use BytesIO instead", DeprecationWarning, 3) 1434 message = message.getvalue() 1435 if isinstance(message, str): 1436 message = self._string_to_bytes(message) 1437 body_start = message.find(b'\n\n') + 2 1438 if body_start - 2 != -1: 1439 self._file.write(message[:body_start].replace(b'\n', linesep)) 1440 self._file.write(b'*** EOOH ***' + linesep) 1441 self._file.write(message[:body_start].replace(b'\n', linesep)) 1442 self._file.write(message[body_start:].replace(b'\n', linesep)) 1443 else: 1444 self._file.write(b'*** EOOH ***' + linesep + linesep) 1445 self._file.write(message.replace(b'\n', linesep)) 1446 elif hasattr(message, 'readline'): 1447 if hasattr(message, 'buffer'): 1448 warnings.warn("Use of text mode files is deprecated, " 1449 "use a binary mode file instead", DeprecationWarning, 3) 1450 message = message.buffer 1451 original_pos = message.tell() 1452 first_pass = True 1453 while True: 1454 line = message.readline() 1455 # Universal newline support. 1456 if line.endswith(b'\r\n'): 1457 line = line[:-2] + b'\n' 1458 elif line.endswith(b'\r'): 1459 line = line[:-1] + b'\n' 1460 self._file.write(line.replace(b'\n', linesep)) 1461 if line == b'\n' or not line: 1462 if first_pass: 1463 first_pass = False 1464 self._file.write(b'*** EOOH ***' + linesep) 1465 message.seek(original_pos) 1466 else: 1467 break 1468 while True: 1469 line = message.readline() 1470 if not line: 1471 break 1472 # Universal newline support. 1473 if line.endswith(b'\r\n'): 1474 line = line[:-2] + linesep 1475 elif line.endswith(b'\r'): 1476 line = line[:-1] + linesep 1477 elif line.endswith(b'\n'): 1478 line = line[:-1] + linesep 1479 self._file.write(line) 1480 else: 1481 raise TypeError('Invalid message type: %s' % type(message)) 1482 stop = self._file.tell() 1483 return (start, stop) 1484 1485 1486class Message(email.message.Message): 1487 """Message with mailbox-format-specific properties.""" 1488 1489 def __init__(self, message=None): 1490 """Initialize a Message instance.""" 1491 if isinstance(message, email.message.Message): 1492 self._become_message(copy.deepcopy(message)) 1493 if isinstance(message, Message): 1494 message._explain_to(self) 1495 elif isinstance(message, bytes): 1496 self._become_message(email.message_from_bytes(message)) 1497 elif isinstance(message, str): 1498 self._become_message(email.message_from_string(message)) 1499 elif isinstance(message, io.TextIOWrapper): 1500 self._become_message(email.message_from_file(message)) 1501 elif hasattr(message, "read"): 1502 self._become_message(email.message_from_binary_file(message)) 1503 elif message is None: 1504 email.message.Message.__init__(self) 1505 else: 1506 raise TypeError('Invalid message type: %s' % type(message)) 1507 1508 def _become_message(self, message): 1509 """Assume the non-format-specific state of message.""" 1510 type_specific = getattr(message, '_type_specific_attributes', []) 1511 for name in message.__dict__: 1512 if name not in type_specific: 1513 self.__dict__[name] = message.__dict__[name] 1514 1515 def _explain_to(self, message): 1516 """Copy format-specific state to message insofar as possible.""" 1517 if isinstance(message, Message): 1518 return # There's nothing format-specific to explain. 1519 else: 1520 raise TypeError('Cannot convert to specified type') 1521 1522 1523class MaildirMessage(Message): 1524 """Message with Maildir-specific properties.""" 1525 1526 _type_specific_attributes = ['_subdir', '_info', '_date'] 1527 1528 def __init__(self, message=None): 1529 """Initialize a MaildirMessage instance.""" 1530 self._subdir = 'new' 1531 self._info = '' 1532 self._date = time.time() 1533 Message.__init__(self, message) 1534 1535 def get_subdir(self): 1536 """Return 'new' or 'cur'.""" 1537 return self._subdir 1538 1539 def set_subdir(self, subdir): 1540 """Set subdir to 'new' or 'cur'.""" 1541 if subdir == 'new' or subdir == 'cur': 1542 self._subdir = subdir 1543 else: 1544 raise ValueError("subdir must be 'new' or 'cur': %s" % subdir) 1545 1546 def get_flags(self): 1547 """Return as a string the flags that are set.""" 1548 if self._info.startswith('2,'): 1549 return self._info[2:] 1550 else: 1551 return '' 1552 1553 def set_flags(self, flags): 1554 """Set the given flags and unset all others.""" 1555 self._info = '2,' + ''.join(sorted(flags)) 1556 1557 def add_flag(self, flag): 1558 """Set the given flag(s) without changing others.""" 1559 self.set_flags(''.join(set(self.get_flags()) | set(flag))) 1560 1561 def remove_flag(self, flag): 1562 """Unset the given string flag(s) without changing others.""" 1563 if self.get_flags(): 1564 self.set_flags(''.join(set(self.get_flags()) - set(flag))) 1565 1566 def get_date(self): 1567 """Return delivery date of message, in seconds since the epoch.""" 1568 return self._date 1569 1570 def set_date(self, date): 1571 """Set delivery date of message, in seconds since the epoch.""" 1572 try: 1573 self._date = float(date) 1574 except ValueError: 1575 raise TypeError("can't convert to float: %s" % date) from None 1576 1577 def get_info(self): 1578 """Get the message's "info" as a string.""" 1579 return self._info 1580 1581 def set_info(self, info): 1582 """Set the message's "info" string.""" 1583 if isinstance(info, str): 1584 self._info = info 1585 else: 1586 raise TypeError('info must be a string: %s' % type(info)) 1587 1588 def _explain_to(self, message): 1589 """Copy Maildir-specific state to message insofar as possible.""" 1590 if isinstance(message, MaildirMessage): 1591 message.set_flags(self.get_flags()) 1592 message.set_subdir(self.get_subdir()) 1593 message.set_date(self.get_date()) 1594 elif isinstance(message, _mboxMMDFMessage): 1595 flags = set(self.get_flags()) 1596 if 'S' in flags: 1597 message.add_flag('R') 1598 if self.get_subdir() == 'cur': 1599 message.add_flag('O') 1600 if 'T' in flags: 1601 message.add_flag('D') 1602 if 'F' in flags: 1603 message.add_flag('F') 1604 if 'R' in flags: 1605 message.add_flag('A') 1606 message.set_from('MAILER-DAEMON', time.gmtime(self.get_date())) 1607 elif isinstance(message, MHMessage): 1608 flags = set(self.get_flags()) 1609 if 'S' not in flags: 1610 message.add_sequence('unseen') 1611 if 'R' in flags: 1612 message.add_sequence('replied') 1613 if 'F' in flags: 1614 message.add_sequence('flagged') 1615 elif isinstance(message, BabylMessage): 1616 flags = set(self.get_flags()) 1617 if 'S' not in flags: 1618 message.add_label('unseen') 1619 if 'T' in flags: 1620 message.add_label('deleted') 1621 if 'R' in flags: 1622 message.add_label('answered') 1623 if 'P' in flags: 1624 message.add_label('forwarded') 1625 elif isinstance(message, Message): 1626 pass 1627 else: 1628 raise TypeError('Cannot convert to specified type: %s' % 1629 type(message)) 1630 1631 1632class _mboxMMDFMessage(Message): 1633 """Message with mbox- or MMDF-specific properties.""" 1634 1635 _type_specific_attributes = ['_from'] 1636 1637 def __init__(self, message=None): 1638 """Initialize an mboxMMDFMessage instance.""" 1639 self.set_from('MAILER-DAEMON', True) 1640 if isinstance(message, email.message.Message): 1641 unixfrom = message.get_unixfrom() 1642 if unixfrom is not None and unixfrom.startswith('From '): 1643 self.set_from(unixfrom[5:]) 1644 Message.__init__(self, message) 1645 1646 def get_from(self): 1647 """Return contents of "From " line.""" 1648 return self._from 1649 1650 def set_from(self, from_, time_=None): 1651 """Set "From " line, formatting and appending time_ if specified.""" 1652 if time_ is not None: 1653 if time_ is True: 1654 time_ = time.gmtime() 1655 from_ += ' ' + time.asctime(time_) 1656 self._from = from_ 1657 1658 def get_flags(self): 1659 """Return as a string the flags that are set.""" 1660 return self.get('Status', '') + self.get('X-Status', '') 1661 1662 def set_flags(self, flags): 1663 """Set the given flags and unset all others.""" 1664 flags = set(flags) 1665 status_flags, xstatus_flags = '', '' 1666 for flag in ('R', 'O'): 1667 if flag in flags: 1668 status_flags += flag 1669 flags.remove(flag) 1670 for flag in ('D', 'F', 'A'): 1671 if flag in flags: 1672 xstatus_flags += flag 1673 flags.remove(flag) 1674 xstatus_flags += ''.join(sorted(flags)) 1675 try: 1676 self.replace_header('Status', status_flags) 1677 except KeyError: 1678 self.add_header('Status', status_flags) 1679 try: 1680 self.replace_header('X-Status', xstatus_flags) 1681 except KeyError: 1682 self.add_header('X-Status', xstatus_flags) 1683 1684 def add_flag(self, flag): 1685 """Set the given flag(s) without changing others.""" 1686 self.set_flags(''.join(set(self.get_flags()) | set(flag))) 1687 1688 def remove_flag(self, flag): 1689 """Unset the given string flag(s) without changing others.""" 1690 if 'Status' in self or 'X-Status' in self: 1691 self.set_flags(''.join(set(self.get_flags()) - set(flag))) 1692 1693 def _explain_to(self, message): 1694 """Copy mbox- or MMDF-specific state to message insofar as possible.""" 1695 if isinstance(message, MaildirMessage): 1696 flags = set(self.get_flags()) 1697 if 'O' in flags: 1698 message.set_subdir('cur') 1699 if 'F' in flags: 1700 message.add_flag('F') 1701 if 'A' in flags: 1702 message.add_flag('R') 1703 if 'R' in flags: 1704 message.add_flag('S') 1705 if 'D' in flags: 1706 message.add_flag('T') 1707 del message['status'] 1708 del message['x-status'] 1709 maybe_date = ' '.join(self.get_from().split()[-5:]) 1710 try: 1711 message.set_date(calendar.timegm(time.strptime(maybe_date, 1712 '%a %b %d %H:%M:%S %Y'))) 1713 except (ValueError, OverflowError): 1714 pass 1715 elif isinstance(message, _mboxMMDFMessage): 1716 message.set_flags(self.get_flags()) 1717 message.set_from(self.get_from()) 1718 elif isinstance(message, MHMessage): 1719 flags = set(self.get_flags()) 1720 if 'R' not in flags: 1721 message.add_sequence('unseen') 1722 if 'A' in flags: 1723 message.add_sequence('replied') 1724 if 'F' in flags: 1725 message.add_sequence('flagged') 1726 del message['status'] 1727 del message['x-status'] 1728 elif isinstance(message, BabylMessage): 1729 flags = set(self.get_flags()) 1730 if 'R' not in flags: 1731 message.add_label('unseen') 1732 if 'D' in flags: 1733 message.add_label('deleted') 1734 if 'A' in flags: 1735 message.add_label('answered') 1736 del message['status'] 1737 del message['x-status'] 1738 elif isinstance(message, Message): 1739 pass 1740 else: 1741 raise TypeError('Cannot convert to specified type: %s' % 1742 type(message)) 1743 1744 1745class mboxMessage(_mboxMMDFMessage): 1746 """Message with mbox-specific properties.""" 1747 1748 1749class MHMessage(Message): 1750 """Message with MH-specific properties.""" 1751 1752 _type_specific_attributes = ['_sequences'] 1753 1754 def __init__(self, message=None): 1755 """Initialize an MHMessage instance.""" 1756 self._sequences = [] 1757 Message.__init__(self, message) 1758 1759 def get_sequences(self): 1760 """Return a list of sequences that include the message.""" 1761 return self._sequences[:] 1762 1763 def set_sequences(self, sequences): 1764 """Set the list of sequences that include the message.""" 1765 self._sequences = list(sequences) 1766 1767 def add_sequence(self, sequence): 1768 """Add sequence to list of sequences including the message.""" 1769 if isinstance(sequence, str): 1770 if not sequence in self._sequences: 1771 self._sequences.append(sequence) 1772 else: 1773 raise TypeError('sequence type must be str: %s' % type(sequence)) 1774 1775 def remove_sequence(self, sequence): 1776 """Remove sequence from the list of sequences including the message.""" 1777 try: 1778 self._sequences.remove(sequence) 1779 except ValueError: 1780 pass 1781 1782 def _explain_to(self, message): 1783 """Copy MH-specific state to message insofar as possible.""" 1784 if isinstance(message, MaildirMessage): 1785 sequences = set(self.get_sequences()) 1786 if 'unseen' in sequences: 1787 message.set_subdir('cur') 1788 else: 1789 message.set_subdir('cur') 1790 message.add_flag('S') 1791 if 'flagged' in sequences: 1792 message.add_flag('F') 1793 if 'replied' in sequences: 1794 message.add_flag('R') 1795 elif isinstance(message, _mboxMMDFMessage): 1796 sequences = set(self.get_sequences()) 1797 if 'unseen' not in sequences: 1798 message.add_flag('RO') 1799 else: 1800 message.add_flag('O') 1801 if 'flagged' in sequences: 1802 message.add_flag('F') 1803 if 'replied' in sequences: 1804 message.add_flag('A') 1805 elif isinstance(message, MHMessage): 1806 for sequence in self.get_sequences(): 1807 message.add_sequence(sequence) 1808 elif isinstance(message, BabylMessage): 1809 sequences = set(self.get_sequences()) 1810 if 'unseen' in sequences: 1811 message.add_label('unseen') 1812 if 'replied' in sequences: 1813 message.add_label('answered') 1814 elif isinstance(message, Message): 1815 pass 1816 else: 1817 raise TypeError('Cannot convert to specified type: %s' % 1818 type(message)) 1819 1820 1821class BabylMessage(Message): 1822 """Message with Babyl-specific properties.""" 1823 1824 _type_specific_attributes = ['_labels', '_visible'] 1825 1826 def __init__(self, message=None): 1827 """Initialize a BabylMessage instance.""" 1828 self._labels = [] 1829 self._visible = Message() 1830 Message.__init__(self, message) 1831 1832 def get_labels(self): 1833 """Return a list of labels on the message.""" 1834 return self._labels[:] 1835 1836 def set_labels(self, labels): 1837 """Set the list of labels on the message.""" 1838 self._labels = list(labels) 1839 1840 def add_label(self, label): 1841 """Add label to list of labels on the message.""" 1842 if isinstance(label, str): 1843 if label not in self._labels: 1844 self._labels.append(label) 1845 else: 1846 raise TypeError('label must be a string: %s' % type(label)) 1847 1848 def remove_label(self, label): 1849 """Remove label from the list of labels on the message.""" 1850 try: 1851 self._labels.remove(label) 1852 except ValueError: 1853 pass 1854 1855 def get_visible(self): 1856 """Return a Message representation of visible headers.""" 1857 return Message(self._visible) 1858 1859 def set_visible(self, visible): 1860 """Set the Message representation of visible headers.""" 1861 self._visible = Message(visible) 1862 1863 def update_visible(self): 1864 """Update and/or sensibly generate a set of visible headers.""" 1865 for header in self._visible.keys(): 1866 if header in self: 1867 self._visible.replace_header(header, self[header]) 1868 else: 1869 del self._visible[header] 1870 for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'): 1871 if header in self and header not in self._visible: 1872 self._visible[header] = self[header] 1873 1874 def _explain_to(self, message): 1875 """Copy Babyl-specific state to message insofar as possible.""" 1876 if isinstance(message, MaildirMessage): 1877 labels = set(self.get_labels()) 1878 if 'unseen' in labels: 1879 message.set_subdir('cur') 1880 else: 1881 message.set_subdir('cur') 1882 message.add_flag('S') 1883 if 'forwarded' in labels or 'resent' in labels: 1884 message.add_flag('P') 1885 if 'answered' in labels: 1886 message.add_flag('R') 1887 if 'deleted' in labels: 1888 message.add_flag('T') 1889 elif isinstance(message, _mboxMMDFMessage): 1890 labels = set(self.get_labels()) 1891 if 'unseen' not in labels: 1892 message.add_flag('RO') 1893 else: 1894 message.add_flag('O') 1895 if 'deleted' in labels: 1896 message.add_flag('D') 1897 if 'answered' in labels: 1898 message.add_flag('A') 1899 elif isinstance(message, MHMessage): 1900 labels = set(self.get_labels()) 1901 if 'unseen' in labels: 1902 message.add_sequence('unseen') 1903 if 'answered' in labels: 1904 message.add_sequence('replied') 1905 elif isinstance(message, BabylMessage): 1906 message.set_visible(self.get_visible()) 1907 for label in self.get_labels(): 1908 message.add_label(label) 1909 elif isinstance(message, Message): 1910 pass 1911 else: 1912 raise TypeError('Cannot convert to specified type: %s' % 1913 type(message)) 1914 1915 1916class MMDFMessage(_mboxMMDFMessage): 1917 """Message with MMDF-specific properties.""" 1918 1919 1920class _ProxyFile: 1921 """A read-only wrapper of a file.""" 1922 1923 def __init__(self, f, pos=None): 1924 """Initialize a _ProxyFile.""" 1925 self._file = f 1926 if pos is None: 1927 self._pos = f.tell() 1928 else: 1929 self._pos = pos 1930 1931 def read(self, size=None): 1932 """Read bytes.""" 1933 return self._read(size, self._file.read) 1934 1935 def read1(self, size=None): 1936 """Read bytes.""" 1937 return self._read(size, self._file.read1) 1938 1939 def readline(self, size=None): 1940 """Read a line.""" 1941 return self._read(size, self._file.readline) 1942 1943 def readlines(self, sizehint=None): 1944 """Read multiple lines.""" 1945 result = [] 1946 for line in self: 1947 result.append(line) 1948 if sizehint is not None: 1949 sizehint -= len(line) 1950 if sizehint <= 0: 1951 break 1952 return result 1953 1954 def __iter__(self): 1955 """Iterate over lines.""" 1956 while True: 1957 line = self.readline() 1958 if not line: 1959 return 1960 yield line 1961 1962 def tell(self): 1963 """Return the position.""" 1964 return self._pos 1965 1966 def seek(self, offset, whence=0): 1967 """Change position.""" 1968 if whence == 1: 1969 self._file.seek(self._pos) 1970 self._file.seek(offset, whence) 1971 self._pos = self._file.tell() 1972 1973 def close(self): 1974 """Close the file.""" 1975 if hasattr(self, '_file'): 1976 try: 1977 if hasattr(self._file, 'close'): 1978 self._file.close() 1979 finally: 1980 del self._file 1981 1982 def _read(self, size, read_method): 1983 """Read size bytes using read_method.""" 1984 if size is None: 1985 size = -1 1986 self._file.seek(self._pos) 1987 result = read_method(size) 1988 self._pos = self._file.tell() 1989 return result 1990 1991 def __enter__(self): 1992 """Context management protocol support.""" 1993 return self 1994 1995 def __exit__(self, *exc): 1996 self.close() 1997 1998 def readable(self): 1999 return self._file.readable() 2000 2001 def writable(self): 2002 return self._file.writable() 2003 2004 def seekable(self): 2005 return self._file.seekable() 2006 2007 def flush(self): 2008 return self._file.flush() 2009 2010 @property 2011 def closed(self): 2012 if not hasattr(self, '_file'): 2013 return True 2014 if not hasattr(self._file, 'closed'): 2015 return False 2016 return self._file.closed 2017 2018 2019class _PartialFile(_ProxyFile): 2020 """A read-only wrapper of part of a file.""" 2021 2022 def __init__(self, f, start=None, stop=None): 2023 """Initialize a _PartialFile.""" 2024 _ProxyFile.__init__(self, f, start) 2025 self._start = start 2026 self._stop = stop 2027 2028 def tell(self): 2029 """Return the position with respect to start.""" 2030 return _ProxyFile.tell(self) - self._start 2031 2032 def seek(self, offset, whence=0): 2033 """Change position, possibly with respect to start or stop.""" 2034 if whence == 0: 2035 self._pos = self._start 2036 whence = 1 2037 elif whence == 2: 2038 self._pos = self._stop 2039 whence = 1 2040 _ProxyFile.seek(self, offset, whence) 2041 2042 def _read(self, size, read_method): 2043 """Read size bytes using read_method, honoring start and stop.""" 2044 remaining = self._stop - self._pos 2045 if remaining <= 0: 2046 return b'' 2047 if size is None or size < 0 or size > remaining: 2048 size = remaining 2049 return _ProxyFile._read(self, size, read_method) 2050 2051 def close(self): 2052 # do *not* close the underlying file object for partial files, 2053 # since it's global to the mailbox object 2054 if hasattr(self, '_file'): 2055 del self._file 2056 2057 2058def _lock_file(f, dotlock=True): 2059 """Lock file f using lockf and dot locking.""" 2060 dotlock_done = False 2061 try: 2062 if fcntl: 2063 try: 2064 fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB) 2065 except OSError as e: 2066 if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS): 2067 raise ExternalClashError('lockf: lock unavailable: %s' % 2068 f.name) 2069 else: 2070 raise 2071 if dotlock: 2072 try: 2073 pre_lock = _create_temporary(f.name + '.lock') 2074 pre_lock.close() 2075 except OSError as e: 2076 if e.errno in (errno.EACCES, errno.EROFS): 2077 return # Without write access, just skip dotlocking. 2078 else: 2079 raise 2080 try: 2081 try: 2082 os.link(pre_lock.name, f.name + '.lock') 2083 dotlock_done = True 2084 except (AttributeError, PermissionError): 2085 os.rename(pre_lock.name, f.name + '.lock') 2086 dotlock_done = True 2087 else: 2088 os.unlink(pre_lock.name) 2089 except FileExistsError: 2090 os.remove(pre_lock.name) 2091 raise ExternalClashError('dot lock unavailable: %s' % 2092 f.name) 2093 except: 2094 if fcntl: 2095 fcntl.lockf(f, fcntl.LOCK_UN) 2096 if dotlock_done: 2097 os.remove(f.name + '.lock') 2098 raise 2099 2100def _unlock_file(f): 2101 """Unlock file f using lockf and dot locking.""" 2102 if fcntl: 2103 fcntl.lockf(f, fcntl.LOCK_UN) 2104 if os.path.exists(f.name + '.lock'): 2105 os.remove(f.name + '.lock') 2106 2107def _create_carefully(path): 2108 """Create a file if it doesn't exist and open for reading and writing.""" 2109 fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o666) 2110 try: 2111 return open(path, 'rb+') 2112 finally: 2113 os.close(fd) 2114 2115def _create_temporary(path): 2116 """Create a temp file based on path and open for reading and writing.""" 2117 return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()), 2118 socket.gethostname(), 2119 os.getpid())) 2120 2121def _sync_flush(f): 2122 """Ensure changes to file f are physically on disk.""" 2123 f.flush() 2124 if hasattr(os, 'fsync'): 2125 os.fsync(f.fileno()) 2126 2127def _sync_close(f): 2128 """Close file f, ensuring all changes are physically on disk.""" 2129 _sync_flush(f) 2130 f.close() 2131 2132 2133class Error(Exception): 2134 """Raised for module-specific errors.""" 2135 2136class NoSuchMailboxError(Error): 2137 """The specified mailbox does not exist and won't be created.""" 2138 2139class NotEmptyError(Error): 2140 """The specified mailbox is not empty and deletion was requested.""" 2141 2142class ExternalClashError(Error): 2143 """Another process caused an action to fail.""" 2144 2145class FormatError(Error): 2146 """A file appears to have an invalid format.""" 2147