Update of /cvsroot/python/python/nondist/sandbox/mailbox In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv6902 Modified Files: mailbox.py test_mailbox.py libmailbox.tex Log Message: * Implement mbox (needs more tests). * Implement mailbox locking. * Replace flush() on mailboxes with close(). Use close() in tests. * Fix numerous non-portable format assumptions by using os.linesep. * Refactor much test code. Index: mailbox.py =================================================================== RCS file: /cvsroot/python/python/nondist/sandbox/mailbox/mailbox.py,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- mailbox.py 30 Jul 2005 22:49:08 -0000 1.2 +++ mailbox.py 2 Aug 2005 03:49:01 -0000 1.3 @@ -12,6 +12,10 @@ import email.Message import email.Generator import rfc822 +try: + import fnctl +except ImportError: + pass __all__ = [ 'open', 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF', 'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage', @@ -162,8 +166,8 @@ if bad_key: raise KeyError, "No message with key(s)" - def flush(self): - """Write any pending changes to disk.""" + def close(self): + """Close mailbox and write any pending changes to disk.""" raise NotImplementedError, "Method must be implemented by subclass" def _dump_message(self, message, target): @@ -301,8 +305,8 @@ self._refresh() return len(self._toc) - def flush(self): - """Write any pending changes to disk.""" + def close(self): + """Close mailbox and write any pending changes to disk.""" return # Maildir changes are always written immediately. def list_folders(self): @@ -396,6 +400,181 @@ raise KeyError, "No message with key '%s'" % key +class mbox(Mailbox): + """A classic mbox mailbox.""" + + def __init__(self, path, factory=None): + """Initialize an mbox instance.""" + Mailbox.__init__(self, path, factory) + try: + f = file(self._path, 'r+') + except IOError, e: + if e.errno == errno.ENOENT: + f = file(self._path, 'w+') + elif e.errno == errno.EACCES: + f = file(self._path, 'r') + else: + raise + try: + _lock_file(f) + except: + f.close() + raise + self._file = f + self._toc = None + self._next_key = 0 + + def add(self, message): + """Add message and return assigned key.""" + self._lookup() + self._toc[self._next_key] = self._append_message(message) + self._next_key += 1 + return self._next_key - 1 + + def remove(self, key): + """Remove the keyed message; raise KeyError if it doesn't exist.""" + del self._toc[key] + + def __setitem__(self, key, message): + """Replace the keyed message; raise KeyError if it doesn't exist.""" + self._lookup(key) + start, stop = self._append_message(message) + self._toc[key] = (start, stop) + + def get_message(self, key): + start, stop = self._lookup(key) + self._file.seek(start) + from_line = self._file.readline() + msg = mboxMessage(self._file.read(stop - self._file.tell())) + msg.set_from(from_line[5:-1]) + return msg + + def get_string(self, key, from_=False): + """Return a string representation or raise a KeyError.""" + start, stop = self._lookup(key) + self._file.seek(start) + if not from_: + self._file.readline() + return self._file.read(stop - self._file.tell()) + + def get_file(self, key, from_=False): + """Return a file-like representation or raise a KeyError.""" + start, stop = self._lookup(key) + self._file.seek(start) + if not from_: + self._file.readline() + return _PartialFile(self._file, self._file.tell(), stop) + + def iterkeys(self): + """Return an iterator over keys.""" + self._lookup() + for key in self._toc.keys(): + yield key + + def has_key(self, key): + """Return True if the keyed message exists, False otherwise.""" + self._lookup() + return key in self._toc + + def __len__(self): + """Return a count of messages in the mailbox.""" + self._lookup() + return len(self._toc) + + def close(self): + """Close mailbox and write any pending changes to disk.""" + self._lookup() + f = file(self._path + '.tmp', 'w') + try: + new_toc = {} + for key in sorted(self._toc.keys()): + start, stop = self._toc[key] + self._file.seek(start) + if f.tell() != 0: + f.write(os.linesep) + new_start = f.tell() + while True: + buffer = self._file.read(min(4096, + stop - self._file.tell())) + if buffer == '': + break + f.write(buffer) + new_toc[key] = (new_start, f.tell()) + finally: + f.close() + try: + os.rename(self._path + '.tmp', self._path) + except oserror, e: + if e.errno == errno.eexist: + # xxx: is this is the exception Windows raises? + os.remove(self._path) + os.rename(self._path + '.tmp', self._path) + else: + raise + self._file.close() + if os.path.exists(self._path + '.lock'): + os.remove(self._path + '.lock') + + def _generate_toc(self): + """Generate key-to-(start, stop) table of contents.""" + starts, stops = [], [] + self._file.seek(0) + prev_line = '' + while True: + pos = self._file.tell() + line = self._file.readline() + if line[:5] == 'From ': + starts.append(pos) + # The preceeding newline is part of the separator, e.g., + # "\nFrom .*\n", not part of the previous message. Ignore it. + if prev_line != '': + stops.append(pos - len(os.linesep)) + elif line == '': + stops.append(pos) + break + prev_line = line + self._toc = dict(enumerate(zip(starts, stops))) + self._next_key = len(self._toc) + + def _lookup(self, key=None): + """Return (start, stop) for given key, or raise a KeyError.""" + if self._toc is None: + self._generate_toc() + if key is not None: + try: + return self._toc[key] + except IndexError: + raise KeyError, "No message with key '%s'" % key + + def _append_message(self, message): + """Append message to mailbox and return (start, stop) offset.""" + from_line = None + if isinstance(message, str) and message[:5] == 'From ': + newline = message.find(os.linesep) + if newline != -1: + from_line = message[:newline] + message = message[newline + len(os.linesep):] + else: + from_line = message + message = '' + elif isinstance(message, _mboxMMDFMessage): + from_line = 'From ' + message.get_from() + elif isinstance(message, email.Message.Message): + from_line = message.get_unixfrom() + if from_line is None: + from_line = 'From MAILER-DAEMON %s' % \ + time.strftime('%a %b %d %H:%M:%S %Y', time.gmtime()) + self._file.seek(0, 2) + if self._file.tell() == 0: + start = self._file.tell() + self._file.write('%s%s' % (from_line, os.linesep)) + else: + start = self._file.tell() + 1 + self._file.write('%s%s%s' % (os.linesep, from_line, os.linesep)) + self._dump_message(message, self._file) + return (start, self._file.tell()) + + class Message(email.Message.Message): """Message with mailbox-format-specific properties.""" @@ -533,6 +712,7 @@ elif 'Return-Path' in message: # XXX: generate "From " line from Return-Path: and Received: pass + Message.__init__(self, message) def get_from(self): @@ -876,3 +1056,53 @@ class Error(Exception): """Raised for module-specific errors.""" + + +def _lock_file(f): + """Use various means to lock f (f.name should be absolute).""" + if 'fcntl' in globals(): + try: + fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError, e: + if e.errno == errno.EAGAIN: + raise Error, 'Failed to acquire exclusive lock ' \ + 'using lockf: %s' % f.name + elif e.errno == errno.EBADF: + try: + fcntl.lockf(f, fcntl.LOCK_SH | fntl.LOCK_NB) + except IOError, e: + if e.errno == errno.EAGAIN: + raise Error, 'Failed to acquire shared lock ' \ + 'using lockf: %s' % f.name + else: + raise + else: + raise + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError, e: + if e.errno == errno.EWOULDBLOCK: + raise Error, 'Failed to aquire exclusive lock ' \ + 'using flock: %s' % f.name + else: + raise + tmp_name = f.name + '.%s.%s' % (socket.gethostname(), os.getpid()) + try: + file(tmp_name, 'w').close() + except IOError, e: + if e.errno == errno.EACCESS: + pass + else: + raise + else: + try: + if hasattr(os, 'link'): + os.link(tmp_name, f.name + '.lock') + os.unlink(tmp_name) + else: + os.rename(tmp_name, f.name + '.lock') + except OSError, e: + if e.errno == errno.EEXIST: + raise Error, 'Failed to acquire dot lock: %s' % f.name + else: + raise Index: test_mailbox.py =================================================================== RCS file: /cvsroot/python/python/nondist/sandbox/mailbox/test_mailbox.py,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- test_mailbox.py 30 Jul 2005 22:49:08 -0000 1.2 +++ test_mailbox.py 2 Aug 2005 03:49:01 -0000 1.3 @@ -10,6 +10,7 @@ from test import test_support import unittest import mailbox +import glob class TestBase(unittest.TestCase): @@ -44,191 +45,172 @@ class TestMailbox(TestBase): _factory = None # Overridden by subclasses to reuse tests [...1009 lines suppressed...] +-- +Gregory K. Johnson +""", +"""H4sICM2D1UIAA3RleHQAC8nILFYAokSFktSKEoW0zJxUPa7wzJIMhZLyfIWczLzUYj0uAHTs +3FYlAAAA +""") def test_main(): - test_support.run_unittest(TestMaildir, TestMessage, TestMaildirMessage, - TestMboxMessage, TestMHMessage, TestBabylMessage, - TestMMDFMessage, TestMessageConversion, - TestProxyFile, TestPartialFile) + tests = (TestMaildir, TestMbox, TestMessage, TestMaildirMessage, + TestMboxMessage, TestMHMessage, TestBabylMessage, TestMMDFMessage, + TestMessageConversion, TestProxyFile, TestPartialFile) + test_support.run_unittest(*tests) if __name__ == '__main__': Index: libmailbox.tex =================================================================== RCS file: /cvsroot/python/python/nondist/sandbox/mailbox/libmailbox.tex,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- libmailbox.tex 30 Jul 2005 22:49:08 -0000 1.3 +++ libmailbox.tex 2 Aug 2005 03:49:01 -0000 1.4 @@ -242,10 +242,10 @@ are not supported.} \end{methoddesc} -\begin{methoddesc}{flush}{} -Write any pending changes to the filesystem. For some \class{Mailbox} -subclasses, this is done automatically and calling \method{flush()} has no -effect. More specific documentation is provided by each subclass. +\begin{methoddesc}{close}{} +Close the mailbox and write any pending changes to the filesystem. For some +\class{Mailbox} subclasses, changes are written immediately even without +calling this method. \end{methoddesc} @@ -316,9 +316,9 @@ these methods to manipulate the same mailbox simultaneously.} \end{methoddesc} -\begin{methoddesc}{flush}{} -All changes to Maildir mailboxes are immediately applied. This method does -nothing. +\begin{methoddesc}{close}{} +All changes to Maildir mailboxes are immediately applied even without calling +this method. \end{methoddesc} \begin{methoddesc}{get_file}{key} @@ -368,6 +368,9 @@ \begin{seealso} \seelink{http://www.qmail.org/man/man5/mbox.html}{mbox man page from qmail}{A specification of the format and its variations.} + \seelink{http://www.tin.org/bin/man.cgi?section=5\&topic=mbox}{mbox man + page from tin}{Another specification of the format, with details on + locking.} \seelink{http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html} {Configuring Netscape Mail on \UNIX{}: Why The Content-Length Format is Bad}{An argument for using the original mbox format rather than a @@ -451,8 +454,9 @@ XXX \end{methoddesc} -\begin{methoddesc}{flush}{} -All changes to MH mailboxes are immediately applied. This method does nothing. +\begin{methoddesc}{close}{} +All changes to MH mailboxes are immediately applied even without calling this +method. \end{methoddesc} \begin{classdesc}{MH}{path\optional{, factory}}