[Python-checkins] python/nondist/sandbox/mailbox mailbox.py, 1.2, 1.3 test_mailbox.py, 1.2, 1.3 libmailbox.tex, 1.3, 1.4
gregorykjohnson@users.sourceforge.net
gregorykjohnson at users.sourceforge.net
Tue Aug 2 05:49:04 CEST 2005
Update of /cvsroot/python/python/nondist/sandbox/mailbox
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv6902
Modified Files:
mailbox.py test_mailbox.py libmailbox.tex
Log Message:
* Implement mbox (needs more tests).
* Implement mailbox locking.
* Replace flush() on mailboxes with close(). Use close() in tests.
* Fix numerous non-portable format assumptions by using os.linesep.
* Refactor much test code.
Index: mailbox.py
===================================================================
RCS file: /cvsroot/python/python/nondist/sandbox/mailbox/mailbox.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mailbox.py 30 Jul 2005 22:49:08 -0000 1.2
+++ mailbox.py 2 Aug 2005 03:49:01 -0000 1.3
@@ -12,6 +12,10 @@
import email.Message
import email.Generator
import rfc822
+try:
+ import fnctl
+except ImportError:
+ pass
__all__ = [ 'open', 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
@@ -162,8 +166,8 @@
if bad_key:
raise KeyError, "No message with key(s)"
- def flush(self):
- """Write any pending changes to disk."""
+ def close(self):
+ """Close mailbox and write any pending changes to disk."""
raise NotImplementedError, "Method must be implemented by subclass"
def _dump_message(self, message, target):
@@ -301,8 +305,8 @@
self._refresh()
return len(self._toc)
- def flush(self):
- """Write any pending changes to disk."""
+ def close(self):
+ """Close mailbox and write any pending changes to disk."""
return # Maildir changes are always written immediately.
def list_folders(self):
@@ -396,6 +400,181 @@
raise KeyError, "No message with key '%s'" % key
+class mbox(Mailbox):
+ """A classic mbox mailbox."""
+
+ def __init__(self, path, factory=None):
+ """Initialize an mbox instance."""
+ Mailbox.__init__(self, path, factory)
+ try:
+ f = file(self._path, 'r+')
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ f = file(self._path, 'w+')
+ elif e.errno == errno.EACCES:
+ f = file(self._path, 'r')
+ else:
+ raise
+ try:
+ _lock_file(f)
+ except:
+ f.close()
+ raise
+ self._file = f
+ self._toc = None
+ self._next_key = 0
+
+ def add(self, message):
+ """Add message and return assigned key."""
+ self._lookup()
+ self._toc[self._next_key] = self._append_message(message)
+ self._next_key += 1
+ return self._next_key - 1
+
+ def remove(self, key):
+ """Remove the keyed message; raise KeyError if it doesn't exist."""
+ del self._toc[key]
+
+ def __setitem__(self, key, message):
+ """Replace the keyed message; raise KeyError if it doesn't exist."""
+ self._lookup(key)
+ start, stop = self._append_message(message)
+ self._toc[key] = (start, stop)
+
+ def get_message(self, key):
+ start, stop = self._lookup(key)
+ self._file.seek(start)
+ from_line = self._file.readline()
+ msg = mboxMessage(self._file.read(stop - self._file.tell()))
+ msg.set_from(from_line[5:-1])
+ return msg
+
+ def get_string(self, key, from_=False):
+ """Return a string representation or raise a KeyError."""
+ start, stop = self._lookup(key)
+ self._file.seek(start)
+ if not from_:
+ self._file.readline()
+ return self._file.read(stop - self._file.tell())
+
+ def get_file(self, key, from_=False):
+ """Return a file-like representation or raise a KeyError."""
+ start, stop = self._lookup(key)
+ self._file.seek(start)
+ if not from_:
+ self._file.readline()
+ return _PartialFile(self._file, self._file.tell(), stop)
+
+ def iterkeys(self):
+ """Return an iterator over keys."""
+ self._lookup()
+ for key in self._toc.keys():
+ yield key
+
+ def has_key(self, key):
+ """Return True if the keyed message exists, False otherwise."""
+ self._lookup()
+ return key in self._toc
+
+ def __len__(self):
+ """Return a count of messages in the mailbox."""
+ self._lookup()
+ return len(self._toc)
+
+ def close(self):
+ """Close mailbox and write any pending changes to disk."""
+ self._lookup()
+ f = file(self._path + '.tmp', 'w')
+ try:
+ new_toc = {}
+ for key in sorted(self._toc.keys()):
+ start, stop = self._toc[key]
+ self._file.seek(start)
+ if f.tell() != 0:
+ f.write(os.linesep)
+ new_start = f.tell()
+ while True:
+ buffer = self._file.read(min(4096,
+ stop - self._file.tell()))
+ if buffer == '':
+ break
+ f.write(buffer)
+ new_toc[key] = (new_start, f.tell())
+ finally:
+ f.close()
+ try:
+ os.rename(self._path + '.tmp', self._path)
+ except oserror, e:
+ if e.errno == errno.eexist:
+ # xxx: is this is the exception Windows raises?
+ os.remove(self._path)
+ os.rename(self._path + '.tmp', self._path)
+ else:
+ raise
+ self._file.close()
+ if os.path.exists(self._path + '.lock'):
+ os.remove(self._path + '.lock')
+
+ def _generate_toc(self):
+ """Generate key-to-(start, stop) table of contents."""
+ starts, stops = [], []
+ self._file.seek(0)
+ prev_line = ''
+ while True:
+ pos = self._file.tell()
+ line = self._file.readline()
+ if line[:5] == 'From ':
+ starts.append(pos)
+ # The preceeding newline is part of the separator, e.g.,
+ # "\nFrom .*\n", not part of the previous message. Ignore it.
+ if prev_line != '':
+ stops.append(pos - len(os.linesep))
+ elif line == '':
+ stops.append(pos)
+ break
+ prev_line = line
+ self._toc = dict(enumerate(zip(starts, stops)))
+ self._next_key = len(self._toc)
+
+ def _lookup(self, key=None):
+ """Return (start, stop) for given key, or raise a KeyError."""
+ if self._toc is None:
+ self._generate_toc()
+ if key is not None:
+ try:
+ return self._toc[key]
+ except IndexError:
+ raise KeyError, "No message with key '%s'" % key
+
+ def _append_message(self, message):
+ """Append message to mailbox and return (start, stop) offset."""
+ from_line = None
+ if isinstance(message, str) and message[:5] == 'From ':
+ newline = message.find(os.linesep)
+ if newline != -1:
+ from_line = message[:newline]
+ message = message[newline + len(os.linesep):]
+ else:
+ from_line = message
+ message = ''
+ elif isinstance(message, _mboxMMDFMessage):
+ from_line = 'From ' + message.get_from()
+ elif isinstance(message, email.Message.Message):
+ from_line = message.get_unixfrom()
+ if from_line is None:
+ from_line = 'From MAILER-DAEMON %s' % \
+ time.strftime('%a %b %d %H:%M:%S %Y', time.gmtime())
+ self._file.seek(0, 2)
+ if self._file.tell() == 0:
+ start = self._file.tell()
+ self._file.write('%s%s' % (from_line, os.linesep))
+ else:
+ start = self._file.tell() + 1
+ self._file.write('%s%s%s' % (os.linesep, from_line, os.linesep))
+ self._dump_message(message, self._file)
+ return (start, self._file.tell())
+
+
class Message(email.Message.Message):
"""Message with mailbox-format-specific properties."""
@@ -533,6 +712,7 @@
elif 'Return-Path' in message:
# XXX: generate "From " line from Return-Path: and Received:
pass
+
Message.__init__(self, message)
def get_from(self):
@@ -876,3 +1056,53 @@
class Error(Exception):
"""Raised for module-specific errors."""
+
+
+def _lock_file(f):
+ """Use various means to lock f (f.name should be absolute)."""
+ if 'fcntl' in globals():
+ try:
+ fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except IOError, e:
+ if e.errno == errno.EAGAIN:
+ raise Error, 'Failed to acquire exclusive lock ' \
+ 'using lockf: %s' % f.name
+ elif e.errno == errno.EBADF:
+ try:
+ fcntl.lockf(f, fcntl.LOCK_SH | fntl.LOCK_NB)
+ except IOError, e:
+ if e.errno == errno.EAGAIN:
+ raise Error, 'Failed to acquire shared lock ' \
+ 'using lockf: %s' % f.name
+ else:
+ raise
+ else:
+ raise
+ try:
+ fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except IOError, e:
+ if e.errno == errno.EWOULDBLOCK:
+ raise Error, 'Failed to aquire exclusive lock ' \
+ 'using flock: %s' % f.name
+ else:
+ raise
+ tmp_name = f.name + '.%s.%s' % (socket.gethostname(), os.getpid())
+ try:
+ file(tmp_name, 'w').close()
+ except IOError, e:
+ if e.errno == errno.EACCESS:
+ pass
+ else:
+ raise
+ else:
+ try:
+ if hasattr(os, 'link'):
+ os.link(tmp_name, f.name + '.lock')
+ os.unlink(tmp_name)
+ else:
+ os.rename(tmp_name, f.name + '.lock')
+ except OSError, e:
+ if e.errno == errno.EEXIST:
+ raise Error, 'Failed to acquire dot lock: %s' % f.name
+ else:
+ raise
Index: test_mailbox.py
===================================================================
RCS file: /cvsroot/python/python/nondist/sandbox/mailbox/test_mailbox.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- test_mailbox.py 30 Jul 2005 22:49:08 -0000 1.2
+++ test_mailbox.py 2 Aug 2005 03:49:01 -0000 1.3
@@ -10,6 +10,7 @@
from test import test_support
import unittest
import mailbox
+import glob
class TestBase(unittest.TestCase):
@@ -44,191 +45,172 @@
class TestMailbox(TestBase):
_factory = None # Overridden by subclasses to reuse tests
[...1009 lines suppressed...]
+--
+Gregory K. Johnson
+""",
+"""H4sICM2D1UIAA3RleHQAC8nILFYAokSFktSKEoW0zJxUPa7wzJIMhZLyfIWczLzUYj0uAHTs
+3FYlAAAA
+""")
def test_main():
- test_support.run_unittest(TestMaildir, TestMessage, TestMaildirMessage,
- TestMboxMessage, TestMHMessage, TestBabylMessage,
- TestMMDFMessage, TestMessageConversion,
- TestProxyFile, TestPartialFile)
+ tests = (TestMaildir, TestMbox, TestMessage, TestMaildirMessage,
+ TestMboxMessage, TestMHMessage, TestBabylMessage, TestMMDFMessage,
+ TestMessageConversion, TestProxyFile, TestPartialFile)
+ test_support.run_unittest(*tests)
if __name__ == '__main__':
Index: libmailbox.tex
===================================================================
RCS file: /cvsroot/python/python/nondist/sandbox/mailbox/libmailbox.tex,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- libmailbox.tex 30 Jul 2005 22:49:08 -0000 1.3
+++ libmailbox.tex 2 Aug 2005 03:49:01 -0000 1.4
@@ -242,10 +242,10 @@
are not supported.}
\end{methoddesc}
-\begin{methoddesc}{flush}{}
-Write any pending changes to the filesystem. For some \class{Mailbox}
-subclasses, this is done automatically and calling \method{flush()} has no
-effect. More specific documentation is provided by each subclass.
+\begin{methoddesc}{close}{}
+Close the mailbox and write any pending changes to the filesystem. For some
+\class{Mailbox} subclasses, changes are written immediately even without
+calling this method.
\end{methoddesc}
@@ -316,9 +316,9 @@
these methods to manipulate the same mailbox simultaneously.}
\end{methoddesc}
-\begin{methoddesc}{flush}{}
-All changes to Maildir mailboxes are immediately applied. This method does
-nothing.
+\begin{methoddesc}{close}{}
+All changes to Maildir mailboxes are immediately applied even without calling
+this method.
\end{methoddesc}
\begin{methoddesc}{get_file}{key}
@@ -368,6 +368,9 @@
\begin{seealso}
\seelink{http://www.qmail.org/man/man5/mbox.html}{mbox man page from
qmail}{A specification of the format and its variations.}
+ \seelink{http://www.tin.org/bin/man.cgi?section=5\&topic=mbox}{mbox man
+ page from tin}{Another specification of the format, with details on
+ locking.}
\seelink{http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html}
{Configuring Netscape Mail on \UNIX{}: Why The Content-Length Format is
Bad}{An argument for using the original mbox format rather than a
@@ -451,8 +454,9 @@
XXX
\end{methoddesc}
-\begin{methoddesc}{flush}{}
-All changes to MH mailboxes are immediately applied. This method does nothing.
+\begin{methoddesc}{close}{}
+All changes to MH mailboxes are immediately applied even without calling this
+method.
\end{methoddesc}
\begin{classdesc}{MH}{path\optional{, factory}}
More information about the Python-checkins
mailing list