SMTP server patch, take 3 This now includes a tested unit test. Feedback is welcome, as always. /Anders Index: twisted/protocols/basic.py =================================================================== RCS file: /cvs/Twisted/twisted/protocols/basic.py,v retrieving revision 1.25 diff -u -u -r1.25 basic.py --- twisted/protocols/basic.py 18 Oct 2002 06:46:52 -0000 1.25 +++ twisted/protocols/basic.py 31 Oct 2002 20:25:03 -0000 @@ -153,12 +153,15 @@ line, self.__buffer = self.__buffer.split(self.delimiter, 1) except ValueError: if len(self.__buffer) > self.MAX_LENGTH: - self.transport.loseConnection() + line, self.__buffer = self.__buffer, '' + self.lineLengthExceeded(line) return break else: - if len(line) > self.MAX_LENGTH: - self.transport.loseConnection() + linelength = len(line) + if linelength > self.MAX_LENGTH: + line, self.__buffer = self.__buffer, '' + self.lineLengthExceeded(line) return self.lineReceived(line) if self.transport.disconnecting: @@ -200,6 +203,12 @@ """Sends a line to the other end of the connection. """ self.transport.write(line + self.delimiter) + + def lineLengthExceeded(self, line): + """Called when the maximum line length has been reached. + Override if it needs to be dealt with in some special way. + """ + self.transport.loseConnection() class Int32StringReceiver(protocol.Protocol): Index: twisted/protocols/smtp.py =================================================================== RCS file: /cvs/Twisted/twisted/protocols/smtp.py,v retrieving revision 1.26 diff -u -u -r1.26 smtp.py --- twisted/protocols/smtp.py 14 Oct 2002 13:21:06 -0000 1.26 +++ twisted/protocols/smtp.py 31 Oct 2002 20:25:04 -0000 @@ -18,10 +18,10 @@ """ from twisted.protocols import basic -from twisted.internet import protocol, defer +from twisted.internet import protocol, defer, reactor from twisted.python import log -import os, time, string, operator +import os, time, string, operator, re class SMTPError(Exception): pass @@ -49,19 +49,110 @@ self.deferred.errback(arg) self.done = 1 +class AddressError(SMTPError): + "Parse error in address" + pass + +# Character classes for parsing addresses +atom = r"-A-Za-z0-9!#$%&'*+/=?^_`{|}~" + +class Address: + """Parse and hold an RFC 2821 address. + + Source routes are stipped and ignored, UUCP-style bang-paths + and %-style routing are not parsed. + """ + + tstring = re.compile(r'''( # A string of + (?:"[^"]*" # quoted string + |\\.i # backslash-escaped characted + |[''' + string.replace(atom,'#',r'\#') + + r'''] # atom character + )+|.) # or any single character''',re.X) + + def __init__(self, addr): + self.local = '' + self.domain = '' + self.addrstr = addr + + # Tokenize + atl = filter(None,self.tstring.split(addr)) + + local = [] + domain = [] + + while atl: + if atl[0] == '<': + if atl[-1] != '>': + raise AddressError, "Unbalanced <>" + atl = atl[1:-1] + elif atl[0] == '@': + atl = atl[1:] + if not local: + # Source route + while atl and atl[0] != ':': + # remove it + atl = atl[1:] + if not atl: + raise AddressError, "Malformed source route" + atl = atl[1:] # remove : + elif domain: + raise AddressError, "Too many @" + else: + # Now in domain + domain = [''] + elif len(atl[0]) == 1 and atl[0] not in atom + '.': + raise AddressError, "Parse error at " + atl[0] + else: + if not domain: + local.append(atl[0]) + else: + domain.append(atl[0]) + atl = atl[1:] + + self.local = ''.join(local) + self.domain = ''.join(domain) + + dequotebs = re.compile(r'\\(.)') + def dequote(self,addr): + "Remove RFC-2821 quotes from address" + res = [] + + atl = filter(None,self.tstring.split(str(addr))) + + for t in atl: + if t[0] == '"' and t[-1] == '"': + res.append(t[1:-1]) + elif '\\' in t: + res.append(self.dequotebs.sub(r'\1',t)) + else: + res.append(t) + + return ''.join(res) + + def __str__(self): + return '%s%s' % (self.local, self.domain and ("@" + self.domain) or "") + + def __repr__(self): + return "%s.%s(%s)" % (self.__module__, self.__class__.__name__, + repr(str(self))) class User: def __init__(self, destination, helo, protocol, orig): - try: - self.name, self.domain = string.split(destination, '@', 1) - except ValueError: - self.name = destination - self.domain = '' + self.dest = Address(destination) self.helo = helo self.protocol = protocol self.orig = orig + def __getstate__(self): + return { 'dest' : self.dest, + 'helo' : self.helo, + 'protocol' : None, + 'orig' : self.orig } + + def __str__(self): + return str(self.dest) class IMessage: @@ -87,19 +178,39 @@ self.mode = COMMAND self.__from = None self.__helo = None - self.__to = () + self.__to = [] + + def timedout(self): + self.sendCode(421, '%s Timeout. Try talking faster next time!' % + self.host) + self.transport.loseConnection() def connectionMade(self): - self.sendCode(220, 'Spammers beware, your ass is on fire') + self.host = self.factory.domain + if hasattr(self.factory, 'timeout'): + self.timeout = self.factory.timeout + else: + self.timeout = 600 + self.sendCode(220, '%s Spammers beware, your ass is on fire' % + self.host) + if self.timeout: + self.timeoutID = reactor.callLater(self.timeout, self.timedout) def sendCode(self, code, message=''): "Send an SMTP code with a message." self.transport.write('%d %s\r\n' % (code, message)) def lineReceived(self, line): + if self.timeout: + self.timeoutID.cancel() + self.timeoutID = reactor.callLater(self.timeout, self.timedout) + if self.mode is DATA: return self.dataLineReceived(line) - command = string.split(line, None, 1)[0] + if line: + command = string.split(line, None, 1)[0] + else: + command = '' method = getattr(self, 'do_'+string.upper(command), None) if method is None: method = self.do_UNKNOWN @@ -107,59 +218,128 @@ line = line[len(command):] return method(string.strip(line)) + def lineLengthExceeded(self, line): + if self.mode is DATA: + for message in self.__messages: + message.connectionLost() + self.mode = COMMAND + del self.__messages + self.sendCode(500, 'Line too long') + + def rawDataReceived(self, data): + "Throw away rest of long line" + rest = string.split(data, '\r\n', 1) + if len(rest) == 2: + self.setLineMode(self.rest[1]) + def do_UNKNOWN(self, rest): - self.sendCode(502, 'Command not implemented') + self.sendCode(500, 'Command not implemented') def do_HELO(self, rest): - self.__helo = rest - self.sendCode(250, 'Nice to meet you') + peer = self.transport.getPeer()[1] + self.__helo = (rest, peer) + self.sendCode(250, '%s Hello %s, nice to meet you' % (self.host, peer)) def do_QUIT(self, rest): self.sendCode(221, 'See you later') self.transport.loseConnection() + # A string of quoted strings, backslash-escaped character or + # atom characters + '@.,:' + qstring = r'("[^"]*"|\\.|[' + string.replace(atom,'#',r'\#') + r'@.,:])+' + + mail_re = re.compile(r'''\s*FROM:\s*(?P<path><> # Empty <> + |<''' + qstring + r'''> # <addr> + |''' + qstring + r''' # addr + )\s*(\s(?P<opts>.*))? # Optional WS + ESMTP options + $''',re.I|re.X) + rcpt_re = re.compile(r'\s*TO:\s*(?P<path><' + qstring + r'''> # <addr> + |''' + qstring + r''' # addr + )\s*(\s(?P<opts>.*))? # Optional WS + ESMTP options + $''',re.I|re.X) + def do_MAIL(self, rest): - from_ = rest[len("MAIL:<"):-len(">")] - self.validateFrom(self.__helo, from_, self._fromValid, - self._fromInvalid) + if self.__from: + self.sendCode(503,"Only one sender per message, please") + return + # Clear old recipient list + self.__to = [] + m = self.mail_re.match(rest) + if not m: + self.sendCode(501, "Syntax error") + return - def _fromValid(self, from_): + try: + addr = Address(m.group('path')) + except AddressError, e: + self.sendCode(553, str(e)) + return + + self.validateFrom(self.__helo, addr, self._fromValid, + self._fromInvalid) + + def _fromValid(self, from_, code=250, msg='From address accepted'): self.__from = from_ - self.sendCode(250, 'From address accepted') + self.sendCode(code, msg) - def _fromInvalid(self, from_): - self.sendCode(550, 'No mail for you!') + def _fromInvalid(self, from_, code=550, msg='No mail for you!'): + self.sendCode(code,msg) def do_RCPT(self, rest): - to = rest[len("TO:<"):-len(">")] - user = User(to, self.__helo, self, self.__from) - self.validateTo(user, self._toValid, self._toInvalid) + if not self.__from: + self.sendCode(503, "Must have sender before recipient") + return + m = self.rcpt_re.match(rest) + if not m: + self.sendCode(501, "Syntax error") + return - def _toValid(self, to): - self.__to = self.__to + (to,) - self.sendCode(250, 'Address recognized') + try: + user = User(m.group('path'), self.__helo, self, self.__from) + except AddressError, e: + self.sendCode(553, str(e)) + return + + self.validateTo(user, self._toValid, self._toInvalid) - def _toInvalid(self, to): - self.sendCode(550, 'Cannot receive for specified address') + def _toValid(self, to, code=250, msg='Address recognized'): + self.__to.append(to) + self.sendCode(code, msg) + + def _toInvalid(self, to, code=550, + msg='Cannot receive for specified address'): + self.sendCode(code, msg) def do_DATA(self, rest): if self.__from is None or not self.__to: - self.sendCode(550, 'Must have valid receiver and originator') + self.sendCode(503, 'Must have valid receiver and originator') return self.mode = DATA helo, origin, recipients = self.__helo, self.__from, self.__to self.__from = None - self.__to = () + self.__to = [] self.__messages = self.startMessage(recipients) + self.__inheader = self.__inbody = 0 + for message in self.__messages: + message.lineReceived(self.receivedHeader(helo, origin, recipients)) self.sendCode(354, 'Continue') def connectionLost(self, reason): + # self.sendCode(421, 'Dropping connection.') # This does nothing... + # Ideally, if we (rather than the other side) lose the connection, + # we should be able to tell the other side that we are going away. + # RFC-2821 requires that we try. if self.mode is DATA: - for message in self.__messages: - message.connectionLost() + try: + for message in self.__messages: + message.connectionLost() + del self.__messages + except AttributeError: + pass def do_RSET(self, rest): - self.__init__() + self.__from = None + self.__to = [] self.sendCode(250, 'I remember nothing.') def dataLineReceived(self, line): @@ -177,8 +357,24 @@ deferred = message.eomReceived() deferred.addCallback(ndeferred.callback) deferred.addErrback(ndeferred.errback) + del self.__messages return line = line[1:] + + # Add a blank line between the generated Received:-header + # and the message body if the message comes in without any + # headers + if not self.__inheader and not self.__inbody: + if ':' in line: + self.__inheaders = 1 + elif line: + for message in self.__messages: + message.lineReceived('') + self.__inbody = 1 + + if not line: + self.__inbody = 1 + for message in self.__messages: message.lineReceived(line) @@ -188,8 +384,27 @@ def _messageNotHandled(self, _): self.sendCode(550, 'Could not send e-mail') + def rfc822date(self): + timeinfo = time.localtime() + if timeinfo[8]: + # DST + tz = -time.altzone + else: + tz = -time.timezone + + return "%s %+2.2d%2.2d" % ( + time.strftime("%a, %d %b %Y %H:%M:%S", timeinfo), + tz / 3600, (tz / 60) % 60) + # overridable methods: + def receivedHeader(self, helo, origin, recipents): + return "Received: From %s ([%s]) by %s; %s" % ( + helo[0], helo[1], self.host, self.rfc822date()) + def validateFrom(self, helo, origin, success, failure): + if not helo: + failure(origin,503,"Who are you? Say HELO first"); + return success(origin) def validateTo(self, user, success, failure): @@ -265,6 +480,7 @@ def smtpCode_354_data(self, line): self.mailFile = self.getMailData() + self.lastsent = '' self.transport.registerProducer(self, 0) def smtpCode_250_afterData(self, line): @@ -277,11 +493,18 @@ chunk = self.mailFile.read(8192) if not chunk: self.transport.unregisterProducer() - self.sendLine('.') + if self.lastsent != '\n': + line = '\r\n.' + else: + line = '.' + self.sendLine(line) self.state = 'afterData' + return chunk = string.replace(chunk, "\n", "\r\n") + chunk = string.replace(chunk, "\r\n.", "\r\n..") self.transport.write(chunk) + self.lastsent = chunk[-1] def pauseProducing(self): pass Index: twisted/test/test_protocols.py =================================================================== RCS file: /cvs/Twisted/twisted/test/test_protocols.py,v retrieving revision 1.17 diff -u -u -r1.17 test_protocols.py --- twisted/test/test_protocols.py 23 Sep 2002 08:51:29 -0000 1.17 +++ twisted/test/test_protocols.py 31 Oct 2002 20:25:04 -0000 @@ -33,11 +33,13 @@ class LineTester(basic.LineReceiver): delimiter = '\n' + MAX_LENGTH = 64 def connectionMade(self): self.received = [] def lineReceived(self, line): + print self.MAX_LENGTH, len(line) self.received.append(line) if line == '': self.setRawMode() @@ -51,6 +53,10 @@ if self.length == 0: self.setLineMode(rest) + def lineLengthExceeded(self, line): + if len(line) > self.MAX_LENGTH+1: + self.setLineMode(line[self.MAX_LENGTH+1:]) + class WireTestCase(unittest.TestCase): def testEcho(self): @@ -103,13 +109,14 @@ 012345678len 0 foo 5 +1234567890123456789012345678901234567890123456789012345678901234567890 len 1 a''' output = ['len 10', '0123456789', 'len 5', '1234\n', 'len 20', 'foo 123', '0123456789\n012345678', - 'len 0', 'foo 5', '', 'len 1', 'a'] + 'len 0', 'foo 5', '', '67890', 'len 1', 'a'] def testBuffer(self): for packet_size in range(1, 10): @@ -175,3 +182,6 @@ r.dataReceived(s) if not r.brokenPeer: raise AssertionError("connection wasn't closed on illegal netstring %s" % repr(s)) + +if __name__ == '__main__': + unittest.main() Index: twisted/test/test_smtp.py =================================================================== RCS file: /cvs/Twisted/twisted/test/test_smtp.py,v retrieving revision 1.16 diff -u -u -r1.16 test_smtp.py --- twisted/test/test_smtp.py 6 Sep 2002 09:09:42 -0000 1.16 +++ twisted/test/test_smtp.py 31 Oct 2002 20:25:04 -0000 @@ -26,7 +26,7 @@ from twisted.protocols import loopback, smtp from twisted.internet import defer, protocol from twisted.test.test_protocols import StringIOWithoutClosing -import string +import string, re from cStringIO import StringIO class DummyMessage: @@ -195,15 +195,17 @@ self.buffer = [] def lineReceived(self, line): - self.buffer.append(line) + # Throw away the generated Received: header + if not re.match('Received: From foo.com \(\[.*\]\) by foo.com;', line): + self.buffer.append(line) def eomReceived(self): message = string.join(self.buffer, '\n')+'\n' - helo, origin = self.users[0].helo, self.users[0].orig + helo, origin = self.users[0].helo[0], str(self.users[0].orig) recipients = [] for user in self.users: - recipients.append(user.name+'@'+user.domain) - self.protocol.messages.append((helo, origin, recipients, message)) + recipients.append(str(user)) + self.protocol.message = (helo, origin, recipients, message) deferred = defer.Deferred() deferred.callback("saved") return deferred @@ -212,51 +214,84 @@ def connectionMade(self): smtp.SMTP.connectionMade(self) - self.messages = [] + self.message = None def startMessage(self, users): return [DummySMTPMessage(self, users)] - class AnotherSMTPTestCase(unittest.TestCase): - messages = [ ('foo.com', 'moshez@foo.com', ['moshez@bar.com'], '''\ + messages = [ ('foo.com', 'moshez@foo.com', ['moshez@bar.com'], + 'moshez@foo.com', ['moshez@bar.com'], '''\ From: Moshe To: Moshe Hi, how are you? '''), - ('foo.com', 'tttt@rrr.com', ['uuu@ooo', 'yyy@eee'], '''\ + ('foo.com', 'tttt@rrr.com', ['uuu@ooo', 'yyy@eee'], + 'tttt@rrr.com', ['uuu@ooo', 'yyy@eee'], '''\ Subject: pass ..rrrr.. -''') +'''), + ('foo.com', '@this,@is,@ignored:foo@bar.com', + ['@ignore,@this,@too:bar@foo.com'], + 'foo@bar.com', ['bar@foo.com'], '''\ +Subject: apa +To: foo + +123 +. +456 +'''), ] - expected_output = '220 Spammers beware, your ass is on fire\015\012250 Nice to meet you\015\012250 From address accepted\015\012250 Address recognized\015\012354 Continue\015\012250 Delivery in progress\015\012250 From address accepted\015\012250 Address recognized\015\012250 Address recognized\015\012354 Continue\015\012250 Delivery in progress\015\012221 See you later\015\012' - - input = 'HELO foo.com\r\n' - for _, from_, to_, message in messages: - input = input + 'MAIL FROM:<%s>\r\n' % from_ - for item in to_: - input = input + 'RCPT TO:<%s>\r\n' % item - input = input + 'DATA\r\n' - for line in string.split(message, '\n')[:-1]: - if line[:1] == '.': line = '.' + line - input = input + line + '\r\n' - input = input + '.' + '\r\n' - input = input + 'QUIT\r\n' + testdata = [ + ('', '220.*\r\n$', None, None), + ('HELO foo.com\r\n', '250.*\r\n$', None, None), + ('RSET\r\n', '250.*\r\n$', None, None), + ] + for helo_, from_, to_, realfrom, realto, msg in messages: + testdata.append(('MAIL FROM:<%s>\r\n' % from_, '250.*\r\n', + None, None)) + for rcpt in to_: + testdata.append(('RCPT TO:<%s>\r\n' % rcpt, '250.*\r\n', + None, None)) + testdata.append(('DATA\r\n','354.*\r\n', + msg, ('250.*\r\n', + (helo_, realfrom, realto, msg)))) + def testBuffer(self): output = StringIOWithoutClosing() a = DummySMTP() - a.makeConnection(protocol.FileWrapper(output)) - a.dataReceived(self.input) - if a.messages != self.messages: - raise AssertionError(a.messages) - if output.getvalue() != self.expected_output: - raise AssertionError(`output.getvalue()`) + class fooFactory: + domain = 'foo.com' + a.factory = fooFactory() + a.makeConnection(protocol.FileWrapper(output)) + for (send, expect, msg, msgexpect) in self.testdata: + if send: + a.dataReceived(send) + data = output.getvalue() + output.truncate(0) + if not re.match(expect, data): + raise AssertionError, (send, expect, data) + if data[:3] == '354': + for line in msg.splitlines(): + if line and line[0] == '.': + line = '.' + line + a.dataReceived(line + '\r\n') + a.dataReceived('.\r\n') + # Special case for DATA. Now we want a 250, and then + # we compare the messages + data = output.getvalue() + output.truncate() + resp, msgdata = msgexpect + if not re.match(resp, data): + raise AssertionError, (resp, data) + if a.message != msgdata: + raise AssertionError, (msgdata, a.message) testCases = [SMTPTestCase, SMTPClientTestCase, LoopbackSMTPTestCase, AnotherSMTPTestCase]