IMAP2IMAP Synchronization -- iis [Was: Re: iis: comparsion of two lists]

Matej Cepl cepl.m at neu.edu
Sat Apr 27 23:46:39 EDT 2002


On Sat, Apr 27, 2002 at 06:51:21AM +0000, Alex Martelli wrote:
> It would, but the first assignment is useless and the approach
> needlessly complicated IMHO.
> 
> erase = [ m for m in proc_old if m not in loc_old ]

Concerning the first assignement, rather be (slightly) slower 
than sorry and your solution seems to work only with Python 2.*. 
I am still on 1.5.2. Do I have to _really_ download multimegabyte 
beast just to get above command?

Concerning the rest of the script, I have yesterday finalized it
and tried to send it to this NG. Unfortunately, the message
somewhere got screwed up (at least on DejaNews, it is not on my
newserver at all; I have no idea, what's up). So, sorry for
reposting (now the script is in the body of message below the
sig), but I would really like to ask people here on the group for 
review (it is my first non-trivial script in Python and I am 
afraid to trust him with my real email traffic).

Therefore, could I ask somebody for review of the below script, 
especially concerning the safety of messages and speed, please? 


Thank you very much.

Matej


-- 
Matej Cepl, cepl.m at neu.edu
138 Highland Ave. #10, Somerville, Ma 02143, (617) 623-1488
 
There's a long-standing bug relating to the x86 architecture that
allows you to install Windows.
	-- Matthew D. Fuller

---------------------------------------------------------------

#!/usr/bin/env python
# $Id: iis.py,v 0.7 2002/04/27 01:32:52 matej Exp $
"""
IMAP-IMAP Synchronization - iis

(always use lower case to avoid terrible misunderstanding :-)

Copyright (c) 2002 Matej Cepl <matej at ceplovi.cz>
   
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.

"""
import ConfigParser
import imaplib
from sys import stderr, exit
from string import split, join
from os.path import expanduser
from time import time, gmtime

# constants
# return errorlevels
DOWNERR = 1
UPERR = 2
SELECTERR = 3
LISTERR = 4
LOGINERR = 5
CONFIGERR = 10

# other constants
IMAPPORT = "143"
LOCALSTR = 'local'

def chatch_error(errmsg,stat_num):
	stderr.write("IMAP Error: %s !\n" % errmsg)
	exit(stat_num)

class Conifg(ConfigParser.ConfigParser):
	"""
	Object containing configuration.
	
	It reads configuration file (in WIN .ini syntax) like this
	
	[localuser1]
	local = password,last_sync
	server1.com = username,password,last_sync (in IMAP shape)
	
	[localuser2]
	local = password,last_sync
	server2.com = username,password,last_sync (in IMAP shape)
	"""
	def __init__(self,fname=".iisrc"):
		ConfigParser.ConfigParser.__ini__(self)
		self.filename = expanduser("~/"+fname)
		self.options = {}
		try:
			self.read(self.filename)
		except ParsingError,e:
			chatch_error(e,CONFIGERR)
		for sect in self.sections():
			tmp_dict = {}
			for opt in self.options(sect)[:-1]:
				tmp_dict[opt] = split(self.get(sect,opt),',')
			self.options[sect] = tmp_dict

	def close(self)
		try:
			f = open(self.filename,'w')
			for sect in self.options.keys():
				f.write('[' + sect + ']\n')
				for opt in self.options[sect].keys():
					f.write(opt + '=')
					f.write(self.optionsp[sect][opt] + '\n')
			f.close()

def comp_lists(processed,local):
	"""
	Compare two lists of messages and select for download/delete.

	INPUT: two objects Server -- one to be processed and other
	OUTPUT: erase, download -- list of messages to delete from
			  the processed server, and list of messages to be
			  downloaded on other server
	What is newer than self.last and it isn't in other, should be
	downloaded from processed, and what is older and isn't on
	other, should be deleted. The same other way around.

	OK, I admitt, that code below is ugly, but it should be as
	fast as possible -- which is what we need.
	"""
	proc_old = map(lambda x: processed.old_msg[x][0],processed.old_msg.keys())
	proc_new = map(lambda x: processed.new_msg[x][0],processed.new_msg.keys())
	loc_old = map(lambda x: local.old_msg[x][0],local.old_msg.keys())
	loc_new = map(lambda x: local.new_msg[x][0],local.new_msg.keys())
	
	erase = e = download = d = ()
	e = filter(lambda m: return not(m in loc_old), proc_old)
	d = filter(lambda m: return not(m in loc_new), proc_new)

	erase = filter(lambda x: processed.old_msg[x][0] in e, \
	processed.old_msg.keys())
	download = filter(lambda x: processed.new_msg[x][0] in d, \
	processed.new_msg.keys())

	return erase,download

def sync(loc,rem):
	"""
	Synchronize two folders on servers.

	## INPUT:
	## OUTPUT:
	## description
	"""
	# delete old on the remote and download new messages
	delete,down = comp_lists(rem,loc)
	try:
		ok, answer = rem.store(join(delete), 'FLAGS', '(\Deleted)')
	except Exception,e:
		stderr.write("Remote messages were not deleted!\n")
		# rather let it be -- it is better not to have
		# something deleted, than to have deleted more than
		# we want
	for msg in down:
		try:
			ok, stuff = rem.fetch(msg,'(RFC822)')
			tmp_msg = rem.new_msg[msg]
			ok, answer = loc.append(loc.curr_folder,tmp_msg[2],tmp_msg[1],stuff)
		except Exception,e:
			catch_error(e,DOWNERR)
	
	# delete old on local and upload new ones
	delete,up = comp_lists(loc,rem)
	try:
		ok, answer = loc.store(join(delete), 'FLAGS', '(\Deleted)')
	except Exception,e:
		stderr.write("Local messages were not deleted!\n")
	for msg in up:
		try:
			stuff = loc.fetch(msg,'(RFC822)')
			tmp_msg = loc.old_msg[msg]
			ok, answer = rem.append(rem.curr_folder,tmp_msg[2],tmp_msg[1],stuff)
		except Exception,e:
			catch_error(e,UPERR)

class Server(imaplib.IMAP4):
	"""
	Main class for managing IMAP servers and folders.
	
	## some description
	"""
	def __init__(self,server='',port=IMAPPORT,user,password,last_sync):
		"""
		Connect to the server and login.
		"""
		self.host=server
		self.port=port
		self.user=user
		self.passwd=password
		self.last=last_sync
		self.new_msg={}
		self.old_msg={}
		self.curr_folder=''
		try:
			imaplib.IMAP4.__init__(self,self.host,self.port)
			self.login(self.user,self.passwd)
		except Exception,e:
			catch_error(e,UPERR)

	def message_lists(self,folder="")
		"""
		Fetch list of messages, that are older than self.last and
		newer than that.
		INPUT: correctly opened IMAP server, name of the folder,
		correctly set self.last (in IMAP representation).
		OUTPUT: sets self.new_msg and self.old_msg.
		"""
		if folder=="":
			folder="INBOX"
		try:
			self.select(folder)
		except Exception,e:
			catch_error(e,SELECTERR)
		self.curr_folder=folder
		try:
			ok, stuff = self.search(None,'( UNDELETED SINCE ' + self.last + ' )')
			if ok == 'OK':
				for msg in split(stuff):
					ok, header = self.fetch(msg,'(UID)')
					if ok == 'OK':
						uid = split(header[0])[2][:-1]
					else
						raise ValueError, 'The field UID is not fetched!',
					ok, header = self.fetch(msg,'(INTERNALDATE)')
					if ok == 'OK':
						temptup = split(header[0])[2:]
						tstr = temptup[0][1:] + temptup[1] + temptup[2][:-2]
						date = imaplib.Internaldate2tuple(tstr)
					else
						raise ValueError, 'The field INTERNALDATE is not fetched!',
					ok, header = self.fetch(msg,'(FLAGS)')
					if ok == 'OK':
						flags = split(header[0])[2]
					else
						raise ValueError, 'The FLAGS are not fetched!',
					self.new_msg[msg]=uid,date,flags
			ok, stuff = self.search(None,'( UNDELETED BEFORE ' + self.last + ' )')
			if ok == 'OK':
				for msg in split(stuff):
					ok, header = self.fetch(msg,'(UID)')
					if ok == 'OK':
						uid = split(header[0])[2][:-1]
					ok, header = self.fetch(msg,'(INTERNALDATE)')
					if ok == 'OK':
						temptup = split(header[0])[2:]
						tstr = temptup[0][1:] + temptup[1] + temptup[2][:-2]
						date = imaplib.Internaldate2tuple(tstr)
					self.old_msg[msg]=uid,date
		except Exception,e:
			catch_error(e,LISTERR)
		self.close()
	def complete(self):
		"""
		leave server
		"""
		self.logout()

if __name__ == '__main__':
	## General outline
	conf = Config()
	for usr in conf.options.keys():
		opts = conf.options[usr][LOCALSTR]
		local = Server(user=usr,password=opts[0],last_sync=opts[1])
		local.message_lists()
		for svr in conf.options[usr].keys():
			if svr != LOCALSTR:
				ropts = conf.options[usr][svr]
				remote = Server(user=ropts[0],password=ropts[1],last_sync=ropts[2])
				remote.message_lists()
				sync(local,remote)
				sync_time = remote.Time2Internaldate(gmtime(time())
				conf.options[usr][svr][2] = sync_time
		conf.options[usr][LOCALSTR][1] = sync_time
	conf.close()

# vim: set ts=3:



More information about the Python-list mailing list