[Mailman-Developers] MHonArc integration

Anand Kumria wildfire@progsoc.uts.edu.au, mailman-developers@python.org
Fri, 4 May 2001 02:41:04 +1000

Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline

Hi folks,

I've just setup another mailman site and once again I wanted
to integrate it with MHonArc. Previously I'd been using shell
script that played games with symlinks and the the archive
files behind Mailman's back.

I wanted something simpler to use, so after much judicious
cutting and pasting of other code I've put together MHonArc.py
which you can use as an almost compatible replacement for

There are a number of things left to do:
	- generate HTML index pages properly
	- store text versions of the archive along with each dir.
	- cleanup the code and document it

This is my first real experience with Python, so no laughing (too
loud). I think I've also noticed a bug where Mailman uses the
udnerlying Pythong rfc822 object when Mailman's is more useful --
see comments in add_article.

I've also simplified the date handling (the current archiving
stuff hyperarch/pipermail convert a to/from unix epoch lots of

Doing this also made clear that a nicer interface in Archive.py
could be done which should handle, imo:
	- message date => volume (and thus path) conversions
	- make the archive date format local to each list
	e.g. I'd like to have 2001/05/01 instead of 20010501
	- generation of HTML index pages
	- make the HTML template editable via the admin interface
	- expose some way to allow list-admins to configure
	any options the archiver might allow (MHonArc allows a lot)

Feedback on all of this welcome.


Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="MHonArc.py"

# Archive interface for MHonArc and Mailman

This attempts to simulate the HpyerArch python script but
using instead MHonArc. By doing this I hope to avoid the need
to have 'list rotation' and 'index generation' as external jobs

1. setup archive directory
	- set ARCHIVE_PERIOD to appropriate value
2. convert mbox into individual messages
	- figure out location to put message
		- convert message date to appropriate directory
		- create directory if needed
3. add message to archive
	- add message to text file in 
	- interperlate -output and -resource  into appropriate directory
	- execute MHonArc
4. create HTML index pages

import sys
import os
import time
import string
from Mailman import mm_cfg
from Mailman import Utils
from Mailman.Logging.Syslog import syslog
from Mailman.Mailbox import Mailbox
from Mailman.Utils import mkdir, open_ex

DIRMODE = 0755      # Mode to give to created directories
FILEMODE = 0644     # Mode to give to created files
INDEX_EXT = ".html" # Extension for indexes

     <title>The %(listname)s Archives</title>
     <META NAME="robots" CONTENT="noindex,follow">
  <BODY BGCOLOR="#ffffff">
     <h1>The %(listname)s Archives </h1>
      You can get <a href="%(listinfo)s">more information about this list</a>
      or you can <a href="%(fullarch)s">download the full raw archive</a>

TOC_entry_template = '''\

              <A href="%(archive)s/thread.html">[ Thread ]</a>
              <A href="%(archive)s/subject.html">[ Subject ]</a>
              <A href="%(archive)s/author.html">[ Author ]</a>
              <A href="%(archive)s/date.html">[ Date ]</a>


def sizeof(filename):
	size = os.path.getsize(filename)
	if size < 1000:
		return ' %d bytes ' % size
	elif size < 1000000:
		return ' %d KB ' % (size / 1000)
	# GB?? :-)
	return ' %d MB ' % (size / 1000000)

class HyperArchive:

	def __init__(self, maillist):
		# make mailist available inside this object
		self.maillist = maillist
		# setup archive attributes
	        if hasattr(self.maillist,'archive_volume_frequency'):
        	    	if self.maillist.archive_volume_frequency == 0:
	            	elif self.maillist.archive_volume_frequency == 2:
		    	elif self.maillist.archive_volume_frequency == 3:
		    	elif self.maillist.archive_volume_frequency == 4:

		self.html_TOC_tmpl = TOC_template
		self.TOC_entry_tmpl = TOC_entry_template    

#	def processUnixMailbox(self, input, articleClass = Article):
	def processUnixMailbox(self, input ):
		mbox = Mailbox(input)
		while 1:
			m = mbox.next()
			if not m:
			self.message("constructed 0 ")
			self.message("constructed 3 ")

	def make_archive(self, path):
	        # If the archive directory doesn't exist, create it
		# recursively
	        except os.error, errdata:
	            errno, errmsg = errdata
	            if errno == 2: 
	                mkdir(path, DIRMODE)
			self.message("created: %s" % path)
	                raise os.error, errdata
	def add_article(self, msg):
		# Using a mailbox give us (eventually) Python's 
		# rfc822 Message object instead of Mailman's.
		# Here we simply reach in and do the necessary
		# but the real fix is to ensure Mailman ships
		# its own version of mailbox.py instead of
		# relying on Python's.
		# convert rfc822 Message object back into text
		txt = msg.unixfrom + string.join(msg.headers,'') + '\n' + msg.fp.read()
		self.ExternalArchive(mm_cfg.MHONARC, txt)

	def message(self, msg):
	            f = sys.stderr
	            if msg[-1:] != '\n':

	def ExternalArchive(self, ar, txt):
	        l = Utils.SafeDict({'listname': self.maillist.internal_name()})
		a = Utils.SafeDict({'archivedir': self.get_list_archive_path()})
	        cmd = ar % l
		cmd = cmd % a
		self.message("cmd = %s" % cmd)
	        extarch = os.popen(cmd, 'w')
	        status = extarch.close()
	        if status:
	            syslog('error', 'external archiver non-zero exit status: %d\n' %
	                   (status & 0xff00) >> 8)

	# The following two methods should be inverses of each other. -ddm
	def dateToVolName(self,date):
#		datetuple=time.localtime(date)
		datetuple = date
		if self.ARCHIVE_PERIOD=='year':
			return time.strftime("%Y",datetuple)
		elif self.ARCHIVE_PERIOD=='quarter':
		    	if datetuple[1] in [1,2,3]:
				return time.strftime("%Yq1",datetuple)
			elif datetuple[1] in [4,5,6]:
				return time.strftime("%Yq2",datetuple)
			elif datetuple[1] in [7,8,9]:
				return time.strftime("%Yq3",datetuple)
				return time.strftime("%Yq4",datetuple)
		elif self.ARCHIVE_PERIOD == 'day':
			return time.strftime("%Y%m%d", datetuple)
		elif self.ARCHIVE_PERIOD == 'week':
	       	     	# Reconstruct "seconds since epoch", and subtract weekday
		        # multiplied by the number of seconds in a day.
	        	monday = time.mktime(datetuple) - datetuple[6] * 24 * 60 * 60
	            	# Build a new datetuple from this "seconds since epoch" value
	            	datetuple = time.localtime(monday)
	            	return time.strftime("Week-of-Mon-%Y%m%d", datetuple)
	        	# month. -ddm
	            	return time.strftime("%Y-%B",datetuple)

### my stuff
	# We convert from string to seconds since 
	# Epoch in order to validate the date.
	def set_msg_date_tuple(self, message):
		if message.has_key('Date'):
			date = message.getdate_tz('Date')
			date, tzoffset = date[:9], date[-1] or 0
				date = time.mktime(date) # - tzoffset
				date = time.localtime(date)
			except (ValueError, OverflowError):
				date = time.localtime(time.time())
			date = time.localtime(time.time())

		self.msg_date_tuple = date

	def get_list_archive_path(self):
	        basedir = self.maillist.archive_dir()
		arch = self.dateToVolName(self.msg_date_tuple)
		archivedir = os.path.join(basedir, arch)

		return archivedir

	def write_TOC(self):
#	        self.sortarchives()
	        basedir = self.maillist.archive_dir()
        	toc=open_ex(os.path.join(basedir, 'index.html'), 'w')

	def html_TOC(self):
		# for know, fudge the fact we have no archives
		self.archives = None
	        listname = self.maillist.internal_name()
	        mbox = os.path.join(self.maillist.archive_directory+'.mbox',
	        d = {"listname": self.maillist.real_name,
	             "listinfo": self.maillist.GetScriptURL('listinfo', absolute=1),
	             "fullarch": '../%s.mbox/%s.mbox' % (listname, listname),
	             "size": sizeof(mbox),
	        if not self.archives:
	            d["noarchive_msg"] = '<P>Currently, there are no archives. </P>'
	            d["archive_listing_start"] = ""
	            d["archive_listing_end"] = ""
	            d["archive_listing"] = ""
	            d["noarchive_msg"] = ""
	            d["archive_listing_start"] = self.arch_listing_start
	            d["archive_listing_end"] = self.arch_listing_end
	            accum = []
	            for a in self.archives:
	            d["archive_listing"] = string.join(accum, '')
	        if not d.has_key("encoding"):
	            d["encoding"] = ""
	        return self.html_TOC_tmpl % d

	def html_TOC_entry(self, arch):
	# Check to see if the archive is gzip'd or not
		txtfile = os.path.join(mm_cfg.PRIVATE_ARCHIVE_FILE_DIR,
					arch + '.txt')
		gzfile = txtfile + '.gz'
		templ = '<td><A href="%(url)s">[ %(fmt)sText%(sz)s]</a></td>'
		# which exists?  .txt.gz first, then .txt
		if os.path.exists(gzfile):
			file = gzfile
			url = arch + '.txt.gz'
			fmt = "Gzip'd "
		elif os.path.exists(txtfile):
			file = txtfile
			url = arch + '.txt'
			fmt = ''
			# neither found?
			file = None
		# in Python 1.5.2 we have an easy way to get the size
		if file:
			textlink = templ % {'url': url,
						'fmt': fmt,
						'sz' : sizeof(file),
			# there's no archive file at all... hmmm.
			textlink = ''
			return self.TOC_entry_tmpl % { 'archive': arch,
							'textlink': textlink }

	def close(self):
#		"Close an archive, save its state, and update any changed archives."
#		self.update_dirty_archives()
#        	self.update_TOC = 0
#		# Save the collective state 
#		self.message('Pickling archive state into ' \
#               	      + os.path.join(self.basedir, 'pipermail.pck'))
#		self.database.close()
#		del self.database
#		f = open(os.path.join(self.basedir, 'pipermail.pck'), 'w')
#		pickle.dump(self.getstate(), f)
#		f.close()