[Distutils] Manifestor.py
Oliver Schoenborn
oliver.schoenborn at utoronto.ca
Mon Nov 1 05:30:43 CET 2004
Hello, I needed some manifest capabilities for my last distrib so I
created a Manifestor.py based on the manifest stuff that's in sdist. It
gives identical functionality and can be reused for other distutils
aspects. For instance, for the data_files parameter to setup.py. E.g. a
setup.py:
dbManif = Manifestor('dbManif', deps=['db/db.py'])
dbManif.loadManifest()
# assume dbManif has a tree of files under a db folder:
# db/db.txt
# db/README.txt
# db/...
# then:
setup(...,
# put all db files in same tree under sys.prefix/Scripts:
data_files=dbManif.getAsDataFiles(dest='Scripts')
)
Here it is in case it is useful to others. Cheers,
Oliver Schoenborn
-------------- next part --------------
"""
Based on manif code from distutils.command.sdist.
Copyright @ 2004 Oliver Schoenborn
License: same as Python
"""
import os # for os.path.*
from sets import Set
from glob import glob
from distutils import dep_util, log
from distutils.text_file import TextFile
from distutils.filelist import FileList
from distutils.errors import DistutilsTemplateError
# if not name given for manifest file, this will be used:
defaultManifName = 'MANIFEST'
class Manifestor:
"""Class to generate a manifest (ie a list of files), either from
a template or list of files. The manifest can be saved to disk or
loaded. See section 9.2 of Python Library Manual for syntax of
template file. """
def __init__(self, manifest=None, forceRegen=False, deps=None,
autoPruneCVS=True, loadNow=False):
"""Parameters:
- manifest: name of the manifest file. If not specified,
defaults to the module-level variable defaultManifName.
- forceRegen: if true, the manifest file will be regenerated
from the assumed manifest template file of same name + ".in"
extension. The syntax of the manifest template is the same
as for distutils's MANIFEST.in.
- deps: list of extra dependencies: if any of them are
existing files that are more recent than the manifest file,
that file is regenerated.
- autoPruneCVS=False means that CVS folders will not be pruned.
- loadNow: if true, will finish construction by calling
loadManifest(), so after construction you can immediately call
getFilenames() or getAsDataList().
"""
self.__forceRegen = forceRegen
if manifest is None:
self.__manifest = defaultManifName
else:
self.__manifest = manifest
self.__template = '%s.in' % self.__manifest
# template is for sure a dependency
self.__deps = Set([self.__template])
# combine with deps given by user:
if deps is not None:
self.__deps.union_update(deps)
# prune CVS-related stuff automatically:
self.__pruneSet = {}
if autoPruneCVS:
pruneCVS = r'%(pathSep)s(RCS|CVS)%(pathSep)s.*'
self.addPrune(pruneCVS, is_regex=1)
# for files added even if not specified using template
self.__defaultFiles = []
# ok, done:
self.__filelist = FileList()
if loadNow:
self.loadManifest()
def addDependency(self, filename):
"""Add filename as a dependency. May cause the manifest file
to be regenerated next time loadManifest() is called. See
class doc for more info. """
self.__deps.add(filename)
def addPrune(self, pattern, **flags):
"""Add a prunning pattern. Parameters are same as for
distutils.filelist.FileList.include_pattern(), with the added
bonus that if pattern contains a substitution item named
pathSep, it gets properly substituted with path separator
suitable for OS. E.g. pattern='%(pathSep)spat' will get
replaced with '/pat' on unix but '\\\\pat' on windows.
"""
if os.path.sep == '\\':
pathSep = os.path.sep*2
else:
pathSep = os.path.sep
pathSub = {'pathSep':pathSep}
#log.debug(flags)
self.__pruneSet[pattern % pathSub] = flags
def addDefaultFile(self, fileGlob, filt=None):
"""Add file(s) obtained from glob(fileGlob) as default file(s)
for the manifest, so it(they) need not be specified in the
manifest template (if one is used). Filt is a filter, e.g.
os.path.isfile if only files are desired. Returns True only
if fileGlob found.
"""
ff = glob(fileGlob)
if filt is not None:
ff = filter(filt, ff)
if ff != []:
self.__defaultFiles.extend(ff)
return True
else:
return False
def addDefaultFileAlt(self, fileGlobs, filt=None):
"""Like addDefaultFile, but fileGlobs is a tuple that
means "add the *first* of these alternatives". Return
True only if one of the file globs was found.
"""
found = None
for ffAltGlob in fileGlobs:
ffAlt = glob(ffAltGlob)
if filt is not None:
ffAlt = filter(filt, ffAlt)
if ffAlt != []:
found = ffAlt
break
# if we get here then we haven't found it:
if found is not None:
self.__defaultFiles.extend(found)
return True
else:
return False
def needRegen(self):
"""Return True only if manifest file needs to be regenerated,
False otherwise. It needs to be regenerated if any of its
dependencies change, if forceRegen=True at construction time,
or if neither a template nor a manifest file exist.
"""
someDepsNewer = False
for ff in self.__deps:
ffExists = os.path.isfile(ff)
if ffExists:
someDepsNewer |= dep_util.newer(ff, self.__manifest)
noManifest = not os.path.isfile(self.__manifest)
noTemplate = not os.path.isfile(self.__template)
neitherExists = noTemplate and noManifest
# Regenerate the manifest if necessary (or if explicitly told to)
return someDepsNewer or neitherExists or self.__forceRegen
def genManifest(self):
"""Generate manifest file from template. Note that this
regenerates it even if not necessary, and saves it to
"manifest.in", where "manifest"=name given at construction
time. """
self.__filelist.findall()
self.__addDefaults()
self.__readTemplate()
self.__pruneFileList()
self.__filelist.sort()
self.__filelist.remove_duplicates()
def loadManifest(self):
"""Load list of file names from manifest file. If
file doesn't exist, it will be generated (from template
if there is one, or from defaults if there were some --
if neither, the manifest will be empty!)."""
if self.needRegen():
self.genManifest()
else:
self.readManifest()
def readManifest(self):
"""Read the manifest file. This raises exception if no
manifest file was created (see genManifest() or loadManifest()).
"""
try:
manifest = file(self.__manifest,'r')
for line in manifest:
self.__filelist.append(line.strip())
finally:
manifest.close()
def writeManifest(self):
"""Write the file names in manifest to manifest file. """
manifest = file(self.__manifest,'w')
print "-> Writing to '%s'" % self.__manifest
try:
manifest.write("\n".join(self.__filelist.files))
finally:
manifest.close()
def delManifestFile(self, raiseIfNoExist=True):
"""Deletes manifest file from disk. If doesn't exist, OSError raised."""
if raiseIfNoExist:
os.remove(self.__manifest)
else:
try: os.remove(self.__manifest)
except OSError: pass
def getFileNames(self):
"""Get list of file names loaded via a call to loadManifest()."""
return self.__filelist.files
def getAsDataList(self, dest=''):
"""Return the files in manifest, in format suitable for
data_files parameter of distutils.setup(). Dest is the
destination, under sys.prefix."""
if dest is not '':
dest += '/'
scmExtDest = ['%s%s'% (dest, os.path.dirname(extFile))
for extFile in self.__filelist.files]
scmExtFiles = [[extFile] for extFile in self.__filelist.files]
return zip(scmExtDest, scmExtFiles)
#---------------------------------------------------------------
def __addDefaults(self):
"""Add files from self.__defaultFiles directly,
without using FileList include/exclude."""
for fname in self.__defaultFiles:
self.__filelist.append(fname)
def __readTemplate(self):
"""Read and parse manifest template file named by
self.__template. The parsing and processing is done by
'self.__filelist', which updates itself accordingly.
"""
template = TextFile(self.__template,
strip_comments=1,
skip_blanks=1,
join_lines=1,
lstrip_ws=1,
rstrip_ws=1,
collapse_join=1)
while 1:
line = template.readline()
if line is None: # end of file
break
try:
self.__filelist.process_template_line(line)
except DistutilsTemplateError, msg:
print "%s, line %d: %s" %\
(template.filename, template.current_line, msg)
def __pruneFileList(self):
"""Prune off branches that might slip into the file list as
created by '__readTemplate()', but really don't belong there.
This includes any RCS or CVS directories, and any patterns
added via addPrune().
"""
for pat, flags in self.__pruneSet.iteritems():
self.__filelist.exclude_pattern(pat, **flags)
#----------------------------------------------------------------------------
if __name__ == '__main__':
def glob(pattern):
# for ('alt1a','alt1b'), return second item globbed,
# whereas for all else, return nothing found
if pattern == 'alt1b':
return ['alt1b1', 'alt1b2']
elif pattern.startswith('alt'):
return [] # nothing found
# for default files, only defFile1 returns a glob
elif pattern == 'defFile1':
return ['defFile1a', 'defFile1b']
elif pattern.startswith('defFile'):
return []
return []
from distutils import filelist
def findall(dir):
fileList = [
'file1',
'file2\\CVS\\Root',
'file3/prune',
'file4'
]
return fileList
filelist.findall = findall
# without force
# need generation: both no exist
man = Manifestor("testManifNoExist") # no template, no manifest
assert man.needRegen()
# manifest exists, no template
manif = file('testManifNoRegen','w')
manif.close()
man = Manifestor("testManifNoRegen")
assert not man.needRegen()
# with force
man = Manifestor("testManifNoRegen",forceRegen=True) # manifest exists
assert man.needRegen(), "Forced regen test failed"
man.delManifestFile()
# pruning, default files
# need generation: newer deps
manifIn = file('testManif.in','w')
manifIn.write('global-include *\n')
manifIn.close()
man = Manifestor("testManif")
assert man.needRegen()
man.addPrune('prune',anchor=0)
ok = man.addDefaultFile('defFile1')
assert ok
ok = man.addDefaultFile('defFile2')
assert not ok
ok = man.addDefaultFileAlt(('alt1a','alt1b'))
assert ok
ok = man.addDefaultFileAlt(('alt2a','alt2b'))
assert not ok
print "Loading manifest (creating from temporary testManif.in)"
man.loadManifest()
expectFiles = ['alt1b1', 'alt1b2', 'defFile1a', 'defFile1b', 'file1','file4']
assert man.getFileNames() == expectFiles, man.getFileNames()
# cleanup:
man.delManifestFile(raiseIfNoExist=False)
os.remove('testManif.in')
More information about the Distutils-SIG
mailing list