[Distutils] Manifestor.py

Oliver Schoenborn oliver.schoenborn at utoronto.ca
Mon Nov 1 05:30:43 CET 2004

Hello, I needed some manifest capabilities for my last distrib so I 
created a Manifestor.py based on the manifest stuff that's in sdist. It 
gives identical functionality and can be reused for other distutils 
aspects. For instance, for the data_files parameter to setup.py. E.g. a 

    dbManif = Manifestor('dbManif', deps=['db/db.py'])
    # assume dbManif has a tree of files under a db folder:
    # db/db.txt
    # db/README.txt
    # db/...
    # then:
             # put all db files in same tree under sys.prefix/Scripts:

Here it is in case it is useful to others. Cheers,
Oliver Schoenborn

-------------- next part --------------
Based on manif code from distutils.command.sdist.
Copyright @ 2004 Oliver Schoenborn
License: same as Python

import os # for os.path.*
from sets import Set
from glob import glob

from distutils import dep_util, log
from distutils.text_file import TextFile
from distutils.filelist import FileList
from distutils.errors import DistutilsTemplateError

# if not name given for manifest file, this will be used: 
defaultManifName = 'MANIFEST'

class Manifestor:
    """Class to generate a manifest (ie a list of files), either from
    a template or list of files. The manifest can be saved to disk or
    loaded. See section 9.2 of Python Library Manual for syntax of
    template file. """
    def __init__(self, manifest=None, forceRegen=False, deps=None,
                 autoPruneCVS=True, loadNow=False):
        - manifest: name of the manifest file. If not specified,
          defaults to the module-level variable defaultManifName.
        - forceRegen: if true, the manifest file will be regenerated
          from the assumed manifest template file of same name + ".in"
          extension. The syntax of the manifest template is the same
          as for distutils's MANIFEST.in.
        - deps: list of extra dependencies: if any of them are
          existing files that are more recent than the manifest file,
          that file is regenerated.
        - autoPruneCVS=False means that CVS folders will not be pruned.
        - loadNow: if true, will finish construction by calling
          loadManifest(), so after construction you can immediately call
          getFilenames() or getAsDataList().
        self.__forceRegen = forceRegen
        if manifest is None:
            self.__manifest = defaultManifName
            self.__manifest = manifest
        self.__template = '%s.in' % self.__manifest
        # template is for sure a dependency
        self.__deps = Set([self.__template])
        # combine with deps given by user:
        if deps is not None:
        # prune CVS-related stuff automatically:
        self.__pruneSet = {}
        if autoPruneCVS:
            pruneCVS = r'%(pathSep)s(RCS|CVS)%(pathSep)s.*'
            self.addPrune(pruneCVS, is_regex=1)

        # for files added even if not specified using template
        self.__defaultFiles = []
        # ok, done:         
        self.__filelist = FileList()
        if loadNow:

    def addDependency(self, filename):
        """Add filename as a dependency. May cause the manifest file
        to be regenerated next time loadManifest() is called. See
        class doc for more info. """

    def addPrune(self, pattern, **flags):
        """Add a prunning pattern. Parameters are same as for
        distutils.filelist.FileList.include_pattern(), with the added
        bonus that if pattern contains a substitution item named
        pathSep, it gets properly substituted with path separator
        suitable for OS. E.g. pattern='%(pathSep)spat' will get
        replaced with '/pat' on unix but '\\\\pat' on windows.
        if os.path.sep == '\\':
            pathSep = os.path.sep*2
            pathSep = os.path.sep
        pathSub = {'pathSep':pathSep}
        self.__pruneSet[pattern % pathSub] = flags

    def addDefaultFile(self, fileGlob, filt=None):
        """Add file(s) obtained from glob(fileGlob) as default file(s)
        for the manifest, so it(they) need not be specified in the
        manifest template (if one is used). Filt is a filter, e.g.
        os.path.isfile if only files are desired. Returns True only
        if fileGlob found. 
        ff = glob(fileGlob)
        if filt is not None:
            ff = filter(filt, ff)
        if ff != []:
            return True
            return False
    def addDefaultFileAlt(self, fileGlobs, filt=None):
        """Like addDefaultFile, but fileGlobs is a tuple that
        means "add the *first* of these alternatives". Return
        True only if one of the file globs was found. 
        found = None
        for ffAltGlob in fileGlobs:
            ffAlt = glob(ffAltGlob)
            if filt is not None:
                ffAlt = filter(filt, ffAlt)
            if ffAlt != []:
                found = ffAlt
        # if we get here then we haven't found it:
        if found is not None:
            return True
            return False
    def needRegen(self):
        """Return True only if manifest file needs to be regenerated,
        False otherwise. It needs to be regenerated if any of its
        dependencies change, if forceRegen=True at construction time,
        or if neither a template nor a manifest file exist.
        someDepsNewer = False
        for ff in self.__deps:
            ffExists = os.path.isfile(ff)
            if ffExists:
                someDepsNewer |= dep_util.newer(ff, self.__manifest)

        noManifest = not os.path.isfile(self.__manifest)
        noTemplate = not os.path.isfile(self.__template)
        neitherExists = noTemplate and noManifest

        # Regenerate the manifest if necessary (or if explicitly told to)
        return someDepsNewer or neitherExists or self.__forceRegen
    def genManifest(self):
        """Generate manifest file from template.  Note that this
        regenerates it even if not necessary, and saves it to
        "manifest.in", where "manifest"=name given at construction
        time. """
    def loadManifest(self):
        """Load list of file names from manifest file. If
        file doesn't exist, it will be generated (from template
        if there is one, or from defaults if there were some --
        if neither, the manifest will be empty!)."""
        if self.needRegen():

    def readManifest(self):
        """Read the manifest file. This raises exception if no
        manifest file was created (see genManifest() or loadManifest()).
            manifest = file(self.__manifest,'r')
            for line in manifest: 

    def writeManifest(self):
        """Write the file names in manifest to manifest file. """
        manifest = file(self.__manifest,'w')
        print "-> Writing to '%s'" % self.__manifest

    def delManifestFile(self, raiseIfNoExist=True):
        """Deletes manifest file from disk. If doesn't exist, OSError raised."""
        if raiseIfNoExist:
            try: os.remove(self.__manifest)
            except OSError: pass
    def getFileNames(self):
        """Get list of file names loaded via a call to loadManifest()."""
        return self.__filelist.files
    def getAsDataList(self, dest=''):
        """Return the files in manifest, in format suitable for
        data_files parameter of distutils.setup(). Dest is the
        destination, under sys.prefix."""
        if dest is not '':
            dest += '/'
        scmExtDest = ['%s%s'% (dest, os.path.dirname(extFile))
                        for extFile in self.__filelist.files]
        scmExtFiles = [[extFile] for extFile in self.__filelist.files]
        return zip(scmExtDest, scmExtFiles)


    def __addDefaults(self):
        """Add files from self.__defaultFiles directly,
        without using FileList include/exclude."""
        for fname in self.__defaultFiles:
    def __readTemplate(self):
        """Read and parse manifest template file named by
        self.__template. The parsing and processing is done by
        'self.__filelist', which updates itself accordingly.
        template = TextFile(self.__template,

        while 1:
            line = template.readline()
            if line is None:            # end of file

            except DistutilsTemplateError, msg:
                print "%s, line %d: %s" %\
                    (template.filename, template.current_line, msg)

    def __pruneFileList(self):
        """Prune off branches that might slip into the file list as
        created by '__readTemplate()', but really don't belong there.
        This includes any RCS or CVS directories, and any patterns
        added via addPrune().
        for pat, flags in self.__pruneSet.iteritems():
            self.__filelist.exclude_pattern(pat, **flags)

if __name__ == '__main__':
    def glob(pattern):
        # for ('alt1a','alt1b'), return second item globbed,
        # whereas for all else, return nothing found
        if pattern == 'alt1b':
            return ['alt1b1', 'alt1b2']
        elif pattern.startswith('alt'):
            return [] # nothing found
        # for default files, only defFile1 returns a glob
        elif pattern == 'defFile1':
            return ['defFile1a', 'defFile1b']
        elif pattern.startswith('defFile'):
            return []
        return []

    from distutils import filelist    
    def findall(dir):
        fileList = [
        return fileList
    filelist.findall = findall
    # without force
    # need generation: both no exist
    man = Manifestor("testManifNoExist") # no template, no manifest
    assert man.needRegen()

    # manifest exists, no template
    manif = file('testManifNoRegen','w')
    man = Manifestor("testManifNoRegen")
    assert not man.needRegen()
    # with force
    man = Manifestor("testManifNoRegen",forceRegen=True) # manifest exists
    assert man.needRegen(), "Forced regen test failed"
    # pruning, default files
    # need generation: newer deps
    manifIn = file('testManif.in','w')
    manifIn.write('global-include *\n')
    man = Manifestor("testManif")
    assert man.needRegen()
    ok = man.addDefaultFile('defFile1')
    assert ok
    ok = man.addDefaultFile('defFile2')
    assert not ok
    ok = man.addDefaultFileAlt(('alt1a','alt1b'))
    assert ok
    ok = man.addDefaultFileAlt(('alt2a','alt2b'))
    assert not ok
    print "Loading manifest (creating from temporary testManif.in)"
    expectFiles = ['alt1b1', 'alt1b2', 'defFile1a', 'defFile1b', 'file1','file4']
    assert man.getFileNames() == expectFiles, man.getFileNames()

    # cleanup:    

More information about the Distutils-SIG mailing list