[Distutils] Manifestor.py

Oliver Schoenborn oliver.schoenborn at utoronto.ca
Mon Nov 1 05:30:43 CET 2004


Hello, I needed some manifest capabilities for my last distrib so I 
created a Manifestor.py based on the manifest stuff that's in sdist. It 
gives identical functionality and can be reused for other distutils 
aspects. For instance, for the data_files parameter to setup.py. E.g. a 
setup.py:

    dbManif = Manifestor('dbManif', deps=['db/db.py'])
    dbManif.loadManifest()
    # assume dbManif has a tree of files under a db folder:
    # db/db.txt
    # db/README.txt
    # db/...
    # then:
    setup(...,
             # put all db files in same tree under sys.prefix/Scripts:
             data_files=dbManif.getAsDataFiles(dest='Scripts')
    )

Here it is in case it is useful to others. Cheers,
Oliver Schoenborn

-------------- next part --------------
"""
Based on manif code from distutils.command.sdist.
Copyright @ 2004 Oliver Schoenborn
License: same as Python
"""

import os # for os.path.*
from sets import Set
from glob import glob

from distutils import dep_util, log
from distutils.text_file import TextFile
from distutils.filelist import FileList
from distutils.errors import DistutilsTemplateError

# if not name given for manifest file, this will be used: 
defaultManifName = 'MANIFEST'

class Manifestor:
    """Class to generate a manifest (ie a list of files), either from
    a template or list of files. The manifest can be saved to disk or
    loaded. See section 9.2 of Python Library Manual for syntax of
    template file. """
    
    def __init__(self, manifest=None, forceRegen=False, deps=None,
                 autoPruneCVS=True, loadNow=False):
        """Parameters:
        - manifest: name of the manifest file. If not specified,
          defaults to the module-level variable defaultManifName.
        - forceRegen: if true, the manifest file will be regenerated
          from the assumed manifest template file of same name + ".in"
          extension. The syntax of the manifest template is the same
          as for distutils's MANIFEST.in.
        - deps: list of extra dependencies: if any of them are
          existing files that are more recent than the manifest file,
          that file is regenerated.
        - autoPruneCVS=False means that CVS folders will not be pruned.
        - loadNow: if true, will finish construction by calling
          loadManifest(), so after construction you can immediately call
          getFilenames() or getAsDataList().
        """
        self.__forceRegen = forceRegen
        if manifest is None:
            self.__manifest = defaultManifName
        else:
            self.__manifest = manifest
            
        self.__template = '%s.in' % self.__manifest
        # template is for sure a dependency
        self.__deps = Set([self.__template])
        # combine with deps given by user:
        if deps is not None:
            self.__deps.union_update(deps)
        # prune CVS-related stuff automatically:
        self.__pruneSet = {}
        if autoPruneCVS:
            pruneCVS = r'%(pathSep)s(RCS|CVS)%(pathSep)s.*'
            self.addPrune(pruneCVS, is_regex=1)

        # for files added even if not specified using template
        self.__defaultFiles = []
        
        # ok, done:         
        self.__filelist = FileList()
        if loadNow:
            self.loadManifest()

    def addDependency(self, filename):
        """Add filename as a dependency. May cause the manifest file
        to be regenerated next time loadManifest() is called. See
        class doc for more info. """
        self.__deps.add(filename)

    def addPrune(self, pattern, **flags):
        """Add a prunning pattern. Parameters are same as for
        distutils.filelist.FileList.include_pattern(), with the added
        bonus that if pattern contains a substitution item named
        pathSep, it gets properly substituted with path separator
        suitable for OS. E.g. pattern='%(pathSep)spat' will get
        replaced with '/pat' on unix but '\\\\pat' on windows.
        """
        if os.path.sep == '\\':
            pathSep = os.path.sep*2
        else:
            pathSep = os.path.sep
        pathSub = {'pathSep':pathSep}
        #log.debug(flags)
        self.__pruneSet[pattern % pathSub] = flags

    def addDefaultFile(self, fileGlob, filt=None):
        """Add file(s) obtained from glob(fileGlob) as default file(s)
        for the manifest, so it(they) need not be specified in the
        manifest template (if one is used). Filt is a filter, e.g.
        os.path.isfile if only files are desired. Returns True only
        if fileGlob found. 
        """
        ff = glob(fileGlob)
        if filt is not None:
            ff = filter(filt, ff)
        if ff != []:
            self.__defaultFiles.extend(ff)
            return True
        else:
            return False
    
    def addDefaultFileAlt(self, fileGlobs, filt=None):
        """Like addDefaultFile, but fileGlobs is a tuple that
        means "add the *first* of these alternatives". Return
        True only if one of the file globs was found. 
        """
        found = None
        for ffAltGlob in fileGlobs:
            ffAlt = glob(ffAltGlob)
            if filt is not None:
                ffAlt = filter(filt, ffAlt)
            if ffAlt != []:
                found = ffAlt
                break
        # if we get here then we haven't found it:
        if found is not None:
            self.__defaultFiles.extend(found)
            return True
        else:
            return False
            
    def needRegen(self):
        """Return True only if manifest file needs to be regenerated,
        False otherwise. It needs to be regenerated if any of its
        dependencies change, if forceRegen=True at construction time,
        or if neither a template nor a manifest file exist.
        """
        someDepsNewer = False
        for ff in self.__deps:
            ffExists = os.path.isfile(ff)
            if ffExists:
                someDepsNewer |= dep_util.newer(ff, self.__manifest)

        noManifest = not os.path.isfile(self.__manifest)
        noTemplate = not os.path.isfile(self.__template)
        neitherExists = noTemplate and noManifest

        # Regenerate the manifest if necessary (or if explicitly told to)
        return someDepsNewer or neitherExists or self.__forceRegen
    
    def genManifest(self):
        """Generate manifest file from template.  Note that this
        regenerates it even if not necessary, and saves it to
        "manifest.in", where "manifest"=name given at construction
        time. """
        self.__filelist.findall()
        
        self.__addDefaults()
        self.__readTemplate()
        self.__pruneFileList()
        
        self.__filelist.sort()
        self.__filelist.remove_duplicates()
        
    def loadManifest(self):
        """Load list of file names from manifest file. If
        file doesn't exist, it will be generated (from template
        if there is one, or from defaults if there were some --
        if neither, the manifest will be empty!)."""
        if self.needRegen():
            self.genManifest()
        else: 
            self.readManifest()

    def readManifest(self):
        """Read the manifest file. This raises exception if no
        manifest file was created (see genManifest() or loadManifest()).
        """
        try: 
            manifest = file(self.__manifest,'r')
            for line in manifest: 
                self.__filelist.append(line.strip())
        finally:
            manifest.close()

    def writeManifest(self):
        """Write the file names in manifest to manifest file. """
        manifest = file(self.__manifest,'w')
        print "-> Writing to '%s'" % self.__manifest
        try: 
            manifest.write("\n".join(self.__filelist.files))
        finally: 
            manifest.close()

    def delManifestFile(self, raiseIfNoExist=True):
        """Deletes manifest file from disk. If doesn't exist, OSError raised."""
        if raiseIfNoExist:
            os.remove(self.__manifest)
        else:
            try: os.remove(self.__manifest)
            except OSError: pass
        
    def getFileNames(self):
        """Get list of file names loaded via a call to loadManifest()."""
        return self.__filelist.files
    
    def getAsDataList(self, dest=''):
        """Return the files in manifest, in format suitable for
        data_files parameter of distutils.setup(). Dest is the
        destination, under sys.prefix."""
        if dest is not '':
            dest += '/'
        scmExtDest = ['%s%s'% (dest, os.path.dirname(extFile))
                        for extFile in self.__filelist.files]
        scmExtFiles = [[extFile] for extFile in self.__filelist.files]
        return zip(scmExtDest, scmExtFiles)

    #---------------------------------------------------------------

    def __addDefaults(self):
        """Add files from self.__defaultFiles directly,
        without using FileList include/exclude."""
        for fname in self.__defaultFiles:
            self.__filelist.append(fname)
        
    def __readTemplate(self):
        """Read and parse manifest template file named by
        self.__template. The parsing and processing is done by
        'self.__filelist', which updates itself accordingly.
        """
        template = TextFile(self.__template,
                            strip_comments=1,
                            skip_blanks=1,
                            join_lines=1,
                            lstrip_ws=1,
                            rstrip_ws=1,
                            collapse_join=1)

        while 1:
            line = template.readline()
            if line is None:            # end of file
                break

            try:
                self.__filelist.process_template_line(line)
            except DistutilsTemplateError, msg:
                print "%s, line %d: %s" %\
                    (template.filename, template.current_line, msg)

    def __pruneFileList(self):
        """Prune off branches that might slip into the file list as
        created by '__readTemplate()', but really don't belong there.
        This includes any RCS or CVS directories, and any patterns
        added via addPrune().
        """
        for pat, flags in self.__pruneSet.iteritems():
            self.__filelist.exclude_pattern(pat, **flags)

#----------------------------------------------------------------------------
if __name__ == '__main__':
    def glob(pattern):
        # for ('alt1a','alt1b'), return second item globbed,
        # whereas for all else, return nothing found
        if pattern == 'alt1b':
            return ['alt1b1', 'alt1b2']
        elif pattern.startswith('alt'):
            return [] # nothing found
        # for default files, only defFile1 returns a glob
        elif pattern == 'defFile1':
            return ['defFile1a', 'defFile1b']
        elif pattern.startswith('defFile'):
            return []
        
        return []

    from distutils import filelist    
    def findall(dir):
        fileList = [
            'file1',
            'file2\\CVS\\Root',
            'file3/prune',
            'file4'
        ]
        return fileList
    filelist.findall = findall
    
    # without force
    # need generation: both no exist
    man = Manifestor("testManifNoExist") # no template, no manifest
    assert man.needRegen()

    # manifest exists, no template
    manif = file('testManifNoRegen','w')
    manif.close()
    man = Manifestor("testManifNoRegen")
    assert not man.needRegen()
    # with force
    man = Manifestor("testManifNoRegen",forceRegen=True) # manifest exists
    assert man.needRegen(), "Forced regen test failed"
    man.delManifestFile()
    
    # pruning, default files
    # need generation: newer deps
    manifIn = file('testManif.in','w')
    manifIn.write('global-include *\n')
    manifIn.close()
    man = Manifestor("testManif")
    assert man.needRegen()
    man.addPrune('prune',anchor=0)
    ok = man.addDefaultFile('defFile1')
    assert ok
    ok = man.addDefaultFile('defFile2')
    assert not ok
    ok = man.addDefaultFileAlt(('alt1a','alt1b'))
    assert ok
    ok = man.addDefaultFileAlt(('alt2a','alt2b'))
    assert not ok
    print "Loading manifest (creating from temporary testManif.in)"
    man.loadManifest()
    
    expectFiles = ['alt1b1', 'alt1b2', 'defFile1a', 'defFile1b', 'file1','file4']
    assert man.getFileNames() == expectFiles, man.getFileNames()

    # cleanup:    
    man.delManifestFile(raiseIfNoExist=False)
    os.remove('testManif.in')
 


More information about the Distutils-SIG mailing list