Hello, I needed some manifest capabilities for my last distrib so I created a Manifestor.py based on the manifest stuff that's in sdist. It gives identical functionality and can be reused for other distutils aspects. For instance, for the data_files parameter to setup.py. E.g. a setup.py: dbManif = Manifestor('dbManif', deps=['db/db.py']) dbManif.loadManifest() # assume dbManif has a tree of files under a db folder: # db/db.txt # db/README.txt # db/... # then: setup(..., # put all db files in same tree under sys.prefix/Scripts: data_files=dbManif.getAsDataFiles(dest='Scripts') ) Here it is in case it is useful to others. Cheers, Oliver Schoenborn """ Based on manif code from distutils.command.sdist. Copyright @ 2004 Oliver Schoenborn License: same as Python """ import os # for os.path.* from sets import Set from glob import glob from distutils import dep_util, log from distutils.text_file import TextFile from distutils.filelist import FileList from distutils.errors import DistutilsTemplateError # if not name given for manifest file, this will be used: defaultManifName = 'MANIFEST' class Manifestor: """Class to generate a manifest (ie a list of files), either from a template or list of files. The manifest can be saved to disk or loaded. See section 9.2 of Python Library Manual for syntax of template file. """ def __init__(self, manifest=None, forceRegen=False, deps=None, autoPruneCVS=True, loadNow=False): """Parameters: - manifest: name of the manifest file. If not specified, defaults to the module-level variable defaultManifName. - forceRegen: if true, the manifest file will be regenerated from the assumed manifest template file of same name + ".in" extension. The syntax of the manifest template is the same as for distutils's MANIFEST.in. - deps: list of extra dependencies: if any of them are existing files that are more recent than the manifest file, that file is regenerated. - autoPruneCVS=False means that CVS folders will not be pruned. - loadNow: if true, will finish construction by calling loadManifest(), so after construction you can immediately call getFilenames() or getAsDataList(). """ self.__forceRegen = forceRegen if manifest is None: self.__manifest = defaultManifName else: self.__manifest = manifest self.__template = '%s.in' % self.__manifest # template is for sure a dependency self.__deps = Set([self.__template]) # combine with deps given by user: if deps is not None: self.__deps.union_update(deps) # prune CVS-related stuff automatically: self.__pruneSet = {} if autoPruneCVS: pruneCVS = r'%(pathSep)s(RCS|CVS)%(pathSep)s.*' self.addPrune(pruneCVS, is_regex=1) # for files added even if not specified using template self.__defaultFiles = [] # ok, done: self.__filelist = FileList() if loadNow: self.loadManifest() def addDependency(self, filename): """Add filename as a dependency. May cause the manifest file to be regenerated next time loadManifest() is called. See class doc for more info. """ self.__deps.add(filename) def addPrune(self, pattern, **flags): """Add a prunning pattern. Parameters are same as for distutils.filelist.FileList.include_pattern(), with the added bonus that if pattern contains a substitution item named pathSep, it gets properly substituted with path separator suitable for OS. E.g. pattern='%(pathSep)spat' will get replaced with '/pat' on unix but '\\\\pat' on windows. """ if os.path.sep == '\\': pathSep = os.path.sep*2 else: pathSep = os.path.sep pathSub = {'pathSep':pathSep} #log.debug(flags) self.__pruneSet[pattern % pathSub] = flags def addDefaultFile(self, fileGlob, filt=None): """Add file(s) obtained from glob(fileGlob) as default file(s) for the manifest, so it(they) need not be specified in the manifest template (if one is used). Filt is a filter, e.g. os.path.isfile if only files are desired. Returns True only if fileGlob found. """ ff = glob(fileGlob) if filt is not None: ff = filter(filt, ff) if ff != []: self.__defaultFiles.extend(ff) return True else: return False def addDefaultFileAlt(self, fileGlobs, filt=None): """Like addDefaultFile, but fileGlobs is a tuple that means "add the *first* of these alternatives". Return True only if one of the file globs was found. """ found = None for ffAltGlob in fileGlobs: ffAlt = glob(ffAltGlob) if filt is not None: ffAlt = filter(filt, ffAlt) if ffAlt != []: found = ffAlt break # if we get here then we haven't found it: if found is not None: self.__defaultFiles.extend(found) return True else: return False def needRegen(self): """Return True only if manifest file needs to be regenerated, False otherwise. It needs to be regenerated if any of its dependencies change, if forceRegen=True at construction time, or if neither a template nor a manifest file exist. """ someDepsNewer = False for ff in self.__deps: ffExists = os.path.isfile(ff) if ffExists: someDepsNewer |= dep_util.newer(ff, self.__manifest) noManifest = not os.path.isfile(self.__manifest) noTemplate = not os.path.isfile(self.__template) neitherExists = noTemplate and noManifest # Regenerate the manifest if necessary (or if explicitly told to) return someDepsNewer or neitherExists or self.__forceRegen def genManifest(self): """Generate manifest file from template. Note that this regenerates it even if not necessary, and saves it to "manifest.in", where "manifest"=name given at construction time. """ self.__filelist.findall() self.__addDefaults() self.__readTemplate() self.__pruneFileList() self.__filelist.sort() self.__filelist.remove_duplicates() def loadManifest(self): """Load list of file names from manifest file. If file doesn't exist, it will be generated (from template if there is one, or from defaults if there were some -- if neither, the manifest will be empty!).""" if self.needRegen(): self.genManifest() else: self.readManifest() def readManifest(self): """Read the manifest file. This raises exception if no manifest file was created (see genManifest() or loadManifest()). """ try: manifest = file(self.__manifest,'r') for line in manifest: self.__filelist.append(line.strip()) finally: manifest.close() def writeManifest(self): """Write the file names in manifest to manifest file. """ manifest = file(self.__manifest,'w') print "-> Writing to '%s'" % self.__manifest try: manifest.write("\n".join(self.__filelist.files)) finally: manifest.close() def delManifestFile(self, raiseIfNoExist=True): """Deletes manifest file from disk. If doesn't exist, OSError raised.""" if raiseIfNoExist: os.remove(self.__manifest) else: try: os.remove(self.__manifest) except OSError: pass def getFileNames(self): """Get list of file names loaded via a call to loadManifest().""" return self.__filelist.files def getAsDataList(self, dest=''): """Return the files in manifest, in format suitable for data_files parameter of distutils.setup(). Dest is the destination, under sys.prefix.""" if dest is not '': dest += '/' scmExtDest = ['%s%s'% (dest, os.path.dirname(extFile)) for extFile in self.__filelist.files] scmExtFiles = [[extFile] for extFile in self.__filelist.files] return zip(scmExtDest, scmExtFiles) #--------------------------------------------------------------- def __addDefaults(self): """Add files from self.__defaultFiles directly, without using FileList include/exclude.""" for fname in self.__defaultFiles: self.__filelist.append(fname) def __readTemplate(self): """Read and parse manifest template file named by self.__template. The parsing and processing is done by 'self.__filelist', which updates itself accordingly. """ template = TextFile(self.__template, strip_comments=1, skip_blanks=1, join_lines=1, lstrip_ws=1, rstrip_ws=1, collapse_join=1) while 1: line = template.readline() if line is None: # end of file break try: self.__filelist.process_template_line(line) except DistutilsTemplateError, msg: print "%s, line %d: %s" %\ (template.filename, template.current_line, msg) def __pruneFileList(self): """Prune off branches that might slip into the file list as created by '__readTemplate()', but really don't belong there. This includes any RCS or CVS directories, and any patterns added via addPrune(). """ for pat, flags in self.__pruneSet.iteritems(): self.__filelist.exclude_pattern(pat, **flags) #---------------------------------------------------------------------------- if __name__ == '__main__': def glob(pattern): # for ('alt1a','alt1b'), return second item globbed, # whereas for all else, return nothing found if pattern == 'alt1b': return ['alt1b1', 'alt1b2'] elif pattern.startswith('alt'): return [] # nothing found # for default files, only defFile1 returns a glob elif pattern == 'defFile1': return ['defFile1a', 'defFile1b'] elif pattern.startswith('defFile'): return [] return [] from distutils import filelist def findall(dir): fileList = [ 'file1', 'file2\\CVS\\Root', 'file3/prune', 'file4' ] return fileList filelist.findall = findall # without force # need generation: both no exist man = Manifestor("testManifNoExist") # no template, no manifest assert man.needRegen() # manifest exists, no template manif = file('testManifNoRegen','w') manif.close() man = Manifestor("testManifNoRegen") assert not man.needRegen() # with force man = Manifestor("testManifNoRegen",forceRegen=True) # manifest exists assert man.needRegen(), "Forced regen test failed" man.delManifestFile() # pruning, default files # need generation: newer deps manifIn = file('testManif.in','w') manifIn.write('global-include *\n') manifIn.close() man = Manifestor("testManif") assert man.needRegen() man.addPrune('prune',anchor=0) ok = man.addDefaultFile('defFile1') assert ok ok = man.addDefaultFile('defFile2') assert not ok ok = man.addDefaultFileAlt(('alt1a','alt1b')) assert ok ok = man.addDefaultFileAlt(('alt2a','alt2b')) assert not ok print "Loading manifest (creating from temporary testManif.in)" man.loadManifest() expectFiles = ['alt1b1', 'alt1b2', 'defFile1a', 'defFile1b', 'file1','file4'] assert man.getFileNames() == expectFiles, man.getFileNames() # cleanup: man.delManifestFile(raiseIfNoExist=False) os.remove('testManif.in')
participants (1)
-
Oliver Schoenborn