Lazy imports (was Re: module naming) (corrected version)

[Peter Funk]

Hi!

Two days ago I posted a followup to a question by Michal Wallace, 
which included some Python code, which was BUGGY!  But see below.

> Michal Wallace (sabren) asks:
> >   That's certainly a lot cleaner! But is there a way to do that and
> > not have it load each one of those files every time? There's actually
> > more than 3, and even though they're all fairly small, I'd rather not
> > have the overhead of loading them until they're needed.
> 
> You could try to adopt the lazy loader concept implemented in 
> the Pmw package.  The idea is to use a small tool, that builds a 
> dictionary of exported features (classes and more) by investigating
> all module files in the package and places this in a file called
> for example 'exports.def'.  Than you can use the following
> __init__.py file to load features (classes, constants, functions,
> or modules, what you want) on demand.  The idea is "stolen" from
> Greg McFarlanes wonderful Pmw package:

As Greg McFarlane pointed out to me, a block of my program got the
wrong indentation, so it will not work as I intended it to.  Here is
an updated (and hopefully more correct version).  Many thanks to
Greg McFarlane for his original work and the time he spent to review
my code.  This time I also included a little helper script used to
automatically generate the 'exports.def' file used by the lazy importer.

Regards, Peter
---- 8< ---- 8< ---- cut here ---- 8< ---- schnipp ---- 8< ---- schnapp ----
#!/usr/bin/env python
## vim:ts=4:et:nowrap
"""__init__ : This file is executed when the package is imported.  It creates
a lazy importer/dynamic loader for the package and replaces the package 
module with it.  This is a very simplified version of the loader supplied 
with Pmw.  All the package version management has been stripped off."""

import sys, os, string, types

_EXP_DEF = 'exports.def'       # export definition file
_BASEMODULE = 'base'           # Name of Base module for the package

class _Loader:
    """An instance of this class will replace the module in sys.modules"""

    def __init__(self, path, package):
        self._path, self._package = path, package
        self._initialised = 0
        
    def _getmodule(self, modpath):
        __import__(modpath)
        mod = sys.modules[modpath]
        return mod

    def _initialise(self):
        # Create attributes for the Base classes and functions.
        basemodule = self._getmodule('_'+self._package+'.'+_BASEMODULE)
        for k,v in basemodule.__dict__.items():
            if k[0] is not '_' and type(v) != types.ModuleType:
                self.__dict__[k] = v
        # Set the package definitions from the exports.def file.
        dict = {
            '_features'     : {},
            '_modules'      : {},
        }
        for name in dict.keys():
            self.__dict__[name] = {}
            d = {}
            execfile(os.path.join(self._path, _EXP_DEF), d)
            for k,v in d.items():
                if dict.has_key(k):
                    if type(v) == types.TupleType:
                        for item in v:
                            ## modpath = self._package + item
                            modpath = item
                            dict[k][item] = modpath
                    elif type(v) == types.DictionaryType:
                        for k1, v1 in v.items():
                            ## modpath = '_'+self._package +'.'+ v1
                            modpath = v1
                            dict[k][k1] = modpath
        self.__dict__.update(dict)
        self._initialised = 1

    def __getattr__(self, name):
        """This will solve references to not yet used features"""
        if not self._initialised:
            self._initialise()
            # Beware: _initialise may have defined 'name'
            if self.__dict__.has_key(name):
                return self.__dict__[name]
        # The requested feature is not yet set. Look it up in the
        # tables set by exports.def, import the appropriate module and
        # set the attribute so that it will be found next time.
        if self._features.has_key(name):
            # The attribute is a feature from one of the modules.
            modname = self._features[name]
            mod  = self._getmodule('_'+self._package+'.'+modname)
            feature = getattr(mod, name)
            self.__dict__[name] = feature
            return feature
        elif self._modules.has_key(name):
            # The attribute is a module
            mod = self._getmodule('_'+self._package+'.'+name)
            self.__dict__[name] = mod
            return mod
        else:
            # The attribute is not known by the package, report an error.
            raise AttributeError, name

# Retrieve the name of the package:
_package = os.path.split(__path__[0])[1]
# Rename (hide) the original package for later perusual:
sys.modules['_'+_package] = sys.modules[_package]
# Create the dynamic loader and install it into sys.modules:
sys.modules[_package] = _Loader(__path__[0], _package)
---- 8< ---- 8< ---- cut here ---- 8< ---- schnipp ---- 8< ---- schnapp ----
#!/usr/bin/env python
## vim:ts=4:et:nowrap                    
"""build_exports.py -- create 'exports.def' helper file for lazy importer

This is a hack to aid changing the substructure of a ggui
package quickly and painless.
"""
import sys, types, pprint

modules = []
features = {}

multiple_defined = {}

template = '''## vim:ts=4:et:nowrap                     
# [Emacs: -*- python -*-]
"""export.def --- This is an exports definition file ---

this was automatically created by %(prog)s

It is invoked by a dynamic import loader in __init__.

    features      : dictionary from feature names to modules names.
    modules       : tuple of module names 
"""
#

_features = %(features)s

_modules = %(modules)s

'''

def spewout(stream=sys.stdout, modules=(), features={}):
    pp = pprint.PrettyPrinter(indent=4)
    d = { 'prog': sys.argv[0], 
          'modules': pp.pformat(tuple(modules)), 
          'features': pp.pformat(features),
    }
    stream.write(template % d)

def inspect(modulename, modules, features, multiple_defined):
    if modulename[-3:] == ".py": modulename = modulename[:-3]
    __import__(modulename)
    mod = sys.modules[modulename]
    for symbol in dir(mod):
        if symbol[:1] != '_' or (symbol == '_' and modulename == 'base'):
            obj = mod.__dict__[symbol]
            if type(obj) == types.ModuleType or symbol == "Pmw":
                if not symbol in modules:
                    modules.append(symbol)
            else:
                if features.has_key(symbol):
                    if multiple_defined.has_key(symbol):
                        multiple_defined[symbol] = multiple_defined[symbol] + \
                            " " + features[symbol]
                    else:
                        multiple_defined[symbol] = features[symbol]
                features[symbol] = modulename

if __name__ == "__main__":
    sys.path.insert(0, '.')
    if len(sys.argv) > 1:
        for arg in sys.argv[1:]:
            inspect(arg, modules, features, multiple_defined)
        outfile = sys.stdout
    else:
        import glob
        l = glob.glob("[a-z]*.py")
        print l
        for module in l:
            inspect(module, modules, features, multiple_defined)
        if multiple_defined == {}:
            outfile = open("exports.def", "w")
    if multiple_defined == {}:
        spewout(outfile, modules,  features)
    else:
        for k, v in multiple_defined.items():
            print k, "has multiple definitions in:", v, features[k]