[Python-checkins] r45497 - python/trunk/Lib/pkgutil.py python/trunk/Lib/runpy.py

phillip.eby python-checkins at python.org
Mon Apr 17 22:17:26 CEST 2006


Author: phillip.eby
Date: Mon Apr 17 22:17:25 2006
New Revision: 45497

Modified:
   python/trunk/Lib/pkgutil.py
   python/trunk/Lib/runpy.py
Log:
First phase of refactoring for runpy, pkgutil, pydoc, and setuptools
to share common PEP 302 support code, as described here:

http://mail.python.org/pipermail/python-dev/2006-April/063724.html

This revision strips all the PEP 302 emulation code from runpy,
replacing it with published API classes and functions in pkgutil,
mostly using setuptools' implementation of common functionality,
but adding features from runpy, and doing some refactoring to make
the layer pydoc needs easier to implement on top of this.

One step down, four to go, although step #4 (adding C versions of
the new APIs to 'imp') may not be able to make it in time for
alpha 2.  We'll see how that goes.


Modified: python/trunk/Lib/pkgutil.py
==============================================================================
--- python/trunk/Lib/pkgutil.py	(original)
+++ python/trunk/Lib/pkgutil.py	Mon Apr 17 22:17:25 2006
@@ -1,7 +1,271 @@
 """Utilities to support packages."""
 
+# NOTE: This module must remain compatible with Python 2.3, as it is shared
+# by setuptools for distribution with Python 2.3 and up.
+
 import os
 import sys
+import imp
+import os.path
+from types import ModuleType
+
+__all__ = [
+    'get_importer', 'iter_importers', 'get_loader', 'find_loader',
+    'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
+]
+
+def read_code(stream):
+    # This helper is needed in order for the PEP 302 emulation to
+    # correctly handle compiled files
+    import marshal
+
+    magic = stream.read(4)
+    if magic != imp.get_magic():
+        return None
+
+    stream.read(4) # Skip timestamp
+    return marshal.load(stream)
+
+
+class ImpImporter:
+    """PEP 302 Importer that wraps Python's "classic" import algorithm
+
+    ImpImporter(dirname) produces a PEP 302 importer that searches that
+    directory.  ImpImporter(None) produces a PEP 302 importer that searches
+    the current sys.path, plus any modules that are frozen or built-in.
+
+    Note that ImpImporter does not currently support being used by placement
+    on sys.meta_path.
+    """
+
+    def __init__(self, path=None):
+        self.path = path
+
+    def find_module(self, fullname, path=None):
+        # Note: we ignore 'path' argument since it is only used via meta_path
+        subname = fullname.split(".")[-1]
+        if subname != fullname and self.path is None:
+            return None
+        if self.path is None:
+            path = None
+        else:
+            path = [self.path]
+        try:
+            file, filename, etc = imp.find_module(subname, path)
+        except ImportError:
+            return None
+        return ImpLoader(fullname, file, filename, etc)
+
+
+class ImpLoader:
+    """PEP 302 Loader that wraps Python's "classic" import algorithm
+    """
+    code = source = None
+
+    def __init__(self, fullname, file, filename, etc):
+        self.file = file
+        self.filename = filename
+        self.fullname = fullname
+        self.etc = etc
+
+    def load_module(self, fullname):
+        self._reopen()
+        try:
+            mod = imp.load_module(fullname, self.file, self.filename, self.etc)
+        finally:
+            if self.file:
+                self.file.close()
+        # Note: we don't set __loader__ because we want the module to look
+        # normal; i.e. this is just a wrapper for standard import machinery
+        return mod
+
+    def get_data(self, pathname):
+        return open(pathname, "rb").read()
+
+    def _reopen(self):
+        if self.file and self.file.closed:
+            if mod_type==imp.PY_SOURCE:
+                self.file = open(self.filename, 'rU')
+            elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
+                self.file = open(self.filename, 'rb')
+
+    def _fix_name(self, fullname):
+        if fullname is None:
+            fullname = self.fullname
+        elif fullname != self.fullname:
+            raise ImportError("Loader for module %s cannot handle "
+                              "module %s" % (self.fullname, fullname))
+        return fullname
+
+    def is_package(self):
+        return self.etc[2]==imp.PKG_DIRECTORY
+
+    def get_code(self, fullname=None):
+        fullname = self._fix_name(fullname)
+        if self.code is None:
+            mod_type = self.etc[2]
+            if mod_type==imp.PY_SOURCE:
+                source = self.get_source(fullname)
+                self.code = compile(source, self.filename, 'exec')
+            elif mod_type==imp.PY_COMPILED:
+                self._reopen()
+                try:
+                    self.code = read_code(self.file)
+                finally:
+                    self.file.close()
+            elif mod_type==imp.PKG_DIRECTORY:
+                self.code = self._get_delegate().get_code()
+        return self.code
+
+    def get_source(self, fullname=None):
+        fullname = self._fix_name(fullname)
+        if self.source is None:
+            mod_type = self.etc[2]
+            if mod_type==imp.PY_SOURCE:
+                self._reopen()
+                try:
+                    self.source = self.file.read()
+                finally:
+                    self.file.close()
+            elif mod_type==imp.PY_COMPILED:
+                if os.path.exists(self.filename[:-1]):
+                    f = open(self.filename[:-1], 'rU')
+                    self.source = f.read()
+                    f.close()
+            elif mod_type==imp.PKG_DIRECTORY:
+                self.source = self._get_delegate().get_source()
+        return self.source
+
+    def _get_delegate(self):
+        return ImpImporter(self.filename).find_module('__init__')
+
+    def get_filename(self, fullname=None):
+        fullname = self._fix_name(fullname)
+        mod_type = self.etc[2]
+        if self.etc[2]==imp.PKG_DIRECTORY:
+            return self._get_delegate().get_filename()
+        elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
+            return self.filename
+        return None
+
+
+def get_importer(path_item):
+    """Retrieve a PEP 302 importer for the given path item
+
+    The returned importer is cached in sys.path_importer_cache
+    if it was newly created by a path hook.
+
+    If there is no importer, a wrapper around the basic import
+    machinery is returned. This wrapper is never inserted into
+    the importer cache (None is inserted instead).
+
+    The cache (or part of it) can be cleared manually if a
+    rescan of sys.path_hooks is necessary.
+    """
+    try:
+        importer = sys.path_importer_cache[path_item]
+    except KeyError:
+        for path_hook in sys.path_hooks:
+            try:
+                importer = path_hook(path_item)
+                break
+            except ImportError:
+                pass
+        else:
+            importer = None
+        sys.path_importer_cache.setdefault(path_item,importer)
+
+    if importer is None:
+        try:
+            importer = ImpImporter(path_item)
+        except ImportError:
+            pass
+    return importer
+
+
+def iter_importers(fullname):
+    """Yield PEP 302 importers for the given module name
+
+    If fullname contains a '.', the importers will be for the package
+    containing fullname, otherwise they will be importers for sys.meta_path,
+    sys.path, and Python's "classic" import machinery, in that order.  If
+    the named module is in a package, that package is imported as a side
+    effect of invoking this function.
+
+    Non PEP 302 mechanisms (e.g. the Windows registry) used by the
+    standard import machinery to find files in alternative locations
+    are partially supported, but are searched AFTER sys.path. Normally,
+    these locations are searched BEFORE sys.path, preventing sys.path
+    entries from shadowing them.
+
+    For this to cause a visible difference in behaviour, there must
+    be a module or package name that is accessible via both sys.path
+    and one of the non PEP 302 file system mechanisms. In this case,
+    the emulation will find the former version, while the builtin
+    import mechanism will find the latter.
+
+    Items of the following types can be affected by this discrepancy:
+        imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
+    """
+    if fullname.startswith('.'):
+        raise ImportError("Relative module names not supported")
+    if '.' in fullname:
+        # Get the containing package's __path__
+        pkg = '.'.join(fullname.split('.')[:-1])
+        if pkg not in sys.modules:
+            __import__(pkg)
+        path = getattr(sys.modules[pkg],'__path__',None) or []
+    else:
+        for importer in sys.meta_path:
+            yield importer
+        path = sys.path
+    for item in path:
+        yield get_importer(item)
+    if '.' not in fullname:
+        yield ImpImporter()
+
+
+def get_loader(module_or_name):
+    """Get a PEP 302 "loader" object for module_or_name
+
+    If the module or package is accessible via the normal import
+    mechanism, a wrapper around the relevant part of that machinery
+    is returned.  Returns None if the module cannot be found or imported.
+    If the named module is not already imported, its containing package
+    (if any) is imported, in order to establish the package __path__.
+
+    This function uses iter_importers(), and is thus subject to the same
+    limitations regarding platform-specific special import locations such
+    as the Windows registry.
+    """
+    if module_or_name in sys.modules:
+        module_or_name = sys.modules[module_or_name]
+    if isinstance(module_or_name, ModuleType):
+        module = module_or_name
+        loader = getattr(module,'__loader__',None)
+        if loader is not None:
+            return loader
+        fullname = module.__name__
+    else:
+        fullname = module_or_name
+    return find_loader(fullname)
+
+
+def find_loader(fullname):
+    """Find a PEP 302 "loader" object for fullname
+
+    If fullname contains dots, path must be the containing package's __path__.
+    Returns None if the module cannot be found or imported. This function uses
+    iter_importers(), and is thus subject to the same limitations regarding
+    platform-specific special import locations such as the Windows registry.
+    """
+    for importer in iter_importers(fullname):
+        loader = importer.find_module(fullname)
+        if loader is not None:
+            return loader
+
+    return None
+
 
 def extend_path(path, name):
     """Extend a package's path.

Modified: python/trunk/Lib/runpy.py
==============================================================================
--- python/trunk/Lib/runpy.py	(original)
+++ python/trunk/Lib/runpy.py	Mon Apr 17 22:17:25 2006
@@ -11,349 +11,15 @@
 
 import sys
 import imp
+try:
+    from imp import get_loader
+except ImportError:
+    from pkgutil import get_loader
 
 __all__ = [
     "run_module",
 ]
 
-try:
-    _get_loader = imp.get_loader
-except AttributeError:
-    # get_loader() is not provided by the imp module, so emulate it
-    # as best we can using the PEP 302 import machinery exposed since
-    # Python 2.3. The emulation isn't perfect, but the differences
-    # in the way names are shadowed shouldn't matter in practice.
-    import os.path
-    import marshal                           # Handle compiled Python files
-
-    # This helper is needed in order for the PEP 302 emulation to
-    # correctly handle compiled files
-    def _read_compiled_file(compiled_file):
-        magic = compiled_file.read(4)
-        if magic != imp.get_magic():
-            return None
-        try:
-            compiled_file.read(4) # Skip timestamp
-            return marshal.load(compiled_file)
-        except Exception:
-            return None
-
-    class _AbsoluteImporter(object):
-        """PEP 302 importer wrapper for top level import machinery"""
-        def find_module(self, mod_name, path=None):
-            if path is not None:
-                return None
-            try:
-                file, filename, mod_info = imp.find_module(mod_name)
-            except ImportError:
-                return None
-            suffix, mode, mod_type = mod_info
-            if mod_type == imp.PY_SOURCE:
-                loader = _SourceFileLoader(mod_name, file,
-                                           filename, mod_info)
-            elif mod_type == imp.PY_COMPILED:
-                loader = _CompiledFileLoader(mod_name, file,
-                                             filename, mod_info)
-            elif mod_type == imp.PKG_DIRECTORY:
-                loader = _PackageDirLoader(mod_name, file,
-                                           filename, mod_info)
-            elif mod_type == imp.C_EXTENSION:
-                loader = _FileSystemLoader(mod_name, file,
-                                           filename, mod_info)
-            else:
-                loader = _BasicLoader(mod_name, file,
-                                      filename, mod_info)
-            return loader
-
-
-    class _FileSystemImporter(object):
-        """PEP 302 importer wrapper for filesystem based imports"""
-        def __init__(self, path_item=None):
-            if path_item is not None:
-                if path_item != '' and not os.path.isdir(path_item):
-                    raise ImportError("%s is not a directory" % path_item)
-                self.path_dir = path_item
-            else:
-                raise ImportError("Filesystem importer requires "
-                                  "a directory name")
-
-        def find_module(self, mod_name, path=None):
-            if path is not None:
-                return None
-            path_dir = self.path_dir
-            if path_dir == '':
-                path_dir = os.getcwd()
-            sub_name = mod_name.rsplit(".", 1)[-1]
-            try:
-                file, filename, mod_info = imp.find_module(sub_name,
-                                                           [path_dir])
-            except ImportError:
-                return None
-            if not filename.startswith(path_dir):
-                return None
-            suffix, mode, mod_type = mod_info
-            if mod_type == imp.PY_SOURCE:
-                loader = _SourceFileLoader(mod_name, file,
-                                           filename, mod_info)
-            elif mod_type == imp.PY_COMPILED:
-                loader = _CompiledFileLoader(mod_name, file,
-                                             filename, mod_info)
-            elif mod_type == imp.PKG_DIRECTORY:
-                loader = _PackageDirLoader(mod_name, file,
-                                           filename, mod_info)
-            elif mod_type == imp.C_EXTENSION:
-                loader = _FileSystemLoader(mod_name, file,
-                                           filename, mod_info)
-            else:
-                loader = _BasicLoader(mod_name, file,
-                                      filename, mod_info)
-            return loader
-
-
-    class _BasicLoader(object):
-        """PEP 302 loader wrapper for top level import machinery"""
-        def __init__(self, mod_name, file, filename, mod_info):
-            self.mod_name = mod_name
-            self.file = file
-            self.filename = filename
-            self.mod_info = mod_info
-
-        def _fix_name(self, mod_name):
-            if mod_name is None:
-                mod_name = self.mod_name
-            elif mod_name != self.mod_name:
-                raise ImportError("Loader for module %s cannot handle "
-                                  "module %s" % (self.mod_name, mod_name))
-            return mod_name
-
-        def load_module(self, mod_name=None):
-            mod_name = self._fix_name(mod_name)
-            mod = imp.load_module(mod_name, self.file,
-                                  self.filename, self.mod_info)
-            mod.__loader__ = self  # for introspection
-            return mod
-
-        def get_code(self, mod_name=None):
-            return None
-
-        def get_source(self, mod_name=None):
-            return None
-
-        def is_package(self, mod_name=None):
-            return False
-
-        def close(self):
-            if self.file:
-                self.file.close()
-
-        def __del__(self):
-            self.close()
-
-
-    class _FileSystemLoader(_BasicLoader):
-        """PEP 302 loader wrapper for filesystem based imports"""
-        def get_code(self, mod_name=None):
-            mod_name = self._fix_name(mod_name)
-            return self._get_code(mod_name)
-
-        def get_data(self, pathname):
-            return open(pathname, "rb").read()
-
-        def get_filename(self, mod_name=None):
-            mod_name = self._fix_name(mod_name)
-            return self._get_filename(mod_name)
-
-        def get_source(self, mod_name=None):
-            mod_name = self._fix_name(mod_name)
-            return self._get_source(mod_name)
-
-        def is_package(self, mod_name=None):
-            mod_name = self._fix_name(mod_name)
-            return self._is_package(mod_name)
-
-        def _get_code(self, mod_name):
-            return None
-
-        def _get_filename(self, mod_name):
-            return self.filename
-
-        def _get_source(self, mod_name):
-            return None
-
-        def _is_package(self, mod_name):
-            return False
-
-    class _PackageDirLoader(_FileSystemLoader):
-        """PEP 302 loader wrapper for PKG_DIRECTORY directories"""
-        def _is_package(self, mod_name):
-            return True
-
-
-    class _SourceFileLoader(_FileSystemLoader):
-        """PEP 302 loader wrapper for PY_SOURCE modules"""
-        def _get_code(self, mod_name):
-            return compile(self._get_source(mod_name),
-                           self.filename, 'exec')
-
-        def _get_source(self, mod_name):
-            f = self.file
-            f.seek(0)
-            return f.read()
-
-
-    class _CompiledFileLoader(_FileSystemLoader):
-        """PEP 302 loader wrapper for PY_COMPILED modules"""
-        def _get_code(self, mod_name):
-            f = self.file
-            f.seek(0)
-            return _read_compiled_file(f)
-
-
-    def _get_importer(path_item):
-        """Retrieve a PEP 302 importer for the given path item
-
-        The returned importer is cached in sys.path_importer_cache
-        if it was newly created by a path hook.
-
-        If there is no importer, a wrapper around the basic import
-        machinery is returned. This wrapper is never inserted into
-        the importer cache (None is inserted instead).
-
-        The cache (or part of it) can be cleared manually if a
-        rescan of sys.path_hooks is necessary.
-        """
-        try:
-            importer = sys.path_importer_cache[path_item]
-        except KeyError:
-            for path_hook in sys.path_hooks:
-                try:
-                    importer = path_hook(path_item)
-                    break
-                except ImportError:
-                    pass
-            else:
-                importer = None
-            sys.path_importer_cache[path_item] = importer
-        if importer is None:
-            try:
-                importer = _FileSystemImporter(path_item)
-            except ImportError:
-                pass
-        return importer
-
-
-    def _get_path_loader(mod_name, path=None):
-        """Retrieve a PEP 302 loader using a path importer"""
-        if path is None:
-            path = sys.path
-            absolute_loader = _AbsoluteImporter().find_module(mod_name)
-            if isinstance(absolute_loader, _FileSystemLoader):
-                # Found in filesystem, so scan path hooks
-                # before accepting this one as the right one
-                loader = None
-            else:
-                # Not found in filesystem, so use top-level loader
-                loader = absolute_loader
-        else:
-            loader = absolute_loader = None
-        if loader is None:
-            for path_item in path:
-                importer = _get_importer(path_item)
-                if importer is not None:
-                    loader = importer.find_module(mod_name)
-                    if loader is not None:
-                        # Found a loader for our module
-                        break
-            else:
-                # No path hook found, so accept the top level loader
-                loader = absolute_loader
-        return loader
-
-    def _get_package(pkg_name):
-        """Retrieve a named package"""
-        pkg = __import__(pkg_name)
-        sub_pkg_names = pkg_name.split(".")
-        for sub_pkg in sub_pkg_names[1:]:
-            pkg = getattr(pkg, sub_pkg)
-        return pkg
-
-    def _get_loader(mod_name, path=None):
-        """Retrieve a PEP 302 loader for the given module or package
-
-        If the module or package is accessible via the normal import
-        mechanism, a wrapper around the relevant part of that machinery
-        is returned.
-
-        Non PEP 302 mechanisms (e.g. the Windows registry) used by the
-        standard import machinery to find files in alternative locations
-        are partially supported, but are searched AFTER sys.path. Normally,
-        these locations are searched BEFORE sys.path, preventing sys.path
-        entries from shadowing them.
-        For this to cause a visible difference in behaviour, there must
-        be a module or package name that is accessible via both sys.path
-        and one of the non PEP 302 file system mechanisms. In this case,
-        the emulation will find the former version, while the builtin
-        import mechanism will find the latter.
-        Items of the following types can be affected by this discrepancy:
-            imp.C_EXTENSION
-            imp.PY_SOURCE
-            imp.PY_COMPILED
-            imp.PKG_DIRECTORY
-        """
-        try:
-            loader = sys.modules[mod_name].__loader__
-        except (KeyError, AttributeError):
-            loader = None
-        if loader is None:
-            imp.acquire_lock()
-            try:
-                # Module not in sys.modules, or uses an unhooked loader
-                parts = mod_name.rsplit(".", 1)
-                if len(parts) == 2:
-                    # Sub package, so use parent package's path
-                    pkg_name, sub_name = parts
-                    if pkg_name and pkg_name[0] != '.':
-                        if path is not None:
-                            raise ImportError("Path argument must be None "
-                                            "for a dotted module name")
-                        pkg = _get_package(pkg_name)
-                        try:
-                            path = pkg.__path__
-                        except AttributeError:
-                            raise ImportError(pkg_name +
-                                            " is not a package")
-                    else:
-                        raise ImportError("Relative import syntax is not "
-                                          "supported by _get_loader()")
-                else:
-                    # Top level module, so stick with default path
-                    sub_name = mod_name
-
-                for importer in sys.meta_path:
-                    loader = importer.find_module(mod_name, path)
-                    if loader is not None:
-                        # Found a metahook to handle the module
-                        break
-                else:
-                    # Handling via the standard path mechanism
-                    loader = _get_path_loader(mod_name, path)
-            finally:
-                imp.release_lock()
-        return loader
-
-
-# This helper is needed due to a missing component in the PEP 302
-# loader protocol (specifically, "get_filename" is non-standard)
-def _get_filename(loader, mod_name):
-    try:
-        get_filename = loader.get_filename
-    except AttributeError:
-        return None
-    else:
-        return get_filename(mod_name)
-
-# ------------------------------------------------------------
-# Done with the import machinery emulation, on with the code!
 
 def _run_code(code, run_globals, init_globals,
               mod_name, mod_fname, mod_loader):
@@ -399,13 +65,24 @@
                          mod_name, mod_fname, mod_loader)
 
 
+# This helper is needed due to a missing component in the PEP 302
+# loader protocol (specifically, "get_filename" is non-standard)
+def _get_filename(loader, mod_name):
+    try:
+        get_filename = loader.get_filename
+    except AttributeError:
+        return None
+    else:
+        return get_filename(mod_name)
+
+
 def run_module(mod_name, init_globals=None,
                          run_name=None, alter_sys=False):
     """Execute a module's code without importing it
 
        Returns the resulting top level namespace dictionary
     """
-    loader = _get_loader(mod_name)
+    loader = get_loader(mod_name)
     if loader is None:
         raise ImportError("No module named " + mod_name)
     code = loader.get_code(mod_name)


More information about the Python-checkins mailing list