[Python-checkins] cpython: Issue #15343: Handle importlib.machinery.FileFinder instances in
nick.coghlan
python-checkins at python.org
Sun Jul 15 13:19:29 CEST 2012
http://hg.python.org/cpython/rev/9101eab6178c
changeset: 78111:9101eab6178c
user: Nick Coghlan <ncoghlan at gmail.com>
date: Sun Jul 15 21:19:18 2012 +1000
summary:
Issue #15343: Handle importlib.machinery.FileFinder instances in pkgutil.walk_packages (et al)
files:
Doc/library/pkgutil.rst | 35 +++++++++++-----------
Lib/pkgutil.py | 43 ++++++++++++++++++++++++++++
Lib/test/test_pkgutil.py | 6 +++-
Lib/test/test_runpy.py | 37 +++++++++++++++++++++++-
Misc/NEWS | 4 ++
5 files changed, 106 insertions(+), 19 deletions(-)
diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst
--- a/Doc/library/pkgutil.rst
+++ b/Doc/library/pkgutil.rst
@@ -81,7 +81,7 @@
.. versionchanged:: 3.3
Updated to be based directly on :mod:`importlib` rather than relying
- on a package internal PEP 302 import emulation.
+ on the package internal PEP 302 import emulation.
.. function:: get_importer(path_item)
@@ -96,7 +96,7 @@
.. versionchanged:: 3.3
Updated to be based directly on :mod:`importlib` rather than relying
- on a package internal PEP 302 import emulation.
+ on the package internal PEP 302 import emulation.
.. function:: get_loader(module_or_name)
@@ -115,7 +115,7 @@
.. versionchanged:: 3.3
Updated to be based directly on :mod:`importlib` rather than relying
- on a package internal PEP 302 import emulation.
+ on the package internal PEP 302 import emulation.
.. function:: iter_importers(fullname='')
@@ -133,12 +133,12 @@
.. versionchanged:: 3.3
Updated to be based directly on :mod:`importlib` rather than relying
- on a package internal PEP 302 import emulation.
+ on the package internal PEP 302 import emulation.
.. function:: iter_modules(path=None, prefix='')
- Yields ``(module_loader, name, ispkg)`` for all submodules on *path*, or, if
+ Yields ``(module_finder, name, ispkg)`` for all submodules on *path*, or, if
path is ``None``, all top-level modules on ``sys.path``.
*path* should be either ``None`` or a list of paths to look for modules in.
@@ -146,19 +146,19 @@
*prefix* is a string to output on the front of every module name on output.
.. note::
- Only works with a :term:`finder` which defines an ``iter_modules()``
- method, which is non-standard but implemented by classes defined in this
- module.
+ Only works for a :term:`finder` which defines an ``iter_modules()``
+ method. This interface is non-standard, so the module also provides
+ implementations for :class:`importlib.machinery.FileFinder` and
+ :class:`zipimport.zipimporter`.
.. versionchanged:: 3.3
- As of Python 3.3, the import system provides finders by default, but they
- do not include the non-standard ``iter_modules()`` method required by this
- function.
+ Updated to be based directly on :mod:`importlib` rather than relying
+ on the package internal PEP 302 import emulation.
.. function:: walk_packages(path=None, prefix='', onerror=None)
- Yields ``(module_loader, name, ispkg)`` for all modules recursively on
+ Yields ``(module_finder, name, ispkg)`` for all modules recursively on
*path*, or, if path is ``None``, all accessible modules.
*path* should be either ``None`` or a list of paths to look for modules in.
@@ -184,13 +184,14 @@
walk_packages(ctypes.__path__, ctypes.__name__ + '.')
.. note::
- Only works for a :term:`finder` which define an ``iter_modules()`` method,
- which is non-standard but implemented by classes defined in this module.
+ Only works for a :term:`finder` which defines an ``iter_modules()``
+ method. This interface is non-standard, so the module also provides
+ implementations for :class:`importlib.machinery.FileFinder` and
+ :class:`zipimport.zipimporter`.
.. versionchanged:: 3.3
- As of Python 3.3, the import system provides finders by default, but they
- do not include the non-standard ``iter_modules()`` method required by this
- function.
+ Updated to be based directly on :mod:`importlib` rather than relying
+ on the package internal PEP 302 import emulation.
.. function:: get_data(package, resource)
diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py
--- a/Lib/pkgutil.py
+++ b/Lib/pkgutil.py
@@ -157,6 +157,49 @@
iter_importer_modules = simplegeneric(iter_importer_modules)
+# Implement a file walker for the normal importlib path hook
+def _iter_file_finder_modules(importer, prefix=''):
+ if importer.path is None or not os.path.isdir(importer.path):
+ return
+
+ yielded = {}
+ import inspect
+ try:
+ filenames = os.listdir(importer.path)
+ except OSError:
+ # ignore unreadable directories like import does
+ filenames = []
+ filenames.sort() # handle packages before same-named modules
+
+ for fn in filenames:
+ modname = inspect.getmodulename(fn)
+ if modname=='__init__' or modname in yielded:
+ continue
+
+ path = os.path.join(importer.path, fn)
+ ispkg = False
+
+ if not modname and os.path.isdir(path) and '.' not in fn:
+ modname = fn
+ try:
+ dircontents = os.listdir(path)
+ except OSError:
+ # ignore unreadable directories like import does
+ dircontents = []
+ for fn in dircontents:
+ subname = inspect.getmodulename(fn)
+ if subname=='__init__':
+ ispkg = True
+ break
+ else:
+ continue # not a package
+
+ if modname and '.' not in modname:
+ yielded[modname] = 1
+ yield prefix + modname, ispkg
+
+iter_importer_modules.register(
+ importlib.machinery.FileFinder, _iter_file_finder_modules)
class ImpImporter:
"""PEP 302 Importer that wraps Python's "classic" import algorithm
diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py
--- a/Lib/test/test_pkgutil.py
+++ b/Lib/test/test_pkgutil.py
@@ -9,7 +9,11 @@
import shutil
import zipfile
-
+# Note: pkgutil.walk_packages is currently tested in test_runpy. This is
+# a hack to get a major issue resolved for 3.3b2. Longer term, it should
+# be moved back here, perhaps by factoring out the helper code for
+# creating interesting package layouts to a separate module.
+# Issue #15348 declares this is indeed a dodgy hack ;)
class PkgutilTests(unittest.TestCase):
diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py
--- a/Lib/test/test_runpy.py
+++ b/Lib/test/test_runpy.py
@@ -13,6 +13,7 @@
from test.script_helper import (
make_pkg, make_script, make_zip_pkg, make_zip_script, temp_dir)
+
import runpy
from runpy import _run_code, _run_module_code, run_module, run_path
# Note: This module can't safely test _run_module_as_main as it
@@ -148,7 +149,7 @@
mod_package)
self.check_code_execution(create_ns, expected_ns)
-
+# TODO: Use self.addCleanup to get rid of a lot of try-finally blocks
class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
"""Unit tests for runpy.run_module"""
@@ -413,6 +414,40 @@
finally:
self._del_pkg(pkg_dir, depth, mod_name)
+ def test_pkgutil_walk_packages(self):
+ # This is a dodgy hack to use the test_runpy infrastructure to test
+ # issue #15343. Issue #15348 declares this is indeed a dodgy hack ;)
+ import pkgutil
+ max_depth = 4
+ base_name = "__runpy_pkg__"
+ package_suffixes = ["uncle", "uncle.cousin"]
+ module_suffixes = ["uncle.cousin.nephew", base_name + ".sibling"]
+ expected_packages = set()
+ expected_modules = set()
+ for depth in range(1, max_depth):
+ pkg_name = ".".join([base_name] * depth)
+ expected_packages.add(pkg_name)
+ for name in package_suffixes:
+ expected_packages.add(pkg_name + "." + name)
+ for name in module_suffixes:
+ expected_modules.add(pkg_name + "." + name)
+ pkg_name = ".".join([base_name] * max_depth)
+ expected_packages.add(pkg_name)
+ expected_modules.add(pkg_name + ".runpy_test")
+ pkg_dir, mod_fname, mod_name = (
+ self._make_pkg("", max_depth))
+ self.addCleanup(self._del_pkg, pkg_dir, max_depth, mod_name)
+ for depth in range(2, max_depth+1):
+ self._add_relative_modules(pkg_dir, "", depth)
+ for finder, mod_name, ispkg in pkgutil.walk_packages([pkg_dir]):
+ self.assertIsInstance(finder,
+ importlib.machinery.FileFinder)
+ if ispkg:
+ expected_packages.remove(mod_name)
+ else:
+ expected_modules.remove(mod_name)
+ self.assertEqual(len(expected_packages), 0, expected_packages)
+ self.assertEqual(len(expected_modules), 0, expected_modules)
class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
"""Unit tests for runpy.run_path"""
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -38,6 +38,10 @@
Library
-------
+- Issue #15343: pkgutil now includes an iter_importer_modules implementation
+ for importlib.machinery.FileFinder (similar to the way it already handled
+ zipimport.zipimporter)
+
- Issue #15314: runpy now sets __main__.__loader__ correctly
- Issue #15357: The import emulation in pkgutil is now deprecated. pkgutil
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list