[Python-checkins] bpo-43428: Sync with importlib_metadata 3.7. (GH-24782)
jaraco
webhook-mailer at python.org
Sat Mar 13 11:32:00 EST 2021
https://github.com/python/cpython/commit/f917efccf8d5aa2b8315d2a832a520339e668187
commit: f917efccf8d5aa2b8315d2a832a520339e668187
branch: master
author: Jason R. Coombs <jaraco at jaraco.com>
committer: jaraco <jaraco at jaraco.com>
date: 2021-03-13T11:31:45-05:00
summary:
bpo-43428: Sync with importlib_metadata 3.7. (GH-24782)
* bpo-43428: Sync with importlib_metadata 3.7.2 (67234b6)
* Add blurb
* Reformat blurb to create separate paragraphs for each change included.
files:
A Lib/importlib/_itertools.py
A Misc/NEWS.d/next/Library/2021-03-07-18-54-39.bpo-43428.br0XmX.rst
M Doc/library/importlib.metadata.rst
M Lib/importlib/metadata.py
M Lib/test/test_importlib/fixtures.py
M Lib/test/test_importlib/test_main.py
M Lib/test/test_importlib/test_metadata_api.py
M Lib/test/test_importlib/test_zip.py
diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst
index 7f154ea02cc4f..ffce1bae979b5 100644
--- a/Doc/library/importlib.metadata.rst
+++ b/Doc/library/importlib.metadata.rst
@@ -74,18 +74,20 @@ This package provides the following functionality via its public API.
Entry points
------------
-The ``entry_points()`` function returns a dictionary of all entry points,
-keyed by group. Entry points are represented by ``EntryPoint`` instances;
+The ``entry_points()`` function returns a collection of entry points.
+Entry points are represented by ``EntryPoint`` instances;
each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and
a ``.load()`` method to resolve the value. There are also ``.module``,
``.attr``, and ``.extras`` attributes for getting the components of the
``.value`` attribute::
>>> eps = entry_points() # doctest: +SKIP
- >>> list(eps) # doctest: +SKIP
+ >>> sorted(eps.groups) # doctest: +SKIP
['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation']
- >>> scripts = eps['console_scripts'] # doctest: +SKIP
- >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] # doctest: +SKIP
+ >>> scripts = eps.select(group='console_scripts') # doctest: +SKIP
+ >>> 'wheel' in scripts.names # doctest: +SKIP
+ True
+ >>> wheel = scripts['wheel'] # doctest: +SKIP
>>> wheel # doctest: +SKIP
EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts')
>>> wheel.module # doctest: +SKIP
@@ -187,6 +189,17 @@ function::
["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"]
+Package distributions
+---------------------
+
+A convience method to resolve the distribution or
+distributions (in the case of a namespace package) for top-level
+Python packages or modules::
+
+ >>> packages_distributions()
+ {'importlib_metadata': ['importlib-metadata'], 'yaml': ['PyYAML'], 'jaraco': ['jaraco.classes', 'jaraco.functools'], ...}
+
+
Distributions
=============
diff --git a/Lib/importlib/_itertools.py b/Lib/importlib/_itertools.py
new file mode 100644
index 0000000000000..dd45f2f096630
--- /dev/null
+++ b/Lib/importlib/_itertools.py
@@ -0,0 +1,19 @@
+from itertools import filterfalse
+
+
+def unique_everseen(iterable, key=None):
+ "List unique elements, preserving order. Remember all elements ever seen."
+ # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+ # unique_everseen('ABBCcAD', str.lower) --> A B C D
+ seen = set()
+ seen_add = seen.add
+ if key is None:
+ for element in filterfalse(seen.__contains__, iterable):
+ seen_add(element)
+ yield element
+ else:
+ for element in iterable:
+ k = key(element)
+ if k not in seen:
+ seen_add(k)
+ yield element
diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py
index 36bb42ee21d9c..8a731858cad70 100644
--- a/Lib/importlib/metadata.py
+++ b/Lib/importlib/metadata.py
@@ -4,20 +4,24 @@
import csv
import sys
import email
+import inspect
import pathlib
import zipfile
import operator
+import warnings
import functools
import itertools
import posixpath
-import collections
+import collections.abc
+
+from ._itertools import unique_everseen
from configparser import ConfigParser
from contextlib import suppress
from importlib import import_module
from importlib.abc import MetaPathFinder
from itertools import starmap
-from typing import Any, List, Optional, Protocol, TypeVar, Union
+from typing import Any, List, Mapping, Optional, Protocol, TypeVar, Union
__all__ = [
@@ -120,18 +124,19 @@ def _from_text(cls, text):
config.read_string(text)
return cls._from_config(config)
- @classmethod
- def _from_text_for(cls, text, dist):
- return (ep._for(dist) for ep in cls._from_text(text))
-
def _for(self, dist):
self.dist = dist
return self
def __iter__(self):
"""
- Supply iter so one may construct dicts of EntryPoints easily.
+ Supply iter so one may construct dicts of EntryPoints by name.
"""
+ msg = (
+ "Construction of dict of EntryPoints is deprecated in "
+ "favor of EntryPoints."
+ )
+ warnings.warn(msg, DeprecationWarning)
return iter((self.name, self))
def __reduce__(self):
@@ -140,6 +145,143 @@ def __reduce__(self):
(self.name, self.value, self.group),
)
+ def matches(self, **params):
+ attrs = (getattr(self, param) for param in params)
+ return all(map(operator.eq, params.values(), attrs))
+
+
+class EntryPoints(tuple):
+ """
+ An immutable collection of selectable EntryPoint objects.
+ """
+
+ __slots__ = ()
+
+ def __getitem__(self, name): # -> EntryPoint:
+ try:
+ return next(iter(self.select(name=name)))
+ except StopIteration:
+ raise KeyError(name)
+
+ def select(self, **params):
+ return EntryPoints(ep for ep in self if ep.matches(**params))
+
+ @property
+ def names(self):
+ return set(ep.name for ep in self)
+
+ @property
+ def groups(self):
+ """
+ For coverage while SelectableGroups is present.
+ >>> EntryPoints().groups
+ set()
+ """
+ return set(ep.group for ep in self)
+
+ @classmethod
+ def _from_text_for(cls, text, dist):
+ return cls(ep._for(dist) for ep in EntryPoint._from_text(text))
+
+
+def flake8_bypass(func):
+ is_flake8 = any('flake8' in str(frame.filename) for frame in inspect.stack()[:5])
+ return func if not is_flake8 else lambda: None
+
+
+class Deprecated:
+ """
+ Compatibility add-in for mapping to indicate that
+ mapping behavior is deprecated.
+
+ >>> recwarn = getfixture('recwarn')
+ >>> class DeprecatedDict(Deprecated, dict): pass
+ >>> dd = DeprecatedDict(foo='bar')
+ >>> dd.get('baz', None)
+ >>> dd['foo']
+ 'bar'
+ >>> list(dd)
+ ['foo']
+ >>> list(dd.keys())
+ ['foo']
+ >>> 'foo' in dd
+ True
+ >>> list(dd.values())
+ ['bar']
+ >>> len(recwarn)
+ 1
+ """
+
+ _warn = functools.partial(
+ warnings.warn,
+ "SelectableGroups dict interface is deprecated. Use select.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
+ def __getitem__(self, name):
+ self._warn()
+ return super().__getitem__(name)
+
+ def get(self, name, default=None):
+ flake8_bypass(self._warn)()
+ return super().get(name, default)
+
+ def __iter__(self):
+ self._warn()
+ return super().__iter__()
+
+ def __contains__(self, *args):
+ self._warn()
+ return super().__contains__(*args)
+
+ def keys(self):
+ self._warn()
+ return super().keys()
+
+ def values(self):
+ self._warn()
+ return super().values()
+
+
+class SelectableGroups(dict):
+ """
+ A backward- and forward-compatible result from
+ entry_points that fully implements the dict interface.
+ """
+
+ @classmethod
+ def load(cls, eps):
+ by_group = operator.attrgetter('group')
+ ordered = sorted(eps, key=by_group)
+ grouped = itertools.groupby(ordered, by_group)
+ return cls((group, EntryPoints(eps)) for group, eps in grouped)
+
+ @property
+ def _all(self):
+ """
+ Reconstruct a list of all entrypoints from the groups.
+ """
+ return EntryPoints(itertools.chain.from_iterable(self.values()))
+
+ @property
+ def groups(self):
+ return self._all.groups
+
+ @property
+ def names(self):
+ """
+ for coverage:
+ >>> SelectableGroups().names
+ set()
+ """
+ return self._all.names
+
+ def select(self, **params):
+ if not params:
+ return self
+ return self._all.select(**params)
+
class PackagePath(pathlib.PurePosixPath):
"""A reference to a path in a package"""
@@ -296,7 +438,7 @@ def version(self):
@property
def entry_points(self):
- return list(EntryPoint._from_text_for(self.read_text('entry_points.txt'), self))
+ return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self)
@property
def files(self):
@@ -485,15 +627,22 @@ class Prepared:
"""
normalized = None
- suffixes = '.dist-info', '.egg-info'
+ suffixes = 'dist-info', 'egg-info'
exact_matches = [''][:0]
+ egg_prefix = ''
+ versionless_egg_name = ''
def __init__(self, name):
self.name = name
if name is None:
return
self.normalized = self.normalize(name)
- self.exact_matches = [self.normalized + suffix for suffix in self.suffixes]
+ self.exact_matches = [
+ self.normalized + '.' + suffix for suffix in self.suffixes
+ ]
+ legacy_normalized = self.legacy_normalize(self.name)
+ self.egg_prefix = legacy_normalized + '-'
+ self.versionless_egg_name = legacy_normalized + '.egg'
@staticmethod
def normalize(name):
@@ -512,8 +661,9 @@ def legacy_normalize(name):
def matches(self, cand, base):
low = cand.lower()
- pre, ext = os.path.splitext(low)
- name, sep, rest = pre.partition('-')
+ # rpartition is faster than splitext and suitable for this purpose.
+ pre, _, ext = low.rpartition('.')
+ name, _, rest = pre.partition('-')
return (
low in self.exact_matches
or ext in self.suffixes
@@ -524,12 +674,9 @@ def matches(self, cand, base):
)
def is_egg(self, base):
- normalized = self.legacy_normalize(self.name or '')
- prefix = normalized + '-' if normalized else ''
- versionless_egg_name = normalized + '.egg' if self.name else ''
return (
- base == versionless_egg_name
- or base.startswith(prefix)
+ base == self.versionless_egg_name
+ or base.startswith(self.egg_prefix)
and base.endswith('.egg')
)
@@ -551,8 +698,9 @@ def find_distributions(cls, context=DistributionFinder.Context()):
@classmethod
def _search_paths(cls, name, paths):
"""Find metadata directories in paths heuristically."""
+ prepared = Prepared(name)
return itertools.chain.from_iterable(
- path.search(Prepared(name)) for path in map(FastPath, paths)
+ path.search(prepared) for path in map(FastPath, paths)
)
@@ -617,16 +765,28 @@ def version(distribution_name):
return distribution(distribution_name).version
-def entry_points():
+def entry_points(**params) -> Union[EntryPoints, SelectableGroups]:
"""Return EntryPoint objects for all installed packages.
- :return: EntryPoint objects for all installed packages.
+ Pass selection parameters (group or name) to filter the
+ result to entry points matching those properties (see
+ EntryPoints.select()).
+
+ For compatibility, returns ``SelectableGroups`` object unless
+ selection parameters are supplied. In the future, this function
+ will return ``EntryPoints`` instead of ``SelectableGroups``
+ even when no selection parameters are supplied.
+
+ For maximum future compatibility, pass selection parameters
+ or invoke ``.select`` with parameters on the result.
+
+ :return: EntryPoints or SelectableGroups for all installed packages.
"""
- eps = itertools.chain.from_iterable(dist.entry_points for dist in distributions())
- by_group = operator.attrgetter('group')
- ordered = sorted(eps, key=by_group)
- grouped = itertools.groupby(ordered, by_group)
- return {group: tuple(eps) for group, eps in grouped}
+ unique = functools.partial(unique_everseen, key=operator.attrgetter('name'))
+ eps = itertools.chain.from_iterable(
+ dist.entry_points for dist in unique(distributions())
+ )
+ return SelectableGroups.load(eps).select(**params)
def files(distribution_name):
@@ -646,3 +806,19 @@ def requires(distribution_name):
packaging.requirement.Requirement.
"""
return distribution(distribution_name).requires
+
+
+def packages_distributions() -> Mapping[str, List[str]]:
+ """
+ Return a mapping of top-level packages to their
+ distributions.
+
+ >>> pkgs = packages_distributions()
+ >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values())
+ True
+ """
+ pkg_to_dist = collections.defaultdict(list)
+ for dist in distributions():
+ for pkg in (dist.read_text('top_level.txt') or '').split():
+ pkg_to_dist[pkg].append(dist.metadata['Name'])
+ return dict(pkg_to_dist)
diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py
index acf6bc87c74e9..429313e9efb98 100644
--- a/Lib/test/test_importlib/fixtures.py
+++ b/Lib/test/test_importlib/fixtures.py
@@ -5,7 +5,6 @@
import tempfile
import textwrap
import contextlib
-import unittest
from test.support.os_helper import FS_NONASCII
from typing import Dict, Union
@@ -221,7 +220,6 @@ def setUp(self):
build_files(self.files)
-
def build_files(file_defs, prefix=pathlib.Path()):
"""Build a set of files/directories, as described by the
@@ -260,9 +258,6 @@ class FileBuilder:
def unicode_filename(self):
return FS_NONASCII or self.skip("File system does not support non-ascii.")
- def skip(self, reason):
- raise unittest.SkipTest(reason)
-
def DALS(str):
"Dedent and left-strip"
diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py
index c937361e8fdd0..02e8a573437a5 100644
--- a/Lib/test/test_importlib/test_main.py
+++ b/Lib/test/test_importlib/test_main.py
@@ -3,6 +3,7 @@
import pickle
import textwrap
import unittest
+import warnings
import importlib.metadata
try:
@@ -58,13 +59,11 @@ def test_import_nonexistent_module(self):
importlib.import_module('does_not_exist')
def test_resolve(self):
- entries = dict(entry_points()['entries'])
- ep = entries['main']
+ ep = entry_points(group='entries')['main']
self.assertEqual(ep.load().__name__, "main")
def test_entrypoint_with_colon_in_name(self):
- entries = dict(entry_points()['entries'])
- ep = entries['ns:sub']
+ ep = entry_points(group='entries')['ns:sub']
self.assertEqual(ep.value, 'mod:main')
def test_resolve_without_attr(self):
@@ -250,7 +249,8 @@ def test_json_dump(self):
json should not expect to be able to dump an EntryPoint
"""
with self.assertRaises(Exception):
- json.dumps(self.ep)
+ with warnings.catch_warnings(record=True):
+ json.dumps(self.ep)
def test_module(self):
assert self.ep.module == 'value'
diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/test_metadata_api.py
index df00ae9375b86..a0f9d511f8433 100644
--- a/Lib/test/test_importlib/test_metadata_api.py
+++ b/Lib/test/test_importlib/test_metadata_api.py
@@ -1,6 +1,7 @@
import re
import textwrap
import unittest
+import warnings
from . import fixtures
from importlib.metadata import (
@@ -64,18 +65,97 @@ def test_read_text(self):
self.assertEqual(top_level.read_text(), 'mod\n')
def test_entry_points(self):
- entries = dict(entry_points()['entries'])
+ eps = entry_points()
+ assert 'entries' in eps.groups
+ entries = eps.select(group='entries')
+ assert 'main' in entries.names
ep = entries['main']
self.assertEqual(ep.value, 'mod:main')
self.assertEqual(ep.extras, [])
def test_entry_points_distribution(self):
- entries = dict(entry_points()['entries'])
+ entries = entry_points(group='entries')
for entry in ("main", "ns:sub"):
ep = entries[entry]
self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg'))
self.assertEqual(ep.dist.version, "1.0.0")
+ def test_entry_points_unique_packages(self):
+ """
+ Entry points should only be exposed for the first package
+ on sys.path with a given name.
+ """
+ alt_site_dir = self.fixtures.enter_context(fixtures.tempdir())
+ self.fixtures.enter_context(self.add_sys_path(alt_site_dir))
+ alt_pkg = {
+ "distinfo_pkg-1.1.0.dist-info": {
+ "METADATA": """
+ Name: distinfo-pkg
+ Version: 1.1.0
+ """,
+ "entry_points.txt": """
+ [entries]
+ main = mod:altmain
+ """,
+ },
+ }
+ fixtures.build_files(alt_pkg, alt_site_dir)
+ entries = entry_points(group='entries')
+ assert not any(
+ ep.dist.name == 'distinfo-pkg' and ep.dist.version == '1.0.0'
+ for ep in entries
+ )
+ # ns:sub doesn't exist in alt_pkg
+ assert 'ns:sub' not in entries
+
+ def test_entry_points_missing_name(self):
+ with self.assertRaises(KeyError):
+ entry_points(group='entries')['missing']
+
+ def test_entry_points_missing_group(self):
+ assert entry_points(group='missing') == ()
+
+ def test_entry_points_dict_construction(self):
+ """
+ Prior versions of entry_points() returned simple lists and
+ allowed casting those lists into maps by name using ``dict()``.
+ Capture this now deprecated use-case.
+ """
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.filterwarnings("default", category=DeprecationWarning)
+ eps = dict(entry_points(group='entries'))
+
+ assert 'main' in eps
+ assert eps['main'] == entry_points(group='entries')['main']
+
+ # check warning
+ expected = next(iter(caught))
+ assert expected.category is DeprecationWarning
+ assert "Construction of dict of EntryPoints is deprecated" in str(expected)
+
+ def test_entry_points_groups_getitem(self):
+ """
+ Prior versions of entry_points() returned a dict. Ensure
+ that callers using '.__getitem__()' are supported but warned to
+ migrate.
+ """
+ with warnings.catch_warnings(record=True):
+ entry_points()['entries'] == entry_points(group='entries')
+
+ with self.assertRaises(KeyError):
+ entry_points()['missing']
+
+ def test_entry_points_groups_get(self):
+ """
+ Prior versions of entry_points() returned a dict. Ensure
+ that callers using '.get()' are supported but warned to
+ migrate.
+ """
+ with warnings.catch_warnings(record=True):
+ entry_points().get('missing', 'default') == 'default'
+ entry_points().get('entries', 'default') == entry_points()['entries']
+ entry_points().get('missing', ()) == ()
+
def test_metadata_for_this_package(self):
md = metadata('egginfo-pkg')
assert md['author'] == 'Steven Ma'
diff --git a/Lib/test/test_importlib/test_zip.py b/Lib/test/test_importlib/test_zip.py
index 74783fc98b99b..83e041385e0e8 100644
--- a/Lib/test/test_importlib/test_zip.py
+++ b/Lib/test/test_importlib/test_zip.py
@@ -41,7 +41,7 @@ def test_zip_version_does_not_match(self):
version('definitely-not-installed')
def test_zip_entry_points(self):
- scripts = dict(entry_points()['console_scripts'])
+ scripts = entry_points(group='console_scripts')
entry_point = scripts['example']
self.assertEqual(entry_point.value, 'example:main')
entry_point = scripts['Example']
diff --git a/Misc/NEWS.d/next/Library/2021-03-07-18-54-39.bpo-43428.br0XmX.rst b/Misc/NEWS.d/next/Library/2021-03-07-18-54-39.bpo-43428.br0XmX.rst
new file mode 100644
index 0000000000000..38361070b39f7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-03-07-18-54-39.bpo-43428.br0XmX.rst
@@ -0,0 +1,17 @@
+Include changes from `importlib_metadata 3.7
+<https://importlib-metadata.readthedocs.io/en/latest/history.html#v3-7-0>`_:
+
+Performance enhancements to distribution discovery.
+
+``entry_points`` only returns unique distributions.
+
+Introduces new ``EntryPoints`` object
+for containing a set of entry points with convenience methods for selecting
+entry points by group or name. ``entry_points`` now returns this object if
+selection parameters are supplied but continues to return a dict object for
+compatibility. Users are encouraged to rely on the selection interface. The
+dict object result is likely to be deprecated in the future.
+
+Added
+packages_distributions function to return a mapping of packages to the
+distributions that provide them.
More information about the Python-checkins
mailing list