Mailman 3 July 2021 - Python-checkins

bpo-30256: Add manager_owned keyword arg to AutoProxy (GH-16341)
by gvanrossum 01 Jul '21

01 Jul '21

https://github.com/python/cpython/commit/85b920498b42c69185540ecc2f5c4907fd… commit: 85b920498b42c69185540ecc2f5c4907fd38d877 branch: main author: finefoot <33361833+finefoot(a)users.noreply.github.com> committer: gvanrossum <gvanrossum(a)gmail.com> date: 2021-07-01T20:45:02-07:00 summary: bpo-30256: Add manager_owned keyword arg to AutoProxy (GH-16341) Co-authored-by: Jordan Speicher <jordan(a)jspeicher.com> files: A Misc/NEWS.d/next/Library/2019-09-25-13-54-41.bpo-30256.wBkzox.rst A Misc/NEWS.d/next/Tests/2019-09-25-18-10-10.bpo-30256.A5i76Q.rst M Lib/multiprocessing/managers.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 6ee9521c76b48f..b981b0e1cb8ed8 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -967,7 +967,7 @@ def MakeProxyType(name, exposed, _cache={}): def AutoProxy(token, serializer, manager=None, authkey=None, - exposed=None, incref=True): + exposed=None, incref=True, manager_owned=False): ''' Return an auto-proxy for `token` ''' @@ -987,7 +987,7 @@ def AutoProxy(token, serializer, manager=None, authkey=None, ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed) proxy = ProxyType(token, serializer, manager=manager, authkey=authkey, - incref=incref) + incref=incref, manager_owned=manager_owned) proxy._isauto = True return proxy diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index ead92cfa2abfea..4c4da24a30c371 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2286,6 +2286,16 @@ def test_dict_proxy_nested(self): self.assertIsInstance(outer[0], list) # Not a ListProxy self.assertEqual(outer[-1][-1]['feed'], 3) + def test_nested_queue(self): + a = self.list() # Test queue inside list + a.append(self.Queue()) + a[0].put(123) + self.assertEqual(a[0].get(), 123) + b = self.dict() # Test queue inside dict + b[0] = self.Queue() + b[0].put(456) + self.assertEqual(b[0].get(), 456) + def test_namespace(self): n = self.Namespace() n.name = 'Bob' diff --git a/Misc/NEWS.d/next/Library/2019-09-25-13-54-41.bpo-30256.wBkzox.rst b/Misc/NEWS.d/next/Library/2019-09-25-13-54-41.bpo-30256.wBkzox.rst new file mode 100644 index 00000000000000..4490803f32642e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-09-25-13-54-41.bpo-30256.wBkzox.rst @@ -0,0 +1 @@ +Pass multiprocessing BaseProxy argument `manager_owned` through AutoProxy diff --git a/Misc/NEWS.d/next/Tests/2019-09-25-18-10-10.bpo-30256.A5i76Q.rst b/Misc/NEWS.d/next/Tests/2019-09-25-18-10-10.bpo-30256.A5i76Q.rst new file mode 100644 index 00000000000000..4a7cfd52fcea22 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2019-09-25-18-10-10.bpo-30256.A5i76Q.rst @@ -0,0 +1,2 @@ +Add test for nested queues when using ``multiprocessing`` shared objects +``AutoProxy[Queue]`` inside ``ListProxy`` and ``DictProxy``

1 0

bpo-43425: Update setup.py not to use distutils.log (GH-26969)
by miss-islington 01 Jul '21

01 Jul '21

https://github.com/python/cpython/commit/956f1fc89475f6c8252a9eb0689bcf0eb6… commit: 956f1fc89475f6c8252a9eb0689bcf0eb69c15ea branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2021-07-01T19:05:11-07:00 summary: bpo-43425: Update setup.py not to use distutils.log (GH-26969) (cherry picked from commit a7e251b07aa5f1a858a3e791a30dea78e93ab728) Co-authored-by: Dong-hee Na <donghee.na(a)python.org> files: M setup.py diff --git a/setup.py b/setup.py index 3857e6887a929..9343d490c1433 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ import importlib._bootstrap import importlib.machinery import importlib.util +import logging import os import re import sys @@ -44,7 +45,6 @@ DeprecationWarning ) - from distutils import log from distutils.command.build_ext import build_ext from distutils.command.build_scripts import build_scripts from distutils.command.install import install @@ -64,6 +64,10 @@ LIST_MODULE_NAMES = False +logging.basicConfig(format='%(message)s', level=logging.INFO) +log = logging.getLogger('setup') + + def get_platform(): # Cross compiling if "_PYTHON_HOST_PLATFORM" in os.environ: @@ -241,6 +245,7 @@ def grep_headers_for(function, headers): return True return False + def find_file(filename, std_dirs, paths): """Searches for the directory where a given file is located, and returns a possibly-empty list of additional directories, or None @@ -259,23 +264,23 @@ def find_file(filename, std_dirs, paths): sysroot = macosx_sdk_root() # Check the standard locations - for dir in std_dirs: - f = os.path.join(dir, filename) + for dir_ in std_dirs: + f = os.path.join(dir_, filename) - if MACOS and is_macosx_sdk_path(dir): - f = os.path.join(sysroot, dir[1:], filename) + if MACOS and is_macosx_sdk_path(dir_): + f = os.path.join(sysroot, dir_[1:], filename) if os.path.exists(f): return [] # Check the additional directories - for dir in paths: - f = os.path.join(dir, filename) + for dir_ in paths: + f = os.path.join(dir_, filename) - if MACOS and is_macosx_sdk_path(dir): - f = os.path.join(sysroot, dir[1:], filename) + if MACOS and is_macosx_sdk_path(dir_): + f = os.path.join(sysroot, dir_[1:], filename) if os.path.exists(f): - return [dir] + return [dir_] # Not found anywhere return None @@ -333,6 +338,7 @@ def find_library_file(compiler, libname, std_dirs, paths): else: assert False, "Internal error: Path not found in std_dirs or paths" + def validate_tzpath(): base_tzpath = sysconfig.get_config_var('TZPATH') if not base_tzpath: @@ -345,15 +351,16 @@ def validate_tzpath(): + f'found:\n{tzpaths!r}\nwith invalid paths:\n' + f'{bad_paths!r}') + def find_module_file(module, dirlist): """Find a module in a set of possible folders. If it is not found return the unadorned filename""" - list = find_file(module, [], dirlist) - if not list: + dirs = find_file(module, [], dirlist) + if not dirs: return module - if len(list) > 1: - log.info("WARNING: multiple copies of %s found", module) - return os.path.join(list[0], module) + if len(dirs) > 1: + log.info(f"WARNING: multiple copies of {module} found") + return os.path.join(dirs[0], module) class PyBuildExt(build_ext): @@ -2667,7 +2674,7 @@ def copy_scripts(self): newfilename = filename + fullversion else: newfilename = filename + minoronly - log.info('renaming %s to %s', filename, newfilename) + log.info(f'renaming {filename} to {newfilename}') os.rename(filename, newfilename) newoutfiles.append(newfilename) if filename in updated_files:

1 0

bpo-43425: Update setup.py not to use distutils.log (GH-26969)
by corona10 01 Jul '21

01 Jul '21

https://github.com/python/cpython/commit/a7e251b07aa5f1a858a3e791a30dea78e9… commit: a7e251b07aa5f1a858a3e791a30dea78e93ab728 branch: main author: Dong-hee Na <donghee.na(a)python.org> committer: corona10 <donghee.na92(a)gmail.com> date: 2021-07-02T10:41:54+09:00 summary: bpo-43425: Update setup.py not to use distutils.log (GH-26969) files: M setup.py diff --git a/setup.py b/setup.py index ce71a966a8194..bc2ea16089569 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ import importlib._bootstrap import importlib.machinery import importlib.util +import logging import os import re import sys @@ -44,7 +45,6 @@ DeprecationWarning ) - from distutils import log from distutils.command.build_ext import build_ext from distutils.command.build_scripts import build_scripts from distutils.command.install import install @@ -64,6 +64,10 @@ LIST_MODULE_NAMES = False +logging.basicConfig(format='%(message)s', level=logging.INFO) +log = logging.getLogger('setup') + + def get_platform(): # Cross compiling if "_PYTHON_HOST_PLATFORM" in os.environ: @@ -241,6 +245,7 @@ def grep_headers_for(function, headers): return True return False + def find_file(filename, std_dirs, paths): """Searches for the directory where a given file is located, and returns a possibly-empty list of additional directories, or None @@ -259,23 +264,23 @@ def find_file(filename, std_dirs, paths): sysroot = macosx_sdk_root() # Check the standard locations - for dir in std_dirs: - f = os.path.join(dir, filename) + for dir_ in std_dirs: + f = os.path.join(dir_, filename) - if MACOS and is_macosx_sdk_path(dir): - f = os.path.join(sysroot, dir[1:], filename) + if MACOS and is_macosx_sdk_path(dir_): + f = os.path.join(sysroot, dir_[1:], filename) if os.path.exists(f): return [] # Check the additional directories - for dir in paths: - f = os.path.join(dir, filename) + for dir_ in paths: + f = os.path.join(dir_, filename) - if MACOS and is_macosx_sdk_path(dir): - f = os.path.join(sysroot, dir[1:], filename) + if MACOS and is_macosx_sdk_path(dir_): + f = os.path.join(sysroot, dir_[1:], filename) if os.path.exists(f): - return [dir] + return [dir_] # Not found anywhere return None @@ -333,6 +338,7 @@ def find_library_file(compiler, libname, std_dirs, paths): else: assert False, "Internal error: Path not found in std_dirs or paths" + def validate_tzpath(): base_tzpath = sysconfig.get_config_var('TZPATH') if not base_tzpath: @@ -345,15 +351,16 @@ def validate_tzpath(): + f'found:\n{tzpaths!r}\nwith invalid paths:\n' + f'{bad_paths!r}') + def find_module_file(module, dirlist): """Find a module in a set of possible folders. If it is not found return the unadorned filename""" - list = find_file(module, [], dirlist) - if not list: + dirs = find_file(module, [], dirlist) + if not dirs: return module - if len(list) > 1: - log.info("WARNING: multiple copies of %s found", module) - return os.path.join(list[0], module) + if len(dirs) > 1: + log.info(f"WARNING: multiple copies of {module} found") + return os.path.join(dirs[0], module) class PyBuildExt(build_ext): @@ -2667,7 +2674,7 @@ def copy_scripts(self): newfilename = filename + fullversion else: newfilename = filename + minoronly - log.info('renaming %s to %s', filename, newfilename) + log.info(f'renaming {filename} to {newfilename}') os.rename(filename, newfilename) newoutfiles.append(newfilename) if filename in updated_files:

1 0

bpo-44313: bump up magic (#26983)
by isidentical 01 Jul '21

01 Jul '21

https://github.com/python/cpython/commit/0d7f61ddb074659d8c18c8f5ac86a6a18e… commit: 0d7f61ddb074659d8c18c8f5ac86a6a18e41f9e5 branch: main author: Batuhan Taskaya <batuhan(a)python.org> committer: isidentical <isidentical(a)gmail.com> date: 2021-07-01T22:25:10+03:00 summary: bpo-44313: bump up magic (#26983) files: M Lib/importlib/_bootstrap_external.py M Python/importlib_external.h diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index c8b8ece5f57ae..c3ce4703a27b8 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -370,7 +370,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3457).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3458).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff --git a/Python/importlib_external.h b/Python/importlib_external.h index f0d4138a0803e..734f78fc037cb 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -362,7 +362,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = { 11,179,9,62,0,190,7,65,22,7,193,6,5,65,12,6, 193,11,1,65,22,7,193,12,7,65,21,13,193,19,3,65, 22,7,193,23,1,65,21,13,193,24,1,65,22,7,114,95, - 0,0,0,105,129,13,0,0,114,45,0,0,0,114,33,0, + 0,0,0,105,130,13,0,0,114,45,0,0,0,114,33,0, 0,0,115,2,0,0,0,13,10,90,11,95,95,112,121,99, 97,99,104,101,95,95,122,4,111,112,116,45,122,3,46,112, 121,122,4,46,112,121,119,122,4,46,112,121,99,41,1,218,

1 0

bpo-43234: Prohibit non-ThreadPoolExecutor in loop.set_default_executor (GH-24540)
by serhiy-storchaka 01 Jul '21

01 Jul '21

https://github.com/python/cpython/commit/ddd5f36971e2ffe20cc3f4b408425c847d… commit: ddd5f36971e2ffe20cc3f4b408425c847d168646 branch: main author: Illia Volochii <illia.volochii(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2021-07-01T17:46:49+03:00 summary: bpo-43234: Prohibit non-ThreadPoolExecutor in loop.set_default_executor (GH-24540) files: A Misc/NEWS.d/next/Library/2021-02-15-22-14-31.bpo-43234.F-vKAT.rst M Doc/library/asyncio-eventloop.rst M Doc/whatsnew/3.11.rst M Lib/asyncio/base_events.py M Lib/test/test_asyncio/test_base_events.py diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 83ab7360a9b6e2..f0f6ae6d98ff99 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1132,16 +1132,12 @@ Executing code in thread or process pools .. method:: loop.set_default_executor(executor) Set *executor* as the default executor used by :meth:`run_in_executor`. - *executor* should be an instance of + *executor* must be an instance of :class:`~concurrent.futures.ThreadPoolExecutor`. - .. deprecated:: 3.8 - Using an executor that is not an instance of - :class:`~concurrent.futures.ThreadPoolExecutor` is deprecated and - will trigger an error in Python 3.9. - - *executor* must be an instance of - :class:`concurrent.futures.ThreadPoolExecutor`. + .. versionchanged:: 3.11 + *executor* must be an instance of + :class:`~concurrent.futures.ThreadPoolExecutor`. Error Handling API diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 94d8bef4d521b0..5b51273ed6b9a2 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -188,6 +188,15 @@ This section lists previously described changes and other bugfixes that may require changes to your code. +Changes in the Python API +------------------------- + +* Prohibited passing non-:class:`concurrent.futures.ThreadPoolExecutor` + executors to :meth:`loop.set_default_executor` following a deprecation in + Python 3.8. + (Contributed by Illia Volochii in :issue:`43234`.) + + C API Changes ============= diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index f789635e0f893a..74d5670bf74edb 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -814,11 +814,7 @@ def run_in_executor(self, executor, func, *args): def set_default_executor(self, executor): if not isinstance(executor, concurrent.futures.ThreadPoolExecutor): - warnings.warn( - 'Using the default executor that is not an instance of ' - 'ThreadPoolExecutor is deprecated and will be prohibited ' - 'in Python 3.9', - DeprecationWarning, 2) + raise TypeError('executor must be ThreadPoolExecutor instance') self._default_executor = executor def _getaddrinfo_debug(self, host, port, family, type, proto, flags): diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 47a9fb98001b4d..adc7bd42c1508d 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -224,14 +224,14 @@ def submit(self, fn, *args, **kwargs): self.loop.set_default_executor(executor) self.assertIs(executor, self.loop._default_executor) - def test_set_default_executor_deprecation_warnings(self): + def test_set_default_executor_error(self): executor = mock.Mock() - with self.assertWarns(DeprecationWarning): + msg = 'executor must be ThreadPoolExecutor instance' + with self.assertRaisesRegex(TypeError, msg): self.loop.set_default_executor(executor) - # Avoid cleaning up the executor mock - self.loop._default_executor = None + self.assertIsNone(self.loop._default_executor) def test_call_soon(self): def cb(): diff --git a/Misc/NEWS.d/next/Library/2021-02-15-22-14-31.bpo-43234.F-vKAT.rst b/Misc/NEWS.d/next/Library/2021-02-15-22-14-31.bpo-43234.F-vKAT.rst new file mode 100644 index 00000000000000..7f195cc752fc13 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-02-15-22-14-31.bpo-43234.F-vKAT.rst @@ -0,0 +1,3 @@ +Prohibit passing non-:class:`concurrent.futures.ThreadPoolExecutor` +executors to :meth:`loop.set_default_executor` following a deprecation in +Python 3.8. Patch by Illia Volochii.

1 0

bpo-43216: Remove @asyncio.coroutine (GH-26369)
by vstinner 01 Jul '21

01 Jul '21

https://github.com/python/cpython/commit/a1092f62492a3fcd6195bea94eccf8d5a3… commit: a1092f62492a3fcd6195bea94eccf8d5a300acb1 branch: main author: Illia Volochii <illia.volochii(a)gmail.com> committer: vstinner <vstinner(a)python.org> date: 2021-07-01T15:13:59+02:00 summary: bpo-43216: Remove @asyncio.coroutine (GH-26369) Remove the @asyncio.coroutine decorator enabling legacy generator-based coroutines to be compatible with async/await code; remove asyncio.coroutines.CoroWrapper used for wrapping legacy coroutine objects in the debug mode. The decorator has been deprecated since Python 3.8 and the removal was initially scheduled for Python 3.10. files: A Misc/NEWS.d/next/Library/2021-05-25-23-26-38.bpo-43216.xTUyyX.rst M Doc/library/asyncio-task.rst M Doc/library/collections.abc.rst M Doc/reference/datamodel.rst M Doc/whatsnew/3.11.rst M Lib/asyncio/coroutines.py M Lib/test/test_asyncio/test_base_events.py M Lib/test/test_asyncio/test_events.py M Lib/test/test_asyncio/test_locks.py M Lib/test/test_asyncio/test_pep492.py M Lib/test/test_asyncio/test_tasks.py diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index bbdef3345a4d42..44ec3cc8826c02 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -148,9 +148,6 @@ other coroutines:: * a *coroutine object*: an object returned by calling a *coroutine function*. -asyncio also supports legacy :ref:`generator-based -<asyncio_generator_based_coro>` coroutines. - .. rubric:: Tasks @@ -1042,60 +1039,3 @@ Task Object in the :func:`repr` output of a task object. .. versionadded:: 3.8 - - -.. _asyncio_generator_based_coro: - -Generator-based Coroutines -========================== - -.. note:: - - Support for generator-based coroutines is **deprecated** and - is scheduled for removal in Python 3.10. - -Generator-based coroutines predate async/await syntax. They are -Python generators that use ``yield from`` expressions to await -on Futures and other coroutines. - -Generator-based coroutines should be decorated with -:func:`@asyncio.coroutine <asyncio.coroutine>`, although this is not -enforced. - - -.. decorator:: coroutine - - Decorator to mark generator-based coroutines. - - This decorator enables legacy generator-based coroutines to be - compatible with async/await code:: - - @asyncio.coroutine - def old_style_coroutine(): - yield from asyncio.sleep(1) - - async def main(): - await old_style_coroutine() - - This decorator should not be used for :keyword:`async def` - coroutines. - - .. deprecated-removed:: 3.8 3.10 - - Use :keyword:`async def` instead. - -.. function:: iscoroutine(obj) - - Return ``True`` if *obj* is a :ref:`coroutine object <coroutine>`. - - This method is different from :func:`inspect.iscoroutine` because - it returns ``True`` for generator-based coroutines. - -.. function:: iscoroutinefunction(func) - - Return ``True`` if *func* is a :ref:`coroutine function - <coroutine>`. - - This method is different from :func:`inspect.iscoroutinefunction` - because it returns ``True`` for generator-based coroutine functions - decorated with :func:`@coroutine <coroutine>`. diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 2345e78a17e4f5..924d0b58d952fd 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -198,7 +198,7 @@ ABC Inherits from Abstract Methods Mixin .. note:: In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine` or :func:`asyncio.coroutine`) are + :func:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`__await__` method. Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -216,7 +216,7 @@ ABC Inherits from Abstract Methods Mixin .. note:: In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine` or :func:`asyncio.coroutine`) are + :func:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`__await__` method. Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index eefdc3d5100b56..2f8ed70e53c735 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2714,7 +2714,7 @@ are awaitable. .. note:: The :term:`generator iterator` objects returned from generators - decorated with :func:`types.coroutine` or :func:`asyncio.coroutine` + decorated with :func:`types.coroutine` are also awaitable, but they do not implement :meth:`__await__`. .. method:: object.__await__(self) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 8e3d5f31754c8f..94d8bef4d521b0 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -171,7 +171,15 @@ Deprecated Removed ======= - +* The :func:`@asyncio.coroutine <asyncio.coroutine>` :term:`decorator` enabling + legacy generator-based coroutines to be compatible with async/await code. + The function has been deprecated since Python 3.8 and the removal was + initially scheduled for Python 3.10. Use :keyword:`async def` instead. + (Contributed by Illia Volochii in :issue:`43216`.) + +* :class:`asyncio.coroutines.CoroWrapper` used for wrapping legacy + generator-based coroutine objects in the debug mode. + (Contributed by Illia Volochii in :issue:`43216`.) Porting to Python 3.11 ====================== diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index 9664ea74d75147..0e4b489f30fd7c 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -1,162 +1,19 @@ -__all__ = 'coroutine', 'iscoroutinefunction', 'iscoroutine' +__all__ = 'iscoroutinefunction', 'iscoroutine' import collections.abc -import functools import inspect import os import sys import traceback import types -import warnings - -from . import base_futures -from . import constants -from . import format_helpers -from .log import logger def _is_debug_mode(): - # If you set _DEBUG to true, @coroutine will wrap the resulting - # generator objects in a CoroWrapper instance (defined below). That - # instance will log a message when the generator is never iterated - # over, which may happen when you forget to use "await" or "yield from" - # with a coroutine call. - # Note that the value of the _DEBUG flag is taken - # when the decorator is used, so to be of any use it must be set - # before you define your coroutines. A downside of using this feature - # is that tracebacks show entries for the CoroWrapper.__next__ method - # when _DEBUG is true. + # See: https://docs.python.org/3/library/asyncio-dev.html#asyncio-debug-mode. return sys.flags.dev_mode or (not sys.flags.ignore_environment and bool(os.environ.get('PYTHONASYNCIODEBUG'))) -_DEBUG = _is_debug_mode() - - -class CoroWrapper: - # Wrapper for coroutine object in _DEBUG mode. - - def __init__(self, gen, func=None): - assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen - self.gen = gen - self.func = func # Used to unwrap @coroutine decorator - self._source_traceback = format_helpers.extract_stack(sys._getframe(1)) - self.__name__ = getattr(gen, '__name__', None) - self.__qualname__ = getattr(gen, '__qualname__', None) - - def __repr__(self): - coro_repr = _format_coroutine(self) - if self._source_traceback: - frame = self._source_traceback[-1] - coro_repr += f', created at {frame[0]}:{frame[1]}' - - return f'<{self.__class__.__name__} {coro_repr}>' - - def __iter__(self): - return self - - def __next__(self): - return self.gen.send(None) - - def send(self, value): - return self.gen.send(value) - - def throw(self, type, value=None, traceback=None): - return self.gen.throw(type, value, traceback) - - def close(self): - return self.gen.close() - - @property - def gi_frame(self): - return self.gen.gi_frame - - @property - def gi_running(self): - return self.gen.gi_running - - @property - def gi_code(self): - return self.gen.gi_code - - def __await__(self): - return self - - @property - def gi_yieldfrom(self): - return self.gen.gi_yieldfrom - - def __del__(self): - # Be careful accessing self.gen.frame -- self.gen might not exist. - gen = getattr(self, 'gen', None) - frame = getattr(gen, 'gi_frame', None) - if frame is not None and frame.f_lasti == -1: - msg = f'{self!r} was never yielded from' - tb = getattr(self, '_source_traceback', ()) - if tb: - tb = ''.join(traceback.format_list(tb)) - msg += (f'\nCoroutine object created at ' - f'(most recent call last, truncated to ' - f'{constants.DEBUG_STACK_DEPTH} last lines):\n') - msg += tb.rstrip() - logger.error(msg) - - -def coroutine(func): - """Decorator to mark coroutines. - - If the coroutine is not yielded from before it is destroyed, - an error message is logged. - """ - warnings.warn('"@coroutine" decorator is deprecated since Python 3.8, use "async def" instead', - DeprecationWarning, - stacklevel=2) - if inspect.iscoroutinefunction(func): - # In Python 3.5 that's all we need to do for coroutines - # defined with "async def". - return func - - if inspect.isgeneratorfunction(func): - coro = func - else: - @functools.wraps(func) - def coro(*args, **kw): - res = func(*args, **kw) - if (base_futures.isfuture(res) or inspect.isgenerator(res) or - isinstance(res, CoroWrapper)): - res = yield from res - else: - # If 'res' is an awaitable, run it. - try: - await_meth = res.__await__ - except AttributeError: - pass - else: - if isinstance(res, collections.abc.Awaitable): - res = yield from await_meth() - return res - - coro = types.coroutine(coro) - if not _DEBUG: - wrapper = coro - else: - @functools.wraps(func) - def wrapper(*args, **kwds): - w = CoroWrapper(coro(*args, **kwds), func=func) - if w._source_traceback: - del w._source_traceback[-1] - # Python < 3.5 does not implement __qualname__ - # on generator objects, so we set it manually. - # We use getattr as some callables (such as - # functools.partial may lack __qualname__). - w.__name__ = getattr(func, '__name__', None) - w.__qualname__ = getattr(func, '__qualname__', None) - return w - - wrapper._is_coroutine = _is_coroutine # For iscoroutinefunction(). - return wrapper - - # A marker for iscoroutinefunction. _is_coroutine = object() @@ -170,7 +27,7 @@ def iscoroutinefunction(func): # Prioritize native coroutine check to speed-up # asyncio.iscoroutine. _COROUTINE_TYPES = (types.CoroutineType, types.GeneratorType, - collections.abc.Coroutine, CoroWrapper) + collections.abc.Coroutine) _iscoroutine_typecache = set() @@ -193,16 +50,11 @@ def iscoroutine(obj): def _format_coroutine(coro): assert iscoroutine(coro) - is_corowrapper = isinstance(coro, CoroWrapper) - def get_name(coro): # Coroutines compiled with Cython sometimes don't have # proper __qualname__ or __name__. While that is a bug # in Cython, asyncio shouldn't crash with an AttributeError # in its __repr__ functions. - if is_corowrapper: - return format_helpers._format_callback(coro.func, (), {}) - if hasattr(coro, '__qualname__') and coro.__qualname__: coro_name = coro.__qualname__ elif hasattr(coro, '__name__') and coro.__name__: @@ -247,18 +99,8 @@ def is_running(coro): filename = coro_code.co_filename or '<empty co_filename>' lineno = 0 - if (is_corowrapper and - coro.func is not None and - not inspect.isgeneratorfunction(coro.func)): - source = format_helpers._get_function_source(coro.func) - if source is not None: - filename, lineno = source - if coro_frame is None: - coro_repr = f'{coro_name} done, defined at {filename}:{lineno}' - else: - coro_repr = f'{coro_name} running, defined at {filename}:{lineno}' - elif coro_frame is not None: + if coro_frame is not None: lineno = coro_frame.f_lineno coro_repr = f'{coro_name} running at {filename}:{lineno}' diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 5691d4250aca9e..47a9fb98001b4d 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -1884,10 +1884,8 @@ def test_accept_connection_exception(self, m_log): MyProto, sock, None, None, mock.ANY, mock.ANY) def test_call_coroutine(self): - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def simple_coroutine(): - pass + async def simple_coroutine(): + pass self.loop.set_debug(True) coro_func = simple_coroutine diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 55fc266cb714b1..e78176997a0591 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -17,6 +17,7 @@ import sys import threading import time +import types import errno import unittest from unittest import mock @@ -2163,8 +2164,7 @@ def test_handle_repr(self): '<Handle cancelled>') # decorated function - with self.assertWarns(DeprecationWarning): - cb = asyncio.coroutine(noop) + cb = types.coroutine(noop) h = asyncio.Handle(cb, (), self.loop) self.assertEqual(repr(h), '<Handle noop() at %s:%s>' diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 6194cd06176dac..441adeea8f8e07 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -38,14 +38,12 @@ def test_repr(self): def test_lock(self): lock = asyncio.Lock() - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def acquire_lock(): - return (yield from lock) + async def acquire_lock(): + return await lock with self.assertRaisesRegex( TypeError, - "object is not iterable" + "object Lock can't be used in 'await' expression" ): self.loop.run_until_complete(acquire_lock()) @@ -78,18 +76,16 @@ def test_lock_by_with_statement(self): asyncio.BoundedSemaphore(), ] - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def test(lock): - yield from asyncio.sleep(0.01) - self.assertFalse(lock.locked()) - with self.assertRaisesRegex( - TypeError, - "object is not iterable" - ): - with (yield from lock): - pass - self.assertFalse(lock.locked()) + async def test(lock): + await asyncio.sleep(0.01) + self.assertFalse(lock.locked()) + with self.assertRaisesRegex( + TypeError, + r"object \w+ can't be used in 'await' expression" + ): + with await lock: + pass + self.assertFalse(lock.locked()) for primitive in primitives: loop.run_until_complete(test(primitive)) @@ -788,14 +784,12 @@ def test_semaphore(self): sem = asyncio.Semaphore() self.assertEqual(1, sem._value) - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def acquire_lock(): - return (yield from sem) + async def acquire_lock(): + return await sem with self.assertRaisesRegex( TypeError, - "'Semaphore' object is not iterable", + "object Semaphore can't be used in 'await' expression", ): self.loop.run_until_complete(acquire_lock()) diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py index 4bd50f4123e579..f833f788dcb98f 100644 --- a/Lib/test/test_asyncio/test_pep492.py +++ b/Lib/test/test_asyncio/test_pep492.py @@ -123,20 +123,6 @@ def test_iscoroutinefunction(self): async def foo(): pass self.assertTrue(asyncio.iscoroutinefunction(foo)) - def test_function_returning_awaitable(self): - class Awaitable: - def __await__(self): - return ('spam',) - - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def func(): - return Awaitable() - - coro = func() - self.assertEqual(coro.send(None), 'spam') - coro.close() - def test_async_def_coroutines(self): async def bar(): return 'spam' diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index a9e4cf53566ca9..86710872ace024 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -33,18 +33,6 @@ async def coroutine_function(): pass -(a)contextlib.contextmanager -def set_coroutine_debug(enabled): - coroutines = asyncio.coroutines - - old_debug = coroutines._DEBUG - try: - coroutines._DEBUG = enabled - yield - finally: - coroutines._DEBUG = old_debug - - def format_coroutine(qualname, state, src, source_traceback, generator=False): if generator: state = '%s' % state @@ -234,43 +222,6 @@ async def test(): self.assertTrue(t.done()) self.assertEqual(t.result(), 'ok') - def test_ensure_future_coroutine_2(self): - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def notmuch(): - return 'ok' - t = asyncio.ensure_future(notmuch(), loop=self.loop) - self.assertIs(t._loop, self.loop) - self.loop.run_until_complete(t) - self.assertTrue(t.done()) - self.assertEqual(t.result(), 'ok') - - a = notmuch() - self.addCleanup(a.close) - with self.assertWarns(DeprecationWarning) as cm: - with self.assertRaisesRegex(RuntimeError, 'There is no current event loop'): - asyncio.ensure_future(a) - self.assertEqual(cm.warnings[0].filename, __file__) - - async def test(): - return asyncio.ensure_future(notmuch()) - t = self.loop.run_until_complete(test()) - self.assertIs(t._loop, self.loop) - self.loop.run_until_complete(t) - self.assertTrue(t.done()) - self.assertEqual(t.result(), 'ok') - - # Deprecated in 3.10 - asyncio.set_event_loop(self.loop) - self.addCleanup(asyncio.set_event_loop, None) - with self.assertWarns(DeprecationWarning) as cm: - t = asyncio.ensure_future(notmuch()) - self.assertEqual(cm.warnings[0].filename, __file__) - self.assertIs(t._loop, self.loop) - self.loop.run_until_complete(t) - self.assertTrue(t.done()) - self.assertEqual(t.result(), 'ok') - def test_ensure_future_future(self): f_orig = self.new_future(self.loop) f_orig.set_result('ko') @@ -318,12 +269,10 @@ class Aw: def __init__(self, coro): self.coro = coro def __await__(self): - return (yield from self.coro) + return self.coro.__await__() - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def coro(): - return 'ok' + async def coro(): + return 'ok' loop = asyncio.new_event_loop() self.set_event_loop(loop) @@ -450,68 +399,6 @@ async def notmuch(): self.assertEqual(t.get_name(), '{6}') self.loop.run_until_complete(t) - def test_task_repr_coro_decorator(self): - self.loop.set_debug(False) - - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def notmuch(): - # notmuch() function doesn't use yield from: it will be wrapped by - # @coroutine decorator - return 123 - - # test coroutine function - self.assertEqual(notmuch.__name__, 'notmuch') - self.assertRegex(notmuch.__qualname__, - r'\w+.test_task_repr_coro_decorator' - r'\.<locals>\.notmuch') - self.assertEqual(notmuch.__module__, __name__) - - # test coroutine object - gen = notmuch() - # On Python >= 3.5, generators now inherit the name of the - # function, as expected, and have a qualified name (__qualname__ - # attribute). - coro_name = 'notmuch' - coro_qualname = ('BaseTaskTests.test_task_repr_coro_decorator' - '.<locals>.notmuch') - self.assertEqual(gen.__name__, coro_name) - self.assertEqual(gen.__qualname__, coro_qualname) - - # test repr(CoroWrapper) - if coroutines._DEBUG: - # format the coroutine object - if coroutines._DEBUG: - filename, lineno = test_utils.get_function_source(notmuch) - frame = gen._source_traceback[-1] - coro = ('%s() running, defined at %s:%s, created at %s:%s' - % (coro_qualname, filename, lineno, - frame[0], frame[1])) - else: - code = gen.gi_code - coro = ('%s() running at %s:%s' - % (coro_qualname, code.co_filename, - code.co_firstlineno)) - - self.assertEqual(repr(gen), '<CoroWrapper %s>' % coro) - - # test pending Task - t = self.new_task(self.loop, gen) - t.add_done_callback(Dummy()) - - # format the coroutine object - if coroutines._DEBUG: - src = '%s:%s' % test_utils.get_function_source(notmuch) - else: - code = gen.gi_code - src = '%s:%s' % (code.co_filename, code.co_firstlineno) - coro = format_coroutine(coro_qualname, 'running', src, - t._source_traceback, - generator=not coroutines._DEBUG) - self.assertEqual(repr(t), - "<Task pending name='TestTask' %s cb=[<Dummy>()]>" % coro) - self.loop.run_until_complete(t) - def test_task_repr_wait_for(self): self.loop.set_debug(False) @@ -527,30 +414,6 @@ async def wait_for(fut): fut.set_result(None) self.loop.run_until_complete(task) - def test_task_repr_partial_corowrapper(self): - # Issue #222: repr(CoroWrapper) must not fail in debug mode if the - # coroutine is a partial function - with set_coroutine_debug(True): - self.loop.set_debug(True) - - async def func(x, y): - await asyncio.sleep(0) - - with self.assertWarns(DeprecationWarning): - partial_func = asyncio.coroutine(functools.partial(func, 1)) - task = self.loop.create_task(partial_func(2)) - - # make warnings quiet - task._log_destroy_pending = False - self.addCleanup(task._coro.close) - - coro_repr = repr(task._coro) - expected = ( - r'<coroutine object \w+\.test_task_repr_partial_corowrapper' - r'\.<locals>\.func at' - ) - self.assertRegex(coro_repr, expected) - def test_task_basics(self): async def outer(): @@ -741,12 +604,10 @@ async def coro(): (asyncio.CancelledError, ('my message',), 2)) def test_cancel_yield(self): - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def task(): - yield - yield - return 12 + async def task(): + await asyncio.sleep(0) + await asyncio.sleep(0) + return 12 t = self.new_task(self.loop, task()) test_utils.run_briefly(self.loop) # start coro @@ -1322,10 +1183,8 @@ async def foo(): def test_wait_duplicate_coroutines(self): - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def coro(s): - return s + async def coro(s): + return s c = coro('test') task = self.new_task( self.loop, @@ -1587,16 +1446,14 @@ def gen(): completed = set() time_shifted = False - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def sleeper(dt, x): - nonlocal time_shifted - yield from asyncio.sleep(dt) - completed.add(x) - if not time_shifted and 'a' in completed and 'b' in completed: - time_shifted = True - loop.advance_time(0.14) - return x + async def sleeper(dt, x): + nonlocal time_shifted + await asyncio.sleep(dt) + completed.add(x) + if not time_shifted and 'a' in completed and 'b' in completed: + time_shifted = True + loop.advance_time(0.14) + return x a = sleeper(0.01, 'a') b = sleeper(0.01, 'b') @@ -1614,10 +1471,6 @@ async def foo(): self.assertTrue('b' in res[:2]) self.assertEqual(res[2], 'c') - # Doing it again should take no time and exercise a different path. - res = loop.run_until_complete(self.new_task(loop, foo())) - self.assertAlmostEqual(0.15, loop.time()) - def test_as_completed_with_timeout(self): def gen(): @@ -1727,19 +1580,15 @@ async def test(): def test_as_completed_duplicate_coroutines(self): - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def coro(s): - return s + async def coro(s): + return s - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def runner(): - result = [] - c = coro('ham') - for f in asyncio.as_completed([c, c, coro('spam')]): - result.append((yield from f)) - return result + async def runner(): + result = [] + c = coro('ham') + for f in asyncio.as_completed([c, c, coro('spam')]): + result.append(await f) + return result fut = self.new_task(self.loop, runner()) self.loop.run_until_complete(fut) @@ -1900,17 +1749,6 @@ async def notmuch(): self.loop.run_until_complete(task), 'ko') - def test_step_result(self): - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def notmuch(): - yield None - yield 1 - return 'ko' - - self.assertRaises( - RuntimeError, self.loop.run_until_complete, notmuch()) - def test_step_result_future(self): # If coroutine returns future, task waits on this future. @@ -1983,53 +1821,15 @@ def fn1(): yield self.assertFalse(asyncio.iscoroutinefunction(fn1)) - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def fn2(): - yield + async def fn2(): + pass self.assertTrue(asyncio.iscoroutinefunction(fn2)) self.assertFalse(asyncio.iscoroutinefunction(mock.Mock())) - def test_yield_vs_yield_from(self): - fut = self.new_future(self.loop) - - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def wait_for_future(): - yield fut - - task = wait_for_future() - with self.assertRaises(RuntimeError): - self.loop.run_until_complete(task) - - self.assertFalse(fut.done()) - - def test_yield_vs_yield_from_generator(self): - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def coro(): - yield - - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def wait_for_future(): - gen = coro() - try: - yield gen - finally: - gen.close() - - task = wait_for_future() - self.assertRaises( - RuntimeError, - self.loop.run_until_complete, task) - def test_coroutine_non_gen_function(self): - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def func(): - return 'test' + async def func(): + return 'test' self.assertTrue(asyncio.iscoroutinefunction(func)) @@ -2042,10 +1842,8 @@ def func(): def test_coroutine_non_gen_function_return_future(self): fut = self.new_future(self.loop) - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def func(): - return fut + async def func(): + return fut async def coro(): fut.set_result('test') @@ -2053,7 +1851,7 @@ async def coro(): t1 = self.new_task(self.loop, func()) t2 = self.new_task(self.loop, coro()) res = self.loop.run_until_complete(t1) - self.assertEqual(res, 'test') + self.assertEqual(res, fut) self.assertIsNone(t2.result()) def test_current_task(self): @@ -2309,136 +2107,15 @@ def test_wait_invalid_args(self): self.assertRaises(ValueError, self.loop.run_until_complete, asyncio.wait([])) - def test_corowrapper_mocks_generator(self): - - def check(): - # A function that asserts various things. - # Called twice, with different debug flag values. - - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def coro(): - # The actual coroutine. - self.assertTrue(gen.gi_running) - yield from fut - - # A completed Future used to run the coroutine. - fut = self.new_future(self.loop) - fut.set_result(None) - - # Call the coroutine. - gen = coro() - - # Check some properties. - self.assertTrue(asyncio.iscoroutine(gen)) - self.assertIsInstance(gen.gi_frame, types.FrameType) - self.assertFalse(gen.gi_running) - self.assertIsInstance(gen.gi_code, types.CodeType) - - # Run it. - self.loop.run_until_complete(gen) - - # The frame should have changed. - self.assertIsNone(gen.gi_frame) - - # Test with debug flag cleared. - with set_coroutine_debug(False): - check() - - # Test with debug flag set. - with set_coroutine_debug(True): - check() - - def test_yield_from_corowrapper(self): - with set_coroutine_debug(True): - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def t1(): - return (yield from t2()) - - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def t2(): - f = self.new_future(self.loop) - self.new_task(self.loop, t3(f)) - return (yield from f) - - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def t3(f): - f.set_result((1, 2, 3)) - - task = self.new_task(self.loop, t1()) - val = self.loop.run_until_complete(task) - self.assertEqual(val, (1, 2, 3)) - - def test_yield_from_corowrapper_send(self): - def foo(): - a = yield - return a - - def call(arg): - cw = asyncio.coroutines.CoroWrapper(foo()) - cw.send(None) - try: - cw.send(arg) - except StopIteration as ex: - return ex.args[0] - else: - raise AssertionError('StopIteration was expected') - - self.assertEqual(call((1, 2)), (1, 2)) - self.assertEqual(call('spam'), 'spam') - - def test_corowrapper_weakref(self): - wd = weakref.WeakValueDictionary() - def foo(): yield from [] - cw = asyncio.coroutines.CoroWrapper(foo()) - wd['cw'] = cw # Would fail without __weakref__ slot. - cw.gen = None # Suppress warning from __del__. - - def test_corowrapper_throw(self): - # Issue 429: CoroWrapper.throw must be compatible with gen.throw - def foo(): - value = None - while True: - try: - value = yield value - except Exception as e: - value = e - - exception = Exception("foo") - cw = asyncio.coroutines.CoroWrapper(foo()) - cw.send(None) - self.assertIs(exception, cw.throw(exception)) - - cw = asyncio.coroutines.CoroWrapper(foo()) - cw.send(None) - self.assertIs(exception, cw.throw(Exception, exception)) - - cw = asyncio.coroutines.CoroWrapper(foo()) - cw.send(None) - exception = cw.throw(Exception, "foo") - self.assertIsInstance(exception, Exception) - self.assertEqual(exception.args, ("foo", )) - - cw = asyncio.coroutines.CoroWrapper(foo()) - cw.send(None) - exception = cw.throw(Exception, "foo", None) - self.assertIsInstance(exception, Exception) - self.assertEqual(exception.args, ("foo", )) - def test_log_destroyed_pending_task(self): Task = self.__class__.Task - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def kill_me(loop): - future = self.new_future(loop) - yield from future - # at this point, the only reference to kill_me() task is - # the Task._wakeup() method in future._callbacks - raise Exception("code never reached") + async def kill_me(loop): + future = self.new_future(loop) + await future + # at this point, the only reference to kill_me() task is + # the Task._wakeup() method in future._callbacks + raise Exception("code never reached") mock_handler = mock.Mock() self.loop.set_debug(True) @@ -2456,8 +2133,6 @@ def kill_me(loop): self.loop._run_once() self.assertEqual(len(self.loop._ready), 0) - # remove the future used in kill_me(), and references to the task - del coro.gi_frame.f_locals['future'] coro = None source_traceback = task._source_traceback task = None @@ -2491,62 +2166,6 @@ async def runner(): loop.run_until_complete(runner()) self.assertFalse(m_log.error.called) - @mock.patch('asyncio.coroutines.logger') - def test_coroutine_never_yielded(self, m_log): - with set_coroutine_debug(True): - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def coro_noop(): - pass - - tb_filename = __file__ - tb_lineno = sys._getframe().f_lineno + 2 - # create a coroutine object but don't use it - coro_noop() - support.gc_collect() - - self.assertTrue(m_log.error.called) - message = m_log.error.call_args[0][0] - func_filename, func_lineno = test_utils.get_function_source(coro_noop) - - regex = (r'^<CoroWrapper %s$?$? .* at %s:%s, .*> ' - r'was never yielded from\n' - r'Coroutine object created at $most recent call last, truncated to \d+ last lines$:\n' - r'.*\n' - r' File "%s", line %s, in test_coroutine_never_yielded\n' - r' coro_noop$' - % (re.escape(coro_noop.__qualname__), - re.escape(func_filename), func_lineno, - re.escape(tb_filename), tb_lineno)) - - self.assertRegex(message, re.compile(regex, re.DOTALL)) - - def test_return_coroutine_from_coroutine(self): - """Return of @asyncio.coroutine()-wrapped function generator object - from @asyncio.coroutine()-wrapped function should have same effect as - returning generator object or Future.""" - def check(): - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def outer_coro(): - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def inner_coro(): - return 1 - - return inner_coro() - - result = self.loop.run_until_complete(outer_coro()) - self.assertEqual(result, 1) - - # Test with debug flag cleared. - with set_coroutine_debug(False): - check() - - # Test with debug flag set. - with set_coroutine_debug(True): - check() - def test_task_source_traceback(self): self.loop.set_debug(True) @@ -2677,10 +2296,8 @@ def call_soon(callback, *args, **kwargs): raise ValueError self.loop.call_soon = call_soon - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def coro(): - pass + async def coro(): + pass self.assertFalse(m_log.error.called) @@ -2708,23 +2325,6 @@ def test_create_task_with_noncoroutine(self): "a coroutine was expected, got 123"): self.new_task(self.loop, 123) - def test_create_task_with_oldstyle_coroutine(self): - - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def coro(): - pass - - task = self.new_task(self.loop, coro()) - self.assertIsInstance(task, self.Task) - self.loop.run_until_complete(task) - - # test it for the second time to ensure that caching - # in asyncio.iscoroutine() doesn't break things. - task = self.new_task(self.loop, coro()) - self.assertIsInstance(task, self.Task) - self.loop.run_until_complete(task) - def test_create_task_with_async_function(self): async def coro(): @@ -3365,7 +2965,7 @@ def test_return_exceptions(self): def test_env_var_debug(self): code = '\n'.join(( 'import asyncio.coroutines', - 'print(asyncio.coroutines._DEBUG)')) + 'print(asyncio.coroutines._is_debug_mode())')) # Test with -E to not fail if the unit test was run with # PYTHONASYNCIODEBUG set to a non-empty string @@ -3542,10 +3142,8 @@ async def coro(): self.other_loop.run_until_complete(fut) def test_duplicate_coroutines(self): - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def coro(s): - return s + async def coro(s): + return s c = coro('abc') fut = self._gather(c, c, coro('def'), c) self._run_loop(self.one_loop) @@ -3633,9 +3231,7 @@ def target(self, fail=False, cancel=False, timeout=None, # otherwise it spills errors and breaks **other** unittests, since # 'target' is interacting with threads. - # With this call, `coro` will be advanced, so that - # CoroWrapper.__del__ won't do anything when asyncio tests run - # in debug mode. + # With this call, `coro` will be advanced. self.loop.call_soon_threadsafe(coro.send, None) try: return future.result(timeout) @@ -3771,54 +3367,6 @@ def tearDown(self): self.loop = None super().tearDown() - def test_yield_from_awaitable(self): - - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def coro(): - yield from asyncio.sleep(0) - return 'ok' - - result = self.loop.run_until_complete(coro()) - self.assertEqual('ok', result) - - def test_await_old_style_coro(self): - - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def coro1(): - return 'ok1' - - with self.assertWarns(DeprecationWarning): - @asyncio.coroutine - def coro2(): - yield from asyncio.sleep(0) - return 'ok2' - - async def inner(): - return await asyncio.gather(coro1(), coro2()) - - result = self.loop.run_until_complete(inner()) - self.assertEqual(['ok1', 'ok2'], result) - - def test_debug_mode_interop(self): - # https://bugs.python.org/issue32636 - code = textwrap.dedent(""" - import asyncio - - async def native_coro(): - pass - - @asyncio.coroutine - def old_style_coro(): - yield from native_coro() - - asyncio.run(old_style_coro()) - """) - - assert_python_ok("-Wignore::DeprecationWarning", "-c", code, - PYTHONASYNCIODEBUG="1") - if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2021-05-25-23-26-38.bpo-43216.xTUyyX.rst b/Misc/NEWS.d/next/Library/2021-05-25-23-26-38.bpo-43216.xTUyyX.rst new file mode 100644 index 00000000000000..845ef95d1ad296 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-05-25-23-26-38.bpo-43216.xTUyyX.rst @@ -0,0 +1,6 @@ +Remove the :func:`@asyncio.coroutine <asyncio.coroutine>` :term:`decorator` +enabling legacy generator-based coroutines to be compatible with async/await +code; remove :class:`asyncio.coroutines.CoroWrapper` used for wrapping +legacy coroutine objects in the debug mode. The decorator has been deprecated +since Python 3.8 and the removal was initially scheduled for Python 3.10. +Patch by Illia Volochii.

1 0

Add file describing how to add or modify specialized families of instructions. (GH-26954)
by markshannon 01 Jul '21

01 Jul '21

https://github.com/python/cpython/commit/3623aaa78cb9c50edb6da5ac37000446f1… commit: 3623aaa78cb9c50edb6da5ac37000446f138b91c branch: main author: Mark Shannon <mark(a)hotpy.org> committer: markshannon <mark(a)hotpy.org> date: 2021-07-01T13:13:05+01:00 summary: Add file describing how to add or modify specialized families of instructions. (GH-26954) files: A Python/adaptive.md M Python/specialize.c diff --git a/Python/adaptive.md b/Python/adaptive.md new file mode 100644 index 0000000000000..66b80a17464bf --- /dev/null +++ b/Python/adaptive.md @@ -0,0 +1,131 @@ +# Adding or extending a family of adaptive instructions. + +## Families of instructions + +The core part of PEP 659 (specializing adaptive interpreter) is the families +of instructions that perform the adaptive specialization. + +A family of instructions has the following fundamental properties: + +* It corresponds to a single instruction in the code + generated by the bytecode compiler. +* It has a single adaptive instruction that records an execution count and, + at regular intervals, attempts to specialize itself. If not specializing, + it executes the non-adaptive instruction. +* It has at least one specialized form of the instruction that is tailored + for a particular value or set of values at runtime. +* All members of the family have access to the same number of cache entries. + Individual family members do not need to use all of the entries. + +The current implementation also requires the following, +although these are not fundamental and may change: + +* If a family uses one or more entries, then the first entry must be a + `_PyAdaptiveEntry` entry. +* If a family uses no cache entries, then the `oparg` is used as the + counter for the adaptive instruction. +* All instruction names should start with the name of the non-adaptive + instruction. +* The adaptive instruction should end in `_ADAPTIVE`. +* Specialized forms should have names describing their specialization. + +## Example family + +The `LOAD_GLOBAL` instruction (in Python/ceval.c) already has an adaptive +family that serves as a relatively simple example. + +The `LOAD_GLOBAL_ADAPTIVE` instruction performs adaptive specialization, +calling `_Py_Specialize_LoadGlobal()` when the counter reaches zero. + +There are two specialized instructions in the family, `LOAD_GLOBAL_MODULE` +which is specialized for global variables in the module, and +`LOAD_GLOBAL_BUILTIN` which is specialized for builtin variables. + +## Performance analysis + +The benefit of a specialization can be assessed with the following formula: +`Tbase/Tadaptive`. + +Where `Tbase` is the mean time to execute the base instruction, +and `Tadaptive` is the mean time to execute the specialized and adaptive forms. + +`Tadaptive = (sum(Ti*Ni) + Tmiss*Nmiss)/(sum(Ni)+Nmiss)` + +`Ti` is the time to execute the `i`th instruction in the family and `Ni` is +the number of times that instruction is executed. +`Tmiss` is the time to process a miss, including de-optimzation +and the time to execute the base instruction. + +The ideal situation is where misses are rare and the specialized +forms are much faster than the base instruction. +`LOAD_GLOBAL` is near ideal, `Nmiss/sum(Ni) ≈ 0`. +In which case we have `Tadaptive ≈ sum(Ti*Ni)`. +Since we can expect the specialized forms `LOAD_GLOBAL_MODULE` and +`LOAD_GLOBAL_BUILTIN` to be much faster than the adaptive base instruction, +we would expect the specialization of `LOAD_GLOBAL` to be profitable. + +## Design considerations + +While `LOAD_GLOBAL` may be ideal, instructions like `LOAD_ATTR` and +`CALL_FUNCTION` are not. For maximum performance we want to keep `Ti` +low for all specialized instructions and `Nmiss` as low as possible. + +Keeping `Nmiss` low means that there should be specializations for almost +all values seen by the base instruction. Keeping `sum(Ti*Ni)` low means +keeping `Ti` low which means minimizing branches and dependent memory +accesses (pointer chasing). These two objectives may be in conflict, +requiring judgement and experimentation to design the family of instructions. + +### Gathering data + +Before choosing how to specialize an instruction, it is important to gather +some data. What are the patterns of usage of the base instruction? +Data can best be gathered by instrumenting the interpreter. Since a +specialization function and adaptive instruction are going to be required, +instrumentation can most easily be added in the specialization function. + +### Choice of specializations + +The performance of the specializing adaptive interpreter relies on the +quality of specialization and keeping the overhead of specialization low. + +Specialized instructions must be fast. In order to be fast, +specialized instructions should be tailored for a particular +set of values that allows them to: +1. Verify that incoming value is part of that set with low overhead. +2. Perform the operation quickly. + +This requires that the set of values is chosen such that membership can be +tested quickly and that membership is sufficient to allow the operation to +performed quickly. + +For example, `LOAD_GLOBAL_MODULE` is specialized for `globals()` +dictionaries that have a keys with the expected version. + +This can be tested quickly: +* `globals->keys->dk_version == expected_version` + +and the operation can be performed quickly: +* `value = globals->keys->entries[index].value`. + +Because it is impossible to measure the performance of an instruction without +also measuring unrelated factors, the assessment of the quality of a +specialization will require some judgement. + +As a general rule, specialized instructions should be much faster than the +base instruction. + +### Implementation of specialized instructions + +In general, specialized instructions should be implemented in two parts: +1. A sequence of guards, each of the form + `DEOPT_IF(guard-condition-is-false, BASE_NAME)`, + followed by a `record_cache_hit()`. +2. The operation, which should ideally have no branches and + a minimum number of dependent memory accesses. + +In practice, the parts may overlap, as data required for guards +can be re-used in the operation. + +If there are branches in the operation, then consider further specialization +to eliminate the branches. diff --git a/Python/specialize.c b/Python/specialize.c index a8ae09ff0e383..3277c6bc9e489 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -7,6 +7,10 @@ #include "opcode.h" #include "structmember.h" // struct PyMemberDef, T_OFFSET_EX +/* For guidance on adding or extending families of instructions see + * ./adaptive.md + */ + /* We layout the quickened data as a bi-directional array: * Instructions upwards, cache entries downwards.

1 0

bpo-43425: Update _osx_support not to use distutils.log (GH-26968)
by miss-islington 30 Jun '21

30 Jun '21

https://github.com/python/cpython/commit/94a4136c8eba349dc7eebe561ddaedbd0a… commit: 94a4136c8eba349dc7eebe561ddaedbd0a89eb91 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2021-06-30T18:20:24-07:00 summary: bpo-43425: Update _osx_support not to use distutils.log (GH-26968) (cherry picked from commit c8979f780e4b7d6db5693cb26a2956cc785abb48) Co-authored-by: Dong-hee Na <donghee.na(a)python.org> files: M Lib/_osx_support.py diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index b7145ee069fa3..326b36d611631 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -428,10 +428,9 @@ def compiler_fixup(compiler_so, cc_args): break if sysroot and not os.path.isdir(sysroot): - from distutils import log - log.warn("Compiling with an SDK that doesn't seem to exist: %s", - sysroot) - log.warn("Please check your Xcode installation") + sys.stderr.write(f"Compiling with an SDK that doesn't seem to exist: {sysroot}\n") + sys.stderr.write("Please check your Xcode installation\n") + sys.stderr.flush() return compiler_so

1 0

bpo-43770: Cleanup _PyObject_GetMethod() (GH-26946)
by vstinner 30 Jun '21

30 Jun '21

https://github.com/python/cpython/commit/dd3adc013b21ec1338bb5fea2e2c04a4fc… commit: dd3adc013b21ec1338bb5fea2e2c04a4fc650306 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2021-07-01T03:11:59+02:00 summary: bpo-43770: Cleanup _PyObject_GetMethod() (GH-26946) _PyObject_GetMethod() now uses _PyType_IsReady() to decide if PyType_Ready() must be called or not, rather than testing if tp->tp_dict is NULL. Move also variable declarations closer to where they are used, and use Py_NewRef(). files: M Objects/object.c diff --git a/Objects/object.c b/Objects/object.c index c87a83f225f14..8a854c7391ef9 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1124,25 +1124,24 @@ _PyObject_NextNotImplemented(PyObject *self) int _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) { - PyTypeObject *tp = Py_TYPE(obj); - PyObject *descr; - descrgetfunc f = NULL; - PyObject **dictptr, *dict; - PyObject *attr; int meth_found = 0; assert(*method == NULL); - if (Py_TYPE(obj)->tp_getattro != PyObject_GenericGetAttr - || !PyUnicode_Check(name)) { - *method = PyObject_GetAttr(obj, name); - return 0; + PyTypeObject *tp = Py_TYPE(obj); + if (!_PyType_IsReady(tp)) { + if (PyType_Ready(tp) < 0) { + return 0; + } } - if (tp->tp_dict == NULL && PyType_Ready(tp) < 0) + if (tp->tp_getattro != PyObject_GenericGetAttr || !PyUnicode_Check(name)) { + *method = PyObject_GetAttr(obj, name); return 0; + } - descr = _PyType_Lookup(tp, name); + PyObject *descr = _PyType_Lookup(tp, name); + descrgetfunc f = NULL; if (descr != NULL) { Py_INCREF(descr); if (_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) { @@ -1157,23 +1156,22 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } } - dictptr = _PyObject_GetDictPtr(obj); + PyObject **dictptr = _PyObject_GetDictPtr(obj); + PyObject *dict; if (dictptr != NULL && (dict = *dictptr) != NULL) { Py_INCREF(dict); - attr = PyDict_GetItemWithError(dict, name); + PyObject *attr = PyDict_GetItemWithError(dict, name); if (attr != NULL) { - Py_INCREF(attr); - *method = attr; + *method = Py_NewRef(attr); Py_DECREF(dict); Py_XDECREF(descr); return 0; } - else { - Py_DECREF(dict); - if (PyErr_Occurred()) { - Py_XDECREF(descr); - return 0; - } + Py_DECREF(dict); + + if (PyErr_Occurred()) { + Py_XDECREF(descr); + return 0; } }

1 0

bpo-43425: Update _osx_support not to use distutils.log (GH-26968)
by corona10 30 Jun '21

30 Jun '21

https://github.com/python/cpython/commit/c8979f780e4b7d6db5693cb26a2956cc78… commit: c8979f780e4b7d6db5693cb26a2956cc785abb48 branch: main author: Dong-hee Na <donghee.na(a)python.org> committer: corona10 <donghee.na92(a)gmail.com> date: 2021-07-01T09:35:10+09:00 summary: bpo-43425: Update _osx_support not to use distutils.log (GH-26968) files: M Lib/_osx_support.py diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index b7145ee069fa3..326b36d611631 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -428,10 +428,9 @@ def compiler_fixup(compiler_so, cc_args): break if sysroot and not os.path.isdir(sysroot): - from distutils import log - log.warn("Compiling with an SDK that doesn't seem to exist: %s", - sysroot) - log.warn("Please check your Xcode installation") + sys.stderr.write(f"Compiling with an SDK that doesn't seem to exist: {sysroot}\n") + sys.stderr.write("Please check your Xcode installation\n") + sys.stderr.flush() return compiler_so

1 0