Mailman 3 January 2025 - Python-checkins

GH-127178: improve compatibility in `_sysconfig_vars_(...).json` (#128558)
by FFY00 Jan. 29, 2025

Jan. 29, 2025

https://github.com/python/cpython/commit/c931d75831e4176b4876875c01e4b989c8… commit: c931d75831e4176b4876875c01e4b989c8915275 branch: main author: Filipe Laíns 🇵🇸 <lains(a)riseup.net> committer: FFY00 <filipe.lains(a)gmail.com> date: 2025-01-29T22:47:20Z summary: GH-127178: improve compatibility in `_sysconfig_vars_(...).json` (#128558) files: M Lib/sysconfig/__init__.py M Lib/sysconfig/__main__.py M Lib/test/test_sysconfig.py diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 3c3c9796ec3307..69f72452c4069a 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -116,8 +116,10 @@ def _getuserbase(): if env_base: return env_base - # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories - if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories. + # Use _PYTHON_HOST_PLATFORM to get the correct platform when cross-compiling. + system_name = os.environ.get('_PYTHON_HOST_PLATFORM', sys.platform).split('-')[0] + if system_name in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -342,6 +344,18 @@ def get_makefile_filename(): return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') +def _import_from_directory(path, name): + if name not in sys.modules: + import importlib.machinery + import importlib.util + + spec = importlib.machinery.PathFinder.find_spec(name, [path]) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[name] = module + return sys.modules[name] + + def _get_sysconfigdata_name(): multiarch = getattr(sys.implementation, '_multiarch', '') return os.environ.get( @@ -349,27 +363,34 @@ def _get_sysconfigdata_name(): f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', ) -def _init_posix(vars): - """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see _generate_posix_vars() + +def _get_sysconfigdata(): + import importlib + name = _get_sysconfigdata_name() + path = os.environ.get('_PYTHON_SYSCONFIGDATA_PATH') + module = _import_from_directory(path, name) if path else importlib.import_module(name) - # For cross builds, the path to the target's sysconfigdata must be specified - # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in - # sys.path can cause crashes when loaded by the host interpreter. - # Rely on truthiness as a valueless env variable is still an empty string. - # See OS X note in _generate_posix_vars re _sysconfigdata. - if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')): - from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES - from importlib.util import module_from_spec - spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name) - _temp = module_from_spec(spec) - spec.loader.exec_module(_temp) - else: - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars + return module.build_time_vars + + +def _installation_is_relocated(): + """Is the Python installation running from a different prefix than what was targetted when building?""" + if os.name != 'posix': + raise NotImplementedError('sysconfig._installation_is_relocated() is currently only supported on POSIX') + + data = _get_sysconfigdata() + return ( + data['prefix'] != getattr(sys, 'base_prefix', '') + or data['exec_prefix'] != getattr(sys, 'base_exec_prefix', '') + ) + + +def _init_posix(vars): + """Initialize the module as appropriate for POSIX systems.""" # GH-126920: Make sure we don't overwrite any of the keys already set - vars.update(build_time_vars | vars) + vars.update(_get_sysconfigdata() | vars) + def _init_non_posix(vars): """Initialize the module as appropriate for NT""" diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index 10728c709e1811..bc2197cfe79402 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -232,10 +232,14 @@ def _generate_posix_vars(): print(f'Written {destfile}') + install_vars = get_config_vars() + # Fix config vars to match the values after install (of the default environment) + install_vars['projectbase'] = install_vars['BINDIR'] + install_vars['srcdir'] = install_vars['LIBPL'] # Write a JSON file with the output of sysconfig.get_config_vars jsonfile = os.path.join(pybuilddir, _get_json_data_name()) with open(jsonfile, 'w') as f: - json.dump(get_config_vars(), f, indent=2) + json.dump(install_vars, f, indent=2) print(f'Written {jsonfile}') diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 1002d90074599a..3950d8888d4c8c 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -650,8 +650,21 @@ def test_sysconfigdata_json(self): system_config_vars = get_config_vars() - # Ignore keys in the check - for key in ('projectbase', 'srcdir'): + ignore_keys = set() + # Keys dependent on Python being run outside the build directrory + if sysconfig.is_python_build(): + ignore_keys |= {'srcdir'} + # Keys dependent on the executable location + if os.path.dirname(sys.executable) != system_config_vars['BINDIR']: + ignore_keys |= {'projectbase'} + # Keys dependent on the environment (different inside virtual environments) + if sys.prefix != sys.base_prefix: + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase'} + # Keys dependent on Python being run from the prefix targetted when building (different on relocatable installs) + if sysconfig._installation_is_relocated(): + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase'} + + for key in ignore_keys: json_config_vars.pop(key) system_config_vars.pop(key)

1 0

GH-128469: warn when libpython was loaded from outside the build directory (#128645)
by FFY00 Jan. 29, 2025

Jan. 29, 2025

https://github.com/python/cpython/commit/29b3ce8355b7d141ef61fa28757b7ff389… commit: 29b3ce8355b7d141ef61fa28757b7ff389189604 branch: main author: Filipe Laíns 🇵🇸 <lains(a)riseup.net> committer: FFY00 <filipe.lains(a)gmail.com> date: 2025-01-29T22:35:55Z summary: GH-128469: warn when libpython was loaded from outside the build directory (#128645) Co-authored-by: Peter Bierma <zintensitydev(a)gmail.com> files: M Lib/test/test_getpath.py M Modules/getpath.py diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index f86df9d0d03485..ca7cee0c39872a 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -1,7 +1,13 @@ import copy import ntpath +import os import pathlib import posixpath +import shutil +import subprocess +import sys +import sysconfig +import tempfile import unittest from test.support import verbose @@ -864,6 +870,37 @@ def test_PYTHONHOME_in_venv(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + +class RealGetPathTests(unittest.TestCase): + @unittest.skipUnless( + sysconfig.is_python_build(), + 'Test only available when running from the buildir', + ) + @unittest.skipUnless( + any(sys.platform.startswith(p) for p in ('linux', 'freebsd', 'centos')), + 'Test only support on Linux-like OS-es (support LD_LIBRARY_PATH)', + ) + @unittest.skipUnless( + sysconfig.get_config_var('LDLIBRARY') != sysconfig.get_config_var('LIBRARY'), + 'Test only available when using a dynamic libpython', + ) + def test_builddir_wrong_library_warning(self): + library_name = sysconfig.get_config_var('INSTSONAME') + with tempfile.TemporaryDirectory() as tmpdir: + shutil.copy2( + os.path.join(sysconfig.get_config_var('srcdir'), library_name), + os.path.join(tmpdir, library_name) + ) + env = os.environ.copy() + env['LD_LIBRARY_PATH'] = tmpdir + process = subprocess.run( + [sys.executable, '-c', ''], + env=env, check=True, capture_output=True, text=True, + ) + error_msg = 'The runtime library has been loaded from outside the build directory' + self.assertTrue(process.stderr.startswith(error_msg), process.stderr) + + # ****************************************************************************** DEFAULT_NAMESPACE = dict( diff --git a/Modules/getpath.py b/Modules/getpath.py index be2210345afbda..9d531e29becbc8 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -785,6 +785,19 @@ def search_up(prefix, *landmarks, test=isfile): base_exec_prefix = config.get('base_exec_prefix') or EXEC_PREFIX or base_prefix +# ****************************************************************************** +# MISC. RUNTIME WARNINGS +# ****************************************************************************** + +# When running Python from the build directory, if libpython is dynamically +# linked, the wrong library might be loaded. +if build_prefix and library and not dirname(abspath(library)).startswith(build_prefix): + msg = f'The runtime library has been loaded from outside the build directory ({library})!' + if os_name == 'posix': + msg += ' Consider setting LD_LIBRARY_PATH=. to force it to be loaded from the build directory.' + warn(msg) + + # ****************************************************************************** # SET pythonpath FROM _PTH FILE # ******************************************************************************

1 0

[3.13] gh-125522: Fix bare except in `test_uuid` (GH-129018) (#129455)
by iritkatriel Jan. 29, 2025

Jan. 29, 2025

https://github.com/python/cpython/commit/8b4a0d641cde667f94ce49f5e64da6bd9d… commit: 8b4a0d641cde667f94ce49f5e64da6bd9d6fbd9c branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel(a)users.noreply.github.com> date: 2025-01-29T21:23:10Z summary: [3.13] gh-125522: Fix bare except in `test_uuid` (GH-129018) (#129455) gh-125522: Fix bare except in `test_uuid` (GH-129018) (cherry picked from commit a1a4e9f39ad86359e148fd193089b3b2a354f71d) Co-authored-by: Tomas R <tomas.roun8(a)gmail.com> files: M Lib/test/test_uuid.py diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index e177464c00f7a6..e7e44c6413c2e2 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -19,7 +19,7 @@ def importable(name): try: __import__(name) return True - except: + except ModuleNotFoundError: return False

1 0

[3.12] gh-125522: Fix bare except in `test_uuid` (GH-129018) (#129456)
by iritkatriel Jan. 29, 2025

Jan. 29, 2025

https://github.com/python/cpython/commit/c0ae83af95df3de49802f02977cc937ad8… commit: c0ae83af95df3de49802f02977cc937ad855fc29 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel(a)users.noreply.github.com> date: 2025-01-29T21:16:47Z summary: [3.12] gh-125522: Fix bare except in `test_uuid` (GH-129018) (#129456) gh-125522: Fix bare except in `test_uuid` (GH-129018) (cherry picked from commit a1a4e9f39ad86359e148fd193089b3b2a354f71d) Co-authored-by: Tomas R <tomas.roun8(a)gmail.com> files: M Lib/test/test_uuid.py diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 9cec1e87fd3c2d..08359a336f2f8d 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -19,7 +19,7 @@ def importable(name): try: __import__(name) return True - except: + except ModuleNotFoundError: return False

1 0

gh-125522: Fix bare except in `test_uuid` (#129018)
by iritkatriel Jan. 29, 2025

Jan. 29, 2025

https://github.com/python/cpython/commit/a1a4e9f39ad86359e148fd193089b3b2a3… commit: a1a4e9f39ad86359e148fd193089b3b2a354f71d branch: main author: Tomas R. <tomas.roun8(a)gmail.com> committer: iritkatriel <1055913+iritkatriel(a)users.noreply.github.com> date: 2025-01-29T20:59:45Z summary: gh-125522: Fix bare except in `test_uuid` (#129018) files: M Lib/test/test_uuid.py diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 74c8e2838efc8f..8216c4dd00e35a 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -21,7 +21,7 @@ def importable(name): try: __import__(name) return True - except: + except ModuleNotFoundError: return False

1 0

gh-126400: Add TCP socket timeout to SysLogHandler to prevent blocking (GH-126716)
by vsajip Jan. 29, 2025

Jan. 29, 2025

https://github.com/python/cpython/commit/fdcedfd3cf1a645634dee354ee349d966c… commit: fdcedfd3cf1a645634dee354ee349d966c56348f branch: main author: Hod <104125115+hodamarr(a)users.noreply.github.com> committer: vsajip <vinay_sajip(a)yahoo.co.uk> date: 2025-01-29T19:37:43Z summary: gh-126400: Add TCP socket timeout to SysLogHandler to prevent blocking (GH-126716) Co-authored-by: Vinay Sajip <vinay_sajip(a)yahoo.co.uk> files: A Misc/NEWS.d/next/Library/2025-01-29-13-37-18.gh-issue-126400.DaBaR3.rst M Doc/library/logging.handlers.rst M Lib/logging/handlers.py M Lib/test/test_logging.py diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 5a081f9e7add99..ffb54591b3563b 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -613,7 +613,7 @@ The :class:`SysLogHandler` class, located in the :mod:`logging.handlers` module, supports sending logging messages to a remote or local Unix syslog. -.. class:: SysLogHandler(address=('localhost', SYSLOG_UDP_PORT), facility=LOG_USER, socktype=socket.SOCK_DGRAM) +.. class:: SysLogHandler(address=('localhost', SYSLOG_UDP_PORT), facility=LOG_USER, socktype=socket.SOCK_DGRAM, timeout=None) Returns a new instance of the :class:`SysLogHandler` class intended to communicate with a remote Unix machine whose address is given by *address* in @@ -626,6 +626,11 @@ supports sending logging messages to a remote or local Unix syslog. *socktype* argument, which defaults to :const:`socket.SOCK_DGRAM` and thus opens a UDP socket. To open a TCP socket (for use with the newer syslog daemons such as rsyslog), specify a value of :const:`socket.SOCK_STREAM`. + If *timeout* is specified, it sets a timeout (in seconds) for the socket operations. + This can help prevent the program from hanging indefinitely if the syslog server is + unreachable. By default, *timeout* is ``None``, meaning no timeout is applied. + + Note that if your server is not listening on UDP port 514, :class:`SysLogHandler` may appear not to work. In that case, check what @@ -645,6 +650,8 @@ supports sending logging messages to a remote or local Unix syslog. .. versionchanged:: 3.2 *socktype* was added. + .. versionchanged:: 3.14 + *timeout* was added. .. method:: close() diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 1cba64fd554100..017c9ab409b7bc 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -855,7 +855,7 @@ class SysLogHandler(logging.Handler): } def __init__(self, address=('localhost', SYSLOG_UDP_PORT), - facility=LOG_USER, socktype=None): + facility=LOG_USER, socktype=None, timeout=None): """ Initialize a handler. @@ -872,6 +872,7 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT), self.address = address self.facility = facility self.socktype = socktype + self.timeout = timeout self.socket = None self.createSocket() @@ -933,6 +934,8 @@ def createSocket(self): err = sock = None try: sock = socket.socket(af, socktype, proto) + if self.timeout: + sock.settimeout(self.timeout) if socktype == socket.SOCK_STREAM: sock.connect(sa) break diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 2e5f6475ae3b1e..e34fe45fd68e52 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -22,6 +22,7 @@ import logging.handlers import logging.config + import codecs import configparser import copy @@ -2095,6 +2096,18 @@ def test_udp_reconnection(self): self.handled.wait(support.LONG_TIMEOUT) self.assertEqual(self.log_output, b'<11>sp\xc3\xa4m\x00') + @patch('socket.socket') + def test_tcp_timeout(self, mock_socket): + instance_mock_sock = mock_socket.return_value + instance_mock_sock.connect.side_effect = socket.timeout + + with self.assertRaises(socket.timeout): + logging.handlers.SysLogHandler(address=('localhost', 514), + socktype=socket.SOCK_STREAM, + timeout=1) + + instance_mock_sock.close.assert_called() + @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") class UnixSysLogHandlerTest(SysLogHandlerTest): diff --git a/Misc/NEWS.d/next/Library/2025-01-29-13-37-18.gh-issue-126400.DaBaR3.rst b/Misc/NEWS.d/next/Library/2025-01-29-13-37-18.gh-issue-126400.DaBaR3.rst new file mode 100644 index 00000000000000..1532faf4b7d6f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-29-13-37-18.gh-issue-126400.DaBaR3.rst @@ -0,0 +1,2 @@ +Add a socket *timeout* keyword argument to +:class:`logging.handlers.SysLogHandler`.

1 0

GH-129386: Use symbolic constants for specialization tests (GH-129415)
by brandtbucher Jan. 29, 2025

Jan. 29, 2025

https://github.com/python/cpython/commit/002c4e2982910937f35a4b80ab94b07c3c… commit: 002c4e2982910937f35a4b80ab94b07c3c16067f branch: main author: Brandt Bucher <brandtbucher(a)microsoft.com> committer: brandtbucher <brandtbucher(a)gmail.com> date: 2025-01-29T10:49:58-08:00 summary: GH-129386: Use symbolic constants for specialization tests (GH-129415) files: M Lib/test/test_call.py M Lib/test/test_dis.py M Lib/test/test_embed.py M Lib/test/test_monitoring.py M Lib/test/test_opcache.py M Lib/test/test_super.py M Lib/test/test_type_cache.py M Modules/_testinternalcapi.c diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 78a706436aea0e..a4115d54ce5995 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,6 +1,6 @@ import unittest from test.support import (cpython_only, is_wasi, requires_limited_api, Py_DEBUG, - set_recursion_limit, skip_on_s390x, skip_emscripten_stack_overflow) + set_recursion_limit, skip_on_s390x, skip_emscripten_stack_overflow, import_helper) try: import _testcapi except ImportError: @@ -616,9 +616,6 @@ def testfunction_kw(self, *, kw): return self -ADAPTIVE_WARMUP_DELAY = 2 - - @unittest.skipIf(_testcapi is None, "requires _testcapi") class TestPEP590(unittest.TestCase): @@ -802,17 +799,18 @@ def __call__(self, *args): def test_setvectorcall(self): from _testcapi import function_setvectorcall + _testinternalcapi = import_helper.import_module("_testinternalcapi") def f(num): return num + 1 assert_equal = self.assertEqual num = 10 assert_equal(11, f(num)) function_setvectorcall(f) - # make sure specializer is triggered by running > 50 times - for _ in range(10 * ADAPTIVE_WARMUP_DELAY): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): assert_equal("overridden", f(num)) def test_setvectorcall_load_attr_specialization_skip(self): from _testcapi import function_setvectorcall + _testinternalcapi = import_helper.import_module("_testinternalcapi") class X: def __getattribute__(self, attr): @@ -824,11 +822,12 @@ def __getattribute__(self, attr): function_setvectorcall(X.__getattribute__) # make sure specialization doesn't trigger # when vectorcall is overridden - for _ in range(ADAPTIVE_WARMUP_DELAY): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): assert_equal("overridden", x.a) def test_setvectorcall_load_attr_specialization_deopt(self): from _testcapi import function_setvectorcall + _testinternalcapi = import_helper.import_module("_testinternalcapi") class X: def __getattribute__(self, attr): @@ -840,12 +839,12 @@ def get_a(x): assert_equal = self.assertEqual x = X() # trigger LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN specialization - for _ in range(ADAPTIVE_WARMUP_DELAY): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): assert_equal("a", get_a(x)) function_setvectorcall(X.__getattribute__) # make sure specialized LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN # gets deopted due to overridden vectorcall - for _ in range(ADAPTIVE_WARMUP_DELAY): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): assert_equal("overridden", get_a(x)) @requires_limited_api diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 2e149b32e5c1ec..bba2ac80aa6769 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -931,8 +931,6 @@ def extended_arg_quick(): """% (extended_arg_quick.__code__.co_firstlineno, extended_arg_quick.__code__.co_firstlineno + 1,) -ADAPTIVE_WARMUP_DELAY = 2 - class DisTestBase(unittest.TestCase): "Common utilities for DisTests and TestDisTraceback" @@ -1259,8 +1257,9 @@ def test__try_compile_no_context_exc_on_error(self): self.assertIsNone(e.__context__) @staticmethod - def code_quicken(f, times=ADAPTIVE_WARMUP_DELAY): - for _ in range(times): + def code_quicken(f): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f() @cpython_only @@ -1306,7 +1305,7 @@ def test_call_specialize(self): @requires_specialization def test_loop_quicken(self): # Loop can trigger a quicken where the loop is located - self.code_quicken(loop_test, 4) + self.code_quicken(loop_test) got = self.get_disassembly(loop_test, adaptive=True) jit = import_helper.import_module("_testinternalcapi").jit_enabled() expected = dis_loop_test_quickened_code.format("JIT" if jit else "NO_JIT") @@ -1315,8 +1314,9 @@ def test_loop_quicken(self): @cpython_only @requires_specialization def test_loop_with_conditional_at_end_is_quickened(self): + _testinternalcapi = import_helper.import_module("_testinternalcapi") def for_loop_true(x): - for i in range(10): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): if x: pass @@ -1325,7 +1325,7 @@ def for_loop_true(x): self.get_disassembly(for_loop_true, adaptive=True)) def for_loop_false(x): - for i in range(10): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): if x: pass @@ -1335,7 +1335,7 @@ def for_loop_false(x): def while_loop(): i = 0 - while i < 10: + while i < _testinternalcapi.SPECIALIZATION_THRESHOLD: i += 1 while_loop() diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index ffb69c87a065aa..f41c45ec4d9cdd 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -384,7 +384,9 @@ def test_simple_initialization_api(self): def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self): # https://github.com/python/cpython/issues/92031 - code = textwrap.dedent("""\ + _testinternalcapi = import_helper.import_module("_testinternalcapi") + + code = textwrap.dedent(f"""\ import dis import importlib._bootstrap import opcode @@ -411,7 +413,7 @@ def is_specialized(f): assert not is_specialized(func), "specialized instructions found" - for i in range(test.test_dis.ADAPTIVE_WARMUP_DELAY): + for _ in range({_testinternalcapi.SPECIALIZATION_THRESHOLD}): func(importlib._bootstrap, ["x"], lambda *args: None) assert is_specialized(func), "no specialized instructions found" diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 364381e7dce00a..3125d190626e38 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -14,6 +14,7 @@ from test.support import requires_specialization_ft, script_helper _testcapi = test.support.import_helper.import_module("_testcapi") +_testinternalcapi = test.support.import_helper.import_module("_testinternalcapi") PAIR = (0,1) @@ -897,13 +898,13 @@ def implicit_stop_iteration(iterator=None): # re-specialize immediately, so that we can we can test the # unspecialized version of the loop first. # Note: this assumes that we don't specialize loops over sets. - implicit_stop_iteration(set(range(100))) + implicit_stop_iteration(set(range(_testinternalcapi.SPECIALIZATION_THRESHOLD))) # This will record a RAISE event for the StopIteration. self.check_events(implicit_stop_iteration, expected, recorders=recorders) # Now specialize, so that we see a STOP_ITERATION event. - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): implicit_stop_iteration() # This will record a STOP_ITERATION event for the StopIteration. @@ -1057,7 +1058,7 @@ def f(): except ValueError: pass - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f() recorders = ( ReturnRecorder, @@ -2033,8 +2034,8 @@ def __init__(self, set_event): sys.monitoring.set_events(TEST_TOOL, E.PY_RESUME) def make_foo_optimized_then_set_event(): - for i in range(100): - Foo(i == 99) + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD + 1): + Foo(i == _testinternalcapi.SPECIALIZATION_THRESHOLD) try: make_foo_optimized_then_set_event() @@ -2106,9 +2107,9 @@ def test_func(recorder): set_events = sys.monitoring.set_events line = E.LINE i = 0 - for i in range(551): - # Turn on events without branching once i reaches 500. - set_events(TEST_TOOL, line * int(i >= 500)) + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD + 51): + # Turn on events without branching once i reaches _testinternalcapi.SPECIALIZATION_THRESHOLD. + set_events(TEST_TOOL, line * int(i >= _testinternalcapi.SPECIALIZATION_THRESHOLD)) pass pass pass diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index dc02d1d7babb23..2defe74892786d 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -45,7 +45,8 @@ def f(self): d = D() - self.assertEqual(d.f(), 1) # warmup + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD - 1): + self.assertEqual(d.f(), 1) # warmup calls.clear() self.assertEqual(d.f(), 1) # try to specialize self.assertEqual(calls, [(d, D)]) @@ -65,7 +66,7 @@ def f(o): return o.x o = C() - for i in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): assert f(o) == 1 Descriptor.__get__ = lambda self, instance, value: 2 @@ -92,13 +93,13 @@ def __set__(self, instance, value): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Descriptor.__get__ = __get__ Descriptor.__set__ = __set__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_descriptor_shadows_class_attribute(self): @@ -113,7 +114,7 @@ class Class(metaclass=Metaclass): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) def test_metaclass_set_descriptor_after_optimization(self): @@ -130,12 +131,12 @@ def attribute(self): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Metaclass.attribute = attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_del_descriptor_after_optimization(self): @@ -150,12 +151,12 @@ class Class(metaclass=Metaclass): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) del Metaclass.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_type_descriptor_shadows_attribute_method(self): @@ -165,7 +166,7 @@ class Class: def f(): return Class.mro - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertIsNone(f()) def test_type_descriptor_shadows_attribute_member(self): @@ -175,7 +176,7 @@ class Class: def f(): return Class.__base__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertIs(f(), object) def test_type_descriptor_shadows_attribute_getset(self): @@ -185,7 +186,7 @@ class Class: def f(): return Class.__name__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertEqual(f(), "Class") def test_metaclass_getattribute(self): @@ -199,7 +200,7 @@ class Class(metaclass=Metaclass): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) def test_metaclass_swap(self): @@ -219,12 +220,12 @@ class Class(metaclass=OldMetaclass): def f(): return Class.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Class.__class__ = NewMetaclass - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_load_shadowing_slot_should_raise_type_error(self): @@ -241,7 +242,7 @@ def f(o): o = Sneaky() o.shadowed = 42 - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): with self.assertRaises(TypeError): f(o) @@ -258,7 +259,7 @@ def f(o): o = Sneaky() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): with self.assertRaises(TypeError): f(o) @@ -274,7 +275,7 @@ def f(o): o = Sneaky() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): with self.assertRaises(TypeError): f(o) @@ -290,7 +291,7 @@ def f(o): o = Sneaky() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): with self.assertRaises(TypeError): f(o) @@ -318,13 +319,13 @@ def attribute(): def f(): return instance.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Descriptor.__get__ = __get__ Descriptor.__set__ = __set__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_descriptor_added_after_optimization(self): @@ -347,13 +348,13 @@ def __set__(self, instance, value): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Descriptor.__get__ = __get__ Descriptor.__set__ = __set__ - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_descriptor_shadows_class_attribute(self): @@ -369,7 +370,7 @@ def attribute(): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) def test_metaclass_set_descriptor_after_optimization(self): @@ -387,12 +388,12 @@ def attribute(self): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Metaclass.attribute = attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_metaclass_del_descriptor_after_optimization(self): @@ -408,12 +409,12 @@ def attribute(): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) del Metaclass.attribute - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) def test_type_descriptor_shadows_attribute_method(self): @@ -424,7 +425,7 @@ def mro(): def f(): return Class.mro() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertEqual(f(), ["Spam", "eggs"]) def test_type_descriptor_shadows_attribute_member(self): @@ -435,7 +436,7 @@ def __base__(): def f(): return Class.__base__() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertNotEqual(f(), "Spam") def test_metaclass_getattribute(self): @@ -450,7 +451,7 @@ def attribute(): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) def test_metaclass_swap(self): @@ -470,12 +471,12 @@ class Class(metaclass=OldMetaclass): def f(): return Class.attribute() - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertTrue(f()) Class.__class__ = NewMetaclass - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertFalse(f()) @@ -490,7 +491,7 @@ def f(): pass f.__defaults__ = (None,) - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f() def test_too_many_defaults_1(self): @@ -498,7 +499,7 @@ def f(x): pass f.__defaults__ = (None, None) - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f(None) f() @@ -507,7 +508,7 @@ def f(x, y): pass f.__defaults__ = (None, None, None) - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): f(None, None) f(None) f() @@ -523,7 +524,7 @@ def instantiate(): return MyClass() # Trigger specialization - for _ in range(1025): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): instantiate() self.assert_specialized(instantiate, "CALL_ALLOC_AND_ENTER_INIT") @@ -541,7 +542,7 @@ def test_push_init_frame_fails(self): def instantiate(): return InitTakesArg() - for _ in range(2): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): with self.assertRaises(TypeError): instantiate() self.assert_specialized(instantiate, "CALL_ALLOC_AND_ENTER_INIT") @@ -566,7 +567,6 @@ class TestRacesDoNotCrash(TestBase): # but you can also burn through a *ton* of type/dict/function versions: ITEMS = 1000 LOOPS = 4 - WARMUPS = 2 WRITERS = 2 @requires_jit_disabled @@ -583,7 +583,7 @@ def assert_races_do_not_crash( else: read.__code__ = read.__code__.replace() # Specialize: - for _ in range(self.WARMUPS): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): read(items) if check_items: for item in items: @@ -1011,7 +1011,7 @@ class C: item = C() item.a = None # Resize into a combined unicode dict: - for i in range(29): + for i in range(_testinternalcapi.SHARED_KEYS_MAX_SIZE - 1): setattr(item, f"_{i}", None) items.append(item) return items @@ -1082,7 +1082,7 @@ class C: for _ in range(self.ITEMS): item = C() # Resize into a combined unicode dict: - for i in range(29): + for i in range(_testinternalcapi.SHARED_KEYS_MAX_SIZE - 1): setattr(item, f"_{i}", None) items.append(item) return items @@ -1178,7 +1178,7 @@ def test_dict_dematerialization(self): c.a = 1 c.b = 2 c.__dict__ - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a self.assertEqual( _testinternalcapi.get_object_dict_values(c), @@ -1190,7 +1190,7 @@ def test_dict_dematerialization_multiple_refs(self): c.a = 1 c.b = 2 d = c.__dict__ - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a self.assertIs(c.__dict__, d) @@ -1199,7 +1199,7 @@ def test_dict_dematerialization_copy(self): c.a = 1 c.b = 2 c2 = copy.copy(c) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a c2.a self.assertEqual( @@ -1211,7 +1211,7 @@ def test_dict_dematerialization_copy(self): (1, 2, '<NULL>') ) c3 = copy.deepcopy(c) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a c3.a self.assertEqual( @@ -1225,7 +1225,7 @@ def test_dict_dematerialization_pickle(self): c.a = 1 c.b = 2 c2 = pickle.loads(pickle.dumps(c)) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a c2.a self.assertEqual( @@ -1243,7 +1243,7 @@ class D(dict): pass c.a = 1 c.b = 2 c.__dict__ = D(c.__dict__) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c.a self.assertIs( _testinternalcapi.get_object_dict_values(c), @@ -1288,7 +1288,7 @@ def f(o, n): for i in range(n): o.b = i # Prime f to store to dict slot 1 - f(c, 100) + f(c, _testinternalcapi.SPECIALIZATION_THRESHOLD) test_obj = NoInlineAorB() test_obj.__dict__ = make_special_dict() @@ -1305,7 +1305,7 @@ class TestSpecializer(TestBase): @requires_specialization_ft def test_binary_op(self): def binary_op_add_int(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = 1, 2 c = a + b self.assertEqual(c, 3) @@ -1315,7 +1315,7 @@ def binary_op_add_int(): self.assert_no_opcode(binary_op_add_int, "BINARY_OP") def binary_op_add_unicode(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = "foo", "bar" c = a + b self.assertEqual(c, "foobar") @@ -1325,7 +1325,7 @@ def binary_op_add_unicode(): self.assert_no_opcode(binary_op_add_unicode, "BINARY_OP") def binary_op_add_extend(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = 6, 3.0 c = a + b self.assertEqual(c, 9.0) @@ -1384,11 +1384,13 @@ def compactlong_rhs(arg): arg / 42, ) nan = float('nan') - self.assertEqual(compactlong_lhs(1.0), (43.0, 41.0, 42.0, 42.0)) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(compactlong_lhs(1.0), (43.0, 41.0, 42.0, 42.0)) + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertTrue(all(filter(lambda x: x is nan, compactlong_lhs(nan)))) - self.assertEqual(compactlong_rhs(42.0), (84.0, 0.0, 84.0, 1.0)) - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(compactlong_rhs(42.0), (84.0, 0.0, 84.0, 1.0)) + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): self.assertTrue(all(filter(lambda x: x is nan, compactlong_rhs(nan)))) self.assert_no_opcode(compactlong_lhs, "BINARY_OP_EXTEND") @@ -1429,7 +1431,7 @@ def __init__(self): meth = super().__init__ super().__init__() - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): A() self.assert_specialized(A.__init__, "LOAD_SUPER_ATTR_ATTR") @@ -1449,7 +1451,7 @@ def init(self): globals()['super'] = fake_super try: # Should be unspecialized after enough calls. - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): A() finally: globals()['super'] = real_super @@ -1462,7 +1464,7 @@ def init(self): @requires_specialization_ft def test_contain_op(self): def contains_op_dict(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = 1, {1: 2, 2: 5} self.assertTrue(a in b) self.assertFalse(3 in b) @@ -1472,7 +1474,7 @@ def contains_op_dict(): self.assert_no_opcode(contains_op_dict, "CONTAINS_OP") def contains_op_set(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = 1, {1, 2} self.assertTrue(a in b) self.assertFalse(3 in b) @@ -1499,7 +1501,7 @@ async def __aexit__(self, *exc): pass async def send_with(): - for i in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): async with CM(): x = 1 @@ -1517,7 +1519,7 @@ def g(): def send_yield_from(): yield from g() - for i in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): list(send_yield_from()) self.assert_specialized(send_yield_from, "SEND_GEN") @@ -1529,19 +1531,19 @@ def test_store_attr_slot(self): class C: __slots__ = ['x'] - def set_slot(): + def set_slot(n): c = C() - for i in range(100): + for i in range(n): c.x = i - set_slot() + set_slot(_testinternalcapi.SPECIALIZATION_THRESHOLD) self.assert_specialized(set_slot, "STORE_ATTR_SLOT") self.assert_no_opcode(set_slot, "STORE_ATTR") # Adding a property for 'x' should unspecialize it. C.x = property(lambda self: None, lambda self, x: None) - set_slot() + set_slot(_testinternalcapi.SPECIALIZATION_COOLDOWN) self.assert_no_opcode(set_slot, "STORE_ATTR_SLOT") @cpython_only @@ -1550,19 +1552,19 @@ def test_store_attr_instance_value(self): class C: pass - def set_value(): + def set_value(n): c = C() - for i in range(100): + for i in range(n): c.x = i - set_value() + set_value(_testinternalcapi.SPECIALIZATION_THRESHOLD) self.assert_specialized(set_value, "STORE_ATTR_INSTANCE_VALUE") self.assert_no_opcode(set_value, "STORE_ATTR") # Adding a property for 'x' should unspecialize it. C.x = property(lambda self: None, lambda self, x: None) - set_value() + set_value(_testinternalcapi.SPECIALIZATION_COOLDOWN) self.assert_no_opcode(set_value, "STORE_ATTR_INSTANCE_VALUE") @cpython_only @@ -1572,21 +1574,21 @@ class C: pass c = C() - for i in range(29): + for i in range(_testinternalcapi.SHARED_KEYS_MAX_SIZE - 1): setattr(c, f"_{i}", None) - def set_value(): - for i in range(100): + def set_value(n): + for i in range(n): c.x = i - set_value() + set_value(_testinternalcapi.SPECIALIZATION_THRESHOLD) self.assert_specialized(set_value, "STORE_ATTR_WITH_HINT") self.assert_no_opcode(set_value, "STORE_ATTR") # Adding a property for 'x' should unspecialize it. C.x = property(lambda self: None, lambda self, x: None) - set_value() + set_value(_testinternalcapi.SPECIALIZATION_COOLDOWN) self.assert_no_opcode(set_value, "STORE_ATTR_WITH_HINT") @cpython_only @@ -1594,14 +1596,15 @@ def set_value(): def test_to_bool(self): def to_bool_bool(): true_cnt, false_cnt = 0, 0 - elems = [e % 2 == 0 for e in range(100)] + elems = [e % 2 == 0 for e in range(_testinternalcapi.SPECIALIZATION_THRESHOLD)] for e in elems: if e: true_cnt += 1 else: false_cnt += 1 - self.assertEqual(true_cnt, 50) - self.assertEqual(false_cnt, 50) + d, m = divmod(_testinternalcapi.SPECIALIZATION_THRESHOLD, 2) + self.assertEqual(true_cnt, d + m) + self.assertEqual(false_cnt, d) to_bool_bool() self.assert_specialized(to_bool_bool, "TO_BOOL_BOOL") @@ -1609,12 +1612,12 @@ def to_bool_bool(): def to_bool_int(): count = 0 - for i in range(100): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): if i: count += 1 else: count -= 1 - self.assertEqual(count, 98) + self.assertEqual(count, _testinternalcapi.SPECIALIZATION_THRESHOLD - 2) to_bool_int() self.assert_specialized(to_bool_int, "TO_BOOL_INT") @@ -1622,11 +1625,11 @@ def to_bool_int(): def to_bool_list(): count = 0 - elems = [1, 2, 3] + elems = list(range(_testinternalcapi.SPECIALIZATION_THRESHOLD)) while elems: count += elems.pop() self.assertEqual(elems, []) - self.assertEqual(count, 6) + self.assertEqual(count, sum(range(_testinternalcapi.SPECIALIZATION_THRESHOLD))) to_bool_list() self.assert_specialized(to_bool_list, "TO_BOOL_LIST") @@ -1634,11 +1637,11 @@ def to_bool_list(): def to_bool_none(): count = 0 - elems = [None, None, None, None] + elems = [None] * _testinternalcapi.SPECIALIZATION_THRESHOLD for e in elems: if not e: count += 1 - self.assertEqual(count, len(elems)) + self.assertEqual(count, _testinternalcapi.SPECIALIZATION_THRESHOLD) to_bool_none() self.assert_specialized(to_bool_none, "TO_BOOL_NONE") @@ -1646,11 +1649,11 @@ def to_bool_none(): def to_bool_str(): count = 0 - elems = ["", "foo", ""] + elems = [""] + ["foo"] * (_testinternalcapi.SPECIALIZATION_THRESHOLD - 1) for e in elems: if e: count += 1 - self.assertEqual(count, 1) + self.assertEqual(count, _testinternalcapi.SPECIALIZATION_THRESHOLD - 1) to_bool_str() self.assert_specialized(to_bool_str, "TO_BOOL_STR") @@ -1660,7 +1663,7 @@ def to_bool_str(): @requires_specialization_ft def test_unpack_sequence(self): def unpack_sequence_two_tuple(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = 1, 2 self.assertEqual(a, 1) self.assertEqual(b, 2) @@ -1671,7 +1674,7 @@ def unpack_sequence_two_tuple(): self.assert_no_opcode(unpack_sequence_two_tuple, "UNPACK_SEQUENCE") def unpack_sequence_tuple(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, = 1, self.assertEqual(a, 1) @@ -1680,7 +1683,7 @@ def unpack_sequence_tuple(): self.assert_no_opcode(unpack_sequence_tuple, "UNPACK_SEQUENCE") def unpack_sequence_list(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = [1, 2] self.assertEqual(a, 1) self.assertEqual(b, 2) @@ -1693,7 +1696,7 @@ def unpack_sequence_list(): @requires_specialization_ft def test_binary_subscr(self): def binary_subscr_list_int(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a = [1, 2, 3] for idx, expected in enumerate(a): self.assertEqual(a[idx], expected) @@ -1704,7 +1707,7 @@ def binary_subscr_list_int(): self.assert_no_opcode(binary_subscr_list_int, "BINARY_SUBSCR") def binary_subscr_tuple_int(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a = (1, 2, 3) for idx, expected in enumerate(a): self.assertEqual(a[idx], expected) @@ -1715,7 +1718,7 @@ def binary_subscr_tuple_int(): self.assert_no_opcode(binary_subscr_tuple_int, "BINARY_SUBSCR") def binary_subscr_dict(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a = {1: 2, 2: 3} self.assertEqual(a[1], 2) self.assertEqual(a[2], 3) @@ -1725,7 +1728,7 @@ def binary_subscr_dict(): self.assert_no_opcode(binary_subscr_dict, "BINARY_SUBSCR") def binary_subscr_str_int(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a = "foobar" for idx, expected in enumerate(a): self.assertEqual(a[idx], expected) @@ -1741,8 +1744,8 @@ def __init__(self, val): def __getitem__(self, item): return self.val - items = [C(i) for i in range(100)] - for i in range(100): + items = [C(i) for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD)] + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertEqual(items[i][i], i) binary_subscr_getitems() @@ -1753,7 +1756,7 @@ def __getitem__(self, item): @requires_specialization_ft def test_compare_op(self): def compare_op_int(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = 1, 2 c = a == b self.assertFalse(c) @@ -1763,7 +1766,7 @@ def compare_op_int(): self.assert_no_opcode(compare_op_int, "COMPARE_OP") def compare_op_float(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = 1.0, 2.0 c = a == b self.assertFalse(c) @@ -1773,7 +1776,7 @@ def compare_op_float(): self.assert_no_opcode(compare_op_float, "COMPARE_OP") def compare_op_str(): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = "spam", "ham" c = a == b self.assertFalse(c) diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 149016635522c3..5cef612a340be9 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -9,9 +9,6 @@ from test.support import import_helper, threading_helper -ADAPTIVE_WARMUP_DELAY = 2 - - class A: def f(self): return 'A' @@ -466,7 +463,8 @@ def test(name): super(MyType, type(mytype)).__setattr__(mytype, "bar", 1) self.assertEqual(mytype.bar, 1) - for _ in range(ADAPTIVE_WARMUP_DELAY): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): test("foo1") def test_reassigned_new(self): @@ -485,7 +483,8 @@ class C(B): def __new__(cls): return super().__new__(cls) - for _ in range(ADAPTIVE_WARMUP_DELAY): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): C() def test_mixed_staticmethod_hierarchy(self): @@ -505,7 +504,8 @@ class C(B): def some(cls): return super().some(cls) - for _ in range(ADAPTIVE_WARMUP_DELAY): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): C.some(C) @threading_helper.requires_working_threading() diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index e109a65741309a..ee64f89358ed55 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -131,7 +131,7 @@ def _all_opnames(self, func): return set(instr.opname for instr in dis.Bytecode(func, adaptive=True)) def _check_specialization(self, func, arg, opname, *, should_specialize): - for _ in range(100): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): func(arg) if should_specialize: diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 9f38a9d7a9a5c2..e44b629897c58a 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2138,6 +2138,21 @@ module_exec(PyObject *module) return 1; } + if (PyModule_Add(module, "SPECIALIZATION_THRESHOLD", + PyLong_FromLong(ADAPTIVE_WARMUP_VALUE + 1)) < 0) { + return 1; + } + + if (PyModule_Add(module, "SPECIALIZATION_COOLDOWN", + PyLong_FromLong(ADAPTIVE_COOLDOWN_VALUE + 1)) < 0) { + return 1; + } + + if (PyModule_Add(module, "SHARED_KEYS_MAX_SIZE", + PyLong_FromLong(SHARED_KEYS_MAX_SIZE)) < 0) { + return 1; + } + return 0; }

1 0

gh-118761: Always lazy import `warnings` in `threading` (#129428)
by AA-Turner Jan. 29, 2025

Jan. 29, 2025

https://github.com/python/cpython/commit/99ed3025fe8fa9079b4c1eac01c5af62ca… commit: 99ed3025fe8fa9079b4c1eac01c5af62caa98c15 branch: main author: Taneli Hukkinen <3275109+hukkin(a)users.noreply.github.com> committer: AA-Turner <9087854+AA-Turner(a)users.noreply.github.com> date: 2025-01-29T17:24:37Z summary: gh-118761: Always lazy import `warnings` in `threading` (#129428) files: A Misc/NEWS.d/next/Library/2025-01-29-11-14-20.gh-issue-118761.gMZwE1.rst M Lib/threading.py diff --git a/Lib/threading.py b/Lib/threading.py index 78e591124278fc..d7cc3ddc44516b 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -3,7 +3,6 @@ import os as _os import sys as _sys import _thread -import warnings from time import monotonic as _time from _weakrefset import WeakSet @@ -133,6 +132,7 @@ def RLock(*args, **kwargs): """ if args or kwargs: + import warnings warnings.warn( 'Passing arguments to RLock is deprecated and will be removed in 3.15', DeprecationWarning, diff --git a/Misc/NEWS.d/next/Library/2025-01-29-11-14-20.gh-issue-118761.gMZwE1.rst b/Misc/NEWS.d/next/Library/2025-01-29-11-14-20.gh-issue-118761.gMZwE1.rst new file mode 100644 index 00000000000000..c2474795d8233e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-29-11-14-20.gh-issue-118761.gMZwE1.rst @@ -0,0 +1,2 @@ +Always lazy import ``warnings`` in :mod:`threading`. Patch by Taneli +Hukkinen.

1 0

[3.13] gh-118761: Improve import time of `subprocess` (GH-129427) (#129447)
by gpshead Jan. 29, 2025

Jan. 29, 2025

https://github.com/python/cpython/commit/f502c8f6a6db4be27c97a0e5466383d117… commit: f502c8f6a6db4be27c97a0e5466383d117859b7f branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: gpshead <greg(a)krypto.org> date: 2025-01-29T16:55:24Z summary: [3.13] gh-118761: Improve import time of `subprocess` (GH-129427) (#129447) gh-118761: Improve import time of `subprocess` (GH-129427) * subprocess: lazy import signal and locale to improve module import time (cherry picked from commit 49f24650e4541456872490ec2b59d6d186204891) Co-authored-by: Taneli Hukkinen <3275109+hukkin(a)users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst M Lib/subprocess.py diff --git a/Lib/subprocess.py b/Lib/subprocess.py index f614c903c8837c..98b98be01ac315 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -43,10 +43,8 @@ import builtins import errno import io -import locale import os import time -import signal import sys import threading import warnings @@ -144,6 +142,8 @@ def __init__(self, returncode, cmd, output=None, stderr=None): def __str__(self): if self.returncode and self.returncode < 0: + # Lazy import to improve module import time + import signal try: return "Command '%s' died with %r." % ( self.cmd, signal.Signals(-self.returncode)) @@ -381,6 +381,8 @@ def _text_encoding(): if sys.flags.utf8_mode: return "utf-8" else: + # Lazy import to improve module import time + import locale return locale.getencoding() @@ -1665,6 +1667,9 @@ def send_signal(self, sig): # Don't signal a process that we know has already died. if self.returncode is not None: return + + # Lazy import to improve module import time + import signal if sig == signal.SIGTERM: self.terminate() elif sig == signal.CTRL_C_EVENT: @@ -1766,6 +1771,9 @@ def _posix_spawn(self, args, executable, env, restore_signals, close_fds, """Execute program using os.posix_spawn().""" kwargs = {} if restore_signals: + # Lazy import to improve module import time + import signal + # See _Py_RestoreSignals() in Python/pylifecycle.c sigset = [] for signame in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'): @@ -2215,9 +2223,13 @@ def send_signal(self, sig): def terminate(self): """Terminate the process with SIGTERM """ + # Lazy import to improve module import time + import signal self.send_signal(signal.SIGTERM) def kill(self): """Kill the process with SIGKILL """ + # Lazy import to improve module import time + import signal self.send_signal(signal.SIGKILL) diff --git a/Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst b/Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst new file mode 100644 index 00000000000000..0762cbe5d63949 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst @@ -0,0 +1,2 @@ +Improve import time of :mod:`subprocess` by lazy importing ``locale`` and +``signal``. Patch by Taneli Hukkinen.

1 0

[3.12] gh-118761: Improve import time of `subprocess` (GH-129427) (#129448)
by gpshead Jan. 29, 2025

Jan. 29, 2025

https://github.com/python/cpython/commit/f65aa0d1bf7b636ab8f9d226429205854b… commit: f65aa0d1bf7b636ab8f9d226429205854b24cd7a branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: gpshead <greg(a)krypto.org> date: 2025-01-29T16:48:48Z summary: [3.12] gh-118761: Improve import time of `subprocess` (GH-129427) (#129448) gh-118761: Improve import time of `subprocess` (GH-129427) * subprocess: lazy import signal and locale to improve module import time (cherry picked from commit 49f24650e4541456872490ec2b59d6d186204891) Co-authored-by: Taneli Hukkinen <3275109+hukkin(a)users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst M Lib/subprocess.py diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 881a9ce800ae99..3ec39ca3e61e7a 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -43,10 +43,8 @@ import builtins import errno import io -import locale import os import time -import signal import sys import threading import warnings @@ -138,6 +136,8 @@ def __init__(self, returncode, cmd, output=None, stderr=None): def __str__(self): if self.returncode and self.returncode < 0: + # Lazy import to improve module import time + import signal try: return "Command '%s' died with %r." % ( self.cmd, signal.Signals(-self.returncode)) @@ -375,6 +375,8 @@ def _text_encoding(): if sys.flags.utf8_mode: return "utf-8" else: + # Lazy import to improve module import time + import locale return locale.getencoding() @@ -1655,6 +1657,9 @@ def send_signal(self, sig): # Don't signal a process that we know has already died. if self.returncode is not None: return + + # Lazy import to improve module import time + import signal if sig == signal.SIGTERM: self.terminate() elif sig == signal.CTRL_C_EVENT: @@ -1759,6 +1764,9 @@ def _posix_spawn(self, args, executable, env, restore_signals, kwargs = {} if restore_signals: + # Lazy import to improve module import time + import signal + # See _Py_RestoreSignals() in Python/pylifecycle.c sigset = [] for signame in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'): @@ -2208,9 +2216,13 @@ def send_signal(self, sig): def terminate(self): """Terminate the process with SIGTERM """ + # Lazy import to improve module import time + import signal self.send_signal(signal.SIGTERM) def kill(self): """Kill the process with SIGKILL """ + # Lazy import to improve module import time + import signal self.send_signal(signal.SIGKILL) diff --git a/Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst b/Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst new file mode 100644 index 00000000000000..0762cbe5d63949 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-29-10-53-32.gh-issue-118761.i8wjpV.rst @@ -0,0 +1,2 @@ +Improve import time of :mod:`subprocess` by lazy importing ``locale`` and +``signal``. Patch by Taneli Hukkinen.

1 0