Mailman 3 March 2025 - Python-checkins

gh-107369: Optimise ``textwrap.indent()`` (#131923)
by AA-Turner April 1, 2025

April 1, 2025

https://github.com/python/cpython/commit/51e0f2b06c042b8742145ce094323a34b6… commit: 51e0f2b06c042b8742145ce094323a34b61f02aa branch: main author: Adam Turner <9087854+AA-Turner(a)users.noreply.github.com> committer: AA-Turner <9087854+AA-Turner(a)users.noreply.github.com> date: 2025-03-31T18:31:39+01:00 summary: gh-107369: Optimise ``textwrap.indent()`` (#131923) Co-authored-by: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2025-03-31-02-06-57.gh-issue-107369.8M-HVz.rst M Lib/textwrap.py diff --git a/Lib/textwrap.py b/Lib/textwrap.py index bb6a1186316275..bac98c99e41df8 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -451,19 +451,20 @@ def indent(text, prefix, predicate=None): it will default to adding 'prefix' to all non-empty lines that do not consist solely of whitespace characters. """ - if predicate is None: - # str.splitlines(True) doesn't produce empty string. - # ''.splitlines(True) => [] - # 'foo\n'.splitlines(True) => ['foo\n'] - # So we can use just `not s.isspace()` here. - predicate = lambda s: not s.isspace() - prefixed_lines = [] - for line in text.splitlines(True): - if predicate(line): - prefixed_lines.append(prefix) - prefixed_lines.append(line) - + if predicate is None: + # str.splitlines(keepends=True) doesn't produce the empty string, + # so we need to use `str.isspace()` rather than a truth test. + # Inlining the predicate leads to a ~30% performance improvement. + for line in text.splitlines(True): + if not line.isspace(): + prefixed_lines.append(prefix) + prefixed_lines.append(line) + else: + for line in text.splitlines(True): + if predicate(line): + prefixed_lines.append(prefix) + prefixed_lines.append(line) return ''.join(prefixed_lines) diff --git a/Misc/NEWS.d/next/Library/2025-03-31-02-06-57.gh-issue-107369.8M-HVz.rst b/Misc/NEWS.d/next/Library/2025-03-31-02-06-57.gh-issue-107369.8M-HVz.rst new file mode 100644 index 00000000000000..9b1bd9e7502bf0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-31-02-06-57.gh-issue-107369.8M-HVz.rst @@ -0,0 +1,2 @@ +Improved performance of :func:`textwrap.dedent` by an average of ~1.3x. +Patch by Adam Turner.

1 0

gh-131936: Strengthen check in `_suggestions._generate_suggestions` (#131945)
by vstinner April 1, 2025

April 1, 2025

https://github.com/python/cpython/commit/511d3440a0bbb19731f4d96dde65dffbf8… commit: 511d3440a0bbb19731f4d96dde65dffbf85cdda5 branch: main author: Peter Bierma <zintensitydev(a)gmail.com> committer: vstinner <vstinner(a)python.org> date: 2025-03-31T19:30:29+02:00 summary: gh-131936: Strengthen check in `_suggestions._generate_suggestions` (#131945) files: M Lib/test/test_traceback.py M Modules/_suggestions.c diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index ac492782cb1a5c..6ae8cf5c9a5130 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -4619,7 +4619,31 @@ def test_levenshtein_distance_short_circuit(self): @cpython_only def test_suggestions_extension(self): # Check that the C extension is available - import _suggestions # noqa: F401 + import _suggestions + + self.assertEqual( + _suggestions._generate_suggestions( + ["hello", "world"], + "hell" + ), + "hello" + ) + self.assertEqual( + _suggestions._generate_suggestions( + ["hovercraft"], + "eels" + ), + None + ) + + # gh-131936: _generate_suggestions() doesn't accept list subclasses + class MyList(list): + pass + + with self.assertRaises(TypeError): + _suggestions._generate_suggestions(MyList(), "") + + class TestColorizedTraceback(unittest.TestCase): diff --git a/Modules/_suggestions.c b/Modules/_suggestions.c index 80c7179c4c251c..b8bc6db2477281 100644 --- a/Modules/_suggestions.c +++ b/Modules/_suggestions.c @@ -21,7 +21,7 @@ _suggestions__generate_suggestions_impl(PyObject *module, /*[clinic end generated code: output=79be7b653ae5e7ca input=ba2a8dddc654e33a]*/ { // Check if dir is a list - if (!PyList_Check(candidates)) { + if (!PyList_CheckExact(candidates)) { PyErr_SetString(PyExc_TypeError, "candidates must be a list"); return NULL; }

1 0

gh-105499: typing: Remove an unused function (#131946)
by JelleZijlstra April 1, 2025

April 1, 2025

https://github.com/python/cpython/commit/0f511d8b44dd9993474402411af8c83f49… commit: 0f511d8b44dd9993474402411af8c83f4964bc95 branch: main author: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2025-03-31T10:27:03-07:00 summary: gh-105499: typing: Remove an unused function (#131946) Leftover from #105511 I believe. GitHub code search found no usages other than copies of typing.py and lists of stdlib functions. files: M Lib/typing.py diff --git a/Lib/typing.py b/Lib/typing.py index e36da7e9f48b71..f7528258f43453 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -29,12 +29,7 @@ import operator import sys import types -from types import ( - WrapperDescriptorType, - MethodWrapperType, - MethodDescriptorType, - GenericAlias, -) +from types import GenericAlias import warnings from _typing import ( @@ -354,26 +349,11 @@ def _deduplicate(params, *, unhashable_fallback=False): if not unhashable_fallback: raise # Happens for cases like `Annotated[dict, {'x': IntValidator()}]` - return _deduplicate_unhashable(params) - -def _deduplicate_unhashable(unhashable_params): - new_unhashable = [] - for t in unhashable_params: - if t not in new_unhashable: - new_unhashable.append(t) - return new_unhashable - -def _compare_args_orderless(first_args, second_args): - first_unhashable = _deduplicate_unhashable(first_args) - second_unhashable = _deduplicate_unhashable(second_args) - t = list(second_unhashable) - try: - for elem in first_unhashable: - t.remove(elem) - except ValueError: - return False - return not t - + new_unhashable = [] + for t in params: + if t not in new_unhashable: + new_unhashable.append(t) + return new_unhashable def _flatten_literal_params(parameters): """Internal helper for Literal creation: flatten Literals among parameters."""

1 0

GH-131296: Fix incompatible pointer type warning in _winapi.c (GH-131600)
by zooba March 31, 2025

March 31, 2025

https://github.com/python/cpython/commit/9127b4602e4e0e110eab7f2f6a8ac54fe6… commit: 9127b4602e4e0e110eab7f2f6a8ac54fe66b0a72 branch: main author: Chris Eibl <138194463+chris-eibl(a)users.noreply.github.com> committer: zooba <steve.dower(a)microsoft.com> date: 2025-03-31T17:33:55+01:00 summary: GH-131296: Fix incompatible pointer type warning in _winapi.c (GH-131600) files: M Modules/_winapi.c diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 9aff28173a5042..02817e09b936f3 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -609,7 +609,7 @@ _winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path, /* overallocate by a few array elements */ LUID_AND_ATTRIBUTES privs[4]; } tp, previousTp; - int previousTpSize = 0; + DWORD previousTpSize = 0; /* Reparse data buffer */ const USHORT prefix_len = 4;

1 0

GH-131296: Fix clang-cl warnings on Windows in sqlite3 (GH-131906)
by zooba March 31, 2025

March 31, 2025

https://github.com/python/cpython/commit/4940d96135a92b90bbc1dcc399f665817f… commit: 4940d96135a92b90bbc1dcc399f665817f956640 branch: main author: Chris Eibl <138194463+chris-eibl(a)users.noreply.github.com> committer: zooba <steve.dower(a)microsoft.com> date: 2025-03-31T16:15:31+01:00 summary: GH-131296: Fix clang-cl warnings on Windows in sqlite3 (GH-131906) files: M PCbuild/sqlite3.vcxproj diff --git a/PCbuild/sqlite3.vcxproj b/PCbuild/sqlite3.vcxproj index 6bcc4e913c8e77..f12ec348b37e29 100644 --- a/PCbuild/sqlite3.vcxproj +++ b/PCbuild/sqlite3.vcxproj @@ -100,6 +100,7 @@ <AdditionalIncludeDirectories>$(sqlite3Dir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <PreprocessorDefinitions>SQLITE_ENABLE_MATH_FUNCTIONS;SQLITE_ENABLE_FTS4;SQLITE_ENABLE_FTS5;SQLITE_ENABLE_RTREE;SQLITE_OMIT_AUTOINIT;SQLITE_API=__declspec(dllexport);%(PreprocessorDefinitions)</PreprocessorDefinitions> <WarningLevel>Level1</WarningLevel> + <AdditionalOptions Condition="$(PlatformToolset) == 'ClangCL'">%(AdditionalOptions) -Wno-unused</AdditionalOptions> </ClCompile> <ResourceCompile> <PreprocessorDefinitions>SQLITE_VERSION=$(SqliteVersion);SQLITE_MAJOR_VERSION=$(SqliteMajorVersion);SQLITE_MINOR_VERSION=$(SqliteMinorVersion);SQLITE_MICRO_VERSION=$(SqliteMicroVersion);SQLITE_PATCH_VERSION=$(SqlitePatchVersion);%(PreprocessorDefinitions)</PreprocessorDefinitions>

1 0

GH-131296: Fix parenthesis warnings on Windows-specific code (GH-131905)
by zooba March 31, 2025

March 31, 2025

https://github.com/python/cpython/commit/7cbcb6efd9e3b2727bc31748bced511def… commit: 7cbcb6efd9e3b2727bc31748bced511def3727c2 branch: main author: Chris Eibl <138194463+chris-eibl(a)users.noreply.github.com> committer: zooba <steve.dower(a)microsoft.com> date: 2025-03-31T15:54:30+01:00 summary: GH-131296: Fix parenthesis warnings on Windows-specific code (GH-131905) files: M Modules/_ssl.c M Modules/posixmodule.c M PC/launcher2.c diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 6615d6d0b99802..e6bfbe46add9e9 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -6111,7 +6111,7 @@ _ssl_enum_certificates_impl(PyObject *module, const char *store_name) return PyErr_SetFromWindowsErr(GetLastError()); } - while (pCertCtx = CertEnumCertificatesInStore(hCollectionStore, pCertCtx)) { + while ((pCertCtx = CertEnumCertificatesInStore(hCollectionStore, pCertCtx))) { cert = PyBytes_FromStringAndSize((const char*)pCertCtx->pbCertEncoded, pCertCtx->cbCertEncoded); if (!cert) { @@ -6210,7 +6210,7 @@ _ssl_enum_crls_impl(PyObject *module, const char *store_name) return PyErr_SetFromWindowsErr(GetLastError()); } - while (pCrlCtx = CertEnumCRLsInStore(hCollectionStore, pCrlCtx)) { + while ((pCrlCtx = CertEnumCRLsInStore(hCollectionStore, pCrlCtx))) { crl = PyBytes_FromStringAndSize((const char*)pCrlCtx->pbCrlEncoded, pCrlCtx->cbCrlEncoded); if (!crl) { diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f3ce1fb632226e..b7300def8dc75f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5372,7 +5372,7 @@ _testFileExistsByName(LPCWSTR path, BOOL followLinks) sizeof(info))) { if (!(info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) || - !followLinks && IsReparseTagNameSurrogate(info.ReparseTag)) + (!followLinks && IsReparseTagNameSurrogate(info.ReparseTag))) { return TRUE; } diff --git a/PC/launcher2.c b/PC/launcher2.c index 72121724726ccb..02ab417bb27ece 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -2760,7 +2760,7 @@ process(int argc, wchar_t ** argv) // We searched earlier, so if we didn't find anything, now we react exitCode = searchExitCode; // If none found, and if permitted, install it - if (exitCode == RC_NO_PYTHON && isEnvVarSet(L"PYLAUNCHER_ALLOW_INSTALL") || + if (((exitCode == RC_NO_PYTHON) && isEnvVarSet(L"PYLAUNCHER_ALLOW_INSTALL")) || isEnvVarSet(L"PYLAUNCHER_ALWAYS_INSTALL")) { exitCode = installEnvironment(&search); if (!exitCode) {

1 0

GH-131296: Fix clang-cl warning on Windows in liblzma (GH-131897)
by zooba March 31, 2025

March 31, 2025

https://github.com/python/cpython/commit/cf839c3373d8a492e87ae1e4337aa0c21f… commit: cf839c3373d8a492e87ae1e4337aa0c21ffaffcd branch: main author: Chris Eibl <138194463+chris-eibl(a)users.noreply.github.com> committer: zooba <steve.dower(a)microsoft.com> date: 2025-03-31T15:50:44+01:00 summary: GH-131296: Fix clang-cl warning on Windows in liblzma (GH-131897) Also removes warnings suppression for MSVC that no longer need to be suppressed. files: M PCbuild/liblzma.vcxproj diff --git a/PCbuild/liblzma.vcxproj b/PCbuild/liblzma.vcxproj index 4dd42ab98a9759..97938692328495 100644 --- a/PCbuild/liblzma.vcxproj +++ b/PCbuild/liblzma.vcxproj @@ -93,7 +93,8 @@ <ClCompile> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> <AdditionalIncludeDirectories>$(lzmaDir)windows/vs2019;$(lzmaDir)src/liblzma/common;$(lzmaDir)src/common;$(lzmaDir)src/liblzma/api;$(lzmaDir)src/liblzma/check;$(lzmaDir)src/liblzma/delta;$(lzmaDir)src/liblzma/lz;$(lzmaDir)src/liblzma/lzma;$(lzmaDir)src/liblzma/rangecoder;$(lzmaDir)src/liblzma/simple;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <DisableSpecificWarnings>4028;4113;4133;4244;4267;4996;%(DisableSpecificWarnings)</DisableSpecificWarnings> + <DisableSpecificWarnings>4244;4267;4996;%(DisableSpecificWarnings)</DisableSpecificWarnings> + <AdditionalOptions Condition="$(PlatformToolset) == 'ClangCL'">%(AdditionalOptions) -Wno-deprecated-declarations</AdditionalOptions> </ClCompile> </ItemDefinitionGroup> <ItemGroup>

1 0

Fix PEP number in `ast_opt.c` for new `finally` check (#131928)
by sobolevn March 31, 2025

March 31, 2025

https://github.com/python/cpython/commit/1e3ec335e01453639f5f475a5a8f917a1c… commit: 1e3ec335e01453639f5f475a5a8f917a1cc83b27 branch: main author: sobolevn <mail(a)sobolevn.me> committer: sobolevn <mail(a)sobolevn.me> date: 2025-03-31T16:29:23+03:00 summary: Fix PEP number in `ast_opt.c` for new `finally` check (#131928) files: M Python/ast_opt.c diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 99a2418da46e35..1b44fa25b9f150 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -21,7 +21,7 @@ typedef struct { int ff_features; int syntax_check_only; - _Py_c_array_t cf_finally; /* context for PEP 678 check */ + _Py_c_array_t cf_finally; /* context for PEP 765 check */ int cf_finally_used; } _PyASTOptimizeState;

1 0

gh-131525: Remove `_HashedSeq` wrapper from `lru_cache` (gh-131922)
by rhettinger March 31, 2025

March 31, 2025

https://github.com/python/cpython/commit/0147be09d585e6b5e013ea2d1b24c77500… commit: 0147be09d585e6b5e013ea2d1b24c77500d9a083 branch: main author: Lukas Geiger <lukas.geiger94(a)gmail.com> committer: rhettinger <rhettinger(a)users.noreply.github.com> date: 2025-03-31T08:23:41-05:00 summary: gh-131525: Remove `_HashedSeq` wrapper from `lru_cache` (gh-131922) files: M Lib/functools.py diff --git a/Lib/functools.py b/Lib/functools.py index e0e45bc336c1ef..714070c6ac9460 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -516,22 +516,6 @@ def _unwrap_partialmethod(func): _CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) -class _HashedSeq(list): - """ This class guarantees that hash() will be called no more than once - per element. This is important because the lru_cache() will hash - the key multiple times on a cache miss. - - """ - - __slots__ = 'hashvalue' - - def __init__(self, tup, hash=hash): - self[:] = tup - self.hashvalue = hash(tup) - - def __hash__(self): - return self.hashvalue - def _make_key(args, kwds, typed, kwd_mark = (object(),), fasttypes = {int, str}, @@ -561,7 +545,7 @@ def _make_key(args, kwds, typed, key += tuple(type(v) for v in kwds.values()) elif len(key) == 1 and type(key[0]) in fasttypes: return key[0] - return _HashedSeq(key) + return key def lru_cache(maxsize=128, typed=False): """Least-recently-used cache decorator.

1 0

GH-131498: Another refactoring of the code generator (GH-131827)
by markshannon March 31, 2025

March 31, 2025

https://github.com/python/cpython/commit/c535a132e40a516a7cca219b2659e85bcc… commit: c535a132e40a516a7cca219b2659e85bccaa0529 branch: main author: Mark Shannon <mark(a)hotpy.org> committer: markshannon <mark(a)hotpy.org> date: 2025-03-31T13:52:48+01:00 summary: GH-131498: Another refactoring of the code generator (GH-131827) * Rename 'defined' attribute to 'in_local' to more accurately reflect how it is used * Make death of variables explicit even for array variables. * Convert in_memory from boolean to stack offset * Don't apply liveness analyis to optimizer generated code * Add 'out' parameter to stack.pop files: M Lib/test/test_generated_cases.py M Python/bytecodes.c M Python/generated_cases.c.h M Tools/cases_generator/cwriter.py M Tools/cases_generator/generators_common.py M Tools/cases_generator/opcode_metadata_generator.py M Tools/cases_generator/optimizer_generator.py M Tools/cases_generator/stack.py M Tools/cases_generator/tier1_generator.py M Tools/cases_generator/tier2_generator.py M Tools/cases_generator/uop_metadata_generator.py diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 856da584ca7e9e..3f396e4da3ef7d 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -68,13 +68,15 @@ def test_effect_sizes(self): StackItem("b", None, "oparg*4"), StackItem("c", None, "1"), ] - stack.pop(z) - stack.pop(y) - stack.pop(x) + null = CWriter.null() + stack.pop(z, null) + stack.pop(y, null) + stack.pop(x, null) for out in outputs: stack.push(Local.undefined(out)) self.assertEqual(stack.base_offset.to_c(), "-1 - oparg - oparg*2") - self.assertEqual(stack.top_offset.to_c(), "1 - oparg - oparg*2 + oparg*4") + self.assertEqual(stack.physical_sp.to_c(), "0") + self.assertEqual(stack.logical_sp.to_c(), "1 - oparg - oparg*2 + oparg*4") class TestGeneratedCases(unittest.TestCase): diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2d18f658702fe7..43424447bb068d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1484,7 +1484,7 @@ dummy_func( (void)counter; } - op(_UNPACK_SEQUENCE, (seq -- output[oparg], top[0])) { + op(_UNPACK_SEQUENCE, (seq -- unused[oparg], top[0])) { PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); Py_DECREF(seq_o); @@ -1533,7 +1533,7 @@ dummy_func( DECREF_INPUTS(); } - inst(UNPACK_EX, (seq -- left[oparg & 0xFF], unused, right[oparg >> 8], top[0])) { + inst(UNPACK_EX, (seq -- unused[oparg & 0xFF], unused, unused[oparg >> 8], top[0])) { PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq_o); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 416035ae29d71c..50db2c867c89e4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2608,9 +2608,9 @@ _PyStackRef *callable; _PyStackRef *self_or_null; _PyStackRef *args; - _PyStackRef kwnames; _PyStackRef kwnames_in; _PyStackRef kwnames_out; + _PyStackRef kwnames; _PyStackRef res; // _SPECIALIZE_CALL_KW { @@ -2791,9 +2791,9 @@ static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef *null; - _PyStackRef kwnames; _PyStackRef *self_or_null; _PyStackRef *args; + _PyStackRef kwnames; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 @@ -2924,9 +2924,9 @@ opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef *callable; - _PyStackRef kwnames; _PyStackRef *self_or_null; _PyStackRef *args; + _PyStackRef kwnames; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -3057,8 +3057,8 @@ static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef *callable; _PyStackRef *self_or_null; - _PyStackRef kwnames; _PyStackRef *args; + _PyStackRef kwnames; _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 @@ -4682,8 +4682,8 @@ PREDICTED_CONTAINS_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - _PyStackRef left; _PyStackRef right; + _PyStackRef left; _PyStackRef b; // _SPECIALIZE_CONTAINS_OP { diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index 07a785e4312fa6..2bda9fe73df83e 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -1,7 +1,7 @@ import contextlib from lexer import Token from typing import TextIO, Iterator - +from io import StringIO class CWriter: "A writer that understands tokens and how to format C code" @@ -18,6 +18,10 @@ def __init__(self, out: TextIO, indent: int, line_directives: bool): self.pending_spill = False self.pending_reload = False + @staticmethod + def null() -> "CWriter": + return CWriter(StringIO(), 0, False) + def set_position(self, tkn: Token) -> None: if self.last_token is not None: if self.last_token.end_line < tkn.line: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index da22cb3181470c..b192738849d18d 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -205,9 +205,9 @@ def error_if( next(tkn_iter) # Semi colon storage.clear_inputs("at ERROR_IF") - c_offset = storage.stack.peek_offset() + c_offset = storage.stack.sp_offset() try: - offset = -int(c_offset) + offset = int(c_offset) except ValueError: offset = -1 self.out.emit(self.goto_error(offset, label, storage)) diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 9cfa30bd3b2afe..02283a0b647e84 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -100,7 +100,7 @@ def generate_stack_effect_functions(analysis: Analysis, out: CWriter) -> None: def add(inst: Instruction | PseudoInstruction) -> None: stack = get_stack_effect(inst) popped = (-stack.base_offset).to_c() - pushed = (stack.top_offset - stack.base_offset).to_c() + pushed = (stack.logical_sp - stack.base_offset).to_c() popped_data.append((inst.name, popped)) pushed_data.append((inst.name, pushed)) diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 3d89448f1fd0f8..182933e9fc4af4 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -72,14 +72,15 @@ def decref_inputs( def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: + null = CWriter.null() for var in reversed(uop.stack.inputs): - stack.pop(var) - top_offset = stack.top_offset.copy() + stack.pop(var, null) + offset = stack.base_offset - stack.physical_sp for var in uop.stack.outputs: if var.is_array() and not var.peek and not var.name == "unused": - c_offset = top_offset.to_c() + c_offset = offset.to_c() out.emit(f"{var.name} = &stack_pointer[{c_offset}];\n") - top_offset.push(var) + offset = offset.push(var) for var in uop.stack.outputs: local = Local.undefined(var) stack.push(local) @@ -123,9 +124,7 @@ def write_uop( try: out.start_line() if override: - code_list, storage = Storage.for_uop(stack, prototype, check_liveness=False) - for code in code_list: - out.emit(code) + storage = Storage.for_uop(stack, prototype, out, check_liveness=False) if debug: args = [] for input in prototype.stack.inputs: diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 94a5d395064191..76dfc48a6b594e 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -30,79 +30,96 @@ def var_size(var: StackItem) -> str: @dataclass -class StackOffset: - "The stack offset of the virtual base of the stack from the physical stack pointer" - - popped: list[str] - pushed: list[str] +class PointerOffset: + """The offset of a pointer from the reference pointer + The 'reference pointer' is the address of the physical stack pointer + at the start of the code section, as if each code section started with + `const PyStackRef *reference = stack_pointer` + """ + numeric: int + positive: tuple[str, ...] + negative: tuple[str, ...] @staticmethod - def empty() -> "StackOffset": - return StackOffset([], []) + def zero() -> "PointerOffset": + return PointerOffset(0, (), ()) - def copy(self) -> "StackOffset": - return StackOffset(self.popped[:], self.pushed[:]) + def pop(self, item: StackItem) -> "PointerOffset": + return self - PointerOffset.from_item(item) - def pop(self, item: StackItem) -> None: - self.popped.append(var_size(item)) + def push(self, item: StackItem) -> "PointerOffset": + return self + PointerOffset.from_item(item) - def push(self, item: StackItem) -> None: - self.pushed.append(var_size(item)) + @staticmethod + def from_item(item: StackItem) -> "PointerOffset": + if not item.size: + return PointerOffset(1, (), ()) + txt = item.size.strip() + n: tuple[str, ...] = () + p: tuple[str, ...] = () + try: + i = int(txt) + except ValueError: + i = 0 + if txt[0] == "+": + txt = txt[1:] + if txt[0] == "-": + n = (txt[1:],) + else: + p = (txt,) + return PointerOffset(i, p, n) - def __sub__(self, other: "StackOffset") -> "StackOffset": - return StackOffset(self.popped + other.pushed, self.pushed + other.popped) + @staticmethod + def create(numeric: int, positive: tuple[str, ...], negative: tuple[str, ...]) -> "PointerOffset": + positive, negative = PointerOffset._simplify(positive, negative) + return PointerOffset(numeric, positive, negative) + + def __sub__(self, other: "PointerOffset") -> "PointerOffset": + return PointerOffset.create( + self.numeric - other.numeric, + self.positive + other.negative, + self.negative + other.positive + ) - def __neg__(self) -> "StackOffset": - return StackOffset(self.pushed, self.popped) + def __add__(self, other: "PointerOffset") -> "PointerOffset": + return PointerOffset.create( + self.numeric + other.numeric, + self.positive + other.positive, + self.negative + other.negative + ) - def simplify(self) -> None: - "Remove matching values from both the popped and pushed list" - if not self.popped: - self.pushed.sort() - return - if not self.pushed: - self.popped.sort() - return - # Sort the list so the lexically largest element is last. - popped = sorted(self.popped) - pushed = sorted(self.pushed) - self.popped = [] - self.pushed = [] - while popped and pushed: - pop = popped.pop() - push = pushed.pop() - if pop == push: - pass - elif pop > push: - # if pop > push, there can be no element in pushed matching pop. - self.popped.append(pop) - pushed.append(push) - else: - self.pushed.append(push) - popped.append(pop) - self.popped.extend(popped) - self.pushed.extend(pushed) - self.pushed.sort() - self.popped.sort() + def __neg__(self) -> "PointerOffset": + return PointerOffset(-self.numeric, self.negative, self.positive) + + @staticmethod + def _simplify(positive: tuple[str, ...], negative: tuple[str, ...]) -> tuple[tuple[str, ...], tuple[str, ...]]: + p_orig: list[str] = sorted(positive) + n_orig: list[str] = sorted(negative) + p_uniq: list[str] = [] + n_uniq: list[str] = [] + while p_orig and n_orig: + p_item = p_orig.pop() + n_item = n_orig.pop() + if p_item > n_item: + # if p_item > n_item, there can be no element in n matching p_item. + p_uniq.append(p_item) + n_orig.append(n_item) + elif p_item < n_item: + n_uniq.append(n_item) + p_orig.append(p_item) + # Otherwise they are the same and cancel each other out + return tuple(p_orig + p_uniq), tuple(n_orig + n_uniq) def to_c(self) -> str: - self.simplify() - int_offset = 0 symbol_offset = "" - for item in self.popped: - try: - int_offset -= int(item) - except ValueError: - symbol_offset += f" - {maybe_parenthesize(item)}" - for item in self.pushed: - try: - int_offset += int(item) - except ValueError: - symbol_offset += f" + {maybe_parenthesize(item)}" - if symbol_offset and not int_offset: + for item in self.negative: + symbol_offset += f" - {maybe_parenthesize(item)}" + for item in self.positive: + symbol_offset += f" + {maybe_parenthesize(item)}" + if symbol_offset and self.numeric == 0: res = symbol_offset else: - res = f"{int_offset}{symbol_offset}" + res = f"{self.numeric}{symbol_offset}" if res.startswith(" + "): res = res[3:] if res.startswith(" - "): @@ -110,53 +127,36 @@ def to_c(self) -> str: return res def as_int(self) -> int | None: - self.simplify() - int_offset = 0 - for item in self.popped: - try: - int_offset -= int(item) - except ValueError: - return None - for item in self.pushed: - try: - int_offset += int(item) - except ValueError: - return None - return int_offset - - def clear(self) -> None: - self.popped = [] - self.pushed = [] + if self.positive or self.negative: + return None + return self.numeric def __bool__(self) -> bool: - self.simplify() - return bool(self.popped) or bool(self.pushed) + return self.numeric != 0 or bool(self.positive) or bool(self.negative) - def __eq__(self, other: object) -> bool: - if not isinstance(other, StackOffset): - return NotImplemented - return self.to_c() == other.to_c() + def __str__(self) -> str: + return self.to_c() + def __repr__(self) -> str: + return f"PointerOffset({self.to_c()})" @dataclass class Local: item: StackItem - memory_offset: StackOffset | None + memory_offset: PointerOffset | None in_local: bool def __repr__(self) -> str: return f"Local('{self.item.name}', mem={self.memory_offset}, local={self.in_local}, array={self.is_array()})" - #def compact_str(self) -> str: - #mtag = "M" if self.memory_offset else "" - #dtag = "D" if self.in_local else "" - #atag = "A" if self.is_array() else "" - #return f"'{self.item.name}'{mtag}{dtag}{atag}" - - compact_str = __repr__ + def compact_str(self) -> str: + mtag = "M" if self.memory_offset else "" + dtag = "D" if self.in_local else "" + atag = "A" if self.is_array() else "" + return f"'{self.item.name}'{mtag}{dtag}{atag}" @staticmethod - def unused(defn: StackItem, offset: StackOffset) -> "Local": + def unused(defn: StackItem, offset: PointerOffset | None) -> "Local": return Local(defn, offset, False) @staticmethod @@ -164,7 +164,7 @@ def undefined(defn: StackItem) -> "Local": return Local(defn, None, False) @staticmethod - def from_memory(defn: StackItem, offset: StackOffset) -> "Local": + def from_memory(defn: StackItem, offset: PointerOffset) -> "Local": return Local(defn, offset, True) def kill(self) -> None: @@ -213,15 +213,15 @@ def array_or_scalar(var: StackItem | Local) -> str: class Stack: def __init__(self, extract_bits: bool=True, cast_type: str = "uintptr_t") -> None: - self.top_offset = StackOffset.empty() - self.base_offset = StackOffset.empty() + self.base_offset = PointerOffset.zero() + self.physical_sp = PointerOffset.zero() + self.logical_sp = PointerOffset.zero() self.variables: list[Local] = [] - self.defined: set[str] = set() self.extract_bits = extract_bits self.cast_type = cast_type def drop(self, var: StackItem, check_liveness: bool) -> None: - self.top_offset.pop(var) + self.logical_sp = self.logical_sp.pop(var) if self.variables: popped = self.variables.pop() if popped.is_dead() or not var.used: @@ -229,8 +229,8 @@ def drop(self, var: StackItem, check_liveness: bool) -> None: if check_liveness: raise StackError(f"Dropping live value '{var.name}'") - def pop(self, var: StackItem) -> tuple[str, Local]: - self.top_offset.pop(var) + def pop(self, var: StackItem, out: CWriter) -> Local: + self.logical_sp = self.logical_sp.pop(var) indirect = "&" if var.is_array() else "" if self.variables: popped = self.variables.pop() @@ -244,125 +244,122 @@ def pop(self, var: StackItem) -> tuple[str, Local]: f"Size mismatch when popping '{popped.name}' from stack to assign to '{var.name}'. " f"Expected {var_size(var)} got {var_size(popped.item)}" ) - if var.name in UNUSED: - if popped.name not in UNUSED and popped.name in self.defined: - raise StackError( - f"Value is declared unused, but is already cached by prior operation as '{popped.name}'" - ) - return "", popped if not var.used: - return "", popped - self.defined.add(var.name) + return popped if popped.name != var.name: rename = f"{var.name} = {popped.name};\n" popped.item = var else: rename = "" if not popped.in_local: - assert popped.memory_offset is not None + if popped.memory_offset is None: + popped.memory_offset = self.logical_sp + assert popped.memory_offset == self.logical_sp, (popped, self.as_comment()) + offset = popped.memory_offset.to_c() if var.is_array(): - defn = f"{var.name} = &stack_pointer[{self.top_offset.to_c()}];\n" + defn = f"{var.name} = &stack_pointer[{offset}];\n" else: - defn = f"{var.name} = stack_pointer[{self.top_offset.to_c()}];\n" + defn = f"{var.name} = stack_pointer[{offset}];\n" popped.in_local = True else: defn = rename - return defn, popped + out.emit(defn) + return popped - self.base_offset.pop(var) + self.base_offset = self.logical_sp if var.name in UNUSED or not var.used: - return "", Local.unused(var, self.base_offset) - self.defined.add(var.name) + return Local.unused(var, self.base_offset) cast = f"({var.type})" if (not indirect and var.type) else "" bits = ".bits" if cast and self.extract_bits else "" - assign = f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}]{bits};" - assign = f"{assign}\n" - return assign, Local.from_memory(var, self.base_offset.copy()) + offset = (self.base_offset - self.physical_sp).to_c() + assign = f"{var.name} = {cast}{indirect}stack_pointer[{offset}]{bits};\n" + out.emit(assign) + return Local.from_memory(var, self.base_offset) def push(self, var: Local) -> None: assert(var not in self.variables) self.variables.append(var) - self.top_offset.push(var.item) - if var.item.used: - self.defined.add(var.name) + self.logical_sp = self.logical_sp.push(var.item) @staticmethod def _do_emit( out: CWriter, var: StackItem, - base_offset: StackOffset, + stack_offset: PointerOffset, cast_type: str, extract_bits: bool, ) -> None: cast = f"({cast_type})" if var.type else "" bits = ".bits" if cast and extract_bits else "" - out.emit(f"stack_pointer[{base_offset.to_c()}]{bits} = {cast}{var.name};\n") + out.emit(f"stack_pointer[{stack_offset.to_c()}]{bits} = {cast}{var.name};\n") - def _adjust_stack_pointer(self, out: CWriter, number: str) -> None: - if number != "0": + def _save_physical_sp(self, out: CWriter) -> None: + if self.physical_sp != self.logical_sp: + diff = self.logical_sp - self.physical_sp out.start_line() - out.emit(f"stack_pointer += {number};\n") + out.emit(f"stack_pointer += {diff.to_c()};\n") out.emit("assert(WITHIN_STACK_BOUNDS());\n") + self.physical_sp = self.logical_sp def flush(self, out: CWriter) -> None: out.start_line() - var_offset = self.base_offset.copy() + var_offset = self.base_offset for var in self.variables: if ( var.in_local and not var.memory_offset and not var.is_array() ): - Stack._do_emit(out, var.item, var_offset, self.cast_type, self.extract_bits) - var.memory_offset = var_offset.copy() - var_offset.push(var.item) - number = self.top_offset.to_c() - self._adjust_stack_pointer(out, number) - self.base_offset -= self.top_offset - self.top_offset.clear() + var.memory_offset = var_offset + stack_offset = var_offset - self.physical_sp + Stack._do_emit(out, var.item, stack_offset, self.cast_type, self.extract_bits) + var_offset = var_offset.push(var.item) + self._save_physical_sp(out) out.start_line() def is_flushed(self) -> bool: - return not self.variables and not self.base_offset and not self.top_offset + for var in self.variables: + if not var.in_memory(): + return False + return self.physical_sp == self.logical_sp - def peek_offset(self) -> str: - return self.top_offset.to_c() + def sp_offset(self) -> str: + return (self.physical_sp - self.logical_sp).to_c() def as_comment(self) -> str: variables = ", ".join([v.compact_str() for v in self.variables]) return ( - f"/* Variables: {variables}. base: {self.base_offset.to_c()}. top: {self.top_offset.to_c()} */" + f"/* Variables: {variables}. base: {self.base_offset.to_c()}. sp: {self.physical_sp.to_c()}. logical_sp: {self.logical_sp.to_c()} */" ) def copy(self) -> "Stack": other = Stack(self.extract_bits, self.cast_type) - other.top_offset = self.top_offset.copy() - other.base_offset = self.base_offset.copy() + other.base_offset = self.base_offset + other.physical_sp = self.physical_sp + other.logical_sp = self.logical_sp other.variables = [var.copy() for var in self.variables] - other.defined = set(self.defined) return other def __eq__(self, other: object) -> bool: if not isinstance(other, Stack): return NotImplemented return ( - self.top_offset == other.top_offset + self.physical_sp == other.physical_sp + and self.logical_sp == other.logical_sp and self.base_offset == other.base_offset and self.variables == other.variables ) def align(self, other: "Stack", out: CWriter) -> None: - if len(self.variables) != len(other.variables): - raise StackError("Cannot align stacks: differing variables") - if self.top_offset == other.top_offset: + if self.logical_sp != other.logical_sp: + raise StackError("Cannot align stacks: differing logical top") + if self.physical_sp == other.physical_sp: return - diff = self.top_offset - other.top_offset - try: - self.top_offset -= diff - self.base_offset -= diff - self._adjust_stack_pointer(out, diff.to_c()) - except ValueError: - raise StackError("Cannot align stacks: cannot adjust stack pointer") + diff = other.physical_sp - self.physical_sp + out.start_line() + out.emit(f"stack_pointer += {diff.to_c()};\n") + out.emit("assert(WITHIN_STACK_BOUNDS());\n") + self.physical_sp = other.physical_sp def merge(self, other: "Stack", out: CWriter) -> None: if len(self.variables) != len(other.variables): @@ -394,15 +391,16 @@ def stacks(inst: Instruction | PseudoInstruction) -> Iterator[StackEffect]: def apply_stack_effect(stack: Stack, effect: StackEffect) -> None: locals: dict[str, Local] = {} + null = CWriter.null() for var in reversed(effect.inputs): - _, local = stack.pop(var) + local = stack.pop(var, null) if var.name != "unused": locals[local.name] = local for var in effect.outputs: if var.name in locals: local = locals[var.name] else: - local = Local.unused(var, stack.base_offset) + local = Local.unused(var, None) stack.push(local) @@ -521,13 +519,11 @@ def reload(self, out: CWriter) -> None: out.emit_reload() @staticmethod - def for_uop(stack: Stack, uop: Uop, check_liveness: bool = True) -> tuple[list[str], "Storage"]: - code_list: list[str] = [] + def for_uop(stack: Stack, uop: Uop, out: CWriter, check_liveness: bool = True) -> "Storage": inputs: list[Local] = [] peeks: list[Local] = [] for input in reversed(uop.stack.inputs): - code, local = stack.pop(input) - code_list.append(code) + local = stack.pop(input, out) if input.peek: peeks.append(local) else: @@ -536,18 +532,16 @@ def for_uop(stack: Stack, uop: Uop, check_liveness: bool = True) -> tuple[list[s peeks.reverse() for peek in peeks: stack.push(peek) - top_offset = stack.top_offset.copy() + offset = stack.logical_sp - stack.physical_sp for ouput in uop.stack.outputs: if ouput.is_array() and ouput.used and not ouput.peek: - c_offset = top_offset.to_c() - top_offset.push(ouput) - code_list.append(f"{ouput.name} = &stack_pointer[{c_offset}];\n") - else: - top_offset.push(ouput) + c_offset = offset.to_c() + out.emit(f"{ouput.name} = &stack_pointer[{c_offset}];\n") + offset = offset.push(ouput) for var in inputs: stack.push(var) outputs = [ Local.undefined(var) for var in uop.stack.outputs if not var.peek ] - return code_list, Storage(stack, inputs, outputs, check_liveness) + return Storage(stack, inputs, outputs, check_liveness) @staticmethod def copy_list(arg: list[Local]) -> list[Local]: @@ -712,7 +706,7 @@ def close_variable(var: Local, overwrite: str) -> None: self.stack.drop(self.inputs[0].item, False) output_in_place = self.outputs and output is self.outputs[0] and lowest.memory_offset is not None if output_in_place: - output.memory_offset = lowest.memory_offset.copy() # type: ignore[union-attr] + output.memory_offset = lowest.memory_offset # type: ignore[union-attr] else: self.stack.flush(out) if output is not None: @@ -721,5 +715,4 @@ def close_variable(var: Local, overwrite: str) -> None: if output_in_place: self.stack.flush(out) if output is not None: - code, output = self.stack.pop(output.item) - out.emit(code) + output = self.stack.pop(output.item, out) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 20cad326a8a2c0..f784b6b3371908 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -48,18 +48,17 @@ def declare_variables(inst: Instruction, out: CWriter) -> None: stack = get_stack_effect(inst) except StackError as ex: raise analysis_error(ex.args[0], inst.where) from None - required = set(stack.defined) - required.discard("unused") + seen = {"unused"} for part in inst.parts: if not isinstance(part, Uop): continue for var in part.stack.inputs: - if var.name in required: - required.remove(var.name) + if var.used and var.name not in seen: + seen.add(var.name) declare_variable(var, out) for var in part.stack.outputs: - if var.name in required: - required.remove(var.name) + if var.used and var.name not in seen: + seen.add(var.name) declare_variable(var, out) @@ -86,10 +85,8 @@ def write_uop( if braces: emitter.out.emit(f"// {uop.name}\n") emitter.emit("{\n") - code_list, storage = Storage.for_uop(stack, uop) + storage = Storage.for_uop(stack, uop, emitter.out) emitter._print_storage(storage) - for code in code_list: - emitter.emit(code) for cache in uop.caches: if cache.name != "unused": diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 8a684c8328ef7b..8c78388ba53481 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -34,11 +34,11 @@ def declare_variable( - var: StackItem, uop: Uop, required: set[str], out: CWriter + var: StackItem, uop: Uop, seen: set[str], out: CWriter ) -> None: - if not var.used or var.name not in required: + if not var.used or var.name in seen: return - required.remove(var.name) + seen.add(var.name) type, null = type_and_null(var) space = " " if type[-1].isalnum() else "" out.emit(f"{type}{space}{var.name};\n") @@ -46,16 +46,16 @@ def declare_variable( def declare_variables(uop: Uop, out: CWriter) -> None: stack = Stack() + null = CWriter.null() for var in reversed(uop.stack.inputs): - stack.pop(var) + stack.pop(var, null) for var in uop.stack.outputs: stack.push(Local.undefined(var)) - required = set(stack.defined) - required.discard("unused") + seen = {"unused"} for var in reversed(uop.stack.inputs): - declare_variable(var, uop, required, out) + declare_variable(var, uop, seen, out) for var in uop.stack.outputs: - declare_variable(var, uop, required, out) + declare_variable(var, uop, seen, out) class Tier2Emitter(Emitter): @@ -143,9 +143,7 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: elif uop.properties.const_oparg >= 0: emitter.emit(f"oparg = {uop.properties.const_oparg};\n") emitter.emit(f"assert(oparg == CURRENT_OPARG());\n") - code_list, storage = Storage.for_uop(stack, uop) - for code in code_list: - emitter.emit(code) + storage = Storage.for_uop(stack, uop, emitter.out) idx = 0 for cache in uop.caches: if cache.name != "unused": diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 6eb022899d6cae..6f995e5c46bfcf 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -47,13 +47,14 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: out.emit("};\n") out.emit("int _PyUop_num_popped(int opcode, int oparg)\n{\n") out.emit("switch(opcode) {\n") + null = CWriter.null() for uop in analysis.uops.values(): if uop.is_viable() and uop.properties.tier != 1: stack = Stack() for var in reversed(uop.stack.inputs): if var.peek: break - stack.pop(var) + stack.pop(var, null) popped = (-stack.base_offset).to_c() out.emit(f"case {uop.name}:\n") out.emit(f" return {popped};\n")

1 0