Mailman 3 January 2025 - Python-checkins

gh-128627: Emscripten: Fix address calculation for wasm-gc trampoline (#128782)
by freakboy3742 Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/960936fe90f9a0b947c7ed10f91582ff66… commit: 960936fe90f9a0b947c7ed10f91582ff6664c0b8 branch: main author: Hood Chatham <roberthoodchatham(a)gmail.com> committer: freakboy3742 <russell(a)keith-magee.com> date: 2025-01-23T08:02:04+08:00 summary: gh-128627: Emscripten: Fix address calculation for wasm-gc trampoline (#128782) Modifies the memory calculation to divide the entire memory address by 4, not just the base address. files: M Python/emscripten_trampoline.c M configure M configure.ac diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c index 2f9648a12ef2e4..0293c7ea0f37ad 100644 --- a/Python/emscripten_trampoline.c +++ b/Python/emscripten_trampoline.c @@ -26,11 +26,9 @@ EM_JS(CountArgsFunc, _PyEM_GetCountArgsPtr, (), { // Binary module for the checks. It has to be done in web assembly because // clang/llvm have no support yet for the reference types yet. In fact, the wasm // binary toolkit doesn't yet support the ref.test instruction either. To -// convert the following module to the binary, my approach is to find and -// replace "ref.test $type" -> "drop i32.const n" on the source text. This -// results in the bytes "0x1a, 0x41, n" where we need the bytes "0xfb, 0x14, n" -// so doing a find and replace on the output from "0x1a, 0x41" -> "0xfb, 0x14" -// gets us the output we need. +// convert the following textual wasm to a binary, you can build wabt from this +// branch: https://github.com/WebAssembly/wabt/pull/2529 and then use that +// wat2wasm binary. // // (module // (type $type0 (func (param) (result i32))) @@ -154,15 +152,15 @@ addOnPreRun(() => { let ptr = 0; try { const mod = new WebAssembly.Module(code); - const inst = new WebAssembly.Instance(mod, {e: {t: wasmTable}}); + const inst = new WebAssembly.Instance(mod, { e: { t: wasmTable } }); ptr = addFunction(inst.exports.f); - } catch(e) { + } catch (e) { // If something goes wrong, we'll null out _PyEM_CountFuncParams and fall // back to the JS trampoline. } Module._PyEM_CountArgsPtr = ptr; - const offset = HEAP32[__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET/4]; - HEAP32[__PyRuntime/4 + offset] = ptr; + const offset = HEAP32[__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET / 4]; + HEAP32[(__PyRuntime + offset) / 4] = ptr; }); ); diff --git a/configure b/configure index ae1b649630171f..885c2cf7828d6c 100755 --- a/configure +++ b/configure @@ -9627,7 +9627,7 @@ fi as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV" - as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version" + as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET" as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" if test "x$enable_wasm_dynamic_linking" = xyes diff --git a/configure.ac b/configure.ac index cc499dfbc502f1..f89a0801948ca5 100644 --- a/configure.ac +++ b/configure.ac @@ -2369,7 +2369,7 @@ AS_CASE([$ac_sys_system], dnl Include file system support AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET"]) AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [

1 0

[3.12] Add colour to GitHub Actions output (GH-129196) (#129199)
by hugovk Jan. 22, 2025

Jan. 22, 2025

https://github.com/python/cpython/commit/880ad1813e8d61c219b7a9449d03786416… commit: 880ad1813e8d61c219b7a9449d03786416509f8e branch: 3.12 author: Hugo van Kemenade <1324225+hugovk(a)users.noreply.github.com> committer: hugovk <1324225+hugovk(a)users.noreply.github.com> date: 2025-01-22T21:50:35Z summary: [3.12] Add colour to GitHub Actions output (GH-129196) (#129199) files: M .github/workflows/build.yml M .github/workflows/reusable-macos.yml M .github/workflows/reusable-tsan.yml M .github/workflows/reusable-ubuntu.yml M .github/workflows/reusable-windows-msi.yml M .github/workflows/reusable-windows.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8c7665f2a14d1d..413600bdd900a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,6 +26,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: check_source: name: Change detection diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 981a866156160e..a7ed573df339bf 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -15,6 +15,9 @@ on: required: true type: string +env: + FORCE_COLOR: 1 + jobs: build_macos: name: build and test (${{ inputs.os }}) diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 52fa3b0eb9c2f7..3aed17213218f0 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -10,6 +10,9 @@ on: required: true type: string +env: + FORCE_COLOR: 1 + jobs: build_tsan_reusable: name: 'Thread sanitizer' diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 5d6f5b2c8bb572..4f6d73cb3729a1 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -12,6 +12,9 @@ on: type: boolean default: false +env: + FORCE_COLOR: 1 + jobs: build_ubuntu_reusable: name: 'build and test' diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index a1c45d954247fb..bc0414d1bbcd8f 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -11,6 +11,9 @@ on: permissions: contents: read +env: + FORCE_COLOR: 1 + jobs: build: name: installer for ${{ inputs.arch }} diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 418b05740fbd88..35c75f4a0877dd 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -14,6 +14,7 @@ on: default: false env: + FORCE_COLOR: 1 IncludeUwp: >- true

1 0

[3.13] Add colour to GitHub Actions output (GH-129196) (#129198)
by hugovk Jan. 22, 2025

Jan. 22, 2025

https://github.com/python/cpython/commit/d674792ba773d78d4ed71698b7d47fafb8… commit: d674792ba773d78d4ed71698b7d47fafb8842d89 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: hugovk <1324225+hugovk(a)users.noreply.github.com> date: 2025-01-22T21:45:08Z summary: [3.13] Add colour to GitHub Actions output (GH-129196) (#129198) Co-authored-by: Hugo van Kemenade <1324225+hugovk(a)users.noreply.github.com> files: M .github/workflows/build.yml M .github/workflows/jit.yml M .github/workflows/reusable-macos.yml M .github/workflows/reusable-tsan.yml M .github/workflows/reusable-ubuntu.yml M .github/workflows/reusable-wasi.yml M .github/workflows/reusable-windows-msi.yml M .github/workflows/reusable-windows.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 11bc517c90cb6d..05a2382fccbe25 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,6 +18,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: check_source: name: Change detection diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 198177408b3b0d..d82c7c69d10302 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -25,6 +25,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: interpreter: name: Interpreter (Debug) diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 310eeb51f7608d..6828c0ba2dff3b 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -15,6 +15,9 @@ on: required: true type: string +env: + FORCE_COLOR: 1 + jobs: build_macos: name: build and test (${{ inputs.os }}) diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 8dad17508ef422..47a63be32560bb 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -18,6 +18,9 @@ on: required: true type: string +env: + FORCE_COLOR: 1 + jobs: build_tsan_reusable: name: 'Thread sanitizer' diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 09044660b432d0..ea9e6f96bc2a8b 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -12,6 +12,9 @@ on: type: boolean default: false +env: + FORCE_COLOR: 1 + jobs: build_ubuntu_reusable: name: 'build and test' @@ -22,7 +25,6 @@ jobs: matrix: os: [ubuntu-24.04, ubuntu-24.04-arm] env: - FORCE_COLOR: 1 OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index a3c4f97a5a69fe..e955e6932d602a 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -7,6 +7,9 @@ on: required: true type: string +env: + FORCE_COLOR: 1 + jobs: build_wasi_reusable: name: 'build and test' diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index a1c45d954247fb..bc0414d1bbcd8f 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -11,6 +11,9 @@ on: permissions: contents: read +env: + FORCE_COLOR: 1 + jobs: build: name: installer for ${{ inputs.arch }} diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 459d2b29e5d42b..bfee3d2722cb44 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -18,6 +18,7 @@ on: default: false env: + FORCE_COLOR: 1 IncludeUwp: >- true

1 0

Add colour to GitHub Actions output (#129196)
by hugovk Jan. 22, 2025

Jan. 22, 2025

https://github.com/python/cpython/commit/298dda57709c45cbcb44831e0d682dc071… commit: 298dda57709c45cbcb44831e0d682dc071af5293 branch: main author: Hugo van Kemenade <1324225+hugovk(a)users.noreply.github.com> committer: hugovk <1324225+hugovk(a)users.noreply.github.com> date: 2025-01-22T21:22:21Z summary: Add colour to GitHub Actions output (#129196) files: M .github/workflows/build.yml M .github/workflows/jit.yml M .github/workflows/reusable-macos.yml M .github/workflows/reusable-tsan.yml M .github/workflows/reusable-ubuntu.yml M .github/workflows/reusable-wasi.yml M .github/workflows/reusable-windows-msi.yml M .github/workflows/reusable-windows.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6667512369716d..0c20b85acbd565 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,6 +18,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: check_source: name: Change detection diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 6f214ad4bc6978..806a8524112d76 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -25,6 +25,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: interpreter: name: Interpreter (Debug) diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index cdbe05e09fb8e7..4e7aced94a321c 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -15,6 +15,9 @@ on: required: true type: string +env: + FORCE_COLOR: 1 + jobs: build_macos: name: build and test (${{ inputs.os }}) diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index b5144ca3e9efc4..269f479849f21e 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -18,6 +18,9 @@ on: required: true type: string +env: + FORCE_COLOR: 1 + jobs: build_tsan_reusable: name: 'Thread sanitizer' diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 686e8fe1abc980..aa8ba00f19d8ca 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -21,13 +21,15 @@ on: required: true type: string +env: + FORCE_COLOR: 1 + jobs: build_ubuntu_reusable: name: build and test (${{ inputs.os }}) timeout-minutes: 60 runs-on: ${{ inputs.os }} env: - FORCE_COLOR: 1 OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 4356d9c1c8795e..4456b83c8e0032 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -7,6 +7,9 @@ on: required: true type: string +env: + FORCE_COLOR: 1 + jobs: build_wasi_reusable: name: 'build and test' diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index a1c45d954247fb..bc0414d1bbcd8f 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -11,6 +11,9 @@ on: permissions: contents: read +env: + FORCE_COLOR: 1 + jobs: build: name: installer for ${{ inputs.arch }} diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 459d2b29e5d42b..bfee3d2722cb44 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -18,6 +18,7 @@ on: default: false env: + FORCE_COLOR: 1 IncludeUwp: >- true

1 0

gh-129192: Use `EnvironmentVarGuard` to restore environment variables (#129193)
by Eclips4 Jan. 22, 2025

Jan. 22, 2025

https://github.com/python/cpython/commit/fc6d4b71eb6bb4dcef3def286302e6bec3… commit: fc6d4b71eb6bb4dcef3def286302e6bec37aec9f branch: main author: Kirill Podoprigora <kirill.bast9(a)mail.ru> committer: Eclips4 <kirill.bast9(a)mail.ru> date: 2025-01-22T20:39:26Z summary: gh-129192: Use `EnvironmentVarGuard` to restore environment variables (#129193) Co-authored-by: Cody Maloney <cmaloney(a)theoreticalchaos.com> files: M Lib/test/test_zipfile/test_core.py diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index ab5fb650084327..76b1de0e3519f9 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -1801,17 +1801,17 @@ def test_write_with_source_date_epoch(self): self.assertAlmostEqual(z_time, g_time, delta=1) def test_write_without_source_date_epoch(self): - if 'SOURCE_DATE_EPOCH' in os.environ: - del os.environ['SOURCE_DATE_EPOCH'] + with os_helper.EnvironmentVarGuard() as env: + del env['SOURCE_DATE_EPOCH'] - with zipfile.ZipFile(TESTFN, "w") as zf: - zf.writestr("test_no_source_date_epoch.txt", "Testing without SOURCE_DATE_EPOCH") + with zipfile.ZipFile(TESTFN, "w") as zf: + zf.writestr("test_no_source_date_epoch.txt", "Testing without SOURCE_DATE_EPOCH") - with zipfile.ZipFile(TESTFN, "r") as zf: - zip_info = zf.getinfo("test_no_source_date_epoch.txt") - current_time = time.localtime()[:6] - for z_time, c_time in zip(zip_info.date_time, current_time): - self.assertAlmostEqual(z_time, c_time, delta=1) + with zipfile.ZipFile(TESTFN, "r") as zf: + zip_info = zf.getinfo("test_no_source_date_epoch.txt") + current_time = time.localtime()[:6] + for z_time, c_time in zip(zip_info.date_time, current_time): + self.assertAlmostEqual(z_time, c_time, delta=1) def test_close(self): """Check that the zipfile is closed after the 'with' block."""

1 0

[3.13] gh-127873: Only check `sys.flags.ignore_environment` for `PYTHON*` env vars (GH-127877) (#129138)
by hugovk Jan. 22, 2025

Jan. 22, 2025

https://github.com/python/cpython/commit/cc3dc8ab24f872c8d31ddbc4fce7ff1213… commit: cc3dc8ab24f872c8d31ddbc4fce7ff1213ec5529 branch: 3.13 author: Hugo van Kemenade <1324225+hugovk(a)users.noreply.github.com> committer: hugovk <1324225+hugovk(a)users.noreply.github.com> date: 2025-01-22T21:09:51+02:00 summary: [3.13] gh-127873: Only check `sys.flags.ignore_environment` for `PYTHON*` env vars (GH-127877) (#129138) files: A Misc/NEWS.d/next/Library/2024-12-12-18-25-50.gh-issue-127873.WJRwfz.rst M .github/CODEOWNERS M Lib/_colorize.py M Lib/test/support/__init__.py M Lib/test/test__colorize.py M Lib/test/test_capi/test_misc.py M Lib/test/test_cmd_line_script.py M Lib/test/test_compileall.py M Lib/test/test_eof.py M Lib/test/test_exceptions.py M Lib/test/test_import/__init__.py M Lib/test/test_inspect/test_inspect.py M Lib/test/test_pyrepl/support.py M Lib/test/test_pyrepl/test_pyrepl.py M Lib/test/test_regrtest.py M Lib/test/test_repl.py M Lib/test/test_runpy.py M Lib/test/test_tracemalloc.py M Lib/test/test_unicodedata.py M Lib/test/test_unittest/test_program.py M Lib/test/test_unittest/test_result.py M Lib/test/test_unittest/test_runner.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f71214f11109f9..4ffbb428bc381d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -267,3 +267,7 @@ Lib/test/test_interpreters/ @ericsnowcurrently # Config Parser Lib/configparser.py @jaraco Lib/test/test_configparser.py @jaraco + +# Colorize +Lib/_colorize.py @hugovk +Lib/test/test__colorize.py @hugovk diff --git a/Lib/_colorize.py b/Lib/_colorize.py index daecf5b16c1965..deb2e854ce1ff8 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -40,15 +40,14 @@ def can_colorize(*, file=None) -> bool: return False if os.environ.get("PYTHON_COLORS") == "1": return True - if "NO_COLOR" in os.environ: - return False + if "NO_COLOR" in os.environ: + return False if not COLORIZE: return False - if not sys.flags.ignore_environment: - if "FORCE_COLOR" in os.environ: - return True - if os.environ.get("TERM") == "dumb": - return False + if "FORCE_COLOR" in os.environ: + return True + if os.environ.get("TERM") == "dumb": + return False if not hasattr(file, "fileno"): return False diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index e6a8ef1ddcc14d..d7fa6096d375c5 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -61,6 +61,7 @@ "without_optimizer", "force_not_colorized", "force_not_colorized_test_class", + "make_clean_env", "BrokenIter", ] @@ -2732,6 +2733,16 @@ def new_setUpClass(cls): return cls +def make_clean_env() -> dict[str, str]: + clean_env = os.environ.copy() + for k in clean_env.copy(): + if k.startswith("PYTHON"): + clean_env.pop(k) + clean_env.pop("FORCE_COLOR", None) + clean_env.pop("NO_COLOR", None) + return clean_env + + def initialized_with_pyrepl(): """Detect whether PyREPL was used during Python initialization.""" # If the main module has a __file__ attribute it's a Python module, which means PyREPL. diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py index 77e74fa3e23c2c..25519ba7e92e1f 100644 --- a/Lib/test/test__colorize.py +++ b/Lib/test/test__colorize.py @@ -3,7 +3,7 @@ import unittest import unittest.mock import _colorize -from test.support import force_not_colorized +from test.support import force_not_colorized, make_clean_env ORIGINAL_CAN_COLORIZE = _colorize.can_colorize @@ -17,6 +17,14 @@ def tearDownModule(): class TestColorizeFunction(unittest.TestCase): + def setUp(self): + # Remove PYTHON* environment variables to isolate from local user + # settings and simulate running with `-E`. Such variables should be + # added to test methods later to patched os.environ. + patcher = unittest.mock.patch("os.environ", new=make_clean_env()) + self.addCleanup(patcher.stop) + patcher.start() + @force_not_colorized def test_colorized_detection_checks_for_environment_variables(self): flags = unittest.mock.MagicMock(ignore_environment=False) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index ec37942bdb2825..f0b872627f6f93 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -75,6 +75,8 @@ class InstanceMethod: id = _testcapi.instancemethod(id) testfunction = _testcapi.instancemethod(testfunction) + +(a)support.force_not_colorized_test_class class CAPITest(unittest.TestCase): def test_instancemethod(self): diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 3a5a8abf81e43d..1ec5e581f81d17 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -88,6 +88,8 @@ def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, importlib.invalidate_caches() return to_return + +(a)support.force_not_colorized_test_class class CmdLineTest(unittest.TestCase): def _check_output(self, script_name, exit_code, data, expected_file, expected_argv0, diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 812ff5e7f84461..21ecebc088d3df 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -766,6 +766,7 @@ def test_d_compile_error(self): rc, out, err = self.assertRunNotOK('-q', '-d', 'dinsdale', self.pkgdir) self.assertRegex(out, b'File "dinsdale') + @support.force_not_colorized def test_d_runtime_error(self): bazfn = script_helper.make_script(self.pkgdir, 'baz', 'raise Exception') self.assertRunOK('-q', '-d', 'dinsdale', self.pkgdir) diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index e377383450e19d..582e5b6de6e687 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -2,7 +2,7 @@ import sys from codecs import BOM_UTF8 -from test import support +from test.support import force_not_colorized from test.support import os_helper from test.support import script_helper from test.support import warnings_helper @@ -44,6 +44,7 @@ def test_EOFS(self): self.assertEqual(cm.exception.text, "ä = '''thîs is ") self.assertEqual(cm.exception.offset, 5) + @force_not_colorized def test_EOFS_with_file(self): expect = ("(<string>, line 1)") with os_helper.temp_dir() as temp_dir: @@ -123,6 +124,7 @@ def test_line_continuation_EOF(self): self.assertEqual(str(cm.exception), expect) @unittest.skipIf(not sys.executable, "sys.executable required") + @force_not_colorized def test_line_continuation_EOF_from_file_bpo2180(self): """Ensure tok_nextc() does not add too many ending newlines.""" with os_helper.temp_dir() as temp_dir: diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 6a29bc38252fdb..c6fb848b8d7fda 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1465,6 +1465,7 @@ def gen(): @cpython_only @unittest.skipIf(_testcapi is None, "requires _testcapi") + @force_not_colorized def test_recursion_normalizing_infinite_exception(self): # Issue #30697. Test that a RecursionError is raised when # maximum recursion depth has been exceeded when creating @@ -2157,6 +2158,7 @@ def test_multiline_not_highlighted(self): self.assertEqual(result[-len(expected):], expected) +(a)support.force_not_colorized_test_class class SyntaxErrorTests(unittest.TestCase): maxDiff = None diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index df712ec5a9d97d..d9246c0ea70a04 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -29,9 +29,20 @@ from test.support import os_helper from test.support import ( - STDLIB_DIR, swap_attr, swap_item, cpython_only, is_apple_mobile, is_emscripten, - is_wasi, run_in_subinterp, run_in_subinterp_with_config, Py_TRACE_REFS, - requires_gil_enabled, Py_GIL_DISABLED) + STDLIB_DIR, + swap_attr, + swap_item, + cpython_only, + is_apple_mobile, + is_emscripten, + is_wasi, + run_in_subinterp, + run_in_subinterp_with_config, + Py_TRACE_REFS, + requires_gil_enabled, + Py_GIL_DISABLED, + force_not_colorized_test_class, +) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, ready_to_import, DirsOnSysPath, CleanImport, import_module) @@ -352,6 +363,7 @@ def _from_subinterp(cls, name, interpid, pipe, script_kwargs): return cls.parse(text.decode()) +@force_not_colorized_test_class class ImportTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 34ae951b38ad58..f30dc7affda11a 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -880,6 +880,7 @@ def test_getsource_stdlib_decimal(self): self.assertEqual(src.splitlines(True), lines) class TestGetsourceInteractive(unittest.TestCase): + @support.force_not_colorized def test_getclasses_interactive(self): # bpo-44648: simulate a REPL session; # there is no `__file__` in the __main__ module diff --git a/Lib/test/test_pyrepl/support.py b/Lib/test/test_pyrepl/support.py index 672d4896c92283..45e3bf758f17de 100644 --- a/Lib/test/test_pyrepl/support.py +++ b/Lib/test/test_pyrepl/support.py @@ -101,16 +101,6 @@ def handle_all_events( ) -def make_clean_env() -> dict[str, str]: - clean_env = os.environ.copy() - for k in clean_env.copy(): - if k.startswith("PYTHON"): - clean_env.pop(k) - clean_env.pop("FORCE_COLOR", None) - clean_env.pop("NO_COLOR", None) - return clean_env - - class FakeConsole(Console): def __init__(self, events, encoding="utf-8") -> None: self.events = iter(events) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index a20e79b674a916..b90b5a05adfa4a 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -10,7 +10,7 @@ import tempfile from unittest import TestCase, skipUnless, skipIf from unittest.mock import patch -from test.support import force_not_colorized +from test.support import force_not_colorized, make_clean_env from test.support import SHORT_TIMEOUT from test.support.import_helper import import_module from test.support.os_helper import unlink @@ -23,7 +23,6 @@ multiline_input, code_to_events, clean_screen, - make_clean_env, ) from _pyrepl.console import Event from _pyrepl.readline import (ReadlineAlikeReader, ReadlineConfig, diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 54b6a16a0dab05..a5c9617bb07fd6 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -789,6 +789,7 @@ def test_finds_expected_number_of_tests(self): f'{", ".join(output.splitlines())}') +(a)support.force_not_colorized_test_class class ProgramsTestCase(BaseTestCase): """ Test various ways to run the Python test suite. Use options close @@ -902,6 +903,7 @@ def test_pcbuild_rt(self): self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) +(a)support.force_not_colorized_test_class class ArgsTestCase(BaseTestCase): """ Test arguments of the Python test suite. diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index e764e60560db23..356ff5b198d637 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -70,6 +70,7 @@ def run_on_interactive_mode(source): return output +(a)support.force_not_colorized_test_class class TestInteractiveInterpreter(unittest.TestCase): @cpython_only @@ -273,6 +274,8 @@ def test_asyncio_repl_is_ok(self): self.assertEqual(exit_code, 0, "".join(output)) + +(a)support.force_not_colorized_test_class class TestInteractiveModeSyntaxErrors(unittest.TestCase): def test_interactive_syntax_error_correct_line(self): diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index b64383f6546f31..ada78ec8e6b0c7 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -12,8 +12,14 @@ import textwrap import unittest import warnings -from test.support import (infinite_recursion, no_tracing, verbose, - requires_subprocess, requires_resource) +from test.support import ( + force_not_colorized_test_class, + infinite_recursion, + no_tracing, + requires_resource, + requires_subprocess, + verbose, +) from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script @@ -758,6 +764,7 @@ def test_encoding(self): self.assertEqual(result['s'], "non-ASCII: h\xe9") +@force_not_colorized_test_class class TestExit(unittest.TestCase): STATUS_CONTROL_C_EXIT = 0xC000013A EXPECTED_CODE = ( diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index a848363fcd1de9..238ae14b388c76 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -981,6 +981,7 @@ def check_sys_xoptions_invalid(self, nframe): return self.fail(f"unexpected output: {stderr!a}") + @force_not_colorized def test_sys_xoptions_invalid(self): for nframe in INVALID_NFRAME: with self.subTest(nframe=nframe): diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index d3bf4ea7c7d437..2cf367a2cfe85b 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -11,8 +11,14 @@ import sys import unicodedata import unittest -from test.support import (open_urlresource, requires_resource, script_helper, - cpython_only, check_disallow_instantiation) +from test.support import ( + open_urlresource, + requires_resource, + script_helper, + cpython_only, + check_disallow_instantiation, + force_not_colorized, +) class UnicodeMethodsTest(unittest.TestCase): @@ -277,6 +283,7 @@ def test_disallow_instantiation(self): # Ensure that the type disallows instantiation (bpo-43916) check_disallow_instantiation(self, unicodedata.UCD) + @force_not_colorized def test_failed_import_during_compiling(self): # Issue 4367 # Decoding \N escapes requires the unicodedata module. If it can't be diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 7241cf59f73d4f..aa7e8b712fd763 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -7,6 +7,7 @@ from test.test_unittest.test_result import BufferedWriter +(a)support.force_not_colorized_test_class class Test_TestProgram(unittest.TestCase): def test_discovery_from_dotted_path(self): diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index 144f20176f8b18..4d552d54e9a6df 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -33,6 +33,7 @@ def bad_cleanup2(): raise ValueError('bad cleanup2') +@force_not_colorized_test_class class Test_TestResult(unittest.TestCase): # Note: there are not separate tests for TestResult.wasSuccessful(), # TestResult.errors, TestResult.failures, TestResult.testsRun or @@ -456,6 +457,7 @@ def test(result): self.assertTrue(stream.getvalue().endswith('\n\nOK\n')) +@force_not_colorized_test_class class Test_TextTestResult(unittest.TestCase): maxDiff = None diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index 1b9cef43e3f9c5..4d3cfd60b8d9c3 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -106,6 +106,7 @@ def cleanup2(*args, **kwargs): self.assertTrue(test.doCleanups()) self.assertEqual(cleanups, [(2, (), {}), (1, (1, 2, 3), dict(four='hello', five='goodbye'))]) + @support.force_not_colorized def testCleanUpWithErrors(self): class TestableTest(unittest.TestCase): def testNothing(self): @@ -249,6 +250,7 @@ def testNothing(self): self.assertEqual(test._cleanups, []) +(a)support.force_not_colorized_test_class class TestClassCleanup(unittest.TestCase): def test_addClassCleanUp(self): class TestableTest(unittest.TestCase): @@ -601,6 +603,7 @@ class EmptyTest(unittest.TestCase): self.assertIn("\nNO TESTS RAN\n", runner.stream.getvalue()) +(a)support.force_not_colorized_test_class class TestModuleCleanUp(unittest.TestCase): def test_add_and_do_ModuleCleanup(self): module_cleanups = [] @@ -1318,6 +1321,7 @@ def MockResultClass(*args): expectedresult = (runner.stream, DESCRIPTIONS, VERBOSITY) self.assertEqual(runner._makeResult(), expectedresult) + @support.force_not_colorized @support.requires_subprocess() def test_warnings(self): """ diff --git a/Misc/NEWS.d/next/Library/2024-12-12-18-25-50.gh-issue-127873.WJRwfz.rst b/Misc/NEWS.d/next/Library/2024-12-12-18-25-50.gh-issue-127873.WJRwfz.rst new file mode 100644 index 00000000000000..d7575c7efb6e88 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-12-18-25-50.gh-issue-127873.WJRwfz.rst @@ -0,0 +1,3 @@ +When ``-E`` is set, only ignore ``PYTHON_COLORS`` and not +``FORCE_COLOR``/``NO_COLOR``/``TERM`` when colourising output. +Patch by Hugo van Kemenade.

1 0

GH-91048: Don't attempt to run on FreeBSD (#129189)
by ambv Jan. 22, 2025

Jan. 22, 2025

https://github.com/python/cpython/commit/67d804b494d5f9f13fff088b50ff488b37… commit: 67d804b494d5f9f13fff088b50ff488b3701979d branch: main author: Łukasz Langa <lukasz(a)langa.pl> committer: ambv <lukasz(a)langa.pl> date: 2025-01-22T19:27:58+01:00 summary: GH-91048: Don't attempt to run on FreeBSD (#129189) files: M Modules/_testexternalinspection.c diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 0c31d1b7a3486c..25e0e9c64ec27d 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -263,9 +263,7 @@ search_map_for_section(pid_t pid, const char* secname, const char* substr) { return 0; } -#endif - -#ifdef __linux__ +#elif defined(__linux__) static uintptr_t find_map_start_address(pid_t pid, char* result_filename, const char* map) { @@ -395,7 +393,12 @@ search_map_for_section(pid_t pid, const char* secname, const char* map) } return result; } - +#else +static uintptr_t +search_map_for_section(pid_t pid, const char* secname, const char* map) +{ + return 0; +} #endif static uintptr_t @@ -1265,7 +1268,6 @@ read_offsets( _Py_DebugOffsets* debug_offsets ) { *runtime_start_address = get_py_runtime(pid); - assert(runtime_start_address != NULL); if ((void*)*runtime_start_address == NULL) { if (!PyErr_Occurred()) { PyErr_SetString(

1 0

[3.12] gh-124363: Treat debug expressions in f-string as raw strings (GH-128399) (#129190)
by pablogsal Jan. 22, 2025

Jan. 22, 2025

https://github.com/python/cpython/commit/9ed04d95162cb5b23663dac706e1646ea1… commit: 9ed04d95162cb5b23663dac706e1646ea195e180 branch: 3.12 author: Pablo Galindo Salgado <Pablogsal(a)gmail.com> committer: pablogsal <Pablogsal(a)gmail.com> date: 2025-01-22T18:08:58Z summary: [3.12] gh-124363: Treat debug expressions in f-string as raw strings (GH-128399) (#129190) (cherry picked from commit 60a3a0dd6fe140fdc87f6e769ee5bb17d92efe4e) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot](a)users.noreply.github.com> files: A Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-19-48-30.gh-issue-124363.vOFhHW.rst M Lib/test/test_fstring.py M Parser/action_helpers.c diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 6cba2e8616eb72..49640f996e0c7a 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1764,6 +1764,14 @@ def __repr__(self): # self.assertEqual(f'X{x =}Y', 'Xx\t='+repr(x)+'Y') # self.assertEqual(f'X{x = }Y', 'Xx\t=\t'+repr(x)+'Y') + def test_debug_expressions_are_raw_strings(self): + + self.assertEqual(f'{b"\N{OX}"=}', 'b"\\N{OX}"=b\'\\\\N{OX}\'') + self.assertEqual(f'{r"\xff"=}', 'r"\\xff"=\'\\\\xff\'') + self.assertEqual(f'{r"\n"=}', 'r"\\n"=\'\\\\n\'') + self.assertEqual(f"{'\''=}", "'\\''=\"'\"") + self.assertEqual(f'{'\xc5'=}', r"'\xc5'='Å'") + def test_walrus(self): x = 20 # This isn't an assignment expression, it's 'x', with a format diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-19-48-30.gh-issue-124363.vOFhHW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-19-48-30.gh-issue-124363.vOFhHW.rst new file mode 100644 index 00000000000000..553aa5a4dd573e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-19-48-30.gh-issue-124363.vOFhHW.rst @@ -0,0 +1 @@ +Treat debug expressions in f-string as raw strings. Patch by Pablo Galindo diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index be52d89495c7dc..e029ae692f53b8 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -909,8 +909,6 @@ _PyPegen_check_fstring_conversion(Parser *p, Token *conv_token, expr_ty conv) { return result_token_with_metadata(p, conv, conv_token->metadata); } -static asdl_expr_seq * -unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions); ResultTokenWithMetadata * _PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, int lineno, int col_offset, int end_lineno, @@ -1192,8 +1190,9 @@ static expr_ty _PyPegen_decode_fstring_part(Parser *p, int is_raw, p->arena); } -static asdl_expr_seq * -unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions) { +expr_ty +_PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* expr, Token*b) { + /* The parser might put multiple f-string values into an individual * JoinedStr node at the top level due to stuff like f-string debugging * expressions. This function flattens those and promotes them to the @@ -1201,44 +1200,14 @@ unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions) { * of the regular output, so this is not necessary if you are not going * to expose the output AST to Python level. */ - Py_ssize_t i, req_size, raw_size; - - req_size = raw_size = asdl_seq_LEN(raw_expressions); - expr_ty expr; - for (i = 0; i < raw_size; i++) { - expr = asdl_seq_GET(raw_expressions, i); - if (expr->kind == JoinedStr_kind) { - req_size += asdl_seq_LEN(expr->v.JoinedStr.values) - 1; - } - } - - asdl_expr_seq *expressions = _Py_asdl_expr_seq_new(req_size, p->arena); - if (expressions == NULL) { - return NULL; - } - - Py_ssize_t raw_index, req_index = 0; - for (raw_index = 0; raw_index < raw_size; raw_index++) { - expr = asdl_seq_GET(raw_expressions, raw_index); - if (expr->kind == JoinedStr_kind) { - asdl_expr_seq *values = expr->v.JoinedStr.values; - for (Py_ssize_t n = 0; n < asdl_seq_LEN(values); n++) { - asdl_seq_SET(expressions, req_index, asdl_seq_GET(values, n)); - req_index++; + Py_ssize_t n_items = asdl_seq_LEN(expr); + Py_ssize_t total_items = n_items; + for (Py_ssize_t i = 0; i < n_items; i++) { + expr_ty item = asdl_seq_GET(expr, i); + if (item->kind == JoinedStr_kind) { + total_items += asdl_seq_LEN(item->v.JoinedStr.values) - 1; } - } else { - asdl_seq_SET(expressions, req_index, expr); - req_index++; - } } - return expressions; -} - -expr_ty _PyPegen_joined_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, - Token *b) { - - asdl_expr_seq *expr = unpack_top_level_joined_strs(p, raw_expressions); - Py_ssize_t n_items = asdl_seq_LEN(expr); const char *quote_str = PyBytes_AsString(a->bytes); if (quote_str == NULL) { @@ -1246,7 +1215,7 @@ expr_ty _PyPegen_joined_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, } int is_raw = strpbrk(quote_str, "rR") != NULL; - asdl_expr_seq *seq = _Py_asdl_expr_seq_new(n_items, p->arena); + asdl_expr_seq *seq = _Py_asdl_expr_seq_new(total_items, p->arena); if (seq == NULL) { return NULL; } @@ -1254,6 +1223,29 @@ expr_ty _PyPegen_joined_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Py_ssize_t index = 0; for (Py_ssize_t i = 0; i < n_items; i++) { expr_ty item = asdl_seq_GET(expr, i); + // This should correspond to a JoinedStr node of two elements + // created _PyPegen_formatted_value. This situation can only be the result of + // a f-string debug expression where the first element is a constant with the text and the second + // a formatted value with the expression. + if (item->kind == JoinedStr_kind) { + asdl_expr_seq *values = item->v.JoinedStr.values; + if (asdl_seq_LEN(values) != 2) { + PyErr_Format(PyExc_SystemError, + "unexpected JoinedStr node without debug data in f-string at line %d", + item->lineno); + return NULL; + } + + expr_ty first = asdl_seq_GET(values, 0); + assert(first->kind == Constant_kind); + asdl_seq_SET(seq, index++, first); + + expr_ty second = asdl_seq_GET(values, 1); + assert(second->kind == FormattedValue_kind); + asdl_seq_SET(seq, index++, second); + + continue; + } if (item->kind == Constant_kind) { item = _PyPegen_decode_fstring_part(p, is_raw, item, b); if (item == NULL) { @@ -1272,7 +1264,7 @@ expr_ty _PyPegen_joined_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, } asdl_expr_seq *resized_exprs; - if (index != n_items) { + if (index != total_items) { resized_exprs = _Py_asdl_expr_seq_new(index, p->arena); if (resized_exprs == NULL) { return NULL;

1 0

[3.13] gh-124363: Treat debug expressions in f-string as raw strings (GH-128399) (#129187)
by pablogsal Jan. 22, 2025

Jan. 22, 2025

https://github.com/python/cpython/commit/f7f8b8b75884377e9db1f5a830a3ecbc2b… commit: f7f8b8b75884377e9db1f5a830a3ecbc2bacbb94 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: pablogsal <Pablogsal(a)gmail.com> date: 2025-01-22T17:19:07Z summary: [3.13] gh-124363: Treat debug expressions in f-string as raw strings (GH-128399) (#129187) gh-124363: Treat debug expressions in f-string as raw strings (GH-128399) (cherry picked from commit 60a3a0dd6fe140fdc87f6e769ee5bb17d92efe4e) Co-authored-by: Pablo Galindo Salgado <Pablogsal(a)gmail.com> Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot](a)users.noreply.github.com> files: A Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-19-48-30.gh-issue-124363.vOFhHW.rst M Lib/test/test_fstring.py M Parser/action_helpers.c diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index e77f76fc55b44ad..6e217ef63be618d 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1650,6 +1650,14 @@ def __repr__(self): #self.assertEqual(f'X{x =}Y', 'Xx\t='+repr(x)+'Y') #self.assertEqual(f'X{x = }Y', 'Xx\t=\t'+repr(x)+'Y') + def test_debug_expressions_are_raw_strings(self): + + self.assertEqual(f'{b"\N{OX}"=}', 'b"\\N{OX}"=b\'\\\\N{OX}\'') + self.assertEqual(f'{r"\xff"=}', 'r"\\xff"=\'\\\\xff\'') + self.assertEqual(f'{r"\n"=}', 'r"\\n"=\'\\\\n\'') + self.assertEqual(f"{'\''=}", "'\\''=\"'\"") + self.assertEqual(f'{'\xc5'=}', r"'\xc5'='Å'") + def test_walrus(self): x = 20 # This isn't an assignment expression, it's 'x', with a format diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-19-48-30.gh-issue-124363.vOFhHW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-19-48-30.gh-issue-124363.vOFhHW.rst new file mode 100644 index 000000000000000..553aa5a4dd573e9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-21-19-48-30.gh-issue-124363.vOFhHW.rst @@ -0,0 +1 @@ +Treat debug expressions in f-string as raw strings. Patch by Pablo Galindo diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 8dfc919da71b5a8..98f9091ee43f902 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -961,8 +961,6 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv) return result_token_with_metadata(p, conv, conv_token->metadata); } -static asdl_expr_seq * -unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions); ResultTokenWithMetadata * _PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) @@ -1271,9 +1269,9 @@ _PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* tok p->arena); } -static asdl_expr_seq * -unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions) -{ +expr_ty +_PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* expr, Token*b) { + /* The parser might put multiple f-string values into an individual * JoinedStr node at the top level due to stuff like f-string debugging * expressions. This function flattens those and promotes them to the @@ -1281,44 +1279,14 @@ unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions) * of the regular output, so this is not necessary if you are not going * to expose the output AST to Python level. */ - Py_ssize_t i, req_size, raw_size; - - req_size = raw_size = asdl_seq_LEN(raw_expressions); - expr_ty expr; - for (i = 0; i < raw_size; i++) { - expr = asdl_seq_GET(raw_expressions, i); - if (expr->kind == JoinedStr_kind) { - req_size += asdl_seq_LEN(expr->v.JoinedStr.values) - 1; - } - } - - asdl_expr_seq *expressions = _Py_asdl_expr_seq_new(req_size, p->arena); - if (expressions == NULL) { - return NULL; - } - - Py_ssize_t raw_index, req_index = 0; - for (raw_index = 0; raw_index < raw_size; raw_index++) { - expr = asdl_seq_GET(raw_expressions, raw_index); - if (expr->kind == JoinedStr_kind) { - asdl_expr_seq *values = expr->v.JoinedStr.values; - for (Py_ssize_t n = 0; n < asdl_seq_LEN(values); n++) { - asdl_seq_SET(expressions, req_index, asdl_seq_GET(values, n)); - req_index++; - } - } else { - asdl_seq_SET(expressions, req_index, expr); - req_index++; + Py_ssize_t n_items = asdl_seq_LEN(expr); + Py_ssize_t total_items = n_items; + for (Py_ssize_t i = 0; i < n_items; i++) { + expr_ty item = asdl_seq_GET(expr, i); + if (item->kind == JoinedStr_kind) { + total_items += asdl_seq_LEN(item->v.JoinedStr.values) - 1; } } - return expressions; -} - -expr_ty -_PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b) { - - asdl_expr_seq *expr = unpack_top_level_joined_strs(p, raw_expressions); - Py_ssize_t n_items = asdl_seq_LEN(expr); const char* quote_str = PyBytes_AsString(a->bytes); if (quote_str == NULL) { @@ -1326,7 +1294,7 @@ _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b } int is_raw = strpbrk(quote_str, "rR") != NULL; - asdl_expr_seq *seq = _Py_asdl_expr_seq_new(n_items, p->arena); + asdl_expr_seq *seq = _Py_asdl_expr_seq_new(total_items, p->arena); if (seq == NULL) { return NULL; } @@ -1334,6 +1302,31 @@ _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b Py_ssize_t index = 0; for (Py_ssize_t i = 0; i < n_items; i++) { expr_ty item = asdl_seq_GET(expr, i); + + // This should correspond to a JoinedStr node of two elements + // created _PyPegen_formatted_value. This situation can only be the result of + // a f-string debug expression where the first element is a constant with the text and the second + // a formatted value with the expression. + if (item->kind == JoinedStr_kind) { + asdl_expr_seq *values = item->v.JoinedStr.values; + if (asdl_seq_LEN(values) != 2) { + PyErr_Format(PyExc_SystemError, + "unexpected JoinedStr node without debug data in f-string at line %d", + item->lineno); + return NULL; + } + + expr_ty first = asdl_seq_GET(values, 0); + assert(first->kind == Constant_kind); + asdl_seq_SET(seq, index++, first); + + expr_ty second = asdl_seq_GET(values, 1); + assert(second->kind == FormattedValue_kind); + asdl_seq_SET(seq, index++, second); + + continue; + } + if (item->kind == Constant_kind) { item = _PyPegen_decode_fstring_part(p, is_raw, item, b); if (item == NULL) { @@ -1352,7 +1345,7 @@ _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b } asdl_expr_seq *resized_exprs; - if (index != n_items) { + if (index != total_items) { resized_exprs = _Py_asdl_expr_seq_new(index, p->arena); if (resized_exprs == NULL) { return NULL;

1 0

[3.13] gh-128636: Fix crash in PyREPL when `os.environ` is overwritten with an invalid value (GH-128653) (#129186)
by pablogsal Jan. 22, 2025

Jan. 22, 2025

https://github.com/python/cpython/commit/0b90dc8e89465ef02bbea06ae3f237cb5a… commit: 0b90dc8e89465ef02bbea06ae3f237cb5a190e57 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: pablogsal <Pablogsal(a)gmail.com> date: 2025-01-22T16:38:13Z summary: [3.13] gh-128636: Fix crash in PyREPL when `os.environ` is overwritten with an invalid value (GH-128653) (#129186) gh-128636: Fix crash in PyREPL when `os.environ` is overwritten with an invalid value (GH-128653) (cherry picked from commit ba9a4b621577b92f36d88388cc9f791c2dc7d7ba) Co-authored-by: Tomas R <tomas.roun8(a)gmail.com> files: A Misc/NEWS.d/next/Library/2025-01-08-22-30-38.gh-issue-128636.jQfWXj.rst M Lib/_pyrepl/unix_console.py M Lib/test/test_pyrepl/test_unix_console.py diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 0f3468421bec949..e69c96b11598aaa 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -449,10 +449,12 @@ def getheightwidth(self): """ try: return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) - except KeyError: - height, width = struct.unpack( - "hhhh", ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8) - )[0:2] + except (KeyError, TypeError, ValueError): + try: + size = ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8) + except OSError: + return 25, 80 + height, width = struct.unpack("hhhh", size)[0:2] if not height: return 25, 80 return height, width @@ -468,7 +470,7 @@ def getheightwidth(self): """ try: return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) - except KeyError: + except (KeyError, TypeError, ValueError): return 25, 80 def forgetinput(self): diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index e3bbabcb0089fb2..15dbf48bcf0f1c4 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -1,7 +1,9 @@ import itertools +import os import sys import unittest from functools import partial +from test.support import os_helper from unittest import TestCase from unittest.mock import MagicMock, call, patch, ANY @@ -312,3 +314,14 @@ def same_console(events): ) console.restore() con.restore() + + def test_getheightwidth_with_invalid_environ(self, _os_write): + # gh-128636 + console = UnixConsole() + with os_helper.EnvironmentVarGuard() as env: + env["LINES"] = "" + self.assertIsInstance(console.getheightwidth(), tuple) + env["COLUMNS"] = "" + self.assertIsInstance(console.getheightwidth(), tuple) + os.environ = [] + self.assertIsInstance(console.getheightwidth(), tuple) diff --git a/Misc/NEWS.d/next/Library/2025-01-08-22-30-38.gh-issue-128636.jQfWXj.rst b/Misc/NEWS.d/next/Library/2025-01-08-22-30-38.gh-issue-128636.jQfWXj.rst new file mode 100644 index 000000000000000..80c9840b5855305 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-08-22-30-38.gh-issue-128636.jQfWXj.rst @@ -0,0 +1,2 @@ +Fix PyREPL failure when :data:`os.environ` is overwritten with an invalid +value.

1 0