Mailman 3 November 2023 - Python-checkins

gh-111972: Make Unicode name C APIcapsule initialization thread-safe (#112249)
by erlend-aasland 30 Nov '23

30 Nov '23

https://github.com/python/cpython/commit/0785c685599aaa052f85d6163872bdecb9… commit: 0785c685599aaa052f85d6163872bdecb9c66486 branch: main author: Kirill Podoprigora <kirill.bast9(a)mail.ru> committer: erlend-aasland <erlend.aasland(a)protonmail.com> date: 2023-11-30T11:12:49+01:00 summary: gh-111972: Make Unicode name C APIcapsule initialization thread-safe (#112249) files: M Include/internal/pycore_ucnhash.h M Objects/unicodeobject.c M Python/codecs.c diff --git a/Include/internal/pycore_ucnhash.h b/Include/internal/pycore_ucnhash.h index 187dd68e7347ff..1561dfbb3150d3 100644 --- a/Include/internal/pycore_ucnhash.h +++ b/Include/internal/pycore_ucnhash.h @@ -28,6 +28,8 @@ typedef struct { } _PyUnicode_Name_CAPI; +extern _PyUnicode_Name_CAPI* _PyUnicode_GetNameCAPI(void); + #ifdef __cplusplus } #endif diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index cffc06297a9aee..10022e23c04abf 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5869,6 +5869,23 @@ PyUnicode_AsUTF16String(PyObject *unicode) return _PyUnicode_EncodeUTF16(unicode, NULL, 0); } +_PyUnicode_Name_CAPI * +_PyUnicode_GetNameCAPI(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyUnicode_Name_CAPI *ucnhash_capi; + + ucnhash_capi = _Py_atomic_load_ptr(&interp->unicode.ucnhash_capi); + if (ucnhash_capi == NULL) { + ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( + PyUnicodeData_CAPSULE_NAME, 1); + + // It's fine if we overwite the value here. It's always the same value. + _Py_atomic_store_ptr(&interp->unicode.ucnhash_capi, ucnhash_capi); + } + return ucnhash_capi; +} + /* --- Unicode Escape Codec ----------------------------------------------- */ PyObject * @@ -5884,7 +5901,6 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, PyObject *errorHandler = NULL; PyObject *exc = NULL; _PyUnicode_Name_CAPI *ucnhash_capi; - PyInterpreterState *interp = _PyInterpreterState_GET(); // so we can remember if we've seen an invalid escape char or not *first_invalid_escape = NULL; @@ -6032,19 +6048,13 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, /* \N{name} */ case 'N': - ucnhash_capi = interp->unicode.ucnhash_capi; + ucnhash_capi = _PyUnicode_GetNameCAPI(); if (ucnhash_capi == NULL) { - /* load the unicode data module */ - ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( - PyUnicodeData_CAPSULE_NAME, 1); - if (ucnhash_capi == NULL) { - PyErr_SetString( + PyErr_SetString( PyExc_UnicodeError, "\\N escapes not supported (can't load unicodedata module)" - ); - goto onError; - } - interp->unicode.ucnhash_capi = ucnhash_capi; + ); + goto onError; } message = "malformed \\N character escape"; diff --git a/Python/codecs.c b/Python/codecs.c index 545bf82e00dca1..d8fe7b22063a80 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -931,8 +931,6 @@ PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) return Py_BuildValue("(Nn)", res, end); } -static _PyUnicode_Name_CAPI *ucnhash_capi = NULL; - PyObject *PyCodec_NameReplaceErrors(PyObject *exc) { if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) { @@ -953,13 +951,9 @@ PyObject *PyCodec_NameReplaceErrors(PyObject *exc) return NULL; if (!(object = PyUnicodeEncodeError_GetObject(exc))) return NULL; - if (!ucnhash_capi) { - /* load the unicode data module */ - ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( - PyUnicodeData_CAPSULE_NAME, 1); - if (!ucnhash_capi) { - return NULL; - } + _PyUnicode_Name_CAPI *ucnhash_capi = _PyUnicode_GetNameCAPI(); + if (ucnhash_capi == NULL) { + return NULL; } for (i = start, ressize = 0; i < end; ++i) { /* object is guaranteed to be "ready" */

1 0

[3.12] IDLE: fix config_key htest (GH-112545) (#112546)
by terryjreedy 30 Nov '23

30 Nov '23

https://github.com/python/cpython/commit/46adf6b701c440e047abf925df9a75aae5… commit: 46adf6b701c440e047abf925df9a75aae50233bf branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: terryjreedy <tjreedy(a)udel.edu> date: 2023-11-30T03:29:59-05:00 summary: [3.12] IDLE: fix config_key htest (GH-112545) (#112546) Change 'Dialog' to 'Window' in two places to match the name of the config_key class being tested. (cherry picked from commit 81261fa67ff82b03c255733b0d1abbbb8a228187) Co-authored-by: Terry Jan Reedy <tjreedy(a)udel.edu> files: M Lib/idlelib/config_key.py M Lib/idlelib/idle_test/htest.py diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index bb07231cd590b..e5f67e8d4069e 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -351,4 +351,4 @@ def cancel(self, event=None): main('idlelib.idle_test.test_config_key', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(GetKeysDialog) + run(GetKeysWindow) diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index d297f8aa0094e..595e51d3699f6 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -152,7 +152,7 @@ def _wrapper(parent): # htest # "Best to close editor first." } -GetKeysDialog_spec = { +GetKeysWindow_spec = { 'file': 'config_key', 'kwds': {'title': 'Test keybindings', 'action': 'find-again',

1 0

[3.11] IDLE: fix config_key htest (GH-112545) (#112547)
by terryjreedy 30 Nov '23

30 Nov '23

https://github.com/python/cpython/commit/03ac9e6ee1aacb1ab42788c40e8465a6e5… commit: 03ac9e6ee1aacb1ab42788c40e8465a6e5c46532 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: terryjreedy <tjreedy(a)udel.edu> date: 2023-11-30T03:29:23-05:00 summary: [3.11] IDLE: fix config_key htest (GH-112545) (#112547) Change 'Dialog' to 'Window' in two places to match the name of the config_key class being tested. (cherry picked from commit 81261fa67ff82b03c255733b0d1abbbb8a228187) Co-authored-by: Terry Jan Reedy <tjreedy(a)udel.edu> files: M Lib/idlelib/config_key.py M Lib/idlelib/idle_test/htest.py diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index bb07231cd590b..e5f67e8d4069e 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -351,4 +351,4 @@ def cancel(self, event=None): main('idlelib.idle_test.test_config_key', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(GetKeysDialog) + run(GetKeysWindow) diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index d297f8aa0094e..595e51d3699f6 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -152,7 +152,7 @@ def _wrapper(parent): # htest # "Best to close editor first." } -GetKeysDialog_spec = { +GetKeysWindow_spec = { 'file': 'config_key', 'kwds': {'title': 'Test keybindings', 'action': 'find-again',

1 0

IDLE: fix config_key htest (#112545)
by terryjreedy 30 Nov '23

30 Nov '23

https://github.com/python/cpython/commit/81261fa67ff82b03c255733b0d1abbbb8a… commit: 81261fa67ff82b03c255733b0d1abbbb8a228187 branch: main author: Terry Jan Reedy <tjreedy(a)udel.edu> committer: terryjreedy <tjreedy(a)udel.edu> date: 2023-11-30T07:08:44Z summary: IDLE: fix config_key htest (#112545) Change 'Dialog' to 'Window' in two places to match the name of the config_key class being tested. files: M Lib/idlelib/config_key.py M Lib/idlelib/idle_test/htest.py diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index bb07231cd590b6..e5f67e8d4069ee 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -351,4 +351,4 @@ def cancel(self, event=None): main('idlelib.idle_test.test_config_key', verbosity=2, exit=False) from idlelib.idle_test.htest import run - run(GetKeysDialog) + run(GetKeysWindow) diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index d297f8aa0094ee..595e51d3699f6e 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -152,7 +152,7 @@ def _wrapper(parent): # htest # "Best to close editor first." } -GetKeysDialog_spec = { +GetKeysWindow_spec = { 'file': 'config_key', 'kwds': {'title': 'Test keybindings', 'action': 'find-again',

1 0

GH-103065, GH-106704, GH-105253: Provide a `Tools/wasm/wasi.py` script to simplify doing a WASI build (GH-112473)
by brettcannon 30 Nov '23

30 Nov '23

https://github.com/python/cpython/commit/37589d76bbe97b0bf13ffafb8dd6aab361… commit: 37589d76bbe97b0bf13ffafb8dd6aab361a0209a branch: main author: Brett Cannon <brett(a)python.org> committer: brettcannon <brett(a)python.org> date: 2023-11-29T16:18:25-08:00 summary: GH-103065, GH-106704, GH-105253: Provide a `Tools/wasm/wasi.py` script to simplify doing a WASI build (GH-112473) files: A Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst A Tools/wasm/wasi.py M .devcontainer/Dockerfile M .gitignore M Makefile.pre.in M Tools/wasm/README.md M Tools/wasm/mypy.ini diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 590d7834b2b8be..9f808af38e69df 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -6,7 +6,7 @@ ENV WASI_SDK_VERSION=20 ENV WASI_SDK_PATH=/opt/wasi-sdk ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=9.0.1 +ENV WASMTIME_VERSION=14.0.4 ENV WASMTIME_CPU_ARCH=x86_64 RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ diff --git a/.gitignore b/.gitignore index 8c8273fc7a3aa3..c424a894c2a6e0 100644 --- a/.gitignore +++ b/.gitignore @@ -125,6 +125,7 @@ Tools/unicode/data/ /config.status.lineno # hendrikmuhs/ccache-action@v1 /.ccache +/cross-build/ /platform /profile-clean-stamp /profile-run-stamp diff --git a/Makefile.pre.in b/Makefile.pre.in index 3d766425abba34..e7f8abce43d648 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2732,6 +2732,7 @@ clobber: clean -rm -rf build platform -rm -rf $(PYTHONFRAMEWORKDIR) -rm -f python-config.py python-config + -rm -rf cross-build # Make things extra clean, before making a distribution: # remove all generated files, even Makefile[.pre] diff --git a/Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst b/Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst new file mode 100644 index 00000000000000..e2240b7c656a2f --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-11-27-13-55-47.gh-issue-103065.o72OiA.rst @@ -0,0 +1 @@ +Introduce ``Tools/wasm/wasi.py`` to simplify doing a WASI build. diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 8ef63c6dcd9ddc..acbece214ed9bd 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -298,100 +298,66 @@ AddType application/wasm wasm ## WASI (wasm32-wasi) -WASI builds require the [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+. -See `.devcontainer/Dockerfile` for an example of how to download and -install the WASI SDK. +**NOTE**: The instructions below assume a Unix-based OS due to cross-compilation for CPython being set up for `./configure`. -### Build +### Prerequisites + +Developing for WASI requires two things to be installed: + +1. The [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+ + (see `.devcontainer/Dockerfile` for an example of how to download and install the WASI SDK) +2. A WASI host/runtime ([wasmtime](https://wasmtime.dev) 14+ is recommended and what the instructions below assume) -The script ``wasi-env`` sets necessary compiler and linker flags as well as -``pkg-config`` overrides. The script assumes that WASI-SDK is installed in -``/opt/wasi-sdk`` or ``$WASI_SDK_PATH``. -There are two scripts you can use to do a WASI build from a source checkout. You can either use: +### Building +Building for WASI requires doing a cross-build where you have a "build" Python to help produce a WASI build of CPython (technically it's a "host x host" cross-build because the build Python is also the target Python while the host build is the WASI build; yes, it's confusing terminology). In the end you should have a build Python in `cross-build/build` and a WASI build in `cross-build/wasm32-wasi`. + +The easiest way to do a build is to use the `wasi.py` script. You can either have it perform the entire build process from start to finish in one step, or you can do it in discrete steps that mirror running `configure` and `make` for each of the two builds of Python you end up producing (which are beneficial when you only need to do a specific step after getting a complete build, e.g. editing some code and you just need to run `make` for the WASI build). + +The discrete steps are: ```shell -./Tools/wasm/wasm_build.py wasi build +python Tools/wasm/wasi.py configure-build-python +python Tools/wasm/wasi.py make-build-python +python Tools/wasm/wasi.py configure-host +python Tools/wasm/wasi.py make-host ``` -or: +To do it in a single command, run: ```shell -./Tools/wasm/build_wasi.sh +python Tools/wasm/wasi.py build ``` -The commands are equivalent to the following steps: - -- Make sure `Modules/Setup.local` exists -- Make sure the necessary build tools are installed: - - [WASI SDK](https://github.com/WebAssembly/wasi-sdk) (which ships with `clang`) - - `make` - - `pkg-config` (on Linux) -- Create the build Python - - `mkdir -p builddir/build` - - `pushd builddir/build` - - Get the build platform - - Python: `sysconfig.get_config_var("BUILD_GNU_TYPE")` - - Shell: `../../config.guess` - - `../../configure -C` - - `make all` - - ```PYTHON_VERSION=`./python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'` ``` - - `popd` -- Create the host/WASI Python - - `mkdir builddir/wasi` - - `pushd builddir/wasi` - - `../../Tools/wasm/wasi-env ../../configure -C --host=wasm32-unknown-wasi --build=$(../../config.guess) --with-build-python=../build/python` - - `CONFIG_SITE=../../Tools/wasm/config.site-wasm32-wasi` - - `HOSTRUNNER="wasmtime run --mapdir /::$(dirname $(dirname $(pwd))) --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION $(pwd)/python.wasm --"` - - Maps the source checkout to `/` in the WASI runtime - - Stdlib gets loaded from `/Lib` - - Gets `_sysconfigdata__wasi_wasm32-wasi.py` on to `sys.path` via `PYTHONPATH` - - Set by `wasi-env` - - `WASI_SDK_PATH` - - `WASI_SYSROOT` - - `CC` - - `CPP` - - `CXX` - - `LDSHARED` - - `AR` - - `RANLIB` - - `CFLAGS` - - `LDFLAGS` - - `PKG_CONFIG_PATH` - - `PKG_CONFIG_LIBDIR` - - `PKG_CONFIG_SYSROOT_DIR` - - `PATH` - - `make all` - +That will: -### Running - -If you followed the instructions above, you can run the interpreter via e.g., `wasmtime` from within the `Tools/wasi` directory (make sure to set/change `$PYTHON_VERSION` and do note the paths are relative to running in`builddir/wasi` for simplicity only): +1. Run `configure` for the build Python (same as `wasi.py configure-build-python`) +2. Run `make` for the build Python (`wasi.py make-build-python`) +3. Run `configure` for the WASI build (`wasi.py configure-host`) +4. Run `make` for the WASI build (`wasi.py make-host`) +See the `--help` for the various options available for each of the subcommands which controls things like the location of the WASI SDK, the command to use with the WASI host/runtime, etc. Also note that you can use `--` as a separtor for any of the `configure`-related commands -- including `build` -- to pass arguments to `configure` itself. For example, if you want a pydebug build that also caches the results from `configure`, you can do: ```shell -wasmtime run --mapdir /::../.. --env PYTHONPATH=/builddir/wasi/build/lib.wasi-wasm32-$PYTHON_VERSION python.wasm -- <args> +python Tools/wasm/wasi.py build -- -C --with-pydebug ``` -There are also helpers provided by `Tools/wasm/wasm_build.py` as listed below. Also, if you used `Tools/wasm/build_wasi.sh`, a `run_wasi.sh` file will be created in `builddir/wasi` which will run the above command for you (it also uses absolute paths, so it can be executed from anywhere). - -#### REPL - +The `wasi.py` script is able to infer details from the build Python, and so you only technically need to specify `--with-pydebug` once for `configure-build-python` and `configure-host` will detect its use if you use the discrete steps: ```shell -./Tools/wasm/wasm_build.py wasi repl +python Tools/wasm/wasi.py configure-build-python -- -C --with-pydebug +python Tools/wasm/wasi.py make-build-python +python Tools/wasm/wasi.py configure-host -- -C +python Tools/wasm/wasi.py make-host ``` -#### Tests +### Running + +If you used `wasi.py` to do your build then there will be a `cross-build/wasm32-wasi/python.sh` file which you can use to run the `python.wasm` file (see the output from the `configure-host` subcommand): ```shell -./Tools/wasm/wasm_build.py wasi test +cross-build/wasm32-wasi/python.sh --version ``` -### Debugging +While you _can_ run `python.wasm` directly, Python will fail to start up without certain things being set (e.g. `PYTHONPATH` for `sysconfig` data). As such, the `python.sh` file records these details for you. -* ``wasmtime run -g`` generates debugging symbols for gdb and lldb. The - feature is currently broken, see - https://github.com/bytecodealliance/wasmtime/issues/4669 . -* The environment variable ``RUST_LOG=wasi_common`` enables debug and - trace logging. ## Detect WebAssembly builds @@ -402,15 +368,17 @@ import os, sys if sys.platform == "emscripten": # Python on Emscripten + ... if sys.platform == "wasi": # Python on WASI + ... if os.name == "posix": # WASM platforms identify as POSIX-like. # Windows does not provide os.uname(). machine = os.uname().machine if machine.startswith("wasm"): - # WebAssembly (wasm32, wasm64 in the future) + # WebAssembly (wasm32, wasm64 potentially in the future) ``` ```python diff --git a/Tools/wasm/mypy.ini b/Tools/wasm/mypy.ini index c62598f89eba69..4de0a30c260f5f 100644 --- a/Tools/wasm/mypy.ini +++ b/Tools/wasm/mypy.ini @@ -1,5 +1,5 @@ [mypy] -files = Tools/wasm +files = Tools/wasm/wasm_*.py pretty = True show_traceback = True @@ -9,6 +9,3 @@ python_version = 3.8 # Be strict... strict = True enable_error_code = truthy-bool,ignore-without-code - -# except for incomplete defs, which are useful for module authors: -disallow_incomplete_defs = False diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py new file mode 100644 index 00000000000000..34c0e9375e24c8 --- /dev/null +++ b/Tools/wasm/wasi.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 + +import argparse +import contextlib +import functools +import os +try: + from os import process_cpu_count as cpu_count +except ImportError: + from os import cpu_count +import pathlib +import shutil +import subprocess +import sys +import sysconfig +import tempfile + + +CHECKOUT = pathlib.Path(__file__).parent.parent.parent +CROSS_BUILD_DIR = CHECKOUT / "cross-build" +BUILD_DIR = CROSS_BUILD_DIR / "build" +HOST_TRIPLE = "wasm32-wasi" +HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE + + +def updated_env(updates={}): + """Create a new dict representing the environment to use. + + The changes made to the execution environment are printed out. + """ + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | os.environ | updates + + env_diff = {} + for key, value in environment.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + print("🌎 Environment changes:") + for key in sorted(env_diff.keys()): + print(f" {key}={env_diff[key]}") + + return environment + + +def subdir(working_dir, *, clean_ok=False): + """Decorator to change to a working directory.""" + def decorator(func): + @functools.wraps(func) + def wrapper(context): + try: + tput_output = subprocess.check_output(["tput", "cols"], + encoding="utf-8") + terminal_width = int(tput_output.strip()) + except subprocess.CalledProcessError: + terminal_width = 80 + print("⎯" * terminal_width) + print("📁", working_dir) + if clean_ok and context.clean and working_dir.exists(): + print(f"🚮 Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator + + +def call(command, *, quiet, **kwargs): + """Execute a command. + + If 'quiet' is true, then redirect stdout and stderr to a temporary file. + """ + print("❯", " ".join(map(str, command))) + if not quiet: + stdout = None + stderr = None + else: + stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8", + delete=False, + prefix="cpython-wasi-", + suffix=".log") + stderr = subprocess.STDOUT + print(f"📝 Logging output to {stdout.name} (--quiet)...") + + subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) + + +def build_platform(): + """The name of the build/host platform.""" + # Can also be found via `config.guess`.` + return sysconfig.get_config_var("BUILD_GNU_TYPE") + + +def build_python_path(): + """The path to the build Python binary.""" + binary = BUILD_DIR / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError("Unable to find `python(.exe)` in " + f"{BUILD_DIR}") + + return binary + + +@subdir(BUILD_DIR, clean_ok=True) +def configure_build_python(context, working_dir): + """Configure the build/host Python.""" + local_setup = CHECKOUT / "Modules" / "Setup.local" + if local_setup.exists(): + print(f"👍 {local_setup} exists ...") + else: + print(f"📝 Touching {local_setup} ...") + local_setup.touch() + + configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] + if context.args: + configure.extend(context.args) + + call(configure, quiet=context.quiet) + + +@subdir(BUILD_DIR) +def make_build_python(context, working_dir): + """Make/build the build Python.""" + call(["make", "--jobs", str(cpu_count()), "all"], + quiet=context.quiet) + + binary = build_python_path() + cmd = [binary, "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + + print(f"🎉 {binary} {version}") + + +def find_wasi_sdk(): + """Find the path to wasi-sdk.""" + if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"): + return pathlib.Path(wasi_sdk_path) + elif (default_path := pathlib.Path("/opt/wasi-sdk")).exists(): + return default_path + + +def wasi_sdk_env(context): + """Calculate environment variables for building with wasi-sdk.""" + wasi_sdk_path = context.wasi_sdk_path + sysroot = wasi_sdk_path / "share" / "wasi-sysroot" + env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", + "LDSHARED": "wasm-ld", "AR": "llvm-ar", "RANLIB": "ranlib"} + + for env_var, binary_name in list(env.items()): + env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) + + if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"): + for compiler in ["CC", "CPP", "CXX"]: + env[compiler] += f" --sysroot={sysroot}" + + env["PKG_CONFIG_PATH"] = "" + env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( + map(os.fsdecode, + [sysroot / "lib" / "pkgconfig", + sysroot / "share" / "pkgconfig"])) + env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) + + env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) + env["WASI_SYSROOT"] = os.fsdecode(sysroot) + + env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"), + os.environ["PATH"]]) + + return env + + +@subdir(HOST_DIR, clean_ok=True) +def configure_wasi_python(context, working_dir): + """Configure the WASI/host build.""" + if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): + raise ValueError("WASI-SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk") + + config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") + + wasi_build_dir = working_dir.relative_to(CHECKOUT) + + python_build_dir = BUILD_DIR / "build" + lib_dirs = list(python_build_dir.glob("lib.*")) + assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" + lib_dir = os.fsdecode(lib_dirs[0]) + pydebug = lib_dir.endswith("-pydebug") + python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] + sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" + if pydebug: + sysconfig_data += "-pydebug" + + # Use PYTHONPATH to include sysconfig data which must be anchored to the + # WASI guest's `/` directory. + host_runner = context.host_runner.format(GUEST_DIR="/", + HOST_DIR=CHECKOUT, + ENV_VAR_NAME="PYTHONPATH", + ENV_VAR_VALUE=f"/{sysconfig_data}", + PYTHON_WASM=working_dir / "python.wasm") + env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + build_python = os.fsdecode(build_python_path()) + # The path to `configure` MUST be relative, else `python.wasm` is unable + # to find the stdlib due to Python not recognizing that it's being + # executed from within a checkout. + configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), + f"--host={HOST_TRIPLE}", + f"--build={build_platform()}", + f"--with-build-python={build_python}"] + if pydebug: + configure.append("--with-pydebug") + if context.args: + configure.extend(context.args) + call(configure, + env=updated_env(env_additions | wasi_sdk_env(context)), + quiet=context.quiet) + + exec_script = working_dir / "python.sh" + with exec_script.open("w", encoding="utf-8") as file: + file.write(f'#!/bin/sh\nexec {host_runner} "$@"\n') + exec_script.chmod(0o755) + print(f"🏃‍♀️ Created {exec_script} ... ") + sys.stdout.flush() + + +@subdir(HOST_DIR) +def make_wasi_python(context, working_dir): + """Run `make` for the WASI/host build.""" + call(["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + quiet=context.quiet) + + exec_script = working_dir / "python.sh" + subprocess.check_call([exec_script, "--version"]) + + +def build_all(context): + """Build everything.""" + steps = [configure_build_python, make_build_python, configure_wasi_python, + make_wasi_python] + for step in steps: + step(context) + + +def main(): + default_host_runner = (f"{shutil.which('wasmtime')} run " + # Make sure the stack size will work for a pydebug + # build. + # The 8388608 value comes from `ulimit -s` under Linux + # which equates to 8291 KiB. + "--wasm max-wasm-stack=8388608 " + # Enable thread support. + "--wasm threads=y --wasi threads=y " + # Map the checkout to / to load the stdlib from /Lib. + "--dir {HOST_DIR}::{GUEST_DIR} " + # Set PYTHONPATH to the sysconfig data. + "--env {ENV_VAR_NAME}={ENV_VAR_VALUE} " + # Path to the WASM binary. + "{PYTHON_WASM}") + + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser("configure-build-python", + help="Run `configure` for the " + "build Python") + make_build = subcommands.add_parser("make-build-python", + help="Run `make` for the build Python") + configure_host = subcommands.add_parser("configure-host", + help="Run `configure` for the " + "host/WASI (pydebug builds " + "are inferred from the build " + "Python)") + make_host = subcommands.add_parser("make-host", + help="Run `make` for the host/WASI") + for subcommand in build, configure_build, make_build, configure_host, make_host: + subcommand.add_argument("--quiet", action="store_true", default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file") + for subcommand in build, configure_build, configure_host: + subcommand.add_argument("--clean", action="store_true", default=False, + dest="clean", + help="Delete any relevant directories before building") + for subcommand in build, configure_build, configure_host: + subcommand.add_argument("args", nargs="*", + help="Extra arguments to pass to `configure`") + for subcommand in build, configure_host: + subcommand.add_argument("--wasi-sdk", type=pathlib.Path, + dest="wasi_sdk_path", + default=find_wasi_sdk(), + help="Path to wasi-sdk; defaults to " + "$WASI_SDK_PATH or /opt/wasi-sdk") + subcommand.add_argument("--host-runner", action="store", + default=default_host_runner, dest="host_runner", + help="Command template for running the WebAssembly " + "code (default meant for wasmtime 14 or newer: " + f"`{default_host_runner}`)") + + context = parser.parse_args() + + dispatch = {"configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "configure-host": configure_wasi_python, + "make-host": make_wasi_python, + "build": build_all} + dispatch[context.subcommand](context) + + +if __name__ == "__main__": + main()

1 0

[3.12] [Enum] update class creation for RuntimeError changes (GH-111815) (GH-112526)
by ethanfurman 29 Nov '23

29 Nov '23

https://github.com/python/cpython/commit/749c8fdafbc924a2b37637d9a28b8c4acc… commit: 749c8fdafbc924a2b37637d9a28b8c4accfdd1ae branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: ethanfurman <ethan(a)stoneleaf.us> date: 2023-11-29T13:49:52-08:00 summary: [3.12] [Enum] update class creation for RuntimeError changes (GH-111815) (GH-112526) (cherry picked from commit f9e6ce03953e9ee988d55324dc715b0ef2303cfb) Co-authored-by: Ethan Furman <ethan(a)stoneleaf.us> files: M Doc/howto/enum.rst M Lib/enum.py diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index a136c76303c8ef..ffdafb749c73a9 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -1439,7 +1439,6 @@ alias:: Traceback (most recent call last): ... ValueError: aliases not allowed in DuplicateFreeEnum: 'GRENE' --> 'GREEN' - Error calling __set_name__ on '_proto_member' instance 'GRENE' in 'Color' .. note:: diff --git a/Lib/enum.py b/Lib/enum.py index 4bd3756ec5bc0c..1502bfe9158520 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -581,12 +581,16 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k try: exc = None enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) - except RuntimeError as e: - # any exceptions raised by member.__new__ will get converted to a - # RuntimeError, so get that original exception back and raise it instead - exc = e.__cause__ or e + except Exception as e: + # since 3.12 the line "Error calling __set_name__ on '_proto_member' instance ..." + # is tacked on to the error instead of raising a RuntimeError + # recreate the exception to discard + exc = type(e)(str(e)) + exc.__cause__ = e.__cause__ + exc.__context__ = e.__context__ + tb = e.__traceback__ if exc is not None: - raise exc + raise exc.with_traceback(tb) # # update classdict with any changes made by __init_subclass__ classdict.update(enum_class.__dict__)

1 0

[3.12] Add more C API tests (GH-112522) (GH-112525)
by serhiy-storchaka 29 Nov '23

29 Nov '23

https://github.com/python/cpython/commit/8f43250c9167a191799e4c570cb3474f9b… commit: 8f43250c9167a191799e4c570cb3474f9b4c5a5b branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2023-11-29T22:02:57+02:00 summary: [3.12] Add more C API tests (GH-112522) (GH-112525) Add tests for PyObject_Str(), PyObject_Repr(), PyObject_ASCII() and PyObject_Bytes(). (cherry picked from commit e0449b9a7fffc0c0eed806bf4cbb8f1f65397bbb) Co-authored-by: Serhiy Storchaka <storchaka(a)gmail.com> files: M Lib/test/test_capi/test_abstract.py M Modules/_testcapi/abstract.c diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py index d01ee66ed4ff2..2fee6811260db 100644 --- a/Lib/test/test_capi/test_abstract.py +++ b/Lib/test/test_capi/test_abstract.py @@ -8,6 +8,30 @@ NULL = None +class StrSubclass(str): + pass + +class BytesSubclass(bytes): + pass + +class WithStr: + def __init__(self, value): + self.value = value + def __str__(self): + return self.value + +class WithRepr: + def __init__(self, value): + self.value = value + def __repr__(self): + return self.value + +class WithBytes: + def __init__(self, value): + self.value = value + def __bytes__(self): + return self.value + class TestObject: @property def evil(self): @@ -44,6 +68,68 @@ def gen(): class CAPITest(unittest.TestCase): + def assertTypedEqual(self, actual, expected): + self.assertIs(type(actual), type(expected)) + self.assertEqual(actual, expected) + + def test_object_str(self): + # Test PyObject_Str() + object_str = _testcapi.object_str + self.assertTypedEqual(object_str(''), '') + self.assertTypedEqual(object_str('abc'), 'abc') + self.assertTypedEqual(object_str('\U0001f40d'), '\U0001f40d') + self.assertTypedEqual(object_str(StrSubclass('abc')), 'abc') + self.assertTypedEqual(object_str(WithStr('abc')), 'abc') + self.assertTypedEqual(object_str(WithStr(StrSubclass('abc'))), StrSubclass('abc')) + self.assertTypedEqual(object_str(WithRepr('<abc>')), '<abc>') + self.assertTypedEqual(object_str(WithRepr(StrSubclass('<abc>'))), StrSubclass('<abc>')) + self.assertTypedEqual(object_str(NULL), '<NULL>') + + def test_object_repr(self): + # Test PyObject_Repr() + object_repr = _testcapi.object_repr + self.assertTypedEqual(object_repr(''), "''") + self.assertTypedEqual(object_repr('abc'), "'abc'") + self.assertTypedEqual(object_repr('\U0001f40d'), "'\U0001f40d'") + self.assertTypedEqual(object_repr(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(object_repr(WithRepr('<abc>')), '<abc>') + self.assertTypedEqual(object_repr(WithRepr(StrSubclass('<abc>'))), StrSubclass('<abc>')) + self.assertTypedEqual(object_repr(WithRepr('<\U0001f40d>')), '<\U0001f40d>') + self.assertTypedEqual(object_repr(WithRepr(StrSubclass('<\U0001f40d>'))), StrSubclass('<\U0001f40d>')) + self.assertTypedEqual(object_repr(NULL), '<NULL>') + + def test_object_ascii(self): + # Test PyObject_ASCII() + object_ascii = _testcapi.object_ascii + self.assertTypedEqual(object_ascii(''), "''") + self.assertTypedEqual(object_ascii('abc'), "'abc'") + self.assertTypedEqual(object_ascii('\U0001f40d'), r"'\U0001f40d'") + self.assertTypedEqual(object_ascii(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(object_ascii(WithRepr('<abc>')), '<abc>') + self.assertTypedEqual(object_ascii(WithRepr(StrSubclass('<abc>'))), StrSubclass('<abc>')) + self.assertTypedEqual(object_ascii(WithRepr('<\U0001f40d>')), r'<\U0001f40d>') + self.assertTypedEqual(object_ascii(WithRepr(StrSubclass('<\U0001f40d>'))), r'<\U0001f40d>') + self.assertTypedEqual(object_ascii(NULL), '<NULL>') + + def test_object_bytes(self): + # Test PyObject_Bytes() + object_bytes = _testcapi.object_bytes + self.assertTypedEqual(object_bytes(b''), b'') + self.assertTypedEqual(object_bytes(b'abc'), b'abc') + self.assertTypedEqual(object_bytes(BytesSubclass(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(WithBytes(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(WithBytes(BytesSubclass(b'abc'))), BytesSubclass(b'abc')) + self.assertTypedEqual(object_bytes(bytearray(b'abc')), b'abc') + self.assertTypedEqual(object_bytes(memoryview(b'abc')), b'abc') + self.assertTypedEqual(object_bytes([97, 98, 99]), b'abc') + self.assertTypedEqual(object_bytes((97, 98, 99)), b'abc') + self.assertTypedEqual(object_bytes(iter([97, 98, 99])), b'abc') + self.assertRaises(TypeError, object_bytes, WithBytes(bytearray(b'abc'))) + self.assertRaises(TypeError, object_bytes, WithBytes([97, 98, 99])) + self.assertRaises(TypeError, object_bytes, 3) + self.assertRaises(TypeError, object_bytes, 'abc') + self.assertRaises(TypeError, object_bytes, object()) + self.assertTypedEqual(object_bytes(NULL), b'<NULL>') def test_object_getattr(self): xgetattr = _testcapi.object_getattr diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c index 1c4c45c6c93d3..56209c06448ea 100644 --- a/Modules/_testcapi/abstract.c +++ b/Modules/_testcapi/abstract.c @@ -3,6 +3,34 @@ #include "util.h" +static PyObject * +object_repr(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Repr(arg); +} + +static PyObject * +object_ascii(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_ASCII(arg); +} + +static PyObject * +object_str(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Str(arg); +} + +static PyObject * +object_bytes(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Bytes(arg); +} + static PyObject * object_getattr(PyObject *self, PyObject *args) { @@ -459,6 +487,11 @@ sequence_tuple(PyObject *self, PyObject *obj) static PyMethodDef test_methods[] = { + {"object_repr", object_repr, METH_O}, + {"object_ascii", object_ascii, METH_O}, + {"object_str", object_str, METH_O}, + {"object_bytes", object_bytes, METH_O}, + {"object_getattr", object_getattr, METH_VARARGS}, {"object_getattrstring", object_getattrstring, METH_VARARGS}, {"object_hasattr", object_hasattr, METH_VARARGS},

1 0

[3.12] gh-112509: Fix keys being present in both required_keys and optional_keys in TypedDict (GH-112512) (#112530)
by JelleZijlstra 29 Nov '23

29 Nov '23

https://github.com/python/cpython/commit/c67812668f228b58f35a46023981c6d22f… commit: c67812668f228b58f35a46023981c6d22fb3b5e9 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2023-11-29T17:55:06Z summary: [3.12] gh-112509: Fix keys being present in both required_keys and optional_keys in TypedDict (GH-112512) (#112530) (cherry picked from commit 403886942376210662610627b01fea6acd77d331) Co-authored-by: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> Co-authored-by: Alex Waygood <Alex.Waygood(a)Gmail.com> files: A Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst M Lib/test/test_typing.py M Lib/typing.py diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index a26e80995fef3..43479e5b53e13 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7509,6 +7509,46 @@ class Cat(Animal): 'voice': str, }) + def test_keys_inheritance_with_same_name(self): + class NotTotal(TypedDict, total=False): + a: int + + class Total(NotTotal): + a: int + + self.assertEqual(NotTotal.__required_keys__, frozenset()) + self.assertEqual(NotTotal.__optional_keys__, frozenset(['a'])) + self.assertEqual(Total.__required_keys__, frozenset(['a'])) + self.assertEqual(Total.__optional_keys__, frozenset()) + + class Base(TypedDict): + a: NotRequired[int] + b: Required[int] + + class Child(Base): + a: Required[int] + b: NotRequired[int] + + self.assertEqual(Base.__required_keys__, frozenset(['b'])) + self.assertEqual(Base.__optional_keys__, frozenset(['a'])) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset(['b'])) + + def test_multiple_inheritance_with_same_key(self): + class Base1(TypedDict): + a: NotRequired[int] + + class Base2(TypedDict): + a: Required[str] + + class Child(Base1, Base2): + pass + + # Last base wins + self.assertEqual(Child.__annotations__, {'a': Required[str]}) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset()) + def test_required_notrequired_keys(self): self.assertEqual(NontotalMovie.__required_keys__, frozenset({"title"})) diff --git a/Lib/typing.py b/Lib/typing.py index 1624300264062..32f0f224827b9 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2846,8 +2846,14 @@ def __new__(cls, name, bases, ns, total=True): for base in bases: annotations.update(base.__dict__.get('__annotations__', {})) - required_keys.update(base.__dict__.get('__required_keys__', ())) - optional_keys.update(base.__dict__.get('__optional_keys__', ())) + + base_required = base.__dict__.get('__required_keys__', set()) + required_keys |= base_required + optional_keys -= base_required + + base_optional = base.__dict__.get('__optional_keys__', set()) + required_keys -= base_optional + optional_keys |= base_optional annotations.update(own_annotations) for annotation_key, annotation_type in own_annotations.items(): @@ -2859,14 +2865,23 @@ def __new__(cls, name, bases, ns, total=True): annotation_origin = get_origin(annotation_type) if annotation_origin is Required: - required_keys.add(annotation_key) + is_required = True elif annotation_origin is NotRequired: - optional_keys.add(annotation_key) - elif total: + is_required = False + else: + is_required = total + + if is_required: required_keys.add(annotation_key) + optional_keys.discard(annotation_key) else: optional_keys.add(annotation_key) + required_keys.discard(annotation_key) + assert required_keys.isdisjoint(optional_keys), ( + f"Required keys overlap with optional keys in {name}:" + f" {required_keys=}, {optional_keys=}" + ) tp_dict.__annotations__ = annotations tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) diff --git a/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst b/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst new file mode 100644 index 0000000000000..a16d67e7776bc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst @@ -0,0 +1,3 @@ +Fix edge cases that could cause a key to be present in both the +``__required_keys__`` and ``__optional_keys__`` attributes of a +:class:`typing.TypedDict`. Patch by Jelle Zijlstra.

1 0

[3.11] gh-112509: Fix keys being present in both required_keys and optional_keys in TypedDict (GH-112512) (#112531)
by JelleZijlstra 29 Nov '23

29 Nov '23

https://github.com/python/cpython/commit/f6edb83fe97f9e0a20a56e18a11a688229… commit: f6edb83fe97f9e0a20a56e18a11a688229d52c76 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2023-11-29T17:51:00Z summary: [3.11] gh-112509: Fix keys being present in both required_keys and optional_keys in TypedDict (GH-112512) (#112531) (cherry picked from commit 403886942376210662610627b01fea6acd77d331) Co-authored-by: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> Co-authored-by: Alex Waygood <Alex.Waygood(a)Gmail.com> files: A Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst M Lib/test/test_typing.py M Lib/typing.py diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 12f78b580effa..f663fbd8679e0 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -6805,6 +6805,46 @@ class Cat(Animal): 'voice': str, }) + def test_keys_inheritance_with_same_name(self): + class NotTotal(TypedDict, total=False): + a: int + + class Total(NotTotal): + a: int + + self.assertEqual(NotTotal.__required_keys__, frozenset()) + self.assertEqual(NotTotal.__optional_keys__, frozenset(['a'])) + self.assertEqual(Total.__required_keys__, frozenset(['a'])) + self.assertEqual(Total.__optional_keys__, frozenset()) + + class Base(TypedDict): + a: NotRequired[int] + b: Required[int] + + class Child(Base): + a: Required[int] + b: NotRequired[int] + + self.assertEqual(Base.__required_keys__, frozenset(['b'])) + self.assertEqual(Base.__optional_keys__, frozenset(['a'])) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset(['b'])) + + def test_multiple_inheritance_with_same_key(self): + class Base1(TypedDict): + a: NotRequired[int] + + class Base2(TypedDict): + a: Required[str] + + class Child(Base1, Base2): + pass + + # Last base wins + self.assertEqual(Child.__annotations__, {'a': Required[str]}) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset()) + def test_required_notrequired_keys(self): self.assertEqual(NontotalMovie.__required_keys__, frozenset({"title"})) diff --git a/Lib/typing.py b/Lib/typing.py index fa812f9bd2355..95f855b5a8845 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2987,8 +2987,14 @@ def __new__(cls, name, bases, ns, total=True): for base in bases: annotations.update(base.__dict__.get('__annotations__', {})) - required_keys.update(base.__dict__.get('__required_keys__', ())) - optional_keys.update(base.__dict__.get('__optional_keys__', ())) + + base_required = base.__dict__.get('__required_keys__', set()) + required_keys |= base_required + optional_keys -= base_required + + base_optional = base.__dict__.get('__optional_keys__', set()) + required_keys -= base_optional + optional_keys |= base_optional annotations.update(own_annotations) for annotation_key, annotation_type in own_annotations.items(): @@ -3000,14 +3006,23 @@ def __new__(cls, name, bases, ns, total=True): annotation_origin = get_origin(annotation_type) if annotation_origin is Required: - required_keys.add(annotation_key) + is_required = True elif annotation_origin is NotRequired: - optional_keys.add(annotation_key) - elif total: + is_required = False + else: + is_required = total + + if is_required: required_keys.add(annotation_key) + optional_keys.discard(annotation_key) else: optional_keys.add(annotation_key) + required_keys.discard(annotation_key) + assert required_keys.isdisjoint(optional_keys), ( + f"Required keys overlap with optional keys in {name}:" + f" {required_keys=}, {optional_keys=}" + ) tp_dict.__annotations__ = annotations tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) diff --git a/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst b/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst new file mode 100644 index 0000000000000..a16d67e7776bc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-28-20-01-33.gh-issue-112509.QtoKed.rst @@ -0,0 +1,3 @@ +Fix edge cases that could cause a key to be present in both the +``__required_keys__`` and ``__optional_keys__`` attributes of a +:class:`typing.TypedDict`. Patch by Jelle Zijlstra.

1 0

gh-104003: Implement PEP 702 (#104004)
by JelleZijlstra 29 Nov '23

29 Nov '23

https://github.com/python/cpython/commit/d4a6229afe10d7e5a9a59bf9472f36d769… commit: d4a6229afe10d7e5a9a59bf9472f36d7698988db branch: main author: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2023-11-29T09:38:29-08:00 summary: gh-104003: Implement PEP 702 (#104004) Co-authored-by: Hugo van Kemenade <hugovk(a)users.noreply.github.com> Co-authored-by: Alex Waygood <Alex.Waygood(a)Gmail.com> files: A Misc/NEWS.d/next/Library/2023-04-29-20-49-13.gh-issue-104003.-8Ruk2.rst M Doc/library/warnings.rst M Doc/whatsnew/3.13.rst M Lib/test/test_warnings/__init__.py M Lib/warnings.py diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 884de08eab1b16..a9c469707e8227 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -522,6 +522,56 @@ Available Functions and calls to :func:`simplefilter`. +.. decorator:: deprecated(msg, *, category=DeprecationWarning, stacklevel=1) + + Decorator to indicate that a class, function or overload is deprecated. + + When this decorator is applied to an object, + deprecation warnings may be emitted at runtime when the object is used. + :term:`static type checkers <static type checker>` + will also generate a diagnostic on usage of the deprecated object. + + Usage:: + + from warnings import deprecated + from typing import overload + + @deprecated("Use B instead") + class A: + pass + + @deprecated("Use g instead") + def f(): + pass + + @overload + @deprecated("int support is deprecated") + def g(x: int) -> int: ... + @overload + def g(x: str) -> int: ... + + The warning specified by *category* will be emitted at runtime + on use of deprecated objects. For functions, that happens on calls; + for classes, on instantiation and on creation of subclasses. + If the *category* is ``None``, no warning is emitted at runtime. + The *stacklevel* determines where the + warning is emitted. If it is ``1`` (the default), the warning + is emitted at the direct caller of the deprecated object; if it + is higher, it is emitted further up the stack. + Static type checker behavior is not affected by the *category* + and *stacklevel* arguments. + + The deprecation message passed to the decorator is saved in the + ``__deprecated__`` attribute on the decorated object. + If applied to an overload, the decorator + must be after the :func:`@overload <typing.overload>` decorator + for the attribute to exist on the overload as returned by + :func:`typing.get_overloads`. + + .. versionadded:: 3.13 + See :pep:`702`. + + Available Context Managers -------------------------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 198ea3a4b57bde..372e4a45468e68 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -348,6 +348,15 @@ venv (using ``--without-scm-ignore-files``). (Contributed by Brett Cannon in :gh:`108125`.) +warnings +-------- + +* The new :func:`warnings.deprecated` decorator provides a way to communicate + deprecations to :term:`static type checkers <static type checker>` and + to warn on usage of deprecated classes and functions. A runtime deprecation + warning may also be emitted when a decorated function or class is used at runtime. + See :pep:`702`. (Contributed by Jelle Zijlstra in :gh:`104003`.) + Optimizations ============= diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 2c523230e7e97f..cd989fe36bf26b 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -5,6 +5,8 @@ import re import sys import textwrap +import types +from typing import overload, get_overloads import unittest from test import support from test.support import import_helper @@ -16,6 +18,7 @@ from test.test_warnings.data import stacklevel as warning_tests import warnings as original_warnings +from warnings import deprecated py_warnings = import_helper.import_fresh_module('warnings', @@ -90,7 +93,7 @@ def test_module_all_attribute(self): self.assertTrue(hasattr(self.module, '__all__')) target_api = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", - "resetwarnings", "catch_warnings"] + "resetwarnings", "catch_warnings", "deprecated"] self.assertSetEqual(set(self.module.__all__), set(target_api)) @@ -1377,6 +1380,283 @@ def test_late_resource_warning(self): self.assertTrue(err.startswith(expected), ascii(err)) +class DeprecatedTests(unittest.TestCase): + def test_dunder_deprecated(self): + @deprecated("A will go away soon") + class A: + pass + + self.assertEqual(A.__deprecated__, "A will go away soon") + self.assertIsInstance(A, type) + + @deprecated("b will go away soon") + def b(): + pass + + self.assertEqual(b.__deprecated__, "b will go away soon") + self.assertIsInstance(b, types.FunctionType) + + @overload + @deprecated("no more ints") + def h(x: int) -> int: ... + @overload + def h(x: str) -> str: ... + def h(x): + return x + + overloads = get_overloads(h) + self.assertEqual(len(overloads), 2) + self.assertEqual(overloads[0].__deprecated__, "no more ints") + + def test_class(self): + @deprecated("A will go away soon") + class A: + pass + + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + A() + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + with self.assertRaises(TypeError): + A(42) + + def test_class_with_init(self): + @deprecated("HasInit will go away soon") + class HasInit: + def __init__(self, x): + self.x = x + + with self.assertWarnsRegex(DeprecationWarning, "HasInit will go away soon"): + instance = HasInit(42) + self.assertEqual(instance.x, 42) + + def test_class_with_new(self): + has_new_called = False + + @deprecated("HasNew will go away soon") + class HasNew: + def __new__(cls, x): + nonlocal has_new_called + has_new_called = True + return super().__new__(cls) + + def __init__(self, x) -> None: + self.x = x + + with self.assertWarnsRegex(DeprecationWarning, "HasNew will go away soon"): + instance = HasNew(42) + self.assertEqual(instance.x, 42) + self.assertTrue(has_new_called) + + def test_class_with_inherited_new(self): + new_base_called = False + + class NewBase: + def __new__(cls, x): + nonlocal new_base_called + new_base_called = True + return super().__new__(cls) + + def __init__(self, x) -> None: + self.x = x + + @deprecated("HasInheritedNew will go away soon") + class HasInheritedNew(NewBase): + pass + + with self.assertWarnsRegex(DeprecationWarning, "HasInheritedNew will go away soon"): + instance = HasInheritedNew(42) + self.assertEqual(instance.x, 42) + self.assertTrue(new_base_called) + + def test_class_with_new_but_no_init(self): + new_called = False + + @deprecated("HasNewNoInit will go away soon") + class HasNewNoInit: + def __new__(cls, x): + nonlocal new_called + new_called = True + obj = super().__new__(cls) + obj.x = x + return obj + + with self.assertWarnsRegex(DeprecationWarning, "HasNewNoInit will go away soon"): + instance = HasNewNoInit(42) + self.assertEqual(instance.x, 42) + self.assertTrue(new_called) + + def test_mixin_class(self): + @deprecated("Mixin will go away soon") + class Mixin: + pass + + class Base: + def __init__(self, a) -> None: + self.a = a + + with self.assertWarnsRegex(DeprecationWarning, "Mixin will go away soon"): + class Child(Base, Mixin): + pass + + instance = Child(42) + self.assertEqual(instance.a, 42) + + def test_existing_init_subclass(self): + @deprecated("C will go away soon") + class C: + def __init_subclass__(cls) -> None: + cls.inited = True + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + C() + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + class D(C): + pass + + self.assertTrue(D.inited) + self.assertIsInstance(D(), D) # no deprecation + + def test_existing_init_subclass_in_base(self): + class Base: + def __init_subclass__(cls, x) -> None: + cls.inited = x + + @deprecated("C will go away soon") + class C(Base, x=42): + pass + + self.assertEqual(C.inited, 42) + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + C() + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + class D(C, x=3): + pass + + self.assertEqual(D.inited, 3) + + def test_init_subclass_has_correct_cls(self): + init_subclass_saw = None + + @deprecated("Base will go away soon") + class Base: + def __init_subclass__(cls) -> None: + nonlocal init_subclass_saw + init_subclass_saw = cls + + self.assertIsNone(init_subclass_saw) + + with self.assertWarnsRegex(DeprecationWarning, "Base will go away soon"): + class C(Base): + pass + + self.assertIs(init_subclass_saw, C) + + def test_init_subclass_with_explicit_classmethod(self): + init_subclass_saw = None + + @deprecated("Base will go away soon") + class Base: + @classmethod + def __init_subclass__(cls) -> None: + nonlocal init_subclass_saw + init_subclass_saw = cls + + self.assertIsNone(init_subclass_saw) + + with self.assertWarnsRegex(DeprecationWarning, "Base will go away soon"): + class C(Base): + pass + + self.assertIs(init_subclass_saw, C) + + def test_function(self): + @deprecated("b will go away soon") + def b(): + pass + + with self.assertWarnsRegex(DeprecationWarning, "b will go away soon"): + b() + + def test_method(self): + class Capybara: + @deprecated("x will go away soon") + def x(self): + pass + + instance = Capybara() + with self.assertWarnsRegex(DeprecationWarning, "x will go away soon"): + instance.x() + + def test_property(self): + class Capybara: + @property + @deprecated("x will go away soon") + def x(self): + pass + + @property + def no_more_setting(self): + return 42 + + @no_more_setting.setter + @deprecated("no more setting") + def no_more_setting(self, value): + pass + + instance = Capybara() + with self.assertWarnsRegex(DeprecationWarning, "x will go away soon"): + instance.x + + with py_warnings.catch_warnings(): + py_warnings.simplefilter("error") + self.assertEqual(instance.no_more_setting, 42) + + with self.assertWarnsRegex(DeprecationWarning, "no more setting"): + instance.no_more_setting = 42 + + def test_category(self): + @deprecated("c will go away soon", category=RuntimeWarning) + def c(): + pass + + with self.assertWarnsRegex(RuntimeWarning, "c will go away soon"): + c() + + def test_turn_off_warnings(self): + @deprecated("d will go away soon", category=None) + def d(): + pass + + with py_warnings.catch_warnings(): + py_warnings.simplefilter("error") + d() + + def test_only_strings_allowed(self): + with self.assertRaisesRegex( + TypeError, + "Expected an object of type str for 'message', not 'type'" + ): + @deprecated + class Foo: ... + + with self.assertRaisesRegex( + TypeError, + "Expected an object of type str for 'message', not 'function'" + ): + @deprecated + def foo(): ... + + def test_no_retained_references_to_wrapper_instance(self): + @deprecated('depr') + def d(): pass + + self.assertFalse(any( + isinstance(cell.cell_contents, deprecated) for cell in d.__closure__ + )) + def setUpModule(): py_warnings.onceregistry.clear() c_warnings.onceregistry.clear() diff --git a/Lib/warnings.py b/Lib/warnings.py index 32e58072b9cc33..924f872172d4d1 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -5,7 +5,7 @@ __all__ = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", - "resetwarnings", "catch_warnings"] + "resetwarnings", "catch_warnings", "deprecated"] def showwarning(message, category, filename, lineno, file=None, line=None): """Hook to write a warning to a file; replace if you like.""" @@ -508,6 +508,135 @@ def __exit__(self, *exc_info): self._module._showwarnmsg_impl = self._showwarnmsg_impl +class deprecated: + """Indicate that a class, function or overload is deprecated. + + When this decorator is applied to an object, the type checker + will generate a diagnostic on usage of the deprecated object. + + Usage: + + @deprecated("Use B instead") + class A: + pass + + @deprecated("Use g instead") + def f(): + pass + + @overload + @deprecated("int support is deprecated") + def g(x: int) -> int: ... + @overload + def g(x: str) -> int: ... + + The warning specified by *category* will be emitted at runtime + on use of deprecated objects. For functions, that happens on calls; + for classes, on instantiation and on creation of subclasses. + If the *category* is ``None``, no warning is emitted at runtime. + The *stacklevel* determines where the + warning is emitted. If it is ``1`` (the default), the warning + is emitted at the direct caller of the deprecated object; if it + is higher, it is emitted further up the stack. + Static type checker behavior is not affected by the *category* + and *stacklevel* arguments. + + The deprecation message passed to the decorator is saved in the + ``__deprecated__`` attribute on the decorated object. + If applied to an overload, the decorator + must be after the ``@overload`` decorator for the attribute to + exist on the overload as returned by ``get_overloads()``. + + See PEP 702 for details. + + """ + def __init__( + self, + message: str, + /, + *, + category: type[Warning] | None = DeprecationWarning, + stacklevel: int = 1, + ) -> None: + if not isinstance(message, str): + raise TypeError( + f"Expected an object of type str for 'message', not {type(message).__name__!r}" + ) + self.message = message + self.category = category + self.stacklevel = stacklevel + + def __call__(self, arg, /): + # Make sure the inner functions created below don't + # retain a reference to self. + msg = self.message + category = self.category + stacklevel = self.stacklevel + if category is None: + arg.__deprecated__ = msg + return arg + elif isinstance(arg, type): + import functools + from types import MethodType + + original_new = arg.__new__ + + @functools.wraps(original_new) + def __new__(cls, *args, **kwargs): + if cls is arg: + warn(msg, category=category, stacklevel=stacklevel + 1) + if original_new is not object.__new__: + return original_new(cls, *args, **kwargs) + # Mirrors a similar check in object.__new__. + elif cls.__init__ is object.__init__ and (args or kwargs): + raise TypeError(f"{cls.__name__}() takes no arguments") + else: + return original_new(cls) + + arg.__new__ = staticmethod(__new__) + + original_init_subclass = arg.__init_subclass__ + # We need slightly different behavior if __init_subclass__ + # is a bound method (likely if it was implemented in Python) + if isinstance(original_init_subclass, MethodType): + original_init_subclass = original_init_subclass.__func__ + + @functools.wraps(original_init_subclass) + def __init_subclass__(*args, **kwargs): + warn(msg, category=category, stacklevel=stacklevel + 1) + return original_init_subclass(*args, **kwargs) + + arg.__init_subclass__ = classmethod(__init_subclass__) + # Or otherwise, which likely means it's a builtin such as + # object's implementation of __init_subclass__. + else: + @functools.wraps(original_init_subclass) + def __init_subclass__(*args, **kwargs): + warn(msg, category=category, stacklevel=stacklevel + 1) + return original_init_subclass(*args, **kwargs) + + arg.__init_subclass__ = __init_subclass__ + + arg.__deprecated__ = __new__.__deprecated__ = msg + __init_subclass__.__deprecated__ = msg + return arg + elif callable(arg): + import functools + + @functools.wraps(arg) + def wrapper(*args, **kwargs): + warn(msg, category=category, stacklevel=stacklevel + 1) + return arg(*args, **kwargs) + + arg.__deprecated__ = wrapper.__deprecated__ = msg + return wrapper + else: + raise TypeError( + "@deprecated decorator with non-None category must be applied to " + f"a class or callable, not {arg!r}" + ) + + _DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}" def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info): diff --git a/Misc/NEWS.d/next/Library/2023-04-29-20-49-13.gh-issue-104003.-8Ruk2.rst b/Misc/NEWS.d/next/Library/2023-04-29-20-49-13.gh-issue-104003.-8Ruk2.rst new file mode 100644 index 00000000000000..82d61ca8b8bc97 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-29-20-49-13.gh-issue-104003.-8Ruk2.rst @@ -0,0 +1,3 @@ +Add :func:`warnings.deprecated`, a decorator to mark deprecated functions to +static type checkers and to warn on usage of deprecated classes and functions. +See :pep:`702`. Patch by Jelle Zijlstra.

1 0