Mailman 3 May 2024 - Python-checkins

statistics.fmean(): speed-up code path for non-sizeable inputs. (gh-119876)
by rhettinger 31 May '24

31 May '24

https://github.com/python/cpython/commit/cc5cd4d93e3e079e897da9ceb1732ef16d… commit: cc5cd4d93e3e079e897da9ceb1732ef16d79d01b branch: main author: Raymond Hettinger <rhettinger(a)users.noreply.github.com> committer: rhettinger <rhettinger(a)users.noreply.github.com> date: 2024-05-31T17:08:55-05:00 summary: statistics.fmean(): speed-up code path for non-sizeable inputs. (gh-119876) files: M Lib/statistics.py diff --git a/Lib/statistics.py b/Lib/statistics.py index c2f4fe8e054d3d..450edfaabe8def 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -505,13 +505,11 @@ def fmean(data, weights=None): n = len(data) except TypeError: # Handle iterators that do not define __len__(). - n = 0 - def count(iterable): - nonlocal n - for n, x in enumerate(iterable, start=1): - yield x - data = count(data) - total = fsum(data) + counter = count() + total = fsum(map(itemgetter(0), zip(data, counter))) + n = next(counter) + else: + total = fsum(data) if not n: raise StatisticsError('fmean requires at least one data point') return total / n

1 0

[3.13] gh-118894: Make asyncio REPL use pyrepl (GH-119433) (#119884)
by ambv 31 May '24

31 May '24

https://github.com/python/cpython/commit/a5272e63efc003a30a2b603b512d367282… commit: a5272e63efc003a30a2b603b512d367282a24209 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: ambv <lukasz(a)langa.pl> date: 2024-05-31T17:15:44-04:00 summary: [3.13] gh-118894: Make asyncio REPL use pyrepl (GH-119433) (#119884) (cherry picked from commit 2237946af0981c46dc7d3886477e425ccfb37f28) Co-authored-by: Łukasz Langa <lukasz(a)langa.pl> files: A Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst M Lib/_pyrepl/commands.py M Lib/_pyrepl/console.py M Lib/_pyrepl/reader.py M Lib/_pyrepl/simple_interact.py M Lib/asyncio/__main__.py M Lib/test/test_pyrepl/test_interact.py diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index ed977f84baac4e..2ef5dada9d9e58 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -219,6 +219,11 @@ def do(self) -> None: os.kill(os.getpid(), signal.SIGINT) +class ctrl_c(Command): + def do(self) -> None: + raise KeyboardInterrupt + + class suspend(Command): def do(self) -> None: import signal diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index fcabf785069ecb..aa0bde865825c9 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -19,10 +19,14 @@ from __future__ import annotations -import sys +import _colorize # type: ignore[import-not-found] from abc import ABC, abstractmethod +import ast +import code from dataclasses import dataclass, field +import os.path +import sys TYPE_CHECKING = False @@ -136,3 +140,54 @@ def wait(self) -> None: @abstractmethod def repaint(self) -> None: ... + + +class InteractiveColoredConsole(code.InteractiveConsole): + def __init__( + self, + locals: dict[str, object] | None = None, + filename: str = "<console>", + *, + local_exit: bool = False, + ) -> None: + super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] + self.can_colorize = _colorize.can_colorize() + + def showsyntaxerror(self, filename=None): + super().showsyntaxerror(colorize=self.can_colorize) + + def showtraceback(self): + super().showtraceback(colorize=self.can_colorize) + + def runsource(self, source, filename="<input>", symbol="single"): + try: + tree = ast.parse(source) + except (SyntaxError, OverflowError, ValueError): + self.showsyntaxerror(filename) + return False + if tree.body: + *_, last_stmt = tree.body + for stmt in tree.body: + wrapper = ast.Interactive if stmt is last_stmt else ast.Module + the_symbol = symbol if stmt is last_stmt else "exec" + item = wrapper([stmt]) + try: + code = self.compile.compiler(item, filename, the_symbol, dont_inherit=True) + except SyntaxError as e: + if e.args[0] == "'await' outside function": + python = os.path.basename(sys.executable) + e.add_note( + f"Try the asyncio REPL ({python} -m asyncio) to use" + f" top-level 'await' and run background asyncio tasks." + ) + self.showsyntaxerror(filename) + return False + except (OverflowError, ValueError): + self.showsyntaxerror(filename) + return False + + if code is None: + return True + + self.runcode(code) + return False diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 8d9a22c272f88b..53e0954a83aa01 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -131,6 +131,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]: ("\\\\", "self-insert"), (r"\x1b[200~", "enable_bracketed_paste"), (r"\x1b[201~", "disable_bracketed_paste"), + (r"\x03", "ctrl-c"), ] + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()] diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index c624f6e12a7094..256bbc7c6d7626 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -25,14 +25,13 @@ from __future__ import annotations -import _colorize # type: ignore[import-not-found] import _sitebuiltins import linecache import sys import code -import ast from types import ModuleType +from .console import InteractiveColoredConsole from .readline import _get_reader, multiline_input _error: tuple[type[Exception], ...] | type[Exception] @@ -74,57 +73,21 @@ def _clear_screen(): "clear": _clear_screen, } -class InteractiveColoredConsole(code.InteractiveConsole): - def __init__( - self, - locals: dict[str, object] | None = None, - filename: str = "<console>", - *, - local_exit: bool = False, - ) -> None: - super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] - self.can_colorize = _colorize.can_colorize() - - def showsyntaxerror(self, filename=None): - super().showsyntaxerror(colorize=self.can_colorize) - - def showtraceback(self): - super().showtraceback(colorize=self.can_colorize) - - def runsource(self, source, filename="<input>", symbol="single"): - try: - tree = ast.parse(source) - except (OverflowError, SyntaxError, ValueError): - self.showsyntaxerror(filename) - return False - if tree.body: - *_, last_stmt = tree.body - for stmt in tree.body: - wrapper = ast.Interactive if stmt is last_stmt else ast.Module - the_symbol = symbol if stmt is last_stmt else "exec" - item = wrapper([stmt]) - try: - code = compile(item, filename, the_symbol, dont_inherit=True) - except (OverflowError, ValueError, SyntaxError): - self.showsyntaxerror(filename) - return False - - if code is None: - return True - - self.runcode(code) - return False - def run_multiline_interactive_console( - mainmodule: ModuleType | None= None, future_flags: int = 0 + mainmodule: ModuleType | None = None, + future_flags: int = 0, + console: code.InteractiveConsole | None = None, ) -> None: import __main__ from .readline import _setup _setup() mainmodule = mainmodule or __main__ - console = InteractiveColoredConsole(mainmodule.__dict__, filename="<stdin>") + if console is None: + console = InteractiveColoredConsole( + mainmodule.__dict__, filename="<stdin>" + ) if future_flags: console.compile.compiler.flags |= future_flags diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 9041b8b8316c1e..91fff9aaee337b 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,42 +1,49 @@ import ast import asyncio -import code import concurrent.futures import inspect +import os import site import sys import threading import types import warnings +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] +from _pyrepl.console import InteractiveColoredConsole + from . import futures -class AsyncIOInteractiveConsole(code.InteractiveConsole): +class AsyncIOInteractiveConsole(InteractiveColoredConsole): def __init__(self, locals, loop): - super().__init__(locals) + super().__init__(locals, filename="<stdin>") self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT self.loop = loop def runcode(self, code): + global return_code future = concurrent.futures.Future() def callback(): + global return_code global repl_future - global repl_future_interrupted + global keyboard_interrupted repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False func = types.FunctionType(code, self.locals) try: coro = func() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except KeyboardInterrupt as ex: - repl_future_interrupted = True + keyboard_interrupted = True future.set_exception(ex) return except BaseException as ex: @@ -57,10 +64,12 @@ def callback(): try: return future.result() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except BaseException: - if repl_future_interrupted: + if keyboard_interrupted: self.write("\nKeyboardInterrupt\n") else: self.showtraceback() @@ -69,18 +78,56 @@ def callback(): class REPLThread(threading.Thread): def run(self): + global return_code + try: banner = ( f'asyncio REPL {sys.version} on {sys.platform}\n' f'Use "await" directly instead of "asyncio.run()".\n' f'Type "help", "copyright", "credits" or "license" ' f'for more information.\n' - f'{getattr(sys, "ps1", ">>> ")}import asyncio' ) - console.interact( - banner=banner, - exitmsg='exiting asyncio REPL...') + console.write(banner) + + if startup_path := os.getenv("PYTHONSTARTUP"): + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code, console.locals) + + ps1 = getattr(sys, "ps1", ">>> ") + if can_colorize(): + ps1 = f"{ANSIColors.BOLD_MAGENTA}{ps1}{ANSIColors.RESET}" + console.write(f"{ps1}import asyncio\n") + + try: + import errno + if os.getenv("PYTHON_BASIC_REPL"): + raise RuntimeError("user environment requested basic REPL") + if not os.isatty(sys.stdin.fileno()): + raise OSError(errno.ENOTTY, "tty required", "stdin") + + # This import will fail on operating systems with no termios. + from _pyrepl.simple_interact import ( + check, + run_multiline_interactive_console, + ) + if err := check(): + raise RuntimeError(err) + except Exception as e: + console.interact(banner="", exitmsg=exit_message) + else: + try: + run_multiline_interactive_console(console=console) + except SystemExit: + # expected via the `exit` and `quit` commands + pass + except BaseException: + # unexpected issue + console.showtraceback() + console.write("Internal error, ") + return_code = 1 finally: warnings.filterwarnings( 'ignore', @@ -91,6 +138,9 @@ def run(self): if __name__ == '__main__': + CAN_USE_PYREPL = True + + return_code = 0 loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -103,7 +153,7 @@ def run(self): console = AsyncIOInteractiveConsole(repl_locals, loop) repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False try: import readline # NoQA @@ -126,7 +176,7 @@ def run(self): completer = rlcompleter.Completer(console.locals) readline.set_completer(completer.complete) - repl_thread = REPLThread() + repl_thread = REPLThread(name="Interactive thread") repl_thread.daemon = True repl_thread.start() @@ -134,9 +184,12 @@ def run(self): try: loop.run_forever() except KeyboardInterrupt: + keyboard_interrupted = True if repl_future and not repl_future.done(): repl_future.cancel() - repl_future_interrupted = True continue else: break + + console.write('exiting asyncio REPL...\n') + sys.exit(return_code) diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index 4d01ea7620109d..df97b1354a168e 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -6,7 +6,7 @@ from test.support import force_not_colorized -from _pyrepl.simple_interact import InteractiveColoredConsole +from _pyrepl.console import InteractiveColoredConsole class TestSimpleInteract(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst b/Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst new file mode 100644 index 00000000000000..ffc4ae336dc54f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst @@ -0,0 +1 @@ +:mod:`asyncio` REPL now has the same capabilities as PyREPL.

1 0

gh-119180: Lazily wrap annotations on classmethod and staticmethod (#119864)
by JelleZijlstra 31 May '24

31 May '24

https://github.com/python/cpython/commit/d28afd3fa064db10a2eb2a65bba33e8ea7… commit: d28afd3fa064db10a2eb2a65bba33e8ea77a8fcf branch: main author: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2024-05-31T14:05:51-07:00 summary: gh-119180: Lazily wrap annotations on classmethod and staticmethod (#119864) files: A Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst M Lib/test/test_descr.py M Objects/funcobject.c diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index c3f292467a6738..7742f075285602 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1593,8 +1593,7 @@ def f(cls, arg): self.fail("classmethod shouldn't accept keyword args") cm = classmethod(f) - cm_dict = {'__annotations__': {}, - '__doc__': ( + cm_dict = {'__doc__': ( "f docstring" if support.HAVE_DOCSTRINGS else None @@ -1610,6 +1609,41 @@ def f(cls, arg): del cm.x self.assertNotHasAttr(cm, "x") + def test_classmethod_staticmethod_annotations(self): + for deco in (classmethod, staticmethod): + @deco + def unannotated(cls): pass + @deco + def annotated(cls) -> int: pass + + for method in (annotated, unannotated): + with self.subTest(deco=deco, method=method): + original_annotations = dict(method.__wrapped__.__annotations__) + self.assertNotIn('__annotations__', method.__dict__) + self.assertEqual(method.__annotations__, original_annotations) + self.assertIn('__annotations__', method.__dict__) + + new_annotations = {"a": "b"} + method.__annotations__ = new_annotations + self.assertEqual(method.__annotations__, new_annotations) + self.assertEqual(method.__wrapped__.__annotations__, original_annotations) + + del method.__annotations__ + self.assertEqual(method.__annotations__, original_annotations) + + original_annotate = method.__wrapped__.__annotate__ + self.assertNotIn('__annotate__', method.__dict__) + self.assertIs(method.__annotate__, original_annotate) + self.assertIn('__annotate__', method.__dict__) + + new_annotate = lambda: {"annotations": 1} + method.__annotate__ = new_annotate + self.assertIs(method.__annotate__, new_annotate) + self.assertIs(method.__wrapped__.__annotate__, original_annotate) + + del method.__annotate__ + self.assertIs(method.__annotate__, original_annotate) + @support.refcount_test def test_refleaks_in_classmethod___init__(self): gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst new file mode 100644 index 00000000000000..1e5ad7d08eed7c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-31-08-23-41.gh-issue-119180.KL4VxZ.rst @@ -0,0 +1,3 @@ +:func:`classmethod` and :func:`staticmethod` now wrap the +:attr:`__annotations__` and :attr:`!__annotate__` attributes of their +underlying callable lazily. See :pep:`649`. Patch by Jelle Zijlstra. diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 4e78252052932c..40211297be20c0 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -1172,12 +1172,57 @@ functools_wraps(PyObject *wrapper, PyObject *wrapped) COPY_ATTR(__name__); COPY_ATTR(__qualname__); COPY_ATTR(__doc__); - COPY_ATTR(__annotations__); return 0; #undef COPY_ATTR } +// Used for wrapping __annotations__ and __annotate__ on classmethod +// and staticmethod objects. +static PyObject * +descriptor_get_wrapped_attribute(PyObject *wrapped, PyObject *dict, PyObject *name) +{ + PyObject *res; + if (PyDict_GetItemRef(dict, name, &res) < 0) { + return NULL; + } + if (res != NULL) { + return res; + } + res = PyObject_GetAttr(wrapped, name); + if (res == NULL) { + return NULL; + } + if (PyDict_SetItem(dict, name, res) < 0) { + Py_DECREF(res); + return NULL; + } + return res; +} + +static int +descriptor_set_wrapped_attribute(PyObject *dict, PyObject *name, PyObject *value, + char *type_name) +{ + if (value == NULL) { + if (PyDict_DelItem(dict, name) < 0) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + PyErr_Format(PyExc_AttributeError, + "'%.200s' object has no attribute '%U'", + type_name, name); + } + else { + return -1; + } + } + return 0; + } + else { + return PyDict_SetItem(dict, name, value); + } +} + /* Class method object */ @@ -1283,10 +1328,37 @@ cm_get___isabstractmethod__(classmethod *cm, void *closure) Py_RETURN_FALSE; } +static PyObject * +cm_get___annotations__(classmethod *cm, void *closure) +{ + return descriptor_get_wrapped_attribute(cm->cm_callable, cm->cm_dict, &_Py_ID(__annotations__)); +} + +static int +cm_set___annotations__(classmethod *cm, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(cm->cm_dict, &_Py_ID(__annotations__), value, "classmethod"); +} + +static PyObject * +cm_get___annotate__(classmethod *cm, void *closure) +{ + return descriptor_get_wrapped_attribute(cm->cm_callable, cm->cm_dict, &_Py_ID(__annotate__)); +} + +static int +cm_set___annotate__(classmethod *cm, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(cm->cm_dict, &_Py_ID(__annotate__), value, "classmethod"); +} + + static PyGetSetDef cm_getsetlist[] = { {"__isabstractmethod__", (getter)cm_get___isabstractmethod__, NULL, NULL, NULL}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {"__annotations__", (getter)cm_get___annotations__, (setter)cm_set___annotations__, NULL, NULL}, + {"__annotate__", (getter)cm_get___annotate__, (setter)cm_set___annotate__, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -1479,10 +1551,36 @@ sm_get___isabstractmethod__(staticmethod *sm, void *closure) Py_RETURN_FALSE; } +static PyObject * +sm_get___annotations__(staticmethod *sm, void *closure) +{ + return descriptor_get_wrapped_attribute(sm->sm_callable, sm->sm_dict, &_Py_ID(__annotations__)); +} + +static int +sm_set___annotations__(staticmethod *sm, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(sm->sm_dict, &_Py_ID(__annotations__), value, "staticmethod"); +} + +static PyObject * +sm_get___annotate__(staticmethod *sm, void *closure) +{ + return descriptor_get_wrapped_attribute(sm->sm_callable, sm->sm_dict, &_Py_ID(__annotate__)); +} + +static int +sm_set___annotate__(staticmethod *sm, PyObject *value, void *closure) +{ + return descriptor_set_wrapped_attribute(sm->sm_dict, &_Py_ID(__annotate__), value, "staticmethod"); +} + static PyGetSetDef sm_getsetlist[] = { {"__isabstractmethod__", (getter)sm_get___isabstractmethod__, NULL, NULL, NULL}, {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {"__annotations__", (getter)sm_get___annotations__, (setter)sm_set___annotations__, NULL, NULL}, + {"__annotate__", (getter)sm_get___annotate__, (setter)sm_set___annotate__, NULL, NULL}, {NULL} /* Sentinel */ };

1 0

gh-119821: Support non-dict globals in LOAD_FROM_DICT_OR_GLOBALS (#119822)
by JelleZijlstra 31 May '24

31 May '24

https://github.com/python/cpython/commit/80a4e3899420faaa012c82b4e82cdb6675… commit: 80a4e3899420faaa012c82b4e82cdb6675a6a944 branch: main author: Jelle Zijlstra <jelle.zijlstra(a)gmail.com> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2024-05-31T14:05:24-07:00 summary: gh-119821: Support non-dict globals in LOAD_FROM_DICT_OR_GLOBALS (#119822) Support non-dict globals in LOAD_FROM_DICT_OR_GLOBALS The implementation basically copies LOAD_GLOBAL. Possibly it could be deduplicated, but that seems like it may get hairy since the two operations have different operands. This is important to fix in 3.14 for PEP 649, but it's a bug in earlier versions too, and we should backport to 3.13 and 3.12 if possible. files: A Misc/NEWS.d/next/Core and Builtins/2024-05-30-23-01-00.gh-issue-119821.jPGfvt.rst M Include/internal/pycore_opcode_metadata.h M Include/internal/pycore_uop_metadata.h M Lib/test/test_type_aliases.py M Python/bytecodes.c M Python/executor_cases.c.h M Python/generated_cases.c.h M Python/optimizer_cases.c.h diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d3535800139a66..0b835230974e39 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1323,7 +1323,6 @@ _PyOpcode_macro_expansion[256] = { [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { _LOAD_FAST_CHECK, 0, 0 } } }, [LOAD_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _LOAD_FAST, 5, 0 }, { _LOAD_FAST, 6, 0 } } }, [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, - [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, [LOAD_GLOBAL] = { .nuops = 1, .uops = { { _LOAD_GLOBAL, 0, 0 } } }, [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 78f0eafaa32042..690ae34a6eef98 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -107,7 +107,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG, [_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG, @@ -439,7 +438,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", - [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", @@ -692,8 +690,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_LOCALS: return 0; - case _LOAD_FROM_DICT_OR_GLOBALS: - return 1; case _LOAD_GLOBAL: return 0; case _GUARD_GLOBALS_VERSION: diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index 9c325bc595f585..f8b395fdc8bb1d 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -1,4 +1,5 @@ import pickle +import textwrap import types import unittest from test.support import check_syntax_error, run_code @@ -328,3 +329,22 @@ def test_pickling_local(self): with self.subTest(thing=thing, proto=proto): with self.assertRaises(pickle.PickleError): pickle.dumps(thing, protocol=proto) + + +class TypeParamsExoticGlobalsTest(unittest.TestCase): + def test_exec_with_unusual_globals(self): + class customdict(dict): + def __missing__(self, key): + return key + + code = compile("type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["Alias"] + self.assertEqual(Alias.__value__, "undefined") + + code = compile("class A: type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["A"].Alias + self.assertEqual(Alias.__value__, "undefined") diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-30-23-01-00.gh-issue-119821.jPGfvt.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-30-23-01-00.gh-issue-119821.jPGfvt.rst new file mode 100644 index 00000000000000..cc25eee6dd6ae4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-30-23-01-00.gh-issue-119821.jPGfvt.rst @@ -0,0 +1,2 @@ +Fix execution of :ref:`annotation scopes <annotation-scopes>` within classes +when ``globals`` is set to a non-dict. Patch by Jelle Zijlstra. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9a8198515dea5e..1c12e1cddbbc10 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1385,18 +1385,35 @@ dummy_func( ERROR_NO_POP(); } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - ERROR_NO_POP(); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (v == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } ERROR_NO_POP(); } + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0, error); if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - ERROR_NO_POP(); + /* namespace 2: builtins */ + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0, error); + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + ERROR_IF(true, error); + } } } } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e862364cb23e7a..0dfe490cb37047 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1405,35 +1405,7 @@ break; } - case _LOAD_FROM_DICT_OR_GLOBALS: { - PyObject *mod_or_class_dict; - PyObject *v; - oparg = CURRENT_OPARG(); - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - JUMP_TO_ERROR(); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - JUMP_TO_ERROR(); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - JUMP_TO_ERROR(); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - JUMP_TO_ERROR(); - } - } - } - Py_DECREF(mod_or_class_dict); - stack_pointer[-1] = v; - break; - } + /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ /* _LOAD_NAME is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4402787d96f12e..1a991608385405 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4401,18 +4401,35 @@ goto error; } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - goto error; - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (v == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } goto error; } + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0) goto pop_1_error; if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - goto error; + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto pop_1_error; + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) goto pop_1_error; + } } } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 1b76f1480b4f11..b3787345ec6714 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -740,12 +740,7 @@ break; } - case _LOAD_FROM_DICT_OR_GLOBALS: { - _Py_UopsSymbol *v; - v = sym_new_not_null(ctx); - stack_pointer[-1] = v; - break; - } + /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ /* _LOAD_NAME is not a viable micro-op for tier 2 */

1 0

gh-118894: Make asyncio REPL use pyrepl (GH-119433)
by ambv 31 May '24

31 May '24

https://github.com/python/cpython/commit/2237946af0981c46dc7d3886477e425ccf… commit: 2237946af0981c46dc7d3886477e425ccfb37f28 branch: main author: Łukasz Langa <lukasz(a)langa.pl> committer: ambv <lukasz(a)langa.pl> date: 2024-05-31T22:26:02+02:00 summary: gh-118894: Make asyncio REPL use pyrepl (GH-119433) files: A Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst M Lib/_pyrepl/commands.py M Lib/_pyrepl/console.py M Lib/_pyrepl/reader.py M Lib/_pyrepl/simple_interact.py M Lib/asyncio/__main__.py M Lib/test/test_pyrepl/test_interact.py diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index ed977f84baac4e..2ef5dada9d9e58 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -219,6 +219,11 @@ def do(self) -> None: os.kill(os.getpid(), signal.SIGINT) +class ctrl_c(Command): + def do(self) -> None: + raise KeyboardInterrupt + + class suspend(Command): def do(self) -> None: import signal diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index fcabf785069ecb..aa0bde865825c9 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -19,10 +19,14 @@ from __future__ import annotations -import sys +import _colorize # type: ignore[import-not-found] from abc import ABC, abstractmethod +import ast +import code from dataclasses import dataclass, field +import os.path +import sys TYPE_CHECKING = False @@ -136,3 +140,54 @@ def wait(self) -> None: @abstractmethod def repaint(self) -> None: ... + + +class InteractiveColoredConsole(code.InteractiveConsole): + def __init__( + self, + locals: dict[str, object] | None = None, + filename: str = "<console>", + *, + local_exit: bool = False, + ) -> None: + super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] + self.can_colorize = _colorize.can_colorize() + + def showsyntaxerror(self, filename=None): + super().showsyntaxerror(colorize=self.can_colorize) + + def showtraceback(self): + super().showtraceback(colorize=self.can_colorize) + + def runsource(self, source, filename="<input>", symbol="single"): + try: + tree = ast.parse(source) + except (SyntaxError, OverflowError, ValueError): + self.showsyntaxerror(filename) + return False + if tree.body: + *_, last_stmt = tree.body + for stmt in tree.body: + wrapper = ast.Interactive if stmt is last_stmt else ast.Module + the_symbol = symbol if stmt is last_stmt else "exec" + item = wrapper([stmt]) + try: + code = self.compile.compiler(item, filename, the_symbol, dont_inherit=True) + except SyntaxError as e: + if e.args[0] == "'await' outside function": + python = os.path.basename(sys.executable) + e.add_note( + f"Try the asyncio REPL ({python} -m asyncio) to use" + f" top-level 'await' and run background asyncio tasks." + ) + self.showsyntaxerror(filename) + return False + except (OverflowError, ValueError): + self.showsyntaxerror(filename) + return False + + if code is None: + return True + + self.runcode(code) + return False diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 0045425cdddb79..5401ae7b0ae32d 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -131,6 +131,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]: ("\\\\", "self-insert"), (r"\x1b[200~", "enable_bracketed_paste"), (r"\x1b[201~", "disable_bracketed_paste"), + (r"\x03", "ctrl-c"), ] + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()] diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index c624f6e12a7094..256bbc7c6d7626 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -25,14 +25,13 @@ from __future__ import annotations -import _colorize # type: ignore[import-not-found] import _sitebuiltins import linecache import sys import code -import ast from types import ModuleType +from .console import InteractiveColoredConsole from .readline import _get_reader, multiline_input _error: tuple[type[Exception], ...] | type[Exception] @@ -74,57 +73,21 @@ def _clear_screen(): "clear": _clear_screen, } -class InteractiveColoredConsole(code.InteractiveConsole): - def __init__( - self, - locals: dict[str, object] | None = None, - filename: str = "<console>", - *, - local_exit: bool = False, - ) -> None: - super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] - self.can_colorize = _colorize.can_colorize() - - def showsyntaxerror(self, filename=None): - super().showsyntaxerror(colorize=self.can_colorize) - - def showtraceback(self): - super().showtraceback(colorize=self.can_colorize) - - def runsource(self, source, filename="<input>", symbol="single"): - try: - tree = ast.parse(source) - except (OverflowError, SyntaxError, ValueError): - self.showsyntaxerror(filename) - return False - if tree.body: - *_, last_stmt = tree.body - for stmt in tree.body: - wrapper = ast.Interactive if stmt is last_stmt else ast.Module - the_symbol = symbol if stmt is last_stmt else "exec" - item = wrapper([stmt]) - try: - code = compile(item, filename, the_symbol, dont_inherit=True) - except (OverflowError, ValueError, SyntaxError): - self.showsyntaxerror(filename) - return False - - if code is None: - return True - - self.runcode(code) - return False - def run_multiline_interactive_console( - mainmodule: ModuleType | None= None, future_flags: int = 0 + mainmodule: ModuleType | None = None, + future_flags: int = 0, + console: code.InteractiveConsole | None = None, ) -> None: import __main__ from .readline import _setup _setup() mainmodule = mainmodule or __main__ - console = InteractiveColoredConsole(mainmodule.__dict__, filename="<stdin>") + if console is None: + console = InteractiveColoredConsole( + mainmodule.__dict__, filename="<stdin>" + ) if future_flags: console.compile.compiler.flags |= future_flags diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 9041b8b8316c1e..91fff9aaee337b 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,42 +1,49 @@ import ast import asyncio -import code import concurrent.futures import inspect +import os import site import sys import threading import types import warnings +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] +from _pyrepl.console import InteractiveColoredConsole + from . import futures -class AsyncIOInteractiveConsole(code.InteractiveConsole): +class AsyncIOInteractiveConsole(InteractiveColoredConsole): def __init__(self, locals, loop): - super().__init__(locals) + super().__init__(locals, filename="<stdin>") self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT self.loop = loop def runcode(self, code): + global return_code future = concurrent.futures.Future() def callback(): + global return_code global repl_future - global repl_future_interrupted + global keyboard_interrupted repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False func = types.FunctionType(code, self.locals) try: coro = func() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except KeyboardInterrupt as ex: - repl_future_interrupted = True + keyboard_interrupted = True future.set_exception(ex) return except BaseException as ex: @@ -57,10 +64,12 @@ def callback(): try: return future.result() - except SystemExit: - raise + except SystemExit as se: + return_code = se.code + self.loop.stop() + return except BaseException: - if repl_future_interrupted: + if keyboard_interrupted: self.write("\nKeyboardInterrupt\n") else: self.showtraceback() @@ -69,18 +78,56 @@ def callback(): class REPLThread(threading.Thread): def run(self): + global return_code + try: banner = ( f'asyncio REPL {sys.version} on {sys.platform}\n' f'Use "await" directly instead of "asyncio.run()".\n' f'Type "help", "copyright", "credits" or "license" ' f'for more information.\n' - f'{getattr(sys, "ps1", ">>> ")}import asyncio' ) - console.interact( - banner=banner, - exitmsg='exiting asyncio REPL...') + console.write(banner) + + if startup_path := os.getenv("PYTHONSTARTUP"): + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code, console.locals) + + ps1 = getattr(sys, "ps1", ">>> ") + if can_colorize(): + ps1 = f"{ANSIColors.BOLD_MAGENTA}{ps1}{ANSIColors.RESET}" + console.write(f"{ps1}import asyncio\n") + + try: + import errno + if os.getenv("PYTHON_BASIC_REPL"): + raise RuntimeError("user environment requested basic REPL") + if not os.isatty(sys.stdin.fileno()): + raise OSError(errno.ENOTTY, "tty required", "stdin") + + # This import will fail on operating systems with no termios. + from _pyrepl.simple_interact import ( + check, + run_multiline_interactive_console, + ) + if err := check(): + raise RuntimeError(err) + except Exception as e: + console.interact(banner="", exitmsg=exit_message) + else: + try: + run_multiline_interactive_console(console=console) + except SystemExit: + # expected via the `exit` and `quit` commands + pass + except BaseException: + # unexpected issue + console.showtraceback() + console.write("Internal error, ") + return_code = 1 finally: warnings.filterwarnings( 'ignore', @@ -91,6 +138,9 @@ def run(self): if __name__ == '__main__': + CAN_USE_PYREPL = True + + return_code = 0 loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -103,7 +153,7 @@ def run(self): console = AsyncIOInteractiveConsole(repl_locals, loop) repl_future = None - repl_future_interrupted = False + keyboard_interrupted = False try: import readline # NoQA @@ -126,7 +176,7 @@ def run(self): completer = rlcompleter.Completer(console.locals) readline.set_completer(completer.complete) - repl_thread = REPLThread() + repl_thread = REPLThread(name="Interactive thread") repl_thread.daemon = True repl_thread.start() @@ -134,9 +184,12 @@ def run(self): try: loop.run_forever() except KeyboardInterrupt: + keyboard_interrupted = True if repl_future and not repl_future.done(): repl_future.cancel() - repl_future_interrupted = True continue else: break + + console.write('exiting asyncio REPL...\n') + sys.exit(return_code) diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index 4d01ea7620109d..df97b1354a168e 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -6,7 +6,7 @@ from test.support import force_not_colorized -from _pyrepl.simple_interact import InteractiveColoredConsole +from _pyrepl.console import InteractiveColoredConsole class TestSimpleInteract(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst b/Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst new file mode 100644 index 00000000000000..ffc4ae336dc54f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-22-21-20-43.gh-issue-118894.xHdxR_.rst @@ -0,0 +1 @@ +:mod:`asyncio` REPL now has the same capabilities as PyREPL.

1 0

gh-119853: Add Include/refcount.h to projects (#119860)
by vstinner 31 May '24

31 May '24

https://github.com/python/cpython/commit/f9d47fed9fbbe9313404838050f6dfe1c7… commit: f9d47fed9fbbe9313404838050f6dfe1c7fe6340 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2024-05-31T21:21:30+02:00 summary: gh-119853: Add Include/refcount.h to projects (#119860) files: M Makefile.pre.in M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters diff --git a/Makefile.pre.in b/Makefile.pre.in index a3fca80d4448ca..9a2fc34f030662 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1055,6 +1055,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/pythread.h \ $(srcdir)/Include/pytypedefs.h \ $(srcdir)/Include/rangeobject.h \ + $(srcdir)/Include/refcount.h \ $(srcdir)/Include/setobject.h \ $(srcdir)/Include/sliceobject.h \ $(srcdir)/Include/structmember.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 16fb424b11c6a8..96960f0579a936 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -349,6 +349,7 @@ <ClInclude Include="..\Include\pythread.h" /> <ClInclude Include="..\Include\pytypedefs.h" /> <ClInclude Include="..\Include\rangeobject.h" /> + <ClInclude Include="..\Include\refcount.h" /> <ClInclude Include="..\Include\setobject.h" /> <ClInclude Include="..\Include\sliceobject.h" /> <ClInclude Include="..\Include\structmember.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index cf9bc0f4bc1c70..2e4bd786be47e9 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -207,6 +207,9 @@ <ClInclude Include="..\Include\rangeobject.h"> <Filter>Include</Filter> </ClInclude> + <ClInclude Include="..\Include\refcount.h"> + <Filter>Include</Filter> + </ClInclude> <ClInclude Include="..\Include\setobject.h"> <Filter>Include</Filter> </ClInclude>

1 0

[3.13] gh-119799: Add missing `_Py_IncRefTotal` to `_Py_NewRefWithLock` (GH-119800) (#119878)
by colesbury 31 May '24

31 May '24

https://github.com/python/cpython/commit/67ac19111f14e9606454d0f36a52132996… commit: 67ac19111f14e9606454d0f36a52132996073db4 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: colesbury <colesbury(a)gmail.com> date: 2024-05-31T19:08:21Z summary: [3.13] gh-119799: Add missing `_Py_IncRefTotal` to `_Py_NewRefWithLock` (GH-119800) (#119878) The free-threaded refleak builds were reporting negative refcount deltas in some tests because of a missing `_Py_NewRefWithLock`. (cherry picked from commit 879d43b705faab0c59f1a6a0042e286f39f3a4ef) Co-authored-by: Sam Gross <colesbury(a)gmail.com> files: M Include/internal/pycore_object.h diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 7602248f956405..f63e1da6fba025 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -497,6 +497,9 @@ _Py_NewRefWithLock(PyObject *op) if (_Py_TryIncrefFast(op)) { return op; } +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif _Py_INCREF_STAT_INC(); for (;;) { Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);

1 0

[3.13] gh-119369: Fix deadlock during thread exit in free-threaded build (GH-119528) (#119868)
by colesbury 31 May '24

31 May '24

https://github.com/python/cpython/commit/a7e81fdfc175bd9cf82dfd8f1e1853cb94… commit: a7e81fdfc175bd9cf82dfd8f1e1853cb942bf0f0 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: colesbury <colesbury(a)gmail.com> date: 2024-05-31T14:19:38-04:00 summary: [3.13] gh-119369: Fix deadlock during thread exit in free-threaded build (GH-119528) (#119868) Release the GIL before calling `_Py_qsbr_unregister`. The deadlock could occur when the GIL was enabled at runtime. The `_Py_qsbr_unregister` call might block while holding the GIL because the thread state was not active, but the GIL was still held. (cherry picked from commit 078b8c8cf2bf68f7484cc4d2e3dd74b6fab55664) Co-authored-by: Sam Gross <colesbury(a)gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-16-52.gh-issue-119369.qBThho.rst M Python/pystate.c M Python/qsbr.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-16-52.gh-issue-119369.qBThho.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-16-52.gh-issue-119369.qBThho.rst new file mode 100644 index 00000000000000..7abdd5cd85ccd6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-24-21-16-52.gh-issue-119369.qBThho.rst @@ -0,0 +1,2 @@ +Fix deadlock during thread deletion in free-threaded build, which could +occur when the GIL was enabled at runtime. diff --git a/Python/pystate.c b/Python/pystate.c index ad7e082ce0d37e..36e4206b4a282e 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1751,7 +1751,7 @@ decrement_stoptheworld_countdown(struct _stoptheworld_state *stw); /* Common code for PyThreadState_Delete() and PyThreadState_DeleteCurrent() */ static void -tstate_delete_common(PyThreadState *tstate) +tstate_delete_common(PyThreadState *tstate, int release_gil) { assert(tstate->_status.cleared && !tstate->_status.finalized); tstate_verify_not_active(tstate); @@ -1793,10 +1793,6 @@ tstate_delete_common(PyThreadState *tstate) HEAD_UNLOCK(runtime); -#ifdef Py_GIL_DISABLED - _Py_qsbr_unregister(tstate); -#endif - // XXX Unbind in PyThreadState_Clear(), or earlier // (and assert not-equal here)? if (tstate->_status.bound_gilstate) { @@ -1807,6 +1803,14 @@ tstate_delete_common(PyThreadState *tstate) // XXX Move to PyThreadState_Clear()? clear_datastack(tstate); + if (release_gil) { + _PyEval_ReleaseLock(tstate->interp, tstate, 1); + } + +#ifdef Py_GIL_DISABLED + _Py_qsbr_unregister(tstate); +#endif + tstate->_status.finalized = 1; } @@ -1818,7 +1822,7 @@ zapthreads(PyInterpreterState *interp) when the threads are all really dead (XXX famous last words). */ while ((tstate = interp->threads.head) != NULL) { tstate_verify_not_active(tstate); - tstate_delete_common(tstate); + tstate_delete_common(tstate, 0); free_threadstate((_PyThreadStateImpl *)tstate); } } @@ -1829,7 +1833,7 @@ PyThreadState_Delete(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); tstate_verify_not_active(tstate); - tstate_delete_common(tstate); + tstate_delete_common(tstate, 0); free_threadstate((_PyThreadStateImpl *)tstate); } @@ -1842,8 +1846,7 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); #endif current_fast_clear(tstate->interp->runtime); - tstate_delete_common(tstate); - _PyEval_ReleaseLock(tstate->interp, tstate, 1); + tstate_delete_common(tstate, 1); // release GIL as part of call free_threadstate((_PyThreadStateImpl *)tstate); } diff --git a/Python/qsbr.c b/Python/qsbr.c index 1e02ff9c2e45f0..9cbce9044e2941 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -236,6 +236,11 @@ _Py_qsbr_unregister(PyThreadState *tstate) struct _qsbr_shared *shared = &tstate->interp->qsbr; struct _PyThreadStateImpl *tstate_imp = (_PyThreadStateImpl*) tstate; + // gh-119369: GIL must be released (if held) to prevent deadlocks, because + // we might not have an active tstate, which means taht blocking on PyMutex + // locks will not implicitly release the GIL. + assert(!tstate->_status.holds_gil); + PyMutex_Lock(&shared->mutex); // NOTE: we must load (or reload) the thread state's qbsr inside the mutex // because the array may have been resized (changing tstate->qsbr) while

1 0

gh-119799: Add missing `_Py_IncRefTotal` to `_Py_NewRefWithLock` (#119800)
by colesbury 31 May '24

31 May '24

https://github.com/python/cpython/commit/879d43b705faab0c59f1a6a0042e286f39… commit: 879d43b705faab0c59f1a6a0042e286f39f3a4ef branch: main author: Sam Gross <colesbury(a)gmail.com> committer: colesbury <colesbury(a)gmail.com> date: 2024-05-31T14:18:24-04:00 summary: gh-119799: Add missing `_Py_IncRefTotal` to `_Py_NewRefWithLock` (#119800) The free-threaded refleak builds were reporting negative refcount deltas in some tests because of a missing `_Py_NewRefWithLock`. files: M Include/internal/pycore_object.h diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 7602248f956405..f63e1da6fba025 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -497,6 +497,9 @@ _Py_NewRefWithLock(PyObject *op) if (_Py_TryIncrefFast(op)) { return op; } +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif _Py_INCREF_STAT_INC(); for (;;) { Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);

1 0

[3.13] doc: Add glossary entry for "free threading" (GH-119865) (#119874)
by colesbury 31 May '24

31 May '24

https://github.com/python/cpython/commit/5e8396e6841e3aae8cc10117583e596ff2… commit: 5e8396e6841e3aae8cc10117583e596ff294d7e0 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: colesbury <colesbury(a)gmail.com> date: 2024-05-31T17:42:36Z summary: [3.13] doc: Add glossary entry for "free threading" (GH-119865) (#119874) (cherry picked from commit 9bc6045842ebc91ec48ab163a9e1e8644231607c) Co-authored-by: Sam Gross <colesbury(a)gmail.com> files: M Doc/glossary.rst diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 1e5bafce861e52..ae9949bc2867c4 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -438,6 +438,12 @@ Glossary division. Note that ``(-11) // 4`` is ``-3`` because that is ``-2.75`` rounded *downward*. See :pep:`238`. + free threading + A threading model where multiple threads can run Python bytecode + simultaneously within the same interpreter. This is in contrast to + the :term:`global interpreter lock` which allows only one thread to + execute Python bytecode at a time. See :pep:`703`. + function A series of statements which returns some value to a caller. It can also be passed zero or more :term:`arguments <argument>` which may be used in

1 0