Mailman 3 January 2025 - Python-checkins

gh-91048: Also clear and set ts->asyncio_running_task with eager tasks (#129197)
by ambv Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/fc3d40016567abd0685131b768c8112e00… commit: fc3d40016567abd0685131b768c8112e0084c238 branch: main author: Łukasz Langa <lukasz(a)langa.pl> committer: ambv <lukasz(a)langa.pl> date: 2025-01-23T19:26:36+01:00 summary: gh-91048: Also clear and set ts->asyncio_running_task with eager tasks (#129197) This was missing from gh-124640. It's already covered by the new test_asyncio/test_free_threading.py in combination with the runtime assertion in set_ts_asyncio_running_task. Co-authored-by: Kumar Aditya <kumaraditya(a)python.org> files: M Modules/_asynciomodule.c diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index bba7416b398101..c821860d9e4f70 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2063,6 +2063,8 @@ static int task_call_step_soon(asyncio_state *state, TaskObj *, PyObject *); static PyObject * task_wakeup(TaskObj *, PyObject *); static PyObject * task_step(asyncio_state *, TaskObj *, PyObject *); static int task_eager_start(asyncio_state *state, TaskObj *task); +static inline void clear_ts_asyncio_running_task(PyObject *loop); +static inline void set_ts_asyncio_running_task(PyObject *loop, PyObject *task); /* ----- Task._step wrapper */ @@ -2236,47 +2238,7 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) assert(task == item); Py_CLEAR(item); - - // This block is needed to enable `asyncio.capture_call_graph()` API. - // We want to be enable debuggers and profilers to be able to quickly - // introspect the asyncio running state from another process. - // When we do that, we need to essentially traverse the address space - // of a Python process and understand what every Python thread in it is - // currently doing, mainly: - // - // * current frame - // * current asyncio task - // - // A naive solution would be to require profilers and debuggers to - // find the current task in the "_asynciomodule" module state, but - // unfortunately that would require a lot of complicated remote - // memory reads and logic, as Python's dict is a notoriously complex - // and ever-changing data structure. - // - // So the easier solution is to put a strong reference to the currently - // running `asyncio.Task` on the interpreter thread state (we already - // have some asyncio state there.) - _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); - if (ts->asyncio_running_loop == loop) { - // Protect from a situation when someone calls this method - // from another thread. This shouldn't ever happen though, - // as `enter_task` and `leave_task` can either be called by: - // - // - `asyncio.Task` itself, in `Task.__step()`. That method - // can only be called by the event loop itself. - // - // - third-party Task "from scratch" implementations, that - // our `capture_call_graph` API doesn't support anyway. - // - // That said, we still want to make sure we don't end up in - // a broken state, so we check that we're in the correct thread - // by comparing the *loop* argument to the event loop running - // in the current thread. If they match we know we're in the - // right thread, as asyncio event loops don't change threads. - assert(ts->asyncio_running_task == NULL); - ts->asyncio_running_task = Py_NewRef(task); - } - + set_ts_asyncio_running_task(loop, task); return 0; } @@ -2308,14 +2270,7 @@ leave_task(asyncio_state *state, PyObject *loop, PyObject *task) // task was not found return err_leave_task(Py_None, task); } - - // See the comment in `enter_task` for the explanation of why - // the following is needed. - _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); - if (ts->asyncio_running_loop == NULL || ts->asyncio_running_loop == loop) { - Py_CLEAR(ts->asyncio_running_task); - } - + clear_ts_asyncio_running_task(loop); return res; } @@ -2342,6 +2297,7 @@ swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) { PyObject *prev_task; + clear_ts_asyncio_running_task(loop); if (task == Py_None) { if (PyDict_Pop(state->current_tasks, loop, &prev_task) < 0) { return NULL; @@ -2361,9 +2317,63 @@ swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) Py_BEGIN_CRITICAL_SECTION(current_tasks); prev_task = swap_current_task_lock_held(current_tasks, loop, hash, task); Py_END_CRITICAL_SECTION(); + set_ts_asyncio_running_task(loop, task); return prev_task; } +static inline void +set_ts_asyncio_running_task(PyObject *loop, PyObject *task) +{ + // We want to enable debuggers and profilers to be able to quickly + // introspect the asyncio running state from another process. + // When we do that, we need to essentially traverse the address space + // of a Python process and understand what every Python thread in it is + // currently doing, mainly: + // + // * current frame + // * current asyncio task + // + // A naive solution would be to require profilers and debuggers to + // find the current task in the "_asynciomodule" module state, but + // unfortunately that would require a lot of complicated remote + // memory reads and logic, as Python's dict is a notoriously complex + // and ever-changing data structure. + // + // So the easier solution is to put a strong reference to the currently + // running `asyncio.Task` on the current thread state (the current loop + // is also stored there.) + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (ts->asyncio_running_loop == loop) { + // Protect from a situation when someone calls this method + // from another thread. This shouldn't ever happen though, + // as `enter_task` and `leave_task` can either be called by: + // + // - `asyncio.Task` itself, in `Task.__step()`. That method + // can only be called by the event loop itself. + // + // - third-party Task "from scratch" implementations, that + // our `capture_call_graph` API doesn't support anyway. + // + // That said, we still want to make sure we don't end up in + // a broken state, so we check that we're in the correct thread + // by comparing the *loop* argument to the event loop running + // in the current thread. If they match we know we're in the + // right thread, as asyncio event loops don't change threads. + assert(ts->asyncio_running_task == NULL); + ts->asyncio_running_task = Py_NewRef(task); + } +} + +static inline void +clear_ts_asyncio_running_task(PyObject *loop) +{ + // See comment in set_ts_asyncio_running_task() for details. + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (ts->asyncio_running_loop == NULL || ts->asyncio_running_loop == loop) { + Py_CLEAR(ts->asyncio_running_task); + } +} + /* ----- Task */ /*[clinic input]

1 0

gh-71339: Use new assertion methods in test_idle (#129213)
by terryjreedy Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/dbb25ce037a8eddc85a9f6ba541aec8b3f… commit: dbb25ce037a8eddc85a9f6ba541aec8b3fa931fa branch: main author: Terry Jan Reedy <tjreedy(a)udel.edu> committer: terryjreedy <tjreedy(a)udel.edu> date: 2025-01-23T13:25:14-05:00 summary: gh-71339: Use new assertion methods in test_idle (#129213) Revise 10 tests in 7 files, with 1 test split into 2. files: M Lib/idlelib/idle_test/test_configdialog.py M Lib/idlelib/idle_test/test_debugger.py M Lib/idlelib/idle_test/test_grep.py M Lib/idlelib/idle_test/test_multicall.py M Lib/idlelib/idle_test/test_query.py M Lib/idlelib/idle_test/test_redirector.py M Lib/idlelib/idle_test/test_sidebar.py diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 5099d093382445..2773ed7ce614b5 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -98,8 +98,8 @@ def test_click_help(self): dialog.buttons['Help'].invoke() title, contents = view.kwds['title'], view.kwds['contents'] self.assertEqual(title, 'Help for IDLE preferences') - self.assertTrue(contents.startswith('When you click') and - contents.endswith('a different name.\n')) + self.assertStartsWith(contents, 'When you click') + self.assertEndsWith(contents,'a different name.\n') class FontPageTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_debugger.py b/Lib/idlelib/idle_test/test_debugger.py index d1c9638dd5d711..9ca3b332648b31 100644 --- a/Lib/idlelib/idle_test/test_debugger.py +++ b/Lib/idlelib/idle_test/test_debugger.py @@ -256,7 +256,7 @@ def test_init(self): flist = None master_window = self.root sv = debugger.StackViewer(master_window, flist, gui) - self.assertTrue(hasattr(sv, 'stack')) + self.assertHasAttr(sv, 'stack') def test_load_stack(self): # Test the .load_stack() method against a fixed test stack. diff --git a/Lib/idlelib/idle_test/test_grep.py b/Lib/idlelib/idle_test/test_grep.py index a0b5b69171879c..d67dba76911fcf 100644 --- a/Lib/idlelib/idle_test/test_grep.py +++ b/Lib/idlelib/idle_test/test_grep.py @@ -143,7 +143,7 @@ def test_found(self): self.assertIn(pat, lines[0]) self.assertIn('py: 1:', lines[1]) # line number 1 self.assertIn('2', lines[3]) # hits found 2 - self.assertTrue(lines[4].startswith('(Hint:')) + self.assertStartsWith(lines[4], '(Hint:') class Default_commandTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_multicall.py b/Lib/idlelib/idle_test/test_multicall.py index b3a3bfb88f9c31..67f28db6b0875c 100644 --- a/Lib/idlelib/idle_test/test_multicall.py +++ b/Lib/idlelib/idle_test/test_multicall.py @@ -27,7 +27,7 @@ def tearDownClass(cls): def test_creator(self): mc = self.mc self.assertIs(multicall._multicall_dict[Text], mc) - self.assertTrue(issubclass(mc, Text)) + self.assertIsSubclass(mc, Text) mc2 = multicall.MultiCallCreator(Text) self.assertIs(mc, mc2) diff --git a/Lib/idlelib/idle_test/test_query.py b/Lib/idlelib/idle_test/test_query.py index bb12b2b08652d5..a6ef858a8c954a 100644 --- a/Lib/idlelib/idle_test/test_query.py +++ b/Lib/idlelib/idle_test/test_query.py @@ -134,10 +134,10 @@ def test_c_source_name(self): def test_good_module_name(self): dialog = self.Dummy_ModuleName('idlelib') - self.assertTrue(dialog.entry_ok().endswith('__init__.py')) + self.assertEndsWith(dialog.entry_ok(), '__init__.py') self.assertEqual(dialog.entry_error['text'], '') dialog = self.Dummy_ModuleName('idlelib.idle') - self.assertTrue(dialog.entry_ok().endswith('idle.py')) + self.assertEndsWith(dialog.entry_ok(), 'idle.py') self.assertEqual(dialog.entry_error['text'], '') @@ -389,7 +389,7 @@ def test_click_module_name(self): self.assertEqual(dialog.text0, 'idlelib') self.assertEqual(dialog.entry.get(), 'idlelib') dialog.button_ok.invoke() - self.assertTrue(dialog.result.endswith('__init__.py')) + self.assertEndsWith(dialog.result, '__init__.py') root.destroy() diff --git a/Lib/idlelib/idle_test/test_redirector.py b/Lib/idlelib/idle_test/test_redirector.py index a97b3002afcf12..bd486d7da66010 100644 --- a/Lib/idlelib/idle_test/test_redirector.py +++ b/Lib/idlelib/idle_test/test_redirector.py @@ -34,7 +34,7 @@ def test_close(self): redir.register('insert', Func) redir.close() self.assertEqual(redir._operations, {}) - self.assertFalse(hasattr(self.text, 'widget')) + self.assertNotHasAttr(self.text, 'widget') class WidgetRedirectorTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_sidebar.py b/Lib/idlelib/idle_test/test_sidebar.py index 605e7a892570d7..4157a4b4dcdd2a 100644 --- a/Lib/idlelib/idle_test/test_sidebar.py +++ b/Lib/idlelib/idle_test/test_sidebar.py @@ -725,7 +725,7 @@ def test_copy(self): text.tag_add('sel', f'{first_line}.0', 'end-1c') selected_text = text.get('sel.first', 'sel.last') - self.assertTrue(selected_text.startswith('if True:\n')) + self.assertStartsWith(selected_text, 'if True:\n') self.assertIn('\n1\n', selected_text) text.event_generate('<<copy>>') @@ -749,7 +749,7 @@ def test_copy_with_prompts(self): text.tag_add('sel', f'{first_line}.3', 'end-1c') selected_text = text.get('sel.first', 'sel.last') - self.assertTrue(selected_text.startswith('True:\n')) + self.assertStartsWith(selected_text, 'True:\n') selected_lines_text = text.get('sel.first linestart', 'sel.last') selected_lines = selected_lines_text.split('\n')

1 0

gh-91048: Improve example in `asyncio-graph.rst` doc (#129224)
by kumaraditya303 Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/c05a851ac59e6fb7bd433677b9c116fb83… commit: c05a851ac59e6fb7bd433677b9c116fb8336a8b1 branch: main author: sobolevn <mail(a)sobolevn.me> committer: kumaraditya303 <kumaraditya(a)python.org> date: 2025-01-23T23:38:51+05:30 summary: gh-91048: Improve example in `asyncio-graph.rst` doc (#129224) files: M Doc/library/asyncio-graph.rst diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index fc8edeb426c567..814bfe6e269bf9 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -59,13 +59,13 @@ and debuggers. async def main(): async with asyncio.TaskGroup() as g: - g.create_task(test()) + g.create_task(test(), name='test') asyncio.run(main()) will print:: - * Task(name='Task-2', id=0x1039f0fe0) + * Task(name='test', id=0x1039f0fe0) + Call stack: | File 't2.py', line 4, in async test() + Awaited by:

1 0

[3.13] gh-128330: Terminal control characters should be restored on repl exit (GH-128331) (#129233)
by pablogsal Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/52ccf264dfc29e28c24b2445c69d63c999… commit: 52ccf264dfc29e28c24b2445c69d63c99989b3c7 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: pablogsal <Pablogsal(a)gmail.com> date: 2025-01-23T17:55:28Z summary: [3.13] gh-128330: Terminal control characters should be restored on repl exit (GH-128331) (#129233) gh-128330: Terminal control characters should be restored on repl exit (GH-128331) (cherry picked from commit 0b15d9c0d2d30c7d3f17ebb90dd822ef32f977cc) Co-authored-by: Andy Fiddaman <andy(a)omnios.org> files: A Misc/NEWS.d/next/Core_and_Builtins/2024-12-29-15-09-21.gh-issue-128330.IaYL7G.rst M Lib/_pyrepl/fancy_termios.py diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py index 5b85cb0f52521f..0468b9a2670267 100644 --- a/Lib/_pyrepl/fancy_termios.py +++ b/Lib/_pyrepl/fancy_termios.py @@ -40,7 +40,9 @@ def as_list(self): self.lflag, self.ispeed, self.ospeed, - self.cc, + # Always return a copy of the control characters list to ensure + # there are not any additional references to self.cc + self.cc[:], ] def copy(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-29-15-09-21.gh-issue-128330.IaYL7G.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-29-15-09-21.gh-issue-128330.IaYL7G.rst new file mode 100644 index 00000000000000..8fe628d18f4bd4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-29-15-09-21.gh-issue-128330.IaYL7G.rst @@ -0,0 +1 @@ +Restore terminal control characters on REPL exit.

1 0

gh-128330: Terminal control characters should be restored on repl exit (#128331)
by pablogsal Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/0b15d9c0d2d30c7d3f17ebb90dd822ef32… commit: 0b15d9c0d2d30c7d3f17ebb90dd822ef32f977cc branch: main author: Andy Fiddaman <andy(a)omnios.org> committer: pablogsal <Pablogsal(a)gmail.com> date: 2025-01-23T17:30:08Z summary: gh-128330: Terminal control characters should be restored on repl exit (#128331) files: A Misc/NEWS.d/next/Core_and_Builtins/2024-12-29-15-09-21.gh-issue-128330.IaYL7G.rst M Lib/_pyrepl/fancy_termios.py diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py index 5b85cb0f52521f..0468b9a2670267 100644 --- a/Lib/_pyrepl/fancy_termios.py +++ b/Lib/_pyrepl/fancy_termios.py @@ -40,7 +40,9 @@ def as_list(self): self.lflag, self.ispeed, self.ospeed, - self.cc, + # Always return a copy of the control characters list to ensure + # there are not any additional references to self.cc + self.cc[:], ] def copy(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-29-15-09-21.gh-issue-128330.IaYL7G.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-29-15-09-21.gh-issue-128330.IaYL7G.rst new file mode 100644 index 00000000000000..8fe628d18f4bd4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-29-15-09-21.gh-issue-128330.IaYL7G.rst @@ -0,0 +1 @@ +Restore terminal control characters on REPL exit.

1 0

[3.9] gh-121277: Allow .. versionadded:: next in docs (GH-121278) (#128117)
by ambv Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/8c8a257dbc0e393125040404f973853d41… commit: 8c8a257dbc0e393125040404f973853d41e81d84 branch: 3.9 author: Petr Viktorin <encukou(a)gmail.com> committer: ambv <lukasz(a)langa.pl> date: 2025-01-23T16:55:08+01:00 summary: [3.9] gh-121277: Allow .. versionadded:: next in docs (GH-121278) (#128117) Make `versionchanged:: next`` expand to current (unreleased) version. When a new CPython release is cut, the release manager will replace all such occurences of "next" with the just-released version. (See the issue for release-tools and devguide PRs.) Co-authored-by: Adam Turner <9087854+AA-Turner(a)users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk(a)users.noreply.github.com> (cherry picked from commit 7d24ea9db3e8fdca52058629c9ba577aba3d8e5c) gh-121277: Raise nice error on `next` as second argument to deprecated-removed (GH-124623) (cherry-picked from e349f73a5ad2856b0a7cbe4aef7cc081c7aed777) (cherry-picked from 3.11: f0895aa9c1d40d0add673cc51bd143556e22100a) (cherry-picked from 3.10: 8773554b717cfb08b4bd11a927813f4ed74762c7) files: A Misc/NEWS.d/next/Documentation/2024-07-19-12-22-48.gh-issue-121277.wF_zKd.rst M Doc/tools/extensions/pyspecific.py diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 4575300f53769d..d4bda83a60634e 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -22,6 +22,7 @@ from sphinx import addnodes from sphinx.builders import Builder +from sphinx.domains.changeset import VersionChange try: from sphinx.errors import NoUri except ImportError: @@ -332,7 +333,22 @@ def run(self): return PyMethod.run(self) -# Support for documenting version of removal in deprecations +# Support for documenting version of changes, additions, deprecations + +def expand_version_arg(argument, release): + """Expand "next" to the current version""" + if argument == 'next': + return translators['sphinx'].gettext('{} (unreleased)').format(release) + return argument + + +class PyVersionChange(VersionChange): + def run(self): + # Replace the 'next' special token with the current development version + self.arguments[0] = expand_version_arg(self.arguments[0], + self.config.release) + return super().run() + class DeprecatedRemoved(Directive): has_content = True @@ -348,18 +364,24 @@ def run(self): node = addnodes.versionmodified() node.document = self.state.document node['type'] = 'deprecated-removed' - version = (self.arguments[0], self.arguments[1]) - node['version'] = version env = self.state.document.settings.env + version = ( + expand_version_arg(self.arguments[0], env.config.release), + self.arguments[1], + ) + if version[1] == 'next': + raise ValueError( + 'deprecated-removed:: second argument cannot be `next`') + node['version'] = version current_version = tuple(int(e) for e in env.config.version.split('.')) - removed_version = tuple(int(e) for e in self.arguments[1].split('.')) + removed_version = tuple(int(e) for e in version[1].split('.')) if current_version < removed_version: label = self._deprecated_label else: label = self._removed_label label = translators['sphinx'].gettext(label) - text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) + text = label.format(deprecated=version[0], removed=version[1]) if len(self.arguments) == 3: inodes, messages = self.state.inline_text(self.arguments[2], self.lineno+1) @@ -607,6 +629,10 @@ def setup(app): app.add_directive('availability', Availability) app.add_directive('audit-event', AuditEvent) app.add_directive('audit-event-table', AuditEventListDirective) + app.add_directive('versionadded', PyVersionChange, override=True) + app.add_directive('versionchanged', PyVersionChange, override=True) + app.add_directive('versionremoved', PyVersionChange, override=True) + app.add_directive('deprecated', PyVersionChange, override=True) app.add_directive('deprecated-removed', DeprecatedRemoved) app.add_builder(PydocTopicsBuilder) app.add_builder(suspicious.CheckSuspiciousMarkupBuilder) diff --git a/Misc/NEWS.d/next/Documentation/2024-07-19-12-22-48.gh-issue-121277.wF_zKd.rst b/Misc/NEWS.d/next/Documentation/2024-07-19-12-22-48.gh-issue-121277.wF_zKd.rst new file mode 100644 index 00000000000000..60f75ae0c21326 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-07-19-12-22-48.gh-issue-121277.wF_zKd.rst @@ -0,0 +1,2 @@ +Writers of CPython's documentation can now use ``next`` as the version for +the ``versionchanged``, ``versionadded``, ``deprecated`` directives.

1 0

gh-128479: fix asyncio staggered race leaking tasks, and logging unhandled exception.append exception (#128475)
by ambv Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/ec91e1c2762412f1408b0dfb5d281873b8… commit: ec91e1c2762412f1408b0dfb5d281873b852affe branch: main author: Thomas Grainger <tagrain(a)gmail.com> committer: ambv <lukasz(a)langa.pl> date: 2025-01-23T16:53:53+01:00 summary: gh-128479: fix asyncio staggered race leaking tasks, and logging unhandled exception.append exception (#128475) Co-authored-by: Peter Bierma <zintensitydev(a)gmail.com> files: A Misc/NEWS.d/next/Library/2025-01-04-11-10-04.gh-issue-128479.jvOrF-.rst M Lib/asyncio/staggered.py M Lib/test/test_asyncio/test_staggered.py diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 0f4df8855a80b9..0afed64fdf9c0f 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -66,8 +66,27 @@ async def staggered_race(coro_fns, delay, *, loop=None): enum_coro_fns = enumerate(coro_fns) winner_result = None winner_index = None + unhandled_exceptions = [] exceptions = [] - running_tasks = [] + running_tasks = set() + on_completed_fut = None + + def task_done(task): + running_tasks.discard(task) + if ( + on_completed_fut is not None + and not on_completed_fut.done() + and not running_tasks + ): + on_completed_fut.set_result(None) + + if task.cancelled(): + return + + exc = task.exception() + if exc is None: + return + unhandled_exceptions.append(exc) async def run_one_coro(ok_to_start, previous_failed) -> None: # in eager tasks this waits for the calling task to append this task @@ -91,11 +110,11 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: this_failed = locks.Event() next_ok_to_start = locks.Event() next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed)) - running_tasks.append(next_task) + running_tasks.add(next_task) + next_task.add_done_callback(task_done) # next_task has been appended to running_tasks so next_task is ok to # start. next_ok_to_start.set() - assert len(running_tasks) == this_index + 2 # Prepare place to put this coroutine's exceptions if not won exceptions.append(None) assert len(exceptions) == this_index + 1 @@ -120,31 +139,36 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: # up as done() == True, cancelled() == False, exception() == # asyncio.CancelledError. This behavior is specified in # https://bugs.python.org/issue30048 - for i, t in enumerate(running_tasks): - if i != this_index: + current_task = tasks.current_task(loop) + for t in running_tasks: + if t is not current_task: t.cancel() - ok_to_start = locks.Event() - first_task = loop.create_task(run_one_coro(ok_to_start, None)) - running_tasks.append(first_task) - # first_task has been appended to running_tasks so first_task is ok to start. - ok_to_start.set() + propagate_cancellation_error = None try: - # Wait for a growing list of tasks to all finish: poor man's version of - # curio's TaskGroup or trio's nursery - done_count = 0 - while done_count != len(running_tasks): - done, _ = await tasks.wait(running_tasks) - done_count = len(done) + ok_to_start = locks.Event() + first_task = loop.create_task(run_one_coro(ok_to_start, None)) + running_tasks.add(first_task) + first_task.add_done_callback(task_done) + # first_task has been appended to running_tasks so first_task is ok to start. + ok_to_start.set() + propagate_cancellation_error = None + # Make sure no tasks are left running if we leave this function + while running_tasks: + on_completed_fut = loop.create_future() + try: + await on_completed_fut + except exceptions_mod.CancelledError as ex: + propagate_cancellation_error = ex + for task in running_tasks: + task.cancel(*ex.args) + on_completed_fut = None + if __debug__ and unhandled_exceptions: # If run_one_coro raises an unhandled exception, it's probably a # programming error, and I want to see it. - if __debug__: - for d in done: - if d.done() and not d.cancelled() and d.exception(): - raise d.exception() + raise ExceptionGroup("staggered race failed", unhandled_exceptions) + if propagate_cancellation_error is not None: + raise propagate_cancellation_error return winner_result, winner_index, exceptions finally: - del exceptions - # Make sure no tasks are left running if we leave this function - for t in running_tasks: - t.cancel() + del exceptions, propagate_cancellation_error, unhandled_exceptions diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py index 3c81b629693596..ad34aa6da01f54 100644 --- a/Lib/test/test_asyncio/test_staggered.py +++ b/Lib/test/test_asyncio/test_staggered.py @@ -122,3 +122,30 @@ async def do_set(): self.assertIsNone(excs[0], None) self.assertIsInstance(excs[1], asyncio.CancelledError) self.assertIsInstance(excs[2], asyncio.CancelledError) + + + async def test_cancelled(self): + log = [] + with self.assertRaises(TimeoutError): + async with asyncio.timeout(None) as cs_outer, asyncio.timeout(None) as cs_inner: + async def coro_fn(): + cs_inner.reschedule(-1) + await asyncio.sleep(0) + try: + await asyncio.sleep(0) + except asyncio.CancelledError: + log.append("cancelled 1") + + cs_outer.reschedule(-1) + await asyncio.sleep(0) + try: + await asyncio.sleep(0) + except asyncio.CancelledError: + log.append("cancelled 2") + try: + await staggered_race([coro_fn], delay=None) + except asyncio.CancelledError: + log.append("cancelled 3") + raise + + self.assertListEqual(log, ["cancelled 1", "cancelled 2", "cancelled 3"]) diff --git a/Misc/NEWS.d/next/Library/2025-01-04-11-10-04.gh-issue-128479.jvOrF-.rst b/Misc/NEWS.d/next/Library/2025-01-04-11-10-04.gh-issue-128479.jvOrF-.rst new file mode 100644 index 00000000000000..fc3b4d5a5273a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-04-11-10-04.gh-issue-128479.jvOrF-.rst @@ -0,0 +1 @@ +Fix :func:`!asyncio.staggered.staggered_race` leaking tasks and issuing an unhandled exception.

1 0

gh-126004: Remove redundant safeguards for codecs handlers tests (#127680)
by picnixz Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/cf0b2da1e6947aa15be119582c2017765a… commit: cf0b2da1e6947aa15be119582c2017765ab46863 branch: main author: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> committer: picnixz <10796600+picnixz(a)users.noreply.github.com> date: 2025-01-23T16:06:16+01:00 summary: gh-126004: Remove redundant safeguards for codecs handlers tests (#127680) We remove the safeguards that were added in `Lib/test/test_capi/test_codecs.py` since they are now redundant (see 32e07fd377f81cbeb8c108fc791a3e7d631319b6 for additional context). Indeed, the codecs handlers now correctly handle the `start` and `end` positions of `UnicodeError` objects and thus should not crash. files: M Lib/test/test_capi/test_codecs.py diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index e368f70c196abf..a0355c7a388c57 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -839,38 +839,33 @@ def test_codec_ignore_errors_handler(self): def test_codec_replace_errors_handler(self): handler = _testcapi.codec_replace_errors - self.do_test_codec_errors_handler(handler, self.all_unicode_errors, - safe=True) + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) def test_codec_xmlcharrefreplace_errors_handler(self): handler = _testcapi.codec_xmlcharrefreplace_errors - self.do_test_codec_errors_handler(handler, self.unicode_encode_errors, - safe=True) + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) def test_codec_backslashreplace_errors_handler(self): handler = _testcapi.codec_backslashreplace_errors - self.do_test_codec_errors_handler(handler, self.all_unicode_errors, - safe=True) + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) def test_codec_namereplace_errors_handler(self): handler = _testlimitedcapi.codec_namereplace_errors self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) - def do_test_codec_errors_handler(self, handler, exceptions, *, safe=False): - at_least_one = False + def do_test_codec_errors_handler(self, handler, exceptions): + self.assertNotEqual(len(exceptions), 0) for exc in exceptions: - # See https://github.com/python/cpython/issues/123378 and related - # discussion and issues for details. - if not safe and self._exception_may_crash(exc): - continue - - at_least_one = True with self.subTest(handler=handler, exc=exc): # test that the handler does not crash - self.assertIsInstance(handler(exc), tuple) - - if exceptions: - self.assertTrue(at_least_one, "all exceptions are crashing") + res = handler(exc) + self.assertIsInstance(res, tuple) + self.assertEqual(len(res), 2) + replacement, continue_from = res + self.assertIsInstance(replacement, str) + self.assertIsInstance(continue_from, int) + self.assertGreaterEqual(continue_from, 0) + self.assertLessEqual(continue_from, len(exc.object)) for bad_exc in ( self.bad_unicode_errors @@ -879,30 +874,6 @@ def do_test_codec_errors_handler(self, handler, exceptions, *, safe=False): with self.subTest('bad type', handler=handler, exc=bad_exc): self.assertRaises(TypeError, handler, bad_exc) - @classmethod - def _exception_may_crash(cls, exc): - """Indicate whether a Unicode exception might currently crash - the interpreter when used by a built-in codecs error handler. - - Until gh-123378 is fixed, we skip the tests for these exceptions. - - This should only be used by "do_test_codec_errors_handler". - """ - message, start, end = exc.object, exc.start, exc.end - match exc: - case UnicodeEncodeError(): - return end < start or (end - start) >= len(message) - case UnicodeDecodeError(): - # The case "end - start >= len(message)" does not crash. - return end < start - case UnicodeTranslateError(): - # Test "end <= start" because PyCodec_ReplaceErrors checks - # the Unicode kind of a 0-length string which by convention - # is PyUnicode_1BYTE_KIND and not PyUnicode_2BYTE_KIND as - # the handler currently expects. - return end <= start or (end - start) >= len(message) - return False - if __name__ == "__main__": unittest.main()

1 0

gh-128690: Update test_embed for getpath.py exec_prefix change (#129137)
by vstinner Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/b23b27bc556857be73ee0f2379441c422b… commit: b23b27bc556857be73ee0f2379441c422b6fee26 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2025-01-23T15:50:04+01:00 summary: gh-128690: Update test_embed for getpath.py exec_prefix change (#129137) files: M Lib/test/test_embed.py diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index bd01913e49d170..ffb69c87a065aa 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -51,6 +51,14 @@ MAX_HASH_SEED = 4294967295 ABI_THREAD = 't' if sysconfig.get_config_var('Py_GIL_DISABLED') else '' +# PLATSTDLIB_LANDMARK copied from Modules/getpath.py +if os.name == 'nt': + PLATSTDLIB_LANDMARK = f'{sys.platlibdir}' +else: + VERSION_MAJOR = sys.version_info.major + VERSION_MINOR = sys.version_info.minor + PLATSTDLIB_LANDMARK = (f'{sys.platlibdir}/python{VERSION_MAJOR}.' + f'{VERSION_MINOR}{ABI_THREAD}/lib-dynload') # If we are running from a build dir, but the stdlib has been installed, @@ -1596,7 +1604,13 @@ def test_init_pyvenv_cfg(self): with self.tmpdir_with_python() as tmpdir, \ tempfile.TemporaryDirectory() as pyvenv_home: + ver = sys.version_info + base_prefix = sysconfig.get_config_var("prefix") + + # gh-128690: base_exec_prefix depends if PLATSTDLIB_LANDMARK exists + platstdlib = os.path.join(base_prefix, PLATSTDLIB_LANDMARK) + change_exec_prefix = not os.path.isdir(platstdlib) if not MS_WINDOWS: lib_dynload = os.path.join(pyvenv_home, @@ -1620,7 +1634,8 @@ def test_init_pyvenv_cfg(self): paths = self.module_search_paths() if not MS_WINDOWS: - paths[-1] = lib_dynload + if change_exec_prefix: + paths[-1] = lib_dynload else: paths = [ os.path.join(tmpdir, os.path.basename(paths[0])), @@ -1630,16 +1645,16 @@ def test_init_pyvenv_cfg(self): executable = self.test_exe base_executable = os.path.join(pyvenv_home, os.path.basename(executable)) - exec_prefix = pyvenv_home config = { - 'base_prefix': sysconfig.get_config_var("prefix"), - 'base_exec_prefix': exec_prefix, + 'base_prefix': base_prefix, 'exec_prefix': tmpdir, 'prefix': tmpdir, 'base_executable': base_executable, 'executable': executable, 'module_search_paths': paths, } + if change_exec_prefix: + config['base_exec_prefix'] = pyvenv_home if MS_WINDOWS: config['base_prefix'] = pyvenv_home config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib')

1 0

gh-118761: Improve import time for `pstats` and `zipfile` (#128981)
by picnixz Jan. 23, 2025

Jan. 23, 2025

https://github.com/python/cpython/commit/a95dca7b9891e673e77f868cac05b2068b… commit: a95dca7b9891e673e77f868cac05b2068bce8b3a branch: main author: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> committer: picnixz <10796600+picnixz(a)users.noreply.github.com> date: 2025-01-23T14:49:36Z summary: gh-118761: Improve import time for `pstats` and `zipfile` (#128981) Importing `pstats` or `zipfile` is now roughly 20% faster. This is achieved by removing type annotations depending on `typing`. files: A Misc/NEWS.d/next/Library/2025-01-18-11-24-02.gh-issue-118761.G8MmxY.rst M Lib/pstats.py M Lib/zipfile/__init__.py diff --git a/Lib/pstats.py b/Lib/pstats.py index 46e18fb7592a77..becaf35580eaee 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -29,7 +29,6 @@ from enum import StrEnum, _simple_enum from functools import cmp_to_key from dataclasses import dataclass -from typing import Dict __all__ = ["Stats", "SortKey", "FunctionProfile", "StatsProfile"] @@ -69,7 +68,7 @@ class FunctionProfile: class StatsProfile: '''Class for keeping track of an item in inventory.''' total_tt: float - func_profiles: Dict[str, FunctionProfile] + func_profiles: dict[str, FunctionProfile] class Stats: """This class is used for creating reports from data generated by the diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 49c40032d848f1..b8b496ad9471f4 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -13,7 +13,6 @@ import sys import threading import time -from typing import Self try: import zlib # We may need its compression method @@ -606,7 +605,7 @@ def from_file(cls, filename, arcname=None, *, strict_timestamps=True): return zinfo - def _for_archive(self, archive: ZipFile) -> Self: + def _for_archive(self, archive): """Resolve suitable defaults from the archive. Resolve the date_time, compression attributes, and external attributes diff --git a/Misc/NEWS.d/next/Library/2025-01-18-11-24-02.gh-issue-118761.G8MmxY.rst b/Misc/NEWS.d/next/Library/2025-01-18-11-24-02.gh-issue-118761.G8MmxY.rst new file mode 100644 index 00000000000000..3b3f3f7d98c5d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-18-11-24-02.gh-issue-118761.G8MmxY.rst @@ -0,0 +1,2 @@ +Reduce import time of :mod:`pstats` and :mod:`zipfile` by up to 20%, by +removing unnecessary imports to :mod:`typing`. Patch by Bénédikt Tran.

1 0