Mailman 3 March 2025 - Python-checkins

gh-128002: use efficient linked list implementation for eager tasks in asyncio (#130518)
by kumaraditya303 March 3, 2025

March 3, 2025

https://github.com/python/cpython/commit/7e3b788e8f3dc986bcccb047ddc3f0a7a9… commit: 7e3b788e8f3dc986bcccb047ddc3f0a7a99bb08c branch: main author: Kumar Aditya <kumaraditya(a)python.org> committer: kumaraditya303 <kumaraditya(a)python.org> date: 2025-03-03T06:36:43Z summary: gh-128002: use efficient linked list implementation for eager tasks in asyncio (#130518) files: M Lib/asyncio/tasks.py M Modules/_asynciomodule.c diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 06d46deda8d647..825e91f5594d98 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -1110,7 +1110,6 @@ def _unregister_eager_task(task): from _asyncio import (_register_task, _register_eager_task, _unregister_task, _unregister_eager_task, _enter_task, _leave_task, _swap_current_task, - _scheduled_tasks, _eager_tasks, current_task, all_tasks) except ImportError: pass diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index c743c246cb4a67..99463562e67286 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -143,8 +143,9 @@ typedef struct { inherit from native asyncio.Task */ PyObject *non_asyncio_tasks; - /* Set containing all eagerly executing tasks. */ - PyObject *eager_tasks; + /* Set containing all 3rd party eagerly executing tasks which don't + inherit from native asyncio.Task */ + PyObject *non_asyncio_eager_tasks; /* An isinstance type cache for the 'is_coroutine()' function. */ PyObject *iscoroutine_typecache; @@ -2180,12 +2181,6 @@ register_task(TaskObj *task) llist_insert_tail(head, &task->task_node); } -static int -register_eager_task(asyncio_state *state, PyObject *task) -{ - return PySet_Add(state->eager_tasks, task); -} - static inline void unregister_task_safe(TaskObj *task) { @@ -2219,12 +2214,6 @@ unregister_task(TaskObj *task) #endif } -static int -unregister_eager_task(asyncio_state *state, PyObject *task) -{ - return PySet_Discard(state->eager_tasks, task); -} - static int enter_task(PyObject *loop, PyObject *task) { @@ -3472,11 +3461,11 @@ task_eager_start(asyncio_state *state, TaskObj *task) if (prevtask == NULL) { return -1; } - - if (register_eager_task(state, (PyObject *)task) == -1) { - Py_DECREF(prevtask); - return -1; - } + // register the task into the linked list of tasks + // if the task completes eagerly (without suspending) then it will unregister itself + // in future_schedule_callbacks when done, otherwise + // it will continue as a regular (non-eager) asyncio task + register_task(task); if (PyContext_Enter(task->task_context) == -1) { Py_DECREF(prevtask); @@ -3506,17 +3495,11 @@ task_eager_start(asyncio_state *state, TaskObj *task) Py_DECREF(curtask); } - if (unregister_eager_task(state, (PyObject *)task) == -1) { - retval = -1; - } - if (PyContext_Exit(task->task_context) == -1) { retval = -1; } - if (task->task_state == STATE_PENDING) { - register_task(task); - } else { + if (task->task_state != STATE_PENDING) { // This seems to really help performance on pyperformance benchmarks clear_task_coro(task); } @@ -3735,9 +3718,18 @@ _asyncio__register_eager_task_impl(PyObject *module, PyObject *task) /*[clinic end generated code: output=dfe1d45367c73f1a input=237f684683398c51]*/ { asyncio_state *state = get_asyncio_state(module); - if (register_eager_task(state, task) < 0) { + + if (Task_Check(state, task)) { + // task is an asyncio.Task instance or subclass, use efficient + // linked-list implementation. + register_task((TaskObj *)task); + Py_RETURN_NONE; + } + + if (PySet_Add(state->non_asyncio_eager_tasks, task) < 0) { return NULL; } + Py_RETURN_NONE; } @@ -3785,9 +3777,17 @@ _asyncio__unregister_eager_task_impl(PyObject *module, PyObject *task) /*[clinic end generated code: output=a426922bd07f23d1 input=9d07401ef14ee048]*/ { asyncio_state *state = get_asyncio_state(module); - if (unregister_eager_task(state, task) < 0) { + if (Task_Check(state, task)) { + // task is an asyncio.Task instance or subclass, use efficient + // linked-list implementation. + unregister_task((TaskObj *)task); + Py_RETURN_NONE; + } + + if (PySet_Discard(state->non_asyncio_eager_tasks, task) < 0) { return NULL; } + Py_RETURN_NONE; } @@ -4041,7 +4041,7 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) Py_DECREF(loop); return NULL; } - if (PyList_Extend(tasks, state->eager_tasks) < 0) { + if (PyList_Extend(tasks, state->non_asyncio_eager_tasks) < 0) { Py_DECREF(tasks); Py_DECREF(loop); return NULL; @@ -4179,7 +4179,7 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->asyncio_CancelledError); Py_VISIT(state->non_asyncio_tasks); - Py_VISIT(state->eager_tasks); + Py_VISIT(state->non_asyncio_eager_tasks); Py_VISIT(state->iscoroutine_typecache); Py_VISIT(state->context_kwname); @@ -4209,7 +4209,7 @@ module_clear(PyObject *mod) Py_CLEAR(state->asyncio_CancelledError); Py_CLEAR(state->non_asyncio_tasks); - Py_CLEAR(state->eager_tasks); + Py_CLEAR(state->non_asyncio_eager_tasks); Py_CLEAR(state->iscoroutine_typecache); Py_CLEAR(state->context_kwname); @@ -4292,8 +4292,8 @@ module_init(asyncio_state *state) goto fail; } - state->eager_tasks = PySet_New(NULL); - if (state->eager_tasks == NULL) { + state->non_asyncio_eager_tasks = PySet_New(NULL); + if (state->non_asyncio_eager_tasks == NULL) { goto fail; } @@ -4363,14 +4363,6 @@ module_exec(PyObject *mod) return -1; } - if (PyModule_AddObjectRef(mod, "_scheduled_tasks", state->non_asyncio_tasks) < 0) { - return -1; - } - - if (PyModule_AddObjectRef(mod, "_eager_tasks", state->eager_tasks) < 0) { - return -1; - } - return 0; }

1 0

gh-128041: Add `terminate_workers` and `kill_workers` methods to ProcessPoolExecutor (GH-128043)
by gpshead March 2, 2025

March 2, 2025

https://github.com/python/cpython/commit/f97e4098ff71a6488fd3411f9f9e6fa7a7… commit: f97e4098ff71a6488fd3411f9f9e6fa7a7bb4efe branch: main author: Charles Machalow <csm10495(a)gmail.com> committer: gpshead <greg(a)krypto.org> date: 2025-03-02T18:01:45-08:00 summary: gh-128041: Add `terminate_workers` and `kill_workers` methods to ProcessPoolExecutor (GH-128043) This adds two new methods to `multiprocessing`'s `ProcessPoolExecutor`: - **`terminate_workers()`**: forcefully terminates worker processes using `Process.terminate()` - **`kill_workers()`**: forcefully kills worker processes using `Process.kill()` These methods provide users with a direct way to stop worker processes without `shutdown()` or relying on implementation details, addressing situations where immediate termination is needed. Co-authored-by: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot](a)users.noreply.github.com> Commit-message-mostly-authored-by: Claude Sonnet 3.7 (because why not -greg) files: A Misc/NEWS.d/next/Library/2024-12-17-18-53-21.gh-issue-128041.W96kAr.rst M Doc/library/concurrent.futures.rst M Doc/whatsnew/3.14.rst M Lib/concurrent/futures/process.py M Lib/test/test_concurrent_futures/test_process_pool.py diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 5a950081a1c98d..dc613f2f8f00cd 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -415,6 +415,30 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. require the *fork* start method for :class:`ProcessPoolExecutor` you must explicitly pass ``mp_context=multiprocessing.get_context("fork")``. + .. method:: terminate_workers() + + Attempt to terminate all living worker processes immediately by calling + :meth:`Process.terminate <multiprocessing.Process.terminate>` on each of them. + Internally, it will also call :meth:`Executor.shutdown` to ensure that all + other resources associated with the executor are freed. + + After calling this method the caller should no longer submit tasks to the + executor. + + .. versionadded:: next + + .. method:: kill_workers() + + Attempt to kill all living worker processes immediately by calling + :meth:`Process.kill <multiprocessing.Process.kill>` on each of them. + Internally, it will also call :meth:`Executor.shutdown` to ensure that all + other resources associated with the executor are freed. + + After calling this method the caller should no longer submit tasks to the + executor. + + .. versionadded:: next + .. _processpoolexecutor-example: ProcessPoolExecutor Example diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index ce75b5fffc0a4c..2875913e4abdf1 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -444,6 +444,11 @@ contextvars * Support context manager protocol by :class:`contextvars.Token`. (Contributed by Andrew Svetlov in :gh:`129889`.) +* Add :meth:`concurrent.futures.ProcessPoolExecutor.terminate_workers` and + :meth:`concurrent.futures.ProcessPoolExecutor.kill_workers` as + ways to terminate or kill all living worker processes in the given pool. + (Contributed by Charles Machalow in :gh:`128043`.) + ctypes ------ diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 42eee72bc1457f..d79d6b959c90d3 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -626,6 +626,14 @@ class BrokenProcessPool(_base.BrokenExecutor): while a future was in the running state. """ +_TERMINATE = "terminate" +_KILL = "kill" + +_SHUTDOWN_CALLBACK_OPERATION = { + _TERMINATE, + _KILL +} + class ProcessPoolExecutor(_base.Executor): def __init__(self, max_workers=None, mp_context=None, @@ -855,3 +863,66 @@ def shutdown(self, wait=True, *, cancel_futures=False): self._executor_manager_thread_wakeup = None shutdown.__doc__ = _base.Executor.shutdown.__doc__ + + def _force_shutdown(self, operation): + """Attempts to terminate or kill the executor's workers based off the + given operation. Iterates through all of the current processes and + performs the relevant task if the process is still alive. + + After terminating workers, the pool will be in a broken state + and no longer usable (for instance, new tasks should not be + submitted). + """ + if operation not in _SHUTDOWN_CALLBACK_OPERATION: + raise ValueError(f"Unsupported operation: {operation!r}") + + processes = {} + if self._processes: + processes = self._processes.copy() + + # shutdown will invalidate ._processes, so we copy it right before + # calling. If we waited here, we would deadlock if a process decides not + # to exit. + self.shutdown(wait=False, cancel_futures=True) + + if not processes: + return + + for proc in processes.values(): + try: + if not proc.is_alive(): + continue + except ValueError: + # The process is already exited/closed out. + continue + + try: + if operation == _TERMINATE: + proc.terminate() + elif operation == _KILL: + proc.kill() + except ProcessLookupError: + # The process just ended before our signal + continue + + def terminate_workers(self): + """Attempts to terminate the executor's workers. + Iterates through all of the current worker processes and terminates + each one that is still alive. + + After terminating workers, the pool will be in a broken state + and no longer usable (for instance, new tasks should not be + submitted). + """ + return self._force_shutdown(operation=_TERMINATE) + + def kill_workers(self): + """Attempts to kill the executor's workers. + Iterates through all of the current worker processes and kills + each one that is still alive. + + After killing workers, the pool will be in a broken state + and no longer usable (for instance, new tasks should not be + submitted). + """ + return self._force_shutdown(operation=_KILL) diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 8b1bdaa33d8f5c..354b7d0a346970 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -1,13 +1,17 @@ import os +import queue +import signal import sys import threading import time import unittest +import unittest.mock from concurrent import futures from concurrent.futures.process import BrokenProcessPool from test import support from test.support import hashlib_helper +from test.test_importlib.metadata.fixtures import parameterize from .executor import ExecutorTest, mul from .util import ( @@ -22,6 +26,19 @@ def __init__(self, mgr): def __del__(self): self.event.set() +TERMINATE_WORKERS = futures.ProcessPoolExecutor.terminate_workers.__name__ +KILL_WORKERS = futures.ProcessPoolExecutor.kill_workers.__name__ +FORCE_SHUTDOWN_PARAMS = [ + dict(function_name=TERMINATE_WORKERS), + dict(function_name=KILL_WORKERS), +] + +def _put_sleep_put(queue): + """ Used as part of test_terminate_workers """ + queue.put('started') + time.sleep(2) + queue.put('finished') + class ProcessPoolExecutorTest(ExecutorTest): @@ -218,6 +235,86 @@ def mock_start_new_thread(func, *args, **kwargs): list(executor.map(mul, [(2, 3)] * 10)) executor.shutdown() + def test_terminate_workers(self): + mock_fn = unittest.mock.Mock() + with self.executor_type(max_workers=1) as executor: + executor._force_shutdown = mock_fn + executor.terminate_workers() + + mock_fn.assert_called_once_with(operation=futures.process._TERMINATE) + + def test_kill_workers(self): + mock_fn = unittest.mock.Mock() + with self.executor_type(max_workers=1) as executor: + executor._force_shutdown = mock_fn + executor.kill_workers() + + mock_fn.assert_called_once_with(operation=futures.process._KILL) + + def test_force_shutdown_workers_invalid_op(self): + with self.executor_type(max_workers=1) as executor: + self.assertRaises(ValueError, + executor._force_shutdown, + operation='invalid operation'), + + @parameterize(*FORCE_SHUTDOWN_PARAMS) + def test_force_shutdown_workers(self, function_name): + manager = self.get_context().Manager() + q = manager.Queue() + + with self.executor_type(max_workers=1) as executor: + executor.submit(_put_sleep_put, q) + + # We should get started, but not finished since we'll terminate the + # workers just after + self.assertEqual(q.get(timeout=5), 'started') + + worker_process = list(executor._processes.values())[0] + getattr(executor, function_name)() + worker_process.join() + + if function_name == TERMINATE_WORKERS or \ + sys.platform == 'win32': + # On windows, kill and terminate both send SIGTERM + self.assertEqual(worker_process.exitcode, -signal.SIGTERM) + elif function_name == KILL_WORKERS: + self.assertEqual(worker_process.exitcode, -signal.SIGKILL) + else: + self.fail(f"Unknown operation: {function_name}") + + self.assertRaises(queue.Empty, q.get, timeout=1) + + @parameterize(*FORCE_SHUTDOWN_PARAMS) + def test_force_shutdown_workers_dead_workers(self, function_name): + with self.executor_type(max_workers=1) as executor: + future = executor.submit(os._exit, 1) + self.assertRaises(BrokenProcessPool, future.result) + + # even though the pool is broken, this shouldn't raise + getattr(executor, function_name)() + + @parameterize(*FORCE_SHUTDOWN_PARAMS) + def test_force_shutdown_workers_not_started_yet(self, function_name): + ctx = self.get_context() + with unittest.mock.patch.object(ctx, 'Process') as mock_process: + with self.executor_type(max_workers=1, mp_context=ctx) as executor: + # The worker has not been started yet, terminate/kill_workers + # should basically no-op + getattr(executor, function_name)() + + mock_process.return_value.kill.assert_not_called() + mock_process.return_value.terminate.assert_not_called() + + @parameterize(*FORCE_SHUTDOWN_PARAMS) + def test_force_shutdown_workers_stops_pool(self, function_name): + with self.executor_type(max_workers=1) as executor: + task = executor.submit(time.sleep, 0) + self.assertIsNone(task.result()) + + getattr(executor, function_name)() + + self.assertRaises(RuntimeError, executor.submit, time.sleep, 0) + create_executor_tests(globals(), ProcessPoolExecutorTest, executor_mixins=(ProcessPoolForkMixin, diff --git a/Misc/NEWS.d/next/Library/2024-12-17-18-53-21.gh-issue-128041.W96kAr.rst b/Misc/NEWS.d/next/Library/2024-12-17-18-53-21.gh-issue-128041.W96kAr.rst new file mode 100644 index 00000000000000..bb9ef96d45eb79 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-17-18-53-21.gh-issue-128041.W96kAr.rst @@ -0,0 +1,4 @@ +Add :meth:`concurrent.futures.ProcessPoolExecutor.terminate_workers` and +:meth:`concurrent.futures.ProcessPoolExecutor.kill_workers` as +ways to terminate or kill all living worker processes in the given pool. +(Contributed by Charles Machalow in :gh:`128043`.)

1 0

GH-130415: Use boolean guards to narrow types to values in the JIT (GH-130659)
by brandtbucher March 2, 2025

March 2, 2025

https://github.com/python/cpython/commit/7afa476874b9a432ad6dbe9fb3e65d62f2… commit: 7afa476874b9a432ad6dbe9fb3e65d62f2999f88 branch: main author: Brandt Bucher <brandtbucher(a)microsoft.com> committer: brandtbucher <brandtbucher(a)gmail.com> date: 2025-03-02T13:21:34-08:00 summary: GH-130415: Use boolean guards to narrow types to values in the JIT (GH-130659) files: A Misc/NEWS.d/next/Core_and_Builtins/2025-02-27-17-05-05.gh-issue-130415.iijvfW.rst M Include/internal/pycore_optimizer.h M Lib/test/test_capi/test_opt.py M Python/optimizer_analysis.c M Python/optimizer_bytecodes.c M Python/optimizer_cases.c.h M Python/optimizer_symbols.c diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 25c3d3e5a22442..79861167f65e63 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -172,6 +172,7 @@ typedef enum _JitSymType { JIT_SYM_KNOWN_CLASS_TAG = 6, JIT_SYM_KNOWN_VALUE_TAG = 7, JIT_SYM_TUPLE_TAG = 8, + JIT_SYM_TRUTHINESS_TAG = 9, } JitSymType; typedef struct _jit_opt_known_class { @@ -198,12 +199,19 @@ typedef struct _jit_opt_tuple { uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; } JitOptTuple; +typedef struct { + uint8_t tag; + bool not; + uint16_t value; +} JitOptTruthiness; + typedef union _jit_opt_symbol { uint8_t tag; JitOptKnownClass cls; JitOptKnownValue value; JitOptKnownVersion version; JitOptTuple tuple; + JitOptTruthiness truthiness; } JitOptSymbol; @@ -245,8 +253,8 @@ typedef struct _JitOptContext { extern bool _Py_uop_sym_is_null(JitOptSymbol *sym); extern bool _Py_uop_sym_is_not_null(JitOptSymbol *sym); -extern bool _Py_uop_sym_is_const(JitOptSymbol *sym); -extern PyObject *_Py_uop_sym_get_const(JitOptSymbol *sym); +extern bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym); +extern PyObject *_Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym); extern JitOptSymbol *_Py_uop_sym_new_unknown(JitOptContext *ctx); extern JitOptSymbol *_Py_uop_sym_new_not_null(JitOptContext *ctx); extern JitOptSymbol *_Py_uop_sym_new_type( @@ -262,12 +270,13 @@ extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeOb extern bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version); extern void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val); extern bool _Py_uop_sym_is_bottom(JitOptSymbol *sym); -extern int _Py_uop_sym_truthiness(JitOptSymbol *sym); +extern int _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym); extern PyTypeObject *_Py_uop_sym_get_type(JitOptSymbol *sym); extern bool _Py_uop_sym_is_immortal(JitOptSymbol *sym); extern JitOptSymbol *_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args); extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item); extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym); +extern JitOptSymbol *_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy); extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 2a9b777862c84a..b7083dbfb89db8 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1437,6 +1437,68 @@ def crash_addition(): crash_addition() + def test_narrow_type_to_constant_bool_false(self): + def f(n): + trace = [] + for i in range(n): + # false is always False, but we can only prove that it's a bool: + false = i == TIER2_THRESHOLD + trace.append("A") + if not false: # Kept. + trace.append("B") + if not false: # Removed! + trace.append("C") + trace.append("D") + if false: # Removed! + trace.append("X") + trace.append("E") + trace.append("F") + if false: # Removed! + trace.append("X") + trace.append("G") + return trace + + trace, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(trace, list("ABCDEFG") * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # Only one guard remains: + self.assertEqual(uops.count("_GUARD_IS_FALSE_POP"), 1) + self.assertEqual(uops.count("_GUARD_IS_TRUE_POP"), 0) + # But all of the appends we care about are still there: + self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) + + def test_narrow_type_to_constant_bool_true(self): + def f(n): + trace = [] + for i in range(n): + # true always True, but we can only prove that it's a bool: + true = i != TIER2_THRESHOLD + trace.append("A") + if true: # Kept. + trace.append("B") + if not true: # Removed! + trace.append("X") + trace.append("C") + if true: # Removed! + trace.append("D") + trace.append("E") + trace.append("F") + if not true: # Removed! + trace.append("X") + trace.append("G") + return trace + + trace, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(trace, list("ABCDEFG") * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # Only one guard remains: + self.assertEqual(uops.count("_GUARD_IS_FALSE_POP"), 0) + self.assertEqual(uops.count("_GUARD_IS_TRUE_POP"), 1) + # But all of the appends we care about are still there: + self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) + def global_identity(x): return x diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-02-27-17-05-05.gh-issue-130415.iijvfW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-27-17-05-05.gh-issue-130415.iijvfW.rst new file mode 100644 index 00000000000000..f5b6d0e9c1a06a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-27-17-05-05.gh-issue-130415.iijvfW.rst @@ -0,0 +1,2 @@ +Improve the experimental JIT's ability to narrow boolean values based on the +results of truthiness tests. diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 29a05088e62c45..67bf8d11b3f9ac 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -136,26 +136,6 @@ incorrect_keys(_PyUOpInstruction *inst, PyObject *obj) return 0; } -static int -check_next_uop(_PyUOpInstruction *buffer, int size, int pc, uint16_t expected) -{ - if (pc + 1 >= size) { - DPRINTF(1, "Cannot rewrite %s at pc %d: buffer too small\n", - _PyOpcode_uop_name[buffer[pc].opcode], pc); - return 0; - } - uint16_t next_opcode = buffer[pc + 1].opcode; - if (next_opcode != expected) { - DPRINTF(1, - "Cannot rewrite %s at pc %d: unexpected next opcode %s, " - "expected %s\n", - _PyOpcode_uop_name[buffer[pc].opcode], pc, - _PyOpcode_uop_name[next_opcode], _PyOpcode_uop_name[expected]); - return 0; - } - return 1; -} - /* Returns 1 if successfully optimized * 0 if the trace is not suitable for optimization (yet) * -1 if there was an error. */ @@ -363,6 +343,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_tuple_getitem _Py_uop_sym_tuple_getitem #define sym_tuple_length _Py_uop_sym_tuple_length #define sym_is_immortal _Py_uop_sym_is_immortal +#define sym_new_truthiness _Py_uop_sym_new_truthiness static int optimize_to_bool( @@ -376,7 +357,7 @@ optimize_to_bool( *result_ptr = value; return 1; } - int truthiness = sym_truthiness(value); + int truthiness = sym_truthiness(ctx, value); if (truthiness >= 0) { PyObject *load = truthiness ? Py_True : Py_False; REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index f3625a1492c47c..c4e4b28e50d211 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -34,6 +34,7 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define sym_tuple_getitem _Py_uop_sym_tuple_getitem #define sym_tuple_length _Py_uop_sym_tuple_length #define sym_is_immortal _Py_uop_sym_is_immortal +#define sym_new_truthiness _Py_uop_sym_new_truthiness extern int optimize_to_bool( @@ -198,11 +199,11 @@ dummy_func(void) { // Case C: res = sym_new_type(ctx, &PyFloat_Type); } - else if (!sym_is_const(right)) { + else if (!sym_is_const(ctx, right)) { // Case A or B... can't know without the sign of the RHS: res = sym_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(right))) { + else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, right))) { // Case B: res = sym_new_type(ctx, &PyFloat_Type); } @@ -223,13 +224,13 @@ dummy_func(void) { } op(_BINARY_OP_ADD_INT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } @@ -244,13 +245,13 @@ dummy_func(void) { } op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } @@ -265,13 +266,13 @@ dummy_func(void) { } op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } @@ -286,14 +287,14 @@ dummy_func(void) { } op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) + - PyFloat_AS_DOUBLE(sym_get_const(right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } @@ -308,14 +309,14 @@ dummy_func(void) { } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) - - PyFloat_AS_DOUBLE(sym_get_const(right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } @@ -330,14 +331,14 @@ dummy_func(void) { } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) * - PyFloat_AS_DOUBLE(sym_get_const(right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } @@ -352,9 +353,9 @@ dummy_func(void) { } op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { - PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); if (temp == NULL) { goto error; } @@ -368,9 +369,9 @@ dummy_func(void) { op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- )) { JitOptSymbol *res; - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { - PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); if (temp == NULL) { goto error; } @@ -391,14 +392,14 @@ dummy_func(void) { op(_TO_BOOL, (value -- res)) { if (!optimize_to_bool(this_instr, ctx, value, &res)) { - res = sym_new_type(ctx, &PyBool_Type); + res = sym_new_truthiness(ctx, value, true); } } op(_TO_BOOL_BOOL, (value -- res)) { if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyBool_Type); - res = value; + res = sym_new_truthiness(ctx, value, true); } } @@ -430,6 +431,11 @@ dummy_func(void) { } } + op(_UNARY_NOT, (value -- res)) { + sym_set_type(value, &PyBool_Type); + res = sym_new_truthiness(ctx, value, false); + } + op(_COMPARE_OP, (left, right -- res)) { if (oparg & 16) { res = sym_new_type(ctx, &PyBool_Type); @@ -521,8 +527,8 @@ dummy_func(void) { (void)dict_version; (void)index; attr = NULL; - if (sym_is_const(owner)) { - PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + if (sym_is_const(ctx, owner)) { + PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { PyObject *dict = mod->md_dict; uint64_t watched_mutations = get_mutations(dict); @@ -599,19 +605,19 @@ dummy_func(void) { } op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { - assert(PyFunction_Check(sym_get_const(callable))); + if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(sym_get_const(ctx, callable))); REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)sym_get_const(callable); + this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); } op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { assert(sym_matches_type(callable, &PyFunction_Type)); - if (sym_is_const(callable)) { + if (sym_is_const(ctx, callable)) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); + PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { REPLACE_OP(this_instr, _NOP, 0 ,0); @@ -812,24 +818,26 @@ dummy_func(void) { } op(_GUARD_IS_TRUE_POP, (flag -- )) { - if (sym_is_const(flag)) { - PyObject *value = sym_get_const(flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_True); } + sym_set_const(flag, Py_True); } op(_GUARD_IS_FALSE_POP, (flag -- )) { - if (sym_is_const(flag)) { - PyObject *value = sym_get_const(flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, value != Py_False); } + sym_set_const(flag, Py_False); } op(_GUARD_IS_NONE_POP, (flag -- )) { - if (sym_is_const(flag)) { - PyObject *value = sym_get_const(flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } @@ -837,11 +845,12 @@ dummy_func(void) { assert(!sym_matches_type(flag, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } + sym_set_const(flag, Py_None); } op(_GUARD_IS_NOT_NONE_POP, (flag -- )) { - if (sym_is_const(flag)) { - PyObject *value = sym_get_const(flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 0372870b94ec0a..397184dd87ad7d 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -140,8 +140,11 @@ } case _UNARY_NOT: { + JitOptSymbol *value; JitOptSymbol *res; - res = sym_new_not_null(ctx); + value = stack_pointer[-1]; + sym_set_type(value, &PyBool_Type); + res = sym_new_truthiness(ctx, value, false); stack_pointer[-1] = res; break; } @@ -151,7 +154,7 @@ JitOptSymbol *res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { - res = sym_new_type(ctx, &PyBool_Type); + res = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; @@ -163,7 +166,7 @@ value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyBool_Type); - res = value; + res = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = res; break; @@ -268,15 +271,15 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } @@ -303,15 +306,15 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } @@ -338,15 +341,15 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) { - assert(PyLong_CheckExact(sym_get_const(left))); - assert(PyLong_CheckExact(sym_get_const(right))); + assert(PyLong_CheckExact(sym_get_const(ctx, left))); + assert(PyLong_CheckExact(sym_get_const(ctx, right))); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), - (PyLongObject *)sym_get_const(right)); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), + (PyLongObject *)sym_get_const(ctx, right)); if (temp == NULL) { goto error; } @@ -404,14 +407,14 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) * - PyFloat_AS_DOUBLE(sym_get_const(right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } @@ -438,14 +441,14 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) + - PyFloat_AS_DOUBLE(sym_get_const(right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } @@ -472,14 +475,14 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) { - assert(PyFloat_CheckExact(sym_get_const(left))); - assert(PyFloat_CheckExact(sym_get_const(right))); + assert(PyFloat_CheckExact(sym_get_const(ctx, left))); + assert(PyFloat_CheckExact(sym_get_const(ctx, right))); PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(left)) - - PyFloat_AS_DOUBLE(sym_get_const(right))); + PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - + PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); if (temp == NULL) { goto error; } @@ -520,9 +523,9 @@ JitOptSymbol *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { - PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); if (temp == NULL) { goto error; } @@ -547,9 +550,9 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; JitOptSymbol *res; - if (sym_is_const(left) && sym_is_const(right) && + if (sym_is_const(ctx, left) && sym_is_const(ctx, right) && sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { - PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); if (temp == NULL) { goto error; } @@ -1159,8 +1162,8 @@ (void)dict_version; (void)index; attr = NULL; - if (sym_is_const(owner)) { - PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + if (sym_is_const(ctx, owner)) { + PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { PyObject *dict = mod->md_dict; stack_pointer[-1] = attr; @@ -1655,10 +1658,10 @@ JitOptSymbol *callable; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand0; - if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) { - assert(PyFunction_Check(sym_get_const(callable))); + if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { + assert(PyFunction_Check(sym_get_const(ctx, callable))); REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); - this_instr->operand1 = (uintptr_t)sym_get_const(callable); + this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable); } sym_set_type(callable, &PyFunction_Type); break; @@ -1724,9 +1727,9 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(sym_matches_type(callable, &PyFunction_Type)); - if (sym_is_const(callable)) { + if (sym_is_const(ctx, callable)) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - PyFunctionObject *func = (PyFunctionObject *)sym_get_const(callable); + PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; if (co->co_argcount == oparg + !sym_is_null(self_or_null)) { REPLACE_OP(this_instr, _NOP, 0 ,0); @@ -2160,12 +2163,12 @@ res = sym_new_type(ctx, &PyFloat_Type); } else { - if (!sym_is_const(right)) { + if (!sym_is_const(ctx, right)) { // Case A or B... can't know without the sign of the RHS: res = sym_new_unknown(ctx); } else { - if (_PyLong_IsNegative((PyLongObject *)sym_get_const(right))) { + if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, right))) { // Case B: res = sym_new_type(ctx, &PyFloat_Type); } @@ -2230,8 +2233,8 @@ case _GUARD_IS_TRUE_POP: { JitOptSymbol *flag; flag = stack_pointer[-1]; - if (sym_is_const(flag)) { - PyObject *value = sym_get_const(flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2239,6 +2242,7 @@ stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } + sym_set_const(flag, Py_True); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2247,8 +2251,8 @@ case _GUARD_IS_FALSE_POP: { JitOptSymbol *flag; flag = stack_pointer[-1]; - if (sym_is_const(flag)) { - PyObject *value = sym_get_const(flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2256,6 +2260,7 @@ stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } + sym_set_const(flag, Py_False); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -2264,8 +2269,8 @@ case _GUARD_IS_NONE_POP: { JitOptSymbol *flag; flag = stack_pointer[-1]; - if (sym_is_const(flag)) { - PyObject *value = sym_get_const(flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2283,14 +2288,15 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } + sym_set_const(flag, Py_None); break; } case _GUARD_IS_NOT_NONE_POP: { JitOptSymbol *flag; flag = stack_pointer[-1]; - if (sym_is_const(flag)) { - PyObject *value = sym_get_const(flag); + if (sym_is_const(ctx, flag)) { + PyObject *value = sym_get_const(ctx, flag); assert(value != NULL); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index dcde8e7ce81577..e8a3b918e56a94 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -48,6 +48,12 @@ static JitOptSymbol NO_SPACE_SYMBOL = { .tag = JIT_SYM_BOTTOM_TAG }; +static JitOptSymbol * +allocation_base(JitOptContext *ctx) +{ + return ctx->t_arena.arena; +} + JitOptSymbol * out_of_space(JitOptContext *ctx) { @@ -70,6 +76,12 @@ sym_new(JitOptContext *ctx) return self; } +static void make_const(JitOptSymbol *sym, PyObject *val) +{ + sym->tag = JIT_SYM_KNOWN_VALUE_TAG; + sym->value.value = Py_NewRef(val); +} + static inline void sym_set_bottom(JitOptContext *ctx, JitOptSymbol *sym) { @@ -90,9 +102,21 @@ _Py_uop_sym_is_not_null(JitOptSymbol *sym) { } bool -_Py_uop_sym_is_const(JitOptSymbol *sym) +_Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym) { - return sym->tag == JIT_SYM_KNOWN_VALUE_TAG; + if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { + return true; + } + if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { + JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; + int truthiness = _Py_uop_sym_truthiness(ctx, value); + if (truthiness < 0) { + return false; + } + make_const(sym, (truthiness ^ sym->truthiness.not) ? Py_True : Py_False); + return true; + } + return false; } bool @@ -103,11 +127,21 @@ _Py_uop_sym_is_null(JitOptSymbol *sym) PyObject * -_Py_uop_sym_get_const(JitOptSymbol *sym) +_Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym) { if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return sym->value.value; } + if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { + JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; + int truthiness = _Py_uop_sym_truthiness(ctx, value); + if (truthiness < 0) { + return NULL; + } + PyObject *res = (truthiness ^ sym->truthiness.not) ? Py_True : Py_False; + make_const(sym, res); + return res; + } return NULL; } @@ -153,6 +187,11 @@ _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) sym->cls.version = 0; sym->cls.type = typ; return; + case JIT_SYM_TRUTHINESS_TAG: + if (typ != &PyBool_Type) { + sym_set_bottom(ctx, sym); + } + return; } } @@ -193,16 +232,16 @@ _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int sym->tag = JIT_SYM_TYPE_VERSION_TAG; sym->version.version = version; return true; + case JIT_SYM_TRUTHINESS_TAG: + if (version != PyBool_Type.tp_version_tag) { + sym_set_bottom(ctx, sym); + return false; + } + return true; } Py_UNREACHABLE(); } -static void make_const(JitOptSymbol *sym, PyObject *val) -{ - sym->tag = JIT_SYM_KNOWN_VALUE_TAG; - sym->value.value = Py_NewRef(val); -} - void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val) { @@ -240,6 +279,29 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val case JIT_SYM_UNKNOWN_TAG: make_const(sym, const_val); return; + case JIT_SYM_TRUTHINESS_TAG: + if (!PyBool_Check(const_val) || + (_Py_uop_sym_is_const(ctx, sym) && + _Py_uop_sym_get_const(ctx, sym) != const_val)) + { + sym_set_bottom(ctx, sym); + return; + } + JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; + PyTypeObject *type = _Py_uop_sym_get_type(value); + if (const_val == (sym->truthiness.not ? Py_False : Py_True)) { + // value is truthy. This is only useful for bool: + if (type == &PyBool_Type) { + _Py_uop_sym_set_const(ctx, value, Py_True); + } + } + // value is falsey: + else if (type == &PyBool_Type) { + _Py_uop_sym_set_const(ctx, value, Py_False); + } + // TODO: More types (GH-130415)! + make_const(sym, const_val); + return; } } @@ -339,6 +401,8 @@ _Py_uop_sym_get_type(JitOptSymbol *sym) return Py_TYPE(sym->value.value); case JIT_SYM_TUPLE_TAG: return &PyTuple_Type; + case JIT_SYM_TRUTHINESS_TAG: + return &PyBool_Type; } Py_UNREACHABLE(); } @@ -361,6 +425,8 @@ _Py_uop_sym_get_type_version(JitOptSymbol *sym) return Py_TYPE(sym->value.value)->tp_version_tag; case JIT_SYM_TUPLE_TAG: return PyTuple_Type.tp_version_tag; + case JIT_SYM_TRUTHINESS_TAG: + return PyBool_Type.tp_version_tag; } Py_UNREACHABLE(); } @@ -379,6 +445,7 @@ _Py_uop_sym_has_type(JitOptSymbol *sym) case JIT_SYM_KNOWN_CLASS_TAG: case JIT_SYM_KNOWN_VALUE_TAG: case JIT_SYM_TUPLE_TAG: + case JIT_SYM_TRUTHINESS_TAG: return true; } Py_UNREACHABLE(); @@ -398,7 +465,7 @@ _Py_uop_sym_matches_type_version(JitOptSymbol *sym, unsigned int version) } int -_Py_uop_sym_truthiness(JitOptSymbol *sym) +_Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) { switch(sym->tag) { case JIT_SYM_NULL_TAG: @@ -416,6 +483,16 @@ _Py_uop_sym_truthiness(JitOptSymbol *sym) break; case JIT_SYM_TUPLE_TAG: return sym->tuple.length != 0; + case JIT_SYM_TRUTHINESS_TAG: + ; + JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; + int truthiness = _Py_uop_sym_truthiness(ctx, value); + if (truthiness < 0) { + return truthiness; + } + truthiness ^= sym->truthiness.not; + make_const(sym, truthiness ? Py_True : Py_False); + return truthiness; } PyObject *value = sym->value.value; /* Only handle a few known safe types */ @@ -435,12 +512,6 @@ _Py_uop_sym_truthiness(JitOptSymbol *sym) return -1; } -static JitOptSymbol * -allocation_base(JitOptContext *ctx) -{ - return ctx->t_arena.arena; -} - JitOptSymbol * _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args) { @@ -503,9 +574,36 @@ _Py_uop_sym_is_immortal(JitOptSymbol *sym) if (sym->tag == JIT_SYM_KNOWN_CLASS_TAG) { return sym->cls.type == &PyBool_Type; } + if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { + return true; + } return false; } +JitOptSymbol * +_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy) +{ + // It's clearer to invert this in the signature: + bool not = !truthy; + if (value->tag == JIT_SYM_TRUTHINESS_TAG && value->truthiness.not == not) { + return value; + } + JitOptSymbol *res = sym_new(ctx); + if (res == NULL) { + return out_of_space(ctx); + } + int truthiness = _Py_uop_sym_truthiness(ctx, value); + if (truthiness < 0) { + res->tag = JIT_SYM_TRUTHINESS_TAG; + res->truthiness.not = not; + res->truthiness.value = (uint16_t)(value - allocation_base(ctx)); + } + else { + make_const(res, (truthiness ^ not) ? Py_True : Py_False); + } + return res; +} + // 0 on success, -1 on error. _Py_UOpsAbstractFrame * _Py_uop_frame_new( @@ -570,7 +668,7 @@ _Py_uop_abstractcontext_fini(JitOptContext *ctx) void _Py_uop_abstractcontext_init(JitOptContext *ctx) { - static_assert(sizeof(JitOptSymbol) <= 2*sizeof(uint64_t)); + static_assert(sizeof(JitOptSymbol) <= 2 * sizeof(uint64_t), "JitOptSymbol has grown"); ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; ctx->n_consumed = ctx->locals_and_stack; #ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. @@ -625,6 +723,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) _Py_uop_abstractcontext_init(ctx); PyObject *val_42 = NULL; PyObject *val_43 = NULL; + PyObject *tuple = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. JitOptSymbol *sym = _Py_uop_sym_new_unknown(ctx); @@ -634,8 +733,8 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "top is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "top is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "top as constant is not NULL"); TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); sym = make_bottom(ctx); @@ -645,8 +744,8 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "bottom is not NULL is not false"); TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "bottom matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "bottom is a constant is not false"); - TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "bottom as constant is not NULL"); TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); @@ -657,8 +756,8 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "int isn't not NULL"); TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "int isn't int"); TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "int matches float"); - TEST_PREDICATE(!_Py_uop_sym_is_const(sym), "int is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(sym) == NULL, "int as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "int is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "int as constant is not NULL"); _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); @@ -679,19 +778,19 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) goto fail; } _Py_uop_sym_set_const(ctx, sym, val_42); - TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 1, "bool(42) is not True"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 1, "bool(42) is not True"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "42 matches float"); - TEST_PREDICATE(_Py_uop_sym_is_const(sym), "42 is not a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(sym) != NULL, "42 as constant is NULL"); - TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "42 as constant isn't 42"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym), "42 is not a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) != NULL, "42 as constant is NULL"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == val_42, "42 as constant isn't 42"); TEST_PREDICATE(_Py_uop_sym_is_immortal(sym), "42 is not immortal"); _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); - TEST_PREDICATE(_Py_uop_sym_get_const(sym) == val_42, "(42 and 42) as constant isn't 42"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == val_42, "(42 and 42) as constant isn't 42"); _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); @@ -709,11 +808,11 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) sym = _Py_uop_sym_new_const(ctx, Py_None); - TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(None) is not False"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(None) is not False"); sym = _Py_uop_sym_new_const(ctx, Py_False); - TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(False) is not False"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(False) is not False"); sym = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); - TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(0) is not False"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(0) is not False"); JitOptSymbol *i1 = _Py_uop_sym_new_type(ctx, &PyFloat_Type); JitOptSymbol *i2 = _Py_uop_sym_new_const(ctx, val_43); @@ -724,17 +823,31 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) "tuple item does not match value used to create tuple" ); TEST_PREDICATE( - _Py_uop_sym_get_const(_Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, "tuple item does not match value used to create tuple" ); PyObject *pair[2] = { val_42, val_43 }; - PyObject *tuple = _PyTuple_FromArray(pair, 2); + tuple = _PyTuple_FromArray(pair, 2); sym = _Py_uop_sym_new_const(ctx, tuple); TEST_PREDICATE( - _Py_uop_sym_get_const(_Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, "tuple item does not match value used to create tuple" ); - + JitOptSymbol *value = _Py_uop_sym_new_type(ctx, &PyBool_Type); + sym = _Py_uop_sym_new_truthiness(ctx, value, false); + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == -1, "truthiness is not unknown"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym) == false, "truthiness is constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "truthiness is not NULL"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == false, "value is constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == NULL, "value is not NULL"); + _Py_uop_sym_set_const(ctx, sym, Py_False); + TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "truthiness is not True"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym) == true, "truthiness is not constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == Py_False, "truthiness is not False"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == true, "value is not constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == Py_True, "value is not True"); _Py_uop_abstractcontext_fini(ctx); Py_DECREF(val_42); Py_DECREF(val_43);

1 0

[3.13] gh-130637: Add validation for numeric response data in `stat()` method (GH-130646) (#130763)
by terryjreedy March 2, 2025

March 2, 2025

https://github.com/python/cpython/commit/e6dfa9d6013018c1f5d8e3a72b73cdab2a… commit: e6dfa9d6013018c1f5d8e3a72b73cdab2a5664c2 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: terryjreedy <tjreedy(a)udel.edu> date: 2025-03-02T20:28:56Z summary: [3.13] gh-130637: Add validation for numeric response data in `stat()` method (GH-130646) (#130763) gh-130637: Add validation for numeric response data in `stat()` method (GH-130646) (cherry picked from commit a42168d316f0c9a4fc5658dab87682dc19054efb) Co-authored-by: Kanishk Pachauri <itskanishkp.py(a)gmail.com> Co-authored-by: Eric V. Smith <ericvsmith(a)users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2025-03-01-02-19-28.gh-issue-130637.swet54w4rs.rst M Lib/poplib.py M Lib/test/test_poplib.py diff --git a/Lib/poplib.py b/Lib/poplib.py index beb93a0d57cf93..4469bff44b4c45 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -226,8 +226,19 @@ def stat(self): retval = self._shortcmd('STAT') rets = retval.split() if self._debugging: print('*stat*', repr(rets)) - numMessages = int(rets[1]) - sizeMessages = int(rets[2]) + + # Check if the response has enough elements + # RFC 1939 requires at least 3 elements (+OK, message count, mailbox size) + # but allows additional data after the required fields + if len(rets) < 3: + raise error_proto("Invalid STAT response format") + + try: + numMessages = int(rets[1]) + sizeMessages = int(rets[2]) + except ValueError: + raise error_proto("Invalid STAT response data: non-numeric values") + return (numMessages, sizeMessages) diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 869f9431b928bb..f1ebbeafe0cfb4 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -289,6 +289,37 @@ def test_pass_(self): def test_stat(self): self.assertEqual(self.client.stat(), (10, 100)) + original_shortcmd = self.client._shortcmd + def mock_shortcmd_invalid_format(cmd): + if cmd == 'STAT': + return b'+OK' + return original_shortcmd(cmd) + + self.client._shortcmd = mock_shortcmd_invalid_format + with self.assertRaises(poplib.error_proto): + self.client.stat() + + def mock_shortcmd_invalid_data(cmd): + if cmd == 'STAT': + return b'+OK abc def' + return original_shortcmd(cmd) + + self.client._shortcmd = mock_shortcmd_invalid_data + with self.assertRaises(poplib.error_proto): + self.client.stat() + + def mock_shortcmd_extra_fields(cmd): + if cmd == 'STAT': + return b'+OK 1 2 3 4 5' + return original_shortcmd(cmd) + + self.client._shortcmd = mock_shortcmd_extra_fields + + result = self.client.stat() + self.assertEqual(result, (1, 2)) + + self.client._shortcmd = original_shortcmd + def test_list(self): self.assertEqual(self.client.list()[1:], ([b'1 1', b'2 2', b'3 3', b'4 4', b'5 5'], diff --git a/Misc/NEWS.d/next/Library/2025-03-01-02-19-28.gh-issue-130637.swet54w4rs.rst b/Misc/NEWS.d/next/Library/2025-03-01-02-19-28.gh-issue-130637.swet54w4rs.rst new file mode 100644 index 00000000000000..83cd6c63c35215 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-01-02-19-28.gh-issue-130637.swet54w4rs.rst @@ -0,0 +1 @@ +Add validation for numeric response data in poplib.POP3.stat() method

1 0

[3.12] gh-130637: Add validation for numeric response data in `stat()` method (GH-130646) (#130764)
by terryjreedy March 2, 2025

March 2, 2025

https://github.com/python/cpython/commit/d9ca98a92d8fbd671d818c2cc0b4811823… commit: d9ca98a92d8fbd671d818c2cc0b481182370f1fc branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: terryjreedy <tjreedy(a)udel.edu> date: 2025-03-02T15:05:46-05:00 summary: [3.12] gh-130637: Add validation for numeric response data in `stat()` method (GH-130646) (#130764) gh-130637: Add validation for numeric response data in `stat()` method (GH-130646) (cherry picked from commit a42168d316f0c9a4fc5658dab87682dc19054efb) Co-authored-by: Kanishk Pachauri <itskanishkp.py(a)gmail.com> Co-authored-by: Eric V. Smith <ericvsmith(a)users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2025-03-01-02-19-28.gh-issue-130637.swet54w4rs.rst M Lib/poplib.py M Lib/test/test_poplib.py diff --git a/Lib/poplib.py b/Lib/poplib.py index 81b01385987c3b..9eb662d0000cb1 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -226,8 +226,19 @@ def stat(self): retval = self._shortcmd('STAT') rets = retval.split() if self._debugging: print('*stat*', repr(rets)) - numMessages = int(rets[1]) - sizeMessages = int(rets[2]) + + # Check if the response has enough elements + # RFC 1939 requires at least 3 elements (+OK, message count, mailbox size) + # but allows additional data after the required fields + if len(rets) < 3: + raise error_proto("Invalid STAT response format") + + try: + numMessages = int(rets[1]) + sizeMessages = int(rets[2]) + except ValueError: + raise error_proto("Invalid STAT response data: non-numeric values") + return (numMessages, sizeMessages) diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 869f9431b928bb..f1ebbeafe0cfb4 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -289,6 +289,37 @@ def test_pass_(self): def test_stat(self): self.assertEqual(self.client.stat(), (10, 100)) + original_shortcmd = self.client._shortcmd + def mock_shortcmd_invalid_format(cmd): + if cmd == 'STAT': + return b'+OK' + return original_shortcmd(cmd) + + self.client._shortcmd = mock_shortcmd_invalid_format + with self.assertRaises(poplib.error_proto): + self.client.stat() + + def mock_shortcmd_invalid_data(cmd): + if cmd == 'STAT': + return b'+OK abc def' + return original_shortcmd(cmd) + + self.client._shortcmd = mock_shortcmd_invalid_data + with self.assertRaises(poplib.error_proto): + self.client.stat() + + def mock_shortcmd_extra_fields(cmd): + if cmd == 'STAT': + return b'+OK 1 2 3 4 5' + return original_shortcmd(cmd) + + self.client._shortcmd = mock_shortcmd_extra_fields + + result = self.client.stat() + self.assertEqual(result, (1, 2)) + + self.client._shortcmd = original_shortcmd + def test_list(self): self.assertEqual(self.client.list()[1:], ([b'1 1', b'2 2', b'3 3', b'4 4', b'5 5'], diff --git a/Misc/NEWS.d/next/Library/2025-03-01-02-19-28.gh-issue-130637.swet54w4rs.rst b/Misc/NEWS.d/next/Library/2025-03-01-02-19-28.gh-issue-130637.swet54w4rs.rst new file mode 100644 index 00000000000000..83cd6c63c35215 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-01-02-19-28.gh-issue-130637.swet54w4rs.rst @@ -0,0 +1 @@ +Add validation for numeric response data in poplib.POP3.stat() method

1 0

[3.13] gh-128481: indicate that the default value for `FrameSummary.end_lineno` changed in 3.13 (GH-130755) (#130767)
by picnixz March 2, 2025

March 2, 2025

https://github.com/python/cpython/commit/3bef4af30124518b439362247709f5ae2b… commit: 3bef4af30124518b439362247709f5ae2bf57c11 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: picnixz <10796600+picnixz(a)users.noreply.github.com> date: 2025-03-02T17:22:48Z summary: [3.13] gh-128481: indicate that the default value for `FrameSummary.end_lineno` changed in 3.13 (GH-130755) (#130767) gh-128481: indicate that the default value for `FrameSummary.end_lineno` changed in 3.13 (GH-130755) The value taken by `FrameSummary.end_lineno` when passing `end_lineno=None` changed in gh-112097. Previously, a `end_lineno` could be specified to be `None` directly but since 939fc6d, passing None makes the constructor use the value of `lineno` instead. (cherry picked from commit c6513f7a627c8918a5e3f3733fa47d34a94ddff9) Co-authored-by: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> files: M Doc/library/traceback.rst diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index ebd691a363959a..291c2f1ce755ac 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -542,6 +542,9 @@ in a :ref:`traceback <traceback-objects>`. The last line number of the source code for this frame. By default, it is set to ``lineno`` and indexation starts from 1. + .. versionchanged:: 3.13 + The default value changed from ``None`` to ``lineno``. + .. attribute:: FrameSummary.colno The column number of the source code for this frame.

1 0

gh-128481: indicate that the default value for `FrameSummary.end_lineno` changed in 3.13 (#130755)
by picnixz March 2, 2025

March 2, 2025

https://github.com/python/cpython/commit/c6513f7a627c8918a5e3f3733fa47d34a9… commit: c6513f7a627c8918a5e3f3733fa47d34a94ddff9 branch: main author: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> committer: picnixz <10796600+picnixz(a)users.noreply.github.com> date: 2025-03-02T18:16:51+01:00 summary: gh-128481: indicate that the default value for `FrameSummary.end_lineno` changed in 3.13 (#130755) The value taken by `FrameSummary.end_lineno` when passing `end_lineno=None` changed in gh-112097. Previously, a `end_lineno` could be specified to be `None` directly but since 939fc6d, passing None makes the constructor use the value of `lineno` instead. files: M Doc/library/traceback.rst diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 757fdf601cd0fe..b5464ac55ddfa9 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -551,6 +551,9 @@ in a :ref:`traceback <traceback-objects>`. The last line number of the source code for this frame. By default, it is set to ``lineno`` and indexation starts from 1. + .. versionchanged:: 3.13 + The default value changed from ``None`` to ``lineno``. + .. attribute:: FrameSummary.colno The column number of the source code for this frame.

1 0

gh-130637: Add validation for numeric response data in `stat()` method (#130646)
by ericvsmith March 2, 2025

March 2, 2025

https://github.com/python/cpython/commit/a42168d316f0c9a4fc5658dab87682dc19… commit: a42168d316f0c9a4fc5658dab87682dc19054efb branch: main author: Kanishk Pachauri <itskanishkp.py(a)gmail.com> committer: ericvsmith <ericvsmith(a)users.noreply.github.com> date: 2025-03-02T08:05:40-05:00 summary: gh-130637: Add validation for numeric response data in `stat()` method (#130646) Co-authored-by: Eric V. Smith <ericvsmith(a)users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2025-03-01-02-19-28.gh-issue-130637.swet54w4rs.rst M Lib/poplib.py M Lib/test/test_poplib.py diff --git a/Lib/poplib.py b/Lib/poplib.py index beb93a0d57cf93..4469bff44b4c45 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -226,8 +226,19 @@ def stat(self): retval = self._shortcmd('STAT') rets = retval.split() if self._debugging: print('*stat*', repr(rets)) - numMessages = int(rets[1]) - sizeMessages = int(rets[2]) + + # Check if the response has enough elements + # RFC 1939 requires at least 3 elements (+OK, message count, mailbox size) + # but allows additional data after the required fields + if len(rets) < 3: + raise error_proto("Invalid STAT response format") + + try: + numMessages = int(rets[1]) + sizeMessages = int(rets[2]) + except ValueError: + raise error_proto("Invalid STAT response data: non-numeric values") + return (numMessages, sizeMessages) diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 869f9431b928bb..f1ebbeafe0cfb4 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -289,6 +289,37 @@ def test_pass_(self): def test_stat(self): self.assertEqual(self.client.stat(), (10, 100)) + original_shortcmd = self.client._shortcmd + def mock_shortcmd_invalid_format(cmd): + if cmd == 'STAT': + return b'+OK' + return original_shortcmd(cmd) + + self.client._shortcmd = mock_shortcmd_invalid_format + with self.assertRaises(poplib.error_proto): + self.client.stat() + + def mock_shortcmd_invalid_data(cmd): + if cmd == 'STAT': + return b'+OK abc def' + return original_shortcmd(cmd) + + self.client._shortcmd = mock_shortcmd_invalid_data + with self.assertRaises(poplib.error_proto): + self.client.stat() + + def mock_shortcmd_extra_fields(cmd): + if cmd == 'STAT': + return b'+OK 1 2 3 4 5' + return original_shortcmd(cmd) + + self.client._shortcmd = mock_shortcmd_extra_fields + + result = self.client.stat() + self.assertEqual(result, (1, 2)) + + self.client._shortcmd = original_shortcmd + def test_list(self): self.assertEqual(self.client.list()[1:], ([b'1 1', b'2 2', b'3 3', b'4 4', b'5 5'], diff --git a/Misc/NEWS.d/next/Library/2025-03-01-02-19-28.gh-issue-130637.swet54w4rs.rst b/Misc/NEWS.d/next/Library/2025-03-01-02-19-28.gh-issue-130637.swet54w4rs.rst new file mode 100644 index 00000000000000..83cd6c63c35215 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-01-02-19-28.gh-issue-130637.swet54w4rs.rst @@ -0,0 +1 @@ +Add validation for numeric response data in poplib.POP3.stat() method

1 0

gh-89083: add support for UUID version 6 (RFC 9562) (#120650)
by picnixz March 2, 2025

March 2, 2025

https://github.com/python/cpython/commit/990ad272f66fe6a50087ad044725bb0f9f… commit: 990ad272f66fe6a50087ad044725bb0f9f8e181d branch: main author: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> committer: picnixz <10796600+picnixz(a)users.noreply.github.com> date: 2025-03-02T12:41:56+01:00 summary: gh-89083: add support for UUID version 6 (RFC 9562) (#120650) Add support for generating UUIDv6 objects according to RFC 9562, §5.6 [1]. The functionality is provided by the `uuid.uuid6()` function which takes as inputs an optional 48-bit hardware address and an optional 14-bit clock sequence. The UUIDv6 temporal fields are ordered differently than those of UUIDv1, thereby providing improved database locality. [1]: https://www.rfc-editor.org/rfc/rfc9562.html#section-5.6 --------- Co-authored-by: Hugo van Kemenade <1324225+hugovk(a)users.noreply.github.com> Co-authored-by: Victor Stinner <vstinner(a)python.org> files: A Misc/NEWS.d/next/Library/2024-06-17-17-31-27.gh-issue-89083.nW00Yq.rst M Doc/library/uuid.rst M Doc/whatsnew/3.14.rst M Lib/test/test_uuid.py M Lib/uuid.py diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index c661fa2e52565c..88e5fae70b76d9 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -12,8 +12,8 @@ This module provides immutable :class:`UUID` objects (the :class:`UUID` class) and the functions :func:`uuid1`, :func:`uuid3`, :func:`uuid4`, :func:`uuid5`, -and :func:`uuid.uuid8` for generating version 1, 3, 4, 5, and 8 UUIDs as -specified in :rfc:`9562` (which supersedes :rfc:`4122`). +:func:`uuid6`, and :func:`uuid8` for generating version 1, 3, 4, 5, 6, +and 8 UUIDs as specified in :rfc:`9562` (which supersedes :rfc:`4122`). If all you want is a unique ID, you should probably call :func:`uuid1` or :func:`uuid4`. Note that :func:`uuid1` may compromise privacy since it creates @@ -153,8 +153,8 @@ which relays any information about the UUID's safety, using this enumeration: The UUID version number (1 through 8, meaningful only when the variant is :const:`RFC_4122`). - .. versionchanged:: 3.14 - Added UUID version 8. + .. versionchanged:: next + Added UUID versions 6 and 8. .. attribute:: UUID.is_safe @@ -212,6 +212,22 @@ The :mod:`uuid` module defines the following functions: that will be encoded using UTF-8). +.. function:: uuid6(node=None, clock_seq=None) + + Generate a UUID from a sequence number and the current time according to + :rfc:`9562`. + This is an alternative to :func:`uuid1` to improve database locality. + + When *node* is not specified, :func:`getnode` is used to obtain the hardware + address as a 48-bit positive integer. When a sequence number *clock_seq* is + not specified, a pseudo-random 14-bit positive integer is generated. + + If *node* or *clock_seq* exceed their expected bit count, only their least + significant bits are kept. + + .. versionadded:: next + + .. function:: uuid8(a=None, b=None, c=None) Generate a pseudo-random UUID according to @@ -314,7 +330,7 @@ The :mod:`uuid` module can be executed as a script from the command line. .. code-block:: sh - python -m uuid [-h] [-u {uuid1,uuid3,uuid4,uuid5,uuid8}] [-n NAMESPACE] [-N NAME] + python -m uuid [-h] [-u {uuid1,uuid3,uuid4,uuid5,uuid6,uuid8}] [-n NAMESPACE] [-N NAME] The following options are accepted: @@ -330,8 +346,8 @@ The following options are accepted: Specify the function name to use to generate the uuid. By default :func:`uuid4` is used. - .. versionadded:: 3.14 - Allow generating UUID version 8. + .. versionchanged:: next + Allow generating UUID versions 6 and 8. .. option:: -n <namespace> --namespace <namespace> diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index dbd59a9d7be150..ce75b5fffc0a4c 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -919,8 +919,8 @@ urllib uuid ---- -* Add support for UUID version 8 via :func:`uuid.uuid8` as specified - in :rfc:`9562`. +* Add support for UUID versions 6 and 8 via :func:`uuid.uuid6` and + :func:`uuid.uuid8` respectively, as specified in :rfc:`9562`. (Contributed by Bénédikt Tran in :gh:`89083`.) * :const:`uuid.NIL` and :const:`uuid.MAX` are now available to represent the diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 8216c4dd00e35a..e284de93fbdfd1 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1,6 +1,3 @@ -import unittest -from test import support -from test.support import import_helper import builtins import contextlib import copy @@ -10,10 +7,14 @@ import pickle import random import sys +import unittest import weakref from itertools import product from unittest import mock +from test import support +from test.support import import_helper + py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = import_helper.import_fresh_module('uuid', fresh=['_uuid']) @@ -724,6 +725,152 @@ def test_uuid5(self): equal(u, self.uuid.UUID(v)) equal(str(u), v) + def test_uuid6(self): + equal = self.assertEqual + u = self.uuid.uuid6() + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 6) + + fake_nanoseconds = 0x1571_20a1_de1a_c533 + fake_node_value = 0x54e1_acf6_da7f + fake_clock_seq = 0x14c5 + with ( + mock.patch.object(self.uuid, '_last_timestamp_v6', None), + mock.patch.object(self.uuid, 'getnode', return_value=fake_node_value), + mock.patch('time.time_ns', return_value=fake_nanoseconds), + mock.patch('random.getrandbits', return_value=fake_clock_seq) + ): + u = self.uuid.uuid6() + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 6) + + # 32 (top) | 16 (mid) | 12 (low) == 60 (timestamp) + equal(u.time, 0x1e901fca_7a55_b92) + equal(u.fields[0], 0x1e901fca) # 32 top bits of time + equal(u.fields[1], 0x7a55) # 16 mid bits of time + # 4 bits of version + 12 low bits of time + equal((u.fields[2] >> 12) & 0xf, 6) + equal((u.fields[2] & 0xfff), 0xb92) + # 2 bits of variant + 6 high bits of clock_seq + equal((u.fields[3] >> 6) & 0xf, 2) + equal(u.fields[3] & 0x3f, fake_clock_seq >> 8) + # 8 low bits of clock_seq + equal(u.fields[4], fake_clock_seq & 0xff) + equal(u.fields[5], fake_node_value) + + def test_uuid6_uniqueness(self): + # Test that UUIDv6-generated values are unique. + + # Unlike UUIDv8, only 62 bits can be randomized for UUIDv6. + # In practice, however, it remains unlikely to generate two + # identical UUIDs for the same 60-bit timestamp if neither + # the node ID nor the clock sequence is specified. + uuids = {self.uuid.uuid6() for _ in range(1000)} + self.assertEqual(len(uuids), 1000) + versions = {u.version for u in uuids} + self.assertSetEqual(versions, {6}) + + timestamp = 0x1ec9414c_232a_b00 + fake_nanoseconds = (timestamp - 0x1b21dd21_3814_000) * 100 + + with mock.patch('time.time_ns', return_value=fake_nanoseconds): + def gen(): + with mock.patch.object(self.uuid, '_last_timestamp_v6', None): + return self.uuid.uuid6(node=0, clock_seq=None) + + # By the birthday paradox, sampling N = 1024 UUIDs with identical + # node IDs and timestamps results in duplicates with probability + # close to 1 (not having a duplicate happens with probability of + # order 1E-15) since only the 14-bit clock sequence is randomized. + N = 1024 + uuids = {gen() for _ in range(N)} + self.assertSetEqual({u.node for u in uuids}, {0}) + self.assertSetEqual({u.time for u in uuids}, {timestamp}) + self.assertLess(len(uuids), N, 'collision property does not hold') + + def test_uuid6_node(self): + # Make sure the given node ID appears in the UUID. + # + # Note: when no node ID is specified, the same logic as for UUIDv1 + # is applied to UUIDv6. In particular, there is no need to test that + # getnode() correctly returns positive integers of exactly 48 bits + # since this is done in test_uuid1_eui64(). + self.assertLessEqual(self.uuid.uuid6().node.bit_length(), 48) + + self.assertEqual(self.uuid.uuid6(0).node, 0) + + # tests with explicit values + max_node = 0xffff_ffff_ffff + self.assertEqual(self.uuid.uuid6(max_node).node, max_node) + big_node = 0xE_1234_5678_ABCD # 52-bit node + res_node = 0x0_1234_5678_ABCD # truncated to 48 bits + self.assertEqual(self.uuid.uuid6(big_node).node, res_node) + + # randomized tests + for _ in range(10): + # node with > 48 bits is truncated + for b in [24, 48, 72]: + node = (1 << (b - 1)) | random.getrandbits(b) + with self.subTest(node=node, bitlen=b): + self.assertEqual(node.bit_length(), b) + u = self.uuid.uuid6(node=node) + self.assertEqual(u.node, node & 0xffff_ffff_ffff) + + def test_uuid6_clock_seq(self): + # Make sure the supplied clock sequence appears in the UUID. + # + # For UUIDv6, clock sequence bits are stored from bit 48 to bit 62, + # with the convention that the least significant bit is bit 0 and + # the most significant bit is bit 127. + get_clock_seq = lambda u: (u.int >> 48) & 0x3fff + + u = self.uuid.uuid6() + self.assertLessEqual(get_clock_seq(u).bit_length(), 14) + + # tests with explicit values + big_clock_seq = 0xffff # 16-bit clock sequence + res_clock_seq = 0x3fff # truncated to 14 bits + u = self.uuid.uuid6(clock_seq=big_clock_seq) + self.assertEqual(get_clock_seq(u), res_clock_seq) + + # some randomized tests + for _ in range(10): + # clock_seq with > 14 bits is truncated + for b in [7, 14, 28]: + node = random.getrandbits(48) + clock_seq = (1 << (b - 1)) | random.getrandbits(b) + with self.subTest(node=node, clock_seq=clock_seq, bitlen=b): + self.assertEqual(clock_seq.bit_length(), b) + u = self.uuid.uuid6(node=node, clock_seq=clock_seq) + self.assertEqual(get_clock_seq(u), clock_seq & 0x3fff) + + def test_uuid6_test_vectors(self): + equal = self.assertEqual + # https://www.rfc-editor.org/rfc/rfc9562#name-test-vectors + # (separators are put at the 12th and 28th bits) + timestamp = 0x1ec9414c_232a_b00 + fake_nanoseconds = (timestamp - 0x1b21dd21_3814_000) * 100 + # https://www.rfc-editor.org/rfc/rfc9562#name-example-of-a-uuidv6-value + node = 0x9f6bdeced846 + clock_seq = (3 << 12) | 0x3c8 + + with ( + mock.patch.object(self.uuid, '_last_timestamp_v6', None), + mock.patch('time.time_ns', return_value=fake_nanoseconds) + ): + u = self.uuid.uuid6(node=node, clock_seq=clock_seq) + equal(str(u).upper(), '1EC9414C-232A-6B00-B3C8-9F6BDECED846') + # 32 16 4 12 2 14 48 + # time_hi | time_mid | ver | time_lo | var | clock_seq | node + equal(u.time, timestamp) + equal(u.int & 0xffff_ffff_ffff, node) + equal((u.int >> 48) & 0x3fff, clock_seq) + equal((u.int >> 62) & 0x3, 0b10) + equal((u.int >> 64) & 0xfff, 0xb00) + equal((u.int >> 76) & 0xf, 0x6) + equal((u.int >> 80) & 0xffff, 0x232a) + equal((u.int >> 96) & 0xffff_ffff, 0x1ec9_414c) + def test_uuid8(self): equal = self.assertEqual u = self.uuid.uuid8() diff --git a/Lib/uuid.py b/Lib/uuid.py index 36809b85cb8ceb..ed69b4de07b53f 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -1,8 +1,8 @@ r"""UUID objects (universally unique identifiers) according to RFC 4122/9562. This module provides immutable UUID objects (class UUID) and the functions -uuid1(), uuid3(), uuid4(), uuid5(), and uuid8() for generating version 1, 3, -4, 5, and 8 UUIDs as specified in RFC 4122/9562. +uuid1(), uuid3(), uuid4(), uuid5(), uuid6(), and uuid8() for generating +version 1, 3, 4, 5, 6, and 8 UUIDs as specified in RFC 4122/9562. If all you want is a unique ID, you should probably call uuid1() or uuid4(). Note that uuid1() may compromise privacy since it creates a UUID containing @@ -101,6 +101,7 @@ class SafeUUID: _RFC_4122_VERSION_3_FLAGS = ((3 << 76) | (0x8000 << 48)) _RFC_4122_VERSION_4_FLAGS = ((4 << 76) | (0x8000 << 48)) _RFC_4122_VERSION_5_FLAGS = ((5 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_6_FLAGS = ((6 << 76) | (0x8000 << 48)) _RFC_4122_VERSION_8_FLAGS = ((8 << 76) | (0x8000 << 48)) @@ -127,7 +128,9 @@ class UUID: fields a tuple of the six integer fields of the UUID, which are also available as six individual attributes - and two derived attributes: + and two derived attributes. The time_* attributes are + only relevant to version 1, while the others are only + relevant to versions 1 and 6: time_low the first 32 bits of the UUID time_mid the next 16 bits of the UUID @@ -353,8 +356,19 @@ def clock_seq_low(self): @property def time(self): - return (((self.time_hi_version & 0x0fff) << 48) | - (self.time_mid << 32) | self.time_low) + if self.version == 6: + # time_hi (32) | time_mid (16) | ver (4) | time_lo (12) | ... (64) + time_hi = self.int >> 96 + time_lo = (self.int >> 64) & 0x0fff + return time_hi << 28 | (self.time_mid << 12) | time_lo + else: + # time_lo (32) | time_mid (16) | ver (4) | time_hi (12) | ... (64) + # + # For compatibility purposes, we do not warn or raise when the + # version is not 1 (timestamp is irrelevant to other versions). + time_hi = (self.int >> 64) & 0x0fff + time_lo = self.int >> 96 + return time_hi << 48 | (self.time_mid << 32) | time_lo @property def clock_seq(self): @@ -756,6 +770,44 @@ def uuid5(namespace, name): int_uuid_5 |= _RFC_4122_VERSION_5_FLAGS return UUID._from_int(int_uuid_5) +_last_timestamp_v6 = None + +def uuid6(node=None, clock_seq=None): + """Similar to :func:`uuid1` but where fields are ordered differently + for improved DB locality. + + More precisely, given a 60-bit timestamp value as specified for UUIDv1, + for UUIDv6 the first 48 most significant bits are stored first, followed + by the 4-bit version (same position), followed by the remaining 12 bits + of the original 60-bit timestamp. + """ + global _last_timestamp_v6 + import time + nanoseconds = time.time_ns() + # 0x01b21dd213814000 is the number of 100-ns intervals between the + # UUID epoch 1582-10-15 00:00:00 and the Unix epoch 1970-01-01 00:00:00. + timestamp = nanoseconds // 100 + 0x01b21dd213814000 + if _last_timestamp_v6 is not None and timestamp <= _last_timestamp_v6: + timestamp = _last_timestamp_v6 + 1 + _last_timestamp_v6 = timestamp + if clock_seq is None: + import random + clock_seq = random.getrandbits(14) # instead of stable storage + time_hi_and_mid = (timestamp >> 12) & 0xffff_ffff_ffff + time_lo = timestamp & 0x0fff # keep 12 bits and clear version bits + clock_s = clock_seq & 0x3fff # keep 14 bits and clear variant bits + if node is None: + node = getnode() + # --- 32 + 16 --- -- 4 -- -- 12 -- -- 2 -- -- 14 --- 48 + # time_hi_and_mid | version | time_lo | variant | clock_seq | node + int_uuid_6 = time_hi_and_mid << 80 + int_uuid_6 |= time_lo << 64 + int_uuid_6 |= clock_s << 48 + int_uuid_6 |= node & 0xffff_ffff_ffff + # by construction, the variant and version bits are already cleared + int_uuid_6 |= _RFC_4122_VERSION_6_FLAGS + return UUID._from_int(int_uuid_6) + def uuid8(a=None, b=None, c=None): """Generate a UUID from three custom blocks. @@ -788,6 +840,7 @@ def main(): "uuid3": uuid3, "uuid4": uuid4, "uuid5": uuid5, + "uuid6": uuid6, "uuid8": uuid8, } uuid_namespace_funcs = ("uuid3", "uuid5") diff --git a/Misc/NEWS.d/next/Library/2024-06-17-17-31-27.gh-issue-89083.nW00Yq.rst b/Misc/NEWS.d/next/Library/2024-06-17-17-31-27.gh-issue-89083.nW00Yq.rst new file mode 100644 index 00000000000000..f4bda53d1a67d5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-17-17-31-27.gh-issue-89083.nW00Yq.rst @@ -0,0 +1,2 @@ +Add :func:`uuid.uuid6` for generating UUIDv6 objects as specified in +:rfc:`9562`. Patch by Bénédikt Tran.

1 0

[3.12] Fix grammar typo in `Doc/c-api/arg.rst` (GH-130741) (#130758)
by hugovk March 2, 2025

March 2, 2025

https://github.com/python/cpython/commit/96e6932bef469599269c15f389e832a56d… commit: 96e6932bef469599269c15f389e832a56d8d094a branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: hugovk <1324225+hugovk(a)users.noreply.github.com> date: 2025-03-02T11:21:57Z summary: [3.12] Fix grammar typo in `Doc/c-api/arg.rst` (GH-130741) (#130758) Fix grammar typo in `Doc/c-api/arg.rst` (GH-130741) (cherry picked from commit 37145cb89fe806377a5e9ed1fdac92dd3a5df2c0) Co-authored-by: Arijit Kumar Das <arijitkrdas2004(a)outlook.com> files: M Doc/c-api/arg.rst diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 76a197bbec73e4..9d53d0e56a4e75 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -5,7 +5,7 @@ Parsing arguments and building values ===================================== -These functions are useful when creating your own extensions functions and +These functions are useful when creating your own extension functions and methods. Additional information and examples are available in :ref:`extending-index`.

1 0