Mailman 3 June 2022 - Python-checkins

[3.11] GH-93354: Use exponential backoff to avoid excessive specialization attempts (GH-93355) (GH-93379)
by ambv 30 Jun '22

30 Jun '22

https://github.com/python/cpython/commit/113b309f18103343a195f7c53c41763acf… commit: 113b309f18103343a195f7c53c41763acf295167 branch: 3.11 author: Mark Shannon <mark(a)hotpy.org> committer: ambv <lukasz(a)langa.pl> date: 2022-06-30T23:03:37+02:00 summary: [3.11] GH-93354: Use exponential backoff to avoid excessive specialization attempts (GH-93355) (GH-93379) Co-authored-by: Mark Shannon <mark(a)hotpy.org> Co-authored-by: Łukasz Langa <lukasz(a)langa.pl> files: A Misc/NEWS.d/next/Core and Builtins/2022-05-30-15-35-42.gh-issue-93354.RZk8gs.rst M Include/internal/pycore_code.h M Python/ceval.c M Python/specialize.c diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 551b9c01e6a98..9528c8996a443 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -233,9 +233,6 @@ extern void _PyLineTable_InitAddressRange( extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range); extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); - -#define ADAPTIVE_CACHE_BACKOFF 64 - /* Specialization functions */ extern int _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, @@ -475,6 +472,50 @@ write_location_entry_start(uint8_t *ptr, int code, int length) } +/** Counters + * The first 16-bit value in each inline cache is a counter. + * When counting misses, the counter is treated as a simple unsigned value. + * + * When counting executions until the next specialization attempt, + * exponential backoff is used to reduce the number of specialization failures. + * The high 12 bits store the counter, the low 4 bits store the backoff exponent. + * On a specialization failure, the backoff exponent is incremented and the + * counter set to (2**backoff - 1). + * Backoff == 6 -> starting counter == 63, backoff == 10 -> starting counter == 1023. + */ + +/* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */ +#define ADAPTIVE_BACKOFF_BITS 4 +/* The initial counter value is 31 == 2**ADAPTIVE_BACKOFF_START - 1 */ +#define ADAPTIVE_BACKOFF_START 5 + +#define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS) + + +static inline uint16_t +adaptive_counter_bits(int value, int backoff) { + return (value << ADAPTIVE_BACKOFF_BITS) | + (backoff & ((1<<ADAPTIVE_BACKOFF_BITS)-1)); +} + +static inline uint16_t +adaptive_counter_start(void) { + unsigned int value = (1 << ADAPTIVE_BACKOFF_START) - 1; + return adaptive_counter_bits(value, ADAPTIVE_BACKOFF_START); +} + +static inline uint16_t +adaptive_counter_backoff(uint16_t counter) { + unsigned int backoff = counter & ((1<<ADAPTIVE_BACKOFF_BITS)-1); + backoff++; + if (backoff > MAX_BACKOFF_VALUE) { + backoff = MAX_BACKOFF_VALUE; + } + unsigned int value = (1 << backoff) - 1; + return adaptive_counter_bits(value, backoff); +} + + /* Line array cache for tracing */ extern int _PyCode_CreateLineArray(PyCodeObject *co); diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-05-30-15-35-42.gh-issue-93354.RZk8gs.rst b/Misc/NEWS.d/next/Core and Builtins/2022-05-30-15-35-42.gh-issue-93354.RZk8gs.rst new file mode 100644 index 0000000000000..dcfe6a9b6ba3a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-05-30-15-35-42.gh-issue-93354.RZk8gs.rst @@ -0,0 +1,3 @@ +Use exponential backoff for specialization counters in the interpreter. Can +reduce the number of failed specializations significantly and avoid slowdown +for those parts of a program that are not suitable for specialization. diff --git a/Python/ceval.c b/Python/ceval.c index b8681617c8c22..a2c1d0bc43985 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1559,7 +1559,11 @@ eval_frame_handle_pending(PyThreadState *tstate) dtrace_function_entry(frame); \ } +#define ADAPTIVE_COUNTER_IS_ZERO(cache) \ + (cache)->counter < (1<<ADAPTIVE_BACKOFF_BITS) +#define DECREMENT_ADAPTIVE_COUNTER(cache) \ + (cache)->counter -= (1<<ADAPTIVE_BACKOFF_BITS) static int trace_function_entry(PyThreadState *tstate, _PyInterpreterFrame *frame) @@ -2154,7 +2158,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(BINARY_SUBSCR_ADAPTIVE) { _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (cache->counter == 0) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *sub = TOP(); PyObject *container = SECOND(); next_instr--; @@ -2165,7 +2169,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } else { STAT_INC(BINARY_SUBSCR, deferred); - cache->counter--; + DECREMENT_ADAPTIVE_COUNTER(cache); JUMP_TO_INSTRUCTION(BINARY_SUBSCR); } } @@ -2319,7 +2323,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(STORE_SUBSCR_ADAPTIVE) { _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (cache->counter == 0) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *sub = TOP(); PyObject *container = SECOND(); next_instr--; @@ -2330,7 +2334,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } else { STAT_INC(STORE_SUBSCR, deferred); - cache->counter--; + DECREMENT_ADAPTIVE_COUNTER(cache); JUMP_TO_INSTRUCTION(STORE_SUBSCR); } } @@ -2812,7 +2816,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(UNPACK_SEQUENCE_ADAPTIVE) { assert(cframe.use_tracing == 0); _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (cache->counter == 0) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *seq = TOP(); next_instr--; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); @@ -2820,7 +2824,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } else { STAT_INC(UNPACK_SEQUENCE, deferred); - cache->counter--; + DECREMENT_ADAPTIVE_COUNTER(cache); JUMP_TO_INSTRUCTION(UNPACK_SEQUENCE); } } @@ -3053,7 +3057,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(LOAD_GLOBAL_ADAPTIVE) { assert(cframe.use_tracing == 0); _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (cache->counter == 0) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *name = GETITEM(names, oparg>>1); next_instr--; if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) { @@ -3063,7 +3067,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } else { STAT_INC(LOAD_GLOBAL, deferred); - cache->counter--; + DECREMENT_ADAPTIVE_COUNTER(cache); JUMP_TO_INSTRUCTION(LOAD_GLOBAL); } } @@ -3477,7 +3481,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(LOAD_ATTR_ADAPTIVE) { assert(cframe.use_tracing == 0); _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (cache->counter == 0) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; @@ -3488,7 +3492,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } else { STAT_INC(LOAD_ATTR, deferred); - cache->counter--; + DECREMENT_ADAPTIVE_COUNTER(cache); JUMP_TO_INSTRUCTION(LOAD_ATTR); } } @@ -3586,7 +3590,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(STORE_ATTR_ADAPTIVE) { assert(cframe.use_tracing == 0); _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (cache->counter == 0) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; @@ -3597,7 +3601,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } else { STAT_INC(STORE_ATTR, deferred); - cache->counter--; + DECREMENT_ADAPTIVE_COUNTER(cache); JUMP_TO_INSTRUCTION(STORE_ATTR); } } @@ -3716,7 +3720,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(COMPARE_OP_ADAPTIVE) { assert(cframe.use_tracing == 0); _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (cache->counter == 0) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *right = TOP(); PyObject *left = SECOND(); next_instr--; @@ -3725,7 +3729,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } else { STAT_INC(COMPARE_OP, deferred); - cache->counter--; + DECREMENT_ADAPTIVE_COUNTER(cache); JUMP_TO_INSTRUCTION(COMPARE_OP); } } @@ -4523,7 +4527,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(LOAD_METHOD_ADAPTIVE) { assert(cframe.use_tracing == 0); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; - if (cache->counter == 0) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; @@ -4534,7 +4538,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } else { STAT_INC(LOAD_METHOD, deferred); - cache->counter--; + DECREMENT_ADAPTIVE_COUNTER(cache); JUMP_TO_INSTRUCTION(LOAD_METHOD); } } @@ -4815,7 +4819,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(CALL_ADAPTIVE) { _PyCallCache *cache = (_PyCallCache *)next_instr; - if (cache->counter == 0) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { next_instr--; int is_meth = is_method(stack_pointer, oparg); int nargs = oparg + is_meth; @@ -4829,7 +4833,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } else { STAT_INC(CALL, deferred); - cache->counter--; + DECREMENT_ADAPTIVE_COUNTER(cache); goto call_function; } } @@ -5560,7 +5564,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int TARGET(BINARY_OP_ADAPTIVE) { assert(cframe.use_tracing == 0); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (cache->counter == 0) { + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { PyObject *lhs = SECOND(); PyObject *rhs = TOP(); next_instr--; @@ -5569,7 +5573,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } else { STAT_INC(BINARY_OP, deferred); - cache->counter--; + DECREMENT_ADAPTIVE_COUNTER(cache); JUMP_TO_INSTRUCTION(BINARY_OP); } } @@ -5690,7 +5694,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(adaptive_opcode); _Py_SET_OPCODE(next_instr[-1], adaptive_opcode); STAT_INC(opcode, deopt); - *counter = ADAPTIVE_CACHE_BACKOFF; + *counter = adaptive_counter_start(); } next_instr--; DISPATCH_GOTO(); diff --git a/Python/specialize.c b/Python/specialize.c index 80d952c90a813..cf7bc32a205be 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -316,7 +316,7 @@ _PyCode_Quicken(PyCodeObject *code) } static inline int -initial_counter_value(void) { +miss_counter_start(void) { /* Starting value for the counter. * This value needs to be not too low, otherwise * it would cause excessive de-optimization. @@ -738,12 +738,12 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(LOAD_ATTR, failure); assert(!PyErr_Occurred()); - cache->counter = ADAPTIVE_CACHE_BACKOFF; + cache->counter = adaptive_counter_backoff(cache->counter); return 0; success: STAT_INC(LOAD_ATTR, success); assert(!PyErr_Occurred()); - cache->counter = initial_counter_value(); + cache->counter = miss_counter_start(); return 0; } @@ -820,12 +820,12 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(STORE_ATTR, failure); assert(!PyErr_Occurred()); - cache->counter = ADAPTIVE_CACHE_BACKOFF; + cache->counter = adaptive_counter_backoff(cache->counter); return 0; success: STAT_INC(STORE_ATTR, success); assert(!PyErr_Occurred()); - cache->counter = initial_counter_value(); + cache->counter = miss_counter_start(); return 0; } @@ -1027,14 +1027,13 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) success: STAT_INC(LOAD_METHOD, success); assert(!PyErr_Occurred()); - cache->counter = initial_counter_value(); + cache->counter = miss_counter_start(); return 0; fail: STAT_INC(LOAD_METHOD, failure); assert(!PyErr_Occurred()); - cache->counter = ADAPTIVE_CACHE_BACKOFF; + cache->counter = adaptive_counter_backoff(cache->counter); return 0; - } int @@ -1110,12 +1109,12 @@ _Py_Specialize_LoadGlobal( fail: STAT_INC(LOAD_GLOBAL, failure); assert(!PyErr_Occurred()); - cache->counter = ADAPTIVE_CACHE_BACKOFF; + cache->counter = adaptive_counter_backoff(cache->counter); return 0; success: STAT_INC(LOAD_GLOBAL, success); assert(!PyErr_Occurred()); - cache->counter = initial_counter_value(); + cache->counter = miss_counter_start(); return 0; } @@ -1239,12 +1238,12 @@ _Py_Specialize_BinarySubscr( fail: STAT_INC(BINARY_SUBSCR, failure); assert(!PyErr_Occurred()); - cache->counter = ADAPTIVE_CACHE_BACKOFF; + cache->counter = adaptive_counter_backoff(cache->counter); return 0; success: STAT_INC(BINARY_SUBSCR, success); assert(!PyErr_Occurred()); - cache->counter = initial_counter_value(); + cache->counter = miss_counter_start(); return 0; } @@ -1343,12 +1342,12 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins fail: STAT_INC(STORE_SUBSCR, failure); assert(!PyErr_Occurred()); - cache->counter = ADAPTIVE_CACHE_BACKOFF; + cache->counter = adaptive_counter_backoff(cache->counter); return 0; success: STAT_INC(STORE_SUBSCR, success); assert(!PyErr_Occurred()); - cache->counter = initial_counter_value(); + cache->counter = miss_counter_start(); return 0; } @@ -1646,12 +1645,12 @@ _Py_Specialize_Precall(PyObject *callable, _Py_CODEUNIT *instr, int nargs, if (fail) { STAT_INC(PRECALL, failure); assert(!PyErr_Occurred()); - cache->counter = ADAPTIVE_CACHE_BACKOFF; + cache->counter = adaptive_counter_backoff(cache->counter); } else { STAT_INC(PRECALL, success); assert(!PyErr_Occurred()); - cache->counter = initial_counter_value(); + cache->counter = miss_counter_start(); } return 0; } @@ -1678,12 +1677,12 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, if (fail) { STAT_INC(CALL, failure); assert(!PyErr_Occurred()); - cache->counter = ADAPTIVE_CACHE_BACKOFF; + cache->counter = adaptive_counter_backoff(cache->counter); } else { STAT_INC(CALL, success); assert(!PyErr_Occurred()); - cache->counter = initial_counter_value(); + cache->counter = miss_counter_start(); } return 0; } @@ -1831,11 +1830,11 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, } SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); STAT_INC(BINARY_OP, failure); - cache->counter = ADAPTIVE_CACHE_BACKOFF; + cache->counter = adaptive_counter_backoff(cache->counter); return; success: STAT_INC(BINARY_OP, success); - cache->counter = initial_counter_value(); + cache->counter = miss_counter_start(); } @@ -1958,11 +1957,11 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); failure: STAT_INC(COMPARE_OP, failure); - cache->counter = ADAPTIVE_CACHE_BACKOFF; + cache->counter = adaptive_counter_backoff(cache->counter); return; success: STAT_INC(COMPARE_OP, success); - cache->counter = initial_counter_value(); + cache->counter = miss_counter_start(); } #ifdef Py_STATS @@ -2008,11 +2007,11 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq)); failure: STAT_INC(UNPACK_SEQUENCE, failure); - cache->counter = ADAPTIVE_CACHE_BACKOFF; + cache->counter = adaptive_counter_backoff(cache->counter); return; success: STAT_INC(UNPACK_SEQUENCE, success); - cache->counter = initial_counter_value(); + cache->counter = miss_counter_start(); } #ifdef Py_STATS

1 0

multiprocessing.spawn doc: Capitalize the p in "Python" (gh-94462)
by miss-islington 30 Jun '22

30 Jun '22

https://github.com/python/cpython/commit/6c405381918a358a62ce72798dd37c2a3d… commit: 6c405381918a358a62ce72798dd37c2a3d2a9be2 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-06-30T11:23:14-07:00 summary: multiprocessing.spawn doc: Capitalize the p in "Python" (gh-94462) (cherry picked from commit 62bb7a3b50150495e215d7bd32f633eef81b3bc2) Co-authored-by: Mariatta Wijaya <Mariatta(a)users.noreply.github.com> files: M Doc/library/multiprocessing.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 2a66b0f65c088..a8e35bc084747 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -96,7 +96,7 @@ Depending on the platform, :mod:`multiprocessing` supports three ways to start a process. These *start methods* are *spawn* - The parent process starts a fresh python interpreter process. The + The parent process starts a fresh Python interpreter process. The child process will only inherit those resources necessary to run the process object's :meth:`~Process.run` method. In particular, unnecessary file descriptors and handles from the parent process

1 0

multiprocessing.spawn doc: Capitalize the p in "Python" (gh-94462)
by miss-islington 30 Jun '22

30 Jun '22

https://github.com/python/cpython/commit/b8544e18e61a0f4a873b3f4b4b22118225… commit: b8544e18e61a0f4a873b3f4b4b2211822584b63f branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-06-30T11:22:58-07:00 summary: multiprocessing.spawn doc: Capitalize the p in "Python" (gh-94462) (cherry picked from commit 62bb7a3b50150495e215d7bd32f633eef81b3bc2) Co-authored-by: Mariatta Wijaya <Mariatta(a)users.noreply.github.com> files: M Doc/library/multiprocessing.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index de8f9718337d7..0de44855a50c0 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -96,7 +96,7 @@ Depending on the platform, :mod:`multiprocessing` supports three ways to start a process. These *start methods* are *spawn* - The parent process starts a fresh python interpreter process. The + The parent process starts a fresh Python interpreter process. The child process will only inherit those resources necessary to run the process object's :meth:`~Process.run` method. In particular, unnecessary file descriptors and handles from the parent process

1 0

gh-84753: Make inspect.iscoroutinefunction() work with AsyncMock (GH-94050) (GH-94460)
by ambv 30 Jun '22

30 Jun '22

https://github.com/python/cpython/commit/9ebec7d46cd52fff68507c291fde34fc86… commit: 9ebec7d46cd52fff68507c291fde34fc865acfc4 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: ambv <lukasz(a)langa.pl> date: 2022-06-30T20:04:42+02:00 summary: gh-84753: Make inspect.iscoroutinefunction() work with AsyncMock (GH-94050) (GH-94460) The inspect version was not working with unittest.mock.AsyncMock. The fix introduces special-casing of AsyncMock in `inspect.iscoroutinefunction` equivalent to the one performed in `asyncio.iscoroutinefunction`. Co-authored-by: Łukasz Langa <lukasz(a)langa.pl> (cherry picked from commit 4261b6bffc0b8bb5c6d4d80578a81b7520f4aefc) Co-authored-by: Mehdi ABAAKOUK <sileht(a)sileht.net> files: A Misc/NEWS.d/next/Library/2022-06-21-11-40-31.gh-issue-84753.FW1pxO.rst M Lib/inspect.py M Lib/test/test_asyncio/test_tasks.py M Lib/test/test_inspect.py M Lib/unittest/mock.py diff --git a/Lib/inspect.py b/Lib/inspect.py index 6e744712f014b..cbc0632484b83 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -395,7 +395,7 @@ def _has_code_flag(f, flag): while ismethod(f): f = f.__func__ f = functools._unwrap_partial(f) - if not isfunction(f): + if not (isfunction(f) or _signature_is_functionlike(f)): return False return bool(f.__code__.co_flags & flag) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 6458859db2d12..b4a86271c64d8 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1653,6 +1653,7 @@ async def fn2(): self.assertTrue(asyncio.iscoroutinefunction(fn2)) self.assertFalse(asyncio.iscoroutinefunction(mock.Mock())) + self.assertTrue(asyncio.iscoroutinefunction(mock.AsyncMock())) def test_coroutine_non_gen_function(self): async def func(): diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index ae1842704d37d..be9f29e04ae11 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -202,6 +202,10 @@ def test_iscoroutine(self): gen_coroutine_function_example)))) self.assertTrue(inspect.isgenerator(gen_coro)) + self.assertFalse( + inspect.iscoroutinefunction(unittest.mock.Mock())) + self.assertTrue( + inspect.iscoroutinefunction(unittest.mock.AsyncMock())) self.assertTrue( inspect.iscoroutinefunction(coroutine_function_example)) self.assertTrue( @@ -210,6 +214,10 @@ def test_iscoroutine(self): coroutine_function_example)))) self.assertTrue(inspect.iscoroutine(coro)) + self.assertFalse( + inspect.isgeneratorfunction(unittest.mock.Mock())) + self.assertFalse( + inspect.isgeneratorfunction(unittest.mock.AsyncMock())) self.assertFalse( inspect.isgeneratorfunction(coroutine_function_example)) self.assertFalse( @@ -218,6 +226,12 @@ def test_iscoroutine(self): coroutine_function_example)))) self.assertFalse(inspect.isgenerator(coro)) + self.assertFalse( + inspect.isasyncgenfunction(unittest.mock.Mock())) + self.assertFalse( + inspect.isasyncgenfunction(unittest.mock.AsyncMock())) + self.assertFalse( + inspect.isasyncgenfunction(coroutine_function_example)) self.assertTrue( inspect.isasyncgenfunction(async_generator_function_example)) self.assertTrue( diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 8262b7170b76b..cd46fea5162ab 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -2175,6 +2175,10 @@ def __init__(self, /, *args, **kwargs): code_mock = NonCallableMock(spec_set=CodeType) code_mock.co_flags = inspect.CO_COROUTINE self.__dict__['__code__'] = code_mock + self.__dict__['__name__'] = 'AsyncMock' + self.__dict__['__defaults__'] = tuple() + self.__dict__['__kwdefaults__'] = {} + self.__dict__['__annotations__'] = None async def _execute_mock_call(self, /, *args, **kwargs): # This is nearly just like super(), except for special handling diff --git a/Misc/NEWS.d/next/Library/2022-06-21-11-40-31.gh-issue-84753.FW1pxO.rst b/Misc/NEWS.d/next/Library/2022-06-21-11-40-31.gh-issue-84753.FW1pxO.rst new file mode 100644 index 0000000000000..f701d2a1afeb8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-06-21-11-40-31.gh-issue-84753.FW1pxO.rst @@ -0,0 +1,3 @@ +:func:`inspect.iscoroutinefunction` now properly returns ``True`` when an instance +of :class:`unittest.mock.AsyncMock` is passed to it. This makes it consistent with +behavior of :func:`asyncio.iscoroutinefunction`. Patch by Mehdi ABAAKOUK.

1 0

GH-94398: TaskGroup: Fail create_task() during shutdown (GH-94400) (GH-94463)
by ambv 30 Jun '22

30 Jun '22

https://github.com/python/cpython/commit/7fe949e5eac3efa8bb0cd9e7274380a044… commit: 7fe949e5eac3efa8bb0cd9e7274380a044a408ea branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: ambv <lukasz(a)langa.pl> date: 2022-06-30T20:02:42+02:00 summary: GH-94398: TaskGroup: Fail create_task() during shutdown (GH-94400) (GH-94463) Once the task group is shutting down, it should not be possible to create a new task. Here "shutting down" means `self._aborting` is set, indicating that at least one task has failed and we have cancelled all others. Co-authored-by: Łukasz Langa <lukasz(a)langa.pl> (cherry picked from commit 594c3699492bfb007650538726d953cbed55de04) Co-authored-by: Guido van Rossum <guido(a)python.org> files: A Misc/NEWS.d/next/Library/2022-06-29-04-42-56.gh-issue-94398.YOq_bJ.rst M Doc/library/asyncio-task.rst M Lib/asyncio/taskgroups.py M Lib/test/test_asyncio/test_taskgroups.py diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 9b76548433b6a..76969762dd9f5 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -320,6 +320,7 @@ no new tasks may be added to the group. The first time any of the tasks belonging to the group fails with an exception other than :exc:`asyncio.CancelledError`, the remaining tasks in the group are cancelled. +No further tasks can then be added to the group. At this point, if the body of the ``async with`` statement is still active (i.e., :meth:`~object.__aexit__` hasn't been called yet), the task directly containing the ``async with`` statement is also cancelled. diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 9e0610deed281..3ca65062efd27 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -138,6 +138,8 @@ def create_task(self, coro, *, name=None, context=None): raise RuntimeError(f"TaskGroup {self!r} has not been entered") if self._exiting and not self._tasks: raise RuntimeError(f"TaskGroup {self!r} is finished") + if self._aborting: + raise RuntimeError(f"TaskGroup {self!r} is shutting down") if context is None: task = self._loop.create_task(coro) else: diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 69369a6100a8f..26fb5e466af92 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -122,10 +122,8 @@ async def runner(): self.assertTrue(t2.cancelled()) async def test_cancel_children_on_child_error(self): - """ - When a child task raises an error, the rest of the children - are cancelled and the errors are gathered into an EG. - """ + # When a child task raises an error, the rest of the children + # are cancelled and the errors are gathered into an EG. NUM = 0 t2_cancel = False @@ -722,6 +720,27 @@ async def coro(val): await t2 self.assertEqual(2, ctx.get(cvar)) + async def test_taskgroup_no_create_task_after_failure(self): + async def coro1(): + await asyncio.sleep(0.001) + 1 / 0 + async def coro2(g): + try: + await asyncio.sleep(1) + except asyncio.CancelledError: + with self.assertRaises(RuntimeError): + g.create_task(c1 := coro1()) + # We still have to await c1 to avoid a warning + with self.assertRaises(ZeroDivisionError): + await c1 + + with self.assertRaises(ExceptionGroup) as cm: + async with taskgroups.TaskGroup() as g: + g.create_task(coro1()) + g.create_task(coro2(g)) + + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-06-29-04-42-56.gh-issue-94398.YOq_bJ.rst b/Misc/NEWS.d/next/Library/2022-06-29-04-42-56.gh-issue-94398.YOq_bJ.rst new file mode 100644 index 0000000000000..c6e7e967d106e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-06-29-04-42-56.gh-issue-94398.YOq_bJ.rst @@ -0,0 +1 @@ +Once a :class:`asyncio.TaskGroup` has started shutting down (i.e., at least one task has failed and the task group has started cancelling the remaining tasks), it should not be possible to add new tasks to the task group.

1 0

gh-84753: Make inspect.iscoroutinefunction() work with AsyncMock (GH-94050) (GH-94461)
by ambv 30 Jun '22

30 Jun '22

https://github.com/python/cpython/commit/47f23b2d8a14a2f182375c840776eadfc3… commit: 47f23b2d8a14a2f182375c840776eadfc371c726 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: ambv <lukasz(a)langa.pl> date: 2022-06-30T20:02:15+02:00 summary: gh-84753: Make inspect.iscoroutinefunction() work with AsyncMock (GH-94050) (GH-94461) The inspect version was not working with unittest.mock.AsyncMock. The fix introduces special-casing of AsyncMock in `inspect.iscoroutinefunction` equivalent to the one performed in `asyncio.iscoroutinefunction`. Co-authored-by: Łukasz Langa <lukasz(a)langa.pl> (cherry picked from commit 4261b6bffc0b8bb5c6d4d80578a81b7520f4aefc) Co-authored-by: Mehdi ABAAKOUK <sileht(a)sileht.net> files: A Misc/NEWS.d/next/Library/2022-06-21-11-40-31.gh-issue-84753.FW1pxO.rst M Lib/inspect.py M Lib/test/test_asyncio/test_tasks.py M Lib/test/test_inspect.py M Lib/unittest/mock.py diff --git a/Lib/inspect.py b/Lib/inspect.py index c5881cc808d21..01df575e4bfee 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -294,7 +294,7 @@ def _has_code_flag(f, flag): while ismethod(f): f = f.__func__ f = functools._unwrap_partial(f) - if not isfunction(f): + if not (isfunction(f) or _signature_is_functionlike(f)): return False return bool(f.__code__.co_flags & flag) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 0007b44a220c4..05a822ba458b7 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1723,6 +1723,7 @@ def fn2(): self.assertTrue(asyncio.iscoroutinefunction(fn2)) self.assertFalse(asyncio.iscoroutinefunction(mock.Mock())) + self.assertTrue(asyncio.iscoroutinefunction(mock.AsyncMock())) def test_yield_vs_yield_from(self): fut = self.new_future(self.loop) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 28e4f5b4a718a..eaefe946e8963 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -188,6 +188,10 @@ def test_iscoroutine(self): gen_coroutine_function_example)))) self.assertTrue(inspect.isgenerator(gen_coro)) + self.assertFalse( + inspect.iscoroutinefunction(unittest.mock.Mock())) + self.assertTrue( + inspect.iscoroutinefunction(unittest.mock.AsyncMock())) self.assertTrue( inspect.iscoroutinefunction(coroutine_function_example)) self.assertTrue( @@ -196,6 +200,10 @@ def test_iscoroutine(self): coroutine_function_example)))) self.assertTrue(inspect.iscoroutine(coro)) + self.assertFalse( + inspect.isgeneratorfunction(unittest.mock.Mock())) + self.assertFalse( + inspect.isgeneratorfunction(unittest.mock.AsyncMock())) self.assertFalse( inspect.isgeneratorfunction(coroutine_function_example)) self.assertFalse( @@ -204,6 +212,12 @@ def test_iscoroutine(self): coroutine_function_example)))) self.assertFalse(inspect.isgenerator(coro)) + self.assertFalse( + inspect.isasyncgenfunction(unittest.mock.Mock())) + self.assertFalse( + inspect.isasyncgenfunction(unittest.mock.AsyncMock())) + self.assertFalse( + inspect.isasyncgenfunction(coroutine_function_example)) self.assertTrue( inspect.isasyncgenfunction(async_generator_function_example)) self.assertTrue( diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 7152f86ed9694..3f7185b9671e4 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -2186,6 +2186,10 @@ def __init__(self, /, *args, **kwargs): code_mock = NonCallableMock(spec_set=CodeType) code_mock.co_flags = inspect.CO_COROUTINE self.__dict__['__code__'] = code_mock + self.__dict__['__name__'] = 'AsyncMock' + self.__dict__['__defaults__'] = tuple() + self.__dict__['__kwdefaults__'] = {} + self.__dict__['__annotations__'] = None async def _execute_mock_call(self, /, *args, **kwargs): # This is nearly just like super(), except for special handling diff --git a/Misc/NEWS.d/next/Library/2022-06-21-11-40-31.gh-issue-84753.FW1pxO.rst b/Misc/NEWS.d/next/Library/2022-06-21-11-40-31.gh-issue-84753.FW1pxO.rst new file mode 100644 index 0000000000000..f701d2a1afeb8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-06-21-11-40-31.gh-issue-84753.FW1pxO.rst @@ -0,0 +1,3 @@ +:func:`inspect.iscoroutinefunction` now properly returns ``True`` when an instance +of :class:`unittest.mock.AsyncMock` is passed to it. This makes it consistent with +behavior of :func:`asyncio.iscoroutinefunction`. Patch by Mehdi ABAAKOUK.

1 0

multiprocessing.spawn doc: Capitalize the p in "Python" (gh-94462)
by Mariatta 30 Jun '22

30 Jun '22

https://github.com/python/cpython/commit/62bb7a3b50150495e215d7bd32f633eef8… commit: 62bb7a3b50150495e215d7bd32f633eef81b3bc2 branch: main author: Mariatta Wijaya <Mariatta(a)users.noreply.github.com> committer: Mariatta <Mariatta(a)users.noreply.github.com> date: 2022-06-30T10:23:04-07:00 summary: multiprocessing.spawn doc: Capitalize the p in "Python" (gh-94462) files: M Doc/library/multiprocessing.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 2a66b0f65c088..a8e35bc084747 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -96,7 +96,7 @@ Depending on the platform, :mod:`multiprocessing` supports three ways to start a process. These *start methods* are *spawn* - The parent process starts a fresh python interpreter process. The + The parent process starts a fresh Python interpreter process. The child process will only inherit those resources necessary to run the process object's :meth:`~Process.run` method. In particular, unnecessary file descriptors and handles from the parent process

1 0

xml.dom.minidom docs: fix typo (GH-93437)
by miss-islington 30 Jun '22

30 Jun '22

https://github.com/python/cpython/commit/d915ed297804041c822a595c0f44e52a4c… commit: d915ed297804041c822a595c0f44e52a4c385302 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-06-30T10:16:06-07:00 summary: xml.dom.minidom docs: fix typo (GH-93437) (cherry picked from commit 639e35108bc8b2b880225862d3571277ad57648b) Co-authored-by: Jean-Christophe Helary <jean.christophe.helary(a)traduction-libre.org> files: M Doc/library/xml.dom.minidom.rst diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 076cf34694ce4..82e5d6aea2310 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -95,7 +95,7 @@ document: the one that holds all others. Here is an example program:: When you are finished with a DOM tree, you may optionally call the :meth:`unlink` method to encourage early cleanup of the now-unneeded objects. :meth:`unlink` is an :mod:`xml.dom.minidom`\ -specific -extension to the DOM API that renders the node and its descendants are +extension to the DOM API that renders the node and its descendants essentially useless. Otherwise, Python's garbage collector will eventually take care of the objects in the tree.

1 0

xml.dom.minidom docs: fix typo (GH-93437)
by miss-islington 30 Jun '22

30 Jun '22

https://github.com/python/cpython/commit/aed28b7feb349cdeb45285d46ae7b36f20… commit: aed28b7feb349cdeb45285d46ae7b36f200c957e branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-06-30T10:15:00-07:00 summary: xml.dom.minidom docs: fix typo (GH-93437) (cherry picked from commit 639e35108bc8b2b880225862d3571277ad57648b) Co-authored-by: Jean-Christophe Helary <jean.christophe.helary(a)traduction-libre.org> files: M Doc/library/xml.dom.minidom.rst diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 076cf34694ce4..82e5d6aea2310 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -95,7 +95,7 @@ document: the one that holds all others. Here is an example program:: When you are finished with a DOM tree, you may optionally call the :meth:`unlink` method to encourage early cleanup of the now-unneeded objects. :meth:`unlink` is an :mod:`xml.dom.minidom`\ -specific -extension to the DOM API that renders the node and its descendants are +extension to the DOM API that renders the node and its descendants essentially useless. Otherwise, Python's garbage collector will eventually take care of the objects in the tree.

1 0

GH-94398: TaskGroup: Fail create_task() during shutdown (GH-94400)
by ambv 30 Jun '22

30 Jun '22

https://github.com/python/cpython/commit/594c3699492bfb007650538726d953cbed… commit: 594c3699492bfb007650538726d953cbed55de04 branch: main author: Guido van Rossum <guido(a)python.org> committer: ambv <lukasz(a)langa.pl> date: 2022-06-30T19:10:46+02:00 summary: GH-94398: TaskGroup: Fail create_task() during shutdown (GH-94400) Once the task group is shutting down, it should not be possible to create a new task. Here "shutting down" means `self._aborting` is set, indicating that at least one task has failed and we have cancelled all others. Co-authored-by: Łukasz Langa <lukasz(a)langa.pl> files: A Misc/NEWS.d/next/Library/2022-06-29-04-42-56.gh-issue-94398.YOq_bJ.rst M Doc/library/asyncio-task.rst M Lib/asyncio/taskgroups.py M Lib/test/test_asyncio/test_taskgroups.py diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 9b76548433b6a..76969762dd9f5 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -320,6 +320,7 @@ no new tasks may be added to the group. The first time any of the tasks belonging to the group fails with an exception other than :exc:`asyncio.CancelledError`, the remaining tasks in the group are cancelled. +No further tasks can then be added to the group. At this point, if the body of the ``async with`` statement is still active (i.e., :meth:`~object.__aexit__` hasn't been called yet), the task directly containing the ``async with`` statement is also cancelled. diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 9e0610deed281..3ca65062efd27 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -138,6 +138,8 @@ def create_task(self, coro, *, name=None, context=None): raise RuntimeError(f"TaskGroup {self!r} has not been entered") if self._exiting and not self._tasks: raise RuntimeError(f"TaskGroup {self!r} is finished") + if self._aborting: + raise RuntimeError(f"TaskGroup {self!r} is shutting down") if context is None: task = self._loop.create_task(coro) else: diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 69369a6100a8f..26fb5e466af92 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -122,10 +122,8 @@ async def runner(): self.assertTrue(t2.cancelled()) async def test_cancel_children_on_child_error(self): - """ - When a child task raises an error, the rest of the children - are cancelled and the errors are gathered into an EG. - """ + # When a child task raises an error, the rest of the children + # are cancelled and the errors are gathered into an EG. NUM = 0 t2_cancel = False @@ -722,6 +720,27 @@ async def coro(val): await t2 self.assertEqual(2, ctx.get(cvar)) + async def test_taskgroup_no_create_task_after_failure(self): + async def coro1(): + await asyncio.sleep(0.001) + 1 / 0 + async def coro2(g): + try: + await asyncio.sleep(1) + except asyncio.CancelledError: + with self.assertRaises(RuntimeError): + g.create_task(c1 := coro1()) + # We still have to await c1 to avoid a warning + with self.assertRaises(ZeroDivisionError): + await c1 + + with self.assertRaises(ExceptionGroup) as cm: + async with taskgroups.TaskGroup() as g: + g.create_task(coro1()) + g.create_task(coro2(g)) + + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-06-29-04-42-56.gh-issue-94398.YOq_bJ.rst b/Misc/NEWS.d/next/Library/2022-06-29-04-42-56.gh-issue-94398.YOq_bJ.rst new file mode 100644 index 0000000000000..c6e7e967d106e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-06-29-04-42-56.gh-issue-94398.YOq_bJ.rst @@ -0,0 +1 @@ +Once a :class:`asyncio.TaskGroup` has started shutting down (i.e., at least one task has failed and the task group has started cancelling the remaining tasks), it should not be possible to add new tasks to the task group.

1 0