Mailman 3 December 2021 - Python-checkins

[3.10] bpo-46120: State that `|` is preferred over `Union` (GH-30222) (GH-30250)
by miss-islington Dec. 29, 2021

Dec. 29, 2021

https://github.com/python/cpython/commit/ed1671ced7c9b951dfc16a0cf32a2b4eab… commit: ed1671ced7c9b951dfc16a0cf32a2b4eab914cf1 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2021-12-29T03:45:14-08:00 summary: [3.10] bpo-46120: State that `|` is preferred over `Union` (GH-30222) (GH-30250) Co-authored-by: Éric <merwok(a)netwok.org> Co-authored-by: Guido van Rossum <gvanrossum(a)gmail.com> (cherry picked from commit 1b30660c3b40da557050948ac8635f2dc50c4ee2) Co-authored-by: Nikita Sobolev <mail(a)sobolevn.me> Automerge-Triggered-By: GH:gpshead files: A Misc/NEWS.d/next/Documentation/2021-12-21-12-45-57.bpo-46120.PE0DmJ.rst M Doc/library/typing.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 3adb495ff7933..29bdb80ad22c6 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -625,7 +625,7 @@ These can be used as types in annotations using ``[]``, each having a unique syn Union type; ``Union[X, Y]`` is equivalent to ``X | Y`` and means either X or Y. - To define a union, use e.g. ``Union[int, str]`` or the shorthand ``int | str``. Details: + To define a union, use e.g. ``Union[int, str]`` or the shorthand ``int | str``. Using that shorthand is recommended. Details: * The arguments must be types and there must be at least one. diff --git a/Misc/NEWS.d/next/Documentation/2021-12-21-12-45-57.bpo-46120.PE0DmJ.rst b/Misc/NEWS.d/next/Documentation/2021-12-21-12-45-57.bpo-46120.PE0DmJ.rst new file mode 100644 index 0000000000000..17f67472e2ab0 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2021-12-21-12-45-57.bpo-46120.PE0DmJ.rst @@ -0,0 +1 @@ +State that ``|`` is preferred for readability over ``Union`` in the :mod:`typing` docs.

1 0

bpo-46090: Allow PyThreadState.datastack_* members to be NULL (GH-30234)
by brandtbucher Dec. 28, 2021

Dec. 28, 2021

https://github.com/python/cpython/commit/77195cd44b2506cda88a3cfc9891852606… commit: 77195cd44b2506cda88a3cfc98918526068b1d46 branch: main author: Brandt Bucher <brandt(a)python.org> committer: brandtbucher <brandtbucher(a)gmail.com> date: 2021-12-28T09:49:48-08:00 summary: bpo-46090: Allow PyThreadState.datastack_* members to be NULL (GH-30234) files: M Include/internal/pycore_frame.h M Python/pystate.c diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 883bef199ba59..8eca39d1ab250 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -174,10 +174,13 @@ static inline InterpreterFrame * _PyThreadState_BumpFramePointer(PyThreadState *tstate, size_t size) { PyObject **base = tstate->datastack_top; - PyObject **top = base + size; - if (top < tstate->datastack_limit) { - tstate->datastack_top = top; - return (InterpreterFrame *)base; + if (base) { + PyObject **top = base + size; + assert(tstate->datastack_limit); + if (top < tstate->datastack_limit) { + tstate->datastack_top = top; + return (InterpreterFrame *)base; + } } return _PyThreadState_BumpFramePointerSlow(tstate, size); } diff --git a/Python/pystate.c b/Python/pystate.c index 0301ce676950b..68fae8d283091 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -749,8 +749,7 @@ free_threadstate(PyThreadState *tstate) static void init_threadstate(PyThreadState *tstate, PyInterpreterState *interp, uint64_t id, - PyThreadState *next, - _PyStackChunk *datastack_chunk) + PyThreadState *next) { if (tstate->_initialized) { Py_FatalError("thread state already initialized"); @@ -784,11 +783,9 @@ init_threadstate(PyThreadState *tstate, tstate->exc_info = &tstate->exc_state; tstate->cframe = &tstate->root_cframe; - assert(datastack_chunk != NULL); - tstate->datastack_chunk = datastack_chunk; - /* If top points to entry 0, then _PyThreadState_PopFrame will try to pop this chunk */ - tstate->datastack_top = &tstate->datastack_chunk->data[1]; - tstate->datastack_limit = (PyObject **)(((char *)tstate->datastack_chunk) + DATA_STACK_CHUNK_SIZE); + tstate->datastack_chunk = NULL; + tstate->datastack_top = NULL; + tstate->datastack_limit = NULL; tstate->_initialized = 1; } @@ -799,11 +796,6 @@ new_threadstate(PyInterpreterState *interp) PyThreadState *tstate; _PyRuntimeState *runtime = interp->runtime; - _PyStackChunk *datastack_chunk = allocate_chunk(DATA_STACK_CHUNK_SIZE, NULL); - if (datastack_chunk == NULL) { - return NULL; - } - /* We serialize concurrent creation to protect global state. */ HEAD_LOCK(runtime); @@ -833,14 +825,13 @@ new_threadstate(PyInterpreterState *interp) } interp->threads.head = tstate; - init_threadstate(tstate, interp, id, old_head, datastack_chunk); + init_threadstate(tstate, interp, id, old_head); HEAD_UNLOCK(runtime); return tstate; error: HEAD_UNLOCK(runtime); - _PyObject_VirtualFree(datastack_chunk, datastack_chunk->size); return NULL; } @@ -2186,8 +2177,6 @@ _Py_GetConfig(void) static PyObject ** push_chunk(PyThreadState *tstate, int size) { - assert(tstate->datastack_top + size >= tstate->datastack_limit); - int allocate_size = DATA_STACK_CHUNK_SIZE; while (allocate_size < (int)sizeof(PyObject*)*(size + MINIMUM_OVERHEAD)) { allocate_size *= 2; @@ -2196,10 +2185,16 @@ push_chunk(PyThreadState *tstate, int size) if (new == NULL) { return NULL; } - tstate->datastack_chunk->top = tstate->datastack_top - &tstate->datastack_chunk->data[0]; + if (tstate->datastack_chunk) { + tstate->datastack_chunk->top = tstate->datastack_top - + &tstate->datastack_chunk->data[0]; + } tstate->datastack_chunk = new; tstate->datastack_limit = (PyObject **)(((char *)new) + allocate_size); - PyObject **res = &new->data[0]; + // When new is the "root" chunk (i.e. new->previous == NULL), we can keep + // _PyThreadState_PopFrame from freeing it later by "skipping" over the + // first element: + PyObject **res = &new->data[new->previous == NULL]; tstate->datastack_top = res + size; return res; } @@ -2212,9 +2207,6 @@ _PyThreadState_BumpFramePointerSlow(PyThreadState *tstate, size_t size) PyObject **top = base + size; if (top >= tstate->datastack_limit) { base = push_chunk(tstate, (int)size); - if (base == NULL) { - return NULL; - } } else { tstate->datastack_top = top; @@ -2244,16 +2236,20 @@ _PyThreadState_PushFrame(PyThreadState *tstate, PyFunctionObject *func, PyObject void _PyThreadState_PopFrame(PyThreadState *tstate, InterpreterFrame * frame) { + assert(tstate->datastack_chunk); PyObject **base = (PyObject **)frame; if (base == &tstate->datastack_chunk->data[0]) { _PyStackChunk *chunk = tstate->datastack_chunk; _PyStackChunk *previous = chunk->previous; + // push_chunk ensures that the root chunk is never popped: + assert(previous); tstate->datastack_top = &previous->data[previous->top]; tstate->datastack_chunk = previous; _PyObject_VirtualFree(chunk, chunk->size); tstate->datastack_limit = (PyObject **)(((char *)previous) + previous->size); } else { + assert(tstate->datastack_top); assert(tstate->datastack_top >= base); tstate->datastack_top = base; }

1 0

bpo-37295: Speed up math.comb(n, k) for 0 <= k <= n <= 67 (GH-30275)
by mdickinson Dec. 28, 2021

Dec. 28, 2021

https://github.com/python/cpython/commit/02b5417f1107415abaf81acab7522f9aa8… commit: 02b5417f1107415abaf81acab7522f9aa84269ea branch: main author: Mark Dickinson <mdickinson(a)enthought.com> committer: mdickinson <dickinsm(a)gmail.com> date: 2021-12-28T12:26:40Z summary: bpo-37295: Speed up math.comb(n, k) for 0 <= k <= n <= 67 (GH-30275) files: A Misc/NEWS.d/next/Library/2021-12-27-15-52-28.bpo-37295.s3LPo0.rst M Modules/mathmodule.c diff --git a/Misc/NEWS.d/next/Library/2021-12-27-15-52-28.bpo-37295.s3LPo0.rst b/Misc/NEWS.d/next/Library/2021-12-27-15-52-28.bpo-37295.s3LPo0.rst new file mode 100644 index 0000000000000..a624f10637002 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-12-27-15-52-28.bpo-37295.s3LPo0.rst @@ -0,0 +1 @@ +Add fast path for ``0 <= k <= n <= 67`` for :func:`math.comb`. diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 011ce0afd3aec..c4a23b6514f6b 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -3450,6 +3450,71 @@ math_perm_impl(PyObject *module, PyObject *n, PyObject *k) return NULL; } +/* least significant 64 bits of the odd part of factorial(n), for n in range(68). + +Python code to generate the values: + + import math + + for n in range(68): + fac = math.factorial(n) + fac_odd_part = fac // (fac & -fac) + reduced_fac_odd_part = fac_odd_part % (2**64) + print(f"{reduced_fac_odd_part:#018x}u") +*/ +static uint64_t reduced_factorial_odd_part[] = { + 0x0000000000000001u, 0x0000000000000001u, 0x0000000000000001u, 0x0000000000000003u, + 0x0000000000000003u, 0x000000000000000fu, 0x000000000000002du, 0x000000000000013bu, + 0x000000000000013bu, 0x0000000000000b13u, 0x000000000000375fu, 0x0000000000026115u, + 0x000000000007233fu, 0x00000000005cca33u, 0x0000000002898765u, 0x00000000260eeeebu, + 0x00000000260eeeebu, 0x0000000286fddd9bu, 0x00000016beecca73u, 0x000001b02b930689u, + 0x00000870d9df20adu, 0x0000b141df4dae31u, 0x00079dd498567c1bu, 0x00af2e19afc5266du, + 0x020d8a4d0f4f7347u, 0x335281867ec241efu, 0x9b3093d46fdd5923u, 0x5e1f9767cc5866b1u, + 0x92dd23d6966aced7u, 0xa30d0f4f0a196e5bu, 0x8dc3e5a1977d7755u, 0x2ab8ce915831734bu, + 0x2ab8ce915831734bu, 0x81d2a0bc5e5fdcabu, 0x9efcac82445da75bu, 0xbc8b95cf58cde171u, + 0xa0e8444a1f3cecf9u, 0x4191deb683ce3ffdu, 0xddd3878bc84ebfc7u, 0xcb39a64b83ff3751u, + 0xf8203f7993fc1495u, 0xbd2a2a78b35f4bddu, 0x84757be6b6d13921u, 0x3fbbcfc0b524988bu, + 0xbd11ed47c8928df9u, 0x3c26b59e41c2f4c5u, 0x677a5137e883fdb3u, 0xff74e943b03b93ddu, + 0xfe5ebbcb10b2bb97u, 0xb021f1de3235e7e7u, 0x33509eb2e743a58fu, 0x390f9da41279fb7du, + 0xe5cb0154f031c559u, 0x93074695ba4ddb6du, 0x81c471caa636247fu, 0xe1347289b5a1d749u, + 0x286f21c3f76ce2ffu, 0x00be84a2173e8ac7u, 0x1595065ca215b88bu, 0xf95877595b018809u, + 0x9c2efe3c5516f887u, 0x373294604679382bu, 0xaf1ff7a888adcd35u, 0x18ddf279a2c5800bu, + 0x18ddf279a2c5800bu, 0x505a90e2542582cbu, 0x5bacad2cd8d5dc2bu, 0xfe3152bcbff89f41u, +}; + +/* inverses of reduced_factorial_odd_part values modulo 2**64. + +Python code to generate the values: + + import math + + for n in range(68): + fac = math.factorial(n) + fac_odd_part = fac // (fac & -fac) + inverted_fac_odd_part = pow(fac_odd_part, -1, 2**64) + print(f"{inverted_fac_odd_part:#018x}u") +*/ +static uint64_t inverted_factorial_odd_part[] = { + 0x0000000000000001u, 0x0000000000000001u, 0x0000000000000001u, 0xaaaaaaaaaaaaaaabu, + 0xaaaaaaaaaaaaaaabu, 0xeeeeeeeeeeeeeeefu, 0x4fa4fa4fa4fa4fa5u, 0x2ff2ff2ff2ff2ff3u, + 0x2ff2ff2ff2ff2ff3u, 0x938cc70553e3771bu, 0xb71c27cddd93e49fu, 0xb38e3229fcdee63du, + 0xe684bb63544a4cbfu, 0xc2f684917ca340fbu, 0xf747c9cba417526du, 0xbb26eb51d7bd49c3u, + 0xbb26eb51d7bd49c3u, 0xb0a7efb985294093u, 0xbe4b8c69f259eabbu, 0x6854d17ed6dc4fb9u, + 0xe1aa904c915f4325u, 0x3b8206df131cead1u, 0x79c6009fea76fe13u, 0xd8c5d381633cd365u, + 0x4841f12b21144677u, 0x4a91ff68200b0d0fu, 0x8f9513a58c4f9e8bu, 0x2b3e690621a42251u, + 0x4f520f00e03c04e7u, 0x2edf84ee600211d3u, 0xadcaa2764aaacdfdu, 0x161f4f9033f4fe63u, + 0x161f4f9033f4fe63u, 0xbada2932ea4d3e03u, 0xcec189f3efaa30d3u, 0xf7475bb68330bf91u, + 0x37eb7bf7d5b01549u, 0x46b35660a4e91555u, 0xa567c12d81f151f7u, 0x4c724007bb2071b1u, + 0x0f4a0cce58a016bdu, 0xfa21068e66106475u, 0x244ab72b5a318ae1u, 0x366ce67e080d0f23u, + 0xd666fdae5dd2a449u, 0xd740ddd0acc06a0du, 0xb050bbbb28e6f97bu, 0x70b003fe890a5c75u, + 0xd03aabff83037427u, 0x13ec4ca72c783bd7u, 0x90282c06afdbd96fu, 0x4414ddb9db4a95d5u, + 0xa2c68735ae6832e9u, 0xbf72d71455676665u, 0xa8469fab6b759b7fu, 0xc1e55b56e606caf9u, + 0x40455630fc4a1cffu, 0x0120a7b0046d16f7u, 0xa7c3553b08faef23u, 0x9f0bfd1b08d48639u, + 0xa433ffce9a304d37u, 0xa22ad1d53915c683u, 0xcb6cbc723ba5dd1du, 0x547fb1b8ab9d0ba3u, + 0x547fb1b8ab9d0ba3u, 0x8f15a826498852e3u, 0x32e1a03f38880283u, 0x3de4cce63283f0c1u, +}; + + /*[clinic input] math.comb @@ -3512,6 +3577,30 @@ math_comb_impl(PyObject *module, PyObject *n, PyObject *k) goto done; } assert(ki >= 0); + + if (ni <= 67) { + /* + For 0 <= k <= n <= 67, comb(n, k) always fits into a uint64_t. + We compute it as + + comb_odd_part << shift + + where 2**shift is the largest power of two dividing comb(n, k) + and comb_odd_part is comb(n, k) >> shift. comb_odd_part can be + calculated efficiently via arithmetic modulo 2**64, using three + lookups and two uint64_t multiplications, while the necessary + shift can be computed via Kummer's theorem: it's the number of + carries when adding k to n - k in binary, which in turn is the + number of set bits of n ^ k ^ (n - k). + */ + uint64_t comb_odd_part = reduced_factorial_odd_part[ni] + * inverted_factorial_odd_part[ki] + * inverted_factorial_odd_part[ni - ki]; + int shift = _Py_popcount32((uint32_t)(ni ^ ki ^ (ni - ki))); + result = PyLong_FromUnsignedLongLong(comb_odd_part << shift); + goto done; + } + ki = Py_MIN(ki, ni - ki); if (ki > 1) { result = perm_comb_small((unsigned long long)ni,

1 0

Update REAME and Docs sidebar for EOL
by ned-deily Dec. 28, 2021

Dec. 28, 2021

https://github.com/python/cpython/commit/8d999cbf4adea053be6dbb612b9844635c… commit: 8d999cbf4adea053be6dbb612b9844635c4dfb8e branch: 3.6 author: Ned Deily <nad(a)python.org> committer: ned-deily <nad(a)python.org> date: 2021-12-28T02:10:16-05:00 summary: Update REAME and Docs sidebar for EOL files: M Doc/tools/static/switchers.js M Doc/tools/templates/indexsidebar.html M README.rst diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js index 30801917ed9eb..3008e1631e793 100644 --- a/Doc/tools/static/switchers.js +++ b/Doc/tools/static/switchers.js @@ -10,8 +10,9 @@ '(?:release/\\d.\\d[\\x\\d\\.]*)']; var all_versions = { - '3.10': 'dev (3.10)', - '3.9': 'pre (3.9)', + '3.11': 'dev (3.11)', + '3.10': '3.10', + '3.9': '3.9', '3.8': '3.8', '3.7': '3.7', '3.6': '3.6', diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 2ef90f094ddaa..f7bf6d8e49117 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -2,12 +2,8 @@ <h3>{% trans %}Download{% endtrans %}</h3> <p><a href="{{ pathto('download') }}">{% trans %}Download these documents{% endtrans %}</a></p> <h3>{% trans %}Docs by version{% endtrans %}</h3> <ul> - <li><a href="https://docs.python.org/3.10/">{% trans %}Python 3.10 (in development){% endtrans %}</a></li> - <li><a href="https://docs.python.org/3.9/">{% trans %}Python 3.9 (pre-release){% endtrans %}</a></li> - <li><a href="https://docs.python.org/3.8/">{% trans %}Python 3.8 (stable){% endtrans %}</a></li> - <li><a href="https://docs.python.org/3.7/">{% trans %}Python 3.7 (stable){% endtrans %}</a></li> - <li><a href="https://docs.python.org/3.6/">{% trans %}Python 3.6 (security-fixes){% endtrans %}</a></li> - <li><a href="https://docs.python.org/2.7/">{% trans %}Python 2.7 (EOL){% endtrans %}</a></li> + <li><a href="https://docs.python.org/">{% trans %}Stable{% endtrans %}</a></li> + <li><a href="https://docs.python.org/dev/">{% trans %}In development{% endtrans %}</a></li> <li><a href="https://www.python.org/doc/versions/">{% trans %}All versions{% endtrans %}</a></li> </ul> @@ -18,4 +14,5 @@ <h3>{% trans %}Other resources{% endtrans %}</h3> <li><a href="https://wiki.python.org/moin/BeginnersGuide">{% trans %}Beginner's Guide{% endtrans %}</a></li> <li><a href="https://wiki.python.org/moin/PythonBooks">{% trans %}Book List{% endtrans %}</a></li> <li><a href="https://www.python.org/doc/av/">{% trans %}Audio/Visual Talks{% endtrans %}</a></li> + <li><a href="https://devguide.python.org/">{% trans %}Python Developer’s Guide{% endtrans %}</a></li> </ul> diff --git a/README.rst b/README.rst index 3e72bad19a34d..570b6f1669db9 100644 --- a/README.rst +++ b/README.rst @@ -236,6 +236,7 @@ All current PEPs, as well as guidelines for submitting a new PEP, are listed at Release Schedule ---------------- +As of 2021-12-23, Python 3.6 has reached ``end-of-life`` status. See :pep:`494` for Python 3.6 release details.

1 0

bpo-45189: Drop the "list_frozen" command from _test_embed. (GH-30273)
by corona10 Dec. 28, 2021

Dec. 28, 2021

https://github.com/python/cpython/commit/196b53eb1e62871ca80dee180e4891b4dd… commit: 196b53eb1e62871ca80dee180e4891b4dd5c52ac branch: main author: Dong-hee Na <donghee.na(a)python.org> committer: corona10 <donghee.na92(a)gmail.com> date: 2021-12-28T11:05:50+09:00 summary: bpo-45189: Drop the "list_frozen" command from _test_embed. (GH-30273) files: M Programs/_testembed.c M Tools/scripts/generate_stdlib_module_names.py diff --git a/Programs/_testembed.c b/Programs/_testembed.c index fc5f44d5eb626..b31781938eb39 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1827,26 +1827,6 @@ static int test_frozenmain(void) } #endif // !MS_WINDOWS - -// List frozen modules. -// Command used by Tools/scripts/generate_stdlib_module_names.py script. -static int list_frozen(void) -{ - const struct _frozen *p; - for (p = _PyImport_FrozenBootstrap; ; p++) { - if (p->name == NULL) - break; - printf("%s\n", p->name); - } - for (p = _PyImport_FrozenStdlib; ; p++) { - if (p->name == NULL) - break; - printf("%s\n", p->name); - } - return 0; -} - - static int test_repeated_init_and_inittab(void) { // bpo-44441: Py_RunMain() must reset PyImport_Inittab at exit. @@ -1960,8 +1940,6 @@ static struct TestCase TestCases[] = { {"test_frozenmain", test_frozenmain}, #endif - // Command - {"list_frozen", list_frozen}, {NULL, NULL} }; diff --git a/Tools/scripts/generate_stdlib_module_names.py b/Tools/scripts/generate_stdlib_module_names.py index 3e896ba737543..fe1e429ebce17 100644 --- a/Tools/scripts/generate_stdlib_module_names.py +++ b/Tools/scripts/generate_stdlib_module_names.py @@ -1,5 +1,6 @@ # This script lists the names of standard library modules # to update Python/stdlib_mod_names.h +import _imp import os.path import re import subprocess @@ -11,7 +12,6 @@ STDLIB_PATH = os.path.join(SRC_DIR, 'Lib') MODULES_SETUP = os.path.join(SRC_DIR, 'Modules', 'Setup') SETUP_PY = os.path.join(SRC_DIR, 'setup.py') -TEST_EMBED = os.path.join(SRC_DIR, 'Programs', '_testembed') IGNORE = { '__init__', @@ -117,16 +117,11 @@ def list_modules_setup_extensions(names): # List frozen modules of the PyImport_FrozenModules list (Python/frozen.c). # Use the "./Programs/_testembed list_frozen" command. def list_frozen(names): - args = [TEST_EMBED, 'list_frozen'] - proc = subprocess.run(args, stdout=subprocess.PIPE, text=True) - exitcode = proc.returncode - if exitcode: - cmd = ' '.join(args) - print(f"{cmd} failed with exitcode {exitcode}") - sys.exit(exitcode) submodules = set() - for line in proc.stdout.splitlines(): - name = line.strip() + for name in _imp._frozen_module_names(): + # To skip __hello__, __hello_alias__ and etc. + if name.startswith('__'): + continue if '.' in name: submodules.add(name) else:

1 0

bpo-46055: Speed up binary shifting operators (GH-30044)
by mdickinson Dec. 27, 2021

Dec. 27, 2021

https://github.com/python/cpython/commit/3581c7abbe15bad6ae08fc38887e5948f8… commit: 3581c7abbe15bad6ae08fc38887e5948f8f39e08 branch: main author: Xinhang Xu <xuxinhang4567(a)126.com> committer: mdickinson <dickinsm(a)gmail.com> date: 2021-12-27T18:36:55Z summary: bpo-46055: Speed up binary shifting operators (GH-30044) Co-authored-by: Mark Dickinson <dickinsm(a)gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2021-12-24-20-21-45.bpo-46055.R0QMVQ.rst M Lib/test/test_long.py M Misc/ACKS M Objects/longobject.c diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index e5c4d7fc4220f..3c8e9e22e17a1 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -946,8 +946,13 @@ def test_huge_lshift(self, size): self.assertEqual(1 << (sys.maxsize + 1000), 1 << 1000 << sys.maxsize) def test_huge_rshift(self): - self.assertEqual(42 >> (1 << 1000), 0) - self.assertEqual((-42) >> (1 << 1000), -1) + huge_shift = 1 << 1000 + self.assertEqual(42 >> huge_shift, 0) + self.assertEqual((-42) >> huge_shift, -1) + self.assertEqual(1123 >> huge_shift, 0) + self.assertEqual((-1123) >> huge_shift, -1) + self.assertEqual(2**128 >> huge_shift, 0) + self.assertEqual(-2**128 >> huge_shift, -1) @support.cpython_only @support.bigmemtest(sys.maxsize + 500, memuse=2/15, dry_run=False) @@ -956,6 +961,60 @@ def test_huge_rshift_of_huge(self, size): self.assertEqual(huge >> (sys.maxsize + 1), (1 << 499) + 5) self.assertEqual(huge >> (sys.maxsize + 1000), 0) + def test_small_rshift(self): + self.assertEqual(42 >> 1, 21) + self.assertEqual((-42) >> 1, -21) + self.assertEqual(43 >> 1, 21) + self.assertEqual((-43) >> 1, -22) + + self.assertEqual(1122 >> 1, 561) + self.assertEqual((-1122) >> 1, -561) + self.assertEqual(1123 >> 1, 561) + self.assertEqual((-1123) >> 1, -562) + + self.assertEqual(2**128 >> 1, 2**127) + self.assertEqual(-2**128 >> 1, -2**127) + self.assertEqual((2**128 + 1) >> 1, 2**127) + self.assertEqual(-(2**128 + 1) >> 1, -2**127 - 1) + + def test_medium_rshift(self): + self.assertEqual(42 >> 9, 0) + self.assertEqual((-42) >> 9, -1) + self.assertEqual(1122 >> 9, 2) + self.assertEqual((-1122) >> 9, -3) + self.assertEqual(2**128 >> 9, 2**119) + self.assertEqual(-2**128 >> 9, -2**119) + + def test_big_rshift(self): + self.assertEqual(42 >> 32, 0) + self.assertEqual((-42) >> 32, -1) + self.assertEqual(1122 >> 32, 0) + self.assertEqual((-1122) >> 32, -1) + self.assertEqual(2**128 >> 32, 2**96) + self.assertEqual(-2**128 >> 32, -2**96) + + def test_small_lshift(self): + self.assertEqual(42 << 1, 84) + self.assertEqual((-42) << 1, -84) + self.assertEqual(561 << 1, 1122) + self.assertEqual((-561) << 1, -1122) + self.assertEqual(2**127 << 1, 2**128) + self.assertEqual(-2**127 << 1, -2**128) + + def test_medium_lshift(self): + self.assertEqual(42 << 9, 21504) + self.assertEqual((-42) << 9, -21504) + self.assertEqual(1122 << 9, 574464) + self.assertEqual((-1122) << 9, -574464) + + def test_big_lshift(self): + self.assertEqual(42 << 32, 42 * 2**32) + self.assertEqual((-42) << 32, -42 * 2**32) + self.assertEqual(1122 << 32, 1122 * 2**32) + self.assertEqual((-1122) << 32, -1122 * 2**32) + self.assertEqual(2**128 << 32, 2**160) + self.assertEqual(-2**128 << 32, -2**160) + @support.cpython_only def test_small_ints_in_huge_calculation(self): a = 2 ** 100 diff --git a/Misc/ACKS b/Misc/ACKS index 94a82a0750622..8baaf7304b603 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1964,6 +1964,7 @@ Doug Wyatt Xiang Zhang Robert Xiao Florent Xicluna +Xinhang Xu Arnon Yaari Alakshendra Yadav Hirokazu Yamamoto diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-12-24-20-21-45.bpo-46055.R0QMVQ.rst b/Misc/NEWS.d/next/Core and Builtins/2021-12-24-20-21-45.bpo-46055.R0QMVQ.rst new file mode 100644 index 0000000000000..124138806f17d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-12-24-20-21-45.bpo-46055.R0QMVQ.rst @@ -0,0 +1,2 @@ +Speed up shifting operation involving integers less than +:c:macro:`PyLong_BASE`. Patch by Xinhang Xu. diff --git a/Objects/longobject.c b/Objects/longobject.c index ddb3def402cdd..09ae9455c5b26 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -4493,6 +4493,15 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) Py_ssize_t newsize, hishift, i, j; twodigits accum; + if (IS_MEDIUM_VALUE(a)) { + stwodigits m, x; + digit shift; + m = medium_value(a); + shift = wordshift == 0 ? remshift : PyLong_SHIFT; + x = m < 0 ? ~(~m >> shift) : m >> shift; + return _PyLong_FromSTwoDigits(x); + } + if (Py_SIZE(a) < 0) { /* Right shifting negative numbers is harder */ PyLongObject *a1, *a2; @@ -4566,11 +4575,17 @@ _PyLong_Rshift(PyObject *a, size_t shiftby) static PyObject * long_lshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) { - /* This version due to Tim Peters */ PyLongObject *z = NULL; Py_ssize_t oldsize, newsize, i, j; twodigits accum; + if (wordshift == 0 && IS_MEDIUM_VALUE(a)) { + stwodigits m = medium_value(a); + // bypass undefined shift operator behavior + stwodigits x = m < 0 ? -(-m << remshift) : m << remshift; + return _PyLong_FromSTwoDigits(x); + } + oldsize = Py_ABS(Py_SIZE(a)); newsize = oldsize + wordshift; if (remshift)

1 0

bpo-46055: Streamline inner loop for right shifts (#30243)
by mdickinson Dec. 27, 2021

Dec. 27, 2021

https://github.com/python/cpython/commit/360fedc2d2ce6ccb0dab554ef45fe83f7a… commit: 360fedc2d2ce6ccb0dab554ef45fe83f7aea1862 branch: main author: Mark Dickinson <mdickinson(a)enthought.com> committer: mdickinson <dickinsm(a)gmail.com> date: 2021-12-27T18:04:36Z summary: bpo-46055: Streamline inner loop for right shifts (#30243) files: M Objects/longobject.c diff --git a/Objects/longobject.c b/Objects/longobject.c index 77434e67642f5..ddb3def402cdd 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -4491,7 +4491,7 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) { PyLongObject *z = NULL; Py_ssize_t newsize, hishift, i, j; - digit lomask, himask; + twodigits accum; if (Py_SIZE(a) < 0) { /* Right shifting negative numbers is harder */ @@ -4511,16 +4511,17 @@ long_rshift1(PyLongObject *a, Py_ssize_t wordshift, digit remshift) if (newsize <= 0) return PyLong_FromLong(0); hishift = PyLong_SHIFT - remshift; - lomask = ((digit)1 << hishift) - 1; - himask = PyLong_MASK ^ lomask; z = _PyLong_New(newsize); if (z == NULL) return NULL; - for (i = 0, j = wordshift; i < newsize; i++, j++) { - z->ob_digit[i] = (a->ob_digit[j] >> remshift) & lomask; - if (i+1 < newsize) - z->ob_digit[i] |= (a->ob_digit[j+1] << hishift) & himask; + j = wordshift; + accum = a->ob_digit[j++] >> remshift; + for (i = 0; j < Py_SIZE(a); i++, j++) { + accum |= (twodigits)a->ob_digit[j] << hishift; + z->ob_digit[i] = (digit)(accum & PyLong_MASK); + accum >>= PyLong_SHIFT; } + z->ob_digit[i] = (digit)accum; z = maybe_small_long(long_normalize(z)); } return (PyObject *)z;

1 0

bpo-42918: Improve built-in function compile() in mode 'single' (GH-29934) (GH-30040)
by ambv Dec. 27, 2021

Dec. 27, 2021

https://github.com/python/cpython/commit/576e38f9db61ca09ca6dc57ad13b02a7bf… commit: 576e38f9db61ca09ca6dc57ad13b02a7bf9d370a branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: ambv <lukasz(a)langa.pl> date: 2021-12-27T17:15:44+01:00 summary: bpo-42918: Improve built-in function compile() in mode 'single' (GH-29934) (GH-30040) Co-authored-by: Alex Waygood <Alex.Waygood(a)Gmail.com> (cherry picked from commit 28179aac796ed1debdce336c4b8ca18e8475d40d) Co-authored-by: Weipeng Hong <hongweichen8888(a)sina.com> files: A Misc/NEWS.d/next/Core and Builtins/2021-12-06-15-32-12.bpo-42918.Czpgtg.rst M Lib/test/test_compile.py M Parser/pegen.c diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 4de54488d84b6..5f80a583450ea 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -502,6 +502,7 @@ def test_single_statement(self): self.compile_single("if x:\n f(x)") self.compile_single("if x:\n f(x)\nelse:\n g(x)") self.compile_single("class T:\n pass") + self.compile_single("c = '''\na=1\nb=2\nc=3\n'''") def test_bad_single_statement(self): self.assertInvalidSingle('1\n2') @@ -512,6 +513,7 @@ def test_bad_single_statement(self): self.assertInvalidSingle('f()\n# blah\nblah()') self.assertInvalidSingle('f()\nxy # blah\nblah()') self.assertInvalidSingle('x = 5 # comment\nx = 6\n') + self.assertInvalidSingle("c = '''\nd=1\n'''\na = 1\n\nb = 2\n") def test_particularly_evil_undecodable(self): # Issue 24022 diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-12-06-15-32-12.bpo-42918.Czpgtg.rst b/Misc/NEWS.d/next/Core and Builtins/2021-12-06-15-32-12.bpo-42918.Czpgtg.rst new file mode 100644 index 0000000000000..f03dadebcf3b3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-12-06-15-32-12.bpo-42918.Czpgtg.rst @@ -0,0 +1,3 @@ +Fix bug where the built-in :func:`compile` function did not always raise a +:exc:`SyntaxError` when passed multiple statements in 'single' mode. Patch by +Weipeng Hong. diff --git a/Parser/pegen.c b/Parser/pegen.c index bae073cb5b347..0504906c947d0 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1131,31 +1131,13 @@ _PyPegen_number_token(Parser *p) t->end_col_offset, p->arena); } -static int // bool -newline_in_string(Parser *p, const char *cur) -{ - for (const char *c = cur; c >= p->tok->buf; c--) { - if (*c == '\'' || *c == '"') { - return 1; - } - } - return 0; -} - /* Check that the source for a single input statement really is a single statement by looking at what is left in the buffer after parsing. Trailing whitespace and comments are OK. */ static int // bool bad_single_statement(Parser *p) { - const char *cur = strchr(p->tok->buf, '\n'); - - /* Newlines are allowed if preceded by a line continuation character - or if they appear inside a string. */ - if (!cur || (cur != p->tok->buf && *(cur - 1) == '\\') - || newline_in_string(p, cur)) { - return 0; - } + char *cur = p->tok->cur; char c = *cur; for (;;) {

1 0

bpo-45496: Allow flexibility in winfo_rgb tests (GH-30185)
by miss-islington Dec. 26, 2021

Dec. 26, 2021

https://github.com/python/cpython/commit/aa056ed472e9d0a79ea21784f6f5171d12… commit: aa056ed472e9d0a79ea21784f6f5171d12a13f85 branch: 3.9 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2021-12-26T04:27:29-08:00 summary: bpo-45496: Allow flexibility in winfo_rgb tests (GH-30185) (cherry picked from commit 2e3e0d23adca8d83722d939d6abd1e467d7578f7) Co-authored-by: E-Paine <63801254+E-Paine(a)users.noreply.github.com> files: M Lib/tkinter/test/test_tkinter/test_misc.py diff --git a/Lib/tkinter/test/test_tkinter/test_misc.py b/Lib/tkinter/test/test_tkinter/test_misc.py index 554ae74d6b415..43ffd46bf0ba0 100644 --- a/Lib/tkinter/test/test_tkinter/test_misc.py +++ b/Lib/tkinter/test/test_tkinter/test_misc.py @@ -193,6 +193,13 @@ def test_clipboard_astral(self): root.clipboard_get() def test_winfo_rgb(self): + + def assertApprox(col1, col2): + # A small amount of flexibility is required (bpo-45496) + # 33 is ~0.05% of 65535, which is a reasonable margin + for col1_channel, col2_channel in zip(col1, col2): + self.assertAlmostEqual(col1_channel, col2_channel, delta=33) + root = self.root rgb = root.winfo_rgb @@ -202,9 +209,9 @@ def test_winfo_rgb(self): # #RGB - extends each 4-bit hex value to be 16-bit. self.assertEqual(rgb('#F0F'), (0xFFFF, 0x0000, 0xFFFF)) # #RRGGBB - extends each 8-bit hex value to be 16-bit. - self.assertEqual(rgb('#4a3c8c'), (0x4a4a, 0x3c3c, 0x8c8c)) + assertApprox(rgb('#4a3c8c'), (0x4a4a, 0x3c3c, 0x8c8c)) # #RRRRGGGGBBBB - self.assertEqual(rgb('#dede14143939'), (0xdede, 0x1414, 0x3939)) + assertApprox(rgb('#dede14143939'), (0xdede, 0x1414, 0x3939)) # Invalid string. with self.assertRaises(tkinter.TclError): rgb('#123456789a')

1 0

[3.9] bpo-46032: Check types in singledispatch's register() at declaration time (GH-30050) (GH-30254) (GH-30255)
by serhiy-storchaka Dec. 26, 2021

Dec. 26, 2021

https://github.com/python/cpython/commit/25a12aac4de819745dfc64664ba183a578… commit: 25a12aac4de819745dfc64664ba183a5784b5a81 branch: 3.9 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2021-12-26T14:23:23+02:00 summary: [3.9] bpo-46032: Check types in singledispatch's register() at declaration time (GH-30050) (GH-30254) (GH-30255) The registry() method of functools.singledispatch() functions checks now the first argument or the first parameter annotation and raises a TypeError if it is not supported. Previously unsupported "types" were ignored (e.g. typing.List[int]) or caused an error at calling time (e.g. list[int]). (cherry picked from commit 078abb676cf759b1e960f78390b6e80f256f0255) (cherry picked from commit 03c7449fbc7c57f5e0365f234a0b65c1dde763f2) Co-authored-by: Serhiy Storchaka <storchaka(a)gmail.com> files: A Misc/NEWS.d/next/Library/2021-12-11-15-45-07.bpo-46032.HmciLT.rst M Lib/functools.py M Lib/test/test_functools.py diff --git a/Lib/functools.py b/Lib/functools.py index 1a290e1c02461..77e5035ebcc9d 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -739,6 +739,7 @@ def _compose_mro(cls, types): # Remove entries which are already present in the __mro__ or unrelated. def is_related(typ): return (typ not in bases and hasattr(typ, '__mro__') + and not isinstance(typ, GenericAlias) and issubclass(cls, typ)) types = [n for n in types if is_related(n)] # Remove entries which are strict bases of other entries (they will end up @@ -836,6 +837,9 @@ def dispatch(cls): dispatch_cache[cls] = impl return impl + def _is_valid_dispatch_type(cls): + return isinstance(cls, type) and not isinstance(cls, GenericAlias) + def register(cls, func=None): """generic_func.register(cls, func) -> func @@ -843,9 +847,15 @@ def register(cls, func=None): """ nonlocal cache_token - if func is None: - if isinstance(cls, type): + if _is_valid_dispatch_type(cls): + if func is None: return lambda f: register(cls, f) + else: + if func is not None: + raise TypeError( + f"Invalid first argument to `register()`. " + f"{cls!r} is not a class." + ) ann = getattr(cls, '__annotations__', {}) if not ann: raise TypeError( @@ -858,11 +868,12 @@ def register(cls, func=None): # only import typing if annotation parsing is necessary from typing import get_type_hints argname, cls = next(iter(get_type_hints(func).items())) - if not isinstance(cls, type): + if not _is_valid_dispatch_type(cls): raise TypeError( f"Invalid annotation for {argname!r}. " f"{cls!r} is not a class." ) + registry[cls] = func if cache_token is None and hasattr(cls, '__abstractmethods__'): cache_token = get_cache_token() diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index b2a7e5a88b422..d7dec3f3d2756 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -2560,7 +2560,7 @@ def decorated_classmethod(cls, arg: int) -> str: self.assertEqual(without_single_dispatch_foo, single_dispatch_foo) self.assertEqual(single_dispatch_foo, '5') - + self.assertEqual( WithoutSingleDispatch.decorated_classmethod(5), WithSingleDispatch.decorated_classmethod(5) @@ -2655,6 +2655,74 @@ def f(*args): with self.assertRaisesRegex(TypeError, msg): f() + def test_register_genericalias(self): + @functools.singledispatch + def f(arg): + return "default" + + with self.assertRaisesRegex(TypeError, "Invalid first argument to "): + f.register(list[int], lambda arg: "types.GenericAlias") + with self.assertRaisesRegex(TypeError, "Invalid first argument to "): + f.register(typing.List[int], lambda arg: "typing.GenericAlias") + with self.assertRaisesRegex(TypeError, "Invalid first argument to "): + f.register(typing.Union[list[int], str], lambda arg: "typing.Union[types.GenericAlias]") + with self.assertRaisesRegex(TypeError, "Invalid first argument to "): + f.register(typing.Union[typing.List[float], bytes], lambda arg: "typing.Union[typing.GenericAlias]") + with self.assertRaisesRegex(TypeError, "Invalid first argument to "): + f.register(typing.Any, lambda arg: "typing.Any") + + self.assertEqual(f([1]), "default") + self.assertEqual(f([1.0]), "default") + self.assertEqual(f(""), "default") + self.assertEqual(f(b""), "default") + + def test_register_genericalias_decorator(self): + @functools.singledispatch + def f(arg): + return "default" + + with self.assertRaisesRegex(TypeError, "Invalid first argument to "): + f.register(list[int]) + with self.assertRaisesRegex(TypeError, "Invalid first argument to "): + f.register(typing.List[int]) + with self.assertRaisesRegex(TypeError, "Invalid first argument to "): + f.register(typing.Union[list[int], str]) + with self.assertRaisesRegex(TypeError, "Invalid first argument to "): + f.register(typing.Union[typing.List[int], str]) + with self.assertRaisesRegex(TypeError, "Invalid first argument to "): + f.register(typing.Any) + + def test_register_genericalias_annotation(self): + @functools.singledispatch + def f(arg): + return "default" + + with self.assertRaisesRegex(TypeError, "Invalid annotation for 'arg'"): + @f.register + def _(arg: list[int]): + return "types.GenericAlias" + with self.assertRaisesRegex(TypeError, "Invalid annotation for 'arg'"): + @f.register + def _(arg: typing.List[float]): + return "typing.GenericAlias" + with self.assertRaisesRegex(TypeError, "Invalid annotation for 'arg'"): + @f.register + def _(arg: typing.Union[list[int], str]): + return "types.UnionType(types.GenericAlias)" + with self.assertRaisesRegex(TypeError, "Invalid annotation for 'arg'"): + @f.register + def _(arg: typing.Union[typing.List[float], bytes]): + return "typing.Union[typing.GenericAlias]" + with self.assertRaisesRegex(TypeError, "Invalid annotation for 'arg'"): + @f.register + def _(arg: typing.Any): + return "typing.Any" + + self.assertEqual(f([1]), "default") + self.assertEqual(f([1.0]), "default") + self.assertEqual(f(""), "default") + self.assertEqual(f(b""), "default") + class CachedCostItem: _cost = 1 diff --git a/Misc/NEWS.d/next/Library/2021-12-11-15-45-07.bpo-46032.HmciLT.rst b/Misc/NEWS.d/next/Library/2021-12-11-15-45-07.bpo-46032.HmciLT.rst new file mode 100644 index 0000000000000..97a553d7ba29f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-12-11-15-45-07.bpo-46032.HmciLT.rst @@ -0,0 +1,5 @@ +The ``registry()`` method of :func:`functools.singledispatch` functions +checks now the first argument or the first parameter annotation and raises a +TypeError if it is not supported. Previously unsupported "types" were +ignored (e.g. ``typing.List[int]``) or caused an error at calling time (e.g. +``list[int]``).

1 0