Mailman 3 May 2020 - Python-checkins

Clean up unused imports for the peg generator module (GH-19891)
by Anthony Shaw 04 May '20

04 May '20

https://github.com/python/cpython/commit/c95e691c904bb5ebd91825efa81b93cb9e… commit: c95e691c904bb5ebd91825efa81b93cb9e354a85 branch: master author: Anthony Shaw <anthony.p.shaw(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-05-04T03:03:05+01:00 summary: Clean up unused imports for the peg generator module (GH-19891) files: M Tools/peg_generator/pegen/build.py M Tools/peg_generator/pegen/c_generator.py M Tools/peg_generator/pegen/first_sets.py M Tools/peg_generator/pegen/grammar.py M Tools/peg_generator/pegen/grammar_visualizer.py M Tools/peg_generator/scripts/ast_timings.py M Tools/peg_generator/scripts/benchmark.py M Tools/peg_generator/scripts/find_max_nesting.py M Tools/peg_generator/scripts/test_parse_directory.py diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 8f9348ddf24ac..931ffc787523b 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -1,12 +1,11 @@ import pathlib import shutil import tokenize -import sys import sysconfig import tempfile import itertools -from typing import Optional, Tuple, List, IO, Iterator, Set, Dict +from typing import Optional, Tuple, List, IO, Set, Dict from pegen.c_generator import CParserGenerator from pegen.grammar import Grammar diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index c9c67067d4677..40004e7875278 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -1,7 +1,7 @@ import ast from dataclasses import dataclass, field import re -from typing import IO, Any, Dict, List, Optional, Set, Text, Tuple +from typing import Any, Dict, IO, Optional, List, Text, Tuple, Set from enum import Enum from pegen import grammar diff --git a/Tools/peg_generator/pegen/first_sets.py b/Tools/peg_generator/pegen/first_sets.py index da30eba99ce5a..71be5a2e7cbf4 100755 --- a/Tools/peg_generator/pegen/first_sets.py +++ b/Tools/peg_generator/pegen/first_sets.py @@ -1,10 +1,9 @@ #!/usr/bin/env python3.8 import argparse -import collections import pprint import sys -from typing import Optional, Set, Dict +from typing import Set, Dict from pegen.build import build_parser from pegen.grammar import ( diff --git a/Tools/peg_generator/pegen/grammar.py b/Tools/peg_generator/pegen/grammar.py index 67039d5a032ab..78edf412ea6e4 100644 --- a/Tools/peg_generator/pegen/grammar.py +++ b/Tools/peg_generator/pegen/grammar.py @@ -4,7 +4,6 @@ from typing import ( AbstractSet, Any, - Callable, Dict, Iterable, Iterator, @@ -13,11 +12,9 @@ Set, Tuple, TYPE_CHECKING, - TypeVar, Union, ) -from pegen.parser import memoize, Parser if TYPE_CHECKING: from pegen.parser_generator import ParserGenerator diff --git a/Tools/peg_generator/pegen/grammar_visualizer.py b/Tools/peg_generator/pegen/grammar_visualizer.py index b1d51d2cdb250..7362ec5fa0f4d 100644 --- a/Tools/peg_generator/pegen/grammar_visualizer.py +++ b/Tools/peg_generator/pegen/grammar_visualizer.py @@ -1,7 +1,7 @@ import argparse import sys -from typing import Any, Iterator, Iterable, Callable +from typing import Any, Iterator, Callable from pegen.build import build_parser from pegen.grammar import Grammar, Rule diff --git a/Tools/peg_generator/scripts/ast_timings.py b/Tools/peg_generator/scripts/ast_timings.py index 7ebd46fdac685..ca252208f283e 100644 --- a/Tools/peg_generator/scripts/ast_timings.py +++ b/Tools/peg_generator/scripts/ast_timings.py @@ -1,8 +1,6 @@ import ast import sys import time -import token -import tokenize from pegen.testutil import print_memstats diff --git a/Tools/peg_generator/scripts/benchmark.py b/Tools/peg_generator/scripts/benchmark.py index 0e9d5bd104813..d39ac3dca79e7 100644 --- a/Tools/peg_generator/scripts/benchmark.py +++ b/Tools/peg_generator/scripts/benchmark.py @@ -4,7 +4,6 @@ import ast import sys import os -import resource from time import time import memory_profiler diff --git a/Tools/peg_generator/scripts/find_max_nesting.py b/Tools/peg_generator/scripts/find_max_nesting.py index a2c41a821342a..f2fdd00bfb7cd 100755 --- a/Tools/peg_generator/scripts/find_max_nesting.py +++ b/Tools/peg_generator/scripts/find_max_nesting.py @@ -13,11 +13,7 @@ Usage: python -m scripts.find_max_nesting """ -import os import sys -from tempfile import TemporaryDirectory -from pathlib import Path -from typing import Any from _peg_parser import parse_string diff --git a/Tools/peg_generator/scripts/test_parse_directory.py b/Tools/peg_generator/scripts/test_parse_directory.py index a6078ef564061..aef9c74b52881 100755 --- a/Tools/peg_generator/scripts/test_parse_directory.py +++ b/Tools/peg_generator/scripts/test_parse_directory.py @@ -4,7 +4,6 @@ import ast import os import sys -import tempfile import time import traceback from glob import glob

1 0

bpo-40334: Set error_indicator in _PyPegen_raise_error (GH-19887)
by Lysandros Nikolaou 04 May '20

04 May '20

https://github.com/python/cpython/commit/7f06af684a1882fdb19d20650825948b1d… commit: 7f06af684a1882fdb19d20650825948b1d7996e5 branch: master author: Lysandros Nikolaou <lisandrosnik(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-05-04T01:20:09+01:00 summary: bpo-40334: Set error_indicator in _PyPegen_raise_error (GH-19887) Due to PyErr_Occurred not being called at the beginning of each rule, we need to set the error indicator, so that rules do not get expanded after an exception has been thrown files: M Parser/pegen/pegen.c diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index 6ff09b3b31f78..9858f71c83c79 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -389,6 +389,7 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int with_col_number, const ch Token *t = p->tokens[p->fill - 1]; Py_ssize_t col_number = !with_col_number; va_list va; + p->error_indicator = 1; va_start(va, errmsg); errstr = PyUnicode_FromFormatV(errmsg, va);

1 0

Remove outdated and confusing advice about setting maxsize (GH-19889) (GH-19890)
by Miss Islington (bot) 04 May '20

04 May '20

https://github.com/python/cpython/commit/9609460ce0b71215eff2d057b6780950e0… commit: 9609460ce0b71215eff2d057b6780950e00be013 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-05-03T17:16:36-07:00 summary: Remove outdated and confusing advice about setting maxsize (GH-19889) (GH-19890) files: M Doc/library/functools.rst diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 3a0b554e923c7..0fb8d900c7362 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -101,8 +101,7 @@ The :mod:`functools` module defines the following functions: return sum(sentence.count(vowel) for vowel in 'aeiou') If *maxsize* is set to ``None``, the LRU feature is disabled and the cache can - grow without bound. The LRU feature performs best when *maxsize* is a - power-of-two. + grow without bound. If *typed* is set to true, function arguments of different types will be cached separately. For example, ``f(3)`` and ``f(3.0)`` will be treated

1 0

Remove outdated and confusing advice about setting maxsize (GH-19889)
by Raymond Hettinger 04 May '20

04 May '20

https://github.com/python/cpython/commit/ad9eaeab74b680830dbefc18e8fe3dec46… commit: ad9eaeab74b680830dbefc18e8fe3dec4677a21b branch: master author: Raymond Hettinger <rhettinger(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-05-03T16:45:13-07:00 summary: Remove outdated and confusing advice about setting maxsize (GH-19889) files: M Doc/library/functools.rst diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index e708a0d99cd00..856c1c790ae36 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -107,8 +107,7 @@ The :mod:`functools` module defines the following functions: return sum(sentence.count(vowel) for vowel in 'aeiou') If *maxsize* is set to ``None``, the LRU feature is disabled and the cache can - grow without bound. The LRU feature performs best when *maxsize* is a - power-of-two. + grow without bound. If *typed* is set to true, function arguments of different types will be cached separately. For example, ``f(3)`` and ``f(3.0)`` will be treated

1 0

Remove out-of-date comment (GH-19886)
by Raymond Hettinger 04 May '20

04 May '20

https://github.com/python/cpython/commit/d699d5e6178adca785a8701c32daf5e18f… commit: d699d5e6178adca785a8701c32daf5e18fad0bf1 branch: master author: Raymond Hettinger <rhettinger(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-05-03T11:25:46-07:00 summary: Remove out-of-date comment (GH-19886) files: M Objects/setobject.c diff --git a/Objects/setobject.c b/Objects/setobject.c index df4a0e1e9420e..0e4e45f60a9cc 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -243,8 +243,7 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) /* Internal routine used by set_table_resize() to insert an item which is -known to be absent from the set. This routine also assumes that -the set contains no deleted entries. Besides the performance benefit, +known to be absent from the set. Besides the performance benefit, there is also safety benefit since using set_add_entry() risks making a callback in the middle of a set_table_resize(), see issue 1456209. The caller is responsible for updating the key's reference count and

1 0

bpo-38870: Don't start generated output with newlines in ast.unparse (GH-19636)
by Batuhan Taskaya 04 May '20

04 May '20

https://github.com/python/cpython/commit/493bf1cc316b0b5bd90779ecd1132878c8… commit: 493bf1cc316b0b5bd90779ecd1132878c881669e branch: master author: Batuhan Taskaya <batuhanosmantaskaya(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-05-03T18:11:51+01:00 summary: bpo-38870: Don't start generated output with newlines in ast.unparse (GH-19636) files: M Lib/ast.py M Lib/test/test_unparse.py diff --git a/Lib/ast.py b/Lib/ast.py index 401af5647a240..5c68c4a66e1dd 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -669,10 +669,16 @@ def items_view(self, traverser, items): else: self.interleave(lambda: self.write(", "), traverser, items) + def maybe_newline(self): + """Adds a newline if it isn't the start of generated source""" + if self._source: + self.write("\n") + def fill(self, text=""): """Indent a piece of text and append it, according to the current indentation level""" - self.write("\n" + " " * self._indent + text) + self.maybe_newline() + self.write(" " * self._indent + text) def write(self, text): """Append a piece of text""" @@ -916,7 +922,7 @@ def visit_ExceptHandler(self, node): self.traverse(node.body) def visit_ClassDef(self, node): - self.write("\n") + self.maybe_newline() for deco in node.decorator_list: self.fill("@") self.traverse(deco) @@ -946,7 +952,7 @@ def visit_AsyncFunctionDef(self, node): self._function_helper(node, "async def") def _function_helper(self, node, fill_suffix): - self.write("\n") + self.maybe_newline() for deco in node.decorator_list: self.fill("@") self.traverse(deco) @@ -1043,7 +1049,7 @@ def _fstring_FormattedValue(self, node, write): write("{") unparser = type(self)() unparser.set_precedence(_Precedence.TEST.next(), node.value) - expr = unparser.visit(node.value).rstrip("\n") + expr = unparser.visit(node.value) if expr.startswith("{"): write(" ") # Separate pair of opening brackets as "{ {" write(expr) diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index d4089a3fc1cdf..2be44b246aa69 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -128,19 +128,17 @@ def check_ast_roundtrip(self, code1, **kwargs): def check_invalid(self, node, raises=ValueError): self.assertRaises(raises, ast.unparse, node) - def get_source(self, code1, code2=None, strip=True): + def get_source(self, code1, code2=None): code2 = code2 or code1 code1 = ast.unparse(ast.parse(code1)) - if strip: - code1 = code1.strip() return code1, code2 - def check_src_roundtrip(self, code1, code2=None, strip=True): - code1, code2 = self.get_source(code1, code2, strip) + def check_src_roundtrip(self, code1, code2=None): + code1, code2 = self.get_source(code1, code2) self.assertEqual(code2, code1) - def check_src_dont_roundtrip(self, code1, code2=None, strip=True): - code1, code2 = self.get_source(code1, code2, strip) + def check_src_dont_roundtrip(self, code1, code2=None): + code1, code2 = self.get_source(code1, code2) self.assertNotEqual(code2, code1) class UnparseTestCase(ASTTestCase):

1 0

Simplify set entry insertion logic. (GH-19881)
by Raymond Hettinger 03 May '20

03 May '20

https://github.com/python/cpython/commit/3dd2157febae5087ca3333d24f69b6de9c… commit: 3dd2157febae5087ca3333d24f69b6de9cbd13cd branch: master author: Raymond Hettinger <rhettinger(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-05-03T04:51:05-07:00 summary: Simplify set entry insertion logic. (GH-19881) files: M Objects/setobject.c diff --git a/Objects/setobject.c b/Objects/setobject.c index bbe013bcfac74..df4a0e1e9420e 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -137,7 +137,6 @@ static int set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) { setentry *table; - setentry *freeslot; setentry *entry; size_t perturb; size_t mask; @@ -158,7 +157,6 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) if (entry->key == NULL) goto found_unused; - freeslot = NULL; perturb = hash; while (1) { @@ -187,14 +185,12 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) goto restart; mask = so->mask; /* help avoid a register spill */ } - else if (entry->hash == -1) - freeslot = entry; if (i + LINEAR_PROBES <= mask) { for (j = 0 ; j < LINEAR_PROBES ; j++) { entry++; if (entry->hash == 0 && entry->key == NULL) - goto found_unused_or_dummy; + goto found_unused; if (entry->hash == hash) { PyObject *startkey = entry->key; assert(startkey != dummy); @@ -216,8 +212,6 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) goto restart; mask = so->mask; } - else if (entry->hash == -1) - freeslot = entry; } } @@ -226,17 +220,9 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) entry = &so->table[i]; if (entry->key == NULL) - goto found_unused_or_dummy; + goto found_unused; } - found_unused_or_dummy: - if (freeslot == NULL) - goto found_unused; - so->used++; - freeslot->key = key; - freeslot->hash = hash; - return 0; - found_unused: so->fill++; so->used++;

1 0

bpo-29587: allow chaining NULL exceptions in _gen_throw() (GH-19877)
by Chris Jerdonek 03 May '20

03 May '20

https://github.com/python/cpython/commit/21893fbb74e8fde2931fbed9b511e2a413… commit: 21893fbb74e8fde2931fbed9b511e2a41362b1ab branch: master author: Chris Jerdonek <chris.jerdonek(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-05-03T00:07:57-07:00 summary: bpo-29587: allow chaining NULL exceptions in _gen_throw() (GH-19877) This is a follow-up to GH-19823 that removes the check that the exception value isn't NULL, prior to calling _PyErr_ChainExceptions(). This enables implicit exception chaining for gen.throw() in more circumstances. The commit also adds a test that a particular code snippet involving gen.throw() doesn't crash. The test shows why the new `gi_exc_state.exc_type != Py_None` check that was added is necessary. Without the new check, the code snippet (as well as a number of other tests) crashes on certain platforms (e.g. Fedora but not Mac). files: M Lib/test/test_generators.py M Objects/genobject.c diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 4d96f44b15062..5824ecd7c37e8 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -332,6 +332,26 @@ def f(): context = cm.exception.__context__ self.assertEqual((type(context), context.args), (KeyError, ('a',))) + def test_throw_after_none_exc_type(self): + def g(): + try: + raise KeyError + except KeyError: + pass + + try: + yield + except Exception: + # Without the `gi_exc_state.exc_type != Py_None` in + # _gen_throw(), this line was causing a crash ("Segmentation + # fault (core dumped)") on e.g. Fedora 32. + raise RuntimeError + + gen = g() + gen.send(None) + with self.assertRaises(RuntimeError) as cm: + gen.throw(ValueError) + class YieldFromTests(unittest.TestCase): def test_generator_gi_yieldfrom(self): diff --git a/Objects/genobject.c b/Objects/genobject.c index 41a63ae2e666a..b27fa929a2625 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -512,11 +512,12 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, } PyErr_Restore(typ, val, tb); - /* XXX Should we also handle the case where exc_type is true and - exc_value is false? */ - if (gen->gi_exc_state.exc_type && gen->gi_exc_state.exc_value) { + /* XXX It seems like we shouldn't have to check not equal to Py_None + here because exc_type should only ever be a class. But not including + this check was causing crashes on certain tests e.g. on Fedora. */ + if (gen->gi_exc_state.exc_type && gen->gi_exc_state.exc_type != Py_None) { Py_INCREF(gen->gi_exc_state.exc_type); - Py_INCREF(gen->gi_exc_state.exc_value); + Py_XINCREF(gen->gi_exc_state.exc_value); Py_XINCREF(gen->gi_exc_state.exc_traceback); _PyErr_ChainExceptions(gen->gi_exc_state.exc_type, gen->gi_exc_state.exc_value, gen->gi_exc_state.exc_traceback);

1 0

Minor code cleanups for statistics (GH-19873)
by Raymond Hettinger 03 May '20

03 May '20

https://github.com/python/cpython/commit/0400a7f2f8abec8d441990e951cc25f69a… commit: 0400a7f2f8abec8d441990e951cc25f69a2a4036 branch: master author: Raymond Hettinger <rhettinger(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-05-02T19:30:24-07:00 summary: Minor code cleanups for statistics (GH-19873) * Minor cleanups: Removed unused code. Move C import near its Python version. * Clean-up whitespace files: M Lib/statistics.py diff --git a/Lib/statistics.py b/Lib/statistics.py index 9beafb341b3ad..c76a6ca519e40 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -894,6 +894,13 @@ def _normal_dist_inv_cdf(p, mu, sigma): return mu + (x * sigma) +# If available, use C implementation +try: + from _statistics import _normal_dist_inv_cdf +except ImportError: + pass + + class NormalDist: "Normal distribution of a random variable" # https://en.wikipedia.org/wiki/Normal_distribution @@ -1111,79 +1118,3 @@ def __hash__(self): def __repr__(self): return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' - -# If available, use C implementation -try: - from _statistics import _normal_dist_inv_cdf -except ImportError: - pass - - -if __name__ == '__main__': - - # Show math operations computed analytically in comparsion - # to a monte carlo simulation of the same operations - - from math import isclose - from operator import add, sub, mul, truediv - from itertools import repeat - import doctest - - g1 = NormalDist(10, 20) - g2 = NormalDist(-5, 25) - - # Test scaling by a constant - assert (g1 * 5 / 5).mean == g1.mean - assert (g1 * 5 / 5).stdev == g1.stdev - - n = 100_000 - G1 = g1.samples(n) - G2 = g2.samples(n) - - for func in (add, sub): - print(f'\nTest {func.__name__} with another NormalDist:') - print(func(g1, g2)) - print(NormalDist.from_samples(map(func, G1, G2))) - - const = 11 - for func in (add, sub, mul, truediv): - print(f'\nTest {func.__name__} with a constant:') - print(func(g1, const)) - print(NormalDist.from_samples(map(func, G1, repeat(const)))) - - const = 19 - for func in (add, sub, mul): - print(f'\nTest constant with {func.__name__}:') - print(func(const, g1)) - print(NormalDist.from_samples(map(func, repeat(const), G1))) - - def assert_close(G1, G2): - assert isclose(G1.mean, G1.mean, rel_tol=0.01), (G1, G2) - assert isclose(G1.stdev, G2.stdev, rel_tol=0.01), (G1, G2) - - X = NormalDist(-105, 73) - Y = NormalDist(31, 47) - s = 32.75 - n = 100_000 - - S = NormalDist.from_samples([x + s for x in X.samples(n)]) - assert_close(X + s, S) - - S = NormalDist.from_samples([x - s for x in X.samples(n)]) - assert_close(X - s, S) - - S = NormalDist.from_samples([x * s for x in X.samples(n)]) - assert_close(X * s, S) - - S = NormalDist.from_samples([x / s for x in X.samples(n)]) - assert_close(X / s, S) - - S = NormalDist.from_samples([x + y for x, y in zip(X.samples(n), - Y.samples(n))]) - assert_close(X + Y, S) - - S = NormalDist.from_samples([x - y for x, y in zip(X.samples(n), - Y.samples(n))]) - assert_close(X - Y, S) - - print(doctest.testmod())

1 0

Fix missing space in docs(GH-19866) (GH-19872)
by Miss Islington (bot) 03 May '20

03 May '20

https://github.com/python/cpython/commit/108e45524d5b2d8aa0d7feb1e593ea061f… commit: 108e45524d5b2d8aa0d7feb1e593ea061fb36ba4 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-05-02T17:05:49-07:00 summary: Fix missing space in docs(GH-19866) (GH-19872) files: M Doc/library/random.rst diff --git a/Doc/library/random.rst b/Doc/library/random.rst index c01b2294b0436..2149e800e1657 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -102,7 +102,7 @@ Bookkeeping functions .. function:: getrandbits(k) Returns a Python integer with *k* random bits. This method is supplied with - the MersenneTwister generator and some other generators may also provide it + the Mersenne Twister generator and some other generators may also provide it as an optional part of the API. When available, :meth:`getrandbits` enables :meth:`randrange` to handle arbitrarily large ranges.

1 0