Mailman 3 May 2018 - Python-checkins

bpo-33718: regrtest keeps filters to re-run fails (GH-7291)
by Victor Stinner 31 May '18

31 May '18

https://github.com/python/cpython/commit/9e24930dfdc28e16dabbfd7dd1ead1336b… commit: 9e24930dfdc28e16dabbfd7dd1ead1336b7b0d6c branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-06-01T00:48:57+02:00 summary: bpo-33718: regrtest keeps filters to re-run fails (GH-7291) * No longer clear filters, like --match, to re-run failed tests in verbose mode (-w option). * Tests result: always indicate if tests have been interrupted. * Enhance tests summary files: M Lib/test/libregrtest/main.py diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 1ab47bf58d67..f0c1631509bd 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -282,7 +282,6 @@ def rerun_failed_tests(self): self.ns.verbose = True self.ns.failfast = False self.ns.verbose3 = False - self.ns.match_tests = None print() print("Re-running failed tests in verbose mode") @@ -313,7 +312,7 @@ def display_result(self): return print() - print("== Tests result ==") + print("== Tests result: %s ==" % self.get_tests_result()) if self.interrupted: print() @@ -324,11 +323,6 @@ def display_result(self): print(count(len(omitted), "test"), "omitted:") printlist(omitted) - if self.rerun: - print() - print(count(len(self.rerun), "test"), "re-run tests:") - printlist(self.rerun) - if self.good and not self.ns.quiet: print() if (not self.bad @@ -361,6 +355,11 @@ def display_result(self): print(count(len(self.skipped), "test"), "skipped:") printlist(self.skipped) + if self.rerun: + print() + print("%s:" % count(len(self.rerun), "re-run test")) + printlist(self.rerun) + def run_tests_sequential(self): if self.ns.trace: import trace @@ -445,6 +444,21 @@ def display_header(self): % (locale.getpreferredencoding(False), sys.getfilesystemencoding())) + def get_tests_result(self): + result = [] + if self.bad: + result.append("FAILURE") + elif self.ns.fail_env_changed and self.environment_changed: + result.append("ENV CHANGED") + + if self.interrupted: + result.append("INTERRUPTED") + + if not result: + result.append("SUCCESS") + + return ', '.join(result) + def run_tests(self): # For a partial run, we do not need to clutter the output. if (self.ns.header @@ -486,16 +500,7 @@ def finalize(self): print() duration = time.monotonic() - self.start_time print("Total duration: %s" % format_duration(duration)) - - if self.bad: - result = "FAILURE" - elif self.interrupted: - result = "INTERRUPTED" - elif self.ns.fail_env_changed and self.environment_changed: - result = "ENV CHANGED" - else: - result = "SUCCESS" - print("Tests result: %s" % result) + print("Tests result: %s" % self.get_tests_result()) if self.ns.runleaks: os.system("leaks %d" % os.getpid())

1 0

bpo-33717: pythoninfo: add CC --version (#7290)
by Victor Stinner 31 May '18

31 May '18

https://github.com/python/cpython/commit/56013218864d5eb81baab4665fcae13400… commit: 56013218864d5eb81baab4665fcae13400934078 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-06-01T00:33:03+02:00 summary: bpo-33717: pythoninfo: add CC --version (#7290) Get the version of the C compiler. files: M Lib/test/pythoninfo.py diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index f1b02336f7e3..776f03f8b58b 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -489,6 +489,34 @@ def collect_test_support(info_add): call_func(info_add, 'test_support.python_is_optimized', support, 'python_is_optimized') +def collect_cc(info_add): + import subprocess + import sysconfig + + CC = sysconfig.get_config_var('CC') + if not CC: + return + + try: + import shlex + args = shlex.split(CC) + except ImportError: + args = CC.split() + args.append('--version') + proc = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + stdout = proc.communicate()[0] + if proc.returncode: + # CC --version failed: ignore error + return + + text = stdout.splitlines()[0] + text = normalize_text(text) + info_add('CC.version', text) + + def collect_info(info): error = False info_add = info.add @@ -515,6 +543,7 @@ def collect_info(info): collect_decimal, collect_testcapi, collect_resource, + collect_cc, # Collecting from tests should be last as they have side effects. collect_test_socket,

1 0

bpo-33615: Temporarily disable a test that is triggering crashes on a few buildbots. (gh-7288)
by Eric Snow 31 May '18

31 May '18

https://github.com/python/cpython/commit/110bc01407ac8c75545d0386577c6e1725… commit: 110bc01407ac8c75545d0386577c6e17254d97d9 branch: master author: Eric Snow <ericsnowcurrently(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2018-05-31T10:17:24-06:00 summary: bpo-33615: Temporarily disable a test that is triggering crashes on a few buildbots. (gh-7288) For bpo-32604 I added some subinterpreter-related tests (see #6914) that are causing crashes on a few buildbots. I'm working on fixing the crashes (see #7251). This change temporarily disables the triggering test. files: M Lib/test/test__xxsubinterpreters.py diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index f66cc9516926..0667f14f7bf4 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -1315,6 +1315,8 @@ def test_run_string_arg_unresolved(self): self.assertEqual(obj, b'spam') self.assertEqual(out.strip(), 'send') + # XXX Fix the crashes. + @unittest.skip('bpo-33615: triggering crashes so temporarily disabled') def test_run_string_arg_resolved(self): cid = interpreters.channel_create() cid = interpreters._channel_id(cid, _resolve=True)

1 0

bpo-33706: Fix pymain_parse_cmdline_impl() (GH-7283)
by Miss Islington (bot) 31 May '18

31 May '18

https://github.com/python/cpython/commit/c6de46e180e81508a3b43341791b56418b… commit: c6de46e180e81508a3b43341791b56418bd811f9 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2018-05-31T06:43:21-07:00 summary: bpo-33706: Fix pymain_parse_cmdline_impl() (GH-7283) Fix a crash in Python initialization when parsing the command line options. Fix memcpy() size parameter: previously, we read one wchar_t after the end of _PyOS_optarg. Moreover, don't copy the trailingg NUL character: we write it manually anyway. Thanks Christoph Gohlke for the bug report and the fix! (cherry picked from commit 58d1683255abb0df4fc70960da6121aeaa41e1d1) Co-authored-by: Victor Stinner <vstinner(a)redhat.com> files: A Misc/NEWS.d/next/Core and Builtins/2018-05-31-14-50-04.bpo-33706.ztlH04.rst M Modules/main.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-05-31-14-50-04.bpo-33706.ztlH04.rst b/Misc/NEWS.d/next/Core and Builtins/2018-05-31-14-50-04.bpo-33706.ztlH04.rst new file mode 100644 index 000000000000..d3b8477b2197 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-05-31-14-50-04.bpo-33706.ztlH04.rst @@ -0,0 +1,2 @@ +Fix a crash in Python initialization when parsing the command line options. +Thanks Christoph Gohlke for the bug report and the fix! diff --git a/Modules/main.c b/Modules/main.c index 9cbb3f1ce003..286ad418fe13 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -761,7 +761,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _Py_CommandLineDetails *cmdline) pymain->err = _Py_INIT_NO_MEMORY(); return -1; } - memcpy(command, _PyOS_optarg, len * sizeof(wchar_t)); + memcpy(command, _PyOS_optarg, (len - 2) * sizeof(wchar_t)); command[len - 2] = '\n'; command[len - 1] = 0; pymain->command = command;

1 0

bpo-33706: Fix pymain_parse_cmdline_impl() (GH-7283)
by Victor Stinner 31 May '18

31 May '18

https://github.com/python/cpython/commit/58d1683255abb0df4fc70960da6121aeaa… commit: 58d1683255abb0df4fc70960da6121aeaa41e1d1 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-05-31T15:09:28+02:00 summary: bpo-33706: Fix pymain_parse_cmdline_impl() (GH-7283) Fix a crash in Python initialization when parsing the command line options. Fix memcpy() size parameter: previously, we read one wchar_t after the end of _PyOS_optarg. Moreover, don't copy the trailingg NUL character: we write it manually anyway. Thanks Christoph Gohlke for the bug report and the fix! files: A Misc/NEWS.d/next/Core and Builtins/2018-05-31-14-50-04.bpo-33706.ztlH04.rst M Modules/main.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-05-31-14-50-04.bpo-33706.ztlH04.rst b/Misc/NEWS.d/next/Core and Builtins/2018-05-31-14-50-04.bpo-33706.ztlH04.rst new file mode 100644 index 000000000000..d3b8477b2197 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-05-31-14-50-04.bpo-33706.ztlH04.rst @@ -0,0 +1,2 @@ +Fix a crash in Python initialization when parsing the command line options. +Thanks Christoph Gohlke for the bug report and the fix! diff --git a/Modules/main.c b/Modules/main.c index 9cbb3f1ce003..286ad418fe13 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -761,7 +761,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _Py_CommandLineDetails *cmdline) pymain->err = _Py_INIT_NO_MEMORY(); return -1; } - memcpy(command, _PyOS_optarg, len * sizeof(wchar_t)); + memcpy(command, _PyOS_optarg, (len - 2) * sizeof(wchar_t)); command[len - 2] = '\n'; command[len - 1] = 0; pymain->command = command;

1 0

Add a crasher to Lib/test for issue #26153. This crasher doesn't crash (#6518)
by T. Wouters 31 May '18

31 May '18

https://github.com/python/cpython/commit/8b98d2a7397cade11393e951db60e4b9ab… commit: 8b98d2a7397cade11393e951db60e4b9abbe1088 branch: 2.7 author: T. Wouters <thomas(a)python.org> committer: GitHub <noreply(a)github.com> date: 2018-05-31T12:22:11+02:00 summary: Add a crasher to Lib/test for issue #26153. This crasher doesn't crash (#6518) Python 3.6, although I've seen the same crash in 3.6 (when involving extension types and more complicated threading setups). files: A Lib/test/crashers/warnings_del_crasher.py diff --git a/Lib/test/crashers/warnings_del_crasher.py b/Lib/test/crashers/warnings_del_crasher.py new file mode 100644 index 000000000000..02e680520204 --- /dev/null +++ b/Lib/test/crashers/warnings_del_crasher.py @@ -0,0 +1,29 @@ +""" +Expose a race in the _warnings module, which is the C backend for the +warnings module. The "_warnings" module tries to access attributes of the +"warnings" module (because of the API it has to support), but doing so +during interpreter shutdown is problematic. Specifically, the call to +PyImport_GetModuleDict() in Python/_warnings.c:get_warnings_attr will +abort() if the modules dict has already been cleaned up. + +This crasher is timing-dependent, and more threads (NUM_THREADS) may be +necessary to expose it reliably on different systems. +""" + +import threading +import warnings + +NUM_THREADS = 10 + +class WarnOnDel(object): + def __del__(self): + warnings.warn("oh no something went wrong", UserWarning) + +def do_work(): + while True: + w = WarnOnDel() + +for i in range(NUM_THREADS): + t = threading.Thread(target=do_work) + t.setDaemon(1) + t.start()

1 0

[3.6] bpo-32591: fix abort in _PyErr_WarnUnawaitedCoroutine during shutdown (GH-5337) (#6536)
by T. Wouters 31 May '18

31 May '18

https://github.com/python/cpython/commit/500a419a7a68c32650717b213f0f5ab046… commit: 500a419a7a68c32650717b213f0f5ab0461bb16b branch: 3.6 author: T. Wouters <thomas(a)python.org> committer: GitHub <noreply(a)github.com> date: 2018-05-31T12:20:46+02:00 summary: [3.6] bpo-32591: fix abort in _PyErr_WarnUnawaitedCoroutine during shutdown (GH-5337) (#6536) When an unawaited coroutine is collected very late in shutdown -- like, during the final GC at the end of PyImport_Cleanup -- then it was triggering an interpreter abort, because we'd try to look up the "warnings" module and not only was it missing (we were prepared for that), but the entire module system was missing (which we were not prepared for). I've tried to fix this at the source, by making the utility function get_warnings_attr robust against this in general. Note that it already has the convention that it can return NULL without setting an error, which is how it signals that the attribute it was asked to fetch is missing, and that all callers already check for NULL returns. There's a similar check for being late in shutdown at the top of warn_explicit, which might be unnecessary after this fix, but I'm not sure so I'm going to leave it.. (cherry picked from commit dba976b8a28d6e5daa66ef31a6a7c694a9193f6a) Co-authored-by: Nathaniel J. Smith <njs(a)pobox.com> files: M Python/_warnings.c diff --git a/Python/_warnings.c b/Python/_warnings.c index 0077b9b1c3d0..e96e154b0d54 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -63,6 +63,13 @@ get_warnings_attr(const char *attr, int try_import) } } else { + /* if we're so late into Python finalization that the module dict is + gone, then we can't even use PyImport_GetModule without triggering + an interpreter abort. + */ + if (!PyThreadState_GET()->interp->modules) { + return NULL; + } all_modules = PyImport_GetModuleDict(); warnings_module = PyDict_GetItem(all_modules, warnings_str);

1 0

Daily reference leaks (4243df51fe43): sum=11
by solipsis＠pitrou.net 31 May '18

31 May '18

results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 0, 7] memory blocks, sum=7 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [-1, 2, -1] memory blocks, sum=0 test_multiprocessing_spawn leaked [-2, 2, 0] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogFiHD02', '--timeout', '7200']

1 0

[2.7] bpo-33677: Fix the signature of tp_clear handler for deque. (GH-7196). (GH-7277)
by Serhiy Storchaka 31 May '18

31 May '18

https://github.com/python/cpython/commit/db1074244d2c12961799f6f9353ae191f5… commit: db1074244d2c12961799f6f9353ae191f59cc497 branch: 2.7 author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2018-05-31T10:32:43+03:00 summary: [2.7] bpo-33677: Fix the signature of tp_clear handler for deque. (GH-7196). (GH-7277) (cherry picked from commit a5c42284e69fb309bdd17ee8c1c120d1be383012) files: M Modules/_collectionsmodule.c diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 2e4da4c211f8..9364aba549b1 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -639,7 +639,7 @@ deque_remove(dequeobject *deque, PyObject *value) PyDoc_STRVAR(remove_doc, "D.remove(value) -- remove first occurrence of value."); -static void +static int deque_clear(dequeobject *deque) { block *b; @@ -650,7 +650,7 @@ deque_clear(dequeobject *deque) PyObject *item; if (deque->len == 0) - return; + return 0; /* During the process of clearing a deque, decrefs can cause the deque to mutate. To avoid fatal confusion, we have to make the @@ -701,7 +701,7 @@ deque_clear(dequeobject *deque) } assert(leftblock->rightlink == NULL); freeblock(leftblock); - return; + return 0; alternate_method: while (deque->len) { @@ -709,6 +709,7 @@ deque_clear(dequeobject *deque) assert (item != NULL); Py_DECREF(item); } + return 0; } static PyObject *

1 0

bpo-32911: Update the historical note about the magic number. (GH-7272)
by Serhiy Storchaka 31 May '18

31 May '18

https://github.com/python/cpython/commit/d6bbb57855da3f92521edd901f49705be4… commit: d6bbb57855da3f92521edd901f49705be46fb203 branch: 3.7 author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2018-05-31T09:12:15+03:00 summary: bpo-32911: Update the historical note about the magic number. (GH-7272) files: M Lib/importlib/_bootstrap_external.py diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index f4be86274cab..53b24ff1b0f2 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -246,7 +246,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.7a2 3391 (update GET_AITER #31709) # Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) # Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) -# Python 3.7rc1 3394 (restored docstring as the firts stmt in the body; +# Python 3.7b5 3394 (restored docstring as the firts stmt in the body; # this might affected the first line number #32911) # # MAGIC must change whenever the bytecode emitted by the compiler may no

1 0