Mailman 3 September 2015 - Python-checkins

cpython: Issue #25220: Use print(flush=True) in libregrtest
by victor.stinner Sept. 30, 2015

Sept. 30, 2015

https://hg.python.org/cpython/rev/7e07c51d8fc6 changeset: 98423:7e07c51d8fc6 user: Victor Stinner <victor.stinner(a)gmail.com> date: Wed Sep 30 00:48:27 2015 +0200 summary: Issue #25220: Use print(flush=True) in libregrtest files: Lib/test/libregrtest/main.py | 7 +++---- Lib/test/libregrtest/refleak.py | 10 ++++------ Lib/test/libregrtest/runtest.py | 14 +++++--------- Lib/test/libregrtest/runtest_mp.py | 9 +++------ 4 files changed, 15 … [View More]insertions(+), 25 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -157,8 +157,8 @@ return fmt = "[{1:{0}}{2}/{3}] {4}" if self.bad else "[{1:{0}}{2}] {4}" print(fmt.format(self.test_count_width, test_index, - self.test_count, len(self.bad), test)) - sys.stdout.flush() + self.test_count, len(self.bad), test), + flush=True) def setup_regrtest(self): if self.ns.huntrleaks: @@ -333,8 +333,7 @@ if self.ns.verbose2 and self.bad: print("Re-running failed tests in verbose mode") for test in self.bad[:]: - print("Re-running test %r in verbose mode" % test) - sys.stdout.flush() + print("Re-running test %r in verbose mode" % test, flush=True) try: self.ns.verbose = True ok = runtest(test, True, self.ns.quiet, self.ns.huntrleaks, diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -44,13 +44,12 @@ alloc_deltas = [0] * repcount print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) - sys.stderr.flush() + print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, + flush=True) for i in range(repcount): indirect_test() alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) - sys.stderr.write('.') - sys.stderr.flush() + print('.', end='', flush=True) if i >= nwarmup: rc_deltas[i] = rc_after - rc_before alloc_deltas[i] = alloc_after - alloc_before @@ -74,8 +73,7 @@ if checker(deltas): msg = '%s leaked %s %s, sum=%s' % ( test, deltas[nwarmup:], item_name, sum(deltas)) - print(msg, file=sys.stderr) - sys.stderr.flush() + print(msg, file=sys.stderr, flush=True) with open(fname, "a") as refrep: print(msg, file=refrep) refrep.flush() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -161,27 +161,23 @@ test_time = time.time() - start_time except support.ResourceDenied as msg: if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() + print(test, "skipped --", msg, flush=True) return RESOURCE_DENIED, test_time except unittest.SkipTest as msg: if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() + print(test, "skipped --", msg, flush=True) return SKIPPED, test_time except KeyboardInterrupt: raise except support.TestFailed as msg: if display_failure: - print("test", test, "failed --", msg, file=sys.stderr) + print("test", test, "failed --", msg, file=sys.stderr, flush=True) else: - print("test", test, "failed", file=sys.stderr) - sys.stderr.flush() + print("test", test, "failed", file=sys.stderr, flush=True) return FAILED, test_time except: msg = traceback.format_exc() - print("test", test, "crashed --", msg, file=sys.stderr) - sys.stderr.flush() + print("test", test, "crashed --", msg, file=sys.stderr, flush=True) return FAILED, test_time else: if refleak: diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -70,9 +70,8 @@ except BaseException as e: traceback.print_exc() result = CHILD_ERROR, str(e) - sys.stdout.flush() print() # Force a newline (just in case) - print(json.dumps(result)) + print(json.dumps(result), flush=True) sys.exit(0) @@ -184,11 +183,9 @@ # Copy stdout and stderr from the child process if stdout: - print(stdout) + print(stdout, flush=True) if stderr: - print(stderr, file=sys.stderr) - sys.stdout.flush() - sys.stderr.flush() + print(stderr, file=sys.stderr, flush=True) if result[0] == INTERRUPTED: raise KeyboardInterrupt -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython: Issue #25220, libregrtest: Call setup_python(ns) in the slaves
by victor.stinner Sept. 30, 2015

Sept. 30, 2015

https://hg.python.org/cpython/rev/b48ae54ed5be changeset: 98427:b48ae54ed5be user: Victor Stinner <victor.stinner(a)gmail.com> date: Wed Sep 30 01:39:28 2015 +0200 summary: Issue #25220, libregrtest: Call setup_python(ns) in the slaves Slaves (child processes running tests for regrtest -jN) now inherit --memlimit/-M, --threshold/-t and --nowindows/-n options. * -M, -t and -n are now supported with -jN * Factorize code to run tests. * run_test_in_subprocess() now pass … [View More]the whole "ns" namespace to the child process. files: Lib/test/libregrtest/cmdline.py | 2 -- Lib/test/libregrtest/main.py | 5 +++-- Lib/test/libregrtest/runtest_mp.py | 8 ++++---- Lib/test/test_regrtest.py | 1 - 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -295,8 +295,6 @@ parser.error("-T and -j don't go together!") if ns.use_mp and ns.findleaks: parser.error("-l and -j don't go together!") - if ns.use_mp and ns.memlimit: - parser.error("-M and -j don't go together!") if ns.failfast and not (ns.verbose or ns.verbose3): parser.error("-G/--failfast needs either -v or -W") diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -354,14 +354,15 @@ def main(self, tests=None, **kwargs): self.ns = self.parse_args(kwargs) - setup_python(self.ns) - if self.ns.slaveargs is not None: from test.libregrtest.runtest_mp import run_tests_slave run_tests_slave(self.ns.slaveargs) + if self.ns.wait: input("Press any key to continue...") + setup_python(self.ns) + self.find_tests(tests) self.run_tests() diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -13,7 +13,8 @@ print("Multiprocess option requires thread support") sys.exit(2) -from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR +from test.libregrtest.runtest import runtest_ns, INTERRUPTED, CHILD_ERROR +from test.libregrtest.setup import setup_python # Minimum duration of a test to display its duration or to mention that @@ -58,11 +59,10 @@ ns_dict, testname = json.loads(slaveargs) ns = types.SimpleNamespace(**ns_dict) - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False + setup_python(ns) try: - result = runtest_ns(testname, ns.verbose, ns.quiet, ns, + result = runtest_ns(testname, ns.verbose, ns, use_resources=ns.use_resources, output_on_failure=ns.verbose3, failfast=ns.failfast, diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -214,7 +214,6 @@ self.checkError([opt, 'foo'], 'invalid int value') self.checkError([opt, '2', '-T'], "don't go together") self.checkError([opt, '2', '-l'], "don't go together") - self.checkError([opt, '2', '-M', '4G'], "don't go together") def test_coverage(self): for opt in '-T', '--coverage': -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython: Issue #25220, libregrtest: Add runtest_ns() function
by victor.stinner Sept. 30, 2015

Sept. 30, 2015

https://hg.python.org/cpython/rev/ff012c1b8068 changeset: 98426:ff012c1b8068 user: Victor Stinner <victor.stinner(a)gmail.com> date: Wed Sep 30 01:32:39 2015 +0200 summary: Issue #25220, libregrtest: Add runtest_ns() function * Factorize code to run tests. * run_test_in_subprocess() now pass the whole "ns" namespace to the child process. files: Lib/test/libregrtest/main.py | 17 ++++-------- Lib/test/libregrtest/runtest.py | 7 +++++ Lib/test/… [View More]libregrtest/runtest_mp.py | 24 +++++++++-------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,7 +7,7 @@ import tempfile import textwrap from test.libregrtest.runtest import ( - findtests, runtest, + findtests, runtest_ns, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) from test.libregrtest.cmdline import _parse_args from test.libregrtest.setup import setup_python @@ -251,8 +251,7 @@ print("Re-running test %r in verbose mode" % test, flush=True) try: self.ns.verbose = True - ok = runtest(test, True, self.ns.quiet, self.ns.huntrleaks, - timeout=self.ns.timeout) + ok = runtest_ns(test, True, self.ns) except KeyboardInterrupt: # print a newline separate from the ^C print() @@ -266,14 +265,10 @@ printlist(self.bad) def run_test(self, test): - result = runtest(test, - self.ns.verbose, - self.ns.quiet, - self.ns.huntrleaks, - output_on_failure=self.ns.verbose3, - timeout=self.ns.timeout, - failfast=self.ns.failfast, - match_tests=self.ns.match_tests) + result = runtest_ns(test, self.ns.verbose, self.ns, + output_on_failure=self.ns.verbose3, + failfast=self.ns.failfast, + match_tests=self.ns.match_tests) self.accumulate_result(test, result) def run_tests_sequential(self): diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -53,6 +53,13 @@ return stdtests + sorted(tests) +def runtest_ns(test, verbose, ns, **kw): + return runtest(test, verbose, ns.quiet, + huntrleaks=ns.huntrleaks, + timeout=ns.timeout, + **kw) + + def runtest(test, verbose, quiet, huntrleaks=False, use_resources=None, output_on_failure=False, failfast=False, match_tests=None, diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -3,6 +3,7 @@ import sys import time import traceback +import types import unittest from queue import Queue from test import support @@ -30,14 +31,8 @@ """ from subprocess import Popen, PIPE - args = (testname, ns.verbose, ns.quiet) - kwargs = dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, - failfast=ns.failfast, - match_tests=ns.match_tests) - slaveargs = (args, kwargs) + ns_dict = vars(ns) + slaveargs = (ns_dict, testname) slaveargs = json.dumps(slaveargs) cmd = [sys.executable, *support.args_from_interpreter_flags(), @@ -60,11 +55,18 @@ def run_tests_slave(slaveargs): - args, kwargs = json.loads(slaveargs) - if kwargs.get('huntrleaks'): + ns_dict, testname = json.loads(slaveargs) + ns = types.SimpleNamespace(**ns_dict) + + if ns.huntrleaks: unittest.BaseTestSuite._cleanup = False + try: - result = runtest(*args, **kwargs) + result = runtest_ns(testname, ns.verbose, ns.quiet, ns, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + failfast=ns.failfast, + match_tests=ns.match_tests) except KeyboardInterrupt: result = INTERRUPTED, '' except BaseException as e: -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython: Issue #25220, libregrtest: Cleanup setup code
by victor.stinner Sept. 30, 2015

Sept. 30, 2015

https://hg.python.org/cpython/rev/e2ed6e9163d5 changeset: 98424:e2ed6e9163d5 user: Victor Stinner <victor.stinner(a)gmail.com> date: Wed Sep 30 00:59:35 2015 +0200 summary: Issue #25220, libregrtest: Cleanup setup code files: Lib/test/libregrtest/main.py | 96 ++++++++++++----------- 1 files changed, 49 insertions(+), 47 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/… [View More]libregrtest/main.py @@ -31,7 +31,7 @@ TEMPDIR = os.path.abspath(TEMPDIR) -def setup_python(): +def setup_python(ns): # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) @@ -80,6 +80,38 @@ newsoft = min(hard, max(soft, 1024*2048)) resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() + + if ns.memlimit is not None: + support.set_memlimit(ns.memlimit) + + if ns.threshold is not None: + if gc is not None: + gc.set_threshold(ns.threshold) + else: + print('No GC available, ignore --threshold.') + + if ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + class Regrtest: """Execute a test suite. @@ -161,36 +193,6 @@ flush=True) def setup_regrtest(self): - if self.ns.huntrleaks: - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - - if self.ns.memlimit is not None: - support.set_memlimit(self.ns.memlimit) - - if self.ns.threshold is not None: - if gc is not None: - gc.set_threshold(self.ns.threshold) - else: - print('No GC available, ignore --threshold.') - - if self.ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - if self.ns.findleaks: if gc is not None: # Uncomment the line below to report garbage that is not @@ -202,19 +204,9 @@ print('No GC available, disabling --findleaks') self.ns.findleaks = False - if self.ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False - # Strip .py extensions. removepy(self.ns.args) - if self.ns.trace: - import trace - self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, - sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) - def find_tests(self, tests): self.tests = tests @@ -278,7 +270,7 @@ except IndexError: pass - # Remove all the self.selected tests that precede start if it's set. + # Remove all the selected tests that precede start if it's set. if self.ns.start: try: del self.selected[:self.selected.index(self.ns.start)] @@ -362,11 +354,18 @@ self.accumulate_result(test, result) def run_tests_sequential(self): + if self.ns.trace: + import trace + self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, + sys.base_exec_prefix, + tempfile.gettempdir()], + trace=False, count=True) + save_modules = sys.modules.keys() for test_index, test in enumerate(self.tests, 1): self.display_progress(test_index, test) - if self.ns.trace: + if self.tracer: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. cmd = 'self.run_test(test)' @@ -426,7 +425,7 @@ else: os.unlink(self.next_single_filename) - if self.ns.trace: + if self.tracer: r = self.tracer.results() r.write_results(show_missing=True, summary=True, coverdir=self.ns.coverdir) @@ -436,15 +435,18 @@ def main(self, tests=None, **kwargs): self.ns = _parse_args(sys.argv[1:], **kwargs) - setup_python() + setup_python(self.ns) self.setup_regrtest() - if self.ns.wait: - input("Press any key to continue...") + if self.ns.slaveargs is not None: from test.libregrtest.runtest_mp import run_tests_slave run_tests_slave(self.ns.slaveargs) + if self.ns.wait: + input("Press any key to continue...") + self.find_tests(tests) self.run_tests() + self.display_result() self.finalize() sys.exit(len(self.bad) > 0 or self.interrupted) -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython: Issue #25274: Workaround test_sys crash just to keep buildbots usable
by victor.stinner Sept. 30, 2015

Sept. 30, 2015

https://hg.python.org/cpython/rev/00c1cd1f0131 changeset: 98428:00c1cd1f0131 user: Victor Stinner <victor.stinner(a)gmail.com> date: Wed Sep 30 02:02:49 2015 +0200 summary: Issue #25274: Workaround test_sys crash just to keep buildbots usable files: Lib/test/test_sys.py | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -208,7 +208,10 @@ … [View More]

1 0

cpython: Issue #25220, libregrtest: Move setup_python() to a new submodule
by victor.stinner Sept. 30, 2015

Sept. 30, 2015

https://hg.python.org/cpython/rev/b7d27c3c9e65 changeset: 98425:b7d27c3c9e65 user: Victor Stinner <victor.stinner(a)gmail.com> date: Wed Sep 30 01:13:53 2015 +0200 summary: Issue #25220, libregrtest: Move setup_python() to a new submodule files: Lib/test/libregrtest/main.py | 125 ++------------------- Lib/test/libregrtest/setup.py | 108 +++++++++++++++++++ 2 files changed, 123 insertions(+), 110 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/… [View More]test/libregrtest/main.py --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,19 +1,16 @@ -import faulthandler import os import platform import random import re -import signal import sys import sysconfig import tempfile import textwrap -import unittest from test.libregrtest.runtest import ( findtests, runtest, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) -from test.libregrtest.refleak import warm_caches from test.libregrtest.cmdline import _parse_args +from test.libregrtest.setup import setup_python from test import support try: import gc @@ -31,88 +28,6 @@ TEMPDIR = os.path.abspath(TEMPDIR) -def setup_python(ns): - # Display the Python traceback on fatal errors (e.g. segfault) - faulthandler.enable(all_threads=True) - - # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] - if hasattr(signal, 'SIGALRM'): - signals.append(signal.SIGALRM) - if hasattr(signal, 'SIGUSR1'): - signals.append(signal.SIGUSR1) - for signum in signals: - faulthandler.register(signum, chain=True) - - replace_stdout() - support.record_original_stdout(sys.stdout) - - # Some times __path__ and __file__ are not absolute (e.g. while running from - # Lib/) and, if we change the CWD to run the tests in a temporary dir, some - # imports might fail. This affects only the modules imported before os.chdir(). - # These modules are searched first in sys.path[0] (so '' -- the CWD) and if - # they are found in the CWD their __file__ and __path__ will be relative (this - # happens before the chdir). All the modules imported after the chdir, are - # not found in the CWD, and since the other paths in sys.path[1:] are absolute - # (site.py absolutize them), the __file__ and __path__ will be absolute too. - # Therefore it is necessary to absolutize manually the __file__ and __path__ of - # the packages to prevent later imports to fail when the CWD is different. - for module in sys.modules.values(): - if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) - for path in module.__path__] - if hasattr(module, '__file__'): - module.__file__ = os.path.abspath(module.__file__) - - # MacOSX (a.k.a. Darwin) has a default stack size that is too small - # for deeply recursive regular expressions. We see this as crashes in - # the Python test suite when running test_re.py and test_sre.py. The - # fix is to set the stack limit to 2048. - # This approach may also be useful for other Unixy platforms that - # suffer from small default stack limits. - if sys.platform == 'darwin': - try: - import resource - except ImportError: - pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) - - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False - - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) - - if ns.threshold is not None: - if gc is not None: - gc.set_threshold(ns.threshold) - else: - print('No GC available, ignore --threshold.') - - if ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - - class Regrtest: """Execute a test suite. @@ -192,8 +107,14 @@ self.test_count, len(self.bad), test), flush=True) - def setup_regrtest(self): - if self.ns.findleaks: + def parse_args(self, kwargs): + ns = _parse_args(sys.argv[1:], **kwargs) + + if ns.threshold is not None and gc is None: + print('No GC available, ignore --threshold.') + ns.threshold = None + + if ns.findleaks: if gc is not None: # Uncomment the line below to report garbage that is not # freeable by reference counting alone. By default only @@ -202,10 +123,12 @@ #gc.set_debug(gc.DEBUG_SAVEALL) else: print('No GC available, disabling --findleaks') - self.ns.findleaks = False + ns.findleaks = False # Strip .py extensions. - removepy(self.ns.args) + removepy(ns.args) + + return ns def find_tests(self, tests): self.tests = tests @@ -434,9 +357,9 @@ os.system("leaks %d" % os.getpid()) def main(self, tests=None, **kwargs): - self.ns = _parse_args(sys.argv[1:], **kwargs) + self.ns = self.parse_args(kwargs) + setup_python(self.ns) - self.setup_regrtest() if self.ns.slaveargs is not None: from test.libregrtest.runtest_mp import run_tests_slave @@ -452,24 +375,6 @@ sys.exit(len(self.bad) > 0 or self.interrupted) -def replace_stdout(): - """Set stdout encoder error handler to backslashreplace (as stderr error - handler) to avoid UnicodeEncodeError when printing a traceback""" - import atexit - - stdout = sys.stdout - sys.stdout = open(stdout.fileno(), 'w', - encoding=stdout.encoding, - errors="backslashreplace", - closefd=False, - newline='\n') - - def restore_stdout(): - sys.stdout.close() - sys.stdout = stdout - atexit.register(restore_stdout) - - def removepy(names): if not names: return diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py new file mode 100644 --- /dev/null +++ b/Lib/test/libregrtest/setup.py @@ -0,0 +1,108 @@ +import atexit +import faulthandler +import os +import signal +import sys +import unittest +from test import support +try: + import gc +except ImportError: + gc = None + +from test.libregrtest.refleak import warm_caches + + +def setup_python(ns): + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + + replace_stdout() + support.record_original_stdout(sys.stdout) + + # Some times __path__ and __file__ are not absolute (e.g. while running from + # Lib/) and, if we change the CWD to run the tests in a temporary dir, some + # imports might fail. This affects only the modules imported before os.chdir(). + # These modules are searched first in sys.path[0] (so '' -- the CWD) and if + # they are found in the CWD their __file__ and __path__ will be relative (this + # happens before the chdir). All the modules imported after the chdir, are + # not found in the CWD, and since the other paths in sys.path[1:] are absolute + # (site.py absolutize them), the __file__ and __path__ will be absolute too. + # Therefore it is necessary to absolutize manually the __file__ and __path__ of + # the packages to prevent later imports to fail when the CWD is different. + for module in sys.modules.values(): + if hasattr(module, '__path__'): + module.__path__ = [os.path.abspath(path) + for path in module.__path__] + if hasattr(module, '__file__'): + module.__file__ = os.path.abspath(module.__file__) + + # MacOSX (a.k.a. Darwin) has a default stack size that is too small + # for deeply recursive regular expressions. We see this as crashes in + # the Python test suite when running test_re.py and test_sre.py. The + # fix is to set the stack limit to 2048. + # This approach may also be useful for other Unixy platforms that + # suffer from small default stack limits. + if sys.platform == 'darwin': + try: + import resource + except ImportError: + pass + else: + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + newsoft = min(hard, max(soft, 1024*2048)) + resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() + + if ns.memlimit is not None: + support.set_memlimit(ns.memlimit) + + if ns.threshold is not None: + gc.set_threshold(ns.threshold) + + if ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + + +def replace_stdout(): + """Set stdout encoder error handler to backslashreplace (as stderr error + handler) to avoid UnicodeEncodeError when printing a traceback""" + stdout = sys.stdout + sys.stdout = open(stdout.fileno(), 'w', + encoding=stdout.encoding, + errors="backslashreplace", + closefd=False, + newline='\n') + + def restore_stdout(): + sys.stdout.close() + sys.stdout = stdout + atexit.register(restore_stdout) -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython: Issue #25220: Enhance regrtest -jN
by victor.stinner Sept. 29, 2015

Sept. 29, 2015

https://hg.python.org/cpython/rev/8bd9422ef41e changeset: 98422:8bd9422ef41e user: Victor Stinner <victor.stinner(a)gmail.com> date: Wed Sep 30 00:33:29 2015 +0200 summary: Issue #25220: Enhance regrtest -jN Running the Python test suite with -jN now: - Display the duration of tests which took longer than 30 seconds - Display the tests currently running since at least 30 seconds - Display the tests we are waiting for when the test suite is interrupted Clenaup also … [View More]run_test_in_subprocess() code. files: Lib/test/libregrtest/runtest_mp.py | 125 ++++++++++++---- 1 files changed, 89 insertions(+), 36 deletions(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -1,7 +1,7 @@ import json import os -import re import sys +import time import traceback import unittest from queue import Queue @@ -15,7 +15,12 @@ from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR -def run_tests_in_subprocess(testname, ns): +# Minimum duration of a test to display its duration or to mention that +# the test is running in background +PROGRESS_MIN_TIME = 30.0 # seconds + + +def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. ns is the option Namespace parsed from command-line arguments. regrtest @@ -24,26 +29,33 @@ 3-tuple. """ from subprocess import Popen, PIPE - base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + - ['-X', 'faulthandler', '-m', 'test.regrtest']) - slaveargs = ( - (testname, ns.verbose, ns.quiet), - dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests)) + args = (testname, ns.verbose, ns.quiet) + kwargs = dict(huntrleaks=ns.huntrleaks, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + timeout=ns.timeout, + failfast=ns.failfast, + match_tests=ns.match_tests) + slaveargs = (args, kwargs) + slaveargs = json.dumps(slaveargs) + + cmd = [sys.executable, *support.args_from_interpreter_flags(), + '-X', 'faulthandler', + '-m', 'test.regrtest', + '--slaveargs', slaveargs] + # Running the child from the same working directory as regrtest's original # invocation ensures that TEMPDIR for the child is the same when # sysconfig.is_python_build() is true. See issue 15300. - popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], + popen = Popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True, close_fds=(os.name != 'nt'), cwd=support.SAVEDCWD) - stdout, stderr = popen.communicate() - retcode = popen.wait() + with popen: + stdout, stderr = popen.communicate() + retcode = popen.wait() return retcode, stdout, stderr @@ -90,30 +102,45 @@ self.pending = pending self.output = output self.ns = ns + self.current_test = None + self.start_time = None + + def _runtest(self): + try: + test = next(self.pending) + except StopIteration: + self.output.put((None, None, None, None)) + return True + + try: + self.start_time = time.monotonic() + self.current_test = test + + retcode, stdout, stderr = run_test_in_subprocess(test, self.ns) + finally: + self.current_test = None + + stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) + return True + + if not result: + self.output.put((None, None, None, None)) + return True + + result = json.loads(result) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) + return False def run(self): - # A worker thread. try: - while True: - try: - test = next(self.pending) - except StopIteration: - self.output.put((None, None, None, None)) - return - retcode, stdout, stderr = run_tests_in_subprocess(test, - self.ns) - stdout, _, result = stdout.strip().rpartition("\n") - if retcode != 0: - result = (CHILD_ERROR, "Exit code %s" % retcode) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), - result)) - return - if not result: - self.output.put((None, None, None, None)) - return - result = json.loads(result) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), - result)) + stop = False + while not stop: + stop = self._runtest() except BaseException: self.output.put((None, None, None, None)) raise @@ -136,13 +163,33 @@ finished += 1 continue regrtest.accumulate_result(test, result) - regrtest.display_progress(test_index, test) + + # Display progress + text = test + ok, test_time = result + if (ok not in (CHILD_ERROR, INTERRUPTED) + and test_time >= PROGRESS_MIN_TIME): + text += ' (%.0f sec)' % test_time + running = [] + for worker in workers: + current_test = worker.current_test + if not current_test: + continue + dt = time.monotonic() - worker.start_time + if dt >= PROGRESS_MIN_TIME: + running.append('%s (%.0f sec)' % (current_test, dt)) + if running: + text += ' -- running: %s' % ', '.join(running) + regrtest.display_progress(test_index, text) + + # Copy stdout and stderr from the child process if stdout: print(stdout) if stderr: print(stderr, file=sys.stderr) sys.stdout.flush() sys.stderr.flush() + if result[0] == INTERRUPTED: raise KeyboardInterrupt if result[0] == CHILD_ERROR: @@ -152,5 +199,11 @@ except KeyboardInterrupt: regrtest.interrupted = True pending.interrupted = True + print() + + running = [worker.current_test for worker in workers] + running = list(filter(bool, running)) + if running: + print("Waiting for %s" % ', '.join(running)) for worker in workers: worker.join() -- Repository URL: https://hg.python.org/cpython [View Less]

1 0

cpython: Don't strip refcount in libregrtest/runtest_mp.py
by victor.stinner Sept. 29, 2015

Sept. 29, 2015

https://hg.python.org/cpython/rev/eddf85b65996 changeset: 98421:eddf85b65996 user: Victor Stinner <victor.stinner(a)gmail.com> date: Tue Sep 29 23:52:33 2015 +0200 summary: Don't strip refcount in libregrtest/runtest_mp.py Python doesn't display the refcount anymore by default. It only displays it when -X showrefcount command line option is used, which is not the case here. regrtest can be run with -X showrefcount, the option is not inherited by child processes. files: … [View More]

1 0

cpython: Issue #25220, libregrtest: Remove unused import
by victor.stinner Sept. 29, 2015

Sept. 29, 2015

https://hg.python.org/cpython/rev/2c53c8dcde3f changeset: 98420:2c53c8dcde3f user: Victor Stinner <victor.stinner(a)gmail.com> date: Tue Sep 29 23:50:19 2015 +0200 summary: Issue #25220, libregrtest: Remove unused import files: Lib/test/libregrtest/main.py | 4 ---- 1 files changed, 0 insertions(+), 4 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -19,10 … [View More]

1 0

cpython: Issue #25220: truncate some long lines in libregrtest/*.py
by victor.stinner Sept. 29, 2015

Sept. 29, 2015

https://hg.python.org/cpython/rev/e6b48bfd6d8e changeset: 98419:e6b48bfd6d8e user: Victor Stinner <victor.stinner(a)gmail.com> date: Tue Sep 29 23:43:33 2015 +0200 summary: Issue #25220: truncate some long lines in libregrtest/*.py files: Lib/test/libregrtest/main.py | 23 ++++++++++++----- Lib/test/libregrtest/runtest_mp.py | 12 ++++++--- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/… [View More]main.py --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -63,7 +63,8 @@ # the packages to prevent later imports to fail when the CWD is different. for module in sys.modules.values(): if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) for path in module.__path__] + module.__path__ = [os.path.abspath(path) + for path in module.__path__] if hasattr(module, '__file__'): module.__file__ = os.path.abspath(module.__file__) @@ -159,8 +160,8 @@ if self.ns.quiet: return fmt = "[{1:{0}}{2}/{3}] {4}" if self.bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - self.test_count_width, test_index, self.test_count, len(self.bad), test)) + print(fmt.format(self.test_count_width, test_index, + self.test_count, len(self.bad), test)) sys.stdout.flush() def setup_regrtest(self): @@ -252,7 +253,10 @@ self.ns.args = [] # For a partial run, we do not need to clutter the output. - if self.ns.verbose or self.ns.header or not (self.ns.quiet or self.ns.single or self.tests or self.ns.args): + if (self.ns.verbose + or self.ns.header + or not (self.ns.quiet or self.ns.single + or self.tests or self.ns.args)): # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) print("== ", platform.platform(aliased=True), @@ -283,7 +287,8 @@ try: del self.selected[:self.selected.index(self.ns.start)] except ValueError: - print("Couldn't find starting test (%s), using all tests" % self.ns.start) + print("Couldn't find starting test (%s), using all tests" + % self.ns.start) if self.ns.randomize: if self.ns.random_seed is None: @@ -297,12 +302,16 @@ # print a newline after ^C print() print("Test suite interrupted by signal SIGINT.") - omitted = set(self.selected) - set(self.good) - set(self.bad) - set(self.skipped) + executed = set(self.good) | set(self.bad) | set(self.skipped) + omitted = set(self.selected) - executed print(count(len(omitted), "test"), "omitted:") printlist(omitted) if self.good and not self.ns.quiet: - if not self.bad and not self.skipped and not self.interrupted and len(self.good) > 1: + if (not self.bad + and not self.skipped + and not self.interrupted + and len(self.good) > 1): print("All", end=' ') print(count(len(self.good), "test"), "OK.") diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -103,20 +103,23 @@ except StopIteration: self.output.put((None, None, None, None)) return - retcode, stdout, stderr = run_tests_in_subprocess(test, self.ns) + retcode, stdout, stderr = run_tests_in_subprocess(test, + self.ns) # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) stdout, _, result = stdout.strip().rpartition("\n") if retcode != 0: result = (CHILD_ERROR, "Exit code %s" % retcode) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) return if not result: self.output.put((None, None, None, None)) return result = json.loads(result) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) except BaseException: self.output.put((None, None, None, None)) raise @@ -149,7 +152,8 @@ if result[0] == INTERRUPTED: raise KeyboardInterrupt if result[0] == CHILD_ERROR: - raise Exception("Child error on {}: {}".format(test, result[1])) + msg = "Child error on {}: {}".format(test, result[1]) + raise Exception(msg) test_index += 1 except KeyboardInterrupt: regrtest.interrupted = True -- Repository URL: https://hg.python.org/cpython [View Less]

1 0