[Python-checkins] gh-98903: Test suite fails with exit code 4 if no tests ran (#98904)
vstinner
webhook-mailer at python.org
Wed Nov 2 10:38:23 EDT 2022
https://github.com/python/cpython/commit/c76db37c0d23174cbffd6fa978d39693890ef020
commit: c76db37c0d23174cbffd6fa978d39693890ef020
branch: main
author: Victor Stinner <vstinner at python.org>
committer: vstinner <vstinner at python.org>
date: 2022-11-02T15:37:40+01:00
summary:
gh-98903: Test suite fails with exit code 4 if no tests ran (#98904)
The Python test suite now fails wit exit code 4 if no tests ran. It
should help detecting typos in test names and test methods.
* Add "EXITCODE_" constants to Lib/test/libregrtest/main.py.
* Fix a typo: "NO TEST RUN" becomes "NO TESTS RAN"
files:
A Misc/NEWS.d/next/Tests/2022-10-31-14-47-49.gh-issue-98903.7KinCV.rst
M Lib/test/libregrtest/main.py
M Lib/test/test_regrtest.py
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 655e4d2e56f8..3eeef029b22d 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -28,6 +28,11 @@
# Must be smaller than buildbot "1200 seconds without output" limit.
EXIT_TIMEOUT = 120.0
+EXITCODE_BAD_TEST = 2
+EXITCODE_INTERRUPTED = 130
+EXITCODE_ENV_CHANGED = 3
+EXITCODE_NO_TESTS_RAN = 4
+
class Regrtest:
"""Execute a test suite.
@@ -493,15 +498,18 @@ def display_header(self):
print("== encodings: locale=%s, FS=%s"
% (locale.getencoding(), sys.getfilesystemencoding()))
+ def no_tests_run(self):
+ return not any((self.good, self.bad, self.skipped, self.interrupted,
+ self.environment_changed))
+
def get_tests_result(self):
result = []
if self.bad:
result.append("FAILURE")
elif self.ns.fail_env_changed and self.environment_changed:
result.append("ENV CHANGED")
- elif not any((self.good, self.bad, self.skipped, self.interrupted,
- self.environment_changed)):
- result.append("NO TEST RUN")
+ elif self.no_tests_run():
+ result.append("NO TESTS RAN")
if self.interrupted:
result.append("INTERRUPTED")
@@ -750,11 +758,13 @@ def _main(self, tests, kwargs):
self.save_xml_result()
if self.bad:
- sys.exit(2)
+ sys.exit(EXITCODE_BAD_TEST)
if self.interrupted:
- sys.exit(130)
+ sys.exit(EXITCODE_INTERRUPTED)
if self.ns.fail_env_changed and self.environment_changed:
- sys.exit(3)
+ sys.exit(EXITCODE_ENV_CHANGED)
+ if self.no_tests_run():
+ sys.exit(EXITCODE_NO_TESTS_RAN)
sys.exit(0)
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index a36d18488a5e..baae4efc2ad7 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -28,6 +28,11 @@
ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'
+EXITCODE_BAD_TEST = 2
+EXITCODE_ENV_CHANGED = 3
+EXITCODE_NO_TESTS_RAN = 4
+EXITCODE_INTERRUPTED = 130
+
TEST_INTERRUPTED = textwrap.dedent("""
from signal import SIGINT, raise_signal
try:
@@ -497,7 +502,7 @@ def list_regex(line_format, tests):
result.append('INTERRUPTED')
if not any((good, result, failed, interrupted, skipped,
env_changed, fail_env_changed)):
- result.append("NO TEST RUN")
+ result.append("NO TESTS RAN")
elif not result:
result.append('SUCCESS')
result = ', '.join(result)
@@ -707,7 +712,7 @@ def test_failing(self):
test_failing = self.create_test('failing', code=code)
tests = [test_ok, test_failing]
- output = self.run_tests(*tests, exitcode=2)
+ output = self.run_tests(*tests, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, tests, failed=test_failing)
def test_resources(self):
@@ -748,13 +753,14 @@ def test_random(self):
test = self.create_test('random', code)
# first run to get the output with the random seed
- output = self.run_tests('-r', test)
+ output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN)
randseed = self.parse_random_seed(output)
match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
test_random = int(match.group(1))
# try to reproduce with the random seed
- output = self.run_tests('-r', '--randseed=%s' % randseed, test)
+ output = self.run_tests('-r', '--randseed=%s' % randseed, test,
+ exitcode=EXITCODE_NO_TESTS_RAN)
randseed2 = self.parse_random_seed(output)
self.assertEqual(randseed2, randseed)
@@ -813,7 +819,7 @@ def test_fromfile(self):
def test_interrupted(self):
code = TEST_INTERRUPTED
test = self.create_test('sigint', code=code)
- output = self.run_tests(test, exitcode=130)
+ output = self.run_tests(test, exitcode=EXITCODE_INTERRUPTED)
self.check_executed_tests(output, test, omitted=test,
interrupted=True)
@@ -838,7 +844,7 @@ def test_slowest_interrupted(self):
args = ("--slowest", "-j2", test)
else:
args = ("--slowest", test)
- output = self.run_tests(*args, exitcode=130)
+ output = self.run_tests(*args, exitcode=EXITCODE_INTERRUPTED)
self.check_executed_tests(output, test,
omitted=test, interrupted=True)
@@ -878,7 +884,7 @@ def test_run(self):
builtins.__dict__['RUN'] = 1
""")
test = self.create_test('forever', code=code)
- output = self.run_tests('--forever', test, exitcode=2)
+ output = self.run_tests('--forever', test, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, [test]*3, failed=test)
def check_leak(self, code, what):
@@ -887,7 +893,7 @@ def check_leak(self, code, what):
filename = 'reflog.txt'
self.addCleanup(os_helper.unlink, filename)
output = self.run_tests('--huntrleaks', '3:3:', test,
- exitcode=2,
+ exitcode=EXITCODE_BAD_TEST,
stderr=subprocess.STDOUT)
self.check_executed_tests(output, [test], failed=test)
@@ -969,7 +975,7 @@ def test_crashed(self):
crash_test = self.create_test(name="crash", code=code)
tests = [crash_test]
- output = self.run_tests("-j2", *tests, exitcode=2)
+ output = self.run_tests("-j2", *tests, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, tests, failed=crash_test,
randomize=True)
@@ -1069,7 +1075,8 @@ def test_env_changed(self):
self.check_executed_tests(output, [testname], env_changed=testname)
# fail with --fail-env-changed
- output = self.run_tests("--fail-env-changed", testname, exitcode=3)
+ output = self.run_tests("--fail-env-changed", testname,
+ exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname], env_changed=testname,
fail_env_changed=True)
@@ -1088,7 +1095,7 @@ def test_fail_always(self):
""")
testname = self.create_test(code=code)
- output = self.run_tests("-w", testname, exitcode=2)
+ output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, [testname],
failed=testname, rerun={testname: "test_fail_always"})
@@ -1123,7 +1130,8 @@ def test_bug(self):
""")
testname = self.create_test(code=code)
- output = self.run_tests(testname, "-m", "nosuchtest", exitcode=0)
+ output = self.run_tests(testname, "-m", "nosuchtest",
+ exitcode=EXITCODE_NO_TESTS_RAN)
self.check_executed_tests(output, [testname], no_test_ran=testname)
def test_no_tests_ran_skip(self):
@@ -1136,7 +1144,7 @@ def test_skipped(self):
""")
testname = self.create_test(code=code)
- output = self.run_tests(testname, exitcode=0)
+ output = self.run_tests(testname)
self.check_executed_tests(output, [testname])
def test_no_tests_ran_multiple_tests_nonexistent(self):
@@ -1150,7 +1158,8 @@ def test_bug(self):
testname = self.create_test(code=code)
testname2 = self.create_test(code=code)
- output = self.run_tests(testname, testname2, "-m", "nosuchtest", exitcode=0)
+ output = self.run_tests(testname, testname2, "-m", "nosuchtest",
+ exitcode=EXITCODE_NO_TESTS_RAN)
self.check_executed_tests(output, [testname, testname2],
no_test_ran=[testname, testname2])
@@ -1198,7 +1207,8 @@ def test_garbage(self):
""")
testname = self.create_test(code=code)
- output = self.run_tests("--fail-env-changed", testname, exitcode=3)
+ output = self.run_tests("--fail-env-changed", testname,
+ exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname],
env_changed=[testname],
fail_env_changed=True)
@@ -1224,7 +1234,8 @@ def test_sleep(self):
""")
testname = self.create_test(code=code)
- output = self.run_tests("-j2", "--timeout=1.0", testname, exitcode=2)
+ output = self.run_tests("-j2", "--timeout=1.0", testname,
+ exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, [testname],
failed=testname)
self.assertRegex(output,
@@ -1256,7 +1267,8 @@ def test_unraisable_exc(self):
""")
testname = self.create_test(code=code)
- output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3)
+ output = self.run_tests("--fail-env-changed", "-v", testname,
+ exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname],
env_changed=[testname],
fail_env_changed=True)
@@ -1287,7 +1299,8 @@ def test_threading_excepthook(self):
""")
testname = self.create_test(code=code)
- output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3)
+ output = self.run_tests("--fail-env-changed", "-v", testname,
+ exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname],
env_changed=[testname],
fail_env_changed=True)
@@ -1328,7 +1341,7 @@ def test_print_warning(self):
for option in ("-v", "-W"):
with self.subTest(option=option):
cmd = ["--fail-env-changed", option, testname]
- output = self.run_tests(*cmd, exitcode=3)
+ output = self.run_tests(*cmd, exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname],
env_changed=[testname],
fail_env_changed=True)
@@ -1373,7 +1386,8 @@ def test_leak_tmp_file(self):
""")
testnames = [self.create_test(code=code) for _ in range(3)]
- output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames, exitcode=3)
+ output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames,
+ exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, testnames,
env_changed=testnames,
fail_env_changed=True,
diff --git a/Misc/NEWS.d/next/Tests/2022-10-31-14-47-49.gh-issue-98903.7KinCV.rst b/Misc/NEWS.d/next/Tests/2022-10-31-14-47-49.gh-issue-98903.7KinCV.rst
new file mode 100644
index 000000000000..65636ab0683f
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2022-10-31-14-47-49.gh-issue-98903.7KinCV.rst
@@ -0,0 +1,2 @@
+The Python test suite now fails wit exit code 4 if no tests ran. It should
+help detecting typos in test names and test methods.
More information about the Python-checkins
mailing list