[Python-checkins] bpo-25894: Always report skipped and failed subtests separately (GH-28082)
ambv
webhook-mailer at python.org
Fri Sep 10 11:55:09 EDT 2021
https://github.com/python/cpython/commit/f0f29f328d8b4568e8c0d4c55c7d120d96f80911
commit: f0f29f328d8b4568e8c0d4c55c7d120d96f80911
branch: main
author: Serhiy Storchaka <storchaka at gmail.com>
committer: ambv <lukasz at langa.pl>
date: 2021-09-10T17:55:05+02:00
summary:
bpo-25894: Always report skipped and failed subtests separately (GH-28082)
* In default mode output separate characters for skipped and failed subtests.
* In verbose mode output separate lines (including description) for skipped
and failed subtests.
* In verbose mode output test description for errors in test cleanup.
files:
A Misc/NEWS.d/next/Library/2021-09-05-13-15-08.bpo-25894.zjbi2f.rst
M Lib/unittest/runner.py
M Lib/unittest/test/test_result.py
diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py
index 45e7e4c0458d4..f316d316601bc 100644
--- a/Lib/unittest/runner.py
+++ b/Lib/unittest/runner.py
@@ -5,6 +5,7 @@
import warnings
from . import result
+from .case import _SubTest
from .signals import registerResult
__unittest = True
@@ -40,6 +41,7 @@ def __init__(self, stream, descriptions, verbosity):
self.showAll = verbosity > 1
self.dots = verbosity == 1
self.descriptions = descriptions
+ self._newline = True
def getDescription(self, test):
doc_first_line = test.shortDescription()
@@ -54,11 +56,39 @@ def startTest(self, test):
self.stream.write(self.getDescription(test))
self.stream.write(" ... ")
self.stream.flush()
+ self._newline = False
+
+ def _write_status(self, test, status):
+ is_subtest = isinstance(test, _SubTest)
+ if is_subtest or self._newline:
+ if not self._newline:
+ self.stream.writeln()
+ if is_subtest:
+ self.stream.write(" ")
+ self.stream.write(self.getDescription(test))
+ self.stream.write(" ... ")
+ self.stream.writeln(status)
+ self._newline = True
+
+ def addSubTest(self, test, subtest, err):
+ if err is not None:
+ if self.showAll:
+ if issubclass(err[0], subtest.failureException):
+ self._write_status(subtest, "FAIL")
+ else:
+ self._write_status(subtest, "ERROR")
+ elif self.dots:
+ if issubclass(err[0], subtest.failureException):
+ self.stream.write('F')
+ else:
+ self.stream.write('E')
+ self.stream.flush()
+ super(TextTestResult, self).addSubTest(test, subtest, err)
def addSuccess(self, test):
super(TextTestResult, self).addSuccess(test)
if self.showAll:
- self.stream.writeln("ok")
+ self._write_status(test, "ok")
elif self.dots:
self.stream.write('.')
self.stream.flush()
@@ -66,7 +96,7 @@ def addSuccess(self, test):
def addError(self, test, err):
super(TextTestResult, self).addError(test, err)
if self.showAll:
- self.stream.writeln("ERROR")
+ self._write_status(test, "ERROR")
elif self.dots:
self.stream.write('E')
self.stream.flush()
@@ -74,7 +104,7 @@ def addError(self, test, err):
def addFailure(self, test, err):
super(TextTestResult, self).addFailure(test, err)
if self.showAll:
- self.stream.writeln("FAIL")
+ self._write_status(test, "FAIL")
elif self.dots:
self.stream.write('F')
self.stream.flush()
@@ -82,7 +112,7 @@ def addFailure(self, test, err):
def addSkip(self, test, reason):
super(TextTestResult, self).addSkip(test, reason)
if self.showAll:
- self.stream.writeln("skipped {0!r}".format(reason))
+ self._write_status(test, "skipped {0!r}".format(reason))
elif self.dots:
self.stream.write("s")
self.stream.flush()
diff --git a/Lib/unittest/test/test_result.py b/Lib/unittest/test/test_result.py
index d6efc7ef0662a..3b9da127764d1 100644
--- a/Lib/unittest/test/test_result.py
+++ b/Lib/unittest/test/test_result.py
@@ -305,12 +305,51 @@ def test_1(self):
self.assertIs(test_case, subtest)
self.assertIn("some recognizable failure", formatted_exc)
+ def testStackFrameTrimming(self):
+ class Frame(object):
+ class tb_frame(object):
+ f_globals = {}
+ result = unittest.TestResult()
+ self.assertFalse(result._is_relevant_tb_level(Frame))
+
+ Frame.tb_frame.f_globals['__unittest'] = True
+ self.assertTrue(result._is_relevant_tb_level(Frame))
+
+ def testFailFast(self):
+ result = unittest.TestResult()
+ result._exc_info_to_string = lambda *_: ''
+ result.failfast = True
+ result.addError(None, None)
+ self.assertTrue(result.shouldStop)
+
+ result = unittest.TestResult()
+ result._exc_info_to_string = lambda *_: ''
+ result.failfast = True
+ result.addFailure(None, None)
+ self.assertTrue(result.shouldStop)
+
+ result = unittest.TestResult()
+ result._exc_info_to_string = lambda *_: ''
+ result.failfast = True
+ result.addUnexpectedSuccess(None)
+ self.assertTrue(result.shouldStop)
+
+ def testFailFastSetByRunner(self):
+ runner = unittest.TextTestRunner(stream=io.StringIO(), failfast=True)
+ def test(result):
+ self.assertTrue(result.failfast)
+ result = runner.run(test)
+
+
+class Test_TextTestResult(unittest.TestCase):
+ maxDiff = None
+
def testGetDescriptionWithoutDocstring(self):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self),
'testGetDescriptionWithoutDocstring (' + __name__ +
- '.Test_TestResult)')
+ '.Test_TextTestResult)')
def testGetSubTestDescriptionWithoutDocstring(self):
with self.subTest(foo=1, bar=2):
@@ -318,13 +357,13 @@ def testGetSubTestDescriptionWithoutDocstring(self):
self.assertEqual(
result.getDescription(self._subtest),
'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
- '.Test_TestResult) (foo=1, bar=2)')
+ '.Test_TextTestResult) (foo=1, bar=2)')
with self.subTest('some message'):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
- '.Test_TestResult) [some message]')
+ '.Test_TextTestResult) [some message]')
def testGetSubTestDescriptionWithoutDocstringAndParams(self):
with self.subTest():
@@ -332,10 +371,10 @@ def testGetSubTestDescriptionWithoutDocstringAndParams(self):
self.assertEqual(
result.getDescription(self._subtest),
'testGetSubTestDescriptionWithoutDocstringAndParams '
- '(' + __name__ + '.Test_TestResult) (<subtest>)')
+ '(' + __name__ + '.Test_TextTestResult) (<subtest>)')
def testGetSubTestDescriptionForFalsyValues(self):
- expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TestResult) [%s]'
+ expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TextTestResult) [%s]'
result = unittest.TextTestResult(None, True, 1)
for arg in [0, None, []]:
with self.subTest(arg):
@@ -351,7 +390,7 @@ def testGetNestedSubTestDescriptionWithoutDocstring(self):
self.assertEqual(
result.getDescription(self._subtest),
'testGetNestedSubTestDescriptionWithoutDocstring '
- '(' + __name__ + '.Test_TestResult) (baz=2, bar=3, foo=1)')
+ '(' + __name__ + '.Test_TextTestResult) (baz=2, bar=3, foo=1)')
def testGetDuplicatedNestedSubTestDescriptionWithoutDocstring(self):
with self.subTest(foo=1, bar=2):
@@ -360,7 +399,7 @@ def testGetDuplicatedNestedSubTestDescriptionWithoutDocstring(self):
self.assertEqual(
result.getDescription(self._subtest),
'testGetDuplicatedNestedSubTestDescriptionWithoutDocstring '
- '(' + __name__ + '.Test_TestResult) (baz=3, bar=4, foo=1)')
+ '(' + __name__ + '.Test_TextTestResult) (baz=3, bar=4, foo=1)')
@unittest.skipIf(sys.flags.optimize >= 2,
"Docstrings are omitted with -O2 and above")
@@ -370,7 +409,7 @@ def testGetDescriptionWithOneLineDocstring(self):
self.assertEqual(
result.getDescription(self),
('testGetDescriptionWithOneLineDocstring '
- '(' + __name__ + '.Test_TestResult)\n'
+ '(' + __name__ + '.Test_TextTestResult)\n'
'Tests getDescription() for a method with a docstring.'))
@unittest.skipIf(sys.flags.optimize >= 2,
@@ -382,7 +421,7 @@ def testGetSubTestDescriptionWithOneLineDocstring(self):
self.assertEqual(
result.getDescription(self._subtest),
('testGetSubTestDescriptionWithOneLineDocstring '
- '(' + __name__ + '.Test_TestResult) (foo=1, bar=2)\n'
+ '(' + __name__ + '.Test_TextTestResult) (foo=1, bar=2)\n'
'Tests getDescription() for a method with a docstring.'))
@unittest.skipIf(sys.flags.optimize >= 2,
@@ -395,7 +434,7 @@ def testGetDescriptionWithMultiLineDocstring(self):
self.assertEqual(
result.getDescription(self),
('testGetDescriptionWithMultiLineDocstring '
- '(' + __name__ + '.Test_TestResult)\n'
+ '(' + __name__ + '.Test_TextTestResult)\n'
'Tests getDescription() for a method with a longer '
'docstring.'))
@@ -410,44 +449,111 @@ def testGetSubTestDescriptionWithMultiLineDocstring(self):
self.assertEqual(
result.getDescription(self._subtest),
('testGetSubTestDescriptionWithMultiLineDocstring '
- '(' + __name__ + '.Test_TestResult) (foo=1, bar=2)\n'
+ '(' + __name__ + '.Test_TextTestResult) (foo=1, bar=2)\n'
'Tests getDescription() for a method with a longer '
'docstring.'))
- def testStackFrameTrimming(self):
- class Frame(object):
- class tb_frame(object):
- f_globals = {}
- result = unittest.TestResult()
- self.assertFalse(result._is_relevant_tb_level(Frame))
-
- Frame.tb_frame.f_globals['__unittest'] = True
- self.assertTrue(result._is_relevant_tb_level(Frame))
-
- def testFailFast(self):
- result = unittest.TestResult()
- result._exc_info_to_string = lambda *_: ''
- result.failfast = True
- result.addError(None, None)
- self.assertTrue(result.shouldStop)
-
- result = unittest.TestResult()
- result._exc_info_to_string = lambda *_: ''
- result.failfast = True
- result.addFailure(None, None)
- self.assertTrue(result.shouldStop)
-
- result = unittest.TestResult()
- result._exc_info_to_string = lambda *_: ''
- result.failfast = True
- result.addUnexpectedSuccess(None)
- self.assertTrue(result.shouldStop)
-
- def testFailFastSetByRunner(self):
- runner = unittest.TextTestRunner(stream=io.StringIO(), failfast=True)
- def test(result):
- self.assertTrue(result.failfast)
- result = runner.run(test)
+ class Test(unittest.TestCase):
+ def testSuccess(self):
+ pass
+ def testSkip(self):
+ self.skipTest('skip')
+ def testFail(self):
+ self.fail('fail')
+ def testError(self):
+ raise Exception('error')
+ def testSubTestSuccess(self):
+ with self.subTest('one', a=1):
+ pass
+ with self.subTest('two', b=2):
+ pass
+ def testSubTestMixed(self):
+ with self.subTest('success', a=1):
+ pass
+ with self.subTest('skip', b=2):
+ self.skipTest('skip')
+ with self.subTest('fail', c=3):
+ self.fail('fail')
+ with self.subTest('error', d=4):
+ raise Exception('error')
+
+ tearDownError = None
+ def tearDown(self):
+ if self.tearDownError is not None:
+ raise self.tearDownError
+
+ def _run_test(self, test_name, verbosity, tearDownError=None):
+ stream = io.StringIO()
+ stream = unittest.runner._WritelnDecorator(stream)
+ result = unittest.TextTestResult(stream, True, verbosity)
+ test = self.Test(test_name)
+ test.tearDownError = tearDownError
+ test.run(result)
+ return stream.getvalue()
+
+ def testDotsOutput(self):
+ self.assertEqual(self._run_test('testSuccess', 1), '.')
+ self.assertEqual(self._run_test('testSkip', 1), 's')
+ self.assertEqual(self._run_test('testFail', 1), 'F')
+ self.assertEqual(self._run_test('testError', 1), 'E')
+
+ def testLongOutput(self):
+ classname = f'{__name__}.{self.Test.__qualname__}'
+ self.assertEqual(self._run_test('testSuccess', 2),
+ f'testSuccess ({classname}) ... ok\n')
+ self.assertEqual(self._run_test('testSkip', 2),
+ f"testSkip ({classname}) ... skipped 'skip'\n")
+ self.assertEqual(self._run_test('testFail', 2),
+ f'testFail ({classname}) ... FAIL\n')
+ self.assertEqual(self._run_test('testError', 2),
+ f'testError ({classname}) ... ERROR\n')
+
+ def testDotsOutputSubTestSuccess(self):
+ self.assertEqual(self._run_test('testSubTestSuccess', 1), '.')
+
+ def testLongOutputSubTestSuccess(self):
+ classname = f'{__name__}.{self.Test.__qualname__}'
+ self.assertEqual(self._run_test('testSubTestSuccess', 2),
+ f'testSubTestSuccess ({classname}) ... ok\n')
+
+ def testDotsOutputSubTestMixed(self):
+ self.assertEqual(self._run_test('testSubTestMixed', 1), 'sFE')
+
+ def testLongOutputSubTestMixed(self):
+ classname = f'{__name__}.{self.Test.__qualname__}'
+ self.assertEqual(self._run_test('testSubTestMixed', 2),
+ f'testSubTestMixed ({classname}) ... \n'
+ f" testSubTestMixed ({classname}) [skip] (b=2) ... skipped 'skip'\n"
+ f' testSubTestMixed ({classname}) [fail] (c=3) ... FAIL\n'
+ f' testSubTestMixed ({classname}) [error] (d=4) ... ERROR\n')
+
+ def testDotsOutputTearDownFail(self):
+ out = self._run_test('testSuccess', 1, AssertionError('fail'))
+ self.assertEqual(out, 'F')
+ out = self._run_test('testError', 1, AssertionError('fail'))
+ self.assertEqual(out, 'EF')
+ out = self._run_test('testFail', 1, Exception('error'))
+ self.assertEqual(out, 'FE')
+ out = self._run_test('testSkip', 1, AssertionError('fail'))
+ self.assertEqual(out, 'sF')
+
+ def testLongOutputTearDownFail(self):
+ classname = f'{__name__}.{self.Test.__qualname__}'
+ out = self._run_test('testSuccess', 2, AssertionError('fail'))
+ self.assertEqual(out,
+ f'testSuccess ({classname}) ... FAIL\n')
+ out = self._run_test('testError', 2, AssertionError('fail'))
+ self.assertEqual(out,
+ f'testError ({classname}) ... ERROR\n'
+ f'testError ({classname}) ... FAIL\n')
+ out = self._run_test('testFail', 2, Exception('error'))
+ self.assertEqual(out,
+ f'testFail ({classname}) ... FAIL\n'
+ f'testFail ({classname}) ... ERROR\n')
+ out = self._run_test('testSkip', 2, AssertionError('fail'))
+ self.assertEqual(out,
+ f"testSkip ({classname}) ... skipped 'skip'\n"
+ f'testSkip ({classname}) ... FAIL\n')
classDict = dict(unittest.TestResult.__dict__)
diff --git a/Misc/NEWS.d/next/Library/2021-09-05-13-15-08.bpo-25894.zjbi2f.rst b/Misc/NEWS.d/next/Library/2021-09-05-13-15-08.bpo-25894.zjbi2f.rst
new file mode 100644
index 0000000000000..b0a036fae6cfa
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-09-05-13-15-08.bpo-25894.zjbi2f.rst
@@ -0,0 +1,4 @@
+:mod:`unittest` now always reports skipped and failed subtests separately:
+separate characters in default mode and separate lines in verbose mode. Also
+the test description is now output for errors in test method, class and
+module cleanups.
More information about the Python-checkins
mailing list