[Python-checkins] bpo-37957: Allow regrtest to receive a file with test (and subtests) to ignore (GH-16989)
Pablo Galindo
webhook-mailer at python.org
Tue Nov 19 18:46:58 EST 2019
https://github.com/python/cpython/commit/e0cd8aa70a3ce19c3d3712568940aa0cbd9aa97b
commit: e0cd8aa70a3ce19c3d3712568940aa0cbd9aa97b
branch: master
author: Pablo Galindo <Pablogsal at gmail.com>
committer: GitHub <noreply at github.com>
date: 2019-11-19T23:46:49Z
summary:
bpo-37957: Allow regrtest to receive a file with test (and subtests) to ignore (GH-16989)
When building Python in some uncommon platforms there are some known tests that will fail. Right now, the test suite has the ability to ignore entire tests using the -x option and to receive a filter file using the --matchfile filter. The problem with the --matchfile option is that it receives a file with patterns to accept and when you want to ignore a couple of tests and subtests, is too cumbersome to lists ALL tests that are not the ones that you want to accept and he problem with -x is that is not easy to ignore just a subtests that fail and the whole test needs to be ignored.
For these reasons, add a new option to allow to ignore a list of test and subtests for these situations.
files:
A Misc/NEWS.d/next/Tests/2019-10-30-00-01-43.bpo-37957.X1r78F.rst
M Lib/test/libregrtest/cmdline.py
M Lib/test/libregrtest/main.py
M Lib/test/libregrtest/runtest.py
M Lib/test/support/__init__.py
M Lib/test/test_regrtest.py
M Lib/test/test_support.py
diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
index c8fedc7ad329b..c0bb051bd0783 100644
--- a/Lib/test/libregrtest/cmdline.py
+++ b/Lib/test/libregrtest/cmdline.py
@@ -207,10 +207,17 @@ def _create_parser():
group.add_argument('-m', '--match', metavar='PAT',
dest='match_tests', action='append',
help='match test cases and methods with glob pattern PAT')
+ group.add_argument('-i', '--ignore', metavar='PAT',
+ dest='ignore_tests', action='append',
+ help='ignore test cases and methods with glob pattern PAT')
group.add_argument('--matchfile', metavar='FILENAME',
dest='match_filename',
help='similar to --match but get patterns from a '
'text file, one pattern per line')
+ group.add_argument('--ignorefile', metavar='FILENAME',
+ dest='ignore_filename',
+ help='similar to --matchfile but it receives patterns '
+ 'from text file to ignore')
group.add_argument('-G', '--failfast', action='store_true',
help='fail as soon as a test fails (only with -v or -W)')
group.add_argument('-u', '--use', metavar='RES1,RES2,...',
@@ -317,7 +324,8 @@ def _parse_args(args, **kwargs):
findleaks=1, use_resources=None, trace=False, coverdir='coverage',
runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
random_seed=None, use_mp=None, verbose3=False, forever=False,
- header=False, failfast=False, match_tests=None, pgo=False)
+ header=False, failfast=False, match_tests=None, ignore_tests=None,
+ pgo=False)
for k, v in kwargs.items():
if not hasattr(ns, k):
raise TypeError('%r is an invalid keyword argument '
@@ -395,6 +403,12 @@ def _parse_args(args, **kwargs):
with open(ns.match_filename) as fp:
for line in fp:
ns.match_tests.append(line.strip())
+ if ns.ignore_filename:
+ if ns.ignore_tests is None:
+ ns.ignore_tests = []
+ with open(ns.ignore_filename) as fp:
+ for line in fp:
+ ns.ignore_tests.append(line.strip())
if ns.forever:
# --forever implies --failfast
ns.failfast = True
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 76ad3359f2d7a..1de51b740a98c 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -287,7 +287,7 @@ def _list_cases(self, suite):
def list_cases(self):
support.verbose = False
- support.set_match_tests(self.ns.match_tests)
+ support.set_match_tests(self.ns.match_tests, self.ns.ignore_tests)
for test_name in self.selected:
abstest = get_abs_module(self.ns, test_name)
diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py
index eeb108bb44739..558f2099c66f5 100644
--- a/Lib/test/libregrtest/runtest.py
+++ b/Lib/test/libregrtest/runtest.py
@@ -123,7 +123,7 @@ def _runtest(ns, test_name):
start_time = time.perf_counter()
try:
- support.set_match_tests(ns.match_tests)
+ support.set_match_tests(ns.match_tests, ns.ignore_tests)
support.junit_xml_list = xml_list = [] if ns.xmlpath else None
if ns.failfast:
support.failfast = True
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 5ad32b8f4081c..7e1b30cc160ca 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -2096,7 +2096,9 @@ def _run_suite(suite):
# By default, don't filter tests
_match_test_func = None
-_match_test_patterns = None
+
+_accept_test_patterns = None
+_ignore_test_patterns = None
def match_test(test):
@@ -2112,18 +2114,45 @@ def _is_full_match_test(pattern):
# as a full test identifier.
# Example: 'test.test_os.FileTests.test_access'.
#
- # Reject patterns which contain fnmatch patterns: '*', '?', '[...]'
- # or '[!...]'. For example, reject 'test_access*'.
+ # ignore patterns which contain fnmatch patterns: '*', '?', '[...]'
+ # or '[!...]'. For example, ignore 'test_access*'.
return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern))
-def set_match_tests(patterns):
- global _match_test_func, _match_test_patterns
+def set_match_tests(accept_patterns=None, ignore_patterns=None):
+ global _match_test_func, _accept_test_patterns, _ignore_test_patterns
- if patterns == _match_test_patterns:
- # No change: no need to recompile patterns.
- return
+ if accept_patterns is None:
+ accept_patterns = ()
+ if ignore_patterns is None:
+ ignore_patterns = ()
+
+ accept_func = ignore_func = None
+
+ if accept_patterns != _accept_test_patterns:
+ accept_patterns, accept_func = _compile_match_function(accept_patterns)
+ if ignore_patterns != _ignore_test_patterns:
+ ignore_patterns, ignore_func = _compile_match_function(ignore_patterns)
+
+ # Create a copy since patterns can be mutable and so modified later
+ _accept_test_patterns = tuple(accept_patterns)
+ _ignore_test_patterns = tuple(ignore_patterns)
+
+ if accept_func is not None or ignore_func is not None:
+ def match_function(test_id):
+ accept = True
+ ignore = False
+ if accept_func:
+ accept = accept_func(test_id)
+ if ignore_func:
+ ignore = ignore_func(test_id)
+ return accept and not ignore
+
+ _match_test_func = match_function
+
+
+def _compile_match_function(patterns):
if not patterns:
func = None
# set_match_tests(None) behaves as set_match_tests(())
@@ -2151,10 +2180,7 @@ def match_test_regex(test_id):
func = match_test_regex
- # Create a copy since patterns can be mutable and so modified later
- _match_test_patterns = tuple(patterns)
- _match_test_func = func
-
+ return patterns, func
def run_unittest(*classes):
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 5df78861db9ce..93f8d44ec6938 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -155,6 +155,24 @@ def test_single(self):
self.assertTrue(ns.single)
self.checkError([opt, '-f', 'foo'], "don't go together")
+ def test_ignore(self):
+ for opt in '-i', '--ignore':
+ with self.subTest(opt=opt):
+ ns = libregrtest._parse_args([opt, 'pattern'])
+ self.assertEqual(ns.ignore_tests, ['pattern'])
+ self.checkError([opt], 'expected one argument')
+
+ self.addCleanup(support.unlink, support.TESTFN)
+ with open(support.TESTFN, "w") as fp:
+ print('matchfile1', file=fp)
+ print('matchfile2', file=fp)
+
+ filename = os.path.abspath(support.TESTFN)
+ ns = libregrtest._parse_args(['-m', 'match',
+ '--ignorefile', filename])
+ self.assertEqual(ns.ignore_tests,
+ ['matchfile1', 'matchfile2'])
+
def test_match(self):
for opt in '-m', '--match':
with self.subTest(opt=opt):
@@ -961,6 +979,42 @@ def parse_methods(self, output):
regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE)
return [match.group(1) for match in regex.finditer(output)]
+ def test_ignorefile(self):
+ code = textwrap.dedent("""
+ import unittest
+
+ class Tests(unittest.TestCase):
+ def test_method1(self):
+ pass
+ def test_method2(self):
+ pass
+ def test_method3(self):
+ pass
+ def test_method4(self):
+ pass
+ """)
+ all_methods = ['test_method1', 'test_method2',
+ 'test_method3', 'test_method4']
+ testname = self.create_test(code=code)
+
+ # only run a subset
+ filename = support.TESTFN
+ self.addCleanup(support.unlink, filename)
+
+ subset = [
+ # only ignore the method name
+ 'test_method1',
+ # ignore the full identifier
+ '%s.Tests.test_method3' % testname]
+ with open(filename, "w") as fp:
+ for name in subset:
+ print(name, file=fp)
+
+ output = self.run_tests("-v", "--ignorefile", filename, testname)
+ methods = self.parse_methods(output)
+ subset = ['test_method2', 'test_method4']
+ self.assertEqual(methods, subset)
+
def test_matchfile(self):
code = textwrap.dedent("""
import unittest
diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
index 8f0746aed8299..e3ce670b8437f 100644
--- a/Lib/test/test_support.py
+++ b/Lib/test/test_support.py
@@ -527,6 +527,7 @@ def id(self):
test_access = Test('test.test_os.FileTests.test_access')
test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir')
+ # Test acceptance
with support.swap_attr(support, '_match_test_func', None):
# match all
support.set_match_tests([])
@@ -534,45 +535,92 @@ def id(self):
self.assertTrue(support.match_test(test_chdir))
# match all using None
- support.set_match_tests(None)
+ support.set_match_tests(None, None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
# match the full test identifier
- support.set_match_tests([test_access.id()])
+ support.set_match_tests([test_access.id()], None)
self.assertTrue(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))
# match the module name
- support.set_match_tests(['test_os'])
+ support.set_match_tests(['test_os'], None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
# Test '*' pattern
- support.set_match_tests(['test_*'])
+ support.set_match_tests(['test_*'], None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
# Test case sensitivity
- support.set_match_tests(['filetests'])
+ support.set_match_tests(['filetests'], None)
self.assertFalse(support.match_test(test_access))
- support.set_match_tests(['FileTests'])
+ support.set_match_tests(['FileTests'], None)
self.assertTrue(support.match_test(test_access))
# Test pattern containing '.' and a '*' metacharacter
- support.set_match_tests(['*test_os.*.test_*'])
+ support.set_match_tests(['*test_os.*.test_*'], None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
# Multiple patterns
- support.set_match_tests([test_access.id(), test_chdir.id()])
+ support.set_match_tests([test_access.id(), test_chdir.id()], None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
- support.set_match_tests(['test_access', 'DONTMATCH'])
+ support.set_match_tests(['test_access', 'DONTMATCH'], None)
self.assertTrue(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))
+ # Test rejection
+ with support.swap_attr(support, '_match_test_func', None):
+ # match all
+ support.set_match_tests(ignore_patterns=[])
+ self.assertTrue(support.match_test(test_access))
+ self.assertTrue(support.match_test(test_chdir))
+
+ # match all using None
+ support.set_match_tests(None, None)
+ self.assertTrue(support.match_test(test_access))
+ self.assertTrue(support.match_test(test_chdir))
+
+ # match the full test identifier
+ support.set_match_tests(None, [test_access.id()])
+ self.assertFalse(support.match_test(test_access))
+ self.assertTrue(support.match_test(test_chdir))
+
+ # match the module name
+ support.set_match_tests(None, ['test_os'])
+ self.assertFalse(support.match_test(test_access))
+ self.assertFalse(support.match_test(test_chdir))
+
+ # Test '*' pattern
+ support.set_match_tests(None, ['test_*'])
+ self.assertFalse(support.match_test(test_access))
+ self.assertFalse(support.match_test(test_chdir))
+
+ # Test case sensitivity
+ support.set_match_tests(None, ['filetests'])
+ self.assertTrue(support.match_test(test_access))
+ support.set_match_tests(None, ['FileTests'])
+ self.assertFalse(support.match_test(test_access))
+
+ # Test pattern containing '.' and a '*' metacharacter
+ support.set_match_tests(None, ['*test_os.*.test_*'])
+ self.assertFalse(support.match_test(test_access))
+ self.assertFalse(support.match_test(test_chdir))
+
+ # Multiple patterns
+ support.set_match_tests(None, [test_access.id(), test_chdir.id()])
+ self.assertFalse(support.match_test(test_access))
+ self.assertFalse(support.match_test(test_chdir))
+
+ support.set_match_tests(None, ['test_access', 'DONTMATCH'])
+ self.assertFalse(support.match_test(test_access))
+ self.assertTrue(support.match_test(test_chdir))
+
def test_fd_count(self):
# We cannot test the absolute value of fd_count(): on old Linux
# kernel or glibc versions, os.urandom() keeps a FD open on
diff --git a/Misc/NEWS.d/next/Tests/2019-10-30-00-01-43.bpo-37957.X1r78F.rst b/Misc/NEWS.d/next/Tests/2019-10-30-00-01-43.bpo-37957.X1r78F.rst
new file mode 100644
index 0000000000000..75e186ef33e07
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-10-30-00-01-43.bpo-37957.X1r78F.rst
@@ -0,0 +1,3 @@
+test.regrtest now can receive a list of test patterns to ignore (using the
+-i/--ignore argument) or a file with a list of patterns to ignore (using the
+--ignore-file argument). Patch by Pablo Galindo.
More information about the Python-checkins
mailing list