[Python-Dev] Avoiding cascading test failures

23 Aug 2007

      When I was fixing tests failing in the py3k branch, I found the number
duplicate failures annoying. Often, a single bug, in an important
method or function, caused a large number of testcase to fail. So, I
thought of a simple mechanism for avoiding such cascading failures.

My solution is to add a notion of dependency to testcases. A typical
usage would look like this:

    @depends('test_getvalue')
    def test_writelines(self):
        ...
        memio.writelines([buf] * 100)
        self.assertEqual(memio.getvalue(), buf * 100)
        ...

Here, running the test is pointless if test_getvalue fails. So by
making test_writelines depends on the success of test_getvalue, we can
ensure that the report won't be polluted with unnecessary failures.

Also, I believe this feature will lead to more orthogonal tests, since
it encourages the user to write smaller test with less dependencies.

I wrote an example implementation (included below) as a proof of
concept. If the idea get enough support, I will implement it and add
it to the unittest module.

-- Alexandre

class CycleError(Exception):
    pass

class TestCase:

    def __init__(self):
        self.graph = {}
        tests = [x for x in dir(self) if x.startswith('test')]
        for testname in tests:
            test = getattr(self, testname)
            if hasattr(test, 'deps'):
                self.graph[testname] = test.deps
            else:
                self.graph[testname] = set()

    def run(self):
        graph = self.graph
        toskip = set()
        msgs = []
        while graph:
            # find tests without any pending dependencies
            source = [test for test, deps in graph.items() if not deps]
            if not source:
                raise CycleError
            for testname in source:
                if testname in toskip:
                    msgs.append("%s... skipped" % testname)
                    resolvedeps(graph, testname)
                    del graph[testname]
                    continue
                test = getattr(self, testname)
                try:
                    test()
                except AssertionError:
                    toskip.update(getrevdeps(graph, testname))
                    msgs.append("%s... failed" % testname)
                except:
                    toskip.update(getrevdeps(graph, testname))
                    msgs.append("%s... error" % testname)
                else:
                    msgs.append("%s... ok" % testname)
                finally:
                    resolvedeps(graph, testname)
                    del graph[testname]
        for msg in sorted(msgs):
            print(msg)

def getrevdeps(graph, testname):
    """Return the reverse depencencies of a test"""
    rdeps = set()
    for x in graph:
        if testname in graph[x]:
            rdeps.add(x)
    if rdeps:
        # propagate depencencies recursively
        for x in rdeps.copy():
            rdeps.update(getrevdeps(graph, x))
    return rdeps

def resolvedeps(graph, testname):
    for test in graph:
        if testname in graph[test]:
            graph[test].remove(testname)

def depends(*args):
    def decorator(test):
        if hasattr(test, 'deps'):
            test.deps.update(args)
        else:
            test.deps = set(args)
        return test
    return decorator

class MyTest(TestCase):

    @depends('test_foo')
    def test_nah(self):
        pass

    @depends('test_bar', 'test_baz')
    def test_foo(self):
        pass

    @depends('test_tin')
    def test_bar(self):
        self.fail()

    def test_baz(self):
        self.error()

    def test_tin(self):
        pass

    def error(self):
        raise ValueError

    def fail(self):
        raise AssertionError

if __name__ == '__main__':
    t = MyTest()
    t.run()

[Python-Dev] Avoiding cascading test failures

Alexandre Vassalotti