Extreme Python Testing

John Dell'Aquila dellaq2 at pdq.net
Tue Jun 8 00:20:14 EDT 1999

I admire the Smalltalk "Extreme Programming" methodology enough that
I'm applying it in my Python development. XP is impossible without an
automated test framework, so the first thing I did was write a Python
version of Kent Beck's Testing Framework.

Shortly thereafter, I saw Tim Peter's brilliant doctest.py and
realized there was no need to write separate TestCase classes. With
doctest, the test case coulde *be* the documentation, inseparably
linked to the object being exercised. After a lot of cutting and
twisting, a very simple framework emerged.

I like the result well enough that I sometimes catch myself doing the
Right Thing - writing test cases *before* starting to code. I attach
xp.py in the hope that others may find it useful and I offer apologies
in advance if my abuse of docStrings treads on *your* abuse.


# Module xp
# John Dell'Aquila <jbd at alum.mit.edu>
# 6 June 1999

# Public domain

# Usual disclaimers apply: "as is", "use at your own risk",
# "no warranty of fitness or merchantability", . . .

""" Module xp - Extreme Python Test Framework.
Inspired by Kent Beck's Testing Framework (see Extreme Programming web
pages at http://c2.com/cgi/wiki?ExtremeProgrammingRoadmap) and derived
from Tim Peter's doctest.py (comp.lang.python archives for March,

xp.Test examines ALL docStrings in specified modules, classes, methods
or functions and *executes* any lines having the Python interpreter
prompt (>>>) as their first non-whitespace.  DocStrings are classified
as passed (no errors), failed (exception raised) or noTest (no
docString or no code within docString).

The general idea is to create test objects and then verify their
correct behavior with 'assert' statements. Multi-line Python
statements are explicitly NOT supported; complex tests must be
implemented as helper functions/methods in the code being
tested. Exception testing IS supported (see TestResult.runDocString)
and so are unqualified imports ('from foo import *' will NOT cause
foo's objects to be tested).

See the docStrings in this file for examples.

    import xp
    tst = xp.Test()           # create Test harness
    tst.check(myModule)       # check some stuff
     . . .
    print tst.result          # print summary
    print tst.result.errors() # display formatted error messages
    print tst.result.passed   # list objects that passed tests
    print tst.result.noTest   # list objects that had no tests
    print tst.result.failed   # list raw error tuples

Sample test summary:
<TestResult: 10 passed, 1 failed, 0 noTest, 0.03 sec>

Sample error message:
DocString:   FUNCTION:__main__.foo
Statement:   assert 1==2
Exception:   AssertionError:

>>> None  # stops noTest snivels, i.e this space intentionally blank

__version__ = '1.0'

import sys, types, time, re, string, traceback

_PS1 = re.compile(r'\s*>>>\s*').match
_summaryFmt = (
    "<TestResult: %(passed)d passed, %(failed)d failed, "
                 "%(noTest)d noTest, %(elapsed)g sec>"
_errorFmt = (
    "DocString:   %s\n"
    "Statement:   %s\n"
    "Exception:   %s\n"

_Module = types.ModuleType
_Class = types.ClassType
_Method = types.MethodType
_Func = types.FunctionType
_Dict = types.DictionaryType
_None = types.NoneType
_Long = types.LongType
_BuiltIn = types.BuiltinFunctionType
_String = types.StringType

_typeAbbrev = {
    _Method: 'METHOD',
    _Long: 'LONG',
    _BuiltIn: 'BUILTIN'

_expect_Fn = """
def _expect_(stmt, exc):
        exec stmt
    except exc:
        raise '_expect_', '%s not raised' % exc.__name__

def moduleName(obj):
    """ Return object's module name, if known, otherwise None.

    >>> assert moduleName(re) == 're'
    >>> assert moduleName(re.compile) == 're'
    >>> assert moduleName(re.RegexObject) == 're'
    >>> assert moduleName(re.RegexObject.match) == 're'
    >>> assert moduleName('foo') == None
    if type(obj) == _Module:
        return obj.__name__
    if type(obj) == _Class:
        return obj.__module__
    if type(obj) == _Method:
        return  obj.im_class.__module__
    if type(obj) == _Func:
        return obj.func_globals['__name__']
    return None

def objRepr(obj):
    """ Return printable representation of object.

    >>> assert objRepr(re) == 'MODULE:re'
    >>> assert objRepr(re.RegexObject) == 'CLASS:re.RegexObject'
    >>> assert objRepr(Test.check) == 'METHOD:%s.Test.check' % __name__
    >>> assert objRepr(objRepr) == 'FUNCTION:%s.objRepr' % __name__
    >>> assert objRepr('foo') == 'STRING:'
    >>> assert objRepr(3L) == 'LONG:'
    >>> assert objRepr(string.lower) == 'BUILTIN:lower'
    typ = _typeAbbrev.get( type(obj),
                           string.upper(type(obj).__name__) )
    if type(obj) == _Module:
        return '%s:%s' % (typ, obj.__name__)
    if type(obj) == _Class:
        return '%s:%s.%s' % (typ, obj.__module__, obj.__name__)
    if type(obj) == _Method:
        return '%s:%s.%s' % (typ, str(obj.im_class), obj.__name__)
    if type(obj) == _Func:
        return '%s:%s.%s' % (typ, obj.func_globals['__name__'],
    if hasattr(obj, '__name__'):
        return '%s:%s' % (typ, obj.__name__)
    return '%s:' % typ

class TestResult:
    """ Accumulate test results. Helper class to Test.
    >>> None

    def __init__(self):
        """ >>> None """
        self.start = self.stop = 0
        self.passed = []
        self.failed = []
        self.noTest = []

    def runDocString(self, obj, env):
        """ Execute obj.__doc__ in a *shallow* copy of the
        environment dictionary, env. If possible, insert
        _expect_ function into environment to allow exception
        testing. Tally results and timestamp.

        >>> tr = TestResult()  # check for *expected* exception
        >>> _expect_('tr.runDocString(3, {})', AssertionError)
        assert type(obj) in (_Module, _Class, _Method, _Func)

        if not self.start:
            self.start = time.time()
        name = objRepr(obj)
        globs = env.copy()
        if not globs.has_key('_expect_'):
            exec _expect_Fn in globs

        status = None
        for line in string.split(obj.__doc__ or '', '\n'):
            matchPS1 = _PS1(line)
            stmt = matchPS1 and line[matchPS1.end(0):]
            if not stmt:
                exec stmt in globs
                status = 1
                etype, evalue = sys.exc_info()[:2]
                edesc = traceback.format_exception_only(etype, evalue)
                edesc = string.strip(string.join(edesc, ''))
                status = 0

        if status == None:
        elif status == 0:
            self.failed.append((name, stmt, edesc))
        elif status == 1:
        self.stop = time.time()

    def __repr__(self):
        """ Display summary results.
        >>> None
        vars = { 'passed': len(self.passed),
                 'failed': len(self.failed),
                 'noTest': len(self.noTest),
                 'start': time.ctime(self.start),
                 'stop': time.ctime(self.stop),
                 'elapsed' : self.stop-self.start }
        return _summaryFmt % vars

    def errors(self):
        """ Return error details as printable string.
        >>> None
        return string.join(map(lambda x, f=_errorFmt: f%x, self.failed),

class Test:
    """ DocString tester.
    >>> None

    def __init__(self):
        """ >>> None """
        self.result = TestResult()

    def check(self, obj, env=None):
        """ Recursively test docStrings of object and any subobjects
        (module or class members). Import safe - does not cross module
        boundaries (use successive calls to check other modules).
        For convenience, Strings are taken as module names, i.e.
        check(__name__) tests the current module.

        DocStrings execute in env dictionary, if supplied, otherwise
        in the appropriate module namespace. A *shallow* copy
        is made for each docString, so temporary variables may be
        used freely, but changing mutable objects can 'pollute'
        environment of succeeding tests.

        >>> None
        assert type(obj) in (_Module, _Class, _Method, _Func, _String)

        if type(obj) == _String:
            thisModuleName, obj = obj, sys.modules[obj]
            thisModuleName = moduleName(obj)
        if not env:
            env = sys.modules[thisModuleName].__dict__

        self.result.runDocString(obj, env)

        if type(obj) not in (_Module, _Class):
            return self
        for name in obj.__dict__.keys():
            nxtObj = getattr(obj, name) # NOT obj.__dict__[name] !!!
            if type(nxtObj) not in (_Class, _Method, _Func):
            if moduleName(nxtObj) == thisModuleName:
                self.check(nxtObj, env)
        return self

if __name__ == '__main__':
    tst = Test().check('__main__')
    print tst.result
    print tst.result.errors()

More information about the Python-list mailing list