[py-svn] py-trunk commit b94b218d6202: adding a logxml plugin and a --xml=path option generating a junit-xml style result log. The xml result log can be parsed nicely by hudson.
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Thu Dec 31 11:25:47 CET 2009
# HG changeset patch -- Bitbucket.org
# Project py-trunk
# URL http://bitbucket.org/hpk42/py-trunk/overview/
# User holger krekel <holger at merlinux.eu>
# Date 1262255107 -3600
# Node ID b94b218d6202bac11ebfa9fcdf961843d32460cf
# Parent 9f60daedd927d002f7c71096df4dd140c6123078
adding a logxml plugin and a --xml=path option generating a junit-xml style result log. The xml result log can be parsed nicely by hudson.
Initial code was based on Ross Lawley's pytest_xmlresult plugin.
--- a/ISSUES.txt
+++ b/ISSUES.txt
@@ -78,6 +78,19 @@ but a remote one fail because the tests
does not contain an "__init__.py". Either give
an error or make it work without the __init__.py
+introduce a "RootCollector"
+----------------------------------------------------------------
+tags: feature 1.2
+
+Currently the top collector is a Directory node and
+there also is the notion of a "topdir". See to refine
+internal handling such that there is a RootCollector
+which holds this topdir (or do away with topdirs?).
+Make sure this leads to an improvement in how
+tests are shown in hudson which currently sometimes
+shows "workspace" and sometimes not as the leading
+name.
+
deprecate ensuretemp / introduce funcargs to setup method
--------------------------------------------------------------
tags: experimental-wish 1.2
--- /dev/null
+++ b/testing/plugin/test_pytest_logxml.py
@@ -0,0 +1,121 @@
+
+from xml.dom import minidom
+
+def runandparse(testdir, *args):
+ resultpath = testdir.tmpdir.join("junit.xml")
+ result = testdir.runpytest("--xml=%s" % resultpath, *args)
+ xmldoc = minidom.parse(str(resultpath))
+ return result, xmldoc
+
+def assert_attr(node, **kwargs):
+ for name, expected in kwargs.items():
+ anode = node.getAttributeNode(name)
+ assert anode, "node %r has no attribute %r" %(node, name)
+ val = anode.value
+ assert val == str(expected)
+
+class TestPython:
+ def test_summing_simple(self, testdir):
+ testdir.makepyfile("""
+ import py
+ def test_pass():
+ pass
+ def test_fail():
+ assert 0
+ def test_skip():
+ py.test.skip("")
+ """)
+ result, dom = runandparse(testdir)
+ assert result.ret
+ node = dom.getElementsByTagName("testsuite")[0]
+ assert_attr(node, errors=0, failures=1, skips=1, tests=3)
+
+ def test_setup_error(self, testdir):
+ testdir.makepyfile("""
+ def pytest_funcarg__arg(request):
+ raise ValueError()
+ def test_function(arg):
+ pass
+ """)
+ result, dom = runandparse(testdir)
+ assert result.ret
+ node = dom.getElementsByTagName("testsuite")[0]
+ assert_attr(node, errors=1, tests=0)
+ tnode = node.getElementsByTagName("testcase")[0]
+ assert_attr(tnode,
+ classname="test_setup_error.test_setup_error",
+ name="test_function")
+ fnode = tnode.getElementsByTagName("error")[0]
+ assert_attr(fnode, message="test setup failure")
+ assert "ValueError" in fnode.toxml()
+
+ def test_internal_error(self, testdir):
+ testdir.makeconftest("def pytest_runtest_protocol(): 0 / 0")
+ testdir.makepyfile("def test_function(): pass")
+ result, dom = runandparse(testdir)
+ assert result.ret
+ node = dom.getElementsByTagName("testsuite")[0]
+ assert_attr(node, errors=1, tests=0)
+ tnode = node.getElementsByTagName("testcase")[0]
+ assert_attr(tnode, classname="pytest", name="internal")
+ fnode = tnode.getElementsByTagName("error")[0]
+ assert_attr(fnode, message="internal error")
+ assert "Division" in fnode.toxml()
+
+ def test_failure_function(self, testdir):
+ testdir.makepyfile("def test_fail(): raise ValueError(42)")
+ result, dom = runandparse(testdir)
+ assert result.ret
+ node = dom.getElementsByTagName("testsuite")[0]
+ assert_attr(node, failures=1, tests=1)
+ tnode = node.getElementsByTagName("testcase")[0]
+ assert_attr(tnode,
+ classname="test_failure_function.test_failure_function",
+ name="test_fail")
+ fnode = tnode.getElementsByTagName("failure")[0]
+ assert_attr(fnode, message="test failure")
+ assert "ValueError" in fnode.toxml()
+
+ def test_collect_error(self, testdir):
+ testdir.makepyfile("syntax error")
+ result, dom = runandparse(testdir)
+ assert result.ret
+ node = dom.getElementsByTagName("testsuite")[0]
+ assert_attr(node, errors=1, tests=0)
+ tnode = node.getElementsByTagName("testcase")[0]
+ assert_attr(tnode,
+ #classname="test_collect_error",
+ name="test_collect_error")
+ fnode = tnode.getElementsByTagName("failure")[0]
+ assert_attr(fnode, message="collection failure")
+ assert "invalid syntax" in fnode.toxml()
+
+class TestNonPython:
+ def test_summing_simple(self, testdir):
+ testdir.makeconftest("""
+ import py
+ def pytest_collect_file(path, parent):
+ if path.ext == ".xyz":
+ return MyItem(path, parent)
+ class MyItem(py.test.collect.Item):
+ def __init__(self, path, parent):
+ super(MyItem, self).__init__(path.basename, parent)
+ self.fspath = path
+ def runtest(self):
+ raise ValueError(42)
+ def repr_failure(self, excinfo):
+ return "custom item runtest failed"
+ """)
+ testdir.tmpdir.join("myfile.xyz").write("hello")
+ result, dom = runandparse(testdir)
+ assert result.ret
+ node = dom.getElementsByTagName("testsuite")[0]
+ assert_attr(node, errors=0, failures=1, skips=0, tests=1)
+ tnode = node.getElementsByTagName("testcase")[0]
+ assert_attr(tnode,
+ #classname="test_collect_error",
+ name="myfile.xyz")
+ fnode = tnode.getElementsByTagName("failure")[0]
+ assert_attr(fnode, message="test failure")
+ assert "custom item runtest failed" in fnode.toxml()
+
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,9 @@
Changes between 1.X and 1.1.1
=====================================
+- new junitxml plugin: --xml=path will generate a junit style xml file
+ which is parseable e.g. by the hudson continous integration server.
+
- new option: --genscript=path will generate a standalone py.test script
which will not need any libraries installed. thanks to Ralf Schmitt.
--- a/pytest_xmlresult.py
+++ /dev/null
@@ -1,189 +0,0 @@
-"""
- xmlresult plugin for machine-readable logging of test results.
- Useful for cruisecontrol integration code.
-
- An adaptation of pytest_resultlog.py
-"""
-
-import time
-
-def pytest_addoption(parser):
- group = parser.getgroup("xmlresult", "xmlresult plugin options")
- group.addoption('--xmlresult', action="store", dest="xmlresult", metavar="path", default=None,
- help="path for machine-readable xml result log.")
-
-def pytest_configure(config):
- xmlresult = config.option.xmlresult
- if xmlresult:
- logfile = open(xmlresult, 'w', 1) # line buffered
- config._xmlresult = XMLResult(logfile)
- config.pluginmanager.register(config._xmlresult)
-
-def pytest_unconfigure(config):
- xmlresult = getattr(config, '_xmlresult', None)
- if xmlresult:
- xmlresult.logfile.close()
- del config._xmlresult
- config.pluginmanager.unregister(xmlresult)
-
-def generic_path(item):
- chain = item.listchain()
- gpath = [chain[0].name]
- fspath = chain[0].fspath
- fspart = False
- for node in chain[1:]:
- newfspath = node.fspath
- if newfspath == fspath:
- if fspart:
- gpath.append(':')
- fspart = False
- else:
- gpath.append('.')
- else:
- gpath.append('/')
- fspart = True
- name = node.name
- if name[0] in '([':
- gpath.pop()
- gpath.append(name)
- fspath = newfspath
- return ''.join(gpath)
-
-class XMLResult(object):
- test_start_time = 0.0
- test_taken_time = 0.0
- test_count = 0
- error_count = 0
- failure_count = 0
- skip_count = 0
-
- def __init__(self, logfile):
- self.logfile = logfile
- self.test_logs = []
-
- def write_log_entry(self, testpath, shortrepr, longrepr):
- self.test_count += 1
- # Create an xml log entry for the tests
- self.test_logs.append('<testcase test_method="%s" name="%s" time="%.3f">' % (testpath.split(':')[-1], testpath, self.test_taken_time))
-
- # Do we have any other data to capture for Errors, Fails and Skips
- if shortrepr in ['E', 'F', 'S']:
-
- if shortrepr == 'E':
- self.error_count += 1
- elif shortrepr == 'F':
- self.failure_count += 1
- elif shortrepr == 'S':
- self.skip_count += 1
-
- tag_map = {'E': 'error', 'F': 'failure', 'S': 'skipped'}
- self.test_logs.append("<%s>" % tag_map[shortrepr])
-
- # Output any more information
- for line in longrepr.splitlines():
- self.test_logs.append("<![CDATA[%s\n]]>" % line)
- self.test_logs.append("</%s>" % tag_map[shortrepr])
- self.test_logs.append("</testcase>")
-
- def log_outcome(self, node, shortrepr, longrepr):
- self.write_log_entry(node.name, shortrepr, longrepr)
-
- def pytest_runtest_logreport(self, report):
- code = report.shortrepr
- if report.passed:
- longrepr = ""
- code = "."
- elif report.failed:
- longrepr = str(report.longrepr)
- code = "F"
- elif report.skipped:
- code = "S"
- longrepr = str(report.longrepr.reprcrash.message)
- self.log_outcome(report.item, code, longrepr)
-
- def pytest_runtest_setup(self, item):
- self.test_start_time = time.time()
-
- def pytest_runtest_teardown(self, item):
- self.test_taken_time = time.time() - self.test_start_time
-
- def pytest_collectreport(self, report):
- if not report.passed:
- if report.failed:
- code = "F"
- else:
- assert report.skipped
- code = "S"
- longrepr = str(report.longrepr.reprcrash)
- self.log_outcome(report.collector, code, longrepr)
-
- def pytest_internalerror(self, excrepr):
- path = excrepr.reprcrash.path
- self.errors += 1
- self.write_log_entry(path, '!', str(excrepr))
-
- def pytest_sessionstart(self, session):
- self.suite_start_time = time.time()
-
- def pytest_sessionfinish(self, session, exitstatus):
- """
- Write the xml output
- """
- suite_stop_time = time.time()
- suite_time_delta = suite_stop_time - self.suite_start_time
- self.logfile.write('<testsuite ')
- self.logfile.write('errors="%i" ' % self.error_count)
- self.logfile.write('failures="%i" ' % self.failure_count)
- self.logfile.write('skips="%i" ' % self.skip_count)
- self.logfile.write('name="" ')
- self.logfile.write('tests="%i" ' % self.test_count)
- self.logfile.write('time="%.3f"' % suite_time_delta)
- self.logfile.write(' >')
- self.logfile.writelines(self.test_logs)
- self.logfile.write('</testsuite>')
- self.logfile.close()
-
-
-# Tests
-def test_generic(testdir, LineMatcher):
- testdir.plugins.append("resultlog")
- testdir.makepyfile("""
- import py
- def test_pass():
- pass
- def test_fail():
- assert 0
- def test_skip():
- py.test.skip("")
- """)
- testdir.runpytest("--xmlresult=result.xml")
- lines = testdir.tmpdir.join("result.xml").readlines(cr=0)
- LineMatcher(lines).fnmatch_lines([
- '*testsuite errors="0" failures="1" skips="1" name="" tests="3"*'
- ])
- LineMatcher(lines).fnmatch_lines([
- '*<failure><![CDATA[def test_fail():*'
- ])
- LineMatcher(lines).fnmatch_lines([
- '*<skipped><![CDATA[Skipped: <Skipped instance>*'
- ])
-
-def test_generic_path():
- from py.__.test.collect import Node, Item, FSCollector
- p1 = Node('a')
- assert p1.fspath is None
- p2 = Node('B', parent=p1)
- p3 = Node('()', parent = p2)
- item = Item('c', parent = p3)
- res = generic_path(item)
- assert res == 'a.B().c'
-
- p0 = FSCollector('proj/test')
- p1 = FSCollector('proj/test/a', parent=p0)
- p2 = Node('B', parent=p1)
- p3 = Node('()', parent = p2)
- p4 = Node('c', parent=p3)
- item = Item('[1]', parent = p4)
-
- res = generic_path(item)
- assert res == 'test/a:B().c[1]'
--- a/AUTHORS
+++ b/AUTHORS
@@ -11,9 +11,10 @@ merlinux GmbH, Germany, office at merlin
Contributors include::
+Ross Lawley
+Ralf Schmitt
Chris Lamb
Harald Armin Massa
-Ralf Schmitt
Martijn Faassen
Ian Bicking
Jan Balster
--- /dev/null
+++ b/py/plugin/pytest_logxml.py
@@ -0,0 +1,147 @@
+"""
+ logxml plugin for machine-readable logging of test results.
+ Based on initial code from Ross Lawley.
+"""
+
+import py
+import time
+
+def pytest_addoption(parser):
+ group = parser.getgroup("general")
+ group.addoption('--xml', action="store", dest="xmlpath",
+ metavar="path", default=None,
+ help="create junit-xml style report file at the given path.")
+
+def pytest_configure(config):
+ xmlpath = config.option.xmlpath
+ if xmlpath:
+ config._xml = LogXML(xmlpath)
+ config.pluginmanager.register(config._xml)
+
+def pytest_unconfigure(config):
+ xml = getattr(config, '_xml', None)
+ if xml:
+ del config._xml
+ config.pluginmanager.unregister(xml)
+
+class LogXML(object):
+ def __init__(self, logfile):
+ self.logfile = logfile
+ self.test_logs = []
+ self.passed = self.skipped = 0
+ self.failed = self.errors = 0
+ self._durations = {}
+
+ def _opentestcase(self, report):
+ node = report.item
+ d = {'time': self._durations.pop(report.item, "0")}
+ names = [x.replace(".py", "") for x in node.listnames()]
+ d['classname'] = ".".join(names[:-1])
+ d['name'] = names[-1]
+ attrs = ['%s="%s"' % item for item in sorted(d.items())]
+ self.test_logs.append("\n<testcase %s>" % " ".join(attrs))
+
+ def _closetestcase(self):
+ self.test_logs.append("</testcase>")
+
+ def append_pass(self, report):
+ self.passed += 1
+ self._opentestcase(report)
+ self._closetestcase()
+
+ def append_failure(self, report):
+ self._opentestcase(report)
+ s = py.xml.escape(str(report.longrepr))
+ #msg = str(report.longrepr.reprtraceback.extraline)
+ self.test_logs.append(
+ '<failure message="test failure">%s</failure>' % (s))
+ self._closetestcase()
+ self.failed += 1
+
+ def _opentestcase_collectfailure(self, report):
+ node = report.collector
+ d = {'time': '???'}
+ names = [x.replace(".py", "") for x in node.listnames()]
+ d['classname'] = ".".join(names[:-1])
+ d['name'] = names[-1]
+ attrs = ['%s="%s"' % item for item in sorted(d.items())]
+ self.test_logs.append("\n<testcase %s>" % " ".join(attrs))
+
+ def append_collect_failure(self, report):
+ self._opentestcase_collectfailure(report)
+ s = py.xml.escape(str(report.longrepr))
+ #msg = str(report.longrepr.reprtraceback.extraline)
+ self.test_logs.append(
+ '<failure message="collection failure">%s</failure>' % (s))
+ self._closetestcase()
+ self.errors += 1
+
+ def append_error(self, report):
+ self._opentestcase(report)
+ s = py.xml.escape(str(report.longrepr))
+ self.test_logs.append(
+ '<error message="test setup failure">%s</error>' % s)
+ self._closetestcase()
+ self.errors += 1
+
+ def append_skipped(self, report):
+ self._opentestcase(report)
+ self.test_logs.append("<skipped/>")
+ self._closetestcase()
+ self.skipped += 1
+
+ def pytest_runtest_logreport(self, report):
+ if report.passed:
+ self.append_pass(report)
+ elif report.failed:
+ if report.when != "call":
+ self.append_error(report)
+ else:
+ self.append_failure(report)
+ elif report.skipped:
+ self.append_skipped(report)
+
+ def pytest_runtest_call(self, item, __multicall__):
+ start = time.time()
+ try:
+ return __multicall__.execute()
+ finally:
+ self._durations[item] = time.time() - start
+
+ def pytest_collectreport(self, report):
+ if not report.passed:
+ if report.failed:
+ self.append_collect_failure(report)
+ else:
+ self.append_collect_skipped(report)
+
+ def pytest_internalerror(self, excrepr):
+ self.errors += 1
+ data = py.xml.escape(str(excrepr))
+ self.test_logs.append(
+ '\n<testcase classname="pytest" name="internal">'
+ ' <error message="internal error">'
+ '%s</error></testcase>' % data)
+
+ def pytest_sessionstart(self, session):
+ self.suite_start_time = time.time()
+
+ def pytest_sessionfinish(self, session, exitstatus, __multicall__):
+ logfile = open(self.logfile, 'w', 1) # line buffered
+ suite_stop_time = time.time()
+ suite_time_delta = suite_stop_time - self.suite_start_time
+ numtests = self.passed + self.skipped + self.failed
+ logfile.write('<testsuite ')
+ logfile.write('name="" ')
+ logfile.write('errors="%i" ' % self.errors)
+ logfile.write('failures="%i" ' % self.failed)
+ logfile.write('skips="%i" ' % self.skipped)
+ logfile.write('tests="%i" ' % numtests)
+ logfile.write('time="%.3f"' % suite_time_delta)
+ logfile.write(' >')
+ logfile.writelines(self.test_logs)
+ logfile.write('</testsuite>')
+ logfile.close()
+ tw = session.config.pluginmanager.getplugin("terminalreporter")._tw
+ tw.line()
+ tw.sep("-", "generated xml file: %s" %(self.logfile))
--- a/py/impl/test/pluginmanager.py
+++ b/py/impl/test/pluginmanager.py
@@ -8,7 +8,8 @@ from py.impl.test.outcome import Skipped
default_plugins = (
"default runner capture terminal mark skipping tmpdir monkeypatch "
- "recwarn pdb pastebin unittest helpconfig nose assertion genscript").split()
+ "recwarn pdb pastebin unittest helpconfig nose assertion genscript "
+ "logxml").split()
def check_old_use(mod, modname):
clsname = modname[len('pytest_'):].capitalize() + "Plugin"
More information about the pytest-commit
mailing list