diff --git a/AUTHORS b/AUTHORS index 20a0fb6bf..957a86973 100644 --- a/AUTHORS +++ b/AUTHORS @@ -118,6 +118,7 @@ Jonas Obrist Jordan Guymon Jordan Moldow Jordan Speicher +Joseph Hunkeler Joshua Bronson Jurko Gospodnetić Justyna Janczyszyn diff --git a/changelog/3547.bugfix.rst b/changelog/3547.bugfix.rst new file mode 100644 index 000000000..b2ec00364 --- /dev/null +++ b/changelog/3547.bugfix.rst @@ -0,0 +1,2 @@ +``--junitxml`` can emit XML compatible with Jenkins xUnit. +``junit_family`` INI option accepts ``legacy|xunit1``, which produces old style output, and ``xunit2`` that conforms more strictly to https://github.com/jenkinsci/xunit-plugin/blob/xunit-2.3.2/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd diff --git a/src/_pytest/junitxml.py b/src/_pytest/junitxml.py index a426f537b..e2e1d43bb 100644 --- a/src/_pytest/junitxml.py +++ b/src/_pytest/junitxml.py @@ -66,12 +66,34 @@ def bin_xml_escape(arg): return py.xml.raw(illegal_xml_re.sub(repl, py.xml.escape(arg))) +def merge_family(left, right): + result = {} + for kl, vl in left.items(): + for kr, vr in right.items(): + if not isinstance(vl, list): + raise TypeError(type(vl)) + result[kl] = vl + vr + left.update(result) + + +families = {} +families["_base"] = {"testcase": ["classname", "name"]} +families["_base_legacy"] = {"testcase": ["file", "line", "url"]} + +# xUnit 1.x inherits legacy attributes +families["xunit1"] = families["_base"].copy() +merge_family(families["xunit1"], families["_base_legacy"]) + +# xUnit 2.x uses strict base attributes +families["xunit2"] = families["_base"] + + class _NodeReporter(object): def __init__(self, nodeid, xml): - self.id = nodeid self.xml = xml self.add_stats = self.xml.add_stats + self.family = self.xml.family self.duration = 0 self.properties = [] self.nodes = [] @@ -119,8 +141,20 @@ class _NodeReporter(object): self.attrs = attrs self.attrs.update(existing_attrs) # restore any user-defined attributes + # Preserve legacy testcase behavior + if self.family == "xunit1": + return + + # Filter out attributes not permitted by this test family. + # Including custom attributes because they are not valid here. + temp_attrs = {} + for key in self.attrs.keys(): + if key in families[self.family]["testcase"]: + temp_attrs[key] = self.attrs[key] + self.attrs = temp_attrs + def to_xml(self): - testcase = Junit.testcase(time=self.duration, **self.attrs) + testcase = Junit.testcase(time="%.3f" % self.duration, **self.attrs) testcase.append(self.make_properties_node()) for node in self.nodes: testcase.append(node) @@ -269,16 +303,26 @@ def record_xml_attribute(request): from _pytest.warning_types import PytestWarning request.node.warn(PytestWarning("record_xml_attribute is an experimental feature")) + + # Declare noop + def add_attr_noop(name, value): + pass + + attr_func = add_attr_noop xml = getattr(request.config, "_xml", None) - if xml is not None: + + if xml is not None and xml.family != "xunit1": + request.node.warn( + PytestWarning( + "record_xml_attribute is incompatible with junit_family: " + "%s (use: legacy|xunit1)" % xml.family + ) + ) + elif xml is not None: node_reporter = xml.node_reporter(request.node.nodeid) - return node_reporter.add_attribute - else: + attr_func = node_reporter.add_attribute - def add_attr_noop(name, value): - pass - - return add_attr_noop + return attr_func def pytest_addoption(parser): @@ -315,6 +359,11 @@ def pytest_addoption(parser): "Duration time to report: one of total|call", default="total", ) # choices=['total', 'call']) + parser.addini( + "junit_family", + "Emit XML for schema: one of legacy|xunit1|xunit2", + default="xunit1", + ) def pytest_configure(config): @@ -327,6 +376,7 @@ def pytest_configure(config): config.getini("junit_suite_name"), config.getini("junit_logging"), config.getini("junit_duration_report"), + config.getini("junit_family"), ) config.pluginmanager.register(config._xml) @@ -361,6 +411,7 @@ class LogXML(object): suite_name="pytest", logging="no", report_duration="total", + family="xunit1", ): logfile = os.path.expanduser(os.path.expandvars(logfile)) self.logfile = os.path.normpath(os.path.abspath(logfile)) @@ -368,6 +419,7 @@ class LogXML(object): self.suite_name = suite_name self.logging = logging self.report_duration = report_duration + self.family = family self.stats = dict.fromkeys(["error", "passed", "failure", "skipped"], 0) self.node_reporters = {} # nodeid -> _NodeReporter self.node_reporters_ordered = [] @@ -376,6 +428,10 @@ class LogXML(object): self.open_reports = [] self.cnt_double_fail_tests = 0 + # Replaces convenience family with real family + if self.family == "legacy": + self.family = "xunit1" + def finalize(self, report): nodeid = getattr(report, "nodeid", report) # local hack to handle xdist report order @@ -545,7 +601,7 @@ class LogXML(object): name=self.suite_name, errors=self.stats["error"], failures=self.stats["failure"], - skips=self.stats["skipped"], + skipped=self.stats["skipped"], tests=numtests, time="%.3f" % suite_time_delta, ).unicode(indent=0) diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index 59c11fa00..2765dfc60 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -107,7 +107,7 @@ class TestPython(object): result, dom = runandparse(testdir) assert result.ret node = dom.find_first_by_tag("testsuite") - node.assert_attr(name="pytest", errors=0, failures=1, skips=2, tests=5) + node.assert_attr(name="pytest", errors=0, failures=1, skipped=2, tests=5) def test_summing_simple_with_errors(self, testdir): testdir.makepyfile( @@ -133,7 +133,7 @@ class TestPython(object): result, dom = runandparse(testdir) assert result.ret node = dom.find_first_by_tag("testsuite") - node.assert_attr(name="pytest", errors=1, failures=2, skips=1, tests=5) + node.assert_attr(name="pytest", errors=1, failures=2, skipped=1, tests=5) def test_timing_function(self, testdir): testdir.makepyfile( @@ -201,12 +201,7 @@ class TestPython(object): node = dom.find_first_by_tag("testsuite") node.assert_attr(errors=1, tests=1) tnode = node.find_first_by_tag("testcase") - tnode.assert_attr( - file="test_setup_error.py", - line="5", - classname="test_setup_error", - name="test_function", - ) + tnode.assert_attr(classname="test_setup_error", name="test_function") fnode = tnode.find_first_by_tag("error") fnode.assert_attr(message="test setup failure") assert "ValueError" in fnode.toxml() @@ -228,12 +223,7 @@ class TestPython(object): assert result.ret node = dom.find_first_by_tag("testsuite") tnode = node.find_first_by_tag("testcase") - tnode.assert_attr( - file="test_teardown_error.py", - line="6", - classname="test_teardown_error", - name="test_function", - ) + tnode.assert_attr(classname="test_teardown_error", name="test_function") fnode = tnode.find_first_by_tag("error") fnode.assert_attr(message="test teardown failure") assert "ValueError" in fnode.toxml() @@ -274,14 +264,9 @@ class TestPython(object): result, dom = runandparse(testdir) assert result.ret == 0 node = dom.find_first_by_tag("testsuite") - node.assert_attr(skips=1) + node.assert_attr(skipped=1) tnode = node.find_first_by_tag("testcase") - tnode.assert_attr( - file="test_skip_contains_name_reason.py", - line="1", - classname="test_skip_contains_name_reason", - name="test_skip", - ) + tnode.assert_attr(classname="test_skip_contains_name_reason", name="test_skip") snode = tnode.find_first_by_tag("skipped") snode.assert_attr(type="pytest.skip", message="hello23") @@ -297,13 +282,10 @@ class TestPython(object): result, dom = runandparse(testdir) assert result.ret == 0 node = dom.find_first_by_tag("testsuite") - node.assert_attr(skips=1) + node.assert_attr(skipped=1) tnode = node.find_first_by_tag("testcase") tnode.assert_attr( - file="test_mark_skip_contains_name_reason.py", - line="1", - classname="test_mark_skip_contains_name_reason", - name="test_skip", + classname="test_mark_skip_contains_name_reason", name="test_skip" ) snode = tnode.find_first_by_tag("skipped") snode.assert_attr(type="pytest.skip", message="hello24") @@ -321,13 +303,10 @@ class TestPython(object): result, dom = runandparse(testdir) assert result.ret == 0 node = dom.find_first_by_tag("testsuite") - node.assert_attr(skips=1) + node.assert_attr(skipped=1) tnode = node.find_first_by_tag("testcase") tnode.assert_attr( - file="test_mark_skipif_contains_name_reason.py", - line="2", - classname="test_mark_skipif_contains_name_reason", - name="test_skip", + classname="test_mark_skipif_contains_name_reason", name="test_skip" ) snode = tnode.find_first_by_tag("skipped") snode.assert_attr(type="pytest.skip", message="hello25") @@ -360,10 +339,7 @@ class TestPython(object): node.assert_attr(failures=1) tnode = node.find_first_by_tag("testcase") tnode.assert_attr( - file="test_classname_instance.py", - line="1", - classname="test_classname_instance.TestClass", - name="test_method", + classname="test_classname_instance.TestClass", name="test_method" ) def test_classname_nested_dir(self, testdir): @@ -374,12 +350,7 @@ class TestPython(object): node = dom.find_first_by_tag("testsuite") node.assert_attr(failures=1) tnode = node.find_first_by_tag("testcase") - tnode.assert_attr( - file=os.path.join("sub", "test_hello.py"), - line="0", - classname="sub.test_hello", - name="test_func", - ) + tnode.assert_attr(classname="sub.test_hello", name="test_func") def test_internal_error(self, testdir): testdir.makeconftest("def pytest_runtest_protocol(): 0 / 0") @@ -415,12 +386,7 @@ class TestPython(object): node = dom.find_first_by_tag("testsuite") node.assert_attr(failures=1, tests=1) tnode = node.find_first_by_tag("testcase") - tnode.assert_attr( - file="test_failure_function.py", - line="3", - classname="test_failure_function", - name="test_fail", - ) + tnode.assert_attr(classname="test_failure_function", name="test_fail") fnode = tnode.find_first_by_tag("failure") fnode.assert_attr(message="ValueError: 42") assert "ValueError" in fnode.toxml() @@ -477,10 +443,7 @@ class TestPython(object): tnode = node.find_nth_by_tag("testcase", index) tnode.assert_attr( - file="test_failure_escape.py", - line="1", - classname="test_failure_escape", - name="test_func[%s]" % char, + classname="test_failure_escape", name="test_func[%s]" % char ) sysout = tnode.find_first_by_tag("system-out") text = sysout.text @@ -501,18 +464,10 @@ class TestPython(object): node = dom.find_first_by_tag("testsuite") node.assert_attr(failures=1, tests=2) tnode = node.find_first_by_tag("testcase") - tnode.assert_attr( - file="test_junit_prefixing.py", - line="0", - classname="xyz.test_junit_prefixing", - name="test_func", - ) + tnode.assert_attr(classname="xyz.test_junit_prefixing", name="test_func") tnode = node.find_nth_by_tag("testcase", 1) tnode.assert_attr( - file="test_junit_prefixing.py", - line="3", - classname="xyz.test_junit_prefixing.TestHello", - name="test_hello", + classname="xyz.test_junit_prefixing.TestHello", name="test_hello" ) def test_xfailure_function(self, testdir): @@ -526,14 +481,9 @@ class TestPython(object): result, dom = runandparse(testdir) assert not result.ret node = dom.find_first_by_tag("testsuite") - node.assert_attr(skips=1, tests=1) + node.assert_attr(skipped=1, tests=1) tnode = node.find_first_by_tag("testcase") - tnode.assert_attr( - file="test_xfailure_function.py", - line="1", - classname="test_xfailure_function", - name="test_xfail", - ) + tnode.assert_attr(classname="test_xfailure_function", name="test_xfail") fnode = tnode.find_first_by_tag("skipped") fnode.assert_attr(message="expected test failure") # assert "ValueError" in fnode.toxml() @@ -569,14 +519,9 @@ class TestPython(object): result, dom = runandparse(testdir) # assert result.ret node = dom.find_first_by_tag("testsuite") - node.assert_attr(skips=0, tests=1) + node.assert_attr(skipped=0, tests=1) tnode = node.find_first_by_tag("testcase") - tnode.assert_attr( - file="test_xfailure_xpass.py", - line="1", - classname="test_xfailure_xpass", - name="test_xpass", - ) + tnode.assert_attr(classname="test_xfailure_xpass", name="test_xpass") def test_xfailure_xpass_strict(self, testdir): testdir.makepyfile( @@ -590,14 +535,9 @@ class TestPython(object): result, dom = runandparse(testdir) # assert result.ret node = dom.find_first_by_tag("testsuite") - node.assert_attr(skips=0, tests=1) + node.assert_attr(skipped=0, tests=1) tnode = node.find_first_by_tag("testcase") - tnode.assert_attr( - file="test_xfailure_xpass_strict.py", - line="1", - classname="test_xfailure_xpass_strict", - name="test_xpass", - ) + tnode.assert_attr(classname="test_xfailure_xpass_strict", name="test_xpass") fnode = tnode.find_first_by_tag("failure") fnode.assert_attr(message="[XPASS(strict)] This needs to fail!") @@ -608,8 +548,6 @@ class TestPython(object): node = dom.find_first_by_tag("testsuite") node.assert_attr(errors=1, tests=1) tnode = node.find_first_by_tag("testcase") - tnode.assert_attr(file="test_collect_error.py", name="test_collect_error") - assert tnode["line"] is None fnode = tnode.find_first_by_tag("error") fnode.assert_attr(message="collection failure") assert "SyntaxError" in fnode.toxml() @@ -792,7 +730,7 @@ class TestNonPython(object): result, dom = runandparse(testdir) assert result.ret node = dom.find_first_by_tag("testsuite") - node.assert_attr(errors=0, failures=1, skips=0, tests=1) + node.assert_attr(errors=0, failures=1, skipped=0, tests=1) tnode = node.find_first_by_tag("testcase") tnode.assert_attr(name="myfile.xyz") fnode = tnode.find_first_by_tag("failure") @@ -1042,6 +980,12 @@ def test_record_property_same_name(testdir): @pytest.mark.filterwarnings("default") def test_record_attribute(testdir): + testdir.makeini( + """ + [pytest] + junit_family = xunit1 + """ + ) testdir.makepyfile( """ import pytest @@ -1063,6 +1007,38 @@ def test_record_attribute(testdir): ) +@pytest.mark.filterwarnings("default") +def test_record_attribute_xunit2(testdir): + """Ensure record_xml_attribute drops values when outside of legacy family + """ + testdir.makeini( + """ + [pytest] + junit_family = xunit2 + """ + ) + testdir.makepyfile( + """ + import pytest + + @pytest.fixture + def other(record_xml_attribute): + record_xml_attribute("bar", 1) + def test_record(record_xml_attribute, other): + record_xml_attribute("foo", "<1"); + """ + ) + + result, dom = runandparse(testdir, "-rw") + result.stdout.fnmatch_lines( + [ + "*test_record_attribute_xunit2.py:6:*record_xml_attribute is an experimental feature", + "*test_record_attribute_xunit2.py:6:*record_xml_attribute is incompatible with " + "junit_family: xunit2 (use: legacy|xunit1)", + ] + ) + + def test_random_report_log_xdist(testdir, monkeypatch): """xdist calls pytest_runtest_logreport as they are executed by the slaves, with nodes from several nodes overlapping, so junitxml must cope with that @@ -1155,20 +1131,18 @@ def test_fancy_items_regression(testdir): assert "INTERNALERROR" not in result.stdout.str() - items = sorted( - "%(classname)s %(name)s %(file)s" % x for x in dom.find_by_tag("testcase") - ) + items = sorted("%(classname)s %(name)s" % x for x in dom.find_by_tag("testcase")) import pprint pprint.pprint(items) assert items == [ - u"conftest a conftest.py", - u"conftest a conftest.py", - u"conftest b conftest.py", - u"test_fancy_items_regression a test_fancy_items_regression.py", - u"test_fancy_items_regression a test_fancy_items_regression.py", - u"test_fancy_items_regression b test_fancy_items_regression.py", - u"test_fancy_items_regression test_pass" u" test_fancy_items_regression.py", + u"conftest a", + u"conftest a", + u"conftest b", + u"test_fancy_items_regression a", + u"test_fancy_items_regression a", + u"test_fancy_items_regression b", + u"test_fancy_items_regression test_pass", ]