Merge pull request #4511 from jhunkeler/junit-strict

Toggle JUnit behavior with INI option

AUTHORS

@@ -118,6 +118,7 @@ Jonas Obrist
 Jordan Guymon
 Jordan Moldow
 Jordan Speicher
+Joseph Hunkeler
 Joshua Bronson
 Jurko Gospodnetić
 Justyna Janczyszyn

changelog/3547.bugfix.rst

@@ -0,0 +1,2 @@
+``--junitxml`` can emit XML compatible with Jenkins xUnit.
+``junit_family`` INI option accepts ``legacy|xunit1``, which produces old style output, and ``xunit2`` that conforms more strictly to https://github.com/jenkinsci/xunit-plugin/blob/xunit-2.3.2/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd

src/_pytest/junitxml.py

@@ -66,12 +66,34 @@ def bin_xml_escape(arg):
     return py.xml.raw(illegal_xml_re.sub(repl, py.xml.escape(arg)))
 
 
+def merge_family(left, right):
+    result = {}
+    for kl, vl in left.items():
+        for kr, vr in right.items():
+            if not isinstance(vl, list):
+                raise TypeError(type(vl))
+            result[kl] = vl + vr
+    left.update(result)
+
+
+families = {}
+families["_base"] = {"testcase": ["classname", "name"]}
+families["_base_legacy"] = {"testcase": ["file", "line", "url"]}
+
+# xUnit 1.x inherits legacy attributes
+families["xunit1"] = families["_base"].copy()
+merge_family(families["xunit1"], families["_base_legacy"])
+
+# xUnit 2.x uses strict base attributes
+families["xunit2"] = families["_base"]
+
+
 class _NodeReporter(object):
     def __init__(self, nodeid, xml):
-
         self.id = nodeid
         self.xml = xml
         self.add_stats = self.xml.add_stats
+        self.family = self.xml.family
         self.duration = 0
         self.properties = []
         self.nodes = []
@@ -119,8 +141,20 @@ class _NodeReporter(object):
         self.attrs = attrs
         self.attrs.update(existing_attrs)  # restore any user-defined attributes
 
+        # Preserve legacy testcase behavior
+        if self.family == "xunit1":
+            return
+
+        # Filter out attributes not permitted by this test family.
+        # Including custom attributes because they are not valid here.
+        temp_attrs = {}
+        for key in self.attrs.keys():
+            if key in families[self.family]["testcase"]:
+                temp_attrs[key] = self.attrs[key]
+        self.attrs = temp_attrs
+
     def to_xml(self):
-        testcase = Junit.testcase(time=self.duration, **self.attrs)
+        testcase = Junit.testcase(time="%.3f" % self.duration, **self.attrs)
         testcase.append(self.make_properties_node())
         for node in self.nodes:
             testcase.append(node)
@@ -269,16 +303,26 @@ def record_xml_attribute(request):
     from _pytest.warning_types import PytestWarning
 
     request.node.warn(PytestWarning("record_xml_attribute is an experimental feature"))
-    xml = getattr(request.config, "_xml", None)
-    if xml is not None:
-        node_reporter = xml.node_reporter(request.node.nodeid)
-        return node_reporter.add_attribute
-    else:
 
+    # Declare noop
     def add_attr_noop(name, value):
         pass
 
-        return add_attr_noop
+    attr_func = add_attr_noop
+    xml = getattr(request.config, "_xml", None)
+
+    if xml is not None and xml.family != "xunit1":
+        request.node.warn(
+            PytestWarning(
+                "record_xml_attribute is incompatible with junit_family: "
+                "%s (use: legacy|xunit1)" % xml.family
+            )
+        )
+    elif xml is not None:
+        node_reporter = xml.node_reporter(request.node.nodeid)
+        attr_func = node_reporter.add_attribute
+
+    return attr_func
 
 
 def pytest_addoption(parser):
@@ -315,6 +359,11 @@ def pytest_addoption(parser):
         "Duration time to report: one of total|call",
         default="total",
     )  # choices=['total', 'call'])
+    parser.addini(
+        "junit_family",
+        "Emit XML for schema: one of legacy|xunit1|xunit2",
+        default="xunit1",
+    )
 
 
 def pytest_configure(config):
@@ -327,6 +376,7 @@ def pytest_configure(config):
             config.getini("junit_suite_name"),
             config.getini("junit_logging"),
             config.getini("junit_duration_report"),
+            config.getini("junit_family"),
         )
         config.pluginmanager.register(config._xml)
 
@@ -361,6 +411,7 @@ class LogXML(object):
         suite_name="pytest",
         logging="no",
         report_duration="total",
+        family="xunit1",
     ):
         logfile = os.path.expanduser(os.path.expandvars(logfile))
         self.logfile = os.path.normpath(os.path.abspath(logfile))
@@ -368,6 +419,7 @@ class LogXML(object):
         self.suite_name = suite_name
         self.logging = logging
         self.report_duration = report_duration
+        self.family = family
         self.stats = dict.fromkeys(["error", "passed", "failure", "skipped"], 0)
         self.node_reporters = {}  # nodeid -> _NodeReporter
         self.node_reporters_ordered = []
@@ -376,6 +428,10 @@ class LogXML(object):
         self.open_reports = []
         self.cnt_double_fail_tests = 0
 
+        # Replaces convenience family with real family
+        if self.family == "legacy":
+            self.family = "xunit1"
+
     def finalize(self, report):
         nodeid = getattr(report, "nodeid", report)
         # local hack to handle xdist report order
@@ -545,7 +601,7 @@ class LogXML(object):
                 name=self.suite_name,
                 errors=self.stats["error"],
                 failures=self.stats["failure"],
-                skips=self.stats["skipped"],
+                skipped=self.stats["skipped"],
                 tests=numtests,
                 time="%.3f" % suite_time_delta,
             ).unicode(indent=0)

testing/test_junitxml.py

@@ -107,7 +107,7 @@ class TestPython(object):
         result, dom = runandparse(testdir)
         assert result.ret
         node = dom.find_first_by_tag("testsuite")
-        node.assert_attr(name="pytest", errors=0, failures=1, skips=2, tests=5)
+        node.assert_attr(name="pytest", errors=0, failures=1, skipped=2, tests=5)
 
     def test_summing_simple_with_errors(self, testdir):
         testdir.makepyfile(
@@ -133,7 +133,7 @@ class TestPython(object):
         result, dom = runandparse(testdir)
         assert result.ret
         node = dom.find_first_by_tag("testsuite")
-        node.assert_attr(name="pytest", errors=1, failures=2, skips=1, tests=5)
+        node.assert_attr(name="pytest", errors=1, failures=2, skipped=1, tests=5)
 
     def test_timing_function(self, testdir):
         testdir.makepyfile(
@@ -201,12 +201,7 @@ class TestPython(object):
         node = dom.find_first_by_tag("testsuite")
         node.assert_attr(errors=1, tests=1)
         tnode = node.find_first_by_tag("testcase")
-        tnode.assert_attr(
+        tnode.assert_attr(classname="test_setup_error", name="test_function")
-            file="test_setup_error.py",
-            line="5",
-            classname="test_setup_error",
-            name="test_function",
-        )
         fnode = tnode.find_first_by_tag("error")
         fnode.assert_attr(message="test setup failure")
         assert "ValueError" in fnode.toxml()
@@ -228,12 +223,7 @@ class TestPython(object):
         assert result.ret
         node = dom.find_first_by_tag("testsuite")
         tnode = node.find_first_by_tag("testcase")
-        tnode.assert_attr(
+        tnode.assert_attr(classname="test_teardown_error", name="test_function")
-            file="test_teardown_error.py",
-            line="6",
-            classname="test_teardown_error",
-            name="test_function",
-        )
         fnode = tnode.find_first_by_tag("error")
         fnode.assert_attr(message="test teardown failure")
         assert "ValueError" in fnode.toxml()
@@ -274,14 +264,9 @@ class TestPython(object):
         result, dom = runandparse(testdir)
         assert result.ret == 0
         node = dom.find_first_by_tag("testsuite")
-        node.assert_attr(skips=1)
+        node.assert_attr(skipped=1)
         tnode = node.find_first_by_tag("testcase")
-        tnode.assert_attr(
+        tnode.assert_attr(classname="test_skip_contains_name_reason", name="test_skip")
-            file="test_skip_contains_name_reason.py",
-            line="1",
-            classname="test_skip_contains_name_reason",
-            name="test_skip",
-        )
         snode = tnode.find_first_by_tag("skipped")
         snode.assert_attr(type="pytest.skip", message="hello23")
 
@@ -297,13 +282,10 @@ class TestPython(object):
         result, dom = runandparse(testdir)
         assert result.ret == 0
         node = dom.find_first_by_tag("testsuite")
-        node.assert_attr(skips=1)
+        node.assert_attr(skipped=1)
         tnode = node.find_first_by_tag("testcase")
         tnode.assert_attr(
-            file="test_mark_skip_contains_name_reason.py",
+            classname="test_mark_skip_contains_name_reason", name="test_skip"
-            line="1",
-            classname="test_mark_skip_contains_name_reason",
-            name="test_skip",
         )
         snode = tnode.find_first_by_tag("skipped")
         snode.assert_attr(type="pytest.skip", message="hello24")
@@ -321,13 +303,10 @@ class TestPython(object):
         result, dom = runandparse(testdir)
         assert result.ret == 0
         node = dom.find_first_by_tag("testsuite")
-        node.assert_attr(skips=1)
+        node.assert_attr(skipped=1)
         tnode = node.find_first_by_tag("testcase")
         tnode.assert_attr(
-            file="test_mark_skipif_contains_name_reason.py",
+            classname="test_mark_skipif_contains_name_reason", name="test_skip"
-            line="2",
-            classname="test_mark_skipif_contains_name_reason",
-            name="test_skip",
         )
         snode = tnode.find_first_by_tag("skipped")
         snode.assert_attr(type="pytest.skip", message="hello25")
@@ -360,10 +339,7 @@ class TestPython(object):
         node.assert_attr(failures=1)
         tnode = node.find_first_by_tag("testcase")
         tnode.assert_attr(
-            file="test_classname_instance.py",
+            classname="test_classname_instance.TestClass", name="test_method"
-            line="1",
-            classname="test_classname_instance.TestClass",
-            name="test_method",
         )
 
     def test_classname_nested_dir(self, testdir):
@@ -374,12 +350,7 @@ class TestPython(object):
         node = dom.find_first_by_tag("testsuite")
         node.assert_attr(failures=1)
         tnode = node.find_first_by_tag("testcase")
-        tnode.assert_attr(
+        tnode.assert_attr(classname="sub.test_hello", name="test_func")
-            file=os.path.join("sub", "test_hello.py"),
-            line="0",
-            classname="sub.test_hello",
-            name="test_func",
-        )
 
     def test_internal_error(self, testdir):
         testdir.makeconftest("def pytest_runtest_protocol(): 0 / 0")
@@ -415,12 +386,7 @@ class TestPython(object):
         node = dom.find_first_by_tag("testsuite")
         node.assert_attr(failures=1, tests=1)
         tnode = node.find_first_by_tag("testcase")
-        tnode.assert_attr(
+        tnode.assert_attr(classname="test_failure_function", name="test_fail")
-            file="test_failure_function.py",
-            line="3",
-            classname="test_failure_function",
-            name="test_fail",
-        )
         fnode = tnode.find_first_by_tag("failure")
         fnode.assert_attr(message="ValueError: 42")
         assert "ValueError" in fnode.toxml()
@@ -477,10 +443,7 @@ class TestPython(object):
 
             tnode = node.find_nth_by_tag("testcase", index)
             tnode.assert_attr(
-                file="test_failure_escape.py",
+                classname="test_failure_escape", name="test_func[%s]" % char
-                line="1",
-                classname="test_failure_escape",
-                name="test_func[%s]" % char,
             )
             sysout = tnode.find_first_by_tag("system-out")
             text = sysout.text
@@ -501,18 +464,10 @@ class TestPython(object):
         node = dom.find_first_by_tag("testsuite")
         node.assert_attr(failures=1, tests=2)
         tnode = node.find_first_by_tag("testcase")
-        tnode.assert_attr(
+        tnode.assert_attr(classname="xyz.test_junit_prefixing", name="test_func")
-            file="test_junit_prefixing.py",
-            line="0",
-            classname="xyz.test_junit_prefixing",
-            name="test_func",
-        )
         tnode = node.find_nth_by_tag("testcase", 1)
         tnode.assert_attr(
-            file="test_junit_prefixing.py",
+            classname="xyz.test_junit_prefixing.TestHello", name="test_hello"
-            line="3",
-            classname="xyz.test_junit_prefixing.TestHello",
-            name="test_hello",
         )
 
     def test_xfailure_function(self, testdir):
@@ -526,14 +481,9 @@ class TestPython(object):
         result, dom = runandparse(testdir)
         assert not result.ret
         node = dom.find_first_by_tag("testsuite")
-        node.assert_attr(skips=1, tests=1)
+        node.assert_attr(skipped=1, tests=1)
         tnode = node.find_first_by_tag("testcase")
-        tnode.assert_attr(
+        tnode.assert_attr(classname="test_xfailure_function", name="test_xfail")
-            file="test_xfailure_function.py",
-            line="1",
-            classname="test_xfailure_function",
-            name="test_xfail",
-        )
         fnode = tnode.find_first_by_tag("skipped")
         fnode.assert_attr(message="expected test failure")
         # assert "ValueError" in fnode.toxml()
@@ -569,14 +519,9 @@ class TestPython(object):
         result, dom = runandparse(testdir)
         # assert result.ret
         node = dom.find_first_by_tag("testsuite")
-        node.assert_attr(skips=0, tests=1)
+        node.assert_attr(skipped=0, tests=1)
         tnode = node.find_first_by_tag("testcase")
-        tnode.assert_attr(
+        tnode.assert_attr(classname="test_xfailure_xpass", name="test_xpass")
-            file="test_xfailure_xpass.py",
-            line="1",
-            classname="test_xfailure_xpass",
-            name="test_xpass",
-        )
 
     def test_xfailure_xpass_strict(self, testdir):
         testdir.makepyfile(
@@ -590,14 +535,9 @@ class TestPython(object):
         result, dom = runandparse(testdir)
         # assert result.ret
         node = dom.find_first_by_tag("testsuite")
-        node.assert_attr(skips=0, tests=1)
+        node.assert_attr(skipped=0, tests=1)
         tnode = node.find_first_by_tag("testcase")
-        tnode.assert_attr(
+        tnode.assert_attr(classname="test_xfailure_xpass_strict", name="test_xpass")
-            file="test_xfailure_xpass_strict.py",
-            line="1",
-            classname="test_xfailure_xpass_strict",
-            name="test_xpass",
-        )
         fnode = tnode.find_first_by_tag("failure")
         fnode.assert_attr(message="[XPASS(strict)] This needs to fail!")
 
@@ -608,8 +548,6 @@ class TestPython(object):
         node = dom.find_first_by_tag("testsuite")
         node.assert_attr(errors=1, tests=1)
         tnode = node.find_first_by_tag("testcase")
-        tnode.assert_attr(file="test_collect_error.py", name="test_collect_error")
-        assert tnode["line"] is None
         fnode = tnode.find_first_by_tag("error")
         fnode.assert_attr(message="collection failure")
         assert "SyntaxError" in fnode.toxml()
@@ -792,7 +730,7 @@ class TestNonPython(object):
         result, dom = runandparse(testdir)
         assert result.ret
         node = dom.find_first_by_tag("testsuite")
-        node.assert_attr(errors=0, failures=1, skips=0, tests=1)
+        node.assert_attr(errors=0, failures=1, skipped=0, tests=1)
         tnode = node.find_first_by_tag("testcase")
         tnode.assert_attr(name="myfile.xyz")
         fnode = tnode.find_first_by_tag("failure")
@@ -1042,6 +980,12 @@ def test_record_property_same_name(testdir):
 
 @pytest.mark.filterwarnings("default")
 def test_record_attribute(testdir):
+    testdir.makeini(
+        """
+        [pytest]
+        junit_family = xunit1
+    """
+    )
     testdir.makepyfile(
         """
         import pytest
@@ -1063,6 +1007,38 @@ def test_record_attribute(testdir):
     )
 
 
+@pytest.mark.filterwarnings("default")
+def test_record_attribute_xunit2(testdir):
+    """Ensure record_xml_attribute drops values when outside of legacy family
+    """
+    testdir.makeini(
+        """
+        [pytest]
+        junit_family = xunit2
+    """
+    )
+    testdir.makepyfile(
+        """
+        import pytest
+
+        @pytest.fixture
+        def other(record_xml_attribute):
+            record_xml_attribute("bar", 1)
+        def test_record(record_xml_attribute, other):
+            record_xml_attribute("foo", "<1");
+    """
+    )
+
+    result, dom = runandparse(testdir, "-rw")
+    result.stdout.fnmatch_lines(
+        [
+            "*test_record_attribute_xunit2.py:6:*record_xml_attribute is an experimental feature",
+            "*test_record_attribute_xunit2.py:6:*record_xml_attribute is incompatible with "
+            "junit_family: xunit2 (use: legacy|xunit1)",
+        ]
+    )
+
+
 def test_random_report_log_xdist(testdir, monkeypatch):
     """xdist calls pytest_runtest_logreport as they are executed by the slaves,
     with nodes from several nodes overlapping, so junitxml must cope with that
@@ -1155,20 +1131,18 @@ def test_fancy_items_regression(testdir):
 
     assert "INTERNALERROR" not in result.stdout.str()
 
-    items = sorted(
+    items = sorted("%(classname)s %(name)s" % x for x in dom.find_by_tag("testcase"))
-        "%(classname)s %(name)s %(file)s" % x for x in dom.find_by_tag("testcase")
-    )
     import pprint
 
     pprint.pprint(items)
     assert items == [
-        u"conftest a conftest.py",
+        u"conftest a",
-        u"conftest a conftest.py",
+        u"conftest a",
-        u"conftest b conftest.py",
+        u"conftest b",
-        u"test_fancy_items_regression a test_fancy_items_regression.py",
+        u"test_fancy_items_regression a",
-        u"test_fancy_items_regression a test_fancy_items_regression.py",
+        u"test_fancy_items_regression a",
-        u"test_fancy_items_regression b test_fancy_items_regression.py",
+        u"test_fancy_items_regression b",
-        u"test_fancy_items_regression test_pass" u" test_fancy_items_regression.py",
+        u"test_fancy_items_regression test_pass",
     ]