Merge pull request #4965 from nicoddemus/serialization-hooks

Serialization hooks

changelog/4965.trivial.rst

@@ -0,0 +1,9 @@
+New ``pytest_report_to_serializable`` and ``pytest_report_from_serializable`` **experimental** hooks.
+
+These hooks will be used by ``pytest-xdist``, ``pytest-subtests``, and the replacement for
+resultlog to serialize and customize reports.
+
+They are experimental, meaning that their details might change or even be removed
+completely in future patch releases without warning.
+
+Feedback is welcome from plugin authors and users alike.

src/_pytest/config/__init__.py

@@ -140,6 +140,7 @@ default_plugins = (
     "stepwise",
     "warnings",
     "logging",
+    "reports",
 )
 
 

src/_pytest/hookspec.py

@@ -375,6 +375,41 @@ def pytest_runtest_logreport(report):
     the respective phase of executing a test. """
 
 
+@hookspec(firstresult=True)
+def pytest_report_to_serializable(config, report):
+    """
+    .. warning::
+        This hook is experimental and subject to change between pytest releases, even
+        bug fixes.
+
+        The intent is for this to be used by plugins maintained by the core-devs, such
+        as ``pytest-xdist``, ``pytest-subtests``, and as a replacement for the internal
+        'resultlog' plugin.
+
+        In the future it might become part of the public hook API.
+
+    Serializes the given report object into a data structure suitable for sending
+    over the wire, e.g. converted to JSON.
+    """
+
+
+@hookspec(firstresult=True)
+def pytest_report_from_serializable(config, data):
+    """
+    .. warning::
+        This hook is experimental and subject to change between pytest releases, even
+        bug fixes.
+
+        The intent is for this to be used by plugins maintained by the core-devs, such
+        as ``pytest-xdist``, ``pytest-subtests``, and as a replacement for the internal
+        'resultlog' plugin.
+
+        In the future it might become part of the public hook API.
+
+    Restores a report object previously serialized with pytest_report_to_serializable().
+    """
+
+
 # -------------------------------------------------------------------------
 # Fixture related hooks
 # -------------------------------------------------------------------------

src/_pytest/reports.py

@@ -1,8 +1,19 @@
+from pprint import pprint
+
 import py
+import six
 
 from _pytest._code.code import ExceptionInfo
+from _pytest._code.code import ReprEntry
+from _pytest._code.code import ReprEntryNative
+from _pytest._code.code import ReprExceptionInfo
+from _pytest._code.code import ReprFileLocation
+from _pytest._code.code import ReprFuncArgs
+from _pytest._code.code import ReprLocals
+from _pytest._code.code import ReprTraceback
 from _pytest._code.code import TerminalRepr
 from _pytest.outcomes import skip
+from _pytest.pathlib import Path
 
 
 def getslaveinfoline(node):
@@ -137,12 +148,136 @@ class BaseReport(object):
             fspath, lineno, domain = self.location
             return domain
 
+    def _to_json(self):
+        """
+        This was originally the serialize_report() function from xdist (ca03269).
+
+        Returns the contents of this report as a dict of builtin entries, suitable for
+        serialization.
+
+        Experimental method.
+        """
+
+        def disassembled_report(rep):
+            reprtraceback = rep.longrepr.reprtraceback.__dict__.copy()
+            reprcrash = rep.longrepr.reprcrash.__dict__.copy()
+
+            new_entries = []
+            for entry in reprtraceback["reprentries"]:
+                entry_data = {
+                    "type": type(entry).__name__,
+                    "data": entry.__dict__.copy(),
+                }
+                for key, value in entry_data["data"].items():
+                    if hasattr(value, "__dict__"):
+                        entry_data["data"][key] = value.__dict__.copy()
+                new_entries.append(entry_data)
+
+            reprtraceback["reprentries"] = new_entries
+
+            return {
+                "reprcrash": reprcrash,
+                "reprtraceback": reprtraceback,
+                "sections": rep.longrepr.sections,
+            }
+
+        d = self.__dict__.copy()
+        if hasattr(self.longrepr, "toterminal"):
+            if hasattr(self.longrepr, "reprtraceback") and hasattr(
+                self.longrepr, "reprcrash"
+            ):
+                d["longrepr"] = disassembled_report(self)
+            else:
+                d["longrepr"] = six.text_type(self.longrepr)
+        else:
+            d["longrepr"] = self.longrepr
+        for name in d:
+            if isinstance(d[name], (py.path.local, Path)):
+                d[name] = str(d[name])
+            elif name == "result":
+                d[name] = None  # for now
+        return d
+
+    @classmethod
+    def _from_json(cls, reportdict):
+        """
+        This was originally the serialize_report() function from xdist (ca03269).
+
+        Factory method that returns either a TestReport or CollectReport, depending on the calling
+        class. It's the callers responsibility to know which class to pass here.
+
+        Experimental method.
+        """
+        if reportdict["longrepr"]:
+            if (
+                "reprcrash" in reportdict["longrepr"]
+                and "reprtraceback" in reportdict["longrepr"]
+            ):
+
+                reprtraceback = reportdict["longrepr"]["reprtraceback"]
+                reprcrash = reportdict["longrepr"]["reprcrash"]
+
+                unserialized_entries = []
+                reprentry = None
+                for entry_data in reprtraceback["reprentries"]:
+                    data = entry_data["data"]
+                    entry_type = entry_data["type"]
+                    if entry_type == "ReprEntry":
+                        reprfuncargs = None
+                        reprfileloc = None
+                        reprlocals = None
+                        if data["reprfuncargs"]:
+                            reprfuncargs = ReprFuncArgs(**data["reprfuncargs"])
+                        if data["reprfileloc"]:
+                            reprfileloc = ReprFileLocation(**data["reprfileloc"])
+                        if data["reprlocals"]:
+                            reprlocals = ReprLocals(data["reprlocals"]["lines"])
+
+                        reprentry = ReprEntry(
+                            lines=data["lines"],
+                            reprfuncargs=reprfuncargs,
+                            reprlocals=reprlocals,
+                            filelocrepr=reprfileloc,
+                            style=data["style"],
+                        )
+                    elif entry_type == "ReprEntryNative":
+                        reprentry = ReprEntryNative(data["lines"])
+                    else:
+                        _report_unserialization_failure(entry_type, cls, reportdict)
+                    unserialized_entries.append(reprentry)
+                reprtraceback["reprentries"] = unserialized_entries
+
+                exception_info = ReprExceptionInfo(
+                    reprtraceback=ReprTraceback(**reprtraceback),
+                    reprcrash=ReprFileLocation(**reprcrash),
+                )
+
+                for section in reportdict["longrepr"]["sections"]:
+                    exception_info.addsection(*section)
+                reportdict["longrepr"] = exception_info
+
+        return cls(**reportdict)
+
+
+def _report_unserialization_failure(type_name, report_class, reportdict):
+    url = "https://github.com/pytest-dev/pytest/issues"
+    stream = py.io.TextIO()
+    pprint("-" * 100, stream=stream)
+    pprint("INTERNALERROR: Unknown entry type returned: %s" % type_name, stream=stream)
+    pprint("report_name: %s" % report_class, stream=stream)
+    pprint(reportdict, stream=stream)
+    pprint("Please report this bug at %s" % url, stream=stream)
+    pprint("-" * 100, stream=stream)
+    raise RuntimeError(stream.getvalue())
+
 
 class TestReport(BaseReport):
     """ Basic test report object (also used for setup and teardown calls if
     they fail).
     """
 
+    __test__ = False
+
     def __init__(
         self,
         nodeid,
@@ -272,3 +407,21 @@ class CollectErrorRepr(TerminalRepr):
 
     def toterminal(self, out):
         out.line(self.longrepr, red=True)
+
+
+def pytest_report_to_serializable(report):
+    if isinstance(report, (TestReport, CollectReport)):
+        data = report._to_json()
+        data["_report_type"] = report.__class__.__name__
+        return data
+
+
+def pytest_report_from_serializable(data):
+    if "_report_type" in data:
+        if data["_report_type"] == "TestReport":
+            return TestReport._from_json(data)
+        elif data["_report_type"] == "CollectReport":
+            return CollectReport._from_json(data)
+        assert False, "Unknown report_type unserialize data: {}".format(
+            data["_report_type"]
+        )

testing/test_reports.py

@@ -0,0 +1,313 @@
+import pytest
+from _pytest.pathlib import Path
+from _pytest.reports import CollectReport
+from _pytest.reports import TestReport
+
+
+class TestReportSerialization(object):
+    def test_xdist_longrepr_to_str_issue_241(self, testdir):
+        """
+        Regarding issue pytest-xdist#241
+
+        This test came originally from test_remote.py in xdist (ca03269).
+        """
+        testdir.makepyfile(
+            """
+            def test_a(): assert False
+            def test_b(): pass
+        """
+        )
+        reprec = testdir.inline_run()
+        reports = reprec.getreports("pytest_runtest_logreport")
+        assert len(reports) == 6
+        test_a_call = reports[1]
+        assert test_a_call.when == "call"
+        assert test_a_call.outcome == "failed"
+        assert test_a_call._to_json()["longrepr"]["reprtraceback"]["style"] == "long"
+        test_b_call = reports[4]
+        assert test_b_call.when == "call"
+        assert test_b_call.outcome == "passed"
+        assert test_b_call._to_json()["longrepr"] is None
+
+    def test_xdist_report_longrepr_reprcrash_130(self, testdir):
+        """Regarding issue pytest-xdist#130
+
+        This test came originally from test_remote.py in xdist (ca03269).
+        """
+        reprec = testdir.inline_runsource(
+            """
+                    def test_fail():
+                        assert False, 'Expected Message'
+                """
+        )
+        reports = reprec.getreports("pytest_runtest_logreport")
+        assert len(reports) == 3
+        rep = reports[1]
+        added_section = ("Failure Metadata", str("metadata metadata"), "*")
+        rep.longrepr.sections.append(added_section)
+        d = rep._to_json()
+        a = TestReport._from_json(d)
+        # Check assembled == rep
+        assert a.__dict__.keys() == rep.__dict__.keys()
+        for key in rep.__dict__.keys():
+            if key != "longrepr":
+                assert getattr(a, key) == getattr(rep, key)
+        assert rep.longrepr.reprcrash.lineno == a.longrepr.reprcrash.lineno
+        assert rep.longrepr.reprcrash.message == a.longrepr.reprcrash.message
+        assert rep.longrepr.reprcrash.path == a.longrepr.reprcrash.path
+        assert rep.longrepr.reprtraceback.entrysep == a.longrepr.reprtraceback.entrysep
+        assert (
+            rep.longrepr.reprtraceback.extraline == a.longrepr.reprtraceback.extraline
+        )
+        assert rep.longrepr.reprtraceback.style == a.longrepr.reprtraceback.style
+        assert rep.longrepr.sections == a.longrepr.sections
+        # Missing section attribute PR171
+        assert added_section in a.longrepr.sections
+
+    def test_reprentries_serialization_170(self, testdir):
+        """Regarding issue pytest-xdist#170
+
+        This test came originally from test_remote.py in xdist (ca03269).
+        """
+        from _pytest._code.code import ReprEntry
+
+        reprec = testdir.inline_runsource(
+            """
+                            def test_repr_entry():
+                                x = 0
+                                assert x
+                        """,
+            "--showlocals",
+        )
+        reports = reprec.getreports("pytest_runtest_logreport")
+        assert len(reports) == 3
+        rep = reports[1]
+        d = rep._to_json()
+        a = TestReport._from_json(d)
+
+        rep_entries = rep.longrepr.reprtraceback.reprentries
+        a_entries = a.longrepr.reprtraceback.reprentries
+        for i in range(len(a_entries)):
+            assert isinstance(rep_entries[i], ReprEntry)
+            assert rep_entries[i].lines == a_entries[i].lines
+            assert rep_entries[i].reprfileloc.lineno == a_entries[i].reprfileloc.lineno
+            assert (
+                rep_entries[i].reprfileloc.message == a_entries[i].reprfileloc.message
+            )
+            assert rep_entries[i].reprfileloc.path == a_entries[i].reprfileloc.path
+            assert rep_entries[i].reprfuncargs.args == a_entries[i].reprfuncargs.args
+            assert rep_entries[i].reprlocals.lines == a_entries[i].reprlocals.lines
+            assert rep_entries[i].style == a_entries[i].style
+
+    def test_reprentries_serialization_196(self, testdir):
+        """Regarding issue pytest-xdist#196
+
+        This test came originally from test_remote.py in xdist (ca03269).
+        """
+        from _pytest._code.code import ReprEntryNative
+
+        reprec = testdir.inline_runsource(
+            """
+                            def test_repr_entry_native():
+                                x = 0
+                                assert x
+                        """,
+            "--tb=native",
+        )
+        reports = reprec.getreports("pytest_runtest_logreport")
+        assert len(reports) == 3
+        rep = reports[1]
+        d = rep._to_json()
+        a = TestReport._from_json(d)
+
+        rep_entries = rep.longrepr.reprtraceback.reprentries
+        a_entries = a.longrepr.reprtraceback.reprentries
+        for i in range(len(a_entries)):
+            assert isinstance(rep_entries[i], ReprEntryNative)
+            assert rep_entries[i].lines == a_entries[i].lines
+
+    def test_itemreport_outcomes(self, testdir):
+        """
+        This test came originally from test_remote.py in xdist (ca03269).
+        """
+        reprec = testdir.inline_runsource(
+            """
+            import py
+            def test_pass(): pass
+            def test_fail(): 0/0
+            @py.test.mark.skipif("True")
+            def test_skip(): pass
+            def test_skip_imperative():
+                py.test.skip("hello")
+            @py.test.mark.xfail("True")
+            def test_xfail(): 0/0
+            def test_xfail_imperative():
+                py.test.xfail("hello")
+        """
+        )
+        reports = reprec.getreports("pytest_runtest_logreport")
+        assert len(reports) == 17  # with setup/teardown "passed" reports
+        for rep in reports:
+            d = rep._to_json()
+            newrep = TestReport._from_json(d)
+            assert newrep.passed == rep.passed
+            assert newrep.failed == rep.failed
+            assert newrep.skipped == rep.skipped
+            if newrep.skipped and not hasattr(newrep, "wasxfail"):
+                assert len(newrep.longrepr) == 3
+            assert newrep.outcome == rep.outcome
+            assert newrep.when == rep.when
+            assert newrep.keywords == rep.keywords
+            if rep.failed:
+                assert newrep.longreprtext == rep.longreprtext
+
+    def test_collectreport_passed(self, testdir):
+        """This test came originally from test_remote.py in xdist (ca03269)."""
+        reprec = testdir.inline_runsource("def test_func(): pass")
+        reports = reprec.getreports("pytest_collectreport")
+        for rep in reports:
+            d = rep._to_json()
+            newrep = CollectReport._from_json(d)
+            assert newrep.passed == rep.passed
+            assert newrep.failed == rep.failed
+            assert newrep.skipped == rep.skipped
+
+    def test_collectreport_fail(self, testdir):
+        """This test came originally from test_remote.py in xdist (ca03269)."""
+        reprec = testdir.inline_runsource("qwe abc")
+        reports = reprec.getreports("pytest_collectreport")
+        assert reports
+        for rep in reports:
+            d = rep._to_json()
+            newrep = CollectReport._from_json(d)
+            assert newrep.passed == rep.passed
+            assert newrep.failed == rep.failed
+            assert newrep.skipped == rep.skipped
+            if rep.failed:
+                assert newrep.longrepr == str(rep.longrepr)
+
+    def test_extended_report_deserialization(self, testdir):
+        """This test came originally from test_remote.py in xdist (ca03269)."""
+        reprec = testdir.inline_runsource("qwe abc")
+        reports = reprec.getreports("pytest_collectreport")
+        assert reports
+        for rep in reports:
+            rep.extra = True
+            d = rep._to_json()
+            newrep = CollectReport._from_json(d)
+            assert newrep.extra
+            assert newrep.passed == rep.passed
+            assert newrep.failed == rep.failed
+            assert newrep.skipped == rep.skipped
+            if rep.failed:
+                assert newrep.longrepr == str(rep.longrepr)
+
+    def test_paths_support(self, testdir):
+        """Report attributes which are py.path or pathlib objects should become strings."""
+        testdir.makepyfile(
+            """
+            def test_a():
+                assert False
+        """
+        )
+        reprec = testdir.inline_run()
+        reports = reprec.getreports("pytest_runtest_logreport")
+        assert len(reports) == 3
+        test_a_call = reports[1]
+        test_a_call.path1 = testdir.tmpdir
+        test_a_call.path2 = Path(testdir.tmpdir)
+        data = test_a_call._to_json()
+        assert data["path1"] == str(testdir.tmpdir)
+        assert data["path2"] == str(testdir.tmpdir)
+
+    def test_unserialization_failure(self, testdir):
+        """Check handling of failure during unserialization of report types."""
+        testdir.makepyfile(
+            """
+            def test_a():
+                assert False
+        """
+        )
+        reprec = testdir.inline_run()
+        reports = reprec.getreports("pytest_runtest_logreport")
+        assert len(reports) == 3
+        test_a_call = reports[1]
+        data = test_a_call._to_json()
+        entry = data["longrepr"]["reprtraceback"]["reprentries"][0]
+        assert entry["type"] == "ReprEntry"
+
+        entry["type"] = "Unknown"
+        with pytest.raises(
+            RuntimeError, match="INTERNALERROR: Unknown entry type returned: Unknown"
+        ):
+            TestReport._from_json(data)
+
+
+class TestHooks:
+    """Test that the hooks are working correctly for plugins"""
+
+    def test_test_report(self, testdir, pytestconfig):
+        testdir.makepyfile(
+            """
+            def test_a(): assert False
+            def test_b(): pass
+        """
+        )
+        reprec = testdir.inline_run()
+        reports = reprec.getreports("pytest_runtest_logreport")
+        assert len(reports) == 6
+        for rep in reports:
+            data = pytestconfig.hook.pytest_report_to_serializable(
+                config=pytestconfig, report=rep
+            )
+            assert data["_report_type"] == "TestReport"
+            new_rep = pytestconfig.hook.pytest_report_from_serializable(
+                config=pytestconfig, data=data
+            )
+            assert new_rep.nodeid == rep.nodeid
+            assert new_rep.when == rep.when
+            assert new_rep.outcome == rep.outcome
+
+    def test_collect_report(self, testdir, pytestconfig):
+        testdir.makepyfile(
+            """
+            def test_a(): assert False
+            def test_b(): pass
+        """
+        )
+        reprec = testdir.inline_run()
+        reports = reprec.getreports("pytest_collectreport")
+        assert len(reports) == 2
+        for rep in reports:
+            data = pytestconfig.hook.pytest_report_to_serializable(
+                config=pytestconfig, report=rep
+            )
+            assert data["_report_type"] == "CollectReport"
+            new_rep = pytestconfig.hook.pytest_report_from_serializable(
+                config=pytestconfig, data=data
+            )
+            assert new_rep.nodeid == rep.nodeid
+            assert new_rep.when == "collect"
+            assert new_rep.outcome == rep.outcome
+
+    @pytest.mark.parametrize(
+        "hook_name", ["pytest_runtest_logreport", "pytest_collectreport"]
+    )
+    def test_invalid_report_types(self, testdir, pytestconfig, hook_name):
+        testdir.makepyfile(
+            """
+            def test_a(): pass
+            """
+        )
+        reprec = testdir.inline_run()
+        reports = reprec.getreports(hook_name)
+        assert reports
+        rep = reports[0]
+        data = pytestconfig.hook.pytest_report_to_serializable(
+            config=pytestconfig, report=rep
+        )
+        data["_report_type"] = "Unknown"
+        with pytest.raises(AssertionError):
+            _ = pytestconfig.hook.pytest_report_from_serializable(
+                config=pytestconfig, data=data
+            )