Merge pull request #218 from nicoddemus/teardown-crash

Identify correct test crashed during teardown and support multiple test logs from plugins

Author: nicoddemus

changelog/124.bugfix

@@ -0,0 +1 @@
+Fix issue where tests were being incorrectly identified if a worker crashed during the ``teardown`` stage of the test.

changelog/206.feature

@@ -0,0 +1,2 @@
+``xdist`` now supports tests to log results multiple times, improving integration with plugins which require
+it like `pytest-rerunfailures <https://github.com/gocept/pytest-rerunfailures>_`.

testing/acceptance_test.py

@@ -471,18 +471,60 @@ def test_hello(myarg):
     assert result.ret
 
 
-def test_crashing_item(testdir):
+@pytest.mark.parametrize('when', ['setup', 'call', 'teardown'])
+def test_crashing_item(testdir, when):
+    """Ensure crashing item is correctly reported during all testing stages"""
+    code = dict(setup='', call='', teardown='')
+    code[when] = 'py.process.kill(os.getpid())'
     p = testdir.makepyfile("""
-        import py
         import os
-        def test_crash():
-            py.process.kill(os.getpid())
-        def test_noncrash():
+        import py
+        import pytest
+
+        @pytest.fixture
+        def fix():
+            {setup}
+            yield
+            {teardown}
+
+        def test_crash(fix):
+            {call}
             pass
-    """)
+
+        def test_ok():
+            pass
+    """.format(**code))
+    passes = 2 if when == 'teardown' else 1
     result = testdir.runpytest("-n2", p)
     result.stdout.fnmatch_lines([
-        "*crashed*test_crash*", "*1 failed*1 passed*"
+        "*crashed*test_crash*",
+        "*1 failed*%d passed*" % passes,
+    ])
+
+
+def test_multiple_log_reports(testdir):
+    """
+    Ensure that pytest-xdist supports plugins that emit multiple logreports
+    (#206).
+    Inspired by pytest-rerunfailures.
+    """
+    testdir.makeconftest("""
+        from _pytest.runner import runtestprotocol
+        def pytest_runtest_protocol(item, nextitem):
+            item.ihook.pytest_runtest_logstart(nodeid=item.nodeid,
+                                               location=item.location)
+            reports = runtestprotocol(item, nextitem=nextitem)
+            for report in reports:
+                item.ihook.pytest_runtest_logreport(report=report)
+            return True
+    """)
+    testdir.makepyfile("""
+        def test():
+            pass
+    """)
+    result = testdir.runpytest("-n1")
+    result.stdout.fnmatch_lines([
+        "*2 passed*",
     ])
 
 

xdist/dsession.py

@@ -230,18 +230,19 @@ def slave_logstart(self, node, nodeid, location):
             nodeid=nodeid, location=location)
 
     def slave_testreport(self, node, rep):
-        """Emitted when a node calls the pytest_runtest_logreport hook.
-
-        If the node indicates it is finished with a test item, remove
-        the item from the pending list in the scheduler.
-        """
-        if rep.when == "call" or (rep.when == "setup" and not rep.passed):
-            self.sched.mark_test_complete(node, rep.item_index, rep.duration)
-        # self.report_line("testreport %s: %s" %(rep.id, rep.status))
+        """Emitted when a node calls the pytest_runtest_logreport hook."""
         rep.node = node
         self.config.hook.pytest_runtest_logreport(report=rep)
         self._handlefailures(rep)
 
+    def slave_runtest_protocol_complete(self, node, item_index, duration):
+        """
+        Emitted when a node fires the 'runtest_protocol_complete' event,
+        signalling that a test has completed the runtestprotocol and should be
+        removed from the pending list in the scheduler.
+        """
+        self.sched.mark_test_complete(node, item_index, duration)
+
     def slave_collectreport(self, node, rep):
         """Emitted when a node calls the pytest_collectreport hook."""
         if rep.failed:

xdist/remote.py

@@ -8,6 +8,7 @@
 
 import sys
 import os
+import time
 import pytest
 
 
@@ -59,23 +60,29 @@ def pytest_runtestloop(self, session):
             self.log("items to run:", torun)
             # only run if we have an item and a next item
             while len(torun) >= 2:
-                self.run_tests(torun)
+                self.run_one_test(torun)
             if name == "shutdown":
                 if torun:
-                    self.run_tests(torun)
+                    self.run_one_test(torun)
                 break
         return True
 
-    def run_tests(self, torun):
+    def run_one_test(self, torun):
         items = self.session.items
         self.item_index = torun.pop(0)
+        item = items[self.item_index]
         if torun:
             nextitem = items[torun[0]]
         else:
             nextitem = None
+
+        start = time.time()
         self.config.hook.pytest_runtest_protocol(
-            item=items[self.item_index],
+            item=item,
             nextitem=nextitem)
+        duration = time.time() - start
+        self.sendevent("runtest_protocol_complete", item_index=self.item_index,
+                       duration=duration)
 
     def pytest_collection_finish(self, session):
         self.sendevent(

xdist/slavemanage.py

@@ -315,6 +315,8 @@ def process_from_remote(self, eventcall):  # noqa too complex
                 self.notify_inproc(eventname, node=self, rep=rep)
             elif eventname == "collectionfinish":
                 self.notify_inproc(eventname, node=self, ids=kwargs['ids'])
+            elif eventname == "runtest_protocol_complete":
+                self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname == "logwarning":
                 self.notify_inproc(eventname, message=kwargs['message'],
                                    code=kwargs['code'], nodeid=kwargs['nodeid'],