Merge pull request #817 from nicoddemus/non-collection-exit-error

return non-zero exit code if no tests are collected

.travis.yml

@@ -28,7 +28,7 @@ env:
     - TESTENV=py35
     - TESTENV=pypy
 
-script: tox --recreate -i ALL=https://devpi.net/hpk/dev/ -e $TESTENV
+script: tox --recreate -e $TESTENV
 
 notifications:
   irc:

AUTHORS

@@ -15,6 +15,7 @@ Bob Ippolito
 Brian Dorsey
 Brian Okken
 Brianna Laugher
+Bruno Oliveira
 Carl Friedrich Bolz
 Charles Cloud
 Chris Lamb

CHANGELOG

@@ -58,6 +58,15 @@
 - Summary bar now is colored yellow for warning
   situations such as: all tests either were skipped or xpass/xfailed,
   or no tests were run at all (this is a partial fix for issue500).
+- fix issue812: pytest now exits with status code 5 in situations where no
+  tests were run at all, such as the directory given in the command line does
+  not contain any tests or as result of a command line option filters
+  all out all tests (-k for example).
+  Thanks Eric Siegerman (issue812) and Bruno Oliveira for the PR.
+
+- Summary bar now is colored yellow for warning
+  situations such as: all tests either were skipped or xpass/xfailed,
+  or no tests were run at all (related to issue500).
   Thanks Eric Siegerman.
 
 - New `testpaths` ini option: list of directories to search for tests

_pytest/main.py

@@ -19,6 +19,7 @@ EXIT_TESTSFAILED = 1
 EXIT_INTERRUPTED = 2
 EXIT_INTERNALERROR = 3
 EXIT_USAGEERROR = 4
+EXIT_NOTESTSCOLLECTED = 5
 
 name_re = re.compile("^[a-zA-Z_]\w*$")
 
@@ -100,8 +101,10 @@ def wrap_session(config, doit):
             if excinfo.errisinstance(SystemExit):
                 sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
         else:
-            if session._testsfailed:
+            if session.testsfailed:
                 session.exitstatus = EXIT_TESTSFAILED
+            elif session.testscollected == 0:
+                session.exitstatus = EXIT_NOTESTSCOLLECTED
     finally:
         excinfo = None  # Explicitly break reference cycle.
         session.startdir.chdir()
@@ -509,7 +512,8 @@ class Session(FSCollector):
         FSCollector.__init__(self, config.rootdir, parent=None,
                              config=config, session=self)
         self._fs2hookproxy = {}
-        self._testsfailed = 0
+        self.testsfailed = 0
+        self.testscollected = 0
         self.shouldstop = False
         self.trace = config.trace.root.get("collection")
         self._norecursepatterns = config.getini("norecursedirs")
@@ -527,11 +531,11 @@ class Session(FSCollector):
     @pytest.hookimpl(tryfirst=True)
     def pytest_runtest_logreport(self, report):
         if report.failed and not hasattr(report, 'wasxfail'):
-            self._testsfailed += 1
+            self.testsfailed += 1
             maxfail = self.config.getvalue("maxfail")
-            if maxfail and self._testsfailed >= maxfail:
+            if maxfail and self.testsfailed >= maxfail:
                 self.shouldstop = "stopping after %d failures" % (
-                    self._testsfailed)
+                    self.testsfailed)
     pytest_collectreport = pytest_runtest_logreport
 
     def isinitpath(self, path):
@@ -564,6 +568,7 @@ class Session(FSCollector):
                 config=self.config, items=items)
         finally:
             hook.pytest_collection_finish(session=self)
+        self.testscollected = len(items)
         return items
 
     def _perform_collect(self, args, genitems):

_pytest/terminal.py

@@ -2,6 +2,8 @@
 
 This is a good source for looking at the various reporting hooks.
 """
+from _pytest.main import EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, \
+    EXIT_USAGEERROR, EXIT_NOTESTSCOLLECTED
 import pytest
 import pluggy
 import py
@@ -355,12 +357,15 @@ class TerminalReporter:
         outcome = yield
         outcome.get_result()
         self._tw.line("")
-        if exitstatus in (0, 1, 2, 4):
+        summary_exit_codes = (
+            EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, EXIT_USAGEERROR,
+            EXIT_NOTESTSCOLLECTED)
+        if exitstatus in summary_exit_codes:
             self.summary_errors()
             self.summary_failures()
             self.summary_warnings()
             self.config.hook.pytest_terminal_summary(terminalreporter=self)
-        if exitstatus == 2:
+        if exitstatus == EXIT_INTERRUPTED:
             self._report_keyboardinterrupt()
             del self._keyboardinterrupt_memo
         self.summary_deselected()

testing/acceptance_test.py

@@ -1,5 +1,7 @@
 import sys
 import py, pytest
+from _pytest.main import EXIT_NOTESTSCOLLECTED, EXIT_USAGEERROR
+
 
 class TestGeneralUsage:
     def test_config_error(self, testdir):
@@ -147,7 +149,7 @@ class TestGeneralUsage:
                 pytest.skip("early")
         """)
         result = testdir.runpytest()
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         result.stdout.fnmatch_lines([
             "*1 skip*"
         ])
@@ -177,7 +179,7 @@ class TestGeneralUsage:
             sys.stderr.write("stder42\\n")
         """)
         result = testdir.runpytest()
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         assert "should not be seen" not in result.stdout.str()
         assert "stderr42" not in result.stderr.str()
 
@@ -212,13 +214,13 @@ class TestGeneralUsage:
         sub2 = testdir.tmpdir.mkdir("sub2")
         sub1.join("conftest.py").write("assert 0")
         result = testdir.runpytest(sub2)
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         sub2.ensure("__init__.py")
         p = sub2.ensure("test_hello.py")
         result = testdir.runpytest(p)
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         result = testdir.runpytest(sub1)
-        assert result.ret != 0
+        assert result.ret == EXIT_USAGEERROR
 
     def test_directory_skipped(self, testdir):
         testdir.makeconftest("""
@@ -228,7 +230,7 @@ class TestGeneralUsage:
         """)
         testdir.makepyfile("def test_hello(): pass")
         result = testdir.runpytest()
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         result.stdout.fnmatch_lines([
             "*1 skipped*"
         ])
@@ -479,7 +481,7 @@ class TestInvocationVariants:
 
     def test_invoke_with_path(self, tmpdir, capsys):
         retcode = pytest.main(tmpdir)
-        assert not retcode
+        assert retcode == EXIT_NOTESTSCOLLECTED
         out, err = capsys.readouterr()
 
     def test_invoke_plugin_api(self, testdir, capsys):

testing/python/collect.py

@@ -1,6 +1,8 @@
 import sys
 from textwrap import dedent
 import pytest, py
+from _pytest.main import EXIT_NOTESTSCOLLECTED
+
 
 class TestModule:
     def test_failing_import(self, testdir):
@@ -906,7 +908,7 @@ def test_unorderable_types(testdir):
     """)
     result = testdir.runpytest()
     assert "TypeError" not in result.stdout.str()
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED    
 
 
 def test_collect_functools_partial(testdir):

testing/test_assertrewrite.py

@@ -12,6 +12,7 @@ if sys.platform.startswith("java"):
 
 from _pytest.assertion import util
 from _pytest.assertion.rewrite import rewrite_asserts, PYTEST_TAG
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 
 
 def setup_module(mod):
@@ -429,7 +430,7 @@ class TestRewriteOnImport:
             import sys
             sys.path.append(%r)
             import test_gum.test_lizard""" % (z_fn,))
-        assert testdir.runpytest().ret == 0
+        assert testdir.runpytest().ret == EXIT_NOTESTSCOLLECTED
 
     def test_readonly(self, testdir):
         sub = testdir.mkdir("testing")
@@ -497,7 +498,7 @@ def test_rewritten():
         pkg = testdir.mkdir('a_package_without_init_py')
         pkg.join('module.py').ensure()
         testdir.makepyfile("import a_package_without_init_py.module")
-        assert testdir.runpytest().ret == 0
+        assert testdir.runpytest().ret == EXIT_NOTESTSCOLLECTED
 
 class TestAssertionRewriteHookDetails(object):
     def test_loader_is_package_false_for_module(self, testdir):

testing/test_capture.py

@@ -10,6 +10,7 @@ import contextlib
 
 from _pytest import capture
 from _pytest.capture import CaptureManager
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 from py.builtin import print_
 
 needsosdup = pytest.mark.xfail("not hasattr(os, 'dup')")
@@ -365,7 +366,7 @@ class TestLoggingInteraction:
         """)
         # make sure that logging is still captured in tests
         result = testdir.runpytest_subprocess("-s", "-p", "no:capturelog")
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         result.stderr.fnmatch_lines([
             "WARNING*hello435*",
         ])

testing/test_collection.py

@@ -1,6 +1,6 @@
 import pytest, py
 
-from _pytest.main import Session
+from _pytest.main import Session, EXIT_NOTESTSCOLLECTED
 
 class TestCollector:
     def test_collect_versus_item(self):
@@ -247,10 +247,10 @@ class TestCustomConftests:
         p = testdir.makepyfile("def test_hello(): pass")
         result = testdir.runpytest(p)
         assert result.ret == 0
-        assert "1 passed" in result.stdout.str()
+        result.stdout.fnmatch_lines("*1 passed*")
         result = testdir.runpytest()
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
-        assert "1 passed" not in result.stdout.str()
+        result.stdout.fnmatch_lines("*collected 0 items*")
 
     def test_collectignore_exclude_on_option(self, testdir):
         testdir.makeconftest("""
@@ -264,7 +264,7 @@ class TestCustomConftests:
         testdir.mkdir("hello")
         testdir.makepyfile(test_world="def test_hello(): pass")
         result = testdir.runpytest()
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         assert "passed" not in result.stdout.str()
         result = testdir.runpytest("--XX")
         assert result.ret == 0

testing/test_config.py

@@ -1,6 +1,7 @@
 import py, pytest
 
 from _pytest.config import getcfg, get_common_ancestor, determine_setup
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 
 class TestParseIni:
     def test_getcfg_and_config(self, testdir, tmpdir):
@@ -343,7 +344,7 @@ def test_invalid_options_show_extra_information(testdir):
 @pytest.mark.skipif("sys.platform == 'win32'")
 def test_toolongargs_issue224(testdir):
     result = testdir.runpytest("-m", "hello" * 500)
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED
 
 def test_notify_exception(testdir, capfd):
     config = testdir.parseconfig()

testing/test_conftest.py

@@ -1,6 +1,7 @@
 from textwrap import dedent
 import py, pytest
 from _pytest.config import PytestPluginManager
+from _pytest.main import EXIT_NOTESTSCOLLECTED, EXIT_USAGEERROR
 
 
 @pytest.fixture(scope="module", params=["global", "inpackage"])
@@ -166,7 +167,10 @@ def test_conftest_confcutdir(testdir):
 def test_no_conftest(testdir):
     testdir.makeconftest("assert 0")
     result = testdir.runpytest("--noconftest")
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED
+
+    result = testdir.runpytest()
+    assert result.ret == EXIT_USAGEERROR
 
 def test_conftest_existing_resultlog(testdir):
     x = testdir.mkdir("tests")

testing/test_helpconfig.py

@@ -1,3 +1,4 @@
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 import pytest
 
 def test_version(testdir, pytestconfig):
@@ -43,7 +44,7 @@ def test_hookvalidation_optional(testdir):
             pass
     """)
     result = testdir.runpytest()
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED
 
 def test_traceconfig(testdir):
     result = testdir.runpytest("--traceconfig")
@@ -54,14 +55,14 @@ def test_traceconfig(testdir):
 
 def test_debug(testdir, monkeypatch):
     result = testdir.runpytest_subprocess("--debug")
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED
     p = testdir.tmpdir.join("pytestdebug.log")
     assert "pytest_sessionstart" in p.read()
 
 def test_PYTEST_DEBUG(testdir, monkeypatch):
     monkeypatch.setenv("PYTEST_DEBUG", "1")
     result = testdir.runpytest_subprocess()
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED
     result.stderr.fnmatch_lines([
         "*pytest_plugin_registered*",
         "*manager*PluginManager*"

testing/test_junitxml.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 from xml.dom import minidom
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 import py, sys, os
 from _pytest.junitxml import LogXML
 
@@ -298,7 +299,7 @@ class TestPython:
     def test_collect_skipped(self, testdir):
         testdir.makepyfile("import pytest; pytest.skip('xyz')")
         result, dom = runandparse(testdir)
-        assert not result.ret
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         node = dom.getElementsByTagName("testsuite")[0]
         assert_attr(node, skips=1, tests=0)
         tnode = node.getElementsByTagName("testcase")[0]

testing/test_pluginmanager.py

@@ -3,6 +3,7 @@ import py
 import os
 
 from _pytest.config import get_config, PytestPluginManager
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 
 @pytest.fixture
 def pytestpm():
@@ -223,7 +224,7 @@ class TestPytestPluginManager:
         p.copy(p.dirpath("skipping2.py"))
         monkeypatch.setenv("PYTEST_PLUGINS", "skipping2")
         result = testdir.runpytest("-rw", "-p", "skipping1", syspathinsert=True)
-        assert result.ret == 0
+        assert result.ret == EXIT_NOTESTSCOLLECTED
         result.stdout.fnmatch_lines([
             "WI1*skipped plugin*skipping1*hello*",
             "WI1*skipped plugin*skipping2*hello*",

testing/test_runner.py

@@ -293,8 +293,8 @@ class TestExecutionForked(BaseFunctionalTests):
 
     def getrunner(self):
         # XXX re-arrange this test to live in pytest-xdist
-        xplugin = pytest.importorskip("xdist.plugin")
+        boxed = pytest.importorskip("xdist.boxed")
-        return xplugin.forked_run_report
+        return boxed.forked_run_report
 
     def test_suicide(self, testdir):
         reports = testdir.runitem("""
@@ -431,6 +431,27 @@ def test_pytest_fail_notrace(testdir):
     ])
     assert 'def teardown_function' not in result.stdout.str()
 
+
+def test_pytest_no_tests_collected_exit_status(testdir):
+    result = testdir.runpytest()
+    result.stdout.fnmatch_lines('*collected 0 items*')
+    assert result.ret == main.EXIT_NOTESTSCOLLECTED
+
+    testdir.makepyfile(test_foo="""
+        def test_foo():
+            assert 1
+    """)
+    result = testdir.runpytest()
+    result.stdout.fnmatch_lines('*collected 1 items*')
+    result.stdout.fnmatch_lines('*1 passed*')
+    assert result.ret == main.EXIT_OK
+
+    result = testdir.runpytest('-k nonmatch')
+    result.stdout.fnmatch_lines('*collected 1 items*')
+    result.stdout.fnmatch_lines('*1 deselected*')
+    assert result.ret == main.EXIT_NOTESTSCOLLECTED
+
+
 def test_exception_printing_skip():
     try:
         pytest.skip("hello")

testing/test_session.py

@@ -1,5 +1,7 @@
 import pytest
 
+from _pytest.main import EXIT_NOTESTSCOLLECTED
+
 class SessionTests:
     def test_basic_testitem_events(self, testdir):
         tfile = testdir.makepyfile("""
@@ -239,4 +241,4 @@ def test_sessionfinish_with_start(testdir):
 
     """)
     res = testdir.runpytest("--collect-only")
-    assert res.ret == 0
+    assert res.ret == EXIT_NOTESTSCOLLECTED

testing/test_terminal.py

@@ -7,6 +7,7 @@ import py
 import pluggy
 import sys
 
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 from _pytest.terminal import TerminalReporter, repr_pythonversion, getreportopt
 from _pytest.terminal import build_summary_stats_line, _plugin_nameversions
 from _pytest import runner
@@ -596,7 +597,7 @@ def test_traceconfig(testdir, monkeypatch):
     result.stdout.fnmatch_lines([
         "*active plugins*"
     ])
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED
 
 
 class TestGenericReporting:

testing/test_unittest.py

@@ -1,3 +1,4 @@
+from _pytest.main import EXIT_NOTESTSCOLLECTED
 import pytest
 
 def test_simple_unittest(testdir):
@@ -41,7 +42,7 @@ def test_isclasscheck_issue53(testdir):
         E = _E()
     """)
     result = testdir.runpytest(testpath)
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED
 
 def test_setup(testdir):
     testpath = testdir.makepyfile("""
@@ -572,7 +573,7 @@ def test_unorderable_types(testdir):
     """)
     result = testdir.runpytest()
     assert "TypeError" not in result.stdout.str()
-    assert result.ret == 0
+    assert result.ret == EXIT_NOTESTSCOLLECTED
 
 def test_unittest_typerror_traceback(testdir):
     testdir.makepyfile("""

tox.ini

@@ -22,7 +22,7 @@ deps=
 [testenv:py27-subprocess]
 changedir=.
 basepython=python2.7
-deps=pytest-xdist
+deps=pytest-xdist>=1.13
     mock
     nose
 commands=
@@ -37,7 +37,7 @@ deps = pytest-flakes>=0.2
 commands = py.test --flakes -m flakes _pytest testing
 
 [testenv:py27-xdist]
-deps=pytest-xdist
+deps=pytest-xdist>=1.13
     mock
     nose
 commands=
@@ -63,7 +63,7 @@ commands=
   py.test -rfsxX test_pdb.py test_terminal.py test_unittest.py
 
 [testenv:py27-nobyte]
-deps=pytest-xdist
+deps=pytest-xdist>=1.13
 distribute=true
 setenv=
     PYTHONDONTWRITEBYTECODE=1