conversion of exit codes to enum + exposure (#5420)

conversion of exit codes to enum  + exposure

Author: nicoddemus

changelog/5125.removal.rst

@@ -0,0 +1,5 @@
+``Session.exitcode`` values are now coded in ``pytest.ExitCode``, an ``IntEnum``. This makes the exit code available for consumer code and are more explicit other than just documentation. User defined exit codes are still valid, but should be used with caution.
+
+The team doesn't expect this change to break test suites or plugins in general, except in esoteric/specific scenarios.
+
+**pytest-xdist** users should upgrade to ``1.29.0`` or later, as ``pytest-xdist`` required a compatibility fix because of this change.

doc/en/reference.rst

@@ -727,6 +727,14 @@ ExceptionInfo
 .. autoclass:: _pytest._code.ExceptionInfo
     :members:
 
+
+pytest.ExitCode
+~~~~~~~~~~~~~~~
+
+.. autoclass:: _pytest.main.ExitCode
+    :members:
+
+
 FixtureDef
 ~~~~~~~~~~
 

doc/en/usage.rst

@@ -33,6 +33,8 @@ Running ``pytest`` can result in six different exit codes:
 :Exit code 4: pytest command line usage error
 :Exit code 5: No tests were collected
 
+They are repressended by the :class:`_pytest.main.ExitCode` enum.
+
 Getting help on version, option names, environment variables
 --------------------------------------------------------------
 

src/_pytest/config/__init__.py

@@ -48,7 +48,7 @@ def main(args=None, plugins=None):
     :arg plugins: list of plugin objects to be auto-registered during
                   initialization.
     """
-    from _pytest.main import EXIT_USAGEERROR
+    from _pytest.main import ExitCode
 
     try:
         try:
@@ -78,7 +78,7 @@ def main(args=None, plugins=None):
         tw = py.io.TerminalWriter(sys.stderr)
         for msg in e.args:
             tw.line("ERROR: {}\n".format(msg), red=True)
-        return EXIT_USAGEERROR
+        return ExitCode.USAGE_ERROR
 
 
 class cmdline:  # compatibility namespace

src/_pytest/main.py

@@ -1,4 +1,5 @@
 """ core implementation of testing process: init, session, runtest loop. """
+import enum
 import fnmatch
 import functools
 import os
@@ -18,13 +19,26 @@
 from _pytest.outcomes import exit
 from _pytest.runner import collect_one_node
 
-# exitcodes for the command line
-EXIT_OK = 0
-EXIT_TESTSFAILED = 1
-EXIT_INTERRUPTED = 2
-EXIT_INTERNALERROR = 3
-EXIT_USAGEERROR = 4
-EXIT_NOTESTSCOLLECTED = 5
+
+class ExitCode(enum.IntEnum):
+    """
+    Encodes the valid exit codes by pytest.
+
+    Currently users and plugins may supply other exit codes as well.
+    """
+
+    #: tests passed
+    OK = 0
+    #: tests failed
+    TESTS_FAILED = 1
+    #: pytest was interrupted
+    INTERRUPTED = 2
+    #: an internal error got in the way
+    INTERNAL_ERROR = 3
+    #: pytest was missused
+    USAGE_ERROR = 4
+    #: pytest couldnt find tests
+    NO_TESTS_COLLECTED = 5
 
 
 def pytest_addoption(parser):
@@ -188,7 +202,7 @@ def pytest_configure(config):
 def wrap_session(config, doit):
     """Skeleton command line program"""
     session = Session(config)
-    session.exitstatus = EXIT_OK
+    session.exitstatus = ExitCode.OK
     initstate = 0
     try:
         try:
@@ -198,13 +212,13 @@ def wrap_session(config, doit):
             initstate = 2
             session.exitstatus = doit(config, session) or 0
         except UsageError:
-            session.exitstatus = EXIT_USAGEERROR
+            session.exitstatus = ExitCode.USAGE_ERROR
             raise
         except Failed:
-            session.exitstatus = EXIT_TESTSFAILED
+            session.exitstatus = ExitCode.TESTS_FAILED
         except (KeyboardInterrupt, exit.Exception):
             excinfo = _pytest._code.ExceptionInfo.from_current()
-            exitstatus = EXIT_INTERRUPTED
+            exitstatus = ExitCode.INTERRUPTED
             if isinstance(excinfo.value, exit.Exception):
                 if excinfo.value.returncode is not None:
                     exitstatus = excinfo.value.returncode
@@ -217,7 +231,7 @@ def wrap_session(config, doit):
         except:  # noqa
             excinfo = _pytest._code.ExceptionInfo.from_current()
             config.notify_exception(excinfo, config.option)
-            session.exitstatus = EXIT_INTERNALERROR
+            session.exitstatus = ExitCode.INTERNAL_ERROR
             if excinfo.errisinstance(SystemExit):
                 sys.stderr.write("mainloop: caught unexpected SystemExit!\n")
 
@@ -243,9 +257,9 @@ def _main(config, session):
     config.hook.pytest_runtestloop(session=session)
 
     if session.testsfailed:
-        return EXIT_TESTSFAILED
+        return ExitCode.TESTS_FAILED
     elif session.testscollected == 0:
-        return EXIT_NOTESTSCOLLECTED
+        return ExitCode.NO_TESTS_COLLECTED
 
 
 def pytest_collection(session):

src/_pytest/pytester.py

@@ -19,8 +19,7 @@
 from _pytest.assertion.rewrite import AssertionRewritingHook
 from _pytest.capture import MultiCapture
 from _pytest.capture import SysCapture
-from _pytest.main import EXIT_INTERRUPTED
-from _pytest.main import EXIT_OK
+from _pytest.main import ExitCode
 from _pytest.main import Session
 from _pytest.monkeypatch import MonkeyPatch
 from _pytest.pathlib import Path
@@ -691,7 +690,7 @@ def getnode(self, config, arg):
         p = py.path.local(arg)
         config.hook.pytest_sessionstart(session=session)
         res = session.perform_collect([str(p)], genitems=False)[0]
-        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
+        config.hook.pytest_sessionfinish(session=session, exitstatus=ExitCode.OK)
         return res
 
     def getpathnode(self, path):
@@ -708,11 +707,11 @@ def getpathnode(self, path):
         x = session.fspath.bestrelpath(path)
         config.hook.pytest_sessionstart(session=session)
         res = session.perform_collect([x], genitems=False)[0]
-        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
+        config.hook.pytest_sessionfinish(session=session, exitstatus=ExitCode.OK)
         return res
 
     def genitems(self, colitems):
-        """Generate all test items from a collection node.
+        """Generate all test items from a collection node.src/_pytest/main.py
 
         This recurses into the collection node and returns a list of all the
         test items contained within.
@@ -841,7 +840,7 @@ class reprec:
 
             # typically we reraise keyboard interrupts from the child run
             # because it's our user requesting interruption of the testing
-            if ret == EXIT_INTERRUPTED and not no_reraise_ctrlc:
+            if ret == ExitCode.INTERRUPTED and not no_reraise_ctrlc:
                 calls = reprec.getcalls("pytest_keyboard_interrupt")
                 if calls and calls[-1].excinfo.type == KeyboardInterrupt:
                     raise KeyboardInterrupt()

src/_pytest/terminal.py

@@ -16,11 +16,7 @@
 
 import pytest
 from _pytest import nodes
-from _pytest.main import EXIT_INTERRUPTED
-from _pytest.main import EXIT_NOTESTSCOLLECTED
-from _pytest.main import EXIT_OK
-from _pytest.main import EXIT_TESTSFAILED
-from _pytest.main import EXIT_USAGEERROR
+from _pytest.main import ExitCode
 
 REPORT_COLLECTING_RESOLUTION = 0.5
 
@@ -654,17 +650,17 @@ def pytest_sessionfinish(self, exitstatus):
         outcome.get_result()
         self._tw.line("")
         summary_exit_codes = (
-            EXIT_OK,
-            EXIT_TESTSFAILED,
-            EXIT_INTERRUPTED,
-            EXIT_USAGEERROR,
-            EXIT_NOTESTSCOLLECTED,
+            ExitCode.OK,
+            ExitCode.TESTS_FAILED,
+            ExitCode.INTERRUPTED,
+            ExitCode.USAGE_ERROR,
+            ExitCode.NO_TESTS_COLLECTED,
         )
         if exitstatus in summary_exit_codes:
             self.config.hook.pytest_terminal_summary(
                 terminalreporter=self, exitstatus=exitstatus, config=self.config
             )
-        if exitstatus == EXIT_INTERRUPTED:
+        if exitstatus == ExitCode.INTERRUPTED:
             self._report_keyboardinterrupt()
             del self._keyboardinterrupt_memo
         self.summary_stats()

src/pytest.py

@@ -15,6 +15,7 @@
 from _pytest.fixtures import fixture
 from _pytest.fixtures import yield_fixture
 from _pytest.freeze_support import freeze_includes
+from _pytest.main import ExitCode
 from _pytest.main import Session
 from _pytest.mark import MARK_GEN as mark
 from _pytest.mark import param
@@ -57,6 +58,7 @@
     "Collector",
     "deprecated_call",
     "exit",
+    "ExitCode",
     "fail",
     "File",
     "fixture",

testing/acceptance_test.py

@@ -8,8 +8,7 @@
 import py
 
 import pytest
-from _pytest.main import EXIT_NOTESTSCOLLECTED
-from _pytest.main import EXIT_USAGEERROR
+from _pytest.main import ExitCode
 from _pytest.warnings import SHOW_PYTEST_WARNINGS_ARG
 
 
@@ -24,7 +23,7 @@ class TestGeneralUsage:
     def test_config_error(self, testdir):
         testdir.copy_example("conftest_usageerror/conftest.py")
         result = testdir.runpytest(testdir.tmpdir)
-        assert result.ret == EXIT_USAGEERROR
+        assert result.ret == ExitCode.USAGE_ERROR
         result.stderr.fnmatch_lines(["*ERROR: hello"])
         result.stdout.fnmatch_lines(["*pytest_unconfigure_called"])
 
@@ -83,7 +82,7 @@ def pytest_unconfigure():
         """
         )
         result = testdir.runpytest("-s", "asd")
-        assert result.ret == 4  # EXIT_USAGEERROR
+        assert result.ret == ExitCode.USAGE_ERROR
         result.stderr.fnmatch_lines(["ERROR: file not found*asd"])
         result.stdout.fnmatch_lines(["*---configure", "*---unconfigure"])
 
@@ -229,7 +228,7 @@ def pytest_collect_directory():
         """
         )
         result = testdir.runpytest()
-        assert result.ret == EXIT_NOTESTSCOLLECTED
+        assert result.ret == ExitCode.NO_TESTS_COLLECTED
         result.stdout.fnmatch_lines(["*1 skip*"])
 
     def test_issue88_initial_file_multinodes(self, testdir):
@@ -247,7 +246,7 @@ def test_issue93_initialnode_importing_capturing(self, testdir):
         """
         )
         result = testdir.runpytest()
-        assert result.ret == EXIT_NOTESTSCOLLECTED
+        assert result.ret == ExitCode.NO_TESTS_COLLECTED
         assert "should not be seen" not in result.stdout.str()
         assert "stderr42" not in result.stderr.str()
 
@@ -290,13 +289,13 @@ def test_issue109_sibling_conftests_not_loaded(self, testdir):
         sub2 = testdir.mkdir("sub2")
         sub1.join("conftest.py").write("assert 0")
         result = testdir.runpytest(sub2)
-        assert result.ret == EXIT_NOTESTSCOLLECTED
+        assert result.ret == ExitCode.NO_TESTS_COLLECTED
         sub2.ensure("__init__.py")
         p = sub2.ensure("test_hello.py")
         result = testdir.runpytest(p)
-        assert result.ret == EXIT_NOTESTSCOLLECTED
+        assert result.ret == ExitCode.NO_TESTS_COLLECTED
         result = testdir.runpytest(sub1)
-        assert result.ret == EXIT_USAGEERROR
+        assert result.ret == ExitCode.USAGE_ERROR
 
     def test_directory_skipped(self, testdir):
         testdir.makeconftest(
@@ -308,7 +307,7 @@ def pytest_ignore_collect():
         )
         testdir.makepyfile("def test_hello(): pass")
         result = testdir.runpytest()
-        assert result.ret == EXIT_NOTESTSCOLLECTED
+        assert result.ret == ExitCode.NO_TESTS_COLLECTED
         result.stdout.fnmatch_lines(["*1 skipped*"])
 
     def test_multiple_items_per_collector_byid(self, testdir):
@@ -410,18 +409,18 @@ def test_a():
     def test_report_all_failed_collections_initargs(self, testdir):
         testdir.makeconftest(
             """
-            from _pytest.main import EXIT_USAGEERROR
+            from _pytest.main import ExitCode
 
             def pytest_sessionfinish(exitstatus):
-                assert exitstatus == EXIT_USAGEERROR
+                assert exitstatus == ExitCode.USAGE_ERROR
                 print("pytest_sessionfinish_called")
             """
         )
         testdir.makepyfile(test_a="def", test_b="def")
         result = testdir.runpytest("test_a.py::a", "test_b.py::b")
         result.stderr.fnmatch_lines(["*ERROR*test_a.py::a*", "*ERROR*test_b.py::b*"])
         result.stdout.fnmatch_lines(["pytest_sessionfinish_called"])
-        assert result.ret == EXIT_USAGEERROR
+        assert result.ret == ExitCode.USAGE_ERROR
 
     @pytest.mark.usefixtures("recwarn")
     def test_namespace_import_doesnt_confuse_import_hook(self, testdir):
@@ -612,7 +611,7 @@ def test_invoke_with_invalid_type(self, capsys):
 
     def test_invoke_with_path(self, tmpdir, capsys):
         retcode = pytest.main(tmpdir)
-        assert retcode == EXIT_NOTESTSCOLLECTED
+        assert retcode == ExitCode.NO_TESTS_COLLECTED
         out, err = capsys.readouterr()
 
     def test_invoke_plugin_api(self, testdir, capsys):
@@ -1160,7 +1159,7 @@ def test_fixture_mock_integration(testdir):
 
 def test_usage_error_code(testdir):
     result = testdir.runpytest("-unknown-option-")
-    assert result.ret == EXIT_USAGEERROR
+    assert result.ret == ExitCode.USAGE_ERROR
 
 
 @pytest.mark.filterwarnings("default")

testing/python/collect.py

@@ -4,7 +4,7 @@
 
 import _pytest._code
 import pytest
-from _pytest.main import EXIT_NOTESTSCOLLECTED
+from _pytest.main import ExitCode
 from _pytest.nodes import Collector
 
 
@@ -246,7 +246,7 @@ def prop(self):
         """
         )
         result = testdir.runpytest()
-        assert result.ret == EXIT_NOTESTSCOLLECTED
+        assert result.ret == ExitCode.NO_TESTS_COLLECTED
 
 
 class TestFunction:
@@ -1140,7 +1140,7 @@ class Test(object):
     )
     result = testdir.runpytest()
     assert "TypeError" not in result.stdout.str()
-    assert result.ret == EXIT_NOTESTSCOLLECTED
+    assert result.ret == ExitCode.NO_TESTS_COLLECTED
 
 
 def test_collect_functools_partial(testdir):

testing/test_assertrewrite.py

@@ -15,7 +15,7 @@
 from _pytest.assertion.rewrite import AssertionRewritingHook
 from _pytest.assertion.rewrite import PYTEST_TAG
 from _pytest.assertion.rewrite import rewrite_asserts
-from _pytest.main import EXIT_NOTESTSCOLLECTED
+from _pytest.main import ExitCode
 
 
 def setup_module(mod):
@@ -692,7 +692,7 @@ def test_zipfile(self, testdir):
             import test_gum.test_lizard"""
             % (z_fn,)
         )
-        assert testdir.runpytest().ret == EXIT_NOTESTSCOLLECTED
+        assert testdir.runpytest().ret == ExitCode.NO_TESTS_COLLECTED
 
     def test_readonly(self, testdir):
         sub = testdir.mkdir("testing")
@@ -792,7 +792,7 @@ def test_package_without__init__py(self, testdir):
         pkg = testdir.mkdir("a_package_without_init_py")
         pkg.join("module.py").ensure()
         testdir.makepyfile("import a_package_without_init_py.module")
-        assert testdir.runpytest().ret == EXIT_NOTESTSCOLLECTED
+        assert testdir.runpytest().ret == ExitCode.NO_TESTS_COLLECTED
 
     def test_rewrite_warning(self, testdir):
         testdir.makeconftest(