From bfc3e48fd57f8ae889efd14b2e9b440b08208dcf Mon Sep 17 00:00:00 2001 From: Eric Siegerman Date: Thu, 2 Jul 2015 13:03:05 -0400 Subject: [PATCH 1/9] Factor out build_summary_stats_line(), and add tests --HG-- branch : esiegerman/summary_colors --- _pytest/terminal.py | 41 +++++++++++++++++-------------- testing/test_terminal.py | 53 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 19 deletions(-) diff --git a/_pytest/terminal.py b/_pytest/terminal.py index 576893bbfdd..82da2045de3 100644 --- a/_pytest/terminal.py +++ b/_pytest/terminal.py @@ -487,26 +487,9 @@ def _outrep_summary(self, rep): def summary_stats(self): session_duration = time.time() - self._sessionstarttime - - keys = ("failed passed skipped deselected " - "xfailed xpassed warnings").split() - for key in self.stats.keys(): - if key not in keys: - keys.append(key) - parts = [] - for key in keys: - if key: # setup/teardown reports have an empty key, ignore them - val = self.stats.get(key, None) - if val: - parts.append("%d %s" % (len(val), key)) - line = ", ".join(parts) + (line, color) = build_summary_stats_line(self.stats) msg = "%s in %.2f seconds" % (line, session_duration) - - markup = {'bold': True} - if 'failed' in self.stats or 'error' in self.stats: - markup = {'red': True, 'bold': True} - else: - markup = {'green': True, 'bold': True} + markup = {color: True, 'bold': True} if self.verbosity >= 0: self.write_sep("=", msg, **markup) @@ -542,3 +525,23 @@ def flatten(l): else: yield x +def build_summary_stats_line(stats): + keys = ("failed passed skipped deselected " + "xfailed xpassed warnings").split() + for key in stats.keys(): + if key not in keys: + keys.append(key) + parts = [] + for key in keys: + if key: # setup/teardown reports have an empty key, ignore them + val = stats.get(key, None) + if val: + parts.append("%d %s" % (len(val), key)) + line = ", ".join(parts) + + if 'failed' in stats or 'error' in stats: + color = 'red' + else: + color = 'green' + + return (line, color) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index fba3021e064..9390ac15310 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -7,6 +7,7 @@ import sys from _pytest.terminal import TerminalReporter, repr_pythonversion, getreportopt +from _pytest.terminal import build_summary_stats_line from _pytest import runner def basic_run_report(item): @@ -718,3 +719,55 @@ def pytest_terminal_summary(terminalreporter): *==== hello ====* world """) + +@pytest.mark.parametrize("exp_color, exp_line, stats_arg", [ + # The method under test only cares about the length of each + # dict value, not the actual contents, so tuples of anything + # suffice + + ("red", "1 failed", {"failed": (1,)}), + ("red", "1 failed, 1 passed", {"failed": (1,), "passed": (1,)}), + + ("red", "1 error", {"error": (1,)}), + ("red", "1 passed, 1 error", {"error": (1,), "passed": (1,)}), + + # (a status that's not known to the code) + ("green", "1 weird", {"weird": (1,)}), + ("green", "1 passed, 1 weird", {"weird": (1,), "passed": (1,)}), + + ("green", "1 warnings", {"warnings": (1,)}), + ("green", "1 passed, 1 warnings", {"warnings": (1,), "passed": (1,)}), + + ("green", "5 passed", {"passed": (1,2,3,4,5)}), + + ("green", "1 skipped", {"skipped": (1,)}), + ("green", "1 passed, 1 skipped", {"skipped": (1,), "passed": (1,)}), + + ("green", "1 deselected", {"deselected": (1,)}), + ("green", "1 passed, 1 deselected", {"deselected": (1,), "passed": (1,)}), + + ("green", "1 xfailed", {"xfailed": (1,)}), + ("green", "1 passed, 1 xfailed", {"xfailed": (1,), "passed": (1,)}), + + ("green", "1 xpassed", {"xpassed": (1,)}), + ("green", "1 passed, 1 xpassed", {"xpassed": (1,), "passed": (1,)}), + + # No tests were found at all + ("green", "", {}), + + + # A couple more complex combinations + ("red", "1 failed, 2 passed, 3 xfailed", + {"passed": (1,2), "failed": (1,), "xfailed": (1,2,3)}), + + ("green", "1 passed, 2 skipped, 3 deselected, 2 xfailed", + {"passed": (1,), + "skipped": (1,2), + "deselected": (1,2,3), + "xfailed": (1,2)}), +]) +def test_summary_stats(exp_line, exp_color, stats_arg): + (line, color) = build_summary_stats_line(stats_arg) + assert line == exp_line + assert color == exp_color + From 7993afae4611fcd3e1add0ee96b64fa00c16b5b8 Mon Sep 17 00:00:00 2001 From: Eric Siegerman Date: Tue, 30 Jun 2015 18:48:49 -0400 Subject: [PATCH 2/9] test_summary_stats() now prints its parameter values This makes it easier to identify failing tests. --- testing/test_terminal.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 9390ac15310..e1436f7f909 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -767,7 +767,10 @@ def pytest_terminal_summary(terminalreporter): "xfailed": (1,2)}), ]) def test_summary_stats(exp_line, exp_color, stats_arg): + print("Based on stats: %s" % stats_arg) + print("Expect summary: \"%s\"; with color \"%s\"" % (exp_line, exp_color)) (line, color) = build_summary_stats_line(stats_arg) + print("Actually got: \"%s\"; with color \"%s\"" % (line, color)) assert line == exp_line assert color == exp_color From e07144aeb4069053c0f9c06bca8ee3d05ab1467b Mon Sep 17 00:00:00 2001 From: Eric Siegerman Date: Tue, 30 Jun 2015 19:12:52 -0400 Subject: [PATCH 3/9] Add tests for the empty-key special case --- testing/test_terminal.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index e1436f7f909..877e9b070fd 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -755,6 +755,10 @@ def pytest_terminal_summary(terminalreporter): # No tests were found at all ("green", "", {}), + # Test the empty-key special case + ("green", "", {"": (1,)}), + ("green", "1 passed", {"": (1,), "passed": (1,)}), + # A couple more complex combinations ("red", "1 failed, 2 passed, 3 xfailed", From 0282da9ddf9d6a257c2b1d5fd4a7e48944e5c734 Mon Sep 17 00:00:00 2001 From: Eric Siegerman Date: Tue, 30 Jun 2015 19:06:28 -0400 Subject: [PATCH 4/9] Refactor slightly Check for the empty-key special case in the first loop, not the second. --- _pytest/terminal.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/_pytest/terminal.py b/_pytest/terminal.py index 82da2045de3..e4181097bdb 100644 --- a/_pytest/terminal.py +++ b/_pytest/terminal.py @@ -530,13 +530,13 @@ def build_summary_stats_line(stats): "xfailed xpassed warnings").split() for key in stats.keys(): if key not in keys: - keys.append(key) + if key: # setup/teardown reports have an empty key, ignore them + keys.append(key) parts = [] for key in keys: - if key: # setup/teardown reports have an empty key, ignore them - val = stats.get(key, None) - if val: - parts.append("%d %s" % (len(val), key)) + val = stats.get(key, None) + if val: + parts.append("%d %s" % (len(val), key)) line = ", ".join(parts) if 'failed' in stats or 'error' in stats: From cb21d844d97ff31eeb8dca8c2e66e9d85d14e0fe Mon Sep 17 00:00:00 2001 From: Eric Siegerman Date: Tue, 30 Jun 2015 19:27:36 -0400 Subject: [PATCH 5/9] Add missing "error" status to the list --- _pytest/terminal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_pytest/terminal.py b/_pytest/terminal.py index e4181097bdb..57560f94c15 100644 --- a/_pytest/terminal.py +++ b/_pytest/terminal.py @@ -527,7 +527,7 @@ def flatten(l): def build_summary_stats_line(stats): keys = ("failed passed skipped deselected " - "xfailed xpassed warnings").split() + "xfailed xpassed warnings error").split() for key in stats.keys(): if key not in keys: if key: # setup/teardown reports have an empty key, ignore them From 044d874c5b7d6c139cdb4e702322ad28f020ad7a Mon Sep 17 00:00:00 2001 From: Eric Siegerman Date: Tue, 30 Jun 2015 19:20:25 -0400 Subject: [PATCH 6/9] If there are warnings, make the status bar yellow Also if we see any statuses the code doesn't know about. --- _pytest/terminal.py | 4 ++++ testing/test_terminal.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/_pytest/terminal.py b/_pytest/terminal.py index 57560f94c15..35293042fa3 100644 --- a/_pytest/terminal.py +++ b/_pytest/terminal.py @@ -528,10 +528,12 @@ def flatten(l): def build_summary_stats_line(stats): keys = ("failed passed skipped deselected " "xfailed xpassed warnings error").split() + unknown_key_seen = False for key in stats.keys(): if key not in keys: if key: # setup/teardown reports have an empty key, ignore them keys.append(key) + unknown_key_seen = True parts = [] for key in keys: val = stats.get(key, None) @@ -541,6 +543,8 @@ def build_summary_stats_line(stats): if 'failed' in stats or 'error' in stats: color = 'red' + elif 'warnings' in stats or unknown_key_seen: + color = 'yellow' else: color = 'green' diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 877e9b070fd..9706ca4c40e 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -732,11 +732,11 @@ def pytest_terminal_summary(terminalreporter): ("red", "1 passed, 1 error", {"error": (1,), "passed": (1,)}), # (a status that's not known to the code) - ("green", "1 weird", {"weird": (1,)}), - ("green", "1 passed, 1 weird", {"weird": (1,), "passed": (1,)}), + ("yellow", "1 weird", {"weird": (1,)}), + ("yellow", "1 passed, 1 weird", {"weird": (1,), "passed": (1,)}), - ("green", "1 warnings", {"warnings": (1,)}), - ("green", "1 passed, 1 warnings", {"warnings": (1,), "passed": (1,)}), + ("yellow", "1 warnings", {"warnings": (1,)}), + ("yellow", "1 passed, 1 warnings", {"warnings": (1,), "passed": (1,)}), ("green", "5 passed", {"passed": (1,2,3,4,5)}), From 6c395cb58c52f7c4f0d55ba034cbf687b056dad2 Mon Sep 17 00:00:00 2001 From: Eric Siegerman Date: Tue, 30 Jun 2015 19:32:33 -0400 Subject: [PATCH 7/9] Default color is now yellow Passing tests override that default, making the color green; but several other "boring" statuses (xfailed, xpassed, deselected, skipped) have no effect. Net effect: if only "boring" tests are seen, or no tests at all, the summary bar is yellow. --- _pytest/terminal.py | 4 +++- testing/test_terminal.py | 20 +++++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/_pytest/terminal.py b/_pytest/terminal.py index 35293042fa3..98ac5556b0a 100644 --- a/_pytest/terminal.py +++ b/_pytest/terminal.py @@ -545,7 +545,9 @@ def build_summary_stats_line(stats): color = 'red' elif 'warnings' in stats or unknown_key_seen: color = 'yellow' - else: + elif 'passed' in stats: color = 'green' + else: + color = 'yellow' return (line, color) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 9706ca4c40e..473bf1d6322 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -725,6 +725,7 @@ def pytest_terminal_summary(terminalreporter): # dict value, not the actual contents, so tuples of anything # suffice + # Important statuses -- the highest priority of these always wins ("red", "1 failed", {"failed": (1,)}), ("red", "1 failed, 1 passed", {"failed": (1,), "passed": (1,)}), @@ -740,23 +741,28 @@ def pytest_terminal_summary(terminalreporter): ("green", "5 passed", {"passed": (1,2,3,4,5)}), - ("green", "1 skipped", {"skipped": (1,)}), + + # "Boring" statuses. These have no effect on the color of the summary + # line. Thus, if *every* test has a boring status, the summary line stays + # at its default color, i.e. yellow, to warn the user that the test run + # produced no useful information + ("yellow", "1 skipped", {"skipped": (1,)}), ("green", "1 passed, 1 skipped", {"skipped": (1,), "passed": (1,)}), - ("green", "1 deselected", {"deselected": (1,)}), + ("yellow", "1 deselected", {"deselected": (1,)}), ("green", "1 passed, 1 deselected", {"deselected": (1,), "passed": (1,)}), - ("green", "1 xfailed", {"xfailed": (1,)}), + ("yellow", "1 xfailed", {"xfailed": (1,)}), ("green", "1 passed, 1 xfailed", {"xfailed": (1,), "passed": (1,)}), - ("green", "1 xpassed", {"xpassed": (1,)}), + ("yellow", "1 xpassed", {"xpassed": (1,)}), ("green", "1 passed, 1 xpassed", {"xpassed": (1,), "passed": (1,)}), - # No tests were found at all - ("green", "", {}), + # Likewise if no tests were found at all + ("yellow", "", {}), # Test the empty-key special case - ("green", "", {"": (1,)}), + ("yellow", "", {"": (1,)}), ("green", "1 passed", {"": (1,), "passed": (1,)}), From afcad74be8d25650ad1fb3a61c54c0eb2ff4cab9 Mon Sep 17 00:00:00 2001 From: Eric Siegerman Date: Thu, 2 Jul 2015 13:05:26 -0400 Subject: [PATCH 8/9] PEP8ify parametrized-test parameters --- testing/test_terminal.py | 50 ++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 473bf1d6322..7ad74a9219d 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -726,55 +726,55 @@ def pytest_terminal_summary(terminalreporter): # suffice # Important statuses -- the highest priority of these always wins - ("red", "1 failed", {"failed": (1,)}), - ("red", "1 failed, 1 passed", {"failed": (1,), "passed": (1,)}), + ("red", "1 failed", {"failed": (1,)}), + ("red", "1 failed, 1 passed", {"failed": (1,), "passed": (1,)}), - ("red", "1 error", {"error": (1,)}), - ("red", "1 passed, 1 error", {"error": (1,), "passed": (1,)}), + ("red", "1 error", {"error": (1,)}), + ("red", "1 passed, 1 error", {"error": (1,), "passed": (1,)}), # (a status that's not known to the code) - ("yellow", "1 weird", {"weird": (1,)}), - ("yellow", "1 passed, 1 weird", {"weird": (1,), "passed": (1,)}), + ("yellow", "1 weird", {"weird": (1,)}), + ("yellow", "1 passed, 1 weird", {"weird": (1,), "passed": (1,)}), - ("yellow", "1 warnings", {"warnings": (1,)}), - ("yellow", "1 passed, 1 warnings", {"warnings": (1,), "passed": (1,)}), + ("yellow", "1 warnings", {"warnings": (1,)}), + ("yellow", "1 passed, 1 warnings", {"warnings": (1,), "passed": (1,)}), - ("green", "5 passed", {"passed": (1,2,3,4,5)}), + ("green", "5 passed", {"passed": (1,2,3,4,5)}), # "Boring" statuses. These have no effect on the color of the summary # line. Thus, if *every* test has a boring status, the summary line stays # at its default color, i.e. yellow, to warn the user that the test run # produced no useful information - ("yellow", "1 skipped", {"skipped": (1,)}), - ("green", "1 passed, 1 skipped", {"skipped": (1,), "passed": (1,)}), + ("yellow", "1 skipped", {"skipped": (1,)}), + ("green", "1 passed, 1 skipped", {"skipped": (1,), "passed": (1,)}), - ("yellow", "1 deselected", {"deselected": (1,)}), - ("green", "1 passed, 1 deselected", {"deselected": (1,), "passed": (1,)}), + ("yellow", "1 deselected", {"deselected": (1,)}), + ("green", "1 passed, 1 deselected", {"deselected": (1,), "passed": (1,)}), - ("yellow", "1 xfailed", {"xfailed": (1,)}), - ("green", "1 passed, 1 xfailed", {"xfailed": (1,), "passed": (1,)}), + ("yellow", "1 xfailed", {"xfailed": (1,)}), + ("green", "1 passed, 1 xfailed", {"xfailed": (1,), "passed": (1,)}), - ("yellow", "1 xpassed", {"xpassed": (1,)}), - ("green", "1 passed, 1 xpassed", {"xpassed": (1,), "passed": (1,)}), + ("yellow", "1 xpassed", {"xpassed": (1,)}), + ("green", "1 passed, 1 xpassed", {"xpassed": (1,), "passed": (1,)}), # Likewise if no tests were found at all - ("yellow", "", {}), + ("yellow", "", {}), # Test the empty-key special case - ("yellow", "", {"": (1,)}), - ("green", "1 passed", {"": (1,), "passed": (1,)}), + ("yellow", "", {"": (1,)}), + ("green", "1 passed", {"": (1,), "passed": (1,)}), # A couple more complex combinations - ("red", "1 failed, 2 passed, 3 xfailed", + ("red", "1 failed, 2 passed, 3 xfailed", {"passed": (1,2), "failed": (1,), "xfailed": (1,2,3)}), - ("green", "1 passed, 2 skipped, 3 deselected, 2 xfailed", - {"passed": (1,), - "skipped": (1,2), + ("green", "1 passed, 2 skipped, 3 deselected, 2 xfailed", + {"passed": (1,), + "skipped": (1,2), "deselected": (1,2,3), - "xfailed": (1,2)}), + "xfailed": (1,2)}), ]) def test_summary_stats(exp_line, exp_color, stats_arg): print("Based on stats: %s" % stats_arg) From 2c419c4790df18224e4c67c0c18458073511d3cb Mon Sep 17 00:00:00 2001 From: Eric Siegerman Date: Thu, 2 Jul 2015 15:47:03 -0400 Subject: [PATCH 9/9] Add CHANGELOG and AUTHORS entries --- AUTHORS | 1 + CHANGELOG | 3 +++ 2 files changed, 4 insertions(+) diff --git a/AUTHORS b/AUTHORS index 75f66b0b95b..2f43698ef3e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -25,6 +25,7 @@ Daniel Nuri Dave Hunt David Mohr Eduardo Schettino +Eric Siegerman Florian Bruhin Edison Gustavo Muenz Floris Bruynooghe diff --git a/CHANGELOG b/CHANGELOG index cba430e14bf..1eec84776df 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,9 @@ 2.8.0.dev (compared to 2.7.X) ----------------------------- +- partial fix for issue500: color the summary bar yellow for warning + situations, including the case where no tests were run + - fix issue713: JUnit XML reports for doctest failures. Thanks Punyashloka Biswal.