tests: Add smart output to aggregate_tests.py

Fix the aggregate_tests.py TextTestRunner buffering so that
output from failing tests is now printed below the failure. This
is done by adding a logging handler that allows TextTestRunner
to modify sys.stderr value between tests.

aggregate_tests.py now accepts the same verbosity flag as individual
test files (e.g. '-vvv' for DEBUG level).

tox sets '-vvv' so failing tests will now get DEBUG output: this
probably makes sense on CI at least.

Fixes #1120

Signed-off-by: Jussi Kukkonen <[email protected]>

Author: Jussi Kukkonen

tests/aggregate_tests.py

@@ -39,6 +39,8 @@
 import glob
 import random
 
+import utils
+
 # Generate a list of pathnames that match a pattern (i.e., that begin with
 # 'test_' and end with '.py'.  A shell-style wildcard is used with glob() to
 # match desired filenames.  All the tests matching the pattern will be loaded
@@ -90,6 +92,7 @@
 random.shuffle(test_modules_to_run)
 
 if __name__ == '__main__':
+  utils.configure_test_logging(sys.argv)
   suite = unittest.TestLoader().loadTestsFromNames(test_modules_to_run)
   all_tests_passed = unittest.TextTestRunner(
       verbosity=1, buffer=True).run(suite).wasSuccessful()

tests/utils.py

@@ -24,12 +24,29 @@
 import errno
 import logging
 import socket
+import sys
 import time
 
 import tuf.log
 
 logger = logging.getLogger(__name__)
 
+# Logging handler that always checks sys.stderr value when it needs the stream:
+# the default handler caches the sys.stderr value and that breaks TextTestRunner
+# buffering (it redirects stderr to a new file object for each individual test)
+class TestLogHandler(logging.StreamHandler):
+  def __init__(self):
+    super().__init__()
+
+  @property
+  def stream(self):
+    return sys.stderr
+
+  @stream.setter
+  def stream(self, value):
+    pass
+
+
 try:
   # is defined in Python 3
   TimeoutError
@@ -93,5 +110,5 @@ def configure_test_logging(argv):
   else:
     loglevel = logging.DEBUG
 
-  logging.basicConfig()
+  logging.basicConfig(handlers=[TestLogHandler()])
   tuf.log.set_log_level(loglevel)

tox.ini

@@ -16,7 +16,7 @@ changedir = tests
 commands =
     pylint {toxinidir}/tuf
     bandit -r {toxinidir}/tuf
-    coverage run aggregate_tests.py
+    coverage run aggregate_tests.py -vvv
     coverage report -m --fail-under 97
 
 deps =
@@ -37,5 +37,5 @@ deps =
     --editable {toxinidir}
 
 commands =
-    coverage run aggregate_tests.py
+    coverage run aggregate_tests.py -vvv
     coverage report -m