From edc1641c424dc1d56157be5c0cbf0b4ecf9a163b Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sun, 6 Nov 2022 21:01:55 +0100 Subject: [PATCH 1/5] properly configure flake8 --- setup.cfg | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..5bcd3eb --- /dev/null +++ b/setup.cfg @@ -0,0 +1,9 @@ +[flake8] +ignore = + E203 # whitespace before ':' - doesn't work well with black + E402 # module level import not at top of file + E501 # line too long - let black worry about that + E731 # do not assign a lambda expression, use a def + W503 # line break before binary operator +exclude= + .eggs From 6bcdc8ff6f3b093f02e44fef9f07129c85bc3bc0 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sun, 6 Nov 2022 21:03:14 +0100 Subject: [PATCH 2/5] same for isort --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f1533a0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,5 @@ +[tool.isort] +profile = "black" +skip_gitignore = true +float_to_top = true +default_section = "THIRDPARTY" From b2582cf58e7552b550c017be10afe787c9dd7137 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sun, 6 Nov 2022 21:03:20 +0100 Subject: [PATCH 3/5] implement a basic "truncating" summary with fallback to condensed summary --- parse_logs.py | 91 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 79 insertions(+), 12 deletions(-) diff --git a/parse_logs.py b/parse_logs.py index c0674ae..4b445f2 100644 --- a/parse_logs.py +++ b/parse_logs.py @@ -3,6 +3,7 @@ import functools import json import pathlib +import re import textwrap from dataclasses import dataclass @@ -33,6 +34,13 @@ def _from_json(cls, json): return cls(**json_) +@dataclass +class PreformattedReport: + name: str + variant: str | None + message: str + + def parse_record(record): report_types = { "TestReport": TestReport, @@ -47,27 +55,43 @@ def parse_record(record): return cls._from_json(record) +nodeid_re = re.compile(r"(?P.+?)(?:\[(?P.+)\])?") + + +def parse_nodeid(nodeid): + match = nodeid_re.fullmatch(nodeid) + if match is None: + raise ValueError(f"unknown test id: {nodeid}") + + return match.groupdict() + + @functools.singledispatch -def format_summary(report): - return f"{report.nodeid}: {report}" +def preformat_report(report): + return PreformattedReport(name=report.nodeid, variant=None, message=str(report)) -@format_summary.register +@preformat_report.register def _(report: TestReport): + parsed = parse_nodeid(report.nodeid) message = report.longrepr.chain[0][1].message - return f"{report.nodeid}: {message}" + return PreformattedReport(message=message, **parsed) -@format_summary.register +@preformat_report.register def _(report: CollectReport): + parsed = parse_nodeid(report.nodeid) message = report.longrepr.split("\n")[-1].removeprefix("E").lstrip() - return f"{report.nodeid}: {message}" + return PreformattedReport(message=message, **parsed) + + +def format_summary(report): + variant = f"[{report.variant}]" if report.variant is not None else "" + return f"{report.name}{variant}: {report.message}" -def format_report(reports, py_version): - newline = "\n" - summaries = newline.join(format_summary(r) for r in reports) - message = textwrap.dedent( +def format_report(summaries, py_version): + template = textwrap.dedent( """\
Python {py_version} Test Summary @@ -77,10 +101,52 @@ def format_report(reports, py_version):
""" - ).format(summaries=summaries, py_version=py_version) + ) + # can't use f-strings because that would format *before* the dedenting + message = template.format(summaries="\n".join(summaries), py_version=py_version) return message +def truncate(reports, max_chars, **formatter_kwargs): + fractions = [0.95, 0.75, 0.5, 0.25, 0.1, 0.01] + + n_reports = len(reports) + for fraction in fractions: + n_selected = int(n_reports * fraction) + selected_reports = reports[: int(n_reports * fraction)] + report_messages = [format_summary(report) for report in selected_reports] + summary = report_messages + [f"+ {n_reports - n_selected} failing tests"] + formatted = format_report(summary, **formatter_kwargs) + if len(formatted) <= max_chars: + return formatted + + return None + + +def summarize(reports): + return f"{len(reports)} failing tests" + + +def compressed_report(reports, max_chars, **formatter_kwargs): + strategies = [ + # merge_variants, + # merge_test_files, + # merge_tests, + truncate, + ] + summaries = [format_summary(report) for report in reports] + formatted = format_report(summaries, **formatter_kwargs) + if len(formatted) <= max_chars: + return formatted + + for strategy in strategies: + formatted = strategy(reports, max_chars=max_chars, **formatter_kwargs) + if formatted is not None and len(formatted) <= max_chars: + return formatted + + return summarize(reports) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("filepath", type=pathlib.Path) @@ -94,8 +160,9 @@ def format_report(reports, py_version): reports = [parse_record(json.loads(line)) for line in lines] failed = [report for report in reports if report.outcome == "failed"] + preformatted = [preformat_report(report) for report in failed] - message = format_report(failed, py_version=py_version) + message = compressed_report(preformatted, max_chars=65535, py_version=py_version) output_file = pathlib.Path("pytest-logs.txt") print(f"Writing output file to: {output_file.absolute()}") From 9f8ceaae7558089acc80c04ef8dddd2238410c2a Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sun, 6 Nov 2022 21:21:18 +0100 Subject: [PATCH 4/5] also split the name into filepath and test name --- parse_logs.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/parse_logs.py b/parse_logs.py index 4b445f2..5ff1f4a 100644 --- a/parse_logs.py +++ b/parse_logs.py @@ -36,6 +36,7 @@ def _from_json(cls, json): @dataclass class PreformattedReport: + filepath: str name: str variant: str | None message: str @@ -55,7 +56,7 @@ def parse_record(record): return cls._from_json(record) -nodeid_re = re.compile(r"(?P.+?)(?:\[(?P.+)\])?") +nodeid_re = re.compile(r"(?P.+)::(?P.+?)(?:\[(?P.+)\])?") def parse_nodeid(nodeid): @@ -68,7 +69,8 @@ def parse_nodeid(nodeid): @functools.singledispatch def preformat_report(report): - return PreformattedReport(name=report.nodeid, variant=None, message=str(report)) + parsed = parse_nodeid(report.nodeid) + return PreformattedReport(message=str(report), **parsed) @preformat_report.register @@ -86,8 +88,10 @@ def _(report: CollectReport): def format_summary(report): - variant = f"[{report.variant}]" if report.variant is not None else "" - return f"{report.name}{variant}: {report.message}" + if report.variant is not None: + return f"{report.filepath}::{report.name}[{report.variant}]: {report.message}" + else: + return f"{report.filepath}::{report.name}: {report.message}" def format_report(summaries, py_version): From 9795715f539a95a940c8a86da66730744f7b0eaf Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sun, 6 Nov 2022 21:21:47 +0100 Subject: [PATCH 5/5] implement the merging of variants with the same name --- parse_logs.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/parse_logs.py b/parse_logs.py index 5ff1f4a..208201a 100644 --- a/parse_logs.py +++ b/parse_logs.py @@ -7,6 +7,7 @@ import textwrap from dataclasses import dataclass +import more_itertools from pytest import CollectReport, TestReport @@ -111,6 +112,24 @@ def format_report(summaries, py_version): return message +def merge_variants(reports, max_chars, **formatter_kwargs): + def format_variant_group(name, group): + filepath, test_name, message = name + + n_variants = len(group) + if n_variants != 0: + return f"{filepath}::{test_name}[{n_variants} failing variants]: {message}" + else: + return f"{filepath}::{test_name}: {message}" + + bucket = more_itertools.bucket(reports, lambda r: (r.filepath, r.name, r.message)) + + summaries = [format_variant_group(name, list(bucket[name])) for name in bucket] + formatted = format_report(summaries, **formatter_kwargs) + + return formatted + + def truncate(reports, max_chars, **formatter_kwargs): fractions = [0.95, 0.75, 0.5, 0.25, 0.1, 0.01] @@ -133,7 +152,7 @@ def summarize(reports): def compressed_report(reports, max_chars, **formatter_kwargs): strategies = [ - # merge_variants, + merge_variants, # merge_test_files, # merge_tests, truncate,