diff --git a/README.rst b/README.rst
index dc2c9b81..84bc0689 100644
--- a/README.rst
+++ b/README.rst
@@ -240,6 +240,45 @@ additional HTML and log output with a notice that the log is empty:
del data[:]
data.append(html.div('No log output captured.', class_='empty log'))
+Grouping test results
+~~~~~~~~~~~~~~~~~~~~~
+
+By default, the generated report will be a flat array containing the results of all the defined
+tests. It is possible to group these results by defining some keys. It also allows defining a group
+hierarchy of undefined depth.
+
+Group keys are defined in the :code:`pytest_runtest_makereport` hook. The following example defines
+2 keys `key1` and `key2` whose value will be the `argx` and `argy` values passed to the test.
+
+.. code-block:: python
+
+ import pytest
+ @pytest.mark.hookwrapper
+ def pytest_runtest_makereport(item, call):
+ outcome = yield
+ report = outcome.get_result()
+ report.key1 = item.funcargs["argx"]
+ report.key2 = item.funcargs["argy"]
+
+Then to generate a grouped report, you can use the `--group-by` option:
+
+.. code-block:: bash
+
+ $ pytest --html=report.html --group-by key1 --group-by key2
+
+Keys will be considered in the order the are given on the command line. The previous example will
+generate the following hierarchy:
+
+.. code-block::
+
+ key1 = "Value 1"
+ |-- key2 = "Value 1.1"
+ |-- key2 = "Value 1.2"
+ key1 = "Value 2"
+ |-- key2 = "Value 2.1"
+ |-- key2 = "Value 2.2"
+
+
Display options
---------------
diff --git a/pytest_html/plugin.py b/pytest_html/plugin.py
index e66840de..93cfbe16 100644
--- a/pytest_html/plugin.py
+++ b/pytest_html/plugin.py
@@ -16,6 +16,7 @@
import bisect
import hashlib
import warnings
+import re
try:
from ansi2html import Ansi2HTMLConverter, style
@@ -58,6 +59,8 @@ def pytest_addoption(parser):
'https://developer.mozilla.org/docs/Web/Security/CSP)')
group.addoption('--css', action='append', metavar='path', default=[],
help='append given css file content to report style file.')
+ group.addoption('--group-by', help='Group report using these attributes',
+ action='append')
def pytest_configure(config):
@@ -83,19 +86,32 @@ def data_uri(content, mime_type='text/plain', charset='utf-8'):
return 'data:{0};charset={1};base64,{2}'.format(mime_type, charset, data)
+def convert_key_to_id(key):
+ # Consider HTML id as ASCII [0-9, A-Z, a-z] + [_.-] strings
+ def remove_non_ascii(matchgroup):
+ return ''
+
+ escaped_id = key.lower().replace(" ", "-")
+ return re.sub(r'[^0-9a-zA-Z._-]', remove_non_ascii, escaped_id)
+
+
class HTMLReport(object):
def __init__(self, logfile, config):
logfile = os.path.expanduser(os.path.expandvars(logfile))
self.logfile = os.path.abspath(logfile)
- self.test_logs = []
- self.results = []
- self.errors = self.failed = 0
- self.passed = self.skipped = 0
- self.xfailed = self.xpassed = 0
+ self.test_logs = {}
+ self.results = {}
+ self.errors = {'_total': 0}
+ self.failed = {'_total': 0}
+ self.passed = {'_total': 0}
+ self.skipped = {'_total': 0}
+ self.xfailed = {'_total': 0}
+ self.xpassed = {'_total': 0}
has_rerun = config.pluginmanager.hasplugin('rerunfailures')
- self.rerun = 0 if has_rerun else None
+ self.rerun = {'_total': 0} if has_rerun else None
self.self_contained = config.getoption('self_contained_html')
+ self.group_report_keys = config.getoption('group_by')
self.config = config
class TestResult:
@@ -264,57 +280,114 @@ def append_log_html(self, report, additional_html):
log.append('No log output captured.')
additional_html.append(log)
+ def _get_index(self, report):
+ test_group_index = ['_default']
+ if self.group_report_keys:
+ test_group_index = [getattr(report, c)
+ for c in self.group_report_keys]
+ return test_group_index
+
+ def _get_indexed_obj(self, obj, index, _type):
+ out = obj
+ last = index[-1]
+ index = index[:-1]
+ while index:
+ key = index[0]
+ if key not in obj:
+ obj[key] = {}
+ out = obj[key]
+ index.pop(0)
+ if last not in out:
+ out[last] = 0 if _type == 'counter' else []
+ return out[last]
+
+ def _get_indexed_list(self, l, index):
+ return self._get_indexed_obj(l, index, 'list')
+
+ def _get_indexed_counter(self, c, index):
+ return self._get_indexed_obj(c, index, 'counter')
+
+ def _increment_counter(self, counter, index):
+ counter['_total'] += 1 # Start by incrementing total
+ last = index[-1]
+ index = index[:-1]
+ while index:
+ key = index[0]
+ if key not in counter:
+ counter[key] = {}
+ counter = counter[key]
+ index.pop(0)
+ if last not in counter:
+ counter[last] = 0
+ counter[last] += 1
+
+ def _count(self, index, *args):
+ total = 0
+ for counter in args:
+ if index == ['_default']:
+ total += counter['_total']
+ else:
+ total += self._get_indexed_counter(counter, index)
+ return total
+
def _appendrow(self, outcome, report):
result = self.TestResult(outcome, report, self.logfile, self.config)
if result.row_table is not None:
- index = bisect.bisect_right(self.results, result)
- self.results.insert(index, result)
+ test_group_index = self._get_index(report)
+ result_list = self._get_indexed_list(self.results,
+ test_group_index)
+ index = bisect.bisect_right(result_list, result)
+ result_list.insert(index, result)
tbody = html.tbody(
result.row_table,
class_='{0} results-table-row'.format(result.outcome.lower()))
if result.row_extra is not None:
tbody.append(result.row_extra)
- self.test_logs.insert(index, tbody)
+ logs = self._get_indexed_list(self.test_logs, test_group_index)
+ logs.insert(index, tbody)
def append_passed(self, report):
if report.when == 'call':
+ test_group_index = self._get_index(report)
if hasattr(report, "wasxfail"):
- self.xpassed += 1
+ self._increment_counter(self.xpassed, test_group_index)
self._appendrow('XPassed', report)
else:
- self.passed += 1
+ self._increment_counter(self.passed, test_group_index)
self._appendrow('Passed', report)
def append_failed(self, report):
+ test_group_index = self._get_index(report)
if getattr(report, 'when', None) == "call":
if hasattr(report, "wasxfail"):
# pytest < 3.0 marked xpasses as failures
- self.xpassed += 1
+ self._increment_counter(self.xpassed, test_group_index)
self._appendrow('XPassed', report)
else:
- self.failed += 1
+ self._increment_counter(self.failed, test_group_index)
self._appendrow('Failed', report)
else:
- self.errors += 1
+ self._increment_counter(self.errors, test_group_index)
self._appendrow('Error', report)
def append_skipped(self, report):
+ test_group_index = self._get_index(report)
if hasattr(report, "wasxfail"):
- self.xfailed += 1
+ self._increment_counter(self.xfailed, test_group_index)
self._appendrow('XFailed', report)
else:
- self.skipped += 1
+ self._increment_counter(self.skipped, test_group_index)
self._appendrow('Skipped', report)
def append_other(self, report):
# For now, the only "other" the plugin give support is rerun
- self.rerun += 1
+ test_group_index = self._get_index(report)
+ self._increment_counter(self.rerun, test_group_index)
self._appendrow('Rerun', report)
def _generate_report(self, session):
suite_stop_time = time.time()
- suite_time_delta = suite_stop_time - self.suite_start_time
- numtests = self.passed + self.failed + self.xpassed + self.xfailed
+ self.suite_time_delta = suite_stop_time - self.suite_start_time
generated = datetime.datetime.now()
self.style_css = pkg_resources.resource_string(
@@ -350,22 +423,109 @@ def _generate_report(self, session):
html.title('Test Report'),
html_css)
+ main_js = pkg_resources.resource_string(
+ __name__, os.path.join('resources', 'main.js'))
+ if PY3:
+ main_js = main_js.decode('utf-8')
+
+ body = html.body(
+ html.script(raw(main_js)),
+ html.h1(os.path.basename(self.logfile)),
+ html.p('Report generated on {0} at {1} by '.format(
+ generated.strftime('%d-%b-%Y'),
+ generated.strftime('%H:%M:%S')),
+ html.a('pytest-html', href=__pypi_url__),
+ ' v{0}'.format(__version__)),
+ onLoad='init()')
+
+ body.extend(self._generate_environment(session.config))
+
+ include_outcome = not self.group_report_keys
+ body.extend(self._generate_summary(session,
+ ['_default'],
+ include_outcome=include_outcome))
+
+ if not self.group_report_keys:
+ results = self._generate_results(session, ['_default'])
+ body.extend(results)
+ else:
+ links_summary, results = self._generate_all_results(session)
+ body.append(html.p('List of test reports:',
+ html.ul(links_summary)))
+ body.extend(results)
+
+ doc = html.html(head, body)
+
+ unicode_doc = u'\n{0}'.format(doc.unicode(indent=2))
+ if PY3:
+ # Fix encoding issues, e.g. with surrogates
+ unicode_doc = unicode_doc.encode('utf-8',
+ errors='xmlcharrefreplace')
+ unicode_doc = unicode_doc.decode('utf-8')
+ return unicode_doc
+
+ def _generate_all_results(self, session,
+ _key=None, _current_level=None, _h_level=2):
+ results = []
+ links_summary = []
+ if _current_level is None:
+ _current_level = self.results
+ if _key is None:
+ _key = []
+ h_result_level = _h_level + 1
+ if _h_level > 6:
+ _h_level = 6
+ h_result_level = 6
+ hx = getattr(html, 'h{}'.format(_h_level))
+ h_result = getattr(html, 'h{}'.format(h_result_level))
+
+ for k, v in _current_level.items():
+ key = _key + [k]
+ prefix_id = '-'.join(key)
+ if prefix_id:
+ prefix_id += '-'
+ prefix_id = convert_key_to_id(prefix_id)
+ results.append(hx(k, id='{}title'.format(prefix_id)))
+ li = html.li(html.a(k.title(), href='#{}title'.format(prefix_id)))
+ if not isinstance(v, list): # Not leaf yet
+ sublinks, subres = self._generate_all_results(session,
+ key,
+ v,
+ _h_level + 1)
+ li.append(html.ul(sublinks))
+ else:
+ subres = self._generate_summary(session, key, prefix_id)
+ subres.extend(self._generate_results(session,
+ key,
+ prefix_id,
+ h_result))
+ links_summary.append(li)
+ results.extend(subres)
+
+ return links_summary, results
+
+ def _generate_summary(self, session, key,
+ prefix_id='', h_summary=html.h2, include_outcome=True):
class Outcome:
def __init__(self, outcome, total=0, label=None,
- test_result=None, class_html=None):
+ test_result=None, class_html=None,
+ prefix_id=None):
self.outcome = outcome
self.label = label or outcome
self.class_html = class_html or outcome
self.total = total
self.test_result = test_result or outcome
+ self.prefix_id = prefix_id
self.generate_checkbox()
self.generate_summary_item()
def generate_checkbox(self):
+ data_pref = convert_key_to_id(self.prefix_id)
checkbox_kwargs = {'data-test-result':
- self.test_result.lower()}
+ self.test_result.lower(),
+ 'data-prefix-id': data_pref}
if self.total == 0:
checkbox_kwargs['disabled'] = 'true'
@@ -382,31 +542,69 @@ def generate_summary_item(self):
format(self.total, self.label),
class_=self.class_html)
- outcomes = [Outcome('passed', self.passed),
- Outcome('skipped', self.skipped),
- Outcome('failed', self.failed),
- Outcome('error', self.errors, label='errors'),
- Outcome('xfailed', self.xfailed,
- label='expected failures'),
- Outcome('xpassed', self.xpassed,
- label='unexpected passes')]
+ outcomes = [Outcome('passed',
+ self._get_indexed_counter(self.passed, key),
+ prefix_id=prefix_id),
+ Outcome('skipped',
+ self._get_indexed_counter(self.skipped, key),
+ prefix_id=prefix_id),
+ Outcome('failed',
+ self._get_indexed_counter(self.failed, key),
+ prefix_id=prefix_id),
+ Outcome('error',
+ self._get_indexed_counter(self.errors, key),
+ label='errors', prefix_id=prefix_id),
+ Outcome('xfailed',
+ self._get_indexed_counter(self.xfailed, key),
+ label='expected failures', prefix_id=prefix_id),
+ Outcome('xpassed',
+ self._get_indexed_counter(self.xpassed, key),
+ label='unexpected passes', prefix_id=prefix_id)]
if self.rerun is not None:
- outcomes.append(Outcome('rerun', self.rerun))
-
- summary = [html.p(
- '{0} tests ran in {1:.2f} seconds. '.format(
- numtests, suite_time_delta)),
- html.p('(Un)check the boxes to filter the results.',
- class_='filter',
- hidden='true')]
+ outcomes.append(Outcome('rerun',
+ self._get_indexed_counter(self.rerun, key),
+ prefix_id=prefix_id))
+ outcome_section = []
for i, outcome in enumerate(outcomes, start=1):
- summary.append(outcome.checkbox)
- summary.append(outcome.summary_item)
+ outcome_section.append(outcome.checkbox)
+ outcome_section.append(outcome.summary_item)
if i < len(outcomes):
- summary.append(', ')
+ outcome_section.append(', ')
+
+ numtests = self._count(key,
+ self.passed,
+ self.failed,
+ self.xpassed,
+ self.xfailed)
+
+ summary_text = '{} tests ran'.format(numtests)
+ if key == ['_default']:
+ summary_text += ' in {:.2f} seconds.'.format(self.suite_time_delta)
+
+ summary = [html.p(summary_text)]
+
+ if include_outcome:
+ summary.append(html.p('(Un)check the boxes to filter the results.',
+ class_='filter',
+ hidden='true'))
+ for i, outcome in enumerate(outcomes, start=1):
+ summary.append(outcome.checkbox)
+ summary.append(outcome.summary_item)
+ if i < len(outcomes):
+ summary.append(', ')
+ summary_prefix, summary_postfix = [], []
+ session.config.hook.pytest_html_results_summary(
+ prefix=summary_prefix, summary=summary, postfix=summary_postfix)
+
+ return ([h_summary('Summary')] +
+ summary_prefix +
+ summary +
+ summary_postfix)
+
+ def _generate_results(self, session, key, prefix_id='', h_result=html.h2):
cells = [
html.th('Result',
class_='sortable result initial-sort',
@@ -416,49 +614,18 @@ def generate_summary_item(self):
html.th('Links')]
session.config.hook.pytest_html_results_table_header(cells=cells)
- results = [html.h2('Results'), html.table([html.thead(
+ results = [h_result('Results'), html.table([html.thead(
html.tr(cells),
html.tr([
html.th('No results found. Try to check the filters',
colspan=len(cells))],
- id='not-found-message', hidden='true'),
- id='results-table-head'),
- self.test_logs], id='results-table')]
-
- main_js = pkg_resources.resource_string(
- __name__, os.path.join('resources', 'main.js'))
- if PY3:
- main_js = main_js.decode('utf-8')
-
- body = html.body(
- html.script(raw(main_js)),
- html.h1(os.path.basename(self.logfile)),
- html.p('Report generated on {0} at {1} by '.format(
- generated.strftime('%d-%b-%Y'),
- generated.strftime('%H:%M:%S')),
- html.a('pytest-html', href=__pypi_url__),
- ' v{0}'.format(__version__)),
- onLoad='init()')
-
- body.extend(self._generate_environment(session.config))
-
- summary_prefix, summary_postfix = [], []
- session.config.hook.pytest_html_results_summary(
- prefix=summary_prefix, summary=summary, postfix=summary_postfix)
- body.extend([html.h2('Summary')] + summary_prefix
- + summary + summary_postfix)
-
- body.extend(results)
+ id=prefix_id + 'not-found-message', hidden='true'),
+ id='-'.join(key) + 'results-table-head'),
+ self._get_indexed_list(self.test_logs, key)],
+ id=prefix_id + 'results-table',
+ class_='results-table')]
- doc = html.html(head, body)
-
- unicode_doc = u'\n{0}'.format(doc.unicode(indent=2))
- if PY3:
- # Fix encoding issues, e.g. with surrogates
- unicode_doc = unicode_doc.encode('utf-8',
- errors='xmlcharrefreplace')
- unicode_doc = unicode_doc.decode('utf-8')
- return unicode_doc
+ return results
def _generate_environment(self, config):
if not hasattr(config, '_metadata') or config._metadata is None:
diff --git a/pytest_html/resources/main.js b/pytest_html/resources/main.js
index dfd0ae19..918226c9 100644
--- a/pytest_html/resources/main.js
+++ b/pytest_html/resources/main.js
@@ -38,12 +38,12 @@ function sort_column(elem) {
sort_table(elem, key(colIndex));
}
-function show_all_extras() {
- find_all('.col-result').forEach(show_extras);
+function show_all_extras(table_id) {
+ find_all('.col-result', find('#' + table_id)).forEach(show_extras);
}
-function hide_all_extras() {
- find_all('.col-result').forEach(hide_extras);
+function hide_all_extras(table_id) {
+ find_all('.col-result', find('#' + table_id)).forEach(hide_extras);
}
function show_extras(colresult_elem) {
@@ -70,11 +70,12 @@ function show_filters() {
function add_collapse() {
// Add links for show/hide all
- var resulttable = find('table#results-table');
- var showhideall = document.createElement("p");
- showhideall.innerHTML = 'Show all details / ' +
- 'Hide all details';
- resulttable.parentElement.insertBefore(showhideall, resulttable);
+ find_all('table.results-table').forEach(function(resulttable) {
+ var showhideall = document.createElement("p");
+ showhideall.innerHTML = 'Show all details / ' +
+ 'Hide all details';
+ resulttable.parentElement.insertBefore(showhideall, resulttable);
+ });
// Add show/hide link to each result
find_all('.col-result').forEach(function(elem) {
@@ -123,21 +124,23 @@ function init () {
};
function sort_table(clicked, key_func) {
- var rows = find_all('.results-table-row');
+ var table = clicked.parentNode.parentNode.parentNode;
+ var previous_sibling = table.previousSibling;
+ var rows = find_all('.results-table-row', table);
var reversed = !clicked.classList.contains('asc');
var sorted_rows = sort(rows, key_func, reversed);
/* Whole table is removed here because browsers acts much slower
* when appending existing elements.
*/
- var thead = document.getElementById("results-table-head");
- document.getElementById('results-table').remove();
+ var thead = find('thead', table);
+ table.remove();
var parent = document.createElement("table");
- parent.id = "results-table";
+ parent.id = table.id;
parent.appendChild(thead);
sorted_rows.forEach(function(elem) {
parent.appendChild(elem);
});
- document.getElementsByTagName("BODY")[0].appendChild(parent);
+ previous_sibling.parentNode.insertBefore(parent, previous_sibling.nextSibling);
}
function sort(items, key_func, reversed) {
@@ -214,15 +217,17 @@ function is_all_rows_hidden(value) {
function filter_table(elem) {
var outcome_att = "data-test-result";
var outcome = elem.getAttribute(outcome_att);
- class_outcome = outcome + " results-table-row";
- var outcome_rows = document.getElementsByClassName(class_outcome);
+ var class_outcome = outcome + " results-table-row";
+ var prefix_id = elem.getAttribute("data-prefix-id");
+ var table = find("#" + prefix_id + "results-table");
+ var outcome_rows = table.getElementsByClassName(class_outcome);
for(var i = 0; i < outcome_rows.length; i++){
outcome_rows[i].hidden = !elem.checked;
}
- var rows = find_all('.results-table-row').filter(is_all_rows_hidden);
+ var rows = find_all('.results-table-row', table).filter(is_all_rows_hidden);
var all_rows_hidden = rows.length == 0 ? true : false;
- var not_found_message = document.getElementById("not-found-message");
+ var not_found_message = document.getElementById(prefix_id + "not-found-message");
not_found_message.hidden = !all_rows_hidden;
}
diff --git a/pytest_html/resources/style.css b/pytest_html/resources/style.css
index 66cab878..8b274d49 100644
--- a/pytest_html/resources/style.css
+++ b/pytest_html/resources/style.css
@@ -67,19 +67,19 @@ span.error, span.failed, span.xpassed, .error .col-result, .failed .col-result,
* 1. Table Layout
*------------------*/
-#results-table {
+.results-table {
border: 1px solid #e6e6e6;
color: #999;
font-size: 12px;
width: 100%
}
-#results-table th, #results-table td {
+.results-table th, .results-table td {
padding: 5px;
border: 1px solid #E6E6E6;
text-align: left
}
-#results-table th {
+.results-table th {
font-weight: bold
}
diff --git a/testing/test_pytest_html.py b/testing/test_pytest_html.py
index eedd44d8..25fa5329 100644
--- a/testing/test_pytest_html.py
+++ b/testing/test_pytest_html.py
@@ -37,6 +37,7 @@ def assert_results_by_outcome(html, test_outcome, test_outcome_number,
# Asserts if the generated checkbox of this outcome is correct
regex_checkbox = ('