Skip to content

Bugfix: junitxml violates Jenkins/xUnit JUnit schema #4493

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Bugfix: junitxml violates Jenkins/xUnit JUnit schema #4493

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AUTHORS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ Jonas Obrist
Jordan Guymon
Jordan Moldow
Jordan Speicher
Joseph Hunkeler
Joshua Bronson
Jurko Gospodnetić
Justyna Janczyszyn
Expand Down
1 change: 1 addition & 0 deletions changelog/3547.bugfix.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
``--junitxml`` emits XML data compatible with JUnit's offical schema releases.
12 changes: 3 additions & 9 deletions src/_pytest/junitxml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,20 +110,14 @@ def record_testreport(self, testreport):
classnames = names[:-1]
if self.xml.prefix:
classnames.insert(0, self.xml.prefix)
attrs = {
"classname": ".".join(classnames),
"name": bin_xml_escape(names[-1]),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unfortunately this one iis a breaking change, well have to sort out the details

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please elaborate. What does it break?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jhunkeler off hand we cant rule out that people rely on the invalid extra attributes we output

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

or to elaborate - junitxml output has been brokenly incorrect for so long that we now are likely to break other peoples code with the fix and we have to account for that in some way (be it a major release for this fix or a opt-in for correct behaviour that will later be a opt out and finally be phased out

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about creating an pytest.ini option to configure this then? junitxml_strict=true?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And this would need to be based on features btw.

"file": testreport.location[0],
}
if testreport.location[1] is not None:
attrs["line"] = testreport.location[1]
attrs = {"classname": ".".join(classnames), "name": bin_xml_escape(names[-1])}
if hasattr(testreport, "url"):
attrs["url"] = testreport.url
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note to self: url cannot be present in testcase either.

self.attrs = attrs
self.attrs.update(existing_attrs) # restore any user-defined attributes

def to_xml(self):
testcase = Junit.testcase(time=self.duration, **self.attrs)
testcase = Junit.testcase(time="%.3f" % self.duration, **self.attrs)
testcase.append(self.make_properties_node())
for node in self.nodes:
testcase.append(node)
Expand Down Expand Up @@ -543,7 +537,7 @@ def pytest_sessionfinish(self):
name=self.suite_name,
errors=self.stats["error"],
failures=self.stats["failure"],
skips=self.stats["skipped"],
skipped=self.stats["skipped"],
tests=numtests,
time="%.3f" % suite_time_delta,
).unicode(indent=0)
Expand Down
126 changes: 31 additions & 95 deletions testing/test_junitxml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ def test_xpass():
result, dom = runandparse(testdir)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(name="pytest", errors=0, failures=1, skips=2, tests=5)
node.assert_attr(name="pytest", errors=0, failures=1, skipped=2, tests=5)

def test_summing_simple_with_errors(self, testdir):
testdir.makepyfile(
Expand All @@ -133,7 +133,7 @@ def test_xpass():
result, dom = runandparse(testdir)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(name="pytest", errors=1, failures=2, skips=1, tests=5)
node.assert_attr(name="pytest", errors=1, failures=2, skipped=1, tests=5)

def test_timing_function(self, testdir):
testdir.makepyfile(
Expand Down Expand Up @@ -170,12 +170,7 @@ def test_function(arg):
node = dom.find_first_by_tag("testsuite")
node.assert_attr(errors=1, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_setup_error.py",
line="5",
classname="test_setup_error",
name="test_function",
)
tnode.assert_attr(classname="test_setup_error", name="test_function")
fnode = tnode.find_first_by_tag("error")
fnode.assert_attr(message="test setup failure")
assert "ValueError" in fnode.toxml()
Expand All @@ -197,12 +192,7 @@ def test_function(arg):
assert result.ret
node = dom.find_first_by_tag("testsuite")
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_teardown_error.py",
line="6",
classname="test_teardown_error",
name="test_function",
)
tnode.assert_attr(classname="test_teardown_error", name="test_function")
fnode = tnode.find_first_by_tag("error")
fnode.assert_attr(message="test teardown failure")
assert "ValueError" in fnode.toxml()
Expand Down Expand Up @@ -243,14 +233,9 @@ def test_skip():
result, dom = runandparse(testdir)
assert result.ret == 0
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=1)
node.assert_attr(skipped=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_skip_contains_name_reason.py",
line="1",
classname="test_skip_contains_name_reason",
name="test_skip",
)
tnode.assert_attr(classname="test_skip_contains_name_reason", name="test_skip")
snode = tnode.find_first_by_tag("skipped")
snode.assert_attr(type="pytest.skip", message="hello23")

Expand All @@ -266,13 +251,10 @@ def test_skip():
result, dom = runandparse(testdir)
assert result.ret == 0
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=1)
node.assert_attr(skipped=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_mark_skip_contains_name_reason.py",
line="1",
classname="test_mark_skip_contains_name_reason",
name="test_skip",
classname="test_mark_skip_contains_name_reason", name="test_skip"
)
snode = tnode.find_first_by_tag("skipped")
snode.assert_attr(type="pytest.skip", message="hello24")
Expand All @@ -290,13 +272,10 @@ def test_skip():
result, dom = runandparse(testdir)
assert result.ret == 0
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=1)
node.assert_attr(skipped=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_mark_skipif_contains_name_reason.py",
line="2",
classname="test_mark_skipif_contains_name_reason",
name="test_skip",
classname="test_mark_skipif_contains_name_reason", name="test_skip"
)
snode = tnode.find_first_by_tag("skipped")
snode.assert_attr(type="pytest.skip", message="hello25")
Expand Down Expand Up @@ -329,10 +308,7 @@ def test_method(self):
node.assert_attr(failures=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_classname_instance.py",
line="1",
classname="test_classname_instance.TestClass",
name="test_method",
classname="test_classname_instance.TestClass", name="test_method"
)

def test_classname_nested_dir(self, testdir):
Expand All @@ -343,12 +319,7 @@ def test_classname_nested_dir(self, testdir):
node = dom.find_first_by_tag("testsuite")
node.assert_attr(failures=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file=os.path.join("sub", "test_hello.py"),
line="0",
classname="sub.test_hello",
name="test_func",
)
tnode.assert_attr(classname="sub.test_hello", name="test_func")

def test_internal_error(self, testdir):
testdir.makeconftest("def pytest_runtest_protocol(): 0 / 0")
Expand Down Expand Up @@ -384,12 +355,7 @@ def test_fail():
node = dom.find_first_by_tag("testsuite")
node.assert_attr(failures=1, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_failure_function.py",
line="3",
classname="test_failure_function",
name="test_fail",
)
tnode.assert_attr(classname="test_failure_function", name="test_fail")
fnode = tnode.find_first_by_tag("failure")
fnode.assert_attr(message="ValueError: 42")
assert "ValueError" in fnode.toxml()
Expand Down Expand Up @@ -446,10 +412,7 @@ def test_func(arg1):

tnode = node.find_nth_by_tag("testcase", index)
tnode.assert_attr(
file="test_failure_escape.py",
line="1",
classname="test_failure_escape",
name="test_func[%s]" % char,
classname="test_failure_escape", name="test_func[%s]" % char
)
sysout = tnode.find_first_by_tag("system-out")
text = sysout.text
Expand All @@ -470,18 +433,10 @@ def test_hello(self):
node = dom.find_first_by_tag("testsuite")
node.assert_attr(failures=1, tests=2)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_junit_prefixing.py",
line="0",
classname="xyz.test_junit_prefixing",
name="test_func",
)
tnode.assert_attr(classname="xyz.test_junit_prefixing", name="test_func")
tnode = node.find_nth_by_tag("testcase", 1)
tnode.assert_attr(
file="test_junit_prefixing.py",
line="3",
classname="xyz.test_junit_prefixing.TestHello",
name="test_hello",
classname="xyz.test_junit_prefixing.TestHello", name="test_hello"
)

def test_xfailure_function(self, testdir):
Expand All @@ -495,14 +450,9 @@ def test_xfail():
result, dom = runandparse(testdir)
assert not result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=1, tests=1)
node.assert_attr(skipped=1, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_xfailure_function.py",
line="1",
classname="test_xfailure_function",
name="test_xfail",
)
tnode.assert_attr(classname="test_xfailure_function", name="test_xfail")
fnode = tnode.find_first_by_tag("skipped")
fnode.assert_attr(message="expected test failure")
# assert "ValueError" in fnode.toxml()
Expand Down Expand Up @@ -538,14 +488,9 @@ def test_xpass():
result, dom = runandparse(testdir)
# assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=0, tests=1)
node.assert_attr(skipped=0, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_xfailure_xpass.py",
line="1",
classname="test_xfailure_xpass",
name="test_xpass",
)
tnode.assert_attr(classname="test_xfailure_xpass", name="test_xpass")

def test_xfailure_xpass_strict(self, testdir):
testdir.makepyfile(
Expand All @@ -559,14 +504,9 @@ def test_xpass():
result, dom = runandparse(testdir)
# assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(skips=0, tests=1)
node.assert_attr(skipped=0, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(
file="test_xfailure_xpass_strict.py",
line="1",
classname="test_xfailure_xpass_strict",
name="test_xpass",
)
tnode.assert_attr(classname="test_xfailure_xpass_strict", name="test_xpass")
fnode = tnode.find_first_by_tag("failure")
fnode.assert_attr(message="[XPASS(strict)] This needs to fail!")

Expand All @@ -577,8 +517,6 @@ def test_collect_error(self, testdir):
node = dom.find_first_by_tag("testsuite")
node.assert_attr(errors=1, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(file="test_collect_error.py", name="test_collect_error")
assert tnode["line"] is None
fnode = tnode.find_first_by_tag("error")
fnode.assert_attr(message="collection failure")
assert "SyntaxError" in fnode.toxml()
Expand Down Expand Up @@ -761,7 +699,7 @@ def repr_failure(self, excinfo):
result, dom = runandparse(testdir)
assert result.ret
node = dom.find_first_by_tag("testsuite")
node.assert_attr(errors=0, failures=1, skips=0, tests=1)
node.assert_attr(errors=0, failures=1, skipped=0, tests=1)
tnode = node.find_first_by_tag("testcase")
tnode.assert_attr(name="myfile.xyz")
fnode = tnode.find_first_by_tag("failure")
Expand Down Expand Up @@ -1123,20 +1061,18 @@ def test_pass():

assert "INTERNALERROR" not in result.stdout.str()

items = sorted(
"%(classname)s %(name)s %(file)s" % x for x in dom.find_by_tag("testcase")
)
items = sorted("%(classname)s %(name)s" % x for x in dom.find_by_tag("testcase"))
import pprint

pprint.pprint(items)
assert items == [
u"conftest a conftest.py",
u"conftest a conftest.py",
u"conftest b conftest.py",
u"test_fancy_items_regression a test_fancy_items_regression.py",
u"test_fancy_items_regression a test_fancy_items_regression.py",
u"test_fancy_items_regression b test_fancy_items_regression.py",
u"test_fancy_items_regression test_pass" u" test_fancy_items_regression.py",
u"conftest a",
u"conftest a",
u"conftest b",
u"test_fancy_items_regression a",
u"test_fancy_items_regression a",
u"test_fancy_items_regression b",
u"test_fancy_items_regression test_pass",
]


Expand Down