diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eef3b42e99e..64a955198ce 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -30,7 +30,7 @@ Features existing ``pytest_enter_pdb`` hook. -- `#4147 `_: Add ``-sw``, ``--stepwise`` as an alternative to ``--lf -x`` for stopping at the first failure, but starting the next test invocation from that test. See `the documentation `__ for more info. +- `#4147 `_: Add ``--sw``, ``--stepwise`` as an alternative to ``--lf -x`` for stopping at the first failure, but starting the next test invocation from that test. See `the documentation `__ for more info. - `#4188 `_: Make ``--color`` emit colorful dots when not running in verbose mode. Earlier, it would only colorize the test-by-test output if ``--verbose`` was also passed. @@ -60,6 +60,8 @@ Bug Fixes - `#611 `_: Naming a fixture ``request`` will now raise a warning: the ``request`` fixture is internal and should not be overwritten as it will lead to internal errors. +- `#4266 `_: Handle (ignore) exceptions raised during collection, e.g. with Django's LazySettings proxy class. + Improved Documentation diff --git a/changelog/4305.trivial.rst b/changelog/4305.trivial.rst new file mode 100644 index 00000000000..2430a5f91d4 --- /dev/null +++ b/changelog/4305.trivial.rst @@ -0,0 +1 @@ +Replace byte/unicode helpers in test_capture with python level syntax. diff --git a/changelog/4306.bugfix.rst b/changelog/4306.bugfix.rst new file mode 100644 index 00000000000..cb2872d3ff8 --- /dev/null +++ b/changelog/4306.bugfix.rst @@ -0,0 +1 @@ +Parse ``minversion`` as an actual version and not as dot-separated strings. diff --git a/changelog/4310.bugfix.rst b/changelog/4310.bugfix.rst new file mode 100644 index 00000000000..24e177c2ea1 --- /dev/null +++ b/changelog/4310.bugfix.rst @@ -0,0 +1 @@ +Fix duplicate collection due to multiple args matching the same packages. diff --git a/doc/4266.bugfix.rst b/doc/4266.bugfix.rst deleted file mode 100644 index f19a7cc1f28..00000000000 --- a/doc/4266.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Handle (ignore) exceptions raised during collection, e.g. with Django's LazySettings proxy class. diff --git a/doc/en/plugins.rst b/doc/en/plugins.rst index 62456e7dd73..3513e79fbd6 100644 --- a/doc/en/plugins.rst +++ b/doc/en/plugins.rst @@ -59,9 +59,9 @@ To see a complete list of all plugins with their latest testing status against different pytest and Python versions, please visit `plugincompat `_. -You may also discover more plugins through a `pytest- pypi.python.org search`_. +You may also discover more plugins through a `pytest- pypi.org search`_. -.. _`pytest- pypi.python.org search`: https://pypi.org/search/?q=pytest- +.. _`pytest- pypi.org search`: https://pypi.org/search/?q=pytest- .. _`available installable plugins`: diff --git a/src/_pytest/assertion/util.py b/src/_pytest/assertion/util.py index 0633410951b..451e454952b 100644 --- a/src/_pytest/assertion/util.py +++ b/src/_pytest/assertion/util.py @@ -11,8 +11,6 @@ import _pytest._code from ..compat import Sequence -u = six.text_type - # The _reprcompare attribute on the util module is used by the new assertion # interpretation code and assertion rewriter to detect this plugin was # loaded and in turn call the hooks defined here as part of the @@ -23,9 +21,9 @@ # the re-encoding is needed for python2 repr # with non-ascii characters (see issue 877 and 1379) def ecu(s): - try: - return u(s, "utf-8", "replace") - except TypeError: + if isinstance(s, bytes): + return s.decode("UTF-8", "replace") + else: return s @@ -42,7 +40,7 @@ def format_explanation(explanation): explanation = ecu(explanation) lines = _split_explanation(explanation) result = _format_lines(lines) - return u("\n").join(result) + return u"\n".join(result) def _split_explanation(explanation): @@ -52,7 +50,7 @@ def _split_explanation(explanation): Any other newlines will be escaped and appear in the line as the literal '\n' characters. """ - raw_lines = (explanation or u("")).split("\n") + raw_lines = (explanation or u"").split("\n") lines = [raw_lines[0]] for values in raw_lines[1:]: if values and values[0] in ["{", "}", "~", ">"]: @@ -77,13 +75,13 @@ def _format_lines(lines): for line in lines[1:]: if line.startswith("{"): if stackcnt[-1]: - s = u("and ") + s = u"and " else: - s = u("where ") + s = u"where " stack.append(len(result)) stackcnt[-1] += 1 stackcnt.append(0) - result.append(u(" +") + u(" ") * (len(stack) - 1) + s + line[1:]) + result.append(u" +" + u" " * (len(stack) - 1) + s + line[1:]) elif line.startswith("}"): stack.pop() stackcnt.pop() @@ -92,7 +90,7 @@ def _format_lines(lines): assert line[0] in ["~", ">"] stack[-1] += 1 indent = len(stack) if line.startswith("~") else len(stack) - 1 - result.append(u(" ") * indent + line[1:]) + result.append(u" " * indent + line[1:]) assert len(stack) == 1 return result @@ -110,7 +108,7 @@ def assertrepr_compare(config, op, left, right): left_repr = py.io.saferepr(left, maxsize=int(width // 2)) right_repr = py.io.saferepr(right, maxsize=width - len(left_repr)) - summary = u("%s %s %s") % (ecu(left_repr), op, ecu(right_repr)) + summary = u"%s %s %s" % (ecu(left_repr), op, ecu(right_repr)) def issequence(x): return isinstance(x, Sequence) and not isinstance(x, basestring) @@ -155,11 +153,9 @@ def isiterable(obj): explanation = _notin_text(left, right, verbose) except Exception: explanation = [ - u( - "(pytest_assertion plugin: representation of details failed. " - "Probably an object has a faulty __repr__.)" - ), - u(_pytest._code.ExceptionInfo()), + u"(pytest_assertion plugin: representation of details failed. " + u"Probably an object has a faulty __repr__.)", + six.text_type(_pytest._code.ExceptionInfo()), ] if not explanation: @@ -203,8 +199,7 @@ def escape_for_readable_diff(binary_text): if i > 42: i -= 10 # Provide some context explanation = [ - u("Skipping %s identical leading characters in diff, use -v to show") - % i + u"Skipping %s identical leading characters in diff, use -v to show" % i ] left = left[i:] right = right[i:] @@ -215,11 +210,8 @@ def escape_for_readable_diff(binary_text): if i > 42: i -= 10 # Provide some context explanation += [ - u( - "Skipping %s identical trailing " - "characters in diff, use -v to show" - ) - % i + u"Skipping {} identical trailing " + u"characters in diff, use -v to show".format(i) ] left = left[:-i] right = right[:-i] @@ -237,21 +229,21 @@ def escape_for_readable_diff(binary_text): def _compare_eq_iterable(left, right, verbose=False): if not verbose: - return [u("Use -v to get the full diff")] + return [u"Use -v to get the full diff"] # dynamic import to speedup pytest import difflib try: left_formatting = pprint.pformat(left).splitlines() right_formatting = pprint.pformat(right).splitlines() - explanation = [u("Full diff:")] + explanation = [u"Full diff:"] except Exception: # hack: PrettyPrinter.pformat() in python 2 fails when formatting items that can't be sorted(), ie, calling # sorted() on a list would raise. See issue #718. # As a workaround, the full diff is generated by using the repr() string of each item of each container. left_formatting = sorted(repr(x) for x in left) right_formatting = sorted(repr(x) for x in right) - explanation = [u("Full diff (fallback to calling repr on each item):")] + explanation = [u"Full diff (fallback to calling repr on each item):"] explanation.extend( line.strip() for line in difflib.ndiff(left_formatting, right_formatting) ) @@ -262,16 +254,16 @@ def _compare_eq_sequence(left, right, verbose=False): explanation = [] for i in range(min(len(left), len(right))): if left[i] != right[i]: - explanation += [u("At index %s diff: %r != %r") % (i, left[i], right[i])] + explanation += [u"At index %s diff: %r != %r" % (i, left[i], right[i])] break if len(left) > len(right): explanation += [ - u("Left contains more items, first extra item: %s") + u"Left contains more items, first extra item: %s" % py.io.saferepr(left[len(right)]) ] elif len(left) < len(right): explanation += [ - u("Right contains more items, first extra item: %s") + u"Right contains more items, first extra item: %s" % py.io.saferepr(right[len(left)]) ] return explanation @@ -282,11 +274,11 @@ def _compare_eq_set(left, right, verbose=False): diff_left = left - right diff_right = right - left if diff_left: - explanation.append(u("Extra items in the left set:")) + explanation.append(u"Extra items in the left set:") for item in diff_left: explanation.append(py.io.saferepr(item)) if diff_right: - explanation.append(u("Extra items in the right set:")) + explanation.append(u"Extra items in the right set:") for item in diff_right: explanation.append(py.io.saferepr(item)) return explanation @@ -297,26 +289,26 @@ def _compare_eq_dict(left, right, verbose=False): common = set(left).intersection(set(right)) same = {k: left[k] for k in common if left[k] == right[k]} if same and verbose < 2: - explanation += [u("Omitting %s identical items, use -vv to show") % len(same)] + explanation += [u"Omitting %s identical items, use -vv to show" % len(same)] elif same: - explanation += [u("Common items:")] + explanation += [u"Common items:"] explanation += pprint.pformat(same).splitlines() diff = {k for k in common if left[k] != right[k]} if diff: - explanation += [u("Differing items:")] + explanation += [u"Differing items:"] for k in diff: explanation += [ py.io.saferepr({k: left[k]}) + " != " + py.io.saferepr({k: right[k]}) ] extra_left = set(left) - set(right) if extra_left: - explanation.append(u("Left contains more items:")) + explanation.append(u"Left contains more items:") explanation.extend( pprint.pformat({k: left[k] for k in extra_left}).splitlines() ) extra_right = set(right) - set(left) if extra_right: - explanation.append(u("Right contains more items:")) + explanation.append(u"Right contains more items:") explanation.extend( pprint.pformat({k: right[k] for k in extra_right}).splitlines() ) @@ -329,14 +321,14 @@ def _notin_text(term, text, verbose=False): tail = text[index + len(term) :] correct_text = head + tail diff = _diff_text(correct_text, text, verbose) - newdiff = [u("%s is contained here:") % py.io.saferepr(term, maxsize=42)] + newdiff = [u"%s is contained here:" % py.io.saferepr(term, maxsize=42)] for line in diff: - if line.startswith(u("Skipping")): + if line.startswith(u"Skipping"): continue - if line.startswith(u("- ")): + if line.startswith(u"- "): continue - if line.startswith(u("+ ")): - newdiff.append(u(" ") + line[2:]) + if line.startswith(u"+ "): + newdiff.append(u" " + line[2:]) else: newdiff.append(line) return newdiff diff --git a/src/_pytest/capture.py b/src/_pytest/capture.py index 99e95a442ff..ec72ae3ecd6 100644 --- a/src/_pytest/capture.py +++ b/src/_pytest/capture.py @@ -504,7 +504,7 @@ class FDCaptureBinary(object): snap() produces `bytes` """ - EMPTY_BUFFER = bytes() + EMPTY_BUFFER = b"" def __init__(self, targetfd, tmpfile=None): self.targetfd = targetfd @@ -630,7 +630,7 @@ def writeorg(self, data): class SysCaptureBinary(SysCapture): - EMPTY_BUFFER = bytes() + EMPTY_BUFFER = b"" def snap(self): res = self.tmpfile.buffer.getvalue() diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index daf9194062e..fdcca5ebdf7 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -11,6 +11,7 @@ import sys import types import warnings +from distutils.version import LooseVersion import py import six @@ -817,9 +818,7 @@ def _checkversion(self): minver = self.inicfg.get("minversion", None) if minver: - ver = minver.split(".") - myver = pytest.__version__.split(".") - if myver < ver: + if LooseVersion(minver) > LooseVersion(pytest.__version__): raise pytest.UsageError( "%s:%d: requires pytest-%s, actual pytest-%s'" % ( diff --git a/src/_pytest/main.py b/src/_pytest/main.py index d626816a822..9674f869b13 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -19,7 +19,6 @@ from _pytest.config import hookimpl from _pytest.config import UsageError from _pytest.outcomes import exit -from _pytest.pathlib import parts from _pytest.runner import collect_one_node @@ -399,6 +398,7 @@ def __init__(self, config): # Keep track of any collected nodes in here, so we don't duplicate fixtures self._node_cache = {} self._bestrelpathcache = _bestrelpath_cache(config.rootdir) + self._pkg_roots = {} self.config.pluginmanager.register(self, name="session") @@ -505,30 +505,26 @@ def _collect(self, arg): names = self._parsearg(arg) argpath = names.pop(0).realpath() - paths = set() - root = self # Start with a Session root, and delve to argpath item (dir or file) # and stack all Packages found on the way. # No point in finding packages when collecting doctests if not self.config.option.doctestmodules: + pm = self.config.pluginmanager for parent in argpath.parts(): - pm = self.config.pluginmanager if pm._confcutdir and pm._confcutdir.relto(parent): continue if parent.isdir(): pkginit = parent.join("__init__.py") if pkginit.isfile(): - if pkginit in self._node_cache: - root = self._node_cache[pkginit][0] - else: - col = root._collectfile(pkginit) + if pkginit not in self._node_cache: + col = self._collectfile(pkginit, handle_dupes=False) if col: if isinstance(col[0], Package): - root = col[0] + self._pkg_roots[parent] = col[0] # always store a list in the cache, matchnodes expects it - self._node_cache[root.fspath] = [root] + self._node_cache[col[0].fspath] = [col[0]] # If it's a directory argument, recurse and look for any Subpackages. # Let the Package collector deal with subnodes, don't collect here. @@ -551,28 +547,34 @@ def filter_(f): ): dirpath = path.dirpath() if dirpath not in seen_dirs: + # Collect packages first. seen_dirs.add(dirpath) pkginit = dirpath.join("__init__.py") - if pkginit.exists() and parts(pkginit.strpath).isdisjoint(paths): - for x in root._collectfile(pkginit): - yield x - paths.add(x.fspath.dirpath()) - - if parts(path.strpath).isdisjoint(paths): - for x in root._collectfile(path): - key = (type(x), x.fspath) - if key in self._node_cache: - yield self._node_cache[key] - else: - self._node_cache[key] = x + if pkginit.exists(): + collect_root = self._pkg_roots.get(dirpath, self) + for x in collect_root._collectfile(pkginit): yield x + if isinstance(x, Package): + self._pkg_roots[dirpath] = x + if dirpath in self._pkg_roots: + # Do not collect packages here. + continue + + for x in self._collectfile(path): + key = (type(x), x.fspath) + if key in self._node_cache: + yield self._node_cache[key] + else: + self._node_cache[key] = x + yield x else: assert argpath.check(file=1) if argpath in self._node_cache: col = self._node_cache[argpath] else: - col = root._collectfile(argpath) + collect_root = self._pkg_roots.get(argpath.dirname, self) + col = collect_root._collectfile(argpath) if col: self._node_cache[argpath] = col m = self.matchnodes(col, names) @@ -586,20 +588,20 @@ def filter_(f): for y in m: yield y - def _collectfile(self, path): + def _collectfile(self, path, handle_dupes=True): ihook = self.gethookproxy(path) if not self.isinitpath(path): if ihook.pytest_ignore_collect(path=path, config=self.config): return () - # Skip duplicate paths. - keepduplicates = self.config.getoption("keepduplicates") - if not keepduplicates: - duplicate_paths = self.config.pluginmanager._duplicatepaths - if path in duplicate_paths: - return () - else: - duplicate_paths.add(path) + if handle_dupes: + keepduplicates = self.config.getoption("keepduplicates") + if not keepduplicates: + duplicate_paths = self.config.pluginmanager._duplicatepaths + if path in duplicate_paths: + return () + else: + duplicate_paths.add(path) return ihook.pytest_collect_file(path=path, parent=self) diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 03a9fe03105..d360e2c8f3e 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -545,11 +545,24 @@ def gethookproxy(self, fspath): proxy = self.config.hook return proxy - def _collectfile(self, path): + def _collectfile(self, path, handle_dupes=True): ihook = self.gethookproxy(path) if not self.isinitpath(path): if ihook.pytest_ignore_collect(path=path, config=self.config): return () + + if handle_dupes: + keepduplicates = self.config.getoption("keepduplicates") + if not keepduplicates: + duplicate_paths = self.config.pluginmanager._duplicatepaths + if path in duplicate_paths: + return () + else: + duplicate_paths.add(path) + + if self.fspath == path: # __init__.py + return [self] + return ihook.pytest_collect_file(path=path, parent=self) def isinitpath(self, path): diff --git a/testing/test_assertion.py b/testing/test_assertion.py index 94fe9d27235..b6c31aba2bf 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -539,11 +539,8 @@ def test_format_nonascii_explanation(self): def test_mojibake(self): # issue 429 - left = "e" - right = "\xc3\xa9" - if not isinstance(left, bytes): - left = bytes(left, "utf-8") - right = bytes(right, "utf-8") + left = b"e" + right = b"\xc3\xa9" expl = callequal(left, right) for line in expl: assert isinstance(line, six.text_type) diff --git a/testing/test_capture.py b/testing/test_capture.py index 4b9b543a5be..39ddd1f79be 100644 --- a/testing/test_capture.py +++ b/testing/test_capture.py @@ -27,24 +27,6 @@ ) -def tobytes(obj): - if isinstance(obj, text_type): - obj = obj.encode("UTF-8") - assert isinstance(obj, bytes) - return obj - - -def totext(obj): - if isinstance(obj, bytes): - obj = text_type(obj, "UTF-8") - assert isinstance(obj, text_type) - return obj - - -def oswritebytes(fd, obj): - os.write(fd, tobytes(obj)) - - def StdCaptureFD(out=True, err=True, in_=True): return capture.MultiCapture(out, err, in_, Capture=capture.FDCapture) @@ -836,10 +818,11 @@ def test_write_bytes_to_buffer(self): def test_bytes_io(): f = py.io.BytesIO() - f.write(tobytes("hello")) - pytest.raises(TypeError, "f.write(totext('hello'))") + f.write(b"hello") + with pytest.raises(TypeError): + f.write(u"hello") s = f.getvalue() - assert s == tobytes("hello") + assert s == b"hello" def test_dontreadfrominput(): @@ -952,7 +935,7 @@ class TestFDCapture(object): def test_simple(self, tmpfile): fd = tmpfile.fileno() cap = capture.FDCapture(fd) - data = tobytes("hello") + data = b"hello" os.write(fd, data) s = cap.snap() cap.done() @@ -992,10 +975,10 @@ def test_stdin(self, tmpfile): cap.start() x = os.read(0, 100).strip() cap.done() - assert x == tobytes("") + assert x == b"" def test_writeorg(self, tmpfile): - data1, data2 = tobytes("foo"), tobytes("bar") + data1, data2 = b"foo", b"bar" cap = capture.FDCapture(tmpfile.fileno()) cap.start() tmpfile.write(data1) @@ -1003,7 +986,7 @@ def test_writeorg(self, tmpfile): cap.writeorg(data2) scap = cap.snap() cap.done() - assert scap == totext(data1) + assert scap == data1.decode("ascii") with open(tmpfile.name, "rb") as stmp_file: stmp = stmp_file.read() assert stmp == data2 @@ -1012,17 +995,17 @@ def test_simple_resume_suspend(self, tmpfile): with saved_fd(1): cap = capture.FDCapture(1) cap.start() - data = tobytes("hello") + data = b"hello" os.write(1, data) sys.stdout.write("whatever") s = cap.snap() assert s == "hellowhatever" cap.suspend() - os.write(1, tobytes("world")) + os.write(1, b"world") sys.stdout.write("qlwkej") assert not cap.snap() cap.resume() - os.write(1, tobytes("but now")) + os.write(1, b"but now") sys.stdout.write(" yes\n") s = cap.snap() assert s == "but now yes\n" @@ -1193,14 +1176,14 @@ def test_x(): def test_intermingling(self): with self.getcapture() as cap: - oswritebytes(1, "1") + os.write(1, b"1") sys.stdout.write(str(2)) sys.stdout.flush() - oswritebytes(1, "3") - oswritebytes(2, "a") + os.write(1, b"3") + os.write(2, b"a") sys.stderr.write("b") sys.stderr.flush() - oswritebytes(2, "c") + os.write(2, b"c") out, err = cap.readouterr() assert out == "123" assert err == "abc" diff --git a/testing/test_collection.py b/testing/test_collection.py index ad55ac98c5f..5747e5ed0f3 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -951,26 +951,58 @@ def test_collect_init_tests(testdir): result = testdir.runpytest(p, "--collect-only") result.stdout.fnmatch_lines( [ - "*", - "*", - "*", - "*", + "collected 2 items", + "", + " ", + " ", + " ", ] ) result = testdir.runpytest("./tests", "--collect-only") result.stdout.fnmatch_lines( [ - "*", - "*", - "*", - "*", + "collected 2 items", + "", + " ", + " ", + " ", + ] + ) + # Ignores duplicates with "." and pkginit (#4310). + result = testdir.runpytest("./tests", ".", "--collect-only") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "", + " ", + " ", + " ", + " ", + ] + ) + # Same as before, but different order. + result = testdir.runpytest(".", "tests", "--collect-only") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "", + " ", + " ", + " ", + " ", ] ) result = testdir.runpytest("./tests/test_foo.py", "--collect-only") - result.stdout.fnmatch_lines(["*", "*"]) + result.stdout.fnmatch_lines( + ["", " ", " "] + ) assert "test_init" not in result.stdout.str() result = testdir.runpytest("./tests/__init__.py", "--collect-only") - result.stdout.fnmatch_lines(["*", "*"]) + result.stdout.fnmatch_lines( + ["", " ", " "] + ) assert "test_foo" not in result.stdout.str()