Merge pull request #4338 from blueyed/merge-master

Merge master into features

CHANGELOG.rst

@@ -30,7 +30,7 @@ Features
   existing ``pytest_enter_pdb`` hook.
 
 
-- `#4147 <https://github.com/pytest-dev/pytest/issues/4147>`_: Add ``-sw``, ``--stepwise`` as an alternative to ``--lf -x`` for stopping at the first failure, but starting the next test invocation from that test.  See `the documentation <https://docs.pytest.org/en/latest/cache.html#stepwise>`__ for more info.
+- `#4147 <https://github.com/pytest-dev/pytest/issues/4147>`_: Add ``--sw``, ``--stepwise`` as an alternative to ``--lf -x`` for stopping at the first failure, but starting the next test invocation from that test.  See `the documentation <https://docs.pytest.org/en/latest/cache.html#stepwise>`__ for more info.
 
 
 - `#4188 <https://github.com/pytest-dev/pytest/issues/4188>`_: Make ``--color`` emit colorful dots when not running in verbose mode. Earlier, it would only colorize the test-by-test output if ``--verbose`` was also passed.
@@ -60,6 +60,8 @@ Bug Fixes
 - `#611 <https://github.com/pytest-dev/pytest/issues/611>`_: Naming a fixture ``request`` will now raise a warning: the ``request`` fixture is internal and
   should not be overwritten as it will lead to internal errors.
 
+- `#4266 <https://github.com/pytest-dev/pytest/issues/4266>`_: Handle (ignore) exceptions raised during collection, e.g. with Django's LazySettings proxy class.
+
 
 
 Improved Documentation

changelog/4305.trivial.rst

@@ -0,0 +1 @@
+Replace byte/unicode helpers in test_capture with python level syntax.

changelog/4306.bugfix.rst

@@ -0,0 +1 @@
+Parse ``minversion`` as an actual version and not as dot-separated strings.

changelog/4310.bugfix.rst

@@ -0,0 +1 @@
+Fix duplicate collection due to multiple args matching the same packages.

doc/4266.bugfix.rst

@@ -1 +0,0 @@
-Handle (ignore) exceptions raised during collection, e.g. with Django's LazySettings proxy class.

doc/en/plugins.rst

@@ -59,9 +59,9 @@ To see a complete list of all plugins with their latest testing
 status against different pytest and Python versions, please visit
 `plugincompat <http://plugincompat.herokuapp.com/>`_.
 
-You may also discover more plugins through a `pytest- pypi.python.org search`_.
+You may also discover more plugins through a `pytest- pypi.org search`_.
 
-.. _`pytest- pypi.python.org search`: https://pypi.org/search/?q=pytest-
+.. _`pytest- pypi.org search`: https://pypi.org/search/?q=pytest-
 
 
 .. _`available installable plugins`:

src/_pytest/assertion/util.py

@@ -11,8 +11,6 @@ import six
 import _pytest._code
 from ..compat import Sequence
 
-u = six.text_type
-
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
 # loaded and in turn call the hooks defined here as part of the
@@ -23,9 +21,9 @@ _reprcompare = None
 # the re-encoding is needed for python2 repr
 # with non-ascii characters (see issue 877 and 1379)
 def ecu(s):
-    try:
+    if isinstance(s, bytes):
-        return u(s, "utf-8", "replace")
+        return s.decode("UTF-8", "replace")
-    except TypeError:
+    else:
         return s
 
 
@@ -42,7 +40,7 @@ def format_explanation(explanation):
     explanation = ecu(explanation)
     lines = _split_explanation(explanation)
     result = _format_lines(lines)
-    return u("\n").join(result)
+    return u"\n".join(result)
 
 
 def _split_explanation(explanation):
@@ -52,7 +50,7 @@ def _split_explanation(explanation):
     Any other newlines will be escaped and appear in the line as the
     literal '\n' characters.
     """
-    raw_lines = (explanation or u("")).split("\n")
+    raw_lines = (explanation or u"").split("\n")
     lines = [raw_lines[0]]
     for values in raw_lines[1:]:
         if values and values[0] in ["{", "}", "~", ">"]:
@@ -77,13 +75,13 @@ def _format_lines(lines):
     for line in lines[1:]:
         if line.startswith("{"):
             if stackcnt[-1]:
-                s = u("and   ")
+                s = u"and   "
             else:
-                s = u("where ")
+                s = u"where "
             stack.append(len(result))
             stackcnt[-1] += 1
             stackcnt.append(0)
-            result.append(u(" +") + u("  ") * (len(stack) - 1) + s + line[1:])
+            result.append(u" +" + u"  " * (len(stack) - 1) + s + line[1:])
         elif line.startswith("}"):
             stack.pop()
             stackcnt.pop()
@@ -92,7 +90,7 @@ def _format_lines(lines):
             assert line[0] in ["~", ">"]
             stack[-1] += 1
             indent = len(stack) if line.startswith("~") else len(stack) - 1
-            result.append(u("  ") * indent + line[1:])
+            result.append(u"  " * indent + line[1:])
     assert len(stack) == 1
     return result
 
@@ -110,7 +108,7 @@ def assertrepr_compare(config, op, left, right):
     left_repr = py.io.saferepr(left, maxsize=int(width // 2))
     right_repr = py.io.saferepr(right, maxsize=width - len(left_repr))
 
-    summary = u("%s %s %s") % (ecu(left_repr), op, ecu(right_repr))
+    summary = u"%s %s %s" % (ecu(left_repr), op, ecu(right_repr))
 
     def issequence(x):
         return isinstance(x, Sequence) and not isinstance(x, basestring)
@@ -155,11 +153,9 @@ def assertrepr_compare(config, op, left, right):
                 explanation = _notin_text(left, right, verbose)
     except Exception:
         explanation = [
-            u(
+            u"(pytest_assertion plugin: representation of details failed.  "
-                "(pytest_assertion plugin: representation of details failed.  "
+            u"Probably an object has a faulty __repr__.)",
-                "Probably an object has a faulty __repr__.)"
+            six.text_type(_pytest._code.ExceptionInfo()),
-            ),
-            u(_pytest._code.ExceptionInfo()),
         ]
 
     if not explanation:
@@ -203,8 +199,7 @@ def _diff_text(left, right, verbose=False):
         if i > 42:
             i -= 10  # Provide some context
             explanation = [
-                u("Skipping %s identical leading characters in diff, use -v to show")
+                u"Skipping %s identical leading characters in diff, use -v to show" % i
-                % i
             ]
             left = left[i:]
             right = right[i:]
@@ -215,11 +210,8 @@ def _diff_text(left, right, verbose=False):
             if i > 42:
                 i -= 10  # Provide some context
                 explanation += [
-                    u(
+                    u"Skipping {} identical trailing "
-                        "Skipping %s identical trailing "
+                    u"characters in diff, use -v to show".format(i)
-                        "characters in diff, use -v to show"
-                    )
-                    % i
                 ]
                 left = left[:-i]
                 right = right[:-i]
@@ -237,21 +229,21 @@ def _diff_text(left, right, verbose=False):
 
 def _compare_eq_iterable(left, right, verbose=False):
     if not verbose:
-        return [u("Use -v to get the full diff")]
+        return [u"Use -v to get the full diff"]
     # dynamic import to speedup pytest
     import difflib
 
     try:
         left_formatting = pprint.pformat(left).splitlines()
         right_formatting = pprint.pformat(right).splitlines()
-        explanation = [u("Full diff:")]
+        explanation = [u"Full diff:"]
     except Exception:
         # hack: PrettyPrinter.pformat() in python 2 fails when formatting items that can't be sorted(), ie, calling
         # sorted() on a list would raise. See issue #718.
         # As a workaround, the full diff is generated by using the repr() string of each item of each container.
         left_formatting = sorted(repr(x) for x in left)
         right_formatting = sorted(repr(x) for x in right)
-        explanation = [u("Full diff (fallback to calling repr on each item):")]
+        explanation = [u"Full diff (fallback to calling repr on each item):"]
     explanation.extend(
         line.strip() for line in difflib.ndiff(left_formatting, right_formatting)
     )
@@ -262,16 +254,16 @@ def _compare_eq_sequence(left, right, verbose=False):
     explanation = []
     for i in range(min(len(left), len(right))):
         if left[i] != right[i]:
-            explanation += [u("At index %s diff: %r != %r") % (i, left[i], right[i])]
+            explanation += [u"At index %s diff: %r != %r" % (i, left[i], right[i])]
             break
     if len(left) > len(right):
         explanation += [
-            u("Left contains more items, first extra item: %s")
+            u"Left contains more items, first extra item: %s"
             % py.io.saferepr(left[len(right)])
         ]
     elif len(left) < len(right):
         explanation += [
-            u("Right contains more items, first extra item: %s")
+            u"Right contains more items, first extra item: %s"
             % py.io.saferepr(right[len(left)])
         ]
     return explanation
@@ -282,11 +274,11 @@ def _compare_eq_set(left, right, verbose=False):
     diff_left = left - right
     diff_right = right - left
     if diff_left:
-        explanation.append(u("Extra items in the left set:"))
+        explanation.append(u"Extra items in the left set:")
         for item in diff_left:
             explanation.append(py.io.saferepr(item))
     if diff_right:
-        explanation.append(u("Extra items in the right set:"))
+        explanation.append(u"Extra items in the right set:")
         for item in diff_right:
             explanation.append(py.io.saferepr(item))
     return explanation
@@ -297,26 +289,26 @@ def _compare_eq_dict(left, right, verbose=False):
     common = set(left).intersection(set(right))
     same = {k: left[k] for k in common if left[k] == right[k]}
     if same and verbose < 2:
-        explanation += [u("Omitting %s identical items, use -vv to show") % len(same)]
+        explanation += [u"Omitting %s identical items, use -vv to show" % len(same)]
     elif same:
-        explanation += [u("Common items:")]
+        explanation += [u"Common items:"]
         explanation += pprint.pformat(same).splitlines()
     diff = {k for k in common if left[k] != right[k]}
     if diff:
-        explanation += [u("Differing items:")]
+        explanation += [u"Differing items:"]
         for k in diff:
             explanation += [
                 py.io.saferepr({k: left[k]}) + " != " + py.io.saferepr({k: right[k]})
             ]
     extra_left = set(left) - set(right)
     if extra_left:
-        explanation.append(u("Left contains more items:"))
+        explanation.append(u"Left contains more items:")
         explanation.extend(
             pprint.pformat({k: left[k] for k in extra_left}).splitlines()
         )
     extra_right = set(right) - set(left)
     if extra_right:
-        explanation.append(u("Right contains more items:"))
+        explanation.append(u"Right contains more items:")
         explanation.extend(
             pprint.pformat({k: right[k] for k in extra_right}).splitlines()
         )
@@ -329,14 +321,14 @@ def _notin_text(term, text, verbose=False):
     tail = text[index + len(term) :]
     correct_text = head + tail
     diff = _diff_text(correct_text, text, verbose)
-    newdiff = [u("%s is contained here:") % py.io.saferepr(term, maxsize=42)]
+    newdiff = [u"%s is contained here:" % py.io.saferepr(term, maxsize=42)]
     for line in diff:
-        if line.startswith(u("Skipping")):
+        if line.startswith(u"Skipping"):
             continue
-        if line.startswith(u("- ")):
+        if line.startswith(u"- "):
             continue
-        if line.startswith(u("+ ")):
+        if line.startswith(u"+ "):
-            newdiff.append(u("  ") + line[2:])
+            newdiff.append(u"  " + line[2:])
         else:
             newdiff.append(line)
     return newdiff

src/_pytest/capture.py

@@ -504,7 +504,7 @@ class FDCaptureBinary(object):
     snap() produces `bytes`
     """
 
-    EMPTY_BUFFER = bytes()
+    EMPTY_BUFFER = b""
 
     def __init__(self, targetfd, tmpfile=None):
         self.targetfd = targetfd
@@ -630,7 +630,7 @@ class SysCapture(object):
 
 
 class SysCaptureBinary(SysCapture):
-    EMPTY_BUFFER = bytes()
+    EMPTY_BUFFER = b""
 
     def snap(self):
         res = self.tmpfile.buffer.getvalue()

src/_pytest/config/__init__.py

@@ -11,6 +11,7 @@ import shlex
 import sys
 import types
 import warnings
+from distutils.version import LooseVersion
 
 import py
 import six
@@ -817,9 +818,7 @@ class Config(object):
 
         minver = self.inicfg.get("minversion", None)
         if minver:
-            ver = minver.split(".")
+            if LooseVersion(minver) > LooseVersion(pytest.__version__):
-            myver = pytest.__version__.split(".")
-            if myver < ver:
                 raise pytest.UsageError(
                     "%s:%d: requires pytest-%s, actual pytest-%s'"
                     % (

src/_pytest/main.py

@@ -19,7 +19,6 @@ from _pytest.config import directory_arg
 from _pytest.config import hookimpl
 from _pytest.config import UsageError
 from _pytest.outcomes import exit
-from _pytest.pathlib import parts
 from _pytest.runner import collect_one_node
 
 
@@ -399,6 +398,7 @@ class Session(nodes.FSCollector):
         # Keep track of any collected nodes in here, so we don't duplicate fixtures
         self._node_cache = {}
         self._bestrelpathcache = _bestrelpath_cache(config.rootdir)
+        self._pkg_roots = {}
 
         self.config.pluginmanager.register(self, name="session")
 
@@ -505,30 +505,26 @@ class Session(nodes.FSCollector):
 
         names = self._parsearg(arg)
         argpath = names.pop(0).realpath()
-        paths = set()
 
-        root = self
         # Start with a Session root, and delve to argpath item (dir or file)
         # and stack all Packages found on the way.
         # No point in finding packages when collecting doctests
         if not self.config.option.doctestmodules:
-            for parent in argpath.parts():
             pm = self.config.pluginmanager
+            for parent in argpath.parts():
                 if pm._confcutdir and pm._confcutdir.relto(parent):
                     continue
 
                 if parent.isdir():
                     pkginit = parent.join("__init__.py")
                     if pkginit.isfile():
-                        if pkginit in self._node_cache:
+                        if pkginit not in self._node_cache:
-                            root = self._node_cache[pkginit][0]
+                            col = self._collectfile(pkginit, handle_dupes=False)
-                        else:
-                            col = root._collectfile(pkginit)
                             if col:
                                 if isinstance(col[0], Package):
-                                    root = col[0]
+                                    self._pkg_roots[parent] = col[0]
                                 # always store a list in the cache, matchnodes expects it
-                                self._node_cache[root.fspath] = [root]
+                                self._node_cache[col[0].fspath] = [col[0]]
 
         # If it's a directory argument, recurse and look for any Subpackages.
         # Let the Package collector deal with subnodes, don't collect here.
@@ -551,15 +547,20 @@ class Session(nodes.FSCollector):
             ):
                 dirpath = path.dirpath()
                 if dirpath not in seen_dirs:
+                    # Collect packages first.
                     seen_dirs.add(dirpath)
                     pkginit = dirpath.join("__init__.py")
-                    if pkginit.exists() and parts(pkginit.strpath).isdisjoint(paths):
+                    if pkginit.exists():
-                        for x in root._collectfile(pkginit):
+                        collect_root = self._pkg_roots.get(dirpath, self)
+                        for x in collect_root._collectfile(pkginit):
                             yield x
-                            paths.add(x.fspath.dirpath())
+                            if isinstance(x, Package):
+                                self._pkg_roots[dirpath] = x
+                if dirpath in self._pkg_roots:
+                    # Do not collect packages here.
+                    continue
 
-                if parts(path.strpath).isdisjoint(paths):
+                for x in self._collectfile(path):
-                    for x in root._collectfile(path):
                     key = (type(x), x.fspath)
                     if key in self._node_cache:
                         yield self._node_cache[key]
@@ -572,7 +573,8 @@ class Session(nodes.FSCollector):
             if argpath in self._node_cache:
                 col = self._node_cache[argpath]
             else:
-                col = root._collectfile(argpath)
+                collect_root = self._pkg_roots.get(argpath.dirname, self)
+                col = collect_root._collectfile(argpath)
                 if col:
                     self._node_cache[argpath] = col
             m = self.matchnodes(col, names)
@@ -586,13 +588,13 @@ class Session(nodes.FSCollector):
             for y in m:
                 yield y
 
-    def _collectfile(self, path):
+    def _collectfile(self, path, handle_dupes=True):
         ihook = self.gethookproxy(path)
         if not self.isinitpath(path):
             if ihook.pytest_ignore_collect(path=path, config=self.config):
                 return ()
 
-        # Skip duplicate paths.
+        if handle_dupes:
             keepduplicates = self.config.getoption("keepduplicates")
             if not keepduplicates:
                 duplicate_paths = self.config.pluginmanager._duplicatepaths

src/_pytest/python.py

@@ -545,11 +545,24 @@ class Package(Module):
             proxy = self.config.hook
         return proxy
 
-    def _collectfile(self, path):
+    def _collectfile(self, path, handle_dupes=True):
         ihook = self.gethookproxy(path)
         if not self.isinitpath(path):
             if ihook.pytest_ignore_collect(path=path, config=self.config):
                 return ()
+
+        if handle_dupes:
+            keepduplicates = self.config.getoption("keepduplicates")
+            if not keepduplicates:
+                duplicate_paths = self.config.pluginmanager._duplicatepaths
+                if path in duplicate_paths:
+                    return ()
+                else:
+                    duplicate_paths.add(path)
+
+        if self.fspath == path:  # __init__.py
+            return [self]
+
         return ihook.pytest_collect_file(path=path, parent=self)
 
     def isinitpath(self, path):

testing/test_assertion.py

@@ -539,11 +539,8 @@ class TestAssert_reprcompare(object):
 
     def test_mojibake(self):
         # issue 429
-        left = "e"
+        left = b"e"
-        right = "\xc3\xa9"
+        right = b"\xc3\xa9"
-        if not isinstance(left, bytes):
-            left = bytes(left, "utf-8")
-            right = bytes(right, "utf-8")
         expl = callequal(left, right)
         for line in expl:
             assert isinstance(line, six.text_type)

testing/test_capture.py

@@ -27,24 +27,6 @@ needsosdup = pytest.mark.skipif(
 )
 
 
-def tobytes(obj):
-    if isinstance(obj, text_type):
-        obj = obj.encode("UTF-8")
-    assert isinstance(obj, bytes)
-    return obj
-
-
-def totext(obj):
-    if isinstance(obj, bytes):
-        obj = text_type(obj, "UTF-8")
-    assert isinstance(obj, text_type)
-    return obj
-
-
-def oswritebytes(fd, obj):
-    os.write(fd, tobytes(obj))
-
-
 def StdCaptureFD(out=True, err=True, in_=True):
     return capture.MultiCapture(out, err, in_, Capture=capture.FDCapture)
 
@@ -836,10 +818,11 @@ class TestCaptureIO(object):
 
 def test_bytes_io():
     f = py.io.BytesIO()
-    f.write(tobytes("hello"))
+    f.write(b"hello")
-    pytest.raises(TypeError, "f.write(totext('hello'))")
+    with pytest.raises(TypeError):
+        f.write(u"hello")
     s = f.getvalue()
-    assert s == tobytes("hello")
+    assert s == b"hello"
 
 
 def test_dontreadfrominput():
@@ -952,7 +935,7 @@ class TestFDCapture(object):
     def test_simple(self, tmpfile):
         fd = tmpfile.fileno()
         cap = capture.FDCapture(fd)
-        data = tobytes("hello")
+        data = b"hello"
         os.write(fd, data)
         s = cap.snap()
         cap.done()
@@ -992,10 +975,10 @@ class TestFDCapture(object):
         cap.start()
         x = os.read(0, 100).strip()
         cap.done()
-        assert x == tobytes("")
+        assert x == b""
 
     def test_writeorg(self, tmpfile):
-        data1, data2 = tobytes("foo"), tobytes("bar")
+        data1, data2 = b"foo", b"bar"
         cap = capture.FDCapture(tmpfile.fileno())
         cap.start()
         tmpfile.write(data1)
@@ -1003,7 +986,7 @@ class TestFDCapture(object):
         cap.writeorg(data2)
         scap = cap.snap()
         cap.done()
-        assert scap == totext(data1)
+        assert scap == data1.decode("ascii")
         with open(tmpfile.name, "rb") as stmp_file:
             stmp = stmp_file.read()
             assert stmp == data2
@@ -1012,17 +995,17 @@ class TestFDCapture(object):
         with saved_fd(1):
             cap = capture.FDCapture(1)
             cap.start()
-            data = tobytes("hello")
+            data = b"hello"
             os.write(1, data)
             sys.stdout.write("whatever")
             s = cap.snap()
             assert s == "hellowhatever"
             cap.suspend()
-            os.write(1, tobytes("world"))
+            os.write(1, b"world")
             sys.stdout.write("qlwkej")
             assert not cap.snap()
             cap.resume()
-            os.write(1, tobytes("but now"))
+            os.write(1, b"but now")
             sys.stdout.write(" yes\n")
             s = cap.snap()
             assert s == "but now yes\n"
@@ -1193,14 +1176,14 @@ class TestStdCaptureFD(TestStdCapture):
 
     def test_intermingling(self):
         with self.getcapture() as cap:
-            oswritebytes(1, "1")
+            os.write(1, b"1")
             sys.stdout.write(str(2))
             sys.stdout.flush()
-            oswritebytes(1, "3")
+            os.write(1, b"3")
-            oswritebytes(2, "a")
+            os.write(2, b"a")
             sys.stderr.write("b")
             sys.stderr.flush()
-            oswritebytes(2, "c")
+            os.write(2, b"c")
             out, err = cap.readouterr()
         assert out == "123"
         assert err == "abc"

testing/test_collection.py

@@ -951,26 +951,58 @@ def test_collect_init_tests(testdir):
     result = testdir.runpytest(p, "--collect-only")
     result.stdout.fnmatch_lines(
         [
-            "*<Module '__init__.py'>",
+            "collected 2 items",
-            "*<Function 'test_init'>",
+            "<Package *",
-            "*<Module 'test_foo.py'>",
+            "  <Module '__init__.py'>",
-            "*<Function 'test_foo'>",
+            "    <Function 'test_init'>",
+            "  <Module 'test_foo.py'>",
+            "    <Function 'test_foo'>",
         ]
     )
     result = testdir.runpytest("./tests", "--collect-only")
     result.stdout.fnmatch_lines(
         [
-            "*<Module '__init__.py'>",
+            "collected 2 items",
-            "*<Function 'test_init'>",
+            "<Package *",
-            "*<Module 'test_foo.py'>",
+            "  <Module '__init__.py'>",
-            "*<Function 'test_foo'>",
+            "    <Function 'test_init'>",
+            "  <Module 'test_foo.py'>",
+            "    <Function 'test_foo'>",
+        ]
+    )
+    # Ignores duplicates with "." and pkginit (#4310).
+    result = testdir.runpytest("./tests", ".", "--collect-only")
+    result.stdout.fnmatch_lines(
+        [
+            "collected 2 items",
+            "<Package */tests'>",
+            "  <Module '__init__.py'>",
+            "    <Function 'test_init'>",
+            "  <Module 'test_foo.py'>",
+            "    <Function 'test_foo'>",
+        ]
+    )
+    # Same as before, but different order.
+    result = testdir.runpytest(".", "tests", "--collect-only")
+    result.stdout.fnmatch_lines(
+        [
+            "collected 2 items",
+            "<Package */tests'>",
+            "  <Module '__init__.py'>",
+            "    <Function 'test_init'>",
+            "  <Module 'test_foo.py'>",
+            "    <Function 'test_foo'>",
         ]
     )
     result = testdir.runpytest("./tests/test_foo.py", "--collect-only")
-    result.stdout.fnmatch_lines(["*<Module 'test_foo.py'>", "*<Function 'test_foo'>"])
+    result.stdout.fnmatch_lines(
+        ["<Package */tests'>", "  <Module 'test_foo.py'>", "    <Function 'test_foo'>"]
+    )
     assert "test_init" not in result.stdout.str()
     result = testdir.runpytest("./tests/__init__.py", "--collect-only")
-    result.stdout.fnmatch_lines(["*<Module '__init__.py'>", "*<Function 'test_init'>"])
+    result.stdout.fnmatch_lines(
+        ["<Package */tests'>", "  <Module '__init__.py'>", "    <Function 'test_init'>"]
+    )
     assert "test_foo" not in result.stdout.str()