fix: don't measure third-party scripts

This finishes the last bit of #905

Also includes tighter logging of the reason for not tracing modules.

Author: nedbat

coverage/inorout.py

@@ -177,7 +177,7 @@ def add_third_party_paths(paths):
         better_scheme = "pypy_posix" if scheme == "pypy" else scheme
         if os.name in better_scheme.split("_"):
             config_paths = sysconfig.get_paths(scheme)
-            for path_name in ["platlib", "purelib"]:
+            for path_name in ["platlib", "purelib", "scripts"]:
                 paths.add(config_paths[path_name])
 
 
@@ -265,9 +265,6 @@ def debug(msg):
                 against.append("modules {!r}".format(self.source_pkgs_match))
             debug("Source matching against " + " and ".join(against))
         else:
-            if self.cover_paths:
-                self.cover_match = TreeMatcher(self.cover_paths, "coverage")
-                debug("Coverage code matching: {!r}".format(self.cover_match))
             if self.pylib_paths:
                 self.pylib_match = TreeMatcher(self.pylib_paths, "pylib")
                 debug("Python stdlib matching: {!r}".format(self.pylib_match))
@@ -277,9 +274,12 @@ def debug(msg):
         if self.omit:
             self.omit_match = FnmatchMatcher(self.omit, "omit")
             debug("Omit matching: {!r}".format(self.omit_match))
-        if self.third_paths:
-            self.third_match = TreeMatcher(self.third_paths, "third")
-            debug("Third-party lib matching: {!r}".format(self.third_match))
+
+        self.cover_match = TreeMatcher(self.cover_paths, "coverage")
+        debug("Coverage code matching: {!r}".format(self.cover_match))
+
+        self.third_match = TreeMatcher(self.third_paths, "third")
+        debug("Third-party lib matching: {!r}".format(self.third_match))
 
         # Check if the source we want to measure has been installed as a
         # third-party package.
@@ -429,27 +429,29 @@ def check_include_omit_etc(self, filename, frame):
                     ok = True
             if not ok:
                 return extra + "falls outside the --source spec"
+            if self.cover_match.match(filename):
+                return "inside --source, but is part of coverage.py"
             if not self.source_in_third:
                 if self.third_match.match(filename):
-                    return "inside --source, but in third-party"
+                    return "inside --source, but is third-party"
         elif self.include_match:
             if not self.include_match.match(filename):
                 return "falls outside the --include trees"
         else:
+            # We exclude the coverage.py code itself, since a little of it
+            # will be measured otherwise.
+            if self.cover_match.match(filename):
+                return "is part of coverage.py"
+
             # If we aren't supposed to trace installed code, then check if this
             # is near the Python standard library and skip it if so.
             if self.pylib_match and self.pylib_match.match(filename):
                 return "is in the stdlib"
 
             # Exclude anything in the third-party installation areas.
-            if self.third_match and self.third_match.match(filename):
+            if self.third_match.match(filename):
                 return "is a third-party module"
 
-            # We exclude the coverage.py code itself, since a little of it
-            # will be measured otherwise.
-            if self.cover_match and self.cover_match.match(filename):
-                return "is part of coverage.py"
-
         # Check the file against the omit pattern.
         if self.omit_match and self.omit_match.match(filename):
             return "is inside an --omit pattern"
@@ -485,6 +487,12 @@ def warn_already_imported_files(self):
                     msg = "Already imported a file that will be measured: {}".format(filename)
                     self.warn(msg, slug="already-imported")
                     warned.add(filename)
+                elif self.debug and self.debug.should('trace'):
+                    self.debug.write(
+                        "Didn't trace already imported file {!r}: {}".format(
+                            disp.original_filename, disp.reason
+                        )
+                    )
 
     def warn_unimported_source(self):
         """Warn about source packages that were of interest, but never traced."""

tests/helpers.py

@@ -240,7 +240,7 @@ def change_dir(new_dir):
 
     """
     old_dir = os.getcwd()
-    os.chdir(new_dir)
+    os.chdir(str(new_dir))
     try:
         yield
     finally:

tests/test_process.py

@@ -1643,30 +1643,33 @@ def test_script_pkg_sub(self):
         self.assert_pth_and_source_work_together('', 'pkg', 'sub')
 
 
-def run_in_venv(args):
-    """Run python with `args` in the "venv" virtualenv.
+def run_in_venv(cmd):
+    r"""Run `cmd` in the virtualenv at `venv`.
+
+    The first word of the command will be adjusted to run it from the
+    venv/bin or venv\Scripts directory.
 
     Returns the text output of the command.
     """
+    words = cmd.split()
     if env.WINDOWS:
-        cmd = r".\venv\Scripts\python.exe "
+        words[0] = r"{}\Scripts\{}.exe".format("venv", words[0])
     else:
-        cmd = "./venv/bin/python "
-    cmd += args
-    status, output = run_command(cmd)
-    print(output)
+        words[0] = "{}/bin/{}".format("venv", words[0])
+    status, output = run_command(" ".join(words))
     assert status == 0
     return output
 
 
-@pytest.fixture(scope="session", name="venv_factory")
-def venv_factory_fixture(tmp_path_factory):
-    """Produce a function which can copy a venv template to a new directory.
+@pytest.fixture(scope="session", name="venv_world")
+def venv_world_fixture(tmp_path_factory):
+    """Create a virtualenv with a few test packages for VirtualenvTest to use.
 
-    The function accepts one argument, the directory to use for the venv.
+    Returns the directory containing the "venv" virtualenv.
     """
-    tmpdir = tmp_path_factory.mktemp("venv_template")
-    with change_dir(str(tmpdir)):
+
+    venv_world = tmp_path_factory.mktemp("venv_world")
+    with change_dir(venv_world):
         # Create a virtualenv.
         run_command("python -m virtualenv venv")
 
@@ -1686,55 +1689,119 @@ def fourth(x):
             """)
 
         # Install the third-party packages.
-        run_in_venv("-m pip install --no-index ./third_pkg")
+        run_in_venv("python -m pip install --no-index ./third_pkg")
+        shutil.rmtree("third_pkg")
 
         # Install coverage.
         coverage_src = nice_file(TESTS_DIR, "..")
-        run_in_venv("-m pip install --no-index {}".format(coverage_src))
+        run_in_venv("python -m pip install --no-index {}".format(coverage_src))
 
-    def factory(dst):
-        """The venv factory function.
+    return venv_world
 
-        Copies the venv template to `dst`.
-        """
-        shutil.copytree(str(tmpdir / "venv"), dst, symlinks=(not env.WINDOWS))
 
-    return factory
+@pytest.fixture(params=[
+    "coverage",
+    "python -m coverage",
+], name="coverage_command")
+def coverage_command_fixture(request):
+    """Parametrized fixture to use multiple forms of "coverage" command."""
+    return request.param
 
 
 class VirtualenvTest(CoverageTest):
     """Tests of virtualenv considerations."""
 
-    def setup_test(self):
-        self.make_file("myproduct.py", """\
-            import third
-            print(third.third(11))
-            """)
-        self.del_environ("COVERAGE_TESTING")    # To avoid needing contracts installed.
-        super(VirtualenvTest, self).setup_test()
-
-    def test_third_party_venv_isnt_measured(self, venv_factory):
-        venv_factory("venv")
-        out = run_in_venv("-m coverage run --source=. myproduct.py")
+    @pytest.fixture(autouse=True)
+    def in_venv_world_fixture(self, venv_world):
+        """For running tests inside venv_world, and cleaning up made files."""
+        with change_dir(venv_world):
+            self.make_file("myproduct.py", """\
+                import colorsys
+                import third
+                print(third.third(11))
+                print(sum(colorsys.rgb_to_hls(1, 0, 0)))
+                """)
+            self.expected_stdout = "33\n1.5\n"      # pylint: disable=attribute-defined-outside-init
+
+            self.del_environ("COVERAGE_TESTING")    # To avoid needing contracts installed.
+            self.set_environ("COVERAGE_DEBUG_FILE", "debug_out.txt")
+            self.set_environ("COVERAGE_DEBUG", "trace")
+
+            yield
+
+            for fname in os.listdir("."):
+                if fname != "venv":
+                    os.remove(fname)
+
+    def get_trace_output(self):
+        """Get the debug output of coverage.py"""
+        with open("debug_out.txt") as f:
+            return f.read()
+
+    def test_third_party_venv_isnt_measured(self, coverage_command):
+        out = run_in_venv(coverage_command + " run --source=. myproduct.py")
         # In particular, this warning doesn't appear:
         # Already imported a file that will be measured: .../coverage/__main__.py
-        assert out == "33\n"
-        out = run_in_venv("-m coverage report")
+        assert out == self.expected_stdout
+
+        # Check that our tracing was accurate. Files are mentioned because
+        # --source refers to a file.
+        debug_out = self.get_trace_output()
+        assert re_lines(
+            debug_out,
+            r"^Not tracing .*\bexecfile.py': inside --source, but is part of coverage.py"
+            )
+        assert re_lines(debug_out, r"^Tracing .*\bmyproduct.py")
+        assert re_lines(
+            debug_out,
+            r"^Not tracing .*\bcolorsys.py': falls outside the --source spec"
+            )
+
+        out = run_in_venv("python -m coverage report")
         assert "myproduct.py" in out
         assert "third" not in out
+        assert "coverage" not in out
+        assert "colorsys" not in out
+
+    def test_us_in_venv_isnt_measured(self, coverage_command):
+        out = run_in_venv(coverage_command + " run --source=third myproduct.py")
+        assert out == self.expected_stdout
+
+        # Check that our tracing was accurate. Modules are mentioned because
+        # --source refers to a module.
+        debug_out = self.get_trace_output()
+        assert re_lines(
+            debug_out,
+            r"^Not tracing .*\bexecfile.py': " +
+            "module 'coverage.execfile' falls outside the --source spec"
+            )
+        print(re_lines(debug_out, "myproduct"))
+        assert re_lines(
+            debug_out,
+            r"^Not tracing .*\bmyproduct.py': module u?'myproduct' falls outside the --source spec"
+            )
+        assert re_lines(
+            debug_out,
+            r"^Not tracing .*\bcolorsys.py': module u?'colorsys' falls outside the --source spec"
+            )
 
-    def test_us_in_venv_is_measured(self, venv_factory):
-        venv_factory("venv")
-        out = run_in_venv("-m coverage run --source=third myproduct.py")
-        assert out == "33\n"
-        out = run_in_venv("-m coverage report")
+        out = run_in_venv("python -m coverage report")
         assert "myproduct.py" not in out
         assert "third" in out
+        assert "coverage" not in out
+        assert "colorsys" not in out
+
+    def test_venv_isnt_measured(self, coverage_command):
+        out = run_in_venv(coverage_command + " run myproduct.py")
+        assert out == self.expected_stdout
+
+        debug_out = self.get_trace_output()
+        assert re_lines(debug_out, r"^Not tracing .*\bexecfile.py': is part of coverage.py")
+        assert re_lines(debug_out, r"^Tracing .*\bmyproduct.py")
+        assert re_lines(debug_out, r"^Not tracing .*\bcolorsys.py': is in the stdlib")
 
-    def test_venv_isnt_measured(self, venv_factory):
-        venv_factory("venv")
-        out = run_in_venv("-m coverage run myproduct.py")
-        assert out == "33\n"
-        out = run_in_venv("-m coverage report")
+        out = run_in_venv("python -m coverage report")
         assert "myproduct.py" in out
         assert "third" not in out
+        assert "coverage" not in out
+        assert "colorsys" not in out