dvcignore: optimize matching x10

On a repo with a dvcignore with 1 pattern and a directory with 400K
files, `dvc status` now takes ~8 sec instead of ~30 sec.

To achieve that, we make some assumptions about the paths formats that
we are dealing with, so we could use simpler logic instead of using
very slow `relpath`, `abspath` etc on every entry in a directory.

It is also clear that CleanTree behavior is inconsistent (even tests
expect very different outputs from it), so we will need to look into
this later.

Author: efiop

dvc/ignore.py

@@ -6,7 +6,6 @@
 from pathspec.patterns import GitWildMatchPattern
 
 from dvc.scm.tree import BaseTree
-from dvc.utils import relpath
 
 logger = logging.getLogger(__name__)
 
@@ -35,12 +34,19 @@ def __call__(self, root, dirs, files):
         return dirs, files
 
     def matches(self, dirname, basename):
-        abs_path = os.path.join(dirname, basename)
-        rel_path = relpath(abs_path, self.dirname)
-
-        if os.pardir + os.sep in rel_path:
+        # NOTE: `relpath` is too slow, so we have to assume that both
+        # `dirname` and `self.dirname` are relative or absolute together.
+        prefix = self.dirname + os.sep
+        if dirname == self.dirname:
+            path = basename
+        elif dirname.startswith(prefix):
+            rel = dirname[len(prefix) :]
+            # NOTE: `os.path.join` is ~x5.5 slower
+            path = f"{rel}{os.sep}{basename}"
+        else:
             return False
-        return self.ignore_spec.match_file(rel_path)
+
+        return self.ignore_spec.match_file(path)
 
     def __hash__(self):
         return hash(self.ignore_file_path)
@@ -135,7 +141,9 @@ def isexec(self, path):
 
     def walk(self, top, topdown=True):
         for root, dirs, files in self.tree.walk(top, topdown):
-            dirs[:], files[:] = self.dvcignore(root, dirs, files)
+            dirs[:], files[:] = self.dvcignore(
+                os.path.abspath(root), dirs, files
+            )
 
             yield root, dirs, files
 

tests/unit/test_ignore.py

@@ -69,7 +69,6 @@ def test_ignore_from_file_should_filter_dirs_and_files():
         ),
         ("dont_ignore.txt", ["dont_ignore"], False),
         ("dont_ignore.txt", ["dont*", "!dont_ignore.txt"], False),
-        ("../../../something.txt", ["**/something.txt"], False),
     ],
 )
 def test_match_ignore_from_file(