Merge remote-tracking branch 'origin/master' into fix_3736

Author: karajan1001

dvc/command/experiments.py

@@ -82,29 +82,25 @@ def _show_experiments(all_experiments, console, precision=None):
 
     metric_names, param_names = _collect_names(all_experiments)
 
-    table = Table(row_styles=["white", "bright_white"])
-    table.add_column("Experiment", header_style="black on grey93")
+    table = Table()
+    table.add_column("Experiment")
     for name in metric_names:
-        table.add_column(
-            name, justify="right", header_style="black on cornsilk1"
-        )
+        table.add_column(name, justify="right")
     for name in param_names:
-        table.add_column(
-            name, justify="left", header_style="black on light_cyan1"
-        )
+        table.add_column(name, justify="left")
 
     for base_rev, experiments in all_experiments.items():
         if Git.is_sha(base_rev):
             base_rev = base_rev[:7]
 
-        for row, style, in _collect_rows(
+        for row, _, in _collect_rows(
             base_rev,
             experiments,
             metric_names,
             param_names,
             precision=precision,
         ):
-            table.add_row(*row, style=style)
+            table.add_row(*row)
 
     console.print(table)
 

dvc/config.py

@@ -314,25 +314,23 @@ def load(self, validate=True):
 
     def _load_config(self, level):
         filename = self.files[level]
-        tree = self.tree if level == "repo" else self.wtree
 
-        if tree.exists(filename, use_dvcignore=False):
-            with tree.open(filename) as fobj:
+        if self.tree.exists(filename, use_dvcignore=False):
+            with self.tree.open(filename) as fobj:
                 conf_obj = configobj.ConfigObj(fobj)
         else:
             conf_obj = configobj.ConfigObj()
         return _parse_remotes(_lower_keys(conf_obj.dict()))
 
     def _save_config(self, level, conf_dict):
         filename = self.files[level]
-        tree = self.tree if level == "repo" else self.wtree
 
         logger.debug(f"Writing '{filename}'.")
 
-        tree.makedirs(os.path.dirname(filename))
+        self.tree.makedirs(os.path.dirname(filename))
 
         config = configobj.ConfigObj(_pack_remotes(conf_dict))
-        with tree.open(filename, "wb") as fobj:
+        with self.tree.open(filename, "wb") as fobj:
             config.write(fobj)
         config.filename = filename
 

dvc/daemon.py

@@ -18,13 +18,17 @@
 def _spawn_windows(cmd, env):
     from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW
 
+    prefix = [sys.executable]
+    if not is_binary():
+        prefix += [sys.argv[0]]
+
     creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
 
     startupinfo = STARTUPINFO()
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
     Popen(
-        cmd,
+        prefix + cmd,
         env=env,
         close_fds=True,
         shell=False,
@@ -34,6 +38,8 @@ def _spawn_windows(cmd, env):
 
 
 def _spawn_posix(cmd, env):
+    from dvc.main import main
+
     # NOTE: using os._exit instead of sys.exit, because dvc built
     # with PyInstaller has trouble with SystemExit exception and throws
     # errors such as "[26338] Failed to execute script __main__"
@@ -59,7 +65,8 @@ def _spawn_posix(cmd, env):
     sys.stdout.close()
     sys.stderr.close()
 
-    Popen(cmd, env=env, close_fds=True, shell=False).communicate()
+    os.environ.update(env)
+    main(cmd)
 
     os._exit(0)  # pylint: disable=protected-access
 
@@ -87,10 +94,7 @@ def daemon(args):
         logger.debug("skipping launching a new daemon.")
         return
 
-    cmd = [sys.executable]
-    if not is_binary():
-        cmd += [sys.argv[0]]
-    cmd += ["daemon", "-q"] + args
+    cmd = ["daemon", "-q"] + args
 
     env = fix_env()
     file_path = os.path.abspath(inspect.stack()[0][1])

dvc/dependency/param.py

@@ -2,6 +2,7 @@
 from collections import defaultdict
 
 import dpath.util
+import toml
 import yaml
 from voluptuous import Any
 
@@ -21,6 +22,8 @@ class ParamsDependency(LocalDependency):
     PARAM_PARAMS = "params"
     PARAM_SCHEMA = {PARAM_PARAMS: Any(dict, list, None)}
     DEFAULT_PARAMS_FILE = "params.yaml"
+    PARAMS_FILE_LOADERS = defaultdict(lambda: yaml.safe_load)
+    PARAMS_FILE_LOADERS.update({".toml": toml.load})
 
     def __init__(self, stage, path, params):
         info = {}
@@ -87,8 +90,10 @@ def read_params(self):
 
         with self.repo.tree.open(self.path_info, "r") as fobj:
             try:
-                config = yaml.safe_load(fobj)
-            except yaml.YAMLError as exc:
+                config = self.PARAMS_FILE_LOADERS[
+                    self.path_info.suffix.lower()
+                ](fobj)
+            except (yaml.YAMLError, toml.TomlDecodeError) as exc:
                 raise BadParamFileError(
                     f"Unable to read parameters from '{self}'"
                 ) from exc

dvc/ignore.py

@@ -2,7 +2,7 @@
 import os
 import re
 from collections import namedtuple
-from itertools import groupby
+from itertools import groupby, takewhile
 
 from pathspec.patterns import GitWildMatchPattern
 from pathspec.util import normalize_file
@@ -183,18 +183,19 @@ def __init__(self, tree, root_dir):
         self.ignores_trie_tree[root_dir] = DvcIgnorePatterns(
             default_ignore_patterns, root_dir
         )
-        for root, dirs, _ in self.tree.walk(self.root_dir):
-            self._update(root)
-            self._update_sub_repo(root, dirs)
-            dirs[:], _ = self(root, dirs, [])
+        self._update(self.root_dir)
 
     def _update(self, dirname):
+        old_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
+        matches = old_pattern.matches(dirname, DvcIgnore.DVCIGNORE_FILE, False)
+
         ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
-        if self.tree.exists(ignore_file_path):
+        if not matches and self.tree.exists(
+            ignore_file_path, use_dvcignore=False
+        ):
             new_pattern = DvcIgnorePatterns.from_files(
                 ignore_file_path, self.tree
             )
-            old_pattern = self._get_trie_pattern(dirname)
             if old_pattern:
                 self.ignores_trie_tree[dirname] = DvcIgnorePatterns(
                     *merge_patterns(
@@ -206,12 +207,19 @@ def _update(self, dirname):
                 )
             else:
                 self.ignores_trie_tree[dirname] = new_pattern
+        elif old_pattern:
+            self.ignores_trie_tree[dirname] = old_pattern
+
+        # NOTE: using `walk` + `break` because tree doesn't have `listdir()`
+        for root, dirs, _ in self.tree.walk(dirname, use_dvcignore=False):
+            self._update_sub_repo(root, dirs)
+            break
 
     def _update_sub_repo(self, root, dirs):
         for d in dirs:
             if self._is_dvc_repo(root, d):
-                old_pattern = self._get_trie_pattern(root)
                 new_pattern = DvcIgnorePatterns(["/{}/".format(d)], root)
+                old_pattern = self.ignores_trie_tree.longest_prefix(root).value
                 if old_pattern:
                     self.ignores_trie_tree[root] = DvcIgnorePatterns(
                         *merge_patterns(
@@ -232,8 +240,28 @@ def __call__(self, root, dirs, files):
             return dirs, files
 
     def _get_trie_pattern(self, dirname):
-        ignore_pattern = self.ignores_trie_tree.longest_prefix(dirname).value
-        return ignore_pattern
+        ignore_pattern = self.ignores_trie_tree.get(dirname)
+        if ignore_pattern:
+            return ignore_pattern
+
+        prefix = self.ignores_trie_tree.longest_prefix(dirname).key
+        if not prefix:
+            # outside of the repo
+            return None
+
+        dirs = list(
+            takewhile(
+                lambda path: path != prefix,
+                (parent.fspath for parent in PathInfo(dirname).parents),
+            )
+        )
+        dirs.reverse()
+        dirs.append(dirname)
+
+        for parent in dirs:
+            self._update(parent)
+
+        return self.ignores_trie_tree.get(dirname)
 
     def _is_ignored(self, path, is_dir=False):
         if self._outside_repo(path):

dvc/repo/params/show.py

@@ -1,5 +1,6 @@
 import logging
 
+import toml
 import yaml
 
 from dvc.dependency.param import ParamsDependency
@@ -34,8 +35,10 @@ def _read_params(repo, configs, rev):
 
         with repo.tree.open(config, "r") as fobj:
             try:
-                res[str(config)] = yaml.safe_load(fobj)
-            except yaml.YAMLError:
+                res[str(config)] = ParamsDependency.PARAMS_FILE_LOADERS[
+                    config.suffix.lower()
+                ](fobj)
+            except (yaml.YAMLError, toml.TomlDecodeError):
                 logger.debug(
                     "failed to read '%s' on '%s'", config, rev, exc_info=True
                 )

dvc/tree/git.py

@@ -46,12 +46,8 @@ def dvcignore(self):
         from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
 
         root = self.dvcignore_root or self.tree_root
-        if not self.use_dvcignore:
-            return DvcIgnoreFilterNoop(self, root)
-        self.use_dvcignore = False
-        ret = DvcIgnoreFilter(self, root)
-        self.use_dvcignore = True
-        return ret
+        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
+        return cls(self, root)
 
     def open(
         self, path, mode="r", encoding="utf-8"
@@ -160,7 +156,7 @@ def _walk(self, tree, topdown=True):
         if not topdown:
             yield os.path.normpath(tree.abspath), dirs, nondirs
 
-    def walk(self, top, topdown=True, onerror=None):
+    def walk(self, top, topdown=True, onerror=None, use_dvcignore=True):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
@@ -178,9 +174,10 @@ def walk(self, top, topdown=True, onerror=None):
             return
 
         for root, dirs, files in self._walk(tree, topdown=topdown):
-            dirs[:], files[:] = self.dvcignore(
-                os.path.abspath(root), dirs, files
-            )
+            if use_dvcignore:
+                dirs[:], files[:] = self.dvcignore(
+                    os.path.abspath(root), dirs, files
+                )
             yield root, dirs, files
 
     def isexec(self, path):

dvc/tree/local.py

@@ -58,12 +58,8 @@ def dvcignore(self):
         from dvc.ignore import DvcIgnoreFilter, DvcIgnoreFilterNoop
 
         root = self.dvcignore_root or self.tree_root
-        if not self.use_dvcignore:
-            return DvcIgnoreFilterNoop(self, root)
-        self.use_dvcignore = False
-        ret = DvcIgnoreFilter(self, root)
-        self.use_dvcignore = True
-        return ret
+        cls = DvcIgnoreFilter if self.use_dvcignore else DvcIgnoreFilterNoop
+        return cls(self, root)
 
     @staticmethod
     def open(path_info, mode="r", encoding=None):
@@ -101,7 +97,7 @@ def iscopy(self, path_info):
             System.is_symlink(path_info) or System.is_hardlink(path_info)
         )
 
-    def walk(self, top, topdown=True, onerror=None):
+    def walk(self, top, topdown=True, onerror=None, use_dvcignore=True):
         """Directory tree generator.
 
         See `os.walk` for the docs. Differences:
@@ -110,9 +106,10 @@ def walk(self, top, topdown=True, onerror=None):
         for root, dirs, files in os.walk(
             top, topdown=topdown, onerror=onerror
         ):
-            dirs[:], files[:] = self.dvcignore(
-                os.path.abspath(root), dirs, files
-            )
+            if use_dvcignore:
+                dirs[:], files[:] = self.dvcignore(
+                    os.path.abspath(root), dirs, files
+                )
 
             yield os.path.normpath(root), dirs, files
 

dvc/utils/.gitignore

@@ -0,0 +1 @@
+/build.py

dvc/utils/fs.py

@@ -96,8 +96,7 @@ def move(src, dst, mode=None):
     tmp = f"{dst}.{uuid()}"
 
     if os.path.islink(src):
-        # readlink does not accept path-like obj for Windows in Python <3.8
-        shutil.copy(os.readlink(os.fspath(src)), tmp)
+        shutil.copy(src, tmp)
         os.unlink(src)
     else:
         shutil.move(src, tmp)

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "1.1.11"
+_BASE_VERSION = "1.2.1"
 
 
 def _generate_version(base_version):

scripts/hooks/hook-dvc.py

@@ -0,0 +1,2 @@
+# https://github.com/pypa/setuptools/issues/1963
+hiddenimports = ["pkg_resources.py2_warn"]

scripts/hooks/hook-google.cloud.storage.py renamed to scripts/hooks/hook-dvc.tree.gs.py

@@ -17,7 +17,7 @@ count=true
 [isort]
 include_trailing_comma=true
 known_first_party=dvc,tests
-known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pygtrie,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
+known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pygtrie,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,toml,tqdm,voluptuous,yaml,zc
 line_length=79
 force_grid_wrap=0
 use_parentheses=True

setup.cfg

@@ -62,6 +62,7 @@ def run(self):
     "appdirs>=1.4.3",
     "PyYAML>=5.1.2,<5.4",  # Compatibility with awscli
     "ruamel.yaml>=0.16.1",
+    "toml>=0.10.1",
     "funcy>=1.14",
     "pathspec>=0.6.0",
     "shortuuid>=0.5.0",
@@ -74,7 +75,7 @@ def run(self):
     "networkx>=2.1,<2.5",
     "pydot>=1.2.4",
     "speedcopy>=2.0.1; python_version < '3.8' and sys_platform == 'win32'",
-    "flatten_json>=0.1.6",
+    "flatten_json>=0.1.6,<0.1.8",
     "tabulate>=0.8.7",
     "pygtrie==2.3.2",
     "dpath>=2.0.1,<3",

setup.py

@@ -14,6 +14,16 @@ def test_show(tmp_dir, dvc):
     assert dvc.params.show() == {"": {"params.yaml": {"foo": "bar"}}}
 
 
+def test_show_toml(tmp_dir, dvc):
+    tmp_dir.gen("params.toml", "[foo]\nbar = 42\nbaz = [1, 2]\n")
+    dvc.run(
+        cmd="echo params.toml", params=["params.toml:foo"], single_stage=True
+    )
+    assert dvc.params.show() == {
+        "": {"params.toml": {"foo": {"bar": 42, "baz": [1, 2]}}}
+    }
+
+
 def test_show_multiple(tmp_dir, dvc):
     tmp_dir.gen("params.yaml", "foo: bar\nbaz: qux\n")
     dvc.run(