Skip to content

Commit 07da9f1

Browse files
efiopefiop
authored
tests: remotes: use TmpDir-like fixtures (#4140)
1 parent 5d2ac9f commit 07da9f1

24 files changed

+792
-694
lines changed

‎setup.cfg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ count=true
1717
[isort]
1818
include_trailing_comma=true
1919
known_first_party=dvc,tests
20-
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,google,grandalf,mock,moto,nanotime,networkx,packaging,paramiko,pathspec,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
20+
known_third_party=PyInstaller,RangeHTTPServer,boto3,colorama,configobj,distro,dpath,flaky,flufl,funcy,git,grandalf,mock,moto,nanotime,networkx,packaging,pathspec,pylint,pytest,requests,ruamel,setuptools,shortuuid,shtab,tqdm,voluptuous,yaml,zc
2121
line_length=79
2222
force_grid_wrap=0
2323
use_parentheses=True

‎tests/func/test_add.py

Lines changed: 88 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -185,27 +185,97 @@ def test_add_file_in_dir(tmp_dir, dvc):
185185
assert stage.outs[0].def_path == "subdata"
186186

187187

188-
class TestAddExternalLocalFile(TestDvc):
189-
def test(self):
190-
from dvc.stage.exceptions import StageExternalOutputsError
188+
@pytest.mark.parametrize(
189+
"workspace, hash_name, hash_value",
190+
[
191+
(
192+
pytest.lazy_fixture("local_cloud"),
193+
"md5",
194+
"8c7dd922ad47494fc02c388e12c00eac",
195+
),
196+
pytest.param(
197+
pytest.lazy_fixture("ssh"),
198+
"md5",
199+
"8c7dd922ad47494fc02c388e12c00eac",
200+
marks=pytest.mark.skipif(
201+
os.name == "nt", reason="disabled on windows"
202+
),
203+
),
204+
(
205+
pytest.lazy_fixture("s3"),
206+
"etag",
207+
"8c7dd922ad47494fc02c388e12c00eac",
208+
),
209+
(pytest.lazy_fixture("gs"), "md5", "8c7dd922ad47494fc02c388e12c00eac"),
210+
(
211+
pytest.lazy_fixture("hdfs"),
212+
"checksum",
213+
"000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e",
214+
),
215+
],
216+
indirect=["workspace"],
217+
)
218+
def test_add_external_file(tmp_dir, dvc, workspace, hash_name, hash_value):
219+
from dvc.stage.exceptions import StageExternalOutputsError
191220

192-
dname = TestDvc.mkdtemp()
193-
fname = os.path.join(dname, "foo")
194-
shutil.copyfile(self.FOO, fname)
221+
workspace.gen("file", "file")
195222

196-
with self.assertRaises(StageExternalOutputsError):
197-
self.dvc.add(fname)
223+
with pytest.raises(StageExternalOutputsError):
224+
dvc.add(workspace.url)
198225

199-
stages = self.dvc.add(fname, external=True)
200-
self.assertEqual(len(stages), 1)
201-
stage = stages[0]
202-
self.assertNotEqual(stage, None)
203-
self.assertEqual(len(stage.deps), 0)
204-
self.assertEqual(len(stage.outs), 1)
205-
self.assertEqual(stage.relpath, "foo.dvc")
206-
self.assertEqual(len(os.listdir(dname)), 1)
207-
self.assertTrue(os.path.isfile(fname))
208-
self.assertTrue(filecmp.cmp(fname, "foo", shallow=False))
226+
dvc.add("remote://workspace/file")
227+
assert (tmp_dir / "file.dvc").read_text() == (
228+
"outs:\n"
229+
f"- {hash_name}: {hash_value}\n"
230+
" path: remote://workspace/file\n"
231+
)
232+
assert (workspace / "file").read_text() == "file"
233+
assert (
234+
workspace / "cache" / hash_value[:2] / hash_value[2:]
235+
).read_text() == "file"
236+
237+
assert dvc.status() == {}
238+
239+
240+
@pytest.mark.parametrize(
241+
"workspace, hash_name, hash_value",
242+
[
243+
(
244+
pytest.lazy_fixture("local_cloud"),
245+
"md5",
246+
"b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
247+
),
248+
pytest.param(
249+
pytest.lazy_fixture("ssh"),
250+
"md5",
251+
"b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
252+
marks=pytest.mark.skipif(
253+
os.name == "nt", reason="disabled on windows"
254+
),
255+
),
256+
(
257+
pytest.lazy_fixture("s3"),
258+
"etag",
259+
"ec602a6ba97b2dd07bd6d2cd89674a60.dir",
260+
),
261+
(
262+
pytest.lazy_fixture("gs"),
263+
"md5",
264+
"b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
265+
),
266+
],
267+
indirect=["workspace"],
268+
)
269+
def test_add_external_dir(tmp_dir, dvc, workspace, hash_name, hash_value):
270+
workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}})
271+
272+
dvc.add("remote://workspace/dir")
273+
assert (tmp_dir / "dir.dvc").read_text() == (
274+
"outs:\n"
275+
f"- {hash_name}: {hash_value}\n"
276+
" path: remote://workspace/dir\n"
277+
)
278+
assert (workspace / "cache" / hash_value[:2] / hash_value[2:]).is_file()
209279

210280

211281
class TestAddLocalRemoteFile(TestDvc):

‎tests/func/test_api.py

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,22 @@ def test_open(tmp_dir, dvc, remote):
6868
assert fd.read() == "foo-text"
6969

7070

71-
@pytest.mark.parametrize("cloud", clouds)
71+
@pytest.mark.parametrize(
72+
"cloud",
73+
[
74+
pytest.lazy_fixture(cloud)
75+
for cloud in [
76+
"real_s3", # NOTE: moto's s3 fails in some tests
77+
"gs",
78+
"azure",
79+
"gdrive",
80+
"oss",
81+
"ssh",
82+
"hdfs",
83+
"http",
84+
]
85+
],
86+
)
7287
def test_open_external(erepo_dir, cloud):
7388
erepo_dir.add_remote(config=cloud.config)
7489

@@ -104,7 +119,22 @@ def test_open_granular(tmp_dir, dvc, remote):
104119
assert fd.read() == "foo-text"
105120

106121

107-
@pytest.mark.parametrize("remote", all_remotes)
122+
@pytest.mark.parametrize(
123+
"remote",
124+
[
125+
pytest.lazy_fixture(f"{cloud}_remote")
126+
for cloud in [
127+
"real_s3", # NOTE: moto's s3 fails in some tests
128+
"gs",
129+
"azure",
130+
"gdrive",
131+
"oss",
132+
"ssh",
133+
"hdfs",
134+
"http",
135+
]
136+
],
137+
)
108138
def test_missing(tmp_dir, dvc, remote):
109139
tmp_dir.dvc_gen("foo", "foo")
110140

‎tests/func/test_gc.py

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,3 +348,45 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
348348
Dvcfile(dvc, PIPELINE_FILE).remove(force=True)
349349
dvc.gc(workspace=True, force=True)
350350
assert _count_files(dvc.cache.local.cache_dir) == 0
351+
352+
353+
@pytest.mark.parametrize(
354+
"workspace",
355+
[
356+
pytest.lazy_fixture("local_cloud"),
357+
pytest.lazy_fixture("s3"),
358+
pytest.lazy_fixture("gs"),
359+
pytest.lazy_fixture("hdfs"),
360+
pytest.param(
361+
pytest.lazy_fixture("ssh"),
362+
marks=pytest.mark.skipif(
363+
os.name == "nt", reason="disabled on windows"
364+
),
365+
),
366+
],
367+
indirect=True,
368+
)
369+
def test_gc_external_output(tmp_dir, dvc, workspace):
370+
workspace.gen({"foo": "foo", "bar": "bar"})
371+
372+
(foo_stage,) = dvc.add("remote://workspace/foo")
373+
(bar_stage,) = dvc.add("remote://workspace/bar")
374+
375+
foo_hash = foo_stage.outs[0].checksum
376+
bar_hash = bar_stage.outs[0].checksum
377+
378+
assert (
379+
workspace / "cache" / foo_hash[:2] / foo_hash[2:]
380+
).read_text() == "foo"
381+
assert (
382+
workspace / "cache" / bar_hash[:2] / bar_hash[2:]
383+
).read_text() == "bar"
384+
385+
(tmp_dir / "foo.dvc").unlink()
386+
387+
dvc.gc(workspace=True)
388+
389+
assert not (workspace / "cache" / foo_hash[:2] / foo_hash[2:]).exists()
390+
assert (
391+
workspace / "cache" / bar_hash[:2] / bar_hash[2:]
392+
).read_text() == "bar"

‎tests/func/test_import_url.py

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,3 +112,56 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
112112
dvc.imp_url(src, ".", no_exec=True)
113113
dst = tmp_dir / "file"
114114
assert not dst.exists()
115+
116+
117+
@pytest.mark.parametrize(
118+
"workspace",
119+
[
120+
pytest.lazy_fixture("local_cloud"),
121+
pytest.lazy_fixture("s3"),
122+
pytest.lazy_fixture("gs"),
123+
pytest.lazy_fixture("hdfs"),
124+
pytest.param(
125+
pytest.lazy_fixture("ssh"),
126+
marks=pytest.mark.skipif(
127+
os.name == "nt", reason="disabled on windows"
128+
),
129+
),
130+
pytest.lazy_fixture("http"),
131+
],
132+
indirect=True,
133+
)
134+
def test_import_url(tmp_dir, dvc, workspace):
135+
workspace.gen("file", "file")
136+
assert not (tmp_dir / "file").exists() # sanity check
137+
dvc.imp_url("remote://workspace/file")
138+
assert (tmp_dir / "file").read_text() == "file"
139+
140+
assert dvc.status() == {}
141+
142+
143+
@pytest.mark.parametrize(
144+
"workspace",
145+
[
146+
pytest.lazy_fixture("local_cloud"),
147+
pytest.lazy_fixture("s3"),
148+
pytest.lazy_fixture("gs"),
149+
pytest.param(
150+
pytest.lazy_fixture("ssh"),
151+
marks=pytest.mark.skipif(
152+
os.name == "nt", reason="disabled on windows"
153+
),
154+
),
155+
],
156+
indirect=True,
157+
)
158+
def test_import_url_dir(tmp_dir, dvc, workspace):
159+
workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}})
160+
assert not (tmp_dir / "dir").exists() # sanity check
161+
dvc.imp_url("remote://workspace/dir")
162+
assert set(os.listdir(tmp_dir / "dir")) == {"file", "subdir"}
163+
assert (tmp_dir / "dir" / "file").read_text() == "file"
164+
assert list(os.listdir(tmp_dir / "dir" / "subdir")) == ["subfile"]
165+
assert (tmp_dir / "dir" / "subdir" / "subfile").read_text() == "subfile"
166+
167+
assert dvc.status() == {}

0 commit comments

Comments
 (0)