Skip to content

Restyle WIP: Fix repro #3646

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 7 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 24 additions & 16 deletions dvc/repo/reproduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,16 +95,21 @@ def reproduce(
)

ret = []
checked_stages = set()
for target in targets:
stages = _reproduce_stages(active_graph, target, **kwargs)
stages, these_checked_stages = _reproduce_stages(
active_graph, target, checked_stages, **kwargs
)
ret.extend(stages)
checked_stages.update(these_checked_stages)

return ret


def _reproduce_stages(
G,
stage,
checked_stages,
downstream=False,
ignore_build_cache=False,
single_item=False,
Expand Down Expand Up @@ -161,19 +166,22 @@ def _reproduce_stages(
pipeline = nx.dfs_postorder_nodes(G, stage)

result = []
these_checked_stages = []
for st in pipeline:
try:
ret = _reproduce_stage(st, **kwargs)

if len(ret) != 0 and ignore_build_cache:
# NOTE: we are walking our pipeline from the top to the
# bottom. If one stage is changed, it will be reproduced,
# which tells us that we should force reproducing all of
# the other stages down below, even if their direct
# dependencies didn't change.
kwargs["force"] = True

result.extend(ret)
except Exception as exc:
raise ReproductionError(st.relpath) from exc
return result
if st not in checked_stages:
try:
ret = _reproduce_stage(st, **kwargs)
these_checked_stages.append(st)

if len(ret) != 0 and ignore_build_cache:
# NOTE: we are walking our pipeline from the top to the
# bottom. If one stage is changed, it will be reproduced,
# which tells us that we should force reproducing all of
# the other stages down below, even if their direct
# dependencies didn't change.
kwargs["force"] = True

result.extend(ret)
except Exception as exc:
raise ReproductionError(st.relpath) from exc
return result, these_checked_stages
18 changes: 18 additions & 0 deletions tests/unit/repo/test_reproduce.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import mock

from dvc.repo.reproduce import _get_active_graph


Expand All @@ -23,3 +25,19 @@ def test_get_active_graph(tmp_dir, dvc):
active_graph = _get_active_graph(graph)
assert set(active_graph.nodes) == {bar_stage, baz_stage}
assert not active_graph.edges


@mock.patch("dvc.repo.reproduce._reproduce_stage", returns=[])
def test_number_reproduces(reproduce_stage_mock, tmp_dir, dvc):
tmp_dir.dvc_gen({"pre-foo": "pre-foo"})

dvc.run(deps=["pre-foo"], outs=["foo"], cmd="echo foo > foo")
dvc.run(deps=["foo"], outs=["bar"], cmd="echo bar > bar")
dvc.run(deps=["foo"], outs=["baz"], cmd="echo baz > baz")
dvc.run(deps=["bar"], outs=["boop"], cmd="echo boop > boop")

reproduce_stage_mock.reset_mock()

dvc.reproduce(all_pipelines=True)

assert reproduce_stage_mock.call_count == 5