From 93a5ff0885aadd84e560c621d7d159012ddb0bc4 Mon Sep 17 00:00:00 2001 From: Eric Redmond Date: Thu, 9 Jan 2014 13:25:07 -0800 Subject: [PATCH 1/4] Add a user local cache to store dep versions ala maven --- src/rebar_deps.erl | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index 5e697094..a79a0607 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -478,13 +478,45 @@ use_source(Config, Dep, Count) -> "with reason:~n~p.\n", [Dep#dep.dir, Reason]) end; false -> - ?CONSOLE("Pulling ~p from ~p\n", [Dep#dep.app, Dep#dep.source]), - require_source_engine(Dep#dep.source), + CacheTargetDir = ensure_local_cache(Dep), {true, TargetDir} = get_deps_dir(Config, Dep#dep.app), - download_source(TargetDir, Dep#dep.source), + rebar_file_utils:cp_r([CacheTargetDir], TargetDir), use_source(Config, Dep#dep { dir = TargetDir }, Count-1) end. +local_cache_base_dir(Dep) -> + {ok, [[Home]]} = init:get_argument(home), + CacheBaseDir = filename:join([Home, ".rebar", "cache", Dep#dep.app]), + filelib:ensure_dir(CacheBaseDir++"/"), + CacheBaseDir. + +local_cache_target_dir(Dep) -> + CacheBaseDir = local_cache_base_dir(Dep), + VersionDirs = case Dep#dep.source of + {_,_,{B,Version}} when is_atom(B) -> [atom_to_list(B), Version]; + {_,_,{B,Version}} -> [B, Version]; + {_,_,Version} -> [Version] + end, + filename:join([CacheBaseDir]++VersionDirs). + +ensure_local_cache(Dep) -> + ?CONSOLE("Checking local cache for ~p from ~p\n", [Dep#dep.app, Dep#dep.source]), + CacheTargetDir = local_cache_target_dir(Dep), + case filelib:is_dir(CacheTargetDir) of + true -> + %% Attempting to update always, and catch timeouts. After N + %% timeouts, set source as 'unreachable' and just carry on + %% In the future, version branches by most recent commit, + %% and compare against remote HEAD + require_source_engine(Dep#dep.source), + update_source(CacheTargetDir, Dep, CacheTargetDir); + false -> + ?CONSOLE("Pulling ~p from ~p to local cache\n", [Dep#dep.app, Dep#dep.source]), + require_source_engine(Dep#dep.source), + download_source(CacheTargetDir, Dep#dep.source) + end, + CacheTargetDir. + download_source(AppDir, {hg, Url, Rev}) -> ok = filelib:ensure_dir(AppDir), rebar_utils:sh(?FMT("hg clone -U ~s ~s", [Url, filename:basename(AppDir)]), @@ -536,11 +568,14 @@ download_source(AppDir, {fossil, Url, Version}) -> []). update_source(Config, Dep) -> + {true, AppDir} = get_deps_dir(Config, Dep#dep.app), + update_source(Config, Dep, AppDir). + +update_source(_Config, Dep, AppDir) -> %% It's possible when updating a source, that a given dep does not have a %% VCS directory, such as when a source archive is built of a project, with %% all deps already downloaded/included. So, verify that the necessary VCS %% directory exists before attempting to do the update. - {true, AppDir} = get_deps_dir(Config, Dep#dep.app), case has_vcs_dir(element(1, Dep#dep.source), AppDir) of true -> ?CONSOLE("Updating ~p from ~p\n", [Dep#dep.app, Dep#dep.source]), From ab7df2b05a284d8f4e209c69d2d8369b4d26a829 Mon Sep 17 00:00:00 2001 From: Eric Redmond Date: Thu, 9 Jan 2014 20:09:51 -0800 Subject: [PATCH 2/4] Missing restest is in dizzys account --- rebar.config.script | 5 ++--- src/rebar_deps.erl | 12 +++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/rebar.config.script b/rebar.config.script index 07feb95d..fb7cfb00 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -2,9 +2,8 @@ %% ex: ts=4 sw=4 ft=erlang et %% TODO: Change temporary retest fork back to dizzyd/retest after merge -%% ExtraDeps = [{retest, ".*", {git, "git://github.com/dizzyd/retest.git"}}], -ExtraDeps = [{retest, ".*", - {git, "git://github.com/tuncer/retest.git", "next"}}], +%% ExtraDeps = [{retest, ".*", {git, "git://github.com/tuncer/retest.git", "next"}}], +ExtraDeps = [{retest, ".*", {git, "git://github.com/dizzyd/retest.git"}}], case os:getenv("REBAR_EXTRA_DEPS") of false -> diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index a79a0607..fef45250 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -478,7 +478,7 @@ use_source(Config, Dep, Count) -> "with reason:~n~p.\n", [Dep#dep.dir, Reason]) end; false -> - CacheTargetDir = ensure_local_cache(Dep), + CacheTargetDir = ensure_local_cache(Config, Dep), {true, TargetDir} = get_deps_dir(Config, Dep#dep.app), rebar_file_utils:cp_r([CacheTargetDir], TargetDir), use_source(Config, Dep#dep { dir = TargetDir }, Count-1) @@ -495,11 +495,12 @@ local_cache_target_dir(Dep) -> VersionDirs = case Dep#dep.source of {_,_,{B,Version}} when is_atom(B) -> [atom_to_list(B), Version]; {_,_,{B,Version}} -> [B, Version]; - {_,_,Version} -> [Version] + {_,_,Version} -> [Version]; + {_,_} -> ["master"] end, filename:join([CacheBaseDir]++VersionDirs). -ensure_local_cache(Dep) -> +ensure_local_cache(Config, Dep) -> ?CONSOLE("Checking local cache for ~p from ~p\n", [Dep#dep.app, Dep#dep.source]), CacheTargetDir = local_cache_target_dir(Dep), case filelib:is_dir(CacheTargetDir) of @@ -509,10 +510,11 @@ ensure_local_cache(Dep) -> %% In the future, version branches by most recent commit, %% and compare against remote HEAD require_source_engine(Dep#dep.source), - update_source(CacheTargetDir, Dep, CacheTargetDir); + update_source(Config, Dep, CacheTargetDir); false -> - ?CONSOLE("Pulling ~p from ~p to local cache\n", [Dep#dep.app, Dep#dep.source]), + ?CONSOLE("Pulling ~p from ~p into local cache\n", [Dep#dep.app, Dep#dep.source]), require_source_engine(Dep#dep.source), + ?CONSOLE("CacheTargetDir: ~p\n", [CacheTargetDir]), download_source(CacheTargetDir, Dep#dep.source) end, CacheTargetDir. From 92ec8769e78c9ad1f96181ebcf9131a88fec5a57 Mon Sep 17 00:00:00 2001 From: Eric Redmond Date: Mon, 13 Jan 2014 17:22:41 -0800 Subject: [PATCH 3/4] Dont cache local repo clones --- src/rebar_deps.erl | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index fef45250..dbab2e6f 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -478,10 +478,21 @@ use_source(Config, Dep, Count) -> "with reason:~n~p.\n", [Dep#dep.dir, Reason]) end; false -> - CacheTargetDir = ensure_local_cache(Config, Dep), - {true, TargetDir} = get_deps_dir(Config, Dep#dep.app), - rebar_file_utils:cp_r([CacheTargetDir], TargetDir), - use_source(Config, Dep#dep { dir = TargetDir }, Count-1) + DepDir = case is_local_source(Dep) of + true -> + %% Repo source is a local directory, skip local cache + ?CONSOLE("Pulling ~p from ~p\n", [Dep#dep.app, Dep#dep.source]), + require_source_engine(Dep#dep.source), + {true, TargetDir} = get_deps_dir(Config, Dep#dep.app), + download_source(TargetDir, Dep#dep.source), + TargetDir; + false -> + CacheTargetDir = ensure_local_cache(Config, Dep), + {true, TargetDir} = get_deps_dir(Config, Dep#dep.app), + rebar_file_utils:cp_r([CacheTargetDir], TargetDir), + TargetDir + end, + use_source(Config, Dep#dep { dir = DepDir }, Count-1) end. local_cache_base_dir(Dep) -> @@ -500,6 +511,21 @@ local_cache_target_dir(Dep) -> end, filename:join([CacheBaseDir]++VersionDirs). +%% the hallmarks of a local file +maybe_local_dir(".") -> true; +maybe_local_dir("/") -> true; +maybe_local_dir("\\") -> true; +maybe_local_dir(_) -> false. + +is_local_source(Dep) -> + case Dep#dep.source of + {_,Local,_} when is_list(Local) -> + maybe_local_dir(string:substr(Local, 1, 1)); + {_,Local} when is_list(Local) -> + maybe_local_dir(string:substr(Local, 1, 1)); + _ -> false + end. + ensure_local_cache(Config, Dep) -> ?CONSOLE("Checking local cache for ~p from ~p\n", [Dep#dep.app, Dep#dep.source]), CacheTargetDir = local_cache_target_dir(Dep), From ebcdc4d14400dc724dcf0b2f74b1bad0f63ee99c Mon Sep 17 00:00:00 2001 From: Eric Redmond Date: Mon, 13 Jan 2014 18:57:29 -0800 Subject: [PATCH 4/4] base directory must exist before cpr --- src/rebar_deps.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index dbab2e6f..5208578c 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -489,6 +489,7 @@ use_source(Config, Dep, Count) -> false -> CacheTargetDir = ensure_local_cache(Config, Dep), {true, TargetDir} = get_deps_dir(Config, Dep#dep.app), + ok = filelib:ensure_dir(TargetDir), rebar_file_utils:cp_r([CacheTargetDir], TargetDir), TargetDir end,