diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 86ad68133ac3..bdb0169dc398 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -213,6 +213,8 @@ pub fn main() !void { builder.verbose_link = true; } else if (mem.eql(u8, arg, "--verbose-air")) { builder.verbose_air = true; + } else if (mem.eql(u8, arg, "--cache-dependencies-locally")) { + builder.cache_dependencies_locally = true; } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { builder.verbose_llvm_ir = "-"; } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) { @@ -1184,6 +1186,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void { \\ --build-file [file] Override path to build.zig \\ --cache-dir [path] Override path to local Zig cache directory \\ --global-cache-dir [path] Override path to global Zig cache directory + \\ --cache-dependencies-locally Cache dependencies locally in zig-deps \\ --zig-lib-dir [arg] Override path to Zig lib directory \\ --build-runner [file] Override path to build runner \\ --seed [integer] For shuffling dependency traversal order (default: random) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 4443fa404c74..17d9978b69f1 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -33,6 +33,7 @@ verbose: bool, verbose_link: bool, verbose_cc: bool, verbose_air: bool, +cache_dependencies_locally: bool, verbose_llvm_ir: ?[]const u8, verbose_llvm_bc: ?[]const u8, verbose_cimport: bool, @@ -254,6 +255,7 @@ pub fn create( .graph = graph, .build_root = build_root, .cache_root = cache_root, + .cache_dependencies_locally = false, .verbose = false, .verbose_link = false, .verbose_cc = false, @@ -380,6 +382,7 @@ fn createChildOnly( .installed_files = ArrayList(InstalledFile).init(allocator), .build_root = build_root, .cache_root = parent.cache_root, + .cache_dependencies_locally = parent.cache_dependencies_locally, .zig_lib_dir = parent.zig_lib_dir, .debug_log_scopes = parent.debug_log_scopes, .debug_compile_errors = parent.debug_compile_errors, diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 36856daf5fb2..87dcc29dd77a 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -1380,7 +1380,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { if (b.debug_compile_errors) { try zig_args.append("--debug-compile-errors"); } - + if (b.cache_dependencies_locally) try zig_args.append("--cache-dependencies-locally"); if (b.verbose_cimport) try zig_args.append("--verbose-cimport"); if (b.verbose_air) try zig_args.append("--verbose-air"); if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path})); diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index f45a5cc93e12..a13d4816b9c9 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -1,9 +1,18 @@ //! Represents one independent job whose responsibility is to: //! -//! 1. Check the global zig package cache to see if the hash already exists. -//! If so, load, parse, and validate the build.zig.zon file therein, and -//! goto step 8. Likewise if the location is a relative path, treat this -//! the same as a cache hit. Otherwise, proceed. +//! 1. if local zig packages is enabled: +//! a. Check the local zig package directory to see if the hash already exists. +//! If so, load, parse, and validate the build.zig.zon file therein, and +//! goto step 8. Likewise if the location is a relative path, treat this +//! the same as a cache hit. Otherwise, proceed. +//! b. Check the global zig package cache to see if the hash already exists. +//! If so, load, parse, and validate the build.zig.zon file therein, copy +//! it to the local zig package directory and goto step 8. Otherwise, proceed. +//! if local zig packages is disabled: +//! Check the global zig package cache to see if the hash already exists. +//! If so, load, parse, and validate the build.zig.zon file therein, and +//! goto step 8. Likewise if the location is a relative path, treat this +//! the same as a cache hit. Otherwise, proceed. //! 2. Fetch and unpack a URL into a temporary directory. //! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed //! for the file to be missing, in which case this fetched package is considered @@ -18,7 +27,8 @@ //! directory. If the hash already exists, delete the temporary directory and //! leave the zig package cache directory untouched as it may be in use by the //! system. This is done even if the hash is invalid, in case the package with -//! the different hash is used in the future. +//! the different hash is used in the future. If local packages is enabled, copy +//! the package locally. //! 7. Validate the computed hash against the expected hash. If invalid, //! this job is done. //! 8. Spawn a new fetch job for each dependency in the manifest file. Use @@ -97,6 +107,7 @@ pub const JobQueue = struct { thread_pool: *ThreadPool, wait_group: WaitGroup = .{}, global_cache: Cache.Directory, + local_package_cache_root: ?Cache.Directory, /// If true then, no fetching occurs, and: /// * The `global_cache` directory is assumed to be the direct parent /// directory of on-disk packages rather than having the "p/" directory @@ -299,7 +310,7 @@ pub fn run(f: *Fetch) RunError!void { const arena = f.arena.allocator(); const gpa = f.arena.child_allocator; const cache_root = f.job_queue.global_cache; - + const local_package_cache = f.job_queue.local_package_cache_root; try eb.init(gpa); // Check the global zig package cache to see if the hash already exists. If @@ -372,41 +383,102 @@ pub fn run(f: *Fetch) RunError!void { const prefixed_pkg_sub_path = "p" ++ s ++ expected_hash; const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0; const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..]; - if (cache_root.handle.access(pkg_sub_path, .{})) |_| { - assert(f.lazy_status != .unavailable); - f.package_root = .{ - .root_dir = cache_root, - .sub_path = try arena.dupe(u8, pkg_sub_path), - }; - try loadManifest(f, f.package_root); - try checkBuildFileExistence(f); - if (!f.job_queue.recursive) return; - return queueJobsForDeps(f); - } else |err| switch (err) { - error.FileNotFound => { - switch (f.lazy_status) { - .eager => {}, - .available => if (!f.job_queue.unlazy_set.contains(expected_hash)) { - f.lazy_status = .unavailable; - return; - }, - .unavailable => unreachable, + var check_cache_root = local_package_cache == null; + if (local_package_cache) |lpc| { + if (lpc.handle.access(&expected_hash, .{})) { + f.package_root = .{ + .root_dir = lpc, + .sub_path = try arena.dupe(u8, &expected_hash), + }; + try loadManifest(f, f.package_root); + try checkBuildFileExistence(f); + if (!f.job_queue.recursive) return; + return queueJobsForDeps(f); + } else |err| switch (err) { + error.FileNotFound => { + check_cache_root = true; + }, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open local package cache directory '{}{s}': {s}", .{ + lpc, &expected_hash, @errorName(e), + }), + }); + return error.FetchFailed; + }, + } + } + if (check_cache_root) { + if (cache_root.handle.access(pkg_sub_path, .{})) |_| { + assert(f.lazy_status != .unavailable); + if (local_package_cache) |lpc| { + var dst_handle = lpc.handle.makeOpenPath(&expected_hash, .{}) catch |err| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open local package cache directory '{}{s}': {s}", .{ + lpc, &expected_hash, @errorName(err), + }), + }); + return error.FetchFailed; + }; + defer dst_handle.close(); + var src_handle = cache_root.handle.openDir(pkg_sub_path, .{}) catch |err| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{ + cache_root, pkg_sub_path, @errorName(err), + }), + }); + return error.FetchFailed; + }; + defer src_handle.close(); + f.recursiveDirectoryCopy(src_handle, dst_handle) catch |err| { + const src = try cache_root.join(arena, &.{pkg_sub_path}); + const dest = try lpc.join(arena, &.{&expected_hash}); + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "sunable to copy into local package directory '{s}' from package cache directory '{s}': {s}", + .{ dest, src, @errorName(err) }, + ) }); + return error.FetchFailed; + }; + f.package_root = .{ + .root_dir = lpc, + .sub_path = try arena.dupe(u8, &expected_hash), + }; + } else { + f.package_root = .{ + .root_dir = cache_root, + .sub_path = try arena.dupe(u8, pkg_sub_path), + }; } - if (f.job_queue.read_only) return f.fail( - f.name_tok, - try eb.printString("package not found at '{}{s}'", .{ - cache_root, pkg_sub_path, - }), - ); - }, - else => |e| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{ - cache_root, pkg_sub_path, @errorName(e), - }), - }); - return error.FetchFailed; - }, + try loadManifest(f, f.package_root); + try checkBuildFileExistence(f); + if (!f.job_queue.recursive) return; + return queueJobsForDeps(f); + } else |err| switch (err) { + error.FileNotFound => { + switch (f.lazy_status) { + .eager => {}, + .available => if (!f.job_queue.unlazy_set.contains(expected_hash)) { + f.lazy_status = .unavailable; + return; + }, + .unavailable => unreachable, + } + if (f.job_queue.read_only) return f.fail( + f.name_tok, + try eb.printString("package not found at '{}{s}'", .{ + cache_root, pkg_sub_path, + }), + ); + }, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{ + cache_root, pkg_sub_path, @errorName(e), + }), + }); + return error.FetchFailed; + }, + } } } else if (f.job_queue.read_only) { try eb.addRootErrorMessage(.{ @@ -444,6 +516,7 @@ fn runResource( const eb = &f.error_bundle; const s = fs.path.sep_str; const cache_root = f.job_queue.global_cache; + const local_package_cache = f.job_queue.local_package_cache_root; const rand_int = std.crypto.random.int(u64); const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); @@ -526,6 +599,36 @@ fn runResource( ) }); return error.FetchFailed; }; + if (local_package_cache) |lpc| { + var dst_handle = lpc.handle.makeOpenPath(&Manifest.hexDigest(f.actual_hash), .{}) catch |err| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open local package cache directory '{}{s}': {s}", .{ + lpc, &Manifest.hexDigest(f.actual_hash), @errorName(err), + }), + }); + return error.FetchFailed; + }; + defer dst_handle.close(); + var src_handle = cache_root.handle.openDir(f.package_root.sub_path, .{}) catch |err| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{ + cache_root, f.package_root.sub_path, @errorName(err), + }), + }); + return error.FetchFailed; + }; + defer src_handle.close(); + f.recursiveDirectoryCopy(src_handle, dst_handle) catch |err| { + const src = try cache_root.join(arena, &.{f.package_root.sub_path}); + const dest = try lpc.join(arena, &.{&Manifest.hexDigest(f.actual_hash)}); + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to copy into local package directory '{s}' from package cache directory '{s}': {s}", + .{ dest, src, @errorName(err) }, + ) }); + return error.FetchFailed; + }; + } + // Remove temporary directory root if not already renamed to global cache. if (!std.mem.eql(u8, package_sub_path, tmp_dir_sub_path)) { cache_root.handle.deleteDir(tmp_dir_sub_path) catch {}; diff --git a/src/main.zig b/src/main.zig index 2eb79877f3aa..b0822be9ee40 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4632,6 +4632,7 @@ const usage_build = \\ --build-file [file] Override path to build.zig \\ --cache-dir [path] Override path to local Zig cache directory \\ --global-cache-dir [path] Override path to global Zig cache directory + \\ --cache-dependencies-locally Cache dependencies locally in zig-deps \\ --zig-lib-dir [arg] Override path to Zig lib directory \\ --build-runner [file] Override path to build runner \\ --prominent-compile-errors Buffer compile errors and display at end @@ -4647,6 +4648,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { var override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena); var override_local_cache_dir: ?[]const u8 = try EnvVar.ZIG_LOCAL_CACHE_DIR.get(arena); var override_build_runner: ?[]const u8 = try EnvVar.ZIG_BUILD_RUNNER.get(arena); + var cache_dependencies_locally = false; var child_argv = std.ArrayList([]const u8).init(arena); var reference_trace: ?u32 = null; var debug_compile_errors = false; @@ -4735,6 +4737,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { i += 1; override_global_cache_dir = args[i]; continue; + } else if (mem.eql(u8, arg, "--cache-dependencies-locally")) { + cache_dependencies_locally = true; + continue; } else if (mem.eql(u8, arg, "-freference-trace")) { reference_trace = 256; } else if (mem.eql(u8, arg, "--fetch")) { @@ -4860,6 +4865,15 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { }; defer global_cache_directory.handle.close(); + var local_package_cache_root: ?Compilation.Directory = null; + if (cache_dependencies_locally) { + local_package_cache_root = .{ + .handle = try fs.cwd().makeOpenPath("zig-deps", .{}), + .path = "zig-deps", + }; + } + defer if (local_package_cache_root) |*lpc| lpc.handle.close(); + child_argv.items[argv_index_global_cache_dir] = global_cache_directory.path orelse cwd_path; var local_cache_directory: Compilation.Directory = l: { @@ -4976,6 +4990,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { .http_client = &http_client, .thread_pool = &thread_pool, .global_cache = global_cache_directory, + .local_package_cache_root = local_package_cache_root, .read_only = false, .recursive = true, .debug_hash = false, @@ -6800,7 +6815,7 @@ const usage_fetch = \\ --save=[name] Add the fetched package to build.zig.zon as name \\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim \\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim - \\ + \\ --cache-dependencies-locally Cache dependencies locally in zig-deps ; fn cmdFetch( @@ -6813,6 +6828,7 @@ fn cmdFetch( EnvVar.ZIG_BTRFS_WORKAROUND.isSet(); var opt_path_or_url: ?[]const u8 = null; var override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena); + var cache_dependencies_locally = false; var debug_hash: bool = false; var save: union(enum) { no, @@ -6837,6 +6853,8 @@ fn cmdFetch( debug_hash = true; } else if (mem.eql(u8, arg, "--save")) { save = .{ .yes = null }; + } else if (mem.eql(u8, arg, "--cache-dependencies-locally")) { + cache_dependencies_locally = true; } else if (mem.startsWith(u8, arg, "--save=")) { save = .{ .yes = arg["--save=".len..] }; } else if (mem.eql(u8, arg, "--save-exact")) { @@ -6879,10 +6897,20 @@ fn cmdFetch( }; defer global_cache_directory.handle.close(); + var local_package_cache_root: ?Compilation.Directory = null; + if (cache_dependencies_locally) { + local_package_cache_root = .{ + .handle = try fs.cwd().makeOpenPath("zig-deps", .{}), + .path = "zig-deps", + }; + } + defer if (local_package_cache_root) |*lpc| lpc.handle.close(); + var job_queue: Package.Fetch.JobQueue = .{ .http_client = &http_client, .thread_pool = &thread_pool, .global_cache = global_cache_directory, + .local_package_cache_root = local_package_cache_root, .recursive = false, .read_only = false, .debug_hash = debug_hash,