Incremental fixes, refactor Zcu.File

build.zig

@@ -8,12 +8,24 @@ const io = std.io;
 const fs = std.fs;
 const InstallDirectoryOptions = std.Build.InstallDirectoryOptions;
 const assert = std.debug.assert;
-const DevEnv = @import("src/dev.zig").Env;
 const ValueInterpretMode = enum { direct, by_name };
 
 const zig_version: std.SemanticVersion = .{ .major = 0, .minor = 15, .patch = 0 };
 const stack_size = 46 * 1024 * 1024;
 
+/// Keep in sync with `Env` in `src/dev.zig`.
+const DevEnv = enum {
+    bootstrap,
+    core,
+    full,
+    c_source,
+    ast_gen,
+    sema,
+    @"x86_64-linux",
+    @"riscv64-linux",
+    wasm,
+};
+
 pub fn build(b: *std.Build) !void {
     const only_c = b.option(bool, "only-c", "Translate the Zig compiler to C code, with only the C backend enabled") orelse false;
     const target = b.standardTargetOptions(.{
@@ -508,18 +520,19 @@ pub fn build(b: *std.Build) !void {
     test_step.dependOn(unit_tests_step);
 
     const unit_tests = b.addTest(.{
-        .root_module = b.createModule(.{
-            .root_source_file = b.path("src/main.zig"),
+        .root_module = addCompilerMod(b, .{
             .optimize = optimize,
             .target = target,
-            .link_libc = link_libc,
             .single_threaded = single_threaded,
         }),
         .filters = test_filters,
         .use_llvm = use_llvm,
         .use_lld = use_llvm,
         .zig_lib_dir = b.path("lib"),
     });
+    if (link_libc) {
+        unit_tests.root_module.link_libc = true;
+    }
     unit_tests.root_module.addOptions("build_options", exe_options);
     unit_tests_step.dependOn(&b.addRunArtifact(unit_tests).step);
 
@@ -645,7 +658,7 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void {
     update_zig1_step.dependOn(&copy_zig_h.step);
 }
 
-const AddCompilerStepOptions = struct {
+const AddCompilerModOptions = struct {
     optimize: std.builtin.OptimizeMode,
     target: std.Build.ResolvedTarget,
     strip: ?bool = null,
@@ -654,7 +667,7 @@ const AddCompilerStepOptions = struct {
     single_threaded: ?bool = null,
 };
 
-fn addCompilerStep(b: *std.Build, options: AddCompilerStepOptions) *std.Build.Step.Compile {
+fn addCompilerMod(b: *std.Build, options: AddCompilerModOptions) *std.Build.Module {
     const compiler_mod = b.createModule(.{
         .root_source_file = b.path("src/main.zig"),
         .target = options.target,
@@ -677,10 +690,14 @@ fn addCompilerStep(b: *std.Build, options: AddCompilerStepOptions) *std.Build.St
     compiler_mod.addImport("aro", aro_mod);
     compiler_mod.addImport("aro_translate_c", aro_translate_c_mod);
 
+    return compiler_mod;
+}
+
+fn addCompilerStep(b: *std.Build, options: AddCompilerModOptions) *std.Build.Step.Compile {
     const exe = b.addExecutable(.{
         .name = "zig",
         .max_rss = 7_800_000_000,
-        .root_module = compiler_mod,
+        .root_module = addCompilerMod(b, options),
     });
     exe.stack_size = stack_size;
 

lib/compiler/aro/aro/Driver/GCCDetector.zig

@@ -1,7 +1,7 @@
 const std = @import("std");
 const Toolchain = @import("../Toolchain.zig");
 const target_util = @import("../target.zig");
-const system_defaults = @import("system_defaults");
+const system_defaults = undefined; // upstream, this imports a module; we don't provide that module
 const GCCVersion = @import("GCCVersion.zig");
 const Multilib = @import("Multilib.zig");
 

lib/compiler/aro/aro/Toolchain.zig

@@ -2,7 +2,7 @@ const std = @import("std");
 const Driver = @import("Driver.zig");
 const Compilation = @import("Compilation.zig");
 const mem = std.mem;
-const system_defaults = @import("system_defaults");
+const system_defaults = undefined; // upstream, this imports a module; we don't provide that module
 const target_util = @import("target.zig");
 const Linux = @import("toolchains/Linux.zig");
 const Multilib = @import("Driver/Multilib.zig");

lib/compiler/aro/aro/toolchains/Linux.zig

@@ -6,7 +6,7 @@ const Toolchain = @import("../Toolchain.zig");
 const Driver = @import("../Driver.zig");
 const Distro = @import("../Driver/Distro.zig");
 const target_util = @import("../target.zig");
-const system_defaults = @import("system_defaults");
+const system_defaults = undefined; // upstream, this imports a module; we don't provide that module
 
 const Linux = @This();
 

lib/compiler/aro/backend/Ir/x86/Renderer.zig

@@ -4,7 +4,7 @@ const assert = std.debug.assert;
 const Interner = @import("../../Interner.zig");
 const Ir = @import("../../Ir.zig");
 const BaseRenderer = Ir.Renderer;
-const zig = @import("zig");
+const zig = undefined; // upstream, this imports a module; we don't provide that module
 const abi = zig.arch.x86_64.abi;
 const bits = zig.arch.x86_64.bits;
 

lib/std/Build/Cache.zig

@@ -276,11 +276,13 @@ pub const HashHelper = struct {
     }
 
     pub fn oneShot(bytes: []const u8) [hex_digest_len]u8 {
+        return binToHex(oneShotBin(bytes));
+    }
+
+    pub fn oneShotBin(bytes: []const u8) BinDigest {
         var hasher: Hasher = hasher_init;
         hasher.update(bytes);
-        var bin_digest: BinDigest = undefined;
-        hasher.final(&bin_digest);
-        return binToHex(bin_digest);
+        return hasher.finalResult();
     }
 };
 

src/Builtin.zig

@@ -19,6 +19,26 @@ code_model: std.builtin.CodeModel,
 omit_frame_pointer: bool,
 wasi_exec_model: std.builtin.WasiExecModel,
 
+/// Compute an abstract hash representing this `Builtin`. This is *not* a hash
+/// of the resulting file contents.
+pub fn hash(opts: @This()) [std.Build.Cache.bin_digest_len]u8 {
+    var h: Cache.Hasher = Cache.hasher_init;
+    inline for (@typeInfo(@This()).@"struct".fields) |f| {
+        if (comptime std.mem.eql(u8, f.name, "target")) {
+            // This needs special handling.
+            std.hash.autoHash(&h, opts.target.cpu);
+            std.hash.autoHash(&h, opts.target.os.tag);
+            std.hash.autoHash(&h, opts.target.os.versionRange());
+            std.hash.autoHash(&h, opts.target.abi);
+            std.hash.autoHash(&h, opts.target.ofmt);
+            std.hash.autoHash(&h, opts.target.dynamic_linker);
+        } else {
+            std.hash.autoHash(&h, @field(opts, f.name));
+        }
+    }
+    return h.finalResult();
+}
+
 pub fn generate(opts: @This(), allocator: Allocator) Allocator.Error![:0]u8 {
     var buffer = std.ArrayList(u8).init(allocator);
     try append(opts, &buffer);
@@ -263,22 +283,49 @@ pub fn append(opts: @This(), buffer: *std.ArrayList(u8)) Allocator.Error!void {
     }
 }
 
-pub fn populateFile(comp: *Compilation, mod: *Module, file: *File) !void {
-    if (mod.root.statFile(mod.root_src_path)) |stat| {
+/// This essentially takes the place of `Zcu.PerThread.updateFile`, but for 'builtin' modules.
+/// Instead of reading the file from disk, its contents are generated in-memory.
+pub fn populateFile(opts: @This(), gpa: Allocator, file: *File) Allocator.Error!void {
+    assert(file.is_builtin);
+    assert(file.status == .never_loaded);
+    assert(file.source == null);
+    assert(file.tree == null);
+    assert(file.zir == null);
+
+    file.source = try opts.generate(gpa);
+
+    log.debug("parsing and generating 'builtin.zig'", .{});
+
+    file.tree = try std.zig.Ast.parse(gpa, file.source.?, .zig);
+    assert(file.tree.?.errors.len == 0); // builtin.zig must parse
+
+    file.zir = try AstGen.generate(gpa, file.tree.?);
+    assert(!file.zir.?.hasCompileErrors()); // builtin.zig must not have astgen errors
+    file.status = .success;
+}
+
+/// After `populateFile` succeeds, call this function to write the generated file out to disk
+/// if necessary. This is useful for external tooling such as debuggers.
+/// Assumes that `file.mod` is correctly set to the builtin module.
+pub fn updateFileOnDisk(file: *File) !void {
+    assert(file.is_builtin);
+    assert(file.status == .success);
+    assert(file.source != null);
+
+    if (file.mod.?.root.statFile("builtin.zig")) |stat| {
         if (stat.size != file.source.?.len) {
             std.log.warn(
-                "the cached file '{}{s}' had the wrong size. Expected {d}, found {d}. " ++
+                "the cached file '{}{c}builtin.zig' had the wrong size. Expected {d}, found {d}. " ++
                     "Overwriting with correct file contents now",
-                .{ mod.root, mod.root_src_path, file.source.?.len, stat.size },
+                .{ file.mod.?.root, std.fs.path.sep, file.source.?.len, stat.size },
             );
-
-            try writeFile(file, mod);
         } else {
             file.stat = .{
                 .size = stat.size,
                 .inode = stat.inode,
                 .mtime = stat.mtime,
             };
+            return;
         }
     } else |err| switch (err) {
         error.BadPathName => unreachable, // it's always "builtin.zig"
@@ -287,26 +334,13 @@ pub fn populateFile(comp: *Compilation, mod: *Module, file: *File) !void {
         error.NoDevice => unreachable, // it's not a pipe
         error.WouldBlock => unreachable, // not asking for non-blocking I/O
 
-        error.FileNotFound => try writeFile(file, mod),
+        error.FileNotFound => {},
 
         else => |e| return e,
     }
 
-    log.debug("parsing and generating '{s}'", .{mod.root_src_path});
-
-    file.tree = try std.zig.Ast.parse(comp.gpa, file.source.?, .zig);
-    assert(file.tree.?.errors.len == 0); // builtin.zig must parse
-
-    file.zir = try AstGen.generate(comp.gpa, file.tree.?);
-    assert(!file.zir.?.hasCompileErrors()); // builtin.zig must not have astgen errors
-    file.status = .success;
-    // Note that whilst we set `zir` here, we populated `path_digest`
-    // all the way back in `Package.Module.create`.
-}
-
-fn writeFile(file: *File, mod: *Module) !void {
     var buf: [std.fs.max_path_bytes]u8 = undefined;
-    var af = try mod.root.atomicFile(mod.root_src_path, .{ .make_path = true }, &buf);
+    var af = try file.mod.?.root.atomicFile("builtin.zig", .{ .make_path = true }, &buf);
     defer af.deinit();
     try af.file.writeAll(file.source.?);
     af.finish() catch |err| switch (err) {
@@ -331,6 +365,7 @@ fn writeFile(file: *File, mod: *Module) !void {
 const builtin = @import("builtin");
 const std = @import("std");
 const Allocator = std.mem.Allocator;
+const Cache = std.Build.Cache;
 const build_options = @import("build_options");
 const Module = @import("Package/Module.zig");
 const assert = std.debug.assert;