Merge pull request ziglang#21078 from jacobly0/new-dwarf

Dwarf: rework self-hosted debug info from scratch

Author: jacobly0

build.zig

@@ -549,6 +549,15 @@ pub fn build(b: *std.Build) !void {
     test_step.dependOn(tests.addStackTraceTests(b, test_filters, optimization_modes));
     test_step.dependOn(tests.addCliTests(b));
     test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filters, optimization_modes));
+    if (tests.addDebuggerTests(b, .{
+        .test_filters = test_filters,
+        .gdb = b.option([]const u8, "gdb", "path to gdb binary"),
+        .lldb = b.option([]const u8, "lldb", "path to lldb binary"),
+        .optimize_modes = optimization_modes,
+        .skip_single_threaded = skip_single_threaded,
+        .skip_non_native = skip_non_native,
+        .skip_libc = skip_libc,
+    })) |test_debugger_step| test_step.dependOn(test_debugger_step);
 
     try addWasiUpdateStep(b, version);
 

ci/x86_64-linux-debug.sh

@@ -64,6 +64,7 @@ stage3-debug/bin/zig build \
 
 stage3-debug/bin/zig build test docs \
   --maxrss 21000000000 \
+  -Dlldb=$HOME/deps/lldb-zig/Debug/bin/lldb \
   -fqemu \
   -fwasmtime \
   -Dstatic-llvm \

ci/x86_64-linux-release.sh

@@ -64,6 +64,7 @@ stage3-release/bin/zig build \
 
 stage3-release/bin/zig build test docs \
   --maxrss 21000000000 \
+  -Dlldb=$HOME/deps/lldb-zig/Release/bin/lldb \
   -fqemu \
   -fwasmtime \
   -Dstatic-llvm \

lib/std/array_list.zig

@@ -359,6 +359,24 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
             return m.len;
         }
 
+        pub const FixedWriter = std.io.Writer(*Self, Allocator.Error, appendWriteFixed);
+
+        /// Initializes a Writer which will append to the list but will return
+        /// `error.OutOfMemory` rather than increasing capacity.
+        pub fn fixedWriter(self: *Self) FixedWriter {
+            return .{ .context = self };
+        }
+
+        /// The purpose of this function existing is to match `std.io.Writer` API.
+        fn appendWriteFixed(self: *Self, m: []const u8) error{OutOfMemory}!usize {
+            const available_capacity = self.capacity - self.items.len;
+            if (m.len > available_capacity)
+                return error.OutOfMemory;
+
+            self.appendSliceAssumeCapacity(m);
+            return m.len;
+        }
+
         /// Append a value to the list `n` times.
         /// Allocates more memory as necessary.
         /// Invalidates element pointers if additional memory is needed.

lib/std/debug.zig

@@ -1360,6 +1360,9 @@ test "manage resources correctly" {
         return error.SkipZigTest;
     }
 
+    // self-hosted debug info is still too buggy
+    if (builtin.zig_backend != .stage2_llvm) return error.SkipZigTest;
+
     const writer = std.io.null_writer;
     var di = try SelfInfo.open(testing.allocator);
     defer di.deinit();

lib/std/dwarf.zig

@@ -95,6 +95,9 @@ pub const LNE = struct {
     pub const set_discriminator = 0x04;
     pub const lo_user = 0x80;
     pub const hi_user = 0xff;
+
+    // Zig extensions
+    pub const ZIG_set_decl = 0xec;
 };
 
 pub const UT = struct {
@@ -118,6 +121,8 @@ pub const LNCT = struct {
 
     pub const lo_user = 0x2000;
     pub const hi_user = 0x3fff;
+
+    pub const LLVM_source = 0x2001;
 };
 
 pub const RLE = struct {
@@ -142,6 +147,37 @@ pub const CC = enum(u8) {
     GNU_renesas_sh = 0x40,
     GNU_borland_fastcall_i386 = 0x41,
 
+    BORLAND_safecall = 0xb0,
+    BORLAND_stdcall = 0xb1,
+    BORLAND_pascal = 0xb2,
+    BORLAND_msfastcall = 0xb3,
+    BORLAND_msreturn = 0xb4,
+    BORLAND_thiscall = 0xb5,
+    BORLAND_fastcall = 0xb6,
+
+    LLVM_vectorcall = 0xc0,
+    LLVM_Win64 = 0xc1,
+    LLVM_X86_64SysV = 0xc2,
+    LLVM_AAPCS = 0xc3,
+    LLVM_AAPCS_VFP = 0xc4,
+    LLVM_IntelOclBicc = 0xc5,
+    LLVM_SpirFunction = 0xc6,
+    LLVM_OpenCLKernel = 0xc7,
+    LLVM_Swift = 0xc8,
+    LLVM_PreserveMost = 0xc9,
+    LLVM_PreserveAll = 0xca,
+    LLVM_X86RegCall = 0xcb,
+    LLVM_M68kRTD = 0xcc,
+    LLVM_PreserveNone = 0xcd,
+    LLVM_RISCVVectorCall = 0xce,
+    LLVM_SwiftTail = 0xcf,
+
     pub const lo_user = 0x40;
     pub const hi_user = 0xff;
 };
+
+pub const ACCESS = struct {
+    pub const public = 0x01;
+    pub const protected = 0x02;
+    pub const private = 0x03;
+};

lib/std/dwarf/AT.zig

@@ -218,6 +218,15 @@ pub const VMS_rtnbeg_pd_address = 0x2201;
 // See http://gcc.gnu.org/wiki/DW_AT_GNAT_descriptive_type .
 pub const use_GNAT_descriptive_type = 0x2301;
 pub const GNAT_descriptive_type = 0x2302;
+
+// Zig extensions.
+pub const ZIG_parent = 0x2ccd;
+pub const ZIG_padding = 0x2cce;
+pub const ZIG_relative_decl = 0x2cd0;
+pub const ZIG_decl_line_relative = 0x2cd1;
+pub const ZIG_is_allowzero = 0x2ce1;
+pub const ZIG_sentinel = 0x2ce2;
+
 // UPC extension.
 pub const upc_threads_scaled = 0x3210;
 // PGI (STMicroelectronics) extensions.

lib/std/dwarf/LANG.zig

@@ -35,6 +35,30 @@ pub const Fortran03 = 0x0022;
 pub const Fortran08 = 0x0023;
 pub const RenderScript = 0x0024;
 pub const BLISS = 0x0025;
+pub const Kotlin = 0x0026;
+pub const Zig = 0x0027;
+pub const Crystal = 0x0028;
+pub const C_plus_plus_17 = 0x002a;
+pub const C_plus_plus_20 = 0x002b;
+pub const C17 = 0x002c;
+pub const Fortran18 = 0x002d;
+pub const Ada2005 = 0x002e;
+pub const Ada2012 = 0x002f;
+pub const HIP = 0x0030;
+pub const Assembly = 0x0031;
+pub const C_sharp = 0x0032;
+pub const Mojo = 0x0033;
+pub const GLSL = 0x0034;
+pub const GLSL_ES = 0x0035;
+pub const HLSL = 0x0036;
+pub const OpenCL_CPP = 0x0037;
+pub const CPP_for_OpenCL = 0x0038;
+pub const SYCL = 0x0039;
+pub const C_plus_plus_23 = 0x003a;
+pub const Odin = 0x003b;
+pub const Ruby = 0x0040;
+pub const Move = 0x0041;
+pub const Hylo = 0x0042;
 
 pub const lo_user = 0x8000;
 pub const hi_user = 0xffff;

lib/std/io.zig

@@ -419,7 +419,7 @@ pub const tty = @import("io/tty.zig");
 /// A Writer that doesn't write to anything.
 pub const null_writer: NullWriter = .{ .context = {} };
 
-const NullWriter = Writer(void, error{}, dummyWrite);
+pub const NullWriter = Writer(void, error{}, dummyWrite);
 fn dummyWrite(context: void, data: []const u8) error{}!usize {
     _ = context;
     return data.len;

lib/std/leb128.zig

@@ -36,10 +36,14 @@ pub fn readUleb128(comptime T: type, reader: anytype) !T {
 pub const readULEB128 = readUleb128;
 
 /// Write a single unsigned integer as unsigned LEB128 to the given writer.
-pub fn writeUleb128(writer: anytype, uint_value: anytype) !void {
-    const T = @TypeOf(uint_value);
-    const U = if (@typeInfo(T).Int.bits < 8) u8 else T;
-    var value: U = @intCast(uint_value);
+pub fn writeUleb128(writer: anytype, arg: anytype) !void {
+    const Arg = @TypeOf(arg);
+    const Int = switch (Arg) {
+        comptime_int => std.math.IntFittingRange(arg, arg),
+        else => Arg,
+    };
+    const Value = if (@typeInfo(Int).Int.bits < 8) u8 else Int;
+    var value: Value = arg;
 
     while (true) {
         const byte: u8 = @truncate(value & 0x7f);
@@ -118,16 +122,19 @@ pub fn readIleb128(comptime T: type, reader: anytype) !T {
 pub const readILEB128 = readIleb128;
 
 /// Write a single signed integer as signed LEB128 to the given writer.
-pub fn writeIleb128(writer: anytype, int_value: anytype) !void {
-    const T = @TypeOf(int_value);
-    const S = if (@typeInfo(T).Int.bits < 8) i8 else T;
-    const U = std.meta.Int(.unsigned, @typeInfo(S).Int.bits);
-
-    var value: S = @intCast(int_value);
+pub fn writeIleb128(writer: anytype, arg: anytype) !void {
+    const Arg = @TypeOf(arg);
+    const Int = switch (Arg) {
+        comptime_int => std.math.IntFittingRange(-arg - 1, arg),
+        else => Arg,
+    };
+    const Signed = if (@typeInfo(Int).Int.bits < 8) i8 else Int;
+    const Unsigned = std.meta.Int(.unsigned, @typeInfo(Signed).Int.bits);
+    var value: Signed = arg;
 
     while (true) {
-        const uvalue: U = @bitCast(value);
-        const byte: u8 = @truncate(uvalue);
+        const unsigned: Unsigned = @bitCast(value);
+        const byte: u8 = @truncate(unsigned);
         value >>= 6;
         if (value == -1 or value == 0) {
             try writer.writeByte(byte & 0x7F);
@@ -147,17 +154,25 @@ pub fn writeIleb128(writer: anytype, int_value: anytype) !void {
 /// "relocatable", meaning that it becomes possible to later go back and patch the number to be a
 /// different value without shifting all the following code.
 pub fn writeUnsignedFixed(comptime l: usize, ptr: *[l]u8, int: std.meta.Int(.unsigned, l * 7)) void {
-    const T = @TypeOf(int);
-    const U = if (@typeInfo(T).Int.bits < 8) u8 else T;
-    var value: U = @intCast(int);
+    writeUnsignedExtended(ptr, int);
+}
 
-    comptime var i = 0;
-    inline while (i < (l - 1)) : (i += 1) {
-        const byte = @as(u8, @truncate(value)) | 0b1000_0000;
+/// Same as `writeUnsignedFixed` but with a runtime-known length.
+/// Asserts `slice.len > 0`.
+pub fn writeUnsignedExtended(slice: []u8, arg: anytype) void {
+    const Arg = @TypeOf(arg);
+    const Int = switch (Arg) {
+        comptime_int => std.math.IntFittingRange(arg, arg),
+        else => Arg,
+    };
+    const Value = if (@typeInfo(Int).Int.bits < 8) u8 else Int;
+    var value: Value = arg;
+
+    for (slice[0 .. slice.len - 1]) |*byte| {
+        byte.* = @truncate(0x80 | value);
         value >>= 7;
-        ptr[i] = byte;
     }
-    ptr[i] = @truncate(value);
+    slice[slice.len - 1] = @as(u7, @intCast(value));
 }
 
 /// Deprecated: use `writeIleb128`

lib/std/math/big/int.zig

@@ -2092,6 +2092,12 @@ pub const Const = struct {
         return bits;
     }
 
+    /// Returns the number of bits required to represent the integer in twos-complement form
+    /// with the given signedness.
+    pub fn bitCountTwosCompForSignedness(self: Const, signedness: std.builtin.Signedness) usize {
+        return self.bitCountTwosComp() + @intFromBool(self.positive and signedness == .signed);
+    }
+
     /// @popCount with two's complement semantics.
     ///
     /// This returns the number of 1 bits set when the value would be represented in
@@ -2147,9 +2153,7 @@ pub const Const = struct {
         if (signedness == .unsigned and !self.positive) {
             return false;
         }
-
-        const req_bits = self.bitCountTwosComp() + @intFromBool(self.positive and signedness == .signed);
-        return bit_count >= req_bits;
+        return bit_count >= self.bitCountTwosCompForSignedness(signedness);
     }
 
     /// Returns whether self can fit into an integer of the requested type.

lib/std/mem.zig

@@ -128,7 +128,7 @@ pub fn alignAllocLen(full_len: usize, alloc_len: usize, len_align: u29) usize {
     assert(full_len >= alloc_len);
     if (len_align == 0)
         return alloc_len;
-    const adjusted = alignBackwardAnyAlign(full_len, len_align);
+    const adjusted = alignBackwardAnyAlign(usize, full_len, len_align);
     assert(adjusted >= alloc_len);
     return adjusted;
 }
@@ -4312,6 +4312,15 @@ test "sliceAsBytes preserves pointer attributes" {
     try testing.expectEqual(in.alignment, out.alignment);
 }
 
+/// Round an address down to the next (or current) aligned address.
+/// Unlike `alignForward`, `alignment` can be any positive number, not just a power of 2.
+pub fn alignForwardAnyAlign(comptime T: type, addr: T, alignment: T) T {
+    if (isValidAlignGeneric(T, alignment))
+        return alignForward(T, addr, alignment);
+    assert(alignment != 0);
+    return alignBackwardAnyAlign(T, addr + (alignment - 1), alignment);
+}
+
 /// Round an address up to the next (or current) aligned address.
 /// The alignment must be a power of 2 and greater than 0.
 /// Asserts that rounding up the address does not cause integer overflow.
@@ -4433,11 +4442,11 @@ test alignForward {
 
 /// Round an address down to the previous (or current) aligned address.
 /// Unlike `alignBackward`, `alignment` can be any positive number, not just a power of 2.
-pub fn alignBackwardAnyAlign(i: usize, alignment: usize) usize {
-    if (isValidAlign(alignment))
-        return alignBackward(usize, i, alignment);
+pub fn alignBackwardAnyAlign(comptime T: type, addr: T, alignment: T) T {
+    if (isValidAlignGeneric(T, alignment))
+        return alignBackward(T, addr, alignment);
     assert(alignment != 0);
-    return i - @mod(i, alignment);
+    return addr - @mod(addr, alignment);
 }
 
 /// Round an address down to the previous (or current) aligned address.

lib/std/zig/AstGen.zig

@@ -4405,7 +4405,6 @@ fn globalVarDecl(
         .decl_line = astgen.source_line,
         .astgen = astgen,
         .is_comptime = true,
-        .anon_name_strategy = .parent,
         .instructions = gz.instructions,
         .instructions_top = gz.instructions.items.len,
     };
@@ -4463,6 +4462,8 @@ fn globalVarDecl(
         else
             .none;
 
+        block_scope.anon_name_strategy = .parent;
+
         const init_inst = try expr(
             &block_scope,
             &block_scope.base,
@@ -4490,6 +4491,8 @@ fn globalVarDecl(
         // Extern variable which has an explicit type.
         const type_inst = try typeExpr(&block_scope, &block_scope.base, var_decl.ast.type_node);
 
+        block_scope.anon_name_strategy = .parent;
+
         const var_inst = try block_scope.addVar(.{
             .var_type = type_inst,
             .lib_name = lib_name,

src/Compilation.zig

@@ -363,6 +363,7 @@ const Job = union(enum) {
         /// It must be deinited when the job is processed.
         air: Air,
     },
+    codegen_type: InternPool.Index,
     /// The `Cau` must be semantically analyzed (and possibly export itself).
     /// This may be its first time being analyzed, or it may be outdated.
     analyze_cau: InternPool.Cau.Index,
@@ -423,6 +424,7 @@ const CodegenJob = union(enum) {
         /// It must be deinited when the job is processed.
         air: Air,
     },
+    type: InternPool.Index,
 };
 
 pub const CObject = struct {
@@ -3712,6 +3714,7 @@ fn processOneJob(tid: usize, comp: *Compilation, job: Job, prog_node: std.Progre
                 .air = func.air,
             } });
         },
+        .codegen_type => |ty| try comp.queueCodegenJob(tid, .{ .type = ty }),
         .analyze_func => |func| {
             const named_frame = tracy.namedFrame("analyze_func");
             defer named_frame.end();
@@ -4001,6 +4004,13 @@ fn processOneCodegenJob(tid: usize, comp: *Compilation, codegen_job: CodegenJob)
             // This call takes ownership of `func.air`.
             try pt.linkerUpdateFunc(func.func, func.air);
         },
+        .type => |ty| {
+            const named_frame = tracy.namedFrame("codegen_type");
+            defer named_frame.end();
+
+            const pt: Zcu.PerThread = .{ .zcu = comp.module.?, .tid = @enumFromInt(tid) };
+            try pt.linkerUpdateContainerType(ty);
+        },
     }
 }
 

src/InternPool.zig

@@ -4003,7 +4003,7 @@ pub fn loadStructType(ip: *const InternPool, index: Index) LoadedStructType {
     }
 }
 
-const LoadedEnumType = struct {
+pub const LoadedEnumType = struct {
     // TODO: the non-fqn will be needed by the new dwarf structure
     /// The name of this enum type.
     name: NullTerminatedString,