Rework AST memory layout for better memory usage and performance

CMakeLists.txt

@@ -370,7 +370,6 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/heap.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/heap/arena_allocator.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/io.zig"
-    "${CMAKE_SOURCE_DIR}/lib/std/io/auto_indenting_stream.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/io/buffered_atomic_file.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/io/buffered_writer.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/io/change_detection_stream.zig"
@@ -408,6 +407,7 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/std/meta.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/meta/trailer_flags.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/meta/trait.zig"
+    "${CMAKE_SOURCE_DIR}/lib/std/multi_array_list.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/os.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/os/bits.zig"
     "${CMAKE_SOURCE_DIR}/lib/std/os/bits/linux.zig"
@@ -573,6 +573,7 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/src/target.zig"
     "${CMAKE_SOURCE_DIR}/src/tracy.zig"
     "${CMAKE_SOURCE_DIR}/src/translate_c.zig"
+    "${CMAKE_SOURCE_DIR}/src/translate_c/ast.zig"
     "${CMAKE_SOURCE_DIR}/src/type.zig"
     "${CMAKE_SOURCE_DIR}/src/value.zig"
     "${CMAKE_SOURCE_DIR}/src/windows_sdk.zig"

README.md

@@ -5,7 +5,7 @@ A general-purpose programming language and toolchain for maintaining
 
 ## Resources
 
- * [Introduction](https://ziglang.org/#Introduction)
+ * [Introduction](https://ziglang.org/learn/#introduction)
  * [Download & Documentation](https://ziglang.org/download)
  * [Chapter 0 - Getting Started | ZigLearn.org](https://ziglearn.org/)
  * [Community](https://github.com/ziglang/zig/wiki/Community)

build.zig

@@ -77,10 +77,12 @@ pub fn build(b: *Builder) !void {
 
     const tracy = b.option([]const u8, "tracy", "Enable Tracy integration. Supply path to Tracy source");
     const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse enable_llvm;
+    const strip = b.option(bool, "strip", "Omit debug information") orelse false;
 
     const main_file = if (is_stage1) "src/stage1.zig" else "src/main.zig";
 
     var exe = b.addExecutable("zig", main_file);
+    exe.strip = strip;
     exe.install();
     exe.setBuildMode(mode);
     exe.setTarget(target);

doc/docgen.zig

@@ -781,106 +781,119 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: anytype, source_token:
         next_tok_is_fn = false;
 
         const token = tokenizer.next();
+        if (mem.indexOf(u8, src[index..token.loc.start], "//")) |comment_start_off| {
+            // render one comment
+            const comment_start = index + comment_start_off;
+            const comment_end_off = mem.indexOf(u8, src[comment_start .. token.loc.start], "\n");
+            const comment_end = if (comment_end_off) |o| comment_start + o else token.loc.start;
+
+            try writeEscaped(out, src[index..comment_start]);
+            try out.writeAll("<span class=\"tok-comment\">");
+            try writeEscaped(out, src[comment_start .. comment_end]);
+            try out.writeAll("</span>");
+            index = comment_end;
+            tokenizer.index = index;
+            continue;
+        }
+
         try writeEscaped(out, src[index..token.loc.start]);
-        switch (token.id) {
-            .Eof => break,
-
-            .Keyword_align,
-            .Keyword_and,
-            .Keyword_asm,
-            .Keyword_async,
-            .Keyword_await,
-            .Keyword_break,
-            .Keyword_catch,
-            .Keyword_comptime,
-            .Keyword_const,
-            .Keyword_continue,
-            .Keyword_defer,
-            .Keyword_else,
-            .Keyword_enum,
-            .Keyword_errdefer,
-            .Keyword_error,
-            .Keyword_export,
-            .Keyword_extern,
-            .Keyword_for,
-            .Keyword_if,
-            .Keyword_inline,
-            .Keyword_noalias,
-            .Keyword_noinline,
-            .Keyword_nosuspend,
-            .Keyword_opaque,
-            .Keyword_or,
-            .Keyword_orelse,
-            .Keyword_packed,
-            .Keyword_anyframe,
-            .Keyword_pub,
-            .Keyword_resume,
-            .Keyword_return,
-            .Keyword_linksection,
-            .Keyword_callconv,
-            .Keyword_struct,
-            .Keyword_suspend,
-            .Keyword_switch,
-            .Keyword_test,
-            .Keyword_threadlocal,
-            .Keyword_try,
-            .Keyword_union,
-            .Keyword_unreachable,
-            .Keyword_usingnamespace,
-            .Keyword_var,
-            .Keyword_volatile,
-            .Keyword_allowzero,
-            .Keyword_while,
-            .Keyword_anytype,
+        switch (token.tag) {
+            .eof => break,
+
+            .keyword_align,
+            .keyword_and,
+            .keyword_asm,
+            .keyword_async,
+            .keyword_await,
+            .keyword_break,
+            .keyword_catch,
+            .keyword_comptime,
+            .keyword_const,
+            .keyword_continue,
+            .keyword_defer,
+            .keyword_else,
+            .keyword_enum,
+            .keyword_errdefer,
+            .keyword_error,
+            .keyword_export,
+            .keyword_extern,
+            .keyword_for,
+            .keyword_if,
+            .keyword_inline,
+            .keyword_noalias,
+            .keyword_noinline,
+            .keyword_nosuspend,
+            .keyword_opaque,
+            .keyword_or,
+            .keyword_orelse,
+            .keyword_packed,
+            .keyword_anyframe,
+            .keyword_pub,
+            .keyword_resume,
+            .keyword_return,
+            .keyword_linksection,
+            .keyword_callconv,
+            .keyword_struct,
+            .keyword_suspend,
+            .keyword_switch,
+            .keyword_test,
+            .keyword_threadlocal,
+            .keyword_try,
+            .keyword_union,
+            .keyword_unreachable,
+            .keyword_usingnamespace,
+            .keyword_var,
+            .keyword_volatile,
+            .keyword_allowzero,
+            .keyword_while,
+            .keyword_anytype,
             => {
                 try out.writeAll("<span class=\"tok-kw\">");
                 try writeEscaped(out, src[token.loc.start..token.loc.end]);
                 try out.writeAll("</span>");
             },
 
-            .Keyword_fn => {
+            .keyword_fn => {
                 try out.writeAll("<span class=\"tok-kw\">");
                 try writeEscaped(out, src[token.loc.start..token.loc.end]);
                 try out.writeAll("</span>");
                 next_tok_is_fn = true;
             },
 
-            .Keyword_undefined,
-            .Keyword_null,
-            .Keyword_true,
-            .Keyword_false,
+            .keyword_undefined,
+            .keyword_null,
+            .keyword_true,
+            .keyword_false,
             => {
                 try out.writeAll("<span class=\"tok-null\">");
                 try writeEscaped(out, src[token.loc.start..token.loc.end]);
                 try out.writeAll("</span>");
             },
 
-            .StringLiteral,
-            .MultilineStringLiteralLine,
-            .CharLiteral,
+            .string_literal,
+            .multiline_string_literal_line,
+            .char_literal,
             => {
                 try out.writeAll("<span class=\"tok-str\">");
                 try writeEscaped(out, src[token.loc.start..token.loc.end]);
                 try out.writeAll("</span>");
             },
 
-            .Builtin => {
+            .builtin => {
                 try out.writeAll("<span class=\"tok-builtin\">");
                 try writeEscaped(out, src[token.loc.start..token.loc.end]);
                 try out.writeAll("</span>");
             },
 
-            .LineComment,
-            .DocComment,
-            .ContainerDocComment,
-            .ShebangLine,
+            .doc_comment,
+            .container_doc_comment,
             => {
                 try out.writeAll("<span class=\"tok-comment\">");
                 try writeEscaped(out, src[token.loc.start..token.loc.end]);
                 try out.writeAll("</span>");
             },
 
-            .Identifier => {
+            .identifier => {
                 if (prev_tok_was_fn) {
                     try out.writeAll("<span class=\"tok-fn\">");
                     try writeEscaped(out, src[token.loc.start..token.loc.end]);
@@ -908,71 +921,71 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: anytype, source_token:
                 }
             },
 
-            .IntegerLiteral,
-            .FloatLiteral,
+            .integer_literal,
+            .float_literal,
             => {
                 try out.writeAll("<span class=\"tok-number\">");
                 try writeEscaped(out, src[token.loc.start..token.loc.end]);
                 try out.writeAll("</span>");
             },
 
-            .Bang,
-            .Pipe,
-            .PipePipe,
-            .PipeEqual,
-            .Equal,
-            .EqualEqual,
-            .EqualAngleBracketRight,
-            .BangEqual,
-            .LParen,
-            .RParen,
-            .Semicolon,
-            .Percent,
-            .PercentEqual,
-            .LBrace,
-            .RBrace,
-            .LBracket,
-            .RBracket,
-            .Period,
-            .PeriodAsterisk,
-            .Ellipsis2,
-            .Ellipsis3,
-            .Caret,
-            .CaretEqual,
-            .Plus,
-            .PlusPlus,
-            .PlusEqual,
-            .PlusPercent,
-            .PlusPercentEqual,
-            .Minus,
-            .MinusEqual,
-            .MinusPercent,
-            .MinusPercentEqual,
-            .Asterisk,
-            .AsteriskEqual,
-            .AsteriskAsterisk,
-            .AsteriskPercent,
-            .AsteriskPercentEqual,
-            .Arrow,
-            .Colon,
-            .Slash,
-            .SlashEqual,
-            .Comma,
-            .Ampersand,
-            .AmpersandEqual,
-            .QuestionMark,
-            .AngleBracketLeft,
-            .AngleBracketLeftEqual,
-            .AngleBracketAngleBracketLeft,
-            .AngleBracketAngleBracketLeftEqual,
-            .AngleBracketRight,
-            .AngleBracketRightEqual,
-            .AngleBracketAngleBracketRight,
-            .AngleBracketAngleBracketRightEqual,
-            .Tilde,
+            .bang,
+            .pipe,
+            .pipe_pipe,
+            .pipe_equal,
+            .equal,
+            .equal_equal,
+            .equal_angle_bracket_right,
+            .bang_equal,
+            .l_paren,
+            .r_paren,
+            .semicolon,
+            .percent,
+            .percent_equal,
+            .l_brace,
+            .r_brace,
+            .l_bracket,
+            .r_bracket,
+            .period,
+            .period_asterisk,
+            .ellipsis2,
+            .ellipsis3,
+            .caret,
+            .caret_equal,
+            .plus,
+            .plus_plus,
+            .plus_equal,
+            .plus_percent,
+            .plus_percent_equal,
+            .minus,
+            .minus_equal,
+            .minus_percent,
+            .minus_percent_equal,
+            .asterisk,
+            .asterisk_equal,
+            .asterisk_asterisk,
+            .asterisk_percent,
+            .asterisk_percent_equal,
+            .arrow,
+            .colon,
+            .slash,
+            .slash_equal,
+            .comma,
+            .ampersand,
+            .ampersand_equal,
+            .question_mark,
+            .angle_bracket_left,
+            .angle_bracket_left_equal,
+            .angle_bracket_angle_bracket_left,
+            .angle_bracket_angle_bracket_left_equal,
+            .angle_bracket_right,
+            .angle_bracket_right_equal,
+            .angle_bracket_angle_bracket_right,
+            .angle_bracket_angle_bracket_right_equal,
+            .tilde,
             => try writeEscaped(out, src[token.loc.start..token.loc.end]),
 
-            .Invalid, .Invalid_ampersands, .Invalid_periodasterisks => return parseError(
+            .invalid, .invalid_ampersands, .invalid_periodasterisks => return parseError(
                 docgen_tokenizer,
                 source_token,
                 "syntax error",