Xtensa ESP32-S3

Author: liamHowatt

.gitignore

@@ -1,7 +1,8 @@
-compile-exe
-execute-exe
+/compile-exe
+/execute-exe
 /*.bin
-x86-32_backend_generated.c
-x86-32_backend_generated.h
-x86-32_engine_asm.o
-__pycache__
+/*_backend_generated.c
+/*_backend_generated.h
+/x86-32_engine_asm.o
+/__pycache__
+/tmp.*

Makefile

@@ -4,17 +4,17 @@ NO_FLAGS =
 
 all: compile-exe execute-exe
 
-x86-32_backend_generated.c: x86_32_backend_generator.py nasm.py
+x86_32_backend_generated.c: x86_32_backend_generator.py backend_generator.py
 	python3 x86_32_backend_generator.py
 
-compile-exe: mcp_forth.h mcp_forth.c compile.c compile-exe.c vm_backend.c x86-32_backend.c x86-32_backend_generated.c x86-32_backend_generated.h elf_linux.c
-	gcc $(NO_FLAGS) -m32 -Wall -fsanitize=address -g mcp_forth.c compile.c compile-exe.c vm_backend.c x86-32_backend.c x86-32_backend_generated.c elf_linux.c -o compile-exe
+compile-exe: mcp_forth.h mcp_forth.c compile.c compile-exe.c vm_backend.c x86-32_backend.c x86_32_backend_generated.c x86_32_backend_generated.h elf_linux.c
+	gcc $(NO_FLAGS) -m32 -Wall -fsanitize=address -g mcp_forth.c compile.c compile-exe.c vm_backend.c x86-32_backend.c x86_32_backend_generated.c elf_linux.c -o compile-exe
 
 x86-32_engine_asm.o: x86-32_engine_asm.s
 	nasm -felf32 -g -o x86-32_engine_asm.o x86-32_engine_asm.s
 
 execute-exe: mcp_forth.h mcp_forth.c global.h execute-exe.c runtime_io.c runtime_time.c runtime_string.c runtime_process.c runtime_file.c runtime_assert.c vm_engine.c x86-32_engine.c x86-32_engine_asm.o
-	gcc $(NO_FLAGS) -m32 -Wall -fsanitize=address -g mcp_forth.c execute-exe.c runtime_io.c runtime_time.c runtime_string.c runtime_process.c runtime_file.c runtime_assert.c runtime_threadutil.c vm_engine.c x86-32_engine.c x86-32_engine_asm.o -lpthread -ldl -o execute-exe
+	gcc $(NO_FLAGS) -m32 -no-pie -Wall -fsanitize=address -g mcp_forth.c execute-exe.c runtime_io.c runtime_time.c runtime_string.c runtime_process.c runtime_file.c runtime_assert.c runtime_threadutil.c vm_engine.c x86-32_engine.c x86-32_engine_asm.o -lpthread -ldl -o execute-exe
 
 test-simple: all
 	find forth_programs/simple -maxdepth 1 -type f | xargs -I{} ./compile-and-run.sh vm {}
@@ -23,4 +23,4 @@ test-simple-x86: all
 	find forth_programs/simple -maxdepth 1 -type f | xargs -I{} ./compile-and-run.sh x86 {}
 
 clean:
-	rm -f compile-exe execute-exe x86-32_backend_generated.c x86-32_backend_generated.h x86-32_engine_asm.o
+	rm -f compile-exe execute-exe x86_32_backend_generated.c x86_32_backend_generated.h x86-32_engine_asm.o

README.md

@@ -1,5 +1,18 @@
 # mcp_forth
 
+mcp_forth is an embeddable Forth compiler.
+
+## Features
+
+- Native machine code
+- No OS or library dependencies
+- Can call C functions
+- Can create C callbacks
+- Multithreading support
+- Small memory footprint
+
+## Quickstart
+
 `gcc-multilib` needs to be installed (unless OS is 32 bit) to compile `execute-exe` as 32 bit.
 
 ```sh
@@ -51,11 +64,11 @@ Currently supported architectures:
 
 - Interpreted bytecode VM (explained above)
 - x86-32
+- Xtensa (ESP32-S3, call8 ABI)
 
 Planned:
 
 - ARM Thumb (Cortex M0+)
-- Xtensa (LX6, i.e. ESP32)
 
 ## 32 Bits
 
@@ -79,42 +92,3 @@ VM runtime because they have no support for 32 bit programs. An emulator such as
   the runtime.
 - Gforth's "compile time only words" can be used outside of functions in mcp-forth.
 - `UNLOOP` is not required and is a no-op
-
-## Minutia
-
-### Iterative "Fragment Solving"
-
-Fragments aka snippets of machine code have variable sizes depending on their operands. If a jump
-instruction jumps somewhere nearby, it may only use 1 byte to encode the offset, otherwise 4 bytes.
-Literal values are similar. An immediate literal may be loaded into a register differently
-depending on its size. Some architectures require multiple instructions to load larger immediate
-literal values.
-
-Given that the jump distance may not be known at the time of a jump fragment's creation, the
-collection of all fragments at the end of compilation must be solved in an iterative way to
-achieve optimal packing.
-
-Question: will iterative solving ever cause the compiler to hang in an infinite loop that it can't solve?
-
-### Optimizing Compiler Memory Usage
-
-The compiler allocates a few arrays which it repeatedly appends elements to during compilation
-and resizes them when their capacities are exceeded. There are no small allocations since the overhead
-of N allocations of a small struct may be greater than an allocated contiguous array of N small structs.
-
-Strings referring to source code tokens are not allocated arrays of bytes.
-They are pointers into the source code and a length.
-
-Since struct references are always being invalidated due to array resizing, struct references
-are stored as array indices instead of pointers.
-
-Question: is it a good or bad idea to reduce memory usage by:
-
-- Storing strings as only a pointer with no length. The strings are whitespace-terminated since they
-  point inside the source code. There is a special case for a string that is the last token in the
-  source with no following whitespace.
-- Extending the previous point, should 16 bit offsets into the source be used instead of 32 bit pointers?
-- For indices into arrays of structs, should 16 bit indices be used instead of 32 bit ints?
-- For a case where less-than-32-bit integer types are used to store offsets, can all the members of
-  an array of offsets be dynamically promoted as needed? The first offset that exceeds 65535 would
-  cause the array to be converted from an array of 16 bit offsets to an array of 32 bit offsets.

backend_generator.py

@@ -0,0 +1,122 @@
+import sys
+import re
+from itertools import chain
+
+templ_c = """\
+/* generated by {generated_by} */
+#include "{name}_backend_generated.h"
+
+const uint8_t m4_{name}_backend_code[{code_array_size}] = {{
+{code_array_content}
+}};
+
+const uint8_t m4_{name}_backend_chore_a[{a_size}] = {{
+    {a_code} /* {a_src} */
+}};
+const uint8_t m4_{name}_backend_chore_b[{b_size}] = {{
+    {b_code} /* {b_src} */
+}};
+const uint8_t m4_{name}_backend_chore_y[{y_size}] = {{
+    {y_code} /* {y_src} */
+}};
+const uint8_t m4_{name}_backend_chore_z[{z_size}] = {{
+    {z_code} /* {z_src} */
+}};
+
+const m4_backend_table_entry_t m4_{name}_backend_table[{code_table_count}] = {{
+{code_table_content}
+}};
+"""
+
+templ_h = """\
+/* generated by {generated_by} */
+#include "mcp_forth.h"
+
+extern const uint8_t m4_{name}_backend_chore_a[{a_size}];
+extern const uint8_t m4_{name}_backend_chore_b[{b_size}];
+extern const uint8_t m4_{name}_backend_chore_y[{y_size}];
+extern const uint8_t m4_{name}_backend_chore_z[{z_size}];
+
+extern const uint8_t m4_{name}_backend_code[{code_array_size}];
+extern const m4_backend_table_entry_t m4_{name}_backend_table[{code_table_count}];
+"""
+
+def generate_builtins(backend_name, builtins, elidables, src_to_code_cb):
+    a_code = src_to_code_cb(elidables[0][0] + "\n")
+    b_code = src_to_code_cb(elidables[1][0] + "\n")
+    y_code = src_to_code_cb(elidables[2][0] + "\n")
+    z_code = src_to_code_cb(elidables[3][0] + "\n")
+
+    code_content = []
+    table_content = []
+    code_cumulative_len = 0
+    for word, flags, src in builtins:
+        src_lines = list(line for line in src.splitlines() if line.strip())
+        flag_a = len(src_lines) > 0 and re.fullmatch(elidables[0][1], src_lines[0]) is not None
+        flag_b = flag_a and len(src_lines) > 1 and re.fullmatch(elidables[1][1], src_lines[1]) is not None
+        flag_clob = flag_b and "clobber" in flags
+        if not flag_clob and "clobber" in flags:
+            print("warning: clobber flag given but criteria not met")
+        flag_z = len(src_lines) > 0 and re.fullmatch(elidables[3][1], src_lines[-1]) is not None
+        flag_y = flag_z and len(src_lines) > 1 and re.fullmatch(elidables[2][1], src_lines[-2]) is not None
+        if(flag_a): del src_lines[0]
+        if(flag_b): del src_lines[0]
+        if(flag_z): del src_lines[-1]
+        if(flag_y): del src_lines[-1]
+        src_reduced = "".join(line + '\n' for line in src_lines)
+        code = src_to_code_cb(src_reduced)
+        code_content.append(
+            f"    /* {word} */ "
+            + "".join(f"0x{b:02x}," for b in code)
+        )
+        pos_str = f"{code_cumulative_len},".ljust(7)
+        len_str = f"{len(code)},".ljust(6)
+        code_cumulative_len += len(code)
+        flags_str = " | ".join(chain.from_iterable((
+            ("M4_BACKEND_FLAG_A",) if flag_a else (),
+            ("M4_BACKEND_FLAG_B",) if flag_b else (),
+            ("M4_BACKEND_FLAG_CLOB",) if flag_clob else (),
+            ("M4_BACKEND_FLAG_Y",) if flag_y else (),
+            ("M4_BACKEND_FLAG_Z",) if flag_z else (),
+        )))
+        table_content.append(
+            f"    {{{pos_str}{len_str}{flags_str}}},"
+        )
+
+    c = templ_c.format(
+        generated_by = sys.argv[0],
+        name = backend_name,
+        code_array_size = code_cumulative_len,
+        code_array_content = "\n".join(code_content),
+        a_size = len(a_code),
+        a_code = ",".join(f"0x{b:02x}" for b in a_code),
+        a_src = elidables[0][0],
+        b_size = len(b_code),
+        b_code = ",".join(f"0x{b:02x}" for b in b_code),
+        b_src = elidables[1][0],
+        y_size = len(y_code),
+        y_code = ",".join(f"0x{b:02x}" for b in y_code),
+        y_src = elidables[2][0],
+        z_size = len(z_code),
+        z_code = ",".join(f"0x{b:02x}" for b in z_code),
+        z_src = elidables[3][0],
+        code_table_count = len(table_content),
+        code_table_content = "\n".join(table_content)
+    )
+
+    h = templ_h.format(
+        generated_by = sys.argv[0],
+        name = backend_name,
+        a_size = len(a_code),
+        b_size = len(b_code),
+        y_size = len(y_code),
+        z_size = len(z_code),
+        code_array_size = code_cumulative_len,
+        code_table_count = len(table_content)
+    )
+
+    with open(f"{backend_name}_backend_generated.c", "w") as f:
+        f.write(c)
+
+    with open(f"{backend_name}_backend_generated.h", "w") as f:
+        f.write(h)

compile-exe.c

@@ -42,7 +42,7 @@ int main(int argc, char ** argv)
 
     uint8_t * bin;
     int error_near = -1;
-    int bin_len = m4_compile(source, source_len, &bin, backend, &error_near);
+    int bin_len = m4_compile(source, source_len, &bin, NULL, backend, &error_near);
     free(source);
     if(bin_len < 0) {
         printf("error %d near %d\n", bin_len, error_near);