Add scu_build option

scu_builders.py

@@ -0,0 +1,286 @@
+"""Functions used to generate scu build source files during build time"""
+
+import glob
+import math
+import os
+from pathlib import Path
+
+base_folder_path = str(Path(__file__).parent) + "/"
+base_folder_only = os.path.basename(os.path.normpath(base_folder_path))
+_verbose = False  # Set manually for debug prints
+_scu_folders = set()
+_max_includes_per_scu = 1024
+
+
+def clear_out_stale_files(output_folder, extension, fresh_files):
+    output_folder = os.path.abspath(output_folder)
+    # print("clear_out_stale_files from folder: " + output_folder)
+
+    if not os.path.isdir(output_folder):
+        # folder does not exist or has not been created yet,
+        # no files to clearout. (this is not an error)
+        return
+
+    for file in glob.glob(output_folder + "/*." + extension):
+        file = Path(file)
+        if file not in fresh_files:
+            # print("removed stale file: " + str(file))
+            os.remove(file)
+
+
+def folder_not_found(folder):
+    abs_folder = base_folder_path + folder + "/"
+    return not os.path.isdir(abs_folder)
+
+
+def find_files_in_folder(folder, sub_folder, include_list, extension, sought_exceptions, found_exceptions):
+    abs_folder = base_folder_path + folder + "/" + sub_folder
+
+    if not os.path.isdir(abs_folder):
+        print(f'SCU: "{abs_folder}" not found.')
+        return include_list, found_exceptions
+
+    os.chdir(abs_folder)
+
+    sub_folder_slashed = ""
+    if sub_folder != "":
+        sub_folder_slashed = sub_folder + "/"
+
+    for file in glob.glob("*." + extension):
+        simple_name = Path(file).stem
+
+        if file.endswith(".gen.cpp"):
+            continue
+
+        li = '#include "' + folder + "/" + sub_folder_slashed + file + '"'
+
+        if simple_name not in sought_exceptions:
+            include_list.append(li)
+        else:
+            found_exceptions.append(li)
+
+    return include_list, found_exceptions
+
+
+def write_output_file(file_count, include_list, start_line, end_line, output_folder, output_filename_prefix, extension):
+    output_folder = os.path.abspath(output_folder)
+
+    if not os.path.isdir(output_folder):
+        # create
+        os.mkdir(output_folder)
+        if not os.path.isdir(output_folder):
+            print(f'SCU: "{output_folder}" could not be created.')
+            return
+        if _verbose:
+            print("SCU: Creating folder: %s" % output_folder)
+
+    file_text = ""
+
+    for i in range(start_line, end_line):
+        if i < len(include_list):
+            line = include_list[i]
+            li = line + "\n"
+            file_text += li
+
+    num_string = ""
+    if file_count > 0:
+        num_string = "_" + str(file_count)
+
+    short_filename = output_filename_prefix + num_string + ".gen." + extension
+    output_filename = output_folder + "/" + short_filename
+    output_path = Path(output_filename)
+
+    if not output_path.exists() or output_path.read_text() != file_text:
+        if _verbose:
+            print("SCU: Generating: %s" % short_filename)
+        output_path.write_text(file_text, encoding="utf8")
+    elif _verbose:
+        print("SCU: Generation not needed for: " + short_filename)
+
+    return output_path
+
+
+def write_exception_output_file(file_count, exception_string, output_folder, output_filename_prefix, extension):
+    output_folder = os.path.abspath(output_folder)
+    if not os.path.isdir(output_folder):
+        print(f"SCU: {output_folder} does not exist.")
+        return
+
+    file_text = exception_string + "\n"
+
+    num_string = ""
+    if file_count > 0:
+        num_string = "_" + str(file_count)
+
+    short_filename = output_filename_prefix + "_exception" + num_string + ".gen." + extension
+    output_filename = output_folder + "/" + short_filename
+
+    output_path = Path(output_filename)
+
+    if not output_path.exists() or output_path.read_text() != file_text:
+        if _verbose:
+            print("SCU: Generating: " + short_filename)
+        output_path.write_text(file_text, encoding="utf8")
+    elif _verbose:
+        print("SCU: Generation not needed for: " + short_filename)
+
+    return output_path
+
+
+def find_section_name(sub_folder):
+    # Construct a useful name for the section from the path for debug logging
+    section_path = os.path.abspath(base_folder_path + sub_folder) + "/"
+
+    folders = []
+    folder = ""
+
+    for i in range(8):
+        folder = os.path.dirname(section_path)
+        folder = os.path.basename(folder)
+        if folder == base_folder_only:
+            break
+        folders.append(folder)
+        section_path += "../"
+        section_path = os.path.abspath(section_path) + "/"
+
+    section_name = ""
+    for n in range(len(folders)):
+        section_name += folders[len(folders) - n - 1]
+        if n != (len(folders) - 1):
+            section_name += "_"
+
+    return section_name
+
+
+# "folders" is a list of folders to add all the files from to add to the SCU
+# "section (like a module)". The name of the scu file will be derived from the first folder
+# (thus e.g. scene/3d becomes scu_scene_3d.gen.cpp)
+
+# "includes_per_scu" limits the number of includes in a single scu file.
+# This allows the module to be built in several translation units instead of just 1.
+# This will usually be slower to compile but will use less memory per compiler instance, which
+# is most relevant in release builds.
+
+# "sought_exceptions" are a list of files (without extension) that contain
+# e.g. naming conflicts, and are therefore not suitable for the scu build.
+# These will automatically be placed in their own separate scu file,
+# which is slow like a normal build, but prevents the naming conflicts.
+# Ideally in these situations, the source code should be changed to prevent naming conflicts.
+
+
+# "extension" will usually be cpp, but can also be set to c (for e.g. third party libraries that use c)
+def process_folder(folders, sought_exceptions=[], includes_per_scu=0, extension="cpp"):
+    if len(folders) == 0:
+        return
+
+    # Construct the filename prefix from the FIRST folder name
+    # e.g. "scene_3d"
+    out_filename = find_section_name(folders[0])
+
+    found_includes = []
+    found_exceptions = []
+
+    main_folder = folders[0]
+    abs_main_folder = base_folder_path + main_folder
+
+    # Keep a record of all folders that have been processed for SCU,
+    # this enables deciding what to do when we call "add_source_files()"
+    global _scu_folders
+    _scu_folders.add(main_folder)
+
+    # main folder (first)
+    found_includes, found_exceptions = find_files_in_folder(
+        main_folder, "", found_includes, extension, sought_exceptions, found_exceptions
+    )
+
+    # sub folders
+    for d in range(1, len(folders)):
+        found_includes, found_exceptions = find_files_in_folder(
+            main_folder, folders[d], found_includes, extension, sought_exceptions, found_exceptions
+        )
+
+    found_includes = sorted(found_includes)
+
+    # calculate how many lines to write in each file
+    total_lines = len(found_includes)
+
+    # adjust number of output files according to whether DEV or release
+    num_output_files = 1
+
+    if includes_per_scu == 0:
+        includes_per_scu = _max_includes_per_scu
+    else:
+        if includes_per_scu > _max_includes_per_scu:
+            includes_per_scu = _max_includes_per_scu
+
+    num_output_files = max(math.ceil(total_lines / float(includes_per_scu)), 1)
+
+    lines_per_file = math.ceil(total_lines / float(num_output_files))
+    lines_per_file = max(lines_per_file, 1)
+
+    start_line = 0
+
+    # These do not vary throughout the loop
+    output_folder = abs_main_folder + "/.scu/"
+    output_filename_prefix = "scu_" + out_filename
+
+    fresh_files = set()
+
+    for file_count in range(0, num_output_files):
+        end_line = start_line + lines_per_file
+
+        # special case to cover rounding error in final file
+        if file_count == (num_output_files - 1):
+            end_line = len(found_includes)
+
+        fresh_file = write_output_file(
+            file_count, found_includes, start_line, end_line, output_folder, output_filename_prefix, extension
+        )
+
+        fresh_files.add(fresh_file)
+
+        start_line = end_line
+
+    # Write the exceptions each in their own scu gen file,
+    # so they can effectively compile in "old style / normal build".
+    for exception_count in range(len(found_exceptions)):
+        fresh_file = write_exception_output_file(
+            exception_count, found_exceptions[exception_count], output_folder, output_filename_prefix, extension
+        )
+
+        fresh_files.add(fresh_file)
+
+    # Clear out any stale file (usually we will be overwriting if necessary,
+    # but we want to remove any that are pre-existing that will not be
+    # overwritten, so as to not compile anything stale).
+    clear_out_stale_files(output_folder, extension, fresh_files)
+
+
+def generate_scu_files(max_includes_per_scu):
+    global _max_includes_per_scu
+    _max_includes_per_scu = max_includes_per_scu
+
+    print("SCU: Generating build files... (max includes per SCU: %d)" % _max_includes_per_scu)
+
+    curr_folder = os.path.abspath("./")
+
+    # check we are running from the correct folder
+    if folder_not_found("src") or folder_not_found("gen"):
+        raise RuntimeError("scu_builders.py must be run from the godot-cpp folder.")
+        return
+
+    process_folder(["src"])
+    process_folder(["src/classes"])
+    process_folder(["src/core"])
+    process_folder(["src/variant"])
+
+    process_folder(["gen/src/classes"])
+    process_folder(["gen/src/variant"])
+
+    # Finally change back the path to the calling folder
+    os.chdir(curr_folder)
+
+    if _verbose:
+        print("SCU: Processed folders: %s" % sorted(_scu_folders))
+
+    return _scu_folders

tools/godotcpp.py

@@ -11,6 +11,7 @@
 from SCons.Variables import BoolVariable, EnumVariable, PathVariable
 from SCons.Variables.BoolVariable import _text2bool
 
+import scu_builders
 from binding_generator import _generate_bindings, _get_file_list, get_file_list
 from build_profile import generate_trimmed_api
 from doc_source_generator import scons_generate_doc_source
@@ -137,6 +138,19 @@ def scons_emit_files(target, source, env):
     if profile_filepath:
         profile_filepath = normalize_path(profile_filepath, env)
 
+    # Clean scu files
+    if not env["gen_scu_build"] and not env["scu_build"]:
+        for root, dirs, files in os.walk(".", topdown=False):
+            if os.path.basename(root) == ".scu":
+                for name in files:
+                    os.remove(os.path.join(root, name))
+                for name in dirs:
+                    os.rmdir(os.path.join(root, name))
+                os.rmdir(root)
+
+            if ".scu" in dirs:
+                dirs.remove(".scu")
+
     # Always clean all files
     env.Clean(target, [env.File(f) for f in get_file_list(str(source[0]), target[0].abspath, True, True)])
 
@@ -377,6 +391,9 @@ def options(opts, env):
     opts.Add(BoolVariable("debug_symbols", "Build with debugging symbols", True))
     opts.Add(BoolVariable("dev_build", "Developer build with dev-only debugging code (DEV_ENABLED)", False))
     opts.Add(BoolVariable("verbose", "Enable verbose output for the compilation", False))
+    opts.Add(BoolVariable("scu_build", "Use single compilation unit build", False))
+    opts.Add(BoolVariable("gen_scu_build", "Generate scu files", False))
+    opts.Add("scu_limit", "Max includes per SCU file when using scu_build (determines RAM use)", "0")
 
     # Add platform options (custom tools can override platforms)
     for pl in sorted(set(platforms + custom_platforms)):
@@ -553,14 +570,34 @@ def _godot_cpp(env):
         env.AlwaysBuild(bindings)
         env.NoCache(bindings)
 
+    # Run SCU file generation script if in a SCU build.
+    if env["gen_scu_build"]:
+        env.AppendUnique(CPPPATH=".")
+        max_includes_per_scu = 8
+        if env.dev_build:
+            max_includes_per_scu = 1024
+
+        read_scu_limit = int(env["scu_limit"])
+        read_scu_limit = max(0, min(read_scu_limit, 1024))
+        if read_scu_limit != 0:
+            max_includes_per_scu = read_scu_limit
+
+        scu_builders.generate_scu_files(max_includes_per_scu)
+
     # Sources to compile
-    sources = [
-        *env.Glob("src/*.cpp"),
-        *env.Glob("src/classes/*.cpp"),
-        *env.Glob("src/core/*.cpp"),
-        *env.Glob("src/variant/*.cpp"),
-        *tuple(f for f in bindings if str(f).endswith(".cpp")),
-    ]
+    if env["scu_build"]:
+        sources = []
+        for f in {"src/classes", "src/variant", "gen/src/variant", "src/core", "gen/src/classes", "src"}:
+            sources.append(env.Glob(f.replace("\\", "/") + "/.scu/*.cpp"))
+        env.AppendUnique(CPPPATH=".")
+    else:
+        sources = [
+            *env.Glob("src/*.cpp"),
+            *env.Glob("src/classes/*.cpp"),
+            *env.Glob("src/core/*.cpp"),
+            *env.Glob("src/variant/*.cpp"),
+            *tuple(f for f in bindings if str(f).endswith(".cpp")),
+        ]
 
     # Includes
     env.AppendUnique(