[Flutter GPU] Runtime shader import.

ci/licenses_golden/excluded_files

@@ -133,6 +133,7 @@
 ../../../flutter/impeller/base/base_unittests.cc
 ../../../flutter/impeller/compiler/README.md
 ../../../flutter/impeller/compiler/compiler_unittests.cc
+../../../flutter/impeller/compiler/shader_bundle_unittests.cc
 ../../../flutter/impeller/compiler/switches_unittests.cc
 ../../../flutter/impeller/core/allocator_unittests.cc
 ../../../flutter/impeller/display_list/dl_unittests.cc

ci/licenses_golden/licenses_flutter

@@ -4933,6 +4933,8 @@ ORIGIN: ../../../flutter/impeller/compiler/reflector.cc + ../../../flutter/LICEN
 ORIGIN: ../../../flutter/impeller/compiler/reflector.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/compiler/runtime_stage_data.cc + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/compiler/runtime_stage_data.h + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/impeller/compiler/shader_bundle.cc + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/impeller/compiler/shader_bundle.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/compiler/shader_lib/flutter/runtime_effect.glsl + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/compiler/shader_lib/impeller/blending.glsl + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/compiler/shader_lib/impeller/branching.glsl + ../../../flutter/LICENSE
@@ -5473,6 +5475,7 @@ ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage.fbs + ../../../flu
 ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage_playground.cc + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage_playground.h + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage_types.fbs + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/scene/animation/animation.cc + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/scene/animation/animation.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/scene/animation/animation_clip.cc + ../../../flutter/LICENSE
@@ -5527,6 +5530,7 @@ ORIGIN: ../../../flutter/impeller/shader_archive/shader_archive_main.cc + ../../
 ORIGIN: ../../../flutter/impeller/shader_archive/shader_archive_types.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/shader_archive/shader_archive_writer.cc + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/shader_archive/shader_archive_writer.h + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/impeller/shader_bundle/shader_bundle.fbs + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/tessellator/c/tessellator.cc + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/tessellator/c/tessellator.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/tessellator/dart/lib/tessellator.dart + ../../../flutter/LICENSE
@@ -7734,6 +7738,8 @@ FILE: ../../../flutter/impeller/compiler/reflector.cc
 FILE: ../../../flutter/impeller/compiler/reflector.h
 FILE: ../../../flutter/impeller/compiler/runtime_stage_data.cc
 FILE: ../../../flutter/impeller/compiler/runtime_stage_data.h
+FILE: ../../../flutter/impeller/compiler/shader_bundle.cc
+FILE: ../../../flutter/impeller/compiler/shader_bundle.h
 FILE: ../../../flutter/impeller/compiler/shader_lib/flutter/runtime_effect.glsl
 FILE: ../../../flutter/impeller/compiler/shader_lib/impeller/blending.glsl
 FILE: ../../../flutter/impeller/compiler/shader_lib/impeller/branching.glsl
@@ -8275,6 +8281,7 @@ FILE: ../../../flutter/impeller/runtime_stage/runtime_stage.fbs
 FILE: ../../../flutter/impeller/runtime_stage/runtime_stage.h
 FILE: ../../../flutter/impeller/runtime_stage/runtime_stage_playground.cc
 FILE: ../../../flutter/impeller/runtime_stage/runtime_stage_playground.h
+FILE: ../../../flutter/impeller/runtime_stage/runtime_stage_types.fbs
 FILE: ../../../flutter/impeller/scene/animation/animation.cc
 FILE: ../../../flutter/impeller/scene/animation/animation.h
 FILE: ../../../flutter/impeller/scene/animation/animation_clip.cc
@@ -8329,6 +8336,7 @@ FILE: ../../../flutter/impeller/shader_archive/shader_archive_main.cc
 FILE: ../../../flutter/impeller/shader_archive/shader_archive_types.h
 FILE: ../../../flutter/impeller/shader_archive/shader_archive_writer.cc
 FILE: ../../../flutter/impeller/shader_archive/shader_archive_writer.h
+FILE: ../../../flutter/impeller/shader_bundle/shader_bundle.fbs
 FILE: ../../../flutter/impeller/tessellator/c/tessellator.cc
 FILE: ../../../flutter/impeller/tessellator/c/tessellator.h
 FILE: ../../../flutter/impeller/tessellator/dart/lib/tessellator.dart

impeller/compiler/BUILD.gn

@@ -46,6 +46,8 @@ impeller_component("compiler_lib") {
     "reflector.h",
     "runtime_stage_data.cc",
     "runtime_stage_data.h",
+    "shader_bundle.cc",
+    "shader_bundle.h",
     "source_options.cc",
     "source_options.h",
     "spirv_compiler.cc",
@@ -66,6 +68,7 @@ impeller_component("compiler_lib") {
     "../geometry",
     "../runtime_stage",
     "//flutter/fml",
+    "//flutter/impeller/shader_bundle:shader_bundle_flatbuffers",
 
     # All third_party deps must be included by the global license script.
     "//third_party/inja",
@@ -102,6 +105,7 @@ impeller_component("compiler_unittests") {
     "compiler_test.cc",
     "compiler_test.h",
     "compiler_unittests.cc",
+    "shader_bundle_unittests.cc",
     "switches_unittests.cc",
   ]
 

impeller/compiler/impellerc_main.cc

@@ -10,6 +10,7 @@
 #include "flutter/fml/file.h"
 #include "flutter/fml/mapping.h"
 #include "impeller/compiler/compiler.h"
+#include "impeller/compiler/shader_bundle.h"
 #include "impeller/compiler/source_options.h"
 #include "impeller/compiler/switches.h"
 #include "impeller/compiler/types.h"
@@ -18,107 +19,52 @@
 namespace impeller {
 namespace compiler {
 
-// Sets the file access mode of the file at path 'p' to 0644.
-static bool SetPermissiveAccess(const std::filesystem::path& p) {
-  auto permissions =
-      std::filesystem::perms::owner_read | std::filesystem::perms::owner_write |
-      std::filesystem::perms::group_read | std::filesystem::perms::others_read;
-  std::error_code error;
-  std::filesystem::permissions(p, permissions, error);
-  if (error) {
-    std::cerr << "Failed to set access on file '" << p
-              << "': " << error.message() << std::endl;
-    return false;
-  }
-  return true;
-}
-
-bool Main(const fml::CommandLine& command_line) {
-  fml::InstallCrashHandler();
-  if (command_line.HasOption("help")) {
-    Switches::PrintHelp(std::cout);
-    return true;
-  }
-
-  Switches switches(command_line);
-  if (!switches.AreValid(std::cerr)) {
-    std::cerr << "Invalid flags specified." << std::endl;
-    Switches::PrintHelp(std::cerr);
-    return false;
-  }
+/// Run the shader compiler to geneate SkSL.
+/// If there is an error, prints error text and returns `nullptr`.
+static std::shared_ptr<fml::Mapping> CompileSkSL(
+    std::shared_ptr<fml::Mapping> source_file_mapping,
+    SourceOptions& options,
+    Reflector::Options& reflector_options) {
+  SourceOptions sksl_options = options;
+  sksl_options.target_platform = TargetPlatform::kSkSL;
 
-  std::shared_ptr<fml::FileMapping> source_file_mapping =
-      fml::FileMapping::CreateReadOnly(switches.source_file_name);
-  if (!source_file_mapping) {
-    std::cerr << "Could not open input file." << std::endl;
-    return false;
-  }
+  Reflector::Options sksl_reflector_options = reflector_options;
+  sksl_reflector_options.target_platform = TargetPlatform::kSkSL;
 
-  SourceOptions options;
-  options.target_platform = switches.target_platform;
-  options.source_language = switches.source_language;
-  if (switches.input_type == SourceType::kUnknown) {
-    options.type = SourceTypeFromFileName(switches.source_file_name);
-  } else {
-    options.type = switches.input_type;
+  Compiler sksl_compiler = Compiler(std::move(source_file_mapping),
+                                    sksl_options, sksl_reflector_options);
+  if (!sksl_compiler.IsValid()) {
+    std::cerr << "Compilation to SkSL failed." << std::endl;
+    std::cerr << sksl_compiler.GetErrorMessages() << std::endl;
+    return nullptr;
   }
-  options.working_directory = switches.working_directory;
-  options.file_name = switches.source_file_name;
-  options.include_dirs = switches.include_directories;
-  options.defines = switches.defines;
-  options.entry_point_name = EntryPointFunctionNameFromSourceName(
-      switches.source_file_name, options.type, options.source_language,
-      switches.entry_point);
-  options.json_format = switches.json_format;
-  options.gles_language_version = switches.gles_language_version;
-  options.metal_version = switches.metal_version;
-  options.use_half_textures = switches.use_half_textures;
-  options.require_framebuffer_fetch = switches.require_framebuffer_fetch;
+  return sksl_compiler.GetSLShaderSource();
+}
 
-  Reflector::Options reflector_options;
-  reflector_options.target_platform = switches.target_platform;
-  reflector_options.entry_point_name = options.entry_point_name;
-  reflector_options.shader_name =
-      InferShaderNameFromPath(switches.source_file_name);
-  reflector_options.header_file_name = Utf8FromPath(
-      std::filesystem::path{switches.reflection_header_name}.filename());
+/// Outputs artifacts for a single compiler invocation and option configuration.
+/// If there is an error, prints error text and returns `false`.
+static bool OutputArtifacts(Compiler& compiler,
+                            Switches& switches,
+                            std::shared_ptr<fml::Mapping> source_file_mapping,
+                            SourceOptions& options,
+                            Reflector::Options& reflector_options) {
+  // --------------------------------------------------------------------------
+  /// 1. Invoke the compiler to generate SkSL if needed.
+  ///
 
-  // Generate SkSL if needed.
   std::shared_ptr<fml::Mapping> sksl_mapping;
-  if (switches.iplr && TargetPlatformBundlesSkSL(switches.target_platform) &&
-      switches.iplr_bundle.empty()) {
-    SourceOptions sksl_options = options;
-    sksl_options.target_platform = TargetPlatform::kSkSL;
-
-    Reflector::Options sksl_reflector_options = reflector_options;
-    sksl_reflector_options.target_platform = TargetPlatform::kSkSL;
-
-    Compiler sksl_compiler =
-        Compiler(source_file_mapping, sksl_options, sksl_reflector_options);
-    if (!sksl_compiler.IsValid()) {
-      std::cerr << "Compilation to SkSL failed." << std::endl;
-      std::cerr << sksl_compiler.GetErrorMessages() << std::endl;
+  if (switches.iplr && TargetPlatformBundlesSkSL(switches.target_platform)) {
+    sksl_mapping =
+        CompileSkSL(std::move(source_file_mapping), options, reflector_options);
+    if (!sksl_mapping) {
       return false;
     }
-    sksl_mapping = sksl_compiler.GetSLShaderSource();
-  }
-
-  Compiler compiler(source_file_mapping, options, reflector_options);
-  if (!compiler.IsValid()) {
-    std::cerr << "Compilation failed." << std::endl;
-    std::cerr << compiler.GetErrorMessages() << std::endl;
-    return false;
   }
 
-  auto spriv_file_name = std::filesystem::absolute(
-      std::filesystem::current_path() / switches.spirv_file_name);
-  if (!fml::WriteAtomically(*switches.working_directory,
-                            Utf8FromPath(spriv_file_name).c_str(),
-                            *compiler.GetSPIRVAssembly())) {
-    std::cerr << "Could not write file to " << switches.spirv_file_name
-              << std::endl;
-    return false;
-  }
+  // --------------------------------------------------------------------------
+  /// 2. Output the source file. When in IPLR/RuntimeStage mode, output the
+  ///    serialized IPLR flatbuffer.
+  ///
 
   auto sl_file_name = std::filesystem::absolute(
       std::filesystem::current_path() / switches.sl_file_name);
@@ -134,7 +80,7 @@ bool Main(const fml::CommandLine& command_line) {
       return false;
     }
     if (sksl_mapping) {
-      stage_data->SetSkSLData(sksl_mapping);
+      stage_data->SetSkSLData(std::move(sksl_mapping));
     }
     auto stage_data_mapping = options.json_format
                                   ? stage_data->CreateJsonMapping()
@@ -166,6 +112,11 @@ bool Main(const fml::CommandLine& command_line) {
     }
   }
 
+  // --------------------------------------------------------------------------
+  /// 3. Output shader reflection data.
+  ///    May include a JSON file, a C++ header, and/or a C++ TU.
+  ///
+
   if (TargetPlatformNeedsReflection(options.target_platform)) {
     if (!switches.reflection_json_name.empty()) {
       auto reflection_json_name = std::filesystem::absolute(
@@ -208,6 +159,10 @@ bool Main(const fml::CommandLine& command_line) {
     }
   }
 
+  // --------------------------------------------------------------------------
+  /// 4. Output a depfile.
+  ///
+
   if (!switches.depfile_path.empty()) {
     std::string result_file;
     switch (switches.target_platform) {
@@ -240,6 +195,89 @@ bool Main(const fml::CommandLine& command_line) {
   return true;
 }
 
+bool Main(const fml::CommandLine& command_line) {
+  fml::InstallCrashHandler();
+  if (command_line.HasOption("help")) {
+    Switches::PrintHelp(std::cout);
+    return true;
+  }
+
+  Switches switches(command_line);
+  if (!switches.AreValid(std::cerr)) {
+    std::cerr << "Invalid flags specified." << std::endl;
+    Switches::PrintHelp(std::cerr);
+    return false;
+  }
+  SourceOptions options;
+  options.target_platform = switches.target_platform;
+  options.source_language = switches.source_language;
+  if (switches.input_type == SourceType::kUnknown) {
+    options.type = SourceTypeFromFileName(switches.source_file_name);
+  } else {
+    options.type = switches.input_type;
+  }
+  options.working_directory = switches.working_directory;
+  options.file_name = switches.source_file_name;
+  options.include_dirs = switches.include_directories;
+  options.defines = switches.defines;
+  options.entry_point_name = EntryPointFunctionNameFromSourceName(
+      switches.source_file_name, options.type, options.source_language,
+      switches.entry_point);
+  options.json_format = switches.json_format;
+  options.gles_language_version = switches.gles_language_version;
+  options.metal_version = switches.metal_version;
+  options.use_half_textures = switches.use_half_textures;
+  options.require_framebuffer_fetch = switches.require_framebuffer_fetch;
+
+  if (!switches.shader_bundle.empty()) {
+    // Invoke the compiler multiple times to build a shader bundle with the
+    // given shader_bundle spec.
+    return GenerateShaderBundle(switches, options);
+  }
+
+  std::shared_ptr<fml::FileMapping> source_file_mapping =
+      fml::FileMapping::CreateReadOnly(switches.source_file_name);
+  if (!source_file_mapping) {
+    std::cerr << "Could not open input file." << std::endl;
+    return false;
+  }
+
+  // Invoke the compiler and generate reflection data for a single shader or
+  // runtime stage IPLR.
+
+  Reflector::Options reflector_options;
+  reflector_options.target_platform = switches.target_platform;
+  reflector_options.entry_point_name = options.entry_point_name;
+  reflector_options.shader_name =
+      InferShaderNameFromPath(switches.source_file_name);
+  reflector_options.header_file_name = Utf8FromPath(
+      std::filesystem::path{switches.reflection_header_name}.filename());
+
+  Compiler compiler(source_file_mapping, options, reflector_options);
+  if (!compiler.IsValid()) {
+    std::cerr << "Compilation failed." << std::endl;
+    std::cerr << compiler.GetErrorMessages() << std::endl;
+    return false;
+  }
+
+  auto spriv_file_name = std::filesystem::absolute(
+      std::filesystem::current_path() / switches.spirv_file_name);
+  if (!fml::WriteAtomically(*switches.working_directory,
+                            Utf8FromPath(spriv_file_name).c_str(),
+                            *compiler.GetSPIRVAssembly())) {
+    std::cerr << "Could not write file to " << switches.spirv_file_name
+              << std::endl;
+    return false;
+  }
+
+  if (!OutputArtifacts(compiler, switches, std::move(source_file_mapping),
+                       options, reflector_options)) {
+    return false;
+  }
+
+  return true;
+}
+
 }  // namespace compiler
 }  // namespace impeller