[C++20] [Modules] Bring Decls Hash to BMI for C++20 Module units

clang/include/clang/AST/ODRHash.h

@@ -101,6 +101,9 @@ class ODRHash {
   // Save booleans until the end to lower the size of data to process.
   void AddBoolean(bool value);
 
+  // Add intergeers to ID.
+  void AddInteger(unsigned value);
+
   static bool isSubDeclToBeProcessed(const Decl *D, const DeclContext *Parent);
 
 private:

clang/include/clang/Basic/DiagnosticDriverKinds.td

@@ -158,6 +158,8 @@ def err_drv_invalid_output_with_multiple_archs : Error<
 def err_drv_no_input_files : Error<"no input files">;
 def err_drv_output_argument_with_multiple_files : Error<
   "cannot specify -o when generating multiple output files">;
+def err_drv_thin_bmi_output_argument_with_multiple_files : Error <
+  "cannot specify -fthinBMI-output when generating multiple module files">;
 def err_drv_out_file_argument_with_multiple_sources : Error<
   "cannot specify '%0%1' when compiling multiple source files">;
 def err_no_external_assembler : Error<

clang/include/clang/Driver/Options.td

@@ -2915,6 +2915,11 @@ def fmodule_output : Flag<["-"], "fmodule-output">, Flags<[NoXarchOption]>,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Save intermediate module file results when compiling a standard C++ module unit.">;
 
+def fthinBMI_output_EQ : Joined<["-"], "fthinBMI-output=">, Group<f_Group>,
+  HelpText<"Specify the output path for the thin BMI for C++20 Named modules">,
+  Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
+  MarshallingInfoString<FrontendOpts<"ThinBMIPath">>;
+
 def fmodules_prune_interval : Joined<["-"], "fmodules-prune-interval=">, Group<i_Group>,
   Visibility<[ClangOption, CC1Option]>, MetaVarName<"<seconds>">,
   HelpText<"Specify the interval (in seconds) between attempts to prune the module cache">,
@@ -5088,6 +5093,9 @@ def muclibc : Flag<["-"], "muclibc">, Group<m_libc_Group>, Flags<[HelpHidden]>;
 def module_file_info : Flag<["-"], "module-file-info">, Flags<[]>,
   Visibility<[ClangOption, CC1Option]>, Group<Action_Group>,
   HelpText<"Provide information about a particular module file">;
+def get_bmi_decls_hash : Flag<["-"], "get-bmi-decls-hash">, Flags<[]>,
+  Visibility<[ClangOption, CC1Option]>, Group<Action_Group>,
+  HelpText<"Get the BMI Decls hash value for a particular module file">;
 def mthumb : Flag<["-"], "mthumb">, Group<m_Group>;
 def mtune_EQ : Joined<["-"], "mtune=">, Group<m_Group>,
   HelpText<"Only supported on AArch64, PowerPC, RISC-V, SystemZ, and X86">;
@@ -7223,7 +7231,9 @@ def ast_view : Flag<["-"], "ast-view">,
 def emit_module : Flag<["-"], "emit-module">,
   HelpText<"Generate pre-compiled module file from a module map">;
 def emit_module_interface : Flag<["-"], "emit-module-interface">,
-  HelpText<"Generate pre-compiled module file from a C++ module interface">;
+  HelpText<"Generate pre-compiled module file from a standard C++ module interface unit">;
+def emit_thin_module_interface : Flag<["-"], "emit-thin-module-interface">,
+  HelpText<"Generate reduced prebuilt module interface from a standard C++ module interface unit">;
 def emit_header_unit : Flag<["-"], "emit-header-unit">,
   HelpText<"Generate C++20 header units from header files">;
 def emit_pch : Flag<["-"], "emit-pch">,

clang/include/clang/Frontend/FrontendActions.h

@@ -118,6 +118,9 @@ class GenerateModuleAction : public ASTFrontendAction {
   CreateOutputFile(CompilerInstance &CI, StringRef InFile) = 0;
 
 protected:
+  std::vector<std::unique_ptr<ASTConsumer>>
+  CreateMultiplexConsumer(CompilerInstance &CI, StringRef InFile);
+
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override;
 
@@ -147,14 +150,27 @@ class GenerateModuleFromModuleMapAction : public GenerateModuleAction {
   CreateOutputFile(CompilerInstance &CI, StringRef InFile) override;
 };
 
+/// Generates fatBMI (which contains full information to generate the object
+/// files) for C++20 Named Modules. Also generates the thin BMI (only contains
+/// necessary information for importers) if `-fthinBMI-output=`.
 class GenerateModuleInterfaceAction : public GenerateModuleAction {
-private:
+protected:
   bool BeginSourceFileAction(CompilerInstance &CI) override;
 
+  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
+                                                 StringRef InFile) override;
+
   std::unique_ptr<raw_pwrite_stream>
   CreateOutputFile(CompilerInstance &CI, StringRef InFile) override;
 };
 
+/// Only generates the thin BMI. This action is mainly used by tests.
+class GenerateThinModuleInterfaceAction : public GenerateModuleInterfaceAction {
+private:
+  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
+                                                 StringRef InFile) override;
+};
+
 class GenerateHeaderUnitAction : public GenerateModuleAction {
 
 private:
@@ -174,28 +190,50 @@ class SyntaxOnlyAction : public ASTFrontendAction {
   bool hasCodeCompletionSupport() const override { return true; }
 };
 
-/// Dump information about the given module file, to be used for
-/// basic debugging and discovery.
-class DumpModuleInfoAction : public ASTFrontendAction {
+// Base action for dumping module informations.
+class DumpModuleInfoActionBase : public ASTFrontendAction {
   // Allow other tools (ex lldb) to direct output for their use.
   std::shared_ptr<llvm::raw_ostream> OutputStream;
 
 protected:
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override;
   bool BeginInvocation(CompilerInstance &CI) override;
-  void ExecuteAction() override;
+  // Setup the output file.
+  llvm::raw_ostream &getOutputStream();
 
 public:
-  DumpModuleInfoAction() = default;
-  explicit DumpModuleInfoAction(std::shared_ptr<llvm::raw_ostream> Out)
+  DumpModuleInfoActionBase() = default;
+  explicit DumpModuleInfoActionBase(std::shared_ptr<llvm::raw_ostream> Out)
       : OutputStream(Out) {}
   bool hasPCHSupport() const override { return false; }
   bool hasASTFileSupport() const override { return true; }
   bool hasIRSupport() const override { return false; }
   bool hasCodeCompletionSupport() const override { return false; }
 };
 
+/// Dump information about the given module file, to be used for
+/// basic debugging and discovery.
+class DumpModuleInfoAction : public DumpModuleInfoActionBase {
+  void ExecuteAction() override;
+
+public:
+  DumpModuleInfoAction() = default;
+  explicit DumpModuleInfoAction(std::shared_ptr<llvm::raw_ostream> Out)
+      : DumpModuleInfoActionBase(Out) {}
+};
+
+/// Get the modules decl hash value action. The information is contained by
+/// DumpModuleInfoAction too. But this should be much faster.
+class GetModuleDeclsHashAction : public DumpModuleInfoActionBase {
+  void ExecuteAction() override;
+
+public:
+  GetModuleDeclsHashAction() = default;
+  explicit GetModuleDeclsHashAction(std::shared_ptr<llvm::raw_ostream> Out)
+      : DumpModuleInfoActionBase(Out) {}
+};
+
 class VerifyPCHAction : public ASTFrontendAction {
 protected:
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,

clang/include/clang/Frontend/FrontendOptions.h

@@ -85,9 +85,13 @@ enum ActionKind {
   /// Generate pre-compiled module from a module map.
   GenerateModule,
 
-  /// Generate pre-compiled module from a C++ module interface file.
+  /// Generate pre-compiled module from a standard C++ module interface unit.
   GenerateModuleInterface,
 
+  /// Generate reduced module interface for a standard C++ module interface
+  /// unit.
+  GenerateThinModuleInterface,
+
   /// Generate a C++20 header unit module from a header file.
   GenerateHeaderUnit,
 
@@ -103,6 +107,9 @@ enum ActionKind {
   /// Dump information about a module file.
   ModuleFileInfo,
 
+  /// Get BMI Decls Hash about a module file.
+  GetBMIDeclsHash,
+
   /// Load and verify that a PCH file is usable.
   VerifyPCH,
 
@@ -549,6 +556,9 @@ class FrontendOptions {
   /// Path which stores the output files for -ftime-trace
   std::string TimeTracePath;
 
+  /// Path to the thin BMI for -fthinbmi-output=
+  std::string ThinBMIPath;
+
 public:
   FrontendOptions()
       : DisableFree(false), RelocatablePCH(false), ShowHelp(false),

clang/include/clang/Serialization/ASTBitCodes.h

@@ -695,6 +695,9 @@ enum ASTRecordTypes {
   /// Record code for an unterminated \#pragma clang assume_nonnull begin
   /// recorded in a preamble.
   PP_ASSUME_NONNULL_LOC = 67,
+
+  /// Record code for the decls hash in the thin BMI.
+  BMI_DECLS_HASH = 68,
 };
 
 /// Record types used within a source manager block.

clang/include/clang/Serialization/ASTReader.h

@@ -937,6 +937,9 @@ class ASTReader
   /// Sema tracks these to emit deferred diags.
   llvm::SmallSetVector<serialization::DeclID, 4> DeclsToCheckForDeferredDiags;
 
+  /// The hash value of read C++20 thin BMI.
+  std::optional<uint64_t> ReadedBMIDeclsHash;
+
 private:
   struct ImportedSubmodule {
     serialization::SubmoduleID ID;
@@ -1794,6 +1797,13 @@ class ASTReader
                                   StringRef ExistingModuleCachePath,
                                   bool RequireStrictOptionMatches = false);
 
+  static std::optional<uint64_t> getBMIHash(StringRef Filename,
+                                            FileManager &FileMgr);
+
+  std::optional<uint64_t> getReadedBMIDeclsHash() const {
+    return ReadedBMIDeclsHash;
+  }
+
   /// Returns the suggested contents of the predefines buffer,
   /// which contains a (typically-empty) subset of the predefines
   /// build prior to including the precompiled header.

clang/include/clang/Serialization/ASTWriter.h

@@ -166,6 +166,15 @@ class ASTWriter : public ASTDeserializationListener,
   /// Indicates that the AST contained compiler errors.
   bool ASTHasCompilerErrors = false;
 
+  /// Indicates that we're going to generate the reduced BMI for C++20
+  /// named modules.
+  bool GeneratingThinBMI = false;
+
+  /// The hash for recorded decls for C++20 named modules. The parts of decls
+  /// which not affecting the ABI may not be recorded. e.g.,
+  /// the function body of a non-inline function.
+  llvm::hash_code BMIDeclsHash = 0;
+
   /// Mapping from input file entries to the index into the
   /// offset table where information about that input file is stored.
   llvm::DenseMap<const FileEntry *, uint32_t> InputFileIDs;
@@ -582,7 +591,8 @@ class ASTWriter : public ASTDeserializationListener,
   ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl<char> &Buffer,
             InMemoryModuleCache &ModuleCache,
             ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
-            bool IncludeTimestamps = true, bool BuildingImplicitModule = false);
+            bool IncludeTimestamps = true, bool BuildingImplicitModule = false,
+            bool GeneratingThinBMI = false);
   ~ASTWriter() override;
 
   ASTContext &getASTContext() const {
@@ -813,14 +823,22 @@ class PCHGenerator : public SemaConsumer {
   const ASTWriter &getWriter() const { return Writer; }
   SmallVectorImpl<char> &getPCH() const { return Buffer->Data; }
 
+  bool isComplete() const { return Buffer->IsComplete; }
+  PCHBuffer *getBufferPtr() { return Buffer.get(); }
+  StringRef getOutputFile() const { return OutputFile; }
+  DiagnosticsEngine &getDiagnostics() const {
+    return SemaPtr->getDiagnostics();
+  }
+
 public:
   PCHGenerator(const Preprocessor &PP, InMemoryModuleCache &ModuleCache,
                StringRef OutputFile, StringRef isysroot,
                std::shared_ptr<PCHBuffer> Buffer,
                ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
                bool AllowASTWithErrors = false, bool IncludeTimestamps = true,
                bool BuildingImplicitModule = false,
-               bool ShouldCacheASTInMemory = false);
+               bool ShouldCacheASTInMemory = false,
+               bool GeneratingThinBMI = false);
   ~PCHGenerator() override;
 
   void InitializeSema(Sema &S) override { SemaPtr = &S; }
@@ -830,6 +848,19 @@ class PCHGenerator : public SemaConsumer {
   bool hasEmittedPCH() const { return Buffer->IsComplete; }
 };
 
+class ThinBMIGenerator : public PCHGenerator {
+public:
+  ThinBMIGenerator(const Preprocessor &PP, InMemoryModuleCache &ModuleCache,
+                   StringRef OutputFile, std::shared_ptr<PCHBuffer> Buffer,
+                   bool IncludeTimestamps);
+
+  void HandleTranslationUnit(ASTContext &Ctx) override;
+};
+
+/// If the definition may impact the ABI. If yes, we're allowed to eliminate
+/// the definition of D in thin BMI.
+bool MayDefAffectABI(const Decl *D);
+
 /// A simple helper class to pack several bits in order into (a) 32 bit
 /// integer(s).
 class BitsPacker {

clang/lib/AST/ODRHash.cpp

@@ -1249,3 +1249,5 @@ void ODRHash::AddQualType(QualType T) {
 void ODRHash::AddBoolean(bool Value) {
   Bools.push_back(Value);
 }
+
+void ODRHash::AddInteger(unsigned Value) { ID.AddInteger(Value); }

clang/lib/Driver/Driver.cpp

@@ -354,6 +354,7 @@ phases::ID Driver::getFinalPhase(const DerivedArgList &DAL,
   } else if ((PhaseArg = DAL.getLastArg(options::OPT_fsyntax_only)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_print_supported_cpus)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_module_file_info)) ||
+             (PhaseArg = DAL.getLastArg(options::OPT_get_bmi_decls_hash)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_verify_pch)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_rewrite_objc)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_rewrite_legacy_objc)) ||
@@ -4086,6 +4087,13 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
     }
   }
 
+  // Diagnose misuse of -fthinBMI-output. It should be an error if we specify
+  // -fthinBMI-output with multiple precompilation jobs. Here we didn't check if
+  // there are multiple module units in the inputs.
+  if (C.getArgs().getLastArg(options::OPT_fthinBMI_output_EQ) &&
+      Inputs.size() > 1)
+    Diag(clang::diag::err_drv_thin_bmi_output_argument_with_multiple_files);
+
   handleArguments(C, Args, Inputs, Actions);
 
   bool UseNewOffloadingDriver =
@@ -4729,7 +4737,8 @@ Action *Driver::ConstructPhaseAction(
       return C.MakeAction<MigrateJobAction>(Input, types::TY_Remap);
     if (Args.hasArg(options::OPT_emit_ast))
       return C.MakeAction<CompileJobAction>(Input, types::TY_AST);
-    if (Args.hasArg(options::OPT_module_file_info))
+    if (Args.hasArg(options::OPT_module_file_info) ||
+        Args.hasArg(options::OPT_get_bmi_decls_hash))
       return C.MakeAction<CompileJobAction>(Input, types::TY_ModuleFile);
     if (Args.hasArg(options::OPT_verify_pch))
       return C.MakeAction<VerifyPCHJobAction>(Input, types::TY_Nothing);
@@ -5811,7 +5820,8 @@ const char *Driver::GetNamedOutputPath(Compilation &C, const JobAction &JA,
   }
 
   if (JA.getType() == types::TY_ModuleFile &&
-      C.getArgs().getLastArg(options::OPT_module_file_info)) {
+      (C.getArgs().getLastArg(options::OPT_module_file_info) ||
+       C.getArgs().getLastArg(options::OPT_get_bmi_decls_hash))) {
     return "-";
   }
 

clang/lib/Driver/ToolChains/Clang.cpp

@@ -3940,6 +3940,8 @@ static bool RenderModulesOptions(Compilation &C, const Driver &D,
   Args.ClaimAllArgs(options::OPT_fmodule_output);
   Args.ClaimAllArgs(options::OPT_fmodule_output_EQ);
 
+  Args.AddLastArg(CmdArgs, options::OPT_fthinBMI_output_EQ);
+
   return HaveModules;
 }
 
@@ -4958,7 +4960,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     } else if (JA.getType() == types::TY_AST) {
       CmdArgs.push_back("-emit-pch");
     } else if (JA.getType() == types::TY_ModuleFile) {
-      CmdArgs.push_back("-module-file-info");
+      if (Args.hasArg(options::OPT_get_bmi_decls_hash))
+        CmdArgs.push_back("-get-bmi-decls-hash");
+      else
+        CmdArgs.push_back("-module-file-info");
     } else if (JA.getType() == types::TY_RewrittenObjC) {
       CmdArgs.push_back("-rewrite-objc");
       rewriteKind = RK_NonFragile;

clang/lib/Frontend/CompilerInvocation.cpp

@@ -2554,12 +2554,14 @@ static const auto &getFrontendActionTable() {
 
       {frontend::GenerateModule, OPT_emit_module},
       {frontend::GenerateModuleInterface, OPT_emit_module_interface},
+      {frontend::GenerateThinModuleInterface, OPT_emit_thin_module_interface},
       {frontend::GenerateHeaderUnit, OPT_emit_header_unit},
       {frontend::GeneratePCH, OPT_emit_pch},
       {frontend::GenerateInterfaceStubs, OPT_emit_interface_stubs},
       {frontend::InitOnly, OPT_init_only},
       {frontend::ParseSyntaxOnly, OPT_fsyntax_only},
       {frontend::ModuleFileInfo, OPT_module_file_info},
+      {frontend::GetBMIDeclsHash, OPT_get_bmi_decls_hash},
       {frontend::VerifyPCH, OPT_verify_pch},
       {frontend::PrintPreamble, OPT_print_preamble},
       {frontend::PrintPreprocessedInput, OPT_E},
@@ -4236,11 +4238,13 @@ static bool isStrictlyPreprocessorAction(frontend::ActionKind Action) {
   case frontend::FixIt:
   case frontend::GenerateModule:
   case frontend::GenerateModuleInterface:
+  case frontend::GenerateThinModuleInterface:
   case frontend::GenerateHeaderUnit:
   case frontend::GeneratePCH:
   case frontend::GenerateInterfaceStubs:
   case frontend::ParseSyntaxOnly:
   case frontend::ModuleFileInfo:
+  case frontend::GetBMIDeclsHash:
   case frontend::VerifyPCH:
   case frontend::PluginAction:
   case frontend::RewriteObjC: