Virtualize swift compiler outputs

include/swift/APIDigester/ModuleAnalyzerNodes.h

@@ -784,8 +784,8 @@ class SwiftDeclCollector: public VisibleDeclConsumer {
   void deSerialize(StringRef Filename);
 
   // Serialize the content of all roots to a given file using JSON format.
-  void serialize(StringRef Filename);
-  static void serialize(StringRef Filename, SDKNode *Root, PayLoad otherInfo);
+  void serialize(llvm::raw_ostream &os);
+  static void serialize(llvm::raw_ostream &os, SDKNode *Root, PayLoad otherInfo);
 
   // After collecting decls, either from imported modules or from a previously
   // serialized JSON file, using this function to get the root of the SDK.
@@ -835,14 +835,15 @@ SDKNodeRoot *getSDKNodeRoot(SDKContext &SDKCtx,
 
 SDKNodeRoot *getEmptySDKNodeRoot(SDKContext &SDKCtx);
 
-void dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, StringRef OutputFile);
-void dumpSDKRoot(SDKNodeRoot *Root, StringRef OutputFile);
+void dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, llvm::raw_ostream &os);
+void dumpSDKRoot(SDKNodeRoot *Root, llvm::raw_ostream &os);
 
 int dumpSDKContent(const CompilerInvocation &InitInvoke,
                    const llvm::StringSet<> &ModuleNames,
-                   StringRef OutputFile, CheckerOptions Opts);
+                   llvm::raw_ostream &os, CheckerOptions Opts);
 
-void dumpModuleContent(ModuleDecl *MD, StringRef OutputFile, bool ABI, bool Empty);
+void dumpModuleContent(ModuleDecl *MD, llvm::raw_ostream &os, bool ABI,
+                       bool Empty);
 
 /// Mostly for testing purposes, this function de-serializes the SDK dump in
 /// dumpPath and re-serialize them to OutputPath. If the tool performs correctly,

include/swift/AST/ASTContext.h

@@ -44,6 +44,7 @@
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/VirtualOutputBackend.h"
 #include <functional>
 #include <memory>
 #include <utility>
@@ -231,6 +232,7 @@ class ASTContext final {
       ClangImporterOptions &ClangImporterOpts,
       symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts,
       SourceManager &SourceMgr, DiagnosticEngine &Diags,
+      llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> OutBackend = nullptr,
       std::function<bool(llvm::StringRef, bool)> PreModuleImportCallback = {});
 
 public:
@@ -248,6 +250,7 @@ class ASTContext final {
       ClangImporterOptions &ClangImporterOpts,
       symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts,
       SourceManager &SourceMgr, DiagnosticEngine &Diags,
+      llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> OutBackend = nullptr,
       std::function<bool(llvm::StringRef, bool)> PreModuleImportCallback = {});
   ~ASTContext();
 
@@ -281,6 +284,9 @@ class ASTContext final {
   /// Diags - The diagnostics engine.
   DiagnosticEngine &Diags;
 
+  /// OutputBackend for writing outputs.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> OutputBackend;
+
   /// If the shared pointer is not a \c nullptr and the pointee is \c true,
   /// all operations working on this ASTContext should be aborted at the next
   /// possible opportunity.
@@ -1516,6 +1522,18 @@ class ASTContext final {
 
   const llvm::StringSet<> &getLoadedPluginLibraryPaths() const;
 
+  /// Get the output backend. The output backend needs to be initialized via
+  /// constructor or `setOutputBackend`.
+  llvm::vfs::OutputBackend &getOutputBackend() const {
+    assert(OutputBackend && "OutputBackend is not setup");
+    return *OutputBackend;
+  }
+  /// Set output backend for virtualized outputs.
+  void setOutputBackend(
+      llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> OutBackend) {
+    OutputBackend = std::move(OutBackend);
+  }
+
 private:
   friend Decl;
 

include/swift/AST/AbstractSourceFileDepGraphFactory.h

@@ -16,6 +16,7 @@
 #include "swift/AST/Decl.h"
 #include "swift/AST/DeclContext.h"
 #include "swift/AST/FineGrainedDependencies.h"
+#include "llvm/Support/VirtualOutputBackend.h"
 
 namespace swift {
 class DiagnosticEngine;
@@ -39,6 +40,9 @@ class AbstractSourceFileDepGraphFactory {
 
   DiagnosticEngine &diags;
 
+  /// OutputBackend.
+  llvm::vfs::OutputBackend &backend;
+
   /// Graph under construction
   SourceFileDepGraph g;
 
@@ -49,7 +53,8 @@ class AbstractSourceFileDepGraphFactory {
                                     StringRef swiftDeps,
                                     Fingerprint fileFingerprint,
                                     bool emitDotFileAfterConstruction,
-                                    DiagnosticEngine &diags);
+                                    DiagnosticEngine &diags,
+                                    llvm::vfs::OutputBackend &outputBackend);
 
   virtual ~AbstractSourceFileDepGraphFactory() = default;
 

include/swift/AST/DiagnosticsCommon.def

@@ -29,6 +29,9 @@ ERROR(not_implemented,none,
 ERROR(error_opening_output,none,
       "error opening '%0' for output: %1", (StringRef, StringRef))
 
+ERROR(error_closing_output,none,
+      "error closing '%0' for output: %1", (StringRef, StringRef))
+
 ERROR(cannot_find_group_info_file,none,
       "cannot find group info file at path: '%0'", (StringRef))
 

include/swift/AST/DiagnosticsFrontend.def

@@ -475,6 +475,16 @@ REMARK(interface_file_backup_used,none,
 
 WARNING(warn_flag_deprecated,none, "flag '%0' is deprecated", (StringRef))
 
+// Output deterministic check
+ERROR(error_nondeterministic_output,none,
+      "output file '%0' is not deterministic: hash value '%1' vs '%2' between two compilations",
+      (StringRef, StringRef, StringRef))
+ERROR(error_output_missing,none,
+      "output file '%0' is missing from %select{first|second}1 compilation for deterministic check",
+      (StringRef, /*SecondRun=*/bool))
+REMARK(matching_output_produced,none,
+      "produced matching output file '%0' for deterministic check: hash '%1'", (StringRef, StringRef))
+
 // Dependency Verifier Diagnostics
 ERROR(missing_member_dependency,none,
       "expected "

include/swift/AST/FileSystem.h

@@ -16,29 +16,42 @@
 #include "swift/Basic/FileSystem.h"
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/DiagnosticsCommon.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputConfig.h"
 
 namespace swift {
-
-/// A wrapper around swift::atomicallyWritingToFile that handles diagnosing any
-/// filesystem errors and asserts the output path is nonempty.
+/// A wrapper around llvm::vfs::OutputBackend to handle diagnosing any file
+/// system errors during output creation.
 ///
 /// \returns true if there were any errors, either from the filesystem
 /// operations or from \p action returning true.
 inline bool
-withOutputFile(DiagnosticEngine &diags, StringRef outputPath,
+withOutputPath(DiagnosticEngine &diags, llvm::vfs::OutputBackend &Backend,
+               StringRef outputPath,
                llvm::function_ref<bool(llvm::raw_pwrite_stream &)> action) {
   assert(!outputPath.empty());
+  llvm::vfs::OutputConfig config;
+  config.setAtomicWrite().setOnlyIfDifferent();
 
-  bool actionFailed = false;
-  std::error_code EC = swift::atomicallyWritingToFile(
-      outputPath,
-      [&](llvm::raw_pwrite_stream &out) { actionFailed = action(out); });
-  if (EC) {
+  auto outputFile = Backend.createFile(outputPath, config);
+  if (!outputFile) {
     diags.diagnose(SourceLoc(), diag::error_opening_output, outputPath,
-                   EC.message());
+                   toString(outputFile.takeError()));
+    return true;
+  }
+
+  bool failed = action(*outputFile);
+  // If there is an error, discard output. Otherwise keep the output file.
+  if (auto error = failed ? outputFile->discard() : outputFile->keep()) {
+    // Don't diagnose discard error.
+    if (failed)
+      consumeError(std::move(error));
+    else
+      diags.diagnose(SourceLoc(), diag::error_closing_output, outputPath,
+                     toString(std::move(error)));
     return true;
   }
-  return actionFailed;
+  return failed;
 }
 } // end namespace swift
 

include/swift/AST/FineGrainedDependencies.h

@@ -24,6 +24,7 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/VirtualOutputBackend.h"
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
@@ -357,8 +358,9 @@ class BiIndexedTwoStageMap {
 /// \Note The returned graph should not be escaped from the callback.
 bool withReferenceDependencies(
     llvm::PointerUnion<const ModuleDecl *, const SourceFile *> MSF,
-    const DependencyTracker &depTracker, StringRef outputPath,
-    bool alsoEmitDotFile, llvm::function_ref<bool(SourceFileDepGraph &&)>);
+    const DependencyTracker &depTracker, llvm::vfs::OutputBackend &backend,
+    StringRef outputPath, bool alsoEmitDotFile,
+    llvm::function_ref<bool(SourceFileDepGraph &&)>);
 
 //==============================================================================
 // MARK: Enums
@@ -895,7 +897,8 @@ class SourceFileDepGraph {
 
   bool verifySequenceNumber() const;
 
-  void emitDotFile(StringRef outputPath, DiagnosticEngine &diags);
+  void emitDotFile(llvm::vfs::OutputBackend &outputBackend,
+                   StringRef outputPath, DiagnosticEngine &diags);
 
   void addNode(SourceFileDepGraphNode *n) {
     n->setSequenceNumber(allNodes.size());

include/swift/AST/FineGrainedDependencyFormat.h

@@ -18,6 +18,9 @@
 
 namespace llvm {
 class MemoryBuffer;
+namespace vfs {
+class OutputBackend;
+}
 }
 
 namespace swift {
@@ -132,6 +135,7 @@ bool readFineGrainedDependencyGraph(llvm::StringRef path,
 /// Tries to write the dependency graph to the given path name.
 /// Returns true if there was an error.
 bool writeFineGrainedDependencyGraphToPath(DiagnosticEngine &diags,
+                                           llvm::vfs::OutputBackend &backend,
                                            llvm::StringRef path,
                                            const SourceFileDepGraph &g);
 

include/swift/AST/IRGenOptions.h

@@ -318,6 +318,9 @@ class IRGenOptions {
   /// Print the LLVM inline tree at the end of the LLVM pass pipeline.
   unsigned PrintInlineTree : 1;
 
+  /// Always recompile the output even if the module hash might match.
+  unsigned AlwaysCompile : 1;
+
   /// Whether we should embed the bitcode file.
   IRGenEmbedMode EmbedMode : 2;
 
@@ -481,7 +484,7 @@ class IRGenOptions {
         DisableLLVMOptzns(false), DisableSwiftSpecificLLVMOptzns(false),
         Playground(false),
         EmitStackPromotionChecks(false), UseSingleModuleLLVMEmission(false),
-        FunctionSections(false), PrintInlineTree(false),
+        FunctionSections(false), PrintInlineTree(false), AlwaysCompile(false),
         EmbedMode(IRGenEmbedMode::None), LLVMLTOKind(IRGenLLVMLTOKind::None),
         SwiftAsyncFramePointer(SwiftAsyncFramePointerKind::Auto),
         HasValueNamesSetting(false), ValueNames(false),

include/swift/AST/ModuleLoader.h

@@ -32,6 +32,9 @@
 
 namespace llvm {
 class FileCollectorBase;
+namespace vfs {
+class OutputBackend;
+}
 }
 
 namespace clang {
@@ -156,6 +159,7 @@ class ModuleInterfaceChecker {
   virtual bool tryEmitForwardingModule(StringRef moduleName,
                                StringRef interfacePath,
                                ArrayRef<std::string> candidates,
+                               llvm::vfs::OutputBackend &backend,
                                StringRef outPath) = 0;
   virtual ~ModuleInterfaceChecker() = default;
 };

include/swift/ClangImporter/ClangImporter.h

@@ -28,6 +28,7 @@ namespace llvm {
   template<typename Fn> class function_ref;
   namespace vfs {
     class FileSystem;
+    class OutputBackend;
   }
 }
 
@@ -394,8 +395,7 @@ class ClangImporter final : public ClangModuleLoader {
   /// replica.
   ///
   /// \sa clang::GeneratePCHAction
-  bool emitBridgingPCH(StringRef headerPath,
-                       StringRef outputPCHPath);
+  bool emitBridgingPCH(StringRef headerPath, StringRef outputPCHPath);
 
   /// Returns true if a clang CompilerInstance can successfully read in a PCH,
   /// assuming it exists, with the current options. This can be used to find out

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -18,6 +18,9 @@
 
 namespace llvm {
 class MemoryBuffer;
+namespace vfs{
+class OutputBackend;
+}
 }
 
 namespace swift {
@@ -187,6 +190,7 @@ bool readInterModuleDependenciesCache(llvm::StringRef path,
 /// Tries to write the dependency graph to the given path name.
 /// Returns true if there was an error.
 bool writeInterModuleDependenciesCache(DiagnosticEngine &diags,
+                                       llvm::vfs::OutputBackend &backend,
                                        llvm::StringRef path,
                                        const SwiftDependencyScanningService &cache);
 

include/swift/Driver/FineGrainedDependencyDriverGraph.h

@@ -20,11 +20,14 @@
 #include "swift/Basic/OptionSet.h"
 #include "swift/Driver/Job.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputBackends.h"
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -201,6 +204,9 @@ class ModuleDepGraph {
                           std::vector<const ModuleDepGraphNode *>>
       dependencyPathsToJobs;
 
+  /// VirtualOutputBackend for emitting graphs.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::OutputBackend> backend;
+
   /// For helping with performance tuning, may be null:
   UnifiedStatsReporter *stats;
 
@@ -306,6 +312,10 @@ class ModuleDepGraph {
                 : None),
         stats(stats) {
     assert(verify() && "ModuleDepGraph should be fine when created");
+
+    // Create a OnDiskOutputBackend for emitting graphs. Currently, this is
+    // only used by driver so the backend is not shared with a CompilerInstance.
+    backend = llvm::makeIntrusiveRefCnt<llvm::vfs::OnDiskOutputBackend>();
   }
 
   /// For unit tests.