From 44d71ab72f9529668e19ca6a8906f33850baed0a Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 5 Apr 2023 23:34:37 +0800 Subject: [PATCH] Virtualize swift compiler outputs (#63206) Using a virutal output backend to capture all the outputs from swift-frontend invocation. This allows redirecting and/or mirroring compiler outputs to multiple location using different OutputBackend. As an example usage for the virtual outputs, teach swift compiler to check its output determinism by running the compiler invocation twice and compare the hash of all its outputs. Virtual output will be used to enable caching in the future. --- .../swift/APIDigester/ModuleAnalyzerNodes.h | 13 +- include/swift/AST/ASTContext.h | 18 ++ .../AST/AbstractSourceFileDepGraphFactory.h | 7 +- include/swift/AST/DiagnosticsCommon.def | 3 + include/swift/AST/DiagnosticsFrontend.def | 10 ++ include/swift/AST/FileSystem.h | 35 ++-- include/swift/AST/FineGrainedDependencies.h | 9 +- .../swift/AST/FineGrainedDependencyFormat.h | 4 + include/swift/AST/IRGenOptions.h | 5 +- include/swift/AST/ModuleLoader.h | 4 + include/swift/ClangImporter/ClangImporter.h | 4 +- .../SerializedModuleDependencyCacheFormat.h | 4 + .../Driver/FineGrainedDependencyDriverGraph.h | 10 ++ include/swift/Frontend/Frontend.h | 21 +++ include/swift/Frontend/FrontendOptions.h | 3 + .../swift/Frontend/ModuleInterfaceLoader.h | 1 + include/swift/Option/FrontendOptions.td | 7 + include/swift/Subsystems.h | 5 + lib/APIDigester/ModuleAnalyzerNodes.cpp | 29 ++-- lib/AST/ASTContext.cpp | 19 +- lib/AST/AbstractSourceFileDepGraphFactory.cpp | 13 +- lib/AST/FineGrainedDependencies.cpp | 12 +- lib/AST/FineGrainedDependencyFormat.cpp | 5 +- lib/AST/FrontendSourceFileDepGraphFactory.cpp | 26 +-- lib/AST/FrontendSourceFileDepGraphFactory.h | 8 +- lib/ClangImporter/ClangImporter.cpp | 30 ++-- lib/DependencyScan/DependencyScanningTool.cpp | 4 +- .../ModuleDependencyCacheSerialization.cpp | 7 +- lib/DependencyScan/ScanDependencies.cpp | 66 ++++--- .../FineGrainedDependencyDriverGraph.cpp | 6 +- lib/DriverTool/modulewrap_main.cpp | 5 +- lib/DriverTool/swift_api_digester_main.cpp | 30 +++- .../ArgsToFrontendOptionsConverter.cpp | 1 + lib/Frontend/CompilerInvocation.cpp | 1 + lib/Frontend/Frontend.cpp | 24 ++- lib/Frontend/ModuleInterfaceBuilder.cpp | 3 +- lib/Frontend/ModuleInterfaceLoader.cpp | 70 ++++---- lib/Frontend/Serialization.cpp | 101 ++++++----- lib/FrontendTool/Dependencies.h | 12 +- lib/FrontendTool/FrontendTool.cpp | 163 ++++++++++++------ lib/FrontendTool/ImportedModules.cpp | 24 ++- lib/FrontendTool/MakeStyleDependencies.cpp | 21 ++- lib/IRGen/IRGen.cpp | 74 ++++---- lib/IRGen/IRGenModule.cpp | 1 - lib/Immediate/Immediate.cpp | 3 +- lib/SymbolGraphGen/SymbolGraphGen.cpp | 12 +- test/Frontend/output_determinism_check.swift | 40 +++++ tools/sil-llvm-gen/SILLLVMGen.cpp | 27 +-- .../swift-dependency-tool.cpp | 6 +- .../ClangImporter/ClangImporterTests.cpp | 2 + .../MockingFineGrainedDependencyGraphs.cpp | 4 +- .../UnitTestSourceFileDepGraphFactory.h | 5 +- 52 files changed, 690 insertions(+), 327 deletions(-) create mode 100644 test/Frontend/output_determinism_check.swift diff --git a/include/swift/APIDigester/ModuleAnalyzerNodes.h b/include/swift/APIDigester/ModuleAnalyzerNodes.h index 367763d5f4870..13b54ac41a405 100644 --- a/include/swift/APIDigester/ModuleAnalyzerNodes.h +++ b/include/swift/APIDigester/ModuleAnalyzerNodes.h @@ -784,8 +784,8 @@ class SwiftDeclCollector: public VisibleDeclConsumer { void deSerialize(StringRef Filename); // Serialize the content of all roots to a given file using JSON format. - void serialize(StringRef Filename); - static void serialize(StringRef Filename, SDKNode *Root, PayLoad otherInfo); + void serialize(llvm::raw_ostream &os); + static void serialize(llvm::raw_ostream &os, SDKNode *Root, PayLoad otherInfo); // After collecting decls, either from imported modules or from a previously // serialized JSON file, using this function to get the root of the SDK. @@ -835,14 +835,15 @@ SDKNodeRoot *getSDKNodeRoot(SDKContext &SDKCtx, SDKNodeRoot *getEmptySDKNodeRoot(SDKContext &SDKCtx); -void dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, StringRef OutputFile); -void dumpSDKRoot(SDKNodeRoot *Root, StringRef OutputFile); +void dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, llvm::raw_ostream &os); +void dumpSDKRoot(SDKNodeRoot *Root, llvm::raw_ostream &os); int dumpSDKContent(const CompilerInvocation &InitInvoke, const llvm::StringSet<> &ModuleNames, - StringRef OutputFile, CheckerOptions Opts); + llvm::raw_ostream &os, CheckerOptions Opts); -void dumpModuleContent(ModuleDecl *MD, StringRef OutputFile, bool ABI, bool Empty); +void dumpModuleContent(ModuleDecl *MD, llvm::raw_ostream &os, bool ABI, + bool Empty); /// Mostly for testing purposes, this function de-serializes the SDK dump in /// dumpPath and re-serialize them to OutputPath. If the tool performs correctly, diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h index 3c763b23a1e49..341a115155748 100644 --- a/include/swift/AST/ASTContext.h +++ b/include/swift/AST/ASTContext.h @@ -44,6 +44,7 @@ #include "llvm/ADT/TinyPtrVector.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/DataTypes.h" +#include "llvm/Support/VirtualOutputBackend.h" #include #include #include @@ -231,6 +232,7 @@ class ASTContext final { ClangImporterOptions &ClangImporterOpts, symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts, SourceManager &SourceMgr, DiagnosticEngine &Diags, + llvm::IntrusiveRefCntPtr OutBackend = nullptr, std::function PreModuleImportCallback = {}); public: @@ -248,6 +250,7 @@ class ASTContext final { ClangImporterOptions &ClangImporterOpts, symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts, SourceManager &SourceMgr, DiagnosticEngine &Diags, + llvm::IntrusiveRefCntPtr OutBackend = nullptr, std::function PreModuleImportCallback = {}); ~ASTContext(); @@ -281,6 +284,9 @@ class ASTContext final { /// Diags - The diagnostics engine. DiagnosticEngine &Diags; + /// OutputBackend for writing outputs. + llvm::IntrusiveRefCntPtr OutputBackend; + /// If the shared pointer is not a \c nullptr and the pointee is \c true, /// all operations working on this ASTContext should be aborted at the next /// possible opportunity. @@ -1516,6 +1522,18 @@ class ASTContext final { const llvm::StringSet<> &getLoadedPluginLibraryPaths() const; + /// Get the output backend. The output backend needs to be initialized via + /// constructor or `setOutputBackend`. + llvm::vfs::OutputBackend &getOutputBackend() const { + assert(OutputBackend && "OutputBackend is not setup"); + return *OutputBackend; + } + /// Set output backend for virtualized outputs. + void setOutputBackend( + llvm::IntrusiveRefCntPtr OutBackend) { + OutputBackend = std::move(OutBackend); + } + private: friend Decl; diff --git a/include/swift/AST/AbstractSourceFileDepGraphFactory.h b/include/swift/AST/AbstractSourceFileDepGraphFactory.h index 6858627dfb5cf..1f44f3f3106cd 100644 --- a/include/swift/AST/AbstractSourceFileDepGraphFactory.h +++ b/include/swift/AST/AbstractSourceFileDepGraphFactory.h @@ -16,6 +16,7 @@ #include "swift/AST/Decl.h" #include "swift/AST/DeclContext.h" #include "swift/AST/FineGrainedDependencies.h" +#include "llvm/Support/VirtualOutputBackend.h" namespace swift { class DiagnosticEngine; @@ -39,6 +40,9 @@ class AbstractSourceFileDepGraphFactory { DiagnosticEngine &diags; + /// OutputBackend. + llvm::vfs::OutputBackend &backend; + /// Graph under construction SourceFileDepGraph g; @@ -49,7 +53,8 @@ class AbstractSourceFileDepGraphFactory { StringRef swiftDeps, Fingerprint fileFingerprint, bool emitDotFileAfterConstruction, - DiagnosticEngine &diags); + DiagnosticEngine &diags, + llvm::vfs::OutputBackend &outputBackend); virtual ~AbstractSourceFileDepGraphFactory() = default; diff --git a/include/swift/AST/DiagnosticsCommon.def b/include/swift/AST/DiagnosticsCommon.def index 535dbd475eabb..f438e7a93b31e 100644 --- a/include/swift/AST/DiagnosticsCommon.def +++ b/include/swift/AST/DiagnosticsCommon.def @@ -29,6 +29,9 @@ ERROR(not_implemented,none, ERROR(error_opening_output,none, "error opening '%0' for output: %1", (StringRef, StringRef)) +ERROR(error_closing_output,none, + "error closing '%0' for output: %1", (StringRef, StringRef)) + ERROR(cannot_find_group_info_file,none, "cannot find group info file at path: '%0'", (StringRef)) diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def index c7316754251f0..4c51234e50b56 100644 --- a/include/swift/AST/DiagnosticsFrontend.def +++ b/include/swift/AST/DiagnosticsFrontend.def @@ -478,6 +478,16 @@ REMARK(interface_file_backup_used,none, WARNING(warn_flag_deprecated,none, "flag '%0' is deprecated", (StringRef)) +// Output deterministic check +ERROR(error_nondeterministic_output,none, + "output file '%0' is not deterministic: hash value '%1' vs '%2' between two compilations", + (StringRef, StringRef, StringRef)) +ERROR(error_output_missing,none, + "output file '%0' is missing from %select{first|second}1 compilation for deterministic check", + (StringRef, /*SecondRun=*/bool)) +REMARK(matching_output_produced,none, + "produced matching output file '%0' for deterministic check: hash '%1'", (StringRef, StringRef)) + // Dependency Verifier Diagnostics ERROR(missing_member_dependency,none, "expected " diff --git a/include/swift/AST/FileSystem.h b/include/swift/AST/FileSystem.h index 15641164c3e08..69f4c5dd7ffce 100644 --- a/include/swift/AST/FileSystem.h +++ b/include/swift/AST/FileSystem.h @@ -16,29 +16,42 @@ #include "swift/Basic/FileSystem.h" #include "swift/AST/DiagnosticEngine.h" #include "swift/AST/DiagnosticsCommon.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputConfig.h" namespace swift { - -/// A wrapper around swift::atomicallyWritingToFile that handles diagnosing any -/// filesystem errors and asserts the output path is nonempty. +/// A wrapper around llvm::vfs::OutputBackend to handle diagnosing any file +/// system errors during output creation. /// /// \returns true if there were any errors, either from the filesystem /// operations or from \p action returning true. inline bool -withOutputFile(DiagnosticEngine &diags, StringRef outputPath, +withOutputPath(DiagnosticEngine &diags, llvm::vfs::OutputBackend &Backend, + StringRef outputPath, llvm::function_ref action) { assert(!outputPath.empty()); + llvm::vfs::OutputConfig config; + config.setAtomicWrite().setOnlyIfDifferent(); - bool actionFailed = false; - std::error_code EC = swift::atomicallyWritingToFile( - outputPath, - [&](llvm::raw_pwrite_stream &out) { actionFailed = action(out); }); - if (EC) { + auto outputFile = Backend.createFile(outputPath, config); + if (!outputFile) { diags.diagnose(SourceLoc(), diag::error_opening_output, outputPath, - EC.message()); + toString(outputFile.takeError())); + return true; + } + + bool failed = action(*outputFile); + // If there is an error, discard output. Otherwise keep the output file. + if (auto error = failed ? outputFile->discard() : outputFile->keep()) { + // Don't diagnose discard error. + if (failed) + consumeError(std::move(error)); + else + diags.diagnose(SourceLoc(), diag::error_closing_output, outputPath, + toString(std::move(error))); return true; } - return actionFailed; + return failed; } } // end namespace swift diff --git a/include/swift/AST/FineGrainedDependencies.h b/include/swift/AST/FineGrainedDependencies.h index 254d3a0d79002..ffac726152c42 100644 --- a/include/swift/AST/FineGrainedDependencies.h +++ b/include/swift/AST/FineGrainedDependencies.h @@ -24,6 +24,7 @@ #include "llvm/ADT/SetVector.h" #include "llvm/Support/MD5.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" #include "llvm/Support/raw_ostream.h" @@ -357,8 +358,9 @@ class BiIndexedTwoStageMap { /// \Note The returned graph should not be escaped from the callback. bool withReferenceDependencies( llvm::PointerUnion MSF, - const DependencyTracker &depTracker, StringRef outputPath, - bool alsoEmitDotFile, llvm::function_ref); + const DependencyTracker &depTracker, llvm::vfs::OutputBackend &backend, + StringRef outputPath, bool alsoEmitDotFile, + llvm::function_ref); //============================================================================== // MARK: Enums @@ -895,7 +897,8 @@ class SourceFileDepGraph { bool verifySequenceNumber() const; - void emitDotFile(StringRef outputPath, DiagnosticEngine &diags); + void emitDotFile(llvm::vfs::OutputBackend &outputBackend, + StringRef outputPath, DiagnosticEngine &diags); void addNode(SourceFileDepGraphNode *n) { n->setSequenceNumber(allNodes.size()); diff --git a/include/swift/AST/FineGrainedDependencyFormat.h b/include/swift/AST/FineGrainedDependencyFormat.h index 0957d03d98353..086de7e60b2c7 100644 --- a/include/swift/AST/FineGrainedDependencyFormat.h +++ b/include/swift/AST/FineGrainedDependencyFormat.h @@ -18,6 +18,9 @@ namespace llvm { class MemoryBuffer; +namespace vfs { +class OutputBackend; +} } namespace swift { @@ -132,6 +135,7 @@ bool readFineGrainedDependencyGraph(llvm::StringRef path, /// Tries to write the dependency graph to the given path name. /// Returns true if there was an error. bool writeFineGrainedDependencyGraphToPath(DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend, llvm::StringRef path, const SourceFileDepGraph &g); diff --git a/include/swift/AST/IRGenOptions.h b/include/swift/AST/IRGenOptions.h index 0c1e8a67c32eb..bfb6f6efd413a 100644 --- a/include/swift/AST/IRGenOptions.h +++ b/include/swift/AST/IRGenOptions.h @@ -318,6 +318,9 @@ class IRGenOptions { /// Print the LLVM inline tree at the end of the LLVM pass pipeline. unsigned PrintInlineTree : 1; + /// Always recompile the output even if the module hash might match. + unsigned AlwaysCompile : 1; + /// Whether we should embed the bitcode file. IRGenEmbedMode EmbedMode : 2; @@ -481,7 +484,7 @@ class IRGenOptions { DisableLLVMOptzns(false), DisableSwiftSpecificLLVMOptzns(false), Playground(false), EmitStackPromotionChecks(false), UseSingleModuleLLVMEmission(false), - FunctionSections(false), PrintInlineTree(false), + FunctionSections(false), PrintInlineTree(false), AlwaysCompile(false), EmbedMode(IRGenEmbedMode::None), LLVMLTOKind(IRGenLLVMLTOKind::None), SwiftAsyncFramePointer(SwiftAsyncFramePointerKind::Auto), HasValueNamesSetting(false), ValueNames(false), diff --git a/include/swift/AST/ModuleLoader.h b/include/swift/AST/ModuleLoader.h index 60ef5ac69693f..0072984634251 100644 --- a/include/swift/AST/ModuleLoader.h +++ b/include/swift/AST/ModuleLoader.h @@ -32,6 +32,9 @@ namespace llvm { class FileCollectorBase; +namespace vfs { +class OutputBackend; +} } namespace clang { @@ -156,6 +159,7 @@ class ModuleInterfaceChecker { virtual bool tryEmitForwardingModule(StringRef moduleName, StringRef interfacePath, ArrayRef candidates, + llvm::vfs::OutputBackend &backend, StringRef outPath) = 0; virtual ~ModuleInterfaceChecker() = default; }; diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h index 3bbeb8f4d1e2e..5ad232c279a6b 100644 --- a/include/swift/ClangImporter/ClangImporter.h +++ b/include/swift/ClangImporter/ClangImporter.h @@ -28,6 +28,7 @@ namespace llvm { template class function_ref; namespace vfs { class FileSystem; + class OutputBackend; } } @@ -394,8 +395,7 @@ class ClangImporter final : public ClangModuleLoader { /// replica. /// /// \sa clang::GeneratePCHAction - bool emitBridgingPCH(StringRef headerPath, - StringRef outputPCHPath); + bool emitBridgingPCH(StringRef headerPath, StringRef outputPCHPath); /// Returns true if a clang CompilerInstance can successfully read in a PCH, /// assuming it exists, with the current options. This can be used to find out diff --git a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h index a8fa5a316140b..254c985fbc85b 100644 --- a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h +++ b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h @@ -18,6 +18,9 @@ namespace llvm { class MemoryBuffer; +namespace vfs{ +class OutputBackend; +} } namespace swift { @@ -187,6 +190,7 @@ bool readInterModuleDependenciesCache(llvm::StringRef path, /// Tries to write the dependency graph to the given path name. /// Returns true if there was an error. bool writeInterModuleDependenciesCache(DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend, llvm::StringRef path, const SwiftDependencyScanningService &cache); diff --git a/include/swift/Driver/FineGrainedDependencyDriverGraph.h b/include/swift/Driver/FineGrainedDependencyDriverGraph.h index e2266ae03a13c..f36f0c269b65c 100644 --- a/include/swift/Driver/FineGrainedDependencyDriverGraph.h +++ b/include/swift/Driver/FineGrainedDependencyDriverGraph.h @@ -20,11 +20,14 @@ #include "swift/Basic/OptionSet.h" #include "swift/Driver/Job.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Support/Path.h" #include "llvm/Support/PointerLikeTypeTraits.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputBackends.h" #include #include #include @@ -201,6 +204,9 @@ class ModuleDepGraph { std::vector> dependencyPathsToJobs; + /// VirtualOutputBackend for emitting graphs. + llvm::IntrusiveRefCntPtr backend; + /// For helping with performance tuning, may be null: UnifiedStatsReporter *stats; @@ -306,6 +312,10 @@ class ModuleDepGraph { : None), stats(stats) { assert(verify() && "ModuleDepGraph should be fine when created"); + + // Create a OnDiskOutputBackend for emitting graphs. Currently, this is + // only used by driver so the backend is not shared with a CompilerInstance. + backend = llvm::makeIntrusiveRefCnt(); } /// For unit tests. diff --git a/include/swift/Frontend/Frontend.h b/include/swift/Frontend/Frontend.h index 30785440fa5fe..9669e98d8488b 100644 --- a/include/swift/Frontend/Frontend.h +++ b/include/swift/Frontend/Frontend.h @@ -45,8 +45,11 @@ #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/SetVector.h" #include "llvm/Option/ArgList.h" +#include "llvm/Support/BLAKE3.h" +#include "llvm/Support/HashingOutputBackend.h" #include "llvm/Support/Host.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackend.h" #include @@ -457,6 +460,13 @@ class CompilerInstance { /// instance has completed its setup, this will be null. std::unique_ptr Stats; + /// Virtual OutputBackend. + llvm::IntrusiveRefCntPtr OutputBackend = nullptr; + + /// The verification output backend. + using HashBackendTy = llvm::vfs::HashingOutputBackend; + llvm::IntrusiveRefCntPtr HashBackend; + mutable ModuleDecl *MainModule = nullptr; SerializedModuleLoaderBase *DefaultSerializedLoader = nullptr; MemoryBufferSerializedModuleLoader *MemoryBufferLoader = nullptr; @@ -507,6 +517,16 @@ class CompilerInstance { SourceMgr.setFileSystem(FS); } + llvm::vfs::OutputBackend &getOutputBackend() const { + return *OutputBackend; + } + void + setOutputBackend(llvm::IntrusiveRefCntPtr Backend) { + OutputBackend = std::move(Backend); + } + using HashingBackendPtrTy = llvm::IntrusiveRefCntPtr; + HashingBackendPtrTy getHashingBackend() { return HashBackend; } + ASTContext &getASTContext() { return *Context; } const ASTContext &getASTContext() const { return *Context; } @@ -629,6 +649,7 @@ class CompilerInstance { bool setUpASTContextIfNeeded(); void setupStatsReporter(); void setupDependencyTrackerIfNeeded(); + void setupOutputBackend(); /// \return false if successful, true on error. bool setupDiagnosticVerifierIfNeeded(); diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h index b1aa4e33db151..76fda8039434e 100644 --- a/include/swift/Frontend/FrontendOptions.h +++ b/include/swift/Frontend/FrontendOptions.h @@ -498,6 +498,9 @@ class FrontendOptions { /// to encode the actual paths into the .swiftmodule file. PathObfuscator serializedPathObfuscator; + /// Whether to run the job twice to check determinism. + bool DeterministicCheck = false; + /// Avoid printing actual module content into the ABI descriptor file. /// This should only be used as a workaround when emitting ABI descriptor files /// crashes the compiler. diff --git a/include/swift/Frontend/ModuleInterfaceLoader.h b/include/swift/Frontend/ModuleInterfaceLoader.h index 1f7a6d9fa1cdd..871cdb7cb313a 100644 --- a/include/swift/Frontend/ModuleInterfaceLoader.h +++ b/include/swift/Frontend/ModuleInterfaceLoader.h @@ -447,6 +447,7 @@ class ModuleInterfaceCheckerImpl: public ModuleInterfaceChecker { bool tryEmitForwardingModule(StringRef moduleName, StringRef interfacePath, ArrayRef candidates, + llvm::vfs::OutputBackend &backend, StringRef outPath) override; bool isCached(StringRef DepPath); }; diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index 0f2713531c53e..8a3ff3da4c9e2 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -1176,6 +1176,13 @@ def enable_emit_generic_class_ro_t_list : HelpText<"Enable emission of a section with references to class_ro_t of " "generic class patterns">; +def enable_deterministic_check : + Flag<["-"], "enable-deterministic-check">, + HelpText<"Check compiler output determinisim by running it twice">; +def always_compile_output_files : + Flag<["-"], "always-compile-output-files">, + HelpText<"Always compile output files even it might not change the results">; + def experimental_spi_only_imports : Flag<["-"], "experimental-spi-only-imports">, HelpText<"Enable use of @_spiOnly imports">; diff --git a/include/swift/Subsystems.h b/include/swift/Subsystems.h index dc3b11ac6210e..b6fcda2b7bd20 100644 --- a/include/swift/Subsystems.h +++ b/include/swift/Subsystems.h @@ -35,6 +35,9 @@ namespace llvm { class Module; class TargetOptions; class TargetMachine; + namespace vfs { + class OutputBackend; + } } namespace swift { @@ -276,6 +279,7 @@ namespace swift { /// \param Module LLVM module to code gen, required. /// \param TargetMachine target of code gen, required. /// \param OutputFilename Filename for output. + /// \param Backend OutputBackend for writing output. bool performLLVM(const IRGenOptions &Opts, DiagnosticEngine &Diags, llvm::sys::Mutex *DiagMutex, @@ -283,6 +287,7 @@ namespace swift { llvm::Module *Module, llvm::TargetMachine *TargetMachine, StringRef OutputFilename, + llvm::vfs::OutputBackend &Backend, UnifiedStatsReporter *Stats); /// Dump YAML describing all fixed-size types imported from the given module. diff --git a/lib/APIDigester/ModuleAnalyzerNodes.cpp b/lib/APIDigester/ModuleAnalyzerNodes.cpp index c3d8caacc2be9..090db50ab61d6 100644 --- a/lib/APIDigester/ModuleAnalyzerNodes.cpp +++ b/lib/APIDigester/ModuleAnalyzerNodes.cpp @@ -2470,11 +2470,10 @@ void SwiftDeclCollector::deSerialize(StringRef Filename) { } // Serialize the content of all roots to a given file using JSON format. -void SwiftDeclCollector::serialize(StringRef Filename, SDKNode *Root, +void SwiftDeclCollector::serialize(llvm::raw_ostream &os, SDKNode *Root, PayLoad OtherInfo) { std::error_code EC; - llvm::raw_fd_ostream fs(Filename, EC, llvm::sys::fs::OF_None); - json::Output yout(fs); + json::Output yout(os); assert(Root->getKind() == SDKNodeKind::Root); SDKNodeRoot &root = *static_cast(Root); yout.beginObject(); @@ -2486,8 +2485,8 @@ void SwiftDeclCollector::serialize(StringRef Filename, SDKNode *Root, } // Serialize the content of all roots to a given file using JSON format. -void SwiftDeclCollector::serialize(StringRef Filename) { - SwiftDeclCollector::serialize(Filename, RootNode, PayLoad()); +void SwiftDeclCollector::serialize(llvm::raw_ostream &os) { + SwiftDeclCollector::serialize(os, RootNode, PayLoad()); } SDKNodeRoot * @@ -2550,28 +2549,26 @@ swift::ide::api::getSDKNodeRoot(SDKContext &SDKCtx, } void swift::ide::api::dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, - StringRef OutputFile) { + llvm::raw_ostream &os) { assert(Root); auto Opts = Root->getSDKContext().getOpts(); if (Opts.Verbose) llvm::errs() << "Dumping SDK...\n"; - SwiftDeclCollector::serialize(OutputFile, Root, load); - if (Opts.Verbose) - llvm::errs() << "Dumped to "<< OutputFile << "\n"; + SwiftDeclCollector::serialize(os, Root, load); } -void swift::ide::api::dumpSDKRoot(SDKNodeRoot *Root, StringRef OutputFile) { - dumpSDKRoot(Root, PayLoad(), OutputFile); +void swift::ide::api::dumpSDKRoot(SDKNodeRoot *Root, llvm::raw_ostream &os) { + dumpSDKRoot(Root, PayLoad(), os); } int swift::ide::api::dumpSDKContent(const CompilerInvocation &InitInvoke, const llvm::StringSet<> &ModuleNames, - StringRef OutputFile, CheckerOptions Opts) { + llvm::raw_ostream &os, CheckerOptions Opts) { SDKContext SDKCtx(Opts); SDKNodeRoot *Root = getSDKNodeRoot(SDKCtx, InitInvoke, ModuleNames); if (!Root) return 1; - dumpSDKRoot(Root, OutputFile); + dumpSDKRoot(Root, os); return 0; } @@ -2589,11 +2586,11 @@ int swift::ide::api::deserializeSDKDump(StringRef dumpPath, StringRef OutputPath SwiftDeclCollector Collector(Ctx); Collector.deSerialize(dumpPath); - Collector.serialize(OutputPath); + Collector.serialize(FS); return 0; } -void swift::ide::api::dumpModuleContent(ModuleDecl *MD, StringRef OutputFile, +void swift::ide::api::dumpModuleContent(ModuleDecl *MD, llvm::raw_ostream &os, bool ABI, bool Empty) { CheckerOptions opts; opts.ABI = ABI; @@ -2610,7 +2607,7 @@ void swift::ide::api::dumpModuleContent(ModuleDecl *MD, StringRef OutputFile, PayLoad payload; SWIFT_DEFER { payload.allContsValues = &extractor.getAllConstValues(); - dumpSDKRoot(collector.getSDKRoot(), payload, OutputFile); + dumpSDKRoot(collector.getSDKRoot(), payload, os); }; if (Empty) { return; diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp index 9b495cdfeb8cc..da794196dbffe 100644 --- a/lib/AST/ASTContext.cpp +++ b/lib/AST/ASTContext.cpp @@ -72,6 +72,8 @@ #include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputBackends.h" #include #include #include @@ -635,6 +637,7 @@ ASTContext *ASTContext::get( ClangImporterOptions &ClangImporterOpts, symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts, SourceManager &SourceMgr, DiagnosticEngine &Diags, + llvm::IntrusiveRefCntPtr OutputBackend, std::function PreModuleImportCallback) { // If more than two data structures are concatentated, then the aggregate // size math needs to become more complicated due to per-struct alignment @@ -646,9 +649,10 @@ ASTContext *ASTContext::get( impl = reinterpret_cast( llvm::alignAddr(impl, llvm::Align(alignof(Implementation)))); new (impl) Implementation(); - return new (mem) ASTContext(langOpts, typecheckOpts, silOpts, SearchPathOpts, - ClangImporterOpts, SymbolGraphOpts, SourceMgr, - Diags, PreModuleImportCallback); + return new (mem) + ASTContext(langOpts, typecheckOpts, silOpts, SearchPathOpts, + ClangImporterOpts, SymbolGraphOpts, SourceMgr, Diags, + std::move(OutputBackend), PreModuleImportCallback); } ASTContext::ASTContext( @@ -657,17 +661,19 @@ ASTContext::ASTContext( ClangImporterOptions &ClangImporterOpts, symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts, SourceManager &SourceMgr, DiagnosticEngine &Diags, + llvm::IntrusiveRefCntPtr OutBackend, std::function PreModuleImportCallback) : LangOpts(langOpts), TypeCheckerOpts(typecheckOpts), SILOpts(silOpts), SearchPathOpts(SearchPathOpts), ClangImporterOpts(ClangImporterOpts), SymbolGraphOpts(SymbolGraphOpts), SourceMgr(SourceMgr), Diags(Diags), - evaluator(Diags, langOpts), TheBuiltinModule(createBuiltinModule(*this)), + OutputBackend(std::move(OutBackend)), evaluator(Diags, langOpts), + TheBuiltinModule(createBuiltinModule(*this)), StdlibModuleName(getIdentifier(STDLIB_NAME)), SwiftShimsModuleName(getIdentifier(SWIFT_SHIMS_NAME)), PreModuleImportCallback(PreModuleImportCallback), TheErrorType(new (*this, AllocationArena::Permanent) ErrorType( *this, Type(), RecursiveTypeProperties::HasError)), - TheUnresolvedType(new (*this, AllocationArena::Permanent) + TheUnresolvedType(new(*this, AllocationArena::Permanent) UnresolvedType(*this)), TheEmptyTupleType(TupleType::get(ArrayRef(), *this)), TheEmptyPackType(PackType::get(*this, {})), @@ -707,6 +713,9 @@ ASTContext::ASTContext( registerNameLookupRequestFunctions(evaluator); createModuleToExecutablePluginMap(); + // Provide a default OnDiskOutputBackend if user didn't supply one. + if (!OutputBackend) + OutputBackend = llvm::makeIntrusiveRefCnt(); } ASTContext::~ASTContext() { diff --git a/lib/AST/AbstractSourceFileDepGraphFactory.cpp b/lib/AST/AbstractSourceFileDepGraphFactory.cpp index 1b73a66b6a42b..78bdec1b0b6ad 100644 --- a/lib/AST/AbstractSourceFileDepGraphFactory.cpp +++ b/lib/AST/AbstractSourceFileDepGraphFactory.cpp @@ -23,6 +23,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" using namespace swift; @@ -33,13 +34,13 @@ using namespace fine_grained_dependencies; //============================================================================== AbstractSourceFileDepGraphFactory::AbstractSourceFileDepGraphFactory( - bool hadCompilationError, StringRef swiftDeps, - Fingerprint fileFingerprint, bool emitDotFileAfterConstruction, - DiagnosticEngine &diags) + bool hadCompilationError, StringRef swiftDeps, Fingerprint fileFingerprint, + bool emitDotFileAfterConstruction, DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend) : hadCompilationError(hadCompilationError), swiftDeps(swiftDeps.str()), fileFingerprint(fileFingerprint), - emitDotFileAfterConstruction(emitDotFileAfterConstruction), diags(diags) { -} + emitDotFileAfterConstruction(emitDotFileAfterConstruction), diags(diags), + backend(backend) {} SourceFileDepGraph AbstractSourceFileDepGraphFactory::construct() { addSourceFileNodesToGraph(); @@ -49,7 +50,7 @@ SourceFileDepGraph AbstractSourceFileDepGraphFactory::construct() { } assert(g.verify()); if (emitDotFileAfterConstruction) - g.emitDotFile(swiftDeps, diags); + g.emitDotFile(backend, swiftDeps, diags); return std::move(g); } diff --git a/lib/AST/FineGrainedDependencies.cpp b/lib/AST/FineGrainedDependencies.cpp index 609a03512b058..39d362ce6d8da 100644 --- a/lib/AST/FineGrainedDependencies.cpp +++ b/lib/AST/FineGrainedDependencies.cpp @@ -373,11 +373,13 @@ void SourceFileDepGraph::verifySame(const SourceFileDepGraph &other) const { #endif } -void SourceFileDepGraph::emitDotFile(StringRef outputPath, +void SourceFileDepGraph::emitDotFile(llvm::vfs::OutputBackend &outputBackend, + StringRef outputPath, DiagnosticEngine &diags) { std::string dotFileName = outputPath.str() + ".dot"; - withOutputFile(diags, dotFileName, [&](llvm::raw_pwrite_stream &out) { - DotFileEmitter(out, *this, false, false).emit(); - return false; - }); + withOutputPath( + diags, outputBackend, dotFileName, [&](llvm::raw_pwrite_stream &out) { + DotFileEmitter(out, *this, false, false).emit(); + return false; + }); } diff --git a/lib/AST/FineGrainedDependencyFormat.cpp b/lib/AST/FineGrainedDependencyFormat.cpp index bab3cdf60a98f..770c8745d8634 100644 --- a/lib/AST/FineGrainedDependencyFormat.cpp +++ b/lib/AST/FineGrainedDependencyFormat.cpp @@ -21,6 +21,7 @@ #include "llvm/Bitstream/BitstreamWriter.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackend.h" using namespace swift; using namespace fine_grained_dependencies; @@ -498,10 +499,10 @@ void swift::fine_grained_dependencies::writeFineGrainedDependencyGraph( } bool swift::fine_grained_dependencies::writeFineGrainedDependencyGraphToPath( - DiagnosticEngine &diags, StringRef path, + DiagnosticEngine &diags, llvm::vfs::OutputBackend &backend, StringRef path, const SourceFileDepGraph &g) { PrettyStackTraceStringAction stackTrace("saving fine-grained dependency graph", path); - return withOutputFile(diags, path, [&](llvm::raw_ostream &out) { + return withOutputPath(diags, backend, path, [&](llvm::raw_ostream &out) { SmallVector Buffer; llvm::BitstreamWriter Writer{Buffer}; writeFineGrainedDependencyGraph(Writer, g, Purpose::ForSwiftDeps); diff --git a/lib/AST/FrontendSourceFileDepGraphFactory.cpp b/lib/AST/FrontendSourceFileDepGraphFactory.cpp index 74afe1a7f810c..7b50752032919 100644 --- a/lib/AST/FrontendSourceFileDepGraphFactory.cpp +++ b/lib/AST/FrontendSourceFileDepGraphFactory.cpp @@ -43,6 +43,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" using namespace swift; @@ -232,18 +233,19 @@ StringRef DependencyKey::Builder::getTopLevelName(const Decl *decl) { bool fine_grained_dependencies::withReferenceDependencies( llvm::PointerUnion MSF, - const DependencyTracker &depTracker, StringRef outputPath, - bool alsoEmitDotFile, + const DependencyTracker &depTracker, llvm::vfs::OutputBackend &backend, + StringRef outputPath, bool alsoEmitDotFile, llvm::function_ref cont) { if (auto *MD = MSF.dyn_cast()) { SourceFileDepGraph g = - ModuleDepGraphFactory(MD, alsoEmitDotFile).construct(); + ModuleDepGraphFactory(backend, MD, alsoEmitDotFile).construct(); return cont(std::move(g)); } else { auto *SF = MSF.get(); - SourceFileDepGraph g = FrontendSourceFileDepGraphFactory( - SF, outputPath, depTracker, alsoEmitDotFile) - .construct(); + SourceFileDepGraph g = + FrontendSourceFileDepGraphFactory(SF, backend, outputPath, depTracker, + alsoEmitDotFile) + .construct(); return cont(std::move(g)); } } @@ -253,11 +255,12 @@ bool fine_grained_dependencies::withReferenceDependencies( //============================================================================== FrontendSourceFileDepGraphFactory::FrontendSourceFileDepGraphFactory( - const SourceFile *SF, StringRef outputPath, - const DependencyTracker &depTracker, const bool alsoEmitDotFile) + const SourceFile *SF, llvm::vfs::OutputBackend &backend, + StringRef outputPath, const DependencyTracker &depTracker, + const bool alsoEmitDotFile) : AbstractSourceFileDepGraphFactory( SF->getASTContext().hadError(), outputPath, SF->getInterfaceHash(), - alsoEmitDotFile, SF->getASTContext().Diags), + alsoEmitDotFile, SF->getASTContext().Diags, backend), SF(SF), depTracker(depTracker) {} //============================================================================== @@ -577,11 +580,12 @@ void FrontendSourceFileDepGraphFactory::addAllUsedDecls() { // MARK: ModuleDepGraphFactory //============================================================================== -ModuleDepGraphFactory::ModuleDepGraphFactory(const ModuleDecl *Mod, +ModuleDepGraphFactory::ModuleDepGraphFactory(llvm::vfs::OutputBackend &backend, + const ModuleDecl *Mod, bool emitDot) : AbstractSourceFileDepGraphFactory( Mod->getASTContext().hadError(), Mod->getNameStr(), - Mod->getFingerprint(), emitDot, Mod->getASTContext().Diags), + Mod->getFingerprint(), emitDot, Mod->getASTContext().Diags, backend), Mod(Mod) {} void ModuleDepGraphFactory::addAllDefinedDecls() { diff --git a/lib/AST/FrontendSourceFileDepGraphFactory.h b/lib/AST/FrontendSourceFileDepGraphFactory.h index d46105e45bc28..9fa24035527f8 100644 --- a/lib/AST/FrontendSourceFileDepGraphFactory.h +++ b/lib/AST/FrontendSourceFileDepGraphFactory.h @@ -14,6 +14,7 @@ #define FrontendSourceFileDepGraphFactory_h #include "swift/AST/AbstractSourceFileDepGraphFactory.h" +#include "llvm/Support/VirtualOutputBackend.h" namespace swift { namespace fine_grained_dependencies { @@ -27,7 +28,9 @@ class FrontendSourceFileDepGraphFactory const DependencyTracker &depTracker; public: - FrontendSourceFileDepGraphFactory(const SourceFile *SF, StringRef outputPath, + FrontendSourceFileDepGraphFactory(const SourceFile *SF, + llvm::vfs::OutputBackend &backend, + StringRef outputPath, const DependencyTracker &depTracker, bool alsoEmitDotFile); @@ -42,7 +45,8 @@ class ModuleDepGraphFactory : public AbstractSourceFileDepGraphFactory { const ModuleDecl *Mod; public: - ModuleDepGraphFactory(const ModuleDecl *Mod, bool emitDot); + ModuleDepGraphFactory(llvm::vfs::OutputBackend &backend, + const ModuleDecl *Mod, bool emitDot); ~ModuleDepGraphFactory() override = default; diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index 6f8cc84be731e..19c506aec03b1 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -66,6 +66,7 @@ #include "clang/Sema/Sema.h" #include "clang/Serialization/ASTReader.h" #include "clang/Serialization/ASTWriter.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Support/CrashRecoveryContext.h" @@ -74,6 +75,7 @@ #include "llvm/Support/Memory.h" #include "llvm/Support/Path.h" #include "llvm/Support/VirtualFileSystem.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include #include @@ -940,10 +942,8 @@ ClangImporter::getPCHFilename(const ClangImporterOptions &ImporterOptions, return PCHFilename.str().str(); } - -Optional -ClangImporter::getOrCreatePCH(const ClangImporterOptions &ImporterOptions, - StringRef SwiftPCHHash) { +Optional ClangImporter::getOrCreatePCH( + const ClangImporterOptions &ImporterOptions, StringRef SwiftPCHHash) { bool isExplicit; auto PCHFilename = getPCHFilename(ImporterOptions, SwiftPCHHash, isExplicit); @@ -959,8 +959,8 @@ ClangImporter::getOrCreatePCH(const ClangImporterOptions &ImporterOptions, << EC.message(); return None; } - auto FailedToEmit = emitBridgingPCH(ImporterOptions.BridgingHeader, - PCHFilename.value()); + auto FailedToEmit = + emitBridgingPCH(ImporterOptions.BridgingHeader, PCHFilename.value()); if (FailedToEmit) { return None; } @@ -1693,13 +1693,13 @@ ClangImporter::cloneCompilerInstanceForPrecompiling() { clonedInstance->setFileManager(&fileManager); clonedInstance->createSourceManager(fileManager); clonedInstance->setTarget(&Impl.Instance->getTarget()); + clonedInstance->setOutputBackend(Impl.SwiftContext.OutputBackend); return clonedInstance; } -bool -ClangImporter::emitBridgingPCH(StringRef headerPath, - StringRef outputPCHPath) { +bool ClangImporter::emitBridgingPCH( + StringRef headerPath, StringRef outputPCHPath) { auto emitInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = emitInstance->getInvocation(); @@ -1728,7 +1728,8 @@ ClangImporter::emitBridgingPCH(StringRef headerPath, return false; } -bool ClangImporter::runPreprocessor(StringRef inputPath, StringRef outputPath) { +bool ClangImporter::runPreprocessor( + StringRef inputPath, StringRef outputPath) { auto emitInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = emitInstance->getInvocation(); auto LangOpts = invocation.getLangOpts(); @@ -1752,9 +1753,8 @@ bool ClangImporter::runPreprocessor(StringRef inputPath, StringRef outputPath) { return emitInstance->getDiagnostics().hasErrorOccurred(); } -bool ClangImporter::emitPrecompiledModule(StringRef moduleMapPath, - StringRef moduleName, - StringRef outputPath) { +bool ClangImporter::emitPrecompiledModule( + StringRef moduleMapPath, StringRef moduleName, StringRef outputPath) { auto emitInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = emitInstance->getInvocation(); @@ -1788,8 +1788,8 @@ bool ClangImporter::emitPrecompiledModule(StringRef moduleMapPath, return false; } -bool ClangImporter::dumpPrecompiledModule(StringRef modulePath, - StringRef outputPath) { +bool ClangImporter::dumpPrecompiledModule( + StringRef modulePath, StringRef outputPath) { auto dumpInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = dumpInstance->getInvocation(); diff --git a/lib/DependencyScan/DependencyScanningTool.cpp b/lib/DependencyScan/DependencyScanningTool.cpp index cc812e4c3a954..d91c9053ec968 100644 --- a/lib/DependencyScan/DependencyScanningTool.cpp +++ b/lib/DependencyScan/DependencyScanningTool.cpp @@ -20,6 +20,7 @@ #include "swift/DependencyScan/DependencyScanImpl.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/VirtualOutputBackends.h" #include @@ -187,8 +188,9 @@ void DependencyScanningTool::serializeCache(llvm::StringRef path) { SourceManager SM; DiagnosticEngine Diags(SM); Diags.addConsumer(CDC); + llvm::vfs::OnDiskOutputBackend Backend; module_dependency_cache_serialization::writeInterModuleDependenciesCache( - Diags, path, *ScanningService); + Diags, Backend, path, *ScanningService); } bool DependencyScanningTool::loadCache(llvm::StringRef path) { diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index e69b3ae5f2462..075666ddc5dbc 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -16,6 +16,7 @@ #include "swift/Basic/Version.h" #include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/Support/VirtualOutputBackend.h" #include using namespace swift; @@ -1146,11 +1147,11 @@ void swift::dependencies::module_dependency_cache_serialization:: bool swift::dependencies::module_dependency_cache_serialization:: writeInterModuleDependenciesCache( - DiagnosticEngine &diags, StringRef path, - const SwiftDependencyScanningService &cache) { + DiagnosticEngine &diags, llvm::vfs::OutputBackend &backend, + StringRef path, const SwiftDependencyScanningService &cache) { PrettyStackTraceStringAction stackTrace( "saving inter-module dependency graph", path); - return withOutputFile(diags, path, [&](llvm::raw_ostream &out) { + return withOutputPath(diags, backend, path, [&](llvm::raw_ostream &out) { SmallVector Buffer; llvm::BitstreamWriter Writer{Buffer}; writeInterModuleDependenciesCache(Writer, cache); diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 84d35f09779cf..3e37c7345ba1b 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -20,6 +20,7 @@ #include "swift/AST/DiagnosticsFrontend.h" #include "swift/AST/DiagnosticsSema.h" #include "swift/AST/DiagnosticsDriver.h" +#include "swift/AST/FileSystem.h" #include "swift/AST/Module.h" #include "swift/AST/ModuleDependencies.h" #include "swift/AST/ModuleLoader.h" @@ -43,8 +44,10 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/StringSaver.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" +#include "llvm/Support/raw_ostream.h" #include #include #include @@ -933,6 +936,25 @@ static void writeJSON(llvm::raw_ostream &out, } } +static bool writePrescanJSONToOutput(DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend, + StringRef path, + const swiftscan_import_set_t importSet) { + return withOutputPath(diags, backend, path, [&](llvm::raw_pwrite_stream &os) { + writePrescanJSON(os, importSet); + return false; + }); +} + +static bool writeJSONToOutput(DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend, StringRef path, + const swiftscan_dependency_graph_t dependencies) { + return withOutputPath(diags, backend, path, [&](llvm::raw_pwrite_stream &os) { + writeJSON(os, dependencies); + return false; + }); +} + static swiftscan_dependency_graph_t generateFullDependencyGraph(CompilerInstance &instance, ModuleDependenciesCache &cache, @@ -1369,7 +1391,7 @@ static void serializeDependencyCache(CompilerInstance &instance, ASTContext &Context = instance.getASTContext(); auto savePath = opts.SerializedDependencyScannerCachePath; module_dependency_cache_serialization::writeInterModuleDependenciesCache( - Context.Diags, savePath, service); + Context.Diags, instance.getOutputBackend(), savePath, service); if (opts.EmitDependencyScannerCacheRemarks) { Context.Diags.diagnose(SourceLoc(), diag::remark_save_cache, savePath); } @@ -1393,15 +1415,6 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { ASTContext &Context = instance.getASTContext(); const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); - std::error_code EC; - llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::OF_None); - if (out.has_error() || EC) { - Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, - EC.message()); - out.clear_error(); - return true; - } - // `-scan-dependencies` invocations use a single new instance // of a module cache SwiftDependencyScanningService service; @@ -1427,8 +1440,10 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { return true; auto dependencies = std::move(*dependenciesOrErr); - // Write out the JSON description. - writeJSON(out, dependencies); + if (writeJSONToOutput(Context.Diags, instance.getOutputBackend(), path, + dependencies)) + return true; + // This process succeeds regardless of whether any errors occurred. // FIXME: We shouldn't need this, but it's masking bugs in our scanning // logic where we don't create a fresh context when scanning Swift interfaces @@ -1441,20 +1456,12 @@ bool swift::dependencies::prescanDependencies(CompilerInstance &instance) { ASTContext &Context = instance.getASTContext(); const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); - std::error_code EC; - llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::OF_None); // `-scan-dependencies` invocations use a single new instance // of a module cache SwiftDependencyScanningService singleUseService; ModuleDependenciesCache cache(singleUseService, instance.getMainModule()->getNameStr().str(), instance.getInvocation().getModuleScanningHash()); - if (out.has_error() || EC) { - Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, - EC.message()); - out.clear_error(); - return true; - } // Execute import prescan, and write JSON output to the output stream auto importSetOrErr = performModulePrescan(instance); @@ -1463,7 +1470,10 @@ bool swift::dependencies::prescanDependencies(CompilerInstance &instance) { auto importSet = std::move(*importSetOrErr); // Serialize and output main module dependencies only and exit. - writePrescanJSON(out, importSet); + if (writePrescanJSONToOutput(Context.Diags, instance.getOutputBackend(), path, + importSet)) + return true; + // This process succeeds regardless of whether any errors occurred. // FIXME: We shouldn't need this, but it's masking bugs in our scanning // logic where we don't create a fresh context when scanning Swift interfaces @@ -1499,12 +1509,13 @@ bool swift::dependencies::batchScanDependencies( auto iresults = batchScanResults.cbegin(); for (; ientries != batchInput->end() and iresults != batchScanResults.end(); ++ientries, ++iresults) { - std::error_code EC; - llvm::raw_fd_ostream out((*ientries).outputPath, EC, llvm::sys::fs::OF_None); if ((*iresults).getError()) return true; - writeJSON(out, **iresults); + if (writeJSONToOutput(instance.getASTContext().Diags, + instance.getOutputBackend(), (*ientries).outputPath, + **iresults)) + return true; } return false; } @@ -1534,12 +1545,13 @@ bool swift::dependencies::batchPrescanDependencies( for (; ientries != batchInput->end() and iresults != batchPrescanResults.end(); ++ientries, ++iresults) { - std::error_code EC; - llvm::raw_fd_ostream out((*ientries).outputPath, EC, llvm::sys::fs::OF_None); if ((*iresults).getError()) return true; - writePrescanJSON(out, **iresults); + if (writePrescanJSONToOutput(instance.getASTContext().Diags, + instance.getOutputBackend(), + (*ientries).outputPath, **iresults)) + return true; } return false; } diff --git a/lib/Driver/FineGrainedDependencyDriverGraph.cpp b/lib/Driver/FineGrainedDependencyDriverGraph.cpp index 1b0c20a5fc849..9e2b36c359570 100644 --- a/lib/Driver/FineGrainedDependencyDriverGraph.cpp +++ b/lib/Driver/FineGrainedDependencyDriverGraph.cpp @@ -29,6 +29,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/SourceMgr.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/raw_ostream.h" #include @@ -529,11 +530,12 @@ void ModuleDepGraph::emitDotFileForJob(DiagnosticEngine &diags, emitDotFile(diags, getSwiftDeps(job)); } -void ModuleDepGraph::emitDotFile(DiagnosticEngine &diags, StringRef baseName) { +void ModuleDepGraph::emitDotFile(DiagnosticEngine &diags, + StringRef baseName) { unsigned seqNo = dotFileSequenceNumber[baseName.str()]++; std::string fullName = baseName.str() + "-post-integration." + std::to_string(seqNo) + ".dot"; - withOutputFile(diags, fullName, [&](llvm::raw_ostream &out) { + withOutputPath(diags, *backend, fullName, [&](llvm::raw_ostream &out) { emitDotFile(out); return false; }); diff --git a/lib/DriverTool/modulewrap_main.cpp b/lib/DriverTool/modulewrap_main.cpp index eff93f616cc5d..676b78e720ae4 100644 --- a/lib/DriverTool/modulewrap_main.cpp +++ b/lib/DriverTool/modulewrap_main.cpp @@ -28,6 +28,7 @@ #include "swift/Subsystems.h" #include "swift/SymbolGraphGen/SymbolGraphOptions.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/Bitstream/BitstreamReader.h" #include "llvm/Option/ArgList.h" #include "llvm/Option/Option.h" @@ -35,6 +36,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/TargetSelect.h" +#include "llvm/Support/VirtualOutputBackends.h" using namespace llvm::opt; using namespace swift; @@ -183,7 +185,8 @@ int modulewrap_main(ArrayRef Args, const char *Argv0, LangOpts.Target = Invocation.getTargetTriple(); ASTContext &ASTCtx = *ASTContext::get( LangOpts, TypeCheckOpts, SILOpts, SearchPathOpts, ClangImporterOpts, - SymbolGraphOpts, SrcMgr, Instance.getDiags()); + SymbolGraphOpts, SrcMgr, Instance.getDiags(), + llvm::makeIntrusiveRefCnt()); registerParseRequestFunctions(ASTCtx.evaluator); registerTypeCheckerRequestFunctions(ASTCtx.evaluator); diff --git a/lib/DriverTool/swift_api_digester_main.cpp b/lib/DriverTool/swift_api_digester_main.cpp index a2b32b4a03e13..7265b48b447e8 100644 --- a/lib/DriverTool/swift_api_digester_main.cpp +++ b/lib/DriverTool/swift_api_digester_main.cpp @@ -36,6 +36,9 @@ #include "swift/IDE/APIDigesterData.h" #include "swift/Option/Options.h" #include "swift/Parse/ParseVersion.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/Support/VirtualOutputBackends.h" +#include "llvm/Support/raw_ostream.h" #include using namespace swift; @@ -2536,13 +2539,18 @@ class SwiftAPIDigesterInvocation { switch (Action) { case ActionType::DumpSDK: { llvm::StringSet<> Modules; - return (prepareForDump(InitInvoke, Modules)) - ? 1 - : dumpSDKContent(InitInvoke, Modules, - getJsonOutputFilePath( - InitInvoke.getLangOptions().Target, - CheckerOpts.ABI, OutputFile, OutputDir), - CheckerOpts); + if (prepareForDump(InitInvoke, Modules)) + return 1; + auto JsonOut = + getJsonOutputFilePath(InitInvoke.getLangOptions().Target, + CheckerOpts.ABI, OutputFile, OutputDir); + std::error_code EC; + llvm::raw_fd_ostream fs(JsonOut, EC); + if (EC) { + llvm::errs() << "Cannot open JSON output file: " << JsonOut << "\n"; + return 1; + } + return dumpSDKContent(InitInvoke, Modules, fs, CheckerOpts); } case ActionType::MigratorGen: case ActionType::DiagnoseSDKs: { @@ -2606,7 +2614,13 @@ class SwiftAPIDigesterInvocation { } case ActionType::GenerateEmptyBaseline: { SDKContext Ctx(CheckerOpts); - dumpSDKRoot(getEmptySDKNodeRoot(Ctx), OutputFile); + std::error_code EC; + llvm::raw_fd_ostream fs(OutputFile, EC); + if (EC) { + llvm::errs() << "Cannot open output file: " << OutputFile << "\n"; + return 1; + } + dumpSDKRoot(getEmptySDKNodeRoot(Ctx), fs); return 0; } case ActionType::FindUsr: { diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index e52caaccc40af..7145d23d80af4 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -350,6 +350,7 @@ bool ArgsToFrontendOptionsConverter::convert( Opts.serializedPathObfuscator.addMapping(SplitMap.first, SplitMap.second); } Opts.emptyABIDescriptor = Args.hasArg(OPT_empty_abi_descriptor); + Opts.DeterministicCheck = Args.hasArg(OPT_enable_deterministic_check); return false; } diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 12e563dffd1ec..806635b18872f 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -2377,6 +2377,7 @@ static bool ParseIRGenArgs(IRGenOptions &Opts, ArgList &Args, Opts.UseProfile = ProfileUse ? ProfileUse->getValue() : ""; Opts.PrintInlineTree |= Args.hasArg(OPT_print_llvm_inline_tree); + Opts.AlwaysCompile |= Args.hasArg(OPT_always_compile_output_files); Opts.EnableDynamicReplacementChaining |= Args.hasArg(OPT_enable_dynamic_replacement_chaining); diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index 5a8ddacecae36..e228ca5db8966 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -38,6 +38,7 @@ #include "swift/Subsystems.h" #include "clang/AST/ASTContext.h" #include "llvm/ADT/Hashing.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Triple.h" #include "llvm/Support/CommandLine.h" @@ -45,6 +46,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/Process.h" +#include "llvm/Support/VirtualOutputBackends.h" #include using namespace swift; @@ -249,7 +251,7 @@ bool CompilerInstance::setUpASTContextIfNeeded() { Invocation.getLangOptions(), Invocation.getTypeCheckerOptions(), Invocation.getSILOptions(), Invocation.getSearchPathOptions(), Invocation.getClangImporterOptions(), Invocation.getSymbolGraphOptions(), - SourceMgr, Diagnostics)); + SourceMgr, Diagnostics, OutputBackend)); if (!Invocation.getFrontendOptions().ModuleAliasMap.empty()) Context->setModuleAliases(Invocation.getFrontendOptions().ModuleAliasMap); @@ -393,11 +395,31 @@ void CompilerInstance::setupDependencyTrackerIfNeeded() { DepTracker->addDependency(path, /*isSystem=*/false); } +void CompilerInstance::setupOutputBackend() { + // Skip if output backend is not setup, default to OnDiskOutputBackend. + if (OutputBackend) + return; + + OutputBackend = + llvm::makeIntrusiveRefCnt(); + + // Setup verification backend. + // Create a mirroring outputbackend to produce hash for output files. + // We cannot skip disk here since swift compiler is expecting to read back + // some output file in later stages. + if (Invocation.getFrontendOptions().DeterministicCheck) { + HashBackend = llvm::makeIntrusiveRefCnt(); + OutputBackend = + llvm::vfs::makeMirroringOutputBackend(OutputBackend, HashBackend); + } +} + bool CompilerInstance::setup(const CompilerInvocation &Invoke, std::string &Error) { Invocation = Invoke; setupDependencyTrackerIfNeeded(); + setupOutputBackend(); // If initializing the overlay file system fails there's no sense in // continuing because the compiler will read the wrong files. diff --git a/lib/Frontend/ModuleInterfaceBuilder.cpp b/lib/Frontend/ModuleInterfaceBuilder.cpp index a898ae822d6ca..1d872b6b3b24d 100644 --- a/lib/Frontend/ModuleInterfaceBuilder.cpp +++ b/lib/Frontend/ModuleInterfaceBuilder.cpp @@ -190,7 +190,8 @@ std::error_code ExplicitModuleInterfaceBuilder::buildSwiftModuleFromInterface( if (Instance.getASTContext() .getModuleInterfaceChecker() ->tryEmitForwardingModule(Invocation.getModuleName(), InterfacePath, - CompiledCandidates, OutputPath)) { + CompiledCandidates, + Instance.getOutputBackend(), OutputPath)) { return std::error_code(); } FrontendOptions &FEOpts = Invocation.getFrontendOptions(); diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index acc34930c72e4..65946488be1d1 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -40,6 +40,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/Errc.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" #include "llvm/Support/xxhash.h" @@ -901,8 +902,8 @@ class ModuleInterfaceLoaderImpl { /// this. If the write was successful, it also updates the /// list of dependencies to match what was written to the forwarding file. bool writeForwardingModuleAndUpdateDeps( - const DiscoveredModule &mod, StringRef outputPath, - SmallVectorImpl &deps) { + const DiscoveredModule &mod, llvm::vfs::OutputBackend &backend, + StringRef outputPath, SmallVectorImpl &deps) { assert(mod.isPrebuilt() && "cannot write forwarding file for non-prebuilt module"); ForwardingModule fwd(mod.path); @@ -938,16 +939,12 @@ class ModuleInterfaceLoaderImpl { depsAdjustedToMTime.push_back(adjustedDep); } - // Create the module cache if we haven't created it yet. - StringRef parentDir = path::parent_path(outputPath); - (void)llvm::sys::fs::create_directories(parentDir); - - auto hadError = withOutputFile(diags, outputPath, - [&](llvm::raw_pwrite_stream &out) { - llvm::yaml::Output yamlWriter(out); - yamlWriter << fwd; - return false; - }); + auto hadError = withOutputPath(diags, backend, outputPath, + [&](llvm::raw_pwrite_stream &out) { + llvm::yaml::Output yamlWriter(out); + yamlWriter << fwd; + return false; + }); if (hadError) return true; @@ -1006,8 +1003,8 @@ class ModuleInterfaceLoaderImpl { // If it's prebuilt, use this time to generate a forwarding module and // update the dependencies to use modification times. if (module.isPrebuilt()) - if (writeForwardingModuleAndUpdateDeps(module, cachedOutputPath, - allDeps)) + if (writeForwardingModuleAndUpdateDeps(module, ctx.getOutputBackend(), + cachedOutputPath, allDeps)) return std::make_error_code(std::errc::not_supported); // Report the module's dependencies to the dependencyTracker @@ -1251,7 +1248,8 @@ ModuleInterfaceCheckerImpl::getCompiledModuleCandidatesForInterface(StringRef mo bool ModuleInterfaceCheckerImpl::tryEmitForwardingModule( StringRef moduleName, StringRef interfacePath, - ArrayRef candidates, StringRef outputPath) { + ArrayRef candidates, llvm::vfs::OutputBackend &backend, + StringRef outputPath) { // Derive .swiftmodule path from the .swiftinterface path. auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile); llvm::SmallString<32> modulePath = interfacePath; @@ -1269,12 +1267,12 @@ bool ModuleInterfaceCheckerImpl::tryEmitForwardingModule( deps, moduleBuffer)) { // If so, emit a forwarding module to the candidate. ForwardingModule FM(mod); - auto hadError = withOutputFile(Ctx.Diags, outputPath, - [&](llvm::raw_pwrite_stream &out) { - llvm::yaml::Output yamlWriter(out); - yamlWriter << FM; - return false; - }); + auto hadError = withOutputPath(Ctx.Diags, backend, outputPath, + [&](llvm::raw_pwrite_stream &out) { + llvm::yaml::Output yamlWriter(out); + yamlWriter << FM; + return false; + }); if (!hadError) return true; } @@ -1402,19 +1400,25 @@ bool ModuleInterfaceLoader::buildExplicitSwiftModuleFromSwiftInterface( ArrayRef CompiledCandidates, DependencyTracker *tracker) { - // First, check if the expected output already exists and possibly up-to-date w.r.t. - // all of the dependencies it was built with. If so, early exit. - UpToDateModuleCheker checker(Instance.getASTContext(), - RequireOSSAModules_t(Instance.getSILOptions())); - ModuleRebuildInfo rebuildInfo; - SmallVector allDeps; - std::unique_ptr moduleBuffer; - if (checker.swiftModuleIsUpToDate(outputPath, rebuildInfo, allDeps, moduleBuffer)) { - if (Instance.getASTContext().LangOpts.EnableSkipExplicitInterfaceModuleBuildRemarks) { - Instance.getDiags().diagnose(SourceLoc(), - diag::explicit_interface_build_skipped, outputPath); + if (!Instance.getInvocation().getIRGenOptions().AlwaysCompile) { + // First, check if the expected output already exists and possibly + // up-to-date w.r.t. all of the dependencies it was built with. If so, early + // exit. + UpToDateModuleCheker checker( + Instance.getASTContext(), + RequireOSSAModules_t(Instance.getSILOptions())); + ModuleRebuildInfo rebuildInfo; + SmallVector allDeps; + std::unique_ptr moduleBuffer; + if (checker.swiftModuleIsUpToDate(outputPath, rebuildInfo, allDeps, + moduleBuffer)) { + if (Instance.getASTContext() + .LangOpts.EnableSkipExplicitInterfaceModuleBuildRemarks) { + Instance.getDiags().diagnose( + SourceLoc(), diag::explicit_interface_build_skipped, outputPath); + } + return false; } - return false; } // Read out the compiler version. diff --git a/lib/Frontend/Serialization.cpp b/lib/Frontend/Serialization.cpp index 3af22d3bf2df0..88ad917fab610 100644 --- a/lib/Frontend/Serialization.cpp +++ b/lib/Frontend/Serialization.cpp @@ -12,11 +12,13 @@ #include "swift/Serialization/Serialization.h" #include "swift/APIDigester/ModuleAnalyzerNodes.h" +#include "swift/AST/DiagnosticsFrontend.h" #include "swift/AST/FileSystem.h" #include "swift/Subsystems.h" #include "swift/SymbolGraphGen/SymbolGraphGen.h" #include "swift/SymbolGraphGen/SymbolGraphOptions.h" #include "llvm/Support/SmallVectorMemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackend.h" using namespace swift; @@ -35,7 +37,23 @@ static void emitABIDescriptor(ModuleOrSourceFile DC, using namespace swift::ide::api; if (!options.ABIDescriptorPath.empty()) { if (DC.is()) { - dumpModuleContent(DC.get(), options.ABIDescriptorPath, true, + auto &OutputBackend = getContext(DC).getOutputBackend(); + auto ABIDesFile = OutputBackend.createFile(options.ABIDescriptorPath); + if (!ABIDesFile) { + getContext(DC).Diags.diagnose(SourceLoc(), diag::error_opening_output, + options.ABIDescriptorPath, + toString(ABIDesFile.takeError())); + return; + } + SWIFT_DEFER { + if (auto E = ABIDesFile->keep()) { + getContext(DC).Diags.diagnose(SourceLoc(), diag::error_closing_output, + options.ABIDescriptorPath, + toString(std::move(E))); + return; + } + }; + dumpModuleContent(DC.get(), *ABIDesFile, true, options.emptyABIDescriptor); } } @@ -56,11 +74,12 @@ void swift::serializeToBuffers( llvm::raw_svector_ostream stream(buf); serialization::writeToStream(stream, DC, M, options, /*dependency info*/ nullptr); - bool hadError = withOutputFile(getContext(DC).Diags, options.OutputPath, - [&](raw_ostream &out) { - out << stream.str(); - return false; - }); + bool hadError = withOutputPath( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.OutputPath, [&](raw_ostream &out) { + out << stream.str(); + return false; + }); if (hadError) return; @@ -77,11 +96,12 @@ void swift::serializeToBuffers( llvm::SmallString<1024> buf; llvm::raw_svector_ostream stream(buf); serialization::writeDocToStream(stream, DC, options.GroupInfoPath); - (void)withOutputFile(getContext(DC).Diags, options.DocOutputPath, - [&](raw_ostream &out) { - out << stream.str(); - return false; - }); + (void)withOutputPath(getContext(DC).Diags, + getContext(DC).getOutputBackend(), + options.DocOutputPath, [&](raw_ostream &out) { + out << stream.str(); + return false; + }); if (moduleDocBuffer) *moduleDocBuffer = std::make_unique( std::move(buf), options.DocOutputPath, @@ -94,11 +114,12 @@ void swift::serializeToBuffers( llvm::SmallString<1024> buf; llvm::raw_svector_ostream stream(buf); serialization::writeSourceInfoToStream(stream, DC); - (void)withOutputFile(getContext(DC).Diags, options.SourceInfoOutputPath, - [&](raw_ostream &out) { - out << stream.str(); - return false; - }); + (void)withOutputPath( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.SourceInfoOutputPath, [&](raw_ostream &out) { + out << stream.str(); + return false; + }); if (moduleSourceInfoBuffer) *moduleSourceInfoBuffer = std::make_unique( std::move(buf), options.SourceInfoOutputPath, @@ -120,37 +141,37 @@ void swift::serialize( return; } - bool hadError = withOutputFile(getContext(DC).Diags, - options.OutputPath, - [&](raw_ostream &out) { - FrontendStatsTracer tracer(getContext(DC).Stats, - "Serialization, swiftmodule"); - serialization::writeToStream(out, DC, M, options, DG); - return false; - }); + bool hadError = withOutputPath( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.OutputPath, [&](raw_ostream &out) { + FrontendStatsTracer tracer(getContext(DC).Stats, + "Serialization, swiftmodule"); + serialization::writeToStream(out, DC, M, options, DG); + return false; + }); if (hadError) return; if (!options.DocOutputPath.empty()) { - (void)withOutputFile(getContext(DC).Diags, - options.DocOutputPath, - [&](raw_ostream &out) { - FrontendStatsTracer tracer(getContext(DC).Stats, - "Serialization, swiftdoc"); - serialization::writeDocToStream(out, DC, options.GroupInfoPath); - return false; - }); + (void)withOutputPath( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.DocOutputPath, [&](raw_ostream &out) { + FrontendStatsTracer tracer(getContext(DC).Stats, + "Serialization, swiftdoc"); + serialization::writeDocToStream(out, DC, options.GroupInfoPath); + return false; + }); } if (!options.SourceInfoOutputPath.empty()) { - (void)withOutputFile(getContext(DC).Diags, - options.SourceInfoOutputPath, - [&](raw_ostream &out) { - FrontendStatsTracer tracer(getContext(DC).Stats, - "Serialization, swiftsourceinfo"); - serialization::writeSourceInfoToStream(out, DC); - return false; - }); + (void)withOutputPath( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.SourceInfoOutputPath, [&](raw_ostream &out) { + FrontendStatsTracer tracer(getContext(DC).Stats, + "Serialization, swiftsourceinfo"); + serialization::writeSourceInfoToStream(out, DC); + return false; + }); } if (!symbolGraphOptions.OutputDir.empty()) { diff --git a/lib/FrontendTool/Dependencies.h b/lib/FrontendTool/Dependencies.h index 54291ebe03f63..91397e642fae7 100644 --- a/lib/FrontendTool/Dependencies.h +++ b/lib/FrontendTool/Dependencies.h @@ -13,6 +13,12 @@ #ifndef SWIFT_FRONTENDTOOL_DEPENDENCIES_H #define SWIFT_FRONTENDTOOL_DEPENDENCIES_H +namespace llvm { +namespace vfs { +class OutputBackend; +} +} // namespace llvm + namespace swift { class ASTContext; @@ -23,11 +29,13 @@ class InputFile; class ModuleDecl; /// Emit the names of the modules imported by \c mainModule. -bool emitImportedModules(ModuleDecl *mainModule, const FrontendOptions &opts); +bool emitImportedModules(ModuleDecl *mainModule, const FrontendOptions &opts, + llvm::vfs::OutputBackend &backend); bool emitMakeDependenciesIfNeeded(DiagnosticEngine &diags, DependencyTracker *depTracker, const FrontendOptions &opts, - const InputFile &input); + const InputFile &input, + llvm::vfs::OutputBackend &backend); bool emitLoadedModuleTraceIfNeeded(ModuleDecl *mainModule, DependencyTracker *depTracker, const FrontendOptions &opts, diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index 12ce64135d364..cad2a25345dbf 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -71,7 +71,10 @@ #include "clang/Lex/Preprocessor.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" @@ -81,6 +84,8 @@ #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputBackends.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Support/FileSystem.h" @@ -105,10 +110,12 @@ static std::string displayName(StringRef MainExecutablePath) { static void emitMakeDependenciesIfNeeded(DiagnosticEngine &diags, DependencyTracker *depTracker, - const FrontendOptions &opts) { + const FrontendOptions &opts, + llvm::vfs::OutputBackend &backend) { opts.InputsAndOutputs.forEachInputProducingSupplementaryOutput( [&](const InputFile &f) -> bool { - return swift::emitMakeDependenciesIfNeeded(diags, depTracker, opts, f); + return swift::emitMakeDependenciesIfNeeded(diags, depTracker, opts, f, + backend); }); } @@ -123,37 +130,22 @@ emitLoadedModuleTraceForAllPrimariesIfNeeded(ModuleDecl *mainModule, }); } -/// Gets an output stream for the provided output filename, or diagnoses to the -/// provided AST Context and returns null if there was an error getting the -/// stream. -static std::unique_ptr -getFileOutputStream(StringRef OutputFilename, ASTContext &Ctx) { - std::error_code errorCode; - auto os = std::make_unique( - OutputFilename, errorCode, llvm::sys::fs::OF_None); - if (errorCode) { - Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_output, - OutputFilename, errorCode.message()); - return nullptr; - } - return os; -} - /// Writes SIL out to the given file. static bool writeSIL(SILModule &SM, ModuleDecl *M, const SILOptions &Opts, - StringRef OutputFilename) { - auto OS = getFileOutputStream(OutputFilename, M->getASTContext()); - if (!OS) return true; - SM.print(*OS, M, Opts); - - return M->getASTContext().hadError(); + StringRef OutputFilename, + llvm::vfs::OutputBackend &Backend) { + return withOutputPath(M->getDiags(), Backend, OutputFilename, + [&](raw_ostream &out) -> bool { + SM.print(out, M, Opts); + return M->getASTContext().hadError(); + }); } static bool writeSIL(SILModule &SM, const PrimarySpecificPaths &PSPs, - const CompilerInstance &Instance, + CompilerInstance &Instance, const SILOptions &Opts) { return writeSIL(SM, Instance.getMainModule(), Opts, - PSPs.OutputFilename); + PSPs.OutputFilename, Instance.getOutputBackend()); } /// Prints the Objective-C "generated header" interface for \p M to \p @@ -166,14 +158,14 @@ static bool writeSIL(SILModule &SM, const PrimarySpecificPaths &PSPs, /// \returns true if there were any errors /// /// \see swift::printAsClangHeader -static bool printAsClangHeaderIfNeeded( +static bool printAsClangHeaderIfNeeded(llvm::vfs::OutputBackend &outputBackend, StringRef outputPath, ModuleDecl *M, StringRef bridgingHeader, const FrontendOptions &frontendOpts, const IRGenOptions &irGenOpts, clang::HeaderSearch &clangHeaderSearchInfo) { if (outputPath.empty()) return false; - return withOutputFile( - M->getDiags(), outputPath, [&](raw_ostream &out) -> bool { + return withOutputPath( + M->getDiags(), outputBackend, outputPath, [&](raw_ostream &out) -> bool { return printAsClangHeader(out, M, bridgingHeader, frontendOpts, irGenOpts, clangHeaderSearchInfo); }); @@ -187,7 +179,8 @@ static bool printAsClangHeaderIfNeeded( /// /// \see swift::emitSwiftInterface static bool -printModuleInterfaceIfNeeded(StringRef outputPath, +printModuleInterfaceIfNeeded(llvm::vfs::OutputBackend &outputBackend, + StringRef outputPath, ModuleInterfaceOptions const &Opts, LangOptions const &LangOpts, ModuleDecl *M) { @@ -205,10 +198,10 @@ printModuleInterfaceIfNeeded(StringRef outputPath, diags.diagnose(SourceLoc(), diag::warn_unsupported_module_interface_library_evolution); } - return withOutputFile(diags, outputPath, + return withOutputPath(diags, outputBackend, outputPath, [M, Opts](raw_ostream &out) -> bool { - return swift::emitSwiftInterface(out, Opts, M); - }); + return swift::emitSwiftInterface(out, Opts, M); + }); } namespace { @@ -372,6 +365,7 @@ static bool precompileBridgingHeader(const CompilerInstance &Instance) { Instance.getASTContext().getClangModuleLoader()); auto &ImporterOpts = Invocation.getClangImporterOptions(); auto &PCHOutDir = ImporterOpts.PrecompiledHeaderOutputDir; + auto OutputBackend = Instance.getOutputBackend().clone(); if (!PCHOutDir.empty()) { // Create or validate a persistent PCH. auto SwiftPCHHash = Invocation.getPCHHash(); @@ -517,8 +511,13 @@ static bool dumpAST(CompilerInstance &Instance) { for (SourceFile *sourceFile: primaryFiles) { auto PSPs = Instance.getPrimarySpecificPathsForSourceFile(*sourceFile); auto OutputFilename = PSPs.OutputFilename; - auto OS = getFileOutputStream(OutputFilename, Instance.getASTContext()); - sourceFile->dump(*OS, /*parseIfNeeded*/ true); + if (withOutputPath(Instance.getASTContext().Diags, + Instance.getOutputBackend(), OutputFilename, + [&](raw_ostream &out) -> bool { + sourceFile->dump(out, /*parseIfNeeded*/ true); + return false; + })) + return true; } } else { // Some invocations don't have primary files. In that case, we default to @@ -543,17 +542,19 @@ static bool emitReferenceDependencies(CompilerInstance &Instance, using SourceFileDepGraph = fine_grained_dependencies::SourceFileDepGraph; return fine_grained_dependencies::withReferenceDependencies( - SF, *Instance.getDependencyTracker(), outputPath, alsoEmitDotFile, - [&](SourceFileDepGraph &&g) -> bool { + SF, *Instance.getDependencyTracker(), Instance.getOutputBackend(), + outputPath, alsoEmitDotFile, [&](SourceFileDepGraph &&g) -> bool { const bool hadError = fine_grained_dependencies::writeFineGrainedDependencyGraphToPath( - Instance.getDiags(), outputPath, g); + Instance.getDiags(), Instance.getOutputBackend(), outputPath, + g); // If path is stdout, cannot read it back, so check for "-" assert(outputPath == "-" || g.verifyReadsWhatIsWritten(outputPath)); if (alsoEmitDotFile) - g.emitDotFile(outputPath, Instance.getDiags()); + g.emitDotFile(Instance.getOutputBackend(), outputPath, + Instance.getDiags()); return hadError; }); } @@ -925,6 +926,7 @@ static bool emitAnyWholeModulePostTypeCheckSupplementaryOutputs( } } hadAnyError |= printAsClangHeaderIfNeeded( + Instance.getOutputBackend(), Invocation.getClangHeaderOutputPathForAtMostOnePrimary(), Instance.getMainModule(), BridgingHeaderPathForPrint, opts, Invocation.getIRGenOptions(), @@ -940,6 +942,7 @@ static bool emitAnyWholeModulePostTypeCheckSupplementaryOutputs( if (opts.InputsAndOutputs.hasModuleInterfaceOutputPath()) { hadAnyError |= printModuleInterfaceIfNeeded( + Instance.getOutputBackend(), Invocation.getModuleInterfaceOutputPathForWholeModule(), Invocation.getModuleInterfaceOptions(), Invocation.getLangOptions(), @@ -953,6 +956,7 @@ static bool emitAnyWholeModulePostTypeCheckSupplementaryOutputs( privOpts.ModulesToSkipInPublicInterface.clear(); hadAnyError |= printModuleInterfaceIfNeeded( + Instance.getOutputBackend(), Invocation.getPrivateModuleInterfaceOutputPathForWholeModule(), privOpts, Invocation.getLangOptions(), @@ -1139,7 +1143,8 @@ static void performEndOfPipelineActions(CompilerInstance &Instance) { // Emit Make-style dependencies. emitMakeDependenciesIfNeeded(Instance.getDiags(), - Instance.getDependencyTracker(), opts); + Instance.getDependencyTracker(), opts, + Instance.getOutputBackend()); // Emit extracted constant values for every file in the batch emitConstValuesForAllPrimaryInputsIfNeeded(Instance); @@ -1333,7 +1338,8 @@ static bool performAction(CompilerInstance &Instance, getPrimaryOrMainSourceFile(Instance).dumpInterfaceHash(llvm::errs()); return Instance.getASTContext().hadError(); case FrontendOptions::ActionType::EmitImportedModules: - return emitImportedModules(Instance.getMainModule(), opts); + return emitImportedModules(Instance.getMainModule(), opts, + Instance.getOutputBackend()); // MARK: Dependency Scanning Actions case FrontendOptions::ActionType::ScanDependencies: @@ -1437,11 +1443,11 @@ static bool serializeModuleSummary(SILModule *SM, const PrimarySpecificPaths &PSPs, const ASTContext &Context) { auto summaryOutputPath = PSPs.SupplementaryOutputs.ModuleSummaryOutputPath; - return withOutputFile(Context.Diags, summaryOutputPath, - [&](llvm::raw_ostream &out) { - out << "Some stuff"; - return false; - }); + return withOutputPath(Context.Diags, Context.getOutputBackend(), + summaryOutputPath, [&](llvm::raw_ostream &out) { + out << "Some stuff"; + return false; + }); } static GeneratedModule @@ -1618,6 +1624,7 @@ static bool generateCode(CompilerInstance &Instance, StringRef OutputFilename, // Now that we have a single IR Module, hand it over to performLLVM. return performLLVM(opts, Instance.getDiags(), nullptr, HashGlobal, IRModule, TargetMachine.get(), OutputFilename, + Instance.getOutputBackend(), Instance.getStatsReporter()); } @@ -1696,7 +1703,8 @@ static bool performCompileStepsPostSILGen(CompilerInstance &Instance, using SourceFileDepGraph = fine_grained_dependencies::SourceFileDepGraph; auto *Mod = MSF.get(); fine_grained_dependencies::withReferenceDependencies( - Mod, *Instance.getDependencyTracker(), Mod->getModuleFilename(), + Mod, *Instance.getDependencyTracker(), + Instance.getOutputBackend(), Mod->getModuleFilename(), alsoEmitDotFile, [&](SourceFileDepGraph &&g) { serialize(MSF, serializationOpts, Invocation.getSymbolGraphOptions(), SM.get(), &g); return false; @@ -1780,9 +1788,9 @@ static bool performCompileStepsPostSILGen(CompilerInstance &Instance, std::vector ParallelOutputFilenames = opts.InputsAndOutputs.copyOutputFilenames(); llvm::GlobalVariable *HashGlobal; - auto IRModule = generateIR( - IRGenOpts, Invocation.getTBDGenOptions(), std::move(SM), PSPs, - OutputFilename, MSF, HashGlobal, ParallelOutputFilenames); + auto IRModule = + generateIR(IRGenOpts, Invocation.getTBDGenOptions(), std::move(SM), PSPs, + OutputFilename, MSF, HashGlobal, ParallelOutputFilenames); // Cancellation check after IRGen. if (Instance.isCancellationRequested()) @@ -2324,6 +2332,23 @@ int swift::performFrontend(ArrayRef Args, PDC.setSuppressOutput(true); } + CompilerInstance::HashingBackendPtrTy HashBackend = nullptr; + if (Invocation.getFrontendOptions().DeterministicCheck) { + // Setup a verfication instance to run. + std::unique_ptr VerifyInstance = + std::make_unique(); + std::string InstanceSetupError; + // This should not fail because it passed already. + (void)VerifyInstance->setup(Invocation, InstanceSetupError); + + // Run the first time without observer and discard return value; + int ReturnValueTest = 0; + (void)performCompile(*VerifyInstance, ReturnValueTest, + /*observer*/ nullptr); + // Get the hashing output backend and free the compiler instance. + HashBackend = VerifyInstance->getHashingBackend(); + } + int ReturnValue = 0; bool HadError = performCompile(*Instance, ReturnValue, observer); @@ -2338,6 +2363,44 @@ int swift::performFrontend(ArrayRef Args, } } + if (Invocation.getFrontendOptions().DeterministicCheck) { + // Collect all output files. + auto ReHashBackend = Instance->getHashingBackend(); + std::set AllOutputs; + llvm::for_each(HashBackend->outputFiles(), [&](StringRef F) { + AllOutputs.insert(F.str()); + }); + llvm::for_each(ReHashBackend->outputFiles(), [&](StringRef F) { + AllOutputs.insert(F.str()); + }); + + DiagnosticEngine &diags = Instance->getDiags(); + for (auto &Filename : AllOutputs) { + auto O1 = HashBackend->getHashValueForFile(Filename); + if (!O1) { + diags.diagnose(SourceLoc(), diag::error_output_missing, Filename, + /*SecondRun=*/false); + HadError = true; + continue; + } + auto O2 = ReHashBackend->getHashValueForFile(Filename); + if (!O2) { + diags.diagnose(SourceLoc(), diag::error_output_missing, Filename, + /*SecondRun=*/true); + HadError = true; + continue; + } + if (*O1 != *O2) { + diags.diagnose(SourceLoc(), diag::error_nondeterministic_output, + Filename, *O1, *O2); + HadError = true; + continue; + } + diags.diagnose(SourceLoc(), diag::matching_output_produced, Filename, + *O1); + } + } + auto r = finishDiagProcessing(HadError ? 1 : ReturnValue, verifierEnabled); if (auto *StatsReporter = Instance->getStatsReporter()) StatsReporter->noteCurrentProcessExitStatus(r); diff --git a/lib/FrontendTool/ImportedModules.cpp b/lib/FrontendTool/ImportedModules.cpp index 595c9351a3b03..39d9cbf58c4bd 100644 --- a/lib/FrontendTool/ImportedModules.cpp +++ b/lib/FrontendTool/ImportedModules.cpp @@ -24,6 +24,7 @@ #include "llvm/ADT/SetVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/VirtualOutputBackend.h" using namespace swift; @@ -43,16 +44,15 @@ static void findAllClangImports(const clang::Module *module, } bool swift::emitImportedModules(ModuleDecl *mainModule, - const FrontendOptions &opts) { + const FrontendOptions &opts, + llvm::vfs::OutputBackend &backend) { auto &Context = mainModule->getASTContext(); std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); - std::error_code EC; - llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::OF_None); - - if (out.has_error() || EC) { - Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, - EC.message()); - out.clear_error(); + auto &diags = Context.Diags; + auto out = backend.createFile(path); + if (!out) { + diags.diagnose(SourceLoc(), diag::error_opening_output, + path, toString(out.takeError())); return true; } @@ -110,7 +110,13 @@ bool swift::emitImportedModules(ModuleDecl *mainModule, } for (auto name : Modules) { - out << name << "\n"; + *out << name << "\n"; + } + + if (auto error = out->keep()) { + diags.diagnose(SourceLoc(), diag::error_closing_output, + path, toString(std::move(error))); + return true; } return false; diff --git a/lib/FrontendTool/MakeStyleDependencies.cpp b/lib/FrontendTool/MakeStyleDependencies.cpp index 38898bbae791f..4aeea9701b47b 100644 --- a/lib/FrontendTool/MakeStyleDependencies.cpp +++ b/lib/FrontendTool/MakeStyleDependencies.cpp @@ -20,6 +20,7 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Support/VirtualOutputBackend.h" using namespace swift; @@ -91,18 +92,16 @@ reversePathSortedFilenames(const Container &elts) { bool swift::emitMakeDependenciesIfNeeded(DiagnosticEngine &diags, DependencyTracker *depTracker, const FrontendOptions &opts, - const InputFile &input) { + const InputFile &input, + llvm::vfs::OutputBackend &backend) { auto dependenciesFilePath = input.getDependenciesFilePath(); if (dependenciesFilePath.empty()) return false; - std::error_code EC; - llvm::raw_fd_ostream out(dependenciesFilePath, EC, llvm::sys::fs::OF_None); - - if (out.has_error() || EC) { + auto out = backend.createFile(dependenciesFilePath); + if (!out) { diags.diagnose(SourceLoc(), diag::error_opening_output, - dependenciesFilePath, EC.message()); - out.clear_error(); + dependenciesFilePath, toString(out.takeError())); return true; } @@ -138,8 +137,14 @@ bool swift::emitMakeDependenciesIfNeeded(DiagnosticEngine &diags, // dependency line. opts.forAllOutputPaths(input, [&](const StringRef targetName) { auto targetNameEscaped = frontend::utils::escapeForMake(targetName, buffer); - out << targetNameEscaped << " :" << dependencyString << '\n'; + *out << targetNameEscaped << " :" << dependencyString << '\n'; }); + if (auto error = out->keep()) { + diags.diagnose(SourceLoc(), diag::error_closing_output, + dependenciesFilePath, toString(std::move(error))); + return true; + } + return false; } diff --git a/lib/IRGen/IRGen.cpp b/lib/IRGen/IRGen.cpp index 2df7a065bf92e..e653747e52546 100644 --- a/lib/IRGen/IRGen.cpp +++ b/lib/IRGen/IRGen.cpp @@ -46,6 +46,7 @@ #include "swift/Subsystems.h" #include "clang/Basic/TargetInfo.h" #include "clang/Frontend/CompilerInstance.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/StringSet.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Bitcode/BitcodeWriter.h" @@ -74,6 +75,8 @@ #include "llvm/Support/FormattedStream.h" #include "llvm/Support/Mutex.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputConfig.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" @@ -534,6 +537,7 @@ bool swift::performLLVM(const IRGenOptions &Opts, llvm::Module *Module, llvm::TargetMachine *TargetMachine, StringRef OutputFilename, + llvm::vfs::OutputBackend &Backend, UnifiedStatsReporter *Stats) { if (Opts.UseIncrementalLLVMCodeGen && HashGlobal) { @@ -552,7 +556,7 @@ bool swift::performLLVM(const IRGenOptions &Opts, ArrayRef HashData(reinterpret_cast(&hash), sizeof(hash)); if (Opts.OutputKind == IRGenOutputKind::ObjectFile && - !Opts.PrintInlineTree && + !Opts.PrintInlineTree && !Opts.AlwaysCompile && !needsRecompile(OutputFilename, HashData, HashGlobal, DiagMutex)) { // The llvm IR did not change. We don't need to re-create the object file. return false; @@ -564,23 +568,28 @@ bool swift::performLLVM(const IRGenOptions &Opts, HashGlobal->setInitializer(HashConstant); } - Optional RawOS; + llvm::Optional OutputFile; + SWIFT_DEFER { + if (!OutputFile) + return; + if (auto E = OutputFile->keep()) { + diagnoseSync(Diags, DiagMutex, SourceLoc(), diag::error_closing_output, + OutputFilename, toString(std::move(E))); + } + }; if (!OutputFilename.empty()) { // Try to open the output file. Clobbering an existing file is fine. // Open in binary mode if we're doing binary output. - llvm::sys::fs::OpenFlags OSFlags = llvm::sys::fs::OF_None; - std::error_code EC; - RawOS.emplace(OutputFilename, EC, OSFlags); - - if (RawOS->has_error() || EC) { - diagnoseSync(Diags, DiagMutex, - SourceLoc(), diag::error_opening_output, - OutputFilename, EC.message()); - RawOS->clear_error(); + llvm::vfs::OutputConfig Config; + if (auto E = + Backend.createFile(OutputFilename, Config).moveInto(OutputFile)) { + diagnoseSync(Diags, DiagMutex, SourceLoc(), diag::error_opening_output, + OutputFilename, toString(std::move(E))); return true; } + if (Opts.OutputKind == IRGenOutputKind::LLVMAssemblyBeforeOptimization) { - Module->print(RawOS.value(), nullptr); + Module->print(*OutputFile, nullptr); return false; } } else { @@ -588,7 +597,7 @@ bool swift::performLLVM(const IRGenOptions &Opts, } performLLVMOptimizations(Opts, Module, TargetMachine, - RawOS ? &*RawOS : nullptr); + OutputFile ? &OutputFile->getOS() : nullptr); if (Stats) { if (DiagMutex) @@ -598,11 +607,11 @@ bool swift::performLLVM(const IRGenOptions &Opts, DiagMutex->unlock(); } - if (!RawOS) + if (OutputFilename.empty()) return false; - return compileAndWriteLLVM(Module, TargetMachine, Opts, Stats, Diags, *RawOS, - DiagMutex); + return compileAndWriteLLVM(Module, TargetMachine, Opts, Stats, Diags, + *OutputFile, DiagMutex); } bool swift::compileAndWriteLLVM(llvm::Module *module, @@ -1241,7 +1250,8 @@ struct LLVMCodeGenThreads { embedBitcode(IGM->getModule(), parent.irgen->Opts); performLLVM(parent.irgen->Opts, IGM->Context.Diags, diagMutex, IGM->ModuleHash, IGM->getModule(), IGM->TargetMachine.get(), - IGM->OutputFilename, IGM->Context.Stats); + IGM->OutputFilename, IGM->Context.getOutputBackend(), + IGM->Context.Stats); if (IGM->Context.Diags.hadAnyError()) return; } @@ -1359,10 +1369,9 @@ static void performParallelIRGeneration(IRGenDescriptor desc) { if (!targetMachine) continue; // Create the IR emitter. - IRGenModule *IGM = - new IRGenModule(irgen, std::move(targetMachine), nextSF, - desc.ModuleName, *OutputIter++, nextSF->getFilename(), - nextSF->getPrivateDiscriminator().str()); + IRGenModule *IGM = new IRGenModule( + irgen, std::move(targetMachine), nextSF, desc.ModuleName, *OutputIter++, + nextSF->getFilename(), nextSF->getPrivateDiscriminator().str()); IGMcreated = true; initLLVMModule(*IGM, *SILMod); @@ -1535,9 +1544,9 @@ GeneratedModule swift::performIRGeneration( const auto *SILModPtr = SILMod.get(); const auto &SILOpts = SILModPtr->getOptions(); auto desc = IRGenDescriptor::forWholeModule( - M, Opts, TBDOpts, SILOpts, SILModPtr->Types, std::move(SILMod), - ModuleName, PSPs, /*symsToEmit*/ None, parallelOutputFilenames, - outModuleHash); + M, Opts, TBDOpts, SILOpts, SILModPtr->Types, + std::move(SILMod), ModuleName, PSPs, /*symsToEmit*/ None, + parallelOutputFilenames, outModuleHash); if (Opts.shouldPerformIRGenerationInParallel() && !parallelOutputFilenames.empty() && @@ -1561,15 +1570,14 @@ performIRGeneration(FileUnit *file, const IRGenOptions &Opts, const auto *SILModPtr = SILMod.get(); const auto &SILOpts = SILModPtr->getOptions(); auto desc = IRGenDescriptor::forFile( - file, Opts, TBDOpts, SILOpts, SILModPtr->Types, std::move(SILMod), - ModuleName, PSPs, PrivateDiscriminator, /*symsToEmit*/ None, - outModuleHash); + file, Opts, TBDOpts, SILOpts, SILModPtr->Types, + std::move(SILMod), ModuleName, PSPs, PrivateDiscriminator, + /*symsToEmit*/ None, outModuleHash); return llvm::cantFail(file->getASTContext().evaluator(IRGenRequest{desc})); } -void -swift::createSwiftModuleObjectFile(SILModule &SILMod, StringRef Buffer, - StringRef OutputPath) { +void swift::createSwiftModuleObjectFile(SILModule &SILMod, StringRef Buffer, + StringRef OutputPath) { auto &Ctx = SILMod.getASTContext(); assert(!Ctx.hadError()); @@ -1620,7 +1628,7 @@ swift::createSwiftModuleObjectFile(SILModule &SILMod, StringRef Buffer, ASTSym->setAlignment(llvm::MaybeAlign(serialization::SWIFTMODULE_ALIGNMENT)); ::performLLVM(Opts, Ctx.Diags, nullptr, nullptr, IGM.getModule(), IGM.TargetMachine.get(), - OutputPath, Ctx.Stats); + OutputPath, Ctx.getOutputBackend(), Ctx.Stats); } bool swift::performLLVM(const IRGenOptions &Opts, ASTContext &Ctx, @@ -1636,8 +1644,8 @@ bool swift::performLLVM(const IRGenOptions &Opts, ASTContext &Ctx, embedBitcode(Module, Opts); if (::performLLVM(Opts, Ctx.Diags, nullptr, nullptr, Module, - TargetMachine.get(), - OutputFilename, Ctx.Stats)) + TargetMachine.get(), OutputFilename, Ctx.getOutputBackend(), + Ctx.Stats)) return true; return false; } diff --git a/lib/IRGen/IRGenModule.cpp b/lib/IRGen/IRGenModule.cpp index 5537135d163ba..75ee76b427684 100644 --- a/lib/IRGen/IRGenModule.cpp +++ b/lib/IRGen/IRGenModule.cpp @@ -2034,7 +2034,6 @@ bool swift::writeEmptyOutputFilesFor( const ASTContext &Context, std::vector& ParallelOutputFilenames, const IRGenOptions &IRGenOpts) { - for (auto fileName : ParallelOutputFilenames) { // The first output file, was use for genuine output. if (fileName == ParallelOutputFilenames[0]) diff --git a/lib/Immediate/Immediate.cpp b/lib/Immediate/Immediate.cpp index 74195af2dff94..5e20fe1485b25 100644 --- a/lib/Immediate/Immediate.cpp +++ b/lib/Immediate/Immediate.cpp @@ -248,7 +248,8 @@ int swift::RunImmediately(CompilerInstance &CI, performLLVM(IRGenOpts, Context.Diags, /*diagMutex*/ nullptr, /*hash*/ nullptr, GenModule.getModule(), GenModule.getTargetMachine(), - PSPs.OutputFilename, Context.Stats); + PSPs.OutputFilename, CI.getOutputBackend(), + Context.Stats); if (Context.hadError()) return -1; diff --git a/lib/SymbolGraphGen/SymbolGraphGen.cpp b/lib/SymbolGraphGen/SymbolGraphGen.cpp index 8256c84d89291..101c34b1b2b11 100644 --- a/lib/SymbolGraphGen/SymbolGraphGen.cpp +++ b/lib/SymbolGraphGen/SymbolGraphGen.cpp @@ -40,11 +40,13 @@ int serializeSymbolGraph(SymbolGraph &SG, SmallString<1024> OutputPath(Options.OutputDir); llvm::sys::path::append(OutputPath, FileName); - return withOutputFile(SG.M.getASTContext().Diags, OutputPath, [&](raw_ostream &OS) { - llvm::json::OStream J(OS, Options.PrettyPrint ? 2 : 0); - SG.serialize(J); - return false; - }); + return withOutputPath( + SG.M.getASTContext().Diags, SG.M.getASTContext().getOutputBackend(), + OutputPath, [&](raw_ostream &OS) { + llvm::json::OStream J(OS, Options.PrettyPrint ? 2 : 0); + SG.serialize(J); + return false; + }); } } // end anonymous namespace diff --git a/test/Frontend/output_determinism_check.swift b/test/Frontend/output_determinism_check.swift new file mode 100644 index 0000000000000..318ffaaead8f5 --- /dev/null +++ b/test/Frontend/output_determinism_check.swift @@ -0,0 +1,40 @@ +// RUN: %empty-directory(%t) +// RUN: %target-swift-frontend -module-name test -emit-module -o %t/test.swiftmodule -primary-file %s -emit-module-doc-path %t/test.docc -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=MODULE_OUTPUT --check-prefix=DOCC_OUTPUT +// RUN: %target-swift-frontend -module-name test -emit-sib -o %t/test.sib -primary-file %s -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=SIB_OUTPUT + +/// object files are "not" deterministic because the second run going to match the mod hash and skip code generation. +// RUN: not %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=OBJECT_MISMATCH +/// object files should match when forcing object generation. +// RUN: %target-swift-frontend -module-name test -emit-dependencies -c -o %t/test.o -primary-file %s -enable-deterministic-check -always-compile-output-files 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT --check-prefix=DEPS_OUTPUT + +/// FIXME: Fine-grain dependencies graph is not deterministics. +/// FAIL: %target-swift-frontend -module-name test -emit-reference-dependencies-path %t/test.swiftdeps -c -o %t/test.o -primary-file %s -enable-deterministic-check -always-compile-output-files + +/// Explicit module build. Check building swiftmodule from interface file. +// RUN: %target-swift-frontend -scan-dependencies -module-name test -o %t/test.json %s -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=DEPSCAN_OUTPUT +/// TODO: Implicit module build use a different compiler instance so it doesn't support checking yet. +// RUN: %target-swift-frontend -typecheck -emit-module-interface-path %t/test.swiftinterface %s -O -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=INTERFACE_OUTPUT +/// Hit cache and not emit the second time. +// RUN: rm %t/test.swiftmodule +// RUN: not %target-swift-frontend -compile-module-from-interface %t/test.swiftinterface -explicit-interface-module-build -o %t/test.swiftmodule -enable-deterministic-check 2>&1 | %FileCheck --check-prefix=MODULE_MISMATCH %s +/// Force swiftmodule generation. +// RUN: %target-swift-frontend -compile-module-from-interface %t/test.swiftinterface -explicit-interface-module-build -o %t/test.swiftmodule -enable-deterministic-check -always-compile-output-files 2>&1 | %FileCheck --check-prefix=MODULE_OUTPUT %s + +// RUN: %target-swift-frontend -scan-dependencies -module-name test %s -o %t/test.deps.json -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=DEPS_JSON_OUTPUT + +// RUN: %target-swift-frontend -emit-pcm -module-name UserClangModule -o %t/test.pcm %S/Inputs/dependencies/module.modulemap -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=PCM_OUTPUT + +// DOCC_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.docc' +// MODULE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftmodule' +// SIB_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.sib' +// DEPS_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.d' +// OBJECT_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.o' +// OBJECT_MISMATCH: error: output file '{{.*}}{{/|\\}}test.o' is missing from second compilation for deterministic check +// DEPSCAN_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.json' +// INTERFACE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftinterface' +// MODULE_MISMATCH: error: output file '{{.*}}{{/|\\}}test.swiftmodule' is missing from second compilation for deterministic check +// DEPS_JSON_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.deps.json' +// PCM_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.pcm' + +public var x = 1 +public func test() {} diff --git a/tools/sil-llvm-gen/SILLLVMGen.cpp b/tools/sil-llvm-gen/SILLLVMGen.cpp index 9b9ff651f456a..b5e47878b7430 100644 --- a/tools/sil-llvm-gen/SILLLVMGen.cpp +++ b/tools/sil-llvm-gen/SILLLVMGen.cpp @@ -34,6 +34,7 @@ #include "swift/Serialization/SerializedSILLoader.h" #include "swift/Strings.h" #include "swift/Subsystems.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/Statistic.h" #include "llvm/IR/Module.h" #include "llvm/Support/CommandLine.h" @@ -43,6 +44,7 @@ #include "llvm/Support/Path.h" #include "llvm/Support/Signals.h" #include "llvm/Support/TargetSelect.h" +#include "llvm/Support/VirtualOutputBackends.h" #include using namespace swift; @@ -181,14 +183,19 @@ int main(int argc, char **argv) { return 1; } - std::error_code EC; - llvm::raw_fd_ostream outStream(OutputFilename, EC, llvm::sys::fs::OF_None); - if (outStream.has_error() || EC) { + llvm::vfs::OnDiskOutputBackend Backend; + auto outFile = Backend.createFile(OutputFilename); + if (!outFile) { CI.getDiags().diagnose(SourceLoc(), diag::error_opening_output, - OutputFilename, EC.message()); - outStream.clear_error(); + OutputFilename, toString(outFile.takeError())); return 1; } + auto closeFile = llvm::make_scope_exit([&]() { + if (auto E = outFile->keep()) { + CI.getDiags().diagnose(SourceLoc(), diag::error_closing_output, + OutputFilename, toString(std::move(E))); + } + }); auto *mod = CI.getMainModule(); assert(mod->getFiles().size() == 1); @@ -205,20 +212,20 @@ int main(int argc, char **argv) { mod, Opts, TBDOpts, SILOpts, SILTypes, /*SILMod*/ nullptr, moduleName, PSPs); } else { - return IRGenDescriptor::forFile(mod->getFiles()[0], Opts, TBDOpts, - SILOpts, SILTypes, /*SILMod*/ nullptr, - moduleName, PSPs, /*discriminator*/ ""); + return IRGenDescriptor::forFile( + mod->getFiles()[0], Opts, TBDOpts, SILOpts, SILTypes, + /*SILMod*/ nullptr, moduleName, PSPs, /*discriminator*/ ""); } }; auto &eval = CI.getASTContext().evaluator; auto desc = getDescriptor(); - desc.out = &outStream; + desc.out = &outFile->getOS(); auto generatedMod = llvm::cantFail(eval(OptimizedIRRequest{desc})); if (!generatedMod) return 1; return compileAndWriteLLVM(generatedMod.getModule(), generatedMod.getTargetMachine(), Opts, - CI.getStatsReporter(), CI.getDiags(), outStream); + CI.getStatsReporter(), CI.getDiags(), *outFile); } diff --git a/tools/swift-dependency-tool/swift-dependency-tool.cpp b/tools/swift-dependency-tool/swift-dependency-tool.cpp index cd4854b2fc9df..35737d832d63e 100644 --- a/tools/swift-dependency-tool/swift-dependency-tool.cpp +++ b/tools/swift-dependency-tool/swift-dependency-tool.cpp @@ -18,6 +18,7 @@ #include "swift/Basic/LLVMInitialize.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackends.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" @@ -208,6 +209,7 @@ int main(int argc, char *argv[]) { SourceManager sourceMgr; DiagnosticEngine diags(sourceMgr); + llvm::vfs::OnDiskOutputBackend outputBackend; switch (options::Action) { case ActionType::None: { @@ -224,7 +226,7 @@ int main(int argc, char *argv[]) { } bool hadError = - withOutputFile(diags, options::OutputFilename, + withOutputPath(diags, outputBackend, options::OutputFilename, [&](llvm::raw_pwrite_stream &out) { out << "# Fine-grained v0\n"; llvm::yaml::Output yamlWriter(out); @@ -256,7 +258,7 @@ int main(int argc, char *argv[]) { } if (writeFineGrainedDependencyGraphToPath( - diags, options::OutputFilename, fg)) { + diags, outputBackend, options::OutputFilename, fg)) { llvm::errs() << "Failed to write binary swiftdeps\n"; return 1; } diff --git a/unittests/ClangImporter/ClangImporterTests.cpp b/unittests/ClangImporter/ClangImporterTests.cpp index 074e660f97f8a..f5f631dc0af12 100644 --- a/unittests/ClangImporter/ClangImporterTests.cpp +++ b/unittests/ClangImporter/ClangImporterTests.cpp @@ -7,8 +7,10 @@ #include "swift/Basic/SourceManager.h" #include "swift/ClangImporter/ClangImporter.h" #include "swift/SymbolGraphGen/SymbolGraphOptions.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackends.h" #include "llvm/Support/raw_ostream.h" #include "gtest/gtest.h" diff --git a/unittests/Driver/MockingFineGrainedDependencyGraphs.cpp b/unittests/Driver/MockingFineGrainedDependencyGraphs.cpp index 327f871af8c50..1ce0f2fe3d534 100644 --- a/unittests/Driver/MockingFineGrainedDependencyGraphs.cpp +++ b/unittests/Driver/MockingFineGrainedDependencyGraphs.cpp @@ -15,6 +15,7 @@ #include "swift/AST/DiagnosticEngine.h" #include "swift/Basic/ReferenceDependencyKeys.h" #include "swift/Basic/SourceManager.h" +#include "llvm/Support/VirtualOutputBackends.h" #include #include @@ -48,12 +49,13 @@ mocking_fine_grained_dependency_graphs::getChangesForSimulatedLoad( SourceManager sm; DiagnosticEngine diags(sm); + llvm::vfs::OnDiskOutputBackend backend; auto sfdg = UnitTestSourceFileDepGraphFactory( hadCompilationError, swiftDeps, interfaceHash, g.emitFineGrainedDependencyDotFileAfterEveryImport, - dependencyDescriptions, diags) + dependencyDescriptions, diags, backend) .construct(); return g.loadFromSourceFileDepGraph(cmd, sfdg, diags); diff --git a/unittests/Driver/UnitTestSourceFileDepGraphFactory.h b/unittests/Driver/UnitTestSourceFileDepGraphFactory.h index 2a53bf89521c2..90f32a1239273 100644 --- a/unittests/Driver/UnitTestSourceFileDepGraphFactory.h +++ b/unittests/Driver/UnitTestSourceFileDepGraphFactory.h @@ -13,6 +13,7 @@ #define UnitTestSourceFileDepGraphFactory_h #include "swift/AST/AbstractSourceFileDepGraphFactory.h" +#include "llvm/Support/VirtualOutputBackend.h" namespace swift { namespace fine_grained_dependencies { @@ -30,10 +31,10 @@ class UnitTestSourceFileDepGraphFactory bool hadCompilationError, StringRef swiftDeps, Fingerprint fileFingerprint, bool emitDotFileAfterConstruction, const DependencyDescriptions &dependencyDescriptions, - DiagnosticEngine &diags) + DiagnosticEngine &diags, llvm::vfs::OutputBackend &backend) : AbstractSourceFileDepGraphFactory( hadCompilationError, swiftDeps, fileFingerprint, - emitDotFileAfterConstruction, diags), + emitDotFileAfterConstruction, diags, backend), dependencyDescriptions(dependencyDescriptions) {} ~UnitTestSourceFileDepGraphFactory() override = default;