diff --git a/include/swift/APIDigester/ModuleAnalyzerNodes.h b/include/swift/APIDigester/ModuleAnalyzerNodes.h index 367763d5f4870..13b54ac41a405 100644 --- a/include/swift/APIDigester/ModuleAnalyzerNodes.h +++ b/include/swift/APIDigester/ModuleAnalyzerNodes.h @@ -784,8 +784,8 @@ class SwiftDeclCollector: public VisibleDeclConsumer { void deSerialize(StringRef Filename); // Serialize the content of all roots to a given file using JSON format. - void serialize(StringRef Filename); - static void serialize(StringRef Filename, SDKNode *Root, PayLoad otherInfo); + void serialize(llvm::raw_ostream &os); + static void serialize(llvm::raw_ostream &os, SDKNode *Root, PayLoad otherInfo); // After collecting decls, either from imported modules or from a previously // serialized JSON file, using this function to get the root of the SDK. @@ -835,14 +835,15 @@ SDKNodeRoot *getSDKNodeRoot(SDKContext &SDKCtx, SDKNodeRoot *getEmptySDKNodeRoot(SDKContext &SDKCtx); -void dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, StringRef OutputFile); -void dumpSDKRoot(SDKNodeRoot *Root, StringRef OutputFile); +void dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, llvm::raw_ostream &os); +void dumpSDKRoot(SDKNodeRoot *Root, llvm::raw_ostream &os); int dumpSDKContent(const CompilerInvocation &InitInvoke, const llvm::StringSet<> &ModuleNames, - StringRef OutputFile, CheckerOptions Opts); + llvm::raw_ostream &os, CheckerOptions Opts); -void dumpModuleContent(ModuleDecl *MD, StringRef OutputFile, bool ABI, bool Empty); +void dumpModuleContent(ModuleDecl *MD, llvm::raw_ostream &os, bool ABI, + bool Empty); /// Mostly for testing purposes, this function de-serializes the SDK dump in /// dumpPath and re-serialize them to OutputPath. If the tool performs correctly, diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h index 3c763b23a1e49..341a115155748 100644 --- a/include/swift/AST/ASTContext.h +++ b/include/swift/AST/ASTContext.h @@ -44,6 +44,7 @@ #include "llvm/ADT/TinyPtrVector.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/DataTypes.h" +#include "llvm/Support/VirtualOutputBackend.h" #include #include #include @@ -231,6 +232,7 @@ class ASTContext final { ClangImporterOptions &ClangImporterOpts, symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts, SourceManager &SourceMgr, DiagnosticEngine &Diags, + llvm::IntrusiveRefCntPtr OutBackend = nullptr, std::function PreModuleImportCallback = {}); public: @@ -248,6 +250,7 @@ class ASTContext final { ClangImporterOptions &ClangImporterOpts, symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts, SourceManager &SourceMgr, DiagnosticEngine &Diags, + llvm::IntrusiveRefCntPtr OutBackend = nullptr, std::function PreModuleImportCallback = {}); ~ASTContext(); @@ -281,6 +284,9 @@ class ASTContext final { /// Diags - The diagnostics engine. DiagnosticEngine &Diags; + /// OutputBackend for writing outputs. + llvm::IntrusiveRefCntPtr OutputBackend; + /// If the shared pointer is not a \c nullptr and the pointee is \c true, /// all operations working on this ASTContext should be aborted at the next /// possible opportunity. @@ -1516,6 +1522,18 @@ class ASTContext final { const llvm::StringSet<> &getLoadedPluginLibraryPaths() const; + /// Get the output backend. The output backend needs to be initialized via + /// constructor or `setOutputBackend`. + llvm::vfs::OutputBackend &getOutputBackend() const { + assert(OutputBackend && "OutputBackend is not setup"); + return *OutputBackend; + } + /// Set output backend for virtualized outputs. + void setOutputBackend( + llvm::IntrusiveRefCntPtr OutBackend) { + OutputBackend = std::move(OutBackend); + } + private: friend Decl; diff --git a/include/swift/AST/AbstractSourceFileDepGraphFactory.h b/include/swift/AST/AbstractSourceFileDepGraphFactory.h index 6858627dfb5cf..1f44f3f3106cd 100644 --- a/include/swift/AST/AbstractSourceFileDepGraphFactory.h +++ b/include/swift/AST/AbstractSourceFileDepGraphFactory.h @@ -16,6 +16,7 @@ #include "swift/AST/Decl.h" #include "swift/AST/DeclContext.h" #include "swift/AST/FineGrainedDependencies.h" +#include "llvm/Support/VirtualOutputBackend.h" namespace swift { class DiagnosticEngine; @@ -39,6 +40,9 @@ class AbstractSourceFileDepGraphFactory { DiagnosticEngine &diags; + /// OutputBackend. + llvm::vfs::OutputBackend &backend; + /// Graph under construction SourceFileDepGraph g; @@ -49,7 +53,8 @@ class AbstractSourceFileDepGraphFactory { StringRef swiftDeps, Fingerprint fileFingerprint, bool emitDotFileAfterConstruction, - DiagnosticEngine &diags); + DiagnosticEngine &diags, + llvm::vfs::OutputBackend &outputBackend); virtual ~AbstractSourceFileDepGraphFactory() = default; diff --git a/include/swift/AST/DiagnosticsCommon.def b/include/swift/AST/DiagnosticsCommon.def index 535dbd475eabb..f438e7a93b31e 100644 --- a/include/swift/AST/DiagnosticsCommon.def +++ b/include/swift/AST/DiagnosticsCommon.def @@ -29,6 +29,9 @@ ERROR(not_implemented,none, ERROR(error_opening_output,none, "error opening '%0' for output: %1", (StringRef, StringRef)) +ERROR(error_closing_output,none, + "error closing '%0' for output: %1", (StringRef, StringRef)) + ERROR(cannot_find_group_info_file,none, "cannot find group info file at path: '%0'", (StringRef)) diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def index c7316754251f0..4c51234e50b56 100644 --- a/include/swift/AST/DiagnosticsFrontend.def +++ b/include/swift/AST/DiagnosticsFrontend.def @@ -478,6 +478,16 @@ REMARK(interface_file_backup_used,none, WARNING(warn_flag_deprecated,none, "flag '%0' is deprecated", (StringRef)) +// Output deterministic check +ERROR(error_nondeterministic_output,none, + "output file '%0' is not deterministic: hash value '%1' vs '%2' between two compilations", + (StringRef, StringRef, StringRef)) +ERROR(error_output_missing,none, + "output file '%0' is missing from %select{first|second}1 compilation for deterministic check", + (StringRef, /*SecondRun=*/bool)) +REMARK(matching_output_produced,none, + "produced matching output file '%0' for deterministic check: hash '%1'", (StringRef, StringRef)) + // Dependency Verifier Diagnostics ERROR(missing_member_dependency,none, "expected " diff --git a/include/swift/AST/FileSystem.h b/include/swift/AST/FileSystem.h index 15641164c3e08..69f4c5dd7ffce 100644 --- a/include/swift/AST/FileSystem.h +++ b/include/swift/AST/FileSystem.h @@ -16,29 +16,42 @@ #include "swift/Basic/FileSystem.h" #include "swift/AST/DiagnosticEngine.h" #include "swift/AST/DiagnosticsCommon.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputConfig.h" namespace swift { - -/// A wrapper around swift::atomicallyWritingToFile that handles diagnosing any -/// filesystem errors and asserts the output path is nonempty. +/// A wrapper around llvm::vfs::OutputBackend to handle diagnosing any file +/// system errors during output creation. /// /// \returns true if there were any errors, either from the filesystem /// operations or from \p action returning true. inline bool -withOutputFile(DiagnosticEngine &diags, StringRef outputPath, +withOutputPath(DiagnosticEngine &diags, llvm::vfs::OutputBackend &Backend, + StringRef outputPath, llvm::function_ref action) { assert(!outputPath.empty()); + llvm::vfs::OutputConfig config; + config.setAtomicWrite().setOnlyIfDifferent(); - bool actionFailed = false; - std::error_code EC = swift::atomicallyWritingToFile( - outputPath, - [&](llvm::raw_pwrite_stream &out) { actionFailed = action(out); }); - if (EC) { + auto outputFile = Backend.createFile(outputPath, config); + if (!outputFile) { diags.diagnose(SourceLoc(), diag::error_opening_output, outputPath, - EC.message()); + toString(outputFile.takeError())); + return true; + } + + bool failed = action(*outputFile); + // If there is an error, discard output. Otherwise keep the output file. + if (auto error = failed ? outputFile->discard() : outputFile->keep()) { + // Don't diagnose discard error. + if (failed) + consumeError(std::move(error)); + else + diags.diagnose(SourceLoc(), diag::error_closing_output, outputPath, + toString(std::move(error))); return true; } - return actionFailed; + return failed; } } // end namespace swift diff --git a/include/swift/AST/FineGrainedDependencies.h b/include/swift/AST/FineGrainedDependencies.h index 254d3a0d79002..ffac726152c42 100644 --- a/include/swift/AST/FineGrainedDependencies.h +++ b/include/swift/AST/FineGrainedDependencies.h @@ -24,6 +24,7 @@ #include "llvm/ADT/SetVector.h" #include "llvm/Support/MD5.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" #include "llvm/Support/raw_ostream.h" @@ -357,8 +358,9 @@ class BiIndexedTwoStageMap { /// \Note The returned graph should not be escaped from the callback. bool withReferenceDependencies( llvm::PointerUnion MSF, - const DependencyTracker &depTracker, StringRef outputPath, - bool alsoEmitDotFile, llvm::function_ref); + const DependencyTracker &depTracker, llvm::vfs::OutputBackend &backend, + StringRef outputPath, bool alsoEmitDotFile, + llvm::function_ref); //============================================================================== // MARK: Enums @@ -895,7 +897,8 @@ class SourceFileDepGraph { bool verifySequenceNumber() const; - void emitDotFile(StringRef outputPath, DiagnosticEngine &diags); + void emitDotFile(llvm::vfs::OutputBackend &outputBackend, + StringRef outputPath, DiagnosticEngine &diags); void addNode(SourceFileDepGraphNode *n) { n->setSequenceNumber(allNodes.size()); diff --git a/include/swift/AST/FineGrainedDependencyFormat.h b/include/swift/AST/FineGrainedDependencyFormat.h index 0957d03d98353..086de7e60b2c7 100644 --- a/include/swift/AST/FineGrainedDependencyFormat.h +++ b/include/swift/AST/FineGrainedDependencyFormat.h @@ -18,6 +18,9 @@ namespace llvm { class MemoryBuffer; +namespace vfs { +class OutputBackend; +} } namespace swift { @@ -132,6 +135,7 @@ bool readFineGrainedDependencyGraph(llvm::StringRef path, /// Tries to write the dependency graph to the given path name. /// Returns true if there was an error. bool writeFineGrainedDependencyGraphToPath(DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend, llvm::StringRef path, const SourceFileDepGraph &g); diff --git a/include/swift/AST/IRGenOptions.h b/include/swift/AST/IRGenOptions.h index 0c1e8a67c32eb..bfb6f6efd413a 100644 --- a/include/swift/AST/IRGenOptions.h +++ b/include/swift/AST/IRGenOptions.h @@ -318,6 +318,9 @@ class IRGenOptions { /// Print the LLVM inline tree at the end of the LLVM pass pipeline. unsigned PrintInlineTree : 1; + /// Always recompile the output even if the module hash might match. + unsigned AlwaysCompile : 1; + /// Whether we should embed the bitcode file. IRGenEmbedMode EmbedMode : 2; @@ -481,7 +484,7 @@ class IRGenOptions { DisableLLVMOptzns(false), DisableSwiftSpecificLLVMOptzns(false), Playground(false), EmitStackPromotionChecks(false), UseSingleModuleLLVMEmission(false), - FunctionSections(false), PrintInlineTree(false), + FunctionSections(false), PrintInlineTree(false), AlwaysCompile(false), EmbedMode(IRGenEmbedMode::None), LLVMLTOKind(IRGenLLVMLTOKind::None), SwiftAsyncFramePointer(SwiftAsyncFramePointerKind::Auto), HasValueNamesSetting(false), ValueNames(false), diff --git a/include/swift/AST/ModuleLoader.h b/include/swift/AST/ModuleLoader.h index 60ef5ac69693f..0072984634251 100644 --- a/include/swift/AST/ModuleLoader.h +++ b/include/swift/AST/ModuleLoader.h @@ -32,6 +32,9 @@ namespace llvm { class FileCollectorBase; +namespace vfs { +class OutputBackend; +} } namespace clang { @@ -156,6 +159,7 @@ class ModuleInterfaceChecker { virtual bool tryEmitForwardingModule(StringRef moduleName, StringRef interfacePath, ArrayRef candidates, + llvm::vfs::OutputBackend &backend, StringRef outPath) = 0; virtual ~ModuleInterfaceChecker() = default; }; diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h index 3bbeb8f4d1e2e..5ad232c279a6b 100644 --- a/include/swift/ClangImporter/ClangImporter.h +++ b/include/swift/ClangImporter/ClangImporter.h @@ -28,6 +28,7 @@ namespace llvm { template class function_ref; namespace vfs { class FileSystem; + class OutputBackend; } } @@ -394,8 +395,7 @@ class ClangImporter final : public ClangModuleLoader { /// replica. /// /// \sa clang::GeneratePCHAction - bool emitBridgingPCH(StringRef headerPath, - StringRef outputPCHPath); + bool emitBridgingPCH(StringRef headerPath, StringRef outputPCHPath); /// Returns true if a clang CompilerInstance can successfully read in a PCH, /// assuming it exists, with the current options. This can be used to find out diff --git a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h index a8fa5a316140b..254c985fbc85b 100644 --- a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h +++ b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h @@ -18,6 +18,9 @@ namespace llvm { class MemoryBuffer; +namespace vfs{ +class OutputBackend; +} } namespace swift { @@ -187,6 +190,7 @@ bool readInterModuleDependenciesCache(llvm::StringRef path, /// Tries to write the dependency graph to the given path name. /// Returns true if there was an error. bool writeInterModuleDependenciesCache(DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend, llvm::StringRef path, const SwiftDependencyScanningService &cache); diff --git a/include/swift/Driver/FineGrainedDependencyDriverGraph.h b/include/swift/Driver/FineGrainedDependencyDriverGraph.h index e2266ae03a13c..f36f0c269b65c 100644 --- a/include/swift/Driver/FineGrainedDependencyDriverGraph.h +++ b/include/swift/Driver/FineGrainedDependencyDriverGraph.h @@ -20,11 +20,14 @@ #include "swift/Basic/OptionSet.h" #include "swift/Driver/Job.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Support/Path.h" #include "llvm/Support/PointerLikeTypeTraits.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputBackends.h" #include #include #include @@ -201,6 +204,9 @@ class ModuleDepGraph { std::vector> dependencyPathsToJobs; + /// VirtualOutputBackend for emitting graphs. + llvm::IntrusiveRefCntPtr backend; + /// For helping with performance tuning, may be null: UnifiedStatsReporter *stats; @@ -306,6 +312,10 @@ class ModuleDepGraph { : None), stats(stats) { assert(verify() && "ModuleDepGraph should be fine when created"); + + // Create a OnDiskOutputBackend for emitting graphs. Currently, this is + // only used by driver so the backend is not shared with a CompilerInstance. + backend = llvm::makeIntrusiveRefCnt(); } /// For unit tests. diff --git a/include/swift/Frontend/Frontend.h b/include/swift/Frontend/Frontend.h index 30785440fa5fe..9669e98d8488b 100644 --- a/include/swift/Frontend/Frontend.h +++ b/include/swift/Frontend/Frontend.h @@ -45,8 +45,11 @@ #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/SetVector.h" #include "llvm/Option/ArgList.h" +#include "llvm/Support/BLAKE3.h" +#include "llvm/Support/HashingOutputBackend.h" #include "llvm/Support/Host.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackend.h" #include @@ -457,6 +460,13 @@ class CompilerInstance { /// instance has completed its setup, this will be null. std::unique_ptr Stats; + /// Virtual OutputBackend. + llvm::IntrusiveRefCntPtr OutputBackend = nullptr; + + /// The verification output backend. + using HashBackendTy = llvm::vfs::HashingOutputBackend; + llvm::IntrusiveRefCntPtr HashBackend; + mutable ModuleDecl *MainModule = nullptr; SerializedModuleLoaderBase *DefaultSerializedLoader = nullptr; MemoryBufferSerializedModuleLoader *MemoryBufferLoader = nullptr; @@ -507,6 +517,16 @@ class CompilerInstance { SourceMgr.setFileSystem(FS); } + llvm::vfs::OutputBackend &getOutputBackend() const { + return *OutputBackend; + } + void + setOutputBackend(llvm::IntrusiveRefCntPtr Backend) { + OutputBackend = std::move(Backend); + } + using HashingBackendPtrTy = llvm::IntrusiveRefCntPtr; + HashingBackendPtrTy getHashingBackend() { return HashBackend; } + ASTContext &getASTContext() { return *Context; } const ASTContext &getASTContext() const { return *Context; } @@ -629,6 +649,7 @@ class CompilerInstance { bool setUpASTContextIfNeeded(); void setupStatsReporter(); void setupDependencyTrackerIfNeeded(); + void setupOutputBackend(); /// \return false if successful, true on error. bool setupDiagnosticVerifierIfNeeded(); diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h index b1aa4e33db151..76fda8039434e 100644 --- a/include/swift/Frontend/FrontendOptions.h +++ b/include/swift/Frontend/FrontendOptions.h @@ -498,6 +498,9 @@ class FrontendOptions { /// to encode the actual paths into the .swiftmodule file. PathObfuscator serializedPathObfuscator; + /// Whether to run the job twice to check determinism. + bool DeterministicCheck = false; + /// Avoid printing actual module content into the ABI descriptor file. /// This should only be used as a workaround when emitting ABI descriptor files /// crashes the compiler. diff --git a/include/swift/Frontend/ModuleInterfaceLoader.h b/include/swift/Frontend/ModuleInterfaceLoader.h index 1f7a6d9fa1cdd..871cdb7cb313a 100644 --- a/include/swift/Frontend/ModuleInterfaceLoader.h +++ b/include/swift/Frontend/ModuleInterfaceLoader.h @@ -447,6 +447,7 @@ class ModuleInterfaceCheckerImpl: public ModuleInterfaceChecker { bool tryEmitForwardingModule(StringRef moduleName, StringRef interfacePath, ArrayRef candidates, + llvm::vfs::OutputBackend &backend, StringRef outPath) override; bool isCached(StringRef DepPath); }; diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index 0f2713531c53e..8a3ff3da4c9e2 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -1176,6 +1176,13 @@ def enable_emit_generic_class_ro_t_list : HelpText<"Enable emission of a section with references to class_ro_t of " "generic class patterns">; +def enable_deterministic_check : + Flag<["-"], "enable-deterministic-check">, + HelpText<"Check compiler output determinisim by running it twice">; +def always_compile_output_files : + Flag<["-"], "always-compile-output-files">, + HelpText<"Always compile output files even it might not change the results">; + def experimental_spi_only_imports : Flag<["-"], "experimental-spi-only-imports">, HelpText<"Enable use of @_spiOnly imports">; diff --git a/include/swift/Subsystems.h b/include/swift/Subsystems.h index dc3b11ac6210e..b6fcda2b7bd20 100644 --- a/include/swift/Subsystems.h +++ b/include/swift/Subsystems.h @@ -35,6 +35,9 @@ namespace llvm { class Module; class TargetOptions; class TargetMachine; + namespace vfs { + class OutputBackend; + } } namespace swift { @@ -276,6 +279,7 @@ namespace swift { /// \param Module LLVM module to code gen, required. /// \param TargetMachine target of code gen, required. /// \param OutputFilename Filename for output. + /// \param Backend OutputBackend for writing output. bool performLLVM(const IRGenOptions &Opts, DiagnosticEngine &Diags, llvm::sys::Mutex *DiagMutex, @@ -283,6 +287,7 @@ namespace swift { llvm::Module *Module, llvm::TargetMachine *TargetMachine, StringRef OutputFilename, + llvm::vfs::OutputBackend &Backend, UnifiedStatsReporter *Stats); /// Dump YAML describing all fixed-size types imported from the given module. diff --git a/lib/APIDigester/ModuleAnalyzerNodes.cpp b/lib/APIDigester/ModuleAnalyzerNodes.cpp index c3d8caacc2be9..090db50ab61d6 100644 --- a/lib/APIDigester/ModuleAnalyzerNodes.cpp +++ b/lib/APIDigester/ModuleAnalyzerNodes.cpp @@ -2470,11 +2470,10 @@ void SwiftDeclCollector::deSerialize(StringRef Filename) { } // Serialize the content of all roots to a given file using JSON format. -void SwiftDeclCollector::serialize(StringRef Filename, SDKNode *Root, +void SwiftDeclCollector::serialize(llvm::raw_ostream &os, SDKNode *Root, PayLoad OtherInfo) { std::error_code EC; - llvm::raw_fd_ostream fs(Filename, EC, llvm::sys::fs::OF_None); - json::Output yout(fs); + json::Output yout(os); assert(Root->getKind() == SDKNodeKind::Root); SDKNodeRoot &root = *static_cast(Root); yout.beginObject(); @@ -2486,8 +2485,8 @@ void SwiftDeclCollector::serialize(StringRef Filename, SDKNode *Root, } // Serialize the content of all roots to a given file using JSON format. -void SwiftDeclCollector::serialize(StringRef Filename) { - SwiftDeclCollector::serialize(Filename, RootNode, PayLoad()); +void SwiftDeclCollector::serialize(llvm::raw_ostream &os) { + SwiftDeclCollector::serialize(os, RootNode, PayLoad()); } SDKNodeRoot * @@ -2550,28 +2549,26 @@ swift::ide::api::getSDKNodeRoot(SDKContext &SDKCtx, } void swift::ide::api::dumpSDKRoot(SDKNodeRoot *Root, PayLoad load, - StringRef OutputFile) { + llvm::raw_ostream &os) { assert(Root); auto Opts = Root->getSDKContext().getOpts(); if (Opts.Verbose) llvm::errs() << "Dumping SDK...\n"; - SwiftDeclCollector::serialize(OutputFile, Root, load); - if (Opts.Verbose) - llvm::errs() << "Dumped to "<< OutputFile << "\n"; + SwiftDeclCollector::serialize(os, Root, load); } -void swift::ide::api::dumpSDKRoot(SDKNodeRoot *Root, StringRef OutputFile) { - dumpSDKRoot(Root, PayLoad(), OutputFile); +void swift::ide::api::dumpSDKRoot(SDKNodeRoot *Root, llvm::raw_ostream &os) { + dumpSDKRoot(Root, PayLoad(), os); } int swift::ide::api::dumpSDKContent(const CompilerInvocation &InitInvoke, const llvm::StringSet<> &ModuleNames, - StringRef OutputFile, CheckerOptions Opts) { + llvm::raw_ostream &os, CheckerOptions Opts) { SDKContext SDKCtx(Opts); SDKNodeRoot *Root = getSDKNodeRoot(SDKCtx, InitInvoke, ModuleNames); if (!Root) return 1; - dumpSDKRoot(Root, OutputFile); + dumpSDKRoot(Root, os); return 0; } @@ -2589,11 +2586,11 @@ int swift::ide::api::deserializeSDKDump(StringRef dumpPath, StringRef OutputPath SwiftDeclCollector Collector(Ctx); Collector.deSerialize(dumpPath); - Collector.serialize(OutputPath); + Collector.serialize(FS); return 0; } -void swift::ide::api::dumpModuleContent(ModuleDecl *MD, StringRef OutputFile, +void swift::ide::api::dumpModuleContent(ModuleDecl *MD, llvm::raw_ostream &os, bool ABI, bool Empty) { CheckerOptions opts; opts.ABI = ABI; @@ -2610,7 +2607,7 @@ void swift::ide::api::dumpModuleContent(ModuleDecl *MD, StringRef OutputFile, PayLoad payload; SWIFT_DEFER { payload.allContsValues = &extractor.getAllConstValues(); - dumpSDKRoot(collector.getSDKRoot(), payload, OutputFile); + dumpSDKRoot(collector.getSDKRoot(), payload, os); }; if (Empty) { return; diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp index 9b495cdfeb8cc..da794196dbffe 100644 --- a/lib/AST/ASTContext.cpp +++ b/lib/AST/ASTContext.cpp @@ -72,6 +72,8 @@ #include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputBackends.h" #include #include #include @@ -635,6 +637,7 @@ ASTContext *ASTContext::get( ClangImporterOptions &ClangImporterOpts, symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts, SourceManager &SourceMgr, DiagnosticEngine &Diags, + llvm::IntrusiveRefCntPtr OutputBackend, std::function PreModuleImportCallback) { // If more than two data structures are concatentated, then the aggregate // size math needs to become more complicated due to per-struct alignment @@ -646,9 +649,10 @@ ASTContext *ASTContext::get( impl = reinterpret_cast( llvm::alignAddr(impl, llvm::Align(alignof(Implementation)))); new (impl) Implementation(); - return new (mem) ASTContext(langOpts, typecheckOpts, silOpts, SearchPathOpts, - ClangImporterOpts, SymbolGraphOpts, SourceMgr, - Diags, PreModuleImportCallback); + return new (mem) + ASTContext(langOpts, typecheckOpts, silOpts, SearchPathOpts, + ClangImporterOpts, SymbolGraphOpts, SourceMgr, Diags, + std::move(OutputBackend), PreModuleImportCallback); } ASTContext::ASTContext( @@ -657,17 +661,19 @@ ASTContext::ASTContext( ClangImporterOptions &ClangImporterOpts, symbolgraphgen::SymbolGraphOptions &SymbolGraphOpts, SourceManager &SourceMgr, DiagnosticEngine &Diags, + llvm::IntrusiveRefCntPtr OutBackend, std::function PreModuleImportCallback) : LangOpts(langOpts), TypeCheckerOpts(typecheckOpts), SILOpts(silOpts), SearchPathOpts(SearchPathOpts), ClangImporterOpts(ClangImporterOpts), SymbolGraphOpts(SymbolGraphOpts), SourceMgr(SourceMgr), Diags(Diags), - evaluator(Diags, langOpts), TheBuiltinModule(createBuiltinModule(*this)), + OutputBackend(std::move(OutBackend)), evaluator(Diags, langOpts), + TheBuiltinModule(createBuiltinModule(*this)), StdlibModuleName(getIdentifier(STDLIB_NAME)), SwiftShimsModuleName(getIdentifier(SWIFT_SHIMS_NAME)), PreModuleImportCallback(PreModuleImportCallback), TheErrorType(new (*this, AllocationArena::Permanent) ErrorType( *this, Type(), RecursiveTypeProperties::HasError)), - TheUnresolvedType(new (*this, AllocationArena::Permanent) + TheUnresolvedType(new(*this, AllocationArena::Permanent) UnresolvedType(*this)), TheEmptyTupleType(TupleType::get(ArrayRef(), *this)), TheEmptyPackType(PackType::get(*this, {})), @@ -707,6 +713,9 @@ ASTContext::ASTContext( registerNameLookupRequestFunctions(evaluator); createModuleToExecutablePluginMap(); + // Provide a default OnDiskOutputBackend if user didn't supply one. + if (!OutputBackend) + OutputBackend = llvm::makeIntrusiveRefCnt(); } ASTContext::~ASTContext() { diff --git a/lib/AST/AbstractSourceFileDepGraphFactory.cpp b/lib/AST/AbstractSourceFileDepGraphFactory.cpp index 1b73a66b6a42b..78bdec1b0b6ad 100644 --- a/lib/AST/AbstractSourceFileDepGraphFactory.cpp +++ b/lib/AST/AbstractSourceFileDepGraphFactory.cpp @@ -23,6 +23,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" using namespace swift; @@ -33,13 +34,13 @@ using namespace fine_grained_dependencies; //============================================================================== AbstractSourceFileDepGraphFactory::AbstractSourceFileDepGraphFactory( - bool hadCompilationError, StringRef swiftDeps, - Fingerprint fileFingerprint, bool emitDotFileAfterConstruction, - DiagnosticEngine &diags) + bool hadCompilationError, StringRef swiftDeps, Fingerprint fileFingerprint, + bool emitDotFileAfterConstruction, DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend) : hadCompilationError(hadCompilationError), swiftDeps(swiftDeps.str()), fileFingerprint(fileFingerprint), - emitDotFileAfterConstruction(emitDotFileAfterConstruction), diags(diags) { -} + emitDotFileAfterConstruction(emitDotFileAfterConstruction), diags(diags), + backend(backend) {} SourceFileDepGraph AbstractSourceFileDepGraphFactory::construct() { addSourceFileNodesToGraph(); @@ -49,7 +50,7 @@ SourceFileDepGraph AbstractSourceFileDepGraphFactory::construct() { } assert(g.verify()); if (emitDotFileAfterConstruction) - g.emitDotFile(swiftDeps, diags); + g.emitDotFile(backend, swiftDeps, diags); return std::move(g); } diff --git a/lib/AST/FineGrainedDependencies.cpp b/lib/AST/FineGrainedDependencies.cpp index 609a03512b058..39d362ce6d8da 100644 --- a/lib/AST/FineGrainedDependencies.cpp +++ b/lib/AST/FineGrainedDependencies.cpp @@ -373,11 +373,13 @@ void SourceFileDepGraph::verifySame(const SourceFileDepGraph &other) const { #endif } -void SourceFileDepGraph::emitDotFile(StringRef outputPath, +void SourceFileDepGraph::emitDotFile(llvm::vfs::OutputBackend &outputBackend, + StringRef outputPath, DiagnosticEngine &diags) { std::string dotFileName = outputPath.str() + ".dot"; - withOutputFile(diags, dotFileName, [&](llvm::raw_pwrite_stream &out) { - DotFileEmitter(out, *this, false, false).emit(); - return false; - }); + withOutputPath( + diags, outputBackend, dotFileName, [&](llvm::raw_pwrite_stream &out) { + DotFileEmitter(out, *this, false, false).emit(); + return false; + }); } diff --git a/lib/AST/FineGrainedDependencyFormat.cpp b/lib/AST/FineGrainedDependencyFormat.cpp index bab3cdf60a98f..770c8745d8634 100644 --- a/lib/AST/FineGrainedDependencyFormat.cpp +++ b/lib/AST/FineGrainedDependencyFormat.cpp @@ -21,6 +21,7 @@ #include "llvm/Bitstream/BitstreamWriter.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackend.h" using namespace swift; using namespace fine_grained_dependencies; @@ -498,10 +499,10 @@ void swift::fine_grained_dependencies::writeFineGrainedDependencyGraph( } bool swift::fine_grained_dependencies::writeFineGrainedDependencyGraphToPath( - DiagnosticEngine &diags, StringRef path, + DiagnosticEngine &diags, llvm::vfs::OutputBackend &backend, StringRef path, const SourceFileDepGraph &g) { PrettyStackTraceStringAction stackTrace("saving fine-grained dependency graph", path); - return withOutputFile(diags, path, [&](llvm::raw_ostream &out) { + return withOutputPath(diags, backend, path, [&](llvm::raw_ostream &out) { SmallVector Buffer; llvm::BitstreamWriter Writer{Buffer}; writeFineGrainedDependencyGraph(Writer, g, Purpose::ForSwiftDeps); diff --git a/lib/AST/FrontendSourceFileDepGraphFactory.cpp b/lib/AST/FrontendSourceFileDepGraphFactory.cpp index 74afe1a7f810c..7b50752032919 100644 --- a/lib/AST/FrontendSourceFileDepGraphFactory.cpp +++ b/lib/AST/FrontendSourceFileDepGraphFactory.cpp @@ -43,6 +43,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" using namespace swift; @@ -232,18 +233,19 @@ StringRef DependencyKey::Builder::getTopLevelName(const Decl *decl) { bool fine_grained_dependencies::withReferenceDependencies( llvm::PointerUnion MSF, - const DependencyTracker &depTracker, StringRef outputPath, - bool alsoEmitDotFile, + const DependencyTracker &depTracker, llvm::vfs::OutputBackend &backend, + StringRef outputPath, bool alsoEmitDotFile, llvm::function_ref cont) { if (auto *MD = MSF.dyn_cast()) { SourceFileDepGraph g = - ModuleDepGraphFactory(MD, alsoEmitDotFile).construct(); + ModuleDepGraphFactory(backend, MD, alsoEmitDotFile).construct(); return cont(std::move(g)); } else { auto *SF = MSF.get(); - SourceFileDepGraph g = FrontendSourceFileDepGraphFactory( - SF, outputPath, depTracker, alsoEmitDotFile) - .construct(); + SourceFileDepGraph g = + FrontendSourceFileDepGraphFactory(SF, backend, outputPath, depTracker, + alsoEmitDotFile) + .construct(); return cont(std::move(g)); } } @@ -253,11 +255,12 @@ bool fine_grained_dependencies::withReferenceDependencies( //============================================================================== FrontendSourceFileDepGraphFactory::FrontendSourceFileDepGraphFactory( - const SourceFile *SF, StringRef outputPath, - const DependencyTracker &depTracker, const bool alsoEmitDotFile) + const SourceFile *SF, llvm::vfs::OutputBackend &backend, + StringRef outputPath, const DependencyTracker &depTracker, + const bool alsoEmitDotFile) : AbstractSourceFileDepGraphFactory( SF->getASTContext().hadError(), outputPath, SF->getInterfaceHash(), - alsoEmitDotFile, SF->getASTContext().Diags), + alsoEmitDotFile, SF->getASTContext().Diags, backend), SF(SF), depTracker(depTracker) {} //============================================================================== @@ -577,11 +580,12 @@ void FrontendSourceFileDepGraphFactory::addAllUsedDecls() { // MARK: ModuleDepGraphFactory //============================================================================== -ModuleDepGraphFactory::ModuleDepGraphFactory(const ModuleDecl *Mod, +ModuleDepGraphFactory::ModuleDepGraphFactory(llvm::vfs::OutputBackend &backend, + const ModuleDecl *Mod, bool emitDot) : AbstractSourceFileDepGraphFactory( Mod->getASTContext().hadError(), Mod->getNameStr(), - Mod->getFingerprint(), emitDot, Mod->getASTContext().Diags), + Mod->getFingerprint(), emitDot, Mod->getASTContext().Diags, backend), Mod(Mod) {} void ModuleDepGraphFactory::addAllDefinedDecls() { diff --git a/lib/AST/FrontendSourceFileDepGraphFactory.h b/lib/AST/FrontendSourceFileDepGraphFactory.h index d46105e45bc28..9fa24035527f8 100644 --- a/lib/AST/FrontendSourceFileDepGraphFactory.h +++ b/lib/AST/FrontendSourceFileDepGraphFactory.h @@ -14,6 +14,7 @@ #define FrontendSourceFileDepGraphFactory_h #include "swift/AST/AbstractSourceFileDepGraphFactory.h" +#include "llvm/Support/VirtualOutputBackend.h" namespace swift { namespace fine_grained_dependencies { @@ -27,7 +28,9 @@ class FrontendSourceFileDepGraphFactory const DependencyTracker &depTracker; public: - FrontendSourceFileDepGraphFactory(const SourceFile *SF, StringRef outputPath, + FrontendSourceFileDepGraphFactory(const SourceFile *SF, + llvm::vfs::OutputBackend &backend, + StringRef outputPath, const DependencyTracker &depTracker, bool alsoEmitDotFile); @@ -42,7 +45,8 @@ class ModuleDepGraphFactory : public AbstractSourceFileDepGraphFactory { const ModuleDecl *Mod; public: - ModuleDepGraphFactory(const ModuleDecl *Mod, bool emitDot); + ModuleDepGraphFactory(llvm::vfs::OutputBackend &backend, + const ModuleDecl *Mod, bool emitDot); ~ModuleDepGraphFactory() override = default; diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index 6f8cc84be731e..19c506aec03b1 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -66,6 +66,7 @@ #include "clang/Sema/Sema.h" #include "clang/Serialization/ASTReader.h" #include "clang/Serialization/ASTWriter.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Support/CrashRecoveryContext.h" @@ -74,6 +75,7 @@ #include "llvm/Support/Memory.h" #include "llvm/Support/Path.h" #include "llvm/Support/VirtualFileSystem.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include #include @@ -940,10 +942,8 @@ ClangImporter::getPCHFilename(const ClangImporterOptions &ImporterOptions, return PCHFilename.str().str(); } - -Optional -ClangImporter::getOrCreatePCH(const ClangImporterOptions &ImporterOptions, - StringRef SwiftPCHHash) { +Optional ClangImporter::getOrCreatePCH( + const ClangImporterOptions &ImporterOptions, StringRef SwiftPCHHash) { bool isExplicit; auto PCHFilename = getPCHFilename(ImporterOptions, SwiftPCHHash, isExplicit); @@ -959,8 +959,8 @@ ClangImporter::getOrCreatePCH(const ClangImporterOptions &ImporterOptions, << EC.message(); return None; } - auto FailedToEmit = emitBridgingPCH(ImporterOptions.BridgingHeader, - PCHFilename.value()); + auto FailedToEmit = + emitBridgingPCH(ImporterOptions.BridgingHeader, PCHFilename.value()); if (FailedToEmit) { return None; } @@ -1693,13 +1693,13 @@ ClangImporter::cloneCompilerInstanceForPrecompiling() { clonedInstance->setFileManager(&fileManager); clonedInstance->createSourceManager(fileManager); clonedInstance->setTarget(&Impl.Instance->getTarget()); + clonedInstance->setOutputBackend(Impl.SwiftContext.OutputBackend); return clonedInstance; } -bool -ClangImporter::emitBridgingPCH(StringRef headerPath, - StringRef outputPCHPath) { +bool ClangImporter::emitBridgingPCH( + StringRef headerPath, StringRef outputPCHPath) { auto emitInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = emitInstance->getInvocation(); @@ -1728,7 +1728,8 @@ ClangImporter::emitBridgingPCH(StringRef headerPath, return false; } -bool ClangImporter::runPreprocessor(StringRef inputPath, StringRef outputPath) { +bool ClangImporter::runPreprocessor( + StringRef inputPath, StringRef outputPath) { auto emitInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = emitInstance->getInvocation(); auto LangOpts = invocation.getLangOpts(); @@ -1752,9 +1753,8 @@ bool ClangImporter::runPreprocessor(StringRef inputPath, StringRef outputPath) { return emitInstance->getDiagnostics().hasErrorOccurred(); } -bool ClangImporter::emitPrecompiledModule(StringRef moduleMapPath, - StringRef moduleName, - StringRef outputPath) { +bool ClangImporter::emitPrecompiledModule( + StringRef moduleMapPath, StringRef moduleName, StringRef outputPath) { auto emitInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = emitInstance->getInvocation(); @@ -1788,8 +1788,8 @@ bool ClangImporter::emitPrecompiledModule(StringRef moduleMapPath, return false; } -bool ClangImporter::dumpPrecompiledModule(StringRef modulePath, - StringRef outputPath) { +bool ClangImporter::dumpPrecompiledModule( + StringRef modulePath, StringRef outputPath) { auto dumpInstance = cloneCompilerInstanceForPrecompiling(); auto &invocation = dumpInstance->getInvocation(); diff --git a/lib/DependencyScan/DependencyScanningTool.cpp b/lib/DependencyScan/DependencyScanningTool.cpp index cc812e4c3a954..d91c9053ec968 100644 --- a/lib/DependencyScan/DependencyScanningTool.cpp +++ b/lib/DependencyScan/DependencyScanningTool.cpp @@ -20,6 +20,7 @@ #include "swift/DependencyScan/DependencyScanImpl.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/VirtualOutputBackends.h" #include @@ -187,8 +188,9 @@ void DependencyScanningTool::serializeCache(llvm::StringRef path) { SourceManager SM; DiagnosticEngine Diags(SM); Diags.addConsumer(CDC); + llvm::vfs::OnDiskOutputBackend Backend; module_dependency_cache_serialization::writeInterModuleDependenciesCache( - Diags, path, *ScanningService); + Diags, Backend, path, *ScanningService); } bool DependencyScanningTool::loadCache(llvm::StringRef path) { diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index e69b3ae5f2462..075666ddc5dbc 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -16,6 +16,7 @@ #include "swift/Basic/Version.h" #include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/Support/VirtualOutputBackend.h" #include using namespace swift; @@ -1146,11 +1147,11 @@ void swift::dependencies::module_dependency_cache_serialization:: bool swift::dependencies::module_dependency_cache_serialization:: writeInterModuleDependenciesCache( - DiagnosticEngine &diags, StringRef path, - const SwiftDependencyScanningService &cache) { + DiagnosticEngine &diags, llvm::vfs::OutputBackend &backend, + StringRef path, const SwiftDependencyScanningService &cache) { PrettyStackTraceStringAction stackTrace( "saving inter-module dependency graph", path); - return withOutputFile(diags, path, [&](llvm::raw_ostream &out) { + return withOutputPath(diags, backend, path, [&](llvm::raw_ostream &out) { SmallVector Buffer; llvm::BitstreamWriter Writer{Buffer}; writeInterModuleDependenciesCache(Writer, cache); diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 84d35f09779cf..3e37c7345ba1b 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -20,6 +20,7 @@ #include "swift/AST/DiagnosticsFrontend.h" #include "swift/AST/DiagnosticsSema.h" #include "swift/AST/DiagnosticsDriver.h" +#include "swift/AST/FileSystem.h" #include "swift/AST/Module.h" #include "swift/AST/ModuleDependencies.h" #include "swift/AST/ModuleLoader.h" @@ -43,8 +44,10 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/StringSaver.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" +#include "llvm/Support/raw_ostream.h" #include #include #include @@ -933,6 +936,25 @@ static void writeJSON(llvm::raw_ostream &out, } } +static bool writePrescanJSONToOutput(DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend, + StringRef path, + const swiftscan_import_set_t importSet) { + return withOutputPath(diags, backend, path, [&](llvm::raw_pwrite_stream &os) { + writePrescanJSON(os, importSet); + return false; + }); +} + +static bool writeJSONToOutput(DiagnosticEngine &diags, + llvm::vfs::OutputBackend &backend, StringRef path, + const swiftscan_dependency_graph_t dependencies) { + return withOutputPath(diags, backend, path, [&](llvm::raw_pwrite_stream &os) { + writeJSON(os, dependencies); + return false; + }); +} + static swiftscan_dependency_graph_t generateFullDependencyGraph(CompilerInstance &instance, ModuleDependenciesCache &cache, @@ -1369,7 +1391,7 @@ static void serializeDependencyCache(CompilerInstance &instance, ASTContext &Context = instance.getASTContext(); auto savePath = opts.SerializedDependencyScannerCachePath; module_dependency_cache_serialization::writeInterModuleDependenciesCache( - Context.Diags, savePath, service); + Context.Diags, instance.getOutputBackend(), savePath, service); if (opts.EmitDependencyScannerCacheRemarks) { Context.Diags.diagnose(SourceLoc(), diag::remark_save_cache, savePath); } @@ -1393,15 +1415,6 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { ASTContext &Context = instance.getASTContext(); const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); - std::error_code EC; - llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::OF_None); - if (out.has_error() || EC) { - Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, - EC.message()); - out.clear_error(); - return true; - } - // `-scan-dependencies` invocations use a single new instance // of a module cache SwiftDependencyScanningService service; @@ -1427,8 +1440,10 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { return true; auto dependencies = std::move(*dependenciesOrErr); - // Write out the JSON description. - writeJSON(out, dependencies); + if (writeJSONToOutput(Context.Diags, instance.getOutputBackend(), path, + dependencies)) + return true; + // This process succeeds regardless of whether any errors occurred. // FIXME: We shouldn't need this, but it's masking bugs in our scanning // logic where we don't create a fresh context when scanning Swift interfaces @@ -1441,20 +1456,12 @@ bool swift::dependencies::prescanDependencies(CompilerInstance &instance) { ASTContext &Context = instance.getASTContext(); const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); - std::error_code EC; - llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::OF_None); // `-scan-dependencies` invocations use a single new instance // of a module cache SwiftDependencyScanningService singleUseService; ModuleDependenciesCache cache(singleUseService, instance.getMainModule()->getNameStr().str(), instance.getInvocation().getModuleScanningHash()); - if (out.has_error() || EC) { - Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, - EC.message()); - out.clear_error(); - return true; - } // Execute import prescan, and write JSON output to the output stream auto importSetOrErr = performModulePrescan(instance); @@ -1463,7 +1470,10 @@ bool swift::dependencies::prescanDependencies(CompilerInstance &instance) { auto importSet = std::move(*importSetOrErr); // Serialize and output main module dependencies only and exit. - writePrescanJSON(out, importSet); + if (writePrescanJSONToOutput(Context.Diags, instance.getOutputBackend(), path, + importSet)) + return true; + // This process succeeds regardless of whether any errors occurred. // FIXME: We shouldn't need this, but it's masking bugs in our scanning // logic where we don't create a fresh context when scanning Swift interfaces @@ -1499,12 +1509,13 @@ bool swift::dependencies::batchScanDependencies( auto iresults = batchScanResults.cbegin(); for (; ientries != batchInput->end() and iresults != batchScanResults.end(); ++ientries, ++iresults) { - std::error_code EC; - llvm::raw_fd_ostream out((*ientries).outputPath, EC, llvm::sys::fs::OF_None); if ((*iresults).getError()) return true; - writeJSON(out, **iresults); + if (writeJSONToOutput(instance.getASTContext().Diags, + instance.getOutputBackend(), (*ientries).outputPath, + **iresults)) + return true; } return false; } @@ -1534,12 +1545,13 @@ bool swift::dependencies::batchPrescanDependencies( for (; ientries != batchInput->end() and iresults != batchPrescanResults.end(); ++ientries, ++iresults) { - std::error_code EC; - llvm::raw_fd_ostream out((*ientries).outputPath, EC, llvm::sys::fs::OF_None); if ((*iresults).getError()) return true; - writePrescanJSON(out, **iresults); + if (writePrescanJSONToOutput(instance.getASTContext().Diags, + instance.getOutputBackend(), + (*ientries).outputPath, **iresults)) + return true; } return false; } diff --git a/lib/Driver/FineGrainedDependencyDriverGraph.cpp b/lib/Driver/FineGrainedDependencyDriverGraph.cpp index 1b0c20a5fc849..9e2b36c359570 100644 --- a/lib/Driver/FineGrainedDependencyDriverGraph.cpp +++ b/lib/Driver/FineGrainedDependencyDriverGraph.cpp @@ -29,6 +29,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/SourceMgr.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/raw_ostream.h" #include @@ -529,11 +530,12 @@ void ModuleDepGraph::emitDotFileForJob(DiagnosticEngine &diags, emitDotFile(diags, getSwiftDeps(job)); } -void ModuleDepGraph::emitDotFile(DiagnosticEngine &diags, StringRef baseName) { +void ModuleDepGraph::emitDotFile(DiagnosticEngine &diags, + StringRef baseName) { unsigned seqNo = dotFileSequenceNumber[baseName.str()]++; std::string fullName = baseName.str() + "-post-integration." + std::to_string(seqNo) + ".dot"; - withOutputFile(diags, fullName, [&](llvm::raw_ostream &out) { + withOutputPath(diags, *backend, fullName, [&](llvm::raw_ostream &out) { emitDotFile(out); return false; }); diff --git a/lib/DriverTool/modulewrap_main.cpp b/lib/DriverTool/modulewrap_main.cpp index eff93f616cc5d..676b78e720ae4 100644 --- a/lib/DriverTool/modulewrap_main.cpp +++ b/lib/DriverTool/modulewrap_main.cpp @@ -28,6 +28,7 @@ #include "swift/Subsystems.h" #include "swift/SymbolGraphGen/SymbolGraphOptions.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/Bitstream/BitstreamReader.h" #include "llvm/Option/ArgList.h" #include "llvm/Option/Option.h" @@ -35,6 +36,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/TargetSelect.h" +#include "llvm/Support/VirtualOutputBackends.h" using namespace llvm::opt; using namespace swift; @@ -183,7 +185,8 @@ int modulewrap_main(ArrayRef Args, const char *Argv0, LangOpts.Target = Invocation.getTargetTriple(); ASTContext &ASTCtx = *ASTContext::get( LangOpts, TypeCheckOpts, SILOpts, SearchPathOpts, ClangImporterOpts, - SymbolGraphOpts, SrcMgr, Instance.getDiags()); + SymbolGraphOpts, SrcMgr, Instance.getDiags(), + llvm::makeIntrusiveRefCnt()); registerParseRequestFunctions(ASTCtx.evaluator); registerTypeCheckerRequestFunctions(ASTCtx.evaluator); diff --git a/lib/DriverTool/swift_api_digester_main.cpp b/lib/DriverTool/swift_api_digester_main.cpp index a2b32b4a03e13..7265b48b447e8 100644 --- a/lib/DriverTool/swift_api_digester_main.cpp +++ b/lib/DriverTool/swift_api_digester_main.cpp @@ -36,6 +36,9 @@ #include "swift/IDE/APIDigesterData.h" #include "swift/Option/Options.h" #include "swift/Parse/ParseVersion.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/Support/VirtualOutputBackends.h" +#include "llvm/Support/raw_ostream.h" #include using namespace swift; @@ -2536,13 +2539,18 @@ class SwiftAPIDigesterInvocation { switch (Action) { case ActionType::DumpSDK: { llvm::StringSet<> Modules; - return (prepareForDump(InitInvoke, Modules)) - ? 1 - : dumpSDKContent(InitInvoke, Modules, - getJsonOutputFilePath( - InitInvoke.getLangOptions().Target, - CheckerOpts.ABI, OutputFile, OutputDir), - CheckerOpts); + if (prepareForDump(InitInvoke, Modules)) + return 1; + auto JsonOut = + getJsonOutputFilePath(InitInvoke.getLangOptions().Target, + CheckerOpts.ABI, OutputFile, OutputDir); + std::error_code EC; + llvm::raw_fd_ostream fs(JsonOut, EC); + if (EC) { + llvm::errs() << "Cannot open JSON output file: " << JsonOut << "\n"; + return 1; + } + return dumpSDKContent(InitInvoke, Modules, fs, CheckerOpts); } case ActionType::MigratorGen: case ActionType::DiagnoseSDKs: { @@ -2606,7 +2614,13 @@ class SwiftAPIDigesterInvocation { } case ActionType::GenerateEmptyBaseline: { SDKContext Ctx(CheckerOpts); - dumpSDKRoot(getEmptySDKNodeRoot(Ctx), OutputFile); + std::error_code EC; + llvm::raw_fd_ostream fs(OutputFile, EC); + if (EC) { + llvm::errs() << "Cannot open output file: " << OutputFile << "\n"; + return 1; + } + dumpSDKRoot(getEmptySDKNodeRoot(Ctx), fs); return 0; } case ActionType::FindUsr: { diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index e52caaccc40af..7145d23d80af4 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -350,6 +350,7 @@ bool ArgsToFrontendOptionsConverter::convert( Opts.serializedPathObfuscator.addMapping(SplitMap.first, SplitMap.second); } Opts.emptyABIDescriptor = Args.hasArg(OPT_empty_abi_descriptor); + Opts.DeterministicCheck = Args.hasArg(OPT_enable_deterministic_check); return false; } diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 12e563dffd1ec..806635b18872f 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -2377,6 +2377,7 @@ static bool ParseIRGenArgs(IRGenOptions &Opts, ArgList &Args, Opts.UseProfile = ProfileUse ? ProfileUse->getValue() : ""; Opts.PrintInlineTree |= Args.hasArg(OPT_print_llvm_inline_tree); + Opts.AlwaysCompile |= Args.hasArg(OPT_always_compile_output_files); Opts.EnableDynamicReplacementChaining |= Args.hasArg(OPT_enable_dynamic_replacement_chaining); diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index 5a8ddacecae36..e228ca5db8966 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -38,6 +38,7 @@ #include "swift/Subsystems.h" #include "clang/AST/ASTContext.h" #include "llvm/ADT/Hashing.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Triple.h" #include "llvm/Support/CommandLine.h" @@ -45,6 +46,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/Process.h" +#include "llvm/Support/VirtualOutputBackends.h" #include using namespace swift; @@ -249,7 +251,7 @@ bool CompilerInstance::setUpASTContextIfNeeded() { Invocation.getLangOptions(), Invocation.getTypeCheckerOptions(), Invocation.getSILOptions(), Invocation.getSearchPathOptions(), Invocation.getClangImporterOptions(), Invocation.getSymbolGraphOptions(), - SourceMgr, Diagnostics)); + SourceMgr, Diagnostics, OutputBackend)); if (!Invocation.getFrontendOptions().ModuleAliasMap.empty()) Context->setModuleAliases(Invocation.getFrontendOptions().ModuleAliasMap); @@ -393,11 +395,31 @@ void CompilerInstance::setupDependencyTrackerIfNeeded() { DepTracker->addDependency(path, /*isSystem=*/false); } +void CompilerInstance::setupOutputBackend() { + // Skip if output backend is not setup, default to OnDiskOutputBackend. + if (OutputBackend) + return; + + OutputBackend = + llvm::makeIntrusiveRefCnt(); + + // Setup verification backend. + // Create a mirroring outputbackend to produce hash for output files. + // We cannot skip disk here since swift compiler is expecting to read back + // some output file in later stages. + if (Invocation.getFrontendOptions().DeterministicCheck) { + HashBackend = llvm::makeIntrusiveRefCnt(); + OutputBackend = + llvm::vfs::makeMirroringOutputBackend(OutputBackend, HashBackend); + } +} + bool CompilerInstance::setup(const CompilerInvocation &Invoke, std::string &Error) { Invocation = Invoke; setupDependencyTrackerIfNeeded(); + setupOutputBackend(); // If initializing the overlay file system fails there's no sense in // continuing because the compiler will read the wrong files. diff --git a/lib/Frontend/ModuleInterfaceBuilder.cpp b/lib/Frontend/ModuleInterfaceBuilder.cpp index a898ae822d6ca..1d872b6b3b24d 100644 --- a/lib/Frontend/ModuleInterfaceBuilder.cpp +++ b/lib/Frontend/ModuleInterfaceBuilder.cpp @@ -190,7 +190,8 @@ std::error_code ExplicitModuleInterfaceBuilder::buildSwiftModuleFromInterface( if (Instance.getASTContext() .getModuleInterfaceChecker() ->tryEmitForwardingModule(Invocation.getModuleName(), InterfacePath, - CompiledCandidates, OutputPath)) { + CompiledCandidates, + Instance.getOutputBackend(), OutputPath)) { return std::error_code(); } FrontendOptions &FEOpts = Invocation.getFrontendOptions(); diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index acc34930c72e4..65946488be1d1 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -40,6 +40,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/Errc.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackend.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" #include "llvm/Support/xxhash.h" @@ -901,8 +902,8 @@ class ModuleInterfaceLoaderImpl { /// this. If the write was successful, it also updates the /// list of dependencies to match what was written to the forwarding file. bool writeForwardingModuleAndUpdateDeps( - const DiscoveredModule &mod, StringRef outputPath, - SmallVectorImpl &deps) { + const DiscoveredModule &mod, llvm::vfs::OutputBackend &backend, + StringRef outputPath, SmallVectorImpl &deps) { assert(mod.isPrebuilt() && "cannot write forwarding file for non-prebuilt module"); ForwardingModule fwd(mod.path); @@ -938,16 +939,12 @@ class ModuleInterfaceLoaderImpl { depsAdjustedToMTime.push_back(adjustedDep); } - // Create the module cache if we haven't created it yet. - StringRef parentDir = path::parent_path(outputPath); - (void)llvm::sys::fs::create_directories(parentDir); - - auto hadError = withOutputFile(diags, outputPath, - [&](llvm::raw_pwrite_stream &out) { - llvm::yaml::Output yamlWriter(out); - yamlWriter << fwd; - return false; - }); + auto hadError = withOutputPath(diags, backend, outputPath, + [&](llvm::raw_pwrite_stream &out) { + llvm::yaml::Output yamlWriter(out); + yamlWriter << fwd; + return false; + }); if (hadError) return true; @@ -1006,8 +1003,8 @@ class ModuleInterfaceLoaderImpl { // If it's prebuilt, use this time to generate a forwarding module and // update the dependencies to use modification times. if (module.isPrebuilt()) - if (writeForwardingModuleAndUpdateDeps(module, cachedOutputPath, - allDeps)) + if (writeForwardingModuleAndUpdateDeps(module, ctx.getOutputBackend(), + cachedOutputPath, allDeps)) return std::make_error_code(std::errc::not_supported); // Report the module's dependencies to the dependencyTracker @@ -1251,7 +1248,8 @@ ModuleInterfaceCheckerImpl::getCompiledModuleCandidatesForInterface(StringRef mo bool ModuleInterfaceCheckerImpl::tryEmitForwardingModule( StringRef moduleName, StringRef interfacePath, - ArrayRef candidates, StringRef outputPath) { + ArrayRef candidates, llvm::vfs::OutputBackend &backend, + StringRef outputPath) { // Derive .swiftmodule path from the .swiftinterface path. auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile); llvm::SmallString<32> modulePath = interfacePath; @@ -1269,12 +1267,12 @@ bool ModuleInterfaceCheckerImpl::tryEmitForwardingModule( deps, moduleBuffer)) { // If so, emit a forwarding module to the candidate. ForwardingModule FM(mod); - auto hadError = withOutputFile(Ctx.Diags, outputPath, - [&](llvm::raw_pwrite_stream &out) { - llvm::yaml::Output yamlWriter(out); - yamlWriter << FM; - return false; - }); + auto hadError = withOutputPath(Ctx.Diags, backend, outputPath, + [&](llvm::raw_pwrite_stream &out) { + llvm::yaml::Output yamlWriter(out); + yamlWriter << FM; + return false; + }); if (!hadError) return true; } @@ -1402,19 +1400,25 @@ bool ModuleInterfaceLoader::buildExplicitSwiftModuleFromSwiftInterface( ArrayRef CompiledCandidates, DependencyTracker *tracker) { - // First, check if the expected output already exists and possibly up-to-date w.r.t. - // all of the dependencies it was built with. If so, early exit. - UpToDateModuleCheker checker(Instance.getASTContext(), - RequireOSSAModules_t(Instance.getSILOptions())); - ModuleRebuildInfo rebuildInfo; - SmallVector allDeps; - std::unique_ptr moduleBuffer; - if (checker.swiftModuleIsUpToDate(outputPath, rebuildInfo, allDeps, moduleBuffer)) { - if (Instance.getASTContext().LangOpts.EnableSkipExplicitInterfaceModuleBuildRemarks) { - Instance.getDiags().diagnose(SourceLoc(), - diag::explicit_interface_build_skipped, outputPath); + if (!Instance.getInvocation().getIRGenOptions().AlwaysCompile) { + // First, check if the expected output already exists and possibly + // up-to-date w.r.t. all of the dependencies it was built with. If so, early + // exit. + UpToDateModuleCheker checker( + Instance.getASTContext(), + RequireOSSAModules_t(Instance.getSILOptions())); + ModuleRebuildInfo rebuildInfo; + SmallVector allDeps; + std::unique_ptr moduleBuffer; + if (checker.swiftModuleIsUpToDate(outputPath, rebuildInfo, allDeps, + moduleBuffer)) { + if (Instance.getASTContext() + .LangOpts.EnableSkipExplicitInterfaceModuleBuildRemarks) { + Instance.getDiags().diagnose( + SourceLoc(), diag::explicit_interface_build_skipped, outputPath); + } + return false; } - return false; } // Read out the compiler version. diff --git a/lib/Frontend/Serialization.cpp b/lib/Frontend/Serialization.cpp index 3af22d3bf2df0..88ad917fab610 100644 --- a/lib/Frontend/Serialization.cpp +++ b/lib/Frontend/Serialization.cpp @@ -12,11 +12,13 @@ #include "swift/Serialization/Serialization.h" #include "swift/APIDigester/ModuleAnalyzerNodes.h" +#include "swift/AST/DiagnosticsFrontend.h" #include "swift/AST/FileSystem.h" #include "swift/Subsystems.h" #include "swift/SymbolGraphGen/SymbolGraphGen.h" #include "swift/SymbolGraphGen/SymbolGraphOptions.h" #include "llvm/Support/SmallVectorMemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackend.h" using namespace swift; @@ -35,7 +37,23 @@ static void emitABIDescriptor(ModuleOrSourceFile DC, using namespace swift::ide::api; if (!options.ABIDescriptorPath.empty()) { if (DC.is()) { - dumpModuleContent(DC.get(), options.ABIDescriptorPath, true, + auto &OutputBackend = getContext(DC).getOutputBackend(); + auto ABIDesFile = OutputBackend.createFile(options.ABIDescriptorPath); + if (!ABIDesFile) { + getContext(DC).Diags.diagnose(SourceLoc(), diag::error_opening_output, + options.ABIDescriptorPath, + toString(ABIDesFile.takeError())); + return; + } + SWIFT_DEFER { + if (auto E = ABIDesFile->keep()) { + getContext(DC).Diags.diagnose(SourceLoc(), diag::error_closing_output, + options.ABIDescriptorPath, + toString(std::move(E))); + return; + } + }; + dumpModuleContent(DC.get(), *ABIDesFile, true, options.emptyABIDescriptor); } } @@ -56,11 +74,12 @@ void swift::serializeToBuffers( llvm::raw_svector_ostream stream(buf); serialization::writeToStream(stream, DC, M, options, /*dependency info*/ nullptr); - bool hadError = withOutputFile(getContext(DC).Diags, options.OutputPath, - [&](raw_ostream &out) { - out << stream.str(); - return false; - }); + bool hadError = withOutputPath( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.OutputPath, [&](raw_ostream &out) { + out << stream.str(); + return false; + }); if (hadError) return; @@ -77,11 +96,12 @@ void swift::serializeToBuffers( llvm::SmallString<1024> buf; llvm::raw_svector_ostream stream(buf); serialization::writeDocToStream(stream, DC, options.GroupInfoPath); - (void)withOutputFile(getContext(DC).Diags, options.DocOutputPath, - [&](raw_ostream &out) { - out << stream.str(); - return false; - }); + (void)withOutputPath(getContext(DC).Diags, + getContext(DC).getOutputBackend(), + options.DocOutputPath, [&](raw_ostream &out) { + out << stream.str(); + return false; + }); if (moduleDocBuffer) *moduleDocBuffer = std::make_unique( std::move(buf), options.DocOutputPath, @@ -94,11 +114,12 @@ void swift::serializeToBuffers( llvm::SmallString<1024> buf; llvm::raw_svector_ostream stream(buf); serialization::writeSourceInfoToStream(stream, DC); - (void)withOutputFile(getContext(DC).Diags, options.SourceInfoOutputPath, - [&](raw_ostream &out) { - out << stream.str(); - return false; - }); + (void)withOutputPath( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.SourceInfoOutputPath, [&](raw_ostream &out) { + out << stream.str(); + return false; + }); if (moduleSourceInfoBuffer) *moduleSourceInfoBuffer = std::make_unique( std::move(buf), options.SourceInfoOutputPath, @@ -120,37 +141,37 @@ void swift::serialize( return; } - bool hadError = withOutputFile(getContext(DC).Diags, - options.OutputPath, - [&](raw_ostream &out) { - FrontendStatsTracer tracer(getContext(DC).Stats, - "Serialization, swiftmodule"); - serialization::writeToStream(out, DC, M, options, DG); - return false; - }); + bool hadError = withOutputPath( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.OutputPath, [&](raw_ostream &out) { + FrontendStatsTracer tracer(getContext(DC).Stats, + "Serialization, swiftmodule"); + serialization::writeToStream(out, DC, M, options, DG); + return false; + }); if (hadError) return; if (!options.DocOutputPath.empty()) { - (void)withOutputFile(getContext(DC).Diags, - options.DocOutputPath, - [&](raw_ostream &out) { - FrontendStatsTracer tracer(getContext(DC).Stats, - "Serialization, swiftdoc"); - serialization::writeDocToStream(out, DC, options.GroupInfoPath); - return false; - }); + (void)withOutputPath( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.DocOutputPath, [&](raw_ostream &out) { + FrontendStatsTracer tracer(getContext(DC).Stats, + "Serialization, swiftdoc"); + serialization::writeDocToStream(out, DC, options.GroupInfoPath); + return false; + }); } if (!options.SourceInfoOutputPath.empty()) { - (void)withOutputFile(getContext(DC).Diags, - options.SourceInfoOutputPath, - [&](raw_ostream &out) { - FrontendStatsTracer tracer(getContext(DC).Stats, - "Serialization, swiftsourceinfo"); - serialization::writeSourceInfoToStream(out, DC); - return false; - }); + (void)withOutputPath( + getContext(DC).Diags, getContext(DC).getOutputBackend(), + options.SourceInfoOutputPath, [&](raw_ostream &out) { + FrontendStatsTracer tracer(getContext(DC).Stats, + "Serialization, swiftsourceinfo"); + serialization::writeSourceInfoToStream(out, DC); + return false; + }); } if (!symbolGraphOptions.OutputDir.empty()) { diff --git a/lib/FrontendTool/Dependencies.h b/lib/FrontendTool/Dependencies.h index 54291ebe03f63..91397e642fae7 100644 --- a/lib/FrontendTool/Dependencies.h +++ b/lib/FrontendTool/Dependencies.h @@ -13,6 +13,12 @@ #ifndef SWIFT_FRONTENDTOOL_DEPENDENCIES_H #define SWIFT_FRONTENDTOOL_DEPENDENCIES_H +namespace llvm { +namespace vfs { +class OutputBackend; +} +} // namespace llvm + namespace swift { class ASTContext; @@ -23,11 +29,13 @@ class InputFile; class ModuleDecl; /// Emit the names of the modules imported by \c mainModule. -bool emitImportedModules(ModuleDecl *mainModule, const FrontendOptions &opts); +bool emitImportedModules(ModuleDecl *mainModule, const FrontendOptions &opts, + llvm::vfs::OutputBackend &backend); bool emitMakeDependenciesIfNeeded(DiagnosticEngine &diags, DependencyTracker *depTracker, const FrontendOptions &opts, - const InputFile &input); + const InputFile &input, + llvm::vfs::OutputBackend &backend); bool emitLoadedModuleTraceIfNeeded(ModuleDecl *mainModule, DependencyTracker *depTracker, const FrontendOptions &opts, diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index 12ce64135d364..cad2a25345dbf 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -71,7 +71,10 @@ #include "clang/Lex/Preprocessor.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringMap.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" @@ -81,6 +84,8 @@ #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputBackends.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Support/FileSystem.h" @@ -105,10 +110,12 @@ static std::string displayName(StringRef MainExecutablePath) { static void emitMakeDependenciesIfNeeded(DiagnosticEngine &diags, DependencyTracker *depTracker, - const FrontendOptions &opts) { + const FrontendOptions &opts, + llvm::vfs::OutputBackend &backend) { opts.InputsAndOutputs.forEachInputProducingSupplementaryOutput( [&](const InputFile &f) -> bool { - return swift::emitMakeDependenciesIfNeeded(diags, depTracker, opts, f); + return swift::emitMakeDependenciesIfNeeded(diags, depTracker, opts, f, + backend); }); } @@ -123,37 +130,22 @@ emitLoadedModuleTraceForAllPrimariesIfNeeded(ModuleDecl *mainModule, }); } -/// Gets an output stream for the provided output filename, or diagnoses to the -/// provided AST Context and returns null if there was an error getting the -/// stream. -static std::unique_ptr -getFileOutputStream(StringRef OutputFilename, ASTContext &Ctx) { - std::error_code errorCode; - auto os = std::make_unique( - OutputFilename, errorCode, llvm::sys::fs::OF_None); - if (errorCode) { - Ctx.Diags.diagnose(SourceLoc(), diag::error_opening_output, - OutputFilename, errorCode.message()); - return nullptr; - } - return os; -} - /// Writes SIL out to the given file. static bool writeSIL(SILModule &SM, ModuleDecl *M, const SILOptions &Opts, - StringRef OutputFilename) { - auto OS = getFileOutputStream(OutputFilename, M->getASTContext()); - if (!OS) return true; - SM.print(*OS, M, Opts); - - return M->getASTContext().hadError(); + StringRef OutputFilename, + llvm::vfs::OutputBackend &Backend) { + return withOutputPath(M->getDiags(), Backend, OutputFilename, + [&](raw_ostream &out) -> bool { + SM.print(out, M, Opts); + return M->getASTContext().hadError(); + }); } static bool writeSIL(SILModule &SM, const PrimarySpecificPaths &PSPs, - const CompilerInstance &Instance, + CompilerInstance &Instance, const SILOptions &Opts) { return writeSIL(SM, Instance.getMainModule(), Opts, - PSPs.OutputFilename); + PSPs.OutputFilename, Instance.getOutputBackend()); } /// Prints the Objective-C "generated header" interface for \p M to \p @@ -166,14 +158,14 @@ static bool writeSIL(SILModule &SM, const PrimarySpecificPaths &PSPs, /// \returns true if there were any errors /// /// \see swift::printAsClangHeader -static bool printAsClangHeaderIfNeeded( +static bool printAsClangHeaderIfNeeded(llvm::vfs::OutputBackend &outputBackend, StringRef outputPath, ModuleDecl *M, StringRef bridgingHeader, const FrontendOptions &frontendOpts, const IRGenOptions &irGenOpts, clang::HeaderSearch &clangHeaderSearchInfo) { if (outputPath.empty()) return false; - return withOutputFile( - M->getDiags(), outputPath, [&](raw_ostream &out) -> bool { + return withOutputPath( + M->getDiags(), outputBackend, outputPath, [&](raw_ostream &out) -> bool { return printAsClangHeader(out, M, bridgingHeader, frontendOpts, irGenOpts, clangHeaderSearchInfo); }); @@ -187,7 +179,8 @@ static bool printAsClangHeaderIfNeeded( /// /// \see swift::emitSwiftInterface static bool -printModuleInterfaceIfNeeded(StringRef outputPath, +printModuleInterfaceIfNeeded(llvm::vfs::OutputBackend &outputBackend, + StringRef outputPath, ModuleInterfaceOptions const &Opts, LangOptions const &LangOpts, ModuleDecl *M) { @@ -205,10 +198,10 @@ printModuleInterfaceIfNeeded(StringRef outputPath, diags.diagnose(SourceLoc(), diag::warn_unsupported_module_interface_library_evolution); } - return withOutputFile(diags, outputPath, + return withOutputPath(diags, outputBackend, outputPath, [M, Opts](raw_ostream &out) -> bool { - return swift::emitSwiftInterface(out, Opts, M); - }); + return swift::emitSwiftInterface(out, Opts, M); + }); } namespace { @@ -372,6 +365,7 @@ static bool precompileBridgingHeader(const CompilerInstance &Instance) { Instance.getASTContext().getClangModuleLoader()); auto &ImporterOpts = Invocation.getClangImporterOptions(); auto &PCHOutDir = ImporterOpts.PrecompiledHeaderOutputDir; + auto OutputBackend = Instance.getOutputBackend().clone(); if (!PCHOutDir.empty()) { // Create or validate a persistent PCH. auto SwiftPCHHash = Invocation.getPCHHash(); @@ -517,8 +511,13 @@ static bool dumpAST(CompilerInstance &Instance) { for (SourceFile *sourceFile: primaryFiles) { auto PSPs = Instance.getPrimarySpecificPathsForSourceFile(*sourceFile); auto OutputFilename = PSPs.OutputFilename; - auto OS = getFileOutputStream(OutputFilename, Instance.getASTContext()); - sourceFile->dump(*OS, /*parseIfNeeded*/ true); + if (withOutputPath(Instance.getASTContext().Diags, + Instance.getOutputBackend(), OutputFilename, + [&](raw_ostream &out) -> bool { + sourceFile->dump(out, /*parseIfNeeded*/ true); + return false; + })) + return true; } } else { // Some invocations don't have primary files. In that case, we default to @@ -543,17 +542,19 @@ static bool emitReferenceDependencies(CompilerInstance &Instance, using SourceFileDepGraph = fine_grained_dependencies::SourceFileDepGraph; return fine_grained_dependencies::withReferenceDependencies( - SF, *Instance.getDependencyTracker(), outputPath, alsoEmitDotFile, - [&](SourceFileDepGraph &&g) -> bool { + SF, *Instance.getDependencyTracker(), Instance.getOutputBackend(), + outputPath, alsoEmitDotFile, [&](SourceFileDepGraph &&g) -> bool { const bool hadError = fine_grained_dependencies::writeFineGrainedDependencyGraphToPath( - Instance.getDiags(), outputPath, g); + Instance.getDiags(), Instance.getOutputBackend(), outputPath, + g); // If path is stdout, cannot read it back, so check for "-" assert(outputPath == "-" || g.verifyReadsWhatIsWritten(outputPath)); if (alsoEmitDotFile) - g.emitDotFile(outputPath, Instance.getDiags()); + g.emitDotFile(Instance.getOutputBackend(), outputPath, + Instance.getDiags()); return hadError; }); } @@ -925,6 +926,7 @@ static bool emitAnyWholeModulePostTypeCheckSupplementaryOutputs( } } hadAnyError |= printAsClangHeaderIfNeeded( + Instance.getOutputBackend(), Invocation.getClangHeaderOutputPathForAtMostOnePrimary(), Instance.getMainModule(), BridgingHeaderPathForPrint, opts, Invocation.getIRGenOptions(), @@ -940,6 +942,7 @@ static bool emitAnyWholeModulePostTypeCheckSupplementaryOutputs( if (opts.InputsAndOutputs.hasModuleInterfaceOutputPath()) { hadAnyError |= printModuleInterfaceIfNeeded( + Instance.getOutputBackend(), Invocation.getModuleInterfaceOutputPathForWholeModule(), Invocation.getModuleInterfaceOptions(), Invocation.getLangOptions(), @@ -953,6 +956,7 @@ static bool emitAnyWholeModulePostTypeCheckSupplementaryOutputs( privOpts.ModulesToSkipInPublicInterface.clear(); hadAnyError |= printModuleInterfaceIfNeeded( + Instance.getOutputBackend(), Invocation.getPrivateModuleInterfaceOutputPathForWholeModule(), privOpts, Invocation.getLangOptions(), @@ -1139,7 +1143,8 @@ static void performEndOfPipelineActions(CompilerInstance &Instance) { // Emit Make-style dependencies. emitMakeDependenciesIfNeeded(Instance.getDiags(), - Instance.getDependencyTracker(), opts); + Instance.getDependencyTracker(), opts, + Instance.getOutputBackend()); // Emit extracted constant values for every file in the batch emitConstValuesForAllPrimaryInputsIfNeeded(Instance); @@ -1333,7 +1338,8 @@ static bool performAction(CompilerInstance &Instance, getPrimaryOrMainSourceFile(Instance).dumpInterfaceHash(llvm::errs()); return Instance.getASTContext().hadError(); case FrontendOptions::ActionType::EmitImportedModules: - return emitImportedModules(Instance.getMainModule(), opts); + return emitImportedModules(Instance.getMainModule(), opts, + Instance.getOutputBackend()); // MARK: Dependency Scanning Actions case FrontendOptions::ActionType::ScanDependencies: @@ -1437,11 +1443,11 @@ static bool serializeModuleSummary(SILModule *SM, const PrimarySpecificPaths &PSPs, const ASTContext &Context) { auto summaryOutputPath = PSPs.SupplementaryOutputs.ModuleSummaryOutputPath; - return withOutputFile(Context.Diags, summaryOutputPath, - [&](llvm::raw_ostream &out) { - out << "Some stuff"; - return false; - }); + return withOutputPath(Context.Diags, Context.getOutputBackend(), + summaryOutputPath, [&](llvm::raw_ostream &out) { + out << "Some stuff"; + return false; + }); } static GeneratedModule @@ -1618,6 +1624,7 @@ static bool generateCode(CompilerInstance &Instance, StringRef OutputFilename, // Now that we have a single IR Module, hand it over to performLLVM. return performLLVM(opts, Instance.getDiags(), nullptr, HashGlobal, IRModule, TargetMachine.get(), OutputFilename, + Instance.getOutputBackend(), Instance.getStatsReporter()); } @@ -1696,7 +1703,8 @@ static bool performCompileStepsPostSILGen(CompilerInstance &Instance, using SourceFileDepGraph = fine_grained_dependencies::SourceFileDepGraph; auto *Mod = MSF.get(); fine_grained_dependencies::withReferenceDependencies( - Mod, *Instance.getDependencyTracker(), Mod->getModuleFilename(), + Mod, *Instance.getDependencyTracker(), + Instance.getOutputBackend(), Mod->getModuleFilename(), alsoEmitDotFile, [&](SourceFileDepGraph &&g) { serialize(MSF, serializationOpts, Invocation.getSymbolGraphOptions(), SM.get(), &g); return false; @@ -1780,9 +1788,9 @@ static bool performCompileStepsPostSILGen(CompilerInstance &Instance, std::vector ParallelOutputFilenames = opts.InputsAndOutputs.copyOutputFilenames(); llvm::GlobalVariable *HashGlobal; - auto IRModule = generateIR( - IRGenOpts, Invocation.getTBDGenOptions(), std::move(SM), PSPs, - OutputFilename, MSF, HashGlobal, ParallelOutputFilenames); + auto IRModule = + generateIR(IRGenOpts, Invocation.getTBDGenOptions(), std::move(SM), PSPs, + OutputFilename, MSF, HashGlobal, ParallelOutputFilenames); // Cancellation check after IRGen. if (Instance.isCancellationRequested()) @@ -2324,6 +2332,23 @@ int swift::performFrontend(ArrayRef Args, PDC.setSuppressOutput(true); } + CompilerInstance::HashingBackendPtrTy HashBackend = nullptr; + if (Invocation.getFrontendOptions().DeterministicCheck) { + // Setup a verfication instance to run. + std::unique_ptr VerifyInstance = + std::make_unique(); + std::string InstanceSetupError; + // This should not fail because it passed already. + (void)VerifyInstance->setup(Invocation, InstanceSetupError); + + // Run the first time without observer and discard return value; + int ReturnValueTest = 0; + (void)performCompile(*VerifyInstance, ReturnValueTest, + /*observer*/ nullptr); + // Get the hashing output backend and free the compiler instance. + HashBackend = VerifyInstance->getHashingBackend(); + } + int ReturnValue = 0; bool HadError = performCompile(*Instance, ReturnValue, observer); @@ -2338,6 +2363,44 @@ int swift::performFrontend(ArrayRef Args, } } + if (Invocation.getFrontendOptions().DeterministicCheck) { + // Collect all output files. + auto ReHashBackend = Instance->getHashingBackend(); + std::set AllOutputs; + llvm::for_each(HashBackend->outputFiles(), [&](StringRef F) { + AllOutputs.insert(F.str()); + }); + llvm::for_each(ReHashBackend->outputFiles(), [&](StringRef F) { + AllOutputs.insert(F.str()); + }); + + DiagnosticEngine &diags = Instance->getDiags(); + for (auto &Filename : AllOutputs) { + auto O1 = HashBackend->getHashValueForFile(Filename); + if (!O1) { + diags.diagnose(SourceLoc(), diag::error_output_missing, Filename, + /*SecondRun=*/false); + HadError = true; + continue; + } + auto O2 = ReHashBackend->getHashValueForFile(Filename); + if (!O2) { + diags.diagnose(SourceLoc(), diag::error_output_missing, Filename, + /*SecondRun=*/true); + HadError = true; + continue; + } + if (*O1 != *O2) { + diags.diagnose(SourceLoc(), diag::error_nondeterministic_output, + Filename, *O1, *O2); + HadError = true; + continue; + } + diags.diagnose(SourceLoc(), diag::matching_output_produced, Filename, + *O1); + } + } + auto r = finishDiagProcessing(HadError ? 1 : ReturnValue, verifierEnabled); if (auto *StatsReporter = Instance->getStatsReporter()) StatsReporter->noteCurrentProcessExitStatus(r); diff --git a/lib/FrontendTool/ImportedModules.cpp b/lib/FrontendTool/ImportedModules.cpp index 595c9351a3b03..39d9cbf58c4bd 100644 --- a/lib/FrontendTool/ImportedModules.cpp +++ b/lib/FrontendTool/ImportedModules.cpp @@ -24,6 +24,7 @@ #include "llvm/ADT/SetVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/VirtualOutputBackend.h" using namespace swift; @@ -43,16 +44,15 @@ static void findAllClangImports(const clang::Module *module, } bool swift::emitImportedModules(ModuleDecl *mainModule, - const FrontendOptions &opts) { + const FrontendOptions &opts, + llvm::vfs::OutputBackend &backend) { auto &Context = mainModule->getASTContext(); std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); - std::error_code EC; - llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::OF_None); - - if (out.has_error() || EC) { - Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, - EC.message()); - out.clear_error(); + auto &diags = Context.Diags; + auto out = backend.createFile(path); + if (!out) { + diags.diagnose(SourceLoc(), diag::error_opening_output, + path, toString(out.takeError())); return true; } @@ -110,7 +110,13 @@ bool swift::emitImportedModules(ModuleDecl *mainModule, } for (auto name : Modules) { - out << name << "\n"; + *out << name << "\n"; + } + + if (auto error = out->keep()) { + diags.diagnose(SourceLoc(), diag::error_closing_output, + path, toString(std::move(error))); + return true; } return false; diff --git a/lib/FrontendTool/MakeStyleDependencies.cpp b/lib/FrontendTool/MakeStyleDependencies.cpp index 38898bbae791f..4aeea9701b47b 100644 --- a/lib/FrontendTool/MakeStyleDependencies.cpp +++ b/lib/FrontendTool/MakeStyleDependencies.cpp @@ -20,6 +20,7 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Support/VirtualOutputBackend.h" using namespace swift; @@ -91,18 +92,16 @@ reversePathSortedFilenames(const Container &elts) { bool swift::emitMakeDependenciesIfNeeded(DiagnosticEngine &diags, DependencyTracker *depTracker, const FrontendOptions &opts, - const InputFile &input) { + const InputFile &input, + llvm::vfs::OutputBackend &backend) { auto dependenciesFilePath = input.getDependenciesFilePath(); if (dependenciesFilePath.empty()) return false; - std::error_code EC; - llvm::raw_fd_ostream out(dependenciesFilePath, EC, llvm::sys::fs::OF_None); - - if (out.has_error() || EC) { + auto out = backend.createFile(dependenciesFilePath); + if (!out) { diags.diagnose(SourceLoc(), diag::error_opening_output, - dependenciesFilePath, EC.message()); - out.clear_error(); + dependenciesFilePath, toString(out.takeError())); return true; } @@ -138,8 +137,14 @@ bool swift::emitMakeDependenciesIfNeeded(DiagnosticEngine &diags, // dependency line. opts.forAllOutputPaths(input, [&](const StringRef targetName) { auto targetNameEscaped = frontend::utils::escapeForMake(targetName, buffer); - out << targetNameEscaped << " :" << dependencyString << '\n'; + *out << targetNameEscaped << " :" << dependencyString << '\n'; }); + if (auto error = out->keep()) { + diags.diagnose(SourceLoc(), diag::error_closing_output, + dependenciesFilePath, toString(std::move(error))); + return true; + } + return false; } diff --git a/lib/IRGen/IRGen.cpp b/lib/IRGen/IRGen.cpp index 2df7a065bf92e..e653747e52546 100644 --- a/lib/IRGen/IRGen.cpp +++ b/lib/IRGen/IRGen.cpp @@ -46,6 +46,7 @@ #include "swift/Subsystems.h" #include "clang/Basic/TargetInfo.h" #include "clang/Frontend/CompilerInstance.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/StringSet.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Bitcode/BitcodeWriter.h" @@ -74,6 +75,8 @@ #include "llvm/Support/FormattedStream.h" #include "llvm/Support/Mutex.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputConfig.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" @@ -534,6 +537,7 @@ bool swift::performLLVM(const IRGenOptions &Opts, llvm::Module *Module, llvm::TargetMachine *TargetMachine, StringRef OutputFilename, + llvm::vfs::OutputBackend &Backend, UnifiedStatsReporter *Stats) { if (Opts.UseIncrementalLLVMCodeGen && HashGlobal) { @@ -552,7 +556,7 @@ bool swift::performLLVM(const IRGenOptions &Opts, ArrayRef HashData(reinterpret_cast(&hash), sizeof(hash)); if (Opts.OutputKind == IRGenOutputKind::ObjectFile && - !Opts.PrintInlineTree && + !Opts.PrintInlineTree && !Opts.AlwaysCompile && !needsRecompile(OutputFilename, HashData, HashGlobal, DiagMutex)) { // The llvm IR did not change. We don't need to re-create the object file. return false; @@ -564,23 +568,28 @@ bool swift::performLLVM(const IRGenOptions &Opts, HashGlobal->setInitializer(HashConstant); } - Optional RawOS; + llvm::Optional OutputFile; + SWIFT_DEFER { + if (!OutputFile) + return; + if (auto E = OutputFile->keep()) { + diagnoseSync(Diags, DiagMutex, SourceLoc(), diag::error_closing_output, + OutputFilename, toString(std::move(E))); + } + }; if (!OutputFilename.empty()) { // Try to open the output file. Clobbering an existing file is fine. // Open in binary mode if we're doing binary output. - llvm::sys::fs::OpenFlags OSFlags = llvm::sys::fs::OF_None; - std::error_code EC; - RawOS.emplace(OutputFilename, EC, OSFlags); - - if (RawOS->has_error() || EC) { - diagnoseSync(Diags, DiagMutex, - SourceLoc(), diag::error_opening_output, - OutputFilename, EC.message()); - RawOS->clear_error(); + llvm::vfs::OutputConfig Config; + if (auto E = + Backend.createFile(OutputFilename, Config).moveInto(OutputFile)) { + diagnoseSync(Diags, DiagMutex, SourceLoc(), diag::error_opening_output, + OutputFilename, toString(std::move(E))); return true; } + if (Opts.OutputKind == IRGenOutputKind::LLVMAssemblyBeforeOptimization) { - Module->print(RawOS.value(), nullptr); + Module->print(*OutputFile, nullptr); return false; } } else { @@ -588,7 +597,7 @@ bool swift::performLLVM(const IRGenOptions &Opts, } performLLVMOptimizations(Opts, Module, TargetMachine, - RawOS ? &*RawOS : nullptr); + OutputFile ? &OutputFile->getOS() : nullptr); if (Stats) { if (DiagMutex) @@ -598,11 +607,11 @@ bool swift::performLLVM(const IRGenOptions &Opts, DiagMutex->unlock(); } - if (!RawOS) + if (OutputFilename.empty()) return false; - return compileAndWriteLLVM(Module, TargetMachine, Opts, Stats, Diags, *RawOS, - DiagMutex); + return compileAndWriteLLVM(Module, TargetMachine, Opts, Stats, Diags, + *OutputFile, DiagMutex); } bool swift::compileAndWriteLLVM(llvm::Module *module, @@ -1241,7 +1250,8 @@ struct LLVMCodeGenThreads { embedBitcode(IGM->getModule(), parent.irgen->Opts); performLLVM(parent.irgen->Opts, IGM->Context.Diags, diagMutex, IGM->ModuleHash, IGM->getModule(), IGM->TargetMachine.get(), - IGM->OutputFilename, IGM->Context.Stats); + IGM->OutputFilename, IGM->Context.getOutputBackend(), + IGM->Context.Stats); if (IGM->Context.Diags.hadAnyError()) return; } @@ -1359,10 +1369,9 @@ static void performParallelIRGeneration(IRGenDescriptor desc) { if (!targetMachine) continue; // Create the IR emitter. - IRGenModule *IGM = - new IRGenModule(irgen, std::move(targetMachine), nextSF, - desc.ModuleName, *OutputIter++, nextSF->getFilename(), - nextSF->getPrivateDiscriminator().str()); + IRGenModule *IGM = new IRGenModule( + irgen, std::move(targetMachine), nextSF, desc.ModuleName, *OutputIter++, + nextSF->getFilename(), nextSF->getPrivateDiscriminator().str()); IGMcreated = true; initLLVMModule(*IGM, *SILMod); @@ -1535,9 +1544,9 @@ GeneratedModule swift::performIRGeneration( const auto *SILModPtr = SILMod.get(); const auto &SILOpts = SILModPtr->getOptions(); auto desc = IRGenDescriptor::forWholeModule( - M, Opts, TBDOpts, SILOpts, SILModPtr->Types, std::move(SILMod), - ModuleName, PSPs, /*symsToEmit*/ None, parallelOutputFilenames, - outModuleHash); + M, Opts, TBDOpts, SILOpts, SILModPtr->Types, + std::move(SILMod), ModuleName, PSPs, /*symsToEmit*/ None, + parallelOutputFilenames, outModuleHash); if (Opts.shouldPerformIRGenerationInParallel() && !parallelOutputFilenames.empty() && @@ -1561,15 +1570,14 @@ performIRGeneration(FileUnit *file, const IRGenOptions &Opts, const auto *SILModPtr = SILMod.get(); const auto &SILOpts = SILModPtr->getOptions(); auto desc = IRGenDescriptor::forFile( - file, Opts, TBDOpts, SILOpts, SILModPtr->Types, std::move(SILMod), - ModuleName, PSPs, PrivateDiscriminator, /*symsToEmit*/ None, - outModuleHash); + file, Opts, TBDOpts, SILOpts, SILModPtr->Types, + std::move(SILMod), ModuleName, PSPs, PrivateDiscriminator, + /*symsToEmit*/ None, outModuleHash); return llvm::cantFail(file->getASTContext().evaluator(IRGenRequest{desc})); } -void -swift::createSwiftModuleObjectFile(SILModule &SILMod, StringRef Buffer, - StringRef OutputPath) { +void swift::createSwiftModuleObjectFile(SILModule &SILMod, StringRef Buffer, + StringRef OutputPath) { auto &Ctx = SILMod.getASTContext(); assert(!Ctx.hadError()); @@ -1620,7 +1628,7 @@ swift::createSwiftModuleObjectFile(SILModule &SILMod, StringRef Buffer, ASTSym->setAlignment(llvm::MaybeAlign(serialization::SWIFTMODULE_ALIGNMENT)); ::performLLVM(Opts, Ctx.Diags, nullptr, nullptr, IGM.getModule(), IGM.TargetMachine.get(), - OutputPath, Ctx.Stats); + OutputPath, Ctx.getOutputBackend(), Ctx.Stats); } bool swift::performLLVM(const IRGenOptions &Opts, ASTContext &Ctx, @@ -1636,8 +1644,8 @@ bool swift::performLLVM(const IRGenOptions &Opts, ASTContext &Ctx, embedBitcode(Module, Opts); if (::performLLVM(Opts, Ctx.Diags, nullptr, nullptr, Module, - TargetMachine.get(), - OutputFilename, Ctx.Stats)) + TargetMachine.get(), OutputFilename, Ctx.getOutputBackend(), + Ctx.Stats)) return true; return false; } diff --git a/lib/IRGen/IRGenModule.cpp b/lib/IRGen/IRGenModule.cpp index 5537135d163ba..75ee76b427684 100644 --- a/lib/IRGen/IRGenModule.cpp +++ b/lib/IRGen/IRGenModule.cpp @@ -2034,7 +2034,6 @@ bool swift::writeEmptyOutputFilesFor( const ASTContext &Context, std::vector& ParallelOutputFilenames, const IRGenOptions &IRGenOpts) { - for (auto fileName : ParallelOutputFilenames) { // The first output file, was use for genuine output. if (fileName == ParallelOutputFilenames[0]) diff --git a/lib/Immediate/Immediate.cpp b/lib/Immediate/Immediate.cpp index 74195af2dff94..5e20fe1485b25 100644 --- a/lib/Immediate/Immediate.cpp +++ b/lib/Immediate/Immediate.cpp @@ -248,7 +248,8 @@ int swift::RunImmediately(CompilerInstance &CI, performLLVM(IRGenOpts, Context.Diags, /*diagMutex*/ nullptr, /*hash*/ nullptr, GenModule.getModule(), GenModule.getTargetMachine(), - PSPs.OutputFilename, Context.Stats); + PSPs.OutputFilename, CI.getOutputBackend(), + Context.Stats); if (Context.hadError()) return -1; diff --git a/lib/SymbolGraphGen/SymbolGraphGen.cpp b/lib/SymbolGraphGen/SymbolGraphGen.cpp index 8256c84d89291..101c34b1b2b11 100644 --- a/lib/SymbolGraphGen/SymbolGraphGen.cpp +++ b/lib/SymbolGraphGen/SymbolGraphGen.cpp @@ -40,11 +40,13 @@ int serializeSymbolGraph(SymbolGraph &SG, SmallString<1024> OutputPath(Options.OutputDir); llvm::sys::path::append(OutputPath, FileName); - return withOutputFile(SG.M.getASTContext().Diags, OutputPath, [&](raw_ostream &OS) { - llvm::json::OStream J(OS, Options.PrettyPrint ? 2 : 0); - SG.serialize(J); - return false; - }); + return withOutputPath( + SG.M.getASTContext().Diags, SG.M.getASTContext().getOutputBackend(), + OutputPath, [&](raw_ostream &OS) { + llvm::json::OStream J(OS, Options.PrettyPrint ? 2 : 0); + SG.serialize(J); + return false; + }); } } // end anonymous namespace diff --git a/test/Frontend/output_determinism_check.swift b/test/Frontend/output_determinism_check.swift new file mode 100644 index 0000000000000..318ffaaead8f5 --- /dev/null +++ b/test/Frontend/output_determinism_check.swift @@ -0,0 +1,40 @@ +// RUN: %empty-directory(%t) +// RUN: %target-swift-frontend -module-name test -emit-module -o %t/test.swiftmodule -primary-file %s -emit-module-doc-path %t/test.docc -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=MODULE_OUTPUT --check-prefix=DOCC_OUTPUT +// RUN: %target-swift-frontend -module-name test -emit-sib -o %t/test.sib -primary-file %s -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=SIB_OUTPUT + +/// object files are "not" deterministic because the second run going to match the mod hash and skip code generation. +// RUN: not %target-swift-frontend -module-name test -c -o %t/test.o -primary-file %s -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=OBJECT_MISMATCH +/// object files should match when forcing object generation. +// RUN: %target-swift-frontend -module-name test -emit-dependencies -c -o %t/test.o -primary-file %s -enable-deterministic-check -always-compile-output-files 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT --check-prefix=DEPS_OUTPUT + +/// FIXME: Fine-grain dependencies graph is not deterministics. +/// FAIL: %target-swift-frontend -module-name test -emit-reference-dependencies-path %t/test.swiftdeps -c -o %t/test.o -primary-file %s -enable-deterministic-check -always-compile-output-files + +/// Explicit module build. Check building swiftmodule from interface file. +// RUN: %target-swift-frontend -scan-dependencies -module-name test -o %t/test.json %s -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=DEPSCAN_OUTPUT +/// TODO: Implicit module build use a different compiler instance so it doesn't support checking yet. +// RUN: %target-swift-frontend -typecheck -emit-module-interface-path %t/test.swiftinterface %s -O -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=INTERFACE_OUTPUT +/// Hit cache and not emit the second time. +// RUN: rm %t/test.swiftmodule +// RUN: not %target-swift-frontend -compile-module-from-interface %t/test.swiftinterface -explicit-interface-module-build -o %t/test.swiftmodule -enable-deterministic-check 2>&1 | %FileCheck --check-prefix=MODULE_MISMATCH %s +/// Force swiftmodule generation. +// RUN: %target-swift-frontend -compile-module-from-interface %t/test.swiftinterface -explicit-interface-module-build -o %t/test.swiftmodule -enable-deterministic-check -always-compile-output-files 2>&1 | %FileCheck --check-prefix=MODULE_OUTPUT %s + +// RUN: %target-swift-frontend -scan-dependencies -module-name test %s -o %t/test.deps.json -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=DEPS_JSON_OUTPUT + +// RUN: %target-swift-frontend -emit-pcm -module-name UserClangModule -o %t/test.pcm %S/Inputs/dependencies/module.modulemap -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=PCM_OUTPUT + +// DOCC_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.docc' +// MODULE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftmodule' +// SIB_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.sib' +// DEPS_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.d' +// OBJECT_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.o' +// OBJECT_MISMATCH: error: output file '{{.*}}{{/|\\}}test.o' is missing from second compilation for deterministic check +// DEPSCAN_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.json' +// INTERFACE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftinterface' +// MODULE_MISMATCH: error: output file '{{.*}}{{/|\\}}test.swiftmodule' is missing from second compilation for deterministic check +// DEPS_JSON_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.deps.json' +// PCM_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.pcm' + +public var x = 1 +public func test() {} diff --git a/tools/sil-llvm-gen/SILLLVMGen.cpp b/tools/sil-llvm-gen/SILLLVMGen.cpp index 9b9ff651f456a..b5e47878b7430 100644 --- a/tools/sil-llvm-gen/SILLLVMGen.cpp +++ b/tools/sil-llvm-gen/SILLLVMGen.cpp @@ -34,6 +34,7 @@ #include "swift/Serialization/SerializedSILLoader.h" #include "swift/Strings.h" #include "swift/Subsystems.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/Statistic.h" #include "llvm/IR/Module.h" #include "llvm/Support/CommandLine.h" @@ -43,6 +44,7 @@ #include "llvm/Support/Path.h" #include "llvm/Support/Signals.h" #include "llvm/Support/TargetSelect.h" +#include "llvm/Support/VirtualOutputBackends.h" #include using namespace swift; @@ -181,14 +183,19 @@ int main(int argc, char **argv) { return 1; } - std::error_code EC; - llvm::raw_fd_ostream outStream(OutputFilename, EC, llvm::sys::fs::OF_None); - if (outStream.has_error() || EC) { + llvm::vfs::OnDiskOutputBackend Backend; + auto outFile = Backend.createFile(OutputFilename); + if (!outFile) { CI.getDiags().diagnose(SourceLoc(), diag::error_opening_output, - OutputFilename, EC.message()); - outStream.clear_error(); + OutputFilename, toString(outFile.takeError())); return 1; } + auto closeFile = llvm::make_scope_exit([&]() { + if (auto E = outFile->keep()) { + CI.getDiags().diagnose(SourceLoc(), diag::error_closing_output, + OutputFilename, toString(std::move(E))); + } + }); auto *mod = CI.getMainModule(); assert(mod->getFiles().size() == 1); @@ -205,20 +212,20 @@ int main(int argc, char **argv) { mod, Opts, TBDOpts, SILOpts, SILTypes, /*SILMod*/ nullptr, moduleName, PSPs); } else { - return IRGenDescriptor::forFile(mod->getFiles()[0], Opts, TBDOpts, - SILOpts, SILTypes, /*SILMod*/ nullptr, - moduleName, PSPs, /*discriminator*/ ""); + return IRGenDescriptor::forFile( + mod->getFiles()[0], Opts, TBDOpts, SILOpts, SILTypes, + /*SILMod*/ nullptr, moduleName, PSPs, /*discriminator*/ ""); } }; auto &eval = CI.getASTContext().evaluator; auto desc = getDescriptor(); - desc.out = &outStream; + desc.out = &outFile->getOS(); auto generatedMod = llvm::cantFail(eval(OptimizedIRRequest{desc})); if (!generatedMod) return 1; return compileAndWriteLLVM(generatedMod.getModule(), generatedMod.getTargetMachine(), Opts, - CI.getStatsReporter(), CI.getDiags(), outStream); + CI.getStatsReporter(), CI.getDiags(), *outFile); } diff --git a/tools/swift-dependency-tool/swift-dependency-tool.cpp b/tools/swift-dependency-tool/swift-dependency-tool.cpp index cd4854b2fc9df..35737d832d63e 100644 --- a/tools/swift-dependency-tool/swift-dependency-tool.cpp +++ b/tools/swift-dependency-tool/swift-dependency-tool.cpp @@ -18,6 +18,7 @@ #include "swift/Basic/LLVMInitialize.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/VirtualOutputBackends.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/YAMLTraits.h" @@ -208,6 +209,7 @@ int main(int argc, char *argv[]) { SourceManager sourceMgr; DiagnosticEngine diags(sourceMgr); + llvm::vfs::OnDiskOutputBackend outputBackend; switch (options::Action) { case ActionType::None: { @@ -224,7 +226,7 @@ int main(int argc, char *argv[]) { } bool hadError = - withOutputFile(diags, options::OutputFilename, + withOutputPath(diags, outputBackend, options::OutputFilename, [&](llvm::raw_pwrite_stream &out) { out << "# Fine-grained v0\n"; llvm::yaml::Output yamlWriter(out); @@ -256,7 +258,7 @@ int main(int argc, char *argv[]) { } if (writeFineGrainedDependencyGraphToPath( - diags, options::OutputFilename, fg)) { + diags, outputBackend, options::OutputFilename, fg)) { llvm::errs() << "Failed to write binary swiftdeps\n"; return 1; } diff --git a/unittests/ClangImporter/ClangImporterTests.cpp b/unittests/ClangImporter/ClangImporterTests.cpp index 074e660f97f8a..f5f631dc0af12 100644 --- a/unittests/ClangImporter/ClangImporterTests.cpp +++ b/unittests/ClangImporter/ClangImporterTests.cpp @@ -7,8 +7,10 @@ #include "swift/Basic/SourceManager.h" #include "swift/ClangImporter/ClangImporter.h" #include "swift/SymbolGraphGen/SymbolGraphOptions.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" +#include "llvm/Support/VirtualOutputBackends.h" #include "llvm/Support/raw_ostream.h" #include "gtest/gtest.h" diff --git a/unittests/Driver/MockingFineGrainedDependencyGraphs.cpp b/unittests/Driver/MockingFineGrainedDependencyGraphs.cpp index 327f871af8c50..1ce0f2fe3d534 100644 --- a/unittests/Driver/MockingFineGrainedDependencyGraphs.cpp +++ b/unittests/Driver/MockingFineGrainedDependencyGraphs.cpp @@ -15,6 +15,7 @@ #include "swift/AST/DiagnosticEngine.h" #include "swift/Basic/ReferenceDependencyKeys.h" #include "swift/Basic/SourceManager.h" +#include "llvm/Support/VirtualOutputBackends.h" #include #include @@ -48,12 +49,13 @@ mocking_fine_grained_dependency_graphs::getChangesForSimulatedLoad( SourceManager sm; DiagnosticEngine diags(sm); + llvm::vfs::OnDiskOutputBackend backend; auto sfdg = UnitTestSourceFileDepGraphFactory( hadCompilationError, swiftDeps, interfaceHash, g.emitFineGrainedDependencyDotFileAfterEveryImport, - dependencyDescriptions, diags) + dependencyDescriptions, diags, backend) .construct(); return g.loadFromSourceFileDepGraph(cmd, sfdg, diags); diff --git a/unittests/Driver/UnitTestSourceFileDepGraphFactory.h b/unittests/Driver/UnitTestSourceFileDepGraphFactory.h index 2a53bf89521c2..90f32a1239273 100644 --- a/unittests/Driver/UnitTestSourceFileDepGraphFactory.h +++ b/unittests/Driver/UnitTestSourceFileDepGraphFactory.h @@ -13,6 +13,7 @@ #define UnitTestSourceFileDepGraphFactory_h #include "swift/AST/AbstractSourceFileDepGraphFactory.h" +#include "llvm/Support/VirtualOutputBackend.h" namespace swift { namespace fine_grained_dependencies { @@ -30,10 +31,10 @@ class UnitTestSourceFileDepGraphFactory bool hadCompilationError, StringRef swiftDeps, Fingerprint fileFingerprint, bool emitDotFileAfterConstruction, const DependencyDescriptions &dependencyDescriptions, - DiagnosticEngine &diags) + DiagnosticEngine &diags, llvm::vfs::OutputBackend &backend) : AbstractSourceFileDepGraphFactory( hadCompilationError, swiftDeps, fileFingerprint, - emitDotFileAfterConstruction, diags), + emitDotFileAfterConstruction, diags, backend), dependencyDescriptions(dependencyDescriptions) {} ~UnitTestSourceFileDepGraphFactory() override = default;