diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h index c0ab977ec71a5..4448b41be61ed 100644 --- a/include/swift/AST/ASTContext.h +++ b/include/swift/AST/ASTContext.h @@ -78,6 +78,7 @@ namespace swift { class LazyContextData; class LazyIterableDeclContextData; class LazyMemberLoader; + class ModuleDependencies; class PatternBindingDecl; class PatternBindingInitializer; class SourceFile; @@ -91,6 +92,7 @@ namespace swift { class Identifier; class InheritedNameSet; class ModuleDecl; + class ModuleDependenciesCache; class ModuleLoader; class NominalTypeDecl; class NormalProtocolConformance; @@ -710,6 +712,15 @@ class ASTContext final { void addModuleLoader(std::unique_ptr loader, bool isClang = false, bool isDWARF = false); + /// Retrieve the module dependencies for the module with the given name. + /// + /// \param isUnderlyingClangModule When true, only look for a Clang module + /// with the given name, ignoring any Swift modules. + Optional getModuleDependencies( + StringRef moduleName, + bool isUnderlyingClangModule, + ModuleDependenciesCache &cache); + /// Load extensions to the given nominal type from the external /// module loaders. /// diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h new file mode 100644 index 0000000000000..5e086bfb7e5a5 --- /dev/null +++ b/include/swift/AST/ModuleDependencies.h @@ -0,0 +1,326 @@ +//===--- ModuleDependencies.h - Module Dependencies -------------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// This file defines data structures for capturing all of the source files +// and modules on which a given module depends, forming a graph of all of the +// modules that need to be present for a given module to be built. +// +//===----------------------------------------------------------------------===// +#ifndef SWIFT_AST_MODULE_DEPENDENCIES_H +#define SWIFT_AST_MODULE_DEPENDENCIES_H + +#include "swift/Basic/LLVM.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringSet.h" +#include +#include + +namespace swift { + +class ClangModuleDependenciesCacheImpl; +class SourceFile; + +/// Which kind of module dependencies we are looking for. +enum class ModuleDependenciesKind : int8_t { + Swift, + Clang, +}; + +/// Base class for the variant storage of ModuleDependencies. +/// +/// This class is mostly an implementation detail for \c ModuleDependencies. +class ModuleDependenciesStorageBase { +public: + const bool isSwiftModule; + + ModuleDependenciesStorageBase(bool isSwiftModule, + const std::string &compiledModulePath) + : isSwiftModule(isSwiftModule), + compiledModulePath(compiledModulePath) { } + + virtual ModuleDependenciesStorageBase *clone() const = 0; + + virtual ~ModuleDependenciesStorageBase(); + + /// The path to the compiled module file. + const std::string compiledModulePath; + + /// The set of modules on which this module depends. + std::vector moduleDependencies; +}; + +/// Describes the dependencies of a Swift module. +/// +/// This class is mostly an implementation detail for \c ModuleDependencies. +class SwiftModuleDependenciesStorage : public ModuleDependenciesStorageBase { +public: + /// The Swift interface file, if it can be used to generate the module file. + const Optional swiftInterfaceFile; + + /// Bridging header file, if there is one. + Optional bridgingHeaderFile; + + /// Swift source files that are part of the Swift module, when known. + std::vector sourceFiles; + + /// Source files on which the bridging header depends. + std::vector bridgingSourceFiles; + + /// (Clang) modules on which the bridging header depends. + std::vector bridgingModuleDependencies; + + SwiftModuleDependenciesStorage( + const std::string &compiledModulePath, + const Optional &swiftInterfaceFile + ) : ModuleDependenciesStorageBase(/*isSwiftModule=*/true, compiledModulePath), + swiftInterfaceFile(swiftInterfaceFile) { } + + ModuleDependenciesStorageBase *clone() const override { + return new SwiftModuleDependenciesStorage(*this); + } + + static bool classof(const ModuleDependenciesStorageBase *base) { + return base->isSwiftModule; + } +}; + +/// Describes the dependencies of a Clang module. +/// +/// This class is mostly an implementation detail for \c ModuleDependencies. +class ClangModuleDependenciesStorage : public ModuleDependenciesStorageBase { +public: + /// The module map file used to generate the Clang module. + const std::string moduleMapFile; + + /// The context hash describing the configuration options for this module. + const std::string contextHash; + + /// Partial (Clang) command line that can be used to build this module. + const std::vector nonPathCommandLine; + + /// The file dependencies + const std::vector fileDependencies; + + ClangModuleDependenciesStorage( + const std::string &compiledModulePath, + const std::string &moduleMapFile, + const std::string &contextHash, + const std::vector &nonPathCommandLine, + const std::vector &fileDependencies + ) : ModuleDependenciesStorageBase(/*isSwiftModule=*/false, + compiledModulePath), + moduleMapFile(moduleMapFile), + contextHash(contextHash), + nonPathCommandLine(nonPathCommandLine), + fileDependencies(fileDependencies) { } + + ModuleDependenciesStorageBase *clone() const override { + return new ClangModuleDependenciesStorage(*this); + } + + static bool classof(const ModuleDependenciesStorageBase *base) { + return !base->isSwiftModule; + } +}; + +/// Describes the dependencies of a given module. +/// +/// The dependencies of a module include all of the source files that go +/// into that module, as well as any modules that are directly imported +/// into the module. +class ModuleDependencies { +private: + std::unique_ptr storage; + + ModuleDependencies(std::unique_ptr &&storage) + : storage(std::move(storage)) { } + +public: + ModuleDependencies(const ModuleDependencies &other) + : storage(other.storage->clone()) { } + ModuleDependencies(ModuleDependencies &&other) = default; + + ModuleDependencies &operator=(const ModuleDependencies &other) { + storage.reset(other.storage->clone()); + return *this; + } + + ModuleDependencies &operator=(ModuleDependencies &&other) = default; + + /// Describe the module dependencies for a Swift module that can be + /// built from a Swift interface file (\c .swiftinterface). + static ModuleDependencies forSwiftInterface( + const std::string &compiledModulePath, + const std::string &swiftInterfaceFile) { + return ModuleDependencies( + std::make_unique( + compiledModulePath, swiftInterfaceFile)); + } + + /// Describe the module dependencies for a serialized or parsed Swift module. + static ModuleDependencies forSwiftModule( + const std::string &compiledModulePath) { + return ModuleDependencies( + std::make_unique( + compiledModulePath, None)); + } + + /// Describe the module dependencies for a Clang module that can be + /// built from a module map and headers. + static ModuleDependencies forClangModule( + const std::string &compiledModulePath, + const std::string &moduleMapFile, + const std::string &contextHash, + const std::vector &nonPathCommandLine, + const std::vector &fileDependencies) { + return ModuleDependencies( + std::make_unique( + compiledModulePath, moduleMapFile, contextHash, nonPathCommandLine, + fileDependencies)); + } + + /// Retrieve the path to the compiled module. + const std::string getCompiledModulePath() const { + return storage->compiledModulePath; + } + + /// Retrieve the module-level dependencies. + ArrayRef getModuleDependencies() const { + return storage->moduleDependencies; + } + + /// Whether the dependencies are for a Swift module. + bool isSwiftModule() const; + + ModuleDependenciesKind getKind() const { + return isSwiftModule() ? ModuleDependenciesKind::Swift + : ModuleDependenciesKind::Clang; + } + /// Retrieve the dependencies for a Swift module. + const SwiftModuleDependenciesStorage *getAsSwiftModule() const; + + /// Retrieve the dependencies for a Clang module. + const ClangModuleDependenciesStorage *getAsClangModule() const; + + /// Add a dependency on the given module, if it was not already in the set. + void addModuleDependency(StringRef module, + llvm::StringSet<> &alreadyAddedModules); + + /// Add all of the module dependencies for the imports in the given source + /// file to the set of module dependencies. + void addModuleDependencies(const SourceFile &sf, + llvm::StringSet<> &alreadyAddedModules); + + /// Get the bridging header. + Optional getBridgingHeader() const; + + /// Add a bridging header to a Swift module's dependencies. + void addBridgingHeader(StringRef bridgingHeader); + + /// Add source files that the bridging header depends on. + void addBridgingSourceFile(StringRef bridgingSourceFile); + + /// Add (Clang) module on which the bridging header depends. + void addBridgingModuleDependency(StringRef module, + llvm::StringSet<> &alreadyAddedModules); +}; + +using ModuleDependencyID = std::pair; + +/// A cache describing the set of module dependencies that has been queried +/// thus far. +class ModuleDependenciesCache { + /// All cached module dependencies, in the order in which they were + /// encountered. + std::vector AllModules; + + /// Dependencies for Swift modules that have already been computed. + llvm::StringMap SwiftModuleDependencies; + + /// Dependencies for Clang modules that have already been computed. + llvm::StringMap ClangModuleDependencies; + + /// Additional information needed for Clang dependency scanning. + ClangModuleDependenciesCacheImpl *clangImpl = nullptr; + + /// Function that will delete \c clangImpl properly. + void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *) = nullptr; + + /// Free up the storage associated with the Clang implementation. + void destroyClangImpl() { + if (this->clangImplDeleter) + this->clangImplDeleter(this->clangImpl); + } + + /// Retrieve the dependencies map that corresponds to the given dependency + /// kind. + llvm::StringMap &getDependenciesMap( + ModuleDependenciesKind kind); + const llvm::StringMap &getDependenciesMap( + ModuleDependenciesKind kind) const; + +public: + ModuleDependenciesCache() { } + + ModuleDependenciesCache(const ModuleDependenciesCache &) = delete; + ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete; + + ~ModuleDependenciesCache() { + destroyClangImpl(); + } + + /// Set the Clang-specific implementation data. + void setClangImpl( + ClangModuleDependenciesCacheImpl *clangImpl, + void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *)) { + destroyClangImpl(); + + this->clangImpl = clangImpl; + this->clangImplDeleter = clangImplDeleter; + } + + /// Retrieve the Clang-specific implementation data; + ClangModuleDependenciesCacheImpl *getClangImpl() const { + return clangImpl; + } + + /// Whether we have cached dependency information for the given module. + bool hasDependencies(StringRef moduleName, + Optional kind) const; + + /// Look for module dependencies for a module with the given name. + /// + /// \returns the cached result, or \c None if there is no cached entry. + Optional findDependencies( + StringRef moduleName, + Optional kind) const; + + /// Record dependencies for the given module. + void recordDependencies(StringRef moduleName, + ModuleDependencies dependencies, + ModuleDependenciesKind kind); + + /// Update stored dependencies for the given module. + void updateDependencies(ModuleDependencyID moduleID, + ModuleDependencies dependencies); + + /// Reference the list of all module dependencies. + const std::vector &getAllModules() const { + return AllModules; + } +}; + +} + +#endif /* SWIFT_AST_MODULE_DEPENDENCIES_H */ diff --git a/include/swift/AST/ModuleLoader.h b/include/swift/AST/ModuleLoader.h index 8e13dae85fdaf..4e38cfac812b0 100644 --- a/include/swift/AST/ModuleLoader.h +++ b/include/swift/AST/ModuleLoader.h @@ -23,6 +23,7 @@ #include "swift/Basic/SourceLoc.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/StringSet.h" #include "llvm/ADT/TinyPtrVector.h" namespace llvm { @@ -41,7 +42,10 @@ class ClangImporterOptions; class ClassDecl; class FileUnit; class ModuleDecl; +class ModuleDependencies; +class ModuleDependenciesCache; class NominalTypeDecl; +class SourceFile; class TypeDecl; enum class KnownProtocolKind : uint8_t; @@ -177,6 +181,12 @@ class ModuleLoader { /// Discover overlays declared alongside this file and add infomation about /// them to it. void findOverlayFiles(SourceLoc diagLoc, ModuleDecl *module, FileUnit *file); + + /// Retrieve the dependencies for the given, named module, or \c None + /// if no such module exists. + virtual Optional getModuleDependencies( + StringRef moduleName, + ModuleDependenciesCache &cache) = 0; }; } // namespace swift diff --git a/include/swift/Basic/FileTypes.def b/include/swift/Basic/FileTypes.def index 1465d14607c74..1435813ae6e6c 100644 --- a/include/swift/Basic/FileTypes.def +++ b/include/swift/Basic/FileTypes.def @@ -71,6 +71,9 @@ TYPE("tbd", TBD, "tbd", "") // Swift section of the internal wiki. TYPE("module-trace", ModuleTrace, "trace.json", "") +// Complete dependency information for the given Swift files as JSON. +TYPE("json-dependencies", JSONDependencies, "dependencies.json", "") + TYPE("index-data", IndexData, "", "") TYPE("yaml-opt-record", YAMLOptRecord, "opt.yaml", "") TYPE("bitstream-opt-record",BitstreamOptRecord, "opt.bitstream", "") diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h index f401e5c5094af..796c9015f4a11 100644 --- a/include/swift/ClangImporter/ClangImporter.h +++ b/include/swift/ClangImporter/ClangImporter.h @@ -369,6 +369,22 @@ class ClangImporter final : public ClangModuleLoader { void verifyAllModules() override; + Optional getModuleDependencies( + StringRef moduleName, ModuleDependenciesCache &cache) override; + + /// Add dependency information for the bridging header. + /// + /// \param moduleName the name of the Swift module whose dependency + /// information will be augmented with information about the given + /// bridging header. + /// + /// \param cache The module dependencies cache to update, with information + /// about new Clang modules discovered along the way. + /// + /// \returns \c true if an error occurred, \c false otherwise + bool addBridgingHeaderDependencies( + StringRef moduleName, ModuleDependenciesCache &cache); + clang::TargetInfo &getTargetInfo() const override; clang::ASTContext &getClangASTContext() const override; clang::Preprocessor &getClangPreprocessor() const override; diff --git a/include/swift/Frontend/Frontend.h b/include/swift/Frontend/Frontend.h index 5624ae0df631b..c58c0cf297407 100644 --- a/include/swift/Frontend/Frontend.h +++ b/include/swift/Frontend/Frontend.h @@ -346,6 +346,9 @@ class CompilerInvocation { return ImplicitStdlibKind::Stdlib; } + /// Whether the Swift -Onone support library should be implicitly imported. + bool shouldImportSwiftONoneSupport() const; + /// Performs input setup common to these tools: /// sil-opt, sil-func-extractor, sil-llvm-gen, and sil-nm. /// Return value includes the buffer so caller can keep it alive. diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h index fc83b750579d6..6b4e03f2506ca 100644 --- a/include/swift/Frontend/FrontendOptions.h +++ b/include/swift/Frontend/FrontendOptions.h @@ -132,6 +132,8 @@ class FrontendOptions { EmitPCM, ///< Emit precompiled Clang module from a module map DumpPCM, ///< Dump information about a precompiled Clang module + + ScanDependencies, ///< Scan dependencies of Swift source files }; /// Indicates the action the user requested that the frontend perform. diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td index 8aa8775751a59..4a8d9883a8541 100644 --- a/include/swift/Option/Options.td +++ b/include/swift/Option/Options.td @@ -1031,6 +1031,10 @@ def sanitize_coverage_EQ : CommaJoined<["-"], "sanitize-coverage=">, HelpText<"Specify the type of coverage instrumentation for Sanitizers and" " additional options separated by commas">; +def scan_dependencies : Flag<["-"], "scan-dependencies">, + HelpText<"Scan dependencies of the given Swift sources">, ModeOpt, + Flags<[FrontendOption, NoInteractiveOption, DoesNotAffectIncrementalBuild]>; + def enable_astscope_lookup : Flag<["-"], "enable-astscope-lookup">, Flags<[FrontendOption]>, HelpText<"Enable ASTScope-based unqualified name lookup">; diff --git a/include/swift/Sema/SourceLoader.h b/include/swift/Sema/SourceLoader.h index cb8f4d19e9bf9..414a802b53cef 100644 --- a/include/swift/Sema/SourceLoader.h +++ b/include/swift/Sema/SourceLoader.h @@ -13,6 +13,7 @@ #ifndef SWIFT_SEMA_SOURCELOADER_H #define SWIFT_SEMA_SOURCELOADER_H +#include "swift/AST/ModuleDependencies.h" #include "swift/AST/ModuleLoader.h" namespace swift { @@ -90,6 +91,12 @@ class SourceLoader : public ModuleLoader { { // Parsing populates the Objective-C method tables. } + + Optional getModuleDependencies( + StringRef moduleName, ModuleDependenciesCache &cache) override { + // FIXME: Implement? + return None; + } }; } diff --git a/include/swift/Serialization/SerializedModuleLoader.h b/include/swift/Serialization/SerializedModuleLoader.h index 11cb775b995a9..3302fba5abbb3 100644 --- a/include/swift/Serialization/SerializedModuleLoader.h +++ b/include/swift/Serialization/SerializedModuleLoader.h @@ -135,6 +135,9 @@ class SerializedModuleLoaderBase : public ModuleLoader { return false; } + /// Scan the given serialized module file to determine dependencies. + llvm::ErrorOr scanModuleFile(Twine modulePath); + public: virtual ~SerializedModuleLoaderBase(); SerializedModuleLoaderBase(const SerializedModuleLoaderBase &) = delete; @@ -189,6 +192,9 @@ class SerializedModuleLoaderBase : public ModuleLoader { llvm::SetVector &results) override; virtual void verifyAllModules() override; + + virtual Optional getModuleDependencies( + StringRef moduleName, ModuleDependenciesCache &cache) override; }; /// Imports serialized Swift modules into an ASTContext. diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp index c08ba61bd559e..5c5b4af8c182e 100644 --- a/lib/AST/ASTContext.cpp +++ b/lib/AST/ASTContext.cpp @@ -32,6 +32,7 @@ #include "swift/AST/IndexSubset.h" #include "swift/AST/KnownProtocols.h" #include "swift/AST/LazyResolver.h" +#include "swift/AST/ModuleDependencies.h" #include "swift/AST/ModuleLoader.h" #include "swift/AST/NameLookup.h" #include "swift/AST/ParameterList.h" @@ -1459,6 +1460,21 @@ void ASTContext::addModuleLoader(std::unique_ptr loader, getImpl().ModuleLoaders.push_back(std::move(loader)); } +Optional ASTContext::getModuleDependencies( + StringRef moduleName, bool isUnderlyingClangModule, + ModuleDependenciesCache &cache) { + for (auto &loader : getImpl().ModuleLoaders) { + if (isUnderlyingClangModule && + loader.get() != getImpl().TheClangModuleLoader) + continue; + + if (auto dependencies = loader->getModuleDependencies(moduleName, cache)) + return dependencies; + } + + return None; +} + void ASTContext::loadExtensions(NominalTypeDecl *nominal, unsigned previousGeneration) { PrettyStackTraceDecl stackTrace("loading extensions for", nominal); diff --git a/lib/AST/CMakeLists.txt b/lib/AST/CMakeLists.txt index 088f638412aa8..5be2d5ae2bfd1 100644 --- a/lib/AST/CMakeLists.txt +++ b/lib/AST/CMakeLists.txt @@ -56,6 +56,7 @@ add_swift_host_library(swiftAST STATIC InlinableText.cpp LayoutConstraint.cpp Module.cpp + ModuleDependencies.cpp ModuleLoader.cpp ModuleNameLookup.cpp NameLookup.cpp diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp new file mode 100644 index 0000000000000..921ed9232e46a --- /dev/null +++ b/lib/AST/ModuleDependencies.cpp @@ -0,0 +1,170 @@ +//===--- ModuleDependencies.h - Module Dependencies -------------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// This file implements data structures for capturing module dependencies. +// +//===----------------------------------------------------------------------===// +#include "swift/AST/ModuleDependencies.h" +#include "swift/AST/Decl.h" +#include "swift/AST/SourceFile.h" +using namespace swift; + +ModuleDependenciesStorageBase::~ModuleDependenciesStorageBase() { } + +bool ModuleDependencies::isSwiftModule() const { + return isa(storage.get()); +} + +/// Retrieve the dependencies for a Swift module. +const SwiftModuleDependenciesStorage * +ModuleDependencies::getAsSwiftModule() const { + return dyn_cast(storage.get()); +} + +/// Retrieve the dependencies for a Clang module. +const ClangModuleDependenciesStorage * +ModuleDependencies::getAsClangModule() const { + return dyn_cast(storage.get()); +} + +void ModuleDependencies::addModuleDependency( + StringRef module, llvm::StringSet<> &alreadyAddedModules) { + if (alreadyAddedModules.insert(module).second) + storage->moduleDependencies.push_back(module); +} + +void ModuleDependencies::addModuleDependencies( + const SourceFile &sf, llvm::StringSet<> &alreadyAddedModules) { + // Add all of the module dependencies. + SmallVector decls; + sf.getTopLevelDecls(decls); + for (auto decl : decls) { + auto importDecl = dyn_cast(decl); + if (!importDecl) + continue; + + addModuleDependency(importDecl->getModulePath().front().Item.str(), + alreadyAddedModules); + } + + auto fileName = sf.getFilename(); + if (fileName.empty()) + return; + + // If the storage is for an interface file, the only source file we + // should see is that interface file. + auto swiftStorage = cast(storage.get()); + if (swiftStorage->swiftInterfaceFile) { + assert(fileName == *swiftStorage->swiftInterfaceFile); + return; + } + + // Otherwise, record the source file. + swiftStorage->sourceFiles.push_back(fileName); +} + +Optional ModuleDependencies::getBridgingHeader() const { + auto swiftStorage = cast(storage.get()); + return swiftStorage->bridgingHeaderFile; +} + +void ModuleDependencies::addBridgingHeader(StringRef bridgingHeader) { + auto swiftStorage = cast(storage.get()); + assert(!swiftStorage->bridgingHeaderFile); + swiftStorage->bridgingHeaderFile = bridgingHeader; +} + +/// Add source files that the bridging header depends on. +void ModuleDependencies::addBridgingSourceFile(StringRef bridgingSourceFile) { + auto swiftStorage = cast(storage.get()); + swiftStorage->bridgingSourceFiles.push_back(bridgingSourceFile); +} + +/// Add (Clang) module on which the bridging header depends. +void ModuleDependencies::addBridgingModuleDependency( + StringRef module, llvm::StringSet<> &alreadyAddedModules) { + auto swiftStorage = cast(storage.get()); + if (alreadyAddedModules.insert(module).second) + swiftStorage->bridgingModuleDependencies.push_back(module); +} + +llvm::StringMap & +ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) { + switch (kind) { + case ModuleDependenciesKind::Swift: + return SwiftModuleDependencies; + + case ModuleDependenciesKind::Clang: + return ClangModuleDependencies; + } +} + +const llvm::StringMap & +ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) const { + switch (kind) { + case ModuleDependenciesKind::Swift: + return SwiftModuleDependencies; + + case ModuleDependenciesKind::Clang: + return ClangModuleDependencies; + } +} + +bool ModuleDependenciesCache::hasDependencies( + StringRef moduleName, + Optional kind) const { + if (!kind) { + return hasDependencies(moduleName, ModuleDependenciesKind::Swift) || + hasDependencies(moduleName, ModuleDependenciesKind::Clang); + } + + const auto &map = getDependenciesMap(*kind); + return map.count(moduleName) > 0; +} + +Optional ModuleDependenciesCache::findDependencies( + StringRef moduleName, + Optional kind) const { + if (!kind) { + if (auto swiftDep = findDependencies( + moduleName, ModuleDependenciesKind::Swift)) + return swiftDep; + + return findDependencies(moduleName, ModuleDependenciesKind::Clang); + } + + const auto &map = getDependenciesMap(*kind); + auto known = map.find(moduleName); + if (known != map.end()) + return known->second; + + return None; +} + +void ModuleDependenciesCache::recordDependencies( + StringRef moduleName, + ModuleDependencies dependencies, + ModuleDependenciesKind kind) { + auto &map = getDependenciesMap(kind); + assert(map.count(moduleName) == 0 && "Already added to map"); + map.insert({moduleName, std::move(dependencies)}); + + AllModules.push_back({moduleName, kind}); +} + +void ModuleDependenciesCache::updateDependencies( + ModuleDependencyID moduleID, ModuleDependencies dependencies) { + auto &map = getDependenciesMap(moduleID.second); + auto known = map.find(moduleID.first); + assert(known != map.end() && "Not yet added to map"); + known->second = std::move(dependencies); +} diff --git a/lib/Basic/FileTypes.cpp b/lib/Basic/FileTypes.cpp index 4539dfc528df9..d2c1570495dd8 100644 --- a/lib/Basic/FileTypes.cpp +++ b/lib/Basic/FileTypes.cpp @@ -82,6 +82,7 @@ bool file_types::isTextual(ID Id) { case file_types::TY_SwiftModuleInterfaceFile: case file_types::TY_PrivateSwiftModuleInterfaceFile: case file_types::TY_SwiftOverlayFile: + case file_types::TY_JSONDependencies: return true; case file_types::TY_Image: case file_types::TY_Object: @@ -151,6 +152,7 @@ bool file_types::isAfterLLVM(ID Id) { case file_types::TY_BitstreamOptRecord: case file_types::TY_SwiftModuleInterfaceFile: case file_types::TY_PrivateSwiftModuleInterfaceFile: + case file_types::TY_JSONDependencies: return false; case file_types::TY_INVALID: llvm_unreachable("Invalid type ID."); @@ -199,6 +201,7 @@ bool file_types::isPartOfSwiftCompilation(ID Id) { case file_types::TY_ModuleTrace: case file_types::TY_YAMLOptRecord: case file_types::TY_BitstreamOptRecord: + case file_types::TY_JSONDependencies: return false; case file_types::TY_INVALID: llvm_unreachable("Invalid type ID."); diff --git a/lib/ClangImporter/CMakeLists.txt b/lib/ClangImporter/CMakeLists.txt index d1fb2a6f8d0a0..97c8bfc6302d3 100644 --- a/lib/ClangImporter/CMakeLists.txt +++ b/lib/ClangImporter/CMakeLists.txt @@ -9,6 +9,7 @@ add_swift_host_library(swiftClangImporter STATIC ClangAdapter.cpp ClangDiagnosticConsumer.cpp ClangImporter.cpp + ClangModuleDependencyScanner.cpp ClangSourceBufferImporter.cpp DWARFImporter.cpp IAMInference.cpp @@ -25,6 +26,9 @@ target_link_libraries(swiftClangImporter PRIVATE swiftParse LLVMBitstreamReader) +target_link_libraries(swiftClangImporter INTERFACE + clangDependencyScanning) + # This property is only set by calls to clang_tablegen. It will not be set on # standalone builds, so it can always be safely passed. get_property(CLANG_TABLEGEN_TARGETS GLOBAL PROPERTY CLANG_TABLEGEN_TARGETS) diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index dd1b55bdc1a38..5b7f5791e2f5b 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -458,10 +458,11 @@ getGlibcModuleMapPath(SearchPathOptions& Opts, llvm::Triple triple, return None; } -static void -getNormalInvocationArguments(std::vector &invocationArgStrs, - ASTContext &ctx, - const ClangImporterOptions &importerOpts) { +void +importer::getNormalInvocationArguments( + std::vector &invocationArgStrs, + ASTContext &ctx, + const ClangImporterOptions &importerOpts) { const auto &LangOpts = ctx.LangOpts; const llvm::Triple &triple = LangOpts.Target; SearchPathOptions &searchPathOpts = ctx.SearchPathOpts; @@ -695,10 +696,11 @@ getEmbedBitcodeInvocationArguments(std::vector &invocationArgStrs, }); } -static void -addCommonInvocationArguments(std::vector &invocationArgStrs, - ASTContext &ctx, - const ClangImporterOptions &importerOpts) { +void +importer::addCommonInvocationArguments( + std::vector &invocationArgStrs, + ASTContext &ctx, + const ClangImporterOptions &importerOpts) { using ImporterImpl = ClangImporter::Implementation; const llvm::Triple &triple = ctx.LangOpts.Target; SearchPathOptions &searchPathOpts = ctx.SearchPathOpts; diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp new file mode 100644 index 0000000000000..097b6d35aaa34 --- /dev/null +++ b/lib/ClangImporter/ClangModuleDependencyScanner.cpp @@ -0,0 +1,341 @@ +//===--- ClangModuleDependencyScanner.cpp - Dependency Scanning -----------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2019 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// This file implements dependency scanning for Clang modules. +// +//===----------------------------------------------------------------------===// +#include "ImporterImpl.h" +#include "swift/AST/ModuleDependencies.h" +#include "swift/ClangImporter/ClangImporter.h" +#include "swift/ClangImporter/ClangImporterOptions.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Signals.h" + +using namespace swift; + +using namespace clang::tooling; +using namespace clang::tooling::dependencies; + +class swift::ClangModuleDependenciesCacheImpl { + /// The name of the file used for the "import hack" to compute module + /// dependencies. + /// FIXME: This should go away once Clang's dependency scanning library + /// can scan by module name. + std::string importHackFile; + +public: + /// Set containing all of the Clang modules that have already been seen. + llvm::StringSet<> alreadySeen; + + DependencyScanningService service; + + DependencyScanningTool tool; + + ClangModuleDependenciesCacheImpl() + : service(ScanningMode::MinimizedSourcePreprocessing, + ScanningOutputFormat::Full), + tool(service) { } + ~ClangModuleDependenciesCacheImpl(); + + /// Retrieve the name of the file used for the "import hack" that is + /// used to scan the dependencies of a Clang module. + llvm::ErrorOr getImportHackFile(); +}; + +ClangModuleDependenciesCacheImpl::~ClangModuleDependenciesCacheImpl() { + if (!importHackFile.empty()) { + llvm::sys::fs::remove(importHackFile); + } +} + +llvm::ErrorOr ClangModuleDependenciesCacheImpl::getImportHackFile() { + if (!importHackFile.empty()) + return importHackFile; + + // Create a temporary file. + int resultFD; + SmallString<128> resultPath; + if (auto error = llvm::sys::fs::createTemporaryFile( + "import-hack", "m", resultFD, resultPath)) + return error; + + llvm::raw_fd_ostream out(resultFD, /*shouldClose=*/true); + out << "@import HACK_MODULE_NAME;\n"; + llvm::sys::RemoveFileOnSignal(resultPath); + importHackFile = resultPath.str().str(); + return importHackFile; +} + +namespace { + class SingleCommandCompilationDatabase : public CompilationDatabase { + public: + SingleCommandCompilationDatabase(CompileCommand Cmd) + : Command(std::move(Cmd)) {} + + virtual std::vector + getCompileCommands(StringRef FilePath) const { + return {Command}; + } + + virtual std::vector getAllCompileCommands() const { + return {Command}; + } + + private: + CompileCommand Command; + }; +} + +// Add search paths. +// Note: This is handled differently for the Clang importer itself, which +// adds search paths to Clang's data structures rather than to its +// command line. +static void addSearchPathInvocationArguments( + std::vector &invocationArgStrs, + ASTContext &ctx, + const ClangImporterOptions &importerOpts) { + SearchPathOptions &searchPathOpts = ctx.SearchPathOpts; + for (const auto &framepath : searchPathOpts.FrameworkSearchPaths) { + invocationArgStrs.push_back(framepath.IsSystem ? "-iframework" : "-F"); + invocationArgStrs.push_back(framepath.Path); + } + + for (auto path : searchPathOpts.ImportSearchPaths) { + invocationArgStrs.push_back("-I"); + invocationArgStrs.push_back(path); + } +} + +/// Create the command line for Clang dependency scanning. +static std::vector getClangDepScanningInvocationArguments( + ASTContext &ctx, + const ClangImporterOptions &importerOpts, + StringRef sourceFileName) { + std::vector commandLineArgs; + + // Form the basic command line. + commandLineArgs.push_back("clang"); + importer::getNormalInvocationArguments(commandLineArgs, ctx, importerOpts); + importer::addCommonInvocationArguments(commandLineArgs, ctx, importerOpts); + addSearchPathInvocationArguments(commandLineArgs, ctx, importerOpts); + + auto sourceFilePos = std::find( + commandLineArgs.begin(), commandLineArgs.end(), + ""); + assert(sourceFilePos != commandLineArgs.end()); + *sourceFilePos = sourceFileName; + + // HACK! Drop the -fmodule-format= argument and the one that + // precedes it. + { + auto moduleFormatPos = std::find_if(commandLineArgs.begin(), + commandLineArgs.end(), + [](StringRef arg) { + return arg.startswith("-fmodule-format="); + }); + assert(moduleFormatPos != commandLineArgs.end()); + assert(moduleFormatPos != commandLineArgs.begin()); + commandLineArgs.erase(moduleFormatPos-1, moduleFormatPos+1); + } + + // HACK: No -fsyntax-only here? + { + auto syntaxOnlyPos = std::find(commandLineArgs.begin(), + commandLineArgs.end(), + "-fsyntax-only"); + assert(syntaxOnlyPos != commandLineArgs.end()); + *syntaxOnlyPos = "-c"; + } + + // HACK: Stolen from ClangScanDeps.cpp + commandLineArgs.push_back("-o"); + commandLineArgs.push_back("/dev/null"); + commandLineArgs.push_back("-M"); + commandLineArgs.push_back("-MT"); + commandLineArgs.push_back("import-hack.o"); + commandLineArgs.push_back("-Xclang"); + commandLineArgs.push_back("-Eonly"); + commandLineArgs.push_back("-Xclang"); + commandLineArgs.push_back("-sys-header-deps"); + commandLineArgs.push_back("-Wno-error"); + + return commandLineArgs; +} + +/// Get or create the Clang-specific +static ClangModuleDependenciesCacheImpl *getOrCreateClangImpl( + ModuleDependenciesCache &cache) { + auto clangImpl = cache.getClangImpl(); + if (!clangImpl) { + clangImpl = new ClangModuleDependenciesCacheImpl(); + cache.setClangImpl(clangImpl, + [](ClangModuleDependenciesCacheImpl *ptr) { + delete ptr; + }); + } + + return clangImpl; +} + +/// Record the module dependencies we found by scanning Clang modules into +/// the module dependencies cache. +static void recordModuleDependencies( + ModuleDependenciesCache &cache, + const FullDependenciesResult &clangDependencies) { + for (const auto &clangModuleDep : clangDependencies.DiscoveredModules) { + // If we've already cached this information, we're done. + if (cache.hasDependencies(clangModuleDep.ModuleName, + ModuleDependenciesKind::Clang)) + continue; + + // File dependencies for this module. + std::vector fileDeps; + for (const auto &fileDep : clangModuleDep.FileDeps) { + fileDeps.push_back(fileDep.getKey()); + } + + // Module-level dependencies. + llvm::StringSet<> alreadyAddedModules; + auto dependencies = ModuleDependencies::forClangModule( + clangModuleDep.ImplicitModulePCMPath, + clangModuleDep.ClangModuleMapFile, + clangModuleDep.ContextHash, + clangModuleDep.NonPathCommandLine, + fileDeps); + for (const auto &moduleName : clangModuleDep.ClangModuleDeps) { + dependencies.addModuleDependency(moduleName.ModuleName, alreadyAddedModules); + } + + cache.recordDependencies(clangModuleDep.ModuleName, + std::move(dependencies), + ModuleDependenciesKind::Clang); + } +} + +Optional ClangImporter::getModuleDependencies( + StringRef moduleName, ModuleDependenciesCache &cache) { + // Check whether there is already a cached result. + if (auto found = cache.findDependencies( + moduleName, ModuleDependenciesKind::Clang)) + return found; + + // Retrieve or create the shared state. + auto clangImpl = getOrCreateClangImpl(cache); + + // HACK! Replace the module import buffer name with the source file hack. + auto importHackFile = clangImpl->getImportHackFile(); + if (!importHackFile) { + // FIXME: Emit a diagnostic here. + return None; + } + + // Reform the Clang importer options. + // FIXME: Just save a reference or copy so we can get this back. + ClangImporterOptions importerOpts; + + // Determine the command-line arguments for dependency scanning. + auto &ctx = Impl.SwiftContext; + std::vector commandLineArgs = + getClangDepScanningInvocationArguments( + ctx, importerOpts, *importHackFile); + + // HACK! Trick out a .m file to use to import the module we name. + std::string moduleNameHackDefine = + ("-DHACK_MODULE_NAME=" + moduleName).str(); + commandLineArgs.push_back(moduleNameHackDefine); + commandLineArgs.push_back("-fmodules-ignore-macro=HACK_MODULE_NAME"); + + std::string workingDir = + ctx.SourceMgr.getFileSystem()->getCurrentWorkingDirectory().get(); + CompileCommand command(workingDir, *importHackFile, commandLineArgs, "-"); + SingleCommandCompilationDatabase database(command); + + auto clangDependencies = clangImpl->tool.getFullDependencies( + database, workingDir, clangImpl->alreadySeen); + + if (!clangDependencies) { + // FIXME: Route this to a normal diagnostic. + llvm::logAllUnhandledErrors(clangDependencies.takeError(), llvm::errs()); + return None; + } + + // Record module dependencies for each module we found. + recordModuleDependencies(cache, *clangDependencies); + + return cache.findDependencies(moduleName, ModuleDependenciesKind::Clang); +} + +bool ClangImporter::addBridgingHeaderDependencies( + StringRef moduleName, + ModuleDependenciesCache &cache) { + auto targetModule = *cache.findDependencies( + moduleName, ModuleDependenciesKind::Swift); + + // If we've already recorded bridging header dependencies, we're done. + auto swiftDeps = targetModule.getAsSwiftModule(); + if (!swiftDeps->bridgingSourceFiles.empty() || + !swiftDeps->bridgingModuleDependencies.empty()) + return false; + + // Retrieve or create the shared state. + auto clangImpl = getOrCreateClangImpl(cache); + + // Reform the Clang importer options. + // FIXME: Just save a reference or copy so we can get this back. + ClangImporterOptions importerOpts; + + // Retrieve the bridging header. + std::string bridgingHeader = *targetModule.getBridgingHeader(); + + // Determine the command-line arguments for dependency scanning. + auto &ctx = Impl.SwiftContext; + std::vector commandLineArgs = + getClangDepScanningInvocationArguments( + ctx, importerOpts, bridgingHeader); + + std::string workingDir = + ctx.SourceMgr.getFileSystem()->getCurrentWorkingDirectory().get(); + CompileCommand command(workingDir, bridgingHeader, commandLineArgs, "-"); + SingleCommandCompilationDatabase database(command); + + auto clangDependencies = clangImpl->tool.getFullDependencies( + database, workingDir, clangImpl->alreadySeen); + + if (!clangDependencies) { + // FIXME: Route this to a normal diagnostic. + llvm::logAllUnhandledErrors(clangDependencies.takeError(), llvm::errs()); + return true; + } + + // Record module dependencies for each module we found. + recordModuleDependencies(cache, *clangDependencies); + + // Record dependencies for the source files the bridging header includes. + for (const auto &fileDep : clangDependencies->FullDeps.FileDeps) + targetModule.addBridgingSourceFile(fileDep); + + // ... and all module dependencies. + llvm::StringSet<> alreadyAddedModules; + for (const auto &moduleDep : clangDependencies->FullDeps.ClangModuleDeps) { + targetModule.addBridgingModuleDependency( + moduleDep.ModuleName, alreadyAddedModules); + } + + // Update the cache with the new information for the module. + cache.updateDependencies( + {moduleName, ModuleDependenciesKind::Swift}, + std::move(targetModule)); + + return false; +} diff --git a/lib/ClangImporter/ImporterImpl.h b/lib/ClangImporter/ImporterImpl.h index 86e1be4d53fa4..b974766d1a0e5 100644 --- a/lib/ClangImporter/ImporterImpl.h +++ b/lib/ClangImporter/ImporterImpl.h @@ -1428,6 +1428,16 @@ bool shouldSuppressDeclImport(const clang::Decl *decl); /// but are now renamed using the swift_name attribute. bool isSpecialUIKitStructZeroProperty(const clang::NamedDecl *decl); +/// Add command-line arguments for a normal import of Clang code. +void getNormalInvocationArguments(std::vector &invocationArgStrs, + ASTContext &ctx, + const ClangImporterOptions &importerOpts); + +/// Add command-line arguments common to all imports of Clang code. +void addCommonInvocationArguments(std::vector &invocationArgStrs, + ASTContext &ctx, + const ClangImporterOptions &importerOpts); + /// Finds a particular kind of nominal by looking through typealiases. template static T *dynCastIgnoringCompatibilityAlias(Decl *D) { diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp index 00eaeb27bd7ca..e77f03e9fbb2e 100644 --- a/lib/Driver/Driver.cpp +++ b/lib/Driver/Driver.cpp @@ -1537,6 +1537,14 @@ void Driver::buildOutputInfo(const ToolChain &TC, const DerivedArgList &Args, OI.CompilerMode = OutputInfo::Mode::SingleCompile; break; + case options::OPT_scan_dependencies: + OI.CompilerOutputType = file_types::TY_JSONDependencies; + // We want the imported modules from the module as a whole, not individual + // files, so let's do it in one invocation rather than having to collate + // later. + OI.CompilerMode = OutputInfo::Mode::SingleCompile; + break; + case options::OPT_index_file: OI.CompilerMode = OutputInfo::Mode::SingleCompile; OI.CompilerOutputType = file_types::TY_IndexData; @@ -1989,6 +1997,7 @@ void Driver::buildActions(SmallVectorImpl &TopLevelActions, case file_types::TY_PrivateSwiftModuleInterfaceFile: case file_types::TY_SwiftCrossImportDir: case file_types::TY_SwiftOverlayFile: + case file_types::TY_JSONDependencies: // We could in theory handle assembly or LLVM input, but let's not. // FIXME: What about LTO? Diags.diagnose(SourceLoc(), diag::error_unexpected_input_file, diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp index 67b92bd60dc0d..54156c76ee306 100644 --- a/lib/Driver/ToolChains.cpp +++ b/lib/Driver/ToolChains.cpp @@ -563,6 +563,8 @@ const char *ToolChain::JobContext::computeFrontendModeForCompile() const { return "-emit-module"; case file_types::TY_ImportedModules: return "-emit-imported-modules"; + case file_types::TY_JSONDependencies: + return "-scan-dependencies"; case file_types::TY_IndexData: return "-typecheck"; case file_types::TY_Remapping: @@ -829,6 +831,7 @@ ToolChain::constructInvocation(const BackendJobAction &job, case file_types::TY_PCH: case file_types::TY_ClangModuleFile: case file_types::TY_IndexData: + case file_types::TY_JSONDependencies: llvm_unreachable("Cannot be output from backend job"); case file_types::TY_Swift: case file_types::TY_dSYM: diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index e45159868a946..85c44e7b837a1 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -83,6 +83,12 @@ bool ArgsToFrontendOptionsConverter::convert( Opts.TrackSystemDeps |= Args.hasArg(OPT_track_system_dependencies); + // Always track system dependencies when scanning dependencies. + if (const Arg *ModeArg = Args.getLastArg(OPT_modes_Group)) { + if (ModeArg->getOption().matches(OPT_scan_dependencies)) + Opts.TrackSystemDeps = true; + } + Opts.SerializeModuleInterfaceDependencyHashes |= Args.hasArg(OPT_serialize_module_interface_dependency_hashes); @@ -345,6 +351,8 @@ ArgsToFrontendOptionsConverter::determineRequestedAction(const ArgList &args) { return FrontendOptions::ActionType::EmitPCH; if (Opt.matches(OPT_emit_imported_modules)) return FrontendOptions::ActionType::EmitImportedModules; + if (Opt.matches(OPT_scan_dependencies)) + return FrontendOptions::ActionType::ScanDependencies; if (Opt.matches(OPT_parse)) return FrontendOptions::ActionType::Parse; if (Opt.matches(OPT_resolve_imports)) diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index 14d1f82a7097f..b1f292ec5a02d 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -687,11 +687,10 @@ std::unique_ptr CompilerInstance::takeSILModule() { /// builds. This allows for use of popular specialized functions /// from the standard library, which makes the non-optimized builds /// execute much faster. -static bool shouldImplicityImportSwiftOnoneSupportModule( - const CompilerInvocation &Invocation) { - if (Invocation.getImplicitStdlibKind() != ImplicitStdlibKind::Stdlib) +bool CompilerInvocation::shouldImportSwiftONoneSupport() const { + if (getImplicitStdlibKind() != ImplicitStdlibKind::Stdlib) return false; - if (Invocation.getSILOptions().shouldOptimize()) + if (getSILOptions().shouldOptimize()) return false; // If we are not executing an action that has a dependency on @@ -706,7 +705,7 @@ static bool shouldImplicityImportSwiftOnoneSupportModule( // // This optimization is disabled by -track-system-dependencies to preserve // the explicit dependency. - const auto &options = Invocation.getFrontendOptions(); + const auto &options = getFrontendOptions(); return options.TrackSystemDeps || FrontendOptions::doesActionGenerateSIL(options.RequestedAction); } @@ -720,7 +719,7 @@ ImplicitImportInfo CompilerInstance::getImplicitImportInfo() const { for (auto &moduleStr : frontendOpts.getImplicitImportModuleNames()) imports.ModuleNames.push_back(Context->getIdentifier(moduleStr)); - if (shouldImplicityImportSwiftOnoneSupportModule(Invocation)) + if (Invocation.shouldImportSwiftONoneSupport()) imports.ModuleNames.push_back(Context->getIdentifier(SWIFT_ONONE_SUPPORT)); imports.ShouldImportUnderlyingModule = frontendOpts.ImportUnderlyingModule; diff --git a/lib/Frontend/FrontendOptions.cpp b/lib/Frontend/FrontendOptions.cpp index 5a447247ad332..ea3632401c8e5 100644 --- a/lib/Frontend/FrontendOptions.cpp +++ b/lib/Frontend/FrontendOptions.cpp @@ -61,6 +61,7 @@ bool FrontendOptions::needsProperModuleName(ActionType action) { case ActionType::EmitImportedModules: case ActionType::DumpTypeInfo: case ActionType::EmitPCM: + case ActionType::ScanDependencies: return true; } llvm_unreachable("Unknown ActionType"); @@ -99,6 +100,7 @@ bool FrontendOptions::isActionImmediate(ActionType action) { case ActionType::DumpTypeInfo: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; } llvm_unreachable("Unknown ActionType"); @@ -111,6 +113,7 @@ bool FrontendOptions::shouldActionOnlyParse(ActionType action) { case FrontendOptions::ActionType::EmitSyntax: case FrontendOptions::ActionType::DumpInterfaceHash: case FrontendOptions::ActionType::EmitImportedModules: + case FrontendOptions::ActionType::ScanDependencies: return true; default: return false; @@ -204,6 +207,9 @@ FrontendOptions::formatForPrincipalOutputFileForAction(ActionType action) { case ActionType::EmitPCM: return TY_ClangModuleFile; + + case ActionType::ScanDependencies: + return TY_JSONDependencies; } llvm_unreachable("unhandled action"); } @@ -240,6 +246,7 @@ bool FrontendOptions::canActionEmitDependencies(ActionType action) { case ActionType::EmitObject: case ActionType::EmitImportedModules: case ActionType::EmitPCM: + case ActionType::ScanDependencies: return true; } llvm_unreachable("unhandled action"); @@ -263,6 +270,7 @@ bool FrontendOptions::canActionEmitReferenceDependencies(ActionType action) { case ActionType::REPL: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::Typecheck: case ActionType::MergeModules: @@ -309,6 +317,7 @@ bool FrontendOptions::canActionEmitObjCHeader(ActionType action) { case ActionType::REPL: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::Typecheck: case ActionType::MergeModules: @@ -344,6 +353,7 @@ bool FrontendOptions::canActionEmitLoadedModuleTrace(ActionType action) { case ActionType::REPL: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::ResolveImports: case ActionType::Typecheck: @@ -385,6 +395,7 @@ bool FrontendOptions::canActionEmitModule(ActionType action) { case ActionType::REPL: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::MergeModules: case ActionType::EmitModuleOnly: @@ -427,6 +438,7 @@ bool FrontendOptions::canActionEmitInterface(ActionType action) { case ActionType::REPL: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::Typecheck: case ActionType::MergeModules: @@ -470,6 +482,7 @@ bool FrontendOptions::doesActionProduceOutput(ActionType action) { case ActionType::DumpTypeInfo: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return true; case ActionType::NoneAction: @@ -513,6 +526,7 @@ bool FrontendOptions::doesActionProduceTextualOutput(ActionType action) { case ActionType::EmitIR: case ActionType::DumpTypeInfo: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return true; } llvm_unreachable("unhandled action"); @@ -536,6 +550,7 @@ bool FrontendOptions::doesActionGenerateSIL(ActionType action) { case ActionType::CompileModuleFromInterface: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::EmitSILGen: case ActionType::EmitSIBGen: @@ -580,6 +595,7 @@ bool FrontendOptions::doesActionGenerateIR(ActionType action) { case ActionType::EmitImportedModules: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::Immediate: case ActionType::REPL: diff --git a/lib/FrontendTool/CMakeLists.txt b/lib/FrontendTool/CMakeLists.txt index 948d0b5ef8b3d..2d06360885e6f 100644 --- a/lib/FrontendTool/CMakeLists.txt +++ b/lib/FrontendTool/CMakeLists.txt @@ -2,6 +2,7 @@ add_swift_host_library(swiftFrontendTool STATIC FrontendTool.cpp ImportedModules.cpp ReferenceDependencies.cpp + ScanDependencies.cpp TBD.cpp) add_dependencies(swiftFrontendTool swift-syntax-generated-headers diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index e78174844befa..b80a03e8ae5a8 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -22,6 +22,7 @@ #include "swift/FrontendTool/FrontendTool.h" #include "ImportedModules.h" +#include "ScanDependencies.h" #include "ReferenceDependencies.h" #include "TBD.h" @@ -1250,8 +1251,10 @@ static bool performCompile(CompilerInstance &Instance, // Disable delayed parsing of type and function bodies when we've been // asked to dump the resulting AST. bool CanDelayBodies = Action != FrontendOptions::ActionType::DumpParse; - Instance.performParseOnly(/*EvaluateConditionals*/ - Action == FrontendOptions::ActionType::EmitImportedModules, + bool EvaluateConditionals = + Action == FrontendOptions::ActionType::EmitImportedModules + || Action == FrontendOptions::ActionType::ScanDependencies; + Instance.performParseOnly(EvaluateConditionals, CanDelayBodies); } else if (Action == FrontendOptions::ActionType::ResolveImports) { Instance.performParseAndResolveImportsOnly(); @@ -1263,10 +1266,15 @@ static bool performCompile(CompilerInstance &Instance, if (Action == FrontendOptions::ActionType::Parse) return Context.hadError(); + if (Action == FrontendOptions::ActionType::ScanDependencies) { + scanDependencies(Instance); + } + (void)emitMakeDependenciesIfNeeded(Context.Diags, Instance.getDependencyTracker(), opts); - if (Action == FrontendOptions::ActionType::ResolveImports) + if (Action == FrontendOptions::ActionType::ResolveImports || + Action == FrontendOptions::ActionType::ScanDependencies) return Context.hadError(); if (observer) diff --git a/lib/FrontendTool/ScanDependencies.cpp b/lib/FrontendTool/ScanDependencies.cpp new file mode 100644 index 0000000000000..a5a266b0a5e30 --- /dev/null +++ b/lib/FrontendTool/ScanDependencies.cpp @@ -0,0 +1,463 @@ +//===--- ScanDependencies.cpp -- Scans the dependencies of a module -------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +#include "ScanDependencies.h" +#include "swift/AST/ASTContext.h" +#include "swift/AST/Decl.h" +#include "swift/AST/DiagnosticEngine.h" +#include "swift/AST/DiagnosticsFrontend.h" +#include "swift/AST/Module.h" +#include "swift/AST/ModuleDependencies.h" +#include "swift/AST/ModuleLoader.h" +#include "swift/AST/SourceFile.h" +#include "swift/ClangImporter/ClangImporter.h" +#include "swift/Basic/Defer.h" +#include "swift/Basic/LLVM.h" +#include "swift/Basic/STLExtras.h" +#include "swift/Frontend/Frontend.h" +#include "swift/Frontend/FrontendOptions.h" +#include "swift/Strings.h" +#include "clang/Basic/Module.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/FileSystem.h" +#include + +using namespace swift; + +namespace { + +} + +/// Find all of the imported Clang modules starting with the given module name. +static void findAllImportedClangModules(ASTContext &ctx, StringRef moduleName, + ModuleDependenciesCache &cache, + std::vector &allModules, + llvm::StringSet<> &knownModules) { + if (!knownModules.insert(moduleName).second) + return; + allModules.push_back(moduleName); + + auto dependencies = cache.findDependencies( + moduleName, ModuleDependenciesKind::Clang); + if (!dependencies) + return; + + for (const auto &dep : dependencies->getModuleDependencies()) { + findAllImportedClangModules(ctx, dep, cache, allModules, knownModules); + } +} + +/// Resolve the direct dependencies of the given module. +static std::vector resolveDirectDependencies( + ASTContext &ctx, ModuleDependencyID module, + ModuleDependenciesCache &cache) { + auto knownDependencies = *cache.findDependencies(module.first, module.second); + auto isSwift = knownDependencies.isSwiftModule(); + + // Find the dependencies of every module this module directly depends on. + std::vector result; + for (auto dependsOn : knownDependencies.getModuleDependencies()) { + // Figure out what kind of module we need. + bool onlyClangModule = !isSwift || module.first == dependsOn; + + // Retrieve the dependencies for this module. + if (auto found = ctx.getModuleDependencies( + dependsOn, onlyClangModule, cache)) { + result.push_back({dependsOn, found->getKind()}); + } + } + + if (isSwift) { + // A record of all of the Clang modules referenced from this Swift module. + std::vector allClangModules; + llvm::StringSet<> knownModules; + + // If the Swift module has a bridging header, add those dependencies. + if (knownDependencies.getBridgingHeader()) { + auto clangImporter = + static_cast(ctx.getClangModuleLoader()); + if (!clangImporter->addBridgingHeaderDependencies(module.first, cache)) { + // Grab the updated module dependencies. + // FIXME: This is such a hack. + knownDependencies = *cache.findDependencies(module.first, module.second); + + // Add the Clang modules referenced from the bridging header to the + // set of Clang modules we know about. + auto swiftDeps = knownDependencies.getAsSwiftModule(); + for (const auto &clangDep : swiftDeps->bridgingModuleDependencies) { + findAllImportedClangModules(ctx, clangDep, cache, allClangModules, + knownModules); + } + } + } + + // Find all of the Clang modules this Swift module depends on. + for (const auto &dep : result) { + if (dep.second != ModuleDependenciesKind::Clang) + continue; + + findAllImportedClangModules(ctx, dep.first, cache, allClangModules, + knownModules); + } + + // Look for overlays for each of the Clang modules. The Swift module + // directly depends on these. + for (const auto &clangDep : allClangModules) { + if (auto found = ctx.getModuleDependencies( + clangDep, /*onlyClangModule=*/false, cache)) { + if (found->getKind() == ModuleDependenciesKind::Swift) + result.push_back({clangDep, found->getKind()}); + } + } + } + + return result; +} + +/// Write a single JSON field. +namespace { + template + void writeJSONSingleField(llvm::raw_ostream &out, + StringRef fieldName, + const T &value, + unsigned indentLevel, + bool trailingComma); + + /// Write a string value as JSON. + void writeJSONValue(llvm::raw_ostream &out, + StringRef value, + unsigned indentLevel) { + out << "\""; + out.write_escaped(value); + out << "\""; + } + + /// Write a module identifier. + void writeJSONValue(llvm::raw_ostream &out, + const ModuleDependencyID &module, + unsigned indentLevel) { + out << "{\n"; + + writeJSONSingleField( + out, + module.second == ModuleDependenciesKind::Swift ? "swift" : "clang", + module.first, + indentLevel + 1, + /*trailingComma=*/false); + + out.indent(indentLevel * 2); + out << "}"; + } + + /// Write a JSON array. + template + void writeJSONValue(llvm::raw_ostream &out, + ArrayRef values, + unsigned indentLevel) { + out << "[\n"; + + for (const auto &value: values) { + + out.indent((indentLevel + 1) * 2); + + writeJSONValue(out, value, indentLevel + 1); + + if (&value != &values.back()) { + out << ","; + } + out << "\n"; + } + + out.indent(indentLevel * 2); + out << "]"; + } + + /// Write a JSON array. + template + void writeJSONValue(llvm::raw_ostream &out, + const std::vector &values, + unsigned indentLevel) { + writeJSONValue(out, llvm::makeArrayRef(values), indentLevel); + } + + /// Write a single JSON field. + template + void writeJSONSingleField(llvm::raw_ostream &out, + StringRef fieldName, + const T &value, + unsigned indentLevel, + bool trailingComma) { + out.indent(indentLevel * 2); + writeJSONValue(out, fieldName, indentLevel); + out << ": "; + writeJSONValue(out, value, indentLevel); + if (trailingComma) + out << ","; + out << "\n"; + } +} + +static void writeJSON(llvm::raw_ostream &out, + ASTContext &ctx, + ModuleDependenciesCache &cache, + ArrayRef allModules) { + // Write out a JSON description of all of the dependencies. + out << "{\n"; + SWIFT_DEFER { + out << "}\n"; + }; + + // Name of the main module. + writeJSONSingleField(out, "mainModuleName", allModules.front().first, + /*indentLevel=*/1, /*trailingComma=*/true); + + // Write out all of the modules. + out << " \"modules\": [\n"; + SWIFT_DEFER { + out << " ]\n"; + }; + for (const auto &module : allModules) { + auto directDependencies = resolveDirectDependencies( + ctx, ModuleDependencyID(module.first, module.second), cache); + + // Grab the completed module dependencies. + auto moduleDeps = *cache.findDependencies(module.first, module.second); + + // The module we are describing. + out.indent(2 * 2); + writeJSONValue(out, module, 2); + out << ",\n"; + + out.indent(2 * 2); + out << "{\n"; + + // Module path. + const char *modulePathSuffix = + moduleDeps.isSwiftModule() ? ".swiftmodule" : ".pcm"; + std::string modulePath = module.first + modulePathSuffix; + writeJSONSingleField(out, "modulePath", modulePath, /*indentLevel=*/3, + /*trailingComma=*/true); + + // Source files. + auto swiftDeps = moduleDeps.getAsSwiftModule(); + auto clangDeps = moduleDeps.getAsClangModule(); + if (swiftDeps) { + writeJSONSingleField(out, "sourceFiles", swiftDeps->sourceFiles, 3, + /*trailingComma=*/true); + } else { + writeJSONSingleField(out, "sourceFiles", clangDeps->fileDependencies, 3, + /*trailingComma=*/true); + } + + // Direct dependencies. + writeJSONSingleField(out, "directDependencies", directDependencies, + 3, /*trailingComma=*/true); + + // Swift and Clang-specific details. + out.indent(3 * 2); + out << "\"details\": {\n"; + out.indent(4 * 2); + if (swiftDeps) { + out << "\"swift\": {\n"; + + /// Swift interface file, if any. + if (swiftDeps->swiftInterfaceFile) { + writeJSONSingleField( + out, "moduleInterfacePath", + *swiftDeps->swiftInterfaceFile, 5, + /*trailingComma=*/swiftDeps->bridgingHeaderFile.hasValue()); + } + + /// Bridging header and its source file dependencies, if any. + if (swiftDeps->bridgingHeaderFile) { + out.indent(5 * 2); + out << "\"bridgingHeader\": {\n"; + writeJSONSingleField(out, "path", + *swiftDeps->bridgingHeaderFile, 6, + /*trailingComma=*/true); + writeJSONSingleField(out, "sourceFiles", + swiftDeps->bridgingSourceFiles, 6, + /*trailingComma=*/true); + writeJSONSingleField(out, "moduleDependencies", + swiftDeps->bridgingModuleDependencies, 6, + /*trailingComma=*/false); + out.indent(5 * 2); + out << "}\n"; + } + } else { + out << "\"clang\": {\n"; + + // Module map file. + writeJSONSingleField(out, "moduleMapPath", + clangDeps->moduleMapFile, 5, + /*trailingComma=*/true); + + // Context hash. + writeJSONSingleField(out, "contextHash", + clangDeps->contextHash, 5, + /*trailingComma=*/true); + + // Command line. + writeJSONSingleField(out, "commandLine", + clangDeps->nonPathCommandLine, 5, + /*trailingComma=*/false); + } + + out.indent(4 * 2); + out << "}\n"; + out.indent(3 * 2); + out << "}\n"; + + out.indent(2 * 2); + out << "}"; + + if (&module != &allModules.back()) + out << ","; + out << "\n"; + } +} + +bool swift::scanDependencies(CompilerInstance &instance) { + ASTContext &Context = instance.getASTContext(); + ModuleDecl *mainModule = instance.getMainModule(); + const CompilerInvocation &invocation = instance.getInvocation(); + const FrontendOptions &opts = invocation.getFrontendOptions(); + + std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); + std::error_code EC; + llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::F_None); + + if (out.has_error() || EC) { + Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, + EC.message()); + out.clear_error(); + return true; + } + + // Main module file name. + auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile); + llvm::SmallString<32> mainModulePath = mainModule->getName().str(); + llvm::sys::path::replace_extension(mainModulePath, newExt); + + // Compute the dependencies of the main module. + auto mainDependencies = + ModuleDependencies::forSwiftModule(mainModulePath.str()); + { + llvm::StringSet<> alreadyAddedModules; + for (auto fileUnit : mainModule->getFiles()) { + auto sf = dyn_cast(fileUnit); + if (!sf) + continue; + + mainDependencies.addModuleDependencies(*sf, alreadyAddedModules); + } + + const auto &importInfo = mainModule->getImplicitImportInfo(); + + // Swift standard library. + switch (importInfo.StdlibKind) { + case ImplicitStdlibKind::None: + case ImplicitStdlibKind::Builtin: + break; + + case ImplicitStdlibKind::Stdlib: + mainDependencies.addModuleDependency("Swift", alreadyAddedModules); + break; + } + + // Swift -Onone support library. + if (invocation.shouldImportSwiftONoneSupport()) { + mainDependencies.addModuleDependency( + SWIFT_ONONE_SUPPORT, alreadyAddedModules); + } + + // Add any implicit module names. + for (const auto &moduleName : importInfo.ModuleNames) { + mainDependencies.addModuleDependency(moduleName.str(), alreadyAddedModules); + } + + // Already-loaded, implicitly imported module names. + for (const auto &module : importInfo.AdditionalModules) { + mainDependencies.addModuleDependency(module.first->getNameStr(), alreadyAddedModules); + } + + // Add the bridging header. + if (!importInfo.BridgingHeaderPath.empty()) { + mainDependencies.addBridgingHeader(importInfo.BridgingHeaderPath); + } + + // If we are to import the underlying Clang module of the same name, + // add a dependency with the same name to trigger the search. + if (importInfo.ShouldImportUnderlyingModule) { + mainDependencies.addModuleDependency(mainModule->getName().str(), + alreadyAddedModules); + } + } + + // Add the main module. + StringRef mainModuleName = mainModule->getNameStr(); + llvm::SetVector, + std::set> allModules; + + allModules.insert({mainModuleName, mainDependencies.getKind()}); + + // Create the module dependency cache. + ModuleDependenciesCache cache; + cache.recordDependencies(mainModuleName, std::move(mainDependencies), + ModuleDependenciesKind::Swift); + + // Explore the dependencies of every module. + for (unsigned currentModuleIdx = 0; + currentModuleIdx < allModules.size(); + ++currentModuleIdx) { + auto module = allModules[currentModuleIdx]; + auto discoveredModules = + resolveDirectDependencies(Context, module, cache); + allModules.insert(discoveredModules.begin(), discoveredModules.end()); + } + + // Write out the JSON description. + writeJSON(out, Context, cache, allModules.getArrayRef()); + + // Update the dependency tracker. + if (auto depTracker = instance.getDependencyTracker()) { + for (auto module : allModules) { + auto deps = cache.findDependencies(module.first, module.second); + if (!deps) + continue; + + if (auto swiftDeps = deps->getAsSwiftModule()) { + if (auto swiftInterfaceFile = swiftDeps->swiftInterfaceFile) + depTracker->addDependency(*swiftInterfaceFile, /*IsSystem=*/false); + for (const auto &sourceFile : swiftDeps->sourceFiles) + depTracker->addDependency(sourceFile, /*IsSystem=*/false); + for (const auto &bridgingSourceFile : swiftDeps->bridgingSourceFiles) + depTracker->addDependency(bridgingSourceFile, /*IsSystem=*/false); + } else { + auto clangDeps = deps->getAsClangModule(); + if (!clangDeps->moduleMapFile.empty()) + depTracker->addDependency(clangDeps->moduleMapFile, /*IsSystem=*/false); + for (const auto &sourceFile : clangDeps->fileDependencies) + depTracker->addDependency(sourceFile, /*IsSystem=*/false); + } + } + } + + // This process succeeds regardless of whether any errors occurred. + // FIXME: We shouldn't need this, but it's masking bugs in our scanning + // logic where we don't create a fresh context when scanning Swift interfaces + // that includes their own command-line flags. + Context.Diags.resetHadAnyError(); + return false; +} diff --git a/lib/FrontendTool/ScanDependencies.h b/lib/FrontendTool/ScanDependencies.h new file mode 100644 index 0000000000000..ece8515097608 --- /dev/null +++ b/lib/FrontendTool/ScanDependencies.h @@ -0,0 +1,25 @@ +//===--- ScanDependencies.h -- Scans the dependencies of a module ------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H +#define SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H + +namespace swift { + +class CompilerInstance; + +/// Scans the dependencies of the main module of \c instance. +bool scanDependencies(CompilerInstance &instance); + +} // end namespace swift + +#endif diff --git a/lib/Serialization/CMakeLists.txt b/lib/Serialization/CMakeLists.txt index ebeb4ab7d3003..90e73ee5ed30c 100644 --- a/lib/Serialization/CMakeLists.txt +++ b/lib/Serialization/CMakeLists.txt @@ -1,6 +1,7 @@ add_swift_host_library(swiftSerialization STATIC Deserialization.cpp DeserializeSIL.cpp + ModuleDependencyScanner.cpp ModuleFile.cpp Serialization.cpp SerializedModuleLoader.cpp diff --git a/lib/Serialization/ModuleDependencyScanner.cpp b/lib/Serialization/ModuleDependencyScanner.cpp new file mode 100644 index 0000000000000..19cd6c3a701ed --- /dev/null +++ b/lib/Serialization/ModuleDependencyScanner.cpp @@ -0,0 +1,148 @@ +//===--- ModuleDependencyScanner.cpp - Compute module dependencies --------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "swift/Serialization/SerializedModuleLoader.h" +#include "swift/AST/ASTContext.h" +#include "swift/AST/DiagnosticSuppression.h" +#include "swift/AST/ModuleDependencies.h" +#include "swift/AST/SourceFile.h" +#include "swift/Basic/FileTypes.h" +#include "swift/Subsystems.h" +using namespace swift; +using llvm::ErrorOr; + +namespace { + +/// A module "loader" that looks for .swiftinterface and .swiftmodule files +/// for the purpose of determining dependencies, but does not attempt to +/// load the module files. +class ModuleDependencyScanner : public SerializedModuleLoaderBase { + /// The module we're scanning dependencies of. + Identifier moduleName; + + /// Scan the given interface file to determine dependencies. + ErrorOr scanInterfaceFile( + Twine moduleInterfacePath); + +public: + Optional dependencies; + + ModuleDependencyScanner(ASTContext &ctx, ModuleLoadingMode LoadMode, + Identifier moduleName) + : SerializedModuleLoaderBase(ctx, nullptr, LoadMode, + /*IgnoreSwiftSourceInfoFile=*/true), + moduleName(moduleName) { } + + virtual std::error_code findModuleFilesInDirectory( + AccessPathElem ModuleID, + const SerializedModuleBaseName &BaseName, + SmallVectorImpl *ModuleInterfacePath, + std::unique_ptr *ModuleBuffer, + std::unique_ptr *ModuleDocBuffer, + std::unique_ptr *ModuleSourceInfoBuffer) override { + using namespace llvm::sys; + + auto &fs = *Ctx.SourceMgr.getFileSystem(); + + // Compute the full path of the module we're looking for. + auto ModPath = BaseName.getName(file_types::TY_SwiftModuleFile); + if (LoadMode == ModuleLoadingMode::OnlySerialized) { + // If there is no module file, there's nothing we can do. + if (!fs.exists(ModPath)) + return std::make_error_code(std::errc::no_such_file_or_directory); + + // The module file will be loaded directly. + auto dependencies = scanModuleFile(ModPath); + if (dependencies) { + this->dependencies = std::move(dependencies.get()); + return std::error_code(); + } + + return dependencies.getError(); + } + + // Check whether the .swiftinterface exists. + auto InPath = BaseName.getName(file_types::TY_SwiftModuleInterfaceFile); + + if (!fs.exists(InPath)) + return std::make_error_code(std::errc::no_such_file_or_directory); + + auto dependencies = scanInterfaceFile(InPath); + if (dependencies) { + this->dependencies = std::move(dependencies.get()); + return std::error_code(); + } + + return dependencies.getError(); + } + + virtual void collectVisibleTopLevelModuleNames( + SmallVectorImpl &names) const override { + llvm_unreachable("Not used"); + } +}; +} + +ErrorOr ModuleDependencyScanner::scanInterfaceFile( + Twine moduleInterfacePath) { + // Open the interface file. + auto &fs = *Ctx.SourceMgr.getFileSystem(); + auto interfaceBuf = fs.getBufferForFile(moduleInterfacePath); + if (!interfaceBuf) + return interfaceBuf.getError(); + + // Create a source file. + unsigned bufferID = Ctx.SourceMgr.addNewSourceBuffer(std::move(interfaceBuf.get())); + auto moduleDecl = ModuleDecl::create(moduleName, Ctx); + auto sourceFile = new (Ctx) SourceFile( + *moduleDecl, SourceFileKind::Interface, bufferID); + + // Create a module filename. + // FIXME: Query the module interface loader to determine an appropriate + // name for the module, which includes an appropriate hash. + auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile); + llvm::SmallString<32> modulePath = moduleName.str(); + llvm::sys::path::replace_extension(modulePath, newExt); + + // Walk the source file to find the import declarations. + llvm::StringSet<> alreadyAddedModules; + auto dependencies = ModuleDependencies::forSwiftInterface( + modulePath.str(), moduleInterfacePath.str()); + + // FIXME: Suppressing diagnostics here is incorrect, and is currently + // used to paper over the fact that we should be creating a fresh + // ASTContext using the command-line arguments from the .swiftinterface + // file. + DiagnosticSuppression suppression(Ctx.Diags); + + dependencies.addModuleDependencies(*sourceFile, alreadyAddedModules); + return dependencies; +} + +Optional SerializedModuleLoaderBase::getModuleDependencies( + StringRef moduleName, ModuleDependenciesCache &cache) { + // Check whether we've cached this result. + if (auto found = cache.findDependencies( + moduleName, ModuleDependenciesKind::Swift)) + return found; + + // Check whether there is a module with this name that we can import. + auto moduleId = Ctx.getIdentifier(moduleName); + ModuleDependencyScanner scanner(Ctx, LoadMode, moduleId); + if (!scanner.canImportModule({moduleId, SourceLoc()})) + return None; + + // Record the dependencies. + cache.recordDependencies(moduleName, *scanner.dependencies, + ModuleDependenciesKind::Swift); + return std::move(scanner.dependencies); +} diff --git a/lib/Serialization/SerializedModuleLoader.cpp b/lib/Serialization/SerializedModuleLoader.cpp index 3dbd972b6cec3..170237a7037d3 100644 --- a/lib/Serialization/SerializedModuleLoader.cpp +++ b/lib/Serialization/SerializedModuleLoader.cpp @@ -14,6 +14,7 @@ #include "ModuleFile.h" #include "swift/AST/ASTContext.h" #include "swift/AST/DiagnosticsSema.h" +#include "swift/AST/ModuleDependencies.h" #include "swift/Basic/Defer.h" #include "swift/Basic/FileTypes.h" #include "swift/Basic/Platform.h" @@ -346,6 +347,46 @@ std::error_code SerializedModuleLoaderBase::openModuleFile( return std::error_code(); } +llvm::ErrorOr SerializedModuleLoaderBase::scanModuleFile( + Twine modulePath) { + // Open the module file + auto &fs = *Ctx.SourceMgr.getFileSystem(); + auto moduleBuf = fs.getBufferForFile(modulePath); + if (!moduleBuf) + return moduleBuf.getError(); + + // Load the module file without validation. + std::unique_ptr loadedModuleFile; + bool isFramework = false; + serialization::ValidationInfo loadInfo = + ModuleFile::load(modulePath.str(), + std::move(moduleBuf.get()), + nullptr, + nullptr, + isFramework, loadedModuleFile, + nullptr); + + // Map the set of dependencies over to the "module dependencies". + auto dependencies = ModuleDependencies::forSwiftModule(modulePath.str()); + llvm::StringSet<> addedModuleNames; + for (const auto &dependency : loadedModuleFile->getDependencies()) { + // FIXME: Record header dependency? + if (dependency.isHeader()) + continue; + + // Find the top-level module name. + auto modulePathStr = dependency.getPrettyPrintedPath(); + StringRef moduleName = modulePathStr; + auto dotPos = moduleName.find('.'); + if (dotPos != std::string::npos) + moduleName = moduleName.slice(0, dotPos); + + dependencies.addModuleDependency(moduleName, addedModuleNames); + } + + return std::move(dependencies); +} + std::error_code SerializedModuleLoader::findModuleFilesInDirectory( AccessPathElem ModuleID, const SerializedModuleBaseName &BaseName, diff --git a/test/ScanDependencies/Inputs/CHeaders/A.h b/test/ScanDependencies/Inputs/CHeaders/A.h new file mode 100644 index 0000000000000..ea06ce27adf7f --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/A.h @@ -0,0 +1 @@ +void funcA(void); diff --git a/test/ScanDependencies/Inputs/CHeaders/B.h b/test/ScanDependencies/Inputs/CHeaders/B.h new file mode 100644 index 0000000000000..dedb1b5524f7f --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/B.h @@ -0,0 +1,4 @@ +#include + +void funcB(void); + diff --git a/test/ScanDependencies/Inputs/CHeaders/Bridging.h b/test/ScanDependencies/Inputs/CHeaders/Bridging.h new file mode 100644 index 0000000000000..c0261c5d9a39b --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/Bridging.h @@ -0,0 +1,3 @@ +#include "BridgingOther.h" + +int bridging_other(void); diff --git a/test/ScanDependencies/Inputs/CHeaders/BridgingOther.h b/test/ScanDependencies/Inputs/CHeaders/BridgingOther.h new file mode 100644 index 0000000000000..ee79a8f5bc79c --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/BridgingOther.h @@ -0,0 +1,3 @@ +#include "F.h" + +int bridging_other(void); diff --git a/test/ScanDependencies/Inputs/CHeaders/C.h b/test/ScanDependencies/Inputs/CHeaders/C.h new file mode 100644 index 0000000000000..8a5a9b119ec9d --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/C.h @@ -0,0 +1,3 @@ +#include + +void funcC(void); diff --git a/test/ScanDependencies/Inputs/CHeaders/D.h b/test/ScanDependencies/Inputs/CHeaders/D.h new file mode 100644 index 0000000000000..2b6ccbb761b9e --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/D.h @@ -0,0 +1 @@ +void funcD(void); diff --git a/test/ScanDependencies/Inputs/CHeaders/F.h b/test/ScanDependencies/Inputs/CHeaders/F.h new file mode 100644 index 0000000000000..c004f15ca23a2 --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/F.h @@ -0,0 +1 @@ +void funcF(void); diff --git a/test/ScanDependencies/Inputs/CHeaders/module.modulemap b/test/ScanDependencies/Inputs/CHeaders/module.modulemap new file mode 100644 index 0000000000000..0ebcb6d029092 --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/module.modulemap @@ -0,0 +1,24 @@ +module A { + header "A.h" + export * +} + +module B { + header "B.h" + export * +} + +module C { + header "C.h" + export * +} + +module D { + header "D.h" + export * +} + +module F { + header "F.h" + export * +} diff --git a/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift b/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift new file mode 100644 index 0000000000000..127126b58aeae --- /dev/null +++ b/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift @@ -0,0 +1,154 @@ +//===--------------- ModuleDependencyGraph.swift --------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import Foundation + +enum ModuleDependencyId: Hashable { + case swift(String) + case clang(String) + + var moduleName: String { + switch self { + case .swift(let name): return name + case .clang(let name): return name + } + } +} + +extension ModuleDependencyId: Codable { + enum CodingKeys: CodingKey { + case swift + case clang + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + do { + let moduleName = try container.decode(String.self, forKey: .swift) + self = .swift(moduleName) + } catch { + let moduleName = try container.decode(String.self, forKey: .clang) + self = .clang(moduleName) + } + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .swift(let moduleName): + try container.encode(moduleName, forKey: .swift) + case .clang(let moduleName): + try container.encode(moduleName, forKey: .clang) + } + } +} + +/// Bridging header +struct BridgingHeader: Codable { + var path: String + var sourceFiles: [String] + var moduleDependencies: [String] +} + +/// Details specific to Swift modules. +struct SwiftModuleDetails: Codable { + /// The module interface from which this module was built, if any. + var moduleInterfacePath: String? + + /// The bridging header, if any. + var bridgingHeader: BridgingHeader? +} + +/// Details specific to Clang modules. +struct ClangModuleDetails: Codable { + /// The path to the module map used to build this module. + var moduleMapPath: String + + /// The context hash used to discriminate this module file. + var contextHash: String + + /// The Clang command line arguments that need to be passed through + /// to the -emit-pcm action to build this module. + var commandLine: [String] = [] +} + +struct ModuleDependencies: Codable { + /// The path for the module. + var modulePath: String + + /// The source files used to build this module. + var sourceFiles: [String] = [] + + /// The set of direct module dependencies of this module. + var directDependencies: [ModuleDependencyId] = [] + + /// Specific details of a particular kind of module. + var details: Details + + /// Specific details of a particular kind of module. + enum Details { + /// Swift modules may be built from a module interface, and may have + /// a bridging header. + case swift(SwiftModuleDetails) + + /// Clang modules are built from a module map file. + case clang(ClangModuleDetails) + } +} + +extension ModuleDependencies.Details: Codable { + enum CodingKeys: CodingKey { + case swift + case clang + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + do { + let details = try container.decode(SwiftModuleDetails.self, forKey: .swift) + self = .swift(details) + } catch { + let details = try container.decode(ClangModuleDetails.self, forKey: .clang) + self = .clang(details) + } + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .swift(let details): + try container.encode(details, forKey: .swift) + case .clang(let details): + try container.encode(details, forKey: .clang) + } + } +} + +/// Describes the complete set of dependencies for a Swift module, including +/// all of the Swift and C modules and source files it depends on. +struct ModuleDependencyGraph: Codable { + /// The name of the main module. + var mainModuleName: String + + /// The complete set of modules discovered + var modules: [ModuleDependencyId: ModuleDependencies] = [:] + + /// Information about the main module. + var mainModule: ModuleDependencies { modules[.swift(mainModuleName)]! } +} + +let fileName = CommandLine.arguments[1] +let data = try! Data(contentsOf: URL(fileURLWithPath: fileName)) + +let decoder = JSONDecoder() +let moduleDependencyGraph = try! decoder.decode( + ModuleDependencyGraph.self, from: data) +print(moduleDependencyGraph) diff --git a/test/ScanDependencies/Inputs/Swift/A.swiftinterface b/test/ScanDependencies/Inputs/Swift/A.swiftinterface new file mode 100644 index 0000000000000..1b6292940c2bf --- /dev/null +++ b/test/ScanDependencies/Inputs/Swift/A.swiftinterface @@ -0,0 +1,5 @@ +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name A +@_exported import A +public func overlayFuncA() { } + diff --git a/test/ScanDependencies/Inputs/Swift/E.swiftinterface b/test/ScanDependencies/Inputs/Swift/E.swiftinterface new file mode 100644 index 0000000000000..824fe883f6b8d --- /dev/null +++ b/test/ScanDependencies/Inputs/Swift/E.swiftinterface @@ -0,0 +1,4 @@ +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name E +import Swift +public func funcE() { } \ No newline at end of file diff --git a/test/ScanDependencies/Inputs/Swift/F.swiftinterface b/test/ScanDependencies/Inputs/Swift/F.swiftinterface new file mode 100644 index 0000000000000..380d87930c853 --- /dev/null +++ b/test/ScanDependencies/Inputs/Swift/F.swiftinterface @@ -0,0 +1,5 @@ +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name F +import Swift +@_exported import F +public func funcF() { } \ No newline at end of file diff --git a/test/ScanDependencies/module_deps.swift b/test/ScanDependencies/module_deps.swift new file mode 100644 index 0000000000000..2a74953e3d26d --- /dev/null +++ b/test/ScanDependencies/module_deps.swift @@ -0,0 +1,132 @@ +// RUN: %empty-directory(%t) +// RUN: mkdir -p %t/clang-module-cache +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h + +// Check the contents of the JSON output +// RUN: %FileCheck %s < %t/deps.json + +// Check the make-style dependencies file +// RUN: %FileCheck %s -check-prefix CHECK-MAKE-DEPS < %t/deps.d + +// Check that the JSON parses correctly into the canonical Swift data +// structures. + +// RUN: %target-build-swift %S/Inputs/ModuleDependencyGraph.swift -o %t/main +// RUN: %target-codesign %t/main +// RUN: %target-run %t/main %t/deps.json + +// REQUIRES: executable_test +// REQUIRES: objc_interop + +import C +import E + +// CHECK: "mainModuleName": "deps" + +/// --------Main module +// CHECK-LABEL: "modulePath": "deps.swiftmodule", +// CHECK-NEXT: sourceFiles +// CHECK-NEXT: module_deps.swift + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "C" +// CHECK-NEXT: } +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "E" +// CHECK-NEXT: } +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" +// CHECK-NEXT: } +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "SwiftOnoneSupport" +// CHECK-NEXT: } +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "F" +// CHECK-NEXT: } +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "A" +// CHECK-NEXT: } + +// CHECK: "bridgingHeader": +// CHECK-NEXT: "path": +// CHECK-SAME: Bridging.h + +// CHECK-NEXT: "sourceFiles": +// CHECK-NEXT: Bridging.h +// CHECK-NEXT: BridgingOther.h + +// CHECK: "moduleDependencies": [ +// CHECK-NEXT: "F" +// CHECK-NEXT: ] + +/// --------Clang module C +// CHECK-LABEL: "modulePath": "C.pcm", + +// CHECK: "sourceFiles": [ +// CHECK-NEXT: module.modulemap +// CHECK-NEXT: C.h + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "B" + +// CHECK: "moduleMapPath" +// CHECK-SAME: module.modulemap + +// CHECK: "contextHash" +// CHECK-SAME: "{{.*}}" + +// CHECK: "commandLine": [ +// CHECK-NEXT: "-remove-preceeding-explicit-module-build-incompatible-options" + +/// --------Swift module E +// CHECK: "swift": "E" +// CHECK-LABEL: modulePath": "E.swiftmodule" +// CHECK: "directDependencies" +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" + +// CHECK: "moduleInterfacePath" +// CHECK-SAME: E.swiftinterface + +/// --------Swift module Swift +// CHECK-LABEL: "modulePath": "Swift.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "SwiftShims" + +/// --------Swift module A +// CHECK-LABEL: "modulePath": "A.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "A" +// CHECK-NEXT: } + +/// --------Clang module B +// CHECK-LABEL: "modulePath": "B.pcm" + +// CHECK-NEXT: sourceFiles +// CHECK-NEXT: B.h +// CHECK-NEXT: module.modulemap + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "A" +// CHECK-NEXT: } + +/// --------Clang module SwiftShims +// CHECK-LABEL: "modulePath": "SwiftShims.pcm", + + +// Check make-style dependencies +// CHECK-MAKE-DEPS: module_deps.swift +// CHECK-MAKE-DEPS-SAME: A.swiftinterface +// CHECK-MAKE-DEPS-SAME: Swift.swiftmodule +// CHECK-MAKE-DEPS-SAME: B.h +// CHECK-MAKE-DEPS-SAME: F.h +// CHECK-MAKE-DEPS-SAME: Bridging.h +// CHECK-MAKE-DEPS-SAME: BridgingOther.h +// CHECK-MAKE-DEPS-SAME: module.modulemap