From 33cdd61835400b8e25a022051a9550eb3399b2a0 Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Sat, 4 Jan 2020 22:49:54 -0800 Subject: [PATCH 01/12] Fast dependency scanning for Swift Implement a new "fast" dependency scanning option, `-scan-dependencies`, in the Swift frontend that determines all of the source file and module dependencies for a given set of Swift sources. It covers four forms of modules: 1) Swift (serialized) module files, by reading the module header 2) Swift interface files, by parsing the source code to find imports 3) Swift source modules, by parsing the source code to find imports 4) Clang modules, using Clang's fast dependency scanning tool A single `-scan-dependencies` operation maps out the full dependency graph for the given Swift source files, including all of the Swift and Clang modules that may need to be built, such that all of the work can be scheduled up front by the Swift driver or any other build system that understands this option. The dependency graph is emitted as JSON, which can be consumed by these other tools. --- include/swift/AST/ASTContext.h | 11 + include/swift/AST/ModuleDependencies.h | 284 +++++++++++++++ include/swift/AST/ModuleLoader.h | 10 + include/swift/Basic/FileTypes.def | 3 + include/swift/ClangImporter/ClangImporter.h | 3 + include/swift/Frontend/FrontendOptions.h | 2 + include/swift/Option/Options.td | 4 + include/swift/Sema/SourceLoader.h | 7 + .../Serialization/SerializedModuleLoader.h | 6 + lib/AST/ASTContext.cpp | 16 + lib/AST/CMakeLists.txt | 1 + lib/AST/ModuleDependencies.cpp | 143 ++++++++ lib/Basic/FileTypes.cpp | 3 + lib/ClangImporter/CMakeLists.txt | 4 + lib/ClangImporter/ClangImporter.cpp | 18 +- .../ClangModuleDependencyScanner.cpp | 241 +++++++++++++ lib/ClangImporter/ImporterImpl.h | 10 + lib/Driver/Driver.cpp | 9 + lib/Driver/ToolChains.cpp | 3 + .../ArgsToFrontendOptionsConverter.cpp | 2 + lib/Frontend/CMakeLists.txt | 1 + lib/Frontend/FrontendOptions.cpp | 16 + lib/Frontend/ModuleDependencyScanner.cpp | 149 ++++++++ lib/FrontendTool/CMakeLists.txt | 1 + lib/FrontendTool/FrontendTool.cpp | 11 +- lib/FrontendTool/ScanDependencies.cpp | 325 ++++++++++++++++++ lib/FrontendTool/ScanDependencies.h | 28 ++ lib/Serialization/SerializedModuleLoader.cpp | 41 +++ test/ScanDependencies/Inputs/CHeaders/A.h | 1 + test/ScanDependencies/Inputs/CHeaders/B.h | 4 + test/ScanDependencies/Inputs/CHeaders/C.h | 3 + test/ScanDependencies/Inputs/CHeaders/D.h | 1 + .../Inputs/CHeaders/module.modulemap | 19 + .../Inputs/Swift/A.swiftinterface | 5 + .../Inputs/Swift/E.swiftinterface | 4 + test/ScanDependencies/module_deps.swift | 75 ++++ 36 files changed, 1454 insertions(+), 10 deletions(-) create mode 100644 include/swift/AST/ModuleDependencies.h create mode 100644 lib/AST/ModuleDependencies.cpp create mode 100644 lib/ClangImporter/ClangModuleDependencyScanner.cpp create mode 100644 lib/Frontend/ModuleDependencyScanner.cpp create mode 100644 lib/FrontendTool/ScanDependencies.cpp create mode 100644 lib/FrontendTool/ScanDependencies.h create mode 100644 test/ScanDependencies/Inputs/CHeaders/A.h create mode 100644 test/ScanDependencies/Inputs/CHeaders/B.h create mode 100644 test/ScanDependencies/Inputs/CHeaders/C.h create mode 100644 test/ScanDependencies/Inputs/CHeaders/D.h create mode 100644 test/ScanDependencies/Inputs/CHeaders/module.modulemap create mode 100644 test/ScanDependencies/Inputs/Swift/A.swiftinterface create mode 100644 test/ScanDependencies/Inputs/Swift/E.swiftinterface create mode 100644 test/ScanDependencies/module_deps.swift diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h index c0ab977ec71a5..4448b41be61ed 100644 --- a/include/swift/AST/ASTContext.h +++ b/include/swift/AST/ASTContext.h @@ -78,6 +78,7 @@ namespace swift { class LazyContextData; class LazyIterableDeclContextData; class LazyMemberLoader; + class ModuleDependencies; class PatternBindingDecl; class PatternBindingInitializer; class SourceFile; @@ -91,6 +92,7 @@ namespace swift { class Identifier; class InheritedNameSet; class ModuleDecl; + class ModuleDependenciesCache; class ModuleLoader; class NominalTypeDecl; class NormalProtocolConformance; @@ -710,6 +712,15 @@ class ASTContext final { void addModuleLoader(std::unique_ptr loader, bool isClang = false, bool isDWARF = false); + /// Retrieve the module dependencies for the module with the given name. + /// + /// \param isUnderlyingClangModule When true, only look for a Clang module + /// with the given name, ignoring any Swift modules. + Optional getModuleDependencies( + StringRef moduleName, + bool isUnderlyingClangModule, + ModuleDependenciesCache &cache); + /// Load extensions to the given nominal type from the external /// module loaders. /// diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h new file mode 100644 index 0000000000000..d1c43b337727b --- /dev/null +++ b/include/swift/AST/ModuleDependencies.h @@ -0,0 +1,284 @@ +//===--- ModuleDependencies.h - Module Dependencies -------------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// This file defines data structures for capturing module dependencies. +// +//===----------------------------------------------------------------------===// +#ifndef SWIFT_AST_MODULE_DEPENDENCIES_H +#define SWIFT_AST_MODULE_DEPENDENCIES_H + +#include "swift/Basic/LLVM.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringSet.h" +#include +#include + +namespace swift { + +class ClangModuleDependenciesCacheImpl; +class SourceFile; + +/// Which kind of module dependencies we are looking for. +enum class ModuleDependenciesKind : int8_t { + Swift, + Clang, +}; + +/// Base class for the variant storage of ModuleDependencies. +class ModuleDependenciesStorageBase { +public: + const bool isSwiftModule; + + ModuleDependenciesStorageBase(bool isSwiftModule, + const std::string &compiledModulePath) + : isSwiftModule(isSwiftModule), + compiledModulePath(compiledModulePath) { } + + virtual ModuleDependenciesStorageBase *clone() const = 0; + + virtual ~ModuleDependenciesStorageBase(); + + /// The path to the compiled module file. + const std::string compiledModulePath; + + /// The set of modules on which this module depends. + std::vector moduleDependencies; +}; + +/// Describes the dependencies of a Swift module. +class SwiftModuleDependenciesStorage : public ModuleDependenciesStorageBase { +public: + /// The Swift interface file, if it can be used to generate the module file. + const Optional swiftInterfaceFile; + + /// Bridging header file, if there is one. + Optional bridgingHeaderFile; + + /// Swift source files that are part of the Swift module, when known. + std::vector sourceFiles; + + /// Source files on which the bridging header depends. + std::vector bridgingSourceFiles; + + SwiftModuleDependenciesStorage( + const std::string &compiledModulePath, + const Optional &swiftInterfaceFile + ) : ModuleDependenciesStorageBase(/*isSwiftModule=*/true, compiledModulePath), + swiftInterfaceFile(swiftInterfaceFile) { } + + ModuleDependenciesStorageBase *clone() const override { + return new SwiftModuleDependenciesStorage(*this); + } + + static bool classof(const ModuleDependenciesStorageBase *base) { + return base->isSwiftModule; + } +}; + +class ClangModuleDependenciesStorage : public ModuleDependenciesStorageBase { +public: + /// The module map file used to generate the Clang module. + const std::string moduleMapFile; + + /// The file dependencies + const std::vector fileDependencies; + + ClangModuleDependenciesStorage( + const std::string &compiledModulePath, + const std::string &moduleMapFile, + const std::vector &fileDependencies + ) : ModuleDependenciesStorageBase(/*isSwiftModule=*/false, + compiledModulePath), + moduleMapFile(moduleMapFile), + fileDependencies(fileDependencies) { } + + ModuleDependenciesStorageBase *clone() const override { + return new ClangModuleDependenciesStorage(*this); + } + + static bool classof(const ModuleDependenciesStorageBase *base) { + return !base->isSwiftModule; + } +}; + +/// Describes the dependencies of a given module. +class ModuleDependencies { +private: + std::unique_ptr storage; + + ModuleDependencies(std::unique_ptr &&storage) + : storage(std::move(storage)) { } + +public: + ModuleDependencies(const ModuleDependencies &other) + : storage(other.storage->clone()) { } + ModuleDependencies(ModuleDependencies &&other) = default; + + ModuleDependencies &operator=(const ModuleDependencies &other) { + storage.reset(other.storage->clone()); + return *this; + } + + ModuleDependencies &operator=(ModuleDependencies &&other) = default; + + /// Describe the module dependencies for a Swift module that can be + /// built from a Swift interface file (\c .swiftinterface). + static ModuleDependencies forSwiftInterface( + const std::string &compiledModulePath, + const std::string &swiftInterfaceFile) { + return ModuleDependencies( + std::make_unique( + compiledModulePath, swiftInterfaceFile)); + } + + /// Describe the module dependencies for a serialized or parsed Swift module. + static ModuleDependencies forSwiftModule( + const std::string &compiledModulePath) { + return ModuleDependencies( + std::make_unique( + compiledModulePath, None)); + } + + /// Describe the module dependencies for a Clang module that can be + /// built from a module map and headers. + static ModuleDependencies forClangModule( + const std::string &compiledModulePath, + const std::string &moduleMapFile, + const std::vector &fileDependencies) { + return ModuleDependencies( + std::make_unique(compiledModulePath, + moduleMapFile, + fileDependencies)); + } + + /// Retrieve the path to the compiled module. + const std::string getCompiledModulePath() const { + return storage->compiledModulePath; + } + + /// Retrieve the module-level dependencies. + ArrayRef getModuleDependencies() const { + return storage->moduleDependencies; + } + + /// Whether the dependencies are for a Swift module. + bool isSwiftModule() const; + + ModuleDependenciesKind getKind() const { + return isSwiftModule() ? ModuleDependenciesKind::Swift + : ModuleDependenciesKind::Clang; + } + /// Retrieve the dependencies for a Swift module. + const SwiftModuleDependenciesStorage *getAsSwiftModule() const; + + /// Retrieve the dependencies for a Clang module. + const ClangModuleDependenciesStorage *getAsClangModule() const; + + /// Add a dependency on the given module, if it was not already in the set. + void addModuleDependency(StringRef module, + llvm::StringSet<> &alreadyAddedModules); + + /// Add all of the module dependencies for the imports in the given source + /// file to the set of module dependencies. + void addModuleDependencies(const SourceFile &sf, + llvm::StringSet<> &alreadyAddedModules); + + /// Add a bridging header to a Swift module's dependencies. + void addBridgingHeader(StringRef bridgingHeader); +}; + +using ModuleDependencyID = std::pair; + +/// A cache describing the set of module dependencies that has been queried +/// thus far. +class ModuleDependenciesCache { + /// All cached module dependencies, in the order in which they were + /// encountered. + std::vector AllModules; + + /// Dependencies for Swift modules that have already been computed. + llvm::StringMap SwiftModuleDependencies; + + /// Dependencies for Clang modules that have already been computed. + llvm::StringMap ClangModuleDependencies; + + /// Additional information needed for Clang dependency scanning. + ClangModuleDependenciesCacheImpl *clangImpl = nullptr; + + /// Function that will delete \c clangImpl properly. + void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *) = nullptr; + + /// Free up the storage associated with the Clang implementation. + void destroyClangImpl() { + if (this->clangImplDeleter) + this->clangImplDeleter(this->clangImpl); + } + + /// Retrieve the dependencies map that corresponds to the given dependency + /// kind. + llvm::StringMap &getDependenciesMap( + ModuleDependenciesKind kind); + const llvm::StringMap &getDependenciesMap( + ModuleDependenciesKind kind) const; + +public: + ModuleDependenciesCache() { } + + ModuleDependenciesCache(const ModuleDependenciesCache &) = delete; + ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete; + + ~ModuleDependenciesCache() { + destroyClangImpl(); + } + + /// Set the Clang-specific implementation data. + void setClangImpl( + ClangModuleDependenciesCacheImpl *clangImpl, + void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *)) { + destroyClangImpl(); + + this->clangImpl = clangImpl; + this->clangImplDeleter = clangImplDeleter; + } + + /// Retrieve the Clang-specific implementation data; + ClangModuleDependenciesCacheImpl *getClangImpl() const { + return clangImpl; + } + + /// Whether we have cached dependency information for the given module. + bool hasDependencies(StringRef moduleName, + Optional kind) const; + + /// Look for module dependencies for a module with the given name. + /// + /// \returns the cached result, or \c None if there is no cached entry. + Optional findDependencies( + StringRef moduleName, + Optional kind) const; + + /// Record dependencies for the given module. + void recordDependencies(StringRef moduleName, + ModuleDependencies dependencies, + ModuleDependenciesKind kind); + + /// Reference the list of all module dependencies. + const std::vector &getAllModules() const { + return AllModules; + } +}; + +} + +#endif /* SWIFT_AST_MODULE_DEPENDENCIES_H */ diff --git a/include/swift/AST/ModuleLoader.h b/include/swift/AST/ModuleLoader.h index 8e13dae85fdaf..4e38cfac812b0 100644 --- a/include/swift/AST/ModuleLoader.h +++ b/include/swift/AST/ModuleLoader.h @@ -23,6 +23,7 @@ #include "swift/Basic/SourceLoc.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/StringSet.h" #include "llvm/ADT/TinyPtrVector.h" namespace llvm { @@ -41,7 +42,10 @@ class ClangImporterOptions; class ClassDecl; class FileUnit; class ModuleDecl; +class ModuleDependencies; +class ModuleDependenciesCache; class NominalTypeDecl; +class SourceFile; class TypeDecl; enum class KnownProtocolKind : uint8_t; @@ -177,6 +181,12 @@ class ModuleLoader { /// Discover overlays declared alongside this file and add infomation about /// them to it. void findOverlayFiles(SourceLoc diagLoc, ModuleDecl *module, FileUnit *file); + + /// Retrieve the dependencies for the given, named module, or \c None + /// if no such module exists. + virtual Optional getModuleDependencies( + StringRef moduleName, + ModuleDependenciesCache &cache) = 0; }; } // namespace swift diff --git a/include/swift/Basic/FileTypes.def b/include/swift/Basic/FileTypes.def index 1465d14607c74..1435813ae6e6c 100644 --- a/include/swift/Basic/FileTypes.def +++ b/include/swift/Basic/FileTypes.def @@ -71,6 +71,9 @@ TYPE("tbd", TBD, "tbd", "") // Swift section of the internal wiki. TYPE("module-trace", ModuleTrace, "trace.json", "") +// Complete dependency information for the given Swift files as JSON. +TYPE("json-dependencies", JSONDependencies, "dependencies.json", "") + TYPE("index-data", IndexData, "", "") TYPE("yaml-opt-record", YAMLOptRecord, "opt.yaml", "") TYPE("bitstream-opt-record",BitstreamOptRecord, "opt.bitstream", "") diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h index f401e5c5094af..15932ce295d85 100644 --- a/include/swift/ClangImporter/ClangImporter.h +++ b/include/swift/ClangImporter/ClangImporter.h @@ -369,6 +369,9 @@ class ClangImporter final : public ClangModuleLoader { void verifyAllModules() override; + Optional getModuleDependencies( + StringRef moduleName, ModuleDependenciesCache &cache) override; + clang::TargetInfo &getTargetInfo() const override; clang::ASTContext &getClangASTContext() const override; clang::Preprocessor &getClangPreprocessor() const override; diff --git a/include/swift/Frontend/FrontendOptions.h b/include/swift/Frontend/FrontendOptions.h index fc83b750579d6..6b4e03f2506ca 100644 --- a/include/swift/Frontend/FrontendOptions.h +++ b/include/swift/Frontend/FrontendOptions.h @@ -132,6 +132,8 @@ class FrontendOptions { EmitPCM, ///< Emit precompiled Clang module from a module map DumpPCM, ///< Dump information about a precompiled Clang module + + ScanDependencies, ///< Scan dependencies of Swift source files }; /// Indicates the action the user requested that the frontend perform. diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td index 8aa8775751a59..4a8d9883a8541 100644 --- a/include/swift/Option/Options.td +++ b/include/swift/Option/Options.td @@ -1031,6 +1031,10 @@ def sanitize_coverage_EQ : CommaJoined<["-"], "sanitize-coverage=">, HelpText<"Specify the type of coverage instrumentation for Sanitizers and" " additional options separated by commas">; +def scan_dependencies : Flag<["-"], "scan-dependencies">, + HelpText<"Scan dependencies of the given Swift sources">, ModeOpt, + Flags<[FrontendOption, NoInteractiveOption, DoesNotAffectIncrementalBuild]>; + def enable_astscope_lookup : Flag<["-"], "enable-astscope-lookup">, Flags<[FrontendOption]>, HelpText<"Enable ASTScope-based unqualified name lookup">; diff --git a/include/swift/Sema/SourceLoader.h b/include/swift/Sema/SourceLoader.h index cb8f4d19e9bf9..414a802b53cef 100644 --- a/include/swift/Sema/SourceLoader.h +++ b/include/swift/Sema/SourceLoader.h @@ -13,6 +13,7 @@ #ifndef SWIFT_SEMA_SOURCELOADER_H #define SWIFT_SEMA_SOURCELOADER_H +#include "swift/AST/ModuleDependencies.h" #include "swift/AST/ModuleLoader.h" namespace swift { @@ -90,6 +91,12 @@ class SourceLoader : public ModuleLoader { { // Parsing populates the Objective-C method tables. } + + Optional getModuleDependencies( + StringRef moduleName, ModuleDependenciesCache &cache) override { + // FIXME: Implement? + return None; + } }; } diff --git a/include/swift/Serialization/SerializedModuleLoader.h b/include/swift/Serialization/SerializedModuleLoader.h index 11cb775b995a9..3302fba5abbb3 100644 --- a/include/swift/Serialization/SerializedModuleLoader.h +++ b/include/swift/Serialization/SerializedModuleLoader.h @@ -135,6 +135,9 @@ class SerializedModuleLoaderBase : public ModuleLoader { return false; } + /// Scan the given serialized module file to determine dependencies. + llvm::ErrorOr scanModuleFile(Twine modulePath); + public: virtual ~SerializedModuleLoaderBase(); SerializedModuleLoaderBase(const SerializedModuleLoaderBase &) = delete; @@ -189,6 +192,9 @@ class SerializedModuleLoaderBase : public ModuleLoader { llvm::SetVector &results) override; virtual void verifyAllModules() override; + + virtual Optional getModuleDependencies( + StringRef moduleName, ModuleDependenciesCache &cache) override; }; /// Imports serialized Swift modules into an ASTContext. diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp index c08ba61bd559e..5c5b4af8c182e 100644 --- a/lib/AST/ASTContext.cpp +++ b/lib/AST/ASTContext.cpp @@ -32,6 +32,7 @@ #include "swift/AST/IndexSubset.h" #include "swift/AST/KnownProtocols.h" #include "swift/AST/LazyResolver.h" +#include "swift/AST/ModuleDependencies.h" #include "swift/AST/ModuleLoader.h" #include "swift/AST/NameLookup.h" #include "swift/AST/ParameterList.h" @@ -1459,6 +1460,21 @@ void ASTContext::addModuleLoader(std::unique_ptr loader, getImpl().ModuleLoaders.push_back(std::move(loader)); } +Optional ASTContext::getModuleDependencies( + StringRef moduleName, bool isUnderlyingClangModule, + ModuleDependenciesCache &cache) { + for (auto &loader : getImpl().ModuleLoaders) { + if (isUnderlyingClangModule && + loader.get() != getImpl().TheClangModuleLoader) + continue; + + if (auto dependencies = loader->getModuleDependencies(moduleName, cache)) + return dependencies; + } + + return None; +} + void ASTContext::loadExtensions(NominalTypeDecl *nominal, unsigned previousGeneration) { PrettyStackTraceDecl stackTrace("loading extensions for", nominal); diff --git a/lib/AST/CMakeLists.txt b/lib/AST/CMakeLists.txt index 088f638412aa8..5be2d5ae2bfd1 100644 --- a/lib/AST/CMakeLists.txt +++ b/lib/AST/CMakeLists.txt @@ -56,6 +56,7 @@ add_swift_host_library(swiftAST STATIC InlinableText.cpp LayoutConstraint.cpp Module.cpp + ModuleDependencies.cpp ModuleLoader.cpp ModuleNameLookup.cpp NameLookup.cpp diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp new file mode 100644 index 0000000000000..62975dd9a1258 --- /dev/null +++ b/lib/AST/ModuleDependencies.cpp @@ -0,0 +1,143 @@ +//===--- ModuleDependencies.h - Module Dependencies -------------*- C++ -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// This file implements data structures for capturing module dependencies. +// +//===----------------------------------------------------------------------===// +#include "swift/AST/ModuleDependencies.h" +#include "swift/AST/Decl.h" +#include "swift/AST/SourceFile.h" +using namespace swift; + +ModuleDependenciesStorageBase::~ModuleDependenciesStorageBase() { } + +bool ModuleDependencies::isSwiftModule() const { + return isa(storage.get()); +} + +/// Retrieve the dependencies for a Swift module. +const SwiftModuleDependenciesStorage * +ModuleDependencies::getAsSwiftModule() const { + return dyn_cast(storage.get()); +} + +/// Retrieve the dependencies for a Clang module. +const ClangModuleDependenciesStorage * +ModuleDependencies::getAsClangModule() const { + return dyn_cast(storage.get()); +} + +void ModuleDependencies::addModuleDependency( + StringRef module, llvm::StringSet<> &alreadyAddedModules) { + if (alreadyAddedModules.insert(module).second) + storage->moduleDependencies.push_back(module); +} + +void ModuleDependencies::addModuleDependencies( + const SourceFile &sf, llvm::StringSet<> &alreadyAddedModules) { + // Add all of the module dependencies. + SmallVector decls; + sf.getTopLevelDecls(decls); + for (auto decl : decls) { + auto importDecl = dyn_cast(decl); + if (!importDecl) + continue; + + addModuleDependency(importDecl->getModulePath().front().Item.str(), + alreadyAddedModules); + } + + auto fileName = sf.getFilename(); + if (fileName.empty()) + return; + + // If the storage is for an interface file, the only source file we + // should see is that interface file. + auto swiftStorage = cast(storage.get()); + if (swiftStorage->swiftInterfaceFile) { + assert(fileName == *swiftStorage->swiftInterfaceFile); + return; + } + + // Otherwise, record the source file. + swiftStorage->sourceFiles.push_back(fileName); +} + +void ModuleDependencies::addBridgingHeader(StringRef bridgingHeader) { + auto swiftStorage = cast(storage.get()); + assert(!swiftStorage->bridgingHeaderFile); + swiftStorage->bridgingHeaderFile = bridgingHeader; +} + +llvm::StringMap & +ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) { + switch (kind) { + case ModuleDependenciesKind::Swift: + return SwiftModuleDependencies; + + case ModuleDependenciesKind::Clang: + return ClangModuleDependencies; + } +} + +const llvm::StringMap & +ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) const { + switch (kind) { + case ModuleDependenciesKind::Swift: + return SwiftModuleDependencies; + + case ModuleDependenciesKind::Clang: + return ClangModuleDependencies; + } +} + +bool ModuleDependenciesCache::hasDependencies( + StringRef moduleName, + Optional kind) const { + if (!kind) { + return hasDependencies(moduleName, ModuleDependenciesKind::Swift) || + hasDependencies(moduleName, ModuleDependenciesKind::Clang); + } + + const auto &map = getDependenciesMap(*kind); + return map.count(moduleName) > 0; +} + +Optional ModuleDependenciesCache::findDependencies( + StringRef moduleName, + Optional kind) const { + if (!kind) { + if (auto swiftDep = findDependencies( + moduleName, ModuleDependenciesKind::Swift)) + return swiftDep; + + return findDependencies(moduleName, ModuleDependenciesKind::Clang); + } + + const auto &map = getDependenciesMap(*kind); + auto known = map.find(moduleName); + if (known != map.end()) + return known->second; + + return None; +} + +void ModuleDependenciesCache::recordDependencies( + StringRef moduleName, + ModuleDependencies dependencies, + ModuleDependenciesKind kind) { + auto &map = getDependenciesMap(kind); + assert(map.count(moduleName) == 0 && "Already added to map"); + map.insert({moduleName, std::move(dependencies)}); + + AllModules.push_back({moduleName, kind}); +} diff --git a/lib/Basic/FileTypes.cpp b/lib/Basic/FileTypes.cpp index 4539dfc528df9..d2c1570495dd8 100644 --- a/lib/Basic/FileTypes.cpp +++ b/lib/Basic/FileTypes.cpp @@ -82,6 +82,7 @@ bool file_types::isTextual(ID Id) { case file_types::TY_SwiftModuleInterfaceFile: case file_types::TY_PrivateSwiftModuleInterfaceFile: case file_types::TY_SwiftOverlayFile: + case file_types::TY_JSONDependencies: return true; case file_types::TY_Image: case file_types::TY_Object: @@ -151,6 +152,7 @@ bool file_types::isAfterLLVM(ID Id) { case file_types::TY_BitstreamOptRecord: case file_types::TY_SwiftModuleInterfaceFile: case file_types::TY_PrivateSwiftModuleInterfaceFile: + case file_types::TY_JSONDependencies: return false; case file_types::TY_INVALID: llvm_unreachable("Invalid type ID."); @@ -199,6 +201,7 @@ bool file_types::isPartOfSwiftCompilation(ID Id) { case file_types::TY_ModuleTrace: case file_types::TY_YAMLOptRecord: case file_types::TY_BitstreamOptRecord: + case file_types::TY_JSONDependencies: return false; case file_types::TY_INVALID: llvm_unreachable("Invalid type ID."); diff --git a/lib/ClangImporter/CMakeLists.txt b/lib/ClangImporter/CMakeLists.txt index d1fb2a6f8d0a0..97c8bfc6302d3 100644 --- a/lib/ClangImporter/CMakeLists.txt +++ b/lib/ClangImporter/CMakeLists.txt @@ -9,6 +9,7 @@ add_swift_host_library(swiftClangImporter STATIC ClangAdapter.cpp ClangDiagnosticConsumer.cpp ClangImporter.cpp + ClangModuleDependencyScanner.cpp ClangSourceBufferImporter.cpp DWARFImporter.cpp IAMInference.cpp @@ -25,6 +26,9 @@ target_link_libraries(swiftClangImporter PRIVATE swiftParse LLVMBitstreamReader) +target_link_libraries(swiftClangImporter INTERFACE + clangDependencyScanning) + # This property is only set by calls to clang_tablegen. It will not be set on # standalone builds, so it can always be safely passed. get_property(CLANG_TABLEGEN_TARGETS GLOBAL PROPERTY CLANG_TABLEGEN_TARGETS) diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index dd1b55bdc1a38..5b7f5791e2f5b 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -458,10 +458,11 @@ getGlibcModuleMapPath(SearchPathOptions& Opts, llvm::Triple triple, return None; } -static void -getNormalInvocationArguments(std::vector &invocationArgStrs, - ASTContext &ctx, - const ClangImporterOptions &importerOpts) { +void +importer::getNormalInvocationArguments( + std::vector &invocationArgStrs, + ASTContext &ctx, + const ClangImporterOptions &importerOpts) { const auto &LangOpts = ctx.LangOpts; const llvm::Triple &triple = LangOpts.Target; SearchPathOptions &searchPathOpts = ctx.SearchPathOpts; @@ -695,10 +696,11 @@ getEmbedBitcodeInvocationArguments(std::vector &invocationArgStrs, }); } -static void -addCommonInvocationArguments(std::vector &invocationArgStrs, - ASTContext &ctx, - const ClangImporterOptions &importerOpts) { +void +importer::addCommonInvocationArguments( + std::vector &invocationArgStrs, + ASTContext &ctx, + const ClangImporterOptions &importerOpts) { using ImporterImpl = ClangImporter::Implementation; const llvm::Triple &triple = ctx.LangOpts.Target; SearchPathOptions &searchPathOpts = ctx.SearchPathOpts; diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp new file mode 100644 index 0000000000000..2afcdc99c2a24 --- /dev/null +++ b/lib/ClangImporter/ClangModuleDependencyScanner.cpp @@ -0,0 +1,241 @@ +//===--- ClangModuleDependencyScanner.cpp - Dependency Scanning -----------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2019 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// This file implements dependency scanning for Clang modules. +// +//===----------------------------------------------------------------------===// +#include "ImporterImpl.h" +#include "swift/AST/ModuleDependencies.h" +#include "swift/ClangImporter/ClangImporter.h" +#include "swift/ClangImporter/ClangImporterOptions.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Signals.h" + +using namespace swift; + +using namespace clang::tooling; +using namespace clang::tooling::dependencies; + +class swift::ClangModuleDependenciesCacheImpl { + /// The name of the file used for the "import hack" to compute module + /// dependencies. + /// FIXME: This should go away once Clang's dependency scanning library + /// can scan by module name. + std::string importHackFile; + +public: + /// Set containing all of the Clang modules that have already been seen. + llvm::StringSet<> alreadySeen; + + DependencyScanningService service; + + DependencyScanningTool tool; + + ClangModuleDependenciesCacheImpl() + : service(ScanningMode::MinimizedSourcePreprocessing, + ScanningOutputFormat::Full), + tool(service) { } + ~ClangModuleDependenciesCacheImpl(); + + /// Retrieve the name of the file used for the "import hack" that is + /// used to scan the dependencies of a Clang module. + llvm::ErrorOr getImportHackFile(); +}; + +ClangModuleDependenciesCacheImpl::~ClangModuleDependenciesCacheImpl() { + if (!importHackFile.empty()) { + llvm::sys::fs::remove(importHackFile); + } +} + +llvm::ErrorOr ClangModuleDependenciesCacheImpl::getImportHackFile() { + if (!importHackFile.empty()) + return importHackFile; + + // Create a temporary file. + int resultFD; + SmallString<128> resultPath; + if (auto error = llvm::sys::fs::createTemporaryFile( + "import-hack", "m", resultFD, resultPath)) + return error; + + llvm::raw_fd_ostream out(resultFD, /*shouldClose=*/true); + out << "@import HACK_MODULE_NAME;\n"; + llvm::sys::RemoveFileOnSignal(resultPath); + importHackFile = resultPath.str().str(); + return importHackFile; +} + +namespace { + class SingleCommandCompilationDatabase : public CompilationDatabase { + public: + SingleCommandCompilationDatabase(CompileCommand Cmd) + : Command(std::move(Cmd)) {} + + virtual std::vector + getCompileCommands(StringRef FilePath) const { + return {Command}; + } + + virtual std::vector getAllCompileCommands() const { + return {Command}; + } + + private: + CompileCommand Command; + }; +} + +Optional ClangImporter::getModuleDependencies( + StringRef moduleName, ModuleDependenciesCache &cache) { + // Check whether there is already a cached result. + if (auto found = cache.findDependencies( + moduleName, ModuleDependenciesKind::Clang)) + return found; + + // Retrieve or create the shared state. + auto clangImpl = cache.getClangImpl(); + if (!clangImpl) { + clangImpl = new ClangModuleDependenciesCacheImpl(); + cache.setClangImpl(clangImpl, + [](ClangModuleDependenciesCacheImpl *ptr) { + delete ptr; + }); + } + + // Reform the Clang importer options. + // FIXME: Just save a reference or copy so we can get this back. + ClangImporterOptions importerOpts; + + // Determine the command-line arguments for dependency scanning. + auto &ctx = Impl.SwiftContext; + std::vector commandLineArgs; + commandLineArgs.push_back("clang"); + importer::getNormalInvocationArguments(commandLineArgs, ctx, importerOpts); + importer::addCommonInvocationArguments(commandLineArgs, ctx, importerOpts); + + // Add search paths. + // Note: This is handled differently for the Clang importer itself, which + // adds search paths to Clang's data structures rather than to its + // command line. + SearchPathOptions &searchPathOpts = ctx.SearchPathOpts; + for (const auto &framepath : searchPathOpts.FrameworkSearchPaths) { + commandLineArgs.push_back(framepath.IsSystem ? "-iframework" : "-F"); + commandLineArgs.push_back(framepath.Path); + } + + for (auto path : searchPathOpts.ImportSearchPaths) { + commandLineArgs.push_back("-I"); + commandLineArgs.push_back(path); + } + + // HACK! Replace the module import buffer name with the source file hack. + auto importHackFile = clangImpl->getImportHackFile(); + if (!importHackFile) { + // FIXME: Emit a diagnostic here. + return None; + } + + auto sourceFilePos = std::find( + commandLineArgs.begin(), commandLineArgs.end(), + ""); + assert(sourceFilePos != commandLineArgs.end()); + *sourceFilePos = *importHackFile; + + // HACK! Drop the -fmodule-format= argument and the one that + // precedes it. + { + auto moduleFormatPos = std::find_if(commandLineArgs.begin(), + commandLineArgs.end(), + [](StringRef arg) { + return arg.startswith("-fmodule-format="); + }); + assert(moduleFormatPos != commandLineArgs.end()); + assert(moduleFormatPos != commandLineArgs.begin()); + commandLineArgs.erase(moduleFormatPos-1, moduleFormatPos+1); + } + + // HACK: No -fsyntax-only here? + { + auto syntaxOnlyPos = std::find(commandLineArgs.begin(), + commandLineArgs.end(), + "-fsyntax-only"); + assert(syntaxOnlyPos != commandLineArgs.end()); + *syntaxOnlyPos = "-c"; + } + + // HACK: Stolen from ClangScanDeps.cpp + commandLineArgs.push_back("-o"); + commandLineArgs.push_back("/dev/null"); + commandLineArgs.push_back("-M"); + commandLineArgs.push_back("-MT"); + commandLineArgs.push_back("import-hack.o"); + commandLineArgs.push_back("-Xclang"); + commandLineArgs.push_back("-Eonly"); + commandLineArgs.push_back("-Xclang"); + commandLineArgs.push_back("-sys-header-deps"); + commandLineArgs.push_back("-Wno-error"); + + // HACK! Trick out a .m file to use to import the module we name. + std::string moduleNameHackDefine = + ("-DHACK_MODULE_NAME=" + moduleName).str(); + commandLineArgs.push_back(moduleNameHackDefine); + commandLineArgs.push_back("-fmodules-ignore-macro=HACK_MODULE_NAME"); + + std::string workingDir = + ctx.SourceMgr.getFileSystem()->getCurrentWorkingDirectory().get(); + CompileCommand command(workingDir, *importHackFile, commandLineArgs, "-"); + SingleCommandCompilationDatabase database(command); + + auto clangDependencies = clangImpl->tool.getFullDependencies( + database, workingDir, clangImpl->alreadySeen); + + if (!clangDependencies) { + // FIXME: Route this to a normal diagnostic. + llvm::logAllUnhandledErrors(clangDependencies.takeError(), llvm::errs()); + return None; + } + + // Record module dependencies for each module we found. + llvm::StringSet<> alreadyAddedModules; + for (const auto &clangModuleDep : clangDependencies->DiscoveredModules) { + // If we've already cached this information, we're done. + if (cache.hasDependencies(clangModuleDep.ModuleName, + ModuleDependenciesKind::Clang)) + continue; + + std::vector fileDeps; + for (const auto &fileDep : clangModuleDep.FileDeps) { + fileDeps.push_back(fileDep.getKey()); + } + + // Create a module filename. + // FIXME: Query Clang to determine an appropriate hashed name for the + // module file. + llvm::SmallString<32> modulePath = moduleName; + llvm::sys::path::replace_extension(modulePath, "pcm"); + + auto dependencies = ModuleDependencies::forClangModule( + modulePath.str(), clangModuleDep.ClangModuleMapFile, fileDeps); + for (const auto &moduleName : clangModuleDep.ClangModuleDeps) { + dependencies.addModuleDependency(moduleName.ModuleName, alreadyAddedModules); + } + + cache.recordDependencies(clangModuleDep.ModuleName, + std::move(dependencies), + ModuleDependenciesKind::Clang); + } + + return cache.findDependencies(moduleName, ModuleDependenciesKind::Clang); +} diff --git a/lib/ClangImporter/ImporterImpl.h b/lib/ClangImporter/ImporterImpl.h index 86e1be4d53fa4..b974766d1a0e5 100644 --- a/lib/ClangImporter/ImporterImpl.h +++ b/lib/ClangImporter/ImporterImpl.h @@ -1428,6 +1428,16 @@ bool shouldSuppressDeclImport(const clang::Decl *decl); /// but are now renamed using the swift_name attribute. bool isSpecialUIKitStructZeroProperty(const clang::NamedDecl *decl); +/// Add command-line arguments for a normal import of Clang code. +void getNormalInvocationArguments(std::vector &invocationArgStrs, + ASTContext &ctx, + const ClangImporterOptions &importerOpts); + +/// Add command-line arguments common to all imports of Clang code. +void addCommonInvocationArguments(std::vector &invocationArgStrs, + ASTContext &ctx, + const ClangImporterOptions &importerOpts); + /// Finds a particular kind of nominal by looking through typealiases. template static T *dynCastIgnoringCompatibilityAlias(Decl *D) { diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp index 00eaeb27bd7ca..e77f03e9fbb2e 100644 --- a/lib/Driver/Driver.cpp +++ b/lib/Driver/Driver.cpp @@ -1537,6 +1537,14 @@ void Driver::buildOutputInfo(const ToolChain &TC, const DerivedArgList &Args, OI.CompilerMode = OutputInfo::Mode::SingleCompile; break; + case options::OPT_scan_dependencies: + OI.CompilerOutputType = file_types::TY_JSONDependencies; + // We want the imported modules from the module as a whole, not individual + // files, so let's do it in one invocation rather than having to collate + // later. + OI.CompilerMode = OutputInfo::Mode::SingleCompile; + break; + case options::OPT_index_file: OI.CompilerMode = OutputInfo::Mode::SingleCompile; OI.CompilerOutputType = file_types::TY_IndexData; @@ -1989,6 +1997,7 @@ void Driver::buildActions(SmallVectorImpl &TopLevelActions, case file_types::TY_PrivateSwiftModuleInterfaceFile: case file_types::TY_SwiftCrossImportDir: case file_types::TY_SwiftOverlayFile: + case file_types::TY_JSONDependencies: // We could in theory handle assembly or LLVM input, but let's not. // FIXME: What about LTO? Diags.diagnose(SourceLoc(), diag::error_unexpected_input_file, diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp index 67b92bd60dc0d..54156c76ee306 100644 --- a/lib/Driver/ToolChains.cpp +++ b/lib/Driver/ToolChains.cpp @@ -563,6 +563,8 @@ const char *ToolChain::JobContext::computeFrontendModeForCompile() const { return "-emit-module"; case file_types::TY_ImportedModules: return "-emit-imported-modules"; + case file_types::TY_JSONDependencies: + return "-scan-dependencies"; case file_types::TY_IndexData: return "-typecheck"; case file_types::TY_Remapping: @@ -829,6 +831,7 @@ ToolChain::constructInvocation(const BackendJobAction &job, case file_types::TY_PCH: case file_types::TY_ClangModuleFile: case file_types::TY_IndexData: + case file_types::TY_JSONDependencies: llvm_unreachable("Cannot be output from backend job"); case file_types::TY_Swift: case file_types::TY_dSYM: diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index e45159868a946..c87fef7b3557a 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -345,6 +345,8 @@ ArgsToFrontendOptionsConverter::determineRequestedAction(const ArgList &args) { return FrontendOptions::ActionType::EmitPCH; if (Opt.matches(OPT_emit_imported_modules)) return FrontendOptions::ActionType::EmitImportedModules; + if (Opt.matches(OPT_scan_dependencies)) + return FrontendOptions::ActionType::ScanDependencies; if (Opt.matches(OPT_parse)) return FrontendOptions::ActionType::Parse; if (Opt.matches(OPT_resolve_imports)) diff --git a/lib/Frontend/CMakeLists.txt b/lib/Frontend/CMakeLists.txt index 0cb3c72b15b96..1b3daf94db1c1 100644 --- a/lib/Frontend/CMakeLists.txt +++ b/lib/Frontend/CMakeLists.txt @@ -8,6 +8,7 @@ add_swift_host_library(swiftFrontend STATIC Frontend.cpp FrontendInputsAndOutputs.cpp FrontendOptions.cpp + ModuleDependencyScanner.cpp ModuleInterfaceBuilder.cpp ModuleInterfaceLoader.cpp ModuleInterfaceSupport.cpp diff --git a/lib/Frontend/FrontendOptions.cpp b/lib/Frontend/FrontendOptions.cpp index 5a447247ad332..ea3632401c8e5 100644 --- a/lib/Frontend/FrontendOptions.cpp +++ b/lib/Frontend/FrontendOptions.cpp @@ -61,6 +61,7 @@ bool FrontendOptions::needsProperModuleName(ActionType action) { case ActionType::EmitImportedModules: case ActionType::DumpTypeInfo: case ActionType::EmitPCM: + case ActionType::ScanDependencies: return true; } llvm_unreachable("Unknown ActionType"); @@ -99,6 +100,7 @@ bool FrontendOptions::isActionImmediate(ActionType action) { case ActionType::DumpTypeInfo: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; } llvm_unreachable("Unknown ActionType"); @@ -111,6 +113,7 @@ bool FrontendOptions::shouldActionOnlyParse(ActionType action) { case FrontendOptions::ActionType::EmitSyntax: case FrontendOptions::ActionType::DumpInterfaceHash: case FrontendOptions::ActionType::EmitImportedModules: + case FrontendOptions::ActionType::ScanDependencies: return true; default: return false; @@ -204,6 +207,9 @@ FrontendOptions::formatForPrincipalOutputFileForAction(ActionType action) { case ActionType::EmitPCM: return TY_ClangModuleFile; + + case ActionType::ScanDependencies: + return TY_JSONDependencies; } llvm_unreachable("unhandled action"); } @@ -240,6 +246,7 @@ bool FrontendOptions::canActionEmitDependencies(ActionType action) { case ActionType::EmitObject: case ActionType::EmitImportedModules: case ActionType::EmitPCM: + case ActionType::ScanDependencies: return true; } llvm_unreachable("unhandled action"); @@ -263,6 +270,7 @@ bool FrontendOptions::canActionEmitReferenceDependencies(ActionType action) { case ActionType::REPL: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::Typecheck: case ActionType::MergeModules: @@ -309,6 +317,7 @@ bool FrontendOptions::canActionEmitObjCHeader(ActionType action) { case ActionType::REPL: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::Typecheck: case ActionType::MergeModules: @@ -344,6 +353,7 @@ bool FrontendOptions::canActionEmitLoadedModuleTrace(ActionType action) { case ActionType::REPL: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::ResolveImports: case ActionType::Typecheck: @@ -385,6 +395,7 @@ bool FrontendOptions::canActionEmitModule(ActionType action) { case ActionType::REPL: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::MergeModules: case ActionType::EmitModuleOnly: @@ -427,6 +438,7 @@ bool FrontendOptions::canActionEmitInterface(ActionType action) { case ActionType::REPL: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::Typecheck: case ActionType::MergeModules: @@ -470,6 +482,7 @@ bool FrontendOptions::doesActionProduceOutput(ActionType action) { case ActionType::DumpTypeInfo: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return true; case ActionType::NoneAction: @@ -513,6 +526,7 @@ bool FrontendOptions::doesActionProduceTextualOutput(ActionType action) { case ActionType::EmitIR: case ActionType::DumpTypeInfo: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return true; } llvm_unreachable("unhandled action"); @@ -536,6 +550,7 @@ bool FrontendOptions::doesActionGenerateSIL(ActionType action) { case ActionType::CompileModuleFromInterface: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::EmitSILGen: case ActionType::EmitSIBGen: @@ -580,6 +595,7 @@ bool FrontendOptions::doesActionGenerateIR(ActionType action) { case ActionType::EmitImportedModules: case ActionType::EmitPCM: case ActionType::DumpPCM: + case ActionType::ScanDependencies: return false; case ActionType::Immediate: case ActionType::REPL: diff --git a/lib/Frontend/ModuleDependencyScanner.cpp b/lib/Frontend/ModuleDependencyScanner.cpp new file mode 100644 index 0000000000000..ea93494ee8654 --- /dev/null +++ b/lib/Frontend/ModuleDependencyScanner.cpp @@ -0,0 +1,149 @@ +//===--- ModuleDependencyScanner.cpp - Compute module dependencies --------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "swift/Serialization/SerializedModuleLoader.h" +#include "swift/AST/ASTContext.h" +#include "swift/AST/DiagnosticSuppression.h" +#include "swift/AST/ModuleDependencies.h" +#include "swift/AST/SourceFile.h" +#include "swift/Basic/FileTypes.h" +#include "swift/Subsystems.h" +using namespace swift; +using llvm::ErrorOr; + +namespace { + +/// A module "loader" that looks for .swiftinterface and .swiftmodule files +/// for the purpose of determining dependencies, but does not attempt to +/// load the module files. +class ModuleDependencyScanner : public SerializedModuleLoaderBase { + /// The module we're scanning dependencies of. + Identifier moduleName; + + /// Scan the given interface file to determine dependencies. + ErrorOr scanInterfaceFile( + Twine moduleInterfacePath); + +public: + Optional dependencies; + + ModuleDependencyScanner(ASTContext &ctx, ModuleLoadingMode LoadMode, + Identifier moduleName) + : SerializedModuleLoaderBase(ctx, nullptr, LoadMode, + /*IgnoreSwiftSourceInfoFile=*/true), + moduleName(moduleName) { } + + virtual std::error_code findModuleFilesInDirectory( + AccessPathElem ModuleID, + const SerializedModuleBaseName &BaseName, + SmallVectorImpl *ModuleInterfacePath, + std::unique_ptr *ModuleBuffer, + std::unique_ptr *ModuleDocBuffer, + std::unique_ptr *ModuleSourceInfoBuffer) override { + using namespace llvm::sys; + + auto &fs = *Ctx.SourceMgr.getFileSystem(); + + // Compute the full path of the module we're looking for. + auto ModPath = BaseName.getName(file_types::TY_SwiftModuleFile); + if (LoadMode == ModuleLoadingMode::OnlySerialized) { + // If there is no module file, there's nothing we can do. + if (!fs.exists(ModPath)) + return std::make_error_code(std::errc::no_such_file_or_directory); + + // The module file will be loaded directly. + auto dependencies = scanModuleFile(ModPath); + if (dependencies) { + this->dependencies = std::move(dependencies.get()); + return std::error_code(); + } + + return dependencies.getError(); + } + + // Check whether the .swiftinterface exists. + auto InPath = BaseName.getName(file_types::TY_SwiftModuleInterfaceFile); + + if (!fs.exists(InPath)) + return std::make_error_code(std::errc::no_such_file_or_directory); + + auto dependencies = scanInterfaceFile(InPath); + if (dependencies) { + this->dependencies = std::move(dependencies.get()); + return std::error_code(); + } + + return dependencies.getError(); + } + + virtual void collectVisibleTopLevelModuleNames( + SmallVectorImpl &names) const override { + llvm_unreachable("Not used"); + } +}; +} + +ErrorOr ModuleDependencyScanner::scanInterfaceFile( + Twine moduleInterfacePath) { + // Open the interface file. + auto &fs = *Ctx.SourceMgr.getFileSystem(); + auto interfaceBuf = fs.getBufferForFile(moduleInterfacePath); + if (!interfaceBuf) + return interfaceBuf.getError(); + + // Create a source file. + unsigned bufferID = Ctx.SourceMgr.addNewSourceBuffer(std::move(interfaceBuf.get())); + auto moduleDecl = ModuleDecl::create(moduleName, Ctx); + auto sourceFile = new (Ctx) SourceFile( + *moduleDecl, SourceFileKind::Interface, bufferID, + SourceFile::ImplicitModuleImportKind::None); + + // Create a module filename. + // FIXME: Query the module interface loader to determine an appropriate + // name for the module, which includes an appropriate hash. + auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile); + llvm::SmallString<32> modulePath = moduleName.str(); + llvm::sys::path::replace_extension(modulePath, newExt); + + // Walk the source file to find the import declarations. + llvm::StringSet<> alreadyAddedModules; + auto dependencies = ModuleDependencies::forSwiftInterface( + modulePath.str(), moduleInterfacePath.str()); + + // FIXME: Suppressing diagnostics here is incorrect, and is currently + // used to paper over the fact that we should be creating a fresh + // ASTContext using the command-line arguments from the .swiftinterface + // file. + DiagnosticSuppression suppression(Ctx.Diags); + + dependencies.addModuleDependencies(*sourceFile, alreadyAddedModules); + return dependencies; +} + +Optional SerializedModuleLoaderBase::getModuleDependencies( + StringRef moduleName, ModuleDependenciesCache &cache) { + // Check whether we've cached this result. + if (auto found = cache.findDependencies( + moduleName, ModuleDependenciesKind::Swift)) + return found; + + // Check whether there is a module with this name that we can import. + auto moduleId = Ctx.getIdentifier(moduleName); + ModuleDependencyScanner scanner(Ctx, LoadMode, moduleId); + if (!scanner.canImportModule({moduleId, SourceLoc()})) + return None; + + // Record the dependencies. + cache.recordDependencies(moduleName, *scanner.dependencies, + ModuleDependenciesKind::Swift); + return std::move(scanner.dependencies); +} diff --git a/lib/FrontendTool/CMakeLists.txt b/lib/FrontendTool/CMakeLists.txt index 948d0b5ef8b3d..2d06360885e6f 100644 --- a/lib/FrontendTool/CMakeLists.txt +++ b/lib/FrontendTool/CMakeLists.txt @@ -2,6 +2,7 @@ add_swift_host_library(swiftFrontendTool STATIC FrontendTool.cpp ImportedModules.cpp ReferenceDependencies.cpp + ScanDependencies.cpp TBD.cpp) add_dependencies(swiftFrontendTool swift-syntax-generated-headers diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index e78174844befa..df2b9ac84abad 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -22,6 +22,7 @@ #include "swift/FrontendTool/FrontendTool.h" #include "ImportedModules.h" +#include "ScanDependencies.h" #include "ReferenceDependencies.h" #include "TBD.h" @@ -922,6 +923,10 @@ static Optional dumpASTIfNeeded(const CompilerInvocation &Invocation, case FrontendOptions::ActionType::EmitImportedModules: emitImportedModules(Context, Instance.getMainModule(), opts); break; + + case FrontendOptions::ActionType::ScanDependencies: + scanDependencies(Context, Instance.getMainModule(), opts); + break; } return Context.hadError(); } @@ -1250,8 +1255,10 @@ static bool performCompile(CompilerInstance &Instance, // Disable delayed parsing of type and function bodies when we've been // asked to dump the resulting AST. bool CanDelayBodies = Action != FrontendOptions::ActionType::DumpParse; - Instance.performParseOnly(/*EvaluateConditionals*/ - Action == FrontendOptions::ActionType::EmitImportedModules, + bool EvaluateConditionals = + Action == FrontendOptions::ActionType::EmitImportedModules + || Action == FrontendOptions::ActionType::ScanDependencies; + Instance.performParseOnly(EvaluateConditionals, CanDelayBodies); } else if (Action == FrontendOptions::ActionType::ResolveImports) { Instance.performParseAndResolveImportsOnly(); diff --git a/lib/FrontendTool/ScanDependencies.cpp b/lib/FrontendTool/ScanDependencies.cpp new file mode 100644 index 0000000000000..6388f38b560bd --- /dev/null +++ b/lib/FrontendTool/ScanDependencies.cpp @@ -0,0 +1,325 @@ +//===--- ScanDependencies.cpp -- Scans the dependencies of a module -------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +#include "ScanDependencies.h" +#include "swift/AST/ASTContext.h" +#include "swift/AST/Decl.h" +#include "swift/AST/DiagnosticEngine.h" +#include "swift/AST/DiagnosticsFrontend.h" +#include "swift/AST/Module.h" +#include "swift/AST/ModuleDependencies.h" +#include "swift/AST/ModuleLoader.h" +#include "swift/AST/SourceFile.h" +#include "swift/Basic/Defer.h" +#include "swift/Basic/LLVM.h" +#include "swift/Basic/STLExtras.h" +#include "swift/ClangImporter/ClangImporter.h" +#include "swift/Frontend/FrontendOptions.h" +#include "clang/Basic/Module.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/FileSystem.h" +#include + +using namespace swift; + +/// Resolve the direct dependencies of the given module. +static std::vector resolveDirectDependencies( + ASTContext &ctx, ModuleDependencyID module, + ModuleDependenciesCache &cache) { + auto knownDependencies = *cache.findDependencies(module.first, module.second); + auto isSwift = knownDependencies.isSwiftModule(); + + // Find the dependencies of every module this module directly depends on. + std::vector result; + for (auto dependsOn : knownDependencies.getModuleDependencies()) { + // Figure out what kind of module we need. + bool onlyClangModule = !isSwift || module.first == dependsOn; + + // Retrieve the dependencies for this module. + if (auto found = ctx.getModuleDependencies( + dependsOn, onlyClangModule, cache)) { + result.push_back({dependsOn, found->getKind()}); + } + } + + // For a Swift module, look for overlays for each of the Clang modules it + // depends on. + if (isSwift) { + // FIXME: Implement this. + } + + return result; +} + +/// Write a single JSON field. +namespace { + template + void writeJSONSingleField(llvm::raw_ostream &out, + StringRef fieldName, + const T &value, + unsigned indentLevel, + bool trailingComma); + + /// Write a string value as JSON. + void writeJSONValue(llvm::raw_ostream &out, + StringRef value, + unsigned indentLevel) { + out << "\""; + out.write_escaped(value); + out << "\""; + } + + /// Write a module identifier. + void writeJSONValue(llvm::raw_ostream &out, + const ModuleDependencyID &module, + unsigned indentLevel) { + out << "{\n"; + + writeJSONSingleField( + out, + module.second == ModuleDependenciesKind::Swift ? "swift" : "clang", + module.first, + indentLevel + 1, + /*trailingComma=*/false); + + out.indent(indentLevel * 2); + out << "}"; + } + + /// Write a JSON array. + template + void writeJSONValue(llvm::raw_ostream &out, + ArrayRef values, + unsigned indentLevel) { + out << "[\n"; + + for (const auto &value: values) { + + out.indent((indentLevel + 1) * 2); + + writeJSONValue(out, value, indentLevel + 1); + + if (&value != &values.back()) { + out << ","; + } + out << "\n"; + } + + out.indent(indentLevel * 2); + out << "]"; + } + + /// Write a JSON array. + template + void writeJSONValue(llvm::raw_ostream &out, + const std::vector &values, + unsigned indentLevel) { + writeJSONValue(out, llvm::makeArrayRef(values), indentLevel); + } + + /// Write a single JSON field. + template + void writeJSONSingleField(llvm::raw_ostream &out, + StringRef fieldName, + const T &value, + unsigned indentLevel, + bool trailingComma) { + out.indent(indentLevel * 2); + writeJSONValue(out, fieldName, indentLevel); + out << ": "; + writeJSONValue(out, value, indentLevel); + if (trailingComma) + out << ","; + out << "\n"; + } +} + +static void writeJSON(llvm::raw_ostream &out, + ASTContext &ctx, + ModuleDependenciesCache &cache, + ArrayRef allModules) { + // Write out a JSON description of all of the dependencies. + out << "{\n"; + SWIFT_DEFER { + out << "}\n"; + }; + + // Name of the main module. + writeJSONSingleField(out, "mainModuleName", allModules.front().first, + /*indentLevel=*/1, /*trailingComma=*/true); + + // Write out all of the modules. + out << " \"modules\": [\n"; + SWIFT_DEFER { + out << " ]\n"; + }; + for (const auto &module : allModules) { + auto moduleDeps = *cache.findDependencies(module.first, module.second); + + // The module we are describing. + out.indent(2 * 2); + writeJSONValue(out, module, 2); + out << ",\n"; + + out.indent(2 * 2); + out << "{\n"; + + // Module path. + const char *modulePathSuffix = + moduleDeps.isSwiftModule() ? ".swiftmodule" : ".pcm"; + std::string modulePath = module.first + modulePathSuffix; + writeJSONSingleField(out, "modulePath", modulePath, /*indentLevel=*/3, + /*trailingComma=*/true); + + // Source files. + auto swiftDeps = moduleDeps.getAsSwiftModule(); + auto clangDeps = moduleDeps.getAsClangModule(); + if (swiftDeps) { + writeJSONSingleField(out, "sourceFiles", swiftDeps->sourceFiles, 3, + /*trailingComma=*/true); + } else { + writeJSONSingleField(out, "sourceFiles", clangDeps->fileDependencies, 3, + /*trailingComma=*/true); + } + + // Direct dependencies. + auto directDependencies = resolveDirectDependencies( + ctx, ModuleDependencyID(module.first, module.second), cache); + writeJSONSingleField(out, "directDependencies", directDependencies, + 3, /*trailingComma=*/true); + + // Swift and Clang-specific details. + out.indent(3 * 2); + out << "\"details\": {\n"; + out.indent(4 * 2); + if (swiftDeps) { + out << "\"swift\": {\n"; + + /// Swift interface file, if any. + if (swiftDeps->swiftInterfaceFile) { + writeJSONSingleField( + out, "moduleInterfacePath", + *swiftDeps->swiftInterfaceFile, 5, + /*trailingComma=*/true); + } + + /// Bridging header and its source file dependencies, if any. + if (swiftDeps->bridgingHeaderFile) { + writeJSONSingleField(out, "bridgingHeaderPath", + *swiftDeps->bridgingHeaderFile, 5, + /*trailingComma=*/true); + } + writeJSONSingleField(out, "bridgingSourceFiles", + swiftDeps->bridgingSourceFiles, 5, + /*trailingComma=*/false); + } else { + out << "\"clang\": {\n"; + + // Module map file. + writeJSONSingleField(out, "moduleMapPath", + clangDeps->moduleMapFile, 5, + /*trailingComma=*/false); + } + + out.indent(4 * 2); + out << "}\n"; + out.indent(3 * 2); + out << "}\n"; + + out.indent(2 * 2); + out << "}"; + + if (&module != &allModules.back()) + out << ","; + out << "\n"; + } +} + +bool swift::scanDependencies(ASTContext &Context, ModuleDecl *mainModule, + const FrontendOptions &opts) { + + std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); + std::error_code EC; + llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::F_None); + + if (out.has_error() || EC) { + Context.Diags.diagnose(SourceLoc(), diag::error_opening_output, path, + EC.message()); + out.clear_error(); + return true; + } + + // Main module file name. + auto newExt = file_types::getExtension(file_types::TY_SwiftModuleFile); + llvm::SmallString<32> mainModulePath = mainModule->getName().str(); + llvm::sys::path::replace_extension(mainModulePath, newExt); + + // Compute the dependencies of the main module. + auto mainDependencies = + ModuleDependencies::forSwiftModule(mainModulePath.str()); + { + llvm::StringSet<> alreadyAddedModules; + for (auto fileUnit : mainModule->getFiles()) { + auto sf = dyn_cast(fileUnit); + if (!sf) + continue; + + mainDependencies.addModuleDependencies(*sf, alreadyAddedModules); + } + + // Add the bridging header. + StringRef implicitHeaderPath = opts.ImplicitObjCHeaderPath; + if (!implicitHeaderPath.empty()) { + mainDependencies.addBridgingHeader(implicitHeaderPath); + } + + // If we are to import the underlying Clang module of the same name, + // add a dependency with the same name to trigger the search. + if (opts.ImportUnderlyingModule) { + mainDependencies.addModuleDependency(mainModule->getName().str(), + alreadyAddedModules); + } + } + + // Add the main module. + StringRef mainModuleName = mainModule->getNameStr(); + llvm::SetVector, + std::set> allModules; + + allModules.insert({mainModuleName, mainDependencies.getKind()}); + + // Create the module dependency cache. + ModuleDependenciesCache cache; + cache.recordDependencies(mainModuleName, std::move(mainDependencies), + ModuleDependenciesKind::Swift); + + // Explore the dependencies of every module. + for (unsigned currentModuleIdx = 0; + currentModuleIdx < allModules.size(); + ++currentModuleIdx) { + auto module = allModules[currentModuleIdx]; + auto discoveredModules = + resolveDirectDependencies(Context, module, cache); + allModules.insert(discoveredModules.begin(), discoveredModules.end()); + } + + // Write out the JSON description. + writeJSON(out, Context, cache, allModules.getArrayRef()); + + // This process succeeds regardless of whether any errors occurred. + // FIXME: We shouldn't need this, but it's masking bugs on our scanning + // logic where we don't create a fresh context when scanning Swift interfaces + // that includes their own command-line flags. + Context.Diags.resetHadAnyError(); + return false; +} diff --git a/lib/FrontendTool/ScanDependencies.h b/lib/FrontendTool/ScanDependencies.h new file mode 100644 index 0000000000000..abc8cde05cfa0 --- /dev/null +++ b/lib/FrontendTool/ScanDependencies.h @@ -0,0 +1,28 @@ +//===--- ScanDependencies.h -- Scans the dependencies of a module ------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H +#define SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H + +namespace swift { + +class ASTContext; +class FrontendOptions; +class ModuleDecl; + +/// Scans the dependencies of \c mainModule. +bool scanDependencies(ASTContext &Context, ModuleDecl *mainModule, + const FrontendOptions &opts); + +} // end namespace swift + +#endif diff --git a/lib/Serialization/SerializedModuleLoader.cpp b/lib/Serialization/SerializedModuleLoader.cpp index 3dbd972b6cec3..170237a7037d3 100644 --- a/lib/Serialization/SerializedModuleLoader.cpp +++ b/lib/Serialization/SerializedModuleLoader.cpp @@ -14,6 +14,7 @@ #include "ModuleFile.h" #include "swift/AST/ASTContext.h" #include "swift/AST/DiagnosticsSema.h" +#include "swift/AST/ModuleDependencies.h" #include "swift/Basic/Defer.h" #include "swift/Basic/FileTypes.h" #include "swift/Basic/Platform.h" @@ -346,6 +347,46 @@ std::error_code SerializedModuleLoaderBase::openModuleFile( return std::error_code(); } +llvm::ErrorOr SerializedModuleLoaderBase::scanModuleFile( + Twine modulePath) { + // Open the module file + auto &fs = *Ctx.SourceMgr.getFileSystem(); + auto moduleBuf = fs.getBufferForFile(modulePath); + if (!moduleBuf) + return moduleBuf.getError(); + + // Load the module file without validation. + std::unique_ptr loadedModuleFile; + bool isFramework = false; + serialization::ValidationInfo loadInfo = + ModuleFile::load(modulePath.str(), + std::move(moduleBuf.get()), + nullptr, + nullptr, + isFramework, loadedModuleFile, + nullptr); + + // Map the set of dependencies over to the "module dependencies". + auto dependencies = ModuleDependencies::forSwiftModule(modulePath.str()); + llvm::StringSet<> addedModuleNames; + for (const auto &dependency : loadedModuleFile->getDependencies()) { + // FIXME: Record header dependency? + if (dependency.isHeader()) + continue; + + // Find the top-level module name. + auto modulePathStr = dependency.getPrettyPrintedPath(); + StringRef moduleName = modulePathStr; + auto dotPos = moduleName.find('.'); + if (dotPos != std::string::npos) + moduleName = moduleName.slice(0, dotPos); + + dependencies.addModuleDependency(moduleName, addedModuleNames); + } + + return std::move(dependencies); +} + std::error_code SerializedModuleLoader::findModuleFilesInDirectory( AccessPathElem ModuleID, const SerializedModuleBaseName &BaseName, diff --git a/test/ScanDependencies/Inputs/CHeaders/A.h b/test/ScanDependencies/Inputs/CHeaders/A.h new file mode 100644 index 0000000000000..ea06ce27adf7f --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/A.h @@ -0,0 +1 @@ +void funcA(void); diff --git a/test/ScanDependencies/Inputs/CHeaders/B.h b/test/ScanDependencies/Inputs/CHeaders/B.h new file mode 100644 index 0000000000000..dedb1b5524f7f --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/B.h @@ -0,0 +1,4 @@ +#include + +void funcB(void); + diff --git a/test/ScanDependencies/Inputs/CHeaders/C.h b/test/ScanDependencies/Inputs/CHeaders/C.h new file mode 100644 index 0000000000000..8a5a9b119ec9d --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/C.h @@ -0,0 +1,3 @@ +#include + +void funcC(void); diff --git a/test/ScanDependencies/Inputs/CHeaders/D.h b/test/ScanDependencies/Inputs/CHeaders/D.h new file mode 100644 index 0000000000000..2b6ccbb761b9e --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/D.h @@ -0,0 +1 @@ +void funcD(void); diff --git a/test/ScanDependencies/Inputs/CHeaders/module.modulemap b/test/ScanDependencies/Inputs/CHeaders/module.modulemap new file mode 100644 index 0000000000000..45d4339198998 --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/module.modulemap @@ -0,0 +1,19 @@ +module A { + header "A.h" + export * +} + +module B { + header "B.h" + export * +} + +module C { + header "C.h" + export * +} + +module D { + header "D.h" + export * +} diff --git a/test/ScanDependencies/Inputs/Swift/A.swiftinterface b/test/ScanDependencies/Inputs/Swift/A.swiftinterface new file mode 100644 index 0000000000000..1b6292940c2bf --- /dev/null +++ b/test/ScanDependencies/Inputs/Swift/A.swiftinterface @@ -0,0 +1,5 @@ +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name A +@_exported import A +public func overlayFuncA() { } + diff --git a/test/ScanDependencies/Inputs/Swift/E.swiftinterface b/test/ScanDependencies/Inputs/Swift/E.swiftinterface new file mode 100644 index 0000000000000..824fe883f6b8d --- /dev/null +++ b/test/ScanDependencies/Inputs/Swift/E.swiftinterface @@ -0,0 +1,4 @@ +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name E +import Swift +public func funcE() { } \ No newline at end of file diff --git a/test/ScanDependencies/module_deps.swift b/test/ScanDependencies/module_deps.swift new file mode 100644 index 0000000000000..3013b1eff6c23 --- /dev/null +++ b/test/ScanDependencies/module_deps.swift @@ -0,0 +1,75 @@ +// RUN: %empty-directory(%t) +// RUN: mkdir -p %t/clang-module-cache +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift +// RUN: %FileCheck %s < %t/deps.json + +import C +import E + +// CHECK: "mainModuleName": "deps" + +/// --------Main module +// CHECK-LABEL: "modulePath": "deps.swiftmodule", +// CHECK-NEXT: sourceFiles +// CHECK-NEXT: module_deps.swift + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "C" +// CHECK-NEXT: } +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "E" + +/// --------Clang module C +// CHECK-LABEL: "modulePath": "C.pcm", + +// CHECK: "sourceFiles": [ +// CHECK-NEXT: module.modulemap +// CHECK-NEXT: C.h + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "B" + +// CHECK: "moduleMapPath" +// CHECK-SAME: module.modulemap + +/// --------Swift module E +// CHECK: "swift": "E" +// CHECK-LABEL: modulePath": "E.swiftmodule" +// CHECK: "directDependencies" +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" + +// CHECK: "moduleInterfacePath" +// CHECK-SAME: E.swiftinterface + +/// --------Clang module B +// CHECK-LABEL: "modulePath": "B.pcm" + +// CHECK-NEXT: sourceFiles +// CHECK-NEXT: B.h +// CHECK-NEXT: module.modulemap + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "A" +// CHECK-NEXT: } + +/// --------Swift module Swift +// CHECK-LABEL: "modulePath": "Swift.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "SwiftShims" + +/// --------Swift module A +// CHECK-LABEL: "modulePath": "A.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "A" +// CHECK-NEXT: } + +/// --------Clang module SwiftShims +// CHECK-LABEL: "modulePath": "SwiftShims.pcm", From b19606caa90e6bd5b32ead2893181712e2e38fe4 Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Sun, 5 Jan 2020 23:48:58 -0800 Subject: [PATCH 02/12] [Scan dependencies] Find overlays associated with imported Clang modules. --- lib/FrontendTool/ScanDependencies.cpp | 43 +++++++++++++++++++++++-- test/ScanDependencies/module_deps.swift | 16 ++++----- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/lib/FrontendTool/ScanDependencies.cpp b/lib/FrontendTool/ScanDependencies.cpp index 6388f38b560bd..538814a00a57b 100644 --- a/lib/FrontendTool/ScanDependencies.cpp +++ b/lib/FrontendTool/ScanDependencies.cpp @@ -21,17 +21,36 @@ #include "swift/Basic/Defer.h" #include "swift/Basic/LLVM.h" #include "swift/Basic/STLExtras.h" -#include "swift/ClangImporter/ClangImporter.h" #include "swift/Frontend/FrontendOptions.h" #include "clang/Basic/Module.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" #include "llvm/Support/FileSystem.h" #include using namespace swift; +/// Find all of the imported Clang modules starting with the given module name. +static void findAllImportedClangModules(ASTContext &ctx, StringRef moduleName, + ModuleDependenciesCache &cache, + std::vector &allModules, + llvm::StringSet<> &knownModules) { + if (!knownModules.insert(moduleName).second) + return; + allModules.push_back(moduleName); + + auto dependencies = cache.findDependencies( + moduleName, ModuleDependenciesKind::Clang); + if (!dependencies) + return; + + for (const auto &dep : dependencies->getModuleDependencies()) { + findAllImportedClangModules(ctx, dep, cache, allModules, knownModules); + } +} + /// Resolve the direct dependencies of the given module. static std::vector resolveDirectDependencies( ASTContext &ctx, ModuleDependencyID module, @@ -55,7 +74,25 @@ static std::vector resolveDirectDependencies( // For a Swift module, look for overlays for each of the Clang modules it // depends on. if (isSwift) { - // FIXME: Implement this. + // Find all of the Clang modules this Swift module depends on. + std::vector allClangModules; + llvm::StringSet<> knownModules; + for (const auto &dep : result) { + if (dep.second != ModuleDependenciesKind::Clang) + continue; + + findAllImportedClangModules(ctx, dep.first, cache, allClangModules, + knownModules); + } + + // Look for overlays for each of the Clang modules. + for (const auto &clangDep : allClangModules) { + if (auto found = ctx.getModuleDependencies( + clangDep, /*onlyClangModule=*/false, cache)) { + if (found->getKind() == ModuleDependenciesKind::Swift) + result.push_back({clangDep, found->getKind()}); + } + } } return result; @@ -317,7 +354,7 @@ bool swift::scanDependencies(ASTContext &Context, ModuleDecl *mainModule, writeJSON(out, Context, cache, allModules.getArrayRef()); // This process succeeds regardless of whether any errors occurred. - // FIXME: We shouldn't need this, but it's masking bugs on our scanning + // FIXME: We shouldn't need this, but it's masking bugs in our scanning // logic where we don't create a fresh context when scanning Swift interfaces // that includes their own command-line flags. Context.Diags.resetHadAnyError(); diff --git a/test/ScanDependencies/module_deps.swift b/test/ScanDependencies/module_deps.swift index 3013b1eff6c23..b6ef7ecf44e56 100644 --- a/test/ScanDependencies/module_deps.swift +++ b/test/ScanDependencies/module_deps.swift @@ -44,6 +44,14 @@ import E // CHECK: "moduleInterfacePath" // CHECK-SAME: E.swiftinterface +/// --------Swift module A +// CHECK-LABEL: "modulePath": "A.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "A" +// CHECK-NEXT: } + /// --------Clang module B // CHECK-LABEL: "modulePath": "B.pcm" @@ -63,13 +71,5 @@ import E // CHECK-NEXT: { // CHECK-NEXT: "clang": "SwiftShims" -/// --------Swift module A -// CHECK-LABEL: "modulePath": "A.swiftmodule", - -// CHECK: directDependencies -// CHECK-NEXT: { -// CHECK-NEXT: "clang": "A" -// CHECK-NEXT: } - /// --------Clang module SwiftShims // CHECK-LABEL: "modulePath": "SwiftShims.pcm", From 08ec8781aa3c5cda2adc6ab64669dd824a8a0ffa Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Mon, 6 Jan 2020 21:50:53 -0800 Subject: [PATCH 03/12] [Dependency scanning] Emit make-style dependency files. If requested, produce the normal make-style dependency file from the fast dependency scanner, using the same dependency-tracking infrastructure. --- lib/FrontendTool/FrontendTool.cpp | 12 +++++++----- lib/FrontendTool/ScanDependencies.cpp | 24 ++++++++++++++++++++++++ lib/FrontendTool/ScanDependencies.h | 2 ++ test/ScanDependencies/module_deps.swift | 12 +++++++++++- 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index df2b9ac84abad..ca671edbad0a1 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -923,10 +923,6 @@ static Optional dumpASTIfNeeded(const CompilerInvocation &Invocation, case FrontendOptions::ActionType::EmitImportedModules: emitImportedModules(Context, Instance.getMainModule(), opts); break; - - case FrontendOptions::ActionType::ScanDependencies: - scanDependencies(Context, Instance.getMainModule(), opts); - break; } return Context.hadError(); } @@ -1270,10 +1266,16 @@ static bool performCompile(CompilerInstance &Instance, if (Action == FrontendOptions::ActionType::Parse) return Context.hadError(); + if (Action == FrontendOptions::ActionType::ScanDependencies) { + scanDependencies(Context, Instance.getMainModule(), + Instance.getDependencyTracker(), opts); + } + (void)emitMakeDependenciesIfNeeded(Context.Diags, Instance.getDependencyTracker(), opts); - if (Action == FrontendOptions::ActionType::ResolveImports) + if (Action == FrontendOptions::ActionType::ResolveImports || + Action == FrontendOptions::ActionType::ScanDependencies) return Context.hadError(); if (observer) diff --git a/lib/FrontendTool/ScanDependencies.cpp b/lib/FrontendTool/ScanDependencies.cpp index 538814a00a57b..bbab2533e3974 100644 --- a/lib/FrontendTool/ScanDependencies.cpp +++ b/lib/FrontendTool/ScanDependencies.cpp @@ -283,6 +283,7 @@ static void writeJSON(llvm::raw_ostream &out, } bool swift::scanDependencies(ASTContext &Context, ModuleDecl *mainModule, + DependencyTracker *depTracker, const FrontendOptions &opts) { std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); @@ -353,6 +354,29 @@ bool swift::scanDependencies(ASTContext &Context, ModuleDecl *mainModule, // Write out the JSON description. writeJSON(out, Context, cache, allModules.getArrayRef()); + // Update the dependency tracker. + if (depTracker) { + for (auto module : allModules) { + auto deps = cache.findDependencies(module.first, module.second); + if (!deps) + continue; + + if (auto swiftDeps = deps->getAsSwiftModule()) { + if (auto swiftInterfaceFile = swiftDeps->swiftInterfaceFile) + depTracker->addDependency(*swiftInterfaceFile, /*IsSystem=*/false); + for (const auto &sourceFile : swiftDeps->sourceFiles) + depTracker->addDependency(sourceFile, /*IsSystem=*/false); + for (const auto &bridgingSourceFile : swiftDeps->bridgingSourceFiles) + depTracker->addDependency(bridgingSourceFile, /*IsSystem=*/false); + } else { + auto clangDeps = deps->getAsClangModule(); + depTracker->addDependency(clangDeps->moduleMapFile, /*IsSystem=*/false); + for (const auto &sourceFile : clangDeps->fileDependencies) + depTracker->addDependency(sourceFile, /*IsSystem=*/false); + } + } + } + // This process succeeds regardless of whether any errors occurred. // FIXME: We shouldn't need this, but it's masking bugs in our scanning // logic where we don't create a fresh context when scanning Swift interfaces diff --git a/lib/FrontendTool/ScanDependencies.h b/lib/FrontendTool/ScanDependencies.h index abc8cde05cfa0..54e47a1d15ba1 100644 --- a/lib/FrontendTool/ScanDependencies.h +++ b/lib/FrontendTool/ScanDependencies.h @@ -16,11 +16,13 @@ namespace swift { class ASTContext; +class DependencyTracker; class FrontendOptions; class ModuleDecl; /// Scans the dependencies of \c mainModule. bool scanDependencies(ASTContext &Context, ModuleDecl *mainModule, + DependencyTracker *depTracker, const FrontendOptions &opts); } // end namespace swift diff --git a/test/ScanDependencies/module_deps.swift b/test/ScanDependencies/module_deps.swift index b6ef7ecf44e56..73d66e5b4e36d 100644 --- a/test/ScanDependencies/module_deps.swift +++ b/test/ScanDependencies/module_deps.swift @@ -1,7 +1,9 @@ // RUN: %empty-directory(%t) // RUN: mkdir -p %t/clang-module-cache -// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d + // RUN: %FileCheck %s < %t/deps.json +// RUN: %FileCheck %s -check-prefix CHECK-MAKE-DEPS < %t/deps.d import C import E @@ -73,3 +75,11 @@ import E /// --------Clang module SwiftShims // CHECK-LABEL: "modulePath": "SwiftShims.pcm", + + +// Check make-style dependencies +// CHECK-MAKE-DEPS: module_deps.swift +// CHECK-MAKE-DEPS-SAME: Swift.swiftmodule +// CHECK-MAKE-DEPS-SAME: A.swiftinterface +// CHECK-MAKE-DEPS-SAME: B.h +// CHECK-MAKE-DEPS-SAME: module.modulemap From d6484c82807f91667bc84c6b04ae37dbf5772368 Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Fri, 10 Jan 2020 21:50:10 -0800 Subject: [PATCH 04/12] [Dependency scanning] Scan and record bridging header dependencies. When there is a bridging header associated with the module, scan and record its dependencies. Note them in a separate structure to capture the specific dependencies of the bridging header. --- include/swift/AST/ModuleDependencies.h | 17 ++ include/swift/ClangImporter/ClangImporter.h | 13 + lib/AST/ModuleDependencies.cpp | 27 ++ .../ClangModuleDependencyScanner.cpp | 230 +++++++++++++----- lib/FrontendTool/ScanDependencies.cpp | 58 ++++- .../Inputs/CHeaders/Bridging.h | 3 + .../Inputs/CHeaders/BridgingOther.h | 3 + test/ScanDependencies/Inputs/CHeaders/F.h | 1 + .../Inputs/CHeaders/module.modulemap | 5 + .../Inputs/Swift/F.swiftinterface | 5 + test/ScanDependencies/module_deps.swift | 24 +- 11 files changed, 309 insertions(+), 77 deletions(-) create mode 100644 test/ScanDependencies/Inputs/CHeaders/Bridging.h create mode 100644 test/ScanDependencies/Inputs/CHeaders/BridgingOther.h create mode 100644 test/ScanDependencies/Inputs/CHeaders/F.h create mode 100644 test/ScanDependencies/Inputs/Swift/F.swiftinterface diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index d1c43b337727b..fbd08a968c85a 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -71,6 +71,9 @@ class SwiftModuleDependenciesStorage : public ModuleDependenciesStorageBase { /// Source files on which the bridging header depends. std::vector bridgingSourceFiles; + /// (Clang) modules on which the bridging header depends. + std::vector bridgingModuleDependencies; + SwiftModuleDependenciesStorage( const std::string &compiledModulePath, const Optional &swiftInterfaceFile @@ -194,8 +197,18 @@ class ModuleDependencies { void addModuleDependencies(const SourceFile &sf, llvm::StringSet<> &alreadyAddedModules); + /// Get the bridging header. + Optional getBridgingHeader() const; + /// Add a bridging header to a Swift module's dependencies. void addBridgingHeader(StringRef bridgingHeader); + + /// Add source files that the bridging header depends on. + void addBridgingSourceFile(StringRef bridgingSourceFile); + + /// Add (Clang) module on which the bridging header depends. + void addBridgingModuleDependency(StringRef module, + llvm::StringSet<> &alreadyAddedModules); }; using ModuleDependencyID = std::pair; @@ -273,6 +286,10 @@ class ModuleDependenciesCache { ModuleDependencies dependencies, ModuleDependenciesKind kind); + /// Update stored dependencies for the given module. + void updateDependencies(ModuleDependencyID moduleID, + ModuleDependencies dependencies); + /// Reference the list of all module dependencies. const std::vector &getAllModules() const { return AllModules; diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h index 15932ce295d85..796c9015f4a11 100644 --- a/include/swift/ClangImporter/ClangImporter.h +++ b/include/swift/ClangImporter/ClangImporter.h @@ -372,6 +372,19 @@ class ClangImporter final : public ClangModuleLoader { Optional getModuleDependencies( StringRef moduleName, ModuleDependenciesCache &cache) override; + /// Add dependency information for the bridging header. + /// + /// \param moduleName the name of the Swift module whose dependency + /// information will be augmented with information about the given + /// bridging header. + /// + /// \param cache The module dependencies cache to update, with information + /// about new Clang modules discovered along the way. + /// + /// \returns \c true if an error occurred, \c false otherwise + bool addBridgingHeaderDependencies( + StringRef moduleName, ModuleDependenciesCache &cache); + clang::TargetInfo &getTargetInfo() const override; clang::ASTContext &getClangASTContext() const override; clang::Preprocessor &getClangPreprocessor() const override; diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index 62975dd9a1258..921ed9232e46a 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -72,12 +72,31 @@ void ModuleDependencies::addModuleDependencies( swiftStorage->sourceFiles.push_back(fileName); } +Optional ModuleDependencies::getBridgingHeader() const { + auto swiftStorage = cast(storage.get()); + return swiftStorage->bridgingHeaderFile; +} + void ModuleDependencies::addBridgingHeader(StringRef bridgingHeader) { auto swiftStorage = cast(storage.get()); assert(!swiftStorage->bridgingHeaderFile); swiftStorage->bridgingHeaderFile = bridgingHeader; } +/// Add source files that the bridging header depends on. +void ModuleDependencies::addBridgingSourceFile(StringRef bridgingSourceFile) { + auto swiftStorage = cast(storage.get()); + swiftStorage->bridgingSourceFiles.push_back(bridgingSourceFile); +} + +/// Add (Clang) module on which the bridging header depends. +void ModuleDependencies::addBridgingModuleDependency( + StringRef module, llvm::StringSet<> &alreadyAddedModules) { + auto swiftStorage = cast(storage.get()); + if (alreadyAddedModules.insert(module).second) + swiftStorage->bridgingModuleDependencies.push_back(module); +} + llvm::StringMap & ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) { switch (kind) { @@ -141,3 +160,11 @@ void ModuleDependenciesCache::recordDependencies( AllModules.push_back({moduleName, kind}); } + +void ModuleDependenciesCache::updateDependencies( + ModuleDependencyID moduleID, ModuleDependencies dependencies) { + auto &map = getDependenciesMap(moduleID.second); + auto known = map.find(moduleID.first); + assert(known != map.end() && "Not yet added to map"); + known->second = std::move(dependencies); +} diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp index 2afcdc99c2a24..4cba31feb2375 100644 --- a/lib/ClangImporter/ClangModuleDependencyScanner.cpp +++ b/lib/ClangImporter/ClangModuleDependencyScanner.cpp @@ -97,61 +97,44 @@ namespace { }; } -Optional ClangImporter::getModuleDependencies( - StringRef moduleName, ModuleDependenciesCache &cache) { - // Check whether there is already a cached result. - if (auto found = cache.findDependencies( - moduleName, ModuleDependenciesKind::Clang)) - return found; - - // Retrieve or create the shared state. - auto clangImpl = cache.getClangImpl(); - if (!clangImpl) { - clangImpl = new ClangModuleDependenciesCacheImpl(); - cache.setClangImpl(clangImpl, - [](ClangModuleDependenciesCacheImpl *ptr) { - delete ptr; - }); - } - - // Reform the Clang importer options. - // FIXME: Just save a reference or copy so we can get this back. - ClangImporterOptions importerOpts; - - // Determine the command-line arguments for dependency scanning. - auto &ctx = Impl.SwiftContext; - std::vector commandLineArgs; - commandLineArgs.push_back("clang"); - importer::getNormalInvocationArguments(commandLineArgs, ctx, importerOpts); - importer::addCommonInvocationArguments(commandLineArgs, ctx, importerOpts); - - // Add search paths. - // Note: This is handled differently for the Clang importer itself, which - // adds search paths to Clang's data structures rather than to its - // command line. +// Add search paths. +// Note: This is handled differently for the Clang importer itself, which +// adds search paths to Clang's data structures rather than to its +// command line. +static void addSearchPathInvocationArguments( + std::vector &invocationArgStrs, + ASTContext &ctx, + const ClangImporterOptions &importerOpts) { SearchPathOptions &searchPathOpts = ctx.SearchPathOpts; for (const auto &framepath : searchPathOpts.FrameworkSearchPaths) { - commandLineArgs.push_back(framepath.IsSystem ? "-iframework" : "-F"); - commandLineArgs.push_back(framepath.Path); + invocationArgStrs.push_back(framepath.IsSystem ? "-iframework" : "-F"); + invocationArgStrs.push_back(framepath.Path); } for (auto path : searchPathOpts.ImportSearchPaths) { - commandLineArgs.push_back("-I"); - commandLineArgs.push_back(path); + invocationArgStrs.push_back("-I"); + invocationArgStrs.push_back(path); } +} - // HACK! Replace the module import buffer name with the source file hack. - auto importHackFile = clangImpl->getImportHackFile(); - if (!importHackFile) { - // FIXME: Emit a diagnostic here. - return None; - } +/// Create the command line for Clang dependency scanning. +static std::vector getClangDepScanningInvocationArguments( + ASTContext &ctx, + const ClangImporterOptions &importerOpts, + StringRef sourceFileName) { + std::vector commandLineArgs; + + // Form the basic command line. + commandLineArgs.push_back("clang"); + importer::getNormalInvocationArguments(commandLineArgs, ctx, importerOpts); + importer::addCommonInvocationArguments(commandLineArgs, ctx, importerOpts); + addSearchPathInvocationArguments(commandLineArgs, ctx, importerOpts); auto sourceFilePos = std::find( commandLineArgs.begin(), commandLineArgs.end(), ""); assert(sourceFilePos != commandLineArgs.end()); - *sourceFilePos = *importHackFile; + *sourceFilePos = sourceFileName; // HACK! Drop the -fmodule-format= argument and the one that // precedes it. @@ -187,34 +170,36 @@ Optional ClangImporter::getModuleDependencies( commandLineArgs.push_back("-sys-header-deps"); commandLineArgs.push_back("-Wno-error"); - // HACK! Trick out a .m file to use to import the module we name. - std::string moduleNameHackDefine = - ("-DHACK_MODULE_NAME=" + moduleName).str(); - commandLineArgs.push_back(moduleNameHackDefine); - commandLineArgs.push_back("-fmodules-ignore-macro=HACK_MODULE_NAME"); - - std::string workingDir = - ctx.SourceMgr.getFileSystem()->getCurrentWorkingDirectory().get(); - CompileCommand command(workingDir, *importHackFile, commandLineArgs, "-"); - SingleCommandCompilationDatabase database(command); - - auto clangDependencies = clangImpl->tool.getFullDependencies( - database, workingDir, clangImpl->alreadySeen); + return commandLineArgs; +} - if (!clangDependencies) { - // FIXME: Route this to a normal diagnostic. - llvm::logAllUnhandledErrors(clangDependencies.takeError(), llvm::errs()); - return None; +/// Get or create the Clang-specific +static ClangModuleDependenciesCacheImpl *getOrCreateClangImpl( + ModuleDependenciesCache &cache) { + auto clangImpl = cache.getClangImpl(); + if (!clangImpl) { + clangImpl = new ClangModuleDependenciesCacheImpl(); + cache.setClangImpl(clangImpl, + [](ClangModuleDependenciesCacheImpl *ptr) { + delete ptr; + }); } - // Record module dependencies for each module we found. - llvm::StringSet<> alreadyAddedModules; - for (const auto &clangModuleDep : clangDependencies->DiscoveredModules) { + return clangImpl; +} + +/// Record the module dependencies we found by scanning Clang modules into +/// the module dependencies cache. +static void recordModuleDependencies( + ModuleDependenciesCache &cache, + const FullDependenciesResult &clangDependencies) { + for (const auto &clangModuleDep : clangDependencies.DiscoveredModules) { // If we've already cached this information, we're done. if (cache.hasDependencies(clangModuleDep.ModuleName, ModuleDependenciesKind::Clang)) continue; + // File dependencies for this module. std::vector fileDeps; for (const auto &fileDep : clangModuleDep.FileDeps) { fileDeps.push_back(fileDep.getKey()); @@ -223,9 +208,11 @@ Optional ClangImporter::getModuleDependencies( // Create a module filename. // FIXME: Query Clang to determine an appropriate hashed name for the // module file. - llvm::SmallString<32> modulePath = moduleName; + llvm::SmallString<32> modulePath(clangModuleDep.ModuleName); llvm::sys::path::replace_extension(modulePath, "pcm"); + // Module-level dependencies. + llvm::StringSet<> alreadyAddedModules; auto dependencies = ModuleDependencies::forClangModule( modulePath.str(), clangModuleDep.ClangModuleMapFile, fileDeps); for (const auto &moduleName : clangModuleDep.ClangModuleDeps) { @@ -236,6 +223,121 @@ Optional ClangImporter::getModuleDependencies( std::move(dependencies), ModuleDependenciesKind::Clang); } +} + +Optional ClangImporter::getModuleDependencies( + StringRef moduleName, ModuleDependenciesCache &cache) { + // Check whether there is already a cached result. + if (auto found = cache.findDependencies( + moduleName, ModuleDependenciesKind::Clang)) + return found; + + // Retrieve or create the shared state. + auto clangImpl = getOrCreateClangImpl(cache); + + // HACK! Replace the module import buffer name with the source file hack. + auto importHackFile = clangImpl->getImportHackFile(); + if (!importHackFile) { + // FIXME: Emit a diagnostic here. + return None; + } + + // Reform the Clang importer options. + // FIXME: Just save a reference or copy so we can get this back. + ClangImporterOptions importerOpts; + + // Determine the command-line arguments for dependency scanning. + auto &ctx = Impl.SwiftContext; + std::vector commandLineArgs = + getClangDepScanningInvocationArguments( + ctx, importerOpts, *importHackFile); + + // HACK! Trick out a .m file to use to import the module we name. + std::string moduleNameHackDefine = + ("-DHACK_MODULE_NAME=" + moduleName).str(); + commandLineArgs.push_back(moduleNameHackDefine); + commandLineArgs.push_back("-fmodules-ignore-macro=HACK_MODULE_NAME"); + + std::string workingDir = + ctx.SourceMgr.getFileSystem()->getCurrentWorkingDirectory().get(); + CompileCommand command(workingDir, *importHackFile, commandLineArgs, "-"); + SingleCommandCompilationDatabase database(command); + + auto clangDependencies = clangImpl->tool.getFullDependencies( + database, workingDir, clangImpl->alreadySeen); + + if (!clangDependencies) { + // FIXME: Route this to a normal diagnostic. + llvm::logAllUnhandledErrors(clangDependencies.takeError(), llvm::errs()); + return None; + } + + // Record module dependencies for each module we found. + recordModuleDependencies(cache, *clangDependencies); return cache.findDependencies(moduleName, ModuleDependenciesKind::Clang); } + +bool ClangImporter::addBridgingHeaderDependencies( + StringRef moduleName, + ModuleDependenciesCache &cache) { + auto targetModule = *cache.findDependencies( + moduleName, ModuleDependenciesKind::Swift); + + // If we've already recorded bridging header dependencies, we're done. + auto swiftDeps = targetModule.getAsSwiftModule(); + if (!swiftDeps->bridgingSourceFiles.empty() || + !swiftDeps->bridgingModuleDependencies.empty()) + return false; + + // Retrieve or create the shared state. + auto clangImpl = getOrCreateClangImpl(cache); + + // Reform the Clang importer options. + // FIXME: Just save a reference or copy so we can get this back. + ClangImporterOptions importerOpts; + + // Retrieve the bridging header. + std::string bridgingHeader = *targetModule.getBridgingHeader(); + + // Determine the command-line arguments for dependency scanning. + auto &ctx = Impl.SwiftContext; + std::vector commandLineArgs = + getClangDepScanningInvocationArguments( + ctx, importerOpts, bridgingHeader); + + std::string workingDir = + ctx.SourceMgr.getFileSystem()->getCurrentWorkingDirectory().get(); + CompileCommand command(workingDir, bridgingHeader, commandLineArgs, "-"); + SingleCommandCompilationDatabase database(command); + + auto clangDependencies = clangImpl->tool.getFullDependencies( + database, workingDir, clangImpl->alreadySeen); + + if (!clangDependencies) { + // FIXME: Route this to a normal diagnostic. + llvm::logAllUnhandledErrors(clangDependencies.takeError(), llvm::errs()); + return true; + } + + // Record module dependencies for each module we found. + recordModuleDependencies(cache, *clangDependencies); + + // Record dependencies for the source files the bridging header includes. + for (const auto &fileDep : clangDependencies->FullDeps.FileDeps) + targetModule.addBridgingSourceFile(fileDep); + + // ... and all module dependencies. + llvm::StringSet<> alreadyAddedModules; + for (const auto &moduleDep : clangDependencies->FullDeps.ClangModuleDeps) { + targetModule.addBridgingModuleDependency( + moduleDep.ModuleName, alreadyAddedModules); + } + + // Update the cache with the new information for the module. + cache.updateDependencies( + {moduleName, ModuleDependenciesKind::Swift}, + std::move(targetModule)); + + return false; +} diff --git a/lib/FrontendTool/ScanDependencies.cpp b/lib/FrontendTool/ScanDependencies.cpp index bbab2533e3974..cf8619303c52f 100644 --- a/lib/FrontendTool/ScanDependencies.cpp +++ b/lib/FrontendTool/ScanDependencies.cpp @@ -18,6 +18,7 @@ #include "swift/AST/ModuleDependencies.h" #include "swift/AST/ModuleLoader.h" #include "swift/AST/SourceFile.h" +#include "swift/ClangImporter/ClangImporter.h" #include "swift/Basic/Defer.h" #include "swift/Basic/LLVM.h" #include "swift/Basic/STLExtras.h" @@ -32,6 +33,10 @@ using namespace swift; +namespace { + +} + /// Find all of the imported Clang modules starting with the given module name. static void findAllImportedClangModules(ASTContext &ctx, StringRef moduleName, ModuleDependenciesCache &cache, @@ -71,12 +76,31 @@ static std::vector resolveDirectDependencies( } } - // For a Swift module, look for overlays for each of the Clang modules it - // depends on. if (isSwift) { - // Find all of the Clang modules this Swift module depends on. + // A record of all of the Clang modules referenced from this Swift module. std::vector allClangModules; llvm::StringSet<> knownModules; + + // If the Swift module has a bridging header, add those dependencies. + if (knownDependencies.getBridgingHeader()) { + auto clangImporter = + static_cast(ctx.getClangModuleLoader()); + if (!clangImporter->addBridgingHeaderDependencies(module.first, cache)) { + // Grab the updated module dependencies. + // FIXME: This is such a hack. + knownDependencies = *cache.findDependencies(module.first, module.second); + + // Add the Clang modules referenced from the bridging header to the + // set of Clang modules we know about. + auto swiftDeps = knownDependencies.getAsSwiftModule(); + for (const auto &clangDep : swiftDeps->bridgingModuleDependencies) { + findAllImportedClangModules(ctx, clangDep, cache, allClangModules, + knownModules); + } + } + } + + // Find all of the Clang modules this Swift module depends on. for (const auto &dep : result) { if (dep.second != ModuleDependenciesKind::Clang) continue; @@ -85,7 +109,8 @@ static std::vector resolveDirectDependencies( knownModules); } - // Look for overlays for each of the Clang modules. + // Look for overlays for each of the Clang modules. The Swift module + // directly depends on these. for (const auto &clangDep : allClangModules) { if (auto found = ctx.getModuleDependencies( clangDep, /*onlyClangModule=*/false, cache)) { @@ -201,6 +226,10 @@ static void writeJSON(llvm::raw_ostream &out, out << " ]\n"; }; for (const auto &module : allModules) { + auto directDependencies = resolveDirectDependencies( + ctx, ModuleDependencyID(module.first, module.second), cache); + + // Grab the completed module dependencies. auto moduleDeps = *cache.findDependencies(module.first, module.second); // The module we are describing. @@ -230,8 +259,6 @@ static void writeJSON(llvm::raw_ostream &out, } // Direct dependencies. - auto directDependencies = resolveDirectDependencies( - ctx, ModuleDependencyID(module.first, module.second), cache); writeJSONSingleField(out, "directDependencies", directDependencies, 3, /*trailingComma=*/true); @@ -247,18 +274,25 @@ static void writeJSON(llvm::raw_ostream &out, writeJSONSingleField( out, "moduleInterfacePath", *swiftDeps->swiftInterfaceFile, 5, - /*trailingComma=*/true); + /*trailingComma=*/swiftDeps->bridgingHeaderFile.hasValue()); } /// Bridging header and its source file dependencies, if any. if (swiftDeps->bridgingHeaderFile) { - writeJSONSingleField(out, "bridgingHeaderPath", - *swiftDeps->bridgingHeaderFile, 5, + out.indent(5 * 2); + out << "\"bridgingHeader\": {\n"; + writeJSONSingleField(out, "path", + *swiftDeps->bridgingHeaderFile, 6, /*trailingComma=*/true); + writeJSONSingleField(out, "sourceFiles", + swiftDeps->bridgingSourceFiles, 6, + /*trailingComma=*/true); + writeJSONSingleField(out, "moduleDependencies", + swiftDeps->bridgingModuleDependencies, 6, + /*trailingComma=*/false); + out.indent(5 * 2); + out << "}\n"; } - writeJSONSingleField(out, "bridgingSourceFiles", - swiftDeps->bridgingSourceFiles, 5, - /*trailingComma=*/false); } else { out << "\"clang\": {\n"; diff --git a/test/ScanDependencies/Inputs/CHeaders/Bridging.h b/test/ScanDependencies/Inputs/CHeaders/Bridging.h new file mode 100644 index 0000000000000..c0261c5d9a39b --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/Bridging.h @@ -0,0 +1,3 @@ +#include "BridgingOther.h" + +int bridging_other(void); diff --git a/test/ScanDependencies/Inputs/CHeaders/BridgingOther.h b/test/ScanDependencies/Inputs/CHeaders/BridgingOther.h new file mode 100644 index 0000000000000..ee79a8f5bc79c --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/BridgingOther.h @@ -0,0 +1,3 @@ +#include "F.h" + +int bridging_other(void); diff --git a/test/ScanDependencies/Inputs/CHeaders/F.h b/test/ScanDependencies/Inputs/CHeaders/F.h new file mode 100644 index 0000000000000..c004f15ca23a2 --- /dev/null +++ b/test/ScanDependencies/Inputs/CHeaders/F.h @@ -0,0 +1 @@ +void funcF(void); diff --git a/test/ScanDependencies/Inputs/CHeaders/module.modulemap b/test/ScanDependencies/Inputs/CHeaders/module.modulemap index 45d4339198998..0ebcb6d029092 100644 --- a/test/ScanDependencies/Inputs/CHeaders/module.modulemap +++ b/test/ScanDependencies/Inputs/CHeaders/module.modulemap @@ -17,3 +17,8 @@ module D { header "D.h" export * } + +module F { + header "F.h" + export * +} diff --git a/test/ScanDependencies/Inputs/Swift/F.swiftinterface b/test/ScanDependencies/Inputs/Swift/F.swiftinterface new file mode 100644 index 0000000000000..380d87930c853 --- /dev/null +++ b/test/ScanDependencies/Inputs/Swift/F.swiftinterface @@ -0,0 +1,5 @@ +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name F +import Swift +@_exported import F +public func funcF() { } \ No newline at end of file diff --git a/test/ScanDependencies/module_deps.swift b/test/ScanDependencies/module_deps.swift index 73d66e5b4e36d..a1b1650f16613 100644 --- a/test/ScanDependencies/module_deps.swift +++ b/test/ScanDependencies/module_deps.swift @@ -1,6 +1,6 @@ // RUN: %empty-directory(%t) // RUN: mkdir -p %t/clang-module-cache -// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h // RUN: %FileCheck %s < %t/deps.json // RUN: %FileCheck %s -check-prefix CHECK-MAKE-DEPS < %t/deps.d @@ -21,6 +21,25 @@ import E // CHECK-NEXT: } // CHECK-NEXT: { // CHECK-NEXT: "swift": "E" +// CHECK-NEXT: } +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "F" +// CHECK-NEXT: } +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "A" +// CHECK-NEXT: } + +// CHECK: "bridgingHeader": +// CHECK-NEXT: "path": +// CHECK-SAME: Bridging.h + +// CHECK-NEXT: "sourceFiles": +// CHECK-NEXT: Bridging.h +// CHECK-NEXT: BridgingOther.h + +// CHECK: "moduleDependencies": [ +// CHECK-NEXT: "F" +// CHECK-NEXT: ] /// --------Clang module C // CHECK-LABEL: "modulePath": "C.pcm", @@ -82,4 +101,7 @@ import E // CHECK-MAKE-DEPS-SAME: Swift.swiftmodule // CHECK-MAKE-DEPS-SAME: A.swiftinterface // CHECK-MAKE-DEPS-SAME: B.h +// CHECK-MAKE-DEPS-SAME: F.h +// CHECK-MAKE-DEPS-SAME: Bridging.h +// CHECK-MAKE-DEPS-SAME: BridgingOther.h // CHECK-MAKE-DEPS-SAME: module.modulemap From 5f953dad371b58be426d43715f37584fc638ada1 Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Fri, 10 Jan 2020 22:22:00 -0800 Subject: [PATCH 05/12] [Dependency scanning] Parse JSON output into Swift data structures. --- .../Inputs/ModuleDependencyGraph.swift | 147 ++++++++++++++++++ test/ScanDependencies/module_deps.swift | 13 ++ 2 files changed, 160 insertions(+) create mode 100644 test/ScanDependencies/Inputs/ModuleDependencyGraph.swift diff --git a/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift b/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift new file mode 100644 index 0000000000000..1eda0710e965c --- /dev/null +++ b/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift @@ -0,0 +1,147 @@ +//===--------------- ModuleDependencyGraph.swift --------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import Foundation + +enum ModuleDependencyId: Hashable { + case swift(String) + case clang(String) + + var moduleName: String { + switch self { + case .swift(let name): return name + case .clang(let name): return name + } + } +} + +extension ModuleDependencyId: Codable { + enum CodingKeys: CodingKey { + case swift + case clang + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + do { + let moduleName = try container.decode(String.self, forKey: .swift) + self = .swift(moduleName) + } catch { + let moduleName = try container.decode(String.self, forKey: .clang) + self = .clang(moduleName) + } + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .swift(let moduleName): + try container.encode(moduleName, forKey: .swift) + case .clang(let moduleName): + try container.encode(moduleName, forKey: .clang) + } + } +} + +/// Bridging header +struct BridgingHeader: Codable { + var path: String + var sourceFiles: [String] + var moduleDependencies: [String] +} + +/// Details specific to Swift modules. +struct SwiftModuleDetails: Codable { + /// The module interface from which this module was built, if any. + var moduleInterfacePath: String? + + /// The bridging header, if any. + var bridgingHeader: BridgingHeader? +} + +/// Details specific to Clang modules. +struct ClangModuleDetails: Codable { + /// The path to the module map used to build this module. + var moduleMapPath: String +} + +struct ModuleDependencies: Codable { + /// The path for the module. + var modulePath: String + + /// The source files used to build this module. + var sourceFiles: [String] = [] + + /// The set of direct module dependencies of this module. + var directDependencies: [ModuleDependencyId] = [] + + /// Specific details of a particular kind of module. + var details: Details + + /// Specific details of a particular kind of module. + enum Details { + /// Swift modules may be built from a module interface, and may have + /// a bridging header. + case swift(SwiftModuleDetails) + + /// Clang modules are built from a module map file. + case clang(ClangModuleDetails) + } +} + +extension ModuleDependencies.Details: Codable { + enum CodingKeys: CodingKey { + case swift + case clang + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + do { + let details = try container.decode(SwiftModuleDetails.self, forKey: .swift) + self = .swift(details) + } catch { + let details = try container.decode(ClangModuleDetails.self, forKey: .clang) + self = .clang(details) + } + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .swift(let details): + try container.encode(details, forKey: .swift) + case .clang(let details): + try container.encode(details, forKey: .clang) + } + } +} + +/// Describes the complete set of dependencies for a Swift module, including +/// all of the Swift and C modules and source files it depends on. +struct ModuleDependencyGraph: Codable { + /// The name of the main module. + var mainModuleName: String + + /// The complete set of modules discovered + var modules: [ModuleDependencyId: ModuleDependencies] = [:] + + /// Information about the main module. + var mainModule: ModuleDependencies { modules[.swift(mainModuleName)]! } +} + +let fileName = CommandLine.arguments[1] +let data = try! Data(contentsOf: URL(fileURLWithPath: fileName)) + +let decoder = JSONDecoder() +let moduleDependencyGraph = try! decoder.decode( + ModuleDependencyGraph.self, from: data) +print(moduleDependencyGraph) diff --git a/test/ScanDependencies/module_deps.swift b/test/ScanDependencies/module_deps.swift index a1b1650f16613..dc4edc8a9532a 100644 --- a/test/ScanDependencies/module_deps.swift +++ b/test/ScanDependencies/module_deps.swift @@ -2,9 +2,22 @@ // RUN: mkdir -p %t/clang-module-cache // RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h +// Check the contents of the JSON output // RUN: %FileCheck %s < %t/deps.json + +// Check the make-style dependencies file // RUN: %FileCheck %s -check-prefix CHECK-MAKE-DEPS < %t/deps.d +// Check that the JSON parses correctly into the canonical Swift data +// structures. + +// RUN: %target-build-swift %S/Inputs/ModuleDependencyGraph.swift -o %t/main +// RUN: %target-codesign %t/main +// RUN: %target-run %t/main %t/deps.json + +// REQUIRES: executable_test +// REQUIRES: objc_interop + import C import E From 71e7d6a9134332332952f10260970a44eb7696f3 Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Fri, 10 Jan 2020 22:55:33 -0800 Subject: [PATCH 06/12] [Dependency scanning] Add implicit Swift and SwiftOnoneSupport dependencies --- include/swift/Frontend/Frontend.h | 3 ++ .../ArgsToFrontendOptionsConverter.cpp | 6 +++ lib/Frontend/Frontend.cpp | 11 ++--- lib/Frontend/ModuleDependencyScanner.cpp | 3 +- lib/FrontendTool/FrontendTool.cpp | 3 +- lib/FrontendTool/ScanDependencies.cpp | 48 +++++++++++++++---- lib/FrontendTool/ScanDependencies.h | 11 ++--- test/ScanDependencies/module_deps.swift | 20 +++++--- 8 files changed, 72 insertions(+), 33 deletions(-) diff --git a/include/swift/Frontend/Frontend.h b/include/swift/Frontend/Frontend.h index 5624ae0df631b..c58c0cf297407 100644 --- a/include/swift/Frontend/Frontend.h +++ b/include/swift/Frontend/Frontend.h @@ -346,6 +346,9 @@ class CompilerInvocation { return ImplicitStdlibKind::Stdlib; } + /// Whether the Swift -Onone support library should be implicitly imported. + bool shouldImportSwiftONoneSupport() const; + /// Performs input setup common to these tools: /// sil-opt, sil-func-extractor, sil-llvm-gen, and sil-nm. /// Return value includes the buffer so caller can keep it alive. diff --git a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp index c87fef7b3557a..85c44e7b837a1 100644 --- a/lib/Frontend/ArgsToFrontendOptionsConverter.cpp +++ b/lib/Frontend/ArgsToFrontendOptionsConverter.cpp @@ -83,6 +83,12 @@ bool ArgsToFrontendOptionsConverter::convert( Opts.TrackSystemDeps |= Args.hasArg(OPT_track_system_dependencies); + // Always track system dependencies when scanning dependencies. + if (const Arg *ModeArg = Args.getLastArg(OPT_modes_Group)) { + if (ModeArg->getOption().matches(OPT_scan_dependencies)) + Opts.TrackSystemDeps = true; + } + Opts.SerializeModuleInterfaceDependencyHashes |= Args.hasArg(OPT_serialize_module_interface_dependency_hashes); diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index 14d1f82a7097f..b1f292ec5a02d 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -687,11 +687,10 @@ std::unique_ptr CompilerInstance::takeSILModule() { /// builds. This allows for use of popular specialized functions /// from the standard library, which makes the non-optimized builds /// execute much faster. -static bool shouldImplicityImportSwiftOnoneSupportModule( - const CompilerInvocation &Invocation) { - if (Invocation.getImplicitStdlibKind() != ImplicitStdlibKind::Stdlib) +bool CompilerInvocation::shouldImportSwiftONoneSupport() const { + if (getImplicitStdlibKind() != ImplicitStdlibKind::Stdlib) return false; - if (Invocation.getSILOptions().shouldOptimize()) + if (getSILOptions().shouldOptimize()) return false; // If we are not executing an action that has a dependency on @@ -706,7 +705,7 @@ static bool shouldImplicityImportSwiftOnoneSupportModule( // // This optimization is disabled by -track-system-dependencies to preserve // the explicit dependency. - const auto &options = Invocation.getFrontendOptions(); + const auto &options = getFrontendOptions(); return options.TrackSystemDeps || FrontendOptions::doesActionGenerateSIL(options.RequestedAction); } @@ -720,7 +719,7 @@ ImplicitImportInfo CompilerInstance::getImplicitImportInfo() const { for (auto &moduleStr : frontendOpts.getImplicitImportModuleNames()) imports.ModuleNames.push_back(Context->getIdentifier(moduleStr)); - if (shouldImplicityImportSwiftOnoneSupportModule(Invocation)) + if (Invocation.shouldImportSwiftONoneSupport()) imports.ModuleNames.push_back(Context->getIdentifier(SWIFT_ONONE_SUPPORT)); imports.ShouldImportUnderlyingModule = frontendOpts.ImportUnderlyingModule; diff --git a/lib/Frontend/ModuleDependencyScanner.cpp b/lib/Frontend/ModuleDependencyScanner.cpp index ea93494ee8654..19cd6c3a701ed 100644 --- a/lib/Frontend/ModuleDependencyScanner.cpp +++ b/lib/Frontend/ModuleDependencyScanner.cpp @@ -104,8 +104,7 @@ ErrorOr ModuleDependencyScanner::scanInterfaceFile( unsigned bufferID = Ctx.SourceMgr.addNewSourceBuffer(std::move(interfaceBuf.get())); auto moduleDecl = ModuleDecl::create(moduleName, Ctx); auto sourceFile = new (Ctx) SourceFile( - *moduleDecl, SourceFileKind::Interface, bufferID, - SourceFile::ImplicitModuleImportKind::None); + *moduleDecl, SourceFileKind::Interface, bufferID); // Create a module filename. // FIXME: Query the module interface loader to determine an appropriate diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index ca671edbad0a1..b80a03e8ae5a8 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -1267,8 +1267,7 @@ static bool performCompile(CompilerInstance &Instance, return Context.hadError(); if (Action == FrontendOptions::ActionType::ScanDependencies) { - scanDependencies(Context, Instance.getMainModule(), - Instance.getDependencyTracker(), opts); + scanDependencies(Instance); } (void)emitMakeDependenciesIfNeeded(Context.Diags, diff --git a/lib/FrontendTool/ScanDependencies.cpp b/lib/FrontendTool/ScanDependencies.cpp index cf8619303c52f..a99efead7f846 100644 --- a/lib/FrontendTool/ScanDependencies.cpp +++ b/lib/FrontendTool/ScanDependencies.cpp @@ -22,7 +22,9 @@ #include "swift/Basic/Defer.h" #include "swift/Basic/LLVM.h" #include "swift/Basic/STLExtras.h" +#include "swift/Frontend/Frontend.h" #include "swift/Frontend/FrontendOptions.h" +#include "swift/Strings.h" #include "clang/Basic/Module.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/StringMap.h" @@ -316,9 +318,11 @@ static void writeJSON(llvm::raw_ostream &out, } } -bool swift::scanDependencies(ASTContext &Context, ModuleDecl *mainModule, - DependencyTracker *depTracker, - const FrontendOptions &opts) { +bool swift::scanDependencies(CompilerInstance &instance) { + ASTContext &Context = instance.getASTContext(); + ModuleDecl *mainModule = instance.getMainModule(); + const CompilerInvocation &invocation = instance.getInvocation(); + const FrontendOptions &opts = invocation.getFrontendOptions(); std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); std::error_code EC; @@ -349,15 +353,43 @@ bool swift::scanDependencies(ASTContext &Context, ModuleDecl *mainModule, mainDependencies.addModuleDependencies(*sf, alreadyAddedModules); } + const auto &importInfo = mainModule->getImplicitImportInfo(); + + // Swift standard library. + switch (importInfo.StdlibKind) { + case ImplicitStdlibKind::None: + case ImplicitStdlibKind::Builtin: + break; + + case ImplicitStdlibKind::Stdlib: + mainDependencies.addModuleDependency("Swift", alreadyAddedModules); + break; + } + + // Swift -Onone support library. + if (invocation.shouldImportSwiftONoneSupport()) { + mainDependencies.addModuleDependency( + SWIFT_ONONE_SUPPORT, alreadyAddedModules); + } + + // Add any implicit module names. + for (const auto &moduleName : importInfo.ModuleNames) { + mainDependencies.addModuleDependency(moduleName.str(), alreadyAddedModules); + } + + // Already-loaded, implicitly imported module names. + for (const auto &module : importInfo.AdditionalModules) { + mainDependencies.addModuleDependency(module.first->getNameStr(), alreadyAddedModules); + } + // Add the bridging header. - StringRef implicitHeaderPath = opts.ImplicitObjCHeaderPath; - if (!implicitHeaderPath.empty()) { - mainDependencies.addBridgingHeader(implicitHeaderPath); + if (!importInfo.BridgingHeaderPath.empty()) { + mainDependencies.addBridgingHeader(importInfo.BridgingHeaderPath); } // If we are to import the underlying Clang module of the same name, // add a dependency with the same name to trigger the search. - if (opts.ImportUnderlyingModule) { + if (importInfo.ShouldImportUnderlyingModule) { mainDependencies.addModuleDependency(mainModule->getName().str(), alreadyAddedModules); } @@ -389,7 +421,7 @@ bool swift::scanDependencies(ASTContext &Context, ModuleDecl *mainModule, writeJSON(out, Context, cache, allModules.getArrayRef()); // Update the dependency tracker. - if (depTracker) { + if (auto depTracker = instance.getDependencyTracker()) { for (auto module : allModules) { auto deps = cache.findDependencies(module.first, module.second); if (!deps) diff --git a/lib/FrontendTool/ScanDependencies.h b/lib/FrontendTool/ScanDependencies.h index 54e47a1d15ba1..ece8515097608 100644 --- a/lib/FrontendTool/ScanDependencies.h +++ b/lib/FrontendTool/ScanDependencies.h @@ -15,15 +15,10 @@ namespace swift { -class ASTContext; -class DependencyTracker; -class FrontendOptions; -class ModuleDecl; +class CompilerInstance; -/// Scans the dependencies of \c mainModule. -bool scanDependencies(ASTContext &Context, ModuleDecl *mainModule, - DependencyTracker *depTracker, - const FrontendOptions &opts); +/// Scans the dependencies of the main module of \c instance. +bool scanDependencies(CompilerInstance &instance); } // end namespace swift diff --git a/test/ScanDependencies/module_deps.swift b/test/ScanDependencies/module_deps.swift index dc4edc8a9532a..2bd5d99e77674 100644 --- a/test/ScanDependencies/module_deps.swift +++ b/test/ScanDependencies/module_deps.swift @@ -36,6 +36,12 @@ import E // CHECK-NEXT: "swift": "E" // CHECK-NEXT: } // CHECK-NEXT: { +// CHECK-NEXT: "swift": "Swift" +// CHECK-NEXT: } +// CHECK-NEXT: { +// CHECK-NEXT: "swift": "SwiftOnoneSupport" +// CHECK-NEXT: } +// CHECK-NEXT: { // CHECK-NEXT: "swift": "F" // CHECK-NEXT: } // CHECK-NEXT: { @@ -78,6 +84,13 @@ import E // CHECK: "moduleInterfacePath" // CHECK-SAME: E.swiftinterface +/// --------Swift module Swift +// CHECK-LABEL: "modulePath": "Swift.swiftmodule", + +// CHECK: directDependencies +// CHECK-NEXT: { +// CHECK-NEXT: "clang": "SwiftShims" + /// --------Swift module A // CHECK-LABEL: "modulePath": "A.swiftmodule", @@ -98,13 +111,6 @@ import E // CHECK-NEXT: "clang": "A" // CHECK-NEXT: } -/// --------Swift module Swift -// CHECK-LABEL: "modulePath": "Swift.swiftmodule", - -// CHECK: directDependencies -// CHECK-NEXT: { -// CHECK-NEXT: "clang": "SwiftShims" - /// --------Clang module SwiftShims // CHECK-LABEL: "modulePath": "SwiftShims.pcm", From a4b0e822555b32479a071acc65476f34e8d6be13 Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Fri, 10 Jan 2020 23:19:18 -0800 Subject: [PATCH 07/12] [Dependency scanning] Cope with inferred module maps --- lib/FrontendTool/ScanDependencies.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/FrontendTool/ScanDependencies.cpp b/lib/FrontendTool/ScanDependencies.cpp index a99efead7f846..4e9b40278f78c 100644 --- a/lib/FrontendTool/ScanDependencies.cpp +++ b/lib/FrontendTool/ScanDependencies.cpp @@ -436,7 +436,8 @@ bool swift::scanDependencies(CompilerInstance &instance) { depTracker->addDependency(bridgingSourceFile, /*IsSystem=*/false); } else { auto clangDeps = deps->getAsClangModule(); - depTracker->addDependency(clangDeps->moduleMapFile, /*IsSystem=*/false); + if (!clangDeps->moduleMapFile.empty()) + depTracker->addDependency(clangDeps->moduleMapFile, /*IsSystem=*/false); for (const auto &sourceFile : clangDeps->fileDependencies) depTracker->addDependency(sourceFile, /*IsSystem=*/false); } From 702c38e91e7af43240b727e04ee4f336d64ca57a Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Sat, 28 Mar 2020 23:46:19 -0700 Subject: [PATCH 08/12] Sink the module dependency scanner down into the Serialization library --- lib/Frontend/CMakeLists.txt | 1 - lib/Serialization/CMakeLists.txt | 1 + lib/{Frontend => Serialization}/ModuleDependencyScanner.cpp | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename lib/{Frontend => Serialization}/ModuleDependencyScanner.cpp (100%) diff --git a/lib/Frontend/CMakeLists.txt b/lib/Frontend/CMakeLists.txt index 1b3daf94db1c1..0cb3c72b15b96 100644 --- a/lib/Frontend/CMakeLists.txt +++ b/lib/Frontend/CMakeLists.txt @@ -8,7 +8,6 @@ add_swift_host_library(swiftFrontend STATIC Frontend.cpp FrontendInputsAndOutputs.cpp FrontendOptions.cpp - ModuleDependencyScanner.cpp ModuleInterfaceBuilder.cpp ModuleInterfaceLoader.cpp ModuleInterfaceSupport.cpp diff --git a/lib/Serialization/CMakeLists.txt b/lib/Serialization/CMakeLists.txt index ebeb4ab7d3003..90e73ee5ed30c 100644 --- a/lib/Serialization/CMakeLists.txt +++ b/lib/Serialization/CMakeLists.txt @@ -1,6 +1,7 @@ add_swift_host_library(swiftSerialization STATIC Deserialization.cpp DeserializeSIL.cpp + ModuleDependencyScanner.cpp ModuleFile.cpp Serialization.cpp SerializedModuleLoader.cpp diff --git a/lib/Frontend/ModuleDependencyScanner.cpp b/lib/Serialization/ModuleDependencyScanner.cpp similarity index 100% rename from lib/Frontend/ModuleDependencyScanner.cpp rename to lib/Serialization/ModuleDependencyScanner.cpp From fdfadf83636cacd181f289d274c14d6638f59d8d Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Mon, 30 Mar 2020 22:50:49 -0700 Subject: [PATCH 09/12] [Dependency scanner] Get the Clang module file name from Clang. --- lib/ClangImporter/ClangModuleDependencyScanner.cpp | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp index 4cba31feb2375..b164dd74d56df 100644 --- a/lib/ClangImporter/ClangModuleDependencyScanner.cpp +++ b/lib/ClangImporter/ClangModuleDependencyScanner.cpp @@ -205,16 +205,11 @@ static void recordModuleDependencies( fileDeps.push_back(fileDep.getKey()); } - // Create a module filename. - // FIXME: Query Clang to determine an appropriate hashed name for the - // module file. - llvm::SmallString<32> modulePath(clangModuleDep.ModuleName); - llvm::sys::path::replace_extension(modulePath, "pcm"); - // Module-level dependencies. llvm::StringSet<> alreadyAddedModules; auto dependencies = ModuleDependencies::forClangModule( - modulePath.str(), clangModuleDep.ClangModuleMapFile, fileDeps); + clangModuleDep.ImplicitModulePCMPath, + clangModuleDep.ClangModuleMapFile, fileDeps); for (const auto &moduleName : clangModuleDep.ClangModuleDeps) { dependencies.addModuleDependency(moduleName.ModuleName, alreadyAddedModules); } @@ -333,7 +328,7 @@ bool ClangImporter::addBridgingHeaderDependencies( targetModule.addBridgingModuleDependency( moduleDep.ModuleName, alreadyAddedModules); } - + // Update the cache with the new information for the module. cache.updateDependencies( {moduleName, ModuleDependenciesKind::Swift}, From 6b5e7efc0e81f10ac34a8eed6312d5435754bdca Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Mon, 30 Mar 2020 23:37:24 -0700 Subject: [PATCH 10/12] [Dependency scanner] Capture Clang context hash and command-line arguments. --- include/swift/AST/ModuleDependencies.h | 18 +++++++++++++++--- .../ClangModuleDependencyScanner.cpp | 5 ++++- lib/FrontendTool/ScanDependencies.cpp | 10 ++++++++++ .../Inputs/ModuleDependencyGraph.swift | 7 +++++++ test/ScanDependencies/module_deps.swift | 6 ++++++ 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index fbd08a968c85a..83691866ba1a9 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -94,16 +94,26 @@ class ClangModuleDependenciesStorage : public ModuleDependenciesStorageBase { /// The module map file used to generate the Clang module. const std::string moduleMapFile; + /// The context hash describing the configuration options for this module. + const std::string contextHash; + + /// Partial (Clang) command line that can be used to build this module. + const std::vector nonPathCommandLine; + /// The file dependencies const std::vector fileDependencies; ClangModuleDependenciesStorage( const std::string &compiledModulePath, const std::string &moduleMapFile, + const std::string &contextHash, + const std::vector &nonPathCommandLine, const std::vector &fileDependencies ) : ModuleDependenciesStorageBase(/*isSwiftModule=*/false, compiledModulePath), moduleMapFile(moduleMapFile), + contextHash(contextHash), + nonPathCommandLine(nonPathCommandLine), fileDependencies(fileDependencies) { } ModuleDependenciesStorageBase *clone() const override { @@ -158,11 +168,13 @@ class ModuleDependencies { static ModuleDependencies forClangModule( const std::string &compiledModulePath, const std::string &moduleMapFile, + const std::string &contextHash, + const std::vector &nonPathCommandLine, const std::vector &fileDependencies) { return ModuleDependencies( - std::make_unique(compiledModulePath, - moduleMapFile, - fileDependencies)); + std::make_unique( + compiledModulePath, moduleMapFile, contextHash, nonPathCommandLine, + fileDependencies)); } /// Retrieve the path to the compiled module. diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp index b164dd74d56df..097b6d35aaa34 100644 --- a/lib/ClangImporter/ClangModuleDependencyScanner.cpp +++ b/lib/ClangImporter/ClangModuleDependencyScanner.cpp @@ -209,7 +209,10 @@ static void recordModuleDependencies( llvm::StringSet<> alreadyAddedModules; auto dependencies = ModuleDependencies::forClangModule( clangModuleDep.ImplicitModulePCMPath, - clangModuleDep.ClangModuleMapFile, fileDeps); + clangModuleDep.ClangModuleMapFile, + clangModuleDep.ContextHash, + clangModuleDep.NonPathCommandLine, + fileDeps); for (const auto &moduleName : clangModuleDep.ClangModuleDeps) { dependencies.addModuleDependency(moduleName.ModuleName, alreadyAddedModules); } diff --git a/lib/FrontendTool/ScanDependencies.cpp b/lib/FrontendTool/ScanDependencies.cpp index 4e9b40278f78c..a5a266b0a5e30 100644 --- a/lib/FrontendTool/ScanDependencies.cpp +++ b/lib/FrontendTool/ScanDependencies.cpp @@ -301,6 +301,16 @@ static void writeJSON(llvm::raw_ostream &out, // Module map file. writeJSONSingleField(out, "moduleMapPath", clangDeps->moduleMapFile, 5, + /*trailingComma=*/true); + + // Context hash. + writeJSONSingleField(out, "contextHash", + clangDeps->contextHash, 5, + /*trailingComma=*/true); + + // Command line. + writeJSONSingleField(out, "commandLine", + clangDeps->nonPathCommandLine, 5, /*trailingComma=*/false); } diff --git a/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift b/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift index 1eda0710e965c..127126b58aeae 100644 --- a/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift +++ b/test/ScanDependencies/Inputs/ModuleDependencyGraph.swift @@ -71,6 +71,13 @@ struct SwiftModuleDetails: Codable { struct ClangModuleDetails: Codable { /// The path to the module map used to build this module. var moduleMapPath: String + + /// The context hash used to discriminate this module file. + var contextHash: String + + /// The Clang command line arguments that need to be passed through + /// to the -emit-pcm action to build this module. + var commandLine: [String] = [] } struct ModuleDependencies: Codable { diff --git a/test/ScanDependencies/module_deps.swift b/test/ScanDependencies/module_deps.swift index 2bd5d99e77674..a4a5f6a424f5c 100644 --- a/test/ScanDependencies/module_deps.swift +++ b/test/ScanDependencies/module_deps.swift @@ -74,6 +74,12 @@ import E // CHECK: "moduleMapPath" // CHECK-SAME: module.modulemap +// CHECK: "contextHash" +// CHECK-SAME: "{{.*}}" + +// CHECK: "commandLine": [ +// CHECK-NEXT: "-remove-preceeding-explicit-module-build-incompatible-options" + /// --------Swift module E // CHECK: "swift": "E" // CHECK-LABEL: modulePath": "E.swiftmodule" From 012878630ad2e7dec6efde9857d37604e4556f5e Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Fri, 24 Apr 2020 12:55:02 -0700 Subject: [PATCH 11/12] [Dependency scanning] Improve overview documentation somewhat. --- include/swift/AST/ModuleDependencies.h | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index 83691866ba1a9..5e086bfb7e5a5 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -2,7 +2,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -10,7 +10,9 @@ // //===----------------------------------------------------------------------===// // -// This file defines data structures for capturing module dependencies. +// This file defines data structures for capturing all of the source files +// and modules on which a given module depends, forming a graph of all of the +// modules that need to be present for a given module to be built. // //===----------------------------------------------------------------------===// #ifndef SWIFT_AST_MODULE_DEPENDENCIES_H @@ -36,6 +38,8 @@ enum class ModuleDependenciesKind : int8_t { }; /// Base class for the variant storage of ModuleDependencies. +/// +/// This class is mostly an implementation detail for \c ModuleDependencies. class ModuleDependenciesStorageBase { public: const bool isSwiftModule; @@ -57,6 +61,8 @@ class ModuleDependenciesStorageBase { }; /// Describes the dependencies of a Swift module. +/// +/// This class is mostly an implementation detail for \c ModuleDependencies. class SwiftModuleDependenciesStorage : public ModuleDependenciesStorageBase { public: /// The Swift interface file, if it can be used to generate the module file. @@ -89,6 +95,9 @@ class SwiftModuleDependenciesStorage : public ModuleDependenciesStorageBase { } }; +/// Describes the dependencies of a Clang module. +/// +/// This class is mostly an implementation detail for \c ModuleDependencies. class ClangModuleDependenciesStorage : public ModuleDependenciesStorageBase { public: /// The module map file used to generate the Clang module. @@ -126,6 +135,10 @@ class ClangModuleDependenciesStorage : public ModuleDependenciesStorageBase { }; /// Describes the dependencies of a given module. +/// +/// The dependencies of a module include all of the source files that go +/// into that module, as well as any modules that are directly imported +/// into the module. class ModuleDependencies { private: std::unique_ptr storage; From 5351d543a253100b72c896bda9a8e01b98c64f74 Mon Sep 17 00:00:00 2001 From: Doug Gregor Date: Fri, 24 Apr 2020 12:57:30 -0700 Subject: [PATCH 12/12] Update test to reflect unimportant change in output results --- test/ScanDependencies/module_deps.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ScanDependencies/module_deps.swift b/test/ScanDependencies/module_deps.swift index a4a5f6a424f5c..2a74953e3d26d 100644 --- a/test/ScanDependencies/module_deps.swift +++ b/test/ScanDependencies/module_deps.swift @@ -123,8 +123,8 @@ import E // Check make-style dependencies // CHECK-MAKE-DEPS: module_deps.swift -// CHECK-MAKE-DEPS-SAME: Swift.swiftmodule // CHECK-MAKE-DEPS-SAME: A.swiftinterface +// CHECK-MAKE-DEPS-SAME: Swift.swiftmodule // CHECK-MAKE-DEPS-SAME: B.h // CHECK-MAKE-DEPS-SAME: F.h // CHECK-MAKE-DEPS-SAME: Bridging.h