diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td index ceb69091b2a51..f965eda7ebf2a 100644 --- a/clang/include/clang/Basic/DiagnosticDriverKinds.td +++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td @@ -587,6 +587,24 @@ def remark_found_cxx20_module_usage : Remark< def remark_performing_driver_managed_module_build : Remark< "performing driver managed module build">, InGroup; +def remark_modules_manifest_not_found : Remark< + "standard modules manifest file not found; import of standard library " + "modules not supported">, + InGroup; +def remark_using_modules_manifest : Remark< + "using standard modules manifest file '%0'">, + InGroup; +def err_modules_manifest_failed_parse : Error< + "failure while parsing standard modules manifest: '%0'">; +def err_failed_dependency_scan : Error< + "failed to perform dependency scan">; +def err_mod_graph_named_module_redefinition : Error< + "duplicate definitions of C++20 named module '%0' in '%1' and '%2'">; +def err_building_dependency_graph : Error< + "failed to construct the module dependency graph">; +def remark_printing_module_graph : Remark< + "printing module dependency graph">, + InGroup; def warn_drv_delayed_template_parsing_after_cxx20 : Warning< "-fdelayed-template-parsing is deprecated after C++20">, diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h index b9b187ada8add..4d32552b7f85f 100644 --- a/clang/include/clang/Driver/Driver.h +++ b/clang/include/clang/Driver/Driver.h @@ -512,9 +512,6 @@ class Driver { /// BuildActions - Construct the list of actions to perform for the /// given arguments, which are only done for a single architecture. - /// If the compilation is an explicit module build, delegates to - /// BuildDriverManagedModuleBuildActions. Otherwise, BuildDefaultActions is - /// used. /// /// \param C - The compilation that is being built. /// \param Args - The input arguments. @@ -799,35 +796,6 @@ class Driver { /// compilation based on which -f(no-)?lto(=.*)? option occurs last. void setLTOMode(const llvm::opt::ArgList &Args); - /// BuildDefaultActions - Constructs the list of actions to perform - /// for the provided arguments, which are only done for a single architecture. - /// - /// \param C - The compilation that is being built. - /// \param Args - The input arguments. - /// \param Actions - The list to store the resulting actions onto. - void BuildDefaultActions(Compilation &C, llvm::opt::DerivedArgList &Args, - const InputList &Inputs, ActionList &Actions) const; - - /// BuildDriverManagedModuleBuildActions - Performs a dependency - /// scan and constructs the list of actions to perform for dependency order - /// and the provided arguments. This is only done for a single a architecture. - /// - /// \param C - The compilation that is being built. - /// \param Args - The input arguments. - /// \param Actions - The list to store the resulting actions onto. - void BuildDriverManagedModuleBuildActions(Compilation &C, - llvm::opt::DerivedArgList &Args, - const InputList &Inputs, - ActionList &Actions) const; - - /// Scans the leading lines of the C++ source inputs to detect C++20 module - /// usage. - /// - /// \returns True if module usage is detected, false otherwise, or an error on - /// read failure. - llvm::ErrorOr - ScanInputsForCXX20ModulesUsage(const InputList &Inputs) const; - /// Retrieves a ToolChain for a particular \p Target triple. /// /// Will cache ToolChains for the life of the driver object, and create them diff --git a/clang/include/clang/Driver/Job.h b/clang/include/clang/Driver/Job.h index 561866197b780..9e24c3ddc54b4 100644 --- a/clang/include/clang/Driver/Job.h +++ b/clang/include/clang/Driver/Job.h @@ -221,6 +221,8 @@ class Command { const char *getExecutable() const { return Executable; } + llvm::opt::ArgStringList &getArguments() { return Arguments; } + const llvm::opt::ArgStringList &getArguments() const { return Arguments; } const std::vector &getInputInfos() const { return InputInfoList; } @@ -279,6 +281,9 @@ class JobList { const list_type &getJobs() const { return Jobs; } + // Returns and transfers ownership of all jobs, leaving this list empty. + list_type takeJobs(); + bool empty() const { return Jobs.empty(); } size_type size() const { return Jobs.size(); } iterator begin() { return Jobs.begin(); } diff --git a/clang/include/clang/Driver/ModulesDriver.h b/clang/include/clang/Driver/ModulesDriver.h new file mode 100644 index 0000000000000..3947e92a1de0f --- /dev/null +++ b/clang/include/clang/Driver/ModulesDriver.h @@ -0,0 +1,112 @@ +//===- ModulesDriver.h - Driver managed module builds --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines functionality to support driver managed builds for +/// compilations which use Clang modules or standard C++20 named modules. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_DRIVER_MODULESDRIVER_H +#define LLVM_CLANG_DRIVER_MODULESDRIVER_H + +#include "clang/Driver/Types.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace vfs { +class FileSystem; +} // namespace vfs +namespace opt { +class Arg; +} // namespace opt +} // namespace llvm + +namespace clang { +class DiagnosticsEngine; +namespace driver { +class Compilation; +} // namespace driver +} // namespace clang + +namespace clang::driver::modules { + +/// A list of inputs and their types for the given arguments. +/// Identical to Driver::InputTy. +using InputTy = std::pair; + +/// A list of inputs and their types for the given arguments. +/// Identical to Driver::InputList. +using InputList = llvm::SmallVector; + +/// Checks whether the -fmodules-driver feature should be implicitly enabled. +/// +/// The modules driver should be enabled if both: +/// 1) the compilation has more than two C++ source inputs; and +/// 2) any C++ source inputs uses C++20 named modules. +/// +/// \param Inputs The input list for the compilation being built. +/// \param VFS The virtual file system to use for all reads. +/// \param Diags The diagnostics engine used for emitting remarks only. +/// +/// \returns True if the modules driver should be enabled, false otherwise, +/// or a llvm::FileError on failure to read a source input. +llvm::Expected shouldUseModulesDriver(const InputList &Inputs, + llvm::vfs::FileSystem &VFS, + DiagnosticsEngine &Diags); + +/// The parsed Standard library module manifest. +struct StdModuleManifest { + struct LocalModuleArgs { + std::vector SystemIncludeDirs; + }; + + struct Module { + bool IsStdlib = false; + std::string LogicalName; + std::string SourcePath; + std::optional LocalArgs; + }; + + std::vector ModuleEntries; +}; + +/// Reads the Standard library module manifest from the specified path. +/// +/// All source file paths in the returned manifest are made absolute. +/// +/// \param StdModuleManifestPath Path to the manifest file. +/// \param VFS The llvm::vfs::FileSystem to be used for all file accesses. +/// +/// \returns The parsed manifest on success, a llvm::FileError on failure to +/// read the manifest, or a llvm::json::ParseError on failure to parse it. +llvm::Expected +readStdModuleManifest(llvm::StringRef StdModuleManifestPath, + llvm::vfs::FileSystem &VFS); + +/// Constructs compilation inputs for each module listed in the provided +/// Standard library module manifest. +/// +/// \param Manifest The standard modules manifest +/// \param C The compilation being built. +/// \param Inputs The input list to which the corresponding input entries are +/// appended. +void buildStdModuleManifestInputs(const StdModuleManifest &Manifest, + Compilation &C, InputList &Inputs); + +/// Reorders and builds compilation jobs based on discovered module +/// dependencies. +/// +/// \param C The compilation being built. +/// \param Manifest The standard modules manifest +void planDriverManagedModuleCompilation(Compilation &C, + const StdModuleManifest &Manifest); + +} // namespace clang::driver::modules + +#endif diff --git a/clang/include/clang/Lex/DependencyDirectivesScanner.h b/clang/include/clang/Lex/DependencyDirectivesScanner.h index c0b742d652a03..bf88ea9ea9de1 100644 --- a/clang/include/clang/Lex/DependencyDirectivesScanner.h +++ b/clang/include/clang/Lex/DependencyDirectivesScanner.h @@ -140,7 +140,7 @@ void printDependencyDirectivesAsSource( /// \param Source The input source buffer. /// /// \returns true if any C++20 named modules related directive was found. -bool scanInputForCXX20ModulesUsage(StringRef Source); +bool scanInputForCXXNamedModulesUsage(StringRef Source); /// Functor that returns the dependency directives for a given file. class DependencyDirectivesGetter { diff --git a/clang/lib/Driver/CMakeLists.txt b/clang/lib/Driver/CMakeLists.txt index 7c4f70b966c48..cef0c058fc2c1 100644 --- a/clang/lib/Driver/CMakeLists.txt +++ b/clang/lib/Driver/CMakeLists.txt @@ -21,6 +21,7 @@ add_clang_library(clangDriver Driver.cpp DriverOptions.cpp Job.cpp + ModulesDriver.cpp Multilib.cpp MultilibBuilder.cpp OffloadBundler.cpp @@ -99,5 +100,6 @@ add_clang_library(clangDriver LINK_LIBS clangBasic clangLex + clangDependencyScanning ${system_libs} ) diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index f110dbab3e5a5..259e149cb0ed3 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -60,6 +60,7 @@ #include "clang/Driver/Compilation.h" #include "clang/Driver/InputInfo.h" #include "clang/Driver/Job.h" +#include "clang/Driver/ModulesDriver.h" #include "clang/Driver/Options.h" #include "clang/Driver/Phases.h" #include "clang/Driver/SanitizerArgs.h" @@ -87,6 +88,7 @@ #include "llvm/Support/FileSystem.h" #include "llvm/Support/FileUtilities.h" #include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/JSON.h" #include "llvm/Support/MD5.h" #include "llvm/Support/Path.h" #include "llvm/Support/PrettyStackTrace.h" @@ -1829,6 +1831,59 @@ Compilation *Driver::BuildCompilation(ArrayRef ArgList) { // Populate the tool chains for the offloading devices, if any. CreateOffloadingDeviceToolChains(*C, Inputs); + modules::StdModuleManifest Manifest; + if (C->getArgs().hasFlag(options::OPT_fmodules_driver, + options::OPT_fno_modules_driver, false)) { + Diags.Report(diag::remark_performing_driver_managed_module_build); + // TODO: When -fmodules-driver is no longer experimental, allow implicit + // activation of the modules driver. For now, keep the detection of whether + // the modules driver should be enabled here for diagnostics only, and do + // not implicitly enable the feature. + auto EnableOrErr = modules::shouldUseModulesDriver(Inputs, getVFS(), Diags); + if (!EnableOrErr) { + llvm::handleAllErrors( + EnableOrErr.takeError(), [&](const llvm::FileError &Err) { + Diags.Report(diag::err_cannot_open_file) + << Err.getFileName() << Err.messageWithoutFileInfo(); + }); + return C; + } + + // Read the standard modules manifest, and if available, add all discovered + // modules to the compilation. Compilation jobs for modules discovered from + // the manifest, which are not imported by any other source input, are + // pruned later. + const auto StdModuleManifestPath = + GetStdModuleManifestPath(*C, C->getDefaultToolChain()); + if (!llvm::sys::fs::exists(StdModuleManifestPath)) { + Diags.Report(diag::remark_modules_manifest_not_found); + } else { + Diags.Report(diag::remark_using_modules_manifest) + << StdModuleManifestPath; + if (auto ManifestOrErr = + modules::readStdModuleManifest(StdModuleManifestPath, getVFS())) { + Manifest = std::move(*ManifestOrErr); + } else { + llvm::handleAllErrors( + ManifestOrErr.takeError(), + [&](llvm::json::ParseError &Err) { + Diags.Report(diag::err_modules_manifest_failed_parse) + << Err.message(); + }, + [&](llvm::FileError &Err) { + Diags.Report(diag::err_cannot_open_file) + << Err.getFileName() << Err.messageWithoutFileInfo(); + }); + } + } + // Only allow on-demand imports of standard library modules. + llvm::erase_if(Manifest.ModuleEntries, [](const auto &ModuleEntry) { + return !ModuleEntry.IsStdlib; + }); + + modules::buildStdModuleManifestInputs(Manifest, *C, Inputs); + } + // Construct the list of abstract actions to perform for this compilation. On // MachO targets this uses the driver-driver and universal actions. if (TC.getTriple().isOSBinFormatMachO()) @@ -1843,6 +1898,11 @@ Compilation *Driver::BuildCompilation(ArrayRef ArgList) { BuildJobs(*C); + if (C->getArgs().hasFlag(options::OPT_fmodules_driver, + options::OPT_fno_modules_driver, false)) { + modules::planDriverManagedModuleCompilation(*C, Manifest); + } + return C; } @@ -4320,33 +4380,6 @@ void Driver::handleArguments(Compilation &C, DerivedArgList &Args, } } -static bool hasCXXModuleInputType(const Driver::InputList &Inputs) { - const auto IsTypeCXXModule = [](const auto &Input) -> bool { - const auto TypeID = Input.first; - return (TypeID == types::TY_CXXModule); - }; - return llvm::any_of(Inputs, IsTypeCXXModule); -} - -llvm::ErrorOr -Driver::ScanInputsForCXX20ModulesUsage(const InputList &Inputs) const { - const auto CXXInputs = llvm::make_filter_range( - Inputs, [](const auto &Input) { return types::isCXX(Input.first); }); - for (const auto &Input : CXXInputs) { - StringRef Filename = Input.second->getSpelling(); - auto ErrOrBuffer = VFS->getBufferForFile(Filename); - if (!ErrOrBuffer) - return ErrOrBuffer.getError(); - const auto Buffer = std::move(*ErrOrBuffer); - - if (scanInputForCXX20ModulesUsage(Buffer->getBuffer())) { - Diags.Report(diag::remark_found_cxx20_module_usage) << Filename; - return true; - } - } - return false; -} - void Driver::BuildActions(Compilation &C, DerivedArgList &Args, const InputList &Inputs, ActionList &Actions) const { llvm::PrettyStackTraceString CrashInfo("Building compilation actions"); @@ -4358,33 +4391,6 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args, handleArguments(C, Args, Inputs, Actions); - if (Args.hasFlag(options::OPT_fmodules_driver, - options::OPT_fno_modules_driver, false)) { - // TODO: Move the logic for implicitly enabling explicit-module-builds out - // of -fmodules-driver once it is no longer experimental. - // Currently, this serves diagnostic purposes only. - bool UsesCXXModules = hasCXXModuleInputType(Inputs); - if (!UsesCXXModules) { - const auto ErrOrScanResult = ScanInputsForCXX20ModulesUsage(Inputs); - if (!ErrOrScanResult) { - Diags.Report(diag::err_cannot_open_file) - << ErrOrScanResult.getError().message(); - return; - } - UsesCXXModules = *ErrOrScanResult; - } - if (UsesCXXModules || Args.hasArg(options::OPT_fmodules)) - BuildDriverManagedModuleBuildActions(C, Args, Inputs, Actions); - return; - } - - BuildDefaultActions(C, Args, Inputs, Actions); -} - -void Driver::BuildDefaultActions(Compilation &C, DerivedArgList &Args, - const InputList &Inputs, - ActionList &Actions) const { - bool UseNewOffloadingDriver = C.isOffloadingHostKind(Action::OFK_OpenMP) || C.isOffloadingHostKind(Action::OFK_SYCL) || @@ -4680,12 +4686,6 @@ void Driver::BuildDefaultActions(Compilation &C, DerivedArgList &Args, Args.ClaimAllArgs(options::OPT_cl_ignored_Group); } -void Driver::BuildDriverManagedModuleBuildActions( - Compilation &C, llvm::opt::DerivedArgList &Args, const InputList &Inputs, - ActionList &Actions) const { - Diags.Report(diag::remark_performing_driver_managed_module_build); -} - /// Returns the canonical name for the offloading architecture when using a HIP /// or CUDA architecture. static StringRef getCanonicalArchString(Compilation &C, diff --git a/clang/lib/Driver/Job.cpp b/clang/lib/Driver/Job.cpp index 880e9e396c41e..8270370575a71 100644 --- a/clang/lib/Driver/Job.cpp +++ b/clang/lib/Driver/Job.cpp @@ -453,3 +453,5 @@ void JobList::Print(raw_ostream &OS, const char *Terminator, bool Quote, } void JobList::clear() { Jobs.clear(); } + +JobList::list_type JobList::takeJobs() { return std::exchange(Jobs, {}); } diff --git a/clang/lib/Driver/ModulesDriver.cpp b/clang/lib/Driver/ModulesDriver.cpp new file mode 100644 index 0000000000000..521173038bd9f --- /dev/null +++ b/clang/lib/Driver/ModulesDriver.cpp @@ -0,0 +1,1520 @@ +//===--- Driver.cpp - Clang GCC Compatible Driver -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines functionality to support driver managed builds for +/// compilations which use Clang modules or standard C++20 named modules. +/// +//===----------------------------------------------------------------------===// + +#include "clang/Driver/ModulesDriver.h" +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/DiagnosticDriver.h" +#include "clang/Basic/SourceManager.h" +#include "clang/Driver/Compilation.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/Job.h" +#include "clang/Driver/Options.h" +#include "clang/Driver/Tool.h" +#include "clang/Lex/DependencyDirectivesScanner.h" +#include "clang/Lex/Lexer.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/DirectedGraph.h" +#include "llvm/ADT/GraphTraits.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallBitVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/TypeSwitch.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/Option.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/DOTGraphTraits.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/GraphWriter.h" +#include "llvm/Support/JSON.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/ThreadPool.h" +#include "llvm/Support/VirtualFileSystem.h" +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace llvm::opt; + +namespace clang::driver::modules { +using JobVector = JobList::list_type; + +// The tooling::deps namespace has conflicting names with clang::driver, we +// therefore introduce only the required tooling::deps namespace members into +// this namespace. +using tooling::dependencies::DependencyActionController; +using tooling::dependencies::DependencyScanningService; +using tooling::dependencies::DependencyScanningWorker; +using tooling::dependencies::FullDependencyConsumer; +using tooling::dependencies::ModuleDeps; +using tooling::dependencies::ModuleDepsGraph; +using tooling::dependencies::ModuleID; +using tooling::dependencies::ModuleOutputKind; +using tooling::dependencies::ScanningMode; +using tooling::dependencies::ScanningOutputFormat; +using tooling::dependencies::TranslationUnitDeps; + +/// Returns true if any source input signals C++ module usage. +static bool hasCXXNamedModuleInput(const InputList &Inputs) { + const auto IsTypeCXXModule = [](const auto &Input) -> bool { + const auto TypeID = Input.first; + return (TypeID == types::TY_CXXModule || TypeID == types::TY_PP_CXXModule); + }; + return any_of(Inputs, IsTypeCXXModule); +} + +/// Scan the leading lines of each C++ source file until C++20 named module +/// usage is detected. +/// +/// \returns true if module usage is detected, false otherwise, or a +/// llvm::FileError on read failure. +static Expected scanForCXXNamedModuleUsage(const InputList &Inputs, + llvm::vfs::FileSystem &VFS, + DiagnosticsEngine &Diags) { + const auto CXXInputs = make_filter_range( + Inputs, [](const InputTy &Input) { return types::isCXX(Input.first); }); + for (const auto &Input : CXXInputs) { + auto Filename = Input.second->getSpelling(); + auto MemBufOrErr = VFS.getBufferForFile(Filename); + if (!MemBufOrErr) + return llvm::createFileError(Filename, MemBufOrErr.getError()); + const auto MemBuf = std::move(*MemBufOrErr); + + // Scan the buffer using the dependency directives scanner. + if (clang::scanInputForCXXNamedModulesUsage(MemBuf->getBuffer())) { + Diags.Report(diag::remark_found_cxx20_module_usage) << Filename; + return true; + } + } + return false; +} + +Expected shouldUseModulesDriver(const InputList &Inputs, + llvm::vfs::FileSystem &FS, + DiagnosticsEngine &Diags) { + if (Inputs.size() < 2) + return false; + if (hasCXXNamedModuleInput(Inputs)) + return true; + return scanForCXXNamedModuleUsage(Inputs, FS, Diags); +} + +static bool fromJSON(const llvm::json::Value &Params, + StdModuleManifest::LocalModuleArgs &LocalArgs, + llvm::json::Path P) { + llvm::json::ObjectMapper O(Params, P); + return O.mapOptional("system-include-directories", + LocalArgs.SystemIncludeDirs); +} + +static bool fromJSON(const llvm::json::Value &Params, + StdModuleManifest::Module &ModuleEntry, + llvm::json::Path P) { + llvm::json::ObjectMapper O(Params, P); + return O.map("is-std-library", ModuleEntry.IsStdlib) && + O.map("logical-name", ModuleEntry.LogicalName) && + O.map("source-path", ModuleEntry.SourcePath) && + O.mapOptional("local-arguments", ModuleEntry.LocalArgs); +} + +static bool fromJSON(const llvm::json::Value &Params, + StdModuleManifest &Manifest, llvm::json::Path P) { + llvm::json::ObjectMapper O(Params, P); + return O.map("modules", Manifest.ModuleEntries); +} + +/// Parses the Standard library module manifest from \c Buffer. +/// +/// The source file paths listed in the manifest are relative to its own +/// path. +static Expected parseStdModuleManifest(StringRef Buffer) { + auto ParsedJsonOrErr = llvm::json::parse(Buffer); + if (!ParsedJsonOrErr) + return ParsedJsonOrErr.takeError(); + + StdModuleManifest Manifest; + llvm::json::Path::Root Root; + if (!fromJSON(*ParsedJsonOrErr, Manifest, Root)) + return Root.getError(); + + return Manifest; +} + +/// Converts all file paths in \c Manifest from paths relative to +/// \c ManifestPath (the manifest's location itself) to absolute. +static void makeStdModuleManifestPathsAbsolute(StdModuleManifest &Manifest, + StringRef ManifestPath) { + SmallString<124> ManifestDir(ManifestPath); + llvm::sys::path::remove_filename(ManifestDir); + + SmallString<256> TempPath; + auto ensureAbsolutePath = [&](std::string &Path) { + if (llvm::sys::path::is_absolute(Path)) + return; + TempPath = ManifestDir; + llvm::sys::path::append(TempPath, Path); + llvm::sys::path::remove_dots(TempPath, true); + Path = std::string(TempPath); + }; + + for (auto &ModuleEntry : Manifest.ModuleEntries) { + ensureAbsolutePath(ModuleEntry.SourcePath); + if (!ModuleEntry.LocalArgs) + continue; + for (auto &IncludeDir : ModuleEntry.LocalArgs->SystemIncludeDirs) + ensureAbsolutePath(IncludeDir); + } +} + +Expected readStdModuleManifest(StringRef ManifestPath, + llvm::vfs::FileSystem &VFS) { + auto MemBufOrErr = VFS.getBufferForFile(ManifestPath); + if (!MemBufOrErr) + return llvm::createFileError(ManifestPath, MemBufOrErr.getError()); + const auto MemBuf = std::move(*MemBufOrErr); + + auto ManifestOrErr = parseStdModuleManifest(MemBuf->getBuffer()); + if (!ManifestOrErr) + return ManifestOrErr.takeError(); + auto Manifest = std::move(*ManifestOrErr); + + // All paths in the manifest are relative to \c ManifestPath. + // Make them absolute. + makeStdModuleManifestPathsAbsolute(Manifest, ManifestPath); + + return Manifest; +} + +/// Appends a compilation input for the given \c Entry of the Standard library +/// module manifest. +static void +appendStdModuleManifestInput(const StdModuleManifest::Module &ModuleEntry, + Compilation &C, InputList &Inputs) { + auto &Args = C.getArgs(); + const auto &Opts = C.getDriver().getOpts(); + + C.getDriver().DiagnoseInputExistence(Args, ModuleEntry.SourcePath, + types::TY_CXXModule, + /*TypoCorrect=*/false); + + auto *A = new Arg(Opts.getOption(options::OPT_INPUT), ModuleEntry.SourcePath, + Args.getBaseArgs().MakeIndex(ModuleEntry.SourcePath), + Args.getBaseArgs().MakeArgString(ModuleEntry.SourcePath)); + Args.AddSynthesizedArg(A); + A->claim(); + Inputs.emplace_back(types::TY_CXXModule, A); +} + +void buildStdModuleManifestInputs(const StdModuleManifest &Manifest, + Compilation &C, InputList &Inputs) { + for (const auto &Module : Manifest.ModuleEntries) + appendStdModuleManifestInput(Module, C, Inputs); +} + +namespace { +/// Represents a CharSourceRange within a StandaloneDiagnostic. +struct SourceOffsetRange { + SourceOffsetRange(CharSourceRange Range, const SourceManager &SrcMgr, + const LangOptions &LangOpts); + unsigned Begin = 0; + unsigned End = 0; + bool IsTokenRange = false; +}; + +/// Represents a FixItHint within a StandaloneDiagnostic. +struct StandaloneFixIt { + StandaloneFixIt(const SourceManager &SrcMgr, const LangOptions &LangOpts, + const FixItHint &FixIt); + + SourceOffsetRange RemoveRange; + SourceOffsetRange InsertFromRange; + std::string CodeToInsert; + bool BeforePreviousInsertions = false; +}; + +/// Represents a StoredDiagnostic in a form that can be retained until after its +/// SourceManager has been destroyed. +/// +/// Source locations are stored as a combination of filename and offsets into +/// that file. +/// To report the diagnostic, it must first be translated back into a +/// StoredDiagnostic with a new associated SourceManager. +struct StandaloneDiagnostic { + explicit StandaloneDiagnostic(const StoredDiagnostic &StoredDiag); + + LangOptions LangOpts; + SrcMgr::CharacteristicKind FileKind; + DiagnosticsEngine::Level Level; + unsigned ID = 0; + unsigned FileOffset = 0; + std::string Filename; + std::string Message; + SmallVector Ranges; + SmallVector FixIts; +}; + +using StandaloneDiagList = SmallVector; +} // anonymous namespace + +SourceOffsetRange::SourceOffsetRange(CharSourceRange Range, + const SourceManager &SrcMgr, + const LangOptions &LangOpts) + : IsTokenRange(Range.isTokenRange()) { + const auto FileRange = Lexer::makeFileCharRange(Range, SrcMgr, LangOpts); + Begin = SrcMgr.getFileOffset(FileRange.getBegin()); + End = SrcMgr.getFileOffset(FileRange.getEnd()); +} + +StandaloneFixIt::StandaloneFixIt(const SourceManager &SrcMgr, + const LangOptions &LangOpts, + const FixItHint &FixIt) + : RemoveRange(FixIt.RemoveRange, SrcMgr, LangOpts), + InsertFromRange(FixIt.InsertFromRange, SrcMgr, LangOpts), + CodeToInsert(FixIt.CodeToInsert), + BeforePreviousInsertions(FixIt.BeforePreviousInsertions) {} + +/// If a custom working directory is set for \c SrcMgr, returns the absolute +/// path of \c Filename to make it independent. Otherwise, returns the original +/// string. +static std::string canonicalizeFilename(const SourceManager &SrcMgr, + StringRef Filename) { + SmallString<256> Abs(Filename); + if (!llvm::sys::path::is_absolute(Abs)) { + if (const auto &CWD = + SrcMgr.getFileManager().getFileSystemOpts().WorkingDir; + !CWD.empty()) + llvm::sys::fs::make_absolute(CWD, Abs); + } + return std::string(Abs.str()); +} + +// FIXME: LangOpts is not properly saved because the LangOptions is not +// copyable! clang/lib/Frontend/SerializedDiagnosticPrinter.cpp does currently +// not serialize LangOpts either. +StandaloneDiagnostic::StandaloneDiagnostic(const StoredDiagnostic &StoredDiag) + : Level(StoredDiag.getLevel()), ID(StoredDiag.getID()), + Message(StoredDiag.getMessage()) { + const FullSourceLoc &FullLoc = StoredDiag.getLocation(); + // This is not an invalid diagnostic; invalid SourceLocations are used to + // represent diagnostics without a specific SourceLocation. + if (FullLoc.isInvalid()) + return; + + const auto &SrcMgr = FullLoc.getManager(); + FileKind = SrcMgr.getFileCharacteristic(static_cast(FullLoc)); + const auto FileLoc = SrcMgr.getFileLoc(static_cast(FullLoc)); + FileOffset = SrcMgr.getFileOffset(FileLoc); + const auto PathRef = SrcMgr.getFilename(FileLoc); + assert(!PathRef.empty() && "diagnostic with location has no source file?"); + Filename = canonicalizeFilename(SrcMgr, PathRef); + + Ranges.reserve(StoredDiag.getRanges().size()); + for (const auto &Range : StoredDiag.getRanges()) + Ranges.emplace_back(Range, SrcMgr, LangOpts); + + FixIts.reserve(StoredDiag.getFixIts().size()); + for (const auto &FixIt : StoredDiag.getFixIts()) + FixIts.emplace_back(SrcMgr, LangOpts, FixIt); +} + +/// Translates \c StandaloneDiag into a StoredDiagnostic, associating it with +/// the provided FileManager and SourceManager. +/// +/// This allows the diagnostic to be emitted using the diagnostics engine, since +/// StandaloneDiagnostics themselfs cannot be emitted directly. +static StoredDiagnostic +translateStandaloneDiag(FileManager &FileMgr, SourceManager &SrcMgr, + StandaloneDiagnostic &&StandaloneDiag) { + const auto FileRef = FileMgr.getOptionalFileRef(StandaloneDiag.Filename); + if (!FileRef) + return StoredDiagnostic(StandaloneDiag.Level, StandaloneDiag.ID, + std::move(StandaloneDiag.Message)); + + const auto FileID = + SrcMgr.getOrCreateFileID(*FileRef, StandaloneDiag.FileKind); + const auto FileLoc = SrcMgr.getLocForStartOfFile(FileID); + assert(FileLoc.isValid() && "StandaloneDiagnostic should only use FilePath " + "for encoding a valid source location."); + const auto DiagLoc = FileLoc.getLocWithOffset(StandaloneDiag.FileOffset); + const FullSourceLoc Loc(DiagLoc, SrcMgr); + + auto ConvertOffsetRange = [&](const SourceOffsetRange &Range) { + return CharSourceRange(SourceRange(FileLoc.getLocWithOffset(Range.Begin), + FileLoc.getLocWithOffset(Range.End)), + Range.IsTokenRange); + }; + + SmallVector TranslatedRanges; + TranslatedRanges.reserve(StandaloneDiag.Ranges.size()); + transform(StandaloneDiag.Ranges, std::back_inserter(TranslatedRanges), + ConvertOffsetRange); + + SmallVector TranslatedFixIts; + TranslatedFixIts.reserve(StandaloneDiag.FixIts.size()); + for (const auto &FixIt : StandaloneDiag.FixIts) { + FixItHint TranslatedFixIt; + TranslatedFixIt.CodeToInsert = std::string(FixIt.CodeToInsert); + TranslatedFixIt.RemoveRange = ConvertOffsetRange(FixIt.RemoveRange); + TranslatedFixIt.InsertFromRange = ConvertOffsetRange(FixIt.InsertFromRange); + TranslatedFixIt.BeforePreviousInsertions = FixIt.BeforePreviousInsertions; + TranslatedFixIts.push_back(std::move(TranslatedFixIt)); + } + + return StoredDiagnostic(StandaloneDiag.Level, StandaloneDiag.ID, + StandaloneDiag.Message, Loc, TranslatedRanges, + TranslatedFixIts); +} + +namespace { +/// RAII utility to report StandaloneDiagnostics through a DiagnosticsEngine. +/// +/// The driver's DiagnosticsEngine usually does not have a SourceManager at this +/// point in building the compilation, in which case the StandaloneDiagReporter +/// supplies its own. +class StandaloneDiagReporter { +public: + explicit StandaloneDiagReporter(DiagnosticsEngine &Diags) : Diags(Diags) { + if (!Diags.hasSourceManager()) { + FileSystemOptions Opts; + Opts.WorkingDir = "."; + OwnedFileMgr = llvm::makeIntrusiveRefCnt(std::move(Opts)); + OwnedSrcMgr = + llvm::makeIntrusiveRefCnt(Diags, *OwnedFileMgr); + } + } + + /// Emits \c StandaloneDiag using the associated DiagnosticsEngine. + void Report(StandaloneDiagnostic &&StandaloneDiag) const { + const auto StoredDiag = translateStandaloneDiag( + getFileManager(), getSourceManager(), std::move(StandaloneDiag)); + Diags.getClient()->BeginSourceFile(StandaloneDiag.LangOpts, nullptr); + Diags.Report(StoredDiag); + Diags.getClient()->EndSourceFile(); + } + + /// Emits all diagnostics in \c StandaloneDiags using the associated + /// DiagnosticsEngine. + void Report(SmallVectorImpl &&StandaloneDiags) const { + for (auto &StandaloneDiag : StandaloneDiags) + Report(std::move(StandaloneDiag)); + } + +private: + DiagnosticsEngine &Diags; + IntrusiveRefCntPtr OwnedFileMgr; + IntrusiveRefCntPtr OwnedSrcMgr; + + FileManager &getFileManager() const { + if (OwnedFileMgr) + return *OwnedFileMgr; + return Diags.getSourceManager().getFileManager(); + } + + SourceManager &getSourceManager() const { + if (OwnedSrcMgr) + return *OwnedSrcMgr; + return Diags.getSourceManager(); + } +}; + +/// Collects diagnostics in a form that can be retained until after their +/// associated SourceManager is destroyed. +class StandaloneDiagCollector : public DiagnosticConsumer { +public: + void BeginSourceFile(const LangOptions &LangOpts, + const Preprocessor *PP = nullptr) override {} + + void HandleDiagnostic(DiagnosticsEngine::Level Level, + const Diagnostic &Info) override { + StoredDiagnostic StoredDiag(Level, Info); + StandaloneDiags.emplace_back(StoredDiag); + DiagnosticConsumer::HandleDiagnostic(Level, Info); + } + + void EndSourceFile() override {} + + StandaloneDiagList takeDiagnostics() { return std::move(StandaloneDiags); } + +private: + StandaloneDiagList StandaloneDiags; +}; +} // anonymous namespace + +namespace { +/// The full dependencies for a single compilation input. +struct InputDependencies { + /// The identifier of the C++20 module this translation unit exports. + /// + /// If the translation unit is not a module then \c ID.ModuleName is empty. + ModuleID ID; + + /// Whether this is a "system" module. + bool IsSystem; + + /// A collection of absolute paths to files that this translation unit + /// directly depends on, not including transitive dependencies. + std::vector FileDeps; + + /// A list of modules this translation unit directly depends on, not including + /// transitive dependencies. + /// + /// This may include modules with a different context hash when it can be + /// determined that the differences are benign for this compilation. + std::vector ClangModuleDeps; + + /// A list of the C++20 named modules this translation unit depends on. + std::vector NamedModuleDeps; + + /// The compiler invocation with modifications to properly import all Clang + /// module dependencies. Does not include argv[0]. + std::vector BuildArgs; +}; + +/// The full dependencies for each compilation input and for all discovered +/// Clang modules. +struct DependencyScanResult { + /// The full dependencies for each compilation input, in the same order as the + /// inputs. + /// + /// System modules inputs that are not imported are represented as + /// std::nullopt. + llvm::SmallVector> InputDeps; + + /// The full Clang module dependenies for this compilation. + SmallVector ClangModuleDeps; +}; + +/// Merges and deterministically orders scan results from multiple threads +/// into a single DependencyScanResult. +class ScanResultCollector { +public: + explicit ScanResultCollector(size_t NumInputs) + : InputDeps(NumInputs), ClangModuleGraphs(NumInputs) {} + + /// Adds the dependency scan result for the input at \c InputIndex. + /// + /// Thread safe, given that each index is written to exactly once. + void handleTUResult(TranslationUnitDeps &&TUDeps, bool IsSystem, + size_t InputIndex); + + /// Finalizes and takes the aggregated results. + /// + /// Not thread-safe. + DependencyScanResult takeScanResults(); + +private: + SmallVector, 0> InputDeps; + llvm::SmallVector, 0> ClangModuleGraphs; +}; +} // anonymous namespace + +void ScanResultCollector::handleTUResult(TranslationUnitDeps &&TUDeps, + bool IsSystem, size_t InputIndex) { + assert(!InputDeps[InputIndex].has_value() && + "Each slot should be written to at most once."); + InputDeps[InputIndex].emplace(); + auto &NewInputDep = InputDeps[InputIndex].value(); + NewInputDep.ID = std::move(TUDeps.ID); + NewInputDep.IsSystem = IsSystem; + NewInputDep.FileDeps = std::move(TUDeps.FileDeps); + NewInputDep.NamedModuleDeps = std::move(TUDeps.NamedModuleDeps); + NewInputDep.ClangModuleDeps = std::move(TUDeps.ClangModuleDeps); + assert(TUDeps.Commands.size() == 1 && "Expected exactly one command"); + NewInputDep.BuildArgs = TUDeps.Commands.front().Arguments; + + assert(ClangModuleGraphs[InputIndex].empty() && + "Each slot should be written to at most once."); + ClangModuleGraphs[InputIndex] = std::move(TUDeps.ModuleGraph); +} + +DependencyScanResult ScanResultCollector::takeScanResults() { + DependencyScanResult Out; + + // Record each module only once, from its first importing input. + // This keeps the output deterministic. + llvm::DenseSet AlreadySeen; + for (auto &ModuleGraph : ClangModuleGraphs) { + for (auto &MD : ModuleGraph) { + auto [It, Inserted] = AlreadySeen.insert(MD.ID); + if (!Inserted) + continue; + Out.ClangModuleDeps.push_back(std::move(MD)); + } + } + + Out.InputDeps = std::move(InputDeps); + return Out; +} + +namespace { +/// Pool of reusable dependency scanning workers and their contexts, with RAII +/// acquire/release semantics. +class ScanningWorkerPool { +public: + ScanningWorkerPool(size_t NumWorkers, DependencyScanningService &S, + llvm::vfs::FileSystem &FS); + + /// Acquires a unique pointer to a dependency scanning worker and its + /// context. + /// + /// The worker bundle automatically released back to the pool when the + /// pointer is destroyed. The pool has to outlive the leased worker bundle. + [[nodiscard]] auto scopedAcquire(); + +private: + /// Releases the worker bundle at \c Index back into the pool. + void release(size_t Index); + + /// A scanning worker with its associated context. + struct WorkerBundle { + WorkerBundle(DependencyScanningService &S, llvm::vfs::FileSystem &FS) + : Worker(S, &FS) {} + + DependencyScanningWorker Worker; + llvm::DenseSet SeenModules; + }; + + std::mutex Lock; + std::condition_variable CV; + SmallVector AvailableSlots; + SmallVector Slots; +}; +} // anonymous namespace + +ScanningWorkerPool::ScanningWorkerPool(size_t NumWorkers, + DependencyScanningService &S, + llvm::vfs::FileSystem &FS) { + Slots.reserve(NumWorkers); + for (size_t I = 0; I < NumWorkers; ++I) + Slots.emplace_back(S, FS); + + AvailableSlots.resize(NumWorkers); + std::iota(AvailableSlots.begin(), AvailableSlots.end(), 0); +} + +[[nodiscard]] auto ScanningWorkerPool::scopedAcquire() { + std::unique_lock UL(Lock); + CV.wait(UL, [&] { return !AvailableSlots.empty(); }); + const auto Index = AvailableSlots.pop_back_val(); + auto ReleaseHandle = [this, Index](WorkerBundle *) { release(Index); }; + return std::unique_ptr(&Slots[Index], + ReleaseHandle); +} + +void ScanningWorkerPool::release(size_t Index) { + { + std::scoped_lock SL(Lock); + AvailableSlots.push_back(Index); + } + CV.notify_one(); +} + +namespace { +/// Thread-safe registry of system modules. +/// +/// Records which system modules have been scheduled, and provides lookup for +/// input indices of system modules that have not yet been seen. +class SystemInputRegistry { +public: + SystemInputRegistry(size_t FirstSystemInputIndex, + const StdModuleManifest &Manifest); + + /// Returns the indices of systems inputs that have not yet been seen. + SmallVector getNewSystemInputs(ArrayRef NamedDeps); + +private: + size_t FirstSystemInputIndex; + std::mutex Lock; + llvm::DenseMap NameToManifestIndex; + llvm::SmallBitVector ScheduledSystemModules; +}; +} // anonymous namespace + +SystemInputRegistry::SystemInputRegistry(size_t FirstSystemInputIndex, + const StdModuleManifest &Manifest) + : FirstSystemInputIndex(FirstSystemInputIndex), + ScheduledSystemModules(Manifest.ModuleEntries.size(), false) { + // Build the mapping from module names to their manifest index. + NameToManifestIndex.reserve(Manifest.ModuleEntries.size()); + for (const auto &[Index, M] : llvm::enumerate(Manifest.ModuleEntries)) + NameToManifestIndex.try_emplace(M.LogicalName, Index); +} + +SmallVector +SystemInputRegistry::getNewSystemInputs(ArrayRef NamedDeps) { + SmallVector ToSchedule; + { + std::scoped_lock SL(Lock); + for (const auto &ModuleName : NamedDeps) { + const auto It = NameToManifestIndex.find(ModuleName); + if (It == NameToManifestIndex.end()) + continue; + const auto ManifestIndex = It->second; + if (ScheduledSystemModules[ManifestIndex]) + continue; + ScheduledSystemModules[ManifestIndex] = true; + ToSchedule.push_back(FirstSystemInputIndex + ManifestIndex); + } + } + return ToSchedule; +} + +/// Construct a path for the explicitly built PCM. +static std::string constructPCMPath(ModuleID ID, StringRef OutputDir) { + assert(!ID.ModuleName.empty() && !ID.ContextHash.empty() && + "Invalid ModuleID"); + SmallString<256> ExplicitPCMPath(OutputDir); + llvm::sys::path::append(ExplicitPCMPath, ID.ContextHash, + ID.ModuleName + "-" + ID.ContextHash + ".pcm"); + return std::string(ExplicitPCMPath); +} + +namespace { +/// A simple dependency action controller that only provides module lookup for +/// Clang modules. +class ModuleLookupController : public DependencyActionController { +public: + ModuleLookupController(StringRef OutputDir) : OutputDir(OutputDir) {} + + std::string lookupModuleOutput(const ModuleDeps &MD, + ModuleOutputKind Kind) override { + if (Kind == tooling::dependencies::ModuleOutputKind::ModuleFile) + return constructPCMPath(MD.ID, OutputDir); + + // Driver command lines that trigger lookups for unsupported + // ModuleOutputKinds are not supported by the modules driver. Those command + // lines should probably be adjusted or rejected in Driver::handleArguments + // or Driver::HandleImmediateArgs. + llvm::reportFatalInternalError( + "call to lookupModuleOutput with unexpected ModuleOutputKind"); + } + +private: + std::string OutputDir; +}; +} // anonymous namespace + +/// Constructs the full -cc1 command line, including executable, for the given +/// driver \c Job. +static std::vector buildCC1CommandLine(const Command &Job) { + assert(StringRef(Job.getCreator().getName()) == "clang"); + const auto &JobArgs = Job.getArguments(); + std::vector CC1CommandLine; + CC1CommandLine.reserve(JobArgs.size() + 1); + CC1CommandLine.emplace_back(Job.getExecutable()); + for (const auto &Arg : JobArgs) + CC1CommandLine.emplace_back(Arg); + return CC1CommandLine; +} + +/// Performs a full dependency scan of the given compilation inputs. +/// +/// Diagnostics are emitted through the driver's diagnostic engine. +/// +/// \returns A \c DependencyScanResult on success, or \c std::nullopt on +/// failure. The returned \c DependencyScanResult is deterministic for the given +/// compilation inputs. +static std::optional scanDependencies( + const JobVector &ScanInputJobs, const StdModuleManifest &Manifest, + size_t FirstSystemInputIndex, Compilation &C, StringRef ModuleDir) { + llvm::PrettyStackTraceString CrashInfo("Performing module dependency scan."); + + DependencyScanningService ScanningService( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Full); + + const auto NumInputs = ScanInputJobs.size(); + + // TODO: Benchmark: Determine the optimal number of worker threads for a given + // number of inputs. How many inputs are required for multi-threading to be + // beneficial? How many inputs should each thread scan? + std::unique_ptr ThreadPool; + size_t WorkerCount; + +#if LLVM_ENABLE_THREADS + const bool HasSystemInputs = !Manifest.ModuleEntries.empty(); + + if (NumInputs <= 2) { + auto S = llvm::optimal_concurrency(1); + ThreadPool = std::make_unique(std::move(S)); + WorkerCount = 1; + } else { + auto S = llvm::optimal_concurrency(NumInputs - + static_cast(HasSystemInputs)); + ThreadPool = std::make_unique(std::move(S)); + const size_t MaxConcurrency = ThreadPool->getMaxConcurrency(); + WorkerCount = std::min( + MaxConcurrency, + NumInputs - (HasSystemInputs && NumInputs < MaxConcurrency ? 1 : 0)); + } +#else + ThreadPool = std::make_unique(); + WorkerCount = 1; +#endif + + ScanningWorkerPool WorkerPool(WorkerCount, ScanningService, + C.getDriver().getVFS()); + + SystemInputRegistry SysInputRegistry(FirstSystemInputIndex, Manifest); + ModuleLookupController LookupController(ModuleDir); + ScanResultCollector ResultCollector(NumInputs); + SmallVector DiagLists(NumInputs); + std::atomic HasError = false; + + std::function ScanFn; + ScanFn = [&](size_t InputIndex, bool IsSystem) { + auto CC1CommandLine = buildCC1CommandLine(*ScanInputJobs[InputIndex]); + StandaloneDiagCollector DiagConsumer; + + std::optional TUDeps; + { + auto WorkerHandle = WorkerPool.scopedAcquire(); + FullDependencyConsumer FullDepsConsumer(WorkerHandle->SeenModules); + if (WorkerHandle->Worker.computeDependencies( + ".", CC1CommandLine, FullDepsConsumer, LookupController, + DiagConsumer)) + TUDeps = FullDepsConsumer.takeTranslationUnitDeps(); + } + + // Always capture diagnostics, since even successful scans may produce + // warnings or notes. + DiagLists[InputIndex] = DiagConsumer.takeDiagnostics(); + + if (!TUDeps) { + HasError.store(true, std::memory_order_relaxed); + return; + } + + // Enqueue scans for system modules newly required by this TU. + for (auto SysInputIndex : + SysInputRegistry.getNewSystemInputs(TUDeps->NamedModuleDeps)) + ThreadPool->async( + [&ScanFn, SysInputIndex]() { ScanFn(SysInputIndex, true); }); + + ResultCollector.handleTUResult(std::move(*TUDeps), IsSystem, InputIndex); + }; + + // Initiate the dependency scan with all user inputs. + for (size_t I = 0; I < FirstSystemInputIndex; ++I) + ThreadPool->async([&ScanFn, I]() { ScanFn(I, /*IsSystem*/ false); }); + ThreadPool->wait(); + + // Report the diagnostics for each dependency scan. + StandaloneDiagReporter DiagReporter(C.getDriver().getDiags()); + for (auto &DiagsList : DiagLists) + DiagReporter.Report(std::move(DiagsList)); + + if (HasError) + return std::nullopt; + + return ResultCollector.takeScanResults(); +} + +namespace { +class MDGNode; +class MDGEdge; +using MDGNodeBase = llvm::DGNode; +using MDGEdgeBase = llvm::DGEdge; +using ModuleDependencyGraphBase = llvm::DirectedGraph; + +/// Abstract base class for all node kinds in the module dependency graph. +class MDGNode : public MDGNodeBase { +public: + enum class NodeKind { + Root, + ClangModule, + NamedModule, + NonModule, + }; + + explicit MDGNode(NodeKind Kind) : Kind(Kind) {} + virtual ~MDGNode() = 0; + + /// Returns this node's kind. + NodeKind getKind() const { return Kind; } + + /// Returns this node's Clang module dependencies. + virtual ArrayRef getClangDeps() const { return {}; } + + /// Returns this node's C++ named module dependencies. + virtual ArrayRef getNamedDeps() const { return {}; } + +private: + const NodeKind Kind; +}; + +MDGNode::~MDGNode() {} + +/// Represents the root node of the module dependency graph. +/// +/// The root node only serves as an entry point for graph traversal. +/// It should have an edge to each node that would otherwise have no incoming +/// edges, ensuring there is always a path from the root to any node in the +/// graph. +/// There should be exactly one such root node in a given graph. +class RootMDGNode final : public MDGNode { +public: + /// Define classof to be able to use isa<>, cast<>, dyn_cast<>, etc. + static bool classof(const MDGNode *N) { + return N->getKind() == NodeKind::Root; + } + +private: + // Only allow the graph itself to creates its own single root node. + friend class ModuleDependencyGraph; + + RootMDGNode() : MDGNode(NodeKind::Root) {} +}; + +class TranslationUnitMDGNode : public MDGNode { +protected: + TranslationUnitMDGNode(NodeKind K, const InputDependencies &D) + : MDGNode(K), InputDeps(D) {} + +public: + StringRef getFilename() const { + assert(!InputDeps.FileDeps.empty() && + "Expected input itself to be the first file dependency."); + return InputDeps.FileDeps.front(); + } + + ArrayRef getClangDeps() const final { + return InputDeps.ClangModuleDeps; + } + + ArrayRef getNamedDeps() const final { + return InputDeps.NamedModuleDeps; + } + + const InputDependencies &InputDeps; +}; + +/// Subclass of MDGNode representing a translation unit that provides no +/// module of any kind. +class NonModuleMDGNode final : public TranslationUnitMDGNode { +public: + explicit NonModuleMDGNode(const InputDependencies &InputDeps) + : TranslationUnitMDGNode(NodeKind::NonModule, InputDeps) {} + + /// Define classof to be able to use isa<>, cast<>, dyn_cast<>, etc. + static bool classof(const MDGNode *N) { + return N->getKind() == NodeKind::NonModule; + } +}; + +/// Subclass of MDGNode representing a translation unit that provides a C++20 +/// named module interface unit. +class CXXNamedModuleMDGNode final : public TranslationUnitMDGNode { +public: + explicit CXXNamedModuleMDGNode(const InputDependencies &InputDeps) + : TranslationUnitMDGNode(NodeKind::NamedModule, InputDeps) {} + + const ModuleID &getModuleID() const { return InputDeps.ID; }; + + /// Define classof to be able to use isa<>, cast<>, dyn_cast<>, etc. + static bool classof(const MDGNode *N) { + return N->getKind() == NodeKind::NamedModule; + } +}; + +/// Subclass of MDGNode representing a Clang module unit. +class ClangModuleMDGNode final : public MDGNode { +public: + explicit ClangModuleMDGNode(const ModuleDeps &ClangModuleDeps) + : MDGNode(NodeKind::ClangModule), ModuleDeps(ClangModuleDeps) {} + + /// Define classof to be able to use isa<>, cast<>, dyn_cast<>, etc. + static bool classof(const MDGNode *N) { + return N->getKind() == NodeKind::ClangModule; + } + const ModuleID &getModuleID() const { return ModuleDeps.ID; } + + ArrayRef getClangDeps() const override { + return ModuleDeps.ClangModuleDeps; + } + + const ModuleDeps &ModuleDeps; +}; + +/// Represents an import relation in the module dependency graph, directed +/// from the imported module to the importer. +class MDGEdge : public MDGEdgeBase { +public: + explicit MDGEdge(MDGNode &N) : MDGEdgeBase(N) {} + MDGEdge() = delete; +}; + +/// A directed graph describing the full dependency relations of this +/// compilation. +/// +/// The graph owns all of its nodes and edges. +/// The graph's root node is initialized on construction. +class ModuleDependencyGraph : public ModuleDependencyGraphBase { +public: + ModuleDependencyGraph() { + Root = new (Alloc.Allocate(sizeof(RootMDGNode), alignof(RootMDGNode))) + RootMDGNode(); + addNode(*Root); + } + + MDGNode *getRoot() { return Root; } + const MDGNode *getRoot() const { return Root; } + + llvm::BumpPtrAllocator &getAllocator() { return Alloc; } + +private: + llvm::BumpPtrAllocator Alloc; + RootMDGNode *Root = nullptr; +}; +} // anonymous namespace + +/// Allocation helper using the graph's allocator. +template +static MDGComponent *makeWithAlloc(llvm::BumpPtrAllocator &Alloc, + Args &&...args) { + return new (Alloc.Allocate(sizeof(MDGComponent), alignof(MDGComponent))) + MDGComponent(std::forward(args)...); +} + +/// Builds all Clang module nodes from \c ClangModuleDeps for \c Graph and +/// registers them in \c ClangModuleMap. +static void buildClangModuleNodes( + ModuleDependencyGraph &Graph, ArrayRef ClangModuleDeps, + llvm::DenseMap &ClangModuleMap) { + auto &Alloc = Graph.getAllocator(); + for (auto &M : ClangModuleDeps) { + auto *Node = makeWithAlloc(Alloc, M); + Graph.addNode(*Node); + const auto [It, Inserted] = ClangModuleMap.try_emplace(M.ID, Node); + assert(Inserted && "Duplicate Clang module in scan result."); + } +} + +/// Builds translation unit nodes from \c InputDeps for \c Graph and registers +/// all named module nodes in \c NamedModuleMap. +/// +/// \returns true on success, if no duplicate modules are detected. +static bool buildTranslationUnitNodes( + ModuleDependencyGraph &Graph, + ArrayRef> InputDeps, + llvm::DenseMap &NamedModuleMap, + DiagnosticsEngine &Diags) { + auto &Alloc = Graph.getAllocator(); + bool HasDuplicate = false; + + for (auto &Dep : InputDeps) { + if (!Dep) + continue; + + MDGNode *Node = nullptr; + if (Dep->ID.ModuleName.empty()) { + // If there is no module name, this is a regular non-module TU. + Node = makeWithAlloc(Alloc, *Dep); + } else { + // There is a module name; this is a named module interface unit. + Node = makeWithAlloc(Alloc, *Dep); + const auto [It, Inserted] = NamedModuleMap.try_emplace( + Dep->ID.ModuleName, static_cast(Node)); + if (!Inserted) { + // We have multiple source files which define the same module. + Diags.Report(diag::err_mod_graph_named_module_redefinition) + << Dep->ID.ModuleName << It->second->InputDeps.FileDeps.front() + << Dep->FileDeps.front(); + HasDuplicate = true; + } + } + + Graph.addNode(*Node); + } + + return !HasDuplicate; +} + +/// Adds edges from \c Importer to nodes referenced in \c Map for each +/// dependency in \c Dependencies. +template +static void addNodeEdges(MDGNode &Importer, ArrayRef Dependencies, + MapTy &Map, llvm::BumpPtrAllocator &Alloc) { + for (const auto &Dep : Dependencies) { + if (auto It = Map.find(Dep); It != Map.end()) { + // TODO: For imports to unknown modules, check if a prebuilt module is + // provided via -fmodule-file. + // FIXME: TranslationUnitDeps::PrebuiltModuleDeps does currently not + // provide this information for named modules provided via + // -fmodule-file. + auto *Edge = makeWithAlloc(Alloc, *It->second); + Importer.addEdge(*Edge); + } + } +} + +/// Connects all nodes in \c Graph to their dependencies and links the root +/// node to nodes with otherwise no incoming edges. +static void connectGraphNodes( + ModuleDependencyGraph &Graph, + const llvm::DenseMap &ClangModuleMap, + const llvm::DenseMap &NamedModuleMap) { + auto &Alloc = Graph.getAllocator(); + + for (auto *Node : Graph) { + addNodeEdges(*Node, Node->getClangDeps(), ClangModuleMap, Alloc); + addNodeEdges(*Node, Node->getNamedDeps(), NamedModuleMap, Alloc); + } + + for (auto *Node : Graph) { + if (Node->getEdges().empty()) { + auto *Edge = makeWithAlloc(Alloc, *Node); + Graph.getRoot()->addEdge(*Edge); + } + } +} + +/// Construct a module dependency graph from the scan result. +static std::optional +buildModuleDependencyGraph(DependencyScanResult &ScanResult, + DiagnosticsEngine &Diags) { + ModuleDependencyGraph Graph; + + // Lookup maps, used for connecting the nodes. + llvm::DenseMap ClangModuleMap; + llvm::DenseMap NamedModuleMap; + + buildClangModuleNodes(Graph, ScanResult.ClangModuleDeps, ClangModuleMap); + if (!buildTranslationUnitNodes(Graph, ScanResult.InputDeps, NamedModuleMap, + Diags)) + return std::nullopt; + connectGraphNodes(Graph, ClangModuleMap, NamedModuleMap); + + return Graph; +} +} // namespace clang::driver::modules + +namespace llvm { +namespace cdm = ::clang::driver::modules; + +/// Non-const versions of the GraphTraits specializations for ModuleDepGraph. +template <> struct GraphTraits { + using NodeRef = cdm::MDGNode *; + + static NodeRef MDGGetTargetNode(cdm::MDGEdgeBase *E) { + return &E->getTargetNode(); + } + + using ChildIteratorType = + mapped_iterator; + using ChildEdgeIteratorType = cdm::MDGNode::iterator; + + static NodeRef getEntryNode(NodeRef N) { return N; } + + static ChildIteratorType child_begin(NodeRef N) { + return ChildIteratorType(N->begin(), &MDGGetTargetNode); + } + + static ChildIteratorType child_end(NodeRef N) { + return ChildIteratorType(N->end(), &MDGGetTargetNode); + } + + static ChildEdgeIteratorType child_edge_begin(NodeRef N) { + return N->begin(); + } + static ChildEdgeIteratorType child_edge_end(NodeRef N) { return N->end(); } +}; + +template <> +struct GraphTraits : GraphTraits { + using GraphRef = cdm::ModuleDependencyGraph *; + using NodeRef = cdm::MDGNode *; + + using nodes_iterator = cdm::ModuleDependencyGraph::iterator; + + static NodeRef getEntryNode(GraphRef G) { return G->getRoot(); } + + static nodes_iterator nodes_begin(GraphRef G) { return G->begin(); } + + static nodes_iterator nodes_end(GraphRef G) { return G->end(); } +}; + +/// Const versions of the GraphTraits specializations for ModuleDepGraph. +template <> struct GraphTraits { + using NodeRef = const cdm::MDGNode *; + + static NodeRef MDGGetTargetNode(const cdm::MDGEdgeBase *E) { + return &E->getTargetNode(); + } + + using ChildIteratorType = mapped_iterator; + using ChildEdgeIteratorType = cdm::MDGNode::const_iterator; + + static NodeRef getEntryNode(NodeRef N) { return N; } + + static ChildIteratorType child_begin(NodeRef N) { + return ChildIteratorType(N->begin(), &MDGGetTargetNode); + } + + static ChildIteratorType child_end(NodeRef N) { + return ChildIteratorType(N->end(), &MDGGetTargetNode); + } + + static ChildEdgeIteratorType child_edge_begin(NodeRef N) { + return N->begin(); + } + + static ChildEdgeIteratorType child_edge_end(NodeRef N) { return N->end(); } +}; + +template <> +struct GraphTraits + : GraphTraits { + using GraphRef = const cdm::ModuleDependencyGraph *; + using NodeRef = const cdm::MDGNode *; + + using nodes_iterator = cdm::ModuleDependencyGraph::const_iterator; + + static NodeRef getEntryNode(GraphRef G) { return G->getRoot(); } + + static nodes_iterator nodes_begin(GraphRef G) { return G->begin(); } + + static nodes_iterator nodes_end(GraphRef G) { return G->end(); } +}; + +template <> +struct DOTGraphTraits + : DefaultDOTGraphTraits { + explicit DOTGraphTraits(bool IsSimple = false) + : DefaultDOTGraphTraits(IsSimple) {} + + static std::string getGraphName(const cdm::ModuleDependencyGraph *) { + return "Module Dependency Graph"; + } + + static std::string getGraphProperties(const cdm::ModuleDependencyGraph *) { + return "\tnode [colorscheme=pastel13,style=filled,shape=Mrecord];\n\tedge " + "[dir=\"back\"];\n"; + } + + static bool isNodeHidden(const cdm::MDGNode *N, + const cdm::ModuleDependencyGraph *G) { + assert(N && "Node must not be null"); + return isa(N); + } + + static std::string getNodeIdentifier(const cdm::MDGNode *N, + const cdm::ModuleDependencyGraph *) { + using namespace cdm; + + assert(N && "Node must not be null"); + return llvm::TypeSwitch(N) + .Case([](const ClangModuleMDGNode *Node) { + const auto &ID = Node->getModuleID(); + return (Twine(ID.ModuleName) + ":" + ID.ContextHash).str(); + }) + .Case([](const CXXNamedModuleMDGNode *Node) { + return Node->getModuleID().ModuleName; + }) + .Case( + [](const NonModuleMDGNode *Node) { return Node->getFilename(); }); + } + + static std::string getNodeLabel(const cdm::MDGNode *N, + const cdm::ModuleDependencyGraph *) { + using namespace cdm; + + assert(N && "Node must not be null"); + SmallString<128> Buf; + raw_svector_ostream OS(Buf); + + auto PrintModuleKind = [&](StringRef Kind) { OS << "Kind: " << Kind; }; + auto PrintModuleName = [&](StringRef Name) { + OS << "Module name: " << Name; + }; + auto PrintFilename = [&](StringRef Filename) { + OS << "Filename: " << Filename; + }; + auto PrintInputOrigin = [&](bool IsSystem) { + OS << "Input origin: " << (IsSystem ? "System" : "User"); + }; + auto PrintContextHash = [&](StringRef Hash) { OS << "Hash: " << Hash; }; + + llvm::TypeSwitch(N) + .Case([&](const ClangModuleMDGNode *Node) { + PrintModuleKind("Clang module"); + OS << " \\| "; + PrintModuleName(Node->getModuleID().ModuleName); + OS << " \\| "; + OS << "Modulemap file: " << Node->ModuleDeps.ClangModuleMapFile; + OS << " \\| "; + PrintInputOrigin(Node->ModuleDeps.IsSystem); + OS << " \\| "; + PrintContextHash(Node->getModuleID().ContextHash); + }) + .Case([&](const CXXNamedModuleMDGNode *Node) { + PrintModuleKind("C++ named module"); + OS << " \\| "; + PrintModuleName(Node->getModuleID().ModuleName); + OS << " \\| "; + PrintFilename(Node->getFilename()); + OS << " \\| "; + PrintInputOrigin(Node->InputDeps.IsSystem); + OS << " \\| "; + PrintContextHash(Node->getModuleID().ContextHash); + }) + .Case([&](const NonModuleMDGNode *Node) { + PrintModuleKind("Non-module"); + OS << " \\| "; + PrintFilename(Node->getFilename()); + }) + .Default([](const MDGNode *) { + llvm_unreachable("Unhandled MDGNode kind in getNodeLabel!"); + }); + + return std::string(OS.str()); + } + + static std::string getNodeAttributes(const cdm::MDGNode *N, + const cdm::ModuleDependencyGraph *) { + using namespace cdm; + + assert(N && "Node must not be null"); + return llvm::TypeSwitch(N) + .Case( + [&](const ClangModuleMDGNode *) { return "fillcolor=1"; }) + .Case( + [&](const CXXNamedModuleMDGNode *) { return "fillcolor=2"; }) + .Case( + [&](const NonModuleMDGNode *) { return "fillcolor=3"; }); + } +}; + +/// GraphWriter specialization for ModuleDepGraph. +/// +/// Uses human-readable identifiers instead of raw pointers and separates node +/// definitions from import edges for a more remark-friendly output. +template <> +class GraphWriter + : public GraphWriterBase> { +public: + using GraphType = const cdm::ModuleDependencyGraph *; + using Base = GraphWriterBase>; + + GraphWriter(llvm::raw_ostream &O, const GraphType &G, bool IsSimple) + : Base(O, G, IsSimple) {} + + void writeNodes(); + +private: + using Base::DOTTraits; + using Base::GTraits; + using Base::NodeRef; + + DenseMap NodeIDMap; + + void writeNodeDefinitions(ArrayRef Nodes); + void writeNodeRelations(ArrayRef Nodes); +}; + +void llvm::GraphWriter::writeNodes() { + auto IsNodeVisible = [&](NodeRef N) { return !DTraits.isNodeHidden(N, G); }; + const auto VisibleNodeRange = make_filter_range(nodes(G), IsNodeVisible); + const SmallVector VisibleNodes(VisibleNodeRange); + + writeNodeDefinitions(VisibleNodes); + writeNodeRelations(VisibleNodes); +} + +void llvm::GraphWriter::writeNodeDefinitions(ArrayRef Nodes) { + for (const auto &Node : Nodes) { + const auto NodeID = DTraits.getNodeIdentifier(Node, G); + const auto NodeLabel = DTraits.getNodeLabel(Node, G); + O << "\t\"" << DOT::EscapeString(NodeID) << "\" [ " + << DTraits.getNodeAttributes(Node, G) << ",label=\"{ " + << DOT::EscapeString(NodeLabel) << " }\"];\n"; + NodeIDMap.try_emplace(Node, std::move(NodeID)); + } + O << "\n"; +} + +void llvm::GraphWriter::writeNodeRelations( + ArrayRef Nodes) { + for (const auto &Node : Nodes) { + const auto &SourceNodeID = NodeIDMap.at(Node); + for (const auto &Edge : Node->getEdges()) { + const auto *TargetNode = GTraits::MDGGetTargetNode(Edge); + const auto &TargetNodeID = NodeIDMap.at(TargetNode); + O << "\t\"" << DOT::EscapeString(SourceNodeID) << "\" -> \"" + << DOT::EscapeString(TargetNodeID) << "\";\n"; + } + } +} +} // namespace llvm + +namespace clang::driver::modules { +/// Returns true if the driver \c Job is eligible as a dependency scan +/// input. +/// +/// A job is eligible if it is a clang \c -cc1 invocation and all its inputs +/// are source files. +static bool isJobForDependencyScan(const Command &Job) { + if (StringRef(Job.getCreator().getName()) != "clang") + return false; + auto IsSrcInput = [](const InputInfo &II) -> bool { + return types::isSrcFile(II.getType()); + }; + return llvm::all_of(Job.getInputInfos(), IsSrcInput); +} + +/// Splits driver jobs into those eligible for dependency scanning and the rest. +/// +/// Non-eligible jobs are either not \c -cc1 jobs or depend on outputs from +/// eligible jobs. The original order of jobs is preserved. +/// +/// \returns the pair: (dependency scan input jobs, rest jobs). +static std::pair +splitCommandsByScanEligibility(JobVector &&Jobs) { + std::pair Result; + auto &[ScanInputJobs, OtherJobs] = Result; + for (auto &Job : Jobs) { + if (isJobForDependencyScan(*Job)) + ScanInputJobs.push_back(std::move(Job)); + else + OtherJobs.push_back(std::move(Job)); + } + return Result; +} + +/// Adds each system include directory in \p SystemIncludeDirs to \p Job's +/// arguments. +static void +addSystemIncludeDirsFromManifest(Compilation &C, Command &Job, + ArrayRef SystemIncludeDirs) { + const auto &TC = Job.getCreator().getToolChain(); + const auto &TCArgs = C.getArgsForToolChain( + &TC, /*BoundArch*/ "", Job.getSource().getOffloadingDeviceKind()); + auto &CC1Args = Job.getArguments(); + + for (const auto &IncludeDir : SystemIncludeDirs) + TC.addSystemInclude(TCArgs, CC1Args, IncludeDir); +} + +/// Applies local module arguments from the \c Manifest to the corresponding +/// system input jobs in \c ScanJobs. +static void applyLocalArgsFromManifest(Compilation &C, JobVector &ScanJobs, + size_t FirstSystemInputIndex, + const StdModuleManifest &Manifest) { + auto SystemInputJobs = ArrayRef(ScanJobs).slice(FirstSystemInputIndex); + for (auto &&[Job, ManifestEntry] : + llvm::zip_equal(SystemInputJobs, Manifest.ModuleEntries)) { + const auto &LocalArgs = ManifestEntry.LocalArgs; + if (LocalArgs) + addSystemIncludeDirsFromManifest(C, *Job, LocalArgs->SystemIncludeDirs); + } +} + +/// Computes the -fmodule-cache-path for this compilation. +static SmallString<128> getModuleCachePath(DerivedArgList &Args) { + SmallString<128> Path; + if (const auto &A = Args.getLastArg(options::OPT_fmodules_cache_path)) + Path = A->getValue(); + else + driver::Driver::getDefaultModuleCachePath(Path); + return Path; +} + +void planDriverManagedModuleCompilation(Compilation &C, + const StdModuleManifest &Manifest) { + llvm::PrettyStackTraceString CrashInfo("Planning modules build."); + auto &Diags = C.getDriver().getDiags(); + + auto [ScanInputJobs, OtherJobs] = + splitCommandsByScanEligibility(C.getJobs().takeJobs()); + + const size_t SystemInputCount = Manifest.ModuleEntries.size(); + const size_t FirstSystemInputIndex = ScanInputJobs.size() - SystemInputCount; + +#ifndef NDEBUG + // System module inputs are appended to the end of the compilation input list, + // in the same order as they appear in the standard module manifest. + // We assume these manifest-provided inputs are appended last. + // If this assumption ever breaks, explicitly partition the jobs by matching + // their inputs against the manifest paths. + auto SysInputJobsForScan = + ArrayRef(ScanInputJobs).take_back(SystemInputCount); + for (const auto &[Job, ManifestEntry] : + llvm::zip_equal(SysInputJobsForScan, Manifest.ModuleEntries)) { + const auto &InputInfos = Job->getInputInfos(); + assert(InputInfos.size() == 1 && "Expected exactly one dependency " + "scanning input for each system module"); + assert(InputInfos.front().getFilename() == ManifestEntry.SourcePath && + "System module input order should match order in manifest!"); + } +#endif + + // Apply manifest-specified local arguments before scanning as they may affect + // the scan results. + applyLocalArgsFromManifest(C, ScanInputJobs, FirstSystemInputIndex, Manifest); + + auto ModuleCachePath = getModuleCachePath(C.getArgs()); + + // Run the dependency scan. + auto ScanResults = scanDependencies( + ScanInputJobs, Manifest, FirstSystemInputIndex, C, ModuleCachePath); + if (!ScanResults) { + C.getDriver().getDiags().Report(diag::err_failed_dependency_scan); + return; + } + + // TODO: Remove driver jobs for module inputs generated from the standard + // library manifest that are unused in this compilation (i.e., each job + // corresponding to a std::nullopt in ScanResults). + + // TODO: Generate driver jobs for each Clang module and update the other + // driver jobs with the scan-generated command lines. + + // Construct the module dependency graph. + // TODO: Attach all jobs to the graph during construction. + auto MaybeModuleGraph = buildModuleDependencyGraph(*ScanResults, Diags); + if (!MaybeModuleGraph) { + Diags.Report(diag::err_building_dependency_graph); + return; + } + Diags.Report(diag::remark_printing_module_graph); + if (!Diags.isLastDiagnosticIgnored()) + llvm::WriteGraph(llvm::errs(), + &(*MaybeModuleGraph)); + + // TODO: Check for cyclic dependencies in the module dependency graph. + + // TODO: Modify each driver job's command line to generate/pass-in the right + // module files. + + // Merge the jobs back into the compilation's job list. + for (auto &Job : + llvm::concat>(ScanInputJobs, OtherJobs)) + C.addCommand(std::move(Job)); +} +} // namespace clang::driver::modules diff --git a/clang/lib/Lex/DependencyDirectivesScanner.cpp b/clang/lib/Lex/DependencyDirectivesScanner.cpp index eee57c786442a..cc6c4e677cd24 100644 --- a/clang/lib/Lex/DependencyDirectivesScanner.cpp +++ b/clang/lib/Lex/DependencyDirectivesScanner.cpp @@ -83,7 +83,7 @@ struct Scanner { /// \returns True on error. bool scan(SmallVectorImpl &Directives); - friend bool clang::scanInputForCXX20ModulesUsage(StringRef Source); + friend bool clang::scanInputForCXXNamedModulesUsage(StringRef Source); private: /// Lexes next token and advances \p First and the \p Lexer. @@ -1095,7 +1095,7 @@ static void skipUntilMaybeCXX20ModuleDirective(const char *&First, } } -bool clang::scanInputForCXX20ModulesUsage(StringRef Source) { +bool clang::scanInputForCXXNamedModulesUsage(StringRef Source) { const char *First = Source.begin(); const char *const End = Source.end(); skipUntilMaybeCXX20ModuleDirective(First, End); diff --git a/clang/test/Driver/modules-driver-dep-graph-with-system-inputs.cpp b/clang/test/Driver/modules-driver-dep-graph-with-system-inputs.cpp new file mode 100644 index 0000000000000..0757f88d0f415 --- /dev/null +++ b/clang/test/Driver/modules-driver-dep-graph-with-system-inputs.cpp @@ -0,0 +1,101 @@ +// This test checks the on-demand scanning of system inputs, in particular: +// 1. Inputs for unused system modules are not scanned. +// 2. Imports between system modules are supported and scanned on demand. + +// RUN: split-file %s %t + +// The standard library modules manifest (libc++.modules.json) is discovered +// relative to the installed C++ standard library runtime libraries +// We need to create them in order for Clang to find the manifest. +// RUN: rm -rf %t && split-file %s %t && cd %t +// RUN: mkdir -p %t/Inputs/usr/lib/x86_64-linux-gnu +// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.so +// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.a + +// RUN: sed "s|DIR|%/t|g" %t/libc++.modules.json.in > \ +// RUN: %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.modules.json + +// RUN: mkdir -p %t/Inputs/usr/lib/share/libc++/v1 +// RUN: mkdir -p %t/Inputs/usr/lib/share/libc++/v2 +// RUN: cat %t/std.cppm > %t/Inputs/usr/lib/share/libc++/v1/std.cppm +// RUN: cat %t/std.compat.cppm > %t/Inputs/usr/lib/share/libc++/v1/std.compat.cppm + +// RUN: %clang -std=c++20 \ +// RUN: -fmodules-driver -Rmodules-driver \ +// RUN: -stdlib=libc++ \ +// RUN: -resource-dir=%t/Inputs/usr/lib/x86_64-linux-gnu \ +// RUN: --target=x86_64-linux-gnu \ +// RUN: -fmodules-cache-path=%t/modules-cache \ +// RUN: %t/main.cpp %t/foo.cpp \ +// RUN: -### 2>&1 \ +// RUN: | sed 's:\\\\\?:/:g' \ +// RUN: | FileCheck %s -DPREFIX=%/t + +// CHECK: remark: using standard modules manifest file '[[PREFIX]]/Inputs/usr/lib/x86_64-linux-gnu/libc++.modules.json' [-Rmodules-driver] +// CHECK: remark: printing module dependency graph [-Rmodules-driver] +// CHECK-NEXT: digraph "Module Dependency Graph" { +// CHECK-NEXT: label="Module Dependency Graph"; + +// CHECK: "[[PREFIX]]/main.cpp" [ fillcolor=[[COLOR1:[0-9]+]],label="{ Kind: Non-module | Filename: [[PREFIX]]/main.cpp }"]; +// CHECK-NEXT: "[[PREFIX]]/foo.cpp" [ fillcolor=[[COLOR1]],label="{ Kind: Non-module | Filename: [[PREFIX]]/foo.cpp }"]; +// CHECK-NEXT: "std" [ fillcolor=[[COLOR2:[0-9]+]],label="{ Kind: C++ named module | Module name: std | Filename: [[PREFIX]]/Inputs/usr/lib/share/libc++/v1/std.cppm | Input origin: System | Hash: {{.*}} }"]; +// CHECK-NEXT: "std.compat" [ fillcolor=[[COLOR2]],label="{ Kind: C++ named module | Module name: std.compat | Filename: [[PREFIX]]/Inputs/usr/lib/share/libc++/v1/std.compat.cppm | Input origin: System | Hash: {{.*}} }"]; +// CHECK-NEXT: "core" [ fillcolor=[[COLOR2]],label="{ Kind: C++ named module | Module name: core | Filename: [[PREFIX]]/core.cxxm | Input origin: System | Hash: {{.*}} }"]; + +// CHECK: "[[PREFIX]]/main.cpp" -> "std"; +// CHECK-NEXT: "[[PREFIX]]/main.cpp" -> "std.compat"; +// CHECK-NEXT: "[[PREFIX]]/foo.cpp" -> "core"; +// CHECK-NEXT: "std.compat" -> "std"; +// CHECK-NEXT: } + + +//--- main.cpp +import std; +import std.compat; + +//--- foo.cpp +import core; + +//--- std.cppm +export module std; + +//--- std.compat.cppm +export module std.compat; +import std; + +// The module 'core' is isn't really a system module in libc++ or libstdc++. +// This is only to test that any module marked with '"is-std-library": true' can +// be imported on demand. +//--- core.cxxm +export module core; + +//--- unused.cppm +export module unused; + +//--- libc++.modules.json.in +{ + "version": 1, + "revision": 1, + "modules": [ + { + "logical-name": "std", + "source-path": "../share/libc++/v1/std.cppm", + "is-std-library": true + }, + { + "logical-name": "std.compat", + "source-path": "../share/libc++/v1/std.compat.cppm", + "is-std-library": true + }, + { + "logical-name": "core", + "source-path": "DIR/core.cxxm", + "is-std-library": true + }, + { + "logical-name": "unused", + "source-path": "DIR/unused.cppm", + "is-std-library": true + } + ] +} diff --git a/clang/test/Driver/modules-driver-dep-graph.cpp b/clang/test/Driver/modules-driver-dep-graph.cpp new file mode 100644 index 0000000000000..39d301c647a3a --- /dev/null +++ b/clang/test/Driver/modules-driver-dep-graph.cpp @@ -0,0 +1,93 @@ +// Tests that the module dependency scan and the module dependency graph +// generation are correct. +// This test does not make use of any system inputs. + +// RUN: split-file %s %t + +// RUN: %clang -std=c++23 -nostdlib -fmodules \ +// RUN: -fmodules-driver -Rmodules-driver \ +// RUN: -fmodule-map-file=%t/module.modulemap %t/main.cpp \ +// RUN: -fmodules-cache-path=%t/modules-cache \ +// RUN: %t/A.cpp %t/A-B.cpp %t/A-C.cpp %t/B.cpp -### 2>&1 \ +// RUN: | sed 's:\\\\\?:/:g' \ +// RUN: | FileCheck -DPREFIX=%/t --check-prefixes=CHECK %s + +// CHECK: clang: remark: printing module dependency graph [-Rmodules-driver] +// CHECK-NEXT: digraph "Module Dependency Graph" { +// CHECK-NEXT: label="Module Dependency Graph"; +// CHECK-NEXT: node [colorscheme={{.*}},style=filled,shape=Mrecord]; +// CHECK-NEXT: edge [dir="back"]; + +// CHECK: "transitive1:[[HASH_TRANSITIVE1:.*]]" [ fillcolor=1,label="{ Kind: Clang module | Module name: transitive1 | Modulemap file: [[PREFIX]]/module.modulemap | Input origin: User | Hash: [[HASH_TRANSITIVE1]] }"]; +// CHECK-NEXT: "transitive2:[[HASH_TRANSITIVE2:.*]]" [ fillcolor=1,label="{ Kind: Clang module | Module name: transitive2 | Modulemap file: [[PREFIX]]/module.modulemap | Input origin: User | Hash: [[HASH_TRANSITIVE2]] }"]; +// CHECK-NEXT: "direct1:[[HASH_DIRECT1:.*]]" [ fillcolor=1,label="{ Kind: Clang module | Module name: direct1 | Modulemap file: [[PREFIX]]/module.modulemap | Input origin: User | Hash: [[HASH_DIRECT1]] }"]; +// CHECK-NEXT: "direct2:[[HASH_DIRECT2:.*]]" [ fillcolor=1,label="{ Kind: Clang module | Module name: direct2 | Modulemap file: [[PREFIX]]/module.modulemap | Input origin: User | Hash: [[HASH_DIRECT2]] }"]; +// CHECK-NEXT: "root:[[HASH_ROOT:.*]]" [ fillcolor=1,label="{ Kind: Clang module | Module name: root | Modulemap file: [[PREFIX]]/module.modulemap | Input origin: User | Hash: [[HASH_ROOT]] }"]; +// CHECK-NEXT: "[[PREFIX]]/main.cpp" [ fillcolor=3,label="{ Kind: Non-module | Filename: [[PREFIX]]/main.cpp }"]; +// CHECK-NEXT: "A" [ fillcolor=2,label="{ Kind: C++ named module | Module name: A | Filename: [[PREFIX]]/A.cpp | Input origin: User | Hash: {{.*}} }"]; +// CHECK-NEXT: "A:B" [ fillcolor=2,label="{ Kind: C++ named module | Module name: A:B | Filename: [[PREFIX]]/A-B.cpp | Input origin: User | Hash: {{.*}} }"]; +// CHECK-NEXT: "A:C" [ fillcolor=2,label="{ Kind: C++ named module | Module name: A:C | Filename: [[PREFIX]]/A-C.cpp | Input origin: User | Hash: {{.*}} }"]; +// CHECK-NEXT: "B" [ fillcolor=2,label="{ Kind: C++ named module | Module name: B | Filename: [[PREFIX]]/B.cpp | Input origin: User | Hash: {{.*}} }"]; + +// CHECK: "direct1:[[HASH_DIRECT1]]" -> "transitive1:[[HASH_TRANSITIVE1]]"; +// CHECK-NEXT: "direct1:[[HASH_DIRECT1]]" -> "transitive2:[[HASH_TRANSITIVE2]]"; +// CHECK-NEXT: "direct2:[[HASH_DIRECT2]]" -> "transitive1:[[HASH_TRANSITIVE1]]"; +// CHECK-NEXT: "root:[[HASH_ROOT]]" -> "direct1:[[HASH_DIRECT1]]"; +// CHECK-NEXT: "root:[[HASH_ROOT]]" -> "direct2:[[HASH_DIRECT2]]"; +// CHECK-NEXT: "[[PREFIX]]/main.cpp" -> "root:[[HASH_ROOT]]"; +// CHECK-NEXT: "[[PREFIX]]/main.cpp" -> "A"; +// CHECK-NEXT: "[[PREFIX]]/main.cpp" -> "B"; +// CHECK-NEXT: "A" -> "A:B"; +// CHECK-NEXT: "A" -> "A:C"; +// CHECK-NEXT: "A:B" -> "direct1:[[HASH_DIRECT1]]"; +// CHECK-NEXT: "B" -> "root:[[HASH_ROOT]]"; +// CHECK-NEXT: "B" -> "A"; +// CHECK-NEXT: } + +//--- module.modulemap +module root { header "root.h" } +module direct1 { header "direct1.h" } +module direct2 { header "direct2.h" } +module transitive1 { header "transitive1.h" } +module transitive2 { header "transitive2.h" } + +//--- root.h +#include "direct1.h" +#include "direct2.h" + +//--- direct1.h +#include "transitive1.h" +#include "transitive2.h" + +//--- direct2.h +#include "transitive1.h" + +//--- transitive1.h +// empty + +//--- transitive2.h +// empty + +//--- A.cpp +export module A; +export import :B; +import :C; + +//--- A-B.cpp +module; +#include "direct1.h" +export module A:B; + +//--- A-C.cpp +export module A:C; + +//--- B.cpp +module; +#include "root.h" +export module B; +import A; + +//--- main.cpp +#include "root.h" +import A; +import B; diff --git a/clang/test/Driver/modules-driver-dep-scan-diagnostics.cpp b/clang/test/Driver/modules-driver-dep-scan-diagnostics.cpp new file mode 100644 index 0000000000000..ff141faf644eb --- /dev/null +++ b/clang/test/Driver/modules-driver-dep-scan-diagnostics.cpp @@ -0,0 +1,25 @@ +// Tests that the module dependency scan properly outputs diagnostics which +// were collected during the dependency scan. + +// RUN: split-file %s %t + +// RUN: not %clang -### -fmodules -fmodules-driver -Rmodules-driver \ +// RUN: -fmodule-map-file=%t/module.modulemap %t/main.cpp 2>&1 \ +// RUN: -fmodules-cache-path=%t/modules-cache \ +// RUN: | FileCheck --check-prefixes=CHECK %s + +//--- module.modulemap +module a { header "a.h" } + +//--- a.h +// Diagnostics collected during the dependency scan need to be translated to +// and from a representation that can outlive the compiler invocation they +// were generated by. +// We test that the diagnostics source location is translated correctly: +//----10|-------20|--------30| +#include /*just some space*/ "doesnotexist.h" +// CHECK: a.h:6:30: fatal error: 'doesnotexist.h' file not found + +//--- main.cpp +#include "a.h" +// CHECK: clang: error: failed to perform dependency scan diff --git a/clang/test/Driver/modules-driver-duplicate-named-module.cpp b/clang/test/Driver/modules-driver-duplicate-named-module.cpp new file mode 100644 index 0000000000000..d131edb388095 --- /dev/null +++ b/clang/test/Driver/modules-driver-duplicate-named-module.cpp @@ -0,0 +1,21 @@ +// Verify that the modules driver rejects ambiguous module definitions. + +// RUN: split-file %s %t + +// RUN: not %clang -std=c++23 -fmodules -fmodules-driver -Rmodules-driver \ +// RUN: -fmodules-cache-path=%t/modules-cache \ +// RUN: %t/main.cpp %t/A1.cpp %t/A2.cpp 2>&1 \ +// RUN: | sed 's:\\\\\?:/:g' \ +// RUN: | FileCheck -DPREFIX=%/t --check-prefixes=CHECK %s + +/// CHECK: clang: error: duplicate definitions of C++20 named module 'A' in '[[PREFIX]]/A1.cpp' and '[[PREFIX]]/A2.cpp' +/// CHECK: clang: error: failed to construct the module dependency graph + +//--- main.cpp +import A; + +//--- A1.cpp +export module A; + +//--- A2.cpp +export module A; diff --git a/clang/test/Driver/modules-driver-malformed-module-manifest.cpp b/clang/test/Driver/modules-driver-malformed-module-manifest.cpp new file mode 100644 index 0000000000000..bda39d808943b --- /dev/null +++ b/clang/test/Driver/modules-driver-malformed-module-manifest.cpp @@ -0,0 +1,43 @@ +// Verify that a malformed standard modules manifest triggers an error. + +// RUN: split-file %s %t + +// The standard library modules manifest (libc++.modules.json) is discovered +// relative to the installed C++ standard library runtime libraries +// We need to create them in order for Clang to find the manifest. +// RUN: rm -rf %t && split-file %s %t && cd %t +// RUN: mkdir -p %t/Inputs/usr/lib/x86_64-linux-gnu +// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.so +// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.a + +// Add the standard module manifest itself. +// RUN: cat %t/libc++.modules.json.in > \ +// RUN: %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.modules.json + +// RUN: not %clang -std=c++20 \ +// RUN: -fmodules-driver -Rmodules-driver \ +// RUN: -stdlib=libc++ \ +// RUN: -resource-dir=%t/Inputs/usr/lib/x86_64-linux-gnu \ +// RUN: --target=x86_64-linux-gnu \ +// RUN: -fmodules-cache-path=%t/modules-cache \ +// RUN: main.cpp \ +// RUN: -### 2>&1 \ +// RUN: | sed 's:\\\\\?:/:g' \ +// RUN: | FileCheck %s -DPREFIX=%/t + +// CHECK: remark: using standard modules manifest file '[[PREFIX]]/Inputs/usr/lib/x86_64-linux-gnu/libc++.modules.json' [-Rmodules-driver] + +// CHECK: error: failure while parsing standard modules manifest: '[5:7, byte=57]: Invalid JSON value (true?)' + +//--- main.cpp +// empty + +//--- libc++.modules.json.in +{ + "version": 1, + "revision": 1, + "modules": [ + this is malformed. + ] +} + diff --git a/clang/test/Driver/modules-driver-manifest-local-system-includes.cpp b/clang/test/Driver/modules-driver-manifest-local-system-includes.cpp new file mode 100644 index 0000000000000..34e20bff5c81f --- /dev/null +++ b/clang/test/Driver/modules-driver-manifest-local-system-includes.cpp @@ -0,0 +1,77 @@ +// Verifies that the local system include directories listed in the standard +// module manifest are added to corresponding driver jobs. + +// RUN: split-file %s %t + +// The standard library modules manifest (libc++.modules.json) is discovered +// relative to the installed C++ standard library runtime libraries +// We need to create them in order for Clang to find the manifest. +// RUN: rm -rf %t && split-file %s %t && cd %t +// RUN: mkdir -p %t/Inputs/usr/lib/x86_64-linux-gnu +// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.so +// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.a + +// RUN: cat %t/libc++.modules.json.in > \ +// RUN: %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.modules.json + +// RUN: mkdir -p %t/Inputs/usr/lib/share/libc++/v1 +// RUN: mkdir -p %t/Inputs/usr/lib/share/libc++/v2 +// RUN: cat %t/std.cppm > %t/Inputs/usr/lib/share/libc++/v1/std.cppm +// RUN: cat %t/std.compat.cppm > %t/Inputs/usr/lib/share/libc++/v1/std.compat.cppm + +// RUN: %clang -std=c++20 \ +// RUN: -fmodules-driver -Rmodules-driver \ +// RUN: -stdlib=libc++ \ +// RUN: -resource-dir=%t/Inputs/usr/lib/x86_64-linux-gnu \ +// RUN: --target=x86_64-linux-gnu \ +// RUN: -fmodules-cache-path=%t/modules-cache \ +// RUN: %t/main.cpp \ +// RUN: -### 2>&1 \ +// RUN: | sed 's:\\\\\?:/:g' \ +// RUN: | FileCheck %s -DPREFIX=%/t + +// CHECK: "-cc1" {{.*}} "-main-file-name" "std.cppm" {{.*}} "-internal-isystem" "[[PREFIX]]/Inputs/usr/lib/share/libc++/v1" +// CHECK: "-cc1" {{.*}} "-main-file-name" "std.compat.cppm" {{.*}} "-internal-isystem" "[[PREFIX]]/Inputs/usr/lib/share/libc++/v1" "-internal-isystem" "[[PREFIX]]/Inputs/usr/lib/share/libc++/v2" + +//--- main.cpp +// Import the system modules to ensure that the corresponding jobs don't get +// deleted. +import std; +import std.compat; + +//--- std.cppm +export module std; + +//--- std.compat.cppm +export module std.compat; +import std; + +//--- libc++.modules.json.in +{ + "version": 1, + "revision": 1, + "modules": [ + { + "logical-name": "std", + "source-path": "../share/libc++/v1/std.cppm", + "is-std-library": true, + "local-arguments": { + "system-include-directories": [ + "../share/libc++/v1" + ] + } + }, + { + "logical-name": "std.compat", + "source-path": "../share/libc++/v1/std.compat.cppm", + "is-std-library": true, + "local-arguments": { + "system-include-directories": [ + "../share/libc++/v1", + "../share/libc++/v2" + ] + } + } + ] +} +