Skip to content

Commit 3b87a20

Browse files
benlangmuirbenlangmuir
committed
[clang][cas] Scaffolding for modules with include-tree
This adds the scaffolding needed for include-tree to push IncludeTreeBuilder stacks, to compute an include-tree for a module and to get fmodule-file-cache-key options for dependencies. Note: this does not yet support building or importing modules, but it lets us compute dependencies in the right way.
1 parent 8f14ad5 commit 3b87a20

19 files changed

+406
-40
lines changed

‎clang/include/clang/Basic/DiagnosticCASKinds.td

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ def err_cas_missing_root_id : Error<
3636
"CAS missing expected root-id '%0'">, DefaultFatal;
3737
def err_cas_cannot_parse_root_id_for_module : Error<
3838
"CAS cannot parse root-id '%0' for module '%1'">, DefaultFatal;
39+
def err_cas_cannot_parse_include_tree_id : Error<
40+
"CAS cannot parse include-tree-id '%0'">, DefaultFatal;
41+
def err_cas_missing_include_tree_id : Error<
42+
"CAS missing expected include-tree '%0'">, DefaultFatal;
3943

4044
def warn_clang_cache_disabled_caching: Warning<
4145
"caching disabled because %0">,

‎clang/include/clang/Basic/Module.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,10 @@ class alignas(8) Module {
209209
/// scanner, if any.
210210
std::optional<std::string> CASFileSystemRootID;
211211

212+
/// The include-tree root ID for implicit modules built with the dependency
213+
/// scanner, if any.
214+
std::optional<std::string> IncludeTreeID;
215+
212216
/// The top-level headers associated with this module.
213217
llvm::SmallSetVector<const FileEntry *, 2> TopHeaders;
214218

@@ -658,6 +662,15 @@ class alignas(8) Module {
658662
getTopLevelModule()->CASFileSystemRootID = std::move(ID);
659663
}
660664

665+
std::optional<std::string> getIncludeTreeID() const {
666+
return getTopLevelModule()->IncludeTreeID;
667+
}
668+
669+
void setIncludeTreeID(std::string ID) {
670+
assert(!getIncludeTreeID() || *getIncludeTreeID() == ID);
671+
getTopLevelModule()->IncludeTreeID = std::move(ID);
672+
}
673+
661674
/// Retrieve the directory for which this module serves as the
662675
/// umbrella.
663676
DirectoryName getUmbrellaDir() const;

‎clang/include/clang/Serialization/ASTBitCodes.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ namespace serialization {
4141
/// Version 4 of AST files also requires that the version control branch and
4242
/// revision match exactly, since there is no backward compatibility of
4343
/// AST files at this time.
44-
const unsigned VERSION_MAJOR = 25;
44+
const unsigned VERSION_MAJOR = 26;
4545

4646
/// AST file minor version number supported by this version of
4747
/// Clang.
@@ -367,6 +367,10 @@ enum ControlRecordTypes {
367367
/// Record code for the (optional) CAS filesystem root ID for implicit modules
368368
/// built with the dependency scanner.
369369
CASFS_ROOT_ID,
370+
371+
/// Record code for the (optional) include-tree ID for implicit modules
372+
/// built with the dependency scanner.
373+
CAS_INCLUDE_TREE_ID,
370374
};
371375

372376
/// Record types that occur within the options block inside

‎clang/include/clang/Serialization/ASTReader.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,11 @@ class ASTReaderListener {
246246
return false;
247247
}
248248

249+
/// Called for each CAS include-tree root ID.
250+
///
251+
/// \returns true to indicate \p RootID is invalid, or false otherwise.
252+
virtual bool readIncludeTreeID(StringRef ID, bool Complain) { return false; }
253+
249254
/// Called for each module cache key.
250255
///
251256
/// \returns true to indicate the key cannot be loaded.
@@ -300,6 +305,7 @@ class ChainedASTReaderListener : public ASTReaderListener {
300305
bool visitInputFile(StringRef Filename, bool isSystem,
301306
bool isOverridden, bool isExplicitModule) override;
302307
bool readCASFileSystemRootID(StringRef RootID, bool Complain) override;
308+
bool readIncludeTreeID(StringRef ID, bool Complain) override;
303309
bool readModuleCacheKey(StringRef ModuleName, StringRef Filename,
304310
StringRef CacheKey) override;
305311
void readModuleFileExtension(

‎clang/include/clang/Serialization/ModuleFile.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,10 @@ class ModuleFile {
143143
/// scanner, or empty.
144144
std::string CASFileSystemRootID;
145145

146+
/// The include-tree root ID for implicit modules built with the dependency
147+
/// scanner, or empty.
148+
std::string IncludeTreeID;
149+
146150
/// The name of the module.
147151
std::string ModuleName;
148152

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ struct TranslationUnitDeps {
7070
/// The CASID for input file dependency tree.
7171
std::optional<std::string> CASFileSystemRootID;
7272

73+
/// The include-tree for input file dependency tree.
74+
std::optional<std::string> IncludeTreeID;
75+
7376
/// The sequence of commands required to build the translation unit. Commands
7477
/// should be executed in order.
7578
///
@@ -141,6 +144,7 @@ class DependencyScanningTool {
141144
Expected<cas::IncludeTreeRoot>
142145
getIncludeTree(cas::ObjectStore &DB,
143146
const std::vector<std::string> &CommandLine, StringRef CWD,
147+
LookupModuleOutputCallback LookupModuleOutput,
144148
const DepscanPrefixMapping &PrefixMapping);
145149

146150
/// If \p DiagGenerationAsCompilation is true it will generate error
@@ -149,7 +153,8 @@ class DependencyScanningTool {
149153
/// \p DiagOpts.DiagnosticSerializationFile setting is set for the invocation.
150154
Expected<cas::IncludeTreeRoot> getIncludeTreeFromCompilerInvocation(
151155
cas::ObjectStore &DB, std::shared_ptr<CompilerInvocation> Invocation,
152-
StringRef CWD, const DepscanPrefixMapping &PrefixMapping,
156+
StringRef CWD, LookupModuleOutputCallback LookupModuleOutput,
157+
const DepscanPrefixMapping &PrefixMapping,
153158
DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS,
154159
bool DiagGenerationAsCompilation);
155160

@@ -246,6 +251,10 @@ class FullDependencyConsumer : public DependencyConsumer {
246251
CASFileSystemRootID = std::move(ID);
247252
}
248253

254+
void handleIncludeTreeID(std::string ID) override {
255+
IncludeTreeID = std::move(ID);
256+
}
257+
249258
TranslationUnitDeps takeTranslationUnitDeps();
250259
ModuleDepsGraph takeModuleGraphDeps();
251260

@@ -257,6 +266,7 @@ class FullDependencyConsumer : public DependencyConsumer {
257266
std::vector<Command> Commands;
258267
std::string ContextHash;
259268
std::optional<std::string> CASFileSystemRootID;
269+
std::optional<std::string> IncludeTreeID;
260270
std::vector<std::string> OutputPaths;
261271
const llvm::StringSet<> &AlreadySeen;
262272
};

‎clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,9 @@ struct ModuleDeps {
139139
/// The CASID for the module input dependency tree, if any.
140140
std::optional<llvm::cas::CASID> CASFileSystemRootID;
141141

142+
/// The CASID for the module include-tree, if any.
143+
std::optional<std::string> IncludeTreeID;
144+
142145
/// The \c ActionCache key for this module, if any.
143146
std::optional<std::string> ModuleCacheKey;
144147

‎clang/lib/Frontend/CompilerInstance.cpp

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -649,10 +649,14 @@ class CompileCacheASTReaderHelper : public ASTReaderListener {
649649
: CAS(CAS), Cache(Cache), ModuleCache(ModuleCache), Diags(Diags) {}
650650

651651
bool readCASFileSystemRootID(StringRef RootID, bool Complain) override;
652+
bool readIncludeTreeID(StringRef ID, bool Complain) override;
652653
bool readModuleCacheKey(StringRef ModuleName, StringRef Filename,
653654
StringRef CacheKey) override;
654655

655656
private:
657+
bool checkCASID(bool Complain, StringRef RootID, unsigned ParseDiagID,
658+
unsigned MissingDiagID);
659+
656660
cas::ObjectStore &CAS;
657661
cas::ActionCache &Cache;
658662
InMemoryModuleCache &ModuleCache;
@@ -2415,21 +2419,36 @@ bool CompileCacheASTReaderHelper::readModuleCacheKey(StringRef ModuleName,
24152419
Filename, CacheKey, "imported module", CAS, Cache, ModuleCache, Diags);
24162420
}
24172421

2418-
bool CompileCacheASTReaderHelper::readCASFileSystemRootID(StringRef RootID,
2419-
bool Complain) {
2420-
// Verify that RootID is in the CAS. Otherwise the module cache probably was
2421-
// created with a different CAS.
2422+
/// Verify that ID is in the CAS. Otherwise the module cache probably was
2423+
/// created with a different CAS.
2424+
bool CompileCacheASTReaderHelper::checkCASID(bool Complain, StringRef RootID,
2425+
unsigned ParseDiagID,
2426+
unsigned MissingDiagID) {
24222427
std::optional<cas::CASID> ID;
24232428
if (errorToBool(CAS.parseID(RootID).moveInto(ID))) {
24242429
if (Complain)
2425-
Diags.Report(diag::err_cas_cannot_parse_root_id) << RootID;
2430+
Diags.Report(ParseDiagID) << RootID;
24262431
return true;
24272432
}
24282433
if (errorToBool(CAS.getProxy(*ID).takeError())) {
24292434
if (Complain) {
2430-
Diags.Report(diag::err_cas_missing_root_id) << RootID;
2435+
Diags.Report(MissingDiagID) << RootID;
24312436
}
24322437
return true;
24332438
}
24342439
return false;
24352440
}
2441+
2442+
bool CompileCacheASTReaderHelper::readCASFileSystemRootID(StringRef RootID,
2443+
bool Complain) {
2444+
return checkCASID(Complain, RootID, diag::err_cas_cannot_parse_root_id,
2445+
diag::err_cas_missing_root_id);
2446+
}
2447+
2448+
bool CompileCacheASTReaderHelper::readIncludeTreeID(StringRef ID,
2449+
bool Complain) {
2450+
// Verify that ID is in the CAS. Otherwise the module cache probably was
2451+
// created with a different CAS.
2452+
return checkCASID(Complain, ID, diag::err_cas_cannot_parse_include_tree_id,
2453+
diag::err_cas_missing_include_tree_id);
2454+
}

‎clang/lib/Serialization/ASTReader.cpp

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,10 @@ bool ChainedASTReaderListener::readCASFileSystemRootID(StringRef RootID,
257257
return First->readCASFileSystemRootID(RootID, Complain) ||
258258
Second->readCASFileSystemRootID(RootID, Complain);
259259
}
260-
260+
bool ChainedASTReaderListener::readIncludeTreeID(StringRef ID, bool Complain) {
261+
return First->readIncludeTreeID(ID, Complain) ||
262+
Second->readIncludeTreeID(ID, Complain);
263+
}
261264
bool ChainedASTReaderListener::readModuleCacheKey(StringRef ModuleName,
262265
StringRef Filename,
263266
StringRef CacheKey) {
@@ -3048,6 +3051,15 @@ ASTReader::ReadControlBlock(ModuleFile &F,
30483051
return OutOfDate;
30493052
}
30503053
break;
3054+
case CAS_INCLUDE_TREE_ID:
3055+
F.IncludeTreeID = Blob.str();
3056+
if (Listener) {
3057+
bool Complain =
3058+
!canRecoverFromOutOfDate(F.FileName, ClientLoadCapabilities);
3059+
if (Listener->readIncludeTreeID(F.IncludeTreeID, Complain))
3060+
return OutOfDate;
3061+
}
3062+
break;
30513063
}
30523064
}
30533065
}
@@ -5694,6 +5706,8 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F,
56945706
CurrentModule->setModuleCacheKey(F.ModuleCacheKey);
56955707
if (!F.CASFileSystemRootID.empty())
56965708
CurrentModule->setCASFileSystemRootID(F.CASFileSystemRootID);
5709+
if (!F.IncludeTreeID.empty())
5710+
CurrentModule->setIncludeTreeID(F.IncludeTreeID);
56975711
}
56985712

56995713
CurrentModule->Kind = Kind;

‎clang/lib/Serialization/ASTWriter.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -801,6 +801,7 @@ void ASTWriter::WriteBlockInfoBlock() {
801801
RECORD(INPUT_FILE_OFFSETS);
802802
RECORD(MODULE_CACHE_KEY);
803803
RECORD(CASFS_ROOT_ID);
804+
RECORD(CAS_INCLUDE_TREE_ID);
804805

805806
BLOCK(OPTIONS_BLOCK);
806807
RECORD(LANGUAGE_OPTIONS);
@@ -1359,6 +1360,15 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
13591360
RecordData::value_type Record[] = {CASFS_ROOT_ID};
13601361
Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID);
13611362
}
1363+
// CAS include-tree id, for the scanner.
1364+
if (auto ID = WritingModule->getIncludeTreeID()) {
1365+
auto Abbrev = std::make_shared<BitCodeAbbrev>();
1366+
Abbrev->Add(BitCodeAbbrevOp(CAS_INCLUDE_TREE_ID));
1367+
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
1368+
unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev));
1369+
RecordData::value_type Record[] = {CAS_INCLUDE_TREE_ID};
1370+
Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID);
1371+
}
13621372
}
13631373

13641374
// Imports

‎clang/lib/Tooling/DependencyScanning/CachingActions.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ class CachingOnDiskFileSystem;
1919
namespace clang::tooling::dependencies {
2020

2121
std::unique_ptr<DependencyActionController>
22-
createIncludeTreeActionController(cas::ObjectStore &DB,
22+
createIncludeTreeActionController(LookupModuleOutputCallback LookupModuleOutput,
23+
cas::ObjectStore &DB,
2324
DepscanPrefixMapping PrefixMapping);
2425

2526
std::unique_ptr<DependencyActionController>

‎clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -190,10 +190,11 @@ DependencyScanningTool::getDependencyTreeFromCompilerInvocation(
190190

191191
Expected<cas::IncludeTreeRoot> DependencyScanningTool::getIncludeTree(
192192
cas::ObjectStore &DB, const std::vector<std::string> &CommandLine,
193-
StringRef CWD, const DepscanPrefixMapping &PrefixMapping) {
193+
StringRef CWD, LookupModuleOutputCallback LookupModuleOutput,
194+
const DepscanPrefixMapping &PrefixMapping) {
194195
GetIncludeTree Consumer(DB);
195-
auto Controller =
196-
createIncludeTreeActionController(DB, std::move(PrefixMapping));
196+
auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB,
197+
std::move(PrefixMapping));
197198
llvm::Error Result =
198199
Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller);
199200
if (Result)
@@ -204,12 +205,13 @@ Expected<cas::IncludeTreeRoot> DependencyScanningTool::getIncludeTree(
204205
Expected<cas::IncludeTreeRoot>
205206
DependencyScanningTool::getIncludeTreeFromCompilerInvocation(
206207
cas::ObjectStore &DB, std::shared_ptr<CompilerInvocation> Invocation,
207-
StringRef CWD, const DepscanPrefixMapping &PrefixMapping,
208+
StringRef CWD, LookupModuleOutputCallback LookupModuleOutput,
209+
const DepscanPrefixMapping &PrefixMapping,
208210
DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS,
209211
bool DiagGenerationAsCompilation) {
210212
GetIncludeTree Consumer(DB);
211-
auto Controller =
212-
createIncludeTreeActionController(DB, std::move(PrefixMapping));
213+
auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB,
214+
std::move(PrefixMapping));
213215
Worker.computeDependenciesFromCompilerInvocation(
214216
std::move(Invocation), CWD, Consumer, *Controller, DiagsConsumer,
215217
VerboseOS, DiagGenerationAsCompilation);
@@ -310,6 +312,7 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() {
310312
TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps);
311313
TU.Commands = std::move(Commands);
312314
TU.CASFileSystemRootID = std::move(CASFileSystemRootID);
315+
TU.IncludeTreeID = std::move(IncludeTreeID);
313316

314317
for (auto &&M : ClangModuleDeps) {
315318
auto &MD = M.second;
@@ -348,7 +351,8 @@ DependencyScanningTool::createActionController(
348351
LookupModuleOutputCallback LookupModuleOutput,
349352
DepscanPrefixMapping PrefixMapping) {
350353
if (Worker.getScanningFormat() == ScanningOutputFormat::FullIncludeTree)
351-
return createIncludeTreeActionController(*Worker.getCAS(), std::move(PrefixMapping));
354+
return createIncludeTreeActionController(
355+
LookupModuleOutput, *Worker.getCAS(), std::move(PrefixMapping));
352356
if (auto CacheFS = Worker.getCASFS())
353357
return createCASFSActionController(LookupModuleOutput, *CacheFS,
354358
std::move(PrefixMapping));

‎clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -439,15 +439,12 @@ class DependencyScanningAction : public tooling::ToolAction {
439439
OriginalInvocation, OptimizeArgs, EagerLoadModules,
440440
Format == ScanningOutputFormat::P1689);
441441
ScanInstance.addDependencyCollector(MDC);
442-
if (CacheFS) {
443-
ScanInstance.setGenModuleActionWrapper(
444-
[CacheFS = CacheFS, &Controller = Controller](
445-
const FrontendOptions &Opts,
446-
std::unique_ptr<FrontendAction> Wrapped) {
447-
return std::make_unique<WrapScanModuleBuildAction>(
448-
std::move(Wrapped), Controller);
449-
});
450-
}
442+
ScanInstance.setGenModuleActionWrapper(
443+
[&Controller = Controller](const FrontendOptions &Opts,
444+
std::unique_ptr<FrontendAction> Wrapped) {
445+
return std::make_unique<WrapScanModuleBuildAction>(
446+
std::move(Wrapped), Controller);
447+
});
451448
break;
452449
}
453450

0 commit comments

Comments
 (0)