Skip to content

Commit 3221056

Browse files
benlangmuirbenlangmuir
committed
[clang][cas] Scaffolding for modules with include-tree
This adds the scaffolding needed for include-tree to push IncludeTreeBuilder stacks, to compute an include-tree for a module and to get fmodule-file-cache-key options for dependencies. Note: this does not yet support building or importing modules, but it lets us compute dependencies in the right way. (cherry picked from commit 3b87a20)
1 parent ca4eed9 commit 3221056

19 files changed

+406
-40
lines changed

‎clang/include/clang/Basic/DiagnosticCASKinds.td

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ def err_cas_missing_root_id : Error<
3636
"CAS missing expected root-id '%0'">, DefaultFatal;
3737
def err_cas_cannot_parse_root_id_for_module : Error<
3838
"CAS cannot parse root-id '%0' for module '%1'">, DefaultFatal;
39+
def err_cas_cannot_parse_include_tree_id : Error<
40+
"CAS cannot parse include-tree-id '%0'">, DefaultFatal;
41+
def err_cas_missing_include_tree_id : Error<
42+
"CAS missing expected include-tree '%0'">, DefaultFatal;
3943

4044
def warn_clang_cache_disabled_caching: Warning<
4145
"caching disabled because %0">,

‎clang/include/clang/Basic/Module.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,10 @@ class alignas(8) Module {
225225
/// scanner, if any.
226226
std::optional<std::string> CASFileSystemRootID;
227227

228+
/// The include-tree root ID for implicit modules built with the dependency
229+
/// scanner, if any.
230+
std::optional<std::string> IncludeTreeID;
231+
228232
/// The top-level headers associated with this module.
229233
llvm::SmallSetVector<const FileEntry *, 2> TopHeaders;
230234

@@ -682,6 +686,15 @@ class alignas(8) Module {
682686
getTopLevelModule()->CASFileSystemRootID = std::move(ID);
683687
}
684688

689+
std::optional<std::string> getIncludeTreeID() const {
690+
return getTopLevelModule()->IncludeTreeID;
691+
}
692+
693+
void setIncludeTreeID(std::string ID) {
694+
assert(!getIncludeTreeID() || *getIncludeTreeID() == ID);
695+
getTopLevelModule()->IncludeTreeID = std::move(ID);
696+
}
697+
685698
/// Retrieve the directory for which this module serves as the
686699
/// umbrella.
687700
DirectoryName getUmbrellaDir() const;

‎clang/include/clang/Serialization/ASTBitCodes.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ namespace serialization {
4141
/// Version 4 of AST files also requires that the version control branch and
4242
/// revision match exactly, since there is no backward compatibility of
4343
/// AST files at this time.
44-
const unsigned VERSION_MAJOR = 25;
44+
const unsigned VERSION_MAJOR = 26;
4545

4646
/// AST file minor version number supported by this version of
4747
/// Clang.
@@ -367,6 +367,10 @@ enum ControlRecordTypes {
367367
/// Record code for the (optional) CAS filesystem root ID for implicit modules
368368
/// built with the dependency scanner.
369369
CASFS_ROOT_ID,
370+
371+
/// Record code for the (optional) include-tree ID for implicit modules
372+
/// built with the dependency scanner.
373+
CAS_INCLUDE_TREE_ID,
370374
};
371375

372376
/// Record types that occur within the options block inside

‎clang/include/clang/Serialization/ASTReader.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,11 @@ class ASTReaderListener {
246246
return false;
247247
}
248248

249+
/// Called for each CAS include-tree root ID.
250+
///
251+
/// \returns true to indicate \p RootID is invalid, or false otherwise.
252+
virtual bool readIncludeTreeID(StringRef ID, bool Complain) { return false; }
253+
249254
/// Called for each module cache key.
250255
///
251256
/// \returns true to indicate the key cannot be loaded.
@@ -300,6 +305,7 @@ class ChainedASTReaderListener : public ASTReaderListener {
300305
bool visitInputFile(StringRef Filename, bool isSystem,
301306
bool isOverridden, bool isExplicitModule) override;
302307
bool readCASFileSystemRootID(StringRef RootID, bool Complain) override;
308+
bool readIncludeTreeID(StringRef ID, bool Complain) override;
303309
bool readModuleCacheKey(StringRef ModuleName, StringRef Filename,
304310
StringRef CacheKey) override;
305311
void readModuleFileExtension(

‎clang/include/clang/Serialization/ModuleFile.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,10 @@ class ModuleFile {
143143
/// scanner, or empty.
144144
std::string CASFileSystemRootID;
145145

146+
/// The include-tree root ID for implicit modules built with the dependency
147+
/// scanner, or empty.
148+
std::string IncludeTreeID;
149+
146150
/// The name of the module.
147151
std::string ModuleName;
148152

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ struct TranslationUnitDeps {
7171
/// The CASID for input file dependency tree.
7272
std::optional<std::string> CASFileSystemRootID;
7373

74+
/// The include-tree for input file dependency tree.
75+
std::optional<std::string> IncludeTreeID;
76+
7477
/// The sequence of commands required to build the translation unit. Commands
7578
/// should be executed in order.
7679
///
@@ -142,6 +145,7 @@ class DependencyScanningTool {
142145
Expected<cas::IncludeTreeRoot>
143146
getIncludeTree(cas::ObjectStore &DB,
144147
const std::vector<std::string> &CommandLine, StringRef CWD,
148+
LookupModuleOutputCallback LookupModuleOutput,
145149
const DepscanPrefixMapping &PrefixMapping);
146150

147151
/// If \p DiagGenerationAsCompilation is true it will generate error
@@ -150,7 +154,8 @@ class DependencyScanningTool {
150154
/// \p DiagOpts.DiagnosticSerializationFile setting is set for the invocation.
151155
Expected<cas::IncludeTreeRoot> getIncludeTreeFromCompilerInvocation(
152156
cas::ObjectStore &DB, std::shared_ptr<CompilerInvocation> Invocation,
153-
StringRef CWD, const DepscanPrefixMapping &PrefixMapping,
157+
StringRef CWD, LookupModuleOutputCallback LookupModuleOutput,
158+
const DepscanPrefixMapping &PrefixMapping,
154159
DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS,
155160
bool DiagGenerationAsCompilation);
156161

@@ -247,6 +252,10 @@ class FullDependencyConsumer : public DependencyConsumer {
247252
CASFileSystemRootID = std::move(ID);
248253
}
249254

255+
void handleIncludeTreeID(std::string ID) override {
256+
IncludeTreeID = std::move(ID);
257+
}
258+
250259
TranslationUnitDeps takeTranslationUnitDeps();
251260
ModuleDepsGraph takeModuleGraphDeps();
252261

@@ -258,6 +267,7 @@ class FullDependencyConsumer : public DependencyConsumer {
258267
std::vector<Command> Commands;
259268
std::string ContextHash;
260269
std::optional<std::string> CASFileSystemRootID;
270+
std::optional<std::string> IncludeTreeID;
261271
std::vector<std::string> OutputPaths;
262272
const llvm::StringSet<> &AlreadySeen;
263273
};

‎clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,9 @@ struct ModuleDeps {
139139
/// The CASID for the module input dependency tree, if any.
140140
std::optional<llvm::cas::CASID> CASFileSystemRootID;
141141

142+
/// The CASID for the module include-tree, if any.
143+
std::optional<std::string> IncludeTreeID;
144+
142145
/// The \c ActionCache key for this module, if any.
143146
std::optional<std::string> ModuleCacheKey;
144147

‎clang/lib/Frontend/CompilerInstance.cpp

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -642,10 +642,14 @@ class CompileCacheASTReaderHelper : public ASTReaderListener {
642642
: CAS(CAS), Cache(Cache), ModuleCache(ModuleCache), Diags(Diags) {}
643643

644644
bool readCASFileSystemRootID(StringRef RootID, bool Complain) override;
645+
bool readIncludeTreeID(StringRef ID, bool Complain) override;
645646
bool readModuleCacheKey(StringRef ModuleName, StringRef Filename,
646647
StringRef CacheKey) override;
647648

648649
private:
650+
bool checkCASID(bool Complain, StringRef RootID, unsigned ParseDiagID,
651+
unsigned MissingDiagID);
652+
649653
cas::ObjectStore &CAS;
650654
cas::ActionCache &Cache;
651655
InMemoryModuleCache &ModuleCache;
@@ -2431,21 +2435,36 @@ bool CompileCacheASTReaderHelper::readModuleCacheKey(StringRef ModuleName,
24312435
Filename, CacheKey, "imported module", CAS, Cache, ModuleCache, Diags);
24322436
}
24332437

2434-
bool CompileCacheASTReaderHelper::readCASFileSystemRootID(StringRef RootID,
2435-
bool Complain) {
2436-
// Verify that RootID is in the CAS. Otherwise the module cache probably was
2437-
// created with a different CAS.
2438+
/// Verify that ID is in the CAS. Otherwise the module cache probably was
2439+
/// created with a different CAS.
2440+
bool CompileCacheASTReaderHelper::checkCASID(bool Complain, StringRef RootID,
2441+
unsigned ParseDiagID,
2442+
unsigned MissingDiagID) {
24382443
std::optional<cas::CASID> ID;
24392444
if (errorToBool(CAS.parseID(RootID).moveInto(ID))) {
24402445
if (Complain)
2441-
Diags.Report(diag::err_cas_cannot_parse_root_id) << RootID;
2446+
Diags.Report(ParseDiagID) << RootID;
24422447
return true;
24432448
}
24442449
if (errorToBool(CAS.getProxy(*ID).takeError())) {
24452450
if (Complain) {
2446-
Diags.Report(diag::err_cas_missing_root_id) << RootID;
2451+
Diags.Report(MissingDiagID) << RootID;
24472452
}
24482453
return true;
24492454
}
24502455
return false;
24512456
}
2457+
2458+
bool CompileCacheASTReaderHelper::readCASFileSystemRootID(StringRef RootID,
2459+
bool Complain) {
2460+
return checkCASID(Complain, RootID, diag::err_cas_cannot_parse_root_id,
2461+
diag::err_cas_missing_root_id);
2462+
}
2463+
2464+
bool CompileCacheASTReaderHelper::readIncludeTreeID(StringRef ID,
2465+
bool Complain) {
2466+
// Verify that ID is in the CAS. Otherwise the module cache probably was
2467+
// created with a different CAS.
2468+
return checkCASID(Complain, ID, diag::err_cas_cannot_parse_include_tree_id,
2469+
diag::err_cas_missing_include_tree_id);
2470+
}

‎clang/lib/Serialization/ASTReader.cpp

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,10 @@ bool ChainedASTReaderListener::readCASFileSystemRootID(StringRef RootID,
257257
return First->readCASFileSystemRootID(RootID, Complain) ||
258258
Second->readCASFileSystemRootID(RootID, Complain);
259259
}
260-
260+
bool ChainedASTReaderListener::readIncludeTreeID(StringRef ID, bool Complain) {
261+
return First->readIncludeTreeID(ID, Complain) ||
262+
Second->readIncludeTreeID(ID, Complain);
263+
}
261264
bool ChainedASTReaderListener::readModuleCacheKey(StringRef ModuleName,
262265
StringRef Filename,
263266
StringRef CacheKey) {
@@ -3027,6 +3030,15 @@ ASTReader::ReadControlBlock(ModuleFile &F,
30273030
return OutOfDate;
30283031
}
30293032
break;
3033+
case CAS_INCLUDE_TREE_ID:
3034+
F.IncludeTreeID = Blob.str();
3035+
if (Listener) {
3036+
bool Complain =
3037+
!canRecoverFromOutOfDate(F.FileName, ClientLoadCapabilities);
3038+
if (Listener->readIncludeTreeID(F.IncludeTreeID, Complain))
3039+
return OutOfDate;
3040+
}
3041+
break;
30303042
}
30313043
}
30323044
}
@@ -5676,6 +5688,8 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F,
56765688
CurrentModule->setModuleCacheKey(F.ModuleCacheKey);
56775689
if (!F.CASFileSystemRootID.empty())
56785690
CurrentModule->setCASFileSystemRootID(F.CASFileSystemRootID);
5691+
if (!F.IncludeTreeID.empty())
5692+
CurrentModule->setIncludeTreeID(F.IncludeTreeID);
56795693
}
56805694

56815695
CurrentModule->Kind = Kind;

‎clang/lib/Serialization/ASTWriter.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -801,6 +801,7 @@ void ASTWriter::WriteBlockInfoBlock() {
801801
RECORD(INPUT_FILE_OFFSETS);
802802
RECORD(MODULE_CACHE_KEY);
803803
RECORD(CASFS_ROOT_ID);
804+
RECORD(CAS_INCLUDE_TREE_ID);
804805

805806
BLOCK(OPTIONS_BLOCK);
806807
RECORD(LANGUAGE_OPTIONS);
@@ -1365,6 +1366,15 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
13651366
RecordData::value_type Record[] = {CASFS_ROOT_ID};
13661367
Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID);
13671368
}
1369+
// CAS include-tree id, for the scanner.
1370+
if (auto ID = WritingModule->getIncludeTreeID()) {
1371+
auto Abbrev = std::make_shared<BitCodeAbbrev>();
1372+
Abbrev->Add(BitCodeAbbrevOp(CAS_INCLUDE_TREE_ID));
1373+
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
1374+
unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev));
1375+
RecordData::value_type Record[] = {CAS_INCLUDE_TREE_ID};
1376+
Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID);
1377+
}
13681378
}
13691379

13701380
// Imports

‎clang/lib/Tooling/DependencyScanning/CachingActions.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ class CachingOnDiskFileSystem;
1919
namespace clang::tooling::dependencies {
2020

2121
std::unique_ptr<DependencyActionController>
22-
createIncludeTreeActionController(cas::ObjectStore &DB,
22+
createIncludeTreeActionController(LookupModuleOutputCallback LookupModuleOutput,
23+
cas::ObjectStore &DB,
2324
DepscanPrefixMapping PrefixMapping);
2425

2526
std::unique_ptr<DependencyActionController>

‎clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -190,10 +190,11 @@ DependencyScanningTool::getDependencyTreeFromCompilerInvocation(
190190

191191
Expected<cas::IncludeTreeRoot> DependencyScanningTool::getIncludeTree(
192192
cas::ObjectStore &DB, const std::vector<std::string> &CommandLine,
193-
StringRef CWD, const DepscanPrefixMapping &PrefixMapping) {
193+
StringRef CWD, LookupModuleOutputCallback LookupModuleOutput,
194+
const DepscanPrefixMapping &PrefixMapping) {
194195
GetIncludeTree Consumer(DB);
195-
auto Controller =
196-
createIncludeTreeActionController(DB, std::move(PrefixMapping));
196+
auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB,
197+
std::move(PrefixMapping));
197198
llvm::Error Result =
198199
Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller);
199200
if (Result)
@@ -204,12 +205,13 @@ Expected<cas::IncludeTreeRoot> DependencyScanningTool::getIncludeTree(
204205
Expected<cas::IncludeTreeRoot>
205206
DependencyScanningTool::getIncludeTreeFromCompilerInvocation(
206207
cas::ObjectStore &DB, std::shared_ptr<CompilerInvocation> Invocation,
207-
StringRef CWD, const DepscanPrefixMapping &PrefixMapping,
208+
StringRef CWD, LookupModuleOutputCallback LookupModuleOutput,
209+
const DepscanPrefixMapping &PrefixMapping,
208210
DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS,
209211
bool DiagGenerationAsCompilation) {
210212
GetIncludeTree Consumer(DB);
211-
auto Controller =
212-
createIncludeTreeActionController(DB, std::move(PrefixMapping));
213+
auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB,
214+
std::move(PrefixMapping));
213215
Worker.computeDependenciesFromCompilerInvocation(
214216
std::move(Invocation), CWD, Consumer, *Controller, DiagsConsumer,
215217
VerboseOS, DiagGenerationAsCompilation);
@@ -310,6 +312,7 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() {
310312
TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps);
311313
TU.Commands = std::move(Commands);
312314
TU.CASFileSystemRootID = std::move(CASFileSystemRootID);
315+
TU.IncludeTreeID = std::move(IncludeTreeID);
313316

314317
for (auto &&M : ClangModuleDeps) {
315318
auto &MD = M.second;
@@ -348,7 +351,8 @@ DependencyScanningTool::createActionController(
348351
LookupModuleOutputCallback LookupModuleOutput,
349352
DepscanPrefixMapping PrefixMapping) {
350353
if (Worker.getScanningFormat() == ScanningOutputFormat::FullIncludeTree)
351-
return createIncludeTreeActionController(*Worker.getCAS(), std::move(PrefixMapping));
354+
return createIncludeTreeActionController(
355+
LookupModuleOutput, *Worker.getCAS(), std::move(PrefixMapping));
352356
if (auto CacheFS = Worker.getCASFS())
353357
return createCASFSActionController(LookupModuleOutput, *CacheFS,
354358
std::move(PrefixMapping));

‎clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -438,15 +438,12 @@ class DependencyScanningAction : public tooling::ToolAction {
438438
OriginalInvocation, OptimizeArgs, EagerLoadModules,
439439
Format == ScanningOutputFormat::P1689);
440440
ScanInstance.addDependencyCollector(MDC);
441-
if (CacheFS) {
442-
ScanInstance.setGenModuleActionWrapper(
443-
[CacheFS = CacheFS, &Controller = Controller](
444-
const FrontendOptions &Opts,
445-
std::unique_ptr<FrontendAction> Wrapped) {
446-
return std::make_unique<WrapScanModuleBuildAction>(
447-
std::move(Wrapped), Controller);
448-
});
449-
}
441+
ScanInstance.setGenModuleActionWrapper(
442+
[&Controller = Controller](const FrontendOptions &Opts,
443+
std::unique_ptr<FrontendAction> Wrapped) {
444+
return std::make_unique<WrapScanModuleBuildAction>(
445+
std::move(Wrapped), Controller);
446+
});
450447
break;
451448
}
452449

0 commit comments

Comments
 (0)