Skip to content

Commit 0ae9977

Browse files
akyrtziakyrtzi
authored
Merge pull request swiftlang#6458 from akyrtzi/pr/distributed-cache-api
[CAS] Introduce changes in `ObjectStore`/`ActionCache` to better accomodate distributed caching
2 parents 6ddb94b + 411d8f0 commit 0ae9977

23 files changed

+623
-130
lines changed

‎clang/include/clang/Frontend/CASDependencyCollector.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,10 @@ class CASDependencyCollector : public DependencyFileGenerator {
3636
/// \param Opts Output options. Only options that affect the output format of
3737
/// a dependency file are signficant.
3838
/// \param CAS The CAS to read the result from.
39-
/// \param DepsRef The dependencies.
39+
/// \param Deps The dependencies.
4040
/// \param OS The output stream to write the dependency file to.
4141
static llvm::Error replay(const DependencyOutputOptions &Opts,
42-
cas::ObjectStore &CAS, cas::ObjectRef DepsRef,
42+
cas::ObjectStore &CAS, cas::ObjectProxy Deps,
4343
llvm::raw_ostream &OS);
4444

4545
private:

‎clang/include/clang/Frontend/CompileJobCacheResult.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,21 @@ class CompileJobCacheResult : public ObjectProxy {
4646
llvm::Error
4747
forEachOutput(llvm::function_ref<llvm::Error(Output)> Callback) const;
4848

49+
/// Loads all outputs concurrently and passes the resulting \c ObjectProxy
50+
/// objects to \p Callback. If there was an error during loading then the
51+
/// callback will not be invoked.
52+
llvm::Error forEachLoadedOutput(
53+
llvm::function_ref<llvm::Error(Output, std::optional<ObjectProxy>)>
54+
Callback);
55+
4956
size_t getNumOutputs() const;
5057

5158
/// Retrieves a specific output specified by \p Kind, if it exists.
5259
std::optional<Output> getOutput(OutputKind Kind) const;
5360

61+
/// \returns a string for the given \p Kind.
62+
static StringRef getOutputKindName(OutputKind Kind);
63+
5464
/// Print this result to \p OS.
5565
llvm::Error print(llvm::raw_ostream &OS);
5666

‎clang/lib/CAS/CASOptions.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ void CASOptions::initCache(DiagnosticsEngine &Diags) const {
8686
StringRef CASPath = Cache.Config.CASPath;
8787

8888
if (!PluginPath.empty()) {
89-
std::pair<std::unique_ptr<ObjectStore>, std::unique_ptr<ActionCache>> DBs;
89+
std::pair<std::shared_ptr<ObjectStore>, std::shared_ptr<ActionCache>> DBs;
9090
if (llvm::Error E =
9191
createPluginCASDatabases(PluginPath, CASPath, PluginOptions)
9292
.moveInto(DBs)) {

‎clang/lib/Frontend/CASDependencyCollector.cpp

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,8 @@ CASDependencyCollector::CASDependencyCollector(
3131
CAS(CAS), Callback(std::move(Callback)) {}
3232

3333
llvm::Error CASDependencyCollector::replay(const DependencyOutputOptions &Opts,
34-
ObjectStore &CAS, ObjectRef DepsRef,
34+
ObjectStore &CAS, ObjectProxy Refs,
3535
llvm::raw_ostream &OS) {
36-
auto Refs = CAS.getProxy(DepsRef);
37-
if (!Refs)
38-
return Refs.takeError();
39-
4036
CASDependencyCollector DC(Opts, CAS, nullptr);
4137

4238
// Add the filenames from DependencyOutputOptions::ExtraDeps. These are kept
@@ -47,7 +43,7 @@ llvm::Error CASDependencyCollector::replay(const DependencyOutputOptions &Opts,
4743
DC.addDependency(Dep.first);
4844
}
4945

50-
auto Err = Refs->forEachReference([&](ObjectRef Ref) -> llvm::Error {
46+
auto Err = Refs.forEachReference([&](ObjectRef Ref) -> llvm::Error {
5147
auto PathHandle = CAS.getProxy(Ref);
5248
if (!PathHandle)
5349
return PathHandle.takeError();

‎clang/lib/Frontend/CompileJobCacheResult.cpp

Lines changed: 46 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,43 @@ Error CompileJobCacheResult::forEachOutput(
3434
return Error::success();
3535
}
3636

37+
Error CompileJobCacheResult::forEachLoadedOutput(
38+
llvm::function_ref<Error(Output, std::optional<ObjectProxy>)> Callback) {
39+
// Kick-off materialization for all outputs concurrently.
40+
SmallVector<std::future<Expected<std::optional<ObjectProxy>>>, 4>
41+
FutureOutputs;
42+
size_t Count = getNumOutputs();
43+
for (size_t I = 0; I < Count; ++I) {
44+
ObjectRef Ref = getOutputObject(I);
45+
FutureOutputs.push_back(getCAS().getProxyAsync(Ref));
46+
}
47+
48+
// Make sure all the outputs have materialized.
49+
std::optional<Error> OccurredError;
50+
SmallVector<std::optional<ObjectProxy>, 4> Outputs;
51+
for (auto &FutureOutput : FutureOutputs) {
52+
auto Obj = FutureOutput.get();
53+
if (!Obj) {
54+
if (!OccurredError)
55+
OccurredError = Obj.takeError();
56+
else
57+
OccurredError =
58+
llvm::joinErrors(std::move(*OccurredError), Obj.takeError());
59+
continue;
60+
}
61+
Outputs.push_back(*Obj);
62+
}
63+
if (OccurredError)
64+
return std::move(*OccurredError);
65+
66+
// Pass the loaded outputs.
67+
for (size_t I = 0; I < Count; ++I) {
68+
if (auto Err = Callback({getOutputObject(I), getOutputKind(I)}, Outputs[I]))
69+
return Err;
70+
}
71+
return Error::success();
72+
}
73+
3774
std::optional<CompileJobCacheResult::Output>
3875
CompileJobCacheResult::getOutput(OutputKind Kind) const {
3976
size_t Count = getNumOutputs();
@@ -45,25 +82,21 @@ CompileJobCacheResult::getOutput(OutputKind Kind) const {
4582
return std::nullopt;
4683
}
4784

48-
static void printOutputKind(llvm::raw_ostream &OS,
49-
CompileJobCacheResult::OutputKind Kind) {
85+
StringRef CompileJobCacheResult::getOutputKindName(OutputKind Kind) {
5086
switch (Kind) {
51-
case CompileJobCacheResult::OutputKind::MainOutput:
52-
OS << "main ";
53-
break;
54-
case CompileJobCacheResult::OutputKind::Dependencies:
55-
OS << "deps ";
56-
break;
57-
case CompileJobCacheResult::OutputKind::SerializedDiagnostics:
58-
OS << "diags ";
59-
break;
87+
case OutputKind::MainOutput:
88+
return "main";
89+
case OutputKind::SerializedDiagnostics:
90+
return "deps";
91+
case OutputKind::Dependencies:
92+
return "diags";
6093
}
6194
}
6295

6396
Error CompileJobCacheResult::print(llvm::raw_ostream &OS) {
6497
return forEachOutput([&](Output O) -> Error {
65-
printOutputKind(OS, O.Kind);
66-
OS << ' ' << getCAS().getID(O.Object) << '\n';
98+
OS << getOutputKindName(O.Kind) << " " << getCAS().getID(O.Object)
99+
<< '\n';
67100
return Error::success();
68101
});
69102
}

‎clang/lib/Frontend/CompilerInstance.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2383,6 +2383,10 @@ static bool addCachedModuleFileToInMemoryCache(
23832383
Result->getOutput(cas::CompileJobCacheResult::OutputKind::MainOutput);
23842384
if (!Output)
23852385
llvm::report_fatal_error("missing main output");
2386+
// FIXME: We wait to materialize each module file before proceeding, which
2387+
// introduces latency for a network CAS. Instead we should collect all the
2388+
// module keys and materialize them concurrently using \c getProxyAsync, for
2389+
// better network utilization.
23862390
auto OutputProxy = CAS.getProxy(Output->Object);
23872391
if (!OutputProxy) {
23882392
Diags.Report(diag::err_cas_cannot_get_module_cache_key)

‎clang/test/CAS/cas-backend.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
//
44
// RUN: %clang -cc1 -triple x86_64-apple-macos11 -fcas-backend \
55
// RUN: -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \
6-
// RUN: -Rcompile-job-cache -emit-obj -o %t/output.o \
6+
// RUN: -Rcompile-job-cache %s -emit-obj -o %t/output.o \
77
// RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb \
88
// RUN: -dependency-file %t/deps.d -MT %t/output.o 2>&1 \
99
// RUN: | FileCheck %s --allow-empty --check-prefix=CACHE-MISS
@@ -14,7 +14,7 @@
1414
// RUN: CLANG_CAS_BACKEND_SAVE_CASID_FILE=1 %clang -cc1 \
1515
// RUN: -triple x86_64-apple-macos11 -fcas-backend \
1616
// RUN: -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \
17-
// RUN: -Rcompile-job-cache -emit-obj -o %t/output.o \
17+
// RUN: -Rcompile-job-cache %s -emit-obj -o %t/output.o \
1818
// RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb \
1919
// RUN: -dependency-file %t/deps.d -MT %t/output.o 2>&1 \
2020
// RUN: | FileCheck %s --check-prefix=CACHE-HIT

‎clang/test/CAS/plugin-cas.c

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,47 @@
22

33
// RUN: rm -rf %t && mkdir -p %t
44

5-
// RUN: %clang -cc1depscan -o %t/t.rsp -fdepscan=inline -cc1-args \
5+
// RUN: %clang -cc1depscan -o %t/t1.rsp -fdepscan=inline -cc1-args \
66
// RUN: -cc1 %s -fcas-path %t/cas \
77
// RUN: -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
8-
// RUN: -fcas-plugin-option first-prefix=myfirst- -fcas-plugin-option second-prefix=mysecond-
9-
// RUN: %clang @%t/t.rsp -emit-obj -o %t/t1.o -Rcompile-job-cache 2>&1 | FileCheck %s --check-prefix=CACHE-MISS
10-
// RUN: %clang @%t/t.rsp -emit-obj -o %t/t2.o -Rcompile-job-cache 2>&1 | FileCheck %s --check-prefix=CACHE-HIT
8+
// RUN: -fcas-plugin-option first-prefix=myfirst- -fcas-plugin-option second-prefix=mysecond- \
9+
// RUN: -fcas-plugin-option upstream-path=%t/cas-upstream
10+
// RUN: %clang @%t/t1.rsp -emit-obj -o %t/t1.o -Rcompile-job-cache 2>&1 | FileCheck %s --check-prefix=CACHE-MISS
11+
12+
// Clear the CAS and check the outputs can still be "downloaded" from upstream.
13+
// RUN: rm -rf %t/cas
14+
// RUN: %clang -cc1depscan -o %t/t2.rsp -fdepscan=inline -cc1-args \
15+
// RUN: -cc1 %s -fcas-path %t/cas \
16+
// RUN: -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
17+
// RUN: -fcas-plugin-option first-prefix=myfirst- -fcas-plugin-option second-prefix=mysecond- \
18+
// RUN: -fcas-plugin-option upstream-path=%t/cas-upstream
19+
// RUN: %clang @%t/t2.rsp -emit-obj -o %t/t2.o -Rcompile-job-cache 2>&1 | FileCheck %s --check-prefix=CACHE-HIT
1120
// RUN: diff %t/t1.o %t/t2.o
1221

22+
// Check that it's a cache miss if outputs are not found in the upstream CAS.
23+
// RUN: rm -rf %t/cas
24+
// RUN: %clang -cc1depscan -o %t/t3.rsp -fdepscan=inline -cc1-args \
25+
// RUN: -cc1 %s -fcas-path %t/cas \
26+
// RUN: -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \
27+
// RUN: -fcas-plugin-option first-prefix=myfirst- -fcas-plugin-option second-prefix=mysecond- \
28+
// RUN: -fcas-plugin-option upstream-path=%t/cas-upstream \
29+
// RUN: -fcas-plugin-option simulate-missing-objects
30+
// RUN: %clang @%t/t3.rsp -emit-obj -o %t/t.o -Rcompile-job-cache 2>&1 | FileCheck %s --check-prefix=CACHE-NOTFOUND
31+
1332
// CACHE-MISS: remark: compile job cache miss for 'myfirst-mysecond-
1433
// CACHE-MISS: warning: some warning
1534

16-
// CACHE-HIT: remark: compile job cache hit for 'myfirst-mysecond-
17-
// CACHE-HIT: warning: some warning
35+
// Check that outputs are downloaded concurrently.
36+
// CACHE-HIT: load_object_async downstream begin:
37+
// CACHE-HIT-NEXT: load_object_async downstream begin:
38+
// CACHE-HIT-NEXT: load_object_async downstream end:
39+
// CACHE-HIT-NEXT: load_object_async downstream end:
40+
// CACHE-HIT-NEXT: remark: compile job cache hit for 'myfirst-mysecond-
41+
// CACHE-HIT-NEXT: warning: some warning
42+
43+
// CACHE-NOTFOUND: remark: compile job cache backend did not find output 'main' for key
44+
// CACHE-NOTFOUND: remark: compile job cache miss
45+
// CACHE-NOTFOUND: warning: some warning
1846

1947
// RUN: not %clang -cc1depscan -o %t/t.rsp -fdepscan=inline -cc1-args \
2048
// RUN: -cc1 %s -fcas-path %t/cas \

‎clang/tools/driver/cc1_main.cpp

Lines changed: 57 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,8 @@ class ObjectStoreCachingOutputs : public CachingOutputs {
282282
/// Replay a cache hit.
283283
///
284284
/// Return status if should exit immediately, otherwise None.
285-
std::optional<int> replayCachedResult(llvm::cas::ObjectRef ResultID,
285+
std::optional<int> replayCachedResult(const llvm::cas::CASID &ResultCacheKey,
286+
llvm::cas::ObjectRef ResultID,
286287
bool JustComputedResult);
287288

288289
const bool ComputedJobNeedsReplay;
@@ -585,7 +586,8 @@ Expected<bool> ObjectStoreCachingOutputs::tryReplayCachedResult(
585586
Diags.Report(diag::remark_compile_job_cache_timing_backend_key_query)
586587
<< llvm::format("%.6fs", Seconds);
587588
});
588-
if (Error E = Cache->get(ResultCacheKey).moveInto(Result))
589+
if (Error E =
590+
Cache->get(ResultCacheKey, /*Globally=*/true).moveInto(Result))
589591
return std::move(E);
590592
}
591593

@@ -607,11 +609,11 @@ Expected<bool> ObjectStoreCachingOutputs::tryReplayCachedResult(
607609
return false;
608610
}
609611

610-
Diags.Report(diag::remark_compile_job_cache_hit)
611-
<< ResultCacheKey.toString() << CAS->getID(*ResultRef).toString();
612-
std::optional<int> Status =
613-
replayCachedResult(*ResultRef, /*JustComputedResult=*/false);
614-
assert(Status && "Expected a status for a cache hit");
612+
// \c replayCachedResult emits remarks for a cache hit or miss.
613+
std::optional<int> Status = replayCachedResult(ResultCacheKey, *ResultRef,
614+
/*JustComputedResult=*/false);
615+
if (!Status)
616+
return false; // cache miss.
615617
assert(*Status == 0 && "Expected success status for a cache hit");
616618
return true;
617619
}
@@ -810,22 +812,23 @@ Error ObjectStoreCachingOutputs::finishComputedResult(
810812
Diags.Report(diag::remark_compile_job_cache_timing_backend_key_update)
811813
<< llvm::format("%.6fs", Seconds);
812814
});
813-
if (llvm::Error E = Cache->put(ResultCacheKey, CAS->getID(*Result)))
815+
if (llvm::Error E =
816+
Cache->put(ResultCacheKey, CAS->getID(*Result), /*Globally=*/true))
814817
return E;
815818
}
816819

817820
// Replay / decanonicalize as necessary.
818-
std::optional<int> Status = replayCachedResult(*Result,
821+
std::optional<int> Status = replayCachedResult(ResultCacheKey, *Result,
819822
/*JustComputedResult=*/true);
820823
(void)Status;
821824
assert(Status == std::nullopt);
822825
return Error::success();
823826
}
824827

825828
/// Replay a result after a cache hit.
826-
std::optional<int>
827-
ObjectStoreCachingOutputs::replayCachedResult(llvm::cas::ObjectRef ResultID,
828-
bool JustComputedResult) {
829+
std::optional<int> ObjectStoreCachingOutputs::replayCachedResult(
830+
const llvm::cas::CASID &ResultCacheKey, llvm::cas::ObjectRef ResultID,
831+
bool JustComputedResult) {
829832
if (JustComputedResult && !ComputedJobNeedsReplay)
830833
return std::nullopt;
831834

@@ -835,19 +838,30 @@ ObjectStoreCachingOutputs::replayCachedResult(llvm::cas::ObjectRef ResultID,
835838
if (Error E = Schema.load(ResultID).moveInto(Result))
836839
llvm::report_fatal_error(std::move(E));
837840

838-
auto Err = Result->forEachOutput([&](clang::cas::CompileJobCacheResult::Output
839-
O) -> Error {
841+
DiagnosticsEngine &Diags = Clang.getDiagnostics();
842+
bool HasMissingOutput = false;
843+
std::optional<llvm::cas::ObjectProxy> SerialDiags;
844+
845+
auto processOutput = [&](clang::cas::CompileJobCacheResult::Output O,
846+
std::optional<llvm::cas::ObjectProxy> Obj) -> Error {
847+
if (!Obj.has_value()) {
848+
Diags.Report(diag::remark_compile_job_cache_backend_output_not_found)
849+
<< clang::cas::CompileJobCacheResult::getOutputKindName(O.Kind)
850+
<< ResultCacheKey.toString() << CAS->getID(O.Object).toString();
851+
HasMissingOutput = true;
852+
return Error::success();
853+
}
854+
if (HasMissingOutput)
855+
return Error::success();
856+
840857
if (O.Kind == OutputKind::SerializedDiagnostics) {
841-
std::optional<llvm::cas::ObjectProxy> DiagsObj;
842-
if (Error E = CAS->getProxy(O.Object).moveInto(DiagsObj))
843-
return E;
844-
return replayCachedDiagnostics(DiagsObj->getData());
858+
SerialDiags = Obj;
859+
return Error::success();
845860
}
846861

847862
std::string Path = std::string(getPathForOutputKind(O.Kind));
848863
if (Path.empty())
849-
// The output may be always generated but not needed with this invocation,
850-
// like the serialized diagnostics file.
864+
// The output may be always generated but not needed with this invocation.
851865
return Error::success(); // continue
852866

853867
// Always create parent directory of outputs, since it is hard to precisely
@@ -878,12 +892,8 @@ ObjectStoreCachingOutputs::replayCachedResult(llvm::cas::ObjectRef ResultID,
878892
return llvm::errorCodeToError(EC);
879893
writeCASIDBuffer(CAS->getID(O.Object), IDOS);
880894
}
881-
std::optional<llvm::cas::ObjectProxy> CASObj;
882-
if (Error E = CAS->getProxy(O.Object).moveInto(CASObj))
883-
return E;
884-
auto Schema =
885-
std::make_unique<llvm::mccasformats::v1::MCSchema>(*CAS);
886-
if (auto E = Schema->serializeObjectFile(*CASObj, OS))
895+
auto Schema = std::make_unique<llvm::mccasformats::v1::MCSchema>(*CAS);
896+
if (auto E = Schema->serializeObjectFile(*Obj, OS))
887897
return E;
888898
}
889899
Contents = ContentsStorage;
@@ -892,14 +902,11 @@ ObjectStoreCachingOutputs::replayCachedResult(llvm::cas::ObjectRef ResultID,
892902
} else if (O.Kind == OutputKind::Dependencies) {
893903
llvm::raw_svector_ostream OS(ContentsStorage);
894904
if (auto E = CASDependencyCollector::replay(
895-
Clang.getDependencyOutputOpts(), *CAS, O.Object, OS))
905+
Clang.getDependencyOutputOpts(), *CAS, *Obj, OS))
896906
return E;
897907
Contents = ContentsStorage;
898908
} else {
899-
std::optional<llvm::cas::ObjectProxy> Bytes;
900-
if (Error E = CAS->getProxy(O.Object).moveInto(Bytes))
901-
return E;
902-
Contents = Bytes->getData();
909+
Contents = Obj->getData();
903910
}
904911

905912
std::unique_ptr<llvm::FileOutputBuffer> Output;
@@ -908,12 +915,29 @@ ObjectStoreCachingOutputs::replayCachedResult(llvm::cas::ObjectRef ResultID,
908915
return E;
909916
llvm::copy(*Contents, Output->getBufferStart());
910917
return Output->commit();
911-
});
918+
};
912919

913920
// FIXME: Stop calling report_fatal_error().
914-
if (Err)
921+
if (auto Err = Result->forEachLoadedOutput(processOutput))
915922
llvm::report_fatal_error(std::move(Err));
916923

924+
if (HasMissingOutput) {
925+
Diags.Report(diag::remark_compile_job_cache_miss)
926+
<< ResultCacheKey.toString();
927+
return std::nullopt;
928+
}
929+
930+
if (!JustComputedResult) {
931+
Diags.Report(diag::remark_compile_job_cache_hit)
932+
<< ResultCacheKey.toString() << CAS->getID(ResultID).toString();
933+
934+
if (SerialDiags) {
935+
// FIXME: Stop calling report_fatal_error().
936+
if (Error E = replayCachedDiagnostics(SerialDiags->getData()))
937+
llvm::report_fatal_error(std::move(E));
938+
}
939+
}
940+
917941
if (JustComputedResult)
918942
return std::nullopt;
919943
return 0;

0 commit comments

Comments
 (0)