From 6f23165ee03a610aa21666a8a61829ca6b6b0385 Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Mon, 12 Aug 2024 09:04:00 -0700 Subject: [PATCH 1/6] [ctx_prof] CtxProfAnalysis: populate module data Continuing from #102084, which introduced the analysis, we now populate it with info about functions contained in the module. When we will update the profile due to e.g. inlined callsites, we'll ingest the callee's counters and callsites to the caller. We'll move those to the caller's respective index space (counter and callers), so we need to know and maintain where those currently end. We also don't need to keep profiles not pertinent to this module. This patch also introduces an arguably much simpler way to track the GUID of a function from the frontend compilation, through ThinLTO, and into the post-thinlink compilation step, which doesn't rely on keeping names around. A separate RFC and patches will discuss extending this to the current PGO (instrumented and sampled) and other consumers as an infrastructural component. --- llvm/include/llvm/Analysis/CtxProfAnalysis.h | 35 +++++- .../Instrumentation/PGOCtxProfLowering.h | 21 ++++ llvm/lib/Analysis/CtxProfAnalysis.cpp | 64 +++++++++- llvm/lib/Passes/PassBuilderPipelines.cpp | 1 + llvm/lib/Passes/PassRegistry.def | 1 + .../Instrumentation/PGOCtxProfLowering.cpp | 36 +++++- .../Analysis/CtxProfAnalysis/full-cycle.ll | 110 ++++++++++++++++++ llvm/test/Analysis/CtxProfAnalysis/load.ll | 108 +++++++++++++---- .../PGOProfile/ctx-instrumentation.ll | 26 +++-- .../PGOProfile/ctx-prof-use-prelink.ll | 4 +- 10 files changed, 366 insertions(+), 40 deletions(-) create mode 100644 llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll diff --git a/llvm/include/llvm/Analysis/CtxProfAnalysis.h b/llvm/include/llvm/Analysis/CtxProfAnalysis.h index d77c81d03582e..18feb88b53b54 100644 --- a/llvm/include/llvm/Analysis/CtxProfAnalysis.h +++ b/llvm/include/llvm/Analysis/CtxProfAnalysis.h @@ -9,10 +9,10 @@ #ifndef LLVM_ANALYSIS_CTXPROFANALYSIS_H #define LLVM_ANALYSIS_CTXPROFANALYSIS_H +#include "llvm/ADT/StringMap.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/PassManager.h" #include "llvm/ProfileData/PGOCtxProfReader.h" -#include namespace llvm { @@ -20,12 +20,27 @@ class CtxProfAnalysis; /// The instrumented contextual profile, produced by the CtxProfAnalysis. class PGOContextualProfile { + friend class CtxProfAnalysis; + friend class CtxProfAnalysisPrinterPass; + struct FunctionInfo { + uint32_t NextCounterIndex = 0; + uint32_t NextCallsiteIndex = 0; + const std::string Name; + + FunctionInfo(StringRef Name) : Name(Name) {} + }; std::optional Profiles; + // For the GUIDs in this module, associate metadata about each function which + // we'll need when we maintain the profiles during IPO transformations. + DenseMap FuncInfo; -public: - explicit PGOContextualProfile(PGOCtxProfContext::CallTargetMapTy &&Profiles) - : Profiles(std::move(Profiles)) {} + GlobalValue::GUID getKnownGUID(const Function &F) const; + + // This is meant to be constructed from CtxProfAnalysis, which will also set + // its state piecemeal. PGOContextualProfile() = default; + +public: PGOContextualProfile(const PGOContextualProfile &) = delete; PGOContextualProfile(PGOContextualProfile &&) = default; @@ -35,6 +50,18 @@ class PGOContextualProfile { return *Profiles; } + bool isFunctionKnown(const Function &F) const { return getKnownGUID(F) != 0; } + + uint32_t allocateNextCounterIndex(const Function &F) { + assert(isFunctionKnown(F)); + return FuncInfo.find(getKnownGUID(F))->second.NextCounterIndex++; + } + + uint32_t allocateNextCallsiteIndex(const Function &F) { + assert(isFunctionKnown(F)); + return FuncInfo.find(getKnownGUID(F))->second.NextCallsiteIndex++; + } + bool invalidate(Module &, const PreservedAnalyses &PA, ModuleAnalysisManager::Invalidator &) { // Check whether the analysis has been explicitly invalidated. Otherwise, diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h b/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h index f127d16b8de12..57486349a7f84 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h +++ b/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h @@ -24,5 +24,26 @@ class PGOCtxProfLoweringPass : public PassInfoMixin { PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); }; + +/// Assign a GUID to functions as metadata. GUID calculation takes linkage into +/// account, which may change especially through and after thinlto. By +/// pre-computing and assigning as metadata, this mechanism is resilient to such +/// changes (as well as name changes e.g. suffix ".llvm." additions). It's +/// arguably a much simpler mechanism than PGO's current GV-based one, and can +/// be made available more broadly. + +// FIXME(mtrofin): we can generalize this mechanism to calculate a GUID early in +// the pass pipeline, associate it with any Global Value, and then use it for +// PGO and ThinLTO. +// At that point, this should be moved elsewhere. +class AssignUniqueIDPass : public PassInfoMixin { +public: + explicit AssignUniqueIDPass() = default; + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); + static const char *GUIDMetadataName; + // This should become GlobalValue::getGUID + static uint64_t getGUID(const Function &F); +}; + } // namespace llvm #endif diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp index fbae705127538..fcbdc4409d210 100644 --- a/llvm/lib/Analysis/CtxProfAnalysis.cpp +++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp @@ -14,12 +14,14 @@ #include "llvm/Analysis/CtxProfAnalysis.h" #include "llvm/ADT/STLExtras.h" #include "llvm/IR/Analysis.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" #include "llvm/ProfileData/PGOCtxProfReader.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/JSON.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h" #define DEBUG_TYPE "ctx_prof" @@ -66,8 +68,8 @@ Value toJSON(const PGOCtxProfContext::CallTargetMapTy &P) { AnalysisKey CtxProfAnalysis::Key; -CtxProfAnalysis::Result CtxProfAnalysis::run(Module &M, - ModuleAnalysisManager &MAM) { +PGOContextualProfile CtxProfAnalysis::run(Module &M, + ModuleAnalysisManager &MAM) { ErrorOr> MB = MemoryBuffer::getFile(Profile); if (auto EC = MB.getError()) { M.getContext().emitError("could not open contextual profile file: " + @@ -81,7 +83,55 @@ CtxProfAnalysis::Result CtxProfAnalysis::run(Module &M, toString(MaybeCtx.takeError())); return {}; } - return Result(std::move(*MaybeCtx)); + + PGOContextualProfile Result; + + for (const auto &F : M) { + if (F.isDeclaration()) + continue; + auto GUID = AssignUniqueIDPass::getGUID(F); + assert(GUID); + const auto &Entry = F.begin(); + uint32_t MaxCounters = 0; // we expect at least a counter. + for (const auto &I : *Entry) + if (auto *C = dyn_cast(&I)) { + MaxCounters = + static_cast(C->getNumCounters()->getZExtValue()); + break; + } + if (!MaxCounters) + continue; + uint32_t MaxCallsites = 0; + for (const auto &BB : F) + for (const auto &I : BB) + if (auto *C = dyn_cast(&I)) { + MaxCallsites = + static_cast(C->getNumCounters()->getZExtValue()); + break; + } + auto [It, Ins] = Result.FuncInfo.insert( + {GUID, PGOContextualProfile::FunctionInfo(F.getName())}); + (void)Ins; + assert(Ins); + It->second.NextCallsiteIndex = MaxCallsites; + It->second.NextCounterIndex = MaxCounters; + } + // If we made it this far, the Result is valid - which we mark by setting + // .Profiles. + // Trim first the roots that aren't in this module. + DenseSet ProfiledGUIDs; + for (auto &[RootGuid, Tree] : llvm::make_early_inc_range(*MaybeCtx)) + if (!Result.FuncInfo.contains(RootGuid)) + MaybeCtx->erase(RootGuid); + Result.Profiles = std::move(*MaybeCtx); + return Result; +} + +GlobalValue::GUID PGOContextualProfile::getKnownGUID(const Function &F) const { + if (auto It = FuncInfo.find(AssignUniqueIDPass::getGUID(F)); + It != FuncInfo.end()) + return It->first; + return 0; } PreservedAnalyses CtxProfAnalysisPrinterPass::run(Module &M, @@ -91,8 +141,16 @@ PreservedAnalyses CtxProfAnalysisPrinterPass::run(Module &M, M.getContext().emitError("Invalid CtxProfAnalysis"); return PreservedAnalyses::all(); } + + OS << "Function Info:\n"; + for (const auto &[Guid, FuncInfo] : C.FuncInfo) + OS << Guid << " : " << FuncInfo.Name + << ". MaxCounterID: " << FuncInfo.NextCounterIndex + << ". MaxCallsiteID: " << FuncInfo.NextCallsiteIndex << "\n"; + const auto JSONed = ::llvm::json::toJSON(C.profiles()); + OS << "\nCurrent Profile:\n"; OS << formatv("{0:2}", JSONed); OS << "\n"; return PreservedAnalyses::all(); diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index 6927a2886b962..e36d105fa1818 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -1196,6 +1196,7 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, // In pre-link, we just want the instrumented IR. We use the contextual // profile in the post-thinlink phase. // The instrumentation will be removed in post-thinlink after IPO. + MPM.addPass(AssignUniqueIDPass()); if (IsCtxProfUse) return MPM; addPostPGOLoopRotation(MPM, Level); diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 95842d15a35bf..27a289616ce1d 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -46,6 +46,7 @@ MODULE_PASS("always-inline", AlwaysInlinerPass()) MODULE_PASS("annotation2metadata", Annotation2MetadataPass()) MODULE_PASS("attributor", AttributorPass()) MODULE_PASS("attributor-light", AttributorLightPass()) +MODULE_PASS("assign-guid", AssignUniqueIDPass()) MODULE_PASS("called-value-propagation", CalledValuePropagationPass()) MODULE_PASS("canonicalize-aliases", CanonicalizeAliasesPass()) MODULE_PASS("check-debugify", NewPMCheckDebugifyPass()) diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp index d6ba12465bb32..b938fad4615df 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp @@ -16,6 +16,7 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" +#include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/CommandLine.h" #include @@ -223,8 +224,8 @@ bool CtxInstrumentationLowerer::lowerFunction(Function &F) { assert(Mark->getIndex()->isZero()); IRBuilder<> Builder(Mark); - // FIXME(mtrofin): use InstrProfSymtab::getCanonicalName - Guid = Builder.getInt64(F.getGUID()); + + Guid = Builder.getInt64(AssignUniqueIDPass::getGUID(F)); // The type of the context of this function is now knowable since we have // NrCallsites and NrCounters. We delcare it here because it's more // convenient - we have the Builder. @@ -349,3 +350,34 @@ bool CtxInstrumentationLowerer::lowerFunction(Function &F) { F.getName()); return true; } + +const char *AssignUniqueIDPass::GUIDMetadataName = "unique_id"; + +PreservedAnalyses AssignUniqueIDPass::run(Module &M, + ModuleAnalysisManager &MAM) { + for (auto &F : M.functions()) { + if (F.isDeclaration()) + continue; + const GlobalValue::GUID GUID = F.getGUID(); + assert(!F.getMetadata(GUIDMetadataName) || + GUID == AssignUniqueIDPass::getGUID(F)); + F.setMetadata(GUIDMetadataName, + MDNode::get(M.getContext(), + {ConstantAsMetadata::get(ConstantInt::get( + Type::getInt64Ty(M.getContext()), GUID))})); + } + return PreservedAnalyses::none(); +} + +GlobalValue::GUID AssignUniqueIDPass::getGUID(const Function &F) { + if (F.isDeclaration()) { + assert(GlobalValue::isExternalLinkage(F.getLinkage())); + return GlobalValue::getGUID(F.getGlobalIdentifier()); + } + auto *MD = F.getMetadata(GUIDMetadataName); + assert(MD); + return cast(cast(MD->getOperand(0)) + ->getValue() + ->stripPointerCasts()) + ->getZExtValue(); +} \ No newline at end of file diff --git a/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll b/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll new file mode 100644 index 0000000000000..282d68b9f0665 --- /dev/null +++ b/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll @@ -0,0 +1,110 @@ +; REQUIRES: x86_64-linux + +; RUN: split-file %s %t +; +; Test that the GUID metadata survives through thinlink. +; +; RUN: llvm-ctxprof-util fromJSON --input=%t/profile.json --output=%t/profile.ctxprofdata +; +; Disable pre-inline to avoid losing the trivial functions to the preinliner. +; +; RUN: opt -module-summary -disable-preinline -passes='thinlto-pre-link' -use-ctx-profile=%t/profile.ctxprofdata -o %t/m1.bc %t/m1.ll +; RUN: opt -module-summary -disable-preinline -passes='thinlto-pre-link' -use-ctx-profile=%t/profile.ctxprofdata -o %t/m2.bc %t/m2.ll +; +; RUN: rm -rf %t/postlink +; RUN: mkdir %t/postlink +; RUN: llvm-lto2 run %t/m1.bc %t/m2.bc -o %t/ -thinlto-distributed-indexes \ +; RUN: -r %t/m1.bc,f1,plx \ +; RUN: -r %t/m2.bc,f1 \ +; RUN: -r %t/m2.bc,entrypoint,plx +; RUN: opt --passes='function-import,require,print' \ +; RUN: -summary-file=%t/m2.bc.thinlto.bc -use-ctx-profile=%t/profile.ctxprofdata %t/m2.bc \ +; RUN: -S -o %t/m2.post.ll 2> %t/profile.txt +; RUN: diff %t/expected.txt %t/profile.txt +;--- m1.ll +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-linux-gnu" + +define private void @f2() { + ret void +} + +define void @f1() { + call void @f2() + ret void +} + +;--- m2.ll +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-linux-gnu" + +declare void @f1() + +define void @entrypoint() { + call void @f1() + ret void +} +;--- profile.json +[ + { + "Callsites": [ + [ + { + "Callsites": [ + [ + { + "Counters": [ + 10 + ], + "Guid": 5967942613276634709 + } + ] + ], + "Counters": [ + 7 + ], + "Guid": 2072045998141807037 + } + ] + ], + "Counters": [ + 1 + ], + "Guid": 10507721908651011566 + } +] +;--- expected.txt +Function Info: +5967942613276634709 : f2.llvm.0. MaxCounterID: 1. MaxCallsiteID: 0 +10507721908651011566 : entrypoint. MaxCounterID: 1. MaxCallsiteID: 1 +2072045998141807037 : f1. MaxCounterID: 1. MaxCallsiteID: 1 + +Current Profile: +[ + { + "Callsites": [ + [ + { + "Callsites": [ + [ + { + "Counters": [ + 10 + ], + "Guid": 5967942613276634709 + } + ] + ], + "Counters": [ + 7 + ], + "Guid": 2072045998141807037 + } + ] + ], + "Counters": [ + 1 + ], + "Guid": 10507721908651011566 + } +] diff --git a/llvm/test/Analysis/CtxProfAnalysis/load.ll b/llvm/test/Analysis/CtxProfAnalysis/load.ll index 9cd78cfef187b..c298c5b2a331a 100644 --- a/llvm/test/Analysis/CtxProfAnalysis/load.ll +++ b/llvm/test/Analysis/CtxProfAnalysis/load.ll @@ -3,14 +3,20 @@ ; RUN: split-file %s %t ; RUN: llvm-ctxprof-util fromJSON --input=%t/profile.json --output=%t/profile.ctxprofdata ; RUN: not opt -passes='require,print' \ -; RUN: %t/empty.ll -S 2>&1 | FileCheck %s --check-prefix=NO-FILE +; RUN: %t/example.ll -S 2>&1 | FileCheck %s --check-prefix=NO-FILE ; RUN: not opt -passes='require,print' \ -; RUN: -use-ctx-profile=does_not_exist.ctxprofdata %t/empty.ll -S 2>&1 | FileCheck %s --check-prefix=NO-FILE +; RUN: -use-ctx-profile=does_not_exist.ctxprofdata %t/example.ll -S 2>&1 | FileCheck %s --check-prefix=NO-FILE +; RUN: opt -module-summary -passes='thinlto-pre-link' \ +; RUN: -use-ctx-profile=%t/profile.ctxprofdata %t/example.ll -S -o %t/prelink.ll + +; disable preinline to get the same functions we started with. +; RUN: opt -module-summary -passes='thinlto-pre-link' -use-ctx-profile=%t/profile.ctxprofdata \ +; RUN: -disable-preinline %t/example.ll -S -o %t/prelink.ll ; RUN: opt -passes='require,print' \ -; RUN: -use-ctx-profile=%t/profile.ctxprofdata %t/empty.ll -S 2> %t/output.json -; RUN: diff %t/profile.json %t/output.json +; RUN: -use-ctx-profile=%t/profile.ctxprofdata %t/prelink.ll -S 2> %t/output.txt +; RUN: diff %t/expected-profile-output.txt %t/output.txt ; NO-FILE: error: could not open contextual profile file ; @@ -18,41 +24,101 @@ ; output it from opt. ;--- profile.json [ + { + "Counters": [ + 9 + ], + "Guid": 12341 + }, + { + "Counters": [ + 5 + ], + "Guid": 12074870348631550642 + }, { "Callsites": [ - [], [ { "Counters": [ - 4, - 5 + 6, + 7 ], - "Guid": 2000 - }, + "Guid": 728453322856651412 + } + ] + ], + "Counters": [ + 1 + ], + "Guid": 11872291593386833696 + } +] +;--- expected-profile-output.txt +Function Info: +4909520559318251808 : an_entrypoint. MaxCounterID: 2. MaxCallsiteID: 1 +12074870348631550642 : another_entrypoint_no_callees. MaxCounterID: 1. MaxCallsiteID: 0 +11872291593386833696 : foo. MaxCounterID: 1. MaxCallsiteID: 1 + +Current Profile: +[ + { + "Callsites": [ + [ { "Counters": [ 6, - 7, - 8 + 7 ], - "Guid": 18446744073709551613 + "Guid": 728453322856651412 } ] ], "Counters": [ - 1, - 2, - 3 + 1 ], - "Guid": 1000 + "Guid": 11872291593386833696 }, { "Counters": [ - 5, - 9, - 10 + 5 ], - "Guid": 18446744073709551612 + "Guid": 12074870348631550642 } ] -;--- empty.ll +;--- example.ll +declare void @bar() + +define private void @foo(i32 %a, ptr %fct) { + %t = icmp eq i32 %a, 0 + br i1 %t, label %yes, label %no +yes: + call void %fct(i32 %a) + br label %exit +no: + call void @bar() + br label %exit +exit: + ret void +} + +define void @an_entrypoint(i32 %a) { + %t = icmp eq i32 %a, 0 + br i1 %t, label %yes, label %no + +yes: + call void @foo(i32 1, ptr null) + ret void +no: + ret void +} + +define void @another_entrypoint_no_callees(i32 %a) { + %t = icmp eq i32 %a, 0 + br i1 %t, label %yes, label %no + +yes: + ret void +no: + ret void +} diff --git a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll index 56c7c7519f694..fa4d0700a11d1 100644 --- a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll +++ b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 4 ; RUN: opt -passes=pgo-instr-gen -profile-context-root=an_entrypoint \ ; RUN: -S < %s | FileCheck --check-prefix=INSTRUMENT %s -; RUN: opt -passes=pgo-instr-gen,ctx-instr-lower -profile-context-root=an_entrypoint \ +; RUN: opt -passes=pgo-instr-gen,assign-guid,ctx-instr-lower -profile-context-root=an_entrypoint \ ; RUN: -profile-context-root=another_entrypoint_no_callees \ ; RUN: -S < %s | FileCheck --check-prefix=LOWERING %s @@ -46,7 +46,7 @@ define void @foo(i32 %a, ptr %fct) { ; INSTRUMENT-NEXT: ret void ; ; LOWERING-LABEL: define void @foo( -; LOWERING-SAME: i32 [[A:%.*]], ptr [[FCT:%.*]]) { +; LOWERING-SAME: i32 [[A:%.*]], ptr [[FCT:%.*]]) !unique_id [[META0:![0-9]+]] { ; LOWERING-NEXT: [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @foo, i64 6699318081062747564, i32 2, i32 2) ; LOWERING-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64 ; LOWERING-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], 1 @@ -104,7 +104,7 @@ define void @an_entrypoint(i32 %a) { ; INSTRUMENT-NEXT: ret void ; ; LOWERING-LABEL: define void @an_entrypoint( -; LOWERING-SAME: i32 [[A:%.*]]) { +; LOWERING-SAME: i32 [[A:%.*]]) !unique_id [[META1:![0-9]+]] { ; LOWERING-NEXT: [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_start_context(ptr @an_entrypoint_ctx_root, i64 4909520559318251808, i32 2, i32 1) ; LOWERING-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64 ; LOWERING-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], 1 @@ -154,7 +154,7 @@ define void @another_entrypoint_no_callees(i32 %a) { ; INSTRUMENT-NEXT: ret void ; ; LOWERING-LABEL: define void @another_entrypoint_no_callees( -; LOWERING-SAME: i32 [[A:%.*]]) { +; LOWERING-SAME: i32 [[A:%.*]]) !unique_id [[META2:![0-9]+]] { ; LOWERING-NEXT: [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_start_context(ptr @another_entrypoint_no_callees_ctx_root, i64 -6371873725078000974, i32 2, i32 0) ; LOWERING-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64 ; LOWERING-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], -2 @@ -188,7 +188,7 @@ define void @simple(i32 %a) { ; INSTRUMENT-NEXT: ret void ; ; LOWERING-LABEL: define void @simple( -; LOWERING-SAME: i32 [[A:%.*]]) { +; LOWERING-SAME: i32 [[A:%.*]]) !unique_id [[META3:![0-9]+]] { ; LOWERING-NEXT: [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @simple, i64 -3006003237940970099, i32 1, i32 0) ; LOWERING-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64 ; LOWERING-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], -2 @@ -212,7 +212,7 @@ define i32 @no_callsites(i32 %a) { ; INSTRUMENT-NEXT: ret i32 0 ; ; LOWERING-LABEL: define i32 @no_callsites( -; LOWERING-SAME: i32 [[A:%.*]]) { +; LOWERING-SAME: i32 [[A:%.*]]) !unique_id [[META4:![0-9]+]] { ; LOWERING-NEXT: [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @no_callsites, i64 5679753335911435902, i32 2, i32 0) ; LOWERING-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64 ; LOWERING-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], -2 @@ -243,7 +243,8 @@ define void @no_counters() { ; INSTRUMENT-NEXT: call void @bar() ; INSTRUMENT-NEXT: ret void ; -; LOWERING-LABEL: define void @no_counters() { +; LOWERING-LABEL: define void @no_counters( +; LOWERING-SAME: ) !unique_id [[META5:![0-9]+]] { ; LOWERING-NEXT: [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @no_counters, i64 5458232184388660970, i32 1, i32 1) ; LOWERING-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64 ; LOWERING-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], 1 @@ -263,8 +264,15 @@ define void @no_counters() { ret void } ;. -; INSTRUMENT: attributes #[[ATTR0:[0-9]+]] = { nounwind } -;. ; LOWERING: attributes #[[ATTR0:[0-9]+]] = { nounwind } ; LOWERING: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } ;. +; INSTRUMENT: attributes #[[ATTR0:[0-9]+]] = { nounwind } +;. +; LOWERING: [[META0]] = !{i64 6699318081062747564} +; LOWERING: [[META1]] = !{i64 4909520559318251808} +; LOWERING: [[META2]] = !{i64 -6371873725078000974} +; LOWERING: [[META3]] = !{i64 -3006003237940970099} +; LOWERING: [[META4]] = !{i64 5679753335911435902} +; LOWERING: [[META5]] = !{i64 5458232184388660970} +;. diff --git a/llvm/test/Transforms/PGOProfile/ctx-prof-use-prelink.ll b/llvm/test/Transforms/PGOProfile/ctx-prof-use-prelink.ll index 18ac2f92aa39d..b0143dccaccd0 100644 --- a/llvm/test/Transforms/PGOProfile/ctx-prof-use-prelink.ll +++ b/llvm/test/Transforms/PGOProfile/ctx-prof-use-prelink.ll @@ -12,7 +12,7 @@ declare void @bar() ;. define void @foo(i32 %a, ptr %fct) { ; CHECK-LABEL: define void @foo( -; CHECK-SAME: i32 [[A:%.*]], ptr [[FCT:%.*]]) local_unnamed_addr { +; CHECK-SAME: i32 [[A:%.*]], ptr [[FCT:%.*]]) local_unnamed_addr !unique_id [[META0:![0-9]+]] { ; CHECK-NEXT: call void @llvm.instrprof.increment(ptr @__profn_foo, i64 728453322856651412, i32 2, i32 0) ; CHECK-NEXT: [[T:%.*]] = icmp eq i32 [[A]], 0 ; CHECK-NEXT: br i1 [[T]], label %[[YES:.*]], label %[[NO:.*]] @@ -42,3 +42,5 @@ exit: ;. ; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind } ;. +; CHECK: [[META0]] = !{i64 6699318081062747564} +;. From 58330d2d0f7fae0c3cba33e6ba4833165f031737 Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Mon, 12 Aug 2024 15:20:23 -0700 Subject: [PATCH 2/6] comments --- llvm/include/llvm/Analysis/CtxProfAnalysis.h | 11 +++++++---- .../Transforms/Instrumentation/PGOCtxProfLowering.h | 4 +--- llvm/lib/Analysis/CtxProfAnalysis.cpp | 5 +++-- .../Transforms/Instrumentation/PGOCtxProfLowering.cpp | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/llvm/include/llvm/Analysis/CtxProfAnalysis.h b/llvm/include/llvm/Analysis/CtxProfAnalysis.h index 18feb88b53b54..a8d82980a798b 100644 --- a/llvm/include/llvm/Analysis/CtxProfAnalysis.h +++ b/llvm/include/llvm/Analysis/CtxProfAnalysis.h @@ -34,7 +34,8 @@ class PGOContextualProfile { // we'll need when we maintain the profiles during IPO transformations. DenseMap FuncInfo; - GlobalValue::GUID getKnownGUID(const Function &F) const; + /// Get the GUID of this Function if it's defined in this module. + GlobalValue::GUID getDefinedFunctionGUID(const Function &F) const; // This is meant to be constructed from CtxProfAnalysis, which will also set // its state piecemeal. @@ -50,16 +51,18 @@ class PGOContextualProfile { return *Profiles; } - bool isFunctionKnown(const Function &F) const { return getKnownGUID(F) != 0; } + bool isFunctionKnown(const Function &F) const { + return getDefinedFunctionGUID(F) != 0; + } uint32_t allocateNextCounterIndex(const Function &F) { assert(isFunctionKnown(F)); - return FuncInfo.find(getKnownGUID(F))->second.NextCounterIndex++; + return FuncInfo.find(getDefinedFunctionGUID(F))->second.NextCounterIndex++; } uint32_t allocateNextCallsiteIndex(const Function &F) { assert(isFunctionKnown(F)); - return FuncInfo.find(getKnownGUID(F))->second.NextCallsiteIndex++; + return FuncInfo.find(getDefinedFunctionGUID(F))->second.NextCallsiteIndex++; } bool invalidate(Module &, const PreservedAnalyses &PA, diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h b/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h index 57486349a7f84..4beb5c265f18b 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h +++ b/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h @@ -28,9 +28,7 @@ class PGOCtxProfLoweringPass : public PassInfoMixin { /// Assign a GUID to functions as metadata. GUID calculation takes linkage into /// account, which may change especially through and after thinlto. By /// pre-computing and assigning as metadata, this mechanism is resilient to such -/// changes (as well as name changes e.g. suffix ".llvm." additions). It's -/// arguably a much simpler mechanism than PGO's current GV-based one, and can -/// be made available more broadly. +/// changes (as well as name changes e.g. suffix ".llvm." additions). // FIXME(mtrofin): we can generalize this mechanism to calculate a GUID early in // the pass pipeline, associate it with any Global Value, and then use it for diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp index fcbdc4409d210..2200f31ef58a8 100644 --- a/llvm/lib/Analysis/CtxProfAnalysis.cpp +++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp @@ -90,7 +90,7 @@ PGOContextualProfile CtxProfAnalysis::run(Module &M, if (F.isDeclaration()) continue; auto GUID = AssignUniqueIDPass::getGUID(F); - assert(GUID); + assert(GUID && "guid not found for defined function"); const auto &Entry = F.begin(); uint32_t MaxCounters = 0; // we expect at least a counter. for (const auto &I : *Entry) @@ -127,7 +127,8 @@ PGOContextualProfile CtxProfAnalysis::run(Module &M, return Result; } -GlobalValue::GUID PGOContextualProfile::getKnownGUID(const Function &F) const { +GlobalValue::GUID +PGOContextualProfile::getDefinedFunctionGUID(const Function &F) const { if (auto It = FuncInfo.find(AssignUniqueIDPass::getGUID(F)); It != FuncInfo.end()) return It->first; diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp index b938fad4615df..2031a5fb731f2 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp @@ -375,7 +375,7 @@ GlobalValue::GUID AssignUniqueIDPass::getGUID(const Function &F) { return GlobalValue::getGUID(F.getGlobalIdentifier()); } auto *MD = F.getMetadata(GUIDMetadataName); - assert(MD); + assert(MD && "unique_id not found for defined function"); return cast(cast(MD->getOperand(0)) ->getValue() ->stripPointerCasts()) From 862842b6582d001b7e5239ec9671930281b21511 Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Tue, 13 Aug 2024 10:29:25 -0700 Subject: [PATCH 3/6] addressed comments --- llvm/include/llvm/Analysis/CtxProfAnalysis.h | 2 +- llvm/lib/Analysis/CtxProfAnalysis.cpp | 2 +- llvm/lib/Passes/PassBuilderPipelines.cpp | 2 ++ llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll | 15 +++++++++------ llvm/test/Analysis/CtxProfAnalysis/load.ll | 7 ++++--- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/llvm/include/llvm/Analysis/CtxProfAnalysis.h b/llvm/include/llvm/Analysis/CtxProfAnalysis.h index a8d82980a798b..f20734bc174e4 100644 --- a/llvm/include/llvm/Analysis/CtxProfAnalysis.h +++ b/llvm/include/llvm/Analysis/CtxProfAnalysis.h @@ -9,7 +9,7 @@ #ifndef LLVM_ANALYSIS_CTXPROFANALYSIS_H #define LLVM_ANALYSIS_CTXPROFANALYSIS_H -#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/PassManager.h" #include "llvm/ProfileData/PGOCtxProfReader.h" diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp index 2200f31ef58a8..ec5dbdeff71b6 100644 --- a/llvm/lib/Analysis/CtxProfAnalysis.cpp +++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp @@ -120,7 +120,7 @@ PGOContextualProfile CtxProfAnalysis::run(Module &M, // .Profiles. // Trim first the roots that aren't in this module. DenseSet ProfiledGUIDs; - for (auto &[RootGuid, Tree] : llvm::make_early_inc_range(*MaybeCtx)) + for (auto &[RootGuid, _] : llvm::make_early_inc_range(*MaybeCtx)) if (!Result.FuncInfo.contains(RootGuid)) MaybeCtx->erase(RootGuid); Result.Profiles = std::move(*MaybeCtx); diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index e36d105fa1818..c14ec3ed1853a 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -1196,6 +1196,8 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, // In pre-link, we just want the instrumented IR. We use the contextual // profile in the post-thinlink phase. // The instrumentation will be removed in post-thinlink after IPO. + // FIXME(mtrofin): move AssignUniqueID if there is agreement to use this + // mechanism for GUIDs. MPM.addPass(AssignUniqueIDPass()); if (IsCtxProfUse) return MPM; diff --git a/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll b/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll index 282d68b9f0665..5878546a5c0d5 100644 --- a/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll +++ b/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll @@ -6,14 +6,15 @@ ; ; RUN: llvm-ctxprof-util fromJSON --input=%t/profile.json --output=%t/profile.ctxprofdata ; -; Disable pre-inline to avoid losing the trivial functions to the preinliner. -; -; RUN: opt -module-summary -disable-preinline -passes='thinlto-pre-link' -use-ctx-profile=%t/profile.ctxprofdata -o %t/m1.bc %t/m1.ll -; RUN: opt -module-summary -disable-preinline -passes='thinlto-pre-link' -use-ctx-profile=%t/profile.ctxprofdata -o %t/m2.bc %t/m2.ll +; RUN: opt -module-summary -passes='thinlto-pre-link' -use-ctx-profile=%t/profile.ctxprofdata -o %t/m1.bc %t/m1.ll +; RUN: opt -module-summary -passes='thinlto-pre-link' -use-ctx-profile=%t/profile.ctxprofdata -o %t/m2.bc %t/m2.ll ; ; RUN: rm -rf %t/postlink ; RUN: mkdir %t/postlink +; +; ; RUN: llvm-lto2 run %t/m1.bc %t/m2.bc -o %t/ -thinlto-distributed-indexes \ +; RUN: -use-ctx-profile=%t/profile.ctxprofdata \ ; RUN: -r %t/m1.bc,f1,plx \ ; RUN: -r %t/m2.bc,f1 \ ; RUN: -r %t/m2.bc,entrypoint,plx @@ -25,15 +26,17 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-linux-gnu" -define private void @f2() { +define private void @f2() #0 { ret void } -define void @f1() { +define void @f1() #0 { call void @f2() ret void } +attributes #0 = { noinline } + ;--- m2.ll target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-linux-gnu" diff --git a/llvm/test/Analysis/CtxProfAnalysis/load.ll b/llvm/test/Analysis/CtxProfAnalysis/load.ll index c298c5b2a331a..27098f780f406 100644 --- a/llvm/test/Analysis/CtxProfAnalysis/load.ll +++ b/llvm/test/Analysis/CtxProfAnalysis/load.ll @@ -11,9 +11,8 @@ ; RUN: opt -module-summary -passes='thinlto-pre-link' \ ; RUN: -use-ctx-profile=%t/profile.ctxprofdata %t/example.ll -S -o %t/prelink.ll -; disable preinline to get the same functions we started with. ; RUN: opt -module-summary -passes='thinlto-pre-link' -use-ctx-profile=%t/profile.ctxprofdata \ -; RUN: -disable-preinline %t/example.ll -S -o %t/prelink.ll +; RUN: %t/example.ll -S -o %t/prelink.ll ; RUN: opt -passes='require,print' \ ; RUN: -use-ctx-profile=%t/profile.ctxprofdata %t/prelink.ll -S 2> %t/output.txt ; RUN: diff %t/expected-profile-output.txt %t/output.txt @@ -89,7 +88,7 @@ Current Profile: ;--- example.ll declare void @bar() -define private void @foo(i32 %a, ptr %fct) { +define private void @foo(i32 %a, ptr %fct) #0 { %t = icmp eq i32 %a, 0 br i1 %t, label %yes, label %no yes: @@ -122,3 +121,5 @@ yes: no: ret void } + +attributes #0 = { noinline } \ No newline at end of file From 9ef952e73f4d6153ada3cc77fe5699f340e28d99 Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Tue, 13 Aug 2024 21:17:22 -0700 Subject: [PATCH 4/6] moved AssignUniqueIDPass to Analysis so that llvm-link can build (also makes more sense) --- llvm/include/llvm/Analysis/CtxProfAnalysis.h | 19 +++++++++++ .../Instrumentation/PGOCtxProfLowering.h | 19 ----------- llvm/lib/Analysis/CtxProfAnalysis.cpp | 30 +++++++++++++++++ llvm/lib/Passes/PassBuilderPipelines.cpp | 1 + .../Instrumentation/PGOCtxProfLowering.cpp | 32 +------------------ 5 files changed, 51 insertions(+), 50 deletions(-) diff --git a/llvm/include/llvm/Analysis/CtxProfAnalysis.h b/llvm/include/llvm/Analysis/CtxProfAnalysis.h index f20734bc174e4..66f1539a24669 100644 --- a/llvm/include/llvm/Analysis/CtxProfAnalysis.h +++ b/llvm/include/llvm/Analysis/CtxProfAnalysis.h @@ -96,5 +96,24 @@ class CtxProfAnalysisPrinterPass PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); static bool isRequired() { return true; } }; + +/// Assign a GUID to functions as metadata. GUID calculation takes linkage into +/// account, which may change especially through and after thinlto. By +/// pre-computing and assigning as metadata, this mechanism is resilient to such +/// changes (as well as name changes e.g. suffix ".llvm." additions). + +// FIXME(mtrofin): we can generalize this mechanism to calculate a GUID early in +// the pass pipeline, associate it with any Global Value, and then use it for +// PGO and ThinLTO. +// At that point, this should be moved elsewhere. +class AssignUniqueIDPass : public PassInfoMixin { +public: + explicit AssignUniqueIDPass() = default; + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); + static const char *GUIDMetadataName; + // This should become GlobalValue::getGUID + static uint64_t getGUID(const Function &F); +}; + } // namespace llvm #endif // LLVM_ANALYSIS_CTXPROFANALYSIS_H diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h b/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h index 4beb5c265f18b..f127d16b8de12 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h +++ b/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h @@ -24,24 +24,5 @@ class PGOCtxProfLoweringPass : public PassInfoMixin { PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); }; - -/// Assign a GUID to functions as metadata. GUID calculation takes linkage into -/// account, which may change especially through and after thinlto. By -/// pre-computing and assigning as metadata, this mechanism is resilient to such -/// changes (as well as name changes e.g. suffix ".llvm." additions). - -// FIXME(mtrofin): we can generalize this mechanism to calculate a GUID early in -// the pass pipeline, associate it with any Global Value, and then use it for -// PGO and ThinLTO. -// At that point, this should be moved elsewhere. -class AssignUniqueIDPass : public PassInfoMixin { -public: - explicit AssignUniqueIDPass() = default; - PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); - static const char *GUIDMetadataName; - // This should become GlobalValue::getGUID - static uint64_t getGUID(const Function &F); -}; - } // namespace llvm #endif diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp index ec5dbdeff71b6..037bfbde380b5 100644 --- a/llvm/lib/Analysis/CtxProfAnalysis.cpp +++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp @@ -66,6 +66,36 @@ Value toJSON(const PGOCtxProfContext::CallTargetMapTy &P) { } // namespace json } // namespace llvm +const char *AssignUniqueIDPass::GUIDMetadataName = "unique_id"; + +PreservedAnalyses AssignUniqueIDPass::run(Module &M, + ModuleAnalysisManager &MAM) { + for (auto &F : M.functions()) { + if (F.isDeclaration()) + continue; + const GlobalValue::GUID GUID = F.getGUID(); + assert(!F.getMetadata(GUIDMetadataName) || + GUID == AssignUniqueIDPass::getGUID(F)); + F.setMetadata(GUIDMetadataName, + MDNode::get(M.getContext(), + {ConstantAsMetadata::get(ConstantInt::get( + Type::getInt64Ty(M.getContext()), GUID))})); + } + return PreservedAnalyses::none(); +} + +GlobalValue::GUID AssignUniqueIDPass::getGUID(const Function &F) { + if (F.isDeclaration()) { + assert(GlobalValue::isExternalLinkage(F.getLinkage())); + return GlobalValue::getGUID(F.getGlobalIdentifier()); + } + auto *MD = F.getMetadata(GUIDMetadataName); + assert(MD && "unique_id not found for defined function"); + return cast(cast(MD->getOperand(0)) + ->getValue() + ->stripPointerCasts()) + ->getZExtValue(); +} AnalysisKey CtxProfAnalysis::Key; PGOContextualProfile CtxProfAnalysis::run(Module &M, diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index c14ec3ed1853a..aefb1d2b334f7 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -18,6 +18,7 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/CGSCCPassManager.h" +#include "llvm/Analysis/CtxProfAnalysis.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/InlineAdvisor.h" #include "llvm/Analysis/ProfileSummaryInfo.h" diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp index 2031a5fb731f2..5068efd21bc21 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp @@ -8,6 +8,7 @@ // #include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h" +#include "llvm/Analysis/CtxProfAnalysis.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/IR/Analysis.h" #include "llvm/IR/DiagnosticInfo.h" @@ -350,34 +351,3 @@ bool CtxInstrumentationLowerer::lowerFunction(Function &F) { F.getName()); return true; } - -const char *AssignUniqueIDPass::GUIDMetadataName = "unique_id"; - -PreservedAnalyses AssignUniqueIDPass::run(Module &M, - ModuleAnalysisManager &MAM) { - for (auto &F : M.functions()) { - if (F.isDeclaration()) - continue; - const GlobalValue::GUID GUID = F.getGUID(); - assert(!F.getMetadata(GUIDMetadataName) || - GUID == AssignUniqueIDPass::getGUID(F)); - F.setMetadata(GUIDMetadataName, - MDNode::get(M.getContext(), - {ConstantAsMetadata::get(ConstantInt::get( - Type::getInt64Ty(M.getContext()), GUID))})); - } - return PreservedAnalyses::none(); -} - -GlobalValue::GUID AssignUniqueIDPass::getGUID(const Function &F) { - if (F.isDeclaration()) { - assert(GlobalValue::isExternalLinkage(F.getLinkage())); - return GlobalValue::getGUID(F.getGlobalIdentifier()); - } - auto *MD = F.getMetadata(GUIDMetadataName); - assert(MD && "unique_id not found for defined function"); - return cast(cast(MD->getOperand(0)) - ->getValue() - ->stripPointerCasts()) - ->getZExtValue(); -} \ No newline at end of file From 4cbb13a1af68cf266fa79f35f7cf7557dec3869d Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Wed, 14 Aug 2024 08:57:46 -0700 Subject: [PATCH 5/6] Make tests resilient to test path AssignUniqueIDPass skips over functions with assigned ID. We can manually set the ID for local linkage functions in test. Otherwise, as their GUID is computed in a way that incorporates the module name (i.e. path), the tests would depend on the build path. --- llvm/lib/Analysis/CtxProfAnalysis.cpp | 4 ++-- llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll | 6 ++++-- llvm/test/Analysis/CtxProfAnalysis/load.ll | 8 +++++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp index 037bfbde380b5..2e407944d309f 100644 --- a/llvm/lib/Analysis/CtxProfAnalysis.cpp +++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp @@ -73,9 +73,9 @@ PreservedAnalyses AssignUniqueIDPass::run(Module &M, for (auto &F : M.functions()) { if (F.isDeclaration()) continue; + if (F.getMetadata(GUIDMetadataName)) + continue; const GlobalValue::GUID GUID = F.getGUID(); - assert(!F.getMetadata(GUIDMetadataName) || - GUID == AssignUniqueIDPass::getGUID(F)); F.setMetadata(GUIDMetadataName, MDNode::get(M.getContext(), {ConstantAsMetadata::get(ConstantInt::get( diff --git a/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll b/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll index 5878546a5c0d5..39d851df1be25 100644 --- a/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll +++ b/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll @@ -1,5 +1,6 @@ ; REQUIRES: x86_64-linux - +; +; RUN: rm -rf %t ; RUN: split-file %s %t ; ; Test that the GUID metadata survives through thinlink. @@ -26,7 +27,7 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-linux-gnu" -define private void @f2() #0 { +define private void @f2() #0 !unique_id !0 { ret void } @@ -36,6 +37,7 @@ define void @f1() #0 { } attributes #0 = { noinline } +!0 = !{ i64 5967942613276634709 } ;--- m2.ll target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" diff --git a/llvm/test/Analysis/CtxProfAnalysis/load.ll b/llvm/test/Analysis/CtxProfAnalysis/load.ll index 27098f780f406..9d2b022615626 100644 --- a/llvm/test/Analysis/CtxProfAnalysis/load.ll +++ b/llvm/test/Analysis/CtxProfAnalysis/load.ll @@ -1,5 +1,6 @@ ; REQUIRES: x86_64-linux - +; +; RUN: rm -rf %t ; RUN: split-file %s %t ; RUN: llvm-ctxprof-util fromJSON --input=%t/profile.json --output=%t/profile.ctxprofdata ; RUN: not opt -passes='require,print' \ @@ -88,7 +89,7 @@ Current Profile: ;--- example.ll declare void @bar() -define private void @foo(i32 %a, ptr %fct) #0 { +define private void @foo(i32 %a, ptr %fct) #0 !unique_id !0 { %t = icmp eq i32 %a, 0 br i1 %t, label %yes, label %no yes: @@ -122,4 +123,5 @@ no: ret void } -attributes #0 = { noinline } \ No newline at end of file +attributes #0 = { noinline } +!0 = !{ i64 11872291593386833696 } \ No newline at end of file From 4a596bda9ec179510497dd88e85300e0e12cb739 Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Wed, 14 Aug 2024 14:09:51 -0700 Subject: [PATCH 6/6] comments: mostly renamed stuff, also setting `source_filename` in test to keep guids stable --- llvm/include/llvm/Analysis/CtxProfAnalysis.h | 7 +++++-- llvm/lib/Analysis/CtxProfAnalysis.cpp | 14 ++++++-------- llvm/lib/Passes/PassBuilderPipelines.cpp | 4 ++-- llvm/lib/Passes/PassRegistry.def | 2 +- .../Instrumentation/PGOCtxProfLowering.cpp | 2 +- llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll | 14 +++++++++----- llvm/test/Analysis/CtxProfAnalysis/load.ll | 2 +- .../Transforms/PGOProfile/ctx-instrumentation.ll | 12 ++++++------ .../Transforms/PGOProfile/ctx-prof-use-prelink.ll | 2 +- 9 files changed, 32 insertions(+), 27 deletions(-) diff --git a/llvm/include/llvm/Analysis/CtxProfAnalysis.h b/llvm/include/llvm/Analysis/CtxProfAnalysis.h index 66f1539a24669..f0e2aeb0f92f7 100644 --- a/llvm/include/llvm/Analysis/CtxProfAnalysis.h +++ b/llvm/include/llvm/Analysis/CtxProfAnalysis.h @@ -106,9 +106,12 @@ class CtxProfAnalysisPrinterPass // the pass pipeline, associate it with any Global Value, and then use it for // PGO and ThinLTO. // At that point, this should be moved elsewhere. -class AssignUniqueIDPass : public PassInfoMixin { +class AssignGUIDPass : public PassInfoMixin { public: - explicit AssignUniqueIDPass() = default; + explicit AssignGUIDPass() = default; + + /// Assign a GUID *if* one is not already assign, as a function metadata named + /// `GUIDMetadataName`. PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); static const char *GUIDMetadataName; // This should become GlobalValue::getGUID diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp index 2e407944d309f..5bf336dd31115 100644 --- a/llvm/lib/Analysis/CtxProfAnalysis.cpp +++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp @@ -66,10 +66,9 @@ Value toJSON(const PGOCtxProfContext::CallTargetMapTy &P) { } // namespace json } // namespace llvm -const char *AssignUniqueIDPass::GUIDMetadataName = "unique_id"; +const char *AssignGUIDPass::GUIDMetadataName = "guid"; -PreservedAnalyses AssignUniqueIDPass::run(Module &M, - ModuleAnalysisManager &MAM) { +PreservedAnalyses AssignGUIDPass::run(Module &M, ModuleAnalysisManager &MAM) { for (auto &F : M.functions()) { if (F.isDeclaration()) continue; @@ -84,13 +83,13 @@ PreservedAnalyses AssignUniqueIDPass::run(Module &M, return PreservedAnalyses::none(); } -GlobalValue::GUID AssignUniqueIDPass::getGUID(const Function &F) { +GlobalValue::GUID AssignGUIDPass::getGUID(const Function &F) { if (F.isDeclaration()) { assert(GlobalValue::isExternalLinkage(F.getLinkage())); return GlobalValue::getGUID(F.getGlobalIdentifier()); } auto *MD = F.getMetadata(GUIDMetadataName); - assert(MD && "unique_id not found for defined function"); + assert(MD && "guid not found for defined function"); return cast(cast(MD->getOperand(0)) ->getValue() ->stripPointerCasts()) @@ -119,7 +118,7 @@ PGOContextualProfile CtxProfAnalysis::run(Module &M, for (const auto &F : M) { if (F.isDeclaration()) continue; - auto GUID = AssignUniqueIDPass::getGUID(F); + auto GUID = AssignGUIDPass::getGUID(F); assert(GUID && "guid not found for defined function"); const auto &Entry = F.begin(); uint32_t MaxCounters = 0; // we expect at least a counter. @@ -159,8 +158,7 @@ PGOContextualProfile CtxProfAnalysis::run(Module &M, GlobalValue::GUID PGOContextualProfile::getDefinedFunctionGUID(const Function &F) const { - if (auto It = FuncInfo.find(AssignUniqueIDPass::getGUID(F)); - It != FuncInfo.end()) + if (auto It = FuncInfo.find(AssignGUIDPass::getGUID(F)); It != FuncInfo.end()) return It->first; return 0; } diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index aefb1d2b334f7..0201e69f3e216 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -1197,9 +1197,9 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, // In pre-link, we just want the instrumented IR. We use the contextual // profile in the post-thinlink phase. // The instrumentation will be removed in post-thinlink after IPO. - // FIXME(mtrofin): move AssignUniqueID if there is agreement to use this + // FIXME(mtrofin): move AssignGUIDPass if there is agreement to use this // mechanism for GUIDs. - MPM.addPass(AssignUniqueIDPass()); + MPM.addPass(AssignGUIDPass()); if (IsCtxProfUse) return MPM; addPostPGOLoopRotation(MPM, Level); diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 6c33e58157f01..efb65d3158068 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -45,9 +45,9 @@ MODULE_ALIAS_ANALYSIS("globals-aa", GlobalsAA()) #endif MODULE_PASS("always-inline", AlwaysInlinerPass()) MODULE_PASS("annotation2metadata", Annotation2MetadataPass()) +MODULE_PASS("assign-guid", AssignGUIDPass()) MODULE_PASS("attributor", AttributorPass()) MODULE_PASS("attributor-light", AttributorLightPass()) -MODULE_PASS("assign-guid", AssignUniqueIDPass()) MODULE_PASS("called-value-propagation", CalledValuePropagationPass()) MODULE_PASS("canonicalize-aliases", CanonicalizeAliasesPass()) MODULE_PASS("check-debugify", NewPMCheckDebugifyPass()) diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp index 5068efd21bc21..9b10cbba84075 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp @@ -226,7 +226,7 @@ bool CtxInstrumentationLowerer::lowerFunction(Function &F) { IRBuilder<> Builder(Mark); - Guid = Builder.getInt64(AssignUniqueIDPass::getGUID(F)); + Guid = Builder.getInt64(AssignGUIDPass::getGUID(F)); // The type of the context of this function is now knowable since we have // NrCallsites and NrCounters. We delcare it here because it's more // convenient - we have the Builder. diff --git a/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll b/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll index 39d851df1be25..0cdf82bd96efc 100644 --- a/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll +++ b/llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll @@ -27,7 +27,9 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-linux-gnu" -define private void @f2() #0 !unique_id !0 { +source_filename = "random_path/m1.cc" + +define private void @f2() #0 !guid !0 { ret void } @@ -37,12 +39,14 @@ define void @f1() #0 { } attributes #0 = { noinline } -!0 = !{ i64 5967942613276634709 } +!0 = !{ i64 3087265239403591524 } ;--- m2.ll target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-linux-gnu" +source_filename = "random_path/m2.cc" + declare void @f1() define void @entrypoint() { @@ -61,7 +65,7 @@ define void @entrypoint() { "Counters": [ 10 ], - "Guid": 5967942613276634709 + "Guid": 3087265239403591524 } ] ], @@ -80,8 +84,8 @@ define void @entrypoint() { ] ;--- expected.txt Function Info: -5967942613276634709 : f2.llvm.0. MaxCounterID: 1. MaxCallsiteID: 0 10507721908651011566 : entrypoint. MaxCounterID: 1. MaxCallsiteID: 1 +3087265239403591524 : f2.llvm.0. MaxCounterID: 1. MaxCallsiteID: 0 2072045998141807037 : f1. MaxCounterID: 1. MaxCallsiteID: 1 Current Profile: @@ -96,7 +100,7 @@ Current Profile: "Counters": [ 10 ], - "Guid": 5967942613276634709 + "Guid": 3087265239403591524 } ] ], diff --git a/llvm/test/Analysis/CtxProfAnalysis/load.ll b/llvm/test/Analysis/CtxProfAnalysis/load.ll index 9d2b022615626..69806e334aaec 100644 --- a/llvm/test/Analysis/CtxProfAnalysis/load.ll +++ b/llvm/test/Analysis/CtxProfAnalysis/load.ll @@ -89,7 +89,7 @@ Current Profile: ;--- example.ll declare void @bar() -define private void @foo(i32 %a, ptr %fct) #0 !unique_id !0 { +define private void @foo(i32 %a, ptr %fct) #0 !guid !0 { %t = icmp eq i32 %a, 0 br i1 %t, label %yes, label %no yes: diff --git a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll index fa4d0700a11d1..a70f94e1521f0 100644 --- a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll +++ b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll @@ -46,7 +46,7 @@ define void @foo(i32 %a, ptr %fct) { ; INSTRUMENT-NEXT: ret void ; ; LOWERING-LABEL: define void @foo( -; LOWERING-SAME: i32 [[A:%.*]], ptr [[FCT:%.*]]) !unique_id [[META0:![0-9]+]] { +; LOWERING-SAME: i32 [[A:%.*]], ptr [[FCT:%.*]]) !guid [[META0:![0-9]+]] { ; LOWERING-NEXT: [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @foo, i64 6699318081062747564, i32 2, i32 2) ; LOWERING-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64 ; LOWERING-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], 1 @@ -104,7 +104,7 @@ define void @an_entrypoint(i32 %a) { ; INSTRUMENT-NEXT: ret void ; ; LOWERING-LABEL: define void @an_entrypoint( -; LOWERING-SAME: i32 [[A:%.*]]) !unique_id [[META1:![0-9]+]] { +; LOWERING-SAME: i32 [[A:%.*]]) !guid [[META1:![0-9]+]] { ; LOWERING-NEXT: [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_start_context(ptr @an_entrypoint_ctx_root, i64 4909520559318251808, i32 2, i32 1) ; LOWERING-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64 ; LOWERING-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], 1 @@ -154,7 +154,7 @@ define void @another_entrypoint_no_callees(i32 %a) { ; INSTRUMENT-NEXT: ret void ; ; LOWERING-LABEL: define void @another_entrypoint_no_callees( -; LOWERING-SAME: i32 [[A:%.*]]) !unique_id [[META2:![0-9]+]] { +; LOWERING-SAME: i32 [[A:%.*]]) !guid [[META2:![0-9]+]] { ; LOWERING-NEXT: [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_start_context(ptr @another_entrypoint_no_callees_ctx_root, i64 -6371873725078000974, i32 2, i32 0) ; LOWERING-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64 ; LOWERING-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], -2 @@ -188,7 +188,7 @@ define void @simple(i32 %a) { ; INSTRUMENT-NEXT: ret void ; ; LOWERING-LABEL: define void @simple( -; LOWERING-SAME: i32 [[A:%.*]]) !unique_id [[META3:![0-9]+]] { +; LOWERING-SAME: i32 [[A:%.*]]) !guid [[META3:![0-9]+]] { ; LOWERING-NEXT: [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @simple, i64 -3006003237940970099, i32 1, i32 0) ; LOWERING-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64 ; LOWERING-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], -2 @@ -212,7 +212,7 @@ define i32 @no_callsites(i32 %a) { ; INSTRUMENT-NEXT: ret i32 0 ; ; LOWERING-LABEL: define i32 @no_callsites( -; LOWERING-SAME: i32 [[A:%.*]]) !unique_id [[META4:![0-9]+]] { +; LOWERING-SAME: i32 [[A:%.*]]) !guid [[META4:![0-9]+]] { ; LOWERING-NEXT: [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @no_callsites, i64 5679753335911435902, i32 2, i32 0) ; LOWERING-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64 ; LOWERING-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], -2 @@ -244,7 +244,7 @@ define void @no_counters() { ; INSTRUMENT-NEXT: ret void ; ; LOWERING-LABEL: define void @no_counters( -; LOWERING-SAME: ) !unique_id [[META5:![0-9]+]] { +; LOWERING-SAME: ) !guid [[META5:![0-9]+]] { ; LOWERING-NEXT: [[TMP1:%.*]] = call ptr @__llvm_ctx_profile_get_context(ptr @no_counters, i64 5458232184388660970, i32 1, i32 1) ; LOWERING-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64 ; LOWERING-NEXT: [[TMP3:%.*]] = and i64 [[TMP2]], 1 diff --git a/llvm/test/Transforms/PGOProfile/ctx-prof-use-prelink.ll b/llvm/test/Transforms/PGOProfile/ctx-prof-use-prelink.ll index b0143dccaccd0..cb8ab78dc0f41 100644 --- a/llvm/test/Transforms/PGOProfile/ctx-prof-use-prelink.ll +++ b/llvm/test/Transforms/PGOProfile/ctx-prof-use-prelink.ll @@ -12,7 +12,7 @@ declare void @bar() ;. define void @foo(i32 %a, ptr %fct) { ; CHECK-LABEL: define void @foo( -; CHECK-SAME: i32 [[A:%.*]], ptr [[FCT:%.*]]) local_unnamed_addr !unique_id [[META0:![0-9]+]] { +; CHECK-SAME: i32 [[A:%.*]], ptr [[FCT:%.*]]) local_unnamed_addr !guid [[META0:![0-9]+]] { ; CHECK-NEXT: call void @llvm.instrprof.increment(ptr @__profn_foo, i64 728453322856651412, i32 2, i32 0) ; CHECK-NEXT: [[T:%.*]] = icmp eq i32 [[A]], 0 ; CHECK-NEXT: br i1 [[T]], label %[[YES:.*]], label %[[NO:.*]]