diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index e765bbf637a66..64f0020a170aa 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -643,7 +643,7 @@ static void addKCFIPass(const Triple &TargetTriple, const LangOptions &LangOpts, // Ensure we lower KCFI operand bundles with -O0. PB.registerOptimizerLastEPCallback( - [&](ModulePassManager &MPM, OptimizationLevel Level) { + [&](ModulePassManager &MPM, OptimizationLevel Level, ThinOrFullLTOPhase) { if (Level == OptimizationLevel::O0 && LangOpts.Sanitize.has(SanitizerKind::KCFI)) MPM.addPass(createModuleToFunctionPassAdaptor(KCFIPass())); @@ -662,8 +662,8 @@ static void addKCFIPass(const Triple &TargetTriple, const LangOptions &LangOpts, static void addSanitizers(const Triple &TargetTriple, const CodeGenOptions &CodeGenOpts, const LangOptions &LangOpts, PassBuilder &PB) { - auto SanitizersCallback = [&](ModulePassManager &MPM, - OptimizationLevel Level) { + auto SanitizersCallback = [&](ModulePassManager &MPM, OptimizationLevel Level, + ThinOrFullLTOPhase) { if (CodeGenOpts.hasSanitizeCoverage()) { auto SancovOpts = getSancovOptsFromCGOpts(CodeGenOpts); MPM.addPass(SanitizerCoveragePass( @@ -749,7 +749,7 @@ static void addSanitizers(const Triple &TargetTriple, PB.registerOptimizerEarlyEPCallback( [SanitizersCallback](ModulePassManager &MPM, OptimizationLevel Level) { ModulePassManager NewMPM; - SanitizersCallback(NewMPM, Level); + SanitizersCallback(NewMPM, Level, ThinOrFullLTOPhase::None); if (!NewMPM.isEmpty()) { // Sanitizers can abandon. NewMPM.addPass(RequireAnalysisPass()); @@ -1018,11 +1018,12 @@ void EmitAssemblyHelper::RunOptimizationPipeline( // TODO: Consider passing the MemoryProfileOutput to the pass builder via // the PGOOptions, and set this up there. if (!CodeGenOpts.MemoryProfileOutput.empty()) { - PB.registerOptimizerLastEPCallback( - [](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass())); - MPM.addPass(ModuleMemProfilerPass()); - }); + PB.registerOptimizerLastEPCallback([](ModulePassManager &MPM, + OptimizationLevel Level, + ThinOrFullLTOPhase) { + MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass())); + MPM.addPass(ModuleMemProfilerPass()); + }); } if (CodeGenOpts.FatLTO) { diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index e1d78a8685aed..ad3901902f784 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -246,8 +246,9 @@ class PassBuilder { /// optimization and code generation without any link-time optimization. It /// typically correspond to frontend "-O[123]" options for optimization /// levels \c O1, \c O2 and \c O3 resp. - ModulePassManager buildPerModuleDefaultPipeline(OptimizationLevel Level, - bool LTOPreLink = false); + ModulePassManager buildPerModuleDefaultPipeline( + OptimizationLevel Level, + ThinOrFullLTOPhase Phase = ThinOrFullLTOPhase::None); /// Build a fat object default optimization pipeline. /// @@ -297,8 +298,9 @@ class PassBuilder { /// Build an O0 pipeline with the minimal semantically required passes. /// /// This should only be used for non-LTO and LTO pre-link pipelines. - ModulePassManager buildO0DefaultPipeline(OptimizationLevel Level, - bool LTOPreLink = false); + ModulePassManager + buildO0DefaultPipeline(OptimizationLevel Level, + ThinOrFullLTOPhase Phase = ThinOrFullLTOPhase::None); /// Build the default `AAManager` with the default alias analysis pipeline /// registered. @@ -497,7 +499,8 @@ class PassBuilder { /// This extension point allows adding optimizations at the very end of the /// function optimization pipeline. void registerOptimizerLastEPCallback( - const std::function &C) { + const std::function &C) { OptimizerLastEPCallbacks.push_back(C); } @@ -630,7 +633,8 @@ class PassBuilder { void invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM, OptimizationLevel Level); void invokeOptimizerLastEPCallbacks(ModulePassManager &MPM, - OptimizationLevel Level); + OptimizationLevel Level, + ThinOrFullLTOPhase Phase); void invokeFullLinkTimeOptimizationEarlyEPCallbacks(ModulePassManager &MPM, OptimizationLevel Level); void invokeFullLinkTimeOptimizationLastEPCallbacks(ModulePassManager &MPM, @@ -755,7 +759,9 @@ class PassBuilder { // Module callbacks SmallVector, 2> OptimizerEarlyEPCallbacks; - SmallVector, 2> + SmallVector, + 2> OptimizerLastEPCallbacks; SmallVector, 2> FullLinkTimeOptimizationEarlyEPCallbacks; diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index adebbb5eeba32..3daa49474887d 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -369,9 +369,10 @@ void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM, C(MPM, Level); } void PassBuilder::invokeOptimizerLastEPCallbacks(ModulePassManager &MPM, - OptimizationLevel Level) { + OptimizationLevel Level, + ThinOrFullLTOPhase Phase) { for (auto &C : OptimizerLastEPCallbacks) - C(MPM, Level); + C(MPM, Level, Phase); } void PassBuilder::invokeFullLinkTimeOptimizationEarlyEPCallbacks( ModulePassManager &MPM, OptimizationLevel Level) { @@ -1539,7 +1540,7 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM), PTO.EagerlyInvalidateAnalyses)); - invokeOptimizerLastEPCallbacks(MPM, Level); + invokeOptimizerLastEPCallbacks(MPM, Level, LTOPhase); // Split out cold code. Splitting is done late to avoid hiding context from // other optimizations and inadvertently regressing performance. The tradeoff @@ -1581,9 +1582,9 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, ModulePassManager PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level, - bool LTOPreLink) { + ThinOrFullLTOPhase Phase) { if (Level == OptimizationLevel::O0) - return buildO0DefaultPipeline(Level, LTOPreLink); + return buildO0DefaultPipeline(Level, Phase); ModulePassManager MPM; @@ -1599,14 +1600,11 @@ PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level, // Apply module pipeline start EP callback. invokePipelineStartEPCallbacks(MPM, Level); - const ThinOrFullLTOPhase LTOPhase = LTOPreLink - ? ThinOrFullLTOPhase::FullLTOPreLink - : ThinOrFullLTOPhase::None; // Add the core simplification pipeline. - MPM.addPass(buildModuleSimplificationPipeline(Level, LTOPhase)); + MPM.addPass(buildModuleSimplificationPipeline(Level, Phase)); // Now add the optimization pipeline. - MPM.addPass(buildModuleOptimizationPipeline(Level, LTOPhase)); + MPM.addPass(buildModuleOptimizationPipeline(Level, Phase)); if (PGOOpt && PGOOpt->PseudoProbeForProfiling && PGOOpt->Action == PGOOptions::SampleUse) @@ -1615,7 +1613,7 @@ PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level, // Emit annotation remarks. addAnnotationRemarksPass(MPM); - if (LTOPreLink) + if (isLTOPreLink(Phase)) addRequiredLTOPreLinkPasses(MPM); return MPM; } @@ -1646,7 +1644,7 @@ PassBuilder::buildFatLTODefaultPipeline(OptimizationLevel Level, bool ThinLTO, ModulePassManager PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) { if (Level == OptimizationLevel::O0) - return buildO0DefaultPipeline(Level, /*LTOPreLink*/true); + return buildO0DefaultPipeline(Level, ThinOrFullLTOPhase::ThinLTOPreLink); ModulePassManager MPM; @@ -1691,7 +1689,8 @@ PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) { // optimization is going to be done in PostLink stage, but clang can't add // callbacks there in case of in-process ThinLTO called by linker. invokeOptimizerEarlyEPCallbacks(MPM, Level); - invokeOptimizerLastEPCallbacks(MPM, Level); + invokeOptimizerLastEPCallbacks(MPM, Level, + ThinOrFullLTOPhase::ThinLTOPreLink); // Emit annotation remarks. addAnnotationRemarksPass(MPM); @@ -1760,7 +1759,7 @@ ModulePassManager PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level) { // FIXME: We should use a customized pre-link pipeline! return buildPerModuleDefaultPipeline(Level, - /* LTOPreLink */ true); + ThinOrFullLTOPhase::FullLTOPreLink); } ModulePassManager @@ -2085,8 +2084,9 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, return MPM; } -ModulePassManager PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level, - bool LTOPreLink) { +ModulePassManager +PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level, + ThinOrFullLTOPhase Phase) { assert(Level == OptimizationLevel::O0 && "buildO0DefaultPipeline should only be used with O0"); @@ -2179,9 +2179,9 @@ ModulePassManager PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level, CoroPM.addPass(GlobalDCEPass()); MPM.addPass(CoroConditionalWrapper(std::move(CoroPM))); - invokeOptimizerLastEPCallbacks(MPM, Level); + invokeOptimizerLastEPCallbacks(MPM, Level, Phase); - if (LTOPreLink) + if (isLTOPreLink(Phase)) addRequiredLTOPreLinkPasses(MPM); MPM.addPass(createModuleToFunctionPassAdaptor(AnnotationRemarksPass())); diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h index 46cc5f349555a..50aef36724f70 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.h +++ b/llvm/lib/Target/AMDGPU/AMDGPU.h @@ -287,8 +287,13 @@ class AMDGPUAttributorPass : public PassInfoMixin { private: TargetMachine &TM; + /// Asserts whether we can assume whole program visibility. + bool HasWholeProgramVisibility = false; + public: - AMDGPUAttributorPass(TargetMachine &TM) : TM(TM){}; + AMDGPUAttributorPass(TargetMachine &TM, + bool HasWholeProgramVisibility = false) + : TM(TM), HasWholeProgramVisibility(HasWholeProgramVisibility) {}; PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp index 29d493db926d9..9557005721cb1 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp @@ -1024,7 +1024,8 @@ static void addPreloadKernArgHint(Function &F, TargetMachine &TM) { } } -static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM) { +static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM, + bool HasWholeProgramVisibility) { SetVector Functions; for (Function &F : M) { if (!F.isIntrinsic()) @@ -1043,6 +1044,7 @@ static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM) { &AAInstanceInfo::ID}); AttributorConfig AC(CGUpdater); + AC.IsClosedWorldModule = HasWholeProgramVisibility; AC.Allowed = &Allowed; AC.IsModulePass = true; AC.DefaultInitializeLiveInternals = false; @@ -1112,7 +1114,7 @@ class AMDGPUAttributorLegacy : public ModulePass { bool runOnModule(Module &M) override { AnalysisGetter AG(this); - return runImpl(M, AG, *TM); + return runImpl(M, AG, *TM, /*HasWholeProgramVisibility=*/false); } void getAnalysisUsage(AnalysisUsage &AU) const override { @@ -1133,8 +1135,9 @@ PreservedAnalyses llvm::AMDGPUAttributorPass::run(Module &M, AnalysisGetter AG(FAM); // TODO: Probably preserves CFG - return runImpl(M, AG, TM) ? PreservedAnalyses::none() - : PreservedAnalyses::all(); + return runImpl(M, AG, TM, HasWholeProgramVisibility) + ? PreservedAnalyses::none() + : PreservedAnalyses::all(); } char AMDGPUAttributorLegacy::ID = 0; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index c8fb68d1c0b0c..50cc2d871d4ec 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -735,12 +735,15 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) { }); // FIXME: Why is AMDGPUAttributor not in CGSCC? - PB.registerOptimizerLastEPCallback( - [this](ModulePassManager &MPM, OptimizationLevel Level) { - if (Level != OptimizationLevel::O0) { - MPM.addPass(AMDGPUAttributorPass(*this)); - } - }); + PB.registerOptimizerLastEPCallback([this](ModulePassManager &MPM, + OptimizationLevel Level, + ThinOrFullLTOPhase Phase) { + if (Level != OptimizationLevel::O0) { + MPM.addPass(AMDGPUAttributorPass( + *this, Phase == ThinOrFullLTOPhase::FullLTOPostLink || + Phase == ThinOrFullLTOPhase::ThinLTOPostLink)); + } + }); PB.registerFullLinkTimeOptimizationLastEPCallback( [this](ModulePassManager &PM, OptimizationLevel Level) { diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp index 91ec905e58f12..acbdbbfa032c1 100644 --- a/llvm/tools/opt/NewPMDriver.cpp +++ b/llvm/tools/opt/NewPMDriver.cpp @@ -306,7 +306,7 @@ static void registerEPCallbacks(PassBuilder &PB) { }); if (tryParsePipelineText(PB, OptimizerLastEPPipeline)) PB.registerOptimizerLastEPCallback( - [&PB](ModulePassManager &PM, OptimizationLevel) { + [&PB](ModulePassManager &PM, OptimizationLevel, ThinOrFullLTOPhase) { ExitOnError Err("Unable to parse OptimizerLastEP pipeline: "); Err(PB.parsePassPipeline(PM, OptimizerLastEPPipeline)); });