diff --git a/llvm/include/llvm/CodeGen/MachinePassManager.h b/llvm/include/llvm/CodeGen/MachinePassManager.h index 662b7e8689717..e69b10b7eaafd 100644 --- a/llvm/include/llvm/CodeGen/MachinePassManager.h +++ b/llvm/include/llvm/CodeGen/MachinePassManager.h @@ -27,6 +27,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/Error.h" +#include "llvm/Target/CGPassBuilderOption.h" #include @@ -143,10 +144,7 @@ class MachineFunctionPassManager using Base = PassManager; public: - MachineFunctionPassManager(bool RequireCodeGenSCCOrder = false, - bool VerifyMachineFunction = false) - : RequireCodeGenSCCOrder(RequireCodeGenSCCOrder), - VerifyMachineFunction(VerifyMachineFunction) {} + MachineFunctionPassManager() : Opt(getCGPassBuilderOption()) {} MachineFunctionPassManager(MachineFunctionPassManager &&) = default; MachineFunctionPassManager & operator=(MachineFunctionPassManager &&) = default; @@ -254,10 +252,7 @@ class MachineFunctionPassManager using PassIndex = decltype(Passes)::size_type; std::map> MachineModulePasses; - // Run codegen in the SCC order. - bool RequireCodeGenSCCOrder; - - bool VerifyMachineFunction; + CGPassBuilderOption Opt; }; } // end namespace llvm diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index c5c53a5bccae7..443daf7504ee8 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -27,8 +27,11 @@ #include "llvm/CodeGen/CallBrPrepare.h" #include "llvm/CodeGen/CodeGenPrepare.h" #include "llvm/CodeGen/DwarfEHPrepare.h" +#include "llvm/CodeGen/ExpandLargeDivRem.h" +#include "llvm/CodeGen/ExpandLargeFpConvert.h" #include "llvm/CodeGen/ExpandMemCmp.h" #include "llvm/CodeGen/ExpandReductions.h" +#include "llvm/CodeGen/ExpandVectorPredication.h" #include "llvm/CodeGen/GCMetadata.h" #include "llvm/CodeGen/GlobalMerge.h" #include "llvm/CodeGen/IndirectBrExpand.h" @@ -53,10 +56,12 @@ #include "llvm/IRPrinter/IRPrintingPasses.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCTargetOptions.h" +#include "llvm/Passes/PassBuilder.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/WithColor.h" #include "llvm/Target/CGPassBuilderOption.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/CFGuard.h" @@ -67,7 +72,10 @@ #include "llvm/Transforms/Scalar/MergeICmps.h" #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" #include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h" +#include "llvm/Transforms/Scalar/TLSVariableHoist.h" +#include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h" #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" +#include "llvm/Transforms/Utils/LowerGlobalDtors.h" #include "llvm/Transforms/Utils/LowerInvoke.h" #include #include @@ -122,11 +130,23 @@ namespace llvm { /// construction. The \c MachinePassRegistry.def file specifies how to construct /// all of the built-in passes, and those may reference these members during /// construction. +/// +/// Target should provide following methods: +/// Parse single target-specific MIR pass +/// @param Name the pass name +/// @return true if failed +/// addPreISel - This method should add any "last minute" LLVM->LLVM +/// passes (which are run just before instruction selector). +/// void addPreISel(AddIRPass &) const; +/// +/// void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const; + template class CodeGenPassBuilder { public: - explicit CodeGenPassBuilder(LLVMTargetMachine &TM, CGPassBuilderOption Opts, + explicit CodeGenPassBuilder(LLVMTargetMachine &TM, PassBuilder &PB, + CGPassBuilderOption Opts, PassInstrumentationCallbacks *PIC) - : TM(TM), Opt(Opts), PIC(PIC) { + : TM(TM), PB(PB), Opt(Opts), PIC(PIC) { // Target could set CGPassBuilderOption::MISchedPostRA to true to achieve // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID) @@ -146,17 +166,6 @@ template class CodeGenPassBuilder { raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType) const; - void registerModuleAnalyses(ModuleAnalysisManager &) const; - void registerFunctionAnalyses(FunctionAnalysisManager &) const; - void registerMachineFunctionAnalyses(MachineFunctionAnalysisManager &) const; - std::pair getPassNameFromLegacyName(StringRef) const; - - void registerAnalyses(MachineFunctionAnalysisManager &MFAM) const { - registerModuleAnalyses(*MFAM.MAM); - registerFunctionAnalyses(*MFAM.FAM); - registerMachineFunctionAnalyses(MFAM); - } - PassInstrumentationCallbacks *getPassInstrumentationCallbacks() const { return PIC; } @@ -238,6 +247,8 @@ template class CodeGenPassBuilder { }); } + MachineFunctionPassManager &getPM() { return PM; } + MachineFunctionPassManager releasePM() { return std::move(PM); } private: @@ -245,18 +256,35 @@ template class CodeGenPassBuilder { const DerivedT &PB; }; + // Find the FSProfile file name. The internal option takes the precedence + // before getting from TargetMachine. + // TODO: Use PGOOptions only. + std::string getFSProfileFile() const { + if (!Opt.FSProfileFile.empty()) + return Opt.FSProfileFile; + const std::optional &PGOOpt = TM.getPGOOption(); + if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) + return std::string(); + return PGOOpt->ProfileFile; + } + + // Find the Profile remapping file name. The internal option takes the + // precedence before getting from TargetMachine. + // TODO: Use PGOOptions only. + std::string getFSRemappingFile() const { + if (!Opt.FSRemappingFile.empty()) + return Opt.FSRemappingFile; + const std::optional &PGOOpt = TM.getPGOOption(); + if (PGOOpt == std::nullopt || PGOOpt->Action != PGOOptions::SampleUse) + return std::string(); + return PGOOpt->ProfileRemappingFile; + } + LLVMTargetMachine &TM; + PassBuilder &PB; CGPassBuilderOption Opt; PassInstrumentationCallbacks *PIC; - /// Target override these hooks to parse target-specific analyses. - void registerTargetAnalysis(ModuleAnalysisManager &) const {} - void registerTargetAnalysis(FunctionAnalysisManager &) const {} - void registerTargetAnalysis(MachineFunctionAnalysisManager &) const {} - std::pair getTargetPassNameFromLegacyName(StringRef) const { - return {"", false}; - } - template TMC &getTM() const { return static_cast(TM); } CodeGenOptLevel getOptLevel() const { return TM.getOptLevel(); } @@ -326,12 +354,14 @@ template class CodeGenPassBuilder { /// immediately before machine code is emitted. void addPreEmitPass(AddMachinePass &) const {} + /// This pass may be implemented by targets that want to run passes + /// immediately after basic block sections are assigned. + void addPostBBSections(AddMachinePass &) const {} + /// Targets may add passes immediately before machine code is emitted in this /// callback. This is called even later than `addPreEmitPass`. - // FIXME: Rename `addPreEmitPass` to something more sensible given its actual - // position and remove the `2` suffix here as this callback is what - // `addPreEmitPass` *should* be but in reality isn't. - void addPreEmitPass2(AddMachinePass &) const {} + /// This function replaces `addPreEmitPass2` in TargetConfig. + void addPrecedingEmitPass(AddMachinePass &) const {} /// {{@ For GlobalISel /// @@ -391,7 +421,7 @@ template class CodeGenPassBuilder { /// representation to the MI representation. /// Adds IR based lowering and target specific optimization passes and finally /// the core instruction selection passes. - void addISelPasses(AddIRPass &) const; + Error addISelPasses(AddIRPass &, AddMachinePass &) const; /// Add the actual instruction selection passes. This does not include /// preparation passes on IR. @@ -431,21 +461,54 @@ template class CodeGenPassBuilder { /// are required for fast register allocation. Error addFastRegAlloc(AddMachinePass &) const; + /// addPostFastRegAllocRewrite - Add passes to the optimized register + /// allocation pipeline after fast register allocation is complete. + void addPostFastRegAllocRewrite(AddMachinePass &) const {} + /// addOptimizedRegAlloc - Add passes related to register allocation. /// LLVMTargetMachine provides standard regalloc passes for most targets. - void addOptimizedRegAlloc(AddMachinePass &) const; + Error addOptimizedRegAlloc(AddMachinePass &) const; /// Add passes that optimize machine instructions after register allocation. void addMachineLateOptimization(AddMachinePass &) const; - /// addGCPasses - Add late codegen passes that analyze code for garbage + /// registerGCPasses - Add late codegen passes that analyze code for garbage /// collection. This should return true if GC info should be printed after /// these passes. - void addGCPasses(AddMachinePass &) const {} + bool registerGCPasses(MachineFunctionAnalysisManager &MFAM) const { + MFAM.registerPass([] { return GCMachineCodeAnalysisPass(); }); + return true; + } /// Add standard basic block placement passes. void addBlockPlacement(AddMachinePass &) const; + /// Add a pass to print the machine function if printing is enabled. + void addPrintPass(AddMachinePass &addPass, const std::string &Banner) const { + if (Opt.PrintAfterISel) + addPass(MachineFunctionPrinterPass(dbgs(), Banner)); + } + + /// Add a pass to perform basic verification of the machine function if + /// verification is enabled. + void addVerifyPass(AddMachinePass &addPass, const std::string &Banner) const { + bool Verify = Opt.VerifyMachineCode.value_or(false); +#ifdef EXPENSIVE_CHECKS + if (!Opt.VerifyMachineCode) + Verify = TM->isMachineVerifierClean(); +#endif + if (Verify) + addPass(MachineVerifierPass(Banner)); + } + + /// printAndVerify - Add a pass to dump then verify the machine function, if + /// those steps are enabled. + void printAndVerify(AddMachinePass &addPass, + const std::string &Banner) const { + addPrintPass(addPass, Banner); + addVerifyPass(addPass, Banner); + } + using CreateMCStreamer = std::function>(MCContext &)>; void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const { @@ -455,18 +518,10 @@ template class CodeGenPassBuilder { /// Utilities for targets to add passes to the pass manager. /// - /// createTargetRegisterAllocator - Create the register allocator pass for - /// this target at the current optimization level. - void addTargetRegisterAllocator(AddMachinePass &, bool Optimized) const; - - /// addMachinePasses helper to create the target-selected or overriden - /// regalloc pass. - void addRegAllocPass(AddMachinePass &, bool Optimized) const; - - /// Add core register alloator passes which do the actual register assignment - /// and rewriting. \returns true if any passes were added. - Error addRegAssignmentFast(AddMachinePass &) const; - Error addRegAssignmentOptimized(AddMachinePass &) const; + /// Add core register allocator passes which do the actual register assignment + /// and rewriting. \returns Error::success() if any passes were added. + Error addRegAssignAndRewriteFast(AddMachinePass &addPass) const; + Error addRegAssignAndRewriteOptimized(AddMachinePass &addPass) const; private: DerivedT &derived() { return static_cast(*this); } @@ -507,10 +562,8 @@ Error CodeGenPassBuilder::buildPipeline( // `ProfileSummaryInfo` is always valid. addIRPass(RequireAnalysisPass()); addIRPass(RequireAnalysisPass()); - addISelPasses(addIRPass); - AddMachinePass addPass(MFPM, derived()); - if (auto Err = addCoreISelPasses(addPass)) + if (auto Err = addISelPasses(addIRPass, addPass)) return std::move(Err); if (auto Err = derived().addMachinePasses(addPass)) @@ -605,93 +658,22 @@ static inline AAManager registerAAAnalyses() { } template -void CodeGenPassBuilder::registerModuleAnalyses( - ModuleAnalysisManager &MAM) const { -#define MODULE_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \ - MAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; }); -#include "MachinePassRegistry.def" - derived().registerTargetAnalysis(MAM); -} - -template -void CodeGenPassBuilder::registerFunctionAnalyses( - FunctionAnalysisManager &FAM) const { - FAM.registerPass([this] { return registerAAAnalyses(); }); - -#define FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \ - FAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; }); -#include "MachinePassRegistry.def" - derived().registerTargetAnalysis(FAM); -} - -template -void CodeGenPassBuilder::registerMachineFunctionAnalyses( - MachineFunctionAnalysisManager &MFAM) const { -#define MACHINE_FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR) \ - MFAM.registerPass([&] { return PASS_NAME CONSTRUCTOR; }); -#include "MachinePassRegistry.def" - derived().registerTargetAnalysis(MFAM); -} - -// FIXME: For new PM, use pass name directly in commandline seems good. -// Translate stringfied pass name to its old commandline name. Returns the -// matching legacy name and a boolean value indicating if the pass is a machine -// pass. -template -std::pair -CodeGenPassBuilder::getPassNameFromLegacyName(StringRef Name) const { - std::pair Ret; - if (Name.empty()) - return Ret; - -#define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ - if (Name == NAME) \ - Ret = {#PASS_NAME, false}; -#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ - if (Name == NAME) \ - Ret = {#PASS_NAME, false}; -#define MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ - if (Name == NAME) \ - Ret = {#PASS_NAME, false}; -#define DUMMY_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ - if (Name == NAME) \ - Ret = {#PASS_NAME, false}; -#define MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ - if (Name == NAME) \ - Ret = {#PASS_NAME, true}; -#define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ - if (Name == NAME) \ - Ret = {#PASS_NAME, true}; -#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ - if (Name == NAME) \ - Ret = {#PASS_NAME, true}; -#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ - if (Name == NAME) \ - Ret = {#PASS_NAME, true}; -#include "llvm/Passes/MachinePassRegistry.def" - - if (Ret.first.empty()) - Ret = derived().getTargetPassNameFromLegacyName(Name); - - if (Ret.first.empty()) - report_fatal_error(Twine('\"') + Twine(Name) + - Twine("\" pass could not be found.")); - - return Ret; -} - -template -void CodeGenPassBuilder::addISelPasses(AddIRPass &addPass) const { +Error CodeGenPassBuilder::addISelPasses( + AddIRPass &addPass, AddMachinePass &addMachinePass) const { derived().addGlobalMergePass(addPass); if (TM.useEmulatedTLS()) addPass(LowerEmuTLSPass()); addPass(PreISelIntrinsicLoweringPass(TM)); + addPass(ExpandLargeDivRemPass(&TM)); + addPass(ExpandLargeFpConvertPass(&TM)); derived().addIRPasses(addPass); derived().addCodeGenPrepare(addPass); addPassesToHandleExceptions(addPass); derived().addISelPrepare(addPass); + + return addCoreISelPasses(addMachinePass); } /// Add common target configurable passes that perform LLVM IR to IR transforms @@ -703,16 +685,15 @@ void CodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const { if (!Opt.DisableVerify) addPass(VerifierPass()); - // Run loop strength reduction before anything else. - if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableLSR) { - addPass(createFunctionToLoopPassAdaptor(LoopStrengthReducePass(), - /*UseMemorySSA=*/true)); - // FIXME: use -stop-after so we could remove PrintLSR - if (Opt.PrintLSR) - addPass(PrintFunctionPass(dbgs(), "\n\n*** Code after LSR ***\n")); - } - if (getOptLevel() != CodeGenOptLevel::None) { + // Run loop strength reduction before anything else. + if (!Opt.DisableLSR) { + addPass(createFunctionToLoopPassAdaptor(CanonicalizeFreezeInLoopsPass(), + /*UseMemorySSA=*/true)); + addPass(createFunctionToLoopPassAdaptor(LoopStrengthReducePass(), + /*UseMemorySSA=*/true)); + } + // The MergeICmpsPass tries to create memcmp calls by grouping sequences of // loads and compares. ExpandMemCmpPass then tries to expand those calls // into optimally-sized loads and compares. The transforms are enabled by a @@ -728,6 +709,12 @@ void CodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const { addPass(ShadowStackGCLoweringPass()); addPass(LowerConstantIntrinsicsPass()); + // For MachO, lower @llvm.global_dtors into @llvm.global_ctors with + // __cxa_atexit() calls to avoid emitting the deprecated __mod_term_func. + if (TM.getTargetTriple().isOSBinFormatMachO() && + !Opt.DisableAtExitBasedGlobalDtorLowering) + addPass(LowerGlobalDtorsPass()); + // Make sure that no unreachable blocks are instruction selected. addPass(UnreachableBlockElimPass()); @@ -744,8 +731,10 @@ void CodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const { !Opt.DisablePartialLibcallInlining) addPass(PartiallyInlineLibCallsPass()); - // Instrument function entry and exit, e.g. with calls to mcount(). - addPass(EntryExitInstrumenterPass(/*PostInlining=*/true)); + // Expand vector predication intrinsics into standard IR instructions. + // This pass has to run before ScalarizeMaskedMemIntrin and ExpandReduction + // passes since it emits those kinds of intrinsics. + addPass(ExpandVectorPredicationPass()); // Add scalarization of target's unsupported masked memory intrinsics pass. // the unsupported intrinsic will be replaced with a chain of basic blocks, @@ -753,7 +742,12 @@ void CodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const { addPass(ScalarizeMaskedMemIntrinPass()); // Expand reduction intrinsics into shuffle sequences if the target wants to. - addPass(ExpandReductionsPass()); + // Allow disabling it for testing purposes. + if (!Opt.DisableExpandReductions) + addPass(ExpandReductionsPass()); + + if (getOptLevel() != CodeGenOptLevel::None) + addPass(TLSVariableHoistPass()); // Convert conditional moves to conditional jumps when profitable. if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableSelectOptimize) @@ -813,8 +807,6 @@ template void CodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const { if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP) addPass(CodeGenPreparePass(&TM)); - // TODO: Default ctor'd RewriteSymbolPass is no-op. - // addPass(RewriteSymbolPass()); } /// Add common passes that perform LLVM IR to IR transforms in preparation for @@ -851,11 +843,10 @@ Error CodeGenPassBuilder::addCoreISelPasses( if (Opt.EnableFastISelOption && *Opt.EnableFastISelOption == true) Selector = SelectorType::FastISel; - else if ((Opt.EnableGlobalISelOption && - *Opt.EnableGlobalISelOption == true) || + + else if (Opt.EnableGlobalISelOption.value_or(false) || (TM.Options.EnableGlobalISel && - (!Opt.EnableGlobalISelOption || - *Opt.EnableGlobalISelOption == false))) + Opt.EnableGlobalISelOption.value_or(true))) Selector = SelectorType::GlobalISel; else if (TM.getOptLevel() == CodeGenOptLevel::None && TM.getO0WantsFastISel()) Selector = SelectorType::FastISel; @@ -874,24 +865,24 @@ Error CodeGenPassBuilder::addCoreISelPasses( // Add instruction selector passes. if (Selector == SelectorType::GlobalISel) { if (auto Err = derived().addIRTranslator(addPass)) - return std::move(Err); + return Err; derived().addPreLegalizeMachineIR(addPass); if (auto Err = derived().addLegalizeMachineIR(addPass)) - return std::move(Err); + return Err; // Before running the register bank selector, ask the target if it // wants to run some passes. derived().addPreRegBankSelect(addPass); if (auto Err = derived().addRegBankSelect(addPass)) - return std::move(Err); + return Err; derived().addPreGlobalInstructionSelect(addPass); if (auto Err = derived().addGlobalInstructionSelect(addPass)) - return std::move(Err); + return Err; // Pass to reset the MachineFunction if the ISel failed. addPass(ResetMachineFunctionPass(reportDiagnosticWhenGlobalISelFallback(), @@ -901,18 +892,16 @@ Error CodeGenPassBuilder::addCoreISelPasses( // not-yet-supported input. if (!isGlobalISelAbortEnabled()) if (auto Err = derived().addInstSelector(addPass)) - return std::move(Err); - + return Err; } else if (auto Err = derived().addInstSelector(addPass)) - return std::move(Err); + return Err; // Expand pseudo-instructions emitted by ISel. Don't run the verifier before // FinalizeISel. addPass(FinalizeISelPass()); // // Print the instruction selected machine code... - // printAndVerify("After Instruction Selection"); - + printAndVerify(addPass, "After Instruction Selection"); return Error::success(); } @@ -949,10 +938,24 @@ Error CodeGenPassBuilder::addMachinePasses( // Run pre-ra passes. derived().addPreRegAlloc(addPass); + // Add a FSDiscriminator pass right before RA, so that we could get + // more precise SampleFDO profile for RA. + if (EnableFSDiscriminator) { + addPass(MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::Pass1)); + const std::string ProfileFile = getFSProfileFile(); + if (!ProfileFile.empty() && !Opt.DisableRAFSProfileLoader) + addPass(MIRProfileLoaderNewPass(ProfileFile, getFSRemappingFile(), + sampleprof::FSDiscriminatorPass::Pass1, + nullptr)); + } + // Run register allocation and passes that are tightly coupled with it, // including phi elimination and scheduling. - if (*Opt.OptimizeRegAlloc) { - derived().addOptimizedRegAlloc(addPass); + bool IsOptimizeRegAlloc = + Opt.OptimizeRegAlloc.value_or(getOptLevel() != CodeGenOptLevel::None); + if (IsOptimizeRegAlloc) { + if (auto Err = derived().addOptimizedRegAlloc(addPass)) + return Err; } else { if (auto Err = derived().addFastRegAlloc(addPass)) return Err; @@ -963,12 +966,16 @@ Error CodeGenPassBuilder::addMachinePasses( addPass(RemoveRedundantDebugValuesPass()); - // Insert prolog/epilog code. Eliminate abstract frame index references... + addPass(FixupStatepointCallerSavedPass()); + + // Insert prolog/epilog code. Eliminate abstract frame index + // references... if (getOptLevel() != CodeGenOptLevel::None) { addPass(PostRAMachineSinkingPass()); addPass(ShrinkWrapPass()); } + // Prolog/Epilog inserter needs a TargetMachine to instantiate. addPass(PrologEpilogInserterPass()); /// Add passes that optimize machine instructions after register allocation. @@ -995,9 +1002,6 @@ Error CodeGenPassBuilder::addMachinePasses( addPass(PostRASchedulerPass()); } - // GC - derived().addGCPasses(addPass); - // Basic block placement. if (getOptLevel() != CodeGenOptLevel::None) derived().addBlockPlacement(addPass); @@ -1031,8 +1035,51 @@ Error CodeGenPassBuilder::addMachinePasses( addPass(MachineOutlinerPass(RunOnAllFunctions)); } + if (Opt.GCEmptyBlocks) + addPass(GCEmptyBasicBlocksPass()); + + if (EnableFSDiscriminator) + addPass( + MIRAddFSDiscriminatorsPass(sampleprof::FSDiscriminatorPass::PassLast)); + + // Machine function splitter uses the basic block sections feature. Both + // cannot be enabled at the same time. Basic block sections takes precedence. + // FIXME: In principle, BasicBlockSection::Labels and splitting can used + // together. Update this check once we have addressed any issues. + if (TM.getBBSectionsType() != llvm::BasicBlockSection::None) { + if (TM.getBBSectionsType() == llvm::BasicBlockSection::List) { + addPass( + BasicBlockSectionsProfileReaderPass(TM.getBBSectionsFuncListBuf())); + } + addPass(BasicBlockSectionsPass()); + } else if (TM.Options.EnableMachineFunctionSplitter || + Opt.EnableMachineFunctionSplitter) { + const std::string ProfileFile = getFSProfileFile(); + if (!ProfileFile.empty()) { + if (EnableFSDiscriminator) { + addPass(MIRProfileLoaderNewPass( + ProfileFile, getFSRemappingFile(), + sampleprof::FSDiscriminatorPass::PassLast, nullptr)); + } else { + // Sample profile is given, but FSDiscriminator is not + // enabled, this may result in performance regression. + WithColor::warning() + << "Using AutoFDO without FSDiscriminator for MFS may regress " + "performance."; + } + } + addPass(MachineFunctionSplitterPass()); + } + + derived().addPostBBSections(addPass); + + if (!Opt.DisableCFIFixup && TM.Options.EnableCFIFixup) + addPass(CFIFixupPass()); + + addPass(StackFrameLayoutAnalysisPass()); + // Add passes that directly emit MI after all other MI passes. - derived().addPreEmitPass2(addPass); + derived().addPrecedingEmitPass(addPass); return Error::success(); } @@ -1082,56 +1129,39 @@ void CodeGenPassBuilder::addMachineSSAOptimization( /// Register Allocation Pass Configuration //===---------------------------------------------------------------------===// -/// Instantiate the default register allocator pass for this target for either -/// the optimized or unoptimized allocation path. This will be added to the pass -/// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc -/// in the optimized case. -/// -/// A target that uses the standard regalloc pass order for fast or optimized -/// allocation may still override this for per-target regalloc -/// selection. But -regalloc=... always takes precedence. template -void CodeGenPassBuilder::addTargetRegisterAllocator( - AddMachinePass &addPass, bool Optimized) const { - if (Optimized) - addPass(RAGreedyPass()); - else - addPass(RAFastPass()); -} +Error CodeGenPassBuilder::addRegAssignAndRewriteFast( + AddMachinePass &addPass) const { + if (Opt.RegAlloc != "default" && !Opt.RegAlloc.starts_with("fast")) + return make_error( + "Must use fast (default) register allocator for unoptimized regalloc.", + inconvertibleErrorCode()); -/// Find and instantiate the register allocation pass requested by this target -/// at the current optimization level. Different register allocators are -/// defined as separate passes because they may require different analysis. -template -void CodeGenPassBuilder::addRegAllocPass(AddMachinePass &addPass, - bool Optimized) const { - // TODO: Parse Opt.RegAlloc to add register allocator. -} + if (Error Err = PB.parseRegAllocPass(addPass.getPM(), Opt.RegAlloc, false)) + return Err; + + // Allow targets to change the register assignments after + // fast register allocation. + addPostFastRegAllocRewrite(addPass); -template -Error CodeGenPassBuilder::addRegAssignmentFast( - AddMachinePass &addPass) const { - // TODO: Ensure allocator is default or fast. - addRegAllocPass(addPass, false); return Error::success(); } -template -Error CodeGenPassBuilder::addRegAssignmentOptimized( +template +Error CodeGenPassBuilder::addRegAssignAndRewriteOptimized( AddMachinePass &addPass) const { // Add the selected register allocation pass. - addRegAllocPass(addPass, true); - + if (Error Err = PB.parseRegAllocPass(addPass.getPM(), Opt.RegAlloc, true)) + return Err; // Allow targets to change the register assignments before rewriting. - derived().addPreRewrite(addPass); + addPreRewrite(addPass); // Finally rewrite virtual registers. addPass(VirtRegRewriterPass()); - // Perform stack slot coloring and post-ra machine LICM. - // - // FIXME: Re-enable coloring with register when it's capable of adding - // kill markers. - addPass(StackSlotColoringPass()); + + // Regalloc scoring for ML-driven eviction - noop except when learning a new + // eviction policy. + addPass(RegAllocScoringPass()); return Error::success(); } @@ -1143,14 +1173,14 @@ Error CodeGenPassBuilder::addFastRegAlloc( AddMachinePass &addPass) const { addPass(PHIEliminationPass()); addPass(TwoAddressInstructionPass()); - return derived().addRegAssignmentFast(addPass); + return derived().addRegAssignAndRewriteFast(addPass); } /// Add standard target-independent passes that are tightly coupled with /// optimized register allocation, including coalescing, machine instruction /// scheduling, and register allocation itself. template -void CodeGenPassBuilder::addOptimizedRegAlloc( +Error CodeGenPassBuilder::addOptimizedRegAlloc( AddMachinePass &addPass) const { addPass(DetectDeadLanesPass()); @@ -1174,20 +1204,22 @@ void CodeGenPassBuilder::addOptimizedRegAlloc( // PreRA instruction scheduling. addPass(MachineSchedulerPass()); - if (derived().addRegAssignmentOptimized(addPass)) { - // Allow targets to expand pseudo instructions depending on the choice of - // registers before MachineCopyPropagation. - derived().addPostRewrite(addPass); + if (Error Err = derived().addRegAssignAndRewriteOptimized(addPass)) + return Err; - // Copy propagate to forward register uses and try to eliminate COPYs that - // were not coalesced. - addPass(MachineCopyPropagationPass()); + // Allow targets to expand pseudo instructions depending on the choice of + // registers before MachineCopyPropagation. + derived().addPostRewrite(addPass); - // Run post-ra machine LICM to hoist reloads / remats. - // - // FIXME: can this move into MachineLateOptimization? - addPass(MachineLICMPass()); - } + // Copy propagate to forward register uses and try to eliminate COPYs that + // were not coalesced. + addPass(MachineCopyPropagationPass()); + + // Run post-ra machine LICM to hoist reloads / remats. + // + // FIXME: can this move into MachineLateOptimization? + addPass(MachineLICMPass()); + return Error::success(); } //===---------------------------------------------------------------------===// diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index 5278c0d1ed7e6..c6131ff630c64 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -129,6 +129,23 @@ MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, // MACHINE_FUNCTION_PASS("free-machine-function", FreeMachineFunctionPass, ()) #undef MACHINE_FUNCTION_PASS +#ifndef RA_PASS +#define RA_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#endif +// RA_PASS("ra-basic", RABasicPass, ()) +#undef RA_PASS + +#ifndef RA_PASS_WITH_PARAMS +#define RA_PASS_WITH_PARAMS(NAME, PASS_NAME, CREATE_PASS, PARSER, PARAMS) +#endif +// RA_PASS_WITH_PARAMS("fast", RAFastPass, +// [](Option Opts){ return RAFastPass(Opts); }, parseRAFastPassOptions, "") +// RA_PASS_WITH_PARAMS("greedy", RAGreedyPass, +// [](Option Opts) { return RAGreedyPass(Opts); }, parseRAGreedyPassOptions, "") +// RA_PASS_WITH_PARAMS("pbqp", RAPBQPPass, +// [](Option Opts) { return RAPBQPPass(Opts); }, parseRAPBQPPassOptions, "") +#undef RA_PASS_WITH_PARAMS + // After a pass is converted to new pass manager, its entry should be moved from // dummy table to the normal one. For example, for a machine function pass, // DUMMY_MACHINE_FUNCTION_PASS to MACHINE_FUNCTION_PASS. @@ -225,15 +242,10 @@ DUMMY_MACHINE_FUNCTION_PASS("processimpdefs", ProcessImplicitDefsPass, ()) DUMMY_MACHINE_FUNCTION_PASS("prologepilog", PrologEpilogInserterPass, ()) DUMMY_MACHINE_FUNCTION_PASS("prologepilog-code", PrologEpilogCodeInserterPass, ()) -DUMMY_MACHINE_FUNCTION_PASS("ra-basic", RABasicPass, ()) -DUMMY_MACHINE_FUNCTION_PASS("ra-fast", RAFastPass, ()) -DUMMY_MACHINE_FUNCTION_PASS("ra-greedy", RAGreedyPass, ()) -DUMMY_MACHINE_FUNCTION_PASS("ra-pbqp", RAPBQPPass, ()) DUMMY_MACHINE_FUNCTION_PASS("reg-usage-collector", RegUsageInfoCollectorPass, ()) DUMMY_MACHINE_FUNCTION_PASS("reg-usage-propagation", RegUsageInfoPropagationPass, ()) -DUMMY_MACHINE_FUNCTION_PASS("regalloc", RegAllocPass, ()) DUMMY_MACHINE_FUNCTION_PASS("regallocscoringpass", RegAllocScoringPass, ()) DUMMY_MACHINE_FUNCTION_PASS("regbankselect", RegBankSelectPass, ()) DUMMY_MACHINE_FUNCTION_PASS("removeredundantdebugvalues", diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 7339b8a988232..56a2a0e56e9f3 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -370,6 +370,11 @@ class PassBuilder { Error parsePassPipeline(MachineFunctionPassManager &MFPM, StringRef PipelineText); + /// Parse textual regalloc pass into the provided \c MachineFunctionPass + /// the PassText should contain only one regalloc pass text. + Error parseRegAllocPass(MachineFunctionPassManager &MFPM, StringRef PassText, + bool Optimized); + /// Parse a textual alias analysis pipeline into the provided AA manager. /// /// The format of the textual AA pipeline is a comma separated list of AA @@ -575,6 +580,12 @@ class PassBuilder { } /// @}} + void registerRegAllocParsingCallback( + const std::function + &C) { + RegAllocPassParsingCallbacks.push_back(C); + } + /// Register a callback for a top-level pipeline entry. /// /// If the PassManager type is not given at the top level of the pipeline @@ -735,6 +746,9 @@ class PassBuilder { MachineFunctionAnalysisRegistrationCallbacks; SmallVector, 2> MachinePipelineParsingCallbacks; + SmallVector< + std::function, 2> + RegAllocPassParsingCallbacks; }; /// This utility template takes care of adding require<> and invalidate<> diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h index a522a12299bb0..38109a106219d 100644 --- a/llvm/include/llvm/Target/TargetMachine.h +++ b/llvm/include/llvm/Target/TargetMachine.h @@ -457,9 +457,9 @@ class LLVMTargetMachine : public TargetMachine { virtual Error buildCodeGenPipeline(ModulePassManager &, MachineFunctionPassManager &, - MachineFunctionAnalysisManager &, - raw_pwrite_stream &, raw_pwrite_stream *, - CodeGenFileType, CGPassBuilderOption, + PassBuilder &, raw_pwrite_stream &, + raw_pwrite_stream *, CodeGenFileType, + CGPassBuilderOption, PassInstrumentationCallbacks *) { return make_error("buildCodeGenPipeline is not overridden", inconvertibleErrorCode()); diff --git a/llvm/lib/CodeGen/MachinePassManager.cpp b/llvm/lib/CodeGen/MachinePassManager.cpp index 914e6b19fde9a..52722cad0c2ce 100644 --- a/llvm/lib/CodeGen/MachinePassManager.cpp +++ b/llvm/lib/CodeGen/MachinePassManager.cpp @@ -22,6 +22,7 @@ template class AllAnalysesOn; template class AnalysisManager; template class PassManager; +// TODO: Add a way to run verifier and debugify passes. Error MachineFunctionPassManager::run(Module &M, MachineFunctionAnalysisManager &MFAM) { // MachineModuleAnalysis is a module analysis pass that is never invalidated @@ -30,26 +31,11 @@ Error MachineFunctionPassManager::run(Module &M, // result of MachineModuleAnalysis. MMI should not be recomputed. auto &MMI = MFAM.getResult(M); - (void)RequireCodeGenSCCOrder; - assert(!RequireCodeGenSCCOrder && "not implemented"); + assert(!Opt.RequiresCodeGenSCCOrder && "not implemented"); // M is unused here PassInstrumentation PI = MFAM.getResult(M); - // Add a PIC to verify machine functions. - if (VerifyMachineFunction) { - // No need to pop this callback later since MIR pipeline is flat which means - // current pipeline is the top-level pipeline. Callbacks are not used after - // current pipeline. - PI.pushBeforeNonSkippedPassCallback([&MFAM](StringRef PassID, Any IR) { - assert(llvm::any_cast(&IR)); - const MachineFunction *MF = llvm::any_cast(IR); - assert(MF && "Machine function should be valid for printing"); - std::string Banner = std::string("After ") + std::string(PassID); - verifyMachineFunction(&MFAM, Banner, *MF); - }); - } - for (auto &F : InitializationFuncs) { if (auto Err = F(M, MFAM)) return Err; diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 2fe167bd439d4..b7609c4c5a03b 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -2081,6 +2081,43 @@ Error PassBuilder::parsePassPipeline(MachineFunctionPassManager &MFPM, return Error::success(); } +Error PassBuilder::parseRegAllocPass(MachineFunctionPassManager &MFPM, + StringRef PassText, bool Optimized) { + // Targets may have their own default pipeline. + for (const auto &C : RegAllocPassParsingCallbacks) { + if (C(PassText, MFPM, Optimized)) + return Error::success(); + } + + if (PassText == "default") { + if (Optimized) { + // TODO: MFPM.addPass(GreedyRegisterAllocatorPass()); + } else { + // TODO: MFPM.addPass(FastRegisterAllocatorPass()); + } + return Error::success(); + } + +#define RA_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (PassText == NAME) { \ + MFPM.addPass(PASS_NAME CONSTRUCTOR); \ + return Error::success(); \ + } +#define RA_PASS_WITH_PARAMS(NAME, PASS_NAME, CREATE_PASS, PARSER, PARAMS) \ + if (checkParametrizedPassName(PassText, NAME)) { \ + auto Params = parsePassParameters(PARSER, PassText, NAME); \ + if (!Params) \ + return Params.takeError(); \ + MFPM.addPass(CREATE_PASS(Params.get())); \ + return Error::success(); \ + } +#include "llvm/Passes/MachinePassRegistry.def" + + return make_error( + formatv("invalid regalloc pass '{0}'", PassText).str(), + inconvertibleErrorCode()); +} + Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) { // If the pipeline just consists of the word 'default' just replace the AA // manager with our default one. diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt index 610999f0cc3cf..a8f8853f5d1fb 100644 --- a/llvm/lib/Target/X86/CMakeLists.txt +++ b/llvm/lib/Target/X86/CMakeLists.txt @@ -102,6 +102,7 @@ add_llvm_target(X86CodeGen ${sources} IRPrinter Instrumentation MC + Passes ProfileData Scalar SelectionDAG diff --git a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp index 4a11dd2e31acd..38550c028ef3e 100644 --- a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp +++ b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp @@ -21,10 +21,10 @@ namespace { class X86CodeGenPassBuilder : public CodeGenPassBuilder { public: - explicit X86CodeGenPassBuilder(LLVMTargetMachine &TM, + explicit X86CodeGenPassBuilder(LLVMTargetMachine &TM, PassBuilder &PB, CGPassBuilderOption Opts, PassInstrumentationCallbacks *PIC) - : CodeGenPassBuilder(TM, Opts, PIC) {} + : CodeGenPassBuilder(TM, PB, Opts, PIC) {} void addPreISel(AddIRPass &addPass) const; void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const; Error addInstSelector(AddMachinePass &) const; @@ -47,10 +47,9 @@ Error X86CodeGenPassBuilder::addInstSelector(AddMachinePass &) const { } // namespace Error X86TargetMachine::buildCodeGenPipeline( - ModulePassManager &MPM, MachineFunctionPassManager &MFPM, - MachineFunctionAnalysisManager &, raw_pwrite_stream &Out, - raw_pwrite_stream *DwoOut, CodeGenFileType FileType, + ModulePassManager &MPM, MachineFunctionPassManager &MFPM, PassBuilder &PB, + raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType, CGPassBuilderOption Opt, PassInstrumentationCallbacks *PIC) { - auto CGPB = X86CodeGenPassBuilder(*this, Opt, PIC); + auto CGPB = X86CodeGenPassBuilder(*this, PB, Opt, PIC); return CGPB.buildPipeline(MPM, MFPM, Out, DwoOut, FileType); } diff --git a/llvm/lib/Target/X86/X86TargetMachine.h b/llvm/lib/Target/X86/X86TargetMachine.h index f31c971df9584..37d6f564ee3b8 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.h +++ b/llvm/lib/Target/X86/X86TargetMachine.h @@ -59,9 +59,9 @@ class X86TargetMachine final : public LLVMTargetMachine { const TargetSubtargetInfo *STI) const override; Error buildCodeGenPipeline(ModulePassManager &, MachineFunctionPassManager &, - MachineFunctionAnalysisManager &, - raw_pwrite_stream &, raw_pwrite_stream *, - CodeGenFileType, CGPassBuilderOption, + PassBuilder &, raw_pwrite_stream &, + raw_pwrite_stream *, CodeGenFileType, + CGPassBuilderOption, PassInstrumentationCallbacks *) override; bool isJIT() const { return IsJIT; } diff --git a/llvm/test/tools/llc/new-pm/pipeline.ll b/llvm/test/tools/llc/new-pm/pipeline.ll index 1ace5963e4ef8..39953c68d97a0 100644 --- a/llvm/test/tools/llc/new-pm/pipeline.ll +++ b/llvm/test/tools/llc/new-pm/pipeline.ll @@ -1,5 +1,4 @@ ; RUN: llc -mtriple=x86_64-pc-linux-gnu -enable-new-pm -print-pipeline-passes -filetype=null %s | FileCheck %s ; CHECK: require,require -; CHECK: MachineSanitizerBinaryMetadata,FreeMachineFunctionPass - +; CHECK: StackFrameLayoutAnalysisPass,FreeMachineFunctionPass diff --git a/llvm/tools/llc/NewPMDriver.cpp b/llvm/tools/llc/NewPMDriver.cpp index 2f979f3081f79..b6338df7070f5 100644 --- a/llvm/tools/llc/NewPMDriver.cpp +++ b/llvm/tools/llc/NewPMDriver.cpp @@ -191,7 +191,7 @@ int llvm::compileModuleWithNewPM( ModulePassManager MPM; MachineFunctionPassManager MFPM; - ExitOnErr(LLVMTM.buildCodeGenPipeline(MPM, MFPM, MFAM, *OS, + ExitOnErr(LLVMTM.buildCodeGenPipeline(MPM, MFPM, PB, *OS, DwoOut ? &DwoOut->os() : nullptr, FileType, Opt, &PIC));