diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h index 882cadea22369..91382fd07aec9 100644 --- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -2088,10 +2088,13 @@ class TargetInstrInfo : public MCInstrInfo { /// Returns a \p outliner::OutlinedFunction struct containing target-specific /// information for a set of outlining candidates. Returns std::nullopt if the - /// candidates are not suitable for outlining. - virtual std::optional getOutliningCandidateInfo( + /// candidates are not suitable for outlining. \p MinRepeats is the minimum + /// number of times the instruction sequence must be repeated. + virtual std::optional> + getOutliningCandidateInfo( const MachineModuleInfo &MMI, - std::vector &RepeatedSequenceLocs) const { + std::vector &RepeatedSequenceLocs, + unsigned MinRepeats) const { llvm_unreachable( "Target didn't implement TargetInstrInfo::getOutliningCandidateInfo!"); } diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp index 4b56a467b8d07..42f410c277179 100644 --- a/llvm/lib/CodeGen/MachineOutliner.cpp +++ b/llvm/lib/CodeGen/MachineOutliner.cpp @@ -456,8 +456,9 @@ struct MachineOutliner : public ModulePass { /// \param Mapper Contains outlining mapping information. /// \param[out] FunctionList Filled with a list of \p OutlinedFunctions /// each type of candidate. - void findCandidates(InstructionMapper &Mapper, - std::vector &FunctionList); + void + findCandidates(InstructionMapper &Mapper, + std::vector> &FunctionList); /// Replace the sequences of instructions represented by \p OutlinedFunctions /// with calls to functions. @@ -465,7 +466,9 @@ struct MachineOutliner : public ModulePass { /// \param M The module we are outlining from. /// \param FunctionList A list of functions to be inserted into the module. /// \param Mapper Contains the instruction mappings for the module. - bool outline(Module &M, std::vector &FunctionList, + /// \param[out] OutlinedFunctionNum The outlined function number. + bool outline(Module &M, + std::vector> &FunctionList, InstructionMapper &Mapper, unsigned &OutlinedFunctionNum); /// Creates a function for \p OF and inserts it into the module. @@ -583,7 +586,8 @@ void MachineOutliner::emitOutlinedFunctionRemark(OutlinedFunction &OF) { } void MachineOutliner::findCandidates( - InstructionMapper &Mapper, std::vector &FunctionList) { + InstructionMapper &Mapper, + std::vector> &FunctionList) { FunctionList.clear(); SuffixTree ST(Mapper.UnsignedVec, OutlinerLeafDescendants); @@ -658,11 +662,12 @@ void MachineOutliner::findCandidates( << "\n"); LLVM_DEBUG(dbgs() << " Candidates kept: " << NumKept << "\n\n"); #endif + unsigned MinRepeats = 2; // We've found something we might want to outline. // Create an OutlinedFunction to store it and check if it'd be beneficial // to outline. - if (CandidatesForRepeatedSeq.size() < 2) + if (CandidatesForRepeatedSeq.size() < MinRepeats) continue; // Arbitrarily choose a TII from the first candidate. @@ -670,21 +675,23 @@ void MachineOutliner::findCandidates( const TargetInstrInfo *TII = CandidatesForRepeatedSeq[0].getMF()->getSubtarget().getInstrInfo(); - std::optional OF = - TII->getOutliningCandidateInfo(*MMI, CandidatesForRepeatedSeq); + std::optional> OF = + TII->getOutliningCandidateInfo(*MMI, CandidatesForRepeatedSeq, + MinRepeats); // If we deleted too many candidates, then there's nothing worth outlining. // FIXME: This should take target-specified instruction sizes into account. - if (!OF || OF->Candidates.size() < 2) + if (!OF.has_value() || OF.value()->Candidates.size() < MinRepeats) continue; // Is it better to outline this candidate than not? - if (OF->getBenefit() < OutlinerBenefitThreshold) { - emitNotOutliningCheaperRemark(StringLen, CandidatesForRepeatedSeq, *OF); + if (OF.value()->getBenefit() < OutlinerBenefitThreshold) { + emitNotOutliningCheaperRemark(StringLen, CandidatesForRepeatedSeq, + *OF.value()); continue; } - FunctionList.push_back(*OF); + FunctionList.emplace_back(std::move(OF.value())); } } @@ -827,10 +834,9 @@ MachineFunction *MachineOutliner::createOutlinedFunction( return &MF; } -bool MachineOutliner::outline(Module &M, - std::vector &FunctionList, - InstructionMapper &Mapper, - unsigned &OutlinedFunctionNum) { +bool MachineOutliner::outline( + Module &M, std::vector> &FunctionList, + InstructionMapper &Mapper, unsigned &OutlinedFunctionNum) { LLVM_DEBUG(dbgs() << "*** Outlining ***\n"); LLVM_DEBUG(dbgs() << "NUMBER OF POTENTIAL FUNCTIONS: " << FunctionList.size() << "\n"); @@ -838,23 +844,23 @@ bool MachineOutliner::outline(Module &M, // Sort by priority where priority := getNotOutlinedCost / getOutliningCost. // The function with highest priority should be outlined first. - stable_sort(FunctionList, - [](const OutlinedFunction &LHS, const OutlinedFunction &RHS) { - return LHS.getNotOutlinedCost() * RHS.getOutliningCost() > - RHS.getNotOutlinedCost() * LHS.getOutliningCost(); - }); + stable_sort(FunctionList, [](const std::unique_ptr &LHS, + const std::unique_ptr &RHS) { + return LHS->getNotOutlinedCost() * RHS->getOutliningCost() > + RHS->getNotOutlinedCost() * LHS->getOutliningCost(); + }); // Walk over each function, outlining them as we go along. Functions are // outlined greedily, based off the sort above. auto *UnsignedVecBegin = Mapper.UnsignedVec.begin(); LLVM_DEBUG(dbgs() << "WALKING FUNCTION LIST\n"); - for (OutlinedFunction &OF : FunctionList) { + for (auto &OF : FunctionList) { #ifndef NDEBUG - auto NumCandidatesBefore = OF.Candidates.size(); + auto NumCandidatesBefore = OF->Candidates.size(); #endif // If we outlined something that overlapped with a candidate in a previous // step, then we can't outline from it. - erase_if(OF.Candidates, [&UnsignedVecBegin](Candidate &C) { + erase_if(OF->Candidates, [&UnsignedVecBegin](Candidate &C) { return std::any_of(UnsignedVecBegin + C.getStartIdx(), UnsignedVecBegin + C.getEndIdx() + 1, [](unsigned I) { return I == static_cast(-1); @@ -862,36 +868,36 @@ bool MachineOutliner::outline(Module &M, }); #ifndef NDEBUG - auto NumCandidatesAfter = OF.Candidates.size(); + auto NumCandidatesAfter = OF->Candidates.size(); LLVM_DEBUG(dbgs() << "PRUNED: " << NumCandidatesBefore - NumCandidatesAfter << "/" << NumCandidatesBefore << " candidates\n"); #endif // If we made it unbeneficial to outline this function, skip it. - if (OF.getBenefit() < OutlinerBenefitThreshold) { - LLVM_DEBUG(dbgs() << "SKIP: Expected benefit (" << OF.getBenefit() + if (OF->getBenefit() < OutlinerBenefitThreshold) { + LLVM_DEBUG(dbgs() << "SKIP: Expected benefit (" << OF->getBenefit() << " B) < threshold (" << OutlinerBenefitThreshold << " B)\n"); continue; } - LLVM_DEBUG(dbgs() << "OUTLINE: Expected benefit (" << OF.getBenefit() + LLVM_DEBUG(dbgs() << "OUTLINE: Expected benefit (" << OF->getBenefit() << " B) > threshold (" << OutlinerBenefitThreshold << " B)\n"); // It's beneficial. Create the function and outline its sequence's // occurrences. - OF.MF = createOutlinedFunction(M, OF, Mapper, OutlinedFunctionNum); - emitOutlinedFunctionRemark(OF); + OF->MF = createOutlinedFunction(M, *OF, Mapper, OutlinedFunctionNum); + emitOutlinedFunctionRemark(*OF); FunctionsCreated++; OutlinedFunctionNum++; // Created a function, move to the next name. - MachineFunction *MF = OF.MF; + MachineFunction *MF = OF->MF; const TargetSubtargetInfo &STI = MF->getSubtarget(); const TargetInstrInfo &TII = *STI.getInstrInfo(); // Replace occurrences of the sequence with calls to the new function. LLVM_DEBUG(dbgs() << "CREATE OUTLINED CALLS\n"); - for (Candidate &C : OF.Candidates) { + for (Candidate &C : OF->Candidates) { MachineBasicBlock &MBB = *C.getMBB(); MachineBasicBlock::iterator StartIt = C.begin(); MachineBasicBlock::iterator EndIt = std::prev(C.end()); @@ -1180,7 +1186,7 @@ bool MachineOutliner::doOutline(Module &M, unsigned &OutlinedFunctionNum) { // Prepare instruction mappings for the suffix tree. populateMapper(Mapper, M); - std::vector FunctionList; + std::vector> FunctionList; // Find all of the outlining candidates. findCandidates(Mapper, FunctionList); diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 697ae510a9565..5fe4173d27d33 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -8684,10 +8684,11 @@ static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a, return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps(); } -std::optional +std::optional> AArch64InstrInfo::getOutliningCandidateInfo( const MachineModuleInfo &MMI, - std::vector &RepeatedSequenceLocs) const { + std::vector &RepeatedSequenceLocs, + unsigned MinRepeats) const { unsigned SequenceSize = 0; for (auto &MI : RepeatedSequenceLocs[0]) SequenceSize += getInstSizeInBytes(MI); @@ -8801,7 +8802,7 @@ AArch64InstrInfo::getOutliningCandidateInfo( llvm::erase_if(RepeatedSequenceLocs, hasIllegalSPModification); // If the sequence doesn't have enough candidates left, then we're done. - if (RepeatedSequenceLocs.size() < 2) + if (RepeatedSequenceLocs.size() < MinRepeats) return std::nullopt; } @@ -9048,7 +9049,7 @@ AArch64InstrInfo::getOutliningCandidateInfo( } // If we dropped all of the candidates, bail out here. - if (RepeatedSequenceLocs.size() < 2) { + if (RepeatedSequenceLocs.size() < MinRepeats) { RepeatedSequenceLocs.clear(); return std::nullopt; } @@ -9091,8 +9092,8 @@ AArch64InstrInfo::getOutliningCandidateInfo( if (FrameID != MachineOutlinerTailCall && CFICount > 0) return std::nullopt; - return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, - NumBytesToCreateFrame, FrameID); + return std::make_unique( + RepeatedSequenceLocs, SequenceSize, NumBytesToCreateFrame, FrameID); } void AArch64InstrInfo::mergeOutliningCandidateAttributes( diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h index a1f2fbff01631..9cda7d86a3a1d 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h @@ -471,9 +471,11 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo { bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override; - std::optional getOutliningCandidateInfo( + std::optional> + getOutliningCandidateInfo( const MachineModuleInfo &MMI, - std::vector &RepeatedSequenceLocs) const override; + std::vector &RepeatedSequenceLocs, + unsigned MinRepeats) const override; void mergeOutliningCandidateAttributes( Function &F, std::vector &Candidates) const override; outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI, diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp index 1199052ca97e9..a87c0ace261a4 100644 --- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp +++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp @@ -5871,10 +5871,11 @@ static bool isLRAvailable(const TargetRegisterInfo &TRI, return !Live; } -std::optional +std::optional> ARMBaseInstrInfo::getOutliningCandidateInfo( const MachineModuleInfo &MMI, - std::vector &RepeatedSequenceLocs) const { + std::vector &RepeatedSequenceLocs, + unsigned MinRepeats) const { unsigned SequenceSize = 0; for (auto &MI : RepeatedSequenceLocs[0]) SequenceSize += getInstSizeInBytes(MI); @@ -5915,7 +5916,7 @@ ARMBaseInstrInfo::getOutliningCandidateInfo( llvm::erase_if(RepeatedSequenceLocs, CantGuaranteeValueAcrossCall); // If the sequence doesn't have enough candidates left, then we're done. - if (RepeatedSequenceLocs.size() < 2) + if (RepeatedSequenceLocs.size() < MinRepeats) return std::nullopt; } @@ -5941,7 +5942,7 @@ ARMBaseInstrInfo::getOutliningCandidateInfo( else RepeatedSequenceLocs.erase(RepeatedSequenceLocs.begin(), NoBTI); - if (RepeatedSequenceLocs.size() < 2) + if (RepeatedSequenceLocs.size() < MinRepeats) return std::nullopt; // Likewise, partition the candidates according to PAC-RET enablement. @@ -5958,7 +5959,7 @@ ARMBaseInstrInfo::getOutliningCandidateInfo( else RepeatedSequenceLocs.erase(RepeatedSequenceLocs.begin(), NoPAC); - if (RepeatedSequenceLocs.size() < 2) + if (RepeatedSequenceLocs.size() < MinRepeats) return std::nullopt; // At this point, we have only "safe" candidates to outline. Figure out @@ -6062,7 +6063,7 @@ ARMBaseInstrInfo::getOutliningCandidateInfo( RepeatedSequenceLocs.size() * Costs.CallDefault) { RepeatedSequenceLocs = CandidatesWithoutStackFixups; FrameID = MachineOutlinerNoLRSave; - if (RepeatedSequenceLocs.size() < 2) + if (RepeatedSequenceLocs.size() < MinRepeats) return std::nullopt; } else SetCandidateCallInfo(MachineOutlinerDefault, Costs.CallDefault); @@ -6088,8 +6089,8 @@ ARMBaseInstrInfo::getOutliningCandidateInfo( NumBytesToCreateFrame += Costs.SaveRestoreLROnStack; } - return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, - NumBytesToCreateFrame, FrameID); + return std::make_unique( + RepeatedSequenceLocs, SequenceSize, NumBytesToCreateFrame, FrameID); } bool ARMBaseInstrInfo::checkAndUpdateStackOffset(MachineInstr *MI, diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h index 8521e3ef91399..baffe78e8d0d2 100644 --- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h +++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h @@ -355,9 +355,11 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo { /// ARM supports the MachineOutliner. bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override; - std::optional getOutliningCandidateInfo( + std::optional> + getOutliningCandidateInfo( const MachineModuleInfo &MMI, - std::vector &RepeatedSequenceLocs) const override; + std::vector &RepeatedSequenceLocs, + unsigned MinRepeats) const override; void mergeOutliningCandidateAttributes( Function &F, std::vector &Candidates) const override; outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI, diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index 9dd79027d7a16..688a72ecfce8d 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -2828,10 +2828,11 @@ bool RISCVInstrInfo::shouldOutlineFromFunctionByDefault( return MF.getFunction().hasMinSize(); } -std::optional +std::optional> RISCVInstrInfo::getOutliningCandidateInfo( const MachineModuleInfo &MMI, - std::vector &RepeatedSequenceLocs) const { + std::vector &RepeatedSequenceLocs, + unsigned MinRepeats) const { // First we need to filter out candidates where the X5 register (IE t0) can't // be used to setup the function call. @@ -2843,7 +2844,7 @@ RISCVInstrInfo::getOutliningCandidateInfo( llvm::erase_if(RepeatedSequenceLocs, CannotInsertCall); // If the sequence doesn't have enough candidates left, then we're done. - if (RepeatedSequenceLocs.size() < 2) + if (RepeatedSequenceLocs.size() < MinRepeats) return std::nullopt; unsigned SequenceSize = 0; @@ -2864,8 +2865,9 @@ RISCVInstrInfo::getOutliningCandidateInfo( .hasStdExtCOrZca()) FrameOverhead = 2; - return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, - FrameOverhead, MachineOutlinerDefault); + return std::make_unique( + RepeatedSequenceLocs, SequenceSize, FrameOverhead, + MachineOutlinerDefault); } outliner::InstrType diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h index ecb7982b3e5e3..e11cf2b2f7670 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h @@ -205,9 +205,11 @@ class RISCVInstrInfo : public RISCVGenInstrInfo { bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override; // Calculate target-specific information for a set of outlining candidates. - std::optional getOutliningCandidateInfo( + std::optional> + getOutliningCandidateInfo( const MachineModuleInfo &MMI, - std::vector &RepeatedSequenceLocs) const override; + std::vector &RepeatedSequenceLocs, + unsigned MinRepeats) const override; // Return if/how a given MachineInstr should be outlined. virtual outliner::InstrType diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp index 39ba7ea777909..9ee00bd8d5270 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -10521,10 +10521,11 @@ FunctionPass *llvm::createCleanupLocalDynamicTLSPass() { /// enum MachineOutlinerClass { MachineOutlinerDefault, MachineOutlinerTailCall }; -std::optional +std::optional> X86InstrInfo::getOutliningCandidateInfo( const MachineModuleInfo &MMI, - std::vector &RepeatedSequenceLocs) const { + std::vector &RepeatedSequenceLocs, + unsigned MinRepeats) const { unsigned SequenceSize = 0; for (auto &MI : RepeatedSequenceLocs[0]) { // FIXME: x86 doesn't implement getInstSizeInBytes, so @@ -10561,9 +10562,10 @@ X86InstrInfo::getOutliningCandidateInfo( for (outliner::Candidate &C : RepeatedSequenceLocs) C.setCallInfo(MachineOutlinerTailCall, 1); - return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, - 0, // Number of bytes to emit frame. - MachineOutlinerTailCall // Type of frame. + return std::make_unique( + RepeatedSequenceLocs, SequenceSize, + 0, // Number of bytes to emit frame. + MachineOutlinerTailCall // Type of frame. ); } @@ -10573,8 +10575,8 @@ X86InstrInfo::getOutliningCandidateInfo( for (outliner::Candidate &C : RepeatedSequenceLocs) C.setCallInfo(MachineOutlinerDefault, 1); - return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, 1, - MachineOutlinerDefault); + return std::make_unique( + RepeatedSequenceLocs, SequenceSize, 1, MachineOutlinerDefault); } bool X86InstrInfo::isFunctionSafeToOutlineFrom( diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h index 3100a9e5699f0..54987a186854d 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.h +++ b/llvm/lib/Target/X86/X86InstrInfo.h @@ -584,9 +584,11 @@ class X86InstrInfo final : public X86GenInstrInfo { ArrayRef> getSerializableDirectMachineOperandTargetFlags() const override; - std::optional getOutliningCandidateInfo( + std::optional> + getOutliningCandidateInfo( const MachineModuleInfo &MMI, - std::vector &RepeatedSequenceLocs) const override; + std::vector &RepeatedSequenceLocs, + unsigned MinRepeats) const override; bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override;