@@ -7634,6 +7634,60 @@ bool BoUpSLP::areAltOperandsProfitable(const InstructionsState &S,
7634
7634
NumAltInsts) < S.getMainOp()->getNumOperands() * VL.size());
7635
7635
}
7636
7636
7637
+ /// Builds the arguments types vector for the given call instruction with the
7638
+ /// given \p ID for the specified vector factor.
7639
+ static SmallVector<Type *>
7640
+ buildIntrinsicArgTypes(const CallInst *CI, const Intrinsic::ID ID,
7641
+ const unsigned VF, unsigned MinBW,
7642
+ const TargetTransformInfo *TTI) {
7643
+ SmallVector<Type *> ArgTys;
7644
+ for (auto [Idx, Arg] : enumerate(CI->args())) {
7645
+ if (ID != Intrinsic::not_intrinsic) {
7646
+ if (isVectorIntrinsicWithScalarOpAtArg(ID, Idx, TTI)) {
7647
+ ArgTys.push_back(Arg->getType());
7648
+ continue;
7649
+ }
7650
+ if (MinBW > 0) {
7651
+ ArgTys.push_back(
7652
+ getWidenedType(IntegerType::get(CI->getContext(), MinBW), VF));
7653
+ continue;
7654
+ }
7655
+ }
7656
+ ArgTys.push_back(getWidenedType(Arg->getType(), VF));
7657
+ }
7658
+ return ArgTys;
7659
+ }
7660
+
7661
+ /// Calculates the costs of vectorized intrinsic (if possible) and vectorized
7662
+ /// function (if possible) calls.
7663
+ static std::pair<InstructionCost, InstructionCost>
7664
+ getVectorCallCosts(CallInst *CI, FixedVectorType *VecTy,
7665
+ TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
7666
+ ArrayRef<Type *> ArgTys) {
7667
+ Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
7668
+
7669
+ // Calculate the cost of the scalar and vector calls.
7670
+ FastMathFlags FMF;
7671
+ if (auto *FPCI = dyn_cast<FPMathOperator>(CI))
7672
+ FMF = FPCI->getFastMathFlags();
7673
+ IntrinsicCostAttributes CostAttrs(ID, VecTy, ArgTys, FMF);
7674
+ auto IntrinsicCost =
7675
+ TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
7676
+
7677
+ auto Shape = VFShape::get(CI->getFunctionType(),
7678
+ ElementCount::getFixed(VecTy->getNumElements()),
7679
+ false /*HasGlobalPred*/);
7680
+ Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
7681
+ auto LibCost = IntrinsicCost;
7682
+ if (!CI->isNoBuiltin() && VecFunc) {
7683
+ // Calculate the cost of the vector library call.
7684
+ // If the corresponding vector call is cheaper, return its cost.
7685
+ LibCost =
7686
+ TTI->getCallInstrCost(nullptr, VecTy, ArgTys, TTI::TCK_RecipThroughput);
7687
+ }
7688
+ return {IntrinsicCost, LibCost};
7689
+ }
7690
+
7637
7691
BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
7638
7692
const InstructionsState &S, ArrayRef<Value *> VL,
7639
7693
bool IsScatterVectorizeUserTE, OrdersType &CurrentOrder,
@@ -9017,34 +9071,6 @@ bool BoUpSLP::areAllUsersVectorized(
9017
9071
});
9018
9072
}
9019
9073
9020
- static std::pair<InstructionCost, InstructionCost>
9021
- getVectorCallCosts(CallInst *CI, FixedVectorType *VecTy,
9022
- TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
9023
- ArrayRef<Type *> ArgTys) {
9024
- Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
9025
-
9026
- // Calculate the cost of the scalar and vector calls.
9027
- FastMathFlags FMF;
9028
- if (auto *FPCI = dyn_cast<FPMathOperator>(CI))
9029
- FMF = FPCI->getFastMathFlags();
9030
- IntrinsicCostAttributes CostAttrs(ID, VecTy, ArgTys, FMF);
9031
- auto IntrinsicCost =
9032
- TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
9033
-
9034
- auto Shape = VFShape::get(CI->getFunctionType(),
9035
- ElementCount::getFixed(VecTy->getNumElements()),
9036
- false /*HasGlobalPred*/);
9037
- Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
9038
- auto LibCost = IntrinsicCost;
9039
- if (!CI->isNoBuiltin() && VecFunc) {
9040
- // Calculate the cost of the vector library call.
9041
- // If the corresponding vector call is cheaper, return its cost.
9042
- LibCost =
9043
- TTI->getCallInstrCost(nullptr, VecTy, ArgTys, TTI::TCK_RecipThroughput);
9044
- }
9045
- return {IntrinsicCost, LibCost};
9046
- }
9047
-
9048
9074
void BoUpSLP::TreeEntry::buildAltOpShuffleMask(
9049
9075
const function_ref<bool(Instruction *)> IsAltOp, SmallVectorImpl<int> &Mask,
9050
9076
SmallVectorImpl<Value *> *OpScalars,
@@ -11045,30 +11071,6 @@ TTI::CastContextHint BoUpSLP::getCastContextHint(const TreeEntry &TE) const {
11045
11071
return TTI::CastContextHint::None;
11046
11072
}
11047
11073
11048
- /// Builds the arguments types vector for the given call instruction with the
11049
- /// given \p ID for the specified vector factor.
11050
- static SmallVector<Type *>
11051
- buildIntrinsicArgTypes(const CallInst *CI, const Intrinsic::ID ID,
11052
- const unsigned VF, unsigned MinBW,
11053
- const TargetTransformInfo *TTI) {
11054
- SmallVector<Type *> ArgTys;
11055
- for (auto [Idx, Arg] : enumerate(CI->args())) {
11056
- if (ID != Intrinsic::not_intrinsic) {
11057
- if (isVectorIntrinsicWithScalarOpAtArg(ID, Idx, TTI)) {
11058
- ArgTys.push_back(Arg->getType());
11059
- continue;
11060
- }
11061
- if (MinBW > 0) {
11062
- ArgTys.push_back(
11063
- getWidenedType(IntegerType::get(CI->getContext(), MinBW), VF));
11064
- continue;
11065
- }
11066
- }
11067
- ArgTys.push_back(getWidenedType(Arg->getType(), VF));
11068
- }
11069
- return ArgTys;
11070
- }
11071
-
11072
11074
InstructionCost
11073
11075
BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
11074
11076
SmallPtrSetImpl<Value *> &CheckedExtracts) {
0 commit comments