Skip to content

Commit f9c9f33

Browse files
fhahnfhahn
committed
[VPlan] Introduce VPWidenIntrinsicRecipe to separate from libcall.
This patch splits off intrinsic hanlding to a new VPWidenIntrinsicRecipe. VPWidenIntrinsicRecipes only need access to the intrinsic ID to widen and the scalar result type (in case the intrinsic is overloaded on the result type). It does not need access to an underlying IR call instruction or function. This means VPWidenIntrinsicRecipe can be created easily without access to underlying IR.
1 parent 79382eb commit f9c9f33

File tree

12 files changed

+194
-101
lines changed

12 files changed

+194
-101
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4367,7 +4367,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
43674367
[](const auto *R) { return Instruction::Store; })
43684368
.Case<VPWidenLoadRecipe>(
43694369
[](const auto *R) { return Instruction::Load; })
4370-
.Case<VPWidenCallRecipe>(
4370+
.Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
43714371
[](const auto *R) { return Instruction::Call; })
43724372
.Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe,
43734373
VPWidenCastRecipe>(
@@ -4392,11 +4392,17 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
43924392
OS << "):";
43934393
if (Opcode == Instruction::Call) {
43944394
auto *WidenCall = dyn_cast<VPWidenCallRecipe>(R);
4395-
Function *CalledFn =
4396-
WidenCall ? WidenCall->getCalledScalarFunction()
4397-
: cast<Function>(R->getOperand(R->getNumOperands() - 1)
4398-
->getLiveInIRValue());
4399-
OS << " call to " << CalledFn->getName();
4395+
StringRef Name = "";
4396+
if (auto *Int = dyn_cast<VPWidenIntrinsicRecipe>(R))
4397+
Name = Int->getIntrinsicName();
4398+
else {
4399+
Function *CalledFn =
4400+
WidenCall ? WidenCall->getCalledScalarFunction()
4401+
: cast<Function>(R->getOperand(R->getNumOperands() - 1)
4402+
->getLiveInIRValue());
4403+
Name = CalledFn->getName();
4404+
}
4405+
OS << " call to " << Name;
44004406
} else
44014407
OS << " " << Instruction::getOpcodeName(Opcode);
44024408
reportVectorizationInfo(OutString, "InvalidCost", ORE, OrigLoop, nullptr,
@@ -4447,6 +4453,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
44474453
case VPDef::VPWidenCanonicalIVSC:
44484454
case VPDef::VPWidenCastSC:
44494455
case VPDef::VPWidenGEPSC:
4456+
case VPDef::VPWidenIntrinsicSC:
44504457
case VPDef::VPWidenSC:
44514458
case VPDef::VPWidenSelectSC:
44524459
case VPDef::VPBlendSC:
@@ -8266,7 +8273,7 @@ VPBlendRecipe *VPRecipeBuilder::tryToBlend(PHINode *Phi,
82668273
return new VPBlendRecipe(Phi, OperandsWithMask);
82678274
}
82688275

8269-
VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
8276+
VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
82708277
ArrayRef<VPValue *> Operands,
82718278
VFRange &Range) {
82728279
bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
@@ -8297,8 +8304,9 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
82978304
},
82988305
Range);
82998306
if (ShouldUseVectorIntrinsic)
8300-
return new VPWidenCallRecipe(CI, make_range(Ops.begin(), Ops.end()), ID,
8301-
CI->getDebugLoc());
8307+
return new VPWidenIntrinsicRecipe(CI, ID,
8308+
make_range(Ops.begin(), Ops.end() - 1),
8309+
CI->getType(), CI->getDebugLoc());
83028310

83038311
Function *Variant = nullptr;
83048312
std::optional<unsigned> MaskPos;
@@ -8350,9 +8358,8 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
83508358
Ops.insert(Ops.begin() + *MaskPos, Mask);
83518359
}
83528360

8353-
return new VPWidenCallRecipe(CI, make_range(Ops.begin(), Ops.end()),
8354-
Intrinsic::not_intrinsic, CI->getDebugLoc(),
8355-
Variant);
8361+
return new VPWidenCallRecipe(
8362+
CI, Variant, make_range(Ops.begin(), Ops.end()), CI->getDebugLoc());
83568363
}
83578364

83588365
return nullptr;
@@ -9218,7 +9225,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92189225
RecurrenceDescriptor::isFMulAddIntrinsic(CurrentLinkI) &&
92199226
"Expected instruction to be a call to the llvm.fmuladd intrinsic");
92209227
assert(((MinVF.isScalar() && isa<VPReplicateRecipe>(CurrentLink)) ||
9221-
isa<VPWidenCallRecipe>(CurrentLink)) &&
9228+
isa<VPWidenIntrinsicRecipe>(CurrentLink)) &&
92229229
CurrentLink->getOperand(2) == PreviousLink &&
92239230
"expected a call where the previous link is the added operand");
92249231

‎llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,9 @@ class VPRecipeBuilder {
9393
VPBlendRecipe *tryToBlend(PHINode *Phi, ArrayRef<VPValue *> Operands);
9494

9595
/// Handle call instructions. If \p CI can be widened for \p Range.Start,
96-
/// return a new VPWidenCallRecipe. Range.End may be decreased to ensure same
97-
/// decision from \p Range.Start to \p Range.End.
98-
VPWidenCallRecipe *tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands,
96+
/// return a new VPWidenCallRecipe or VPWidenIntrinsicRecipe. Range.End may be
97+
/// decreased to ensure same decision from \p Range.Start to \p Range.End.
98+
VPSingleDefRecipe *tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands,
9999
VFRange &Range);
100100

101101
/// Check if \p I has an opcode that can be widened and return a VPWidenRecipe

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 54 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -886,6 +886,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
886886
case VPRecipeBase::VPWidenCanonicalIVSC:
887887
case VPRecipeBase::VPWidenCastSC:
888888
case VPRecipeBase::VPWidenGEPSC:
889+
case VPRecipeBase::VPWidenIntrinsicSC:
889890
case VPRecipeBase::VPWidenSC:
890891
case VPRecipeBase::VPWidenEVLSC:
891892
case VPRecipeBase::VPWidenSelectSC:
@@ -1608,24 +1609,63 @@ class VPScalarCastRecipe : public VPSingleDefRecipe {
16081609
}
16091610
};
16101611

1611-
/// A recipe for widening Call instructions.
1612-
class VPWidenCallRecipe : public VPSingleDefRecipe {
1613-
/// ID of the vector intrinsic to call when widening the call. If set the
1614-
/// Intrinsic::not_intrinsic, a library call will be used instead.
1612+
/// A recipe for widening vector intrinsics.
1613+
class VPWidenIntrinsicRecipe : public VPSingleDefRecipe {
1614+
/// ID of the vector intrinsic to widen.
16151615
Intrinsic::ID VectorIntrinsicID;
1616-
/// If this recipe represents a library call, Variant stores a pointer to
1617-
/// the chosen function. There is a 1:1 mapping between a given VF and the
1618-
/// chosen vectorized variant, so there will be a different vplan for each
1619-
/// VF with a valid variant.
1616+
1617+
/// Scalar type of the result produced by the intrinsic.
1618+
Type *ResultTy;
1619+
1620+
public:
1621+
template <typename IterT>
1622+
VPWidenIntrinsicRecipe(Value *UV, Intrinsic::ID VectorIntrinsicID,
1623+
iterator_range<IterT> CallArguments, Type *Ty,
1624+
DebugLoc DL = {})
1625+
: VPSingleDefRecipe(VPDef::VPWidenIntrinsicSC, CallArguments, UV, DL),
1626+
VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {}
1627+
1628+
~VPWidenIntrinsicRecipe() override = default;
1629+
1630+
VPWidenIntrinsicRecipe *clone() override {
1631+
return new VPWidenIntrinsicRecipe(getUnderlyingValue(), VectorIntrinsicID,
1632+
operands(), ResultTy, getDebugLoc());
1633+
}
1634+
1635+
VP_CLASSOF_IMPL(VPDef::VPWidenIntrinsicSC)
1636+
1637+
/// Produce a widened version of the vector intrinsic.
1638+
void execute(VPTransformState &State) override;
1639+
1640+
/// Return the cost of this vector intrinsic.
1641+
InstructionCost computeCost(ElementCount VF,
1642+
VPCostContext &Ctx) const override;
1643+
1644+
Type *getResultTy() const { return ResultTy; }
1645+
1646+
/// Return to name of the intrinsic as string.
1647+
StringRef getIntrinsicName() const;
1648+
1649+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1650+
/// Print the recipe.
1651+
void print(raw_ostream &O, const Twine &Indent,
1652+
VPSlotTracker &SlotTracker) const override;
1653+
#endif
1654+
};
1655+
1656+
/// A recipe for widening Call instructions using library calls.
1657+
class VPWidenCallRecipe : public VPSingleDefRecipe {
1658+
/// Variant stores a pointer to the chosen function. There is a 1:1 mapping
1659+
/// between a given VF and the chosen vectorized variant, so there will be a
1660+
/// different VPlan for each VF with a valid variant.
16201661
Function *Variant;
16211662

16221663
public:
16231664
template <typename IterT>
1624-
VPWidenCallRecipe(Value *UV, iterator_range<IterT> CallArguments,
1625-
Intrinsic::ID VectorIntrinsicID, DebugLoc DL = {},
1626-
Function *Variant = nullptr)
1665+
VPWidenCallRecipe(Value *UV, Function *Variant,
1666+
iterator_range<IterT> CallArguments, DebugLoc DL = {})
16271667
: VPSingleDefRecipe(VPDef::VPWidenCallSC, CallArguments, UV, DL),
1628-
VectorIntrinsicID(VectorIntrinsicID), Variant(Variant) {
1668+
Variant(Variant) {
16291669
assert(
16301670
isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
16311671
"last operand must be the called function");
@@ -1634,8 +1674,8 @@ class VPWidenCallRecipe : public VPSingleDefRecipe {
16341674
~VPWidenCallRecipe() override = default;
16351675

16361676
VPWidenCallRecipe *clone() override {
1637-
return new VPWidenCallRecipe(getUnderlyingValue(), operands(),
1638-
VectorIntrinsicID, getDebugLoc(), Variant);
1677+
return new VPWidenCallRecipe(getUnderlyingValue(), Variant, operands(),
1678+
getDebugLoc());
16391679
}
16401680

16411681
VP_CLASSOF_IMPL(VPDef::VPWidenCallSC)

‎llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,8 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
268268
VPReplicateRecipe, VPWidenCallRecipe, VPWidenMemoryRecipe,
269269
VPWidenSelectRecipe>(
270270
[this](const auto *R) { return inferScalarTypeForRecipe(R); })
271+
.Case<VPWidenIntrinsicRecipe>(
272+
[](const VPWidenIntrinsicRecipe *R) { return R->getResultTy(); })
271273
.Case<VPInterleaveRecipe>([V](const VPInterleaveRecipe *R) {
272274
// TODO: Use info from interleave group.
273275
return V->getUnderlyingValue()->getType();

0 commit comments

Comments
 (0)