Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions llvm/include/llvm/Analysis/TargetTransformInfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -961,12 +961,10 @@ class TargetTransformInfo {
TTI::TargetCostKind CostKind, bool ForPoisonSrc = true,
ArrayRef<Value *> VL = {}) const;

/// Estimate the overhead of scalarizing an instructions unique
/// non-constant operands. The (potentially vector) types to use for each of
/// argument are passes via Tys.
/// Estimate the overhead of scalarizing operands with the given types. The
/// (potentially vector) types to use for each of argument are passes via Tys.
LLVM_ABI InstructionCost getOperandsScalarizationOverhead(
ArrayRef<const Value *> Args, ArrayRef<Type *> Tys,
TTI::TargetCostKind CostKind) const;
ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind) const;

/// If target has efficient vector element load/store instructions, it can
/// return true here so that insertion/extraction costs are not added to
Expand Down
3 changes: 1 addition & 2 deletions llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
Original file line number Diff line number Diff line change
Expand Up @@ -459,8 +459,7 @@ class TargetTransformInfoImplBase {
}

virtual InstructionCost
getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
ArrayRef<Type *> Tys,
getOperandsScalarizationOverhead(ArrayRef<Type *> Tys,
TTI::TargetCostKind CostKind) const {
return 0;
}
Expand Down
46 changes: 28 additions & 18 deletions llvm/include/llvm/CodeGen/BasicTTIImpl.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

#include "llvm/ADT/APInt.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/LoopInfo.h"
Expand Down Expand Up @@ -347,6 +348,21 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return Cost;
}

/// Filter out constant and duplicated entries in \p Ops and return a vector
/// containing the types from \p Tys corresponding to the remaining operands.
static SmallVector<Type *, 4>
filterConstantAndDuplicatedOperands(ArrayRef<const Value *> Ops,
ArrayRef<Type *> Tys) {
SmallPtrSet<const Value *, 4> UniqueOperands;
SmallVector<Type *, 4> FilteredTys;
for (const auto &[Op, Ty] : zip_equal(Ops, Tys)) {
if (isa<Constant>(Op) || !UniqueOperands.insert(Op).second)
continue;
FilteredTys.push_back(Ty);
}
return FilteredTys;
}

protected:
explicit BasicTTIImplBase(const TargetMachine *TM, const DataLayout &DL)
: BaseT(DL) {}
Expand Down Expand Up @@ -935,29 +951,21 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
CostKind);
}

/// Estimate the overhead of scalarizing an instructions unique
/// non-constant operands. The (potentially vector) types to use for each of
/// Estimate the overhead of scalarizing an instruction's
/// operands. The (potentially vector) types to use for each of
/// argument are passes via Tys.
InstructionCost getOperandsScalarizationOverhead(
ArrayRef<const Value *> Args, ArrayRef<Type *> Tys,
TTI::TargetCostKind CostKind) const override {
assert(Args.size() == Tys.size() && "Expected matching Args and Tys");

ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind) const override {
InstructionCost Cost = 0;
SmallPtrSet<const Value*, 4> UniqueOperands;
for (int I = 0, E = Args.size(); I != E; I++) {
for (Type *Ty : Tys) {
// Disregard things like metadata arguments.
const Value *A = Args[I];
Type *Ty = Tys[I];
if (!Ty->isIntOrIntVectorTy() && !Ty->isFPOrFPVectorTy() &&
!Ty->isPtrOrPtrVectorTy())
continue;

if (!isa<Constant>(A) && UniqueOperands.insert(A).second) {
if (auto *VecTy = dyn_cast<VectorType>(Ty))
Cost += getScalarizationOverhead(VecTy, /*Insert*/ false,
/*Extract*/ true, CostKind);
}
if (auto *VecTy = dyn_cast<VectorType>(Ty))
Cost += getScalarizationOverhead(VecTy, /*Insert*/ false,
/*Extract*/ true, CostKind);
}

return Cost;
Expand All @@ -974,7 +982,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
InstructionCost Cost = getScalarizationOverhead(
RetTy, /*Insert*/ true, /*Extract*/ false, CostKind);
if (!Args.empty())
Cost += getOperandsScalarizationOverhead(Args, Tys, CostKind);
Cost += getOperandsScalarizationOverhead(
filterConstantAndDuplicatedOperands(Args, Tys), CostKind);
else
// When no information on arguments is provided, we add the cost
// associated with one argument as a heuristic.
Expand Down Expand Up @@ -2170,8 +2179,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
/*Insert=*/true, /*Extract=*/false, CostKind);
}
}
ScalarizationCost +=
getOperandsScalarizationOverhead(Args, ICA.getArgTypes(), CostKind);
ScalarizationCost += getOperandsScalarizationOverhead(
filterConstantAndDuplicatedOperands(Args, ICA.getArgTypes()),
CostKind);
}

IntrinsicCostAttributes Attrs(IID, RetTy, ICA.getArgTypes(), FMF, I,
Expand Down
5 changes: 2 additions & 3 deletions llvm/lib/Analysis/TargetTransformInfo.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -637,9 +637,8 @@ InstructionCost TargetTransformInfo::getScalarizationOverhead(
}

InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead(
ArrayRef<const Value *> Args, ArrayRef<Type *> Tys,
TTI::TargetCostKind CostKind) const {
return TTIImpl->getOperandsScalarizationOverhead(Args, Tys, CostKind);
ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind) const {
return TTIImpl->getOperandsScalarizationOverhead(Tys, CostKind);
}

bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const {
Expand Down
15 changes: 11 additions & 4 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1697,8 +1697,16 @@ class LoopVectorizationCostModel {
/// Returns a range containing only operands needing to be extracted.
SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
ElementCount VF) const {
return SmallVector<Value *, 4>(make_filter_range(
Ops, [this, VF](Value *V) { return this->needsExtract(V, VF); }));

SmallPtrSet<const Value *, 4> UniqueOperands;
SmallVector<Value *, 4> Res;
for (Value *Op : Ops) {
if (isa<Constant>(Op) || !UniqueOperands.insert(Op).second ||
!needsExtract(Op, VF))
continue;
Res.push_back(Op);
}
return Res;
}

public:
Expand Down Expand Up @@ -5610,8 +5618,7 @@ LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
SmallVector<Type *> Tys;
for (auto *V : filterExtractingOperands(Ops, VF))
Tys.push_back(maybeVectorizeType(V->getType(), VF));
return Cost + TTI.getOperandsScalarizationOverhead(
filterExtractingOperands(Ops, VF), Tys, CostKind);
return Cost + TTI.getOperandsScalarizationOverhead(Tys, CostKind);
}

void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
Expand Down