diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 001c8987667df..193bca7e5d49a 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4363,7 +4363,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks( [](const auto *R) { return Instruction::Store; }) .Case<VPWidenLoadRecipe>( [](const auto *R) { return Instruction::Load; }) - .Case<VPWidenCallRecipe>( + .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>( [](const auto *R) { return Instruction::Call; }) .Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe, VPWidenCastRecipe>( @@ -4387,12 +4387,18 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks( OS << (Pair.second == Subset.front().second ? "" : ", ") << Pair.second; OS << "):"; if (Opcode == Instruction::Call) { - auto *WidenCall = dyn_cast<VPWidenCallRecipe>(R); - Function *CalledFn = - WidenCall ? WidenCall->getCalledScalarFunction() - : cast<Function>(R->getOperand(R->getNumOperands() - 1) - ->getLiveInIRValue()); - OS << " call to " << CalledFn->getName(); + StringRef Name = ""; + if (auto *Int = dyn_cast<VPWidenIntrinsicRecipe>(R)) { + Name = Int->getIntrinsicName(); + } else { + auto *WidenCall = dyn_cast<VPWidenCallRecipe>(R); + Function *CalledFn = + WidenCall ? WidenCall->getCalledScalarFunction() + : cast<Function>(R->getOperand(R->getNumOperands() - 1) + ->getLiveInIRValue()); + Name = CalledFn->getName(); + } + OS << " call to " << Name; } else OS << " " << Instruction::getOpcodeName(Opcode); reportVectorizationInfo(OutString, "InvalidCost", ORE, OrigLoop, nullptr, @@ -4443,6 +4449,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF, case VPDef::VPWidenCanonicalIVSC: case VPDef::VPWidenCastSC: case VPDef::VPWidenGEPSC: + case VPDef::VPWidenIntrinsicSC: case VPDef::VPWidenSC: case VPDef::VPWidenSelectSC: case VPDef::VPBlendSC: @@ -8266,7 +8273,7 @@ VPBlendRecipe *VPRecipeBuilder::tryToBlend(PHINode *Phi, return new VPBlendRecipe(Phi, OperandsWithMask); } -VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, +VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands, VFRange &Range) { bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange( @@ -8286,7 +8293,6 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, return nullptr; SmallVector<VPValue *, 4> Ops(Operands.take_front(CI->arg_size())); - Ops.push_back(Operands.back()); // Is it beneficial to perform intrinsic call compared to lib call? bool ShouldUseVectorIntrinsic = @@ -8297,8 +8303,8 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, }, Range); if (ShouldUseVectorIntrinsic) - return new VPWidenCallRecipe(CI, make_range(Ops.begin(), Ops.end()), ID, - CI->getDebugLoc()); + return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), + CI->getDebugLoc()); Function *Variant = nullptr; std::optional<unsigned> MaskPos; @@ -8350,9 +8356,8 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, Ops.insert(Ops.begin() + *MaskPos, Mask); } - return new VPWidenCallRecipe(CI, make_range(Ops.begin(), Ops.end()), - Intrinsic::not_intrinsic, CI->getDebugLoc(), - Variant); + Ops.push_back(Operands.back()); + return new VPWidenCallRecipe(CI, Variant, Ops, CI->getDebugLoc()); } return nullptr; @@ -9225,7 +9230,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( RecurrenceDescriptor::isFMulAddIntrinsic(CurrentLinkI) && "Expected instruction to be a call to the llvm.fmuladd intrinsic"); assert(((MinVF.isScalar() && isa<VPReplicateRecipe>(CurrentLink)) || - isa<VPWidenCallRecipe>(CurrentLink)) && + isa<VPWidenIntrinsicRecipe>(CurrentLink)) && CurrentLink->getOperand(2) == PreviousLink && "expected a call where the previous link is the added operand"); diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h index 02ef6f7b6fb98..5d4a3b555981c 100644 --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -93,9 +93,9 @@ class VPRecipeBuilder { VPBlendRecipe *tryToBlend(PHINode *Phi, ArrayRef<VPValue *> Operands); /// Handle call instructions. If \p CI can be widened for \p Range.Start, - /// return a new VPWidenCallRecipe. Range.End may be decreased to ensure same - /// decision from \p Range.Start to \p Range.End. - VPWidenCallRecipe *tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands, + /// return a new VPWidenCallRecipe or VPWidenIntrinsicRecipe. Range.End may be + /// decreased to ensure same decision from \p Range.Start to \p Range.End. + VPSingleDefRecipe *tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands, VFRange &Range); /// Check if \p I has an opcode that can be widened and return a VPWidenRecipe diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 8c5246d613c13..3aecfaffaf2b8 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -886,6 +886,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue { case VPRecipeBase::VPWidenCanonicalIVSC: case VPRecipeBase::VPWidenCastSC: case VPRecipeBase::VPWidenGEPSC: + case VPRecipeBase::VPWidenIntrinsicSC: case VPRecipeBase::VPWidenSC: case VPRecipeBase::VPWidenEVLSC: case VPRecipeBase::VPWidenSelectSC: @@ -1613,25 +1614,85 @@ class VPScalarCastRecipe : public VPSingleDefRecipe { } }; -/// A recipe for widening Call instructions. -class VPWidenCallRecipe : public VPRecipeWithIRFlags { - /// ID of the vector intrinsic to call when widening the call. If set the - /// Intrinsic::not_intrinsic, a library call will be used instead. +/// A recipe for widening vector intrinsics. +class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags { + /// ID of the vector intrinsic to widen. Intrinsic::ID VectorIntrinsicID; - /// If this recipe represents a library call, Variant stores a pointer to - /// the chosen function. There is a 1:1 mapping between a given VF and the - /// chosen vectorized variant, so there will be a different vplan for each - /// VF with a valid variant. + + /// Scalar return type of the intrinsic. + Type *ResultTy; + + /// True if the intrinsic may read from memory. + bool MayReadFromMemory; + + /// True if the intrinsic may read write to memory. + bool MayWriteToMemory; + + /// True if the intrinsic may have side-effects. + bool MayHaveSideEffects; + +public: + VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID, + ArrayRef<VPValue *> CallArguments, Type *Ty, + DebugLoc DL = {}) + : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), + VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), + MayReadFromMemory(CI.mayReadFromMemory()), + MayWriteToMemory(CI.mayWriteToMemory()), + MayHaveSideEffects(CI.mayHaveSideEffects()) {} + + ~VPWidenIntrinsicRecipe() override = default; + + VPWidenIntrinsicRecipe *clone() override { + return new VPWidenIntrinsicRecipe(*cast<CallInst>(getUnderlyingValue()), + VectorIntrinsicID, {op_begin(), op_end()}, + ResultTy, getDebugLoc()); + } + + VP_CLASSOF_IMPL(VPDef::VPWidenIntrinsicSC) + + /// Produce a widened version of the vector intrinsic. + void execute(VPTransformState &State) override; + + /// Return the cost of this vector intrinsic. + InstructionCost computeCost(ElementCount VF, + VPCostContext &Ctx) const override; + + /// Return the scalar return type of the intrinsic. + Type *getResultType() const { return ResultTy; } + + /// Return to name of the intrinsic as string. + StringRef getIntrinsicName() const; + + /// Returns true if the intrinsic may read from memory. + bool mayReadFromMemory() const { return MayReadFromMemory; } + + /// Returns true if the intrinsic may write to memory. + bool mayWriteToMemory() const { return MayWriteToMemory; } + + /// Returns true if the intrinsic may have side-effects. + bool mayHaveSideEffects() const { return MayHaveSideEffects; } + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + /// Print the recipe. + void print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const override; +#endif +}; + +/// A recipe for widening Call instructions using library calls. +class VPWidenCallRecipe : public VPRecipeWithIRFlags { + /// Variant stores a pointer to the chosen function. There is a 1:1 mapping + /// between a given VF and the chosen vectorized variant, so there will be a + /// different VPlan for each VF with a valid variant. Function *Variant; public: - template <typename IterT> - VPWidenCallRecipe(Value *UV, iterator_range<IterT> CallArguments, - Intrinsic::ID VectorIntrinsicID, DebugLoc DL = {}, - Function *Variant = nullptr) + VPWidenCallRecipe(Value *UV, Function *Variant, + ArrayRef<VPValue *> CallArguments, DebugLoc DL = {}) : VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments, *cast<Instruction>(UV)), - VectorIntrinsicID(VectorIntrinsicID), Variant(Variant) { + Variant(Variant) { assert( isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) && "last operand must be the called function"); @@ -1640,8 +1701,8 @@ class VPWidenCallRecipe : public VPRecipeWithIRFlags { ~VPWidenCallRecipe() override = default; VPWidenCallRecipe *clone() override { - return new VPWidenCallRecipe(getUnderlyingValue(), operands(), - VectorIntrinsicID, getDebugLoc(), Variant); + return new VPWidenCallRecipe(getUnderlyingValue(), Variant, + {op_begin(), op_end()}, getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenCallSC) diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp index 277df0637372d..5a5b3ac19c46a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp @@ -268,6 +268,9 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) { VPReplicateRecipe, VPWidenCallRecipe, VPWidenMemoryRecipe, VPWidenSelectRecipe>( [this](const auto *R) { return inferScalarTypeForRecipe(R); }) + .Case<VPWidenIntrinsicRecipe>([](const VPWidenIntrinsicRecipe *R) { + return R->getResultType(); + }) .Case<VPInterleaveRecipe>([V](const VPInterleaveRecipe *R) { // TODO: Use info from interleave group. return V->getUnderlyingValue()->getType(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index f31db3c17e5ca..4e983214318ab 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -79,6 +79,8 @@ bool VPRecipeBase::mayWriteToMemory() const { return !cast<VPWidenCallRecipe>(this) ->getCalledScalarFunction() ->onlyReadsMemory(); + case VPWidenIntrinsicSC: + return cast<VPWidenIntrinsicRecipe>(this)->mayWriteToMemory(); case VPBranchOnMaskSC: case VPScalarIVStepsSC: case VPPredInstPHISC: @@ -120,6 +122,8 @@ bool VPRecipeBase::mayReadFromMemory() const { return !cast<VPWidenCallRecipe>(this) ->getCalledScalarFunction() ->onlyWritesMemory(); + case VPWidenIntrinsicSC: + return cast<VPWidenIntrinsicRecipe>(this)->mayReadFromMemory(); case VPBranchOnMaskSC: case VPPredInstPHISC: case VPScalarIVStepsSC: @@ -161,6 +165,8 @@ bool VPRecipeBase::mayHaveSideEffects() const { Function *Fn = cast<VPWidenCallRecipe>(this)->getCalledScalarFunction(); return mayWriteToMemory() || !Fn->doesNotThrow() || !Fn->willReturn(); } + case VPWidenIntrinsicSC: + return cast<VPWidenIntrinsicRecipe>(this)->mayHaveSideEffects(); case VPBlendSC: case VPReductionEVLSC: case VPReductionSC: @@ -880,56 +886,103 @@ void VPIRInstruction::print(raw_ostream &O, const Twine &Indent, void VPWidenCallRecipe::execute(VPTransformState &State) { assert(State.VF.isVector() && "not widening"); - Function *CalledScalarFn = getCalledScalarFunction(); - assert(!isDbgInfoIntrinsic(CalledScalarFn->getIntrinsicID()) && - "DbgInfoIntrinsic should have been dropped during VPlan construction"); State.setDebugLocFrom(getDebugLoc()); - bool UseIntrinsic = VectorIntrinsicID != Intrinsic::not_intrinsic; - FunctionType *VFTy = nullptr; - if (Variant) - VFTy = Variant->getFunctionType(); - SmallVector<Type *, 2> TysForDecl; + FunctionType *VFTy = Variant->getFunctionType(); // Add return type if intrinsic is overloaded on it. - if (UseIntrinsic && - isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1)) - TysForDecl.push_back(VectorType::get( - CalledScalarFn->getReturnType()->getScalarType(), State.VF)); SmallVector<Value *, 4> Args; for (const auto &I : enumerate(arg_operands())) { - // Some intrinsics have a scalar argument - don't replace it with a - // vector. Value *Arg; - if (UseIntrinsic && - isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index())) - Arg = State.get(I.value(), VPLane(0)); // Some vectorized function variants may also take a scalar argument, // e.g. linear parameters for pointers. This needs to be the scalar value // from the start of the respective part when interleaving. - else if (VFTy && !VFTy->getParamType(I.index())->isVectorTy()) + if (!VFTy->getParamType(I.index())->isVectorTy()) Arg = State.get(I.value(), VPLane(0)); else - Arg = State.get(I.value()); - if (UseIntrinsic && - isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index())) - TysForDecl.push_back(Arg->getType()); + Arg = State.get(I.value(), onlyFirstLaneUsed(I.value())); Args.push_back(Arg); } - Function *VectorF; - if (UseIntrinsic) { - // Use vector version of the intrinsic. - Module *M = State.Builder.GetInsertBlock()->getModule(); - VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl); - assert(VectorF && "Can't retrieve vector intrinsic."); - } else { -#ifndef NDEBUG - assert(Variant != nullptr && "Can't create vector function."); + assert(Variant != nullptr && "Can't create vector function."); + + auto *CI = cast_or_null<CallInst>(getUnderlyingValue()); + SmallVector<OperandBundleDef, 1> OpBundles; + if (CI) + CI->getOperandBundlesAsDefs(OpBundles); + + CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles); + setFlags(V); + + if (!V->getType()->isVoidTy()) + State.set(this, V); + State.addMetadata(V, CI); +} + +InstructionCost VPWidenCallRecipe::computeCost(ElementCount VF, + VPCostContext &Ctx) const { + TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; + return Ctx.TTI.getCallInstrCost(nullptr, Variant->getReturnType(), + Variant->getFunctionType()->params(), + CostKind); +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const { + O << Indent << "WIDEN-CALL "; + + Function *CalledFn = getCalledScalarFunction(); + if (CalledFn->getReturnType()->isVoidTy()) + O << "void "; + else { + printAsOperand(O, SlotTracker); + O << " = "; + } + + O << "call"; + printFlags(O); + O << " @" << CalledFn->getName() << "("; + interleaveComma(arg_operands(), O, [&O, &SlotTracker](VPValue *Op) { + Op->printAsOperand(O, SlotTracker); + }); + O << ")"; + + O << " (using library function"; + if (Variant->hasName()) + O << ": " << Variant->getName(); + O << ")"; +} #endif - VectorF = Variant; + +void VPWidenIntrinsicRecipe::execute(VPTransformState &State) { + assert(State.VF.isVector() && "not widening"); + State.setDebugLocFrom(getDebugLoc()); + + SmallVector<Type *, 2> TysForDecl; + // Add return type if intrinsic is overloaded on it. + if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1)) + TysForDecl.push_back(VectorType::get(getResultType(), State.VF)); + SmallVector<Value *, 4> Args; + for (const auto &I : enumerate(operands())) { + // Some intrinsics have a scalar argument - don't replace it with a + // vector. + Value *Arg; + if (isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index())) + Arg = State.get(I.value(), VPLane(0)); + else + Arg = State.get(I.value(), onlyFirstLaneUsed(I.value())); + if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index())) + TysForDecl.push_back(Arg->getType()); + Args.push_back(Arg); } - auto *CI = cast_or_null<CallInst>(getUnderlyingInstr()); + // Use vector version of the intrinsic. + Module *M = State.Builder.GetInsertBlock()->getModule(); + Function *VectorF = + Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl); + assert(VectorF && "Can't retrieve vector intrinsic."); + + auto *CI = cast_or_null<CallInst>(getUnderlyingValue()); SmallVector<OperandBundleDef, 1> OpBundles; if (CI) CI->getOperandBundlesAsDefs(OpBundles); @@ -943,14 +996,9 @@ void VPWidenCallRecipe::execute(VPTransformState &State) { State.addMetadata(V, CI); } -InstructionCost VPWidenCallRecipe::computeCost(ElementCount VF, - VPCostContext &Ctx) const { +InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF, + VPCostContext &Ctx) const { TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; - if (Variant) { - return Ctx.TTI.getCallInstrCost(nullptr, Variant->getReturnType(), - Variant->getFunctionType()->params(), - CostKind); - } // Some backends analyze intrinsic arguments to determine cost. Use the // underlying value for the operand if it has one. Otherwise try to use the @@ -986,35 +1034,29 @@ InstructionCost VPWidenCallRecipe::computeCost(ElementCount VF, return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, CostKind); } -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent, - VPSlotTracker &SlotTracker) const { - O << Indent << "WIDEN-CALL "; +StringRef VPWidenIntrinsicRecipe::getIntrinsicName() const { + return Intrinsic::getBaseName(VectorIntrinsicID); +} - Function *CalledFn = getCalledScalarFunction(); - if (CalledFn->getReturnType()->isVoidTy()) +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +void VPWidenIntrinsicRecipe::print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const { + O << Indent << "WIDEN-INTRINSIC "; + if (ResultTy->isVoidTy()) { O << "void "; - else { + } else { printAsOperand(O, SlotTracker); O << " = "; } O << "call"; printFlags(O); - O << " @" << CalledFn->getName() << "("; - interleaveComma(arg_operands(), O, [&O, &SlotTracker](VPValue *Op) { + O << getIntrinsicName() << "("; + + interleaveComma(operands(), O, [&O, &SlotTracker](VPValue *Op) { Op->printAsOperand(O, SlotTracker); }); O << ")"; - - if (VectorIntrinsicID) - O << " (using vector intrinsic)"; - else { - O << " (using library function"; - if (Variant->hasName()) - O << ": " << Variant->getName(); - O << ")"; - } } #endif diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 9ca14fc781213..379bfc0a4394b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -82,8 +82,9 @@ void VPlanTransforms::VPInstructionsToVPRecipes( } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) { NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands()); } else if (CallInst *CI = dyn_cast<CallInst>(Inst)) { - NewRecipe = new VPWidenCallRecipe( - CI, Ingredient.operands(), getVectorIntrinsicIDForCall(CI, &TLI), + NewRecipe = new VPWidenIntrinsicRecipe( + *CI, getVectorIntrinsicIDForCall(CI, &TLI), + {Ingredient.op_begin(), Ingredient.op_end() - 1}, CI->getType(), CI->getDebugLoc()); } else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) { NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands()); diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index 4c383244f96f1..f2978b0a758b6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -350,6 +350,7 @@ class VPDef { VPWidenCanonicalIVSC, VPWidenCastSC, VPWidenGEPSC, + VPWidenIntrinsicSC, VPWidenLoadEVLSC, VPWidenLoadSC, VPWidenStoreEVLSC, diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll index 7797c0bce0baa..a55e53cc9b0bd 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll @@ -102,7 +102,7 @@ for.end: ; CHECK-REMARKS: UserVF ignored because of invalid costs. ; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load -; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin.f32 +; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin ; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store define void @vec_sin_no_mapping(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) { ; CHECK: @vec_sin_no_mapping @@ -129,8 +129,8 @@ for.cond.cleanup: ; preds = %for.body ; CHECK-REMARKS: UserVF ignored because of invalid costs. ; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load ; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): fadd -; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin.f32 -; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin.f32 +; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin +; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin ; CHECK-REMARKS-NEXT: t.c:3:40: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store define void @vec_sin_no_mapping_ite(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) { ; CHECK: @vec_sin_no_mapping_ite @@ -166,7 +166,7 @@ for.cond.cleanup: ; preds = %for.body ; CHECK-REMARKS: UserVF ignored because of invalid costs. ; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load -; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin.f32 +; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin ; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store define void @vec_sin_fixed_mapping(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) { ; CHECK: @vec_sin_fixed_mapping diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll index a43e1a6755ed7..e9303ec9d3eb7 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll @@ -60,7 +60,7 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.src> ; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]> ; CHECK-NEXT: WIDEN-CAST ir<%conv> = fpext ir<%l> to double -; CHECK-NEXT: WIDEN-CALL ir<%s> = call reassoc nnan ninf nsz arcp contract afn @llvm.sin.f64(ir<%conv>) (using vector intrinsic) +; CHECK-NEXT: WIDEN-INTRINSIC ir<%s> = call reassoc nnan ninf nsz arcp contract afn llvm.sin(ir<%conv>) ; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> ; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll index d872fb187a3bc..68dd47537fdfd 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll @@ -31,7 +31,7 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" + ; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer ir\<%arrayidx\>\l" + ; CHECK-NEXT: " WIDEN ir\<%lv\> = load vp\<[[VEC_PTR]]\>\l" + -; CHECK-NEXT: " WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>) (using vector intrinsic)\l" + +; CHECK-NEXT: " WIDEN-INTRINSIC ir\<%call\> = call llvm.sqrt(ir\<%lv\>)\l" + ; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" + ; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer ir\<%arrayidx2\>\l" + ; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, ir\<%call\>\l" + diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll index bb8bdd2fb52e3..0dde507d08be7 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -23,7 +23,7 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> ; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> ; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>) +; CHECK-NEXT: WIDEN-INTRINSIC ir<%call> = call llvm.sqrt(ir<%lv>) ; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> ; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2> ; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index a5d44e9a57246..372c5aaea5938 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -901,7 +901,7 @@ TEST(VPRecipeTest, CastVPWidenCallRecipeToVPUserAndVPDef) { Args.push_back(&Op1); Args.push_back(&Op2); Args.push_back(&CalledFn); - VPWidenCallRecipe Recipe(Call, make_range(Args.begin(), Args.end()), false); + VPWidenCallRecipe Recipe(Call, Fn, Args); EXPECT_TRUE(isa<VPUser>(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa<VPUser>(BaseR)); @@ -1182,7 +1182,7 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { Args.push_back(&Op1); Args.push_back(&Op2); Args.push_back(&CalledFn); - VPWidenCallRecipe Recipe(Call, make_range(Args.begin(), Args.end()), false); + VPWidenCallRecipe Recipe(Call, Fn, Args); EXPECT_TRUE(Recipe.mayHaveSideEffects()); EXPECT_TRUE(Recipe.mayReadFromMemory()); EXPECT_TRUE(Recipe.mayWriteToMemory()); @@ -1205,7 +1205,7 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { Args.push_back(&Op1); Args.push_back(&Op2); Args.push_back(&CalledFn); - VPWidenCallRecipe Recipe(Call, make_range(Args.begin(), Args.end()), false); + VPWidenCallRecipe Recipe(Call, TheFn, Args); EXPECT_FALSE(Recipe.mayHaveSideEffects()); EXPECT_FALSE(Recipe.mayReadFromMemory()); EXPECT_FALSE(Recipe.mayWriteToMemory());