diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index e9e1d0ac6c196..b784cc701cd4b 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9757,7 +9757,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { Legal->getUncountableEarlyExitingBlock()) { VPlanTransforms::runPass(VPlanTransforms::handleUncountableEarlyExit, *Plan, *PSE.getSE(), OrigLoop, UncountableExitingBlock, - RecipeBuilder); + RecipeBuilder, Range); } DenseMap IVEndValues; addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index d0cb1c3ac590f..46cd9174e1119 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -17,6 +17,7 @@ #include "VPlanAnalysis.h" #include "VPlanCFG.h" #include "VPlanDominatorTree.h" +#include "VPlanHelpers.h" #include "VPlanPatternMatch.h" #include "VPlanUtils.h" #include "VPlanVerifier.h" @@ -2445,7 +2446,8 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan, void VPlanTransforms::handleUncountableEarlyExit( VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop, - BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder) { + BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder, + VFRange &Range) { VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion(); auto *LatchVPBB = cast(LoopRegion->getExiting()); VPBuilder Builder(LatchVPBB->getTerminator()); @@ -2500,8 +2502,14 @@ void VPlanTransforms::handleUncountableEarlyExit( ExitIRI->addOperand(IncomingFromLatch); ExitIRI->extractLastLaneOfOperand(MiddleBuilder); } - // Add the incoming value from the early exit. - if (!IncomingFromEarlyExit->isLiveIn() && !Plan.hasScalarVFOnly()) { + + auto IsVector = [](ElementCount VF) { return VF.isVector(); }; + // When the VFs are vectors, need to add `extract` to get the incoming value + // from early exit. When the range contains scalar VF, limit the range to + // scalar VF to prevent mis-compilation for the range containing both scalar + // and vector VFs. + if (!IncomingFromEarlyExit->isLiveIn() && + LoopVectorizationPlanner::getDecisionAndClampRange(IsVector, Range)) { VPValue *FirstActiveLane = EarlyExitB.createNaryOp( VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr, "first.active.lane"); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index a9461b261ddb6..1200e8eaab0ba 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -28,6 +28,7 @@ class PredicatedScalarEvolution; class TargetLibraryInfo; class VPBuilder; class VPRecipeBuilder; +class VFRange; extern cl::opt VerifyEachVPlan; @@ -173,7 +174,8 @@ struct VPlanTransforms { static void handleUncountableEarlyExit(VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop, BasicBlock *UncountableExitingBlock, - VPRecipeBuilder &RecipeBuilder); + VPRecipeBuilder &RecipeBuilder, + VFRange &Range); /// Lower abstract recipes to concrete ones, that can be codegen'd. Use \p /// CanonicalIVTy as type for all un-typed live-ins in VPTypeAnalysis.