[VPlan] Replace VPRegionBlock with explicit CFG before execute (NFCI). #117506
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
@llvm/pr-subscribers-backend-risc-v @llvm/pr-subscribers-llvm-transforms Author: Florian Hahn (fhahn) ChangesBuilding on top of #114305, Patch is 25.58 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/117506.diff 8 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d13770a35c108f..ec48f90ba4fcca 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2368,12 +2368,6 @@ void InnerLoopVectorizer::scalarizeInstruction(const Instruction *Instr,
// End if-block.
VPRegionBlock *Parent = RepRecipe->getParent()->getParent();
bool IfPredicateInstr = Parent ? Parent->isReplicator() : false;
- assert((Parent || all_of(RepRecipe->operands(),
- [](VPValue *Op) {
- return Op->isDefinedOutsideLoopRegions();
- })) &&
- "Expected a recipe is either within a region or all of its operands "
- "are defined outside the vectorized region.");
if (IfPredicateInstr)
PredicatedInstructions.push_back(Cloned);
}
@@ -2969,8 +2963,8 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
for (Instruction *PI : PredicatedInstructions)
sinkScalarOperands(&*PI);
- VPRegionBlock *VectorRegion = State.Plan->getVectorLoopRegion();
- VPBasicBlock *HeaderVPBB = VectorRegion->getEntryBasicBlock();
+ VPBasicBlock *HeaderVPBB = cast<VPBasicBlock>(
+ State.Plan->getVectorPreheader()->getSingleSuccessor());
BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB];
// Remove redundant induction instructions.
@@ -7742,6 +7736,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
BestVPlan.prepareToExecute(ILV.getTripCount(),
ILV.getOrCreateVectorTripCount(nullptr),
CanonicalIVStartValue, State);
+ VPlanTransforms::prepareToExecute(BestVPlan);
BestVPlan.execute(&State);
@@ -7763,7 +7758,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
LLVMLoopVectorizeFollowupVectorized});
VPBasicBlock *HeaderVPBB =
- BestVPlan.getVectorLoopRegion()->getEntryBasicBlock();
+ cast<VPBasicBlock>(BestVPlan.getVectorPreheader()->getSingleSuccessor());
Loop *L = LI->getLoopFor(State.CFG.VPBB2IRBB[HeaderVPBB]);
if (VectorizedLoopID)
L->setLoopID(*VectorizedLoopID);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 529108a5aaa97f..a3e11c7f2f5f8d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -352,8 +352,8 @@ Value *VPTransformState::get(VPValue *Def, bool NeedsScalar) {
}
BasicBlock *VPTransformState::CFGState::getPreheaderBBFor(VPRecipeBase *R) {
- VPRegionBlock *LoopRegion = R->getParent()->getEnclosingLoopRegion();
- return VPBB2IRBB[LoopRegion->getPreheaderVPBB()];
+
+ return VPBB2IRBB[cast<VPBasicBlock>(R->getParent()->getPredecessors()[0])];
}
void VPTransformState::addNewMetadata(Instruction *To,
@@ -425,6 +425,8 @@ void VPBasicBlock::connectToPredecessors(VPTransformState::CFGState &CFG) {
VPBasicBlock *PredVPBB = PredVPBlock->getExitingBasicBlock();
auto &PredVPSuccessors = PredVPBB->getHierarchicalSuccessors();
BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB];
+ if (!PredBB)
+ continue;
assert(PredBB && "Predecessor basic-block not found building successor.");
auto *PredBBTerminator = PredBB->getTerminator();
@@ -432,6 +434,8 @@ void VPBasicBlock::connectToPredecessors(VPTransformState::CFGState &CFG) {
auto *TermBr = dyn_cast<BranchInst>(PredBBTerminator);
if (isa<UnreachableInst>(PredBBTerminator)) {
+ if (PredVPSuccessors.size() == 2)
+ continue;
assert(PredVPSuccessors.size() == 1 &&
"Predecessor ending w/o branch must have single successor.");
DebugLoc DL = PredBBTerminator->getDebugLoc();
@@ -480,6 +484,21 @@ void VPBasicBlock::execute(VPTransformState *State) {
bool Replica = bool(State->Lane);
BasicBlock *NewBB = State->CFG.PrevBB; // Reuse it if possible.
+ if (isHeader()) {
+ // Create and register the new vector loop.
+ State->CurrentVectorLoop = State->LI->AllocateLoop();
+ BasicBlock *VectorPH =
+ State->CFG.VPBB2IRBB[cast<VPBasicBlock>(getPredecessors()[0])];
+ Loop *ParentLoop = State->LI->getLoopFor(VectorPH);
+
+ // Insert the new loop into the loop nest and register the new basic blocks
+ // before calling any utilities such as SCEV that require valid LoopInfo.
+ if (ParentLoop)
+ ParentLoop->addChildLoop(State->CurrentVectorLoop);
+ else
+ State->LI->addTopLevelLoop(State->CurrentVectorLoop);
+ }
+
auto IsReplicateRegion = [](VPBlockBase *BB) {
auto *R = dyn_cast_or_null<VPRegionBlock>(BB);
return R && R->isReplicator();
@@ -718,37 +737,13 @@ void VPRegionBlock::dropAllReferences(VPValue *NewValue) {
}
void VPRegionBlock::execute(VPTransformState *State) {
- ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>>
- RPOT(Entry);
-
- if (!isReplicator()) {
- // Create and register the new vector loop.
- Loop *PrevLoop = State->CurrentVectorLoop;
- State->CurrentVectorLoop = State->LI->AllocateLoop();
- BasicBlock *VectorPH = State->CFG.VPBB2IRBB[getPreheaderVPBB()];
- Loop *ParentLoop = State->LI->getLoopFor(VectorPH);
-
- // Insert the new loop into the loop nest and register the new basic blocks
- // before calling any utilities such as SCEV that require valid LoopInfo.
- if (ParentLoop)
- ParentLoop->addChildLoop(State->CurrentVectorLoop);
- else
- State->LI->addTopLevelLoop(State->CurrentVectorLoop);
-
- // Visit the VPBlocks connected to "this", starting from it.
- for (VPBlockBase *Block : RPOT) {
- LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n');
- Block->execute(State);
- }
-
- State->CurrentVectorLoop = PrevLoop;
- return;
- }
-
+ assert(isReplicator() &&
+ "Loop regions should have been lowered to plain CFG");
assert(!State->Lane && "Replicating a Region with non-null instance.");
-
- // Enter replicating mode.
assert(!State->VF.isScalable() && "VF is assumed to be non scalable.");
+
+ ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
+ Entry);
State->Lane = VPLane(0);
for (unsigned Lane = 0, VF = State->VF.getKnownMinValue(); Lane < VF;
++Lane) {
@@ -823,6 +818,26 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
}
#endif
+void VPRegionBlock::removeRegion() {
+ auto *Header = cast<VPBasicBlock>(getEntry());
+ VPBlockBase *Preheader = getSinglePredecessor();
+ auto *Exiting = cast<VPBasicBlock>(getExiting());
+
+ VPBlockBase *Middle = getSingleSuccessor();
+ VPBlockUtils::disconnectBlocks(Preheader, this);
+ VPBlockUtils::disconnectBlocks(this, Middle);
+
+ for (VPBlockBase *VPB : vp_depth_first_shallow(Entry))
+ VPB->setParent(nullptr);
+
+ VPBlockUtils::connectBlocks(Preheader, Header);
+ VPBlockUtils::connectBlocks(Exiting, Middle);
+
+ // Set LoopRegion's Entry to nullptr, as the CFG from LoopRegion shouldn't
+ // be deleted when the region is deleted.
+ Entry = nullptr;
+}
+
VPlan::~VPlan() {
if (Entry) {
VPValue DummyValue;
@@ -1032,51 +1047,55 @@ void VPlan::execute(VPTransformState *State) {
for (VPBlockBase *Block : vp_depth_first_shallow(Entry))
Block->execute(State);
- VPBasicBlock *LatchVPBB = getVectorLoopRegion()->getExitingBasicBlock();
- BasicBlock *VectorLatchBB = State->CFG.VPBB2IRBB[LatchVPBB];
-
// Fix the latch value of canonical, reduction and first-order recurrences
// phis in the vector loop.
- VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock();
- for (VPRecipeBase &R : Header->phis()) {
- // Skip phi-like recipes that generate their backedege values themselves.
- if (isa<VPWidenPHIRecipe>(&R))
+ for (VPBasicBlock *Header :
+ VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow(Entry))) {
+ if (!Header->isHeader())
continue;
+ for (VPRecipeBase &R : Header->phis()) {
+ VPBasicBlock *LatchVPBB =
+ cast<VPBasicBlock>(Header->getPredecessors()[1]);
+ BasicBlock *VectorLatchBB = State->CFG.VPBB2IRBB[LatchVPBB];
- if (isa<VPWidenPointerInductionRecipe>(&R) ||
- isa<VPWidenIntOrFpInductionRecipe>(&R)) {
- PHINode *Phi = nullptr;
- if (isa<VPWidenIntOrFpInductionRecipe>(&R)) {
- Phi = cast<PHINode>(State->get(R.getVPSingleValue()));
- } else {
- auto *WidenPhi = cast<VPWidenPointerInductionRecipe>(&R);
- assert(!WidenPhi->onlyScalarsGenerated(State->VF.isScalable()) &&
- "recipe generating only scalars should have been replaced");
- auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi));
- Phi = cast<PHINode>(GEP->getPointerOperand());
- }
-
- Phi->setIncomingBlock(1, VectorLatchBB);
+ // Skip phi-like recipes that generate their backedege values themselves.
+ if (isa<VPWidenPHIRecipe>(&R))
+ continue;
- // Move the last step to the end of the latch block. This ensures
- // consistent placement of all induction updates.
- Instruction *Inc = cast<Instruction>(Phi->getIncomingValue(1));
- Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode());
+ if (isa<VPWidenPointerInductionRecipe>(&R) ||
+ isa<VPWidenIntOrFpInductionRecipe>(&R)) {
+ PHINode *Phi = nullptr;
+ if (isa<VPWidenIntOrFpInductionRecipe>(&R)) {
+ Phi = cast<PHINode>(State->get(R.getVPSingleValue()));
+ } else {
+ auto *WidenPhi = cast<VPWidenPointerInductionRecipe>(&R);
+ assert(!WidenPhi->onlyScalarsGenerated(State->VF.isScalable()) &&
+ "recipe generating only scalars should have been replaced");
+ auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi));
+ Phi = cast<PHINode>(GEP->getPointerOperand());
+ }
+
+ Phi->setIncomingBlock(1, VectorLatchBB);
+
+ // Move the last step to the end of the latch block. This ensures
+ // consistent placement of all induction updates.
+ Instruction *Inc = cast<Instruction>(Phi->getIncomingValue(1));
+ Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode());
+
+ // Use the steps for the last part as backedge value for the induction.
+ if (auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
+ Inc->setOperand(0, State->get(IV->getLastUnrolledPartOperand()));
+ continue;
+ }
- // Use the steps for the last part as backedge value for the induction.
- if (auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
- Inc->setOperand(0, State->get(IV->getLastUnrolledPartOperand()));
- continue;
+ auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
+ bool NeedsScalar = isa<VPScalarPHIRecipe>(PhiR) ||
+ (isa<VPReductionPHIRecipe>(PhiR) &&
+ cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
+ Value *Phi = State->get(PhiR, NeedsScalar);
+ Value *Val = State->get(PhiR->getBackedgeValue(), NeedsScalar);
+ cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
}
-
- auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
- bool NeedsScalar =
- isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe>(PhiR) ||
- (isa<VPReductionPHIRecipe>(PhiR) &&
- cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
- Value *Phi = State->get(PhiR, NeedsScalar);
- Value *Val = State->get(PhiR->getBackedgeValue(), NeedsScalar);
- cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
}
State->CFG.DTU.flush();
@@ -1418,8 +1437,13 @@ void VPlanIngredient::print(raw_ostream &O) const {
#endif
bool VPValue::isDefinedOutsideLoopRegions() const {
- return !hasDefiningRecipe() ||
- !getDefiningRecipe()->getParent()->getEnclosingLoopRegion();
+ auto *DefR = getDefiningRecipe();
+ if (!DefR)
+ return true;
+
+ const VPBasicBlock *DefVPBB = DefR->getParent();
+ auto *Plan = DefVPBB->getPlan();
+ return DefVPBB == Plan->getPreheader() || DefVPBB == Plan->getEntry();
}
void VPValue::replaceAllUsesWith(VPValue *New) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 6f7c5c3580114d..58e57db618ddd8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2239,6 +2239,45 @@ class VPWidenPointerInductionRecipe : public VPHeaderPHIRecipe,
#endif
};
+/// Recipe to generate a scalar PHI. Used to generate code for recipes that
+/// produce scalar header phis, including VPCanonicalIVPHIRecipe and
+/// VPEVLBasedIVPHIRecipe.
+class VPScalarPHIRecipe : public VPHeaderPHIRecipe {
+ std::string Name;
+
+public:
+ VPScalarPHIRecipe(VPValue *Start, VPValue *BackedgeValue, DebugLoc DL,
+ StringRef Name)
+ : VPHeaderPHIRecipe(VPDef::VPScalarPHISC, nullptr, Start, DL),
+ Name(Name.str()) {
+ addOperand(BackedgeValue);
+ }
+
+ ~VPScalarPHIRecipe() override = default;
+
+ VPScalarPHIRecipe *clone() override {
+ llvm_unreachable("cloning not implemented yet");
+ }
+
+ VP_CLASSOF_IMPL(VPDef::VPScalarPHISC)
+
+ /// Generate the phi/select nodes.
+ void execute(VPTransformState &State) override;
+
+ /// Returns true if the recipe only uses the first lane of operand \p Op.
+ bool onlyFirstLaneUsed(const VPValue *Op) const override {
+ assert(is_contained(operands(), Op) &&
+ "Op must be an operand of the recipe");
+ return true;
+ }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
/// A recipe for handling phis that are widened in the vector loop.
/// In the VPlan native path, all incoming VPValues & VPBasicBlock pairs are
/// managed in the recipe directly.
@@ -3134,8 +3173,10 @@ class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
return D->getVPDefID() == VPDef::VPCanonicalIVPHISC;
}
- /// Generate the canonical scalar induction phi of the vector loop.
- void execute(VPTransformState &State) override;
+ void execute(VPTransformState &State) override {
+ llvm_unreachable(
+ "cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
+ }
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
@@ -3231,9 +3272,10 @@ class VPEVLBasedIVPHIRecipe : public VPHeaderPHIRecipe {
return D->getVPDefID() == VPDef::VPEVLBasedIVPHISC;
}
- /// Generate phi for handling IV based on EVL over iterations correctly.
- /// TODO: investigate if it can share the code with VPCanonicalIVPHIRecipe.
- void execute(VPTransformState &State) override;
+ void execute(VPTransformState &State) override {
+ llvm_unreachable(
+ "cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
+ }
/// Return the cost of this VPEVLBasedIVPHIRecipe.
InstructionCost computeCost(ElementCount VF,
@@ -3543,6 +3585,8 @@ class VPBasicBlock : public VPBlockBase {
return NewBlock;
}
+ bool isHeader() { return any_of(phis(), IsaPred<VPHeaderPHIRecipe>); }
+
protected:
/// Execute the recipes in the IR basic block \p BB.
void executeRecipes(VPTransformState *State, BasicBlock *BB);
@@ -3700,6 +3744,10 @@ class VPRegionBlock : public VPBlockBase {
/// Clone all blocks in the single-entry single-exit region of the block and
/// their recipes without updating the operands of the cloned recipes.
VPRegionBlock *clone() override;
+
+ /// Remove the current region from its VPlan, connecting its predecessor to
+ /// its entry and exiting block to its successor.
+ void removeRegion();
};
/// VPlan models a candidate for vectorization, encoding various decisions take
@@ -3833,10 +3881,10 @@ class VPlan {
/// whether to execute the scalar tail loop or the exit block from the loop
/// latch.
const VPBasicBlock *getMiddleBlock() const {
- return cast<VPBasicBlock>(getVectorLoopRegion()->getSingleSuccessor());
+ return cast<VPBasicBlock>(getScalarPreheader()->getSinglePredecessor());
}
VPBasicBlock *getMiddleBlock() {
- return cast<VPBasicBlock>(getVectorLoopRegion()->getSingleSuccessor());
+ return cast<VPBasicBlock>(getScalarPreheader()->getSinglePredecessor());
}
/// Return an iterator range over the VPIRBasicBlock wrapping the exit blocks
@@ -3958,9 +4006,7 @@ class VPlan {
}
/// Returns the preheader of the vector loop region.
- VPBasicBlock *getVectorPreheader() {
- return cast<VPBasicBlock>(getVectorLoopRegion()->getSinglePredecessor());
- }
+ VPBasicBlock *getVectorPreheader() { return cast<VPBasicBlock>(getEntry()); }
/// Returns the canonical induction recipe of the vector loop.
VPCanonicalIVPHIRecipe *getCanonicalIV() {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index cb42cfe8159b04..969d07b229e469 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -213,14 +213,14 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
.Case<VPActiveLaneMaskPHIRecipe, VPCanonicalIVPHIRecipe,
VPFirstOrderRecurrencePHIRecipe, VPReductionPHIRecipe,
- VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe>(
- [this](const auto *R) {
- // Handle header phi recipes, except VPWidenIntOrFpInduction
- // which needs special handling due it being possibly truncated.
- // TODO: consider inferring/caching type of siblings, e.g.,
- // backedge value, here and in cases below.
- return inferScalarType(R->getStartValue());
- })
+ VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe,
+ VPScalarPHIRecipe>([this](const auto *R) {
+ // Handle header phi recipes, except VPWidenIntOrFpInduction
+ // which needs special handling due it being possibly truncated.
+ // TODO: consider inferring/caching type of siblings, e.g.,
+ // backedge value, here and in cases below.
+ return inferScalarType(R->getStartValue());
+ })
.Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>(
[](const auto *R) { return R->getScalarType(); })
.Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b2ee31c3e240a1..dc8a4435818475 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -491,11 +491,10 @@ Value *VPInstruction::generate(VPTransformState &State) {
CondBr->setSuccessor(0, nullptr);
Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
- if (!getParent()->isExiting())
+ VPBasicBlock *Header = cast<VPBasicBlock>(getParent()->getSuccessors()[1]);
+ if (!State.CFG.VPBB2IRBB.contains(Header))
return CondBr;
- VPRegionBlock *ParentRegion = getParent()->getParent();
- VPBasicBlock *Header = ParentRegion->getEntryBasicBlock();
CondBr->setSuccessor(1, State.CFG.VPBB2IRBB[Header]);
return CondBr;
}
@@ -506,9 +505,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
Value *Cond = Builder.CreateICmpEQ(IV, TC);
// Now create the branch.
- auto *Plan = getParent()->getPlan();
- VPRegionBlock *TopRegion = Plan->getVectorLoopRegion();
- VPBasicBlock *Header = TopRegion->getEntry()->getEntryBasicBlock();
+ VPBasicBlock *Header = cast<VPBasicBlock>(getParent()->getSuccessors()[1]);
// Replace the temporary unreachable terminator with a new conditional
// branch, hooking it up to backward destination (the header) now and to the
@@ -3099,17 +3096,6 @@ InstructionCost VPInterleaveRecipe::computeCost(ElementCount VF,
VectorTy, std::nullopt, CostKind, 0);
}
-void VPCanonicalIVPHIRecipe::execute(VPTransformState &State) {
- Value *Start = getStartValue()->getLiveInIRValue();
- PHINode *Phi = PHINode::Create(Start->getType(), 2, "index");...
[truncated]
|
|
@llvm/pr-subscribers-vectorizers Author: Florian Hahn (fhahn) ChangesBuilding on top of #114305, Patch is 25.58 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/117506.diff 8 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d13770a35c108f..ec48f90ba4fcca 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2368,12 +2368,6 @@ void InnerLoopVectorizer::scalarizeInstruction(const Instruction *Instr,
// End if-block.
VPRegionBlock *Parent = RepRecipe->getParent()->getParent();
bool IfPredicateInstr = Parent ? Parent->isReplicator() : false;
- assert((Parent || all_of(RepRecipe->operands(),
- [](VPValue *Op) {
- return Op->isDefinedOutsideLoopRegions();
- })) &&
- "Expected a recipe is either within a region or all of its operands "
- "are defined outside the vectorized region.");
if (IfPredicateInstr)
PredicatedInstructions.push_back(Cloned);
}
@@ -2969,8 +2963,8 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
for (Instruction *PI : PredicatedInstructions)
sinkScalarOperands(&*PI);
- VPRegionBlock *VectorRegion = State.Plan->getVectorLoopRegion();
- VPBasicBlock *HeaderVPBB = VectorRegion->getEntryBasicBlock();
+ VPBasicBlock *HeaderVPBB = cast<VPBasicBlock>(
+ State.Plan->getVectorPreheader()->getSingleSuccessor());
BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB];
// Remove redundant induction instructions.
@@ -7742,6 +7736,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
BestVPlan.prepareToExecute(ILV.getTripCount(),
ILV.getOrCreateVectorTripCount(nullptr),
CanonicalIVStartValue, State);
+ VPlanTransforms::prepareToExecute(BestVPlan);
BestVPlan.execute(&State);
@@ -7763,7 +7758,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
LLVMLoopVectorizeFollowupVectorized});
VPBasicBlock *HeaderVPBB =
- BestVPlan.getVectorLoopRegion()->getEntryBasicBlock();
+ cast<VPBasicBlock>(BestVPlan.getVectorPreheader()->getSingleSuccessor());
Loop *L = LI->getLoopFor(State.CFG.VPBB2IRBB[HeaderVPBB]);
if (VectorizedLoopID)
L->setLoopID(*VectorizedLoopID);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 529108a5aaa97f..a3e11c7f2f5f8d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -352,8 +352,8 @@ Value *VPTransformState::get(VPValue *Def, bool NeedsScalar) {
}
BasicBlock *VPTransformState::CFGState::getPreheaderBBFor(VPRecipeBase *R) {
- VPRegionBlock *LoopRegion = R->getParent()->getEnclosingLoopRegion();
- return VPBB2IRBB[LoopRegion->getPreheaderVPBB()];
+
+ return VPBB2IRBB[cast<VPBasicBlock>(R->getParent()->getPredecessors()[0])];
}
void VPTransformState::addNewMetadata(Instruction *To,
@@ -425,6 +425,8 @@ void VPBasicBlock::connectToPredecessors(VPTransformState::CFGState &CFG) {
VPBasicBlock *PredVPBB = PredVPBlock->getExitingBasicBlock();
auto &PredVPSuccessors = PredVPBB->getHierarchicalSuccessors();
BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB];
+ if (!PredBB)
+ continue;
assert(PredBB && "Predecessor basic-block not found building successor.");
auto *PredBBTerminator = PredBB->getTerminator();
@@ -432,6 +434,8 @@ void VPBasicBlock::connectToPredecessors(VPTransformState::CFGState &CFG) {
auto *TermBr = dyn_cast<BranchInst>(PredBBTerminator);
if (isa<UnreachableInst>(PredBBTerminator)) {
+ if (PredVPSuccessors.size() == 2)
+ continue;
assert(PredVPSuccessors.size() == 1 &&
"Predecessor ending w/o branch must have single successor.");
DebugLoc DL = PredBBTerminator->getDebugLoc();
@@ -480,6 +484,21 @@ void VPBasicBlock::execute(VPTransformState *State) {
bool Replica = bool(State->Lane);
BasicBlock *NewBB = State->CFG.PrevBB; // Reuse it if possible.
+ if (isHeader()) {
+ // Create and register the new vector loop.
+ State->CurrentVectorLoop = State->LI->AllocateLoop();
+ BasicBlock *VectorPH =
+ State->CFG.VPBB2IRBB[cast<VPBasicBlock>(getPredecessors()[0])];
+ Loop *ParentLoop = State->LI->getLoopFor(VectorPH);
+
+ // Insert the new loop into the loop nest and register the new basic blocks
+ // before calling any utilities such as SCEV that require valid LoopInfo.
+ if (ParentLoop)
+ ParentLoop->addChildLoop(State->CurrentVectorLoop);
+ else
+ State->LI->addTopLevelLoop(State->CurrentVectorLoop);
+ }
+
auto IsReplicateRegion = [](VPBlockBase *BB) {
auto *R = dyn_cast_or_null<VPRegionBlock>(BB);
return R && R->isReplicator();
@@ -718,37 +737,13 @@ void VPRegionBlock::dropAllReferences(VPValue *NewValue) {
}
void VPRegionBlock::execute(VPTransformState *State) {
- ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>>
- RPOT(Entry);
-
- if (!isReplicator()) {
- // Create and register the new vector loop.
- Loop *PrevLoop = State->CurrentVectorLoop;
- State->CurrentVectorLoop = State->LI->AllocateLoop();
- BasicBlock *VectorPH = State->CFG.VPBB2IRBB[getPreheaderVPBB()];
- Loop *ParentLoop = State->LI->getLoopFor(VectorPH);
-
- // Insert the new loop into the loop nest and register the new basic blocks
- // before calling any utilities such as SCEV that require valid LoopInfo.
- if (ParentLoop)
- ParentLoop->addChildLoop(State->CurrentVectorLoop);
- else
- State->LI->addTopLevelLoop(State->CurrentVectorLoop);
-
- // Visit the VPBlocks connected to "this", starting from it.
- for (VPBlockBase *Block : RPOT) {
- LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n');
- Block->execute(State);
- }
-
- State->CurrentVectorLoop = PrevLoop;
- return;
- }
-
+ assert(isReplicator() &&
+ "Loop regions should have been lowered to plain CFG");
assert(!State->Lane && "Replicating a Region with non-null instance.");
-
- // Enter replicating mode.
assert(!State->VF.isScalable() && "VF is assumed to be non scalable.");
+
+ ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
+ Entry);
State->Lane = VPLane(0);
for (unsigned Lane = 0, VF = State->VF.getKnownMinValue(); Lane < VF;
++Lane) {
@@ -823,6 +818,26 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
}
#endif
+void VPRegionBlock::removeRegion() {
+ auto *Header = cast<VPBasicBlock>(getEntry());
+ VPBlockBase *Preheader = getSinglePredecessor();
+ auto *Exiting = cast<VPBasicBlock>(getExiting());
+
+ VPBlockBase *Middle = getSingleSuccessor();
+ VPBlockUtils::disconnectBlocks(Preheader, this);
+ VPBlockUtils::disconnectBlocks(this, Middle);
+
+ for (VPBlockBase *VPB : vp_depth_first_shallow(Entry))
+ VPB->setParent(nullptr);
+
+ VPBlockUtils::connectBlocks(Preheader, Header);
+ VPBlockUtils::connectBlocks(Exiting, Middle);
+
+ // Set LoopRegion's Entry to nullptr, as the CFG from LoopRegion shouldn't
+ // be deleted when the region is deleted.
+ Entry = nullptr;
+}
+
VPlan::~VPlan() {
if (Entry) {
VPValue DummyValue;
@@ -1032,51 +1047,55 @@ void VPlan::execute(VPTransformState *State) {
for (VPBlockBase *Block : vp_depth_first_shallow(Entry))
Block->execute(State);
- VPBasicBlock *LatchVPBB = getVectorLoopRegion()->getExitingBasicBlock();
- BasicBlock *VectorLatchBB = State->CFG.VPBB2IRBB[LatchVPBB];
-
// Fix the latch value of canonical, reduction and first-order recurrences
// phis in the vector loop.
- VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock();
- for (VPRecipeBase &R : Header->phis()) {
- // Skip phi-like recipes that generate their backedege values themselves.
- if (isa<VPWidenPHIRecipe>(&R))
+ for (VPBasicBlock *Header :
+ VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow(Entry))) {
+ if (!Header->isHeader())
continue;
+ for (VPRecipeBase &R : Header->phis()) {
+ VPBasicBlock *LatchVPBB =
+ cast<VPBasicBlock>(Header->getPredecessors()[1]);
+ BasicBlock *VectorLatchBB = State->CFG.VPBB2IRBB[LatchVPBB];
- if (isa<VPWidenPointerInductionRecipe>(&R) ||
- isa<VPWidenIntOrFpInductionRecipe>(&R)) {
- PHINode *Phi = nullptr;
- if (isa<VPWidenIntOrFpInductionRecipe>(&R)) {
- Phi = cast<PHINode>(State->get(R.getVPSingleValue()));
- } else {
- auto *WidenPhi = cast<VPWidenPointerInductionRecipe>(&R);
- assert(!WidenPhi->onlyScalarsGenerated(State->VF.isScalable()) &&
- "recipe generating only scalars should have been replaced");
- auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi));
- Phi = cast<PHINode>(GEP->getPointerOperand());
- }
-
- Phi->setIncomingBlock(1, VectorLatchBB);
+ // Skip phi-like recipes that generate their backedege values themselves.
+ if (isa<VPWidenPHIRecipe>(&R))
+ continue;
- // Move the last step to the end of the latch block. This ensures
- // consistent placement of all induction updates.
- Instruction *Inc = cast<Instruction>(Phi->getIncomingValue(1));
- Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode());
+ if (isa<VPWidenPointerInductionRecipe>(&R) ||
+ isa<VPWidenIntOrFpInductionRecipe>(&R)) {
+ PHINode *Phi = nullptr;
+ if (isa<VPWidenIntOrFpInductionRecipe>(&R)) {
+ Phi = cast<PHINode>(State->get(R.getVPSingleValue()));
+ } else {
+ auto *WidenPhi = cast<VPWidenPointerInductionRecipe>(&R);
+ assert(!WidenPhi->onlyScalarsGenerated(State->VF.isScalable()) &&
+ "recipe generating only scalars should have been replaced");
+ auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi));
+ Phi = cast<PHINode>(GEP->getPointerOperand());
+ }
+
+ Phi->setIncomingBlock(1, VectorLatchBB);
+
+ // Move the last step to the end of the latch block. This ensures
+ // consistent placement of all induction updates.
+ Instruction *Inc = cast<Instruction>(Phi->getIncomingValue(1));
+ Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode());
+
+ // Use the steps for the last part as backedge value for the induction.
+ if (auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
+ Inc->setOperand(0, State->get(IV->getLastUnrolledPartOperand()));
+ continue;
+ }
- // Use the steps for the last part as backedge value for the induction.
- if (auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
- Inc->setOperand(0, State->get(IV->getLastUnrolledPartOperand()));
- continue;
+ auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
+ bool NeedsScalar = isa<VPScalarPHIRecipe>(PhiR) ||
+ (isa<VPReductionPHIRecipe>(PhiR) &&
+ cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
+ Value *Phi = State->get(PhiR, NeedsScalar);
+ Value *Val = State->get(PhiR->getBackedgeValue(), NeedsScalar);
+ cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
}
-
- auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
- bool NeedsScalar =
- isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe>(PhiR) ||
- (isa<VPReductionPHIRecipe>(PhiR) &&
- cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
- Value *Phi = State->get(PhiR, NeedsScalar);
- Value *Val = State->get(PhiR->getBackedgeValue(), NeedsScalar);
- cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
}
State->CFG.DTU.flush();
@@ -1418,8 +1437,13 @@ void VPlanIngredient::print(raw_ostream &O) const {
#endif
bool VPValue::isDefinedOutsideLoopRegions() const {
- return !hasDefiningRecipe() ||
- !getDefiningRecipe()->getParent()->getEnclosingLoopRegion();
+ auto *DefR = getDefiningRecipe();
+ if (!DefR)
+ return true;
+
+ const VPBasicBlock *DefVPBB = DefR->getParent();
+ auto *Plan = DefVPBB->getPlan();
+ return DefVPBB == Plan->getPreheader() || DefVPBB == Plan->getEntry();
}
void VPValue::replaceAllUsesWith(VPValue *New) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 6f7c5c3580114d..58e57db618ddd8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2239,6 +2239,45 @@ class VPWidenPointerInductionRecipe : public VPHeaderPHIRecipe,
#endif
};
+/// Recipe to generate a scalar PHI. Used to generate code for recipes that
+/// produce scalar header phis, including VPCanonicalIVPHIRecipe and
+/// VPEVLBasedIVPHIRecipe.
+class VPScalarPHIRecipe : public VPHeaderPHIRecipe {
+ std::string Name;
+
+public:
+ VPScalarPHIRecipe(VPValue *Start, VPValue *BackedgeValue, DebugLoc DL,
+ StringRef Name)
+ : VPHeaderPHIRecipe(VPDef::VPScalarPHISC, nullptr, Start, DL),
+ Name(Name.str()) {
+ addOperand(BackedgeValue);
+ }
+
+ ~VPScalarPHIRecipe() override = default;
+
+ VPScalarPHIRecipe *clone() override {
+ llvm_unreachable("cloning not implemented yet");
+ }
+
+ VP_CLASSOF_IMPL(VPDef::VPScalarPHISC)
+
+ /// Generate the phi/select nodes.
+ void execute(VPTransformState &State) override;
+
+ /// Returns true if the recipe only uses the first lane of operand \p Op.
+ bool onlyFirstLaneUsed(const VPValue *Op) const override {
+ assert(is_contained(operands(), Op) &&
+ "Op must be an operand of the recipe");
+ return true;
+ }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
/// A recipe for handling phis that are widened in the vector loop.
/// In the VPlan native path, all incoming VPValues & VPBasicBlock pairs are
/// managed in the recipe directly.
@@ -3134,8 +3173,10 @@ class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
return D->getVPDefID() == VPDef::VPCanonicalIVPHISC;
}
- /// Generate the canonical scalar induction phi of the vector loop.
- void execute(VPTransformState &State) override;
+ void execute(VPTransformState &State) override {
+ llvm_unreachable(
+ "cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
+ }
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
@@ -3231,9 +3272,10 @@ class VPEVLBasedIVPHIRecipe : public VPHeaderPHIRecipe {
return D->getVPDefID() == VPDef::VPEVLBasedIVPHISC;
}
- /// Generate phi for handling IV based on EVL over iterations correctly.
- /// TODO: investigate if it can share the code with VPCanonicalIVPHIRecipe.
- void execute(VPTransformState &State) override;
+ void execute(VPTransformState &State) override {
+ llvm_unreachable(
+ "cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
+ }
/// Return the cost of this VPEVLBasedIVPHIRecipe.
InstructionCost computeCost(ElementCount VF,
@@ -3543,6 +3585,8 @@ class VPBasicBlock : public VPBlockBase {
return NewBlock;
}
+ bool isHeader() { return any_of(phis(), IsaPred<VPHeaderPHIRecipe>); }
+
protected:
/// Execute the recipes in the IR basic block \p BB.
void executeRecipes(VPTransformState *State, BasicBlock *BB);
@@ -3700,6 +3744,10 @@ class VPRegionBlock : public VPBlockBase {
/// Clone all blocks in the single-entry single-exit region of the block and
/// their recipes without updating the operands of the cloned recipes.
VPRegionBlock *clone() override;
+
+ /// Remove the current region from its VPlan, connecting its predecessor to
+ /// its entry and exiting block to its successor.
+ void removeRegion();
};
/// VPlan models a candidate for vectorization, encoding various decisions take
@@ -3833,10 +3881,10 @@ class VPlan {
/// whether to execute the scalar tail loop or the exit block from the loop
/// latch.
const VPBasicBlock *getMiddleBlock() const {
- return cast<VPBasicBlock>(getVectorLoopRegion()->getSingleSuccessor());
+ return cast<VPBasicBlock>(getScalarPreheader()->getSinglePredecessor());
}
VPBasicBlock *getMiddleBlock() {
- return cast<VPBasicBlock>(getVectorLoopRegion()->getSingleSuccessor());
+ return cast<VPBasicBlock>(getScalarPreheader()->getSinglePredecessor());
}
/// Return an iterator range over the VPIRBasicBlock wrapping the exit blocks
@@ -3958,9 +4006,7 @@ class VPlan {
}
/// Returns the preheader of the vector loop region.
- VPBasicBlock *getVectorPreheader() {
- return cast<VPBasicBlock>(getVectorLoopRegion()->getSinglePredecessor());
- }
+ VPBasicBlock *getVectorPreheader() { return cast<VPBasicBlock>(getEntry()); }
/// Returns the canonical induction recipe of the vector loop.
VPCanonicalIVPHIRecipe *getCanonicalIV() {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index cb42cfe8159b04..969d07b229e469 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -213,14 +213,14 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
.Case<VPActiveLaneMaskPHIRecipe, VPCanonicalIVPHIRecipe,
VPFirstOrderRecurrencePHIRecipe, VPReductionPHIRecipe,
- VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe>(
- [this](const auto *R) {
- // Handle header phi recipes, except VPWidenIntOrFpInduction
- // which needs special handling due it being possibly truncated.
- // TODO: consider inferring/caching type of siblings, e.g.,
- // backedge value, here and in cases below.
- return inferScalarType(R->getStartValue());
- })
+ VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe,
+ VPScalarPHIRecipe>([this](const auto *R) {
+ // Handle header phi recipes, except VPWidenIntOrFpInduction
+ // which needs special handling due it being possibly truncated.
+ // TODO: consider inferring/caching type of siblings, e.g.,
+ // backedge value, here and in cases below.
+ return inferScalarType(R->getStartValue());
+ })
.Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>(
[](const auto *R) { return R->getScalarType(); })
.Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b2ee31c3e240a1..dc8a4435818475 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -491,11 +491,10 @@ Value *VPInstruction::generate(VPTransformState &State) {
CondBr->setSuccessor(0, nullptr);
Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
- if (!getParent()->isExiting())
+ VPBasicBlock *Header = cast<VPBasicBlock>(getParent()->getSuccessors()[1]);
+ if (!State.CFG.VPBB2IRBB.contains(Header))
return CondBr;
- VPRegionBlock *ParentRegion = getParent()->getParent();
- VPBasicBlock *Header = ParentRegion->getEntryBasicBlock();
CondBr->setSuccessor(1, State.CFG.VPBB2IRBB[Header]);
return CondBr;
}
@@ -506,9 +505,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
Value *Cond = Builder.CreateICmpEQ(IV, TC);
// Now create the branch.
- auto *Plan = getParent()->getPlan();
- VPRegionBlock *TopRegion = Plan->getVectorLoopRegion();
- VPBasicBlock *Header = TopRegion->getEntry()->getEntryBasicBlock();
+ VPBasicBlock *Header = cast<VPBasicBlock>(getParent()->getSuccessors()[1]);
// Replace the temporary unreachable terminator with a new conditional
// branch, hooking it up to backward destination (the header) now and to the
@@ -3099,17 +3096,6 @@ InstructionCost VPInterleaveRecipe::computeCost(ElementCount VF,
VectorTy, std::nullopt, CostKind, 0);
}
-void VPCanonicalIVPHIRecipe::execute(VPTransformState &State) {
- Value *Start = getStartValue()->getLiveInIRValue();
- PHINode *Phi = PHINode::Create(Start->getType(), 2, "index");...
[truncated]
|
|
✅ With the latest revision this PR passed the C/C++ code formatter. |
| @@ -2969,8 +2963,8 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) { | |||
| for (Instruction *PI : PredicatedInstructions) | |||
| sinkScalarOperands(&*PI); | |||
|
|
|||
| VPRegionBlock *VectorRegion = State.Plan->getVectorLoopRegion(); | |||
| VPBasicBlock *HeaderVPBB = VectorRegion->getEntryBasicBlock(); | |||
| VPBasicBlock *HeaderVPBB = cast<VPBasicBlock>( | |||
There was a problem hiding this comment.
Suggested change
| VPBasicBlock *HeaderVPBB = cast<VPBasicBlock>( | |
| auto *HeaderVPBB = cast<VPBasicBlock>( |
Comment on lines
7760
to
7761
| VPBasicBlock *HeaderVPBB = | ||
| BestVPlan.getVectorLoopRegion()->getEntryBasicBlock(); | ||
| cast<VPBasicBlock>(BestVPlan.getVectorPreheader()->getSingleSuccessor()); |
There was a problem hiding this comment.
Suggested change
| VPBasicBlock *HeaderVPBB = | |
| BestVPlan.getVectorLoopRegion()->getEntryBasicBlock(); | |
| cast<VPBasicBlock>(BestVPlan.getVectorPreheader()->getSingleSuccessor()); | |
| auto *HeaderVPBB = | |
| cast<VPBasicBlock>(BestVPlan.getVectorPreheader()->getSingleSuccessor()); |
| @@ -425,13 +425,17 @@ void VPBasicBlock::connectToPredecessors(VPTransformState::CFGState &CFG) { | |||
| VPBasicBlock *PredVPBB = PredVPBlock->getExitingBasicBlock(); | |||
| auto &PredVPSuccessors = PredVPBB->getHierarchicalSuccessors(); | |||
| BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB]; | |||
There was a problem hiding this comment.
This kind of accessing may create nullptr entry in CFG.VPBB2IRBB. Is this ok?
There was a problem hiding this comment.
Updated to use lookup, thanks~
Comment on lines
1057
to
1058
| VPBasicBlock *LatchVPBB = | ||
| cast<VPBasicBlock>(Header->getPredecessors()[1]); |
There was a problem hiding this comment.
Suggested change
| VPBasicBlock *LatchVPBB = | |
| cast<VPBasicBlock>(Header->getPredecessors()[1]); | |
| auto *LatchVPBB = | |
| cast<VPBasicBlock>(Header->getPredecessors()[1]); |
Comment on lines
1065
to
1066
| if (isa<VPWidenPointerInductionRecipe>(&R) || | ||
| isa<VPWidenIntOrFpInductionRecipe>(&R)) { |
There was a problem hiding this comment.
Suggested change
| if (isa<VPWidenPointerInductionRecipe>(&R) || | |
| isa<VPWidenIntOrFpInductionRecipe>(&R)) { | |
| if (isa<VPWidenPointerInductionRecipe,VPWidenIntOrFpInductionRecipe>(&R)) { |
Comment on lines
1093
to
1027
| (isa<VPReductionPHIRecipe>(PhiR) && | ||
| cast<VPReductionPHIRecipe>(PhiR)->isInLoop()); |
There was a problem hiding this comment.
Use dyn_cast instead of isa/cast
There was a problem hiding this comment.
dyn_cast would need a separate assignment I think? Left as is for now, as this just moves the existing code
There was a problem hiding this comment.
It doesn't even move the existing code, merely indents it.
There was a problem hiding this comment.
Changes should be gone now, thanks
| /// produce scalar header phis, including VPCanonicalIVPHIRecipe and | ||
| /// VPEVLBasedIVPHIRecipe. | ||
| class VPScalarPHIRecipe : public VPHeaderPHIRecipe { | ||
| std::string Name; |
There was a problem hiding this comment.
Suggested change
| std::string Name; | |
| SmallString Name; |
| @@ -3543,6 +3585,8 @@ class VPBasicBlock : public VPBlockBase { | |||
| return NewBlock; | |||
| } | |||
|
|
|||
| bool isHeader() { return any_of(phis(), IsaPred<VPHeaderPHIRecipe>); } | |||
There was a problem hiding this comment.
Suggested change
| bool isHeader() { return any_of(phis(), IsaPred<VPHeaderPHIRecipe>); } | |
| bool isHeader() const { return any_of(phis(), IsaPred<VPHeaderPHIRecipe>); } |
There was a problem hiding this comment.
Marked as const now, thanks
| VPBasicBlock *getVectorPreheader() { | ||
| return cast<VPBasicBlock>(getVectorLoopRegion()->getSinglePredecessor()); | ||
| } | ||
| VPBasicBlock *getVectorPreheader() { return cast<VPBasicBlock>(getEntry()); } |
There was a problem hiding this comment.
Suggested change
| VPBasicBlock *getVectorPreheader() { return cast<VPBasicBlock>(getEntry()); } | |
| VPBasicBlock *getVectorPreheader() const { return cast<VPBasicBlock>(getEntry()); } |
fhahn
force-pushed
the
vplan-remove-region-before-cg
branch
from
7f1ef83 to
d31b3f6
Compare
fhahn
commented
Mar 18, 2025
| @@ -425,13 +425,17 @@ void VPBasicBlock::connectToPredecessors(VPTransformState::CFGState &CFG) { | |||
| VPBasicBlock *PredVPBB = PredVPBlock->getExitingBasicBlock(); | |||
| auto &PredVPSuccessors = PredVPBB->getHierarchicalSuccessors(); | |||
| BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB]; | |||
There was a problem hiding this comment.
Updated to use lookup, thanks~
Comment on lines
1057
to
1058
| VPBasicBlock *LatchVPBB = | ||
| cast<VPBasicBlock>(Header->getPredecessors()[1]); |
Comment on lines
1093
to
1027
| (isa<VPReductionPHIRecipe>(PhiR) && | ||
| cast<VPReductionPHIRecipe>(PhiR)->isInLoop()); |
There was a problem hiding this comment.
dyn_cast would need a separate assignment I think? Left as is for now, as this just moves the existing code
| @@ -3543,6 +3585,8 @@ class VPBasicBlock : public VPBlockBase { | |||
| return NewBlock; | |||
| } | |||
|
|
|||
| bool isHeader() { return any_of(phis(), IsaPred<VPHeaderPHIRecipe>); } | |||
There was a problem hiding this comment.
Marked as const now, thanks
fhahn
force-pushed
the
vplan-remove-region-before-cg
branch
from
d31b3f6 to
1c6fc7a
Compare
fhahn
added a commit
that referenced
this pull request
Mar 30, 2025
Currently should be NFC, but will be used by #117506.
llvm-sync bot
pushed a commit
to arm/arm-toolchain
that referenced
this pull request
Mar 30, 2025
…torization. Currently should be NFC, but will be used by llvm/llvm-project#117506.
SchrodingerZhu
pushed a commit
to SchrodingerZhu/llvm-project
that referenced
this pull request
Mar 31, 2025
Currently should be NFC, but will be used by llvm#117506.
fhahn
added a commit
to fhahn/llvm-project
that referenced
this pull request
Apr 29, 2025
Generalize isUniformAfterVectorization check to not rely on the region, but purely work on checking operands and opcodes. This will be needed when disolving the vector region (llvm#117506) and improves codegen slightly in some cases.
fhahn
added a commit
to fhahn/llvm-project
that referenced
this pull request
May 2, 2025
Generalize isUniformAfterVectorization check to not rely on the region, but purely work on checking operands and opcodes. This will be needed when disolving the vector region (llvm#117506) and improves codegen slightly in some cases.
fhahn
force-pushed
the
vplan-remove-region-before-cg
branch
from
1c6fc7a to
6894aa5
Compare
fhahn
force-pushed
the
vplan-remove-region-before-cg
branch
from
6894aa5 to
34b096d
Compare
IanWood1
pushed a commit
to IanWood1/llvm-project
that referenced
this pull request
May 6, 2025
…lvm#137883) Generalize isUniformAfterVectorization check to not rely on the region, but purely work on checking operands and opcodes. This will be needed when disolving the vector region (llvm#117506) and improves codegen slightly in some cases. PR: llvm#137883
IanWood1
pushed a commit
to IanWood1/llvm-project
that referenced
this pull request
May 6, 2025
…lvm#137883) Generalize isUniformAfterVectorization check to not rely on the region, but purely work on checking operands and opcodes. This will be needed when disolving the vector region (llvm#117506) and improves codegen slightly in some cases. PR: llvm#137883
IanWood1
pushed a commit
to IanWood1/llvm-project
that referenced
this pull request
May 6, 2025
…lvm#137883) Generalize isUniformAfterVectorization check to not rely on the region, but purely work on checking operands and opcodes. This will be needed when disolving the vector region (llvm#117506) and improves codegen slightly in some cases. PR: llvm#137883
llvm-sync bot
pushed a commit
to arm/arm-toolchain
that referenced
this pull request
May 6, 2025
…ization. (#137883) Generalize isUniformAfterVectorization check to not rely on the region, but purely work on checking operands and opcodes. This will be needed when disolving the vector region (llvm/llvm-project#117506) and improves codegen slightly in some cases. PR: llvm/llvm-project#137883
GeorgeARM
pushed a commit
to GeorgeARM/llvm-project
that referenced
this pull request
May 7, 2025
…lvm#137883) Generalize isUniformAfterVectorization check to not rely on the region, but purely work on checking operands and opcodes. This will be needed when disolving the vector region (llvm#117506) and improves codegen slightly in some cases. PR: llvm#137883
!fixup update more tests.
fhahn
force-pushed
the
vplan-remove-region-before-cg
branch
from
34b096d to
c274eea
Compare
ayalz
reviewed
May 11, 2025
| VPB->setParent(nullptr); | ||
|
|
||
| VPBlockUtils::connectBlocks(Preheader, Header); | ||
| VPBlockUtils::connectBlocks(Exiting, Middle); |
There was a problem hiding this comment.
Should the backedge be added too?
There was a problem hiding this comment.
Yep, moved here, thanks!
| auto *Plan = DefVPBB->getPlan(); | ||
| if (Plan->getVectorLoopRegion()) | ||
| return !DefR->getParent()->getEnclosingLoopRegion(); | ||
| return DefVPBB == Plan->getEntry(); |
There was a problem hiding this comment.
The only out-of-loop place that values can be defined in when regions are gone - is in Plan's entry block?
There was a problem hiding this comment.
All relevant hoisting should already have happened before disolving the regions; updated to only check non-live-ins if the top-level loop region still exists.
| vp_depth_first_shallow(State.Plan->getEntry()), | ||
| [&State](VPBlockBase *VPB, bool) { | ||
| auto *VPBB = dyn_cast<VPBasicBlock>(VPB); | ||
| return VPBB && VPBB->isHeader(State.VPDT) ? VPBB : nullptr; |
There was a problem hiding this comment.
At this point when IRBB's have been created for VPBB's and LI is maintained for the former, can lookup VPBB2IRBB for each VPBB and then check if LI->isLoopHeader(), before retrieving LI->getLoopFor().
| return; | ||
|
|
||
| VPRegionBlock *VectorRegion = State.Plan->getVectorLoopRegion(); | ||
| VPBasicBlock *HeaderVPBB = VectorRegion->getEntryBasicBlock(); | ||
| BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB]; | ||
|
|
||
| // Remove redundant induction instructions. |
There was a problem hiding this comment.
Could cse() be applied as a VPlan2VPlan transformation rather than to the generated IR below? Possibly even before dismantling regions, when header blocks are easier to find.
There was a problem hiding this comment.
Yep, this should also be moved to a VPlan-transform. The limitation to the loop region isn't really material for the VPlan version.
| @@ -2874,7 +2870,7 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) { | |||
| } | |||
There was a problem hiding this comment.
Could setProfileInfoAfterUnrolling() be applied to the branch recipe as part of its VPIRMetadata, rather than above to the branch instruction it generates?
There was a problem hiding this comment.
Yep, but it would also need adding metadata to the branch in the scalar loop.
There was a problem hiding this comment.
Which calls for expanding VPlan's scope...
There was a problem hiding this comment.
Yep, although it will require modifying the original scalar loop in VPlan, which probably needs more thought what kind of modifications we need. Maybe adding metadata would be sufficient?
| // Don't apply optimizations below when no vector loop remains, as they all | ||
| // require one at the moment. | ||
| VPBasicBlock *HeaderVPBB = | ||
| getHeaderForMainVectorLoop(*State.Plan, State.VPDT); |
There was a problem hiding this comment.
Note that here we could ask if LI->isHeader() instead as we'll be calling LI->getLoopFor() next, i.e., rely on the IRBB's generated from VPBB's and their mappings in VPBB2IRBB and LI.
There was a problem hiding this comment.
We could, although it wouldnt' be much simpler? I left it as is for now
| return DefR && (!DefR->getParent()->getPlan()->getVectorLoopRegion() || | ||
| DefR->getParent()->getEnclosingLoopRegion()); | ||
| } | ||
|
|
||
| bool VPValue::isDefinedOutsideLoopRegions() const { |
There was a problem hiding this comment.
Suggested change
| bool VPValue::isDefinedOutsideLoopRegions() const { | |
| bool VPValue::isDefinedOutsideLoop() const { |
whether loop is represented as a region or not?
| @@ -583,11 +583,10 @@ Value *VPInstruction::generate(VPTransformState &State) { | |||
| CondBr->setSuccessor(0, nullptr); | |||
| Builder.GetInsertBlock()->getTerminator()->eraseFromParent(); | |||
|
|
|||
| if (!getParent()->isExiting()) | |||
| VPBasicBlock *Header = cast<VPBasicBlock>(getParent()->getSuccessors()[1]); | |||
There was a problem hiding this comment.
Comment above:
"hooking it up to backward destination for exiting blocks now"
"hooking it up to backward destination (header) for latch blocks now"
?
| auto *Plan = getParent()->getPlan(); | ||
| VPRegionBlock *TopRegion = Plan->getVectorLoopRegion(); | ||
| VPBasicBlock *Header = TopRegion->getEntry()->getEntryBasicBlock(); | ||
| VPBasicBlock *Header = cast<VPBasicBlock>(getParent()->getSuccessors()[1]); | ||
|
|
||
| // Replace the temporary unreachable terminator with a new conditional | ||
| // branch, hooking it up to backward destination (the header) now and to the |
Comment on lines
442
to
443
| if (PredVPSuccessors.size() == 2) | ||
| continue; |
There was a problem hiding this comment.
Add comment explaining this newly ignored case?
There was a problem hiding this comment.
Check not needed for the latest version, removed thanks.
ayalz
reviewed
May 13, 2025
| @@ -2874,7 +2870,7 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) { | |||
| } | |||
There was a problem hiding this comment.
Which calls for expanding VPlan's scope...
| @@ -7859,6 +7855,9 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan( | |||
| BestVPlan, BestVF, | |||
| TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)); | |||
| VPlanTransforms::removeDeadRecipes(BestVPlan); | |||
|
|
|||
| VPBasicBlock *MiddleVPBB = | |||
| BestVPlan.getVectorLoopRegion() ? BestVPlan.getMiddleBlock() : nullptr; | |||
There was a problem hiding this comment.
Ah, right. Deserves a comment.
Also worth noting phase ordering aspects of disolveLoopRegions() - intentionally after optimizeForVFAndUF()?
| auto *VPBB = dyn_cast<VPBasicBlock>(VPB); | ||
| if (!VPBB) | ||
| return false; | ||
| if (auto *R = VPBB->getParent()) |
There was a problem hiding this comment.
Suggested change
| if (auto *R = VPBB->getParent()) | |
| // If VPBB is in a region R, VPBB is a loop header if R is a loop region with VPBB as its entry, i.e., free of predecessors. | |
| if (auto *R = VPBB->getParent()) |
| if (auto *R = VPBB->getParent()) | ||
| return !R->isReplicator() && VPBB->getNumPredecessors() == 0; | ||
|
|
||
| assert(!VPB->getParent() && "checking blocks in regions not implemented yet"); |
There was a problem hiding this comment.
Implemented above?
Suggested change
| assert(!VPB->getParent() && "checking blocks in regions not implemented yet"); |
There was a problem hiding this comment.
Ah yes, removed, thanks
| return !R->isReplicator() && VPBB->getNumPredecessors() == 0; | ||
|
|
||
| assert(!VPB->getParent() && "checking blocks in regions not implemented yet"); | ||
| return VPB->getPredecessors().size() == 2 && |
There was a problem hiding this comment.
Suggested change
| return VPB->getPredecessors().size() == 2 && | |
| // A header dominates its second predecessor (the latch), with the other predecessor being the preheader. | |
| return VPB->getPredecessors().size() == 2 && |
| @@ -514,6 +546,11 @@ void VPBasicBlock::execute(VPTransformState *State) { | |||
|
|
|||
| // 2. Fill the IR basic block with IR instructions. | |||
| executeRecipes(State, NewBB); | |||
|
|
|||
| // If this block is a latch, update CurrentParentLoop. | |||
There was a problem hiding this comment.
Suggested change
| // If this block is a latch, update CurrentParentLoop. | |
| // If this block is a latch, update CurrentParentLoop. The second successor of a latch is a header, with the other successor being an exit/middle. |
Wrapping in VPBlockUtils::isLatch() may be clearer?
Comment on lines
1405
to
1409
| // For non-live-ins, check if is in a region only if the top-level loop region | ||
| // still exits. | ||
| const VPBasicBlock *DefVPBB = DefR->getParent(); | ||
| auto *Plan = DefVPBB->getPlan(); | ||
| return Plan->getVectorLoopRegion() && !DefVPBB->getEnclosingLoopRegion(); |
There was a problem hiding this comment.
How/Does this differ from the existing isDefinedOutsideLoopRegions(), if it still depends on (loop) regions?
There was a problem hiding this comment.
The previousisDefinedOutsideLoopRegions would return true if there are no loop regions (because the only possible case was when it had been removed). Now we need to be more conservative, because the regions will be removed, but loops still remain.
There was a problem hiding this comment.
De Morgan'izing previous isDefinedOutsideLoopRegions():
!(DefR && (!DefR->getParent()->getPlan()->getVectorLoopRegion() ||
DefR->getParent()->getEnclosingLoopRegion()))
==
!DefR || !(!DefR->getParent()->getPlan()->getVectorLoopRegion() ||
DefR->getParent()->getEnclosingLoopRegion())
==
!DefR || (DefR->getParent()->getPlan()->getVectorLoopRegion() &&
!DefR->getParent()->getEnclosingLoopRegion())
which appears to be what the current isDefinedOutsideLoop() is doing?
This is already conservative in the sense that answering false means it may be defined inside loops - those that remain after loop regions are removed? Perhaps more accurately called mustBeDefinedOutsideLoops(), instead of negating isDefinedInsideLoopRegions() which is precisely documented to return "true if there is a vector loop region and \p VPV is defined in a loop region".
There was a problem hiding this comment.
Ah yes, messed something up on my side when trying to restore, should be done now. Still kept the new name isDefinedOutsideLoop.
I've not adjusted the name of isDefinedInsideLoopRegions yet, as it's code is now not touched in the PR. can do separately or pull in here.
| // branch, hooking it up to backward destination for exiting blocks now and | ||
| // to forward destination(s) later when they are created. | ||
| // branch, hooking it up to backward destination (header) for latch blocks | ||
| // now to forward destination(s) later when they are created. |
There was a problem hiding this comment.
Suggested change
| // now to forward destination(s) later when they are created. | |
| // now, and to forward destination(s) later when they are created. |
| // Note that CreateCondBr expects a valid BB as first argument, so we need | ||
| // to set it to nullptr later. | ||
| // branch, hooking it up to backward destination (the header) for latch | ||
| // blocks now forward destination (the exit/middle block) later when it is |
There was a problem hiding this comment.
Suggested change
| // blocks now forward destination (the exit/middle block) later when it is | |
| // blocks now, and to forward destination (the exit/middle block) later when it is |
| @@ -1124,10 +1121,6 @@ void VPInstructionWithType::print(raw_ostream &O, const Twine &Indent, | |||
|
|
|||
| void VPPhi::execute(VPTransformState &State) { | |||
| State.setDebugLocFrom(getDebugLoc()); | |||
| assert(getParent() == | |||
| getParent()->getPlan()->getVectorLoopRegion()->getEntry() && | |||
| "VPInstructions with PHI opcodes must be used for header phis only " | |||
There was a problem hiding this comment.
Ah, ok, the assert is essentially still valid, but harder to detect - if VPPhi is in a header or not.
ayalz
reviewed
May 14, 2025
|
|
||
| bool VPBlockUtils::isLatch(const VPBlockBase *VPB, | ||
| const VPDominatorTree &VPDT) { | ||
| return VPB->getNumSuccessors() == 2 && |
There was a problem hiding this comment.
Suggested change
| return VPB->getNumSuccessors() == 2 && | |
| // A latch has a header as its second successor, with its other successor leaving the loop. | |
| // A preheader OTOH has a header as its first (and only) successor. | |
| return VPB->getNumSuccessors() == 2 && |
| assert(PredBB && "Predecessor basic-block not found building successor."); | ||
| assert(CFG.VPBB2IRBB.contains(PredVPBB) && | ||
| "Predecessor basic-block not found building successor."); | ||
| BasicBlock *PredBB = CFG.VPBB2IRBB.lookup(PredVPBB); |
There was a problem hiding this comment.
can retrain
Suggested change
| BasicBlock *PredBB = CFG.VPBB2IRBB.lookup(PredVPBB); | |
| BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB]; |
?
| // branch, hooking it up to backward destination for exiting blocks now and | ||
| // to forward destination(s) later when they are created. | ||
| // branch, hooking it up to backward destination (header) for latch blocks | ||
| // now, and to forward destination(s) later when they are created. | ||
| BranchInst *CondBr = | ||
| Builder.CreateCondBr(Cond, Builder.GetInsertBlock(), nullptr); | ||
| CondBr->setSuccessor(0, nullptr); |
There was a problem hiding this comment.
Suggested change
| CondBr->setSuccessor(0, nullptr); | |
| // First successor is always forward. | |
| CondBr->setSuccessor(0, nullptr); |
| // branch, hooking it up to backward destination for exiting blocks now and | ||
| // to forward destination(s) later when they are created. | ||
| // branch, hooking it up to backward destination (header) for latch blocks | ||
| // now, and to forward destination(s) later when they are created. |
There was a problem hiding this comment.
Could this be more consistent with BranchOnCount below, as in
Suggested change
| // now, and to forward destination(s) later when they are created. | |
| // now, and to forward destination(s) later when they are created. | |
| // Second successor may be backwards - iff it is already in VPBB2IRBB. | |
| VPBasicBlock *SecondVPSucc = cast<VPBasicBlock>(getParent()->getSuccessors()[1]); | |
| BasicBlock *SecondIRSucc = State.CFG.VPBB2IRBB.lookup(SecondVPSucc); | |
| // Note that CreateCondBr expects a valid BB as first argument, but first successor is always forwards, so we reset it to nullptr. | |
| BranchInst *CondBr = | |
| Builder.CreateCondBr(Cond, Builder.GetInsertBlock(), SecondIRSucc); | |
| CondBr->setSuccessor(0, nullptr); |
There was a problem hiding this comment.
Done and moved to a helper used for both, thanks
| // branch, hooking it up to backward destination (the header) for latch | ||
| // blocks now, and to forward destination (the exit/middle block) later when | ||
| // it is created. Note that CreateCondBr expects a valid BB as first | ||
| // argument, so we need to set it to nullptr later. |
There was a problem hiding this comment.
Suggested change
| // argument, so we need to set it to nullptr later. | |
| // argument, but first successor is forwards, so we reset it to nullptr. |
There was a problem hiding this comment.
Should be adjusted in the unified version, thanks
Comment on lines
1405
to
1409
| // For non-live-ins, check if is in a region only if the top-level loop region | ||
| // still exits. | ||
| const VPBasicBlock *DefVPBB = DefR->getParent(); | ||
| auto *Plan = DefVPBB->getPlan(); | ||
| return Plan->getVectorLoopRegion() && !DefVPBB->getEnclosingLoopRegion(); |
There was a problem hiding this comment.
De Morgan'izing previous isDefinedOutsideLoopRegions():
!(DefR && (!DefR->getParent()->getPlan()->getVectorLoopRegion() ||
DefR->getParent()->getEnclosingLoopRegion()))
==
!DefR || !(!DefR->getParent()->getPlan()->getVectorLoopRegion() ||
DefR->getParent()->getEnclosingLoopRegion())
==
!DefR || (DefR->getParent()->getPlan()->getVectorLoopRegion() &&
!DefR->getParent()->getEnclosingLoopRegion())
which appears to be what the current isDefinedOutsideLoop() is doing?
This is already conservative in the sense that answering false means it may be defined inside loops - those that remain after loop regions are removed? Perhaps more accurately called mustBeDefinedOutsideLoops(), instead of negating isDefinedInsideLoopRegions() which is precisely documented to return "true if there is a vector loop region and \p VPV is defined in a loop region".
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Building on top of #114305,
replace VPRegionBlocks with explicit CFG before executing.
This will enable further simplifications of phi handling during execution
and transformations that do not have to preserve the canonical IV required
by loop regions. This for example could include replacing the canonical
IV with an EVL based phi while completely removing the original canonical
IV.