Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
b357af4
[VPlan] Add VPPhiAccessors to provide interface for phi recipes (NFC)
fhahn Feb 27, 2025
9113022
Merge remote-tracking branch 'origin/main' into vplan-phi-accessors
fhahn Mar 13, 2025
c61233f
!fixup address latest comments, thanks
fhahn Mar 13, 2025
a73a137
Merge remote-tracking branch 'origin/main' into vplan-phi-accessors
fhahn Apr 23, 2025
90b2d4d
Merge remote-tracking branch 'origin/main' into vplan-phi-accessors
fhahn Apr 26, 2025
f768b97
!fixup address latest comments, thanks
fhahn Apr 26, 2025
d6d6bb5
Merge remote-tracking branch 'origin/main' into vplan-phi-accessors
fhahn Apr 28, 2025
8dfd569
Merge remote-tracking branch 'origin/main' into vplan-phi-accessors
fhahn May 1, 2025
595b057
!fixup address latest comments, thanks
fhahn May 1, 2025
a2b4ebb
Merge remote-tracking branch 'origin/main' into vplan-phi-accessors
fhahn May 3, 2025
46780ab
!fixup address comments, make getAsRecipe pure virtual.
fhahn May 3, 2025
5fee077
Use virtual
fhahn May 2, 2025
450901e
Merge remote-tracking branch 'origin/main' into vplan-verify-def-use-phi
fhahn May 4, 2025
996af30
[VPlan] Add VPPhiAccessors::incoming_values_and_blocks (NFC).
fhahn May 2, 2025
904998b
Merge remote-tracking branch 'origin/main' into vplan-phi-iterators
fhahn May 26, 2025
bcd9064
!fixup after merge
fhahn May 26, 2025
7291b0e
Merge remote-tracking branch 'origin/main' into vplan-phi-iterators
fhahn May 26, 2025
2c318e6
Merge remote-tracking branch 'origin/main' into vplan-phi-iterators
fhahn Aug 1, 2025
260e727
!fixup use in VPlanPredicator.
fhahn Aug 1, 2025
fc8cf8b
Merge remote-tracking branch 'origin/main' into vplan-phi-iterators
fhahn Aug 6, 2025
4b705f8
!fixup use map_range with index_range
fhahn Aug 6, 2025
14a6b44
Merge branch 'main' into vplan-phi-iterators
fhahn Aug 11, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2687,11 +2687,8 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) {
PHINode *NewPhi = cast<PHINode>(State.get(VPPhi));
// Make sure the builder has a valid insert point.
Builder.SetInsertPoint(NewPhi);
for (unsigned Idx = 0; Idx < VPPhi->getNumIncoming(); ++Idx) {
VPValue *Inc = VPPhi->getIncomingValue(Idx);
const VPBasicBlock *VPBB = VPPhi->getIncomingBlock(Idx);
for (const auto &[Inc, VPBB] : VPPhi->incoming_values_and_blocks())
NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]);
}
}
}
}
Expand Down
30 changes: 30 additions & 0 deletions llvm/lib/Transforms/Vectorize/VPlan.h
Original file line number Diff line number Diff line change
Expand Up @@ -1230,6 +1230,31 @@ class VPPhiAccessors {
return getAsRecipe()->getNumOperands();
}

/// Returns an interator range over the incoming values.
VPUser::const_operand_range incoming_values() const {
return make_range(getAsRecipe()->op_begin(),
getAsRecipe()->op_begin() + getNumIncoming());
}

using const_incoming_blocks_range = iterator_range<mapped_iterator<
detail::index_iterator, std::function<const VPBasicBlock *(size_t)>>>;

/// Returns an iterator range over the incoming blocks.
const_incoming_blocks_range incoming_blocks() const {
std::function<const VPBasicBlock *(size_t)> GetBlock = [this](size_t Idx) {
return getIncomingBlock(Idx);
};
return map_range(index_range(0, getNumIncoming()), GetBlock);
}

/// Returns an iterator range over pairs of incoming values and corresponding
/// incoming blocks.
detail::zippy<llvm::detail::zip_first, VPUser::const_operand_range,
const_incoming_blocks_range>
incoming_values_and_blocks() const {
return zip_equal(incoming_values(), incoming_blocks());
}

/// Removes the incoming value for \p IncomingBlock, which must be a
/// predecessor.
void removeIncomingValueFor(VPBlockBase *IncomingBlock) const;
Expand Down Expand Up @@ -2301,6 +2326,11 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
VPSlotTracker &SlotTracker) const override;
#endif

/// Returns the number of incoming values, also number of incoming blocks.
/// Note that at the moment, VPWidenPointerInductionRecipe only has a single
/// incoming value, its start value.
unsigned getNumIncoming() const override { return 2; }

/// Returns the recurrence kind of the reduction.
RecurKind getRecurrenceKind() const { return Kind; }

Expand Down
11 changes: 4 additions & 7 deletions llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -238,14 +238,11 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
// optimizations will clean it up.

SmallVector<VPValue *, 2> OperandsWithMask;
unsigned NumIncoming = PhiR->getNumIncoming();
for (unsigned In = 0; In < NumIncoming; In++) {
const VPBasicBlock *Pred = PhiR->getIncomingBlock(In);
OperandsWithMask.push_back(PhiR->getIncomingValue(In));
VPValue *EdgeMask = getEdgeMask(Pred, VPBB);
for (const auto &[InVPV, InVPBB] : PhiR->incoming_values_and_blocks()) {
OperandsWithMask.push_back(InVPV);
VPValue *EdgeMask = getEdgeMask(InVPBB, VPBB);
if (!EdgeMask) {
assert(In == 0 && "Both null and non-null edge masks found");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess there's not an easy way to preserve this assert?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nope unfortunately not :(

assert(all_equal(PhiR->operands()) &&
assert(all_equal(PhiR->incoming_values()) &&
"Distinct incoming values with one having a full mask");
break;
}
Expand Down
12 changes: 6 additions & 6 deletions llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1444,12 +1444,12 @@ void VPIRPhi::print(raw_ostream &O, const Twine &Indent,

if (getNumOperands() != 0) {
O << " (extra operand" << (getNumOperands() > 1 ? "s" : "") << ": ";
interleaveComma(
enumerate(operands()), O, [this, &O, &SlotTracker](auto Op) {
Op.value()->printAsOperand(O, SlotTracker);
O << " from ";
getParent()->getPredecessors()[Op.index()]->printAsOperand(O);
});
interleaveComma(incoming_values_and_blocks(), O,
[&O, &SlotTracker](auto Op) {
std::get<0>(Op)->printAsOperand(O, SlotTracker);
O << " from ";
std::get<1>(Op)->printAsOperand(O);
});
O << ")";
}
}
Expand Down
8 changes: 3 additions & 5 deletions llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -250,17 +250,15 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
for (const VPUser *U : V->users()) {
auto *UI = cast<VPRecipeBase>(U);
if (auto *Phi = dyn_cast<VPPhiAccessors>(UI)) {
for (unsigned Idx = 0; Idx != Phi->getNumIncoming(); ++Idx) {
VPValue *IncomingVPV = Phi->getIncomingValue(Idx);
for (const auto &[IncomingVPV, IncomingVPBB] :
Phi->incoming_values_and_blocks()) {
Comment on lines +253 to +254
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not majorly important but if you wanted to preserve the error message you could do const auto &[Idx, IncomingVPV, IncomingVPBB] : enumerate(Phi->incoming_values(), Phi->incoming_blocks())

if (IncomingVPV != V)
continue;

const VPBasicBlock *IncomingVPBB = Phi->getIncomingBlock(Idx);
if (VPDT.dominates(VPBB, IncomingVPBB))
continue;

errs() << "Incoming def at index " << Idx
<< " does not dominate incoming block!\n";
errs() << "Incoming def does not dominate incoming block!\n";
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
VPSlotTracker Tracker(VPBB->getPlan());
IncomingVPV->getDefiningRecipe()->print(errs(), " ", Tracker);
Expand Down
7 changes: 3 additions & 4 deletions llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -170,15 +170,14 @@ TEST_F(VPVerifierTest, VPPhiIncomingValueDoesntDominateIncomingBlock) {
EXPECT_FALSE(verifyVPlanIsValid(Plan));
#if GTEST_HAS_STREAM_REDIRECTION
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
EXPECT_STREQ("Incoming def at index 0 does not dominate incoming block!\n"
EXPECT_STREQ("Incoming def does not dominate incoming block!\n"
" EMIT vp<%2> = add ir<0>\n"
" does not dominate preheader for\n"
" EMIT-SCALAR vp<%1> = phi [ vp<%2>, preheader ]",
::testing::internal::GetCapturedStderr().c_str());
#else
EXPECT_STREQ("Incoming def at index 0 does not dominate incoming block!\n", ::
testing::internal::GetCapturedStderr()
.c_str());
EXPECT_STREQ("Incoming def does not dominate incoming block!\n",
::testing::internal::GetCapturedStderr().c_str());
#endif
#endif
}
Expand Down