1818#include " VPlanDominatorTree.h"
1919#include " llvm/ADT/DepthFirstIterator.h"
2020#include " llvm/ADT/SmallPtrSet.h"
21+ #include " llvm/ADT/TypeSwitch.h"
2122#include " llvm/Support/CommandLine.h"
2223
2324#define DEBUG_TYPE " loop-vectorize"
@@ -35,6 +36,11 @@ class VPlanVerifier {
3536 // VPHeaderPHIRecipes.
3637 bool verifyPhiRecipes (const VPBasicBlock *VPBB);
3738
39+ // Verify that \p EVL is used correctly. The user must be either in EVL-based
40+ // recipes as a last operand or VPInstruction::Add which is incoming value
41+ // into EVL's recipe.
42+ bool verifyEVLRecipe (const VPInstruction &EVL) const ;
43+
3844 bool verifyVPBasicBlock (const VPBasicBlock *VPBB);
3945
4046 bool verifyBlock (const VPBlockBase *VPB);
@@ -114,6 +120,64 @@ bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) {
114120 return true ;
115121}
116122
123+ bool VPlanVerifier::verifyEVLRecipe (const VPInstruction &EVL) const {
124+ if (EVL.getOpcode () != VPInstruction::ExplicitVectorLength) {
125+ errs () << " verifyEVLRecipe should only be called on "
126+ " VPInstruction::ExplicitVectorLength\n "
127+ return false ;
128+ }
129+ auto VerifyEVLUse = [&](const VPRecipeBase &R,
130+ const unsigned ExpectedIdx) -> bool {
131+ SmallVector<const VPValue *> Ops (R.operands ());
132+ unsigned UseCount = count (Ops, &EVL);
133+ if (UseCount != 1 || Ops[ExpectedIdx] != &EVL) {
134+ errs () << " EVL is used as non-last operand in EVL-based recipe\n "
135+ return false ;
136+ }
137+ return true ;
138+ };
139+ for (const VPUser *U : EVL.users ()) {
140+ if (!TypeSwitch<const VPUser *, bool >(U)
141+ .Case <VPWidenStoreEVLRecipe>([&](const VPWidenStoreEVLRecipe *S) {
142+ return VerifyEVLUse (*S, 2 );
143+ })
144+ .Case <VPWidenLoadEVLRecipe>([&](const VPWidenLoadEVLRecipe *L) {
145+ return VerifyEVLUse (*L, 1 );
146+ })
147+ .Case <VPWidenEVLRecipe>([&](const VPWidenEVLRecipe *W) {
148+ return VerifyEVLUse (
149+ *W, Instruction::isUnaryOp (W->getOpcode ()) ? 1 : 2 );
150+ })
151+ .Case <VPScalarCastRecipe>(
152+ [&](const VPScalarCastRecipe *S) { return true ; })
153+ .Case <VPInstruction>([&](const VPInstruction *I) {
154+ if (I->getOpcode () != Instruction::Add) {
155+ errs ()
156+ << " EVL is used as an operand in non-VPInstruction::Add\n "
157+ return false ;
158+ }
159+ if (I->getNumUsers () != 1 ) {
160+ errs () << " EVL is used in VPInstruction:Add with multiple "
161+ " users\n "
162+ return false ;
163+ }
164+ if (!isa<VPEVLBasedIVPHIRecipe>(*I->users ().begin ())) {
165+ errs () << " Result of VPInstruction::Add with EVL operand is "
166+ " not used by VPEVLBasedIVPHIRecipe\n "
167+ return false ;
168+ }
169+ return true ;
170+ })
171+ .Default ([&](const VPUser *U) {
172+ errs () << " EVL has unexpected user\n "
173+ return false ;
174+ })) {
175+ return false ;
176+ }
177+ }
178+ return true ;
179+ }
180+
117181bool VPlanVerifier::verifyVPBasicBlock (const VPBasicBlock *VPBB) {
118182 if (!verifyPhiRecipes (VPBB))
119183 return false ;
@@ -150,6 +214,13 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
150214 }
151215 }
152216 }
217+ if (const auto *EVL = dyn_cast<VPInstruction>(&R)) {
218+ if (EVL->getOpcode () == VPInstruction::ExplicitVectorLength &&
219+ !verifyEVLRecipe (*EVL)) {
220+ errs () << " EVL VPValue is not used correctly\n "
221+ return false ;
222+ }
223+ }
153224 }
154225
155226 auto *IRBB = dyn_cast<VPIRBasicBlock>(VPBB);
0 commit comments