diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 32ddd82d9adbd..d09bf3872f04f 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -9036,41 +9036,45 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, ArrayRef VL, "Expected only single user of the gather node."); // TODO: currently checking only for Scalars in the tree entry, need to count // reused elements too for better cost estimation. - Instruction &UserInst = - getLastInstructionInBundle(TE->UserTreeIndices.front().UserTE); - BasicBlock *ParentBB = nullptr; + const EdgeInfo &TEUseEI = TE->UserTreeIndices.front(); + const Instruction *TEInsertPt = &getLastInstructionInBundle(TEUseEI.UserTE); + const BasicBlock *TEInsertBlock = nullptr; // Main node of PHI entries keeps the correct order of operands/incoming // blocks. - if (auto *PHI = - dyn_cast(TE->UserTreeIndices.front().UserTE->getMainOp())) { - ParentBB = PHI->getIncomingBlock(TE->UserTreeIndices.front().EdgeIdx); + if (auto *PHI = dyn_cast(TEUseEI.UserTE->getMainOp())) { + TEInsertBlock = PHI->getIncomingBlock(TEUseEI.EdgeIdx); } else { - ParentBB = UserInst.getParent(); + TEInsertBlock = TEInsertPt->getParent(); } - auto *NodeUI = DT->getNode(ParentBB); + auto *NodeUI = DT->getNode(TEInsertBlock); assert(NodeUI && "Should only process reachable instructions"); SmallPtrSet GatheredScalars(VL.begin(), VL.end()); - auto CheckOrdering = [&](Instruction *LastEI) { - // Check if the user node of the TE comes after user node of EntryPtr, - // otherwise EntryPtr depends on TE. - // Gather nodes usually are not scheduled and inserted before their first - // user node. So, instead of checking dependency between the gather nodes - // themselves, we check the dependency between their user nodes. - // If one user node comes before the second one, we cannot use the second - // gather node as the source vector for the first gather node, because in - // the list of instructions it will be emitted later. - auto *EntryParent = LastEI->getParent(); - auto *NodeEUI = DT->getNode(EntryParent); + auto CheckOrdering = [&](const Instruction *InsertPt) { + // Argument InsertPt is an instruction where vector code for some other + // tree entry (one that shares one or more scalars with TE) is going to be + // generated. This lambda returns true if insertion point of vector code + // for the TE dominates that point (otherwise dependency is the other way + // around). The other node is not limited to be of a gather kind. Gather + // nodes are not scheduled and their vector code is inserted before their + // first user. If user is PHI, that is supposed to be at the end of a + // predecessor block. Otherwise it is the last instruction among scalars of + // the user node. So, instead of checking dependency between instructions + // themselves, we check dependency between their insertion points for vector + // code (since each scalar instruction ends up as a lane of a vector + // instruction). + const BasicBlock *InsertBlock = InsertPt->getParent(); + auto *NodeEUI = DT->getNode(InsertBlock); if (!NodeEUI) return false; assert((NodeUI == NodeEUI) == (NodeUI->getDFSNumIn() == NodeEUI->getDFSNumIn()) && "Different nodes should have different DFS numbers"); // Check the order of the gather nodes users. - if (UserInst.getParent() != EntryParent && + if (TEInsertPt->getParent() != InsertBlock && (DT->dominates(NodeUI, NodeEUI) || !DT->dominates(NodeEUI, NodeUI))) return false; - if (UserInst.getParent() == EntryParent && UserInst.comesBefore(LastEI)) + if (TEInsertPt->getParent() == InsertBlock && + TEInsertPt->comesBefore(InsertPt)) return false; return true; }; @@ -9095,49 +9099,35 @@ BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, ArrayRef VL, [&](Value *V) { return GatheredScalars.contains(V); }) && "Must contain at least single gathered value."); assert(TEPtr->UserTreeIndices.size() == 1 && - "Expected only single user of the gather node."); - PHINode *EntryPHI = - dyn_cast(TEPtr->UserTreeIndices.front().UserTE->getMainOp()); - Instruction *EntryUserInst = - EntryPHI ? nullptr - : &getLastInstructionInBundle( - TEPtr->UserTreeIndices.front().UserTE); - if (&UserInst == EntryUserInst) { - assert(!EntryPHI && "Unexpected phi node entry."); - // If 2 gathers are operands of the same entry, compare operands - // indices, use the earlier one as the base. - if (TE->UserTreeIndices.front().UserTE == - TEPtr->UserTreeIndices.front().UserTE && - TE->UserTreeIndices.front().EdgeIdx < - TEPtr->UserTreeIndices.front().EdgeIdx) + "Expected only single user of a gather node."); + const EdgeInfo &UseEI = TEPtr->UserTreeIndices.front(); + + PHINode *UserPHI = dyn_cast(UseEI.UserTE->getMainOp()); + const Instruction *InsertPt = + UserPHI ? UserPHI->getIncomingBlock(UseEI.EdgeIdx)->getTerminator() + : &getLastInstructionInBundle(UseEI.UserTE); + if (!UserPHI && TEInsertPt == InsertPt) { + // If 2 gathers are operands of the same non-PHI entry, + // compare operands indices, use the earlier one as the base. + if (TEUseEI.UserTE == UseEI.UserTE && TEUseEI.EdgeIdx < UseEI.EdgeIdx) continue; // If the user instruction is used for some reason in different // vectorized nodes - make it depend on index. - if (TE->UserTreeIndices.front().UserTE != - TEPtr->UserTreeIndices.front().UserTE && - TE->Idx < TEPtr->Idx) + if (TEUseEI.UserTE != UseEI.UserTE && TE->Idx < TEPtr->Idx) continue; } - // Check if the user node of the TE comes after user node of EntryPtr, - // otherwise EntryPtr depends on TE. - auto *EntryI = - EntryPHI - ? EntryPHI - ->getIncomingBlock(TEPtr->UserTreeIndices.front().EdgeIdx) - ->getTerminator() - : EntryUserInst; - if ((ParentBB != EntryI->getParent() || - TE->UserTreeIndices.front().EdgeIdx < - TEPtr->UserTreeIndices.front().EdgeIdx || - TE->UserTreeIndices.front().UserTE != - TEPtr->UserTreeIndices.front().UserTE) && - !CheckOrdering(EntryI)) + + // Check if the user node of the TE comes after user node of TEPtr, + // otherwise TEPtr depends on TE. + if ((TEInsertBlock != InsertPt->getParent() || + TEUseEI.EdgeIdx < UseEI.EdgeIdx || TEUseEI.UserTE != UseEI.UserTE) && + !CheckOrdering(InsertPt)) continue; VToTEs.insert(TEPtr); } if (const TreeEntry *VTE = getTreeEntry(V)) { - Instruction &EntryUserInst = getLastInstructionInBundle(VTE); - if (&EntryUserInst == &UserInst || !CheckOrdering(&EntryUserInst)) + Instruction &LastBundleInst = getLastInstructionInBundle(VTE); + if (&LastBundleInst == TEInsertPt || !CheckOrdering(&LastBundleInst)) continue; VToTEs.insert(VTE); }