diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 4b8aec8e8a5dd..76e8d1166ae0c 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -853,6 +853,9 @@ class CombinerHelper { bool matchExtractVectorElementWithDifferentIndices(const MachineOperand &MO, BuildFnTy &MatchInfo); + /// Combine insert vector element OOB. + bool matchInsertVectorElementOOB(MachineInstr &MI, BuildFnTy &MatchInfo); + private: /// Checks for legality of an indexed variant of \p LdSt. bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h index c4174cee5e10c..70421a518ab72 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h @@ -559,5 +559,31 @@ void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI); /// having only floating-point operands. bool isPreISelGenericFloatingPointOpcode(unsigned Opc); +/// Returns true if \p Reg can create undef or poison from non-undef & +/// non-poison operands. \p ConsiderFlagsAndMetadata controls whether poison +/// producing flags and metadata on the instruction are considered. This can be +/// used to see if the instruction could still introduce undef or poison even +/// without poison generating flags and metadata which might be on the +/// instruction. +bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI, + bool ConsiderFlagsAndMetadata = true); + +/// Returns true if \p Reg can create poison from non-poison operands. +bool canCreatePoison(Register Reg, const MachineRegisterInfo &MRI, + bool ConsiderFlagsAndMetadata = true); + +/// Returns true if \p Reg cannot be poison and undef. +bool isGuaranteedNotToBeUndefOrPoison(Register Reg, + const MachineRegisterInfo &MRI, + unsigned Depth = 0); + +/// Returns true if \p Reg cannot be poison, but may be undef. +bool isGuaranteedNotToBePoison(Register Reg, const MachineRegisterInfo &MRI, + unsigned Depth = 0); + +/// Returns true if \p Reg cannot be undef, but may be poison. +bool isGuaranteedNotToBeUndef(Register Reg, const MachineRegisterInfo &MRI, + unsigned Depth = 0); + } // End namespace llvm. #endif diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index 31b903e63d99b..dbbb3abaa8304 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -1525,11 +1525,39 @@ def combine_shuffle_concat : GICombineRule< [{ return Helper.matchCombineShuffleConcat(*${root}, ${matchinfo}); }]), (apply [{ Helper.applyCombineShuffleConcat(*${root}, ${matchinfo}); }])>; -// match_extract_of_element must be the first! +def insert_vector_element_idx_undef : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_IMPLICIT_DEF $idx), + (G_INSERT_VECTOR_ELT $root, $src, $elt, $idx)), + (apply (G_IMPLICIT_DEF $root))>; + +def insert_vector_element_elt_undef : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_IMPLICIT_DEF $elt), + (G_INSERT_VECTOR_ELT $root, $src, $elt, $idx), + [{ return isGuaranteedNotToBePoison(${src}.getReg(), MRI); }]), + (apply (GIReplaceReg $root, $src))>; + +def insert_vector_element_extract_vector_element : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (G_EXTRACT_VECTOR_ELT $elt, $src, $idx), + (G_INSERT_VECTOR_ELT $root, $src, $elt, $idx)), + (apply (GIReplaceReg $root, $src))>; + +def insert_vector_elt_oob : GICombineRule< + (defs root:$root, build_fn_matchinfo:$matchinfo), + (match (wip_match_opcode G_INSERT_VECTOR_ELT):$root, + [{ return Helper.matchInsertVectorElementOOB(*${root}, ${matchinfo}); }]), + (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>; + +// match_extract_of_element and insert_vector_elt_oob must be the first! def vector_ops_combines: GICombineGroup<[ match_extract_of_element_undef_vector, match_extract_of_element_undef_index, +insert_vector_element_idx_undef, +insert_vector_element_elt_undef, match_extract_of_element, +insert_vector_elt_oob, extract_vector_element_not_const, extract_vector_element_different_indices, extract_vector_element_build_vector2, @@ -1553,7 +1581,8 @@ extract_vector_element_build_vector_trunc5, extract_vector_element_build_vector_trunc6, extract_vector_element_build_vector_trunc7, extract_vector_element_build_vector_trunc8, -extract_vector_element_freeze +extract_vector_element_freeze, +insert_vector_element_extract_vector_element ]>; // FIXME: These should use the custom predicate feature once it lands. diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp index 123bf21f657c3..fb33801a3a33f 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp @@ -77,7 +77,7 @@ bool CombinerHelper::matchExtractVectorElement(MachineInstr &MI, // Fold extractVectorElement(Vector, TOOLARGE) -> undef if (IndexC && VectorTy.isFixedVector() && - IndexC->getZExtValue() >= VectorTy.getNumElements() && + IndexC->uge(VectorTy.getNumElements()) && isLegalOrBeforeLegalizer({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) { // For fixed-length vectors, it's invalid to extract out-of-range elements. MatchInfo = [=](MachineIRBuilder &B) { B.buildUndef(Dst); }; @@ -324,3 +324,26 @@ bool CombinerHelper::matchExtractVectorElementWithBuildVectorTrunc( return true; } + +bool CombinerHelper::matchInsertVectorElementOOB(MachineInstr &MI, + BuildFnTy &MatchInfo) { + GInsertVectorElement *Insert = cast(&MI); + + Register Dst = Insert->getReg(0); + LLT DstTy = MRI.getType(Dst); + Register Index = Insert->getIndexReg(); + + if (!DstTy.isFixedVector()) + return false; + + std::optional MaybeIndex = + getIConstantVRegValWithLookThrough(Index, MRI); + + if (MaybeIndex && MaybeIndex->Value.uge(DstTy.getNumElements()) && + isLegalOrBeforeLegalizer({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) { + MatchInfo = [=](MachineIRBuilder &B) { B.buildUndef(Dst); }; + return true; + } + + return false; +} diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp index ae43e9ccf6112..4e3781cb4e9d5 100644 --- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp +++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp @@ -12,6 +12,7 @@ #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/APInt.h" +#include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/CodeGenCommonISel.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" @@ -28,6 +29,7 @@ #include "llvm/CodeGen/StackProtector.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" +#include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/IR/Constants.h" @@ -1709,3 +1711,84 @@ bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) { return false; } } + +namespace { +enum class UndefPoisonKind { + PoisonOnly = (1 << 0), + UndefOnly = (1 << 1), + UndefOrPoison = PoisonOnly | UndefOnly, +}; +} + +[[maybe_unused]] static bool includesPoison(UndefPoisonKind Kind) { + return (unsigned(Kind) & unsigned(UndefPoisonKind::PoisonOnly)) != 0; +} + +[[maybe_unused]] static bool includesUndef(UndefPoisonKind Kind) { + return (unsigned(Kind) & unsigned(UndefPoisonKind::UndefOnly)) != 0; +} + +static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI, + bool ConsiderFlagsAndMetadata, + UndefPoisonKind Kind) { + MachineInstr *RegDef = MRI.getVRegDef(Reg); + + switch (RegDef->getOpcode()) { + case TargetOpcode::G_FREEZE: + return false; + default: + return true; + } +} + +static bool isGuaranteedNotToBeUndefOrPoison(Register Reg, + const MachineRegisterInfo &MRI, + unsigned Depth, + UndefPoisonKind Kind) { + if (Depth >= MaxAnalysisRecursionDepth) + return false; + + MachineInstr *RegDef = MRI.getVRegDef(Reg); + + switch (RegDef->getOpcode()) { + case TargetOpcode::G_FREEZE: + return true; + case TargetOpcode::G_IMPLICIT_DEF: + return !includesUndef(Kind); + default: + return false; + } +} + +bool llvm::canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI, + bool ConsiderFlagsAndMetadata) { + return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata, + UndefPoisonKind::UndefOrPoison); +} + +bool canCreatePoison(Register Reg, const MachineRegisterInfo &MRI, + bool ConsiderFlagsAndMetadata = true) { + return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata, + UndefPoisonKind::PoisonOnly); +} + +bool llvm::isGuaranteedNotToBeUndefOrPoison(Register Reg, + const MachineRegisterInfo &MRI, + unsigned Depth) { + return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth, + UndefPoisonKind::UndefOrPoison); +} + +bool llvm::isGuaranteedNotToBePoison(Register Reg, + const MachineRegisterInfo &MRI, + unsigned Depth) { + return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth, + UndefPoisonKind::PoisonOnly); +} + +bool llvm::isGuaranteedNotToBeUndef(Register Reg, + const MachineRegisterInfo &MRI, + unsigned Depth) { + return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth, + UndefPoisonKind::UndefOnly); +} diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-insert-vec-elt.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-insert-vec-elt.mir index 06fb2ce161c20..0c67a867580cc 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-insert-vec-elt.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-insert-vec-elt.mir @@ -201,3 +201,113 @@ body: | RET_ReallyLR ... +--- +name: test_idx_undef +body: | + bb.1: + liveins: $x0 + ; CHECK-LABEL: name: test_idx_undef + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: RET_ReallyLR + %3:_(s8) = G_CONSTANT i8 127 + %2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8) + %4:_(s8) = G_CONSTANT i8 -128 + %5:_(s64) = G_IMPLICIT_DEF + %0:_(p0) = COPY $x0 + %1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %2, %4(s8), %5(s64) + G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>)) + RET_ReallyLR + +... +--- +name: test_elt_undef +body: | + bb.1: + liveins: $x0 + ; CHECK-LABEL: name: test_elt_undef + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 127 + ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s8>) = G_BUILD_VECTOR [[C]](s8), [[C]](s8), [[C]](s8), [[DEF]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8) + ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<32 x s8>), [[COPY]](p0) :: (store (<32 x s8>)) + ; CHECK-NEXT: RET_ReallyLR + %3:_(s8) = G_CONSTANT i8 127 + %2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8) + %4:_(s8) = G_IMPLICIT_DEF + %5:_(s64) = G_CONSTANT i64 3 + %0:_(p0) = COPY $x0 + %1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %2, %4(s8), %5(s64) + G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>)) + RET_ReallyLR + +... +--- +name: test_elt_undef_with_freeze +body: | + bb.1: + liveins: $x0 + ; CHECK-LABEL: name: test_elt_undef_with_freeze + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 127 + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s8>) = G_BUILD_VECTOR [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8) + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(<32 x s8>) = G_FREEZE [[BUILD_VECTOR]] + ; CHECK-NEXT: G_STORE [[FREEZE]](<32 x s8>), [[COPY]](p0) :: (store (<32 x s8>)) + ; CHECK-NEXT: RET_ReallyLR + %3:_(s8) = G_CONSTANT i8 127 + %2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8) + %4:_(s8) = G_IMPLICIT_DEF + %5:_(s64) = G_CONSTANT i64 3 + %0:_(p0) = COPY $x0 + %9:_(<32 x s8>) = G_FREEZE %2 + %1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %9, %4(s8), %5(s64) + G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>)) + RET_ReallyLR + +... +--- +name: test_insert_extract +body: | + bb.1: + liveins: $x0 + ; CHECK-LABEL: name: test_insert_extract + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 127 + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s8>) = G_BUILD_VECTOR [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8) + ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<32 x s8>), [[COPY]](p0) :: (store (<32 x s8>)) + ; CHECK-NEXT: RET_ReallyLR + %3:_(s8) = G_CONSTANT i8 127 + %2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8) + %5:_(s64) = G_CONSTANT i64 3 + %4:_(s8) = G_EXTRACT_VECTOR_ELT %2, %5 + %0:_(p0) = COPY $x0 + %1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %2, %4(s8), %5(s64) + G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>)) + RET_ReallyLR + +... +--- +name: test_idx_oob +body: | + bb.1: + liveins: $x0 + ; CHECK-LABEL: name: test_idx_oob + ; CHECK: liveins: $x0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: RET_ReallyLR + %3:_(s8) = G_CONSTANT i8 127 + %2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8) + %4:_(s8) = G_CONSTANT i8 -128 + %5:_(s64) = G_CONSTANT i64 1024 + %0:_(p0) = COPY $x0 + %1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %2, %4(s8), %5(s64) + G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>)) + RET_ReallyLR + +...