Skip to content

[GlobalIsel] combine insert vector element #89363

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 27, 2024
Merged

[GlobalIsel] combine insert vector element #89363

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -853,6 +853,9 @@ class CombinerHelper {
bool matchExtractVectorElementWithDifferentIndices(const MachineOperand &MO,
BuildFnTy &MatchInfo);

/// Combine insert vector element OOB.
bool matchInsertVectorElementOOB(MachineInstr &MI, BuildFnTy &MatchInfo);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please clarify what OOB means, it is not obvious.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

out of bound


private:
/// Checks for legality of an indexed variant of \p LdSt.
bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;
Expand Down
26 changes: 26 additions & 0 deletions llvm/include/llvm/CodeGen/GlobalISel/Utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -559,5 +559,31 @@ void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI);
/// having only floating-point operands.
bool isPreISelGenericFloatingPointOpcode(unsigned Opc);

/// Returns true if \p Reg can create undef or poison from non-undef &
/// non-poison operands. \p ConsiderFlagsAndMetadata controls whether poison
/// producing flags and metadata on the instruction are considered. This can be
/// used to see if the instruction could still introduce undef or poison even
/// without poison generating flags and metadata which might be on the
/// instruction.
bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
bool ConsiderFlagsAndMetadata = true);

/// Returns true if \p Reg can create poison from non-poison operands.
bool canCreatePoison(Register Reg, const MachineRegisterInfo &MRI,
bool ConsiderFlagsAndMetadata = true);

/// Returns true if \p Reg cannot be poison and undef.
bool isGuaranteedNotToBeUndefOrPoison(Register Reg,
const MachineRegisterInfo &MRI,
unsigned Depth = 0);

/// Returns true if \p Reg cannot be poison, but may be undef.
bool isGuaranteedNotToBePoison(Register Reg, const MachineRegisterInfo &MRI,
unsigned Depth = 0);

/// Returns true if \p Reg cannot be undef, but may be poison.
bool isGuaranteedNotToBeUndef(Register Reg, const MachineRegisterInfo &MRI,
unsigned Depth = 0);

} // End namespace llvm.
#endif
33 changes: 31 additions & 2 deletions llvm/include/llvm/Target/GlobalISel/Combine.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1525,11 +1525,39 @@ def combine_shuffle_concat : GICombineRule<
[{ return Helper.matchCombineShuffleConcat(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyCombineShuffleConcat(*${root}, ${matchinfo}); }])>;

// match_extract_of_element must be the first!
def insert_vector_element_idx_undef : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_IMPLICIT_DEF $idx),
(G_INSERT_VECTOR_ELT $root, $src, $elt, $idx)),
(apply (G_IMPLICIT_DEF $root))>;

def insert_vector_element_elt_undef : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_IMPLICIT_DEF $elt),
(G_INSERT_VECTOR_ELT $root, $src, $elt, $idx),
[{ return isGuaranteedNotToBePoison(${src}.getReg(), MRI); }]),
(apply (GIReplaceReg $root, $src))>;

def insert_vector_element_extract_vector_element : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_EXTRACT_VECTOR_ELT $elt, $src, $idx),
(G_INSERT_VECTOR_ELT $root, $src, $elt, $idx)),
(apply (GIReplaceReg $root, $src))>;

def insert_vector_elt_oob : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
(match (wip_match_opcode G_INSERT_VECTOR_ELT):$root,
[{ return Helper.matchInsertVectorElementOOB(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;

// match_extract_of_element and insert_vector_elt_oob must be the first!
def vector_ops_combines: GICombineGroup<[
match_extract_of_element_undef_vector,
match_extract_of_element_undef_index,
insert_vector_element_idx_undef,
insert_vector_element_elt_undef,
match_extract_of_element,
insert_vector_elt_oob,
extract_vector_element_not_const,
extract_vector_element_different_indices,
extract_vector_element_build_vector2,
Expand All @@ -1553,7 +1581,8 @@ extract_vector_element_build_vector_trunc5,
extract_vector_element_build_vector_trunc6,
extract_vector_element_build_vector_trunc7,
extract_vector_element_build_vector_trunc8,
extract_vector_element_freeze
extract_vector_element_freeze,
insert_vector_element_extract_vector_element
]>;

// FIXME: These should use the custom predicate feature once it lands.
Expand Down
25 changes: 24 additions & 1 deletion llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ bool CombinerHelper::matchExtractVectorElement(MachineInstr &MI,

// Fold extractVectorElement(Vector, TOOLARGE) -> undef
if (IndexC && VectorTy.isFixedVector() &&
IndexC->getZExtValue() >= VectorTy.getNumElements() &&
IndexC->uge(VectorTy.getNumElements()) &&
isLegalOrBeforeLegalizer({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) {
// For fixed-length vectors, it's invalid to extract out-of-range elements.
MatchInfo = [=](MachineIRBuilder &B) { B.buildUndef(Dst); };
Expand Down Expand Up @@ -324,3 +324,26 @@ bool CombinerHelper::matchExtractVectorElementWithBuildVectorTrunc(

return true;
}

bool CombinerHelper::matchInsertVectorElementOOB(MachineInstr &MI,
BuildFnTy &MatchInfo) {
GInsertVectorElement *Insert = cast<GInsertVectorElement>(&MI);

Register Dst = Insert->getReg(0);
LLT DstTy = MRI.getType(Dst);
Register Index = Insert->getIndexReg();

if (!DstTy.isFixedVector())
return false;

std::optional<ValueAndVReg> MaybeIndex =
getIConstantVRegValWithLookThrough(Index, MRI);

if (MaybeIndex && MaybeIndex->Value.uge(DstTy.getNumElements()) &&
isLegalOrBeforeLegalizer({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) {
MatchInfo = [=](MachineIRBuilder &B) { B.buildUndef(Dst); };
return true;
}

return false;
}
83 changes: 83 additions & 0 deletions llvm/lib/CodeGen/GlobalISel/Utils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
Expand All @@ -28,6 +29,7 @@
#include "llvm/CodeGen/StackProtector.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/Constants.h"
Expand Down Expand Up @@ -1709,3 +1711,84 @@ bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) {
return false;
}
}

namespace {
enum class UndefPoisonKind {
PoisonOnly = (1 << 0),
UndefOnly = (1 << 1),
UndefOrPoison = PoisonOnly | UndefOnly,
};
}

[[maybe_unused]] static bool includesPoison(UndefPoisonKind Kind) {
return (unsigned(Kind) & unsigned(UndefPoisonKind::PoisonOnly)) != 0;
}

[[maybe_unused]] static bool includesUndef(UndefPoisonKind Kind) {
return (unsigned(Kind) & unsigned(UndefPoisonKind::UndefOnly)) != 0;
}

static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
bool ConsiderFlagsAndMetadata,
UndefPoisonKind Kind) {
MachineInstr *RegDef = MRI.getVRegDef(Reg);

switch (RegDef->getOpcode()) {
case TargetOpcode::G_FREEZE:
return false;
default:
return true;
}
}

static bool isGuaranteedNotToBeUndefOrPoison(Register Reg,
const MachineRegisterInfo &MRI,
unsigned Depth,
UndefPoisonKind Kind) {
if (Depth >= MaxAnalysisRecursionDepth)
return false;

MachineInstr *RegDef = MRI.getVRegDef(Reg);

switch (RegDef->getOpcode()) {
case TargetOpcode::G_FREEZE:
return true;
case TargetOpcode::G_IMPLICIT_DEF:
return !includesUndef(Kind);
default:
return false;
}
}

bool llvm::canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
bool ConsiderFlagsAndMetadata) {
return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,
UndefPoisonKind::UndefOrPoison);
}

bool canCreatePoison(Register Reg, const MachineRegisterInfo &MRI,
bool ConsiderFlagsAndMetadata = true) {
return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,
UndefPoisonKind::PoisonOnly);
}

bool llvm::isGuaranteedNotToBeUndefOrPoison(Register Reg,
const MachineRegisterInfo &MRI,
unsigned Depth) {
return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
UndefPoisonKind::UndefOrPoison);
}

bool llvm::isGuaranteedNotToBePoison(Register Reg,
const MachineRegisterInfo &MRI,
unsigned Depth) {
return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
UndefPoisonKind::PoisonOnly);
}

bool llvm::isGuaranteedNotToBeUndef(Register Reg,
const MachineRegisterInfo &MRI,
unsigned Depth) {
return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
UndefPoisonKind::UndefOnly);
}
110 changes: 110 additions & 0 deletions llvm/test/CodeGen/AArch64/GlobalISel/combine-insert-vec-elt.mir
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,3 +201,113 @@ body: |
RET_ReallyLR

...
---
name: test_idx_undef
body: |
bb.1:
liveins: $x0
; CHECK-LABEL: name: test_idx_undef
; CHECK: liveins: $x0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: RET_ReallyLR
%3:_(s8) = G_CONSTANT i8 127
%2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8)
%4:_(s8) = G_CONSTANT i8 -128
%5:_(s64) = G_IMPLICIT_DEF
%0:_(p0) = COPY $x0
%1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %2, %4(s8), %5(s64)
G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>))
RET_ReallyLR

...
---
name: test_elt_undef
body: |
bb.1:
liveins: $x0
; CHECK-LABEL: name: test_elt_undef
; CHECK: liveins: $x0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 127
; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s8>) = G_BUILD_VECTOR [[C]](s8), [[C]](s8), [[C]](s8), [[DEF]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8)
; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<32 x s8>), [[COPY]](p0) :: (store (<32 x s8>))
; CHECK-NEXT: RET_ReallyLR
%3:_(s8) = G_CONSTANT i8 127
%2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8)
%4:_(s8) = G_IMPLICIT_DEF
%5:_(s64) = G_CONSTANT i64 3
%0:_(p0) = COPY $x0
%1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %2, %4(s8), %5(s64)
G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>))
RET_ReallyLR

...
---
name: test_elt_undef_with_freeze
body: |
bb.1:
liveins: $x0
; CHECK-LABEL: name: test_elt_undef_with_freeze
; CHECK: liveins: $x0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 127
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s8>) = G_BUILD_VECTOR [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8)
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(<32 x s8>) = G_FREEZE [[BUILD_VECTOR]]
; CHECK-NEXT: G_STORE [[FREEZE]](<32 x s8>), [[COPY]](p0) :: (store (<32 x s8>))
; CHECK-NEXT: RET_ReallyLR
%3:_(s8) = G_CONSTANT i8 127
%2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8)
%4:_(s8) = G_IMPLICIT_DEF
%5:_(s64) = G_CONSTANT i64 3
%0:_(p0) = COPY $x0
%9:_(<32 x s8>) = G_FREEZE %2
%1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %9, %4(s8), %5(s64)
G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>))
RET_ReallyLR

...
---
name: test_insert_extract
body: |
bb.1:
liveins: $x0
; CHECK-LABEL: name: test_insert_extract
; CHECK: liveins: $x0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 127
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s8>) = G_BUILD_VECTOR [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8)
; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<32 x s8>), [[COPY]](p0) :: (store (<32 x s8>))
; CHECK-NEXT: RET_ReallyLR
%3:_(s8) = G_CONSTANT i8 127
%2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8)
%5:_(s64) = G_CONSTANT i64 3
%4:_(s8) = G_EXTRACT_VECTOR_ELT %2, %5
%0:_(p0) = COPY $x0
%1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %2, %4(s8), %5(s64)
G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>))
RET_ReallyLR

...
---
name: test_idx_oob
body: |
bb.1:
liveins: $x0
; CHECK-LABEL: name: test_idx_oob
; CHECK: liveins: $x0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: RET_ReallyLR
%3:_(s8) = G_CONSTANT i8 127
%2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8)
%4:_(s8) = G_CONSTANT i8 -128
%5:_(s64) = G_CONSTANT i64 1024
%0:_(p0) = COPY $x0
%1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %2, %4(s8), %5(s64)
G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>))
RET_ReallyLR

...
Loading