Skip to content

Commit 4c48b3c

Browse files
dc03-workdc03-work
authored
[GISel][CombinerHelper] Push freeze through non-poison-producing operands (#90618)
This combine matches the existing fold in InstCombine, i.e. InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating. It tries to push freeze through an operand if the operand has only one maybe-poison operand and all other operands are guaranteed non-poison, and if the operation itself cannot generate poison (eg. add with nsw can generate poison, even with non-poison operands). This is beneficial because it can potentially enable other optimizations to occur that would otherwise be blocked because of the freeze.
1 parent 781b135 commit 4c48b3c

File tree

9 files changed

+139
-28
lines changed

9 files changed

+139
-28
lines changed

‎llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -869,6 +869,9 @@ class CombinerHelper {
869869
/// Combine insert vector element OOB.
870870
bool matchInsertVectorElementOOB(MachineInstr &MI, BuildFnTy &MatchInfo);
871871

872+
bool matchFreezeOfSingleMaybePoisonOperand(MachineInstr &MI,
873+
BuildFnTy &MatchInfo);
874+
872875
private:
873876
/// Checks for legality of an indexed variant of \p LdSt.
874877
bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;

‎llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,17 @@ class GenericMachineInstr : public MachineInstr {
3434
static bool classof(const MachineInstr *MI) {
3535
return isPreISelGenericOpcode(MI->getOpcode());
3636
}
37+
38+
bool hasPoisonGeneratingFlags() const {
39+
return getFlags() & (NoUWrap | NoSWrap | IsExact | Disjoint | NonNeg |
40+
FmNoNans | FmNoInfs);
41+
}
42+
43+
void dropPoisonGeneratingFlags() {
44+
clearFlags(NoUWrap | NoSWrap | IsExact | Disjoint | NonNeg | FmNoNans |
45+
FmNoInfs);
46+
assert(!hasPoisonGeneratingFlags());
47+
}
3748
};
3849

3950
/// Provides common memory operand functionality.

‎llvm/include/llvm/CodeGen/MachineInstr.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,12 @@ class MachineInstr
416416
Flags &= ~((uint32_t)Flag);
417417
}
418418

419+
void clearFlags(unsigned flags) {
420+
assert(isUInt<LLVM_MI_FLAGS_BITS>(flags) &&
421+
"flags to be cleared are out of range for the Flags field");
422+
Flags &= ~flags;
423+
}
424+
419425
/// Return true if MI is in a bundle (but not the first MI in a bundle).
420426
///
421427
/// A bundle looks like this before it's finalized:

‎llvm/include/llvm/Target/GlobalISel/Combine.td

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,13 @@ def idempotent_prop : GICombineRule<
220220
(match (idempotent_prop_frags $dst, $src)),
221221
(apply (GIReplaceReg $dst, $src))>;
222222

223+
// Convert freeze(Op(Op0, NonPoisonOps...)) to Op(freeze(Op0), NonPoisonOps...)
224+
// when Op0 is not guaranteed non-poison
225+
def push_freeze_to_prevent_poison_from_propagating : GICombineRule<
226+
(defs root:$root, build_fn_matchinfo:$matchinfo),
227+
(match (G_FREEZE $dst, $src):$root,
228+
[{ return !isGuaranteedNotToBePoison(${src}.getReg(), MRI) && Helper.matchFreezeOfSingleMaybePoisonOperand(*${root}, ${matchinfo}); }]),
229+
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
223230

224231
def extending_loads : GICombineRule<
225232
(defs root:$root, extending_load_matchdata:$matchinfo),
@@ -1713,7 +1720,8 @@ def all_combines : GICombineGroup<[trivial_combines, vector_ops_combines,
17131720
sub_add_reg, select_to_minmax, redundant_binop_in_equality,
17141721
fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
17151722
combine_concat_vector, double_icmp_zero_and_or_combine, match_addos,
1716-
sext_trunc, zext_trunc, combine_shuffle_concat]>;
1723+
sext_trunc, zext_trunc, combine_shuffle_concat,
1724+
push_freeze_to_prevent_poison_from_propagating]>;
17171725

17181726
// A combine group used to for prelegalizer combiners at -O0. The combines in
17191727
// this group have been selected based on experiments to balance code size and

‎llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,70 @@ void CombinerHelper::applyCombineCopy(MachineInstr &MI) {
223223
replaceRegWith(MRI, DstReg, SrcReg);
224224
}
225225

226+
bool CombinerHelper::matchFreezeOfSingleMaybePoisonOperand(
227+
MachineInstr &MI, BuildFnTy &MatchInfo) {
228+
// Ported from InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating.
229+
Register DstOp = MI.getOperand(0).getReg();
230+
Register OrigOp = MI.getOperand(1).getReg();
231+
232+
if (!MRI.hasOneNonDBGUse(OrigOp))
233+
return false;
234+
235+
MachineInstr *OrigDef = MRI.getUniqueVRegDef(OrigOp);
236+
// Even if only a single operand of the PHI is not guaranteed non-poison,
237+
// moving freeze() backwards across a PHI can cause optimization issues for
238+
// other users of that operand.
239+
//
240+
// Moving freeze() from one of the output registers of a G_UNMERGE_VALUES to
241+
// the source register is unprofitable because it makes the freeze() more
242+
// strict than is necessary (it would affect the whole register instead of
243+
// just the subreg being frozen).
244+
if (OrigDef->isPHI() || isa<GUnmerge>(OrigDef))
245+
return false;
246+
247+
if (canCreateUndefOrPoison(OrigOp, MRI,
248+
/*ConsiderFlagsAndMetadata=*/false))
249+
return false;
250+
251+
std::optional<MachineOperand> MaybePoisonOperand;
252+
for (MachineOperand &Operand : OrigDef->uses()) {
253+
if (!Operand.isReg())
254+
return false;
255+
256+
if (isGuaranteedNotToBeUndefOrPoison(Operand.getReg(), MRI))
257+
continue;
258+
259+
if (!MaybePoisonOperand)
260+
MaybePoisonOperand = Operand;
261+
else {
262+
// We have more than one maybe-poison operand. Moving the freeze is
263+
// unsafe.
264+
return false;
265+
}
266+
}
267+
268+
cast<GenericMachineInstr>(OrigDef)->dropPoisonGeneratingFlags();
269+
270+
// Eliminate freeze if all operands are guaranteed non-poison.
271+
if (!MaybePoisonOperand) {
272+
MatchInfo = [=](MachineIRBuilder &B) { MRI.replaceRegWith(DstOp, OrigOp); };
273+
return true;
274+
}
275+
276+
Register MaybePoisonOperandReg = MaybePoisonOperand->getReg();
277+
LLT MaybePoisonOperandRegTy = MRI.getType(MaybePoisonOperandReg);
278+
279+
MatchInfo = [=](MachineIRBuilder &B) mutable {
280+
B.setInsertPt(*OrigDef->getParent(), OrigDef->getIterator());
281+
auto Freeze = B.buildFreeze(MaybePoisonOperandRegTy, MaybePoisonOperandReg);
282+
replaceRegOpWith(
283+
MRI, *OrigDef->findRegisterUseOperand(MaybePoisonOperandReg, TRI),
284+
Freeze.getReg(0));
285+
replaceRegWith(MRI, DstOp, OrigOp);
286+
};
287+
return true;
288+
}
289+
226290
bool CombinerHelper::matchCombineConcatVectors(MachineInstr &MI,
227291
SmallVector<Register> &Ops) {
228292
assert(MI.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&

‎llvm/lib/CodeGen/GlobalISel/Utils.cpp

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1745,11 +1745,20 @@ static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
17451745
UndefPoisonKind Kind) {
17461746
MachineInstr *RegDef = MRI.getVRegDef(Reg);
17471747

1748+
if (auto *GMI = dyn_cast<GenericMachineInstr>(RegDef)) {
1749+
if (ConsiderFlagsAndMetadata && includesPoison(Kind) &&
1750+
GMI->hasPoisonGeneratingFlags())
1751+
return true;
1752+
} else {
1753+
// Conservatively return true.
1754+
return true;
1755+
}
1756+
17481757
switch (RegDef->getOpcode()) {
17491758
case TargetOpcode::G_FREEZE:
17501759
return false;
17511760
default:
1752-
return true;
1761+
return !isa<GCastOp>(RegDef) && !isa<GBinOp>(RegDef);
17531762
}
17541763
}
17551764

@@ -1767,8 +1776,17 @@ static bool isGuaranteedNotToBeUndefOrPoison(Register Reg,
17671776
return true;
17681777
case TargetOpcode::G_IMPLICIT_DEF:
17691778
return !includesUndef(Kind);
1770-
default:
1771-
return false;
1779+
default: {
1780+
auto MOCheck = [&](const MachineOperand &MO) {
1781+
if (!MO.isReg())
1782+
return true;
1783+
return ::isGuaranteedNotToBeUndefOrPoison(MO.getReg(), MRI, Depth + 1,
1784+
Kind);
1785+
};
1786+
return !::canCreateUndefOrPoison(Reg, MRI,
1787+
/*ConsiderFlagsAndMetadata=*/true, Kind) &&
1788+
all_of(RegDef->uses(), MOCheck);
1789+
}
17721790
}
17731791
}
17741792

‎llvm/lib/Target/AArch64/AArch64Combine.td

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -295,5 +295,6 @@ def AArch64PostLegalizerCombiner
295295
ptr_add_immed_chain, overlapping_and,
296296
split_store_zero_128, undef_combines,
297297
select_to_minmax, or_to_bsp, combine_concat_vector,
298-
commute_constant_to_rhs]> {
298+
commute_constant_to_rhs,
299+
push_freeze_to_prevent_poison_from_propagating]> {
299300
}

‎llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,9 @@ body: |
117117
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
118118
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x2
119119
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
120-
; CHECK-NEXT: %f:_(s1) = G_TRUNC [[COPY1]](s64)
121-
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %f
122-
; CHECK-NEXT: %sel:_(s1) = G_OR %c, [[FREEZE]]
120+
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
121+
; CHECK-NEXT: %f:_(s1) = G_TRUNC [[FREEZE]](s64)
122+
; CHECK-NEXT: %sel:_(s1) = G_OR %c, %f
123123
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
124124
; CHECK-NEXT: $w0 = COPY %ext(s32)
125125
%0:_(s64) = COPY $x0
@@ -144,9 +144,9 @@ body: |
144144
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
145145
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x2
146146
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
147-
; CHECK-NEXT: %f:_(s1) = G_TRUNC [[COPY1]](s64)
148-
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %f
149-
; CHECK-NEXT: %sel:_(s1) = G_OR %c, [[FREEZE]]
147+
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
148+
; CHECK-NEXT: %f:_(s1) = G_TRUNC [[FREEZE]](s64)
149+
; CHECK-NEXT: %sel:_(s1) = G_OR %c, %f
150150
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
151151
; CHECK-NEXT: $w0 = COPY %ext(s32)
152152
%0:_(s64) = COPY $x0
@@ -172,9 +172,9 @@ body: |
172172
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
173173
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d2
174174
; CHECK-NEXT: %c:_(<2 x s1>) = G_TRUNC [[COPY]](<2 x s32>)
175-
; CHECK-NEXT: %f:_(<2 x s1>) = G_TRUNC [[COPY1]](<2 x s32>)
176-
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(<2 x s1>) = G_FREEZE %f
177-
; CHECK-NEXT: %sel:_(<2 x s1>) = G_OR %c, [[FREEZE]]
175+
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(<2 x s32>) = G_FREEZE [[COPY1]]
176+
; CHECK-NEXT: %f:_(<2 x s1>) = G_TRUNC [[FREEZE]](<2 x s32>)
177+
; CHECK-NEXT: %sel:_(<2 x s1>) = G_OR %c, %f
178178
; CHECK-NEXT: %ext:_(<2 x s32>) = G_ANYEXT %sel(<2 x s1>)
179179
; CHECK-NEXT: $d0 = COPY %ext(<2 x s32>)
180180
%0:_(<2 x s32>) = COPY $d0
@@ -201,9 +201,9 @@ body: |
201201
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
202202
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
203203
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
204-
; CHECK-NEXT: %t:_(s1) = G_TRUNC [[COPY1]](s64)
205-
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %t
206-
; CHECK-NEXT: %sel:_(s1) = G_AND %c, [[FREEZE]]
204+
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
205+
; CHECK-NEXT: %t:_(s1) = G_TRUNC [[FREEZE]](s64)
206+
; CHECK-NEXT: %sel:_(s1) = G_AND %c, %t
207207
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
208208
; CHECK-NEXT: $w0 = COPY %ext(s32)
209209
%0:_(s64) = COPY $x0
@@ -229,9 +229,9 @@ body: |
229229
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
230230
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
231231
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
232-
; CHECK-NEXT: %t:_(s1) = G_TRUNC [[COPY1]](s64)
233-
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %t
234-
; CHECK-NEXT: %sel:_(s1) = G_AND %c, [[FREEZE]]
232+
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
233+
; CHECK-NEXT: %t:_(s1) = G_TRUNC [[FREEZE]](s64)
234+
; CHECK-NEXT: %sel:_(s1) = G_AND %c, %t
235235
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
236236
; CHECK-NEXT: $w0 = COPY %ext(s32)
237237
%0:_(s64) = COPY $x0
@@ -257,11 +257,11 @@ body: |
257257
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
258258
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
259259
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
260-
; CHECK-NEXT: %t:_(s1) = G_TRUNC [[COPY1]](s64)
260+
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
261+
; CHECK-NEXT: %t:_(s1) = G_TRUNC [[FREEZE]](s64)
261262
; CHECK-NEXT: %one:_(s1) = G_CONSTANT i1 true
262263
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR %c, %one
263-
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %t
264-
; CHECK-NEXT: %sel:_(s1) = G_OR [[XOR]], [[FREEZE]]
264+
; CHECK-NEXT: %sel:_(s1) = G_OR [[XOR]], %t
265265
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
266266
; CHECK-NEXT: $w0 = COPY %ext(s32)
267267
%0:_(s64) = COPY $x0
@@ -287,11 +287,11 @@ body: |
287287
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
288288
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x2
289289
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
290-
; CHECK-NEXT: %f:_(s1) = G_TRUNC [[COPY1]](s64)
290+
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
291+
; CHECK-NEXT: %f:_(s1) = G_TRUNC [[FREEZE]](s64)
291292
; CHECK-NEXT: [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
292293
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR %c, [[C]]
293-
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %f
294-
; CHECK-NEXT: %sel:_(s1) = G_AND [[XOR]], [[FREEZE]]
294+
; CHECK-NEXT: %sel:_(s1) = G_AND [[XOR]], %f
295295
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
296296
; CHECK-NEXT: $w0 = COPY %ext(s32)
297297
%0:_(s64) = COPY $x0

‎llvm/test/CodeGen/AArch64/pr58431.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
define i32 @f(i64 %0) {
55
; CHECK-LABEL: f:
66
; CHECK: // %bb.0:
7-
; CHECK-NEXT: mov w8, #10
8-
; CHECK-NEXT: mov w9, w0
7+
; CHECK-NEXT: mov w8, #10 // =0xa
8+
; CHECK-NEXT: and x9, x0, #0xffffffff
99
; CHECK-NEXT: udiv x10, x9, x8
1010
; CHECK-NEXT: msub x0, x10, x8, x9
1111
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0

0 commit comments

Comments
 (0)