[InstCombine] Handle ceil division idiom #100977

antoniofrighetto · 2024-07-29T06:27:12Z

The expression add (udiv (sub A, Bias), B), Bias can be folded to udiv (add A, B - 1), B) when the sum between A and B is known not to overflow, and Bias = A != 0.

Fixes: #95652.

Proof: https://alive2.llvm.org/ce/z/hiWHQA.

llvmbot · 2024-07-29T06:27:50Z

@llvm/pr-subscribers-llvm-transforms

Author: Antonio Frighetto (antoniofrighetto)

Changes

The expression add (udiv (sub A, Bias), B), Bias can be folded to udiv (add A, B - 1), B) when the sum between A and B is known not to overflow, and Bias = A != 0.

Fixes: #95652.

Proof: https://alive2.llvm.org/ce/z/hiWHQA.

Full diff: https://github.com/llvm/llvm-project/pull/100977.diff

2 Files Affected:

(modified) llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp (+71-24)
(added) llvm/test/Transforms/InstCombine/fold-ceil-div-idiom.ll (+253)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 3bd086230cbec..aded338982fcf 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1250,6 +1250,75 @@ static Instruction *foldToUnsignedSaturatedAdd(BinaryOperator &I) {
   return nullptr;
 }
 
+static Value *foldCeilIdioms(BinaryOperator &I, InstCombinerImpl &IC) {
+  assert(I.getOpcode() == Instruction::Add && "Expecting add instruction.");
+  Value *A, *B;
+  ICmpInst::Predicate Pred;
+  auto &ICB = IC.Builder;
+
+  // Fold the log2 ceil idiom:
+  // zext (ctpop(A) >u/!= 1) + (ctlz (A, true) ^ (BW - 1))
+  //      -> BW - ctlz (A - 1, false)
+  const APInt *XorC;
+  if (match(&I,
+            m_c_Add(
+                m_ZExt(m_ICmp(Pred, m_Intrinsic<Intrinsic::ctpop>(m_Value(A)),
+                              m_One())),
+                m_OneUse(m_ZExtOrSelf(m_OneUse(m_Xor(
+                    m_OneUse(m_TruncOrSelf(m_OneUse(
+                        m_Intrinsic<Intrinsic::ctlz>(m_Deferred(A), m_One())))),
+                    m_APInt(XorC))))))) &&
+      (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_NE) &&
+      *XorC == A->getType()->getScalarSizeInBits() - 1) {
+    Value *Sub = ICB.CreateAdd(A, Constant::getAllOnesValue(A->getType()));
+    Value *Ctlz = ICB.CreateIntrinsic(Intrinsic::ctlz, {A->getType()},
+                                      {Sub, ICB.getFalse()});
+    Value *Ret = ICB.CreateSub(
+        ConstantInt::get(A->getType(), A->getType()->getScalarSizeInBits()),
+        Ctlz, "", /*HasNUW*/ true, /*HasNSW*/ true);
+    return ICB.CreateZExtOrTrunc(Ret, I.getType());
+  }
+
+  // Fold the ceil division idiom:
+  // add (udiv (sub A, Bias), B), Bias
+  //      -> udiv (add A, B - 1), B)
+  // with Bias = A != 0; A + B not to overflow
+  auto MatchDivision = [&IC](Instruction *Div, Value *&DivOp0, Value *&DivOp1) {
+    if (match(Div, m_UDiv(m_Value(DivOp0), m_Value(DivOp1))))
+      return true;
+
+    Value *N;
+    const APInt *C;
+    if (match(Div, m_LShr(m_Value(DivOp0), m_Value(N))) &&
+        match(N,
+              m_OneUse(m_Sub(m_APInt(C), m_Intrinsic<Intrinsic::ctlz>(
+                                             m_Specific(DivOp1), m_Zero())))) &&
+        (*C == Div->getType()->getScalarSizeInBits() - 1) &&
+        IC.isKnownToBeAPowerOfTwo(DivOp1, true, 0, Div))
+      return true;
+
+    return false;
+  };
+
+  Instruction *Div;
+  Value *Bias, *Sub;
+  if (match(&I, m_c_Add(m_Instruction(Div), m_Value(Bias))) &&
+      MatchDivision(Div, Sub, B) &&
+      match(Sub, m_Sub(m_Value(A), m_Value(Bias))) &&
+      match(Bias, m_ZExt(m_ICmp(Pred, m_Specific(A), m_ZeroInt()))) &&
+      Pred == ICmpInst::ICMP_NE && Bias->hasNUses(2)) {
+    WithCache<const Value *> LHSCache(A), RHSCache(B);
+    auto OR = IC.computeOverflowForUnsignedAdd(LHSCache, RHSCache, &I);
+    if (OR == OverflowResult::NeverOverflows) {
+      auto *BMinusOne =
+          ICB.CreateAdd(B, Constant::getAllOnesValue(I.getType()));
+      return ICB.CreateUDiv(ICB.CreateAdd(A, BMinusOne), B);
+    }
+  }
+
+  return nullptr;
+}
+
 // Transform:
 //  (add A, (shl (neg B), Y))
 //      -> (sub A, (shl B, Y))
@@ -1785,30 +1854,8 @@ Instruction *InstCombinerImpl::visitAdd(BinaryOperator &I) {
         I, Builder.CreateIntrinsic(Intrinsic::ctpop, {I.getType()},
                                    {Builder.CreateOr(A, B)}));
 
-  // Fold the log2_ceil idiom:
-  // zext(ctpop(A) >u/!= 1) + (ctlz(A, true) ^ (BW - 1))
-  // -->
-  // BW - ctlz(A - 1, false)
-  const APInt *XorC;
-  ICmpInst::Predicate Pred;
-  if (match(&I,
-            m_c_Add(
-                m_ZExt(m_ICmp(Pred, m_Intrinsic<Intrinsic::ctpop>(m_Value(A)),
-                              m_One())),
-                m_OneUse(m_ZExtOrSelf(m_OneUse(m_Xor(
-                    m_OneUse(m_TruncOrSelf(m_OneUse(
-                        m_Intrinsic<Intrinsic::ctlz>(m_Deferred(A), m_One())))),
-                    m_APInt(XorC))))))) &&
-      (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_NE) &&
-      *XorC == A->getType()->getScalarSizeInBits() - 1) {
-    Value *Sub = Builder.CreateAdd(A, Constant::getAllOnesValue(A->getType()));
-    Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {A->getType()},
-                                          {Sub, Builder.getFalse()});
-    Value *Ret = Builder.CreateSub(
-        ConstantInt::get(A->getType(), A->getType()->getScalarSizeInBits()),
-        Ctlz, "", /*HasNUW*/ true, /*HasNSW*/ true);
-    return replaceInstUsesWith(I, Builder.CreateZExtOrTrunc(Ret, I.getType()));
-  }
+  if (Value *V = foldCeilIdioms(I, *this))
+    return replaceInstUsesWith(I, V);
 
   if (Instruction *Res = foldSquareSumInt(I))
     return Res;
diff --git a/llvm/test/Transforms/InstCombine/fold-ceil-div-idiom.ll b/llvm/test/Transforms/InstCombine/fold-ceil-div-idiom.ll
new file mode 100644
index 0000000000000..b9cc0fa6ce050
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fold-ceil-div-idiom.ll
@@ -0,0 +1,253 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define i8 @ceil_div_idiom(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @ceil_div_idiom(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[WO:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[X]], i8 [[Y]])
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[WO]], 1
+; CHECK-NEXT:    [[OV_NOT:%.*]] = xor i1 [[OV]], true
+; CHECK-NEXT:    call void @llvm.assume(i1 [[OV_NOT]])
+; CHECK-NEXT:    [[NONZERO:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    [[BIAS:%.*]] = zext i1 [[NONZERO]] to i8
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[X]], [[BIAS]]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 [[SUB]], [[Y]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[DIV]], [[BIAS]]
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %wo = call {i8, i1} @llvm.uadd.with.overflow(i8 %x, i8 %y)
+  %ov = extractvalue {i8, i1} %wo, 1
+  %ov.not = xor i1 %ov, true
+  call void @llvm.assume(i1 %ov.not)
+
+  %nonzero = icmp ne i8 %x, 0
+  %bias = zext i1 %nonzero to i8
+  %sub = sub i8 %x, %bias
+  %div = udiv i8 %sub, %y
+  %add = add i8 %div, %bias
+  ret i8 %add
+}
+
+define i8 @ceil_div_idiom_2(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @ceil_div_idiom_2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[OV_NOT:%.*]] = add nuw i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[OV_NOT]] to i1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[TRUNC]])
+; CHECK-NEXT:    [[NONZERO:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    [[BIAS:%.*]] = zext i1 [[NONZERO]] to i8
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[X]], [[BIAS]]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 [[SUB]], [[Y]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[DIV]], [[BIAS]]
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %ov.not = add nuw i8 %x, %y
+  %trunc = trunc i8 %ov.not to i1
+  call void @llvm.assume(i1 %trunc)
+
+  %nonzero = icmp ne i8 %x, 0
+  %bias = zext i1 %nonzero to i8
+  %sub = sub i8 %x, %bias
+  %div = udiv i8 %sub, %y
+  %add = add i8 %div, %bias
+  ret i8 %add
+}
+
+define i8 @ceil_div_idiom_with_lshr(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @ceil_div_idiom_with_lshr(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[WO:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[X]], i8 [[Y]])
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[WO]], 1
+; CHECK-NEXT:    [[OV_NOT:%.*]] = xor i1 [[OV]], true
+; CHECK-NEXT:    call void @llvm.assume(i1 [[OV_NOT]])
+; CHECK-NEXT:    [[CTPOPULATION:%.*]] = call range(i8 0, 9) i8 @llvm.ctpop.i8(i8 [[Y]])
+; CHECK-NEXT:    [[IS_POW_2:%.*]] = icmp eq i8 [[CTPOPULATION]], 1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[IS_POW_2]])
+; CHECK-NEXT:    [[NONZERO:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    [[BIAS:%.*]] = zext i1 [[NONZERO]] to i8
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[X]], [[BIAS]]
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call range(i8 0, 9) i8 @llvm.ctlz.i8(i8 [[Y]], i1 true)
+; CHECK-NEXT:    [[N:%.*]] = xor i8 [[CTLZ]], 7
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i8 [[SUB]], [[N]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[DIV]], [[BIAS]]
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %wo = call {i8, i1} @llvm.uadd.with.overflow(i8 %x, i8 %y)
+  %ov = extractvalue {i8, i1} %wo, 1
+  %ov.not = xor i1 %ov, true
+  call void @llvm.assume(i1 %ov.not)
+
+  %ctpopulation = call i8 @llvm.ctpop.i8(i8 %y)
+  %is_pow_2 = icmp eq i8 %ctpopulation, 1
+  call void @llvm.assume(i1 %is_pow_2)
+
+  %nonzero = icmp ne i8 %x, 0
+  %bias = zext i1 %nonzero to i8
+  %sub = sub i8 %x, %bias
+  %ctlz = tail call i8 @llvm.ctlz.i8(i8 %y, i1 true)
+  %n = sub i8 7, %ctlz
+  %div = lshr i8 %sub, %n
+  %add = add i8 %div, %bias
+  ret i8 %add
+}
+
+define i8 @ceil_div_idiom_add_may_overflow(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @ceil_div_idiom_add_may_overflow(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NONZERO:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    [[BIAS:%.*]] = zext i1 [[NONZERO]] to i8
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[X]], [[BIAS]]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 [[SUB]], [[Y]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[DIV]], [[BIAS]]
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %nonzero = icmp ne i8 %x, 0
+  %bias = zext i1 %nonzero to i8
+  %sub = sub i8 %x, %bias
+  %div = udiv i8 %sub, %y
+  %add = add i8 %div, %bias
+  ret i8 %add
+}
+
+define i8 @ceil_div_idiom_multiuse_bias(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @ceil_div_idiom_multiuse_bias(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[WO:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[X]], i8 [[Y]])
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[WO]], 1
+; CHECK-NEXT:    [[OV_NOT:%.*]] = xor i1 [[OV]], true
+; CHECK-NEXT:    call void @llvm.assume(i1 [[OV_NOT]])
+; CHECK-NEXT:    [[NONZERO:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    [[BIAS:%.*]] = zext i1 [[NONZERO]] to i8
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[X]], [[BIAS]]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 [[SUB]], [[Y]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[DIV]], [[BIAS]]
+; CHECK-NEXT:    call void @use(i8 [[BIAS]])
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %wo = call {i8, i1} @llvm.uadd.with.overflow(i8 %x, i8 %y)
+  %ov = extractvalue {i8, i1} %wo, 1
+  %ov.not = xor i1 %ov, true
+  call void @llvm.assume(i1 %ov.not)
+
+  %nonzero = icmp ne i8 %x, 0
+  %bias = zext i1 %nonzero to i8
+  %sub = sub i8 %x, %bias
+  %div = udiv i8 %sub, %y
+  %add = add i8 %div, %bias
+  call void @use(i8 %bias)
+  ret i8 %add
+}
+
+define i8 @ceil_div_idiom_with_lshr_not_power_2(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @ceil_div_idiom_with_lshr_not_power_2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[WO:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[X]], i8 [[Y]])
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[WO]], 1
+; CHECK-NEXT:    [[OV_NOT:%.*]] = xor i1 [[OV]], true
+; CHECK-NEXT:    call void @llvm.assume(i1 [[OV_NOT]])
+; CHECK-NEXT:    [[NONZERO:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    [[BIAS:%.*]] = zext i1 [[NONZERO]] to i8
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[X]], [[BIAS]]
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call range(i8 0, 9) i8 @llvm.ctlz.i8(i8 [[Y]], i1 true)
+; CHECK-NEXT:    [[N:%.*]] = xor i8 [[CTLZ]], 7
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i8 [[SUB]], [[N]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[DIV]], [[BIAS]]
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %wo = call {i8, i1} @llvm.uadd.with.overflow(i8 %x, i8 %y)
+  %ov = extractvalue {i8, i1} %wo, 1
+  %ov.not = xor i1 %ov, true
+  call void @llvm.assume(i1 %ov.not)
+
+  %nonzero = icmp ne i8 %x, 0
+  %bias = zext i1 %nonzero to i8
+  %sub = sub i8 %x, %bias
+  %ctlz = tail call i8 @llvm.ctlz.i8(i8 %y, i1 true)
+  %n = sub i8 7, %ctlz
+  %div = lshr i8 %sub, %n
+  %add = add i8 %div, %bias
+  ret i8 %add
+}
+
+define i8 @ceil_div_idiom_with_lshr_wrong_bw(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @ceil_div_idiom_with_lshr_wrong_bw(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[WO:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[X]], i8 [[Y]])
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[WO]], 1
+; CHECK-NEXT:    [[OV_NOT:%.*]] = xor i1 [[OV]], true
+; CHECK-NEXT:    call void @llvm.assume(i1 [[OV_NOT]])
+; CHECK-NEXT:    [[CTPOPULATION:%.*]] = call range(i8 0, 9) i8 @llvm.ctpop.i8(i8 [[Y]])
+; CHECK-NEXT:    [[IS_POW_2:%.*]] = icmp eq i8 [[CTPOPULATION]], 1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[IS_POW_2]])
+; CHECK-NEXT:    [[NONZERO:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    [[BIAS:%.*]] = zext i1 [[NONZERO]] to i8
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[X]], [[BIAS]]
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call range(i8 0, 9) i8 @llvm.ctlz.i8(i8 [[Y]], i1 true)
+; CHECK-NEXT:    [[N:%.*]] = sub nuw nsw i8 8, [[CTLZ]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i8 [[SUB]], [[N]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[DIV]], [[BIAS]]
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %wo = call {i8, i1} @llvm.uadd.with.overflow(i8 %x, i8 %y)
+  %ov = extractvalue {i8, i1} %wo, 1
+  %ov.not = xor i1 %ov, true
+  call void @llvm.assume(i1 %ov.not)
+
+  %ctpopulation = call i8 @llvm.ctpop.i8(i8 %y)
+  %is_pow_2 = icmp eq i8 %ctpopulation, 1
+  call void @llvm.assume(i1 %is_pow_2)
+
+  %nonzero = icmp ne i8 %x, 0
+  %bias = zext i1 %nonzero to i8
+  %sub = sub i8 %x, %bias
+  %ctlz = tail call i8 @llvm.ctlz.i8(i8 %y, i1 true)
+  %n = sub i8 8, %ctlz
+  %div = lshr i8 %sub, %n
+  %add = add i8 %div, %bias
+  ret i8 %add
+}
+
+define i8 @ceil_div_idiom_with_lshr_multiuse_n(i8 %x, i8 %y) {
+; CHECK-LABEL: define i8 @ceil_div_idiom_with_lshr_multiuse_n(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[WO:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[X]], i8 [[Y]])
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[WO]], 1
+; CHECK-NEXT:    [[OV_NOT:%.*]] = xor i1 [[OV]], true
+; CHECK-NEXT:    call void @llvm.assume(i1 [[OV_NOT]])
+; CHECK-NEXT:    [[CTPOPULATION:%.*]] = call range(i8 0, 9) i8 @llvm.ctpop.i8(i8 [[Y]])
+; CHECK-NEXT:    [[IS_POW_2:%.*]] = icmp eq i8 [[CTPOPULATION]], 1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[IS_POW_2]])
+; CHECK-NEXT:    [[NONZERO:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    [[BIAS:%.*]] = zext i1 [[NONZERO]] to i8
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[X]], [[BIAS]]
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call range(i8 0, 9) i8 @llvm.ctlz.i8(i8 [[Y]], i1 true)
+; CHECK-NEXT:    [[N:%.*]] = sub nuw nsw i8 8, [[CTLZ]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i8 [[SUB]], [[N]]
+; CHECK-NEXT:    call void @use(i8 [[N]])
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[DIV]], [[BIAS]]
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %wo = call {i8, i1} @llvm.uadd.with.overflow(i8 %x, i8 %y)
+  %ov = extractvalue {i8, i1} %wo, 1
+  %ov.not = xor i1 %ov, true
+  call void @llvm.assume(i1 %ov.not)
+
+  %ctpopulation = call i8 @llvm.ctpop.i8(i8 %y)
+  %is_pow_2 = icmp eq i8 %ctpopulation, 1
+  call void @llvm.assume(i1 %is_pow_2)
+
+  %nonzero = icmp ne i8 %x, 0
+  %bias = zext i1 %nonzero to i8
+  %sub = sub i8 %x, %bias
+  %ctlz = tail call i8 @llvm.ctlz.i8(i8 %y, i1 true)
+  %n = sub i8 8, %ctlz
+  %div = lshr i8 %sub, %n
+  call void @use(i8 %n)
+  %add = add i8 %div, %bias
+  ret i8 %add
+}
+
+declare { i8, i1 } @llvm.uadd.with.overflow.i8(i8, i8)
+declare i8 @llvm.ctpop.i8(i8)
+declare void @llvm.assume(i1)
+declare void @use(i8)

antoniofrighetto · 2024-07-29T06:33:05Z

Unless I'm missing something, ValueTracking seems to be lacking support of considering the possible overflow coming from *.with.overflow intrinsics, as computeOverflow currently returns MayOverflow when is guaranteed not to overflow. I'll be taking a look.

dtcxzyw · 2024-07-29T06:34:53Z

llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp

+  // Fold the log2 ceil idiom:
+  // zext (ctpop(A) >u/!= 1) + (ctlz (A, true) ^ (BW - 1))
+  //      -> BW - ctlz (A - 1, false)
+  const APInt *XorC;


Why did you merge the log2 ceil part into this function? I don't think they share common code.

They do not, but they do both involve the ceil part, thus I thought it was nice to batch them together under a single foldCeilIdioms.

dtcxzyw · 2024-07-29T06:39:30Z

Unless I'm missing something, ValueTracking seems to be lacking support of considering the possible overflow coming from *.with.overflow intrinsics, as computeOverflow currently returns MayOverflow when is guaranteed not to overflow. I'll be taking a look.

Yeah, ValueTracking doesn't support this assumption. It only exists in some rust applications. See also rust-lang/hashbrown#509.

llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp

goldsteinn · 2024-07-29T12:57:07Z

llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp

+        match(N, m_OneUse(m_Sub(m_APInt(C), m_Intrinsic<Intrinsic::ctlz>(
+                                                m_Value(DivOp1), m_Zero())))) &&
+        (*C == Div->getType()->getScalarSizeInBits() - 1) &&
+        IC.isKnownToBeAPowerOfTwo(DivOp1, true, 0, Div))


OrZero needs to be false here. (Also comment the constant).

Technically, I believe it should be allowed, but changed this to false.

goldsteinn · 2024-07-29T12:58:04Z

llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp

+    Value *N;
+    const APInt *C;
+    if (match(Div, m_LShr(m_Value(DivOp0), m_Value(N))) &&
+        match(N, m_OneUse(m_Sub(m_APInt(C), m_Intrinsic<Intrinsic::ctlz>(


Why does the sub need to be oneuse?

I just thought it could make sense to restrain the shifted amount as one-use only, although now I realize that N could be used for other log2 calculations, so maybe better to drop it.

llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp

PR Link: llvm/llvm-project#100977

antoniofrighetto · 2024-07-30T15:22:44Z

@dtcxzyw I think we should rebase to #84016 in order to see the actual impact over the opt-benchmark.

dtcxzyw · 2024-07-30T15:46:25Z

@nikic Does this pattern exist in real-world code? Can you provide an example without llvm.assume?

nikic · 2024-08-01T10:28:02Z

@dtcxzyw The motivation for this were the regressions we saw from #95556. I think most divideCeil() uses have a constant power of two RHS though, so they will produce a lshr pattern, not the udiv one handled here.

But it's possible that we don't actually know that the addition won't overflow in the hot cases. The key one is probably this in DataLayout:

  TypeSize getTypeStoreSize(Type *Ty) const {
    TypeSize BaseSize = getTypeSizeInBits(Ty);
    return {divideCeil(BaseSize.getKnownMinValue(), 8), BaseSize.isScalable()};
  }

This works on uint64_t and nothing tells us that it will not overflow :(

antoniofrighetto · 2024-08-27T13:58:06Z

The key one is probably this in DataLayout:
  TypeSize getTypeStoreSize(Type *Ty) const {
    TypeSize BaseSize = getTypeSizeInBits(Ty);
    return {divideCeil(BaseSize.getKnownMinValue(), 8), BaseSize.isScalable()};
  }
This works on uint64_t and nothing tells us that it will not overflow :(

Would it make sense by any chance to change divideCeil to somehow take into account when we know it will not overflow? It seems there are a handful of such cases.

nikic · 2024-08-27T14:05:56Z

The key one is probably this in DataLayout:
  TypeSize getTypeStoreSize(Type *Ty) const {
    TypeSize BaseSize = getTypeSizeInBits(Ty);
    return {divideCeil(BaseSize.getKnownMinValue(), 8), BaseSize.isScalable()};
  }
This works on uint64_t and nothing tells us that it will not overflow :(
Would it make sense by any chance to change divideCeil to somehow take into account when we know it will not overflow? It seems there are a handful of such cases.

I think a good way to solve this one would be to invert the relationship between getTypeStoreSize and getTypeStoreSizeInBits and use alignToPowerOf2 + plain divide.

dtcxzyw · 2025-01-05T13:15:49Z

#84016 has landed.

antoniofrighetto · 2025-01-06T12:31:13Z

#84016 has landed.

I think we are still missing to preserve the no overflow guarantee. By the time we visit the last addition, both add nuw i8 %x, %y and the assume(no_overflow) are optimized out (the latter is just constant folded to assume(true)). Shouldn't we emit an assume in the form of x+y < uint_max true instead?

The expression `add (udiv (sub A, Bias), B), Bias` can be folded to `udiv (add A, B - 1), B)` when the sum between `A` and `B` is known not to overflow, and `Bias = A != 0`. Proof: https://alive2.llvm.org/ce/z/hiWHQA.

antoniofrighetto · 2025-01-06T14:55:44Z

#84016 has landed.

I think we are still missing to preserve the no overflow guarantee. By the time we visit the last addition, both add nuw i8 %x, %y and the assume(no_overflow) are optimized out (the latter is just constant folded to assume(true)). Shouldn't we emit an assume in the form of x+y < uint_max true instead?

Arguably, this should just happen because WO has no uses (and provided that I'm not missing anything within the AssumptionCache or other ways to query the no overflow guarantee).

dtcxzyw · 2025-01-09T12:00:05Z

provided that I'm not missing anything within the AssumptionCache or other ways to query the no overflow guarantee

Currently we don't infer no overflow from overflow checking idioms (e.g., extract (uadd_overflow x, y), 1 <-> ~x <u y: https://alive2.llvm.org/ce/z/w5sc2k).

Shouldn't we emit an assume in the form of x+y < uint_max true instead?

Agree that we should emit overflow checking idioms instead of !(extract (uadd_overflow x, y), 1), if we only care about the overflow flag. However, for the rustc case handled by #84016, both the result and the overflow flag are used.

To address the motivating case in #95652, we should:

convert extract (xxx_overflow x, y), 1 into canonical forms
identify these idioms and add them into DomConditionCache
handle these idioms in computeOverflowForXXX

As @nikic pointed out, we have yet to encounter a case for divideCeil(A, B) where A+B doesn't overflow. So I don't know if this patch has real-world usefulness :(

antoniofrighetto · 2025-01-11T17:35:38Z

Thank you for clarifying this! Closing this as I'd rather avoid introducing patterns with dubious real-world usefulness. Happy to reopen it in the future, should it turn out to be needed.

antoniofrighetto requested review from nikic, dtcxzyw and goldsteinn July 29, 2024 06:27

llvmbot added the llvm:transforms label Jul 29, 2024

dtcxzyw reviewed Jul 29, 2024

View reviewed changes

goldsteinn reviewed Jul 29, 2024

View reviewed changes

llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp Outdated Show resolved Hide resolved

goldsteinn reviewed Jul 29, 2024

View reviewed changes

dtcxzyw reviewed Jul 30, 2024

View reviewed changes

llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp Outdated Show resolved Hide resolved

dtcxzyw reviewed Jul 30, 2024

View reviewed changes

llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp Outdated Show resolved Hide resolved

dtcxzyw added a commit to dtcxzyw/llvm-opt-benchmark that referenced this pull request Jul 30, 2024

pre-commit: test PR100977

9d7459a

PR Link: llvm/llvm-project#100977

dtcxzyw mentioned this pull request Jul 30, 2024

pre-commit: test PR100977 dtcxzyw/llvm-opt-benchmark#1108

Closed

nikic mentioned this pull request Aug 30, 2024

[InstCombine] Optimize ceil division idiom #95652

Closed

antoniofrighetto force-pushed the feature/ic-ceil-div-idiom branch from cca259c to edb1352 Compare January 6, 2025 14:28

llvmbot added the llvm:instcombine label Jan 6, 2025

antoniofrighetto added 2 commits January 6, 2025 15:28

[InstCombine] Introduce test for PR100977 (NFC)

87aa097

[InstCombine] Handle ceil division idiom

0ae6431

The expression `add (udiv (sub A, Bias), B), Bias` can be folded to `udiv (add A, B - 1), B)` when the sum between `A` and `B` is known not to overflow, and `Bias = A != 0`. Proof: https://alive2.llvm.org/ce/z/hiWHQA.

antoniofrighetto force-pushed the feature/ic-ceil-div-idiom branch from edb1352 to 0ae6431 Compare January 6, 2025 14:29

antoniofrighetto closed this Jan 11, 2025

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[InstCombine] Handle ceil division idiom #100977

[InstCombine] Handle ceil division idiom #100977

antoniofrighetto commented Jul 29, 2024

llvmbot commented Jul 29, 2024

antoniofrighetto commented Jul 29, 2024

dtcxzyw Jul 29, 2024

antoniofrighetto Jul 29, 2024

dtcxzyw commented Jul 29, 2024

goldsteinn Jul 29, 2024

antoniofrighetto Jul 29, 2024

goldsteinn Jul 29, 2024

antoniofrighetto Jul 29, 2024

antoniofrighetto commented Jul 30, 2024

dtcxzyw commented Jul 30, 2024

nikic commented Aug 1, 2024

antoniofrighetto commented Aug 27, 2024

nikic commented Aug 27, 2024

dtcxzyw commented Jan 5, 2025

antoniofrighetto commented Jan 6, 2025

antoniofrighetto commented Jan 6, 2025 •

edited

Loading

dtcxzyw commented Jan 9, 2025

antoniofrighetto commented Jan 11, 2025

[InstCombine] Handle ceil division idiom #100977

[InstCombine] Handle ceil division idiom #100977

Conversation

antoniofrighetto commented Jul 29, 2024

llvmbot commented Jul 29, 2024

antoniofrighetto commented Jul 29, 2024

dtcxzyw Jul 29, 2024

Choose a reason for hiding this comment

antoniofrighetto Jul 29, 2024

Choose a reason for hiding this comment

dtcxzyw commented Jul 29, 2024

goldsteinn Jul 29, 2024

Choose a reason for hiding this comment

antoniofrighetto Jul 29, 2024

Choose a reason for hiding this comment

goldsteinn Jul 29, 2024

Choose a reason for hiding this comment

antoniofrighetto Jul 29, 2024

Choose a reason for hiding this comment

antoniofrighetto commented Jul 30, 2024

dtcxzyw commented Jul 30, 2024

nikic commented Aug 1, 2024

antoniofrighetto commented Aug 27, 2024

nikic commented Aug 27, 2024

dtcxzyw commented Jan 5, 2025

antoniofrighetto commented Jan 6, 2025

antoniofrighetto commented Jan 6, 2025 • edited Loading

dtcxzyw commented Jan 9, 2025

antoniofrighetto commented Jan 11, 2025

antoniofrighetto commented Jan 6, 2025 •

edited

Loading