From 82d3666162ff715f8bd7992d279721decb3dd89a Mon Sep 17 00:00:00 2001 From: Han-Kuan Chen Date: Thu, 25 Jul 2024 17:22:19 -0700 Subject: [PATCH 1/6] [SLP][REVEC] Pre-commit test. --- llvm/test/Transforms/SLPVectorizer/revec.ll | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll index 9c8a98ba47b62..2cd227734938e 100644 --- a/llvm/test/Transforms/SLPVectorizer/revec.ll +++ b/llvm/test/Transforms/SLPVectorizer/revec.ll @@ -238,3 +238,20 @@ define void @test7() { store <8 x i16> %3, ptr null, align 2 ret void } + +define void @test5() { +entry: + br i1 false, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body + +for.cond.cleanup.loopexit.unr-lcssa: ; preds = %for.body, %entry + %0 = phi <2 x float> [ zeroinitializer, %entry ], [ %4, %for.body ] + %1 = phi <2 x float> [ zeroinitializer, %entry ], [ %5, %for.body ] + %2 = phi <2 x float> [ zeroinitializer, %entry ], [ %4, %for.body ] + %3 = phi <2 x float> [ zeroinitializer, %entry ], [ %5, %for.body ] + ret void + +for.body: ; preds = %for.body, %entry + %4 = phi <2 x float> [ %4, %for.body ], [ zeroinitializer, %entry ] + %5 = phi <2 x float> [ %5, %for.body ], [ zeroinitializer, %entry ] + br i1 false, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body +} From cde39bcf41930fe871235db9f6f207f851f4b609 Mon Sep 17 00:00:00 2001 From: Han-Kuan Chen Date: Thu, 4 Jul 2024 22:25:52 -0700 Subject: [PATCH 2/6] [SLP][REVEC] Use VL.front()->getType() as ScalarTy. VL.front()->getType() may be FixedVectorType when revec is enabled. Fix "Expected item in MinBWs.". --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 ++-- llvm/test/Transforms/SLPVectorizer/revec.ll | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 39cfd08976b48..a6fc858f8f12c 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -12586,8 +12586,8 @@ Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx, } if (IsSameVE) { auto FinalShuffle = [&](Value *V, ArrayRef Mask) { - ShuffleInstructionBuilder ShuffleBuilder( - cast(V->getType())->getElementType(), Builder, *this); + ShuffleInstructionBuilder ShuffleBuilder(VL.front()->getType(), Builder, + *this); ShuffleBuilder.add(V, Mask); return ShuffleBuilder.finalize(std::nullopt); }; diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll index 2cd227734938e..12a2096e22419 100644 --- a/llvm/test/Transforms/SLPVectorizer/revec.ll +++ b/llvm/test/Transforms/SLPVectorizer/revec.ll @@ -240,6 +240,23 @@ define void @test7() { } define void @test5() { +; CHECK-LABEL: @test5( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v2f32(<8 x float> poison, <2 x float> zeroinitializer, i64 0) +; CHECK-NEXT: [[TMP1:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v2f32(<8 x float> [[TMP0]], <2 x float> zeroinitializer, i64 2) +; CHECK-NEXT: [[TMP2:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v2f32(<8 x float> [[TMP1]], <2 x float> zeroinitializer, i64 4) +; CHECK-NEXT: [[TMP3:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v2f32(<8 x float> [[TMP2]], <2 x float> zeroinitializer, i64 6) +; CHECK-NEXT: [[TMP4:%.*]] = call <4 x float> @llvm.vector.insert.v4f32.v2f32(<4 x float> poison, <2 x float> zeroinitializer, i64 0) +; CHECK-NEXT: [[TMP5:%.*]] = call <4 x float> @llvm.vector.insert.v4f32.v2f32(<4 x float> [[TMP4]], <2 x float> zeroinitializer, i64 2) +; CHECK-NEXT: br i1 false, label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA:%.*]], label [[FOR_BODY:%.*]] +; CHECK: for.cond.cleanup.loopexit.unr-lcssa: +; CHECK-NEXT: [[TMP6:%.*]] = phi <8 x float> [ [[TMP3]], [[ENTRY:%.*]] ], [ [[TMP8:%.*]], [[FOR_BODY]] ] +; CHECK-NEXT: ret void +; CHECK: for.body: +; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x float> [ [[TMP7]], [[FOR_BODY]] ], [ [[TMP5]], [[ENTRY]] ] +; CHECK-NEXT: [[TMP8]] = shufflevector <4 x float> [[TMP7]], <4 x float> poison, <8 x i32> +; CHECK-NEXT: br i1 false, label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]], label [[FOR_BODY]] +; entry: br i1 false, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body From d5bdd5030473b8284d6f4e5ffbf411baefddea33 Mon Sep 17 00:00:00 2001 From: Han-Kuan Chen Date: Thu, 8 Aug 2024 06:43:02 -0700 Subject: [PATCH 3/6] [SLP][REVEC] Apply comments. --- llvm/test/Transforms/SLPVectorizer/revec.ll | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll index 12a2096e22419..f81dfe9a22c15 100644 --- a/llvm/test/Transforms/SLPVectorizer/revec.ll +++ b/llvm/test/Transforms/SLPVectorizer/revec.ll @@ -248,27 +248,27 @@ define void @test5() { ; CHECK-NEXT: [[TMP3:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v2f32(<8 x float> [[TMP2]], <2 x float> zeroinitializer, i64 6) ; CHECK-NEXT: [[TMP4:%.*]] = call <4 x float> @llvm.vector.insert.v4f32.v2f32(<4 x float> poison, <2 x float> zeroinitializer, i64 0) ; CHECK-NEXT: [[TMP5:%.*]] = call <4 x float> @llvm.vector.insert.v4f32.v2f32(<4 x float> [[TMP4]], <2 x float> zeroinitializer, i64 2) -; CHECK-NEXT: br i1 false, label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA:%.*]], label [[FOR_BODY:%.*]] -; CHECK: for.cond.cleanup.loopexit.unr-lcssa: +; CHECK-NEXT: br i1 false, label [[FOR0:%.*]], label [[FOR_BODY:%.*]] +; CHECK: for0: ; CHECK-NEXT: [[TMP6:%.*]] = phi <8 x float> [ [[TMP3]], [[ENTRY:%.*]] ], [ [[TMP8:%.*]], [[FOR_BODY]] ] ; CHECK-NEXT: ret void ; CHECK: for.body: ; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x float> [ [[TMP7]], [[FOR_BODY]] ], [ [[TMP5]], [[ENTRY]] ] ; CHECK-NEXT: [[TMP8]] = shufflevector <4 x float> [[TMP7]], <4 x float> poison, <8 x i32> -; CHECK-NEXT: br i1 false, label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]], label [[FOR_BODY]] +; CHECK-NEXT: br i1 false, label [[FOR0]], label [[FOR_BODY]] ; entry: - br i1 false, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body + br i1 false, label %for0, label %for.body -for.cond.cleanup.loopexit.unr-lcssa: ; preds = %for.body, %entry +for0: %0 = phi <2 x float> [ zeroinitializer, %entry ], [ %4, %for.body ] %1 = phi <2 x float> [ zeroinitializer, %entry ], [ %5, %for.body ] %2 = phi <2 x float> [ zeroinitializer, %entry ], [ %4, %for.body ] %3 = phi <2 x float> [ zeroinitializer, %entry ], [ %5, %for.body ] ret void -for.body: ; preds = %for.body, %entry +for.body: %4 = phi <2 x float> [ %4, %for.body ], [ zeroinitializer, %entry ] %5 = phi <2 x float> [ %5, %for.body ], [ zeroinitializer, %entry ] - br i1 false, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body + br i1 false, label %for0, label %for.body } From 33c633e446f47a5ad771af9f431bf9eba915add6 Mon Sep 17 00:00:00 2001 From: Han-Kuan Chen Date: Mon, 12 Aug 2024 23:23:48 -0700 Subject: [PATCH 4/6] [SLP][REVEC] Solve conflicts. --- llvm/test/Transforms/SLPVectorizer/revec.ll | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll index f81dfe9a22c15..8f43d7725ff78 100644 --- a/llvm/test/Transforms/SLPVectorizer/revec.ll +++ b/llvm/test/Transforms/SLPVectorizer/revec.ll @@ -239,8 +239,8 @@ define void @test7() { ret void } -define void @test5() { -; CHECK-LABEL: @test5( +define void @test7() { +; CHECK-LABEL: @test7( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v2f32(<8 x float> poison, <2 x float> zeroinitializer, i64 0) ; CHECK-NEXT: [[TMP1:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v2f32(<8 x float> [[TMP0]], <2 x float> zeroinitializer, i64 2) From 40acd6a77c3175fd8cd5f6ae032d207b6b0f8afe Mon Sep 17 00:00:00 2001 From: Han-Kuan Chen Date: Tue, 13 Aug 2024 02:33:58 -0700 Subject: [PATCH 5/6] [SLP][REVEC] DO not break MinBWs flow. --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index a6fc858f8f12c..8a08d766a8ed4 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -12586,8 +12586,15 @@ Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx, } if (IsSameVE) { auto FinalShuffle = [&](Value *V, ArrayRef Mask) { - ShuffleInstructionBuilder ShuffleBuilder(VL.front()->getType(), Builder, - *this); + // V may be affected by MinBWs. + // We want ShuffleInstructionBuilder to correctly support REVEC. The key + // factor is the number of elements, not their type. + Type *ScalarTy = cast(V->getType())->getElementType(); + unsigned NumElements = getNumElements(VL.front()->getType()); + ShuffleInstructionBuilder ShuffleBuilder( + NumElements != 1 ? FixedVectorType::get(ScalarTy, NumElements) + : ScalarTy, + Builder, *this); ShuffleBuilder.add(V, Mask); return ShuffleBuilder.finalize(std::nullopt); }; From 3ef091e64b9d05d16499fa3b4064b693a89e4a5e Mon Sep 17 00:00:00 2001 From: Han-Kuan Chen Date: Tue, 13 Aug 2024 04:42:01 -0700 Subject: [PATCH 6/6] [SLP][REVEC] Solve conflicts. --- llvm/test/Transforms/SLPVectorizer/revec.ll | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll index 8f43d7725ff78..9ce9db1d360cc 100644 --- a/llvm/test/Transforms/SLPVectorizer/revec.ll +++ b/llvm/test/Transforms/SLPVectorizer/revec.ll @@ -239,8 +239,8 @@ define void @test7() { ret void } -define void @test7() { -; CHECK-LABEL: @test7( +define void @test8() { +; CHECK-LABEL: @test8( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v2f32(<8 x float> poison, <2 x float> zeroinitializer, i64 0) ; CHECK-NEXT: [[TMP1:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v2f32(<8 x float> [[TMP0]], <2 x float> zeroinitializer, i64 2)