-
Notifications
You must be signed in to change notification settings - Fork 13.6k
[HLSL] Implement WaveActiveAllTrue
Intrinsic
#117245
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Conversation
@llvm/pr-subscribers-hlsl @llvm/pr-subscribers-backend-directx Author: Ashley Coleman (V-FEXrt) ChangesResolves #99161
Full diff: https://github.com/llvm/llvm-project/pull/117245.diff 12 Files Affected:
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 290feb58754adb..8aa5719d736f94 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -4750,6 +4750,12 @@ def HLSLAny : LangBuiltin<"HLSL_LANG"> {
let Prototype = "bool(...)";
}
+def HLSLWaveActiveAllTrue : LangBuiltin<"HLSL_LANG"> {
+ let Spellings = ["__builtin_hlsl_wave_active_all_true"];
+ let Attributes = [NoThrow, Const];
+ let Prototype = "bool(bool)";
+}
+
def HLSLWaveActiveAnyTrue : LangBuiltin<"HLSL_LANG"> {
let Spellings = ["__builtin_hlsl_wave_active_any_true"];
let Attributes = [NoThrow, Const];
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index ff7132fd8bc1e7..522a6ce3b999e3 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -19282,6 +19282,16 @@ case Builtin::BI__builtin_hlsl_elementwise_isinf: {
/*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getStepIntrinsic(),
ArrayRef<Value *>{Op0, Op1}, nullptr, "hlsl.step");
}
+ case Builtin::BI__builtin_hlsl_wave_active_all_true: {
+ Value *Op = EmitScalarExpr(E->getArg(0));
+ llvm::Type *Ty = Op->getType();
+ assert(Ty->isIntegerTy(1) &&
+ "Intrinsic WaveActiveAllTrue operand must be a bool");
+
+ Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
+ return EmitRuntimeCall(
+ Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID), {Op});
+ }
case Builtin::BI__builtin_hlsl_wave_active_any_true: {
Value *Op = EmitScalarExpr(E->getArg(0));
assert(Op->getType()->isIntegerTy(1) &&
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index a8e0ed42b79a35..29c0fe1acabccc 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -91,6 +91,7 @@ class CGHLSLRuntime {
GENERATE_HLSL_INTRINSIC_FUNCTION(UDot, udot)
GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddI8Packed, dot4add_i8packed)
GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddU8Packed, dot4add_u8packed)
+ GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAllTrue, wave_all)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAnyTrue, wave_any)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveCountBits, wave_active_countbits)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveIsFirstLane, wave_is_first_lane)
diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index a484d04155d6b2..a99b3b7e4202b8 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -2223,6 +2223,15 @@ float4 trunc(float4);
// Wave* builtins
//===----------------------------------------------------------------------===//
+/// \brief Returns true if the expression is true in all active lanes in the
+/// current wave.
+///
+/// \param Val The boolean expression to evaluate.
+/// \return True if the expression is true in all lanes.
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_all_true)
+__attribute__((convergent)) bool WaveActiveAllTrue(bool Val);
+
/// \brief Returns true if the expression is true in any active lane in the
/// current wave.
///
diff --git a/clang/test/CodeGenHLSL/builtins/WaveActiveAllTrue.hlsl b/clang/test/CodeGenHLSL/builtins/WaveActiveAllTrue.hlsl
new file mode 100644
index 00000000000000..df530a9cee561a
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/WaveActiveAllTrue.hlsl
@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 -finclude-default-header -fnative-half-type -triple \
+// RUN: dxil-pc-shadermodel6.3-compute %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-DXIL
+// RUN: %clang_cc1 -finclude-default-header -fnative-half-type -triple \
+// RUN: spirv-pc-vulkan-compute %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-SPIRV
+
+// Test basic lowering to runtime function call for int values.
+
+// CHECK-LABEL: define {{.*}}test
+bool test(bool p1) {
+ // CHECK-SPIRV: %[[#entry_tok0:]] = call token @llvm.experimental.convergence.entry()
+ // CHECK-SPIRV: %[[RET:.*]] = call spir_func i1 @llvm.spv.wave.all(i1 %{{[a-zA-Z0-9]+}}) [ "convergencectrl"(token %[[#entry_tok0]]) ]
+ // CHECK-DXIL: %[[RET:.*]] = call i1 @llvm.dx.wave.all(i1 %{{[a-zA-Z0-9]+}})
+ // CHECK: ret i1 %[[RET]]
+ return WaveActiveAllTrue(p1);
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/WaveActiveAllTrue-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/WaveActiveAllTrue-errors.hlsl
new file mode 100644
index 00000000000000..b0d0fdfca5e18d
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/WaveActiveAllTrue-errors.hlsl
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify
+
+bool test_too_few_arg() {
+ return __builtin_hlsl_wave_active_all_true();
+ // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
+}
+
+bool test_too_many_arg(bool p0) {
+ return __builtin_hlsl_wave_active_all_true(p0, p0);
+ // expected-error@-1 {{too many arguments to function call, expected 1, have 2}}
+}
+
+struct Foo
+{
+ int a;
+};
+
+bool test_type_check(Foo p0) {
+ return __builtin_hlsl_wave_active_all_true(p0);
+ // expected-error@-1 {{no viable conversion from 'Foo' to 'bool'}}
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 62688eae36e9a8..8bb70d6a9b84a3 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -94,6 +94,7 @@ def int_dx_umad : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLV
def int_dx_normalize : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty], [IntrNoMem]>;
def int_dx_rsqrt : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
def int_dx_wave_active_countbits : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
+def int_dx_wave_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
def int_dx_wave_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
def int_dx_wave_getlaneindex : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrConvergent, IntrNoMem]>;
def int_dx_wave_is_first_lane : DefaultAttrsIntrinsic<[llvm_i1_ty], [], [IntrConvergent]>;
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index e1157085832866..63539fa9466098 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -86,6 +86,7 @@ let TargetPrefix = "spv" in {
def int_spv_dot4add_i8packed : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
def int_spv_dot4add_u8packed : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
def int_spv_wave_active_countbits : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
+ def int_spv_wave_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
def int_spv_wave_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
def int_spv_wave_is_first_lane : DefaultAttrsIntrinsic<[llvm_i1_ty], [], [IntrConvergent]>;
def int_spv_wave_readlane : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>, llvm_i32_ty], [IntrConvergent, IntrNoMem]>;
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 36228a5e0dce18..5b2e2d116fe0aa 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -861,6 +861,14 @@ def CreateHandleFromBinding : DXILOp<217, createHandleFromBinding> {
let stages = [Stages<DXIL1_6, [all_stages]>];
}
+def WaveActiveAllTrue : DXILOp<334, waveAllTrue> {
+ let Doc = "returns true if the expression is true in all of the active lanes in the current wave";
+ let LLVMIntrinsic = int_dx_wave_all;
+ let arguments = [Int1Ty];
+ let result = Int1Ty;
+ let stages = [Stages<DXIL1_0, [all_stages]>];
+}
+
def WaveActiveAnyTrue : DXILOp<113, waveAnyTrue> {
let Doc = "returns true if the expression is true in any of the active lanes in the current wave";
let LLVMIntrinsic = int_dx_wave_any;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 6b23b0d6b6f787..fcbd1429144b58 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -2830,6 +2830,8 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp);
case Intrinsic::spv_wave_active_countbits:
return selectWaveActiveCountBits(ResVReg, ResType, I);
+ case Intrinsic::spv_wave_all:
+ return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll);
case Intrinsic::spv_wave_any:
return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny);
case Intrinsic::spv_wave_is_first_lane:
diff --git a/llvm/test/CodeGen/DirectX/WaveActiveAllTrue.ll b/llvm/test/CodeGen/DirectX/WaveActiveAllTrue.ll
new file mode 100644
index 00000000000000..73b3e4f7843845
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/WaveActiveAllTrue.ll
@@ -0,0 +1,10 @@
+; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-compute %s | FileCheck %s
+
+define noundef i1 @wave_all_simple(i1 noundef %p1) {
+entry:
+; CHECK: call i1 @dx.op.waveAllTrue(i32 334, i1 %p1)
+ %ret = call i1 @llvm.dx.wave.all(i1 %p1)
+ ret i1 %ret
+}
+
+declare i1 @llvm.dx.wave.all(i1)
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveActiveAllTrue.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveActiveAllTrue.ll
new file mode 100644
index 00000000000000..d030353ad93b4f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveActiveAllTrue.ll
@@ -0,0 +1,21 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#bool:]] = OpTypeBool
+; CHECK-DAG: %[[#uint:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#scope:]] = OpConstant %[[#uint]] 3
+; CHECK-DAG: OpCapability GroupNonUniformVote
+
+; CHECK-LABEL: Begin function test_wave_all
+define i1 @test_wave_all(i1 %p1) #0 {
+entry:
+; CHECK: %[[#param:]] = OpFunctionParameter %[[#bool]]
+; CHECK: %{{.+}} = OpGroupNonUniformAll %[[#bool]] %[[#scope]] %[[#param]]
+ %0 = call token @llvm.experimental.convergence.entry()
+ %ret = call i1 @llvm.spv.wave.all(i1 %p1) [ "convergencectrl"(token %0) ]
+ ret i1 %ret
+}
+
+declare i1 @llvm.spv.wave.all(i1) #0
+
+attributes #0 = { convergent }
|
@llvm/pr-subscribers-backend-x86 Author: Ashley Coleman (V-FEXrt) ChangesResolves #99161
Full diff: https://github.com/llvm/llvm-project/pull/117245.diff 12 Files Affected:
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 290feb58754adb..8aa5719d736f94 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -4750,6 +4750,12 @@ def HLSLAny : LangBuiltin<"HLSL_LANG"> {
let Prototype = "bool(...)";
}
+def HLSLWaveActiveAllTrue : LangBuiltin<"HLSL_LANG"> {
+ let Spellings = ["__builtin_hlsl_wave_active_all_true"];
+ let Attributes = [NoThrow, Const];
+ let Prototype = "bool(bool)";
+}
+
def HLSLWaveActiveAnyTrue : LangBuiltin<"HLSL_LANG"> {
let Spellings = ["__builtin_hlsl_wave_active_any_true"];
let Attributes = [NoThrow, Const];
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index ff7132fd8bc1e7..522a6ce3b999e3 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -19282,6 +19282,16 @@ case Builtin::BI__builtin_hlsl_elementwise_isinf: {
/*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getStepIntrinsic(),
ArrayRef<Value *>{Op0, Op1}, nullptr, "hlsl.step");
}
+ case Builtin::BI__builtin_hlsl_wave_active_all_true: {
+ Value *Op = EmitScalarExpr(E->getArg(0));
+ llvm::Type *Ty = Op->getType();
+ assert(Ty->isIntegerTy(1) &&
+ "Intrinsic WaveActiveAllTrue operand must be a bool");
+
+ Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
+ return EmitRuntimeCall(
+ Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID), {Op});
+ }
case Builtin::BI__builtin_hlsl_wave_active_any_true: {
Value *Op = EmitScalarExpr(E->getArg(0));
assert(Op->getType()->isIntegerTy(1) &&
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index a8e0ed42b79a35..29c0fe1acabccc 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -91,6 +91,7 @@ class CGHLSLRuntime {
GENERATE_HLSL_INTRINSIC_FUNCTION(UDot, udot)
GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddI8Packed, dot4add_i8packed)
GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddU8Packed, dot4add_u8packed)
+ GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAllTrue, wave_all)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAnyTrue, wave_any)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveCountBits, wave_active_countbits)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveIsFirstLane, wave_is_first_lane)
diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index a484d04155d6b2..a99b3b7e4202b8 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -2223,6 +2223,15 @@ float4 trunc(float4);
// Wave* builtins
//===----------------------------------------------------------------------===//
+/// \brief Returns true if the expression is true in all active lanes in the
+/// current wave.
+///
+/// \param Val The boolean expression to evaluate.
+/// \return True if the expression is true in all lanes.
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_all_true)
+__attribute__((convergent)) bool WaveActiveAllTrue(bool Val);
+
/// \brief Returns true if the expression is true in any active lane in the
/// current wave.
///
diff --git a/clang/test/CodeGenHLSL/builtins/WaveActiveAllTrue.hlsl b/clang/test/CodeGenHLSL/builtins/WaveActiveAllTrue.hlsl
new file mode 100644
index 00000000000000..df530a9cee561a
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/WaveActiveAllTrue.hlsl
@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 -finclude-default-header -fnative-half-type -triple \
+// RUN: dxil-pc-shadermodel6.3-compute %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-DXIL
+// RUN: %clang_cc1 -finclude-default-header -fnative-half-type -triple \
+// RUN: spirv-pc-vulkan-compute %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-SPIRV
+
+// Test basic lowering to runtime function call for int values.
+
+// CHECK-LABEL: define {{.*}}test
+bool test(bool p1) {
+ // CHECK-SPIRV: %[[#entry_tok0:]] = call token @llvm.experimental.convergence.entry()
+ // CHECK-SPIRV: %[[RET:.*]] = call spir_func i1 @llvm.spv.wave.all(i1 %{{[a-zA-Z0-9]+}}) [ "convergencectrl"(token %[[#entry_tok0]]) ]
+ // CHECK-DXIL: %[[RET:.*]] = call i1 @llvm.dx.wave.all(i1 %{{[a-zA-Z0-9]+}})
+ // CHECK: ret i1 %[[RET]]
+ return WaveActiveAllTrue(p1);
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/WaveActiveAllTrue-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/WaveActiveAllTrue-errors.hlsl
new file mode 100644
index 00000000000000..b0d0fdfca5e18d
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/WaveActiveAllTrue-errors.hlsl
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify
+
+bool test_too_few_arg() {
+ return __builtin_hlsl_wave_active_all_true();
+ // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
+}
+
+bool test_too_many_arg(bool p0) {
+ return __builtin_hlsl_wave_active_all_true(p0, p0);
+ // expected-error@-1 {{too many arguments to function call, expected 1, have 2}}
+}
+
+struct Foo
+{
+ int a;
+};
+
+bool test_type_check(Foo p0) {
+ return __builtin_hlsl_wave_active_all_true(p0);
+ // expected-error@-1 {{no viable conversion from 'Foo' to 'bool'}}
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 62688eae36e9a8..8bb70d6a9b84a3 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -94,6 +94,7 @@ def int_dx_umad : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLV
def int_dx_normalize : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty], [IntrNoMem]>;
def int_dx_rsqrt : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
def int_dx_wave_active_countbits : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
+def int_dx_wave_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
def int_dx_wave_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
def int_dx_wave_getlaneindex : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrConvergent, IntrNoMem]>;
def int_dx_wave_is_first_lane : DefaultAttrsIntrinsic<[llvm_i1_ty], [], [IntrConvergent]>;
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index e1157085832866..63539fa9466098 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -86,6 +86,7 @@ let TargetPrefix = "spv" in {
def int_spv_dot4add_i8packed : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
def int_spv_dot4add_u8packed : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
def int_spv_wave_active_countbits : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
+ def int_spv_wave_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
def int_spv_wave_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
def int_spv_wave_is_first_lane : DefaultAttrsIntrinsic<[llvm_i1_ty], [], [IntrConvergent]>;
def int_spv_wave_readlane : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>, llvm_i32_ty], [IntrConvergent, IntrNoMem]>;
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 36228a5e0dce18..5b2e2d116fe0aa 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -861,6 +861,14 @@ def CreateHandleFromBinding : DXILOp<217, createHandleFromBinding> {
let stages = [Stages<DXIL1_6, [all_stages]>];
}
+def WaveActiveAllTrue : DXILOp<334, waveAllTrue> {
+ let Doc = "returns true if the expression is true in all of the active lanes in the current wave";
+ let LLVMIntrinsic = int_dx_wave_all;
+ let arguments = [Int1Ty];
+ let result = Int1Ty;
+ let stages = [Stages<DXIL1_0, [all_stages]>];
+}
+
def WaveActiveAnyTrue : DXILOp<113, waveAnyTrue> {
let Doc = "returns true if the expression is true in any of the active lanes in the current wave";
let LLVMIntrinsic = int_dx_wave_any;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 6b23b0d6b6f787..fcbd1429144b58 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -2830,6 +2830,8 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp);
case Intrinsic::spv_wave_active_countbits:
return selectWaveActiveCountBits(ResVReg, ResType, I);
+ case Intrinsic::spv_wave_all:
+ return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll);
case Intrinsic::spv_wave_any:
return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny);
case Intrinsic::spv_wave_is_first_lane:
diff --git a/llvm/test/CodeGen/DirectX/WaveActiveAllTrue.ll b/llvm/test/CodeGen/DirectX/WaveActiveAllTrue.ll
new file mode 100644
index 00000000000000..73b3e4f7843845
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/WaveActiveAllTrue.ll
@@ -0,0 +1,10 @@
+; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-compute %s | FileCheck %s
+
+define noundef i1 @wave_all_simple(i1 noundef %p1) {
+entry:
+; CHECK: call i1 @dx.op.waveAllTrue(i32 334, i1 %p1)
+ %ret = call i1 @llvm.dx.wave.all(i1 %p1)
+ ret i1 %ret
+}
+
+declare i1 @llvm.dx.wave.all(i1)
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveActiveAllTrue.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveActiveAllTrue.ll
new file mode 100644
index 00000000000000..d030353ad93b4f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveActiveAllTrue.ll
@@ -0,0 +1,21 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#bool:]] = OpTypeBool
+; CHECK-DAG: %[[#uint:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#scope:]] = OpConstant %[[#uint]] 3
+; CHECK-DAG: OpCapability GroupNonUniformVote
+
+; CHECK-LABEL: Begin function test_wave_all
+define i1 @test_wave_all(i1 %p1) #0 {
+entry:
+; CHECK: %[[#param:]] = OpFunctionParameter %[[#bool]]
+; CHECK: %{{.+}} = OpGroupNonUniformAll %[[#bool]] %[[#scope]] %[[#param]]
+ %0 = call token @llvm.experimental.convergence.entry()
+ %ret = call i1 @llvm.spv.wave.all(i1 %p1) [ "convergencectrl"(token %0) ]
+ ret i1 %ret
+}
+
+declare i1 @llvm.spv.wave.all(i1) #0
+
+attributes #0 = { convergent }
|
@llvm/pr-subscribers-clang Author: Ashley Coleman (V-FEXrt) ChangesResolves #99161
Full diff: https://github.com/llvm/llvm-project/pull/117245.diff 12 Files Affected:
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 290feb58754adb..8aa5719d736f94 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -4750,6 +4750,12 @@ def HLSLAny : LangBuiltin<"HLSL_LANG"> {
let Prototype = "bool(...)";
}
+def HLSLWaveActiveAllTrue : LangBuiltin<"HLSL_LANG"> {
+ let Spellings = ["__builtin_hlsl_wave_active_all_true"];
+ let Attributes = [NoThrow, Const];
+ let Prototype = "bool(bool)";
+}
+
def HLSLWaveActiveAnyTrue : LangBuiltin<"HLSL_LANG"> {
let Spellings = ["__builtin_hlsl_wave_active_any_true"];
let Attributes = [NoThrow, Const];
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index ff7132fd8bc1e7..522a6ce3b999e3 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -19282,6 +19282,16 @@ case Builtin::BI__builtin_hlsl_elementwise_isinf: {
/*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getStepIntrinsic(),
ArrayRef<Value *>{Op0, Op1}, nullptr, "hlsl.step");
}
+ case Builtin::BI__builtin_hlsl_wave_active_all_true: {
+ Value *Op = EmitScalarExpr(E->getArg(0));
+ llvm::Type *Ty = Op->getType();
+ assert(Ty->isIntegerTy(1) &&
+ "Intrinsic WaveActiveAllTrue operand must be a bool");
+
+ Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
+ return EmitRuntimeCall(
+ Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID), {Op});
+ }
case Builtin::BI__builtin_hlsl_wave_active_any_true: {
Value *Op = EmitScalarExpr(E->getArg(0));
assert(Op->getType()->isIntegerTy(1) &&
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index a8e0ed42b79a35..29c0fe1acabccc 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -91,6 +91,7 @@ class CGHLSLRuntime {
GENERATE_HLSL_INTRINSIC_FUNCTION(UDot, udot)
GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddI8Packed, dot4add_i8packed)
GENERATE_HLSL_INTRINSIC_FUNCTION(Dot4AddU8Packed, dot4add_u8packed)
+ GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAllTrue, wave_all)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveAnyTrue, wave_any)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveActiveCountBits, wave_active_countbits)
GENERATE_HLSL_INTRINSIC_FUNCTION(WaveIsFirstLane, wave_is_first_lane)
diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index a484d04155d6b2..a99b3b7e4202b8 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -2223,6 +2223,15 @@ float4 trunc(float4);
// Wave* builtins
//===----------------------------------------------------------------------===//
+/// \brief Returns true if the expression is true in all active lanes in the
+/// current wave.
+///
+/// \param Val The boolean expression to evaluate.
+/// \return True if the expression is true in all lanes.
+_HLSL_AVAILABILITY(shadermodel, 6.0)
+_HLSL_BUILTIN_ALIAS(__builtin_hlsl_wave_active_all_true)
+__attribute__((convergent)) bool WaveActiveAllTrue(bool Val);
+
/// \brief Returns true if the expression is true in any active lane in the
/// current wave.
///
diff --git a/clang/test/CodeGenHLSL/builtins/WaveActiveAllTrue.hlsl b/clang/test/CodeGenHLSL/builtins/WaveActiveAllTrue.hlsl
new file mode 100644
index 00000000000000..df530a9cee561a
--- /dev/null
+++ b/clang/test/CodeGenHLSL/builtins/WaveActiveAllTrue.hlsl
@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 -finclude-default-header -fnative-half-type -triple \
+// RUN: dxil-pc-shadermodel6.3-compute %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-DXIL
+// RUN: %clang_cc1 -finclude-default-header -fnative-half-type -triple \
+// RUN: spirv-pc-vulkan-compute %s -emit-llvm -disable-llvm-passes -o - | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-SPIRV
+
+// Test basic lowering to runtime function call for int values.
+
+// CHECK-LABEL: define {{.*}}test
+bool test(bool p1) {
+ // CHECK-SPIRV: %[[#entry_tok0:]] = call token @llvm.experimental.convergence.entry()
+ // CHECK-SPIRV: %[[RET:.*]] = call spir_func i1 @llvm.spv.wave.all(i1 %{{[a-zA-Z0-9]+}}) [ "convergencectrl"(token %[[#entry_tok0]]) ]
+ // CHECK-DXIL: %[[RET:.*]] = call i1 @llvm.dx.wave.all(i1 %{{[a-zA-Z0-9]+}})
+ // CHECK: ret i1 %[[RET]]
+ return WaveActiveAllTrue(p1);
+}
diff --git a/clang/test/SemaHLSL/BuiltIns/WaveActiveAllTrue-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/WaveActiveAllTrue-errors.hlsl
new file mode 100644
index 00000000000000..b0d0fdfca5e18d
--- /dev/null
+++ b/clang/test/SemaHLSL/BuiltIns/WaveActiveAllTrue-errors.hlsl
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.6-library %s -emit-llvm-only -disable-llvm-passes -verify
+
+bool test_too_few_arg() {
+ return __builtin_hlsl_wave_active_all_true();
+ // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
+}
+
+bool test_too_many_arg(bool p0) {
+ return __builtin_hlsl_wave_active_all_true(p0, p0);
+ // expected-error@-1 {{too many arguments to function call, expected 1, have 2}}
+}
+
+struct Foo
+{
+ int a;
+};
+
+bool test_type_check(Foo p0) {
+ return __builtin_hlsl_wave_active_all_true(p0);
+ // expected-error@-1 {{no viable conversion from 'Foo' to 'bool'}}
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 62688eae36e9a8..8bb70d6a9b84a3 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -94,6 +94,7 @@ def int_dx_umad : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLV
def int_dx_normalize : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty], [IntrNoMem]>;
def int_dx_rsqrt : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
def int_dx_wave_active_countbits : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
+def int_dx_wave_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
def int_dx_wave_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
def int_dx_wave_getlaneindex : DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrConvergent, IntrNoMem]>;
def int_dx_wave_is_first_lane : DefaultAttrsIntrinsic<[llvm_i1_ty], [], [IntrConvergent]>;
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index e1157085832866..63539fa9466098 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -86,6 +86,7 @@ let TargetPrefix = "spv" in {
def int_spv_dot4add_i8packed : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
def int_spv_dot4add_u8packed : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
def int_spv_wave_active_countbits : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
+ def int_spv_wave_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
def int_spv_wave_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i1_ty], [IntrConvergent, IntrNoMem]>;
def int_spv_wave_is_first_lane : DefaultAttrsIntrinsic<[llvm_i1_ty], [], [IntrConvergent]>;
def int_spv_wave_readlane : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>, llvm_i32_ty], [IntrConvergent, IntrNoMem]>;
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 36228a5e0dce18..5b2e2d116fe0aa 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -861,6 +861,14 @@ def CreateHandleFromBinding : DXILOp<217, createHandleFromBinding> {
let stages = [Stages<DXIL1_6, [all_stages]>];
}
+def WaveActiveAllTrue : DXILOp<334, waveAllTrue> {
+ let Doc = "returns true if the expression is true in all of the active lanes in the current wave";
+ let LLVMIntrinsic = int_dx_wave_all;
+ let arguments = [Int1Ty];
+ let result = Int1Ty;
+ let stages = [Stages<DXIL1_0, [all_stages]>];
+}
+
def WaveActiveAnyTrue : DXILOp<113, waveAnyTrue> {
let Doc = "returns true if the expression is true in any of the active lanes in the current wave";
let LLVMIntrinsic = int_dx_wave_any;
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 6b23b0d6b6f787..fcbd1429144b58 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -2830,6 +2830,8 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp);
case Intrinsic::spv_wave_active_countbits:
return selectWaveActiveCountBits(ResVReg, ResType, I);
+ case Intrinsic::spv_wave_all:
+ return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll);
case Intrinsic::spv_wave_any:
return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny);
case Intrinsic::spv_wave_is_first_lane:
diff --git a/llvm/test/CodeGen/DirectX/WaveActiveAllTrue.ll b/llvm/test/CodeGen/DirectX/WaveActiveAllTrue.ll
new file mode 100644
index 00000000000000..73b3e4f7843845
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/WaveActiveAllTrue.ll
@@ -0,0 +1,10 @@
+; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-compute %s | FileCheck %s
+
+define noundef i1 @wave_all_simple(i1 noundef %p1) {
+entry:
+; CHECK: call i1 @dx.op.waveAllTrue(i32 334, i1 %p1)
+ %ret = call i1 @llvm.dx.wave.all(i1 %p1)
+ ret i1 %ret
+}
+
+declare i1 @llvm.dx.wave.all(i1)
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveActiveAllTrue.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveActiveAllTrue.ll
new file mode 100644
index 00000000000000..d030353ad93b4f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveActiveAllTrue.ll
@@ -0,0 +1,21 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#bool:]] = OpTypeBool
+; CHECK-DAG: %[[#uint:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#scope:]] = OpConstant %[[#uint]] 3
+; CHECK-DAG: OpCapability GroupNonUniformVote
+
+; CHECK-LABEL: Begin function test_wave_all
+define i1 @test_wave_all(i1 %p1) #0 {
+entry:
+; CHECK: %[[#param:]] = OpFunctionParameter %[[#bool]]
+; CHECK: %{{.+}} = OpGroupNonUniformAll %[[#bool]] %[[#scope]] %[[#param]]
+ %0 = call token @llvm.experimental.convergence.entry()
+ %ret = call i1 @llvm.spv.wave.all(i1 %p1) [ "convergencectrl"(token %0) ]
+ ret i1 %ret
+}
+
+declare i1 @llvm.spv.wave.all(i1) #0
+
+attributes #0 = { convergent }
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
LGTM, just a nit and your self-review
73afef2
to
47a582d
Compare
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
LGTM
LLVM Buildbot has detected a new failure on builder Full details are available at: https://lab.llvm.org/buildbot/#/builders/41/builds/4142 Here is the relevant piece of the build log for the reference
|
Resolves #99161
WaveActiveAllTrue
clang builtin,WaveActiveAllTrue
clang builtin withhlsl_intrinsics.h
WaveActiveAllTrue
toCheckHLSLBuiltinFunctionCall
inSemaChecking.cpp
WaveActiveAllTrue
toEmitHLSLBuiltinExpr
inCGBuiltin.cpp
clang/test/CodeGenHLSL/builtins/WaveActiveAllTrue.hlsl
clang/test/SemaHLSL/BuiltIns/WaveActiveAllTrue-errors.hlsl
int_dx_WaveActiveAllTrue
intrinsic inIntrinsicsDirectX.td
DXILOpMapping
ofint_dx_WaveActiveAllTrue
to114
inDXIL.td
WaveActiveAllTrue.ll
andWaveActiveAllTrue_errors.ll
tests inllvm/test/CodeGen/DirectX/
int_spv_WaveActiveAllTrue
intrinsic inIntrinsicsSPIRV.td
WaveActiveAllTrue
lowering and map it toint_spv_WaveActiveAllTrue
inSPIRVInstructionSelector::selectIntrinsic
.llvm/test/CodeGen/SPIRV/hlsl-intrinsics/WaveActiveAllTrue.ll