diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp index 35d17bbe2bd9a..15df6216f89a4 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp @@ -226,13 +226,6 @@ void AMDGPUAtomicOptimizerImpl::visitAtomicRMWInst(AtomicRMWInst &I) { bool ValDivergent = UA->isDivergentUse(I.getOperandUse(ValIdx)); - if ((Op == AtomicRMWInst::FAdd || Op == AtomicRMWInst::FSub) && - !I.use_empty()) { - // Disable the uniform return value calculation using fmul because it - // mishandles infinities, NaNs and signed zeros. FIXME. - ValDivergent = true; - } - // If the value operand is divergent, each lane is contributing a different // value to the atomic calculation. We can only optimize divergent values if // we have DPP available on our subtarget, and the atomic operation is 32 @@ -937,18 +930,25 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I, break; case AtomicRMWInst::FAdd: case AtomicRMWInst::FSub: { - // FIXME: This path is currently disabled in visitAtomicRMWInst because - // of problems calculating the first active lane of the result (where - // Mbcnt is 0): - // - If V is infinity or NaN we will return NaN instead of BroadcastI. - // - If BroadcastI is -0.0 and V is positive we will return +0.0 instead - // of -0.0. LaneOffset = B.CreateFMul(V, Mbcnt); break; } } } - Value *const Result = buildNonAtomicBinOp(B, Op, BroadcastI, LaneOffset); + Value *Result = buildNonAtomicBinOp(B, Op, BroadcastI, LaneOffset); + if (isAtomicFloatingPointTy) { + // For fadd/fsub the first active lane of LaneOffset should be the + // identity (-0.0 for fadd or +0.0 for fsub) but the value we calculated + // is V * +0.0 which might have the wrong sign or might be nan (if V is + // inf or nan). + // + // For all floating point ops if the in-memory value was a nan then the + // binop we just built might have quieted it or changed its payload. + // + // Correct all these problems by using BroadcastI as the result in the + // first active lane. + Result = B.CreateSelect(Cond, BroadcastI, Result); + } if (IsPixelShader) { // Need a final PHI to reconverge to above the helper lane branch mask. diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-rtn.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-rtn.ll index 504735b4985ad..2dc40c38343a9 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-rtn.ll @@ -221,7 +221,7 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace ; GFX90A-NEXT: bb.4.Flow: ; GFX90A-NEXT: successors: %bb.6(0x80000000) ; GFX90A-NEXT: {{ $}} - ; GFX90A-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI %42, %bb.5, [[DEF]], %bb.1 + ; GFX90A-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI %43, %bb.5, [[DEF]], %bb.1 ; GFX90A-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec ; GFX90A-NEXT: S_BRANCH %bb.6 ; GFX90A-NEXT: {{ $}} @@ -234,9 +234,11 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace ; GFX90A-NEXT: [[STRICT_WWM1:%[0-9]+]]:vgpr_32 = STRICT_WWM [[V_MOV_B32_dpp6]], implicit $exec ; GFX90A-NEXT: [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[V_READFIRSTLANE_B32_]] ; GFX90A-NEXT: [[V_ADD_F32_e64_6:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[COPY21]], 0, [[STRICT_WWM1]], 0, 0, implicit $mode, implicit $exec + ; GFX90A-NEXT: [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[V_READFIRSTLANE_B32_]] + ; GFX90A-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_ADD_F32_e64_6]], 0, [[COPY22]], [[V_CMP_EQ_U32_e64_]], implicit $exec ; GFX90A-NEXT: S_BRANCH %bb.4 ; GFX90A-NEXT: {{ $}} - ; GFX90A-NEXT: bb.6 (%ir-block.40): + ; GFX90A-NEXT: bb.6 (%ir-block.41): ; GFX90A-NEXT: $vgpr0 = COPY [[PHI]] ; GFX90A-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0 ; @@ -312,7 +314,7 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace ; GFX940-NEXT: bb.4.Flow: ; GFX940-NEXT: successors: %bb.6(0x80000000) ; GFX940-NEXT: {{ $}} - ; GFX940-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI %41, %bb.5, [[DEF]], %bb.1 + ; GFX940-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI %42, %bb.5, [[DEF]], %bb.1 ; GFX940-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec ; GFX940-NEXT: S_BRANCH %bb.6 ; GFX940-NEXT: {{ $}} @@ -325,9 +327,11 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace ; GFX940-NEXT: [[STRICT_WWM1:%[0-9]+]]:vgpr_32 = STRICT_WWM [[V_MOV_B32_dpp6]], implicit $exec ; GFX940-NEXT: [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[V_READFIRSTLANE_B32_]] ; GFX940-NEXT: [[V_ADD_F32_e64_6:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[COPY21]], 0, [[STRICT_WWM1]], 0, 0, implicit $mode, implicit $exec + ; GFX940-NEXT: [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[V_READFIRSTLANE_B32_]] + ; GFX940-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_ADD_F32_e64_6]], 0, [[COPY22]], [[V_CMP_EQ_U32_e64_]], implicit $exec ; GFX940-NEXT: S_BRANCH %bb.4 ; GFX940-NEXT: {{ $}} - ; GFX940-NEXT: bb.6 (%ir-block.40): + ; GFX940-NEXT: bb.6 (%ir-block.41): ; GFX940-NEXT: $vgpr0 = COPY [[PHI]] ; GFX940-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0 ; @@ -398,7 +402,7 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace ; GFX11-NEXT: bb.4.Flow: ; GFX11-NEXT: successors: %bb.6(0x80000000) ; GFX11-NEXT: {{ $}} - ; GFX11-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI %40, %bb.5, [[DEF]], %bb.1 + ; GFX11-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI %41, %bb.5, [[DEF]], %bb.1 ; GFX11-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec ; GFX11-NEXT: S_BRANCH %bb.6 ; GFX11-NEXT: {{ $}} @@ -411,9 +415,11 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace ; GFX11-NEXT: [[STRICT_WWM1:%[0-9]+]]:vgpr_32 = STRICT_WWM [[V_WRITELANE_B32_]], implicit $exec ; GFX11-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[V_READFIRSTLANE_B32_]] ; GFX11-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[COPY15]], 0, [[STRICT_WWM1]], 0, 0, implicit $mode, implicit $exec + ; GFX11-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[V_READFIRSTLANE_B32_]] + ; GFX11-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_ADD_F32_e64_5]], 0, [[COPY16]], [[V_CMP_EQ_U32_e64_]], implicit $exec ; GFX11-NEXT: S_BRANCH %bb.4 ; GFX11-NEXT: {{ $}} - ; GFX11-NEXT: bb.6 (%ir-block.37): + ; GFX11-NEXT: bb.6 (%ir-block.38): ; GFX11-NEXT: $vgpr0 = COPY [[PHI]] ; GFX11-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0 %ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll index 1b49b68fc5ba3..5e541ec6661ff 100644 --- a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll @@ -199,22 +199,23 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace ; GFX90A-NEXT: [[V_READLANE_B32_:%[0-9]+]]:sreg_32 = V_READLANE_B32 [[V_ADD_F32_e64_5]], killed [[S_MOV_B32_1]] ; GFX90A-NEXT: early-clobber %2:sgpr_32 = STRICT_WWM killed [[V_READLANE_B32_]], implicit $exec ; GFX90A-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 killed [[V_MBCNT_HI_U32_B32_e64_]], [[S_MOV_B32_]], implicit $exec + ; GFX90A-NEXT: [[COPY8:%[0-9]+]]:vreg_1 = COPY [[V_CMP_EQ_U32_e64_]] ; GFX90A-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF - ; GFX90A-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64 = SI_IF killed [[V_CMP_EQ_U32_e64_]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec + ; GFX90A-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64 = SI_IF [[V_CMP_EQ_U32_e64_]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX90A-NEXT: S_BRANCH %bb.2 ; GFX90A-NEXT: {{ $}} ; GFX90A-NEXT: bb.2 (%ir-block.32): ; GFX90A-NEXT: successors: %bb.4(0x80000000) ; GFX90A-NEXT: {{ $}} ; GFX90A-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec - ; GFX90A-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY %2 - ; GFX90A-NEXT: [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_SADDR_RTN killed [[V_MOV_B32_e32_1]], [[COPY8]], [[COPY3]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1) + ; GFX90A-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY %2 + ; GFX90A-NEXT: [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_SADDR_RTN killed [[V_MOV_B32_e32_1]], [[COPY9]], [[COPY3]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1) ; GFX90A-NEXT: S_BRANCH %bb.4 ; GFX90A-NEXT: {{ $}} ; GFX90A-NEXT: bb.3.Flow: ; GFX90A-NEXT: successors: %bb.5(0x80000000) ; GFX90A-NEXT: {{ $}} - ; GFX90A-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, %7, %bb.4 + ; GFX90A-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, %8, %bb.4 ; GFX90A-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX90A-NEXT: S_BRANCH %bb.5 ; GFX90A-NEXT: {{ $}} @@ -224,11 +225,14 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace ; GFX90A-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[DEF1]], %bb.1, [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN]], %bb.2 ; GFX90A-NEXT: SI_END_CF [[SI_IF1]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX90A-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[PHI1]], implicit $exec - ; GFX90A-NEXT: early-clobber %44:vgpr_32 = STRICT_WWM [[V_MOV_B32_dpp6]], implicit $exec - ; GFX90A-NEXT: [[V_ADD_F32_e64_6:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[V_READFIRSTLANE_B32_]], 0, killed %44, 0, 0, implicit $mode, implicit $exec + ; GFX90A-NEXT: early-clobber %45:vgpr_32 = STRICT_WWM [[V_MOV_B32_dpp6]], implicit $exec + ; GFX90A-NEXT: [[V_ADD_F32_e64_6:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_READFIRSTLANE_B32_]], 0, killed %45, 0, 0, implicit $mode, implicit $exec + ; GFX90A-NEXT: [[COPY10:%[0-9]+]]:sreg_64_xexec = COPY [[COPY8]] + ; GFX90A-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[V_READFIRSTLANE_B32_]] + ; GFX90A-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD_F32_e64_6]], 0, [[COPY11]], [[COPY10]], implicit $exec ; GFX90A-NEXT: S_BRANCH %bb.3 ; GFX90A-NEXT: {{ $}} - ; GFX90A-NEXT: bb.5 (%ir-block.40): + ; GFX90A-NEXT: bb.5 (%ir-block.41): ; GFX90A-NEXT: $vgpr0 = COPY [[PHI]] ; GFX90A-NEXT: SI_RETURN_TO_EPILOG $vgpr0 ; @@ -276,22 +280,23 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace ; GFX940-NEXT: [[V_READLANE_B32_:%[0-9]+]]:sreg_32 = V_READLANE_B32 [[V_ADD_F32_e64_5]], killed [[S_MOV_B32_1]] ; GFX940-NEXT: early-clobber %2:sgpr_32 = STRICT_WWM killed [[V_READLANE_B32_]], implicit $exec ; GFX940-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 killed [[V_MBCNT_HI_U32_B32_e64_]], [[S_MOV_B32_]], implicit $exec + ; GFX940-NEXT: [[COPY8:%[0-9]+]]:vreg_1 = COPY [[V_CMP_EQ_U32_e64_]] ; GFX940-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF - ; GFX940-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64 = SI_IF killed [[V_CMP_EQ_U32_e64_]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec + ; GFX940-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64 = SI_IF [[V_CMP_EQ_U32_e64_]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX940-NEXT: S_BRANCH %bb.2 ; GFX940-NEXT: {{ $}} ; GFX940-NEXT: bb.2 (%ir-block.32): ; GFX940-NEXT: successors: %bb.4(0x80000000) ; GFX940-NEXT: {{ $}} ; GFX940-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec - ; GFX940-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY %2 - ; GFX940-NEXT: [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_SADDR_RTN killed [[V_MOV_B32_e32_1]], [[COPY8]], [[COPY3]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1) + ; GFX940-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY %2 + ; GFX940-NEXT: [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_SADDR_RTN killed [[V_MOV_B32_e32_1]], [[COPY9]], [[COPY3]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1) ; GFX940-NEXT: S_BRANCH %bb.4 ; GFX940-NEXT: {{ $}} ; GFX940-NEXT: bb.3.Flow: ; GFX940-NEXT: successors: %bb.5(0x80000000) ; GFX940-NEXT: {{ $}} - ; GFX940-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, %7, %bb.4 + ; GFX940-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, %8, %bb.4 ; GFX940-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX940-NEXT: S_BRANCH %bb.5 ; GFX940-NEXT: {{ $}} @@ -301,11 +306,14 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace ; GFX940-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[DEF1]], %bb.1, [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN]], %bb.2 ; GFX940-NEXT: SI_END_CF [[SI_IF1]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX940-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[PHI1]], implicit $exec - ; GFX940-NEXT: early-clobber %43:vgpr_32 = STRICT_WWM [[V_MOV_B32_dpp6]], implicit $exec - ; GFX940-NEXT: [[V_ADD_F32_e64_6:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[V_READFIRSTLANE_B32_]], 0, killed %43, 0, 0, implicit $mode, implicit $exec + ; GFX940-NEXT: early-clobber %44:vgpr_32 = STRICT_WWM [[V_MOV_B32_dpp6]], implicit $exec + ; GFX940-NEXT: [[V_ADD_F32_e64_6:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_READFIRSTLANE_B32_]], 0, killed %44, 0, 0, implicit $mode, implicit $exec + ; GFX940-NEXT: [[COPY10:%[0-9]+]]:sreg_64_xexec = COPY [[COPY8]] + ; GFX940-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[V_READFIRSTLANE_B32_]] + ; GFX940-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD_F32_e64_6]], 0, [[COPY11]], [[COPY10]], implicit $exec ; GFX940-NEXT: S_BRANCH %bb.3 ; GFX940-NEXT: {{ $}} - ; GFX940-NEXT: bb.5 (%ir-block.40): + ; GFX940-NEXT: bb.5 (%ir-block.41): ; GFX940-NEXT: $vgpr0 = COPY [[PHI]] ; GFX940-NEXT: SI_RETURN_TO_EPILOG $vgpr0 ; @@ -353,22 +361,23 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace ; GFX11-NEXT: [[V_READLANE_B32_1:%[0-9]+]]:sreg_32 = V_READLANE_B32 [[V_ADD_F32_e64_4]], killed [[S_MOV_B32_4]] ; GFX11-NEXT: early-clobber %2:sgpr_32 = STRICT_WWM killed [[V_READLANE_B32_1]], implicit $exec ; GFX11-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 killed [[V_MBCNT_LO_U32_B32_e64_]], [[S_MOV_B32_]], implicit $exec + ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vreg_1 = COPY [[V_CMP_EQ_U32_e64_]] ; GFX11-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF - ; GFX11-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32 = SI_IF killed [[V_CMP_EQ_U32_e64_]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec + ; GFX11-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32 = SI_IF [[V_CMP_EQ_U32_e64_]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX11-NEXT: S_BRANCH %bb.2 ; GFX11-NEXT: {{ $}} ; GFX11-NEXT: bb.2 (%ir-block.29): ; GFX11-NEXT: successors: %bb.4(0x80000000) ; GFX11-NEXT: {{ $}} ; GFX11-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec - ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY %2 - ; GFX11-NEXT: [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_SADDR_RTN killed [[V_MOV_B32_e32_1]], [[COPY5]], [[COPY3]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1) + ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY %2 + ; GFX11-NEXT: [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_F32_SADDR_RTN killed [[V_MOV_B32_e32_1]], [[COPY6]], [[COPY3]], 0, 1, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1) ; GFX11-NEXT: S_BRANCH %bb.4 ; GFX11-NEXT: {{ $}} ; GFX11-NEXT: bb.3.Flow: ; GFX11-NEXT: successors: %bb.5(0x80000000) ; GFX11-NEXT: {{ $}} - ; GFX11-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, %7, %bb.4 + ; GFX11-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[DEF]], %bb.0, %8, %bb.4 ; GFX11-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX11-NEXT: S_BRANCH %bb.5 ; GFX11-NEXT: {{ $}} @@ -378,11 +387,13 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace ; GFX11-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[DEF1]], %bb.1, [[GLOBAL_ATOMIC_ADD_F32_SADDR_RTN]], %bb.2 ; GFX11-NEXT: SI_END_CF [[SI_IF1]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec ; GFX11-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[PHI1]], implicit $exec - ; GFX11-NEXT: early-clobber %43:vgpr_32 = STRICT_WWM [[V_WRITELANE_B32_]], implicit $exec - ; GFX11-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[V_READFIRSTLANE_B32_]], 0, killed %43, 0, 0, implicit $mode, implicit $exec + ; GFX11-NEXT: early-clobber %44:vgpr_32 = STRICT_WWM [[V_WRITELANE_B32_]], implicit $exec + ; GFX11-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_READFIRSTLANE_B32_]], 0, killed %44, 0, 0, implicit $mode, implicit $exec + ; GFX11-NEXT: [[COPY7:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[COPY5]] + ; GFX11-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD_F32_e64_5]], 0, [[V_READFIRSTLANE_B32_]], [[COPY7]], implicit $exec ; GFX11-NEXT: S_BRANCH %bb.3 ; GFX11-NEXT: {{ $}} - ; GFX11-NEXT: bb.5 (%ir-block.37): + ; GFX11-NEXT: bb.5 (%ir-block.38): ; GFX11-NEXT: $vgpr0 = COPY [[PHI]] ; GFX11-NEXT: SI_RETURN_TO_EPILOG $vgpr0 %ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll index 32c2078f08fc0..be6f8a4375163 100644 --- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll @@ -4,55 +4,44 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_wrong_subtarget(ptr addrspace(1) %ptr) #1 { ; GCN-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget: ; GCN: ; %bb.0: -; GCN-NEXT: s_mov_b64 s[2:3], exec -; GCN-NEXT: v_bfrev_b32_e32 v1, 1 -; GCN-NEXT: v_mov_b32_e32 v2, 4.0 -; GCN-NEXT: ; implicit-def: $vgpr0 -; GCN-NEXT: .LBB0_1: ; %ComputeLoop -; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 -; GCN-NEXT: s_ff1_i32_b64 s6, s[2:3] -; GCN-NEXT: s_lshl_b64 s[4:5], 1, s6 -; GCN-NEXT: v_readfirstlane_b32 s7, v1 -; GCN-NEXT: v_readlane_b32 s8, v2, s6 -; GCN-NEXT: s_mov_b32 m0, s6 -; GCN-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5] -; GCN-NEXT: v_writelane_b32 v0, s7, m0 -; GCN-NEXT: s_cmp_lg_u64 s[2:3], 0 -; GCN-NEXT: v_add_f32_e32 v1, s8, v1 -; GCN-NEXT: s_cbranch_scc1 .LBB0_1 -; GCN-NEXT: ; %bb.2: ; %ComputeEnd -; GCN-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 -; GCN-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 -; GCN-NEXT: ; implicit-def: $vgpr2 +; GCN-NEXT: s_mov_b64 s[6:7], exec +; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0 +; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0 +; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GCN-NEXT: ; implicit-def: $vgpr1 ; GCN-NEXT: s_and_saveexec_b64 s[2:3], vcc -; GCN-NEXT: s_xor_b64 s[2:3], exec, s[2:3] -; GCN-NEXT: s_cbranch_execz .LBB0_6 -; GCN-NEXT: ; %bb.3: -; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 -; GCN-NEXT: s_mov_b64 s[4:5], 0 -; GCN-NEXT: v_mov_b32_e32 v3, 0 +; GCN-NEXT: s_cbranch_execz .LBB0_4 +; GCN-NEXT: ; %bb.1: +; GCN-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GCN-NEXT: s_bcnt1_i32_b64 s1, s[6:7] +; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s1 +; GCN-NEXT: s_mov_b64 s[6:7], 0 +; GCN-NEXT: v_mul_f32_e32 v2, 4.0, v1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_load_dword s6, s[0:1], 0x0 +; GCN-NEXT: s_load_dword s0, s[4:5], 0x0 +; GCN-NEXT: v_mov_b32_e32 v3, 0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: v_mov_b32_e32 v2, s6 -; GCN-NEXT: .LBB0_4: ; %atomicrmw.start +; GCN-NEXT: v_mov_b32_e32 v1, s0 +; GCN-NEXT: .LBB0_2: ; %atomicrmw.start ; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 -; GCN-NEXT: v_mov_b32_e32 v5, v2 -; GCN-NEXT: v_add_f32_e32 v4, v5, v1 -; GCN-NEXT: global_atomic_cmpswap v2, v3, v[4:5], s[0:1] glc +; GCN-NEXT: v_mov_b32_e32 v5, v1 +; GCN-NEXT: v_add_f32_e32 v4, v5, v2 +; GCN-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[4:5] glc ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: buffer_wbinvl1 -; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v2, v5 -; GCN-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GCN-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GCN-NEXT: s_cbranch_execnz .LBB0_4 -; GCN-NEXT: ; %bb.5: ; %Flow -; GCN-NEXT: s_or_b64 exec, exec, s[4:5] -; GCN-NEXT: .LBB0_6: ; %Flow4 +; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v5 +; GCN-NEXT: s_or_b64 s[6:7], s[0:1], s[6:7] +; GCN-NEXT: s_andn2_b64 exec, exec, s[6:7] +; GCN-NEXT: s_cbranch_execnz .LBB0_2 +; GCN-NEXT: ; %bb.3: ; %Flow +; GCN-NEXT: s_or_b64 exec, exec, s[6:7] +; GCN-NEXT: .LBB0_4: ; %Flow2 ; GCN-NEXT: s_or_b64 exec, exec, s[2:3] -; GCN-NEXT: v_readfirstlane_b32 s0, v2 -; GCN-NEXT: v_add_f32_e32 v0, s0, v0 +; GCN-NEXT: v_readfirstlane_b32 s0, v1 +; GCN-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GCN-NEXT: v_mov_b32_e32 v1, s0 +; GCN-NEXT: v_mad_f32 v0, v0, 4.0, s0 +; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc ; GCN-NEXT: global_store_dword v[0:1], v0, off ; GCN-NEXT: s_endpgm %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst diff --git a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll index 18142c108ed58..dab5e991d7d43 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll @@ -7,87 +7,36 @@ ; strategies are valid for only divergent values. This optimization is valid for divergent addresses. Test also covers different scopes. define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, float inreg %val) #0 { -; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe( -; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() -; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP16:%.*]] -; IR-ITERATIVE: 2: -; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) -; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 -; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 -; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 -; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) -; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) -; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) -; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]] -; IR-ITERATIVE: 10: -; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP22:%.*]] syncscope("agent") monotonic, align 4 -; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]] -; IR-ITERATIVE: 12: -; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ] -; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) -; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = fadd float [[TMP14]], [[TMP21:%.*]] -; IR-ITERATIVE-NEXT: br label [[TMP16]] -; IR-ITERATIVE: 16: -; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP15]], [[TMP12]] ] -; IR-ITERATIVE-NEXT: ret float [[TMP17]] -; IR-ITERATIVE: ComputeLoop: -; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP21]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP25:%.*]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) -; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 -; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP19]]) -; IR-ITERATIVE-NEXT: [[TMP21]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP19]], float [[OLDVALUEPHI]]) -; IR-ITERATIVE-NEXT: [[TMP22]] = fadd float [[ACCUMULATOR]], [[TMP20]] -; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = shl i64 1, [[TMP18]] -; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = xor i64 [[TMP23]], -1 -; IR-ITERATIVE-NEXT: [[TMP25]] = and i64 [[ACTIVEBITS]], [[TMP24]] -; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP26]], label [[COMPUTEEND]], label [[COMPUTELOOP]] -; IR-ITERATIVE: ComputeEnd: -; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[TMP10]], label [[TMP12]] -; -; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe( -; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() -; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP33:%.*]] -; IR-DPP: 2: -; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) -; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 -; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 -; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 -; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) -; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) -; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) -; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) -; IR-DPP-NEXT: [[TMP11:%.*]] = fadd float [[TMP9]], [[TMP10]] -; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) -; IR-DPP-NEXT: [[TMP13:%.*]] = fadd float [[TMP11]], [[TMP12]] -; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) -; IR-DPP-NEXT: [[TMP15:%.*]] = fadd float [[TMP13]], [[TMP14]] -; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) -; IR-DPP-NEXT: [[TMP17:%.*]] = fadd float [[TMP15]], [[TMP16]] -; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) -; IR-DPP-NEXT: [[TMP19:%.*]] = fadd float [[TMP17]], [[TMP18]] -; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) -; IR-DPP-NEXT: [[TMP21:%.*]] = fadd float [[TMP19]], [[TMP20]] -; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) -; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) -; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) -; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]] -; IR-DPP: 26: -; IR-DPP-NEXT: [[TMP27:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP24]] syncscope("agent") monotonic, align 4 -; IR-DPP-NEXT: br label [[TMP28]] -; IR-DPP: 28: -; IR-DPP-NEXT: [[TMP29:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ] -; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) -; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) -; IR-DPP-NEXT: [[TMP32:%.*]] = fadd float [[TMP30]], [[TMP31]] -; IR-DPP-NEXT: br label [[TMP33]] -; IR-DPP: 33: -; IR-DPP-NEXT: [[TMP34:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP32]], [[TMP28]] ] -; IR-DPP-NEXT: ret float [[TMP34]] +; IR-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) +; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to float +; IR-NEXT: [[TMP12:%.*]] = fmul float [[VAL:%.*]], [[TMP11]] +; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR: 14: +; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4 +; IR-NEXT: br label [[TMP16]] +; IR: 16: +; IR-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) +; IR-NEXT: [[TMP19:%.*]] = uitofp i32 [[TMP8]] to float +; IR-NEXT: [[TMP20:%.*]] = fmul float [[VAL]], [[TMP19]] +; IR-NEXT: [[TMP21:%.*]] = fadd float [[TMP18]], [[TMP20]] +; IR-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]] +; IR-NEXT: br label [[TMP23]] +; IR: 23: +; IR-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-NEXT: ret float [[TMP24]] ; %result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic, align 4 ret float %result @@ -96,7 +45,7 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_agent_scope_uns define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_scope_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, float %val) #0 { ; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_div_value_scope_agent_scope_unsafe( ; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() -; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] ; IR-ITERATIVE: 2: ; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) ; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -107,37 +56,38 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_scope_agent_sco ; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) ; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]] ; IR-ITERATIVE: 10: -; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP22:%.*]] syncscope("agent") monotonic, align 4 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] syncscope("agent") monotonic, align 4 ; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]] ; IR-ITERATIVE: 12: ; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ] ; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) -; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = fadd float [[TMP14]], [[TMP21:%.*]] -; IR-ITERATIVE-NEXT: br label [[TMP16]] -; IR-ITERATIVE: 16: -; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP15]], [[TMP12]] ] -; IR-ITERATIVE-NEXT: ret float [[TMP17]] +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = fadd float [[TMP14]], [[TMP22:%.*]] +; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]] +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP16]], [[TMP12]] ] +; IR-ITERATIVE-NEXT: ret float [[TMP18]] ; IR-ITERATIVE: ComputeLoop: -; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP21]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP25:%.*]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) -; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 -; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP19]]) -; IR-ITERATIVE-NEXT: [[TMP21]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP19]], float [[OLDVALUEPHI]]) -; IR-ITERATIVE-NEXT: [[TMP22]] = fadd float [[ACCUMULATOR]], [[TMP20]] -; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = shl i64 1, [[TMP18]] -; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = xor i64 [[TMP23]], -1 -; IR-ITERATIVE-NEXT: [[TMP25]] = and i64 [[ACTIVEBITS]], [[TMP24]] -; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP26]], label [[COMPUTEEND]], label [[COMPUTELOOP]] +; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) +; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) +; IR-ITERATIVE-NEXT: [[TMP23]] = fadd float [[ACCUMULATOR]], [[TMP21]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]] +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1 +; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]] +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i64 [[TMP26]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[COMPUTEEND]], label [[COMPUTELOOP]] ; IR-ITERATIVE: ComputeEnd: -; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[TMP10]], label [[TMP12]] +; IR-ITERATIVE-NEXT: [[TMP28]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]] ; ; IR-DPP-LABEL: @global_atomic_fadd_uni_address_div_value_scope_agent_scope_unsafe( ; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() -; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP33:%.*]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]] ; IR-DPP: 2: ; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) ; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -171,10 +121,11 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_scope_agent_sco ; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) ; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) ; IR-DPP-NEXT: [[TMP32:%.*]] = fadd float [[TMP30]], [[TMP31]] -; IR-DPP-NEXT: br label [[TMP33]] -; IR-DPP: 33: -; IR-DPP-NEXT: [[TMP34:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP32]], [[TMP28]] ] -; IR-DPP-NEXT: ret float [[TMP34]] +; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]] +; IR-DPP-NEXT: br label [[TMP34]] +; IR-DPP: 34: +; IR-DPP-NEXT: [[TMP35:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP33]], [[TMP28]] ] +; IR-DPP-NEXT: ret float [[TMP35]] ; %result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic, align 4 ret float %result @@ -183,7 +134,7 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_scope_agent_sco define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #1 { ; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp( ; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7:[0-9]+]] -; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] ; IR-ITERATIVE: 2: ; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -191,40 +142,30 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_one_as_scope_un ; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 ; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] -; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]] -; IR-ITERATIVE: 10: -; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP22:%.*]] syncscope("one-as") monotonic, align 4 -; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]] -; IR-ITERATIVE: 12: -; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ] -; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP21:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("one-as") monotonic, align 4 ; IR-ITERATIVE-NEXT: br label [[TMP16]] ; IR-ITERATIVE: 16: -; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP15]], [[TMP12]] ] -; IR-ITERATIVE-NEXT: ret float [[TMP17]] -; IR-ITERATIVE: ComputeLoop: -; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP21]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP25:%.*]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 -; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP19]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP21]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP19]], float [[OLDVALUEPHI]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = shl i64 1, [[TMP18]] -; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = xor i64 [[TMP23]], -1 -; IR-ITERATIVE-NEXT: [[TMP25]] = and i64 [[ACTIVEBITS]], [[TMP24]] -; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP26]], label [[COMPUTEEND]], label [[COMPUTELOOP]] -; IR-ITERATIVE: ComputeEnd: -; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[TMP10]], label [[TMP12]] +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]] +; IR-ITERATIVE-NEXT: br label [[TMP23]] +; IR-ITERATIVE: 23: +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret float [[TMP24]] ; ; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp( ; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8:[0-9]+]] -; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP33:%.*]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] ; IR-DPP: 2: ; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -232,36 +173,26 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_one_as_scope_un ; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 ; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]] -; IR-DPP: 26: -; IR-DPP-NEXT: [[TMP27:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP24]] syncscope("one-as") monotonic, align 4 -; IR-DPP-NEXT: br label [[TMP28]] -; IR-DPP: 28: -; IR-DPP-NEXT: [[TMP29:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ] -; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: br label [[TMP33]] -; IR-DPP: 33: -; IR-DPP-NEXT: [[TMP34:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP32]], [[TMP28]] ] -; IR-DPP-NEXT: ret float [[TMP34]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("one-as") monotonic, align 4 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]] +; IR-DPP-NEXT: br label [[TMP23]] +; IR-DPP: 23: +; IR-DPP-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-DPP-NEXT: ret float [[TMP24]] ; %result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("one-as") monotonic ret float %result @@ -270,7 +201,7 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_one_as_scope_un define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float %val) #1 { ; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp( ; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] -; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] ; IR-ITERATIVE: 2: ; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -281,37 +212,38 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_un ; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] ; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]] ; IR-ITERATIVE: 10: -; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP22:%.*]] syncscope("one-as") monotonic, align 4 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] syncscope("one-as") monotonic, align 4 ; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]] ; IR-ITERATIVE: 12: ; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ] ; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP21:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] -; IR-ITERATIVE-NEXT: br label [[TMP16]] -; IR-ITERATIVE: 16: -; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP15]], [[TMP12]] ] -; IR-ITERATIVE-NEXT: ret float [[TMP17]] +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]] +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP16]], [[TMP12]] ] +; IR-ITERATIVE-NEXT: ret float [[TMP18]] ; IR-ITERATIVE: ComputeLoop: -; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP21]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP25:%.*]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 -; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP19]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP21]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP19]], float [[OLDVALUEPHI]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = shl i64 1, [[TMP18]] -; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = xor i64 [[TMP23]], -1 -; IR-ITERATIVE-NEXT: [[TMP25]] = and i64 [[ACTIVEBITS]], [[TMP24]] -; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP26]], label [[COMPUTEEND]], label [[COMPUTELOOP]] +; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]] +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1 +; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]] +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i64 [[TMP26]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[COMPUTEEND]], label [[COMPUTELOOP]] ; IR-ITERATIVE: ComputeEnd: -; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[TMP10]], label [[TMP12]] +; IR-ITERATIVE-NEXT: [[TMP28]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]] ; ; IR-DPP-LABEL: @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp( ; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] -; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP33:%.*]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]] ; IR-DPP: 2: ; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -345,10 +277,11 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_un ; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: br label [[TMP33]] -; IR-DPP: 33: -; IR-DPP-NEXT: [[TMP34:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP32]], [[TMP28]] ] -; IR-DPP-NEXT: ret float [[TMP34]] +; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]] +; IR-DPP-NEXT: br label [[TMP34]] +; IR-DPP: 34: +; IR-DPP-NEXT: [[TMP35:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP33]], [[TMP28]] ] +; IR-DPP-NEXT: ret float [[TMP35]] ; %result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("one-as") monotonic ret float %result @@ -357,7 +290,7 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_un define amdgpu_ps float @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #2 { ; IR-ITERATIVE-LABEL: @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp( ; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] -; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] ; IR-ITERATIVE: 2: ; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -365,40 +298,30 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_uni_value_agent_scope_str ; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 ; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] -; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]] -; IR-ITERATIVE: 10: -; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP22:%.*]] syncscope("agent") monotonic, align 4 -; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]] -; IR-ITERATIVE: 12: -; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ] -; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP21:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4 ; IR-ITERATIVE-NEXT: br label [[TMP16]] ; IR-ITERATIVE: 16: -; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP15]], [[TMP12]] ] -; IR-ITERATIVE-NEXT: ret float [[TMP17]] -; IR-ITERATIVE: ComputeLoop: -; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP21]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP25:%.*]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 -; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP19]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP21]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP19]], float [[OLDVALUEPHI]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = shl i64 1, [[TMP18]] -; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = xor i64 [[TMP23]], -1 -; IR-ITERATIVE-NEXT: [[TMP25]] = and i64 [[ACTIVEBITS]], [[TMP24]] -; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP26]], label [[COMPUTEEND]], label [[COMPUTELOOP]] -; IR-ITERATIVE: ComputeEnd: -; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[TMP10]], label [[TMP12]] +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]] +; IR-ITERATIVE-NEXT: br label [[TMP23]] +; IR-ITERATIVE: 23: +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret float [[TMP24]] ; ; IR-DPP-LABEL: @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp( ; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] -; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP33:%.*]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] ; IR-DPP: 2: ; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -406,36 +329,26 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_uni_value_agent_scope_str ; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 ; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]] -; IR-DPP: 26: -; IR-DPP-NEXT: [[TMP27:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP24]] syncscope("agent") monotonic, align 4 -; IR-DPP-NEXT: br label [[TMP28]] -; IR-DPP: 28: -; IR-DPP-NEXT: [[TMP29:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ] -; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: br label [[TMP33]] -; IR-DPP: 33: -; IR-DPP-NEXT: [[TMP34:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP32]], [[TMP28]] ] -; IR-DPP-NEXT: ret float [[TMP34]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]] +; IR-DPP-NEXT: br label [[TMP23]] +; IR-DPP: 23: +; IR-DPP-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-DPP-NEXT: ret float [[TMP24]] ; %result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic ret float %result @@ -444,7 +357,7 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_uni_value_agent_scope_str define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, float %val) #2 { ; IR-ITERATIVE-LABEL: @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp( ; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] -; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] ; IR-ITERATIVE: 2: ; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -455,37 +368,38 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_str ; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] ; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]] ; IR-ITERATIVE: 10: -; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP22:%.*]] syncscope("agent") monotonic, align 4 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] syncscope("agent") monotonic, align 4 ; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]] ; IR-ITERATIVE: 12: ; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ] ; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fsub.f32(float [[TMP14]], float [[TMP21:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] -; IR-ITERATIVE-NEXT: br label [[TMP16]] -; IR-ITERATIVE: 16: -; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP15]], [[TMP12]] ] -; IR-ITERATIVE-NEXT: ret float [[TMP17]] +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fsub.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]] +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP16]], [[TMP12]] ] +; IR-ITERATIVE-NEXT: ret float [[TMP18]] ; IR-ITERATIVE: ComputeLoop: -; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP21]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP25:%.*]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 -; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP19]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP21]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP19]], float [[OLDVALUEPHI]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = shl i64 1, [[TMP18]] -; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = xor i64 [[TMP23]], -1 -; IR-ITERATIVE-NEXT: [[TMP25]] = and i64 [[ACTIVEBITS]], [[TMP24]] -; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP26]], label [[COMPUTEEND]], label [[COMPUTELOOP]] +; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]] +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1 +; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]] +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i64 [[TMP26]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[COMPUTEEND]], label [[COMPUTELOOP]] ; IR-ITERATIVE: ComputeEnd: -; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[TMP10]], label [[TMP12]] +; IR-ITERATIVE-NEXT: [[TMP28]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]] ; ; IR-DPP-LABEL: @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp( ; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] -; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP33:%.*]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]] ; IR-DPP: 2: ; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -519,10 +433,11 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_str ; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fsub.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: br label [[TMP33]] -; IR-DPP: 33: -; IR-DPP-NEXT: [[TMP34:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP32]], [[TMP28]] ] -; IR-DPP-NEXT: ret float [[TMP34]] +; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]] +; IR-DPP-NEXT: br label [[TMP34]] +; IR-DPP: 34: +; IR-DPP-NEXT: [[TMP35:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP33]], [[TMP28]] ] +; IR-DPP-NEXT: ret float [[TMP35]] ; %result = atomicrmw fsub ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic ret float %result @@ -531,7 +446,7 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_str define amdgpu_ps float @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, float inreg %val) #0 { ; IR-LABEL: @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe( ; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() -; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP18:%.*]] +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]] ; IR: 2: ; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) ; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -550,10 +465,11 @@ define amdgpu_ps float @global_atomic_fmin_uni_address_uni_value_agent_scope_uns ; IR-NEXT: [[TMP15:%.*]] = uitofp i32 [[TMP8]] to float ; IR-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], float 0x7FF8000000000000, float [[VAL]] ; IR-NEXT: [[TMP17:%.*]] = call float @llvm.minnum.f32(float [[TMP14]], float [[TMP16]]) -; IR-NEXT: br label [[TMP18]] -; IR: 18: -; IR-NEXT: [[TMP19:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP17]], [[TMP12]] ] -; IR-NEXT: ret float [[TMP19]] +; IR-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], float [[TMP14]], float [[TMP17]] +; IR-NEXT: br label [[TMP19]] +; IR: 19: +; IR-NEXT: [[TMP20:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ] +; IR-NEXT: ret float [[TMP20]] ; %result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic ret float %result @@ -562,7 +478,7 @@ define amdgpu_ps float @global_atomic_fmin_uni_address_uni_value_agent_scope_uns define amdgpu_ps float @global_atomic_fmin_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, float %val) #0 { ; IR-ITERATIVE-LABEL: @global_atomic_fmin_uni_address_div_value_agent_scope_unsafe( ; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() -; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] ; IR-ITERATIVE: 2: ; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) ; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -573,37 +489,38 @@ define amdgpu_ps float @global_atomic_fmin_uni_address_div_value_agent_scope_uns ; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) ; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]] ; IR-ITERATIVE: 10: -; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP22:%.*]] syncscope("agent") monotonic, align 4 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] syncscope("agent") monotonic, align 4 ; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]] ; IR-ITERATIVE: 12: ; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ] ; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) -; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.minnum.f32(float [[TMP14]], float [[TMP21:%.*]]) -; IR-ITERATIVE-NEXT: br label [[TMP16]] -; IR-ITERATIVE: 16: -; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP15]], [[TMP12]] ] -; IR-ITERATIVE-NEXT: ret float [[TMP17]] +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.minnum.f32(float [[TMP14]], float [[TMP22:%.*]]) +; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]] +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP16]], [[TMP12]] ] +; IR-ITERATIVE-NEXT: ret float [[TMP18]] ; IR-ITERATIVE: ComputeLoop: -; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ 0x7FF8000000000000, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP21]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP25:%.*]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) -; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 -; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP19]]) -; IR-ITERATIVE-NEXT: [[TMP21]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP19]], float [[OLDVALUEPHI]]) -; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.minnum.f32(float [[ACCUMULATOR]], float [[TMP20]]) -; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = shl i64 1, [[TMP18]] -; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = xor i64 [[TMP23]], -1 -; IR-ITERATIVE-NEXT: [[TMP25]] = and i64 [[ACTIVEBITS]], [[TMP24]] -; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP26]], label [[COMPUTEEND]], label [[COMPUTELOOP]] +; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ 0x7FF8000000000000, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) +; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) +; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.minnum.f32(float [[ACCUMULATOR]], float [[TMP21]]) +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]] +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1 +; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]] +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i64 [[TMP26]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[COMPUTEEND]], label [[COMPUTELOOP]] ; IR-ITERATIVE: ComputeEnd: -; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[TMP10]], label [[TMP12]] +; IR-ITERATIVE-NEXT: [[TMP28]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]] ; ; IR-DPP-LABEL: @global_atomic_fmin_uni_address_div_value_agent_scope_unsafe( ; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() -; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP33:%.*]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]] ; IR-DPP: 2: ; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) ; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -637,10 +554,11 @@ define amdgpu_ps float @global_atomic_fmin_uni_address_div_value_agent_scope_uns ; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) ; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) ; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.minnum.f32(float [[TMP30]], float [[TMP31]]) -; IR-DPP-NEXT: br label [[TMP33]] -; IR-DPP: 33: -; IR-DPP-NEXT: [[TMP34:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP32]], [[TMP28]] ] -; IR-DPP-NEXT: ret float [[TMP34]] +; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]] +; IR-DPP-NEXT: br label [[TMP34]] +; IR-DPP: 34: +; IR-DPP-NEXT: [[TMP35:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP33]], [[TMP28]] ] +; IR-DPP-NEXT: ret float [[TMP35]] ; %result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic ret float %result @@ -649,7 +567,7 @@ define amdgpu_ps float @global_atomic_fmin_uni_address_div_value_agent_scope_uns define amdgpu_ps float @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #1{ ; IR-ITERATIVE-LABEL: @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp( ; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] -; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP18:%.*]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]] ; IR-ITERATIVE: 2: ; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -668,14 +586,15 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_uni_value_agent_scope_uns ; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], float 0x7FF8000000000000, float [[VAL]] ; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR7]] -; IR-ITERATIVE-NEXT: br label [[TMP18]] -; IR-ITERATIVE: 18: -; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP17]], [[TMP12]] ] -; IR-ITERATIVE-NEXT: ret float [[TMP19]] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], float [[TMP14]], float [[TMP17]] +; IR-ITERATIVE-NEXT: br label [[TMP19]] +; IR-ITERATIVE: 19: +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ] +; IR-ITERATIVE-NEXT: ret float [[TMP20]] ; ; IR-DPP-LABEL: @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp( ; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] -; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP18:%.*]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]] ; IR-DPP: 2: ; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -694,10 +613,11 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_uni_value_agent_scope_uns ; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] ; IR-DPP-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], float 0x7FF8000000000000, float [[VAL]] ; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: br label [[TMP18]] -; IR-DPP: 18: -; IR-DPP-NEXT: [[TMP19:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP17]], [[TMP12]] ] -; IR-DPP-NEXT: ret float [[TMP19]] +; IR-DPP-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], float [[TMP14]], float [[TMP17]] +; IR-DPP-NEXT: br label [[TMP19]] +; IR-DPP: 19: +; IR-DPP-NEXT: [[TMP20:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ] +; IR-DPP-NEXT: ret float [[TMP20]] ; %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic ret float %result @@ -706,7 +626,7 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_uni_value_agent_scope_uns define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float %val) #1{ ; IR-ITERATIVE-LABEL: @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp( ; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] -; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] ; IR-ITERATIVE: 2: ; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -717,37 +637,38 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_uns ; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] ; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]] ; IR-ITERATIVE: 10: -; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[TMP22:%.*]] syncscope("agent") monotonic, align 4 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] syncscope("agent") monotonic, align 4 ; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]] ; IR-ITERATIVE: 12: ; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ] ; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP21:%.*]], metadata !"fpexcept.strict") #[[ATTR7]] -; IR-ITERATIVE-NEXT: br label [[TMP16]] -; IR-ITERATIVE: 16: -; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP15]], [[TMP12]] ] -; IR-ITERATIVE-NEXT: ret float [[TMP17]] +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]] +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP16]], [[TMP12]] ] +; IR-ITERATIVE-NEXT: ret float [[TMP18]] ; IR-ITERATIVE: ComputeLoop: -; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ 0x7FF8000000000000, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP21]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP25:%.*]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 -; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP19]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP21]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP19]], float [[OLDVALUEPHI]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.experimental.constrained.maxnum.f32(float [[ACCUMULATOR]], float [[TMP20]], metadata !"fpexcept.strict") #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = shl i64 1, [[TMP18]] -; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = xor i64 [[TMP23]], -1 -; IR-ITERATIVE-NEXT: [[TMP25]] = and i64 [[ACTIVEBITS]], [[TMP24]] -; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP26]], label [[COMPUTEEND]], label [[COMPUTELOOP]] +; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ 0x7FF8000000000000, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.maxnum.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]] +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1 +; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]] +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i64 [[TMP26]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[COMPUTEEND]], label [[COMPUTELOOP]] ; IR-ITERATIVE: ComputeEnd: -; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[TMP10]], label [[TMP12]] +; IR-ITERATIVE-NEXT: [[TMP28]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]] ; ; IR-DPP-LABEL: @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp( ; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] -; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP33:%.*]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]] ; IR-DPP: 2: ; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -781,10 +702,11 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_uns ; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP30]], float [[TMP31]], metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: br label [[TMP33]] -; IR-DPP: 33: -; IR-DPP-NEXT: [[TMP34:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP32]], [[TMP28]] ] -; IR-DPP-NEXT: ret float [[TMP34]] +; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]] +; IR-DPP-NEXT: br label [[TMP34]] +; IR-DPP: 34: +; IR-DPP-NEXT: [[TMP35:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP33]], [[TMP28]] ] +; IR-DPP-NEXT: ret float [[TMP35]] ; %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic ret float %result @@ -793,7 +715,7 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_uns define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #2 { ; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp( ; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] -; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] ; IR-ITERATIVE: 2: ; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -801,40 +723,30 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_system_scope_st ; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 ; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] -; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]] -; IR-ITERATIVE: 10: -; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP22:%.*]] monotonic, align 4 -; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]] -; IR-ITERATIVE: 12: -; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ] -; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP21:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] monotonic, align 4 ; IR-ITERATIVE-NEXT: br label [[TMP16]] ; IR-ITERATIVE: 16: -; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP15]], [[TMP12]] ] -; IR-ITERATIVE-NEXT: ret float [[TMP17]] -; IR-ITERATIVE: ComputeLoop: -; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP21]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP25:%.*]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 -; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP19]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP21]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP19]], float [[OLDVALUEPHI]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = shl i64 1, [[TMP18]] -; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = xor i64 [[TMP23]], -1 -; IR-ITERATIVE-NEXT: [[TMP25]] = and i64 [[ACTIVEBITS]], [[TMP24]] -; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP26]], label [[COMPUTEEND]], label [[COMPUTELOOP]] -; IR-ITERATIVE: ComputeEnd: -; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[TMP10]], label [[TMP12]] +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]] +; IR-ITERATIVE-NEXT: br label [[TMP23]] +; IR-ITERATIVE: 23: +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret float [[TMP24]] ; ; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp( ; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] -; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP33:%.*]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] ; IR-DPP: 2: ; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -842,36 +754,26 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_system_scope_st ; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 ; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]] -; IR-DPP: 26: -; IR-DPP-NEXT: [[TMP27:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP24]] monotonic, align 4 -; IR-DPP-NEXT: br label [[TMP28]] -; IR-DPP: 28: -; IR-DPP-NEXT: [[TMP29:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ] -; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]] -; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: br label [[TMP33]] -; IR-DPP: 33: -; IR-DPP-NEXT: [[TMP34:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP32]], [[TMP28]] ] -; IR-DPP-NEXT: ret float [[TMP34]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] monotonic, align 4 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]] +; IR-DPP-NEXT: br label [[TMP23]] +; IR-DPP: 23: +; IR-DPP-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-DPP-NEXT: ret float [[TMP24]] ; %result = atomicrmw fadd ptr addrspace(1) %ptr, float %val monotonic, align 4 ret float %result @@ -880,7 +782,7 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_system_scope_st define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, float %val) #2 { ; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_div_value_system_scope_strictfp( ; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] -; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] ; IR-ITERATIVE: 2: ; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -891,37 +793,38 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_system_scope_st ; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] ; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]] ; IR-ITERATIVE: 10: -; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP22:%.*]] monotonic, align 4 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] monotonic, align 4 ; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]] ; IR-ITERATIVE: 12: ; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ] ; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP21:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] -; IR-ITERATIVE-NEXT: br label [[TMP16]] -; IR-ITERATIVE: 16: -; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP15]], [[TMP12]] ] -; IR-ITERATIVE-NEXT: ret float [[TMP17]] +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]] +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP16]], [[TMP12]] ] +; IR-ITERATIVE-NEXT: ret float [[TMP18]] ; IR-ITERATIVE: ComputeLoop: -; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP21]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP25:%.*]], [[COMPUTELOOP]] ] -; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 -; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP19]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP21]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP19]], float [[OLDVALUEPHI]]) #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] -; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = shl i64 1, [[TMP18]] -; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = xor i64 [[TMP23]], -1 -; IR-ITERATIVE-NEXT: [[TMP25]] = and i64 [[ACTIVEBITS]], [[TMP24]] -; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP26]], label [[COMPUTEEND]], label [[COMPUTELOOP]] +; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]] +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1 +; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]] +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i64 [[TMP26]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[COMPUTEEND]], label [[COMPUTELOOP]] ; IR-ITERATIVE: ComputeEnd: -; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = icmp eq i32 [[TMP8]], 0 -; IR-ITERATIVE-NEXT: br i1 [[TMP27]], label [[TMP10]], label [[TMP12]] +; IR-ITERATIVE-NEXT: [[TMP28]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]] ; ; IR-DPP-LABEL: @global_atomic_fadd_uni_address_div_value_system_scope_strictfp( ; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] -; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP33:%.*]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]] ; IR-DPP: 2: ; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -955,10 +858,11 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_system_scope_st ; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: br label [[TMP33]] -; IR-DPP: 33: -; IR-DPP-NEXT: [[TMP34:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP32]], [[TMP28]] ] -; IR-DPP-NEXT: ret float [[TMP34]] +; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]] +; IR-DPP-NEXT: br label [[TMP34]] +; IR-DPP: 34: +; IR-DPP-NEXT: [[TMP35:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP33]], [[TMP28]] ] +; IR-DPP-NEXT: ret float [[TMP35]] ; %result = atomicrmw fadd ptr addrspace(1) %ptr, float %val monotonic, align 4 ret float %result @@ -1074,8 +978,35 @@ define amdgpu_ps float @global_atomic_fadd_div_address_div_value_system_scope_st define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { ; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe( -; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 -; IR-NEXT: ret double [[RESULT]] +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) +; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to double +; IR-NEXT: [[TMP12:%.*]] = fmul double [[VAL:%.*]], [[TMP11]] +; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR: 14: +; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 4 +; IR-NEXT: br label [[TMP16]] +; IR: 16: +; IR-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) +; IR-NEXT: [[TMP19:%.*]] = uitofp i32 [[TMP8]] to double +; IR-NEXT: [[TMP20:%.*]] = fmul double [[VAL]], [[TMP19]] +; IR-NEXT: [[TMP21:%.*]] = fadd double [[TMP18]], [[TMP20]] +; IR-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]] +; IR-NEXT: br label [[TMP23]] +; IR: 23: +; IR-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-NEXT: ret double [[TMP24]] ; %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 ret double %result @@ -1091,9 +1022,67 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_scope_a } define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1 { -; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp( -; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 -; IR-NEXT: ret double [[RESULT]] +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]] +; IR-ITERATIVE-NEXT: br label [[TMP23]] +; IR-ITERATIVE: 23: +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP24]] +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]] +; IR-DPP-NEXT: br label [[TMP23]] +; IR-DPP: 23: +; IR-DPP-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-DPP-NEXT: ret double [[TMP24]] ; %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic ret double %result @@ -1109,9 +1098,67 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_ } define amdgpu_ps double @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { -; IR-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( -; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 -; IR-NEXT: ret double [[RESULT]] +; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]] +; IR-ITERATIVE-NEXT: br label [[TMP23]] +; IR-ITERATIVE: 23: +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP24]] +; +; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]] +; IR-DPP-NEXT: br label [[TMP23]] +; IR-DPP: 23: +; IR-DPP-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-DPP-NEXT: ret double [[TMP24]] ; %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic ret double %result @@ -1129,7 +1176,7 @@ define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_s define amdgpu_ps double @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { ; IR-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe( ; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() -; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP18:%.*]] +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]] ; IR: 2: ; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) ; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -1148,10 +1195,11 @@ define amdgpu_ps double @global_atomic_fmin_double_uni_address_uni_value_agent_s ; IR-NEXT: [[TMP15:%.*]] = uitofp i32 [[TMP8]] to double ; IR-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], double 0x7FF8000000000000, double [[VAL]] ; IR-NEXT: [[TMP17:%.*]] = call double @llvm.minnum.f64(double [[TMP14]], double [[TMP16]]) -; IR-NEXT: br label [[TMP18]] -; IR: 18: -; IR-NEXT: [[TMP19:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP17]], [[TMP12]] ] -; IR-NEXT: ret double [[TMP19]] +; IR-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], double [[TMP14]], double [[TMP17]] +; IR-NEXT: br label [[TMP19]] +; IR: 19: +; IR-NEXT: [[TMP20:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ] +; IR-NEXT: ret double [[TMP20]] ; %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic ret double %result @@ -1169,7 +1217,7 @@ define amdgpu_ps double @global_atomic_fmin_double_uni_address_div_value_agent_s define amdgpu_ps double @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1{ ; IR-ITERATIVE-LABEL: @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp( ; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] -; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP18:%.*]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]] ; IR-ITERATIVE: 2: ; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -1188,14 +1236,15 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_uni_value_agent_ ; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] ; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], double 0x7FF8000000000000, double [[VAL]] ; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR7]] -; IR-ITERATIVE-NEXT: br label [[TMP18]] -; IR-ITERATIVE: 18: -; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP17]], [[TMP12]] ] -; IR-ITERATIVE-NEXT: ret double [[TMP19]] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], double [[TMP14]], double [[TMP17]] +; IR-ITERATIVE-NEXT: br label [[TMP19]] +; IR-ITERATIVE: 19: +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP20]] ; ; IR-DPP-LABEL: @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp( ; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] -; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP18:%.*]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]] ; IR-DPP: 2: ; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] ; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 @@ -1214,10 +1263,11 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_uni_value_agent_ ; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] ; IR-DPP-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], double 0x7FF8000000000000, double [[VAL]] ; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]] -; IR-DPP-NEXT: br label [[TMP18]] -; IR-DPP: 18: -; IR-DPP-NEXT: [[TMP19:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP17]], [[TMP12]] ] -; IR-DPP-NEXT: ret double [[TMP19]] +; IR-DPP-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], double [[TMP14]], double [[TMP17]] +; IR-DPP-NEXT: br label [[TMP19]] +; IR-DPP: 19: +; IR-DPP-NEXT: [[TMP20:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ] +; IR-DPP-NEXT: ret double [[TMP20]] ; %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic ret double %result @@ -1233,9 +1283,67 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_ } define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { -; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( -; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 -; IR-NEXT: ret double [[RESULT]] +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]] +; IR-ITERATIVE-NEXT: br label [[TMP23]] +; IR-ITERATIVE: 23: +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP24]] +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]] +; IR-DPP-NEXT: br label [[TMP23]] +; IR-DPP: 23: +; IR-DPP-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ] +; IR-DPP-NEXT: ret double [[TMP24]] ; %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 ret double %result diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll index 9744bd42786ea..03ee6a325fbbc 100644 --- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll @@ -6979,95 +6979,88 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %ptrf, i32 %idx) { ; GFX12-LABEL: local_ds_fadd: ; GFX12: ; %bb.0: -; GFX12-NEXT: v_mov_b32_e32 v1, 0x42280000 -; GFX12-NEXT: s_mov_b32 s2, exec_lo -; GFX12-NEXT: s_brev_b32 s4, 1 -; GFX12-NEXT: ; implicit-def: $vgpr0 -; GFX12-NEXT: .LBB28_1: ; %ComputeLoop -; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-NEXT: s_ctz_i32_b32 s3, s2 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) -; GFX12-NEXT: v_readlane_b32 s5, v1, s3 -; GFX12-NEXT: s_lshl_b32 s6, 1, s3 -; GFX12-NEXT: v_writelane_b32 v0, s4, s3 -; GFX12-NEXT: s_and_not1_b32 s2, s2, s6 -; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX12-NEXT: s_cmp_lg_u32 s2, 0 -; GFX12-NEXT: s_add_f32 s4, s4, s5 -; GFX12-NEXT: s_cbranch_scc1 .LBB28_1 -; GFX12-NEXT: ; %bb.2: ; %ComputeEnd -; GFX12-NEXT: s_load_b64 s[2:3], s[0:1], 0x8 -; GFX12-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1) -; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 +; GFX12-NEXT: s_load_b64 s[4:5], s[0:1], 0x8 +; GFX12-NEXT: s_mov_b32 s6, exec_lo ; GFX12-NEXT: ; implicit-def: $vgpr1 +; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0 +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 ; GFX12-NEXT: s_wait_kmcnt 0x0 -; GFX12-NEXT: s_add_co_i32 s3, s3, 4 -; GFX12-NEXT: s_and_saveexec_b32 s5, vcc_lo -; GFX12-NEXT: s_xor_b32 s5, exec_lo, s5 -; GFX12-NEXT: s_cbranch_execz .LBB28_4 -; GFX12-NEXT: ; %bb.3: -; GFX12-NEXT: s_lshl_b32 s6, s3, 3 +; GFX12-NEXT: s_add_co_i32 s3, s5, 4 +; GFX12-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX12-NEXT: s_cbranch_execz .LBB28_2 +; GFX12-NEXT: ; %bb.1: +; GFX12-NEXT: s_bcnt1_i32_b32 s5, s6 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v1, s6 -; GFX12-NEXT: ds_add_rtn_f32 v1, v1, v2 +; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s5 +; GFX12-NEXT: s_lshl_b32 s5, s3, 3 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) +; GFX12-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mul_f32 v1, 0x42280000, v1 +; GFX12-NEXT: ds_add_rtn_f32 v1, v2, v1 ; GFX12-NEXT: s_wait_dscnt 0x0 ; GFX12-NEXT: global_inv scope:SCOPE_SE -; GFX12-NEXT: .LBB28_4: -; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: .LBB28_2: +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX12-NEXT: s_mov_b32 s6, exec_lo +; GFX12-NEXT: s_mov_b32 s7, exec_lo ; GFX12-NEXT: v_readfirstlane_b32 s5, v1 -; GFX12-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0 -; GFX12-NEXT: s_mov_b32 s4, exec_lo +; GFX12-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0 +; GFX12-NEXT: s_mov_b32 s6, exec_lo ; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v2 -; GFX12-NEXT: s_cbranch_execz .LBB28_6 -; GFX12-NEXT: ; %bb.5: -; GFX12-NEXT: s_bcnt1_i32_b32 s6, s6 -; GFX12-NEXT: s_lshl_b32 s3, s3, 4 -; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1 +; GFX12-NEXT: s_cbranch_execz .LBB28_4 +; GFX12-NEXT: ; %bb.3: +; GFX12-NEXT: s_bcnt1_i32_b32 s2, s7 +; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX12-NEXT: s_lshl_b32 s2, s3, 4 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) +; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mul_f32 v1, 0x42280000, v1 ; GFX12-NEXT: ds_add_f32 v2, v1 ; GFX12-NEXT: s_wait_dscnt 0x0 ; GFX12-NEXT: global_inv scope:SCOPE_SE -; GFX12-NEXT: .LBB28_6: -; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4 -; GFX12-NEXT: v_add_f32_e32 v1, s5, v0 -; GFX12-NEXT: s_mov_b32 s4, exec_lo -; GFX12-NEXT: s_brev_b32 s3, 1 +; GFX12-NEXT: .LBB28_4: +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s6 +; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX12-NEXT: s_mov_b32 s3, exec_lo +; GFX12-NEXT: s_brev_b32 s2, 1 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX12-NEXT: v_add_f32_e32 v0, s5, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e64 v1, v0, s5, vcc_lo ; GFX12-NEXT: ; implicit-def: $vgpr0 -; GFX12-NEXT: .LBB28_7: ; %ComputeLoop1 +; GFX12-NEXT: .LBB28_5: ; %ComputeLoop ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-NEXT: s_ctz_i32_b32 s5, s4 +; GFX12-NEXT: s_ctz_i32_b32 s5, s3 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) ; GFX12-NEXT: v_readlane_b32 s6, v1, s5 ; GFX12-NEXT: s_lshl_b32 s7, 1, s5 -; GFX12-NEXT: v_writelane_b32 v0, s3, s5 -; GFX12-NEXT: s_and_not1_b32 s4, s4, s7 +; GFX12-NEXT: v_writelane_b32 v0, s2, s5 +; GFX12-NEXT: s_and_not1_b32 s3, s3, s7 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX12-NEXT: s_cmp_lg_u32 s4, 0 -; GFX12-NEXT: s_add_f32 s3, s3, s6 -; GFX12-NEXT: s_cbranch_scc1 .LBB28_7 -; GFX12-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX12-NEXT: s_cmp_lg_u32 s3, 0 +; GFX12-NEXT: s_add_f32 s2, s2, s6 +; GFX12-NEXT: s_cbranch_scc1 .LBB28_5 +; GFX12-NEXT: ; %bb.6: ; %ComputeEnd ; GFX12-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 ; GFX12-NEXT: ; implicit-def: $vgpr1 -; GFX12-NEXT: s_and_saveexec_b32 s4, vcc_lo -; GFX12-NEXT: s_xor_b32 s4, exec_lo, s4 -; GFX12-NEXT: s_cbranch_execz .LBB28_10 -; GFX12-NEXT: ; %bb.9: -; GFX12-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3 +; GFX12-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX12-NEXT: s_xor_b32 s3, exec_lo, s3 +; GFX12-NEXT: s_cbranch_execz .LBB28_8 +; GFX12-NEXT: ; %bb.7: +; GFX12-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s2 ; GFX12-NEXT: ds_add_rtn_f32 v1, v1, v2 ; GFX12-NEXT: s_wait_dscnt 0x0 ; GFX12-NEXT: global_inv scope:SCOPE_SE -; GFX12-NEXT: .LBB28_10: -; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4 +; GFX12-NEXT: .LBB28_8: +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s3 ; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX12-NEXT: v_readfirstlane_b32 s2, v1 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v0 +; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo ; GFX12-NEXT: s_wait_kmcnt 0x0 ; GFX12-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX12-NEXT: s_nop 0 @@ -7076,191 +7069,174 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) ; ; GFX940-LABEL: local_ds_fadd: ; GFX940: ; %bb.0: +; GFX940-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 ; GFX940-NEXT: s_mov_b64 s[2:3], exec -; GFX940-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX940-NEXT: v_mov_b32_e32 v2, 0x42280000 -; GFX940-NEXT: ; implicit-def: $vgpr0 -; GFX940-NEXT: .LBB28_1: ; %ComputeLoop -; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX940-NEXT: s_ff1_i32_b64 s6, s[2:3] -; GFX940-NEXT: s_lshl_b64 s[4:5], 1, s6 -; GFX940-NEXT: v_readfirstlane_b32 s7, v1 -; GFX940-NEXT: v_readlane_b32 s8, v2, s6 -; GFX940-NEXT: s_mov_b32 m0, s6 -; GFX940-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5] -; GFX940-NEXT: v_writelane_b32 v0, s7, m0 -; GFX940-NEXT: s_cmp_lg_u64 s[2:3], 0 -; GFX940-NEXT: v_add_f32_e32 v1, s8, v1 -; GFX940-NEXT: s_cbranch_scc1 .LBB28_1 -; GFX940-NEXT: ; %bb.2: ; %ComputeEnd -; GFX940-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 -; GFX940-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 -; GFX940-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 -; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 -; GFX940-NEXT: ; implicit-def: $vgpr2 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: s_add_i32 s5, s5, 4 +; GFX940-NEXT: ; implicit-def: $vgpr1 +; GFX940-NEXT: s_and_saveexec_b64 s[6:7], vcc +; GFX940-NEXT: s_cbranch_execz .LBB28_2 +; GFX940-NEXT: ; %bb.1: +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: s_lshl_b32 s8, s5, 3 +; GFX940-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX940-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX940-NEXT: v_mov_b32_e32 v2, s8 +; GFX940-NEXT: ds_add_rtn_f32 v1, v2, v1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: s_add_i32 s3, s3, 4 -; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX940-NEXT: s_xor_b64 s[4:5], exec, s[4:5] +; GFX940-NEXT: .LBB28_2: +; GFX940-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX940-NEXT: s_mov_b64 s[8:9], exec +; GFX940-NEXT: v_readfirstlane_b32 s10, v1 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1 +; GFX940-NEXT: v_cmp_eq_u32_e64 s[2:3], 0, v1 +; GFX940-NEXT: s_and_saveexec_b64 s[6:7], s[2:3] ; GFX940-NEXT: s_cbranch_execz .LBB28_4 ; GFX940-NEXT: ; %bb.3: -; GFX940-NEXT: s_lshl_b32 s6, s3, 3 -; GFX940-NEXT: v_mov_b32_e32 v2, s6 -; GFX940-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: .LBB28_4: -; GFX940-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX940-NEXT: s_mov_b64 s[6:7], exec -; GFX940-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0 -; GFX940-NEXT: v_mbcnt_hi_u32_b32 v1, s7, v1 -; GFX940-NEXT: v_readfirstlane_b32 s8, v2 -; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 -; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX940-NEXT: s_cbranch_execz .LBB28_6 -; GFX940-NEXT: ; %bb.5: -; GFX940-NEXT: s_bcnt1_i32_b64 s6, s[6:7] -; GFX940-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX940-NEXT: s_lshl_b32 s3, s3, 4 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[8:9] +; GFX940-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX940-NEXT: s_lshl_b32 s2, s5, 4 ; GFX940-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 -; GFX940-NEXT: v_mov_b32_e32 v2, s3 +; GFX940-NEXT: v_mov_b32_e32 v2, s2 ; GFX940-NEXT: ds_add_f32 v2, v1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: .LBB28_6: -; GFX940-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX940-NEXT: s_mov_b64 s[4:5], exec -; GFX940-NEXT: v_add_f32_e32 v2, s8, v0 +; GFX940-NEXT: .LBB28_4: +; GFX940-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX940-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX940-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX940-NEXT: v_add_f32_e32 v0, s10, v0 +; GFX940-NEXT: v_mov_b32_e32 v1, s10 +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc ; GFX940-NEXT: v_bfrev_b32_e32 v1, 1 ; GFX940-NEXT: ; implicit-def: $vgpr0 -; GFX940-NEXT: .LBB28_7: ; %ComputeLoop1 +; GFX940-NEXT: .LBB28_5: ; %ComputeLoop ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX940-NEXT: s_ff1_i32_b64 s3, s[4:5] -; GFX940-NEXT: s_lshl_b64 s[6:7], 1, s3 +; GFX940-NEXT: s_ff1_i32_b64 s5, s[2:3] +; GFX940-NEXT: s_lshl_b64 s[6:7], 1, s5 ; GFX940-NEXT: v_readfirstlane_b32 s8, v1 -; GFX940-NEXT: v_readlane_b32 s9, v2, s3 -; GFX940-NEXT: s_mov_b32 m0, s3 -; GFX940-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7] +; GFX940-NEXT: v_readlane_b32 s9, v2, s5 +; GFX940-NEXT: s_mov_b32 m0, s5 +; GFX940-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7] ; GFX940-NEXT: v_writelane_b32 v0, s8, m0 -; GFX940-NEXT: s_cmp_lg_u64 s[4:5], 0 +; GFX940-NEXT: s_cmp_lg_u64 s[2:3], 0 ; GFX940-NEXT: v_add_f32_e32 v1, s9, v1 -; GFX940-NEXT: s_cbranch_scc1 .LBB28_7 -; GFX940-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX940-NEXT: s_cbranch_scc1 .LBB28_5 +; GFX940-NEXT: ; %bb.6: ; %ComputeEnd ; GFX940-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 ; GFX940-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 ; GFX940-NEXT: ; implicit-def: $vgpr2 -; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX940-NEXT: s_xor_b64 s[4:5], exec, s[4:5] -; GFX940-NEXT: s_cbranch_execz .LBB28_10 -; GFX940-NEXT: ; %bb.9: -; GFX940-NEXT: v_mov_b32_e32 v2, s2 +; GFX940-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX940-NEXT: s_xor_b64 s[2:3], exec, s[2:3] +; GFX940-NEXT: s_cbranch_execz .LBB28_8 +; GFX940-NEXT: ; %bb.7: +; GFX940-NEXT: v_mov_b32_e32 v2, s4 ; GFX940-NEXT: ds_add_rtn_f32 v2, v2, v1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: .LBB28_10: -; GFX940-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX940-NEXT: .LBB28_8: +; GFX940-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX940-NEXT: v_readfirstlane_b32 s2, v2 ; GFX940-NEXT: v_mov_b32_e32 v1, 0 ; GFX940-NEXT: s_nop 0 ; GFX940-NEXT: v_add_f32_e32 v0, s2, v0 +; GFX940-NEXT: v_mov_b32_e32 v2, s2 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_store_dword v1, v0, s[0:1] sc0 sc1 ; GFX940-NEXT: s_endpgm ; ; GFX11-LABEL: local_ds_fadd: ; GFX11: ; %bb.0: -; GFX11-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX11-NEXT: v_mov_b32_e32 v2, 0x42280000 -; GFX11-NEXT: s_mov_b32 s2, exec_lo -; GFX11-NEXT: ; implicit-def: $vgpr0 -; GFX11-NEXT: .LBB28_1: ; %ComputeLoop -; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x8 +; GFX11-NEXT: s_mov_b32 s6, exec_lo +; GFX11-NEXT: ; implicit-def: $vgpr1 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: s_ctz_i32_b32 s3, s2 -; GFX11-NEXT: v_readfirstlane_b32 s4, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-NEXT: v_readlane_b32 s5, v2, s3 -; GFX11-NEXT: s_lshl_b32 s6, 1, s3 -; GFX11-NEXT: s_and_not1_b32 s2, s2, s6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_writelane_b32 v0, s4, s3 -; GFX11-NEXT: v_add_f32_e32 v1, s5, v1 -; GFX11-NEXT: s_cmp_lg_u32 s2, 0 -; GFX11-NEXT: s_cbranch_scc1 .LBB28_1 -; GFX11-NEXT: ; %bb.2: ; %ComputeEnd -; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x8 -; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1) -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 -; GFX11-NEXT: ; implicit-def: $vgpr2 +; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0 +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_add_i32 s3, s3, 4 -; GFX11-NEXT: s_and_saveexec_b32 s4, vcc_lo -; GFX11-NEXT: s_xor_b32 s4, exec_lo, s4 -; GFX11-NEXT: s_cbranch_execz .LBB28_4 -; GFX11-NEXT: ; %bb.3: -; GFX11-NEXT: s_lshl_b32 s5, s3, 3 +; GFX11-NEXT: s_add_i32 s3, s5, 4 +; GFX11-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX11-NEXT: s_cbranch_execz .LBB28_2 +; GFX11-NEXT: ; %bb.1: +; GFX11-NEXT: s_bcnt1_i32_b32 s5, s6 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-NEXT: v_mov_b32_e32 v2, s5 -; GFX11-NEXT: ds_add_rtn_f32 v2, v2, v1 +; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s5 +; GFX11-NEXT: s_lshl_b32 s5, s3, 3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mul_f32 v1, 0x42280000, v1 +; GFX11-NEXT: ds_add_rtn_f32 v1, v2, v1 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: buffer_gl0_inv -; GFX11-NEXT: .LBB28_4: -; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4 +; GFX11-NEXT: .LBB28_2: +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: s_mov_b32 s7, exec_lo +; GFX11-NEXT: v_readfirstlane_b32 s5, v1 +; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0 ; GFX11-NEXT: s_mov_b32 s6, exec_lo -; GFX11-NEXT: v_readfirstlane_b32 s4, v2 -; GFX11-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0 -; GFX11-NEXT: s_mov_b32 s5, exec_lo -; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v1 -; GFX11-NEXT: s_cbranch_execz .LBB28_6 -; GFX11-NEXT: ; %bb.5: -; GFX11-NEXT: s_bcnt1_i32_b32 s6, s6 -; GFX11-NEXT: s_lshl_b32 s3, s3, 4 -; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1 +; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX11-NEXT: s_cbranch_execz .LBB28_4 +; GFX11-NEXT: ; %bb.3: +; GFX11-NEXT: s_bcnt1_i32_b32 s2, s7 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX11-NEXT: s_lshl_b32 s2, s3, 4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mul_f32 v1, 0x42280000, v1 ; GFX11-NEXT: ds_add_f32 v2, v1 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: buffer_gl0_inv -; GFX11-NEXT: .LBB28_6: -; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-NEXT: v_add_f32_e32 v2, s4, v0 +; GFX11-NEXT: .LBB28_4: +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s6 +; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 ; GFX11-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX11-NEXT: s_mov_b32 s3, exec_lo +; GFX11-NEXT: s_mov_b32 s2, exec_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX11-NEXT: v_add_f32_e32 v0, s5, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e64 v2, v0, s5, vcc_lo ; GFX11-NEXT: ; implicit-def: $vgpr0 -; GFX11-NEXT: .LBB28_7: ; %ComputeLoop1 +; GFX11-NEXT: .LBB28_5: ; %ComputeLoop ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: s_ctz_i32_b32 s4, s3 +; GFX11-NEXT: s_ctz_i32_b32 s3, s2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_readfirstlane_b32 s5, v1 -; GFX11-NEXT: v_readlane_b32 s6, v2, s4 -; GFX11-NEXT: s_lshl_b32 s7, 1, s4 +; GFX11-NEXT: v_readlane_b32 s6, v2, s3 +; GFX11-NEXT: s_lshl_b32 s7, 1, s3 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: s_and_not1_b32 s3, s3, s7 -; GFX11-NEXT: v_writelane_b32 v0, s5, s4 +; GFX11-NEXT: s_and_not1_b32 s2, s2, s7 +; GFX11-NEXT: v_writelane_b32 v0, s5, s3 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-NEXT: v_add_f32_e32 v1, s6, v1 -; GFX11-NEXT: s_cmp_lg_u32 s3, 0 -; GFX11-NEXT: s_cbranch_scc1 .LBB28_7 -; GFX11-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX11-NEXT: s_cmp_lg_u32 s2, 0 +; GFX11-NEXT: s_cbranch_scc1 .LBB28_5 +; GFX11-NEXT: ; %bb.6: ; %ComputeEnd ; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 ; GFX11-NEXT: ; implicit-def: $vgpr2 -; GFX11-NEXT: s_and_saveexec_b32 s3, vcc_lo -; GFX11-NEXT: s_xor_b32 s3, exec_lo, s3 -; GFX11-NEXT: s_cbranch_execz .LBB28_10 -; GFX11-NEXT: ; %bb.9: -; GFX11-NEXT: v_mov_b32_e32 v2, s2 +; GFX11-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX11-NEXT: s_xor_b32 s2, exec_lo, s2 +; GFX11-NEXT: s_cbranch_execz .LBB28_8 +; GFX11-NEXT: ; %bb.7: +; GFX11-NEXT: v_mov_b32_e32 v2, s4 ; GFX11-NEXT: ds_add_rtn_f32 v2, v2, v1 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: buffer_gl0_inv -; GFX11-NEXT: .LBB28_10: -; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3 +; GFX11-NEXT: .LBB28_8: +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: v_readfirstlane_b32 s2, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v0 +; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX11-NEXT: s_nop 0 @@ -7269,368 +7245,335 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) ; ; GFX10-LABEL: local_ds_fadd: ; GFX10: ; %bb.0: -; GFX10-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX10-NEXT: v_mov_b32_e32 v2, 0x42280000 -; GFX10-NEXT: s_mov_b32 s2, exec_lo -; GFX10-NEXT: ; implicit-def: $vgpr0 -; GFX10-NEXT: .LBB28_1: ; %ComputeLoop -; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: s_ff1_i32_b32 s3, s2 -; GFX10-NEXT: v_readfirstlane_b32 s4, v1 -; GFX10-NEXT: v_readlane_b32 s5, v2, s3 -; GFX10-NEXT: s_lshl_b32 s6, 1, s3 -; GFX10-NEXT: s_andn2_b32 s2, s2, s6 -; GFX10-NEXT: v_writelane_b32 v0, s4, s3 -; GFX10-NEXT: v_add_f32_e32 v1, s5, v1 -; GFX10-NEXT: s_cmp_lg_u32 s2, 0 -; GFX10-NEXT: s_cbranch_scc1 .LBB28_1 -; GFX10-NEXT: ; %bb.2: ; %ComputeEnd -; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 -; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 -; GFX10-NEXT: ; implicit-def: $vgpr2 +; GFX10-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 +; GFX10-NEXT: s_mov_b32 s6, exec_lo +; GFX10-NEXT: ; implicit-def: $vgpr1 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_add_i32 s3, s3, 4 -; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo -; GFX10-NEXT: s_xor_b32 s4, exec_lo, s4 -; GFX10-NEXT: s_cbranch_execz .LBB28_4 -; GFX10-NEXT: ; %bb.3: +; GFX10-NEXT: s_add_i32 s3, s5, 4 +; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB28_2 +; GFX10-NEXT: ; %bb.1: +; GFX10-NEXT: s_bcnt1_i32_b32 s5, s6 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s5 ; GFX10-NEXT: s_lshl_b32 s5, s3, 3 ; GFX10-NEXT: v_mov_b32_e32 v2, s5 -; GFX10-NEXT: ds_add_rtn_f32 v2, v2, v1 +; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX10-NEXT: ds_add_rtn_f32 v1, v2, v1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: buffer_gl0_inv -; GFX10-NEXT: .LBB28_4: +; GFX10-NEXT: .LBB28_2: ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4 -; GFX10-NEXT: s_mov_b32 s6, exec_lo -; GFX10-NEXT: v_readfirstlane_b32 s4, v2 -; GFX10-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0 -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 -; GFX10-NEXT: s_and_saveexec_b32 s5, vcc_lo -; GFX10-NEXT: s_cbranch_execz .LBB28_6 -; GFX10-NEXT: ; %bb.5: -; GFX10-NEXT: s_bcnt1_i32_b32 s6, s6 -; GFX10-NEXT: s_lshl_b32 s3, s3, 4 -; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX10-NEXT: v_mov_b32_e32 v2, s3 +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2 +; GFX10-NEXT: s_mov_b32 s7, exec_lo +; GFX10-NEXT: v_readfirstlane_b32 s5, v1 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0 +; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v2 +; GFX10-NEXT: s_and_saveexec_b32 s6, s2 +; GFX10-NEXT: s_cbranch_execz .LBB28_4 +; GFX10-NEXT: ; %bb.3: +; GFX10-NEXT: s_bcnt1_i32_b32 s2, s7 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX10-NEXT: s_lshl_b32 s2, s3, 4 +; GFX10-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX10-NEXT: ds_add_f32 v2, v1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: buffer_gl0_inv -; GFX10-NEXT: .LBB28_6: +; GFX10-NEXT: .LBB28_4: ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX10-NEXT: v_add_f32_e32 v2, s4, v0 +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 ; GFX10-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX10-NEXT: s_mov_b32 s3, exec_lo +; GFX10-NEXT: s_mov_b32 s2, exec_lo +; GFX10-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX10-NEXT: v_add_f32_e32 v0, s5, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v2, v0, s5, vcc_lo ; GFX10-NEXT: ; implicit-def: $vgpr0 -; GFX10-NEXT: .LBB28_7: ; %ComputeLoop1 +; GFX10-NEXT: .LBB28_5: ; %ComputeLoop ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: s_ff1_i32_b32 s4, s3 +; GFX10-NEXT: s_ff1_i32_b32 s3, s2 ; GFX10-NEXT: v_readfirstlane_b32 s5, v1 -; GFX10-NEXT: v_readlane_b32 s6, v2, s4 -; GFX10-NEXT: s_lshl_b32 s7, 1, s4 -; GFX10-NEXT: s_andn2_b32 s3, s3, s7 -; GFX10-NEXT: v_writelane_b32 v0, s5, s4 +; GFX10-NEXT: v_readlane_b32 s6, v2, s3 +; GFX10-NEXT: s_lshl_b32 s7, 1, s3 +; GFX10-NEXT: s_andn2_b32 s2, s2, s7 +; GFX10-NEXT: v_writelane_b32 v0, s5, s3 ; GFX10-NEXT: v_add_f32_e32 v1, s6, v1 -; GFX10-NEXT: s_cmp_lg_u32 s3, 0 -; GFX10-NEXT: s_cbranch_scc1 .LBB28_7 -; GFX10-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX10-NEXT: s_cmp_lg_u32 s2, 0 +; GFX10-NEXT: s_cbranch_scc1 .LBB28_5 +; GFX10-NEXT: ; %bb.6: ; %ComputeEnd ; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 ; GFX10-NEXT: ; implicit-def: $vgpr2 -; GFX10-NEXT: s_and_saveexec_b32 s3, vcc_lo -; GFX10-NEXT: s_xor_b32 s3, exec_lo, s3 -; GFX10-NEXT: s_cbranch_execz .LBB28_10 -; GFX10-NEXT: ; %bb.9: -; GFX10-NEXT: v_mov_b32_e32 v2, s2 +; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX10-NEXT: s_xor_b32 s2, exec_lo, s2 +; GFX10-NEXT: s_cbranch_execz .LBB28_8 +; GFX10-NEXT: ; %bb.7: +; GFX10-NEXT: v_mov_b32_e32 v2, s4 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX10-NEXT: ds_add_rtn_f32 v2, v2, v1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: buffer_gl0_inv -; GFX10-NEXT: .LBB28_10: +; GFX10-NEXT: .LBB28_8: ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3 +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX10-NEXT: v_readfirstlane_b32 s2, v2 ; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_add_f32_e32 v0, s2, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_store_dword v1, v0, s[0:1] ; GFX10-NEXT: s_endpgm ; ; GFX90A-LABEL: local_ds_fadd: ; GFX90A: ; %bb.0: +; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 ; GFX90A-NEXT: s_mov_b64 s[2:3], exec -; GFX90A-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX90A-NEXT: v_mov_b32_e32 v2, 0x42280000 -; GFX90A-NEXT: ; implicit-def: $vgpr0 -; GFX90A-NEXT: .LBB28_1: ; %ComputeLoop -; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: s_ff1_i32_b64 s6, s[2:3] -; GFX90A-NEXT: s_lshl_b64 s[4:5], 1, s6 -; GFX90A-NEXT: v_readfirstlane_b32 s7, v1 -; GFX90A-NEXT: v_readlane_b32 s8, v2, s6 -; GFX90A-NEXT: s_mov_b32 m0, s6 -; GFX90A-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5] -; GFX90A-NEXT: v_writelane_b32 v0, s7, m0 -; GFX90A-NEXT: s_cmp_lg_u64 s[2:3], 0 -; GFX90A-NEXT: v_add_f32_e32 v1, s8, v1 -; GFX90A-NEXT: s_cbranch_scc1 .LBB28_1 -; GFX90A-NEXT: ; %bb.2: ; %ComputeEnd -; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 -; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 -; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 -; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 -; GFX90A-NEXT: ; implicit-def: $vgpr2 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: s_add_i32 s3, s3, 4 -; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5] +; GFX90A-NEXT: s_add_i32 s5, s5, 4 +; GFX90A-NEXT: ; implicit-def: $vgpr1 +; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB28_2 +; GFX90A-NEXT: ; %bb.1: +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: s_lshl_b32 s8, s5, 3 +; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX90A-NEXT: v_mov_b32_e32 v2, s8 +; GFX90A-NEXT: ds_add_rtn_f32 v1, v2, v1 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: .LBB28_2: +; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX90A-NEXT: s_mov_b64 s[8:9], exec +; GFX90A-NEXT: v_readfirstlane_b32 s10, v1 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1 +; GFX90A-NEXT: v_cmp_eq_u32_e64 s[2:3], 0, v1 +; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], s[2:3] ; GFX90A-NEXT: s_cbranch_execz .LBB28_4 ; GFX90A-NEXT: ; %bb.3: -; GFX90A-NEXT: s_lshl_b32 s6, s3, 3 -; GFX90A-NEXT: v_mov_b32_e32 v2, s6 -; GFX90A-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: .LBB28_4: -; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX90A-NEXT: s_mov_b64 s[6:7], exec -; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0 -; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, s7, v1 -; GFX90A-NEXT: v_readfirstlane_b32 s8, v2 -; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 -; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX90A-NEXT: s_cbranch_execz .LBB28_6 -; GFX90A-NEXT: ; %bb.5: -; GFX90A-NEXT: s_bcnt1_i32_b64 s6, s[6:7] -; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX90A-NEXT: s_lshl_b32 s3, s3, 4 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[8:9] +; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX90A-NEXT: s_lshl_b32 s2, s5, 4 ; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 -; GFX90A-NEXT: v_mov_b32_e32 v2, s3 +; GFX90A-NEXT: v_mov_b32_e32 v2, s2 ; GFX90A-NEXT: ds_add_f32 v2, v1 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: .LBB28_6: -; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX90A-NEXT: s_mov_b64 s[4:5], exec -; GFX90A-NEXT: v_add_f32_e32 v2, s8, v0 +; GFX90A-NEXT: .LBB28_4: +; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX90A-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX90A-NEXT: v_add_f32_e32 v0, s10, v0 +; GFX90A-NEXT: v_mov_b32_e32 v1, s10 +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc ; GFX90A-NEXT: v_bfrev_b32_e32 v1, 1 ; GFX90A-NEXT: ; implicit-def: $vgpr0 -; GFX90A-NEXT: .LBB28_7: ; %ComputeLoop1 +; GFX90A-NEXT: .LBB28_5: ; %ComputeLoop ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: s_ff1_i32_b64 s3, s[4:5] -; GFX90A-NEXT: s_lshl_b64 s[6:7], 1, s3 +; GFX90A-NEXT: s_ff1_i32_b64 s5, s[2:3] +; GFX90A-NEXT: s_lshl_b64 s[6:7], 1, s5 ; GFX90A-NEXT: v_readfirstlane_b32 s8, v1 -; GFX90A-NEXT: v_readlane_b32 s9, v2, s3 -; GFX90A-NEXT: s_mov_b32 m0, s3 -; GFX90A-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7] +; GFX90A-NEXT: v_readlane_b32 s9, v2, s5 +; GFX90A-NEXT: s_mov_b32 m0, s5 +; GFX90A-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7] ; GFX90A-NEXT: v_writelane_b32 v0, s8, m0 -; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0 +; GFX90A-NEXT: s_cmp_lg_u64 s[2:3], 0 ; GFX90A-NEXT: v_add_f32_e32 v1, s9, v1 -; GFX90A-NEXT: s_cbranch_scc1 .LBB28_7 -; GFX90A-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX90A-NEXT: s_cbranch_scc1 .LBB28_5 +; GFX90A-NEXT: ; %bb.6: ; %ComputeEnd ; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 ; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 ; GFX90A-NEXT: ; implicit-def: $vgpr2 -; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5] -; GFX90A-NEXT: s_cbranch_execz .LBB28_10 -; GFX90A-NEXT: ; %bb.9: -; GFX90A-NEXT: v_mov_b32_e32 v2, s2 +; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX90A-NEXT: s_xor_b64 s[2:3], exec, s[2:3] +; GFX90A-NEXT: s_cbranch_execz .LBB28_8 +; GFX90A-NEXT: ; %bb.7: +; GFX90A-NEXT: v_mov_b32_e32 v2, s4 ; GFX90A-NEXT: ds_add_rtn_f32 v2, v2, v1 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: .LBB28_10: -; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX90A-NEXT: .LBB28_8: +; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX90A-NEXT: v_readfirstlane_b32 s2, v2 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0 ; GFX90A-NEXT: v_add_f32_e32 v0, s2, v0 +; GFX90A-NEXT: v_mov_b32_e32 v2, s2 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: global_store_dword v1, v0, s[0:1] ; GFX90A-NEXT: s_endpgm ; ; GFX908-LABEL: local_ds_fadd: ; GFX908: ; %bb.0: +; GFX908-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 ; GFX908-NEXT: s_mov_b64 s[2:3], exec -; GFX908-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX908-NEXT: v_mov_b32_e32 v2, 0x42280000 -; GFX908-NEXT: ; implicit-def: $vgpr0 -; GFX908-NEXT: .LBB28_1: ; %ComputeLoop -; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX908-NEXT: s_ff1_i32_b64 s6, s[2:3] -; GFX908-NEXT: s_lshl_b64 s[4:5], 1, s6 -; GFX908-NEXT: v_readfirstlane_b32 s7, v1 -; GFX908-NEXT: v_readlane_b32 s8, v2, s6 -; GFX908-NEXT: s_mov_b32 m0, s6 -; GFX908-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5] -; GFX908-NEXT: v_writelane_b32 v0, s7, m0 -; GFX908-NEXT: s_cmp_lg_u64 s[2:3], 0 -; GFX908-NEXT: v_add_f32_e32 v1, s8, v1 -; GFX908-NEXT: s_cbranch_scc1 .LBB28_1 -; GFX908-NEXT: ; %bb.2: ; %ComputeEnd -; GFX908-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 -; GFX908-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 -; GFX908-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 -; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 -; GFX908-NEXT: ; implicit-def: $vgpr2 +; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: s_add_i32 s3, s3, 4 -; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5] +; GFX908-NEXT: s_add_i32 s5, s5, 4 +; GFX908-NEXT: ; implicit-def: $vgpr1 +; GFX908-NEXT: s_and_saveexec_b64 s[6:7], vcc +; GFX908-NEXT: s_cbranch_execz .LBB28_2 +; GFX908-NEXT: ; %bb.1: +; GFX908-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX908-NEXT: s_lshl_b32 s8, s5, 3 +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX908-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-NEXT: ds_add_rtn_f32 v1, v2, v1 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: .LBB28_2: +; GFX908-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX908-NEXT: s_mov_b64 s[8:9], exec +; GFX908-NEXT: v_readfirstlane_b32 s10, v1 +; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0 +; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1 +; GFX908-NEXT: v_cmp_eq_u32_e64 s[2:3], 0, v1 +; GFX908-NEXT: s_and_saveexec_b64 s[6:7], s[2:3] ; GFX908-NEXT: s_cbranch_execz .LBB28_4 ; GFX908-NEXT: ; %bb.3: -; GFX908-NEXT: s_lshl_b32 s6, s3, 3 -; GFX908-NEXT: v_mov_b32_e32 v2, s6 -; GFX908-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: .LBB28_4: -; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX908-NEXT: s_mov_b64 s[6:7], exec -; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0 -; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, s7, v1 -; GFX908-NEXT: v_readfirstlane_b32 s8, v2 -; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 -; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX908-NEXT: s_cbranch_execz .LBB28_6 -; GFX908-NEXT: ; %bb.5: -; GFX908-NEXT: s_bcnt1_i32_b64 s6, s[6:7] -; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX908-NEXT: s_lshl_b32 s3, s3, 4 +; GFX908-NEXT: s_bcnt1_i32_b64 s2, s[8:9] +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX908-NEXT: s_lshl_b32 s2, s5, 4 ; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 -; GFX908-NEXT: v_mov_b32_e32 v2, s3 +; GFX908-NEXT: v_mov_b32_e32 v2, s2 ; GFX908-NEXT: ds_add_f32 v2, v1 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: .LBB28_6: -; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX908-NEXT: s_mov_b64 s[4:5], exec -; GFX908-NEXT: v_add_f32_e32 v2, s8, v0 +; GFX908-NEXT: .LBB28_4: +; GFX908-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX908-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX908-NEXT: v_add_f32_e32 v0, s10, v0 +; GFX908-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-NEXT: s_mov_b64 s[2:3], exec +; GFX908-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc ; GFX908-NEXT: v_bfrev_b32_e32 v1, 1 ; GFX908-NEXT: ; implicit-def: $vgpr0 -; GFX908-NEXT: .LBB28_7: ; %ComputeLoop1 +; GFX908-NEXT: .LBB28_5: ; %ComputeLoop ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX908-NEXT: s_ff1_i32_b64 s3, s[4:5] -; GFX908-NEXT: s_lshl_b64 s[6:7], 1, s3 +; GFX908-NEXT: s_ff1_i32_b64 s5, s[2:3] +; GFX908-NEXT: s_lshl_b64 s[6:7], 1, s5 ; GFX908-NEXT: v_readfirstlane_b32 s8, v1 -; GFX908-NEXT: v_readlane_b32 s9, v2, s3 -; GFX908-NEXT: s_mov_b32 m0, s3 -; GFX908-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7] +; GFX908-NEXT: v_readlane_b32 s9, v2, s5 +; GFX908-NEXT: s_mov_b32 m0, s5 +; GFX908-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7] ; GFX908-NEXT: v_writelane_b32 v0, s8, m0 -; GFX908-NEXT: s_cmp_lg_u64 s[4:5], 0 +; GFX908-NEXT: s_cmp_lg_u64 s[2:3], 0 ; GFX908-NEXT: v_add_f32_e32 v1, s9, v1 -; GFX908-NEXT: s_cbranch_scc1 .LBB28_7 -; GFX908-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX908-NEXT: s_cbranch_scc1 .LBB28_5 +; GFX908-NEXT: ; %bb.6: ; %ComputeEnd ; GFX908-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 ; GFX908-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 ; GFX908-NEXT: ; implicit-def: $vgpr2 -; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5] -; GFX908-NEXT: s_cbranch_execz .LBB28_10 -; GFX908-NEXT: ; %bb.9: -; GFX908-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX908-NEXT: s_xor_b64 s[2:3], exec, s[2:3] +; GFX908-NEXT: s_cbranch_execz .LBB28_8 +; GFX908-NEXT: ; %bb.7: +; GFX908-NEXT: v_mov_b32_e32 v2, s4 ; GFX908-NEXT: ds_add_rtn_f32 v2, v2, v1 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: .LBB28_10: -; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX908-NEXT: .LBB28_8: +; GFX908-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX908-NEXT: v_readfirstlane_b32 s2, v2 -; GFX908-NEXT: v_mov_b32_e32 v1, 0 ; GFX908-NEXT: v_add_f32_e32 v0, s2, v0 +; GFX908-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-NEXT: v_mov_b32_e32 v1, 0 +; GFX908-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GFX908-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-NEXT: global_store_dword v1, v0, s[0:1] ; GFX908-NEXT: s_endpgm ; ; GFX8-LABEL: local_ds_fadd: ; GFX8: ; %bb.0: +; GFX8-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 ; GFX8-NEXT: s_mov_b64 s[2:3], exec -; GFX8-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX8-NEXT: v_mov_b32_e32 v2, 0x42280000 -; GFX8-NEXT: ; implicit-def: $vgpr0 -; GFX8-NEXT: .LBB28_1: ; %ComputeLoop -; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX8-NEXT: s_ff1_i32_b64 s6, s[2:3] -; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s6 -; GFX8-NEXT: v_readfirstlane_b32 s7, v1 -; GFX8-NEXT: v_readlane_b32 s8, v2, s6 -; GFX8-NEXT: s_mov_b32 m0, s6 -; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5] -; GFX8-NEXT: v_writelane_b32 v0, s7, m0 -; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0 -; GFX8-NEXT: v_add_f32_e32 v1, s8, v1 -; GFX8-NEXT: s_cbranch_scc1 .LBB28_1 -; GFX8-NEXT: ; %bb.2: ; %ComputeEnd -; GFX8-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 -; GFX8-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0 -; GFX8-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 -; GFX8-NEXT: ; implicit-def: $vgpr2 +; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX8-NEXT: s_waitcnt lgkmcnt(0) +; GFX8-NEXT: s_add_i32 s5, s5, 4 +; GFX8-NEXT: ; implicit-def: $vgpr1 ; GFX8-NEXT: s_mov_b32 m0, -1 +; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc +; GFX8-NEXT: s_cbranch_execz .LBB28_2 +; GFX8-NEXT: ; %bb.1: +; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX8-NEXT: s_lshl_b32 s8, s5, 3 +; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX8-NEXT: v_mov_b32_e32 v2, s8 +; GFX8-NEXT: ds_add_rtn_f32 v1, v2, v1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_add_i32 s3, s3, 4 -; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5] +; GFX8-NEXT: .LBB28_2: +; GFX8-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX8-NEXT: s_mov_b64 s[8:9], exec +; GFX8-NEXT: v_readfirstlane_b32 s10, v1 +; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0 +; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 0, v1 +; GFX8-NEXT: s_and_saveexec_b64 s[6:7], s[2:3] ; GFX8-NEXT: s_cbranch_execz .LBB28_4 ; GFX8-NEXT: ; %bb.3: -; GFX8-NEXT: s_lshl_b32 s6, s3, 3 -; GFX8-NEXT: v_mov_b32_e32 v2, s6 -; GFX8-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: .LBB28_4: -; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX8-NEXT: s_mov_b64 s[6:7], exec -; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0 -; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, s7, v1 -; GFX8-NEXT: v_readfirstlane_b32 s8, v2 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 -; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX8-NEXT: s_cbranch_execz .LBB28_6 -; GFX8-NEXT: ; %bb.5: -; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7] -; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX8-NEXT: s_lshl_b32 s3, s3, 4 +; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[8:9] +; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX8-NEXT: s_lshl_b32 s2, s5, 4 ; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 -; GFX8-NEXT: v_mov_b32_e32 v2, s3 +; GFX8-NEXT: v_mov_b32_e32 v2, s2 ; GFX8-NEXT: ds_add_f32 v2, v1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: .LBB28_6: -; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX8-NEXT: s_mov_b64 s[4:5], exec -; GFX8-NEXT: v_add_f32_e32 v2, s8, v0 +; GFX8-NEXT: .LBB28_4: +; GFX8-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX8-NEXT: v_add_f32_e32 v0, s10, v0 +; GFX8-NEXT: v_mov_b32_e32 v1, s10 +; GFX8-NEXT: s_mov_b64 s[2:3], exec +; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc ; GFX8-NEXT: v_bfrev_b32_e32 v1, 1 ; GFX8-NEXT: ; implicit-def: $vgpr0 -; GFX8-NEXT: .LBB28_7: ; %ComputeLoop1 +; GFX8-NEXT: .LBB28_5: ; %ComputeLoop ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX8-NEXT: s_ff1_i32_b64 s3, s[4:5] -; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s3 +; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3] +; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5 ; GFX8-NEXT: v_readfirstlane_b32 s8, v1 -; GFX8-NEXT: v_readlane_b32 s9, v2, s3 -; GFX8-NEXT: s_mov_b32 m0, s3 -; GFX8-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7] +; GFX8-NEXT: v_readlane_b32 s9, v2, s5 +; GFX8-NEXT: s_mov_b32 m0, s5 +; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7] ; GFX8-NEXT: v_writelane_b32 v0, s8, m0 -; GFX8-NEXT: s_cmp_lg_u64 s[4:5], 0 +; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0 ; GFX8-NEXT: v_add_f32_e32 v1, s9, v1 -; GFX8-NEXT: s_cbranch_scc1 .LBB28_7 -; GFX8-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX8-NEXT: s_cbranch_scc1 .LBB28_5 +; GFX8-NEXT: ; %bb.6: ; %ComputeEnd ; GFX8-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 ; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 ; GFX8-NEXT: ; implicit-def: $vgpr2 -; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5] -; GFX8-NEXT: s_cbranch_execz .LBB28_10 -; GFX8-NEXT: ; %bb.9: -; GFX8-NEXT: v_mov_b32_e32 v2, s2 +; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX8-NEXT: s_xor_b64 s[2:3], exec, s[2:3] +; GFX8-NEXT: s_cbranch_execz .LBB28_8 +; GFX8-NEXT: ; %bb.7: +; GFX8-NEXT: v_mov_b32_e32 v2, s4 ; GFX8-NEXT: s_mov_b32 m0, -1 ; GFX8-NEXT: ds_add_rtn_f32 v2, v2, v1 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: .LBB28_10: -; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX8-NEXT: .LBB28_8: +; GFX8-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX8-NEXT: v_readfirstlane_b32 s2, v2 -; GFX8-NEXT: v_add_f32_e32 v2, s2, v0 +; GFX8-NEXT: v_add_f32_e32 v0, s2, v0 +; GFX8-NEXT: v_mov_b32_e32 v1, s2 +; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s1 @@ -7639,153 +7582,186 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) ; ; GFX7-LABEL: local_ds_fadd: ; GFX7: ; %bb.0: -; GFX7-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2 -; GFX7-NEXT: s_mov_b32 m0, -1 +; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2 +; GFX7-NEXT: s_mov_b64 s[2:3], exec +; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_lshl_b32 s4, s3, 3 -; GFX7-NEXT: v_mov_b32_e32 v0, s4 -; GFX7-NEXT: ds_read_b32 v0, v0 offset:32 -; GFX7-NEXT: s_add_i32 s3, s3, 4 -; GFX7-NEXT: s_lshl_b32 s6, s3, 3 -; GFX7-NEXT: s_mov_b64 s[4:5], 0 -; GFX7-NEXT: v_mov_b32_e32 v1, s6 -; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start +; GFX7-NEXT: s_add_i32 s5, s5, 4 +; GFX7-NEXT: ; implicit-def: $vgpr1 +; GFX7-NEXT: s_mov_b32 m0, -1 +; GFX7-NEXT: s_and_saveexec_b64 s[6:7], vcc +; GFX7-NEXT: s_cbranch_execz .LBB28_4 +; GFX7-NEXT: ; %bb.1: +; GFX7-NEXT: s_lshl_b32 s8, s5, 3 +; GFX7-NEXT: v_mov_b32_e32 v2, s8 +; GFX7-NEXT: ds_read_b32 v1, v2 +; GFX7-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s2 +; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3 +; GFX7-NEXT: s_mov_b64 s[8:9], 0 +; GFX7-NEXT: .LBB28_2: ; %atomicrmw.start ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_mov_b32_e32 v2, v0 -; GFX7-NEXT: v_add_f32_e32 v0, 0x42280000, v2 -; GFX7-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0 +; GFX7-NEXT: v_mov_b32_e32 v4, v1 +; GFX7-NEXT: v_add_f32_e32 v1, v4, v3 +; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2 -; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX7-NEXT: s_cbranch_execnz .LBB28_1 -; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX7-NEXT: s_mov_b64 s[6:7], exec -; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s6, 0 -; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s7, v1 -; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 -; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX7-NEXT: s_cbranch_execz .LBB28_5 -; GFX7-NEXT: ; %bb.3: -; GFX7-NEXT: s_lshl_b32 s3, s3, 4 -; GFX7-NEXT: v_mov_b32_e32 v1, s3 +; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], v1, v4 +; GFX7-NEXT: s_or_b64 s[8:9], s[2:3], s[8:9] +; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GFX7-NEXT: s_cbranch_execnz .LBB28_2 +; GFX7-NEXT: ; %bb.3: ; %Flow18 +; GFX7-NEXT: s_or_b64 exec, exec, s[8:9] +; GFX7-NEXT: .LBB28_4: ; %Flow19 +; GFX7-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX7-NEXT: s_mov_b64 s[8:9], exec +; GFX7-NEXT: v_readfirstlane_b32 s10, v1 +; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0 +; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1 +; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 0, v1 +; GFX7-NEXT: s_and_saveexec_b64 s[6:7], s[2:3] +; GFX7-NEXT: s_cbranch_execz .LBB28_7 +; GFX7-NEXT: ; %bb.5: +; GFX7-NEXT: s_lshl_b32 s2, s5, 4 +; GFX7-NEXT: v_mov_b32_e32 v1, s2 ; GFX7-NEXT: ds_read_b32 v3, v1 -; GFX7-NEXT: s_bcnt1_i32_b64 s3, s[6:7] -; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s3 +; GFX7-NEXT: s_bcnt1_i32_b64 s2, s[8:9] +; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s2 ; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2 -; GFX7-NEXT: s_mov_b64 s[6:7], 0 -; GFX7-NEXT: .LBB28_4: ; %atomicrmw.start2 +; GFX7-NEXT: s_mov_b64 s[8:9], 0 +; GFX7-NEXT: .LBB28_6: ; %atomicrmw.start2 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: v_add_f32_e32 v4, v3, v2 ; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3 -; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], v4, v3 +; GFX7-NEXT: s_or_b64 s[8:9], s[2:3], s[8:9] ; GFX7-NEXT: v_mov_b32_e32 v3, v4 -; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7] -; GFX7-NEXT: s_cbranch_execnz .LBB28_4 -; GFX7-NEXT: .LBB28_5: ; %Flow17 -; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX7-NEXT: v_mov_b32_e32 v2, s2 -; GFX7-NEXT: ds_read_b32 v1, v2 +; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GFX7-NEXT: s_cbranch_execnz .LBB28_6 +; GFX7-NEXT: .LBB28_7: ; %Flow17 +; GFX7-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX7-NEXT: v_mov_b32_e32 v1, s4 +; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v0 +; GFX7-NEXT: ds_read_b32 v0, v1 +; GFX7-NEXT: v_add_f32_e32 v2, s10, v2 +; GFX7-NEXT: v_mov_b32_e32 v3, s10 +; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ; GFX7-NEXT: s_mov_b64 s[2:3], 0 -; GFX7-NEXT: .LBB28_6: ; %atomicrmw.start8 +; GFX7-NEXT: .LBB28_8: ; %atomicrmw.start8 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_mov_b32_e32 v3, v1 -; GFX7-NEXT: v_add_f32_e32 v1, v3, v0 -; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1 +; GFX7-NEXT: v_mov_b32_e32 v3, v0 +; GFX7-NEXT: v_add_f32_e32 v0, v3, v2 +; GFX7-NEXT: ds_cmpst_rtn_b32 v0, v1, v3, v0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 +; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3 ; GFX7-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX7-NEXT: s_cbranch_execnz .LBB28_6 -; GFX7-NEXT: ; %bb.7: ; %atomicrmw.end7 +; GFX7-NEXT: s_cbranch_execnz .LBB28_8 +; GFX7-NEXT: ; %bb.9: ; %atomicrmw.end7 ; GFX7-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: buffer_store_dword v1, off, s[0:3], 0 +; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: s_endpgm ; ; GFX6-LABEL: local_ds_fadd: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2 -; GFX6-NEXT: s_mov_b32 m0, -1 +; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2 +; GFX6-NEXT: s_mov_b64 s[2:3], exec +; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_lshl_b32 s4, s3, 3 -; GFX6-NEXT: s_add_i32 s4, s4, 32 -; GFX6-NEXT: v_mov_b32_e32 v0, s4 -; GFX6-NEXT: ds_read_b32 v0, v0 -; GFX6-NEXT: s_add_i32 s3, s3, 4 -; GFX6-NEXT: s_lshl_b32 s6, s3, 3 -; GFX6-NEXT: s_mov_b64 s[4:5], 0 -; GFX6-NEXT: v_mov_b32_e32 v1, s6 -; GFX6-NEXT: .LBB28_1: ; %atomicrmw.start +; GFX6-NEXT: s_add_i32 s5, s5, 4 +; GFX6-NEXT: ; implicit-def: $vgpr1 +; GFX6-NEXT: s_mov_b32 m0, -1 +; GFX6-NEXT: s_and_saveexec_b64 s[6:7], vcc +; GFX6-NEXT: s_cbranch_execz .LBB28_4 +; GFX6-NEXT: ; %bb.1: +; GFX6-NEXT: s_lshl_b32 s8, s5, 3 +; GFX6-NEXT: v_mov_b32_e32 v2, s8 +; GFX6-NEXT: ds_read_b32 v1, v2 +; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v3, s2 +; GFX6-NEXT: v_mul_f32_e32 v3, 0x42280000, v3 +; GFX6-NEXT: s_mov_b64 s[8:9], 0 +; GFX6-NEXT: .LBB28_2: ; %atomicrmw.start ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v2, v0 -; GFX6-NEXT: v_add_f32_e32 v0, 0x42280000, v2 -; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0 +; GFX6-NEXT: v_mov_b32_e32 v4, v1 +; GFX6-NEXT: v_add_f32_e32 v1, v4, v3 +; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2 -; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX6-NEXT: s_cbranch_execnz .LBB28_1 -; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX6-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX6-NEXT: s_mov_b64 s[6:7], exec -; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s6, 0 -; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s7, v1 -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 -; GFX6-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX6-NEXT: s_cbranch_execz .LBB28_5 -; GFX6-NEXT: ; %bb.3: -; GFX6-NEXT: s_lshl_b32 s3, s3, 4 -; GFX6-NEXT: v_mov_b32_e32 v1, s3 +; GFX6-NEXT: v_cmp_eq_u32_e64 s[2:3], v1, v4 +; GFX6-NEXT: s_or_b64 s[8:9], s[2:3], s[8:9] +; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GFX6-NEXT: s_cbranch_execnz .LBB28_2 +; GFX6-NEXT: ; %bb.3: ; %Flow16 +; GFX6-NEXT: s_or_b64 exec, exec, s[8:9] +; GFX6-NEXT: .LBB28_4: ; %Flow17 +; GFX6-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX6-NEXT: s_mov_b64 s[8:9], exec +; GFX6-NEXT: v_readfirstlane_b32 s10, v1 +; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0 +; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1 +; GFX6-NEXT: v_cmp_eq_u32_e64 s[2:3], 0, v1 +; GFX6-NEXT: s_and_saveexec_b64 s[6:7], s[2:3] +; GFX6-NEXT: s_cbranch_execz .LBB28_7 +; GFX6-NEXT: ; %bb.5: +; GFX6-NEXT: s_lshl_b32 s2, s5, 4 +; GFX6-NEXT: v_mov_b32_e32 v1, s2 ; GFX6-NEXT: ds_read_b32 v3, v1 -; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[6:7] -; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s3 +; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[8:9] +; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s2 ; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2 -; GFX6-NEXT: s_mov_b64 s[6:7], 0 -; GFX6-NEXT: .LBB28_4: ; %atomicrmw.start2 +; GFX6-NEXT: s_mov_b64 s[8:9], 0 +; GFX6-NEXT: .LBB28_6: ; %atomicrmw.start2 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: v_add_f32_e32 v4, v3, v2 ; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3 -; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; GFX6-NEXT: v_cmp_eq_u32_e64 s[2:3], v4, v3 +; GFX6-NEXT: s_or_b64 s[8:9], s[2:3], s[8:9] ; GFX6-NEXT: v_mov_b32_e32 v3, v4 -; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7] -; GFX6-NEXT: s_cbranch_execnz .LBB28_4 -; GFX6-NEXT: .LBB28_5: ; %Flow15 -; GFX6-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX6-NEXT: v_mov_b32_e32 v2, s2 -; GFX6-NEXT: ds_read_b32 v1, v2 +; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GFX6-NEXT: s_cbranch_execnz .LBB28_6 +; GFX6-NEXT: .LBB28_7: ; %Flow15 +; GFX6-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX6-NEXT: v_mov_b32_e32 v1, s4 +; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v0 +; GFX6-NEXT: ds_read_b32 v0, v1 +; GFX6-NEXT: v_add_f32_e32 v2, s10, v2 +; GFX6-NEXT: v_mov_b32_e32 v3, s10 +; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ; GFX6-NEXT: s_mov_b64 s[2:3], 0 -; GFX6-NEXT: .LBB28_6: ; %atomicrmw.start8 +; GFX6-NEXT: .LBB28_8: ; %atomicrmw.start8 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v3, v1 -; GFX6-NEXT: v_add_f32_e32 v1, v3, v0 -; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1 +; GFX6-NEXT: v_mov_b32_e32 v3, v0 +; GFX6-NEXT: v_add_f32_e32 v0, v3, v2 +; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v3, v0 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 +; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3 ; GFX6-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX6-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX6-NEXT: s_cbranch_execnz .LBB28_6 -; GFX6-NEXT: ; %bb.7: ; %atomicrmw.end7 +; GFX6-NEXT: s_cbranch_execnz .LBB28_8 +; GFX6-NEXT: ; %bb.9: ; %atomicrmw.end7 ; GFX6-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 ; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: buffer_store_dword v1, off, s[0:3], 0 +; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX6-NEXT: s_endpgm %idx.add = add nuw i32 %idx, 4 %shl0 = shl i32 %idx.add, 3 @@ -7802,91 +7778,84 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspace(3) %ptrf, i32 %idx) { ; GFX12-LABEL: local_ds_fadd_one_as: ; GFX12: ; %bb.0: -; GFX12-NEXT: v_mov_b32_e32 v1, 0x42280000 -; GFX12-NEXT: s_mov_b32 s2, exec_lo -; GFX12-NEXT: s_brev_b32 s4, 1 -; GFX12-NEXT: ; implicit-def: $vgpr0 -; GFX12-NEXT: .LBB29_1: ; %ComputeLoop -; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-NEXT: s_ctz_i32_b32 s3, s2 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) -; GFX12-NEXT: v_readlane_b32 s5, v1, s3 -; GFX12-NEXT: s_lshl_b32 s6, 1, s3 -; GFX12-NEXT: v_writelane_b32 v0, s4, s3 -; GFX12-NEXT: s_and_not1_b32 s2, s2, s6 -; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX12-NEXT: s_cmp_lg_u32 s2, 0 -; GFX12-NEXT: s_add_f32 s4, s4, s5 -; GFX12-NEXT: s_cbranch_scc1 .LBB29_1 -; GFX12-NEXT: ; %bb.2: ; %ComputeEnd -; GFX12-NEXT: s_load_b64 s[2:3], s[0:1], 0x8 -; GFX12-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1) -; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 +; GFX12-NEXT: s_load_b64 s[4:5], s[0:1], 0x8 +; GFX12-NEXT: s_mov_b32 s6, exec_lo ; GFX12-NEXT: ; implicit-def: $vgpr1 +; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0 +; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 ; GFX12-NEXT: s_wait_kmcnt 0x0 -; GFX12-NEXT: s_add_co_i32 s3, s3, 4 -; GFX12-NEXT: s_and_saveexec_b32 s5, vcc_lo -; GFX12-NEXT: s_xor_b32 s5, exec_lo, s5 -; GFX12-NEXT: s_cbranch_execz .LBB29_4 -; GFX12-NEXT: ; %bb.3: -; GFX12-NEXT: s_lshl_b32 s6, s3, 3 +; GFX12-NEXT: s_add_co_i32 s3, s5, 4 +; GFX12-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX12-NEXT: s_cbranch_execz .LBB29_2 +; GFX12-NEXT: ; %bb.1: +; GFX12-NEXT: s_bcnt1_i32_b32 s5, s6 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v1, s6 -; GFX12-NEXT: ds_add_rtn_f32 v1, v1, v2 -; GFX12-NEXT: .LBB29_4: -; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5 +; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s5 +; GFX12-NEXT: s_lshl_b32 s5, s3, 3 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) +; GFX12-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mul_f32 v1, 0x42280000, v1 +; GFX12-NEXT: ds_add_rtn_f32 v1, v2, v1 +; GFX12-NEXT: .LBB29_2: +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX12-NEXT: s_mov_b32 s6, exec_lo +; GFX12-NEXT: s_mov_b32 s7, exec_lo ; GFX12-NEXT: s_wait_dscnt 0x0 ; GFX12-NEXT: v_readfirstlane_b32 s5, v1 -; GFX12-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0 -; GFX12-NEXT: s_mov_b32 s4, exec_lo +; GFX12-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0 +; GFX12-NEXT: s_mov_b32 s6, exec_lo ; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v2 -; GFX12-NEXT: s_cbranch_execz .LBB29_6 -; GFX12-NEXT: ; %bb.5: -; GFX12-NEXT: s_bcnt1_i32_b32 s6, s6 -; GFX12-NEXT: s_lshl_b32 s3, s3, 4 -; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1 +; GFX12-NEXT: s_cbranch_execz .LBB29_4 +; GFX12-NEXT: ; %bb.3: +; GFX12-NEXT: s_bcnt1_i32_b32 s2, s7 +; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX12-NEXT: s_lshl_b32 s2, s3, 4 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) +; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mul_f32 v1, 0x42280000, v1 ; GFX12-NEXT: ds_add_f32 v2, v1 -; GFX12-NEXT: .LBB29_6: -; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4 -; GFX12-NEXT: v_add_f32_e32 v1, s5, v0 -; GFX12-NEXT: s_mov_b32 s4, exec_lo -; GFX12-NEXT: s_brev_b32 s3, 1 +; GFX12-NEXT: .LBB29_4: +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s6 +; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX12-NEXT: s_mov_b32 s3, exec_lo +; GFX12-NEXT: s_brev_b32 s2, 1 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX12-NEXT: v_add_f32_e32 v0, s5, v0 +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: v_cndmask_b32_e64 v1, v0, s5, vcc_lo ; GFX12-NEXT: ; implicit-def: $vgpr0 -; GFX12-NEXT: .LBB29_7: ; %ComputeLoop1 +; GFX12-NEXT: .LBB29_5: ; %ComputeLoop ; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX12-NEXT: s_ctz_i32_b32 s5, s4 +; GFX12-NEXT: s_ctz_i32_b32 s5, s3 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) ; GFX12-NEXT: v_readlane_b32 s6, v1, s5 ; GFX12-NEXT: s_lshl_b32 s7, 1, s5 -; GFX12-NEXT: v_writelane_b32 v0, s3, s5 -; GFX12-NEXT: s_and_not1_b32 s4, s4, s7 +; GFX12-NEXT: v_writelane_b32 v0, s2, s5 +; GFX12-NEXT: s_and_not1_b32 s3, s3, s7 ; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX12-NEXT: s_cmp_lg_u32 s4, 0 -; GFX12-NEXT: s_add_f32 s3, s3, s6 -; GFX12-NEXT: s_cbranch_scc1 .LBB29_7 -; GFX12-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX12-NEXT: s_cmp_lg_u32 s3, 0 +; GFX12-NEXT: s_add_f32 s2, s2, s6 +; GFX12-NEXT: s_cbranch_scc1 .LBB29_5 +; GFX12-NEXT: ; %bb.6: ; %ComputeEnd ; GFX12-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0 ; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) ; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 ; GFX12-NEXT: ; implicit-def: $vgpr1 -; GFX12-NEXT: s_and_saveexec_b32 s4, vcc_lo -; GFX12-NEXT: s_xor_b32 s4, exec_lo, s4 -; GFX12-NEXT: s_cbranch_execz .LBB29_10 -; GFX12-NEXT: ; %bb.9: -; GFX12-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3 +; GFX12-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX12-NEXT: s_xor_b32 s3, exec_lo, s3 +; GFX12-NEXT: s_cbranch_execz .LBB29_8 +; GFX12-NEXT: ; %bb.7: +; GFX12-NEXT: v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s2 ; GFX12-NEXT: ds_add_rtn_f32 v1, v1, v2 -; GFX12-NEXT: .LBB29_10: -; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4 +; GFX12-NEXT: .LBB29_8: +; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s3 ; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX12-NEXT: s_wait_dscnt 0x0 ; GFX12-NEXT: v_readfirstlane_b32 s2, v1 -; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v0 +; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo ; GFX12-NEXT: s_wait_kmcnt 0x0 ; GFX12-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX12-NEXT: s_nop 0 @@ -7895,185 +7864,168 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs ; ; GFX940-LABEL: local_ds_fadd_one_as: ; GFX940: ; %bb.0: +; GFX940-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 ; GFX940-NEXT: s_mov_b64 s[2:3], exec -; GFX940-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX940-NEXT: v_mov_b32_e32 v2, 0x42280000 -; GFX940-NEXT: ; implicit-def: $vgpr0 -; GFX940-NEXT: .LBB29_1: ; %ComputeLoop -; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX940-NEXT: s_ff1_i32_b64 s6, s[2:3] -; GFX940-NEXT: s_lshl_b64 s[4:5], 1, s6 -; GFX940-NEXT: v_readfirstlane_b32 s7, v1 -; GFX940-NEXT: v_readlane_b32 s8, v2, s6 -; GFX940-NEXT: s_mov_b32 m0, s6 -; GFX940-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5] -; GFX940-NEXT: v_writelane_b32 v0, s7, m0 -; GFX940-NEXT: s_cmp_lg_u64 s[2:3], 0 -; GFX940-NEXT: v_add_f32_e32 v1, s8, v1 -; GFX940-NEXT: s_cbranch_scc1 .LBB29_1 -; GFX940-NEXT: ; %bb.2: ; %ComputeEnd -; GFX940-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 -; GFX940-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 -; GFX940-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 -; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 -; GFX940-NEXT: ; implicit-def: $vgpr2 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: s_add_i32 s3, s3, 4 -; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX940-NEXT: s_xor_b64 s[4:5], exec, s[4:5] +; GFX940-NEXT: s_add_i32 s5, s5, 4 +; GFX940-NEXT: ; implicit-def: $vgpr1 +; GFX940-NEXT: s_and_saveexec_b64 s[6:7], vcc +; GFX940-NEXT: s_cbranch_execz .LBB29_2 +; GFX940-NEXT: ; %bb.1: +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: s_lshl_b32 s8, s5, 3 +; GFX940-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX940-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX940-NEXT: v_mov_b32_e32 v2, s8 +; GFX940-NEXT: ds_add_rtn_f32 v1, v2, v1 +; GFX940-NEXT: .LBB29_2: +; GFX940-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX940-NEXT: s_mov_b64 s[8:9], exec +; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: v_readfirstlane_b32 s10, v1 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1 +; GFX940-NEXT: v_cmp_eq_u32_e64 s[2:3], 0, v1 +; GFX940-NEXT: s_and_saveexec_b64 s[6:7], s[2:3] ; GFX940-NEXT: s_cbranch_execz .LBB29_4 ; GFX940-NEXT: ; %bb.3: -; GFX940-NEXT: s_lshl_b32 s6, s3, 3 -; GFX940-NEXT: v_mov_b32_e32 v2, s6 -; GFX940-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX940-NEXT: .LBB29_4: -; GFX940-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX940-NEXT: s_mov_b64 s[6:7], exec -; GFX940-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0 -; GFX940-NEXT: v_mbcnt_hi_u32_b32 v1, s7, v1 -; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_readfirstlane_b32 s8, v2 -; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 -; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX940-NEXT: s_cbranch_execz .LBB29_6 -; GFX940-NEXT: ; %bb.5: -; GFX940-NEXT: s_bcnt1_i32_b64 s6, s[6:7] -; GFX940-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX940-NEXT: s_lshl_b32 s3, s3, 4 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[8:9] +; GFX940-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX940-NEXT: s_lshl_b32 s2, s5, 4 ; GFX940-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 -; GFX940-NEXT: v_mov_b32_e32 v2, s3 +; GFX940-NEXT: v_mov_b32_e32 v2, s2 ; GFX940-NEXT: ds_add_f32 v2, v1 -; GFX940-NEXT: .LBB29_6: -; GFX940-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX940-NEXT: s_mov_b64 s[4:5], exec -; GFX940-NEXT: v_add_f32_e32 v2, s8, v0 +; GFX940-NEXT: .LBB29_4: +; GFX940-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX940-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX940-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX940-NEXT: v_add_f32_e32 v0, s10, v0 +; GFX940-NEXT: v_mov_b32_e32 v1, s10 +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc ; GFX940-NEXT: v_bfrev_b32_e32 v1, 1 ; GFX940-NEXT: ; implicit-def: $vgpr0 -; GFX940-NEXT: .LBB29_7: ; %ComputeLoop1 +; GFX940-NEXT: .LBB29_5: ; %ComputeLoop ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX940-NEXT: s_ff1_i32_b64 s3, s[4:5] -; GFX940-NEXT: s_lshl_b64 s[6:7], 1, s3 +; GFX940-NEXT: s_ff1_i32_b64 s5, s[2:3] +; GFX940-NEXT: s_lshl_b64 s[6:7], 1, s5 ; GFX940-NEXT: v_readfirstlane_b32 s8, v1 -; GFX940-NEXT: v_readlane_b32 s9, v2, s3 -; GFX940-NEXT: s_mov_b32 m0, s3 -; GFX940-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7] +; GFX940-NEXT: v_readlane_b32 s9, v2, s5 +; GFX940-NEXT: s_mov_b32 m0, s5 +; GFX940-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7] ; GFX940-NEXT: v_writelane_b32 v0, s8, m0 -; GFX940-NEXT: s_cmp_lg_u64 s[4:5], 0 +; GFX940-NEXT: s_cmp_lg_u64 s[2:3], 0 ; GFX940-NEXT: v_add_f32_e32 v1, s9, v1 -; GFX940-NEXT: s_cbranch_scc1 .LBB29_7 -; GFX940-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX940-NEXT: s_cbranch_scc1 .LBB29_5 +; GFX940-NEXT: ; %bb.6: ; %ComputeEnd ; GFX940-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 ; GFX940-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 ; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 ; GFX940-NEXT: ; implicit-def: $vgpr2 -; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX940-NEXT: s_xor_b64 s[4:5], exec, s[4:5] -; GFX940-NEXT: s_cbranch_execz .LBB29_10 -; GFX940-NEXT: ; %bb.9: -; GFX940-NEXT: v_mov_b32_e32 v2, s2 +; GFX940-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX940-NEXT: s_xor_b64 s[2:3], exec, s[2:3] +; GFX940-NEXT: s_cbranch_execz .LBB29_8 +; GFX940-NEXT: ; %bb.7: +; GFX940-NEXT: v_mov_b32_e32 v2, s4 ; GFX940-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX940-NEXT: .LBB29_10: -; GFX940-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX940-NEXT: .LBB29_8: +; GFX940-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: v_readfirstlane_b32 s2, v2 ; GFX940-NEXT: v_mov_b32_e32 v1, 0 ; GFX940-NEXT: s_nop 0 ; GFX940-NEXT: v_add_f32_e32 v0, s2, v0 +; GFX940-NEXT: v_mov_b32_e32 v2, s2 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GFX940-NEXT: global_store_dword v1, v0, s[0:1] sc0 sc1 ; GFX940-NEXT: s_endpgm ; ; GFX11-LABEL: local_ds_fadd_one_as: ; GFX11: ; %bb.0: -; GFX11-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX11-NEXT: v_mov_b32_e32 v2, 0x42280000 -; GFX11-NEXT: s_mov_b32 s2, exec_lo -; GFX11-NEXT: ; implicit-def: $vgpr0 -; GFX11-NEXT: .LBB29_1: ; %ComputeLoop -; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x8 +; GFX11-NEXT: s_mov_b32 s6, exec_lo +; GFX11-NEXT: ; implicit-def: $vgpr1 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: s_ctz_i32_b32 s3, s2 -; GFX11-NEXT: v_readfirstlane_b32 s4, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) -; GFX11-NEXT: v_readlane_b32 s5, v2, s3 -; GFX11-NEXT: s_lshl_b32 s6, 1, s3 -; GFX11-NEXT: s_and_not1_b32 s2, s2, s6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_writelane_b32 v0, s4, s3 -; GFX11-NEXT: v_add_f32_e32 v1, s5, v1 -; GFX11-NEXT: s_cmp_lg_u32 s2, 0 -; GFX11-NEXT: s_cbranch_scc1 .LBB29_1 -; GFX11-NEXT: ; %bb.2: ; %ComputeEnd -; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x8 -; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1) -; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 -; GFX11-NEXT: ; implicit-def: $vgpr2 +; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0 +; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_add_i32 s3, s3, 4 -; GFX11-NEXT: s_and_saveexec_b32 s4, vcc_lo -; GFX11-NEXT: s_xor_b32 s4, exec_lo, s4 -; GFX11-NEXT: s_cbranch_execz .LBB29_4 -; GFX11-NEXT: ; %bb.3: -; GFX11-NEXT: s_lshl_b32 s5, s3, 3 +; GFX11-NEXT: s_add_i32 s3, s5, 4 +; GFX11-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX11-NEXT: s_cbranch_execz .LBB29_2 +; GFX11-NEXT: ; %bb.1: +; GFX11-NEXT: s_bcnt1_i32_b32 s5, s6 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-NEXT: v_mov_b32_e32 v2, s5 -; GFX11-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX11-NEXT: .LBB29_4: -; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4 +; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s5 +; GFX11-NEXT: s_lshl_b32 s5, s3, 3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mul_f32 v1, 0x42280000, v1 +; GFX11-NEXT: ds_add_rtn_f32 v1, v2, v1 +; GFX11-NEXT: .LBB29_2: +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1) -; GFX11-NEXT: s_mov_b32 s6, exec_lo +; GFX11-NEXT: s_mov_b32 s7, exec_lo ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_readfirstlane_b32 s4, v2 -; GFX11-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0 -; GFX11-NEXT: s_mov_b32 s5, exec_lo -; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v1 -; GFX11-NEXT: s_cbranch_execz .LBB29_6 -; GFX11-NEXT: ; %bb.5: -; GFX11-NEXT: s_bcnt1_i32_b32 s6, s6 -; GFX11-NEXT: s_lshl_b32 s3, s3, 4 -; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1 +; GFX11-NEXT: v_readfirstlane_b32 s5, v1 +; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0 +; GFX11-NEXT: s_mov_b32 s6, exec_lo +; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX11-NEXT: s_cbranch_execz .LBB29_4 +; GFX11-NEXT: ; %bb.3: +; GFX11-NEXT: s_bcnt1_i32_b32 s2, s7 +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX11-NEXT: s_lshl_b32 s2, s3, 4 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mul_f32 v1, 0x42280000, v1 ; GFX11-NEXT: ds_add_f32 v2, v1 -; GFX11-NEXT: .LBB29_6: -; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX11-NEXT: v_add_f32_e32 v2, s4, v0 +; GFX11-NEXT: .LBB29_4: +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s6 +; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 ; GFX11-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX11-NEXT: s_mov_b32 s3, exec_lo +; GFX11-NEXT: s_mov_b32 s2, exec_lo +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX11-NEXT: v_add_f32_e32 v0, s5, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_cndmask_b32_e64 v2, v0, s5, vcc_lo ; GFX11-NEXT: ; implicit-def: $vgpr0 -; GFX11-NEXT: .LBB29_7: ; %ComputeLoop1 +; GFX11-NEXT: .LBB29_5: ; %ComputeLoop ; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: s_ctz_i32_b32 s4, s3 +; GFX11-NEXT: s_ctz_i32_b32 s3, s2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_readfirstlane_b32 s5, v1 -; GFX11-NEXT: v_readlane_b32 s6, v2, s4 -; GFX11-NEXT: s_lshl_b32 s7, 1, s4 +; GFX11-NEXT: v_readlane_b32 s6, v2, s3 +; GFX11-NEXT: s_lshl_b32 s7, 1, s3 ; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-NEXT: s_and_not1_b32 s3, s3, s7 -; GFX11-NEXT: v_writelane_b32 v0, s5, s4 +; GFX11-NEXT: s_and_not1_b32 s2, s2, s7 +; GFX11-NEXT: v_writelane_b32 v0, s5, s3 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX11-NEXT: v_add_f32_e32 v1, s6, v1 -; GFX11-NEXT: s_cmp_lg_u32 s3, 0 -; GFX11-NEXT: s_cbranch_scc1 .LBB29_7 -; GFX11-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX11-NEXT: s_cmp_lg_u32 s2, 0 +; GFX11-NEXT: s_cbranch_scc1 .LBB29_5 +; GFX11-NEXT: ; %bb.6: ; %ComputeEnd ; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) ; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 ; GFX11-NEXT: ; implicit-def: $vgpr2 -; GFX11-NEXT: s_and_saveexec_b32 s3, vcc_lo -; GFX11-NEXT: s_xor_b32 s3, exec_lo, s3 -; GFX11-NEXT: s_cbranch_execz .LBB29_10 -; GFX11-NEXT: ; %bb.9: -; GFX11-NEXT: v_mov_b32_e32 v2, s2 +; GFX11-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX11-NEXT: s_xor_b32 s2, exec_lo, s2 +; GFX11-NEXT: s_cbranch_execz .LBB29_8 +; GFX11-NEXT: ; %bb.7: +; GFX11-NEXT: v_mov_b32_e32 v2, s4 ; GFX11-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX11-NEXT: .LBB29_10: -; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3 +; GFX11-NEXT: .LBB29_8: +; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) ; GFX11-NEXT: v_readfirstlane_b32 s2, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v0 +; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo ; GFX11-NEXT: global_store_b32 v1, v0, s[0:1] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -8081,357 +8033,324 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs ; ; GFX10-LABEL: local_ds_fadd_one_as: ; GFX10: ; %bb.0: -; GFX10-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX10-NEXT: v_mov_b32_e32 v2, 0x42280000 -; GFX10-NEXT: s_mov_b32 s2, exec_lo -; GFX10-NEXT: ; implicit-def: $vgpr0 -; GFX10-NEXT: .LBB29_1: ; %ComputeLoop -; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: s_ff1_i32_b32 s3, s2 -; GFX10-NEXT: v_readfirstlane_b32 s4, v1 -; GFX10-NEXT: v_readlane_b32 s5, v2, s3 -; GFX10-NEXT: s_lshl_b32 s6, 1, s3 -; GFX10-NEXT: s_andn2_b32 s2, s2, s6 -; GFX10-NEXT: v_writelane_b32 v0, s4, s3 -; GFX10-NEXT: v_add_f32_e32 v1, s5, v1 -; GFX10-NEXT: s_cmp_lg_u32 s2, 0 -; GFX10-NEXT: s_cbranch_scc1 .LBB29_1 -; GFX10-NEXT: ; %bb.2: ; %ComputeEnd -; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 -; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 -; GFX10-NEXT: ; implicit-def: $vgpr2 +; GFX10-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 +; GFX10-NEXT: s_mov_b32 s6, exec_lo +; GFX10-NEXT: ; implicit-def: $vgpr1 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0 +; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_add_i32 s3, s3, 4 -; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo -; GFX10-NEXT: s_xor_b32 s4, exec_lo, s4 -; GFX10-NEXT: s_cbranch_execz .LBB29_4 -; GFX10-NEXT: ; %bb.3: +; GFX10-NEXT: s_add_i32 s3, s5, 4 +; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX10-NEXT: s_cbranch_execz .LBB29_2 +; GFX10-NEXT: ; %bb.1: +; GFX10-NEXT: s_bcnt1_i32_b32 s5, s6 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s5 ; GFX10-NEXT: s_lshl_b32 s5, s3, 3 ; GFX10-NEXT: v_mov_b32_e32 v2, s5 -; GFX10-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX10-NEXT: .LBB29_4: +; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX10-NEXT: ds_add_rtn_f32 v1, v2, v1 +; GFX10-NEXT: .LBB29_2: ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4 -; GFX10-NEXT: s_mov_b32 s6, exec_lo +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2 +; GFX10-NEXT: s_mov_b32 s7, exec_lo ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_readfirstlane_b32 s4, v2 -; GFX10-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0 -; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 -; GFX10-NEXT: s_and_saveexec_b32 s5, vcc_lo -; GFX10-NEXT: s_cbranch_execz .LBB29_6 -; GFX10-NEXT: ; %bb.5: -; GFX10-NEXT: s_bcnt1_i32_b32 s6, s6 -; GFX10-NEXT: s_lshl_b32 s3, s3, 4 -; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX10-NEXT: v_mov_b32_e32 v2, s3 +; GFX10-NEXT: v_readfirstlane_b32 s5, v1 +; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0 +; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v2 +; GFX10-NEXT: s_and_saveexec_b32 s6, s2 +; GFX10-NEXT: s_cbranch_execz .LBB29_4 +; GFX10-NEXT: ; %bb.3: +; GFX10-NEXT: s_bcnt1_i32_b32 s2, s7 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX10-NEXT: s_lshl_b32 s2, s3, 4 +; GFX10-NEXT: v_mov_b32_e32 v2, s2 ; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 ; GFX10-NEXT: ds_add_f32 v2, v1 -; GFX10-NEXT: .LBB29_6: +; GFX10-NEXT: .LBB29_4: ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5 -; GFX10-NEXT: v_add_f32_e32 v2, s4, v0 +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6 +; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 ; GFX10-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX10-NEXT: s_mov_b32 s3, exec_lo +; GFX10-NEXT: s_mov_b32 s2, exec_lo +; GFX10-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX10-NEXT: v_add_f32_e32 v0, s5, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v2, v0, s5, vcc_lo ; GFX10-NEXT: ; implicit-def: $vgpr0 -; GFX10-NEXT: .LBB29_7: ; %ComputeLoop1 +; GFX10-NEXT: .LBB29_5: ; %ComputeLoop ; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX10-NEXT: s_ff1_i32_b32 s4, s3 +; GFX10-NEXT: s_ff1_i32_b32 s3, s2 ; GFX10-NEXT: v_readfirstlane_b32 s5, v1 -; GFX10-NEXT: v_readlane_b32 s6, v2, s4 -; GFX10-NEXT: s_lshl_b32 s7, 1, s4 -; GFX10-NEXT: s_andn2_b32 s3, s3, s7 -; GFX10-NEXT: v_writelane_b32 v0, s5, s4 +; GFX10-NEXT: v_readlane_b32 s6, v2, s3 +; GFX10-NEXT: s_lshl_b32 s7, 1, s3 +; GFX10-NEXT: s_andn2_b32 s2, s2, s7 +; GFX10-NEXT: v_writelane_b32 v0, s5, s3 ; GFX10-NEXT: v_add_f32_e32 v1, s6, v1 -; GFX10-NEXT: s_cmp_lg_u32 s3, 0 -; GFX10-NEXT: s_cbranch_scc1 .LBB29_7 -; GFX10-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX10-NEXT: s_cmp_lg_u32 s2, 0 +; GFX10-NEXT: s_cbranch_scc1 .LBB29_5 +; GFX10-NEXT: ; %bb.6: ; %ComputeEnd ; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 ; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2 ; GFX10-NEXT: ; implicit-def: $vgpr2 -; GFX10-NEXT: s_and_saveexec_b32 s3, vcc_lo -; GFX10-NEXT: s_xor_b32 s3, exec_lo, s3 -; GFX10-NEXT: s_cbranch_execz .LBB29_10 -; GFX10-NEXT: ; %bb.9: -; GFX10-NEXT: v_mov_b32_e32 v2, s2 +; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo +; GFX10-NEXT: s_xor_b32 s2, exec_lo, s2 +; GFX10-NEXT: s_cbranch_execz .LBB29_8 +; GFX10-NEXT: ; %bb.7: +; GFX10-NEXT: v_mov_b32_e32 v2, s4 ; GFX10-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX10-NEXT: .LBB29_10: +; GFX10-NEXT: .LBB29_8: ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3 +; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: v_readfirstlane_b32 s2, v2 ; GFX10-NEXT: v_mov_b32_e32 v1, 0 ; GFX10-NEXT: v_add_f32_e32 v0, s2, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX10-NEXT: global_store_dword v1, v0, s[0:1] ; GFX10-NEXT: s_endpgm ; ; GFX90A-LABEL: local_ds_fadd_one_as: ; GFX90A: ; %bb.0: +; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 ; GFX90A-NEXT: s_mov_b64 s[2:3], exec -; GFX90A-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX90A-NEXT: v_mov_b32_e32 v2, 0x42280000 -; GFX90A-NEXT: ; implicit-def: $vgpr0 -; GFX90A-NEXT: .LBB29_1: ; %ComputeLoop -; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: s_ff1_i32_b64 s6, s[2:3] -; GFX90A-NEXT: s_lshl_b64 s[4:5], 1, s6 -; GFX90A-NEXT: v_readfirstlane_b32 s7, v1 -; GFX90A-NEXT: v_readlane_b32 s8, v2, s6 -; GFX90A-NEXT: s_mov_b32 m0, s6 -; GFX90A-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5] -; GFX90A-NEXT: v_writelane_b32 v0, s7, m0 -; GFX90A-NEXT: s_cmp_lg_u64 s[2:3], 0 -; GFX90A-NEXT: v_add_f32_e32 v1, s8, v1 -; GFX90A-NEXT: s_cbranch_scc1 .LBB29_1 -; GFX90A-NEXT: ; %bb.2: ; %ComputeEnd -; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 -; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 -; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 -; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 -; GFX90A-NEXT: ; implicit-def: $vgpr2 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_add_i32 s5, s5, 4 +; GFX90A-NEXT: ; implicit-def: $vgpr1 +; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB29_2 +; GFX90A-NEXT: ; %bb.1: +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: s_lshl_b32 s8, s5, 3 +; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX90A-NEXT: v_mov_b32_e32 v2, s8 +; GFX90A-NEXT: ds_add_rtn_f32 v1, v2, v1 +; GFX90A-NEXT: .LBB29_2: +; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX90A-NEXT: s_mov_b64 s[8:9], exec ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: s_add_i32 s3, s3, 4 -; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5] +; GFX90A-NEXT: v_readfirstlane_b32 s10, v1 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1 +; GFX90A-NEXT: v_cmp_eq_u32_e64 s[2:3], 0, v1 +; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], s[2:3] ; GFX90A-NEXT: s_cbranch_execz .LBB29_4 ; GFX90A-NEXT: ; %bb.3: -; GFX90A-NEXT: s_lshl_b32 s6, s3, 3 -; GFX90A-NEXT: v_mov_b32_e32 v2, s6 -; GFX90A-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX90A-NEXT: .LBB29_4: -; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX90A-NEXT: s_mov_b64 s[6:7], exec -; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0 -; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, s7, v1 -; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_readfirstlane_b32 s8, v2 -; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 -; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX90A-NEXT: s_cbranch_execz .LBB29_6 -; GFX90A-NEXT: ; %bb.5: -; GFX90A-NEXT: s_bcnt1_i32_b64 s6, s[6:7] -; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX90A-NEXT: s_lshl_b32 s3, s3, 4 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[8:9] +; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX90A-NEXT: s_lshl_b32 s2, s5, 4 ; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 -; GFX90A-NEXT: v_mov_b32_e32 v2, s3 +; GFX90A-NEXT: v_mov_b32_e32 v2, s2 ; GFX90A-NEXT: ds_add_f32 v2, v1 -; GFX90A-NEXT: .LBB29_6: -; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX90A-NEXT: s_mov_b64 s[4:5], exec -; GFX90A-NEXT: v_add_f32_e32 v2, s8, v0 +; GFX90A-NEXT: .LBB29_4: +; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX90A-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX90A-NEXT: v_add_f32_e32 v0, s10, v0 +; GFX90A-NEXT: v_mov_b32_e32 v1, s10 +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc ; GFX90A-NEXT: v_bfrev_b32_e32 v1, 1 ; GFX90A-NEXT: ; implicit-def: $vgpr0 -; GFX90A-NEXT: .LBB29_7: ; %ComputeLoop1 +; GFX90A-NEXT: .LBB29_5: ; %ComputeLoop ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: s_ff1_i32_b64 s3, s[4:5] -; GFX90A-NEXT: s_lshl_b64 s[6:7], 1, s3 +; GFX90A-NEXT: s_ff1_i32_b64 s5, s[2:3] +; GFX90A-NEXT: s_lshl_b64 s[6:7], 1, s5 ; GFX90A-NEXT: v_readfirstlane_b32 s8, v1 -; GFX90A-NEXT: v_readlane_b32 s9, v2, s3 -; GFX90A-NEXT: s_mov_b32 m0, s3 -; GFX90A-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7] +; GFX90A-NEXT: v_readlane_b32 s9, v2, s5 +; GFX90A-NEXT: s_mov_b32 m0, s5 +; GFX90A-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7] ; GFX90A-NEXT: v_writelane_b32 v0, s8, m0 -; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 0 +; GFX90A-NEXT: s_cmp_lg_u64 s[2:3], 0 ; GFX90A-NEXT: v_add_f32_e32 v1, s9, v1 -; GFX90A-NEXT: s_cbranch_scc1 .LBB29_7 -; GFX90A-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX90A-NEXT: s_cbranch_scc1 .LBB29_5 +; GFX90A-NEXT: ; %bb.6: ; %ComputeEnd ; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 ; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 ; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 ; GFX90A-NEXT: ; implicit-def: $vgpr2 -; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5] -; GFX90A-NEXT: s_cbranch_execz .LBB29_10 -; GFX90A-NEXT: ; %bb.9: -; GFX90A-NEXT: v_mov_b32_e32 v2, s2 +; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX90A-NEXT: s_xor_b64 s[2:3], exec, s[2:3] +; GFX90A-NEXT: s_cbranch_execz .LBB29_8 +; GFX90A-NEXT: ; %bb.7: +; GFX90A-NEXT: v_mov_b32_e32 v2, s4 ; GFX90A-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX90A-NEXT: .LBB29_10: -; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX90A-NEXT: .LBB29_8: +; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_readfirstlane_b32 s2, v2 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0 ; GFX90A-NEXT: v_add_f32_e32 v0, s2, v0 +; GFX90A-NEXT: v_mov_b32_e32 v2, s2 +; GFX90A-NEXT: v_mov_b32_e32 v1, 0 +; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GFX90A-NEXT: global_store_dword v1, v0, s[0:1] ; GFX90A-NEXT: s_endpgm ; ; GFX908-LABEL: local_ds_fadd_one_as: ; GFX908: ; %bb.0: +; GFX908-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 ; GFX908-NEXT: s_mov_b64 s[2:3], exec -; GFX908-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX908-NEXT: v_mov_b32_e32 v2, 0x42280000 -; GFX908-NEXT: ; implicit-def: $vgpr0 -; GFX908-NEXT: .LBB29_1: ; %ComputeLoop -; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX908-NEXT: s_ff1_i32_b64 s6, s[2:3] -; GFX908-NEXT: s_lshl_b64 s[4:5], 1, s6 -; GFX908-NEXT: v_readfirstlane_b32 s7, v1 -; GFX908-NEXT: v_readlane_b32 s8, v2, s6 -; GFX908-NEXT: s_mov_b32 m0, s6 -; GFX908-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5] -; GFX908-NEXT: v_writelane_b32 v0, s7, m0 -; GFX908-NEXT: s_cmp_lg_u64 s[2:3], 0 -; GFX908-NEXT: v_add_f32_e32 v1, s8, v1 -; GFX908-NEXT: s_cbranch_scc1 .LBB29_1 -; GFX908-NEXT: ; %bb.2: ; %ComputeEnd -; GFX908-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 -; GFX908-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 -; GFX908-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 -; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 -; GFX908-NEXT: ; implicit-def: $vgpr2 +; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX908-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-NEXT: s_add_i32 s5, s5, 4 +; GFX908-NEXT: ; implicit-def: $vgpr1 +; GFX908-NEXT: s_and_saveexec_b64 s[6:7], vcc +; GFX908-NEXT: s_cbranch_execz .LBB29_2 +; GFX908-NEXT: ; %bb.1: +; GFX908-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX908-NEXT: s_lshl_b32 s8, s5, 3 +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX908-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-NEXT: ds_add_rtn_f32 v1, v2, v1 +; GFX908-NEXT: .LBB29_2: +; GFX908-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX908-NEXT: s_mov_b64 s[8:9], exec ; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: s_add_i32 s3, s3, 4 -; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5] +; GFX908-NEXT: v_readfirstlane_b32 s10, v1 +; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0 +; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1 +; GFX908-NEXT: v_cmp_eq_u32_e64 s[2:3], 0, v1 +; GFX908-NEXT: s_and_saveexec_b64 s[6:7], s[2:3] ; GFX908-NEXT: s_cbranch_execz .LBB29_4 ; GFX908-NEXT: ; %bb.3: -; GFX908-NEXT: s_lshl_b32 s6, s3, 3 -; GFX908-NEXT: v_mov_b32_e32 v2, s6 -; GFX908-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX908-NEXT: .LBB29_4: -; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX908-NEXT: s_mov_b64 s[6:7], exec -; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0 -; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, s7, v1 -; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: v_readfirstlane_b32 s8, v2 -; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 -; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX908-NEXT: s_cbranch_execz .LBB29_6 -; GFX908-NEXT: ; %bb.5: -; GFX908-NEXT: s_bcnt1_i32_b64 s6, s[6:7] -; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX908-NEXT: s_lshl_b32 s3, s3, 4 +; GFX908-NEXT: s_bcnt1_i32_b64 s2, s[8:9] +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX908-NEXT: s_lshl_b32 s2, s5, 4 ; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 -; GFX908-NEXT: v_mov_b32_e32 v2, s3 +; GFX908-NEXT: v_mov_b32_e32 v2, s2 ; GFX908-NEXT: ds_add_f32 v2, v1 -; GFX908-NEXT: .LBB29_6: -; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX908-NEXT: s_mov_b64 s[4:5], exec -; GFX908-NEXT: v_add_f32_e32 v2, s8, v0 +; GFX908-NEXT: .LBB29_4: +; GFX908-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX908-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX908-NEXT: v_add_f32_e32 v0, s10, v0 +; GFX908-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-NEXT: s_mov_b64 s[2:3], exec +; GFX908-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc ; GFX908-NEXT: v_bfrev_b32_e32 v1, 1 ; GFX908-NEXT: ; implicit-def: $vgpr0 -; GFX908-NEXT: .LBB29_7: ; %ComputeLoop1 +; GFX908-NEXT: .LBB29_5: ; %ComputeLoop ; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX908-NEXT: s_ff1_i32_b64 s3, s[4:5] -; GFX908-NEXT: s_lshl_b64 s[6:7], 1, s3 +; GFX908-NEXT: s_ff1_i32_b64 s5, s[2:3] +; GFX908-NEXT: s_lshl_b64 s[6:7], 1, s5 ; GFX908-NEXT: v_readfirstlane_b32 s8, v1 -; GFX908-NEXT: v_readlane_b32 s9, v2, s3 -; GFX908-NEXT: s_mov_b32 m0, s3 -; GFX908-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7] +; GFX908-NEXT: v_readlane_b32 s9, v2, s5 +; GFX908-NEXT: s_mov_b32 m0, s5 +; GFX908-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7] ; GFX908-NEXT: v_writelane_b32 v0, s8, m0 -; GFX908-NEXT: s_cmp_lg_u64 s[4:5], 0 +; GFX908-NEXT: s_cmp_lg_u64 s[2:3], 0 ; GFX908-NEXT: v_add_f32_e32 v1, s9, v1 -; GFX908-NEXT: s_cbranch_scc1 .LBB29_7 -; GFX908-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX908-NEXT: s_cbranch_scc1 .LBB29_5 +; GFX908-NEXT: ; %bb.6: ; %ComputeEnd ; GFX908-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 ; GFX908-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 ; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 ; GFX908-NEXT: ; implicit-def: $vgpr2 -; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5] -; GFX908-NEXT: s_cbranch_execz .LBB29_10 -; GFX908-NEXT: ; %bb.9: -; GFX908-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX908-NEXT: s_xor_b64 s[2:3], exec, s[2:3] +; GFX908-NEXT: s_cbranch_execz .LBB29_8 +; GFX908-NEXT: ; %bb.7: +; GFX908-NEXT: v_mov_b32_e32 v2, s4 ; GFX908-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX908-NEXT: .LBB29_10: -; GFX908-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX908-NEXT: .LBB29_8: +; GFX908-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-NEXT: v_readfirstlane_b32 s2, v2 -; GFX908-NEXT: v_mov_b32_e32 v1, 0 ; GFX908-NEXT: v_add_f32_e32 v0, s2, v0 +; GFX908-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-NEXT: v_mov_b32_e32 v1, 0 +; GFX908-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc ; GFX908-NEXT: global_store_dword v1, v0, s[0:1] ; GFX908-NEXT: s_endpgm ; ; GFX8-LABEL: local_ds_fadd_one_as: ; GFX8: ; %bb.0: +; GFX8-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x8 ; GFX8-NEXT: s_mov_b64 s[2:3], exec -; GFX8-NEXT: v_bfrev_b32_e32 v1, 1 -; GFX8-NEXT: v_mov_b32_e32 v2, 0x42280000 -; GFX8-NEXT: ; implicit-def: $vgpr0 -; GFX8-NEXT: .LBB29_1: ; %ComputeLoop -; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX8-NEXT: s_ff1_i32_b64 s6, s[2:3] -; GFX8-NEXT: s_lshl_b64 s[4:5], 1, s6 -; GFX8-NEXT: v_readfirstlane_b32 s7, v1 -; GFX8-NEXT: v_readlane_b32 s8, v2, s6 -; GFX8-NEXT: s_mov_b32 m0, s6 -; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5] -; GFX8-NEXT: v_writelane_b32 v0, s7, m0 -; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0 -; GFX8-NEXT: v_add_f32_e32 v1, s8, v1 -; GFX8-NEXT: s_cbranch_scc1 .LBB29_1 -; GFX8-NEXT: ; %bb.2: ; %ComputeEnd -; GFX8-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8 -; GFX8-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0 -; GFX8-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 -; GFX8-NEXT: ; implicit-def: $vgpr2 +; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX8-NEXT: s_waitcnt lgkmcnt(0) +; GFX8-NEXT: s_add_i32 s5, s5, 4 +; GFX8-NEXT: ; implicit-def: $vgpr1 ; GFX8-NEXT: s_mov_b32 m0, -1 +; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc +; GFX8-NEXT: s_cbranch_execz .LBB29_2 +; GFX8-NEXT: ; %bb.1: +; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX8-NEXT: s_lshl_b32 s8, s5, 3 +; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 +; GFX8-NEXT: v_mov_b32_e32 v2, s8 +; GFX8-NEXT: ds_add_rtn_f32 v1, v2, v1 +; GFX8-NEXT: .LBB29_2: +; GFX8-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX8-NEXT: s_mov_b64 s[8:9], exec ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_add_i32 s3, s3, 4 -; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5] +; GFX8-NEXT: v_readfirstlane_b32 s10, v1 +; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0 +; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 0, v1 +; GFX8-NEXT: s_and_saveexec_b64 s[6:7], s[2:3] ; GFX8-NEXT: s_cbranch_execz .LBB29_4 ; GFX8-NEXT: ; %bb.3: -; GFX8-NEXT: s_lshl_b32 s6, s3, 3 -; GFX8-NEXT: v_mov_b32_e32 v2, s6 -; GFX8-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX8-NEXT: .LBB29_4: -; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX8-NEXT: s_mov_b64 s[6:7], exec -; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, s6, 0 -; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, s7, v1 -; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_readfirstlane_b32 s8, v2 -; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 -; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX8-NEXT: s_cbranch_execz .LBB29_6 -; GFX8-NEXT: ; %bb.5: -; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7] -; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s6 -; GFX8-NEXT: s_lshl_b32 s3, s3, 4 +; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[8:9] +; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s2 +; GFX8-NEXT: s_lshl_b32 s2, s5, 4 ; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1 -; GFX8-NEXT: v_mov_b32_e32 v2, s3 +; GFX8-NEXT: v_mov_b32_e32 v2, s2 ; GFX8-NEXT: ds_add_f32 v2, v1 -; GFX8-NEXT: .LBB29_6: -; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX8-NEXT: s_mov_b64 s[4:5], exec -; GFX8-NEXT: v_add_f32_e32 v2, s8, v0 +; GFX8-NEXT: .LBB29_4: +; GFX8-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0 +; GFX8-NEXT: v_add_f32_e32 v0, s10, v0 +; GFX8-NEXT: v_mov_b32_e32 v1, s10 +; GFX8-NEXT: s_mov_b64 s[2:3], exec +; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc ; GFX8-NEXT: v_bfrev_b32_e32 v1, 1 ; GFX8-NEXT: ; implicit-def: $vgpr0 -; GFX8-NEXT: .LBB29_7: ; %ComputeLoop1 +; GFX8-NEXT: .LBB29_5: ; %ComputeLoop ; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX8-NEXT: s_ff1_i32_b64 s3, s[4:5] -; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s3 +; GFX8-NEXT: s_ff1_i32_b64 s5, s[2:3] +; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s5 ; GFX8-NEXT: v_readfirstlane_b32 s8, v1 -; GFX8-NEXT: v_readlane_b32 s9, v2, s3 -; GFX8-NEXT: s_mov_b32 m0, s3 -; GFX8-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7] +; GFX8-NEXT: v_readlane_b32 s9, v2, s5 +; GFX8-NEXT: s_mov_b32 m0, s5 +; GFX8-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7] ; GFX8-NEXT: v_writelane_b32 v0, s8, m0 -; GFX8-NEXT: s_cmp_lg_u64 s[4:5], 0 +; GFX8-NEXT: s_cmp_lg_u64 s[2:3], 0 ; GFX8-NEXT: v_add_f32_e32 v1, s9, v1 -; GFX8-NEXT: s_cbranch_scc1 .LBB29_7 -; GFX8-NEXT: ; %bb.8: ; %ComputeEnd2 +; GFX8-NEXT: s_cbranch_scc1 .LBB29_5 +; GFX8-NEXT: ; %bb.6: ; %ComputeEnd ; GFX8-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 ; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 ; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2 ; GFX8-NEXT: ; implicit-def: $vgpr2 -; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5] -; GFX8-NEXT: s_cbranch_execz .LBB29_10 -; GFX8-NEXT: ; %bb.9: -; GFX8-NEXT: v_mov_b32_e32 v2, s2 +; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX8-NEXT: s_xor_b64 s[2:3], exec, s[2:3] +; GFX8-NEXT: s_cbranch_execz .LBB29_8 +; GFX8-NEXT: ; %bb.7: +; GFX8-NEXT: v_mov_b32_e32 v2, s4 ; GFX8-NEXT: s_mov_b32 m0, -1 ; GFX8-NEXT: ds_add_rtn_f32 v2, v2, v1 -; GFX8-NEXT: .LBB29_10: -; GFX8-NEXT: s_or_b64 exec, exec, s[4:5] +; GFX8-NEXT: .LBB29_8: +; GFX8-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: v_readfirstlane_b32 s2, v2 -; GFX8-NEXT: v_add_f32_e32 v2, s2, v0 +; GFX8-NEXT: v_add_f32_e32 v0, s2, v0 +; GFX8-NEXT: v_mov_b32_e32 v1, s2 +; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc ; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: flat_store_dword v[0:1], v2 @@ -8439,153 +8358,186 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs ; ; GFX7-LABEL: local_ds_fadd_one_as: ; GFX7: ; %bb.0: -; GFX7-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2 -; GFX7-NEXT: s_mov_b32 m0, -1 +; GFX7-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2 +; GFX7-NEXT: s_mov_b64 s[2:3], exec +; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_lshl_b32 s4, s3, 3 -; GFX7-NEXT: v_mov_b32_e32 v0, s4 -; GFX7-NEXT: ds_read_b32 v0, v0 offset:32 -; GFX7-NEXT: s_add_i32 s3, s3, 4 -; GFX7-NEXT: s_lshl_b32 s6, s3, 3 -; GFX7-NEXT: s_mov_b64 s[4:5], 0 -; GFX7-NEXT: v_mov_b32_e32 v1, s6 -; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start +; GFX7-NEXT: s_add_i32 s5, s5, 4 +; GFX7-NEXT: ; implicit-def: $vgpr1 +; GFX7-NEXT: s_mov_b32 m0, -1 +; GFX7-NEXT: s_and_saveexec_b64 s[6:7], vcc +; GFX7-NEXT: s_cbranch_execz .LBB29_4 +; GFX7-NEXT: ; %bb.1: +; GFX7-NEXT: s_lshl_b32 s8, s5, 3 +; GFX7-NEXT: v_mov_b32_e32 v2, s8 +; GFX7-NEXT: ds_read_b32 v1, v2 +; GFX7-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s2 +; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3 +; GFX7-NEXT: s_mov_b64 s[8:9], 0 +; GFX7-NEXT: .LBB29_2: ; %atomicrmw.start ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_mov_b32_e32 v2, v0 -; GFX7-NEXT: v_add_f32_e32 v0, 0x42280000, v2 -; GFX7-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0 +; GFX7-NEXT: v_mov_b32_e32 v4, v1 +; GFX7-NEXT: v_add_f32_e32 v1, v4, v3 +; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2 -; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX7-NEXT: s_cbranch_execnz .LBB29_1 -; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX7-NEXT: s_mov_b64 s[6:7], exec -; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s6, 0 -; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s7, v1 -; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 -; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX7-NEXT: s_cbranch_execz .LBB29_5 -; GFX7-NEXT: ; %bb.3: -; GFX7-NEXT: s_lshl_b32 s3, s3, 4 -; GFX7-NEXT: v_mov_b32_e32 v1, s3 +; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], v1, v4 +; GFX7-NEXT: s_or_b64 s[8:9], s[2:3], s[8:9] +; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GFX7-NEXT: s_cbranch_execnz .LBB29_2 +; GFX7-NEXT: ; %bb.3: ; %Flow18 +; GFX7-NEXT: s_or_b64 exec, exec, s[8:9] +; GFX7-NEXT: .LBB29_4: ; %Flow19 +; GFX7-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX7-NEXT: s_mov_b64 s[8:9], exec +; GFX7-NEXT: v_readfirstlane_b32 s10, v1 +; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0 +; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1 +; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 0, v1 +; GFX7-NEXT: s_and_saveexec_b64 s[6:7], s[2:3] +; GFX7-NEXT: s_cbranch_execz .LBB29_7 +; GFX7-NEXT: ; %bb.5: +; GFX7-NEXT: s_lshl_b32 s2, s5, 4 +; GFX7-NEXT: v_mov_b32_e32 v1, s2 ; GFX7-NEXT: ds_read_b32 v3, v1 -; GFX7-NEXT: s_bcnt1_i32_b64 s3, s[6:7] -; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s3 +; GFX7-NEXT: s_bcnt1_i32_b64 s2, s[8:9] +; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s2 ; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2 -; GFX7-NEXT: s_mov_b64 s[6:7], 0 -; GFX7-NEXT: .LBB29_4: ; %atomicrmw.start2 +; GFX7-NEXT: s_mov_b64 s[8:9], 0 +; GFX7-NEXT: .LBB29_6: ; %atomicrmw.start2 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: v_add_f32_e32 v4, v3, v2 ; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3 -; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], v4, v3 +; GFX7-NEXT: s_or_b64 s[8:9], s[2:3], s[8:9] ; GFX7-NEXT: v_mov_b32_e32 v3, v4 -; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7] -; GFX7-NEXT: s_cbranch_execnz .LBB29_4 -; GFX7-NEXT: .LBB29_5: ; %Flow17 -; GFX7-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX7-NEXT: v_mov_b32_e32 v2, s2 -; GFX7-NEXT: ds_read_b32 v1, v2 +; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GFX7-NEXT: s_cbranch_execnz .LBB29_6 +; GFX7-NEXT: .LBB29_7: ; %Flow17 +; GFX7-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX7-NEXT: v_mov_b32_e32 v1, s4 +; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v0 +; GFX7-NEXT: ds_read_b32 v0, v1 +; GFX7-NEXT: v_add_f32_e32 v2, s10, v2 +; GFX7-NEXT: v_mov_b32_e32 v3, s10 +; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ; GFX7-NEXT: s_mov_b64 s[2:3], 0 -; GFX7-NEXT: .LBB29_6: ; %atomicrmw.start8 +; GFX7-NEXT: .LBB29_8: ; %atomicrmw.start8 ; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_mov_b32_e32 v3, v1 -; GFX7-NEXT: v_add_f32_e32 v1, v3, v0 -; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1 +; GFX7-NEXT: v_mov_b32_e32 v3, v0 +; GFX7-NEXT: v_add_f32_e32 v0, v3, v2 +; GFX7-NEXT: ds_cmpst_rtn_b32 v0, v1, v3, v0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 +; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3 ; GFX7-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX7-NEXT: s_cbranch_execnz .LBB29_6 -; GFX7-NEXT: ; %bb.7: ; %atomicrmw.end7 +; GFX7-NEXT: s_cbranch_execnz .LBB29_8 +; GFX7-NEXT: ; %bb.9: ; %atomicrmw.end7 ; GFX7-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: buffer_store_dword v1, off, s[0:3], 0 +; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: s_endpgm ; ; GFX6-LABEL: local_ds_fadd_one_as: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2 -; GFX6-NEXT: s_mov_b32 m0, -1 +; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2 +; GFX6-NEXT: s_mov_b64 s[2:3], exec +; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_lshl_b32 s4, s3, 3 -; GFX6-NEXT: s_add_i32 s4, s4, 32 -; GFX6-NEXT: v_mov_b32_e32 v0, s4 -; GFX6-NEXT: ds_read_b32 v0, v0 -; GFX6-NEXT: s_add_i32 s3, s3, 4 -; GFX6-NEXT: s_lshl_b32 s6, s3, 3 -; GFX6-NEXT: s_mov_b64 s[4:5], 0 -; GFX6-NEXT: v_mov_b32_e32 v1, s6 -; GFX6-NEXT: .LBB29_1: ; %atomicrmw.start +; GFX6-NEXT: s_add_i32 s5, s5, 4 +; GFX6-NEXT: ; implicit-def: $vgpr1 +; GFX6-NEXT: s_mov_b32 m0, -1 +; GFX6-NEXT: s_and_saveexec_b64 s[6:7], vcc +; GFX6-NEXT: s_cbranch_execz .LBB29_4 +; GFX6-NEXT: ; %bb.1: +; GFX6-NEXT: s_lshl_b32 s8, s5, 3 +; GFX6-NEXT: v_mov_b32_e32 v2, s8 +; GFX6-NEXT: ds_read_b32 v1, v2 +; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v3, s2 +; GFX6-NEXT: v_mul_f32_e32 v3, 0x42280000, v3 +; GFX6-NEXT: s_mov_b64 s[8:9], 0 +; GFX6-NEXT: .LBB29_2: ; %atomicrmw.start ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v2, v0 -; GFX6-NEXT: v_add_f32_e32 v0, 0x42280000, v2 -; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0 +; GFX6-NEXT: v_mov_b32_e32 v4, v1 +; GFX6-NEXT: v_add_f32_e32 v1, v4, v3 +; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2 -; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5] -; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5] -; GFX6-NEXT: s_cbranch_execnz .LBB29_1 -; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end -; GFX6-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX6-NEXT: s_mov_b64 s[6:7], exec -; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s6, 0 -; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s7, v1 -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 -; GFX6-NEXT: s_and_saveexec_b64 s[4:5], vcc -; GFX6-NEXT: s_cbranch_execz .LBB29_5 -; GFX6-NEXT: ; %bb.3: -; GFX6-NEXT: s_lshl_b32 s3, s3, 4 -; GFX6-NEXT: v_mov_b32_e32 v1, s3 +; GFX6-NEXT: v_cmp_eq_u32_e64 s[2:3], v1, v4 +; GFX6-NEXT: s_or_b64 s[8:9], s[2:3], s[8:9] +; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GFX6-NEXT: s_cbranch_execnz .LBB29_2 +; GFX6-NEXT: ; %bb.3: ; %Flow16 +; GFX6-NEXT: s_or_b64 exec, exec, s[8:9] +; GFX6-NEXT: .LBB29_4: ; %Flow17 +; GFX6-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX6-NEXT: s_mov_b64 s[8:9], exec +; GFX6-NEXT: v_readfirstlane_b32 s10, v1 +; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0 +; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1 +; GFX6-NEXT: v_cmp_eq_u32_e64 s[2:3], 0, v1 +; GFX6-NEXT: s_and_saveexec_b64 s[6:7], s[2:3] +; GFX6-NEXT: s_cbranch_execz .LBB29_7 +; GFX6-NEXT: ; %bb.5: +; GFX6-NEXT: s_lshl_b32 s2, s5, 4 +; GFX6-NEXT: v_mov_b32_e32 v1, s2 ; GFX6-NEXT: ds_read_b32 v3, v1 -; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[6:7] -; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s3 +; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[8:9] +; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s2 ; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2 -; GFX6-NEXT: s_mov_b64 s[6:7], 0 -; GFX6-NEXT: .LBB29_4: ; %atomicrmw.start2 +; GFX6-NEXT: s_mov_b64 s[8:9], 0 +; GFX6-NEXT: .LBB29_6: ; %atomicrmw.start2 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: v_add_f32_e32 v4, v3, v2 ; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3 -; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7] +; GFX6-NEXT: v_cmp_eq_u32_e64 s[2:3], v4, v3 +; GFX6-NEXT: s_or_b64 s[8:9], s[2:3], s[8:9] ; GFX6-NEXT: v_mov_b32_e32 v3, v4 -; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7] -; GFX6-NEXT: s_cbranch_execnz .LBB29_4 -; GFX6-NEXT: .LBB29_5: ; %Flow15 -; GFX6-NEXT: s_or_b64 exec, exec, s[4:5] -; GFX6-NEXT: v_mov_b32_e32 v2, s2 -; GFX6-NEXT: ds_read_b32 v1, v2 +; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9] +; GFX6-NEXT: s_cbranch_execnz .LBB29_6 +; GFX6-NEXT: .LBB29_7: ; %Flow15 +; GFX6-NEXT: s_or_b64 exec, exec, s[6:7] +; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, v0 +; GFX6-NEXT: v_mov_b32_e32 v1, s4 +; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v0 +; GFX6-NEXT: ds_read_b32 v0, v1 +; GFX6-NEXT: v_add_f32_e32 v2, s10, v2 +; GFX6-NEXT: v_mov_b32_e32 v3, s10 +; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ; GFX6-NEXT: s_mov_b64 s[2:3], 0 -; GFX6-NEXT: .LBB29_6: ; %atomicrmw.start8 +; GFX6-NEXT: .LBB29_8: ; %atomicrmw.start8 ; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: v_mov_b32_e32 v3, v1 -; GFX6-NEXT: v_add_f32_e32 v1, v3, v0 -; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1 +; GFX6-NEXT: v_mov_b32_e32 v3, v0 +; GFX6-NEXT: v_add_f32_e32 v0, v3, v2 +; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v3, v0 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3 +; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3 ; GFX6-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX6-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX6-NEXT: s_cbranch_execnz .LBB29_6 -; GFX6-NEXT: ; %bb.7: ; %atomicrmw.end7 +; GFX6-NEXT: s_cbranch_execnz .LBB29_8 +; GFX6-NEXT: ; %bb.9: ; %atomicrmw.end7 ; GFX6-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 ; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: buffer_store_dword v1, off, s[0:3], 0 +; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX6-NEXT: s_endpgm %idx.add = add nuw i32 %idx, 4 %shl0 = shl i32 %idx.add, 3