diff --git a/llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp b/llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp index b348cb56c1361..18a532b55ee5a 100644 --- a/llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp @@ -571,11 +571,11 @@ bool LoongArchExpandAtomicPseudo::expandAtomicCmpXchg( BuildMI(LoopTailMBB, DL, TII->get(LoongArch::B)).addMBB(DoneMBB); } - AtomicOrdering Ordering = + AtomicOrdering FailureOrdering = static_cast(MI.getOperand(IsMasked ? 6 : 5).getImm()); int hint; - switch (Ordering) { + switch (FailureOrdering) { case AtomicOrdering::Acquire: case AtomicOrdering::AcquireRelease: case AtomicOrdering::SequentiallyConsistent: diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index ecdbaf92e56c1..334daccab1e8b 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -4184,8 +4184,9 @@ LoongArchTargetLowering::shouldExpandAtomicCmpXchgInIR( Value *LoongArchTargetLowering::emitMaskedAtomicCmpXchgIntrinsic( IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const { - Value *Ordering = - Builder.getIntN(Subtarget.getGRLen(), static_cast(Ord)); + AtomicOrdering FailOrd = CI->getFailureOrdering(); + Value *FailureOrdering = + Builder.getIntN(Subtarget.getGRLen(), static_cast(FailOrd)); // TODO: Support cmpxchg on LA32. Intrinsic::ID CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i64; @@ -4196,7 +4197,7 @@ Value *LoongArchTargetLowering::emitMaskedAtomicCmpXchgIntrinsic( Function *MaskedCmpXchg = Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys); Value *Result = Builder.CreateCall( - MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering}); + MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, FailureOrdering}); Result = Builder.CreateTrunc(Result, Builder.getInt32Ty()); return Result; } diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td index a892ae65f610d..3f67494bb284a 100644 --- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td +++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td @@ -1814,7 +1814,7 @@ def PseudoMaskedAtomicLoadMin32 : PseudoMaskedAMMinMax; class PseudoCmpXchg : Pseudo<(outs GPR:$res, GPR:$scratch), - (ins GPR:$addr, GPR:$cmpval, GPR:$newval, grlenimm:$ordering)> { + (ins GPR:$addr, GPR:$cmpval, GPR:$newval, grlenimm:$fail_order)> { let Constraints = "@earlyclobber $res,@earlyclobber $scratch"; let mayLoad = 1; let mayStore = 1; @@ -1828,7 +1828,7 @@ def PseudoCmpXchg64 : PseudoCmpXchg; def PseudoMaskedCmpXchg32 : Pseudo<(outs GPR:$res, GPR:$scratch), (ins GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, - grlenimm:$ordering)> { + grlenimm:$fail_order)> { let Constraints = "@earlyclobber $res,@earlyclobber $scratch"; let mayLoad = 1; let mayStore = 1; @@ -1846,6 +1846,43 @@ class AtomicPat : Pat<(intrin GPR:$addr, GPR:$incr, GPR:$mask, timm:$ordering), (AMInst GPR:$addr, GPR:$incr, GPR:$mask, timm:$ordering)>; +// These atomic cmpxchg PatFrags only care about the failure ordering. +// The PatFrags defined by multiclass `ternary_atomic_op_ord` in +// TargetSelectionDAG.td care about the merged memory ordering that is the +// stronger one between success and failure. But for LoongArch LL-SC we only +// need to care about the failure ordering as explained in PR #67391. So we +// define these PatFrags that will be used to define cmpxchg pats below. +multiclass ternary_atomic_op_failure_ord { + def NAME#_failure_monotonic : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast(NAME) node:$ptr, node:$cmp, node:$val), [{ + AtomicOrdering Ordering = cast(N)->getFailureOrdering(); + return Ordering == AtomicOrdering::Monotonic; + }]>; + def NAME#_failure_acquire : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast(NAME) node:$ptr, node:$cmp, node:$val), [{ + AtomicOrdering Ordering = cast(N)->getFailureOrdering(); + return Ordering == AtomicOrdering::Acquire; + }]>; + def NAME#_failure_release : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast(NAME) node:$ptr, node:$cmp, node:$val), [{ + AtomicOrdering Ordering = cast(N)->getFailureOrdering(); + return Ordering == AtomicOrdering::Release; + }]>; + def NAME#_failure_acq_rel : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast(NAME) node:$ptr, node:$cmp, node:$val), [{ + AtomicOrdering Ordering = cast(N)->getFailureOrdering(); + return Ordering == AtomicOrdering::AcquireRelease; + }]>; + def NAME#_failure_seq_cst : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast(NAME) node:$ptr, node:$cmp, node:$val), [{ + AtomicOrdering Ordering = cast(N)->getFailureOrdering(); + return Ordering == AtomicOrdering::SequentiallyConsistent; + }]>; +} + +defm atomic_cmp_swap_32 : ternary_atomic_op_failure_ord; +defm atomic_cmp_swap_64 : ternary_atomic_op_failure_ord; + let Predicates = [IsLA64] in { def : AtomicPat; @@ -1908,24 +1945,24 @@ def : AtomicPat { - def : Pat<(vt (!cast(Op#"_monotonic") GPR:$addr, GPR:$cmp, GPR:$new)), + def : Pat<(vt (!cast(Op#"_failure_monotonic") GPR:$addr, GPR:$cmp, GPR:$new)), (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 2)>; - def : Pat<(vt (!cast(Op#"_acquire") GPR:$addr, GPR:$cmp, GPR:$new)), + def : Pat<(vt (!cast(Op#"_failure_acquire") GPR:$addr, GPR:$cmp, GPR:$new)), (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 4)>; - def : Pat<(vt (!cast(Op#"_release") GPR:$addr, GPR:$cmp, GPR:$new)), + def : Pat<(vt (!cast(Op#"_failure_release") GPR:$addr, GPR:$cmp, GPR:$new)), (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 5)>; - def : Pat<(vt (!cast(Op#"_acq_rel") GPR:$addr, GPR:$cmp, GPR:$new)), + def : Pat<(vt (!cast(Op#"_failure_acq_rel") GPR:$addr, GPR:$cmp, GPR:$new)), (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 6)>; - def : Pat<(vt (!cast(Op#"_seq_cst") GPR:$addr, GPR:$cmp, GPR:$new)), + def : Pat<(vt (!cast(Op#"_failure_seq_cst") GPR:$addr, GPR:$cmp, GPR:$new)), (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 7)>; } defm : PseudoCmpXchgPat<"atomic_cmp_swap_32", PseudoCmpXchg32>; defm : PseudoCmpXchgPat<"atomic_cmp_swap_64", PseudoCmpXchg64, i64>; def : Pat<(int_loongarch_masked_cmpxchg_i64 - GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$ordering), + GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$fail_order), (PseudoMaskedCmpXchg32 - GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$ordering)>; + GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$fail_order)>; def : PseudoMaskedAMMinMaxPat; diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll index 76f9ebed0d93b..417c865f6383f 100644 --- a/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll +++ b/llvm/test/CodeGen/LoongArch/ir-instruction/atomic-cmpxchg.ll @@ -129,7 +129,7 @@ define void @cmpxchg_i8_acquire_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind { ; LA64-NEXT: beqz $a5, .LBB4_1 ; LA64-NEXT: b .LBB4_4 ; LA64-NEXT: .LBB4_3: -; LA64-NEXT: dbar 20 +; LA64-NEXT: dbar 1792 ; LA64-NEXT: .LBB4_4: ; LA64-NEXT: ret %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val acquire monotonic @@ -162,7 +162,7 @@ define void @cmpxchg_i16_acquire_monotonic(ptr %ptr, i16 %cmp, i16 %val) nounwin ; LA64-NEXT: beqz $a5, .LBB5_1 ; LA64-NEXT: b .LBB5_4 ; LA64-NEXT: .LBB5_3: -; LA64-NEXT: dbar 20 +; LA64-NEXT: dbar 1792 ; LA64-NEXT: .LBB5_4: ; LA64-NEXT: ret %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val acquire monotonic @@ -181,7 +181,7 @@ define void @cmpxchg_i32_acquire_monotonic(ptr %ptr, i32 %cmp, i32 %val) nounwin ; LA64-NEXT: beqz $a4, .LBB6_1 ; LA64-NEXT: b .LBB6_4 ; LA64-NEXT: .LBB6_3: -; LA64-NEXT: dbar 20 +; LA64-NEXT: dbar 1792 ; LA64-NEXT: .LBB6_4: ; LA64-NEXT: ret %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val acquire monotonic @@ -200,7 +200,7 @@ define void @cmpxchg_i64_acquire_monotonic(ptr %ptr, i64 %cmp, i64 %val) nounwin ; LA64-NEXT: beqz $a4, .LBB7_1 ; LA64-NEXT: b .LBB7_4 ; LA64-NEXT: .LBB7_3: -; LA64-NEXT: dbar 20 +; LA64-NEXT: dbar 1792 ; LA64-NEXT: .LBB7_4: ; LA64-NEXT: ret %res = cmpxchg ptr %ptr, i64 %cmp, i64 %val acquire monotonic