diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 3db777f904df0..e6ea4205623d3 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -872,6 +872,15 @@ bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const
   return false;
 }
 
+TargetLowering::AtomicExpansionKind
+SystemZTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
+  return (RMW->isFloatingPointOperation() ||
+          RMW->getOperation() == AtomicRMWInst::UIncWrap ||
+          RMW->getOperation() == AtomicRMWInst::UDecWrap)
+             ? AtomicExpansionKind::CmpXChg
+             : AtomicExpansionKind::None;
+}
+
 bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
   // We can use CGFI or CLGFI.
   return isInt<32>(Imm) || isUInt<32>(Imm);
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 0d4b4873e9d73..fd951b935702a 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -452,6 +452,8 @@ class SystemZTargetLowering : public TargetLowering {
     return VT != MVT::f64;
   }
   bool hasInlineStackProbe(const MachineFunction &MF) const override;
+  AtomicExpansionKind
+  shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const override;
   bool isLegalICmpImmediate(int64_t Imm) const override;
   bool isLegalAddImmediate(int64_t Imm) const override;
   bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
index 8235446da65c4..186494ad2ac61 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
@@ -226,6 +226,8 @@ void SystemZPassConfig::addIRPasses() {
     addPass(createLoopDataPrefetchPass());
   }
 
+  addPass(createAtomicExpandPass());
+
   TargetPassConfig::addIRPasses();
 }
 
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-06.ll b/llvm/test/CodeGen/SystemZ/atomic-load-06.ll
new file mode 100644
index 0000000000000..c9c5504520345
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-06.ll
@@ -0,0 +1,13 @@
+; Test float atomic loads.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK: lgf [[R:%r[0-9]+]], 0(%r2)
+; CHECK: sllg [[R]], [[R]], 32
+; CHECK: ldgr %f0, [[R]]
+; CHECK: br %r14
+  %val = load atomic float, ptr %src seq_cst, align 4
+  ret float %val
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-07.ll b/llvm/test/CodeGen/SystemZ/atomic-load-07.ll
new file mode 100644
index 0000000000000..d183cb6af3d20
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-07.ll
@@ -0,0 +1,11 @@
+; Test double atomic loads.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK: ld %f0, 0(%r2)
+; CHECK: br %r14
+  %val = load atomic double, ptr %src seq_cst, align 8
+  ret double %val
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-08.ll b/llvm/test/CodeGen/SystemZ/atomic-load-08.ll
new file mode 100644
index 0000000000000..069d2168e19af
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-08.ll
@@ -0,0 +1,20 @@
+; Test long double atomic loads. Expect a libcall.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[RET:%r[0-9]+]], %r2
+; CHECK: la %r4, 160(%r15)
+; CHECK: lghi %r2, 16
+; CHECK: lhi %r5, 5
+; CHECK: brasl %r14, __atomic_load@PLT
+; CHECK: ld [[FL:%f[0-9]+]], 160(%r15)
+; CHECK: ld [[FH:%f[0-9]+]], 168(%r15)
+; CHECK: std [[FL]], 0([[RET]])
+; CHECK: std [[FH]], 8([[RET]])
+; CHECK: br %r14
+  %val = load atomic fp128, ptr %src seq_cst, align 8
+  store fp128 %val, ptr %ret, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-06.ll b/llvm/test/CodeGen/SystemZ/atomic-store-06.ll
new file mode 100644
index 0000000000000..fd39793faefc8
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-06.ll
@@ -0,0 +1,13 @@
+; Test float atomic loads.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %src, float %val) {
+; CHECK-LABEL: f1:
+; CHECK: lgdr [[R:%r[0-9]+]], %f0
+; CHECK: srlg [[R]], [[R]], 32
+; CHECK: st [[R]], 0(%r2)
+; CHECK: br %r14
+  store atomic float %val, ptr %src seq_cst, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-07.ll b/llvm/test/CodeGen/SystemZ/atomic-store-07.ll
new file mode 100644
index 0000000000000..c904b738f2c57
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-07.ll
@@ -0,0 +1,11 @@
+; Test double atomic stores.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %dst, double %val) {
+; CHECK-LABEL: f1:
+; CHECK: std %f0, 0(%r2)
+; CHECK: br %r14
+  store atomic double %val, ptr %dst seq_cst, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-08.ll b/llvm/test/CodeGen/SystemZ/atomic-store-08.ll
new file mode 100644
index 0000000000000..b33b283e8dbd7
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-08.ll
@@ -0,0 +1,20 @@
+; Test long double atomic stores. Expect a libcall.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %dst, ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK: ld [[FL:%f[0-9]+]], 0(%r3)
+; CHECK: ld [[FH:%f[0-9]+]], 8(%r3)
+; CHECK: lgr %r3, %r2
+; CHECK: std [[FL]], 160(%r15)
+; CHECK: std [[FH]], 168(%r15)
+; CHECK: la %r4, 160(%r15)
+; CHECK: lghi %r2, 16
+; CHECK: lhi %r5, 5
+; CHECK: brasl %r14, __atomic_store@PLT
+; CHECK: br %r14
+  %val = load fp128, ptr %src, align 8
+  store atomic fp128 %val, ptr %dst seq_cst, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-01.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-01.ll
new file mode 100644
index 0000000000000..1bfa055781c98
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-01.ll
@@ -0,0 +1,22 @@
+; Test atomic float addition. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src, float %b) {
+; CHECK-LABEL: f1:
+; CHECK: le [[F:%f[0-9]+]], 0(%r2)
+; CHECK: [[L:\.L.+]]:
+; CHECK: lgdr [[RI:%r[0-9]+]], [[F]]
+; CHECK: aebr [[F]], %f0
+; CHECK: lgdr [[RO:%r[0-9]+]], [[F]]
+; CHECK: srlg [[RO]], [[RO]], 32
+; CHECK: srlg [[RI]], [[RI]], 32
+; CHECK: cs [[RI]], [[RO]], 0(%r2)
+; CHECK: sllg [[RI]], [[RI]], 32
+; CHECK: ldgr [[F]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ler %f0, [[F]]
+; CHECK: br %r14
+  %res = atomicrmw fadd ptr %src, float %b seq_cst
+  ret float %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-02.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-02.ll
new file mode 100644
index 0000000000000..7f9ee3cbec22e
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-02.ll
@@ -0,0 +1,19 @@
+; Test atomic double addition. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src, double %b) {
+; CHECK-LABEL: f1:
+; CHECK: ld [[F:%f[0-9]+]], 0(%r2)
+; CHECK: [[L:\.L.+]]:
+; CHECK: lgdr [[RI:%r[0-9]+]], [[F]]
+; CHECK: adbr [[F]], %f0
+; CHECK: lgdr [[RO:%r[0-9]+]], [[F]]
+; CHECK: csg [[RI]], [[RO]], 0(%r2)
+; CHECK: ldgr [[F]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ldr %f0, [[F]]
+; CHECK: br %r14
+  %res = atomicrmw fadd ptr %src, double %b seq_cst
+  ret double %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-03.ll
new file mode 100644
index 0000000000000..729fcbc4ac1e7
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fadd-03.ll
@@ -0,0 +1,30 @@
+; Test atomic long double addition. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src, ptr %b) {
+; CHECK-LABEL: f1:
+; CHECK: [[FBL:%f[0-9]+]], 0(%r4)
+; CHECK: [[FBH:%f[0-9]+]], 8(%r4)
+; CHECK: [[FSL:%f[0-9]+]], 0(%r3)
+; CHECK: [[FSH:%f[0-9]+]], 8(%r3)
+; CHECK: [[LABEL:\.L.+]]:
+; CHECK: lgdr [[RISH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[RISL:%r[0-9]+]], [[FSL]]
+; CHECK: axbr [[FSL]], [[FBL]]
+; CHECK: lgdr [[ROSH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[ROSL:%r[0-9]+]], [[FSL]]
+; CHECK: cdsg [[RISL]], [[ROSL]], 0(%r3)
+; CHECK: stg [[RISH]], 168(%r15)
+; CHECK: stg [[RISL]], 160(%r15)
+; CHECK: ld [[FSL]], 160(%r15)
+; CHECK: ld [[FSH]], 168(%r15)
+; CHECK: jl [[LABEL]]
+; CHECK: std [[FSL]], 0(%r2)
+; CHECK: std [[FSH]], 8(%r2)
+; CHECK: br %r14
+  %val = load fp128, ptr %b
+  %res = atomicrmw fadd ptr %src, fp128 %val seq_cst
+  store fp128 %res, ptr %ret
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-01.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-01.ll
new file mode 100644
index 0000000000000..80c43137e3a03
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-01.ll
@@ -0,0 +1,27 @@
+; Test atomic float maximum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src, float %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r2
+; CHECK: le [[FSRC:%f[0-9]+]], 0(%r2)
+; CHECK: ler [[FB:%f[0-9]+]], %f0
+; CHECK: [[L:\.L.+]]:
+; CHECK: ler %f0, [[FSRC]]
+; CHECK: ler %f2, [[FB]]
+; CHECK: brasl %r14, fmaxf@PLT
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: srlg [[RO]], [[RO]], 32
+; CHECK: lgdr [[RI:%r[0-9]+]], [[FSRC]]
+; CHECK: srlg [[RI]], [[RI]], 32
+; CHECK: cs [[RI]], [[RO]], 0([[SRC]])
+; CHECK: sllg [[RO]], [[RI]], 32
+; CHECK: ldgr [[FSRC]], [[RO]]
+; CHECK: jl [[L]]
+; CHECK: ler %f0, [[FSRC]]
+; CHECK: br %r14
+  %res = atomicrmw fmax ptr %src, float %b seq_cst
+  ret float %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-02.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-02.ll
new file mode 100644
index 0000000000000..8c245bb049f3f
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-02.ll
@@ -0,0 +1,24 @@
+; Test atomic double maximum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src, double %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[RB:%r[0-9]+]], %r2
+; CHECK: ld [[FB:%f[0-9]+]], 0(%r2)
+; CHECK: ldr [[FSRC:%f[0-9]+]], %f0
+; CHECK: [[L:\.L.+]]:
+; CHECK: ldr %f0, [[FB]]
+; CHECK: ldr %f2, [[FSRC]]
+; CHECK: brasl %r14, fmax@PLT
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: lgdr [[RI:%r[0-9]+]], [[FB]]
+; CHECK: csg [[RI]], [[RO]], 0([[RB]])
+; CHECK: ldgr [[FB]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ldr %f0, [[FB]]
+; CHECK: br %r14
+  %res = atomicrmw fmax ptr %src, double %b seq_cst
+  ret double %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-03.ll
new file mode 100644
index 0000000000000..3c8ea19f86f86
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fmax-03.ll
@@ -0,0 +1,42 @@
+; Test atomic long double maximum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src, ptr %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r3
+; CHECK: ld [[FBL:%f[0-9]+]], 0(%r4)
+; CHECK: ld [[FBH:%f[0-9]+]], 8(%r4)
+; CHECK: ld [[FSL:%f[0-9]+]], 0(%r3)
+; CHECK: ld [[FSH:%f[0-9]+]], 8(%r3)
+; CHECK: lgr [[RET:%r[0-9]+]], %r2
+; CHECK: [[L:\.L.+]]:
+; CHECK: std [[FBL]], 160(%r15)
+; CHECK: std [[FBH]], 168(%r15)
+; CHECK: la %r2, 192(%r15)
+; CHECK: la %r3, 176(%r15)
+; CHECK: la %r4, 160(%r15)
+; CHECK: std [[FSL]], 176(%r15)
+; CHECK: std [[FSH]], 184(%r15)
+; CHECK: brasl %r14, fmaxl@PLT
+; CHECK: ld [[FL:%f[0-9]+]], 192(%r15)
+; CHECK: ld [[FH:%f[0-9]+]], 200(%r15)
+; CHECK: lgdr [[RH:%r[0-9]+]], [[FH]]
+; CHECK: lgdr [[RL:%r[0-9]+]], [[FL]]
+; CHECK: lgdr [[RSH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[RSL:%r[0-9]+]], [[FSL]]
+; CHECK: cdsg [[RSL]], [[RL]], 0([[SRC]])
+; CHECK: stg [[RSH]], 216(%r15)
+; CHECK: stg [[RSL]], 208(%r15)
+; CHECK: ld [[FSL]], 208(%r15)
+; CHECK: ld [[FSH]], 216(%r15)
+; CHECK: jl [[L]]
+; CHECK: std [[FSL]], 0([[RET]])
+; CHECK: std [[FSH]], 8([[RET]])
+; CHECK: br %r14
+  %val = load fp128, ptr %b
+  %res = atomicrmw fmax ptr %src, fp128 %val seq_cst
+  store fp128 %res, ptr %ret
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-01.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-01.ll
new file mode 100644
index 0000000000000..c67b02e688de3
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-01.ll
@@ -0,0 +1,27 @@
+; Test atomic float minimum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src, float %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r2
+; CHECK: le [[FSRC:%f[0-9]+]], 0(%r2)
+; CHECK: ler [[FB:%f[0-9]+]], %f0
+; CHECK: [[L:\.L.+]]:
+; CHECK: ler %f0, [[FSRC]]
+; CHECK: ler %f2, [[FB]]
+; CHECK: brasl %r14, fminf@PLT
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: srlg [[RO]], [[RO]], 32
+; CHECK: lgdr [[RI:%r[0-9]+]], [[FSRC]]
+; CHECK: srlg [[RI]], [[RI]], 32
+; CHECK: cs [[RI]], [[RO]], 0([[SRC]])
+; CHECK: sllg [[RO]], [[RI]], 32
+; CHECK: ldgr [[FSRC]], [[RO]]
+; CHECK: jl [[L]]
+; CHECK: ler %f0, [[FSRC]]
+; CHECK: br %r14
+  %res = atomicrmw fmin ptr %src, float %b seq_cst
+  ret float %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-02.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-02.ll
new file mode 100644
index 0000000000000..6691a8b21d11b
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-02.ll
@@ -0,0 +1,24 @@
+; Test atomic double minimum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src, double %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r2
+; CHECK: ld [[FSRC:%f[0-9]+]], 0(%r2)
+; CHECK: ldr [[FB:%f[0-9]+]], %f0
+; CHECK: [[L:\.L.+]]:
+; CHECK: ldr %f0, [[FSRC]]
+; CHECK: ldr %f2, [[FB]]
+; CHECK: brasl %r14, fmin@PLT
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: lgdr [[RI:%r[0-9]+]], [[FSRC]]
+; CHECK: csg [[RI]], [[RO]], 0([[SRC]])
+; CHECK: ldgr [[FSRC]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ldr %f0, [[FSRC]]
+; CHECK: br %r14
+  %res = atomicrmw fmin ptr %src, double %b seq_cst
+  ret double %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-03.ll
new file mode 100644
index 0000000000000..dfa2cc021d166
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fmin-03.ll
@@ -0,0 +1,42 @@
+; Test atomic long double minimum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src, ptr %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r3
+; CHECK: ld [[FBL:%f[0-9]+]], 0(%r4)
+; CHECK: ld [[FBH:%f[0-9]+]], 8(%r4)
+; CHECK: ld [[FSL:%f[0-9]+]], 0(%r3)
+; CHECK: ld [[FSH:%f[0-9]+]], 8(%r3)
+; CHECK: lgr [[RET:%r[0-9]+]], %r2
+; CHECK: [[L:\.L.+]]:
+; CHECK: std [[FBL]], 160(%r15)
+; CHECK: std [[FBH]], 168(%r15)
+; CHECK: la %r2, 192(%r15)
+; CHECK: la %r3, 176(%r15)
+; CHECK: la %r4, 160(%r15)
+; CHECK: std [[FSL]], 176(%r15)
+; CHECK: std [[FSH]], 184(%r15)
+; CHECK: brasl %r14, fminl@PLT
+; CHECK: ld [[FL:%f[0-9]+]], 192(%r15)
+; CHECK: ld [[FH:%f[0-9]+]], 200(%r15)
+; CHECK: lgdr [[RH:%r[0-9]+]], [[FH]]
+; CHECK: lgdr [[RL:%r[0-9]+]], [[FL]]
+; CHECK: lgdr [[RSH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[RSL:%r[0-9]+]], [[FSL]]
+; CHECK: cdsg [[RSL]], [[RL]], 0([[SRC]])
+; CHECK: stg [[RSH]], 216(%r15)
+; CHECK: stg [[RSL]], 208(%r15)
+; CHECK: ld [[FSL]], 208(%r15)
+; CHECK: ld [[FSH]], 216(%r15)
+; CHECK: jl [[L]]
+; CHECK: std [[FSL]], 0([[RET]])
+; CHECK: std [[FSH]], 8([[RET]])
+; CHECK: br %r14
+  %val = load fp128, ptr %b
+  %res = atomicrmw fmin ptr %src, fp128 %val seq_cst
+  store fp128 %res, ptr %ret
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-01.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-01.ll
new file mode 100644
index 0000000000000..3f4ad31762753
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-01.ll
@@ -0,0 +1,22 @@
+; Test atomic float subtraction. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src, float %b) {
+; CHECK-LABEL: f1:
+; CHECK: le [[F:%f[0-9]+]], 0(%r2)
+; CHECK: [[L:\.L.+]]:
+; CHECK: lgdr [[RI:%r[0-9]+]], [[F]]
+; CHECK: sebr [[F]], %f0
+; CHECK: lgdr [[RO:%r[0-9]+]], [[F]]
+; CHECK: srlg [[RO]], [[RO]], 32
+; CHECK: srlg [[RI]], [[RI]], 32
+; CHECK: cs [[RI]], [[RO]], 0(%r2)
+; CHECK: sllg [[RI]], [[RI]], 32
+; CHECK: ldgr [[F]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ler %f0, [[F]]
+; CHECK: br %r14
+  %res = atomicrmw fsub ptr %src, float %b seq_cst
+  ret float %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-02.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-02.ll
new file mode 100644
index 0000000000000..69071f9cbe029
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-02.ll
@@ -0,0 +1,19 @@
+; Test atomic double subtraction. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src, double %b) {
+; CHECK-LABEL: f1:
+; CHECK: ld [[F:%f[0-9]+]], 0(%r2)
+; CHECK: [[L:\.L.+]]:
+; CHECK: lgdr [[RI:%r[0-9]+]], [[F]]
+; CHECK: sdbr [[F]], %f0
+; CHECK: lgdr [[RO:%r[0-9]+]], [[F]]
+; CHECK: csg [[RI]], [[RO]], 0(%r2)
+; CHECK: ldgr [[F]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ldr %f0, [[F]]
+; CHECK: br %r14
+  %res = atomicrmw fsub ptr %src, double %b seq_cst
+  ret double %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-03.ll
new file mode 100644
index 0000000000000..aacbc942f1a56
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-fsub-03.ll
@@ -0,0 +1,30 @@
+; Test atomic long double subtraction. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src, ptr %b) {
+; CHECK-LABEL: f1:
+; CHECK: [[FBL:%f[0-9]+]], 0(%r4)
+; CHECK: [[FBH:%f[0-9]+]], 8(%r4)
+; CHECK: [[FSL:%f[0-9]+]], 0(%r3)
+; CHECK: [[FSH:%f[0-9]+]], 8(%r3)
+; CHECK: [[LABEL:\.L.+]]:
+; CHECK: lgdr [[RISH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[RISL:%r[0-9]+]], [[FSL]]
+; CHECK: sxbr [[FSL]], [[FBL]]
+; CHECK: lgdr [[ROSH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[ROSL:%r[0-9]+]], [[FSL]]
+; CHECK: cdsg [[RISL]], [[ROSL]], 0(%r3)
+; CHECK: stg [[RISH]], 168(%r15)
+; CHECK: stg [[RISL]], 160(%r15)
+; CHECK: ld [[FSL]], 160(%r15)
+; CHECK: ld [[FSH]], 168(%r15)
+; CHECK: jl [[LABEL]]
+; CHECK: std [[FSL]], 0(%r2)
+; CHECK: std [[FSH]], 8(%r2)
+; CHECK: br %r14
+  %val = load fp128, ptr %b
+  %res = atomicrmw fsub ptr %src, fp128 %val seq_cst
+  store fp128 %res, ptr %ret
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-udec_wrap.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-udec_wrap.ll
new file mode 100644
index 0000000000000..d6427fc29f051
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-udec_wrap.ll
@@ -0,0 +1,28 @@
+; Test decrementing until to a minimum value. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define i64 @f1(ptr %src, i64 %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r2
+; CHECK: lg [[RI:%r[0-9]+]], 0(%r2)
+; CHECK: j [[L2:\.L.+]]
+; CHECK: [[L1:\.L.+]]:
+; CHECK: csg [[RI]], [[RO:%r[0-9]+]], 0([[SRC]])
+; CHECK: je [[L4:\.L.+]]
+; CHECK: [[L2]]:
+; CHECK: lgr [[RO]], [[RI]]
+; CHECK: slgfi [[RO]], 1
+; CHECK: lgr [[RB:%r[0-9]+]], %r3
+; CHECK: clgrjh [[RI]], %r3, [[L3:\.L.+]]
+; CHECK: lgr [[RB]], [[RO]]
+; CHECK: [[L3]]:
+; CHECK: lgr [[RO]], [[RI]]
+; CHECK: slgfi [[RO]], 1
+; CHECK: lgr [[RO]], %r3
+; CHECK: jle [[L1]]
+; CHECK: [[L4]]:
+; CHECK: br %r14
+  %res = atomicrmw udec_wrap ptr %src, i64 %b seq_cst
+  ret i64 %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-uinc_wrap.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-uinc_wrap.ll
new file mode 100644
index 0000000000000..cf90d75653595
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-uinc_wrap.ll
@@ -0,0 +1,22 @@
+; Test incrementing up to a maximum value. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define i64 @f1(ptr %src, i64 %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r2
+; CHECK: lg [[RI:%r[0-9]+]], 0(%r2)
+; CHECK: j [[L2:\.L.+]]
+; CHECK: [[L1:\.L.+]]:
+; CHECK: csg [[RI]], [[RO:%r[0-9]+]], 0([[SRC]])
+; CHECK: je [[L3:\.L.+]]
+; CHECK: [[L2]]:
+; CHECK: lghi [[RO]], 0
+; CHECK: clgrjhe [[RI]], %r3, [[L1]]
+; CHECK: la [[RO]], 1([[RI]])
+; CHECK: j [[L1]]
+; CHECK: [[L3]]:
+; CHECK: br %r14
+  %res = atomicrmw uinc_wrap ptr %src, i64 %b seq_cst
+  ret i64 %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-05.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-05.ll
new file mode 100644
index 0000000000000..d86e476115c38
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-05.ll
@@ -0,0 +1,17 @@
+; Test float atomic exchange.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src, float %b) {
+; CHECK-LABEL: f1:
+; CHECK: l [[RI:%r[0-9]+]], 0(%r2)
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: [[LABEL:\.[^:]*]]:
+; CHECK: cs [[RI]], [[RO]], 0(%r2)
+; CHECK: jl [[LABEL]]
+; CHECK: sllg [[RI]], [[RI]], 32
+; CHECK: ldgr %f0, [[RI]]
+; CHECK: br %r14
+  %res = atomicrmw xchg ptr %src, float %b seq_cst
+  ret float %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-06.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-06.ll
new file mode 100644
index 0000000000000..9b78e783035dd
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-06.ll
@@ -0,0 +1,16 @@
+; Test double atomic exchange.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src, double %b) {
+; CHECK-LABEL: f1:
+; CHECK: lg [[RI:%r[0-9]+]], 0(%r2)
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: [[LABEL:\.[^:]*]]:
+; CHECK: csg [[RI]], [[RO]], 0(%r2)
+; CHECK: jl [[LABEL]]
+; CHECK: ldgr %f0, [[RI]]
+; CHECK: br %r14
+  %res = atomicrmw xchg ptr %src, double %b seq_cst
+  ret double %res
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-07.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-07.ll
new file mode 100644
index 0000000000000..80cc85158e45e
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-07.ll
@@ -0,0 +1,24 @@
+; Test long double atomic exchange.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src, ptr %b) {
+; CHECK-LABEL: f1:
+; CHECK: lg [[RH:%r[0-9]+]], 8(%r4)
+; CHECK: lgr [[RET:%r[0-9]+]], %r2
+; CHECK: lg [[RL:%r[0-9]+]], 0(%r4)
+; CHECK: stg [[RH]], 168(%r15)
+; CHECK: la %r2, 176(%r15)
+; CHECK: la %r4, 160(%r15)
+; CHECK: stg [[RL]], 160(%r15)
+; CHECK: brasl %r14, __sync_lock_test_and_set_16@PLT
+; CHECK: lg [[RH2:%r[0-9]+]], 184(%r15)
+; CHECK: lg [[RL2:%r[0-9]+]], 176(%r15)
+; CHECK: stg [[RH]], 8([[RET]])
+; CHECK: stg [[RL]], 0([[RET]])
+; CHECK: br %r14
+  %val = load fp128, ptr %b, align 8
+  %res = atomicrmw xchg ptr %src, fp128 %val seq_cst
+  store fp128 %res, ptr %ret, align 8
+  ret void
+}