opal/asm: add opal_atomic_compare_exchange_strong functions

This commit adds a new set of compare-and-exchange functions. These
functions have a signature similar to the functions found in C11. The
old cmpset functions are now deprecated and defined in terms of the
new compare-and-exchange functions. All asm backends have been
updated.

Signed-off-by: Nathan Hjelm <[email protected]>

Author: hjelmn

opal/class/opal_fifo.h

@@ -76,7 +76,7 @@ static inline bool opal_fifo_is_empty( opal_fifo_t* fifo )
     return opal_fifo_head (fifo) == &fifo->opal_fifo_ghost;
 }
 
-#if OPAL_HAVE_ATOMIC_CMPSET_128
+#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
 
 /* Add one element to the FIFO. We will return the last head of the list
  * to allow the upper level to detect if this element is the first one in the

opal/class/opal_lifo.h

@@ -36,8 +36,8 @@
 BEGIN_C_DECLS
 
 /* NTH: temporarily suppress warnings about this not being defined */
-#if !defined(OPAL_HAVE_ATOMIC_CMPSET_128)
-#define OPAL_HAVE_ATOMIC_CMPSET_128 0
+#if !defined(OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128)
+#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
 #endif
 
 /**
@@ -50,7 +50,7 @@ union opal_counted_pointer_t {
         /** list item pointer */
         volatile opal_list_item_t * volatile item;
     } data;
-#if OPAL_HAVE_ATOMIC_CMPSET_128 && HAVE_OPAL_INT128_T
+#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 && HAVE_OPAL_INT128_T
     /** used for atomics when there is a cmpset that can operate on
      * two 64-bit values */
     opal_int128_t value;
@@ -59,7 +59,7 @@ union opal_counted_pointer_t {
 typedef union opal_counted_pointer_t opal_counted_pointer_t;
 
 
-#if OPAL_HAVE_ATOMIC_CMPSET_128
+#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
 
 /* Add one element to the FIFO. We will return the last head of the list
  * to allow the upper level to detect if this element is the first one in the
@@ -110,7 +110,7 @@ static inline bool opal_lifo_is_empty( opal_lifo_t* lifo )
 }
 
 
-#if OPAL_HAVE_ATOMIC_CMPSET_128
+#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
 
 /* Add one element to the LIFO. We will return the last head of the list
  * to allow the upper level to detect if this element is the first one in the

opal/include/opal/sys/arm/atomic.h

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -11,6 +12,8 @@
  *                         All rights reserved.
  * Copyright (c) 2010      IBM Corporation.  All rights reserved.
  * Copyright (c) 2010      ARM ltd.  All rights reserved.
+ * Copyright (c) 2017      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -104,12 +107,12 @@ void opal_atomic_isync(void)
 
 #if (OPAL_GCC_INLINE_ASSEMBLY && (OPAL_ASM_ARM_VERSION >= 6))
 
-#define OPAL_HAVE_ATOMIC_CMPSET_32 1
+#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
 #define OPAL_HAVE_ATOMIC_MATH_32 1
-static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
-                                              int32_t oldval, int32_t newval)
+static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
 {
-  int32_t ret, tmp;
+  int32_t prev, tmp;
+  bool ret;
 
    __asm__ __volatile__ (
                          "1:  ldrex   %0, [%2]        \n"
@@ -120,87 +123,86 @@ static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
                          "    bne     1b              \n"
                          "2:                          \n"
 
-                         : "=&r" (ret), "=&r" (tmp)
-                         : "r" (addr), "r" (oldval), "r" (newval)
+                         : "=&r" (prev), "=&r" (tmp)
+                         : "r" (addr), "r" (*oldval), "r" (newval)
                          : "cc", "memory");
 
-   return (ret == oldval);
+   ret = (prev == *oldval);
+   *oldval = prev;
+   return ret;
 }
 
 /* these two functions aren't inlined in the non-gcc case because then
    there would be two function calls (since neither cmpset_32 nor
    atomic_?mb can be inlined).  Instead, we "inline" them by hand in
    the assembly, meaning there is one function call overhead instead
    of two */
-static inline bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr,
-                                                  int32_t oldval, int32_t newval)
+static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
 {
     bool rc;
 
-    rc = opal_atomic_bool_cmpset_32(addr, oldval, newval);
+    rc = opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
     opal_atomic_rmb();
 
     return rc;
 }
 
 
-static inline bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr,
-                                                  int32_t oldval, int32_t newval)
+static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
 {
     opal_atomic_wmb();
-    return opal_atomic_bool_cmpset_32(addr, oldval, newval);
+    return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
 }
 
 #if (OPAL_ASM_SUPPORT_64BIT == 1)
 
-#define OPAL_HAVE_ATOMIC_CMPSET_64 1
-static inline bool opal_atomic_bool_cmpset_64(volatile int64_t *addr,
-                                              int64_t oldval, int64_t newval)
+#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
+static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
 {
-  int64_t ret;
-  int tmp;
-
-
-   __asm__ __volatile__ (
-                         "1:  ldrexd  %0, %H0, [%2]           \n"
-                         "    cmp     %0, %3                  \n"
-                         "    it      eq                      \n"
-                         "    cmpeq   %H0, %H3                \n"
-                         "    bne     2f                      \n"
-                         "    strexd  %1, %4, %H4, [%2]       \n"
-                         "    cmp     %1, #0                  \n"
-                         "    bne     1b                      \n"
-                         "2:                                    \n"
-
-                         : "=&r" (ret), "=&r" (tmp)
-                         : "r" (addr), "r" (oldval), "r" (newval)
-                         : "cc", "memory");
-
-   return (ret == oldval);
+    int64_t prev;
+    int tmp;
+    bool ret;
+
+    __asm__ __volatile__ (
+                          "1:  ldrexd  %0, %H0, [%2]           \n"
+                          "    cmp     %0, %3                  \n"
+                          "    it      eq                      \n"
+                          "    cmpeq   %H0, %H3                \n"
+                          "    bne     2f                      \n"
+                          "    strexd  %1, %4, %H4, [%2]       \n"
+                          "    cmp     %1, #0                  \n"
+                          "    bne     1b                      \n"
+                          "2:                                    \n"
+
+                          : "=&r" (prev), "=&r" (tmp)
+                          : "r" (addr), "r" (*oldval), "r" (newval)
+                          : "cc", "memory");
+
+    ret = (prev == *oldval);
+    *oldval = prev;
+    return ret;
 }
 
 /* these two functions aren't inlined in the non-gcc case because then
    there would be two function calls (since neither cmpset_64 nor
    atomic_?mb can be inlined).  Instead, we "inline" them by hand in
    the assembly, meaning there is one function call overhead instead
    of two */
-static inline bool opal_atomic_bool_cmpset_acq_64(volatile int64_t *addr,
-                                                  int64_t oldval, int64_t newval)
+static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
 {
     bool rc;
 
-    rc = opal_atomic_bool_cmpset_64(addr, oldval, newval);
+    rc = opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
     opal_atomic_rmb();
 
     return rc;
 }
 
 
-static inline bool opal_atomic_bool_cmpset_rel_64(volatile int64_t *addr,
-                                                  int64_t oldval, int64_t newval)
+static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
 {
     opal_atomic_wmb();
-    return opal_atomic_bool_cmpset_64(addr, oldval, newval);
+    return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
 }
 
 #endif
@@ -247,30 +249,6 @@ static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int dec)
    return t;
 }
 
-#else /* OPAL_ASM_ARM_VERSION <=5 or no GCC inline assembly */
-
-#define OPAL_HAVE_ATOMIC_CMPSET_32 1
-#define __kuser_cmpxchg (*((int (*)(int, int, volatile int*))(0xffff0fc0)))
-static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
-                                              int32_t oldval, int32_t newval)
-{
-    return !(__kuser_cmpxchg(oldval, newval, addr));
-}
-
-static inline bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr,
-                                                  int32_t oldval, int32_t newval)
-{
-    /* kernel function includes all necessary memory barriers */
-    return opal_atomic_bool_cmpset_32(addr, oldval, newval);
-}
-
-static inline bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr,
-                                                  int32_t oldval, int32_t newval)
-{
-    /* kernel function includes all necessary memory barriers */
-    return opal_atomic_bool_cmpset_32(addr, oldval, newval);
-}
-
 #endif
 
 #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */