Refine atomic operations for Linux/ARM.

base/atomicops_internals_arm_gcc.h

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // This file is an internal atomic implementation, use base/atomicops.h instead.
 //
 // LinuxKernelCmpxchg and Barrier_AtomicIncrement are from Google Gears.
 
 #ifndef BASE_ATOMICOPS_INTERNALS_ARM_GCC_H_
 #define BASE_ATOMICOPS_INTERNALS_ARM_GCC_H_
@@ skipping 52 matching lines @@
   int reloop;
   do {
     // The following is equivalent to:
     //
     //   prev_value = LDREX(ptr)
     //   reloop = 0
     //   if (prev_value != old_value)
     //      reloop = STREX(ptr, new_value)
     __asm__ __volatile__("    ldrex %0, [%3]\n"
                          "    mov %1, #0\n"
-                         "    teq %0, %4\n"
+                         "    cmp %0, %4\n"
 #ifdef __thumb2__
                          "    it eq\n"
 #endif
                          "    strexeq %1, %5, [%3]\n"
                          : "=&r"(prev_value), "=&r"(reloop), "+m"(*ptr)
                          : "r"(ptr), "r"(old_value), "r"(new_value)
                          : "cc", "memory");
   } while (reloop != 0);
   return prev_value;
 }
@@ skipping 44 matching lines @@
   MemoryBarrier();
   return result;
 }
 
 inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr,
                                          Atomic32 new_value) {
   Atomic32 old_value;
   int reloop;
   do {
     // old_value = LDREX(ptr)
-    // fail = STREX(ptr, new_value)
+    // reloop = STREX(ptr, new_value)
     __asm__ __volatile__("   ldrex %0, [%3]\n"
                          "   strex %1, %4, [%3]\n"
                          : "=&r"(old_value), "=&r"(reloop), "+m"(*ptr)
                          : "r"(ptr), "r"(new_value)
                          : "cc", "memory");
   } while (reloop != 0);
   return old_value;
 }
 
 // This tests against any known ARMv5 variant.
@@ skipping 76 matching lines @@
       return prev_value;
     }
     if (!LinuxKernelCmpxchg(old_value, new_value, ptr))
       return old_value;
   }
 }
 
 inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr,
                                        Atomic32 old_value,
                                        Atomic32 new_value) {
-  // Use NoBarrier_CompareAndSwap(), because its implementation
-  // ensures that all stores happen through the kernel helper
-  // which always implement a full barrier.
-  return NoBarrier_CompareAndSwap(ptr, old_value, new_value);
+  // This could be implemented as:
+  //    MemoryBarrier();
+  //    return NoBarrier_CompareAndSwap();
+  //
+  // But would use 3 barriers per succesful CAS. To save performance,
+  // use Acquire_CompareAndSwap(). Its implementation guarantees that:
+  // - A succesful swap uses only 2 barriers (in the kernel helper).
+  // - An early return due to (prev_value != old_value) performs
+  //   a memory barrier with no store, which is equivalent to the
+  //   generic implementation above.
+  return Acquire_CompareAndSwap(ptr, old_value, new_value);

[Dmitry Vyukov, 2013/07/15 14:42:32]

A failing Release_CompareAndSwap() does not need any memory barriers. Release
makes sense only when an acquire operation loads the value stored by side effect
of the release operation. A failing release CAS does not mutate anything, so
nobody can possibly observe it's side effect.

See the relevant part of the C++ standard:

29.6.5/21
Effects: Atomically, compares the contents of the memory pointed to by object or
by this for equality
with that in expected, and if true, replaces the contents of the memory pointed
to by object or by
this with that in desired, and if false, updates the contents of the memory in
expected with the
contents of the memory pointed to by object or by this. Further, if the
comparison is true, memory
is affected according to the value of success, and if the comparison is false,
memory is affected
according to the value of failure. When only one memory_order argument is
supplied, the value of
success is order, and the value of failure is order except that a value of
memory_order_acq_rel
shall be replaced by the value memory_order_acquire and a value of
memory_order_release shall
be replaced by the value memory_order_relaxed. If the operation returns true,
these operations are
atomic read-modify-write operations (1.10). Otherwise, these operations are
atomic load operations.

 }
 
 #else
 #  error "Your CPU's ARM architecture is not supported yet"
 #endif
 
 // NOTE: Atomicity of the following load and store operations is only
 // guaranteed in case of 32-bit alignement of |ptr| values.
 
 inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) {
@@ skipping 20 matching lines @@
 
 inline Atomic32 Release_Load(volatile const Atomic32* ptr) {
   MemoryBarrier();
   return *ptr;
 }
 
 }  // namespace base::subtle
 }  // namespace base
 
 #endif  // BASE_ATOMICOPS_INTERNALS_ARM_GCC_H_