| OLD | NEW |
| 1 // Copyright (c) 2011, Google Inc. | 1 // Copyright (c) 2011, Google Inc. |
| 2 // All rights reserved. | 2 // All rights reserved. |
| 3 // | 3 // |
| 4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
| 5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
| 6 // met: | 6 // met: |
| 7 // | 7 // |
| 8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
| 9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
| 10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
| (...skipping 24 matching lines...) Expand all Loading... |
| 35 // | 35 // |
| 36 // This code implements ARM atomics for architectures V6 and newer. | 36 // This code implements ARM atomics for architectures V6 and newer. |
| 37 | 37 |
| 38 #ifndef BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ | 38 #ifndef BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ |
| 39 #define BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ | 39 #define BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ |
| 40 | 40 |
| 41 #include <stdio.h> | 41 #include <stdio.h> |
| 42 #include <stdlib.h> | 42 #include <stdlib.h> |
| 43 #include "base/basictypes.h" // For COMPILE_ASSERT | 43 #include "base/basictypes.h" // For COMPILE_ASSERT |
| 44 | 44 |
| 45 // The LDREXD and STREXD instructions in ARM all v7 variants or above. In v6, | |
| 46 // only some variants support it. For simplicity, we only use exclusive | |
| 47 // 64-bit load/store in V7 or above. | |
| 48 #if defined(ARMV7) | |
| 49 # define BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD | |
| 50 #endif | |
| 51 | |
| 52 typedef int32_t Atomic32; | 45 typedef int32_t Atomic32; |
| 53 | 46 |
| 54 namespace base { | 47 namespace base { |
| 55 namespace subtle { | 48 namespace subtle { |
| 56 | 49 |
| 57 typedef int64_t Atomic64; | 50 typedef int64_t Atomic64; |
| 58 | 51 |
| 59 // 32-bit low-level ops | 52 // 32-bit low-level ops |
| 60 | 53 |
| 61 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, | 54 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, |
| 62 Atomic32 old_value, | 55 Atomic32 old_value, |
| 63 Atomic32 new_value) { | 56 Atomic32 new_value) { |
| 64 Atomic32 oldval, res; | 57 Atomic32 oldval, res; |
| 65 do { | 58 do { |
| 66 __asm__ __volatile__( | 59 __asm__ __volatile__( |
| 67 "ldrex %1, [%3]\n" | 60 "ldrex %1, [%3]\n" |
| 68 "mov %0, #0\n" | 61 "mov %0, #0\n" |
| 69 "teq %1, %4\n" | 62 "teq %1, %4\n" |
| 70 // The following IT (if-then) instruction is needed for the subsequent | |
| 71 // conditional instruction STREXEQ when compiling in THUMB mode. | |
| 72 // In ARM mode, the compiler/assembler will not generate any code for it. | |
| 73 "it eq\n" | |
| 74 "strexeq %0, %5, [%3]\n" | 63 "strexeq %0, %5, [%3]\n" |
| 75 : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) | 64 : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) |
| 76 : "r" (ptr), "Ir" (old_value), "r" (new_value) | 65 : "r" (ptr), "Ir" (old_value), "r" (new_value) |
| 77 : "cc"); | 66 : "cc"); |
| 78 } while (res); | 67 } while (res); |
| 79 return oldval; | 68 return oldval; |
| 80 } | 69 } |
| 81 | 70 |
| 82 inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, | 71 inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, |
| 83 Atomic32 new_value) { | 72 Atomic32 new_value) { |
| (...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 168 Atomic32 value = *ptr; | 157 Atomic32 value = *ptr; |
| 169 MemoryBarrier(); | 158 MemoryBarrier(); |
| 170 return value; | 159 return value; |
| 171 } | 160 } |
| 172 | 161 |
| 173 inline Atomic32 Release_Load(volatile const Atomic32* ptr) { | 162 inline Atomic32 Release_Load(volatile const Atomic32* ptr) { |
| 174 MemoryBarrier(); | 163 MemoryBarrier(); |
| 175 return *ptr; | 164 return *ptr; |
| 176 } | 165 } |
| 177 | 166 |
| 178 // 64-bit versions are only available if LDREXD and STREXD instructions | 167 // 64-bit versions are not implemented yet. |
| 179 // are available. | |
| 180 #ifdef BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD | |
| 181 | |
| 182 #define BASE_HAS_ATOMIC64 1 | |
| 183 | |
| 184 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, | |
| 185 Atomic64 old_value, | |
| 186 Atomic64 new_value) { | |
| 187 Atomic64 oldval, res; | |
| 188 do { | |
| 189 __asm__ __volatile__( | |
| 190 "ldrexd %1, [%3]\n" | |
| 191 "mov %0, #0\n" | |
| 192 "teq %Q1, %Q4\n" | |
| 193 // The following IT (if-then) instructions are needed for the subsequent | |
| 194 // conditional instructions when compiling in THUMB mode. | |
| 195 // In ARM mode, the compiler/assembler will not generate any code for it. | |
| 196 "it eq\n" | |
| 197 "teqeq %R1, %R4\n" | |
| 198 "it eq\n" | |
| 199 "strexdeq %0, %5, [%3]\n" | |
| 200 : "=&r" (res), "=&r" (oldval), "+Q" (*ptr) | |
| 201 : "r" (ptr), "Ir" (old_value), "r" (new_value) | |
| 202 : "cc"); | |
| 203 } while (res); | |
| 204 return oldval; | |
| 205 } | |
| 206 | |
| 207 inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, | |
| 208 Atomic64 new_value) { | |
| 209 int store_failed; | |
| 210 Atomic64 old; | |
| 211 __asm__ __volatile__( | |
| 212 "1:\n" | |
| 213 "ldrexd %1, [%2]\n" | |
| 214 "strexd %0, %3, [%2]\n" | |
| 215 "teq %0, #0\n" | |
| 216 "bne 1b" | |
| 217 : "=&r" (store_failed), "=&r" (old) | |
| 218 : "r" (ptr), "r" (new_value) | |
| 219 : "cc", "memory"); | |
| 220 return old; | |
| 221 } | |
| 222 | |
| 223 inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, | |
| 224 Atomic64 increment) { | |
| 225 int store_failed; | |
| 226 Atomic64 res; | |
| 227 __asm__ __volatile__( | |
| 228 "1:\n" | |
| 229 "ldrexd %1, [%2]\n" | |
| 230 "adds %Q1, %Q1, %Q3\n" | |
| 231 "adc %R1, %R1, %R3\n" | |
| 232 "strexd %0, %1, [%2]\n" | |
| 233 "teq %0, #0\n" | |
| 234 "bne 1b" | |
| 235 : "=&r" (store_failed), "=&r"(res) | |
| 236 : "r" (ptr), "r"(increment) | |
| 237 : "cc", "memory"); | |
| 238 return res; | |
| 239 } | |
| 240 | |
| 241 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, | |
| 242 Atomic64 increment) { | |
| 243 int store_failed; | |
| 244 Atomic64 res; | |
| 245 __asm__ __volatile__( | |
| 246 "1:\n" | |
| 247 "ldrexd %1, [%2]\n" | |
| 248 "adds %Q1, %Q1, %Q3\n" | |
| 249 "adc %R1, %R1, %R3\n" | |
| 250 "dmb\n" | |
| 251 "strexd %0, %1, [%2]\n" | |
| 252 "teq %0, #0\n" | |
| 253 "bne 1b" | |
| 254 : "=&r" (store_failed), "=&r"(res) | |
| 255 : "r" (ptr), "r"(increment) | |
| 256 : "cc", "memory"); | |
| 257 return res; | |
| 258 } | |
| 259 | |
| 260 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { | |
| 261 int store_failed; | |
| 262 Atomic64 dummy; | |
| 263 __asm__ __volatile__( | |
| 264 "1:\n" | |
| 265 // Dummy load to lock cache line. | |
| 266 "ldrexd %1, [%3]\n" | |
| 267 "strexd %0, %2, [%3]\n" | |
| 268 "teq %0, #0\n" | |
| 269 "bne 1b" | |
| 270 : "=&r" (store_failed), "=&r"(dummy) | |
| 271 : "r"(value), "r" (ptr) | |
| 272 : "cc", "memory"); | |
| 273 } | |
| 274 | |
| 275 inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { | |
| 276 Atomic64 res; | |
| 277 __asm__ __volatile__( | |
| 278 "ldrexd %0, [%1]\n" | |
| 279 "clrex\n" | |
| 280 : "=r" (res) | |
| 281 : "r"(ptr), "Q"(*ptr)); | |
| 282 return res; | |
| 283 } | |
| 284 | |
| 285 #else // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD | |
| 286 | 168 |
| 287 inline void NotImplementedFatalError(const char *function_name) { | 169 inline void NotImplementedFatalError(const char *function_name) { |
| 288 fprintf(stderr, "64-bit %s() not implemented on this platform\n", | 170 fprintf(stderr, "64-bit %s() not implemented on this platform\n", |
| 289 function_name); | 171 function_name); |
| 290 abort(); | 172 abort(); |
| 291 } | 173 } |
| 292 | 174 |
| 293 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, | 175 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, |
| 294 Atomic64 old_value, | 176 Atomic64 old_value, |
| 295 Atomic64 new_value) { | 177 Atomic64 new_value) { |
| (...skipping 16 matching lines...) Expand all Loading... |
| 312 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, | 194 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, |
| 313 Atomic64 increment) { | 195 Atomic64 increment) { |
| 314 NotImplementedFatalError("Barrier_AtomicIncrement"); | 196 NotImplementedFatalError("Barrier_AtomicIncrement"); |
| 315 return 0; | 197 return 0; |
| 316 } | 198 } |
| 317 | 199 |
| 318 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { | 200 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { |
| 319 NotImplementedFatalError("NoBarrier_Store"); | 201 NotImplementedFatalError("NoBarrier_Store"); |
| 320 } | 202 } |
| 321 | 203 |
| 204 inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { |
| 205 NotImplementedFatalError("Acquire_Store64"); |
| 206 } |
| 207 |
| 208 inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { |
| 209 NotImplementedFatalError("Release_Store"); |
| 210 } |
| 211 |
| 322 inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { | 212 inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { |
| 323 NotImplementedFatalError("NoBarrier_Load"); | 213 NotImplementedFatalError("NoBarrier_Load"); |
| 324 return 0; | 214 return 0; |
| 325 } | 215 } |
| 326 | 216 |
| 327 #endif // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD | |
| 328 | |
| 329 inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { | |
| 330 NoBarrier_Store(ptr, value); | |
| 331 MemoryBarrier(); | |
| 332 } | |
| 333 | |
| 334 inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { | |
| 335 MemoryBarrier(); | |
| 336 NoBarrier_Store(ptr, value); | |
| 337 } | |
| 338 | |
| 339 inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { | 217 inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { |
| 340 Atomic64 value = NoBarrier_Load(ptr); | 218 NotImplementedFatalError("Atomic64 Acquire_Load"); |
| 341 MemoryBarrier(); | 219 return 0; |
| 342 return value; | |
| 343 } | 220 } |
| 344 | 221 |
| 345 inline Atomic64 Release_Load(volatile const Atomic64* ptr) { | 222 inline Atomic64 Release_Load(volatile const Atomic64* ptr) { |
| 346 MemoryBarrier(); | 223 NotImplementedFatalError("Atomic64 Release_Load"); |
| 347 return NoBarrier_Load(ptr); | 224 return 0; |
| 348 } | 225 } |
| 349 | 226 |
| 350 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, | 227 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, |
| 351 Atomic64 old_value, | 228 Atomic64 old_value, |
| 352 Atomic64 new_value) { | 229 Atomic64 new_value) { |
| 353 Atomic64 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); | 230 NotImplementedFatalError("Atomic64 Acquire_CompareAndSwap"); |
| 354 MemoryBarrier(); | 231 return 0; |
| 355 return value; | |
| 356 } | 232 } |
| 357 | 233 |
| 358 inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, | 234 inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, |
| 359 Atomic64 old_value, | 235 Atomic64 old_value, |
| 360 Atomic64 new_value) { | 236 Atomic64 new_value) { |
| 361 MemoryBarrier(); | 237 NotImplementedFatalError("Atomic64 Release_CompareAndSwap"); |
| 362 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); | 238 return 0; |
| 363 } | 239 } |
| 364 | 240 |
| 365 } // namespace subtle ends | 241 } // namespace subtle ends |
| 366 } // namespace base ends | 242 } // namespace base ends |
| 367 | 243 |
| 368 #endif // BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ | 244 #endif // BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ |
| OLD | NEW |