1
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
2
// Use of this source code is governed by a BSD-style license that can be
3
// found in the LICENSE file. See the AUTHORS file for names of contributors.
5
// AtomicPointer provides storage for a lock-free pointer.
6
// Platform-dependent implementation of AtomicPointer:
7
// - If the platform provides a cheap barrier, we use it with raw pointers
8
// - If cstdatomic is present (on newer versions of gcc, it is), we use
9
// a cstdatomic-based AtomicPointer. However we prefer the memory
10
// barrier based version, because at least on a gcc 4.4 32-bit build
11
// on linux, we have encountered a buggy <cstdatomic>
12
// implementation. Also, some <cstdatomic> implementations are much
13
// slower than a memory-barrier based implementation (~16ns for
14
// <cstdatomic> based acquire-load vs. ~1ns for a barrier based
16
// This code is based on atomicops-internals-* in Google's perftools:
17
// http://code.google.com/p/google-perftools/source/browse/#svn%2Ftrunk%2Fsrc%2Fbase
19
#ifndef PORT_ATOMIC_POINTER_H_
20
#define PORT_ATOMIC_POINTER_H_
23
#ifdef LEVELDB_CSTDATOMIC_PRESENT
30
#include <libkern/OSAtomic.h>
33
#if defined(_M_X64) || defined(__x86_64__)
34
#define ARCH_CPU_X86_FAMILY 1
35
#elif defined(_M_IX86) || defined(__i386__) || defined(__i386)
36
#define ARCH_CPU_X86_FAMILY 1
37
#elif defined(__ARMEL__)
38
#define ARCH_CPU_ARM_FAMILY 1
39
#elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__)
40
#define ARCH_CPU_PPC_FAMILY 1
41
#elif defined(__ia64__)
42
#define ARCH_CPU_IA64_FAMILY 1
43
#elif defined(__alpha__)
44
#define ARCH_CPU_ALPHA_FAMILY 1
45
#elif defined(__s390x__) || defined(__s390__)
46
#define ARCH_CPU_S390_FAMILY 1
47
#elif defined(__sparc__) || defined(__sparc64__)
48
#define ARCH_CPU_SPARC_FAMILY 1
49
#elif defined(__mips__) || defined(__mips64__)
50
#define ARCH_CPU_MIPS_FAMILY 1
56
// Define MemoryBarrier() if available
58
#if defined(OS_WIN) && defined(COMPILER_MSVC) && defined(ARCH_CPU_X86_FAMILY)
59
// windows.h already provides a MemoryBarrier(void) macro
60
// http://msdn.microsoft.com/en-us/library/ms684208(v=vs.85).aspx
61
#define LEVELDB_HAVE_MEMORY_BARRIER
63
#define ReadMemoryBarrier MemoryBarrier()
64
#define WriteMemoryBarrier MemoryBarrier()
67
#elif defined(ARCH_CPU_X86_FAMILY) && defined(__GNUC__)
68
inline void ReadMemoryBarrier() {
69
// See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on
70
// this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering.
71
__asm__ __volatile__("" : : : "memory");
73
inline void WriteMemoryBarrier() {
74
// See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on
75
// this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering.
76
__asm__ __volatile__("" : : : "memory");
78
#define LEVELDB_HAVE_MEMORY_BARRIER
81
#elif defined(ARCH_CPU_X86_FAMILY) && defined(__SUNPRO_CC)
82
inline void ReadMemoryBarrier() {
83
// See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on
84
// this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering.
85
asm volatile("" : : : "memory");
87
inline void WriteMemoryBarrier() {
88
// See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on
89
// this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering.
90
asm volatile("" : : : "memory");
92
#define LEVELDB_HAVE_MEMORY_BARRIER
95
#elif defined(OS_MACOSX)
96
inline void ReadMemoryBarrier() {
99
inline void WriteMemoryBarrier() {
102
#define LEVELDB_HAVE_MEMORY_BARRIER
105
#elif defined(ARCH_CPU_ARM_FAMILY) && defined(__linux__)
106
typedef void (*LinuxKernelMemoryBarrierFunc)(void);
107
// The Linux ARM kernel provides a highly optimized device-specific memory
108
// barrier function at a fixed memory address that is mapped in every
109
// user-level process.
111
// This beats using CPU-specific instructions which are, on single-core
112
// devices, un-necessary and very costly (e.g. ARMv7-A "dmb" takes more
113
// than 180ns on a Cortex-A8 like the one on a Nexus One). Benchmarking
114
// shows that the extra function call cost is completely negligible on
115
// multi-core devices.
117
inline void ReadMemoryBarrier() {
118
(*(LinuxKernelMemoryBarrierFunc)0xffff0fa0)();
120
inline void WriteMemoryBarrier() {
121
(*(LinuxKernelMemoryBarrierFunc)0xffff0fa0)();
123
#define LEVELDB_HAVE_MEMORY_BARRIER
126
#elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__)
128
inline void ReadMemoryBarrier() {
130
__asm__ __volatile__ ("lwsync" : : : "memory");
132
__asm__ __volatile__ ("sync" : : : "memory");
135
inline void WriteMemoryBarrier() {
136
__asm__ __volatile__ ("sync" : : : "memory");
138
#define LEVELDB_HAVE_MEMORY_BARRIER
141
#elif defined(ARCH_CPU_IA64_FAMILY)
142
inline void ReadMemoryBarrier() {
143
__asm__ __volatile__ ("mf" : : : "memory");
145
inline void WriteMemoryBarrier() {
146
__asm__ __volatile__ ("mf" : : : "memory");
148
#define LEVELDB_HAVE_MEMORY_BARRIER
151
#elif defined(ARCH_CPU_ALPHA_FAMILY)
153
inline void ReadMemoryBarrier() {
154
__asm__ __volatile__("mb" : : : "memory");
156
inline void WriteMemoryBarrier() {
157
__asm__ __volatile__("wmb" : : : "memory");
159
#define LEVELDB_HAVE_MEMORY_BARRIER
162
#elif defined(ARCH_CPU_S390_FAMILY)
164
inline void ReadMemoryBarrier() {
165
asm volatile("bcr 15,0" : : : "memory");
167
inline void WriteMemoryBarrier() {
168
asm volatile("bcr 15,0" : : : "memory");
170
#define LEVELDB_HAVE_MEMORY_BARRIER
173
#elif defined(ARCH_CPU_SPARC_FAMILY)
175
inline void ReadMemoryBarrier() {
176
__asm__ __volatile__("" : : : "memory");
178
inline void WriteMemoryBarrier() {
179
__asm__ __volatile__("" : : : "memory");
181
#define LEVELDB_HAVE_MEMORY_BARRIER
184
#elif defined(ARCH_CPU_MIPS_FAMILY)
186
inline void ReadMemoryBarrier() {
187
__asm__ __volatile__("" : : : "memory");
189
inline void WriteMemoryBarrier() {
190
__asm__ __volatile__("" : : : "memory");
192
#define LEVELDB_HAVE_MEMORY_BARRIER
196
// AtomicPointer built using platform-specific MemoryBarrier()
197
#if defined(LEVELDB_HAVE_MEMORY_BARRIER)
198
class AtomicPointer {
203
explicit AtomicPointer(void* p) : rep_(p) {}
204
inline void* NoBarrier_Load() const { return rep_; }
205
inline void NoBarrier_Store(void* v) { rep_ = v; }
206
inline void* Acquire_Load() const {
211
inline void Release_Store(void* v) {
212
WriteMemoryBarrier();
217
// AtomicPointer based on <cstdatomic>
218
#elif defined(LEVELDB_CSTDATOMIC_PRESENT)
219
class AtomicPointer {
221
std::atomic<void*> rep_;
224
explicit AtomicPointer(void* v) : rep_(v) { }
225
inline void* Acquire_Load() const {
226
return rep_.load(std::memory_order_acquire);
228
inline void Release_Store(void* v) {
229
rep_.store(v, std::memory_order_release);
231
inline void* NoBarrier_Load() const {
232
return rep_.load(std::memory_order_relaxed);
234
inline void NoBarrier_Store(void* v) {
235
rep_.store(v, std::memory_order_relaxed);
239
// Atomic pointer based on sparc memory barriers
240
#elif defined(__sparcv9) && defined(__GNUC__)
241
class AtomicPointer {
246
explicit AtomicPointer(void* v) : rep_(v) { }
247
inline void* Acquire_Load() const {
249
__asm__ __volatile__ (
250
"ldx [%[rep_]], %[val] \n\t"
251
"membar #LoadLoad|#LoadStore \n\t"
257
inline void Release_Store(void* v) {
258
__asm__ __volatile__ (
259
"membar #LoadStore|#StoreStore \n\t"
260
"stx %[v], [%[rep_]] \n\t"
262
: [rep_] "r" (&rep_), [v] "r" (v)
265
inline void* NoBarrier_Load() const { return rep_; }
266
inline void NoBarrier_Store(void* v) { rep_ = v; }
269
// Atomic pointer based on ia64 acq/rel
270
#elif defined(__ia64) && defined(__GNUC__)
271
class AtomicPointer {
276
explicit AtomicPointer(void* v) : rep_(v) { }
277
inline void* Acquire_Load() const {
279
__asm__ __volatile__ (
280
"ld8.acq %[val] = [%[rep_]] \n\t"
287
inline void Release_Store(void* v) {
288
__asm__ __volatile__ (
289
"st8.rel [%[rep_]] = %[v] \n\t"
291
: [rep_] "r" (&rep_), [v] "r" (v)
295
inline void* NoBarrier_Load() const { return rep_; }
296
inline void NoBarrier_Store(void* v) { rep_ = v; }
299
// We have neither MemoryBarrier(), nor <cstdatomic>
301
#error Please implement AtomicPointer for this platform.
305
#undef LEVELDB_HAVE_MEMORY_BARRIER
306
#undef ARCH_CPU_X86_FAMILY
307
#undef ARCH_CPU_ARM_FAMILY
308
#undef ARCH_CPU_PPC_FAMILY
309
#undef ARCH_CPU_IA64_FAMILY
310
#undef ARCH_CPU_ALPHA_FAMILY
311
#undef ARCH_CPU_S390_FAMILY
312
#undef ARCH_CPU_SPARC_FAMILY
313
#undef ARCH_CPU_MIPS_FAMILY
316
} // namespace leveldb
318
#endif // PORT_ATOMIC_POINTER_H_