1
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
2
// Use of this source code is governed by a BSD-style license that can be
3
// found in the LICENSE file. See the AUTHORS file for names of contributors.
5
// AtomicPointer provides storage for a lock-free pointer.
6
// Platform-dependent implementation of AtomicPointer:
7
// - If the platform provides a cheap barrier, we use it with raw pointers
8
// - If cstdatomic is present (on newer versions of gcc, it is), we use
9
// a cstdatomic-based AtomicPointer. However we prefer the memory
10
// barrier based version, because at least on a gcc 4.4 32-bit build
11
// on linux, we have encountered a buggy <cstdatomic>
12
// implementation. Also, some <cstdatomic> implementations are much
13
// slower than a memory-barrier based implementation (~16ns for
14
// <cstdatomic> based acquire-load vs. ~1ns for a barrier based
16
// This code is based on atomicops-internals-* in Google's perftools:
17
// http://code.google.com/p/google-perftools/source/browse/#svn%2Ftrunk%2Fsrc%2Fbase
19
#ifndef PORT_ATOMIC_POINTER_H_
20
#define PORT_ATOMIC_POINTER_H_
23
#ifdef LEVELDB_CSTDATOMIC_PRESENT
30
#include <libkern/OSAtomic.h>
33
#if defined(_M_X64) || defined(__x86_64__)
34
#define ARCH_CPU_X86_FAMILY 1
35
#elif defined(_M_IX86) || defined(__i386__) || defined(__i386)
36
#define ARCH_CPU_X86_FAMILY 1
37
#elif defined(__ARMEL__)
38
#define ARCH_CPU_ARM_FAMILY 1
39
#elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__)
40
#define ARCH_CPU_PPC_FAMILY 1
46
// Define MemoryBarrier() if available
48
#if defined(OS_WIN) && defined(COMPILER_MSVC) && defined(ARCH_CPU_X86_FAMILY)
49
// windows.h already provides a MemoryBarrier(void) macro
50
// http://msdn.microsoft.com/en-us/library/ms684208(v=vs.85).aspx
51
#define LEVELDB_HAVE_MEMORY_BARRIER
54
#elif defined(ARCH_CPU_X86_FAMILY) && defined(__GNUC__)
55
inline void MemoryBarrier() {
56
// See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on
57
// this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering.
58
__asm__ __volatile__("" : : : "memory");
60
#define LEVELDB_HAVE_MEMORY_BARRIER
63
#elif defined(ARCH_CPU_X86_FAMILY) && defined(__SUNPRO_CC)
64
inline void MemoryBarrier() {
65
// See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on
66
// this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering.
67
asm volatile("" : : : "memory");
69
#define LEVELDB_HAVE_MEMORY_BARRIER
72
#elif defined(OS_MACOSX)
73
inline void MemoryBarrier() {
76
#define LEVELDB_HAVE_MEMORY_BARRIER
79
#elif defined(ARCH_CPU_ARM_FAMILY) && defined(__linux__)
80
typedef void (*LinuxKernelMemoryBarrierFunc)(void);
81
// The Linux ARM kernel provides a highly optimized device-specific memory
82
// barrier function at a fixed memory address that is mapped in every
83
// user-level process.
85
// This beats using CPU-specific instructions which are, on single-core
86
// devices, un-necessary and very costly (e.g. ARMv7-A "dmb" takes more
87
// than 180ns on a Cortex-A8 like the one on a Nexus One). Benchmarking
88
// shows that the extra function call cost is completely negligible on
89
// multi-core devices.
91
inline void MemoryBarrier() {
92
(*(LinuxKernelMemoryBarrierFunc)0xffff0fa0)();
94
#define LEVELDB_HAVE_MEMORY_BARRIER
97
#elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__)
98
inline void MemoryBarrier() {
99
// TODO for some powerpc expert: is there a cheaper suitable variant?
100
// Perhaps by having separate barriers for acquire and release ops.
101
asm volatile("sync" : : : "memory");
103
#define LEVELDB_HAVE_MEMORY_BARRIER
107
// AtomicPointer built using platform-specific MemoryBarrier()
108
#if defined(LEVELDB_HAVE_MEMORY_BARRIER)
109
class AtomicPointer {
114
explicit AtomicPointer(void* p) : rep_(p) {}
115
inline void* NoBarrier_Load() const { return rep_; }
116
inline void NoBarrier_Store(void* v) { rep_ = v; }
117
inline void* Acquire_Load() const {
122
inline void Release_Store(void* v) {
128
// AtomicPointer based on <cstdatomic>
129
#elif defined(LEVELDB_CSTDATOMIC_PRESENT)
130
class AtomicPointer {
132
std::atomic<void*> rep_;
135
explicit AtomicPointer(void* v) : rep_(v) { }
136
inline void* Acquire_Load() const {
137
return rep_.load(std::memory_order_acquire);
139
inline void Release_Store(void* v) {
140
rep_.store(v, std::memory_order_release);
142
inline void* NoBarrier_Load() const {
143
return rep_.load(std::memory_order_relaxed);
145
inline void NoBarrier_Store(void* v) {
146
rep_.store(v, std::memory_order_relaxed);
150
// Atomic pointer based on sparc memory barriers
151
#elif defined(__sparcv9) && defined(__GNUC__)
152
class AtomicPointer {
157
explicit AtomicPointer(void* v) : rep_(v) { }
158
inline void* Acquire_Load() const {
160
__asm__ __volatile__ (
161
"ldx [%[rep_]], %[val] \n\t"
162
"membar #LoadLoad|#LoadStore \n\t"
168
inline void Release_Store(void* v) {
169
__asm__ __volatile__ (
170
"membar #LoadStore|#StoreStore \n\t"
171
"stx %[v], [%[rep_]] \n\t"
173
: [rep_] "r" (&rep_), [v] "r" (v)
176
inline void* NoBarrier_Load() const { return rep_; }
177
inline void NoBarrier_Store(void* v) { rep_ = v; }
180
// Atomic pointer based on ia64 acq/rel
181
#elif defined(__ia64) && defined(__GNUC__)
182
class AtomicPointer {
187
explicit AtomicPointer(void* v) : rep_(v) { }
188
inline void* Acquire_Load() const {
190
__asm__ __volatile__ (
191
"ld8.acq %[val] = [%[rep_]] \n\t"
198
inline void Release_Store(void* v) {
199
__asm__ __volatile__ (
200
"st8.rel [%[rep_]] = %[v] \n\t"
202
: [rep_] "r" (&rep_), [v] "r" (v)
206
inline void* NoBarrier_Load() const { return rep_; }
207
inline void NoBarrier_Store(void* v) { rep_ = v; }
210
// We have neither MemoryBarrier(), nor <cstdatomic>
212
#error Please implement AtomicPointer for this platform.
216
#undef LEVELDB_HAVE_MEMORY_BARRIER
217
#undef ARCH_CPU_X86_FAMILY
218
#undef ARCH_CPU_ARM_FAMILY
219
#undef ARCH_CPU_PPC_FAMILY
222
} // namespace leveldb
224
#endif // PORT_ATOMIC_POINTER_H_