GNU Radio's TEST Package
atomicops.h
Go to the documentation of this file.
1 // ©2013-2015 Cameron Desrochers.
2 // Distributed under the simplified BSD license (see the license file that
3 // should have come with this header).
4 // Uses Jeff Preshing's semaphore implementation (under the terms of its
5 // separate zlib license, embedded below).
6 
7 #pragma once
8 
9 // Provides portable (VC++2010+, Intel ICC 13, GCC 4.7+, and anything C++11 compliant) implementation
10 // of low-level memory barriers, plus a few semi-portable utility macros (for inlining and alignment).
11 // Also has a basic atomic type (limited to hardware-supported atomics with no memory ordering guarantees).
12 // Uses the AE_* prefix for macros (historical reasons), and the "moodycamel" namespace for symbols.
13 
14 #include <cassert>
15 #include <type_traits>
16 
17 
18 // Platform detection
19 #if defined(__INTEL_COMPILER)
20 #define AE_ICC
21 #elif defined(_MSC_VER)
22 #define AE_VCPP
23 #elif defined(__GNUC__)
24 #define AE_GCC
25 #endif
26 
27 #if defined(_M_IA64) || defined(__ia64__)
28 #define AE_ARCH_IA64
29 #elif defined(_WIN64) || defined(__amd64__) || defined(_M_X64) || defined(__x86_64__)
30 #define AE_ARCH_X64
31 #elif defined(_M_IX86) || defined(__i386__)
32 #define AE_ARCH_X86
33 #elif defined(_M_PPC) || defined(__powerpc__)
34 #define AE_ARCH_PPC
35 #else
36 #define AE_ARCH_UNKNOWN
37 #endif
38 
39 
40 // AE_UNUSED
41 #define AE_UNUSED(x) ((void)x)
42 
43 
44 // AE_FORCEINLINE
45 #if defined(AE_VCPP) || defined(AE_ICC)
46 #define AE_FORCEINLINE __forceinline
47 #elif defined(AE_GCC)
48 //#define AE_FORCEINLINE __attribute__((always_inline))
49 #define AE_FORCEINLINE inline
50 #else
51 #define AE_FORCEINLINE inline
52 #endif
53 
54 
55 // AE_ALIGN
56 #if defined(AE_VCPP) || defined(AE_ICC)
57 #define AE_ALIGN(x) __declspec(align(x))
58 #elif defined(AE_GCC)
59 #define AE_ALIGN(x) __attribute__((aligned(x)))
60 #else
61 // Assume GCC compliant syntax...
62 #define AE_ALIGN(x) __attribute__((aligned(x)))
63 #endif
64 
65 
66 // Portable atomic fences implemented below:
67 
68 namespace moodycamel {
69 
70 enum memory_order {
71  memory_order_relaxed,
72  memory_order_acquire,
73  memory_order_release,
74  memory_order_acq_rel,
75  memory_order_seq_cst,
76 
77  // memory_order_sync: Forces a full sync:
78  // #LoadLoad, #LoadStore, #StoreStore, and most significantly, #StoreLoad
79  memory_order_sync = memory_order_seq_cst
80 };
81 
82 } // end namespace moodycamel
83 
84 #if (defined(AE_VCPP) && (_MSC_VER < 1700 || defined(__cplusplus_cli))) || defined(AE_ICC)
85 // VS2010 and ICC13 don't support std::atomic_*_fence, implement our own fences
86 
87 #include <intrin.h>
88 
89 #if defined(AE_ARCH_X64) || defined(AE_ARCH_X86)
90 #define AeFullSync _mm_mfence
91 #define AeLiteSync _mm_mfence
92 #elif defined(AE_ARCH_IA64)
93 #define AeFullSync __mf
94 #define AeLiteSync __mf
95 #elif defined(AE_ARCH_PPC)
96 #include <ppcintrinsics.h>
97 #define AeFullSync __sync
98 #define AeLiteSync __lwsync
99 #endif
100 
101 
102 #ifdef AE_VCPP
103 #pragma warning(push)
104 #pragma warning(disable: 4365) // Disable erroneous 'conversion from long to unsigned int, signed/unsigned mismatch' error when using `assert`
105 #ifdef __cplusplus_cli
106 #pragma managed(push, off)
107 #endif
108 #endif
109 
110 namespace moodycamel {
111 
112 AE_FORCEINLINE void compiler_fence(memory_order order)
113 {
114  switch (order) {
115  case memory_order_relaxed: break;
116  case memory_order_acquire: _ReadBarrier(); break;
117  case memory_order_release: _WriteBarrier(); break;
118  case memory_order_acq_rel: _ReadWriteBarrier(); break;
119  case memory_order_seq_cst: _ReadWriteBarrier(); break;
120  default: assert(false);
121  }
122 }
123 
124 // x86/x64 have a strong memory model -- all loads and stores have
125 // acquire and release semantics automatically (so only need compiler
126 // barriers for those).
127 #if defined(AE_ARCH_X86) || defined(AE_ARCH_X64)
128 AE_FORCEINLINE void fence(memory_order order)
129 {
130  switch (order) {
131  case memory_order_relaxed: break;
132  case memory_order_acquire: _ReadBarrier(); break;
133  case memory_order_release: _WriteBarrier(); break;
134  case memory_order_acq_rel: _ReadWriteBarrier(); break;
135  case memory_order_seq_cst:
136  _ReadWriteBarrier();
137  AeFullSync();
138  _ReadWriteBarrier();
139  break;
140  default: assert(false);
141  }
142 }
143 #else
144 AE_FORCEINLINE void fence(memory_order order)
145 {
146  // Non-specialized arch, use heavier memory barriers everywhere just in case :-(
147  switch (order) {
148  case memory_order_relaxed:
149  break;
150  case memory_order_acquire:
151  _ReadBarrier();
152  AeLiteSync();
153  _ReadBarrier();
154  break;
155  case memory_order_release:
156  _WriteBarrier();
157  AeLiteSync();
158  _WriteBarrier();
159  break;
160  case memory_order_acq_rel:
161  _ReadWriteBarrier();
162  AeLiteSync();
163  _ReadWriteBarrier();
164  break;
165  case memory_order_seq_cst:
166  _ReadWriteBarrier();
167  AeFullSync();
168  _ReadWriteBarrier();
169  break;
170  default: assert(false);
171  }
172 }
173 #endif
174 } // end namespace moodycamel
175 #else
176 // Use standard library of atomics
177 #include <atomic>
178 
179 namespace moodycamel {
180 
181 AE_FORCEINLINE void compiler_fence(memory_order order)
182 {
183  switch (order) {
184  case memory_order_relaxed: break;
185  case memory_order_acquire: std::atomic_signal_fence(std::memory_order_acquire); break;
186  case memory_order_release: std::atomic_signal_fence(std::memory_order_release); break;
187  case memory_order_acq_rel: std::atomic_signal_fence(std::memory_order_acq_rel); break;
188  case memory_order_seq_cst: std::atomic_signal_fence(std::memory_order_seq_cst); break;
189  default: assert(false);
190  }
191 }
192 
193 AE_FORCEINLINE void fence(memory_order order)
194 {
195  switch (order) {
196  case memory_order_relaxed: break;
197  case memory_order_acquire: std::atomic_thread_fence(std::memory_order_acquire); break;
198  case memory_order_release: std::atomic_thread_fence(std::memory_order_release); break;
199  case memory_order_acq_rel: std::atomic_thread_fence(std::memory_order_acq_rel); break;
200  case memory_order_seq_cst: std::atomic_thread_fence(std::memory_order_seq_cst); break;
201  default: assert(false);
202  }
203 }
204 
205 } // end namespace moodycamel
206 
207 #endif
208 
209 
210 #if !defined(AE_VCPP) || (_MSC_VER >= 1700 && !defined(__cplusplus_cli))
211 #define AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC
212 #endif
213 
214 #ifdef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC
215 #include <atomic>
216 #endif
217 #include <utility>
218 
219 // WARNING: *NOT* A REPLACEMENT FOR std::atomic. READ CAREFULLY:
220 // Provides basic support for atomic variables -- no memory ordering guarantees are provided.
221 // The guarantee of atomicity is only made for types that already have atomic load and store guarantees
222 // at the hardware level -- on most platforms this generally means aligned pointers and integers (only).
223 namespace moodycamel {
224 template<typename T>
225 class weak_atomic
226 {
227 public:
228  weak_atomic() { }
229 #ifdef AE_VCPP
230 #pragma warning(disable: 4100) // Get rid of (erroneous) 'unreferenced formal parameter' warning
231 #endif
232  template<typename U> weak_atomic(U&& x) : value(std::forward<U>(x)) { }
233 #ifdef __cplusplus_cli
234  // Work around bug with universal reference/nullptr combination that only appears when /clr is on
235  weak_atomic(nullptr_t) : value(nullptr) { }
236 #endif
237  weak_atomic(weak_atomic const& other) : value(other.value) { }
238  weak_atomic(weak_atomic&& other) : value(std::move(other.value)) { }
239 #ifdef AE_VCPP
240 #pragma warning(default: 4100)
241 #endif
242 
243  AE_FORCEINLINE operator T() const { return load(); }
244 
245 
246 #ifndef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC
247  template<typename U> AE_FORCEINLINE weak_atomic const& operator=(U&& x) { value = std::forward<U>(x); return *this; }
248  AE_FORCEINLINE weak_atomic const& operator=(weak_atomic const& other) { value = other.value; return *this; }
249 
250  AE_FORCEINLINE T load() const { return value; }
251 
252  AE_FORCEINLINE T fetch_add_acquire(T increment)
253  {
254 #if defined(AE_ARCH_X64) || defined(AE_ARCH_X86)
255  if (sizeof(T) == 4) return _InterlockedExchangeAdd((long volatile*)&value, (long)increment);
256 #if defined(_M_AMD64)
257  else if (sizeof(T) == 8) return _InterlockedExchangeAdd64((long long volatile*)&value, (long long)increment);
258 #endif
259 #else
260 #error Unsupported platform
261 #endif
262  assert(false && "T must be either a 32 or 64 bit type");
263  return value;
264  }
265 
266  AE_FORCEINLINE T fetch_add_release(T increment)
267  {
268 #if defined(AE_ARCH_X64) || defined(AE_ARCH_X86)
269  if (sizeof(T) == 4) return _InterlockedExchangeAdd((long volatile*)&value, (long)increment);
270 #if defined(_M_AMD64)
271  else if (sizeof(T) == 8) return _InterlockedExchangeAdd64((long long volatile*)&value, (long long)increment);
272 #endif
273 #else
274 #error Unsupported platform
275 #endif
276  assert(false && "T must be either a 32 or 64 bit type");
277  return value;
278  }
279 #else
280  template<typename U>
281  AE_FORCEINLINE weak_atomic const& operator=(U&& x)
282  {
283  value.store(std::forward<U>(x), std::memory_order_relaxed);
284  return *this;
285  }
286 
287  AE_FORCEINLINE weak_atomic const& operator=(weak_atomic const& other)
288  {
289  value.store(other.value.load(std::memory_order_relaxed), std::memory_order_relaxed);
290  return *this;
291  }
292 
293  AE_FORCEINLINE T load() const { return value.load(std::memory_order_relaxed); }
294 
295  AE_FORCEINLINE T fetch_add_acquire(T increment)
296  {
297  return value.fetch_add(increment, std::memory_order_acquire);
298  }
299 
300  AE_FORCEINLINE T fetch_add_release(T increment)
301  {
302  return value.fetch_add(increment, std::memory_order_release);
303  }
304 #endif
305 
306 
307 private:
308 #ifndef AE_USE_STD_ATOMIC_FOR_WEAK_ATOMIC
309  // No std::atomic support, but still need to circumvent compiler optimizations.
310  // `volatile` will make memory access slow, but is guaranteed to be reliable.
311  volatile T value;
312 #else
313  std::atomic<T> value;
314 #endif
315 };
316 
317 } // end namespace moodycamel
318 
319 
320 
321 // Portable single-producer, single-consumer semaphore below:
322 
323 #if defined(_WIN32)