/src/rocksdb/util/atomic.h
Line | Count | Source |
1 | | // Copyright (c) Meta Platforms, Inc. and affiliates. |
2 | | // This source code is licensed under both the GPLv2 (found in the |
3 | | // COPYING file in the root directory) and Apache 2.0 License |
4 | | // (found in the LICENSE.Apache file in the root directory). |
5 | | |
6 | | #pragma once |
7 | | |
8 | | #include <atomic> |
9 | | #include <memory> |
10 | | |
11 | | #include "rocksdb/rocksdb_namespace.h" |
12 | | |
13 | | #if defined(__clang__) |
14 | | #define ROCKSDB_SUPPRESS_SHARED_PTR_ATOMIC_DEPRECATION_PUSH \ |
15 | | _Pragma("clang diagnostic push") \ |
16 | | _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") |
17 | | #define ROCKSDB_SUPPRESS_SHARED_PTR_ATOMIC_DEPRECATION_POP \ |
18 | | _Pragma("clang diagnostic pop") |
19 | | #elif defined(__GNUC__) |
20 | | #define ROCKSDB_SUPPRESS_SHARED_PTR_ATOMIC_DEPRECATION_PUSH \ |
21 | | _Pragma("GCC diagnostic push") \ |
22 | | _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") |
23 | | #define ROCKSDB_SUPPRESS_SHARED_PTR_ATOMIC_DEPRECATION_POP \ |
24 | | _Pragma("GCC diagnostic pop") |
25 | | #else |
26 | | #define ROCKSDB_SUPPRESS_SHARED_PTR_ATOMIC_DEPRECATION_PUSH |
27 | | #define ROCKSDB_SUPPRESS_SHARED_PTR_ATOMIC_DEPRECATION_POP |
28 | | #endif |
29 | | |
30 | | namespace ROCKSDB_NAMESPACE { |
31 | | |
32 | | // Background: |
33 | | // std::atomic is somewhat easy to misuse: |
34 | | // * Implicit conversion to T makes it easy to use an unnecessarily strong |
35 | | // memory ordering (std::memory_order_seq_cst) and to hide atomic operations |
36 | | // that should be evident on reading the code. |
37 | | // * Similarly, defaulting to std::memory_order_seq_cst for atomic operations |
38 | | // makes it easy to use unnecessarily strong orderings. (It's always safe if |
39 | | // some ordering is safe, but it's better to be intentional and thoughtful when |
40 | | // carefully optimizing code with atomics.) Legitimate needs for seq_cst vs. |
41 | | // acq_rel are rare, such as drawing inferences across two atomics in |
42 | | // implementing hazard pointers. |
43 | | // * It's easy to use nonsensical (UB) combinations like store with |
44 | | // std::memory_order_acquire. Getting these right in development is an |
45 | | // unnecessary cognitive overhead even if they are caught by UBSAN. |
46 | | // |
47 | | // For such reasons, we provide wrappers below to make clear and explicit |
48 | | // usage of atomics easier. |
49 | | |
50 | | // Wrapper around std::atomic for better code clarity (see Background above). |
51 | | // |
52 | | // This relaxed-only wrapper is intended for atomics that are not used to |
53 | | // synchronize other data across threads (only the atomic data), so can always |
54 | | // used relaxed memory ordering. For example, a cross-thread counter that never |
55 | | // returns the same result can be a RelaxedAtomic. |
56 | | template <typename T> |
57 | | class RelaxedAtomic { |
58 | | public: |
59 | 9.27M | explicit RelaxedAtomic(T initial = {}) : v_(initial) {}rocksdb::RelaxedAtomic<unsigned long>::RelaxedAtomic(unsigned long) Line | Count | Source | 59 | 8.33M | explicit RelaxedAtomic(T initial = {}) : v_(initial) {} |
rocksdb::RelaxedAtomic<bool>::RelaxedAtomic(bool) Line | Count | Source | 59 | 433k | explicit RelaxedAtomic(T initial = {}) : v_(initial) {} |
rocksdb::RelaxedAtomic<unsigned int>::RelaxedAtomic(unsigned int) Line | Count | Source | 59 | 122k | explicit RelaxedAtomic(T initial = {}) : v_(initial) {} |
rocksdb::RelaxedAtomic<char const*>::RelaxedAtomic(char const*) Line | Count | Source | 59 | 128k | explicit RelaxedAtomic(T initial = {}) : v_(initial) {} |
rocksdb::RelaxedAtomic<int>::RelaxedAtomic(int) Line | Count | Source | 59 | 256k | explicit RelaxedAtomic(T initial = {}) : v_(initial) {} |
Unexecuted instantiation: rocksdb::RelaxedAtomic<rocksdb::SkipList<char const*, rocksdb::MemTableRep::KeyComparator const&>::Node*>::RelaxedAtomic(rocksdb::SkipList<char const*, rocksdb::MemTableRep::KeyComparator const&>::Node*) Unexecuted instantiation: rocksdb::RelaxedAtomic<long>::RelaxedAtomic(long) Unexecuted instantiation: rocksdb::RelaxedAtomic<rocksdb::SkipList<rocksdb::WriteBatchIndexEntry*, rocksdb::WriteBatchEntryComparator const&>::Node*>::RelaxedAtomic(rocksdb::SkipList<rocksdb::WriteBatchIndexEntry*, rocksdb::WriteBatchEntryComparator const&>::Node*) |
60 | 4.64M | void StoreRelaxed(T desired) { v_.store(desired, std::memory_order_relaxed); }rocksdb::RelaxedAtomic<unsigned long>::StoreRelaxed(unsigned long) Line | Count | Source | 60 | 1.82M | void StoreRelaxed(T desired) { v_.store(desired, std::memory_order_relaxed); } |
rocksdb::RelaxedAtomic<bool>::StoreRelaxed(bool) Line | Count | Source | 60 | 657k | void StoreRelaxed(T desired) { v_.store(desired, std::memory_order_relaxed); } |
rocksdb::RelaxedAtomic<rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::Node*>::StoreRelaxed(rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::Node*) Line | Count | Source | 60 | 2.04M | void StoreRelaxed(T desired) { v_.store(desired, std::memory_order_relaxed); } |
rocksdb::RelaxedAtomic<unsigned int>::StoreRelaxed(unsigned int) Line | Count | Source | 60 | 123k | void StoreRelaxed(T desired) { v_.store(desired, std::memory_order_relaxed); } |
Unexecuted instantiation: rocksdb::RelaxedAtomic<int>::StoreRelaxed(int) Unexecuted instantiation: rocksdb::RelaxedAtomic<rocksdb::SkipList<char const*, rocksdb::MemTableRep::KeyComparator const&>::Node*>::StoreRelaxed(rocksdb::SkipList<char const*, rocksdb::MemTableRep::KeyComparator const&>::Node*) Unexecuted instantiation: rocksdb::RelaxedAtomic<rocksdb::SkipList<rocksdb::WriteBatchIndexEntry*, rocksdb::WriteBatchEntryComparator const&>::Node*>::StoreRelaxed(rocksdb::SkipList<rocksdb::WriteBatchIndexEntry*, rocksdb::WriteBatchEntryComparator const&>::Node*) |
61 | 7.79M | T LoadRelaxed() const { return v_.load(std::memory_order_relaxed); }rocksdb::RelaxedAtomic<unsigned long>::LoadRelaxed() const Line | Count | Source | 61 | 3.69M | T LoadRelaxed() const { return v_.load(std::memory_order_relaxed); } |
rocksdb::RelaxedAtomic<bool>::LoadRelaxed() const Line | Count | Source | 61 | 2.40M | T LoadRelaxed() const { return v_.load(std::memory_order_relaxed); } |
rocksdb::RelaxedAtomic<unsigned int>::LoadRelaxed() const Line | Count | Source | 61 | 123k | T LoadRelaxed() const { return v_.load(std::memory_order_relaxed); } |
rocksdb::RelaxedAtomic<int>::LoadRelaxed() const Line | Count | Source | 61 | 1.56M | T LoadRelaxed() const { return v_.load(std::memory_order_relaxed); } |
Unexecuted instantiation: rocksdb::RelaxedAtomic<rocksdb::SkipList<char const*, rocksdb::MemTableRep::KeyComparator const&>::Node*>::LoadRelaxed() const Unexecuted instantiation: rocksdb::RelaxedAtomic<long>::LoadRelaxed() const Unexecuted instantiation: rocksdb::RelaxedAtomic<rocksdb::SkipList<rocksdb::WriteBatchIndexEntry*, rocksdb::WriteBatchEntryComparator const&>::Node*>::LoadRelaxed() const |
62 | 33.9k | bool CasWeakRelaxed(T& expected, T desired) { |
63 | 33.9k | return v_.compare_exchange_weak(expected, desired, |
64 | 33.9k | std::memory_order_relaxed); |
65 | 33.9k | } rocksdb::RelaxedAtomic<unsigned long>::CasWeakRelaxed(unsigned long&, unsigned long) Line | Count | Source | 62 | 1.14k | bool CasWeakRelaxed(T& expected, T desired) { | 63 | 1.14k | return v_.compare_exchange_weak(expected, desired, | 64 | 1.14k | std::memory_order_relaxed); | 65 | 1.14k | } |
rocksdb::RelaxedAtomic<int>::CasWeakRelaxed(int&, int) Line | Count | Source | 62 | 32.8k | bool CasWeakRelaxed(T& expected, T desired) { | 63 | 32.8k | return v_.compare_exchange_weak(expected, desired, | 64 | 32.8k | std::memory_order_relaxed); | 65 | 32.8k | } |
|
66 | | bool CasStrongRelaxed(T& expected, T desired) { |
67 | | return v_.compare_exchange_strong(expected, desired, |
68 | | std::memory_order_relaxed); |
69 | | } |
70 | 0 | T ExchangeRelaxed(T desired) { |
71 | 0 | return v_.exchange(desired, std::memory_order_relaxed); |
72 | 0 | } |
73 | 5.55M | T FetchAddRelaxed(T operand) { |
74 | 5.55M | return v_.fetch_add(operand, std::memory_order_relaxed); |
75 | 5.55M | } rocksdb::RelaxedAtomic<unsigned long>::FetchAddRelaxed(unsigned long) Line | Count | Source | 73 | 5.55M | T FetchAddRelaxed(T operand) { | 74 | 5.55M | return v_.fetch_add(operand, std::memory_order_relaxed); | 75 | 5.55M | } |
Unexecuted instantiation: rocksdb::RelaxedAtomic<unsigned int>::FetchAddRelaxed(unsigned int) Unexecuted instantiation: rocksdb::RelaxedAtomic<long>::FetchAddRelaxed(long) |
76 | 18.5k | T FetchSubRelaxed(T operand) { |
77 | 18.5k | return v_.fetch_sub(operand, std::memory_order_relaxed); |
78 | 18.5k | } Unexecuted instantiation: rocksdb::RelaxedAtomic<unsigned int>::FetchSubRelaxed(unsigned int) rocksdb::RelaxedAtomic<unsigned long>::FetchSubRelaxed(unsigned long) Line | Count | Source | 76 | 18.5k | T FetchSubRelaxed(T operand) { | 77 | 18.5k | return v_.fetch_sub(operand, std::memory_order_relaxed); | 78 | 18.5k | } |
Unexecuted instantiation: rocksdb::RelaxedAtomic<long>::FetchSubRelaxed(long) |
79 | | T FetchAndRelaxed(T operand) { |
80 | | return v_.fetch_and(operand, std::memory_order_relaxed); |
81 | | } |
82 | 0 | T FetchOrRelaxed(T operand) { |
83 | 0 | return v_.fetch_or(operand, std::memory_order_relaxed); |
84 | 0 | } |
85 | | T FetchXorRelaxed(T operand) { |
86 | | return v_.fetch_xor(operand, std::memory_order_relaxed); |
87 | | } |
88 | | |
89 | | protected: |
90 | | std::atomic<T> v_; |
91 | | }; |
92 | | |
93 | | // A reasonably general-purpose wrapper around std::atomic for better code |
94 | | // clarity (see Background above). |
95 | | // |
96 | | // Operations use std::memory_order_acq_rel by default (or just acquire or just |
97 | | // release for read-only and write-only operations), but relaxed operations are |
98 | | // also available and can be mixed in when appropriate. |
99 | | // |
100 | | // Future: add std::memory_order_seqcst variants like StoreSeqCst if/when |
101 | | // there's a need for them (rare). No distinct type is needed because the |
102 | | // distinction between acq_rel and seq_cst is more about where it is used in |
103 | | // combination with other atomics than the atomic itself. |
104 | | template <typename T> |
105 | | class Atomic : public RelaxedAtomic<T> { |
106 | | public: |
107 | 2.70M | explicit Atomic(T initial = {}) : RelaxedAtomic<T>(initial) {}rocksdb::Atomic<unsigned long>::Atomic(unsigned long) Line | Count | Source | 107 | 2.57M | explicit Atomic(T initial = {}) : RelaxedAtomic<T>(initial) {} |
rocksdb::Atomic<char const*>::Atomic(char const*) Line | Count | Source | 107 | 128k | explicit Atomic(T initial = {}) : RelaxedAtomic<T>(initial) {} |
Unexecuted instantiation: rocksdb::Atomic<rocksdb::SkipList<char const*, rocksdb::MemTableRep::KeyComparator const&>::Node*>::Atomic(rocksdb::SkipList<char const*, rocksdb::MemTableRep::KeyComparator const&>::Node*) Unexecuted instantiation: rocksdb::Atomic<rocksdb::SkipList<rocksdb::WriteBatchIndexEntry*, rocksdb::WriteBatchEntryComparator const&>::Node*>::Atomic(rocksdb::SkipList<rocksdb::WriteBatchIndexEntry*, rocksdb::WriteBatchEntryComparator const&>::Node*) |
108 | 4.27M | void Store(T desired) { |
109 | 4.27M | RelaxedAtomic<T>::v_.store(desired, std::memory_order_release); |
110 | 4.27M | } rocksdb::Atomic<rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::Node*>::Store(rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::Node*) Line | Count | Source | 108 | 4.27M | void Store(T desired) { | 109 | 4.27M | RelaxedAtomic<T>::v_.store(desired, std::memory_order_release); | 110 | 4.27M | } |
Unexecuted instantiation: rocksdb::Atomic<rocksdb::SkipList<char const*, rocksdb::MemTableRep::KeyComparator const&>::Node*>::Store(rocksdb::SkipList<char const*, rocksdb::MemTableRep::KeyComparator const&>::Node*) Unexecuted instantiation: rocksdb::Atomic<rocksdb::SkipList<rocksdb::WriteBatchIndexEntry*, rocksdb::WriteBatchEntryComparator const&>::Node*>::Store(rocksdb::SkipList<rocksdb::WriteBatchIndexEntry*, rocksdb::WriteBatchEntryComparator const&>::Node*) |
111 | 42.1M | T Load() const { |
112 | 42.1M | return RelaxedAtomic<T>::v_.load(std::memory_order_acquire); |
113 | 42.1M | } rocksdb::Atomic<unsigned long>::Load() const Line | Count | Source | 111 | 4.63M | T Load() const { | 112 | 4.63M | return RelaxedAtomic<T>::v_.load(std::memory_order_acquire); | 113 | 4.63M | } |
Unexecuted instantiation: rocksdb::Atomic<char const*>::Load() const rocksdb::Atomic<rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::Node*>::Load() const Line | Count | Source | 111 | 37.5M | T Load() const { | 112 | 37.5M | return RelaxedAtomic<T>::v_.load(std::memory_order_acquire); | 113 | 37.5M | } |
Unexecuted instantiation: rocksdb::Atomic<rocksdb::SkipList<char const*, rocksdb::MemTableRep::KeyComparator const&>::Node*>::Load() const Unexecuted instantiation: rocksdb::Atomic<rocksdb::SkipList<rocksdb::WriteBatchIndexEntry*, rocksdb::WriteBatchEntryComparator const&>::Node*>::Load() const |
114 | 0 | bool CasWeak(T& expected, T desired) { |
115 | 0 | return RelaxedAtomic<T>::v_.compare_exchange_weak( |
116 | 0 | expected, desired, std::memory_order_acq_rel); |
117 | 0 | } |
118 | 843k | bool CasStrong(T& expected, T desired) { |
119 | 843k | return RelaxedAtomic<T>::v_.compare_exchange_strong( |
120 | 843k | expected, desired, std::memory_order_acq_rel); |
121 | 843k | } rocksdb::Atomic<unsigned long>::CasStrong(unsigned long&, unsigned long) Line | Count | Source | 118 | 1.92k | bool CasStrong(T& expected, T desired) { | 119 | 1.92k | return RelaxedAtomic<T>::v_.compare_exchange_strong( | 120 | 1.92k | expected, desired, std::memory_order_acq_rel); | 121 | 1.92k | } |
rocksdb::Atomic<rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::Node*>::CasStrong(rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::Node*&, rocksdb::InlineSkipList<rocksdb::MemTableRep::KeyComparator const&>::Node*) Line | Count | Source | 118 | 841k | bool CasStrong(T& expected, T desired) { | 119 | 841k | return RelaxedAtomic<T>::v_.compare_exchange_strong( | 120 | 841k | expected, desired, std::memory_order_acq_rel); | 121 | 841k | } |
|
122 | | T Exchange(T desired) { |
123 | | return RelaxedAtomic<T>::v_.exchange(desired, std::memory_order_acq_rel); |
124 | | } |
125 | 105k | T FetchAdd(T operand) { |
126 | 105k | return RelaxedAtomic<T>::v_.fetch_add(operand, std::memory_order_acq_rel); |
127 | 105k | } |
128 | 16.1k | T FetchSub(T operand) { |
129 | 16.1k | return RelaxedAtomic<T>::v_.fetch_sub(operand, std::memory_order_acq_rel); |
130 | 16.1k | } |
131 | | T FetchAnd(T operand) { |
132 | | return RelaxedAtomic<T>::v_.fetch_and(operand, std::memory_order_acq_rel); |
133 | | } |
134 | | T FetchOr(T operand) { |
135 | | return RelaxedAtomic<T>::v_.fetch_or(operand, std::memory_order_acq_rel); |
136 | | } |
137 | | T FetchXor(T operand) { |
138 | | return RelaxedAtomic<T>::v_.fetch_xor(operand, std::memory_order_acq_rel); |
139 | | } |
140 | | }; |
141 | | |
142 | | // Atomic shared_ptr helper wrappers. |
143 | | // |
144 | | // The C++20 replacement is std::atomic<std::shared_ptr<T>>, but the fbcode |
145 | | // make build still uses a libstdc++ 11.x toolchain where that specialization |
146 | | // is unavailable while the older free-function API is already deprecated. |
147 | | // Centralize the workaround here so callers can keep a single code path until |
148 | | // the build toolchain can support std::atomic<std::shared_ptr<T>> directly. |
149 | | template <typename T> |
150 | | inline std::shared_ptr<T> AtomicSharedPtrLoad(const std::shared_ptr<T>* ptr, |
151 | 2.87k | std::memory_order order) { |
152 | 2.87k | ROCKSDB_SUPPRESS_SHARED_PTR_ATOMIC_DEPRECATION_PUSH |
153 | 2.87k | auto result = std::atomic_load_explicit(ptr, order); |
154 | 2.87k | ROCKSDB_SUPPRESS_SHARED_PTR_ATOMIC_DEPRECATION_POP |
155 | 2.87k | return result; |
156 | 2.87k | } |
157 | | |
158 | | template <typename T> |
159 | | inline void AtomicSharedPtrStore(std::shared_ptr<T>* ptr, |
160 | | std::shared_ptr<T> desired, |
161 | 24.3M | std::memory_order order) { |
162 | 24.3M | ROCKSDB_SUPPRESS_SHARED_PTR_ATOMIC_DEPRECATION_PUSH |
163 | 24.3M | std::atomic_store_explicit(ptr, std::move(desired), order); |
164 | 24.3M | ROCKSDB_SUPPRESS_SHARED_PTR_ATOMIC_DEPRECATION_POP |
165 | 24.3M | } |
166 | | |
167 | | } // namespace ROCKSDB_NAMESPACE |
168 | | |
169 | | #undef ROCKSDB_SUPPRESS_SHARED_PTR_ATOMIC_DEPRECATION_PUSH |
170 | | #undef ROCKSDB_SUPPRESS_SHARED_PTR_ATOMIC_DEPRECATION_POP |