/src/rocksdb/db/job_context.h
Line | Count | Source |
1 | | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | | // This source code is licensed under both the GPLv2 (found in the |
3 | | // COPYING file in the root directory) and Apache 2.0 License |
4 | | // (found in the LICENSE.Apache file in the root directory). |
5 | | // |
6 | | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. |
7 | | // Use of this source code is governed by a BSD-style license that can be |
8 | | // found in the LICENSE file. See the AUTHORS file for names of contributors. |
9 | | |
10 | | #pragma once |
11 | | |
12 | | #include <limits> |
13 | | #include <string> |
14 | | #include <vector> |
15 | | |
16 | | #include "db/column_family.h" |
17 | | #include "db/log_writer.h" |
18 | | #include "db/version_set.h" |
19 | | #include "util/autovector.h" |
20 | | #include "util/hash_containers.h" |
21 | | |
22 | | namespace ROCKSDB_NAMESPACE { |
23 | | |
24 | | class MemTable; |
25 | | struct SuperVersion; |
26 | | |
27 | | // The purpose of this struct is to simplify pushing work such as |
28 | | // allocation/construction, de-allocation/destruction, and notifications to |
29 | | // outside of holding the DB mutex. |
30 | | struct SuperVersionContext { |
31 | | struct WriteStallNotification { |
32 | | WriteStallInfo write_stall_info; |
33 | | const ImmutableOptions* immutable_options; |
34 | | }; |
35 | | |
36 | | autovector<SuperVersion*> superversions_to_free; |
37 | | #ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION |
38 | | autovector<WriteStallNotification> write_stall_notifications; |
39 | | #endif |
40 | | std::unique_ptr<SuperVersion> |
41 | | new_superversion; // if nullptr no new superversion |
42 | | |
43 | | explicit SuperVersionContext(bool create_superversion = false) |
44 | 1.11M | : new_superversion(create_superversion ? new SuperVersion() : nullptr) {} |
45 | | |
46 | | SuperVersionContext(SuperVersionContext&& other) noexcept |
47 | 218k | : superversions_to_free(std::move(other.superversions_to_free)), |
48 | | #ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION |
49 | 218k | write_stall_notifications(std::move(other.write_stall_notifications)), |
50 | | #endif |
51 | 218k | new_superversion(std::move(other.new_superversion)) { |
52 | 218k | } |
53 | | // No copies |
54 | | SuperVersionContext(const SuperVersionContext& other) = delete; |
55 | | void operator=(const SuperVersionContext& other) = delete; |
56 | | void operator=(SuperVersionContext&& other) = delete; |
57 | | |
58 | 1.81k | void NewSuperVersion() { |
59 | 1.81k | new_superversion = std::unique_ptr<SuperVersion>(new SuperVersion()); |
60 | 1.81k | } |
61 | | |
62 | 7.79k | inline bool HaveSomethingToDelete() const { |
63 | 7.79k | #ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION |
64 | 7.79k | return !superversions_to_free.empty() || !write_stall_notifications.empty(); |
65 | | #else |
66 | | return !superversions_to_free.empty(); |
67 | | #endif |
68 | 7.79k | } |
69 | | |
70 | | void PushWriteStallNotification(WriteStallCondition old_cond, |
71 | | WriteStallCondition new_cond, |
72 | | const std::string& name, |
73 | 0 | const ImmutableOptions* ioptions) { |
74 | 0 | #if !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION) |
75 | 0 | WriteStallNotification notif; |
76 | 0 | notif.write_stall_info.cf_name = name; |
77 | 0 | notif.write_stall_info.condition.prev = old_cond; |
78 | 0 | notif.write_stall_info.condition.cur = new_cond; |
79 | 0 | notif.immutable_options = ioptions; |
80 | 0 | write_stall_notifications.push_back(notif); |
81 | | #else |
82 | | (void)old_cond; |
83 | | (void)new_cond; |
84 | | (void)name; |
85 | | (void)ioptions; |
86 | | #endif // !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION) |
87 | 0 | } |
88 | | |
89 | 1.11M | void Clean() { |
90 | 1.11M | #if !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION) |
91 | | // notify listeners on changed write stall conditions |
92 | 1.11M | for (auto& notif : write_stall_notifications) { |
93 | 0 | for (auto& listener : notif.immutable_options->listeners) { |
94 | 0 | listener->OnStallConditionsChanged(notif.write_stall_info); |
95 | 0 | } |
96 | 0 | } |
97 | 1.11M | write_stall_notifications.clear(); |
98 | 1.11M | #endif |
99 | | // free superversions |
100 | 1.11M | for (auto s : superversions_to_free) { |
101 | 8.90k | delete s; |
102 | 8.90k | } |
103 | 1.11M | superversions_to_free.clear(); |
104 | 1.11M | } |
105 | | |
106 | 1.33M | ~SuperVersionContext() { |
107 | 1.33M | #ifndef ROCKSDB_DISABLE_STALL_NOTIFICATION |
108 | 1.33M | assert(write_stall_notifications.empty()); |
109 | 1.33M | #endif |
110 | 1.33M | assert(superversions_to_free.empty()); |
111 | 1.33M | } |
112 | | }; |
113 | | |
114 | | struct JobContext { |
115 | 332k | inline bool HaveSomethingToDelete() const { |
116 | 332k | return !(full_scan_candidate_files.empty() && sst_delete_files.empty() && |
117 | 142k | blob_delete_files.empty() && log_delete_files.empty() && |
118 | 139k | manifest_delete_files.empty()); |
119 | 332k | } |
120 | | |
121 | 7.79k | inline bool HaveSomethingToClean() const { |
122 | 7.79k | bool sv_have_sth = false; |
123 | 7.79k | for (const auto& sv_ctx : superversion_contexts) { |
124 | 7.79k | if (sv_ctx.HaveSomethingToDelete()) { |
125 | 7.09k | sv_have_sth = true; |
126 | 7.09k | break; |
127 | 7.09k | } |
128 | 7.79k | } |
129 | 7.79k | return memtables_to_free.size() > 0 || wals_to_free.size() > 0 || |
130 | 5.98k | job_snapshot != nullptr || sv_have_sth; |
131 | 7.79k | } |
132 | | |
133 | 4.58k | SequenceNumber GetJobSnapshotSequence() const { |
134 | 4.58k | if (job_snapshot) { |
135 | 0 | assert(job_snapshot->snapshot()); |
136 | 0 | return job_snapshot->snapshot()->GetSequenceNumber(); |
137 | 0 | } |
138 | 4.58k | return kMaxSequenceNumber; |
139 | 4.58k | } |
140 | | |
141 | 2.77k | SequenceNumber GetLatestSnapshotSequence() const { |
142 | 2.77k | assert(snapshot_context_initialized); |
143 | 2.77k | if (snapshot_seqs.empty()) { |
144 | 2.77k | return 0; |
145 | 2.77k | } |
146 | 0 | return snapshot_seqs.back(); |
147 | 2.77k | } |
148 | | |
149 | 4.58k | SequenceNumber GetEarliestSnapshotSequence() const { |
150 | 4.58k | assert(snapshot_context_initialized); |
151 | 4.58k | if (snapshot_seqs.empty()) { |
152 | 4.58k | return kMaxSequenceNumber; |
153 | 4.58k | } |
154 | 0 | return snapshot_seqs.front(); |
155 | 4.58k | } |
156 | | |
157 | | void InitSnapshotContext(SnapshotChecker* checker, |
158 | | std::unique_ptr<ManagedSnapshot> managed_snapshot, |
159 | | SequenceNumber earliest_write_conflict, |
160 | 4.58k | std::vector<SequenceNumber>&& snapshots) { |
161 | 4.58k | if (snapshot_context_initialized) { |
162 | 0 | return; |
163 | 0 | } |
164 | 4.58k | snapshot_context_initialized = true; |
165 | 4.58k | snapshot_checker = checker; |
166 | 4.58k | assert(!job_snapshot); |
167 | 4.58k | job_snapshot = std::move(managed_snapshot); |
168 | 4.58k | earliest_write_conflict_snapshot = earliest_write_conflict; |
169 | 4.58k | snapshot_seqs = std::move(snapshots); |
170 | 4.58k | } |
171 | | |
172 | | // Structure to store information for candidate files to delete. |
173 | | struct CandidateFileInfo { |
174 | | std::string file_name; |
175 | | std::string file_path; |
176 | | CandidateFileInfo(std::string name, std::string path) |
177 | 1.61M | : file_name(std::move(name)), file_path(std::move(path)) {} |
178 | 1.52M | bool operator==(const CandidateFileInfo& other) const { |
179 | 1.52M | return file_name == other.file_name && file_path == other.file_path; |
180 | 1.52M | } |
181 | | }; |
182 | | |
183 | | // a list of all files that we'll consider deleting |
184 | | // (every once in a while this is filled up with all files |
185 | | // in the DB directory) |
186 | | // (filled only if we're doing full scan) |
187 | | std::vector<CandidateFileInfo> full_scan_candidate_files; |
188 | | |
189 | | // the list of all live sst files that cannot be deleted |
190 | | std::vector<uint64_t> sst_live; |
191 | | |
192 | | // the list of sst files that we need to delete |
193 | | std::vector<ObsoleteFileInfo> sst_delete_files; |
194 | | |
195 | | // the list of all live blob files that cannot be deleted |
196 | | std::vector<uint64_t> blob_live; |
197 | | |
198 | | // the list of blob files that we need to delete |
199 | | std::vector<ObsoleteBlobFileInfo> blob_delete_files; |
200 | | |
201 | | // a list of log files that we need to delete |
202 | | std::vector<uint64_t> log_delete_files; |
203 | | |
204 | | // a list of log files that we need to preserve during full purge since they |
205 | | // will be reused later |
206 | | std::vector<uint64_t> log_recycle_files; |
207 | | |
208 | | // Files quarantined from deletion. This list contains file numbers for files |
209 | | // that are in an ambiguous states. This includes newly generated SST files |
210 | | // and blob files from flush and compaction job whose VersionEdits' persist |
211 | | // state in Manifest are unclear. An old manifest file whose immediately |
212 | | // following new manifest file's CURRENT file creation is in an unclear state. |
213 | | // WAL logs don't have this premature deletion risk since |
214 | | // min_log_number_to_keep is only updated after successful manifest commits. |
215 | | // So this data structure doesn't track log files. |
216 | | autovector<uint64_t> files_to_quarantine; |
217 | | |
218 | | // Blob file numbers that PurgeObsoleteFiles must keep. This includes both |
219 | | // actively written direct-write files and sealed direct-write files that are |
220 | | // still reachable through live memtables / old SuperVersions. |
221 | | // Collected under db_mutex_ in FindObsoleteFiles so PurgeObsoleteFiles can |
222 | | // safely use the snapshot without taking DB mutex. |
223 | | UnorderedSet<uint64_t> active_blob_direct_write_files; |
224 | | |
225 | | // Snapshot of VersionSet's next file number taken before collecting active |
226 | | // direct-write blob files. This keeps the current purge pass from racing a |
227 | | // concurrently created blob file that was not yet part of the active set. |
228 | | uint64_t min_blob_file_number_to_keep = std::numeric_limits<uint64_t>::max(); |
229 | | |
230 | | // a list of manifest files that we need to delete |
231 | | std::vector<std::string> manifest_delete_files; |
232 | | |
233 | | // a list of memtables to be free |
234 | | autovector<ReadOnlyMemTable*> memtables_to_free; |
235 | | |
236 | | // contexts for installing superversions for multiple column families |
237 | | std::vector<SuperVersionContext> superversion_contexts; |
238 | | |
239 | | autovector<log::Writer*> wals_to_free; |
240 | | |
241 | | // the current manifest_file_number, log_number and prev_log_number |
242 | | // that corresponds to the set of files in 'live'. |
243 | | uint64_t manifest_file_number = 0; |
244 | | uint64_t pending_manifest_file_number = 0; |
245 | | |
246 | | // Used for remote compaction. To prevent OPTIONS files from getting |
247 | | // purged by PurgeObsoleteFiles() of the primary host |
248 | | uint64_t min_options_file_number; |
249 | | // Exact OPTIONS file committed by the current MANIFEST. Unlike |
250 | | // min_options_file_number, this protects one authoritative file even when |
251 | | // higher-numbered uncommitted OPTIONS files exist. |
252 | | uint64_t manifest_options_file_number = 0; |
253 | | uint64_t log_number = 0; |
254 | | uint64_t prev_log_number = 0; |
255 | | |
256 | | uint64_t min_pending_output = 0; |
257 | | uint64_t prev_wals_total_size = 0; |
258 | | size_t num_alive_wal_files = 0; |
259 | | uint64_t size_log_to_delete = 0; |
260 | | |
261 | | // Snapshot taken before flush/compaction job. |
262 | | std::unique_ptr<ManagedSnapshot> job_snapshot; |
263 | | SnapshotChecker* snapshot_checker = nullptr; |
264 | | std::vector<SequenceNumber> snapshot_seqs; |
265 | | // This is the earliest snapshot that could be used for write-conflict |
266 | | // checking by a transaction. For any user-key newer than this snapshot, we |
267 | | // should make sure not to remove evidence that a write occurred. |
268 | | SequenceNumber earliest_write_conflict_snapshot = kMaxSequenceNumber; |
269 | | |
270 | | // Unique job id |
271 | | int job_id; |
272 | | |
273 | | bool snapshot_context_initialized = false; |
274 | | |
275 | 218k | explicit JobContext(int _job_id, bool create_superversion = false) { |
276 | 218k | job_id = _job_id; |
277 | 218k | superversion_contexts.emplace_back( |
278 | 218k | SuperVersionContext(create_superversion)); |
279 | 218k | } |
280 | | |
281 | | // Delete the default constructor |
282 | | JobContext() = delete; |
283 | | |
284 | | // No copying allowed |
285 | | JobContext(const JobContext&) = delete; |
286 | | JobContext& operator=(const JobContext&) = delete; |
287 | | JobContext(JobContext&&) = delete; |
288 | | JobContext& operator=(JobContext&&) = delete; |
289 | | |
290 | | // For non-empty JobContext Clean() has to be called at least once before |
291 | | // before destruction (see asserts in ~JobContext()). Should be called with |
292 | | // unlocked DB mutex. Destructor doesn't call Clean() to avoid accidentally |
293 | | // doing potentially slow Clean() with locked DB mutex. |
294 | 218k | void Clean() { |
295 | | // free superversions |
296 | 218k | for (auto& sv_context : superversion_contexts) { |
297 | 218k | sv_context.Clean(); |
298 | 218k | } |
299 | | // free pending memtables |
300 | 218k | for (auto m : memtables_to_free) { |
301 | 0 | delete m; |
302 | 0 | } |
303 | 218k | for (auto l : wals_to_free) { |
304 | 1.81k | delete l; |
305 | 1.81k | } |
306 | | |
307 | 218k | memtables_to_free.clear(); |
308 | 218k | wals_to_free.clear(); |
309 | 218k | job_snapshot.reset(); |
310 | 218k | } |
311 | | |
312 | 218k | ~JobContext() { |
313 | 218k | assert(memtables_to_free.size() == 0); |
314 | | assert(wals_to_free.size() == 0); |
315 | 218k | } |
316 | | }; |
317 | | |
318 | | } // namespace ROCKSDB_NAMESPACE |