/src/rocksdb/db/internal_stats.cc
Line | Count | Source |
1 | | // This source code is licensed under both the GPLv2 (found in the |
2 | | // COPYING file in the root directory) and Apache 2.0 License |
3 | | // (found in the LICENSE.Apache file in the root directory). |
4 | | // |
5 | | // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. |
6 | | // |
7 | | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. |
8 | | // Use of this source code is governed by a BSD-style license that can be |
9 | | // found in the LICENSE file. See the AUTHORS file for names of contributors. |
10 | | |
11 | | #include "db/internal_stats.h" |
12 | | |
13 | | #include <algorithm> |
14 | | #include <cinttypes> |
15 | | #include <cstddef> |
16 | | #include <limits> |
17 | | #include <sstream> |
18 | | #include <string> |
19 | | #include <unordered_map> |
20 | | #include <utility> |
21 | | #include <vector> |
22 | | |
23 | | #include "cache/cache_entry_roles.h" |
24 | | #include "cache/cache_entry_stats.h" |
25 | | #include "db/column_family.h" |
26 | | #include "db/db_impl/db_impl.h" |
27 | | #include "db/write_stall_stats.h" |
28 | | #include "port/port.h" |
29 | | #include "rocksdb/system_clock.h" |
30 | | #include "rocksdb/table.h" |
31 | | #include "table/block_based/cachable_entry.h" |
32 | | #include "util/hash_containers.h" |
33 | | #include "util/string_util.h" |
34 | | |
35 | | namespace ROCKSDB_NAMESPACE { |
36 | | |
37 | | const std::map<LevelStatType, LevelStat> InternalStats::compaction_level_stats = |
38 | | { |
39 | | {LevelStatType::NUM_FILES, LevelStat{"NumFiles", "Files"}}, |
40 | | {LevelStatType::COMPACTED_FILES, |
41 | | LevelStat{"CompactedFiles", "CompactedFiles"}}, |
42 | | {LevelStatType::SIZE_BYTES, LevelStat{"SizeBytes", "Size"}}, |
43 | | {LevelStatType::SCORE, LevelStat{"Score", "Score"}}, |
44 | | {LevelStatType::READ_GB, LevelStat{"ReadGB", "Read(GB)"}}, |
45 | | {LevelStatType::RN_GB, LevelStat{"RnGB", "Rn(GB)"}}, |
46 | | {LevelStatType::RNP1_GB, LevelStat{"Rnp1GB", "Rnp1(GB)"}}, |
47 | | {LevelStatType::WRITE_GB, LevelStat{"WriteGB", "Write(GB)"}}, |
48 | | {LevelStatType::WRITE_PRE_COMP_GB, |
49 | | LevelStat{"WPreCompGB", "WPreComp(GB)"}}, |
50 | | {LevelStatType::W_NEW_GB, LevelStat{"WnewGB", "Wnew(GB)"}}, |
51 | | {LevelStatType::MOVED_GB, LevelStat{"MovedGB", "Moved(GB)"}}, |
52 | | {LevelStatType::WRITE_AMP, LevelStat{"WriteAmp", "W-Amp"}}, |
53 | | {LevelStatType::READ_MBPS, LevelStat{"ReadMBps", "Rd(MB/s)"}}, |
54 | | {LevelStatType::WRITE_MBPS, LevelStat{"WriteMBps", "Wr(MB/s)"}}, |
55 | | {LevelStatType::COMP_SEC, LevelStat{"CompSec", "Comp(sec)"}}, |
56 | | {LevelStatType::COMP_CPU_SEC, |
57 | | LevelStat{"CompMergeCPU", "CompMergeCPU(sec)"}}, |
58 | | {LevelStatType::COMP_COUNT, LevelStat{"CompCount", "Comp(cnt)"}}, |
59 | | {LevelStatType::AVG_SEC, LevelStat{"AvgSec", "Avg(sec)"}}, |
60 | | {LevelStatType::KEY_IN, LevelStat{"KeyIn", "KeyIn"}}, |
61 | | {LevelStatType::KEY_DROP, LevelStat{"KeyDrop", "KeyDrop"}}, |
62 | | {LevelStatType::R_BLOB_GB, LevelStat{"RblobGB", "Rblob(GB)"}}, |
63 | | {LevelStatType::W_BLOB_GB, LevelStat{"WblobGB", "Wblob(GB)"}}, |
64 | | }; |
65 | | |
66 | | const std::map<InternalStats::InternalDBStatsType, DBStatInfo> |
67 | | InternalStats::db_stats_type_to_info = { |
68 | | {InternalStats::kIntStatsWalFileBytes, |
69 | | DBStatInfo{"db.wal_bytes_written"}}, |
70 | | {InternalStats::kIntStatsWalFileSynced, DBStatInfo{"db.wal_syncs"}}, |
71 | | {InternalStats::kIntStatsBytesWritten, |
72 | | DBStatInfo{"db.user_bytes_written"}}, |
73 | | {InternalStats::kIntStatsNumKeysWritten, |
74 | | DBStatInfo{"db.user_keys_written"}}, |
75 | | {InternalStats::kIntStatsWriteDoneByOther, |
76 | | DBStatInfo{"db.user_writes_by_other"}}, |
77 | | {InternalStats::kIntStatsWriteDoneBySelf, |
78 | | DBStatInfo{"db.user_writes_by_self"}}, |
79 | | {InternalStats::kIntStatsWriteWithWal, |
80 | | DBStatInfo{"db.user_writes_with_wal"}}, |
81 | | {InternalStats::kIntStatsWriteStallMicros, |
82 | | DBStatInfo{"db.user_write_stall_micros"}}, |
83 | | {InternalStats::kIntStatsWriteBufferManagerLimitStopsCounts, |
84 | | DBStatInfo{WriteStallStatsMapKeys::CauseConditionCount( |
85 | | WriteStallCause::kWriteBufferManagerLimit, |
86 | | WriteStallCondition::kStopped)}}, |
87 | | }; |
88 | | |
89 | | namespace { |
90 | | const double kMB = 1048576.0; |
91 | | const double kGB = kMB * 1024; |
92 | | const double kMicrosInSec = 1000000.0; |
93 | | |
94 | | void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name, |
95 | 708 | const std::string& group_by) { |
96 | 708 | int written_size = |
97 | 708 | snprintf(buf, len, "\n** Compaction Stats [%s] **\n", cf_name.c_str()); |
98 | 708 | written_size = std::min(written_size, static_cast<int>(len)); |
99 | 14.8k | auto hdr = [](LevelStatType t) { |
100 | 14.8k | return InternalStats::compaction_level_stats.at(t).header_name.c_str(); |
101 | 14.8k | }; |
102 | 708 | int line_size = snprintf( |
103 | 708 | buf + written_size, len - written_size, |
104 | 708 | "%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s " |
105 | 708 | "%s " |
106 | 708 | "%s\n", |
107 | | // Note that we skip COMPACTED_FILES and merge it with Files column |
108 | 708 | group_by.c_str(), hdr(LevelStatType::NUM_FILES), |
109 | 708 | hdr(LevelStatType::SIZE_BYTES), hdr(LevelStatType::SCORE), |
110 | 708 | hdr(LevelStatType::READ_GB), hdr(LevelStatType::RN_GB), |
111 | 708 | hdr(LevelStatType::RNP1_GB), hdr(LevelStatType::WRITE_GB), |
112 | 708 | hdr(LevelStatType::WRITE_PRE_COMP_GB), hdr(LevelStatType::W_NEW_GB), |
113 | 708 | hdr(LevelStatType::MOVED_GB), hdr(LevelStatType::WRITE_AMP), |
114 | 708 | hdr(LevelStatType::READ_MBPS), hdr(LevelStatType::WRITE_MBPS), |
115 | 708 | hdr(LevelStatType::COMP_SEC), hdr(LevelStatType::COMP_CPU_SEC), |
116 | 708 | hdr(LevelStatType::COMP_COUNT), hdr(LevelStatType::AVG_SEC), |
117 | 708 | hdr(LevelStatType::KEY_IN), hdr(LevelStatType::KEY_DROP), |
118 | 708 | hdr(LevelStatType::R_BLOB_GB), hdr(LevelStatType::W_BLOB_GB)); |
119 | | |
120 | 708 | written_size += line_size; |
121 | 708 | written_size = std::min(written_size, static_cast<int>(len)); |
122 | 708 | snprintf(buf + written_size, len - written_size, "%s\n", |
123 | 708 | std::string(line_size, '-').c_str()); |
124 | 708 | } |
125 | | |
126 | | void PrepareLevelStats(std::map<LevelStatType, double>* level_stats, |
127 | | int num_files, int being_compacted, |
128 | | double total_file_size, double score, double w_amp, |
129 | 967 | const InternalStats::CompactionStats& stats) { |
130 | 967 | const uint64_t bytes_read = stats.bytes_read_non_output_levels + |
131 | 967 | stats.bytes_read_output_level + |
132 | 967 | stats.bytes_read_blob; |
133 | 967 | const uint64_t bytes_written = stats.bytes_written + stats.bytes_written_blob; |
134 | 967 | const int64_t bytes_new = stats.bytes_written - stats.bytes_read_output_level; |
135 | 967 | const double elapsed = (stats.micros + 1) / kMicrosInSec; |
136 | | |
137 | 967 | (*level_stats)[LevelStatType::NUM_FILES] = num_files; |
138 | 967 | (*level_stats)[LevelStatType::COMPACTED_FILES] = being_compacted; |
139 | 967 | (*level_stats)[LevelStatType::SIZE_BYTES] = total_file_size; |
140 | 967 | (*level_stats)[LevelStatType::SCORE] = score; |
141 | 967 | (*level_stats)[LevelStatType::READ_GB] = bytes_read / kGB; |
142 | 967 | (*level_stats)[LevelStatType::RN_GB] = |
143 | 967 | stats.bytes_read_non_output_levels / kGB; |
144 | 967 | (*level_stats)[LevelStatType::RNP1_GB] = stats.bytes_read_output_level / kGB; |
145 | 967 | (*level_stats)[LevelStatType::WRITE_GB] = stats.bytes_written / kGB; |
146 | 967 | (*level_stats)[LevelStatType::WRITE_PRE_COMP_GB] = |
147 | 967 | stats.bytes_written_pre_comp / kGB; |
148 | 967 | (*level_stats)[LevelStatType::W_NEW_GB] = bytes_new / kGB; |
149 | 967 | (*level_stats)[LevelStatType::MOVED_GB] = stats.bytes_moved / kGB; |
150 | 967 | (*level_stats)[LevelStatType::WRITE_AMP] = w_amp; |
151 | 967 | (*level_stats)[LevelStatType::READ_MBPS] = bytes_read / kMB / elapsed; |
152 | 967 | (*level_stats)[LevelStatType::WRITE_MBPS] = bytes_written / kMB / elapsed; |
153 | 967 | (*level_stats)[LevelStatType::COMP_SEC] = stats.micros / kMicrosInSec; |
154 | 967 | (*level_stats)[LevelStatType::COMP_CPU_SEC] = stats.cpu_micros / kMicrosInSec; |
155 | 967 | (*level_stats)[LevelStatType::COMP_COUNT] = stats.count; |
156 | 967 | (*level_stats)[LevelStatType::AVG_SEC] = |
157 | 967 | stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count; |
158 | 967 | (*level_stats)[LevelStatType::KEY_IN] = |
159 | 967 | static_cast<double>(stats.num_input_records); |
160 | 967 | (*level_stats)[LevelStatType::KEY_DROP] = |
161 | 967 | static_cast<double>(stats.num_dropped_records); |
162 | 967 | (*level_stats)[LevelStatType::R_BLOB_GB] = stats.bytes_read_blob / kGB; |
163 | 967 | (*level_stats)[LevelStatType::W_BLOB_GB] = stats.bytes_written_blob / kGB; |
164 | 967 | } |
165 | | |
166 | | void PrintLevelStats(char* buf, size_t len, const std::string& name, |
167 | 967 | const std::map<LevelStatType, double>& stat_value) { |
168 | 967 | snprintf( |
169 | 967 | buf, len, |
170 | 967 | "%4s " /* Level */ |
171 | 967 | "%6d/%-3d " /* Files */ |
172 | 967 | "%10s " /* Size */ |
173 | 967 | "%5.1f " /* Score */ |
174 | 967 | "%8.1f " /* Read(GB) */ |
175 | 967 | "%7.1f " /* Rn(GB) */ |
176 | 967 | "%8.1f " /* Rnp1(GB) */ |
177 | 967 | "%9.1f " /* Write(GB) */ |
178 | 967 | "%9.1f " /* WPreComp(GB) */ |
179 | 967 | "%8.1f " /* Wnew(GB) */ |
180 | 967 | "%9.1f " /* Moved(GB) */ |
181 | 967 | "%5.1f " /* W-Amp */ |
182 | 967 | "%8.1f " /* Rd(MB/s) */ |
183 | 967 | "%8.1f " /* Wr(MB/s) */ |
184 | 967 | "%9.2f " /* Comp(sec) */ |
185 | 967 | "%17.2f " /* CompMergeCPU(sec) */ |
186 | 967 | "%9d " /* Comp(cnt) */ |
187 | 967 | "%8.3f " /* Avg(sec) */ |
188 | 967 | "%7s " /* KeyIn */ |
189 | 967 | "%6s " /* KeyDrop */ |
190 | 967 | "%9.1f " /* Rblob(GB) */ |
191 | 967 | "%9.1f\n", /* Wblob(GB) */ |
192 | 967 | name.c_str(), static_cast<int>(stat_value.at(LevelStatType::NUM_FILES)), |
193 | 967 | static_cast<int>(stat_value.at(LevelStatType::COMPACTED_FILES)), |
194 | 967 | BytesToHumanString( |
195 | 967 | static_cast<uint64_t>(stat_value.at(LevelStatType::SIZE_BYTES))) |
196 | 967 | .c_str(), |
197 | 967 | stat_value.at(LevelStatType::SCORE), |
198 | 967 | stat_value.at(LevelStatType::READ_GB), |
199 | 967 | stat_value.at(LevelStatType::RN_GB), |
200 | 967 | stat_value.at(LevelStatType::RNP1_GB), |
201 | 967 | stat_value.at(LevelStatType::WRITE_GB), |
202 | 967 | stat_value.at(LevelStatType::WRITE_PRE_COMP_GB), |
203 | 967 | stat_value.at(LevelStatType::W_NEW_GB), |
204 | 967 | stat_value.at(LevelStatType::MOVED_GB), |
205 | 967 | stat_value.at(LevelStatType::WRITE_AMP), |
206 | 967 | stat_value.at(LevelStatType::READ_MBPS), |
207 | 967 | stat_value.at(LevelStatType::WRITE_MBPS), |
208 | 967 | stat_value.at(LevelStatType::COMP_SEC), |
209 | 967 | stat_value.at(LevelStatType::COMP_CPU_SEC), |
210 | 967 | static_cast<int>(stat_value.at(LevelStatType::COMP_COUNT)), |
211 | 967 | stat_value.at(LevelStatType::AVG_SEC), |
212 | 967 | NumberToHumanString( |
213 | 967 | static_cast<std::int64_t>(stat_value.at(LevelStatType::KEY_IN))) |
214 | 967 | .c_str(), |
215 | 967 | NumberToHumanString( |
216 | 967 | static_cast<std::int64_t>(stat_value.at(LevelStatType::KEY_DROP))) |
217 | 967 | .c_str(), |
218 | 967 | stat_value.at(LevelStatType::R_BLOB_GB), |
219 | 967 | stat_value.at(LevelStatType::W_BLOB_GB)); |
220 | 967 | } |
221 | | |
222 | | void PrintLevelStats(char* buf, size_t len, const std::string& name, |
223 | | int num_files, int being_compacted, double total_file_size, |
224 | | double score, double w_amp, |
225 | 354 | const InternalStats::CompactionStats& stats) { |
226 | 354 | std::map<LevelStatType, double> level_stats; |
227 | 354 | PrepareLevelStats(&level_stats, num_files, being_compacted, total_file_size, |
228 | 354 | score, w_amp, stats); |
229 | 354 | PrintLevelStats(buf, len, name, level_stats); |
230 | 354 | } |
231 | | |
232 | | // Assumes that trailing numbers represent an optional argument. This requires |
233 | | // property names to not end with numbers. |
234 | 2.86k | std::pair<Slice, Slice> GetPropertyNameAndArg(const Slice& property) { |
235 | 2.86k | Slice name = property; |
236 | 2.86k | Slice arg = property; |
237 | 2.86k | size_t sfx_len = 0; |
238 | 2.95k | while (sfx_len < property.size() && |
239 | 1.46k | isdigit(property[property.size() - sfx_len - 1])) { |
240 | 97 | ++sfx_len; |
241 | 97 | } |
242 | 2.86k | name.remove_suffix(sfx_len); |
243 | 2.86k | arg.remove_prefix(property.size() - sfx_len); |
244 | 2.86k | return {name, arg}; |
245 | 2.86k | } |
246 | | } // anonymous namespace |
247 | | |
248 | | static const std::string rocksdb_prefix = "rocksdb."; |
249 | | |
250 | | static const std::string num_files_at_level_prefix = "num-files-at-level"; |
251 | | static const std::string compression_ratio_at_level_prefix = |
252 | | "compression-ratio-at-level"; |
253 | | static const std::string allstats = "stats"; |
254 | | static const std::string sstables = "sstables"; |
255 | | static const std::string cfstats = "cfstats"; |
256 | | static const std::string cfstats_no_file_histogram = |
257 | | "cfstats-no-file-histogram"; |
258 | | static const std::string cf_file_histogram = "cf-file-histogram"; |
259 | | static const std::string cf_write_stall_stats = "cf-write-stall-stats"; |
260 | | static const std::string dbstats = "dbstats"; |
261 | | static const std::string db_write_stall_stats = "db-write-stall-stats"; |
262 | | static const std::string levelstats = "levelstats"; |
263 | | static const std::string block_cache_entry_stats = "block-cache-entry-stats"; |
264 | | static const std::string fast_block_cache_entry_stats = |
265 | | "fast-block-cache-entry-stats"; |
266 | | static const std::string num_immutable_mem_table = "num-immutable-mem-table"; |
267 | | static const std::string num_immutable_mem_table_flushed = |
268 | | "num-immutable-mem-table-flushed"; |
269 | | static const std::string mem_table_flush_pending = "mem-table-flush-pending"; |
270 | | static const std::string compaction_pending = "compaction-pending"; |
271 | | static const std::string background_errors = "background-errors"; |
272 | | static const std::string cur_size_active_mem_table = |
273 | | "cur-size-active-mem-table"; |
274 | | static const std::string cur_size_all_mem_tables = "cur-size-all-mem-tables"; |
275 | | static const std::string size_all_mem_tables = "size-all-mem-tables"; |
276 | | static const std::string num_entries_active_mem_table = |
277 | | "num-entries-active-mem-table"; |
278 | | static const std::string num_entries_imm_mem_tables = |
279 | | "num-entries-imm-mem-tables"; |
280 | | static const std::string num_deletes_active_mem_table = |
281 | | "num-deletes-active-mem-table"; |
282 | | static const std::string num_deletes_imm_mem_tables = |
283 | | "num-deletes-imm-mem-tables"; |
284 | | static const std::string estimate_num_keys = "estimate-num-keys"; |
285 | | static const std::string estimate_table_readers_mem = |
286 | | "estimate-table-readers-mem"; |
287 | | static const std::string is_file_deletions_enabled = |
288 | | "is-file-deletions-enabled"; |
289 | | static const std::string num_snapshots = "num-snapshots"; |
290 | | static const std::string oldest_snapshot_time = "oldest-snapshot-time"; |
291 | | static const std::string oldest_snapshot_sequence = "oldest-snapshot-sequence"; |
292 | | static const std::string num_live_versions = "num-live-versions"; |
293 | | static const std::string current_version_number = |
294 | | "current-super-version-number"; |
295 | | static const std::string estimate_live_data_size = "estimate-live-data-size"; |
296 | | static const std::string min_log_number_to_keep_str = "min-log-number-to-keep"; |
297 | | static const std::string min_obsolete_sst_number_to_keep_str = |
298 | | "min-obsolete-sst-number-to-keep"; |
299 | | static const std::string base_level_str = "base-level"; |
300 | | static const std::string total_sst_files_size = "total-sst-files-size"; |
301 | | static const std::string live_sst_files_size = "live-sst-files-size"; |
302 | | static const std::string obsolete_sst_files_size = "obsolete-sst-files-size"; |
303 | | static const std::string live_sst_files_size_at_temperature = |
304 | | "live-sst-files-size-at-temperature"; |
305 | | static const std::string estimate_pending_comp_bytes = |
306 | | "estimate-pending-compaction-bytes"; |
307 | | static const std::string aggregated_table_properties = |
308 | | "aggregated-table-properties"; |
309 | | static const std::string aggregated_table_properties_at_level = |
310 | | aggregated_table_properties + "-at-level"; |
311 | | static const std::string num_running_compactions = "num-running-compactions"; |
312 | | static const std::string num_running_remote_compactions = |
313 | | "num-running-remote-compactions"; |
314 | | static const std::string num_remote_compaction_pool_threads = |
315 | | "num-remote-compaction-pool-threads"; |
316 | | static const std::string num_running_bottom_compactions = |
317 | | "num-running-bottom-compactions"; |
318 | | static const std::string num_running_compaction_sorted_runs = |
319 | | "num-running-compaction-sorted-runs"; |
320 | | static const std::string compaction_abort_count = "compaction-abort-count"; |
321 | | static const std::string num_unscheduled_compactions = |
322 | | "num-unscheduled-compactions"; |
323 | | static const std::string num_running_flushes = "num-running-flushes"; |
324 | | static const std::string actual_delayed_write_rate = |
325 | | "actual-delayed-write-rate"; |
326 | | static const std::string is_write_stopped = "is-write-stopped"; |
327 | | static const std::string estimate_oldest_key_time = "estimate-oldest-key-time"; |
328 | | static const std::string block_cache_capacity = "block-cache-capacity"; |
329 | | static const std::string block_cache_usage = "block-cache-usage"; |
330 | | static const std::string block_cache_pinned_usage = "block-cache-pinned-usage"; |
331 | | static const std::string options_statistics = "options-statistics"; |
332 | | static const std::string num_blob_files = "num-blob-files"; |
333 | | static const std::string blob_stats = "blob-stats"; |
334 | | static const std::string total_blob_file_size = "total-blob-file-size"; |
335 | | static const std::string live_blob_file_size = "live-blob-file-size"; |
336 | | static const std::string live_blob_file_garbage_size = |
337 | | "live-blob-file-garbage-size"; |
338 | | static const std::string blob_cache_capacity = "blob-cache-capacity"; |
339 | | static const std::string blob_cache_usage = "blob-cache-usage"; |
340 | | static const std::string blob_cache_pinned_usage = "blob-cache-pinned-usage"; |
341 | | |
342 | | const std::string DB::Properties::kNumFilesAtLevelPrefix = |
343 | | rocksdb_prefix + num_files_at_level_prefix; |
344 | | const std::string DB::Properties::kCompressionRatioAtLevelPrefix = |
345 | | rocksdb_prefix + compression_ratio_at_level_prefix; |
346 | | const std::string DB::Properties::kStats = rocksdb_prefix + allstats; |
347 | | const std::string DB::Properties::kSSTables = rocksdb_prefix + sstables; |
348 | | const std::string DB::Properties::kCFStats = rocksdb_prefix + cfstats; |
349 | | const std::string DB::Properties::kCFStatsNoFileHistogram = |
350 | | rocksdb_prefix + cfstats_no_file_histogram; |
351 | | const std::string DB::Properties::kCFFileHistogram = |
352 | | rocksdb_prefix + cf_file_histogram; |
353 | | const std::string DB::Properties::kCFWriteStallStats = |
354 | | rocksdb_prefix + cf_write_stall_stats; |
355 | | const std::string DB::Properties::kDBWriteStallStats = |
356 | | rocksdb_prefix + db_write_stall_stats; |
357 | | const std::string DB::Properties::kDBStats = rocksdb_prefix + dbstats; |
358 | | const std::string DB::Properties::kLevelStats = rocksdb_prefix + levelstats; |
359 | | const std::string DB::Properties::kBlockCacheEntryStats = |
360 | | rocksdb_prefix + block_cache_entry_stats; |
361 | | const std::string DB::Properties::kFastBlockCacheEntryStats = |
362 | | rocksdb_prefix + fast_block_cache_entry_stats; |
363 | | const std::string DB::Properties::kNumImmutableMemTable = |
364 | | rocksdb_prefix + num_immutable_mem_table; |
365 | | const std::string DB::Properties::kNumImmutableMemTableFlushed = |
366 | | rocksdb_prefix + num_immutable_mem_table_flushed; |
367 | | const std::string DB::Properties::kMemTableFlushPending = |
368 | | rocksdb_prefix + mem_table_flush_pending; |
369 | | const std::string DB::Properties::kCompactionPending = |
370 | | rocksdb_prefix + compaction_pending; |
371 | | const std::string DB::Properties::kNumRunningCompactions = |
372 | | rocksdb_prefix + num_running_compactions; |
373 | | const std::string DB::Properties::kNumRunningRemoteCompactions = |
374 | | rocksdb_prefix + num_running_remote_compactions; |
375 | | const std::string DB::Properties::kNumRemoteCompactionPoolThreads = |
376 | | rocksdb_prefix + num_remote_compaction_pool_threads; |
377 | | const std::string DB::Properties::kNumRunningBottomCompactions = |
378 | | rocksdb_prefix + num_running_bottom_compactions; |
379 | | const std::string DB::Properties::kNumRunningCompactionSortedRuns = |
380 | | rocksdb_prefix + num_running_compaction_sorted_runs; |
381 | | const std::string DB::Properties::kCompactionAbortCount = |
382 | | rocksdb_prefix + compaction_abort_count; |
383 | | const std::string DB::Properties::kNumUnscheduledCompactions = |
384 | | rocksdb_prefix + num_unscheduled_compactions; |
385 | | const std::string DB::Properties::kNumRunningFlushes = |
386 | | rocksdb_prefix + num_running_flushes; |
387 | | const std::string DB::Properties::kBackgroundErrors = |
388 | | rocksdb_prefix + background_errors; |
389 | | const std::string DB::Properties::kCurSizeActiveMemTable = |
390 | | rocksdb_prefix + cur_size_active_mem_table; |
391 | | const std::string DB::Properties::kCurSizeAllMemTables = |
392 | | rocksdb_prefix + cur_size_all_mem_tables; |
393 | | const std::string DB::Properties::kSizeAllMemTables = |
394 | | rocksdb_prefix + size_all_mem_tables; |
395 | | const std::string DB::Properties::kNumEntriesActiveMemTable = |
396 | | rocksdb_prefix + num_entries_active_mem_table; |
397 | | const std::string DB::Properties::kNumEntriesImmMemTables = |
398 | | rocksdb_prefix + num_entries_imm_mem_tables; |
399 | | const std::string DB::Properties::kNumDeletesActiveMemTable = |
400 | | rocksdb_prefix + num_deletes_active_mem_table; |
401 | | const std::string DB::Properties::kNumDeletesImmMemTables = |
402 | | rocksdb_prefix + num_deletes_imm_mem_tables; |
403 | | const std::string DB::Properties::kEstimateNumKeys = |
404 | | rocksdb_prefix + estimate_num_keys; |
405 | | const std::string DB::Properties::kEstimateTableReadersMem = |
406 | | rocksdb_prefix + estimate_table_readers_mem; |
407 | | const std::string DB::Properties::kIsFileDeletionsEnabled = |
408 | | rocksdb_prefix + is_file_deletions_enabled; |
409 | | const std::string DB::Properties::kNumSnapshots = |
410 | | rocksdb_prefix + num_snapshots; |
411 | | const std::string DB::Properties::kOldestSnapshotTime = |
412 | | rocksdb_prefix + oldest_snapshot_time; |
413 | | const std::string DB::Properties::kOldestSnapshotSequence = |
414 | | rocksdb_prefix + oldest_snapshot_sequence; |
415 | | const std::string DB::Properties::kNumLiveVersions = |
416 | | rocksdb_prefix + num_live_versions; |
417 | | const std::string DB::Properties::kCurrentSuperVersionNumber = |
418 | | rocksdb_prefix + current_version_number; |
419 | | const std::string DB::Properties::kEstimateLiveDataSize = |
420 | | rocksdb_prefix + estimate_live_data_size; |
421 | | const std::string DB::Properties::kMinLogNumberToKeep = |
422 | | rocksdb_prefix + min_log_number_to_keep_str; |
423 | | const std::string DB::Properties::kMinObsoleteSstNumberToKeep = |
424 | | rocksdb_prefix + min_obsolete_sst_number_to_keep_str; |
425 | | const std::string DB::Properties::kTotalSstFilesSize = |
426 | | rocksdb_prefix + total_sst_files_size; |
427 | | const std::string DB::Properties::kLiveSstFilesSize = |
428 | | rocksdb_prefix + live_sst_files_size; |
429 | | const std::string DB::Properties::kObsoleteSstFilesSize = |
430 | | rocksdb_prefix + obsolete_sst_files_size; |
431 | | const std::string DB::Properties::kBaseLevel = rocksdb_prefix + base_level_str; |
432 | | const std::string DB::Properties::kEstimatePendingCompactionBytes = |
433 | | rocksdb_prefix + estimate_pending_comp_bytes; |
434 | | const std::string DB::Properties::kAggregatedTableProperties = |
435 | | rocksdb_prefix + aggregated_table_properties; |
436 | | const std::string DB::Properties::kAggregatedTablePropertiesAtLevel = |
437 | | rocksdb_prefix + aggregated_table_properties_at_level; |
438 | | const std::string DB::Properties::kActualDelayedWriteRate = |
439 | | rocksdb_prefix + actual_delayed_write_rate; |
440 | | const std::string DB::Properties::kIsWriteStopped = |
441 | | rocksdb_prefix + is_write_stopped; |
442 | | const std::string DB::Properties::kEstimateOldestKeyTime = |
443 | | rocksdb_prefix + estimate_oldest_key_time; |
444 | | const std::string DB::Properties::kBlockCacheCapacity = |
445 | | rocksdb_prefix + block_cache_capacity; |
446 | | const std::string DB::Properties::kBlockCacheUsage = |
447 | | rocksdb_prefix + block_cache_usage; |
448 | | const std::string DB::Properties::kBlockCachePinnedUsage = |
449 | | rocksdb_prefix + block_cache_pinned_usage; |
450 | | const std::string DB::Properties::kOptionsStatistics = |
451 | | rocksdb_prefix + options_statistics; |
452 | | const std::string DB::Properties::kLiveSstFilesSizeAtTemperature = |
453 | | rocksdb_prefix + live_sst_files_size_at_temperature; |
454 | | const std::string DB::Properties::kNumBlobFiles = |
455 | | rocksdb_prefix + num_blob_files; |
456 | | const std::string DB::Properties::kBlobStats = rocksdb_prefix + blob_stats; |
457 | | const std::string DB::Properties::kTotalBlobFileSize = |
458 | | rocksdb_prefix + total_blob_file_size; |
459 | | const std::string DB::Properties::kLiveBlobFileSize = |
460 | | rocksdb_prefix + live_blob_file_size; |
461 | | const std::string DB::Properties::kLiveBlobFileGarbageSize = |
462 | | rocksdb_prefix + live_blob_file_garbage_size; |
463 | | const std::string DB::Properties::kBlobCacheCapacity = |
464 | | rocksdb_prefix + blob_cache_capacity; |
465 | | const std::string DB::Properties::kBlobCacheUsage = |
466 | | rocksdb_prefix + blob_cache_usage; |
467 | | const std::string DB::Properties::kBlobCachePinnedUsage = |
468 | | rocksdb_prefix + blob_cache_pinned_usage; |
469 | | |
470 | | const std::string InternalStats::kPeriodicCFStats = |
471 | | DB::Properties::kCFStats + ".periodic"; |
472 | | const int InternalStats::kMaxNoChangePeriodSinceDump = 8; |
473 | | |
474 | | const UnorderedMap<std::string, DBPropertyInfo> |
475 | | InternalStats::ppt_name_to_info = { |
476 | | {DB::Properties::kNumFilesAtLevelPrefix, |
477 | | {false, &InternalStats::HandleNumFilesAtLevel, nullptr, nullptr, |
478 | | nullptr}}, |
479 | | {DB::Properties::kCompressionRatioAtLevelPrefix, |
480 | | {false, &InternalStats::HandleCompressionRatioAtLevelPrefix, nullptr, |
481 | | nullptr, nullptr}}, |
482 | | {DB::Properties::kLevelStats, |
483 | | {false, &InternalStats::HandleLevelStats, nullptr, nullptr, nullptr}}, |
484 | | {DB::Properties::kStats, |
485 | | {false, &InternalStats::HandleStats, nullptr, nullptr, nullptr}}, |
486 | | {DB::Properties::kCFStats, |
487 | | {false, &InternalStats::HandleCFStats, nullptr, |
488 | | &InternalStats::HandleCFMapStats, nullptr}}, |
489 | | {InternalStats::kPeriodicCFStats, |
490 | | {false, &InternalStats::HandleCFStatsPeriodic, nullptr, nullptr, |
491 | | nullptr}}, |
492 | | {DB::Properties::kCFStatsNoFileHistogram, |
493 | | {false, &InternalStats::HandleCFStatsNoFileHistogram, nullptr, nullptr, |
494 | | nullptr}}, |
495 | | {DB::Properties::kCFFileHistogram, |
496 | | {false, &InternalStats::HandleCFFileHistogram, nullptr, nullptr, |
497 | | nullptr}}, |
498 | | {DB::Properties::kCFWriteStallStats, |
499 | | {false, &InternalStats::HandleCFWriteStallStats, nullptr, |
500 | | &InternalStats::HandleCFWriteStallStatsMap, nullptr}}, |
501 | | {DB::Properties::kDBStats, |
502 | | {false, &InternalStats::HandleDBStats, nullptr, |
503 | | &InternalStats::HandleDBMapStats, nullptr}}, |
504 | | {DB::Properties::kDBWriteStallStats, |
505 | | {false, &InternalStats::HandleDBWriteStallStats, nullptr, |
506 | | &InternalStats::HandleDBWriteStallStatsMap, nullptr}}, |
507 | | {DB::Properties::kBlockCacheEntryStats, |
508 | | {true, &InternalStats::HandleBlockCacheEntryStats, nullptr, |
509 | | &InternalStats::HandleBlockCacheEntryStatsMap, nullptr}}, |
510 | | {DB::Properties::kFastBlockCacheEntryStats, |
511 | | {true, &InternalStats::HandleFastBlockCacheEntryStats, nullptr, |
512 | | &InternalStats::HandleFastBlockCacheEntryStatsMap, nullptr}}, |
513 | | {DB::Properties::kSSTables, |
514 | | {false, &InternalStats::HandleSsTables, nullptr, nullptr, nullptr}}, |
515 | | {DB::Properties::kAggregatedTableProperties, |
516 | | {false, &InternalStats::HandleAggregatedTableProperties, nullptr, |
517 | | &InternalStats::HandleAggregatedTablePropertiesMap, nullptr}}, |
518 | | {DB::Properties::kAggregatedTablePropertiesAtLevel, |
519 | | {false, &InternalStats::HandleAggregatedTablePropertiesAtLevel, |
520 | | nullptr, &InternalStats::HandleAggregatedTablePropertiesAtLevelMap, |
521 | | nullptr}}, |
522 | | {DB::Properties::kNumImmutableMemTable, |
523 | | {false, nullptr, &InternalStats::HandleNumImmutableMemTable, nullptr, |
524 | | nullptr}}, |
525 | | {DB::Properties::kNumImmutableMemTableFlushed, |
526 | | {false, nullptr, &InternalStats::HandleNumImmutableMemTableFlushed, |
527 | | nullptr, nullptr}}, |
528 | | {DB::Properties::kMemTableFlushPending, |
529 | | {false, nullptr, &InternalStats::HandleMemTableFlushPending, nullptr, |
530 | | nullptr}}, |
531 | | {DB::Properties::kCompactionPending, |
532 | | {false, nullptr, &InternalStats::HandleCompactionPending, nullptr, |
533 | | nullptr}}, |
534 | | {DB::Properties::kBackgroundErrors, |
535 | | {false, nullptr, &InternalStats::HandleBackgroundErrors, nullptr, |
536 | | nullptr}}, |
537 | | {DB::Properties::kCurSizeActiveMemTable, |
538 | | {false, nullptr, &InternalStats::HandleCurSizeActiveMemTable, nullptr, |
539 | | nullptr}}, |
540 | | {DB::Properties::kCurSizeAllMemTables, |
541 | | {false, nullptr, &InternalStats::HandleCurSizeAllMemTables, nullptr, |
542 | | nullptr}}, |
543 | | {DB::Properties::kSizeAllMemTables, |
544 | | {false, nullptr, &InternalStats::HandleSizeAllMemTables, nullptr, |
545 | | nullptr}}, |
546 | | {DB::Properties::kNumEntriesActiveMemTable, |
547 | | {false, nullptr, &InternalStats::HandleNumEntriesActiveMemTable, |
548 | | nullptr, nullptr}}, |
549 | | {DB::Properties::kNumEntriesImmMemTables, |
550 | | {false, nullptr, &InternalStats::HandleNumEntriesImmMemTables, nullptr, |
551 | | nullptr}}, |
552 | | {DB::Properties::kNumDeletesActiveMemTable, |
553 | | {false, nullptr, &InternalStats::HandleNumDeletesActiveMemTable, |
554 | | nullptr, nullptr}}, |
555 | | {DB::Properties::kNumDeletesImmMemTables, |
556 | | {false, nullptr, &InternalStats::HandleNumDeletesImmMemTables, nullptr, |
557 | | nullptr}}, |
558 | | {DB::Properties::kEstimateNumKeys, |
559 | | {false, nullptr, &InternalStats::HandleEstimateNumKeys, nullptr, |
560 | | nullptr}}, |
561 | | {DB::Properties::kEstimateTableReadersMem, |
562 | | {true, nullptr, &InternalStats::HandleEstimateTableReadersMem, nullptr, |
563 | | nullptr}}, |
564 | | {DB::Properties::kIsFileDeletionsEnabled, |
565 | | {false, nullptr, &InternalStats::HandleIsFileDeletionsEnabled, nullptr, |
566 | | nullptr}}, |
567 | | {DB::Properties::kNumSnapshots, |
568 | | {false, nullptr, &InternalStats::HandleNumSnapshots, nullptr, |
569 | | nullptr}}, |
570 | | {DB::Properties::kOldestSnapshotTime, |
571 | | {false, nullptr, &InternalStats::HandleOldestSnapshotTime, nullptr, |
572 | | nullptr}}, |
573 | | {DB::Properties::kOldestSnapshotSequence, |
574 | | {false, nullptr, &InternalStats::HandleOldestSnapshotSequence, nullptr, |
575 | | nullptr}}, |
576 | | {DB::Properties::kNumLiveVersions, |
577 | | {false, nullptr, &InternalStats::HandleNumLiveVersions, nullptr, |
578 | | nullptr}}, |
579 | | {DB::Properties::kCurrentSuperVersionNumber, |
580 | | {false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber, |
581 | | nullptr, nullptr}}, |
582 | | {DB::Properties::kEstimateLiveDataSize, |
583 | | {true, nullptr, &InternalStats::HandleEstimateLiveDataSize, nullptr, |
584 | | nullptr}}, |
585 | | {DB::Properties::kMinLogNumberToKeep, |
586 | | {false, nullptr, &InternalStats::HandleMinLogNumberToKeep, nullptr, |
587 | | nullptr}}, |
588 | | {DB::Properties::kMinObsoleteSstNumberToKeep, |
589 | | {false, nullptr, &InternalStats::HandleMinObsoleteSstNumberToKeep, |
590 | | nullptr, nullptr}}, |
591 | | {DB::Properties::kBaseLevel, |
592 | | {false, nullptr, &InternalStats::HandleBaseLevel, nullptr, nullptr}}, |
593 | | {DB::Properties::kTotalSstFilesSize, |
594 | | {false, nullptr, &InternalStats::HandleTotalSstFilesSize, nullptr, |
595 | | nullptr}}, |
596 | | {DB::Properties::kLiveSstFilesSize, |
597 | | {false, nullptr, &InternalStats::HandleLiveSstFilesSize, nullptr, |
598 | | nullptr}}, |
599 | | {DB::Properties::kLiveSstFilesSizeAtTemperature, |
600 | | {false, &InternalStats::HandleLiveSstFilesSizeAtTemperature, nullptr, |
601 | | nullptr, nullptr}}, |
602 | | {DB::Properties::kObsoleteSstFilesSize, |
603 | | {false, nullptr, &InternalStats::HandleObsoleteSstFilesSize, nullptr, |
604 | | nullptr}}, |
605 | | {DB::Properties::kEstimatePendingCompactionBytes, |
606 | | {false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes, |
607 | | nullptr, nullptr}}, |
608 | | {DB::Properties::kNumRunningFlushes, |
609 | | {false, nullptr, &InternalStats::HandleNumRunningFlushes, nullptr, |
610 | | nullptr}}, |
611 | | {DB::Properties::kNumRunningCompactions, |
612 | | {false, nullptr, &InternalStats::HandleNumRunningCompactions, nullptr, |
613 | | nullptr}}, |
614 | | {DB::Properties::kNumRunningRemoteCompactions, |
615 | | {false, nullptr, &InternalStats::HandleNumRunningRemoteCompactions, |
616 | | nullptr, nullptr}}, |
617 | | {DB::Properties::kNumRemoteCompactionPoolThreads, |
618 | | {false, nullptr, &InternalStats::HandleNumRemoteCompactionPoolThreads, |
619 | | nullptr, nullptr}}, |
620 | | {DB::Properties::kNumRunningBottomCompactions, |
621 | | {false, nullptr, &InternalStats::HandleNumRunningBottomCompactions, |
622 | | nullptr, nullptr}}, |
623 | | {DB::Properties::kNumRunningCompactionSortedRuns, |
624 | | {false, nullptr, &InternalStats::HandleNumRunningCompactionSortedRuns, |
625 | | nullptr, nullptr}}, |
626 | | {DB::Properties::kCompactionAbortCount, |
627 | | {false, nullptr, &InternalStats::HandleCompactionAbortCount, nullptr, |
628 | | nullptr}}, |
629 | | {DB::Properties::kNumUnscheduledCompactions, |
630 | | {false, nullptr, &InternalStats::HandleNumUnscheduledCompactions, |
631 | | nullptr, nullptr}}, |
632 | | {DB::Properties::kActualDelayedWriteRate, |
633 | | {false, nullptr, &InternalStats::HandleActualDelayedWriteRate, nullptr, |
634 | | nullptr}}, |
635 | | {DB::Properties::kIsWriteStopped, |
636 | | {false, nullptr, &InternalStats::HandleIsWriteStopped, nullptr, |
637 | | nullptr}}, |
638 | | {DB::Properties::kEstimateOldestKeyTime, |
639 | | {false, nullptr, &InternalStats::HandleEstimateOldestKeyTime, nullptr, |
640 | | nullptr}}, |
641 | | {DB::Properties::kBlockCacheCapacity, |
642 | | {false, nullptr, &InternalStats::HandleBlockCacheCapacity, nullptr, |
643 | | nullptr}}, |
644 | | {DB::Properties::kBlockCacheUsage, |
645 | | {false, nullptr, &InternalStats::HandleBlockCacheUsage, nullptr, |
646 | | nullptr}}, |
647 | | {DB::Properties::kBlockCachePinnedUsage, |
648 | | {false, nullptr, &InternalStats::HandleBlockCachePinnedUsage, nullptr, |
649 | | nullptr}}, |
650 | | {DB::Properties::kOptionsStatistics, |
651 | | {true, nullptr, nullptr, nullptr, |
652 | | &DBImpl::GetPropertyHandleOptionsStatistics}}, |
653 | | {DB::Properties::kNumBlobFiles, |
654 | | {false, nullptr, &InternalStats::HandleNumBlobFiles, nullptr, |
655 | | nullptr}}, |
656 | | {DB::Properties::kBlobStats, |
657 | | {false, &InternalStats::HandleBlobStats, nullptr, nullptr, nullptr}}, |
658 | | {DB::Properties::kTotalBlobFileSize, |
659 | | {false, nullptr, &InternalStats::HandleTotalBlobFileSize, nullptr, |
660 | | nullptr}}, |
661 | | {DB::Properties::kLiveBlobFileSize, |
662 | | {false, nullptr, &InternalStats::HandleLiveBlobFileSize, nullptr, |
663 | | nullptr}}, |
664 | | {DB::Properties::kLiveBlobFileGarbageSize, |
665 | | {false, nullptr, &InternalStats::HandleLiveBlobFileGarbageSize, |
666 | | nullptr, nullptr}}, |
667 | | {DB::Properties::kBlobCacheCapacity, |
668 | | {false, nullptr, &InternalStats::HandleBlobCacheCapacity, nullptr, |
669 | | nullptr}}, |
670 | | {DB::Properties::kBlobCacheUsage, |
671 | | {false, nullptr, &InternalStats::HandleBlobCacheUsage, nullptr, |
672 | | nullptr}}, |
673 | | {DB::Properties::kBlobCachePinnedUsage, |
674 | | {false, nullptr, &InternalStats::HandleBlobCachePinnedUsage, nullptr, |
675 | | nullptr}}, |
676 | | }; |
677 | | |
678 | | InternalStats::InternalStats(int num_levels, SystemClock* clock, |
679 | | ColumnFamilyData* cfd) |
680 | 87.5k | : db_stats_{}, |
681 | 87.5k | cf_stats_value_{}, |
682 | 87.5k | cf_stats_count_{}, |
683 | 87.5k | comp_stats_(num_levels), |
684 | 87.5k | comp_stats_by_pri_(Env::Priority::TOTAL), |
685 | 87.5k | file_read_latency_(num_levels), |
686 | 87.5k | has_cf_change_since_dump_(true), |
687 | 87.5k | bg_error_count_(0), |
688 | 87.5k | num_running_compaction_sorted_runs_(0), |
689 | 87.5k | number_levels_(num_levels), |
690 | 87.5k | clock_(clock), |
691 | 87.5k | cfd_(cfd), |
692 | 87.5k | started_at_(clock->NowMicros()) { |
693 | 87.5k | Cache* block_cache = GetBlockCacheForStats(); |
694 | 87.5k | if (block_cache) { |
695 | | // Extract or create stats collector. Could fail in rare cases. |
696 | 87.5k | Status s = CacheEntryStatsCollector<CacheEntryRoleStats>::GetShared( |
697 | 87.5k | block_cache, clock_, &cache_entry_stats_collector_); |
698 | 87.5k | if (s.ok()) { |
699 | 87.5k | assert(cache_entry_stats_collector_); |
700 | 87.5k | } else { |
701 | 0 | assert(!cache_entry_stats_collector_); |
702 | 0 | } |
703 | 87.5k | } |
704 | 87.5k | } |
705 | | |
706 | | void InternalStats::TEST_GetCacheEntryRoleStats(CacheEntryRoleStats* stats, |
707 | 0 | bool foreground) { |
708 | 0 | CollectCacheEntryStats(foreground); |
709 | 0 | if (cache_entry_stats_collector_) { |
710 | 0 | cache_entry_stats_collector_->GetStats(stats); |
711 | 0 | } |
712 | 0 | } |
713 | | |
714 | 354 | void InternalStats::CollectCacheEntryStats(bool foreground) { |
715 | | // This function is safe to call from any thread because |
716 | | // cache_entry_stats_collector_ field is const after constructor |
717 | | // and ->GetStats does its own synchronization, which also suffices for |
718 | | // cache_entry_stats_. |
719 | | |
720 | 354 | if (!cache_entry_stats_collector_) { |
721 | 0 | return; // nothing to do (e.g. no block cache) |
722 | 0 | } |
723 | | |
724 | | // For "background" collections, strictly cap the collection time by |
725 | | // expanding effective cache TTL. For foreground, be more aggressive about |
726 | | // getting latest data. |
727 | 354 | int min_interval_seconds = foreground ? 10 : 180; |
728 | | // 1/500 = max of 0.2% of one CPU thread |
729 | 354 | int min_interval_factor = foreground ? 10 : 500; |
730 | 354 | cache_entry_stats_collector_->CollectStats(min_interval_seconds, |
731 | 354 | min_interval_factor); |
732 | 354 | } |
733 | | |
734 | | std::function<void(const Slice& key, Cache::ObjectPtr value, size_t charge, |
735 | | const Cache::CacheItemHelper* helper)> |
736 | 354 | InternalStats::CacheEntryRoleStats::GetEntryCallback() { |
737 | 354 | return [&](const Slice& /*key*/, Cache::ObjectPtr /*value*/, size_t charge, |
738 | 420 | const Cache::CacheItemHelper* helper) -> void { |
739 | 420 | size_t role_idx = |
740 | 420 | static_cast<size_t>(helper ? helper->role : CacheEntryRole::kMisc); |
741 | 420 | entry_counts[role_idx]++; |
742 | 420 | total_charges[role_idx] += charge; |
743 | 420 | }; |
744 | 354 | } |
745 | | |
746 | | void InternalStats::CacheEntryRoleStats::BeginCollection( |
747 | 354 | Cache* cache, SystemClock*, uint64_t start_time_micros) { |
748 | 354 | Clear(); |
749 | 354 | last_start_time_micros_ = start_time_micros; |
750 | 354 | ++collection_count; |
751 | 354 | std::ostringstream str; |
752 | 354 | str << cache->Name() << "@" << static_cast<void*>(cache) << "#" |
753 | 354 | << port::GetProcessID(); |
754 | 354 | cache_id = str.str(); |
755 | 354 | cache_capacity = cache->GetCapacity(); |
756 | 354 | cache_usage = cache->GetUsage(); |
757 | 354 | table_size = cache->GetTableAddressCount(); |
758 | 354 | occupancy = cache->GetOccupancyCount(); |
759 | 354 | hash_seed = cache->GetHashSeed(); |
760 | 354 | } |
761 | | |
762 | | void InternalStats::CacheEntryRoleStats::EndCollection( |
763 | 354 | Cache*, SystemClock*, uint64_t end_time_micros) { |
764 | 354 | last_end_time_micros_ = end_time_micros; |
765 | 354 | } |
766 | | |
767 | 0 | void InternalStats::CacheEntryRoleStats::SkippedCollection() { |
768 | 0 | ++copies_of_last_collection; |
769 | 0 | } |
770 | | |
771 | 354 | uint64_t InternalStats::CacheEntryRoleStats::GetLastDurationMicros() const { |
772 | 354 | if (last_end_time_micros_ > last_start_time_micros_) { |
773 | 354 | return last_end_time_micros_ - last_start_time_micros_; |
774 | 354 | } else { |
775 | 0 | return 0U; |
776 | 0 | } |
777 | 354 | } |
778 | | |
779 | | std::string InternalStats::CacheEntryRoleStats::ToString( |
780 | 354 | SystemClock* clock) const { |
781 | 354 | std::ostringstream str; |
782 | 354 | str << "Block cache " << cache_id |
783 | 354 | << " capacity: " << BytesToHumanString(cache_capacity) |
784 | 354 | << " seed: " << hash_seed << " usage: " << BytesToHumanString(cache_usage) |
785 | 354 | << " table_size: " << table_size << " occupancy: " << occupancy |
786 | 354 | << " collections: " << collection_count |
787 | 354 | << " last_copies: " << copies_of_last_collection |
788 | 354 | << " last_secs: " << (GetLastDurationMicros() / 1000000.0) |
789 | 354 | << " secs_since: " |
790 | 354 | << ((clock->NowMicros() - last_end_time_micros_) / 1000000U) << "\n"; |
791 | 354 | str << "Block cache entry stats(count,size,portion):"; |
792 | 5.31k | for (size_t i = 0; i < kNumCacheEntryRoles; ++i) { |
793 | 4.95k | if (entry_counts[i] > 0) { |
794 | 371 | str << " " << kCacheEntryRoleToCamelString[i] << "(" << entry_counts[i] |
795 | 371 | << "," << BytesToHumanString(total_charges[i]) << "," |
796 | 371 | << (100.0 * total_charges[i] / cache_capacity) << "%)"; |
797 | 371 | } |
798 | 4.95k | } |
799 | 354 | str << "\n"; |
800 | 354 | return str.str(); |
801 | 354 | } |
802 | | |
803 | | void InternalStats::CacheEntryRoleStats::ToMap( |
804 | 0 | std::map<std::string, std::string>* values, SystemClock* clock) const { |
805 | 0 | values->clear(); |
806 | 0 | auto& v = *values; |
807 | 0 | v[BlockCacheEntryStatsMapKeys::CacheId()] = cache_id; |
808 | 0 | v[BlockCacheEntryStatsMapKeys::CacheCapacityBytes()] = |
809 | 0 | std::to_string(cache_capacity); |
810 | 0 | v[BlockCacheEntryStatsMapKeys::LastCollectionDurationSeconds()] = |
811 | 0 | std::to_string(GetLastDurationMicros() / 1000000.0); |
812 | 0 | v[BlockCacheEntryStatsMapKeys::LastCollectionAgeSeconds()] = |
813 | 0 | std::to_string((clock->NowMicros() - last_end_time_micros_) / 1000000U); |
814 | 0 | for (size_t i = 0; i < kNumCacheEntryRoles; ++i) { |
815 | 0 | auto role = static_cast<CacheEntryRole>(i); |
816 | 0 | v[BlockCacheEntryStatsMapKeys::EntryCount(role)] = |
817 | 0 | std::to_string(entry_counts[i]); |
818 | 0 | v[BlockCacheEntryStatsMapKeys::UsedBytes(role)] = |
819 | 0 | std::to_string(total_charges[i]); |
820 | 0 | v[BlockCacheEntryStatsMapKeys::UsedPercent(role)] = |
821 | 0 | std::to_string(100.0 * total_charges[i] / cache_capacity); |
822 | 0 | } |
823 | 0 | } |
824 | | |
825 | | bool InternalStats::HandleBlockCacheEntryStatsInternal(std::string* value, |
826 | 0 | bool fast) { |
827 | 0 | if (!cache_entry_stats_collector_) { |
828 | 0 | return false; |
829 | 0 | } |
830 | 0 | CollectCacheEntryStats(!fast /* foreground */); |
831 | 0 | CacheEntryRoleStats stats; |
832 | 0 | cache_entry_stats_collector_->GetStats(&stats); |
833 | 0 | *value = stats.ToString(clock_); |
834 | 0 | return true; |
835 | 0 | } |
836 | | |
837 | | bool InternalStats::HandleBlockCacheEntryStatsMapInternal( |
838 | 0 | std::map<std::string, std::string>* values, bool fast) { |
839 | 0 | if (!cache_entry_stats_collector_) { |
840 | 0 | return false; |
841 | 0 | } |
842 | 0 | CollectCacheEntryStats(!fast /* foreground */); |
843 | 0 | CacheEntryRoleStats stats; |
844 | 0 | cache_entry_stats_collector_->GetStats(&stats); |
845 | 0 | stats.ToMap(values, clock_); |
846 | 0 | return true; |
847 | 0 | } |
848 | | |
849 | | bool InternalStats::HandleBlockCacheEntryStats(std::string* value, |
850 | 0 | Slice /*suffix*/) { |
851 | 0 | return HandleBlockCacheEntryStatsInternal(value, false /* fast */); |
852 | 0 | } |
853 | | |
854 | | bool InternalStats::HandleBlockCacheEntryStatsMap( |
855 | 0 | std::map<std::string, std::string>* values, Slice /*suffix*/) { |
856 | 0 | return HandleBlockCacheEntryStatsMapInternal(values, false /* fast */); |
857 | 0 | } |
858 | | |
859 | | bool InternalStats::HandleFastBlockCacheEntryStats(std::string* value, |
860 | 0 | Slice /*suffix*/) { |
861 | 0 | return HandleBlockCacheEntryStatsInternal(value, true /* fast */); |
862 | 0 | } |
863 | | |
864 | | bool InternalStats::HandleFastBlockCacheEntryStatsMap( |
865 | 0 | std::map<std::string, std::string>* values, Slice /*suffix*/) { |
866 | 0 | return HandleBlockCacheEntryStatsMapInternal(values, true /* fast */); |
867 | 0 | } |
868 | | |
869 | | bool InternalStats::HandleLiveSstFilesSizeAtTemperature(std::string* value, |
870 | 0 | Slice suffix) { |
871 | 0 | uint64_t temperature; |
872 | 0 | bool ok = ConsumeDecimalNumber(&suffix, &temperature) && suffix.empty(); |
873 | 0 | if (!ok) { |
874 | 0 | return false; |
875 | 0 | } |
876 | | |
877 | 0 | uint64_t size = 0; |
878 | 0 | const auto* vstorage = cfd_->current()->storage_info(); |
879 | 0 | for (int level = 0; level < vstorage->num_levels(); level++) { |
880 | 0 | for (const auto& file_meta : vstorage->LevelFiles(level)) { |
881 | 0 | if (static_cast<uint8_t>(file_meta->temperature) == temperature) { |
882 | 0 | size += file_meta->fd.GetFileSize(); |
883 | 0 | } |
884 | 0 | } |
885 | 0 | } |
886 | |
|
887 | 0 | *value = std::to_string(size); |
888 | 0 | return true; |
889 | 0 | } |
890 | | |
891 | | bool InternalStats::HandleNumBlobFiles(uint64_t* value, DBImpl* /*db*/, |
892 | 0 | Version* /*version*/) { |
893 | 0 | assert(value); |
894 | 0 | assert(cfd_); |
895 | |
|
896 | 0 | const auto* current = cfd_->current(); |
897 | 0 | assert(current); |
898 | |
|
899 | 0 | const auto* vstorage = current->storage_info(); |
900 | 0 | assert(vstorage); |
901 | |
|
902 | 0 | const auto& blob_files = vstorage->GetBlobFiles(); |
903 | |
|
904 | 0 | *value = blob_files.size(); |
905 | |
|
906 | 0 | return true; |
907 | 0 | } |
908 | | |
909 | 0 | bool InternalStats::HandleBlobStats(std::string* value, Slice /*suffix*/) { |
910 | 0 | assert(value); |
911 | 0 | assert(cfd_); |
912 | |
|
913 | 0 | const auto* current = cfd_->current(); |
914 | 0 | assert(current); |
915 | |
|
916 | 0 | const auto* vstorage = current->storage_info(); |
917 | 0 | assert(vstorage); |
918 | |
|
919 | 0 | const auto blob_st = vstorage->GetBlobStats(); |
920 | |
|
921 | 0 | std::ostringstream oss; |
922 | |
|
923 | 0 | oss << "Number of blob files: " << vstorage->GetBlobFiles().size() |
924 | 0 | << "\nTotal size of blob files: " << blob_st.total_file_size |
925 | 0 | << "\nTotal size of garbage in blob files: " << blob_st.total_garbage_size |
926 | 0 | << "\nBlob file space amplification: " << blob_st.space_amp << '\n'; |
927 | |
|
928 | 0 | value->append(oss.str()); |
929 | |
|
930 | 0 | return true; |
931 | 0 | } |
932 | | |
933 | | bool InternalStats::HandleTotalBlobFileSize(uint64_t* value, DBImpl* /*db*/, |
934 | 0 | Version* /*version*/) { |
935 | 0 | assert(value); |
936 | 0 | assert(cfd_); |
937 | |
|
938 | 0 | *value = cfd_->GetTotalBlobFileSize(); |
939 | |
|
940 | 0 | return true; |
941 | 0 | } |
942 | | |
943 | | bool InternalStats::HandleLiveBlobFileSize(uint64_t* value, DBImpl* /*db*/, |
944 | 0 | Version* /*version*/) { |
945 | 0 | assert(value); |
946 | 0 | assert(cfd_); |
947 | |
|
948 | 0 | const auto* current = cfd_->current(); |
949 | 0 | assert(current); |
950 | |
|
951 | 0 | const auto* vstorage = current->storage_info(); |
952 | 0 | assert(vstorage); |
953 | |
|
954 | 0 | *value = vstorage->GetBlobStats().total_file_size; |
955 | |
|
956 | 0 | return true; |
957 | 0 | } |
958 | | |
959 | | bool InternalStats::HandleLiveBlobFileGarbageSize(uint64_t* value, |
960 | | DBImpl* /*db*/, |
961 | 0 | Version* /*version*/) { |
962 | 0 | assert(value); |
963 | 0 | assert(cfd_); |
964 | |
|
965 | 0 | const auto* current = cfd_->current(); |
966 | 0 | assert(current); |
967 | |
|
968 | 0 | const auto* vstorage = current->storage_info(); |
969 | 0 | assert(vstorage); |
970 | |
|
971 | 0 | *value = vstorage->GetBlobStats().total_garbage_size; |
972 | |
|
973 | 0 | return true; |
974 | 0 | } |
975 | | |
976 | 0 | Cache* InternalStats::GetBlobCacheForStats() { |
977 | 0 | return cfd_->ioptions().blob_cache.get(); |
978 | 0 | } |
979 | | |
980 | | bool InternalStats::HandleBlobCacheCapacity(uint64_t* value, DBImpl* /*db*/, |
981 | 0 | Version* /*version*/) { |
982 | 0 | Cache* blob_cache = GetBlobCacheForStats(); |
983 | 0 | if (blob_cache) { |
984 | 0 | *value = static_cast<uint64_t>(blob_cache->GetCapacity()); |
985 | 0 | return true; |
986 | 0 | } |
987 | 0 | return false; |
988 | 0 | } |
989 | | |
990 | | bool InternalStats::HandleBlobCacheUsage(uint64_t* value, DBImpl* /*db*/, |
991 | 0 | Version* /*version*/) { |
992 | 0 | Cache* blob_cache = GetBlobCacheForStats(); |
993 | 0 | if (blob_cache) { |
994 | 0 | *value = static_cast<uint64_t>(blob_cache->GetUsage()); |
995 | 0 | return true; |
996 | 0 | } |
997 | 0 | return false; |
998 | 0 | } |
999 | | |
1000 | | bool InternalStats::HandleBlobCachePinnedUsage(uint64_t* value, DBImpl* /*db*/, |
1001 | 0 | Version* /*version*/) { |
1002 | 0 | Cache* blob_cache = GetBlobCacheForStats(); |
1003 | 0 | if (blob_cache) { |
1004 | 0 | *value = static_cast<uint64_t>(blob_cache->GetPinnedUsage()); |
1005 | 0 | return true; |
1006 | 0 | } |
1007 | 0 | return false; |
1008 | 0 | } |
1009 | | |
1010 | 2.26k | const DBPropertyInfo* GetPropertyInfo(const Slice& property) { |
1011 | 2.26k | std::string ppt_name = GetPropertyNameAndArg(property).first.ToString(); |
1012 | 2.26k | auto ppt_info_iter = InternalStats::ppt_name_to_info.find(ppt_name); |
1013 | 2.26k | if (ppt_info_iter == InternalStats::ppt_name_to_info.end()) { |
1014 | 1.76k | return nullptr; |
1015 | 1.76k | } |
1016 | 494 | return &ppt_info_iter->second; |
1017 | 2.26k | } |
1018 | | |
1019 | | bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info, |
1020 | | const Slice& property, |
1021 | 601 | std::string* value) { |
1022 | 601 | assert(value != nullptr); |
1023 | 601 | assert(property_info.handle_string != nullptr); |
1024 | 601 | Slice arg = GetPropertyNameAndArg(property).second; |
1025 | 601 | return (this->*(property_info.handle_string))(value, arg); |
1026 | 601 | } |
1027 | | |
1028 | | bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info, |
1029 | | const Slice& property, |
1030 | 0 | std::map<std::string, std::string>* value) { |
1031 | 0 | assert(value != nullptr); |
1032 | 0 | assert(property_info.handle_map != nullptr); |
1033 | 0 | Slice arg = GetPropertyNameAndArg(property).second; |
1034 | 0 | return (this->*(property_info.handle_map))(value, arg); |
1035 | 0 | } |
1036 | | |
1037 | | bool InternalStats::GetIntProperty(const DBPropertyInfo& property_info, |
1038 | 0 | uint64_t* value, DBImpl* db) { |
1039 | 0 | assert(value != nullptr); |
1040 | 0 | assert(property_info.handle_int != nullptr && |
1041 | 0 | !property_info.need_out_of_mutex); |
1042 | 0 | db->mutex_.AssertHeld(); |
1043 | 0 | return (this->*(property_info.handle_int))(value, db, nullptr /* version */); |
1044 | 0 | } |
1045 | | |
1046 | | bool InternalStats::GetIntPropertyOutOfMutex( |
1047 | 0 | const DBPropertyInfo& property_info, Version* version, uint64_t* value) { |
1048 | 0 | assert(value != nullptr); |
1049 | 0 | assert(property_info.handle_int != nullptr && |
1050 | 0 | property_info.need_out_of_mutex); |
1051 | 0 | return (this->*(property_info.handle_int))(value, nullptr /* db */, version); |
1052 | 0 | } |
1053 | | |
1054 | 0 | bool InternalStats::HandleNumFilesAtLevel(std::string* value, Slice suffix) { |
1055 | 0 | uint64_t level; |
1056 | 0 | const auto* vstorage = cfd_->current()->storage_info(); |
1057 | 0 | bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty(); |
1058 | 0 | if (!ok || static_cast<int>(level) >= number_levels_) { |
1059 | 0 | return false; |
1060 | 0 | } else { |
1061 | 0 | char buf[100]; |
1062 | 0 | snprintf(buf, sizeof(buf), "%d", |
1063 | 0 | vstorage->NumLevelFiles(static_cast<int>(level))); |
1064 | 0 | *value = buf; |
1065 | 0 | return true; |
1066 | 0 | } |
1067 | 0 | } |
1068 | | |
1069 | | bool InternalStats::HandleCompressionRatioAtLevelPrefix(std::string* value, |
1070 | 0 | Slice suffix) { |
1071 | 0 | uint64_t level; |
1072 | 0 | const auto* vstorage = cfd_->current()->storage_info(); |
1073 | 0 | bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty(); |
1074 | 0 | if (!ok || level >= static_cast<uint64_t>(number_levels_)) { |
1075 | 0 | return false; |
1076 | 0 | } |
1077 | 0 | *value = std::to_string( |
1078 | 0 | vstorage->GetEstimatedCompressionRatioAtLevel(static_cast<int>(level))); |
1079 | 0 | return true; |
1080 | 0 | } |
1081 | | |
1082 | 0 | bool InternalStats::HandleLevelStats(std::string* value, Slice /*suffix*/) { |
1083 | 0 | char buf[1000]; |
1084 | 0 | const auto* vstorage = cfd_->current()->storage_info(); |
1085 | 0 | snprintf(buf, sizeof(buf), |
1086 | 0 | "Level Files Size(MB)\n" |
1087 | 0 | "--------------------\n"); |
1088 | 0 | value->append(buf); |
1089 | |
|
1090 | 0 | for (int level = 0; level < number_levels_; level++) { |
1091 | 0 | snprintf(buf, sizeof(buf), "%3d %8d %8.0f\n", level, |
1092 | 0 | vstorage->NumLevelFiles(level), |
1093 | 0 | vstorage->NumLevelBytes(level) / kMB); |
1094 | 0 | value->append(buf); |
1095 | 0 | } |
1096 | 0 | return true; |
1097 | 0 | } |
1098 | | |
1099 | 0 | bool InternalStats::HandleStats(std::string* value, Slice suffix) { |
1100 | 0 | if (!HandleCFStats(value, suffix)) { |
1101 | 0 | return false; |
1102 | 0 | } |
1103 | 0 | if (!HandleDBStats(value, suffix)) { |
1104 | 0 | return false; |
1105 | 0 | } |
1106 | 0 | return true; |
1107 | 0 | } |
1108 | | |
1109 | | bool InternalStats::HandleCFMapStats( |
1110 | 0 | std::map<std::string, std::string>* cf_stats, Slice /*suffix*/) { |
1111 | 0 | DumpCFMapStats(cf_stats); |
1112 | 0 | return true; |
1113 | 0 | } |
1114 | | |
1115 | 0 | bool InternalStats::HandleCFStats(std::string* value, Slice /*suffix*/) { |
1116 | 0 | DumpCFStats(value); |
1117 | 0 | return true; |
1118 | 0 | } |
1119 | | |
1120 | | bool InternalStats::HandleCFStatsPeriodic(std::string* value, |
1121 | 354 | Slice /*suffix*/) { |
1122 | 354 | bool has_change = has_cf_change_since_dump_; |
1123 | 354 | if (!has_change) { |
1124 | | // If file histogram changes, there is activity in this period too. |
1125 | 0 | uint64_t new_histogram_num = 0; |
1126 | 0 | for (int level = 0; level < number_levels_; level++) { |
1127 | 0 | new_histogram_num += file_read_latency_[level].num(); |
1128 | 0 | } |
1129 | 0 | new_histogram_num += blob_file_read_latency_.num(); |
1130 | 0 | if (new_histogram_num != last_histogram_num) { |
1131 | 0 | has_change = true; |
1132 | 0 | last_histogram_num = new_histogram_num; |
1133 | 0 | } |
1134 | 0 | } |
1135 | 354 | if (has_change) { |
1136 | 354 | no_cf_change_period_since_dump_ = 0; |
1137 | 354 | has_cf_change_since_dump_ = false; |
1138 | 354 | } else if (no_cf_change_period_since_dump_++ > 0) { |
1139 | | // Not ready to sync |
1140 | 0 | if (no_cf_change_period_since_dump_ == kMaxNoChangePeriodSinceDump) { |
1141 | | // Next periodic, we need to dump stats even if there is no change. |
1142 | 0 | no_cf_change_period_since_dump_ = 0; |
1143 | 0 | } |
1144 | 0 | return true; |
1145 | 0 | } |
1146 | | |
1147 | 354 | DumpCFStatsNoFileHistogram(/*is_periodic=*/true, value); |
1148 | 354 | DumpCFFileHistogram(value); |
1149 | 354 | return true; |
1150 | 354 | } |
1151 | | |
1152 | | bool InternalStats::HandleCFStatsNoFileHistogram(std::string* value, |
1153 | 0 | Slice /*suffix*/) { |
1154 | 0 | DumpCFStatsNoFileHistogram(/*is_periodic=*/false, value); |
1155 | 0 | return true; |
1156 | 0 | } |
1157 | | |
1158 | | bool InternalStats::HandleCFFileHistogram(std::string* value, |
1159 | 0 | Slice /*suffix*/) { |
1160 | 0 | DumpCFFileHistogram(value); |
1161 | 0 | return true; |
1162 | 0 | } |
1163 | | |
1164 | | bool InternalStats::HandleCFWriteStallStats(std::string* value, |
1165 | 0 | Slice /*suffix*/) { |
1166 | 0 | DumpCFStatsWriteStall(value); |
1167 | 0 | return true; |
1168 | 0 | } |
1169 | | |
1170 | | bool InternalStats::HandleCFWriteStallStatsMap( |
1171 | 0 | std::map<std::string, std::string>* value, Slice /*suffix*/) { |
1172 | 0 | DumpCFMapStatsWriteStall(value); |
1173 | 0 | return true; |
1174 | 0 | } |
1175 | | |
1176 | | bool InternalStats::HandleDBMapStats( |
1177 | 0 | std::map<std::string, std::string>* db_stats, Slice /*suffix*/) { |
1178 | 0 | DumpDBMapStats(db_stats); |
1179 | 0 | return true; |
1180 | 0 | } |
1181 | | |
1182 | 247 | bool InternalStats::HandleDBStats(std::string* value, Slice /*suffix*/) { |
1183 | 247 | DumpDBStats(value); |
1184 | 247 | return true; |
1185 | 247 | } |
1186 | | |
1187 | | bool InternalStats::HandleDBWriteStallStats(std::string* value, |
1188 | 0 | Slice /*suffix*/) { |
1189 | 0 | DumpDBStatsWriteStall(value); |
1190 | 0 | return true; |
1191 | 0 | } |
1192 | | |
1193 | | bool InternalStats::HandleDBWriteStallStatsMap( |
1194 | 0 | std::map<std::string, std::string>* value, Slice /*suffix*/) { |
1195 | 0 | DumpDBMapStatsWriteStall(value); |
1196 | 0 | return true; |
1197 | 0 | } |
1198 | | |
1199 | 0 | bool InternalStats::HandleSsTables(std::string* value, Slice /*suffix*/) { |
1200 | 0 | auto* current = cfd_->current(); |
1201 | 0 | *value = current->DebugString(true, true); |
1202 | 0 | return true; |
1203 | 0 | } |
1204 | | |
1205 | | bool InternalStats::HandleAggregatedTableProperties(std::string* value, |
1206 | 0 | Slice /*suffix*/) { |
1207 | 0 | std::shared_ptr<const TableProperties> tp; |
1208 | | // TODO: plumb Env::IOActivity, Env::IOPriority |
1209 | 0 | const ReadOptions read_options; |
1210 | 0 | auto s = cfd_->current()->GetAggregatedTableProperties(read_options, &tp); |
1211 | 0 | if (!s.ok()) { |
1212 | 0 | return false; |
1213 | 0 | } |
1214 | 0 | *value = tp->ToString(); |
1215 | 0 | return true; |
1216 | 0 | } |
1217 | | |
1218 | | static std::map<std::string, std::string> MapUint64ValuesToString( |
1219 | 0 | const std::map<std::string, uint64_t>& from) { |
1220 | 0 | std::map<std::string, std::string> to; |
1221 | 0 | for (const auto& e : from) { |
1222 | 0 | to[e.first] = std::to_string(e.second); |
1223 | 0 | } |
1224 | 0 | return to; |
1225 | 0 | } |
1226 | | |
1227 | | bool InternalStats::HandleAggregatedTablePropertiesMap( |
1228 | 0 | std::map<std::string, std::string>* values, Slice /*suffix*/) { |
1229 | 0 | std::shared_ptr<const TableProperties> tp; |
1230 | | // TODO: plumb Env::IOActivity, Env::IOPriority |
1231 | 0 | const ReadOptions read_options; |
1232 | 0 | auto s = cfd_->current()->GetAggregatedTableProperties(read_options, &tp); |
1233 | 0 | if (!s.ok()) { |
1234 | 0 | return false; |
1235 | 0 | } |
1236 | 0 | *values = MapUint64ValuesToString(tp->GetAggregatablePropertiesAsMap()); |
1237 | 0 | return true; |
1238 | 0 | } |
1239 | | |
1240 | | bool InternalStats::HandleAggregatedTablePropertiesAtLevel(std::string* values, |
1241 | 0 | Slice suffix) { |
1242 | 0 | uint64_t level; |
1243 | 0 | bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty(); |
1244 | 0 | if (!ok || static_cast<int>(level) >= number_levels_) { |
1245 | 0 | return false; |
1246 | 0 | } |
1247 | 0 | std::shared_ptr<const TableProperties> tp; |
1248 | | // TODO: plumb Env::IOActivity, Env::IOPriority |
1249 | 0 | const ReadOptions read_options; |
1250 | 0 | auto s = cfd_->current()->GetAggregatedTableProperties( |
1251 | 0 | read_options, &tp, static_cast<int>(level)); |
1252 | 0 | if (!s.ok()) { |
1253 | 0 | return false; |
1254 | 0 | } |
1255 | 0 | *values = tp->ToString(); |
1256 | 0 | return true; |
1257 | 0 | } |
1258 | | |
1259 | | bool InternalStats::HandleAggregatedTablePropertiesAtLevelMap( |
1260 | 0 | std::map<std::string, std::string>* values, Slice suffix) { |
1261 | 0 | uint64_t level; |
1262 | 0 | bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty(); |
1263 | 0 | if (!ok || static_cast<int>(level) >= number_levels_) { |
1264 | 0 | return false; |
1265 | 0 | } |
1266 | 0 | std::shared_ptr<const TableProperties> tp; |
1267 | | // TODO: plumb Env::IOActivity, Env::IOPriority |
1268 | 0 | const ReadOptions read_options; |
1269 | 0 | auto s = cfd_->current()->GetAggregatedTableProperties( |
1270 | 0 | read_options, &tp, static_cast<int>(level)); |
1271 | 0 | if (!s.ok()) { |
1272 | 0 | return false; |
1273 | 0 | } |
1274 | 0 | *values = MapUint64ValuesToString(tp->GetAggregatablePropertiesAsMap()); |
1275 | 0 | return true; |
1276 | 0 | } |
1277 | | |
1278 | | bool InternalStats::HandleNumImmutableMemTable(uint64_t* value, DBImpl* /*db*/, |
1279 | 0 | Version* /*version*/) { |
1280 | 0 | *value = cfd_->imm()->NumNotFlushed(); |
1281 | 0 | return true; |
1282 | 0 | } |
1283 | | |
1284 | | bool InternalStats::HandleNumImmutableMemTableFlushed(uint64_t* value, |
1285 | | DBImpl* /*db*/, |
1286 | 0 | Version* /*version*/) { |
1287 | 0 | *value = cfd_->imm()->NumFlushed(); |
1288 | 0 | return true; |
1289 | 0 | } |
1290 | | |
1291 | | bool InternalStats::HandleMemTableFlushPending(uint64_t* value, DBImpl* /*db*/, |
1292 | 0 | Version* /*version*/) { |
1293 | 0 | *value = (cfd_->imm()->IsFlushPending() ? 1 : 0); |
1294 | 0 | return true; |
1295 | 0 | } |
1296 | | |
1297 | | bool InternalStats::HandleNumRunningFlushes(uint64_t* value, DBImpl* db, |
1298 | 0 | Version* /*version*/) { |
1299 | 0 | *value = db->num_running_flushes(); |
1300 | 0 | return true; |
1301 | 0 | } |
1302 | | |
1303 | | bool InternalStats::HandleCompactionPending(uint64_t* value, DBImpl* /*db*/, |
1304 | 0 | Version* /*version*/) { |
1305 | | // 1 if the system already determines at least one compaction is needed. |
1306 | | // 0 otherwise, |
1307 | 0 | const auto* vstorage = cfd_->current()->storage_info(); |
1308 | 0 | *value = (cfd_->compaction_picker()->NeedsCompaction(vstorage) ? 1 : 0); |
1309 | 0 | return true; |
1310 | 0 | } |
1311 | | |
1312 | | bool InternalStats::HandleNumRunningCompactions(uint64_t* value, DBImpl* db, |
1313 | 0 | Version* /*version*/) { |
1314 | 0 | *value = db->num_running_compactions_; |
1315 | 0 | return true; |
1316 | 0 | } |
1317 | | |
1318 | | bool InternalStats::HandleNumRunningRemoteCompactions(uint64_t* value, |
1319 | | DBImpl* db, |
1320 | 0 | Version* /*version*/) { |
1321 | 0 | const int num_running = |
1322 | 0 | db->num_running_remote_compactions_.load(std::memory_order_relaxed); |
1323 | 0 | assert(num_running >= 0); |
1324 | 0 | *value = static_cast<uint64_t>(num_running); |
1325 | 0 | return true; |
1326 | 0 | } |
1327 | | |
1328 | | bool InternalStats::HandleNumRemoteCompactionPoolThreads(uint64_t* value, |
1329 | | DBImpl* /*db*/, |
1330 | 0 | Version* /*version*/) { |
1331 | 0 | auto pool = RemoteCompactionThreadPool::GetShared(false); |
1332 | 0 | *value = pool ? static_cast<uint64_t>(pool->GetBackgroundThreads()) : 0; |
1333 | 0 | return true; |
1334 | 0 | } |
1335 | | |
1336 | | bool InternalStats::HandleNumRunningBottomCompactions(uint64_t* value, |
1337 | | DBImpl* db, |
1338 | 0 | Version* /*version*/) { |
1339 | 0 | *value = static_cast<uint64_t>(db->num_running_bottom_compactions_); |
1340 | 0 | return true; |
1341 | 0 | } |
1342 | | |
1343 | | bool InternalStats::HandleNumRunningCompactionSortedRuns(uint64_t* value, |
1344 | | DBImpl* db, |
1345 | 0 | Version* /*version*/) { |
1346 | 0 | db->mutex()->AssertHeld(); |
1347 | 0 | uint64_t sorted_runs = 0; |
1348 | 0 | for (auto* cfd : *db->versions_->GetColumnFamilySet()) { |
1349 | 0 | sorted_runs += cfd->internal_stats()->NumRunningCompactionSortedRuns(); |
1350 | 0 | } |
1351 | 0 | *value = sorted_runs; |
1352 | 0 | return true; |
1353 | 0 | } |
1354 | | |
1355 | | bool InternalStats::HandleCompactionAbortCount(uint64_t* value, DBImpl* db, |
1356 | 0 | Version* /*version*/) { |
1357 | 0 | *value = static_cast<uint64_t>( |
1358 | 0 | db->compaction_aborted_.load(std::memory_order_acquire)); |
1359 | 0 | return true; |
1360 | 0 | } |
1361 | | |
1362 | | bool InternalStats::HandleNumUnscheduledCompactions(uint64_t* value, DBImpl* db, |
1363 | 0 | Version* /*version*/) { |
1364 | 0 | db->mutex()->AssertHeld(); |
1365 | 0 | assert(db->unscheduled_compactions_ >= 0); |
1366 | 0 | *value = static_cast<uint64_t>(db->unscheduled_compactions_); |
1367 | 0 | return true; |
1368 | 0 | } |
1369 | | |
1370 | | bool InternalStats::HandleBackgroundErrors(uint64_t* value, DBImpl* /*db*/, |
1371 | 0 | Version* /*version*/) { |
1372 | | // Accumulated number of errors in background flushes or compactions. |
1373 | 0 | *value = GetBackgroundErrorCount(); |
1374 | 0 | return true; |
1375 | 0 | } |
1376 | | |
1377 | | bool InternalStats::HandleCurSizeActiveMemTable(uint64_t* value, DBImpl* /*db*/, |
1378 | 0 | Version* /*version*/) { |
1379 | | // Current size of the active memtable |
1380 | | // Using ApproximateMemoryUsageFast to avoid the need for synchronization |
1381 | 0 | *value = cfd_->mem()->ApproximateMemoryUsageFast(); |
1382 | 0 | return true; |
1383 | 0 | } |
1384 | | |
1385 | | bool InternalStats::HandleCurSizeAllMemTables(uint64_t* value, DBImpl* /*db*/, |
1386 | 0 | Version* /*version*/) { |
1387 | | // Current size of the active memtable + immutable memtables |
1388 | | // Using ApproximateMemoryUsageFast to avoid the need for synchronization |
1389 | 0 | *value = cfd_->mem()->ApproximateMemoryUsageFast() + |
1390 | 0 | cfd_->imm()->ApproximateUnflushedMemTablesMemoryUsage(); |
1391 | 0 | return true; |
1392 | 0 | } |
1393 | | |
1394 | | bool InternalStats::HandleSizeAllMemTables(uint64_t* value, DBImpl* /*db*/, |
1395 | 0 | Version* /*version*/) { |
1396 | | // Using ApproximateMemoryUsageFast to avoid the need for synchronization |
1397 | 0 | *value = cfd_->mem()->ApproximateMemoryUsageFast() + |
1398 | 0 | cfd_->imm()->ApproximateMemoryUsage(); |
1399 | 0 | return true; |
1400 | 0 | } |
1401 | | |
1402 | | bool InternalStats::HandleNumEntriesActiveMemTable(uint64_t* value, |
1403 | | DBImpl* /*db*/, |
1404 | 0 | Version* /*version*/) { |
1405 | | // Current number of entires in the active memtable |
1406 | 0 | *value = cfd_->mem()->NumEntries(); |
1407 | 0 | return true; |
1408 | 0 | } |
1409 | | |
1410 | | bool InternalStats::HandleNumEntriesImmMemTables(uint64_t* value, |
1411 | | DBImpl* /*db*/, |
1412 | 0 | Version* /*version*/) { |
1413 | | // Current number of entries in the immutable memtables |
1414 | 0 | *value = cfd_->imm()->current()->GetTotalNumEntries(); |
1415 | 0 | return true; |
1416 | 0 | } |
1417 | | |
1418 | | bool InternalStats::HandleNumDeletesActiveMemTable(uint64_t* value, |
1419 | | DBImpl* /*db*/, |
1420 | 0 | Version* /*version*/) { |
1421 | | // Current number of entires in the active memtable |
1422 | 0 | *value = cfd_->mem()->NumDeletion(); |
1423 | 0 | return true; |
1424 | 0 | } |
1425 | | |
1426 | | bool InternalStats::HandleNumDeletesImmMemTables(uint64_t* value, |
1427 | | DBImpl* /*db*/, |
1428 | 0 | Version* /*version*/) { |
1429 | | // Current number of entries in the immutable memtables |
1430 | 0 | *value = cfd_->imm()->current()->GetTotalNumDeletes(); |
1431 | 0 | return true; |
1432 | 0 | } |
1433 | | |
1434 | | bool InternalStats::HandleEstimateNumKeys(uint64_t* value, DBImpl* /*db*/, |
1435 | 0 | Version* /*version*/) { |
1436 | | // Estimate number of entries in the column family: |
1437 | | // Use estimated entries in tables + total entries in memtables. |
1438 | 0 | const auto* vstorage = cfd_->current()->storage_info(); |
1439 | 0 | uint64_t estimate_keys = cfd_->mem()->NumEntries() + |
1440 | 0 | cfd_->imm()->current()->GetTotalNumEntries() + |
1441 | 0 | vstorage->GetEstimatedActiveKeys(); |
1442 | 0 | uint64_t estimate_deletes = |
1443 | 0 | cfd_->mem()->NumDeletion() + cfd_->imm()->current()->GetTotalNumDeletes(); |
1444 | 0 | *value = estimate_keys > estimate_deletes * 2 |
1445 | 0 | ? estimate_keys - (estimate_deletes * 2) |
1446 | 0 | : 0; |
1447 | 0 | return true; |
1448 | 0 | } |
1449 | | |
1450 | | bool InternalStats::HandleNumSnapshots(uint64_t* value, DBImpl* db, |
1451 | 0 | Version* /*version*/) { |
1452 | 0 | *value = db->snapshots().count(); |
1453 | 0 | return true; |
1454 | 0 | } |
1455 | | |
1456 | | bool InternalStats::HandleOldestSnapshotTime(uint64_t* value, DBImpl* db, |
1457 | 0 | Version* /*version*/) { |
1458 | 0 | *value = static_cast<uint64_t>(db->snapshots().GetOldestSnapshotTime()); |
1459 | 0 | return true; |
1460 | 0 | } |
1461 | | |
1462 | | bool InternalStats::HandleOldestSnapshotSequence(uint64_t* value, DBImpl* db, |
1463 | 0 | Version* /*version*/) { |
1464 | 0 | *value = static_cast<uint64_t>(db->snapshots().GetOldestSnapshotSequence()); |
1465 | 0 | return true; |
1466 | 0 | } |
1467 | | |
1468 | | bool InternalStats::HandleNumLiveVersions(uint64_t* value, DBImpl* /*db*/, |
1469 | 0 | Version* /*version*/) { |
1470 | 0 | *value = cfd_->GetNumLiveVersions(); |
1471 | 0 | return true; |
1472 | 0 | } |
1473 | | |
1474 | | bool InternalStats::HandleCurrentSuperVersionNumber(uint64_t* value, |
1475 | | DBImpl* /*db*/, |
1476 | 0 | Version* /*version*/) { |
1477 | 0 | *value = cfd_->GetSuperVersionNumber(); |
1478 | 0 | return true; |
1479 | 0 | } |
1480 | | |
1481 | | bool InternalStats::HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db, |
1482 | 0 | Version* /*version*/) { |
1483 | 0 | *value = db->IsFileDeletionsEnabled() ? 1 : 0; |
1484 | 0 | return true; |
1485 | 0 | } |
1486 | | |
1487 | | bool InternalStats::HandleBaseLevel(uint64_t* value, DBImpl* /*db*/, |
1488 | 0 | Version* /*version*/) { |
1489 | 0 | const auto* vstorage = cfd_->current()->storage_info(); |
1490 | 0 | *value = vstorage->base_level(); |
1491 | 0 | return true; |
1492 | 0 | } |
1493 | | |
1494 | | bool InternalStats::HandleTotalSstFilesSize(uint64_t* value, DBImpl* /*db*/, |
1495 | 0 | Version* /*version*/) { |
1496 | 0 | *value = cfd_->GetTotalSstFilesSize(); |
1497 | 0 | return true; |
1498 | 0 | } |
1499 | | |
1500 | | bool InternalStats::HandleLiveSstFilesSize(uint64_t* value, DBImpl* /*db*/, |
1501 | 0 | Version* /*version*/) { |
1502 | 0 | *value = cfd_->GetLiveSstFilesSize(); |
1503 | 0 | return true; |
1504 | 0 | } |
1505 | | |
1506 | | bool InternalStats::HandleObsoleteSstFilesSize(uint64_t* value, DBImpl* db, |
1507 | 0 | Version* /*version*/) { |
1508 | 0 | *value = db->GetObsoleteSstFilesSize(); |
1509 | 0 | return true; |
1510 | 0 | } |
1511 | | |
1512 | | bool InternalStats::HandleEstimatePendingCompactionBytes(uint64_t* value, |
1513 | | DBImpl* /*db*/, |
1514 | 0 | Version* /*version*/) { |
1515 | 0 | const auto* vstorage = cfd_->current()->storage_info(); |
1516 | 0 | *value = vstorage->estimated_compaction_needed_bytes(); |
1517 | 0 | return true; |
1518 | 0 | } |
1519 | | |
1520 | | bool InternalStats::HandleEstimateTableReadersMem(uint64_t* value, |
1521 | | DBImpl* /*db*/, |
1522 | 0 | Version* version) { |
1523 | | // TODO: plumb Env::IOActivity, Env::IOPriority |
1524 | 0 | const ReadOptions read_options; |
1525 | 0 | *value = (version == nullptr) |
1526 | 0 | ? 0 |
1527 | 0 | : version->GetMemoryUsageByTableReaders(read_options); |
1528 | 0 | return true; |
1529 | 0 | } |
1530 | | |
1531 | | bool InternalStats::HandleEstimateLiveDataSize(uint64_t* value, DBImpl* /*db*/, |
1532 | 0 | Version* version) { |
1533 | 0 | const auto* vstorage = version->storage_info(); |
1534 | 0 | *value = vstorage->EstimateLiveDataSize(); |
1535 | 0 | return true; |
1536 | 0 | } |
1537 | | |
1538 | | bool InternalStats::HandleMinLogNumberToKeep(uint64_t* value, DBImpl* db, |
1539 | 0 | Version* /*version*/) { |
1540 | 0 | *value = db->MinLogNumberToKeep(); |
1541 | 0 | return true; |
1542 | 0 | } |
1543 | | |
1544 | | bool InternalStats::HandleMinObsoleteSstNumberToKeep(uint64_t* value, |
1545 | | DBImpl* db, |
1546 | 0 | Version* /*version*/) { |
1547 | 0 | *value = db->MinObsoleteSstNumberToKeep(); |
1548 | 0 | return true; |
1549 | 0 | } |
1550 | | |
1551 | | bool InternalStats::HandleActualDelayedWriteRate(uint64_t* value, DBImpl* db, |
1552 | 0 | Version* /*version*/) { |
1553 | 0 | const WriteController& wc = db->write_controller(); |
1554 | 0 | if (!wc.NeedsDelay()) { |
1555 | 0 | *value = 0; |
1556 | 0 | } else { |
1557 | 0 | *value = wc.delayed_write_rate(); |
1558 | 0 | } |
1559 | 0 | return true; |
1560 | 0 | } |
1561 | | |
1562 | | bool InternalStats::HandleIsWriteStopped(uint64_t* value, DBImpl* db, |
1563 | 0 | Version* /*version*/) { |
1564 | 0 | *value = db->write_controller().IsStopped() ? 1 : 0; |
1565 | 0 | return true; |
1566 | 0 | } |
1567 | | |
1568 | | bool InternalStats::HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* /*db*/, |
1569 | 0 | Version* /*version*/) { |
1570 | | // TODO(yiwu): The property is currently available for fifo compaction |
1571 | | // with allow_compaction = false. This is because we don't propagate |
1572 | | // oldest_key_time on compaction. |
1573 | 0 | if (cfd_->ioptions().compaction_style != kCompactionStyleFIFO || |
1574 | 0 | cfd_->GetCurrentMutableCFOptions() |
1575 | 0 | .compaction_options_fifo.allow_compaction) { |
1576 | 0 | return false; |
1577 | 0 | } |
1578 | | // TODO: plumb Env::IOActivity, Env::IOPriority |
1579 | 0 | const ReadOptions read_options; |
1580 | 0 | TablePropertiesCollection collection; |
1581 | 0 | auto s = cfd_->current()->GetPropertiesOfAllTables(read_options, &collection); |
1582 | 0 | if (!s.ok()) { |
1583 | 0 | return false; |
1584 | 0 | } |
1585 | 0 | *value = std::numeric_limits<uint64_t>::max(); |
1586 | 0 | for (auto& p : collection) { |
1587 | 0 | *value = std::min(*value, p.second->oldest_key_time); |
1588 | 0 | if (*value == 0) { |
1589 | 0 | break; |
1590 | 0 | } |
1591 | 0 | } |
1592 | 0 | if (*value > 0) { |
1593 | 0 | *value = std::min({cfd_->mem()->ApproximateOldestKeyTime(), |
1594 | 0 | cfd_->imm()->ApproximateOldestKeyTime(), *value}); |
1595 | 0 | } |
1596 | 0 | return *value > 0 && *value < std::numeric_limits<uint64_t>::max(); |
1597 | 0 | } |
1598 | | |
1599 | 87.5k | Cache* InternalStats::GetBlockCacheForStats() { |
1600 | | // NOTE: called in startup before GetCurrentMutableCFOptions() is ready |
1601 | 87.5k | auto* table_factory = cfd_->GetLatestMutableCFOptions().table_factory.get(); |
1602 | 87.5k | assert(table_factory != nullptr); |
1603 | | // FIXME: need to a shared_ptr if/when block_cache is going to be mutable |
1604 | 87.5k | return table_factory->GetOptions<Cache>(TableFactory::kBlockCacheOpts()); |
1605 | 87.5k | } |
1606 | | |
1607 | | bool InternalStats::HandleBlockCacheCapacity(uint64_t* value, DBImpl* /*db*/, |
1608 | 0 | Version* /*version*/) { |
1609 | 0 | Cache* block_cache = GetBlockCacheForStats(); |
1610 | 0 | if (block_cache) { |
1611 | 0 | *value = static_cast<uint64_t>(block_cache->GetCapacity()); |
1612 | 0 | return true; |
1613 | 0 | } |
1614 | 0 | return false; |
1615 | 0 | } |
1616 | | |
1617 | | bool InternalStats::HandleBlockCacheUsage(uint64_t* value, DBImpl* /*db*/, |
1618 | 0 | Version* /*version*/) { |
1619 | 0 | Cache* block_cache = GetBlockCacheForStats(); |
1620 | 0 | if (block_cache) { |
1621 | 0 | *value = static_cast<uint64_t>(block_cache->GetUsage()); |
1622 | 0 | return true; |
1623 | 0 | } |
1624 | 0 | return false; |
1625 | 0 | } |
1626 | | |
1627 | | bool InternalStats::HandleBlockCachePinnedUsage(uint64_t* value, DBImpl* /*db*/, |
1628 | 0 | Version* /*version*/) { |
1629 | 0 | Cache* block_cache = GetBlockCacheForStats(); |
1630 | 0 | if (block_cache) { |
1631 | 0 | *value = static_cast<uint64_t>(block_cache->GetPinnedUsage()); |
1632 | 0 | return true; |
1633 | 0 | } |
1634 | 0 | return false; |
1635 | 0 | } |
1636 | | |
1637 | | void InternalStats::DumpDBMapStats( |
1638 | 0 | std::map<std::string, std::string>* db_stats) { |
1639 | 0 | for (int i = 0; i < static_cast<int>(kIntStatsNumMax); ++i) { |
1640 | 0 | InternalDBStatsType type = static_cast<InternalDBStatsType>(i); |
1641 | 0 | (*db_stats)[db_stats_type_to_info.at(type).property_name] = |
1642 | 0 | std::to_string(GetDBStats(type)); |
1643 | 0 | } |
1644 | 0 | double seconds_up = (clock_->NowMicros() - started_at_) / kMicrosInSec; |
1645 | 0 | (*db_stats)["db.uptime"] = std::to_string(seconds_up); |
1646 | 0 | } |
1647 | | |
1648 | 247 | void InternalStats::DumpDBStats(std::string* value) { |
1649 | 247 | char buf[1000]; |
1650 | | // DB-level stats, only available from default column family |
1651 | 247 | double seconds_up = (clock_->NowMicros() - started_at_) / kMicrosInSec; |
1652 | 247 | double interval_seconds_up = seconds_up - db_stats_snapshot_.seconds_up; |
1653 | 247 | snprintf(buf, sizeof(buf), |
1654 | 247 | "\n** DB Stats **\nUptime(secs): %.1f total, %.1f interval\n", |
1655 | 247 | seconds_up, interval_seconds_up); |
1656 | 247 | value->append(buf); |
1657 | | // Cumulative |
1658 | 247 | uint64_t user_bytes_written = |
1659 | 247 | GetDBStats(InternalStats::kIntStatsBytesWritten); |
1660 | 247 | uint64_t num_keys_written = |
1661 | 247 | GetDBStats(InternalStats::kIntStatsNumKeysWritten); |
1662 | 247 | uint64_t write_other = GetDBStats(InternalStats::kIntStatsWriteDoneByOther); |
1663 | 247 | uint64_t write_self = GetDBStats(InternalStats::kIntStatsWriteDoneBySelf); |
1664 | 247 | uint64_t wal_bytes = GetDBStats(InternalStats::kIntStatsWalFileBytes); |
1665 | 247 | uint64_t wal_synced = GetDBStats(InternalStats::kIntStatsWalFileSynced); |
1666 | 247 | uint64_t write_with_wal = GetDBStats(InternalStats::kIntStatsWriteWithWal); |
1667 | 247 | uint64_t write_stall_micros = |
1668 | 247 | GetDBStats(InternalStats::kIntStatsWriteStallMicros); |
1669 | | |
1670 | 247 | const int kHumanMicrosLen = 32; |
1671 | 247 | char human_micros[kHumanMicrosLen]; |
1672 | | |
1673 | | // Data |
1674 | | // writes: total number of write requests. |
1675 | | // keys: total number of key updates issued by all the write requests |
1676 | | // commit groups: number of group commits issued to the DB. Each group can |
1677 | | // contain one or more writes. |
1678 | | // so writes/keys is the average number of put in multi-put or put |
1679 | | // writes/groups is the average group commit size. |
1680 | | // |
1681 | | // The format is the same for interval stats. |
1682 | 247 | snprintf(buf, sizeof(buf), |
1683 | 247 | "Cumulative writes: %s writes, %s keys, %s commit groups, " |
1684 | 247 | "%.1f writes per commit group, ingest: %.2f GB, %.2f MB/s\n", |
1685 | 247 | NumberToHumanString(write_other + write_self).c_str(), |
1686 | 247 | NumberToHumanString(num_keys_written).c_str(), |
1687 | 247 | NumberToHumanString(write_self).c_str(), |
1688 | 247 | (write_other + write_self) / |
1689 | 247 | std::max(1.0, static_cast<double>(write_self)), |
1690 | 247 | user_bytes_written / kGB, |
1691 | 247 | user_bytes_written / kMB / std::max(seconds_up, 0.001)); |
1692 | 247 | value->append(buf); |
1693 | | // WAL |
1694 | 247 | snprintf(buf, sizeof(buf), |
1695 | 247 | "Cumulative WAL: %s writes, %s syncs, " |
1696 | 247 | "%.2f writes per sync, written: %.2f GB, %.2f MB/s\n", |
1697 | 247 | NumberToHumanString(write_with_wal).c_str(), |
1698 | 247 | NumberToHumanString(wal_synced).c_str(), |
1699 | 247 | write_with_wal / std::max(1.0, static_cast<double>(wal_synced)), |
1700 | 247 | wal_bytes / kGB, wal_bytes / kMB / std::max(seconds_up, 0.001)); |
1701 | 247 | value->append(buf); |
1702 | | // Stall |
1703 | 247 | AppendHumanMicros(write_stall_micros, human_micros, kHumanMicrosLen, true); |
1704 | 247 | snprintf(buf, sizeof(buf), "Cumulative stall: %s, %.1f percent\n", |
1705 | 247 | human_micros, |
1706 | | // 10000 = divide by 1M to get secs, then multiply by 100 for pct |
1707 | 247 | write_stall_micros / 10000.0 / std::max(seconds_up, 0.001)); |
1708 | 247 | value->append(buf); |
1709 | | |
1710 | | // Interval |
1711 | 247 | uint64_t interval_write_other = write_other - db_stats_snapshot_.write_other; |
1712 | 247 | uint64_t interval_write_self = write_self - db_stats_snapshot_.write_self; |
1713 | 247 | uint64_t interval_num_keys_written = |
1714 | 247 | num_keys_written - db_stats_snapshot_.num_keys_written; |
1715 | 247 | snprintf( |
1716 | 247 | buf, sizeof(buf), |
1717 | 247 | "Interval writes: %s writes, %s keys, %s commit groups, " |
1718 | 247 | "%.1f writes per commit group, ingest: %.2f MB, %.2f MB/s\n", |
1719 | 247 | NumberToHumanString(interval_write_other + interval_write_self).c_str(), |
1720 | 247 | NumberToHumanString(interval_num_keys_written).c_str(), |
1721 | 247 | NumberToHumanString(interval_write_self).c_str(), |
1722 | 247 | static_cast<double>(interval_write_other + interval_write_self) / |
1723 | 247 | std::max(1.0, static_cast<double>(interval_write_self)), |
1724 | 247 | (user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB, |
1725 | 247 | (user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB / |
1726 | 247 | std::max(interval_seconds_up, 0.001)), |
1727 | 247 | value->append(buf); |
1728 | | |
1729 | 247 | uint64_t interval_write_with_wal = |
1730 | 247 | write_with_wal - db_stats_snapshot_.write_with_wal; |
1731 | 247 | uint64_t interval_wal_synced = wal_synced - db_stats_snapshot_.wal_synced; |
1732 | 247 | uint64_t interval_wal_bytes = wal_bytes - db_stats_snapshot_.wal_bytes; |
1733 | | |
1734 | 247 | snprintf(buf, sizeof(buf), |
1735 | 247 | "Interval WAL: %s writes, %s syncs, " |
1736 | 247 | "%.2f writes per sync, written: %.2f GB, %.2f MB/s\n", |
1737 | 247 | NumberToHumanString(interval_write_with_wal).c_str(), |
1738 | 247 | NumberToHumanString(interval_wal_synced).c_str(), |
1739 | 247 | interval_write_with_wal / |
1740 | 247 | std::max(1.0, static_cast<double>(interval_wal_synced)), |
1741 | 247 | interval_wal_bytes / kGB, |
1742 | 247 | interval_wal_bytes / kMB / std::max(interval_seconds_up, 0.001)); |
1743 | 247 | value->append(buf); |
1744 | | |
1745 | | // Stall |
1746 | 247 | AppendHumanMicros(write_stall_micros - db_stats_snapshot_.write_stall_micros, |
1747 | 247 | human_micros, kHumanMicrosLen, true); |
1748 | 247 | snprintf(buf, sizeof(buf), "Interval stall: %s, %.1f percent\n", human_micros, |
1749 | | // 10000 = divide by 1M to get secs, then multiply by 100 for pct |
1750 | 247 | (write_stall_micros - db_stats_snapshot_.write_stall_micros) / |
1751 | 247 | 10000.0 / std::max(interval_seconds_up, 0.001)); |
1752 | 247 | value->append(buf); |
1753 | | |
1754 | 247 | std::string write_stall_stats; |
1755 | 247 | DumpDBStatsWriteStall(&write_stall_stats); |
1756 | 247 | value->append(write_stall_stats); |
1757 | | |
1758 | 247 | db_stats_snapshot_.seconds_up = seconds_up; |
1759 | 247 | db_stats_snapshot_.ingest_bytes = user_bytes_written; |
1760 | 247 | db_stats_snapshot_.write_other = write_other; |
1761 | 247 | db_stats_snapshot_.write_self = write_self; |
1762 | 247 | db_stats_snapshot_.num_keys_written = num_keys_written; |
1763 | 247 | db_stats_snapshot_.wal_bytes = wal_bytes; |
1764 | 247 | db_stats_snapshot_.wal_synced = wal_synced; |
1765 | 247 | db_stats_snapshot_.write_with_wal = write_with_wal; |
1766 | 247 | db_stats_snapshot_.write_stall_micros = write_stall_micros; |
1767 | 247 | } |
1768 | | |
1769 | | void InternalStats::DumpDBMapStatsWriteStall( |
1770 | 247 | std::map<std::string, std::string>* value) { |
1771 | 247 | constexpr uint32_t max_db_scope_write_stall_cause = |
1772 | 247 | static_cast<uint32_t>(WriteStallCause::kDBScopeWriteStallCauseEnumMax); |
1773 | | |
1774 | 247 | for (uint32_t i = |
1775 | 247 | max_db_scope_write_stall_cause - kNumDBScopeWriteStallCauses; |
1776 | 494 | i < max_db_scope_write_stall_cause; ++i) { |
1777 | 247 | for (uint32_t j = 0; |
1778 | 741 | j < static_cast<uint32_t>(WriteStallCondition::kNormal); ++j) { |
1779 | 494 | WriteStallCause cause = static_cast<WriteStallCause>(i); |
1780 | 494 | WriteStallCondition condition = static_cast<WriteStallCondition>(j); |
1781 | 494 | InternalStats::InternalDBStatsType internal_db_stat = |
1782 | 494 | InternalDBStat(cause, condition); |
1783 | | |
1784 | 494 | if (internal_db_stat == InternalStats::kIntStatsNumMax) { |
1785 | 247 | continue; |
1786 | 247 | } |
1787 | | |
1788 | 247 | std::string name = |
1789 | 247 | WriteStallStatsMapKeys::CauseConditionCount(cause, condition); |
1790 | 247 | uint64_t stat = |
1791 | 247 | db_stats_[static_cast<std::size_t>(internal_db_stat)].load( |
1792 | 247 | std::memory_order_relaxed); |
1793 | 247 | (*value)[name] = std::to_string(stat); |
1794 | 247 | } |
1795 | 247 | } |
1796 | 247 | } |
1797 | | |
1798 | 247 | void InternalStats::DumpDBStatsWriteStall(std::string* value) { |
1799 | 247 | assert(value); |
1800 | | |
1801 | 247 | std::map<std::string, std::string> write_stall_stats_map; |
1802 | 247 | DumpDBMapStatsWriteStall(&write_stall_stats_map); |
1803 | | |
1804 | 247 | std::ostringstream str; |
1805 | 247 | str << "Write Stall (count): "; |
1806 | | |
1807 | 247 | for (auto write_stall_stats_map_iter = write_stall_stats_map.begin(); |
1808 | 494 | write_stall_stats_map_iter != write_stall_stats_map.end(); |
1809 | 247 | write_stall_stats_map_iter++) { |
1810 | 247 | const auto& name_and_stat = *write_stall_stats_map_iter; |
1811 | 247 | str << name_and_stat.first << ": " << name_and_stat.second; |
1812 | 247 | if (std::next(write_stall_stats_map_iter) == write_stall_stats_map.end()) { |
1813 | 247 | str << "\n"; |
1814 | 247 | } else { |
1815 | 0 | str << ", "; |
1816 | 0 | } |
1817 | 247 | } |
1818 | 247 | *value = str.str(); |
1819 | 247 | } |
1820 | | |
1821 | | /** |
1822 | | * Dump Compaction Level stats to a map of stat name with "compaction." prefix |
1823 | | * to value in double as string. The level in stat name is represented with |
1824 | | * a prefix "Lx" where "x" is the level number. A special level "Sum" |
1825 | | * represents the sum of a stat for all levels. |
1826 | | * The result also contains IO stall counters which keys start with "io_stalls." |
1827 | | * and values represent uint64 encoded as strings. |
1828 | | */ |
1829 | | void InternalStats::DumpCFMapStats( |
1830 | 0 | std::map<std::string, std::string>* cf_stats) { |
1831 | 0 | const VersionStorageInfo* vstorage = cfd_->current()->storage_info(); |
1832 | 0 | CompactionStats compaction_stats_sum; |
1833 | 0 | std::map<int, std::map<LevelStatType, double>> levels_stats; |
1834 | 0 | DumpCFMapStats(vstorage, &levels_stats, &compaction_stats_sum); |
1835 | 0 | for (auto const& level_ent : levels_stats) { |
1836 | 0 | auto level_str = |
1837 | 0 | level_ent.first == -1 ? "Sum" : "L" + std::to_string(level_ent.first); |
1838 | 0 | for (auto const& stat_ent : level_ent.second) { |
1839 | 0 | auto stat_type = stat_ent.first; |
1840 | 0 | auto key_str = |
1841 | 0 | "compaction." + level_str + "." + |
1842 | 0 | InternalStats::compaction_level_stats.at(stat_type).property_name; |
1843 | 0 | (*cf_stats)[key_str] = std::to_string(stat_ent.second); |
1844 | 0 | } |
1845 | 0 | } |
1846 | |
|
1847 | 0 | DumpCFMapStatsWriteStall(cf_stats); |
1848 | 0 | } |
1849 | | |
1850 | | void InternalStats::DumpCFMapStats( |
1851 | | const VersionStorageInfo* vstorage, |
1852 | | std::map<int, std::map<LevelStatType, double>>* levels_stats, |
1853 | 354 | CompactionStats* compaction_stats_sum) { |
1854 | 354 | assert(vstorage); |
1855 | | |
1856 | 354 | int num_levels_to_check = |
1857 | 354 | (cfd_->ioptions().compaction_style == kCompactionStyleLevel) |
1858 | 354 | ? vstorage->num_levels() - 1 |
1859 | 354 | : 1; |
1860 | | |
1861 | | // Compaction scores are sorted based on its value. Restore them to the |
1862 | | // level order |
1863 | 354 | std::vector<double> compaction_score(number_levels_, 0); |
1864 | 2.47k | for (int i = 0; i < num_levels_to_check; ++i) { |
1865 | 2.12k | compaction_score[vstorage->CompactionScoreLevel(i)] = |
1866 | 2.12k | vstorage->CompactionScore(i); |
1867 | 2.12k | } |
1868 | | // Count # of files being compacted for each level |
1869 | 354 | std::vector<int> files_being_compacted(number_levels_, 0); |
1870 | 2.83k | for (int level = 0; level < number_levels_; ++level) { |
1871 | 2.47k | for (auto* f : vstorage->LevelFiles(level)) { |
1872 | 407 | if (f->being_compacted) { |
1873 | 43 | ++files_being_compacted[level]; |
1874 | 43 | } |
1875 | 407 | } |
1876 | 2.47k | } |
1877 | | |
1878 | 354 | int total_files = 0; |
1879 | 354 | int total_files_being_compacted = 0; |
1880 | 354 | double total_file_size = 0; |
1881 | 354 | uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED]; |
1882 | 354 | uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE]; |
1883 | 354 | uint64_t curr_ingest = flush_ingest + add_file_ingest; |
1884 | 2.83k | for (int level = 0; level < number_levels_; level++) { |
1885 | 2.47k | int files = vstorage->NumLevelFiles(level); |
1886 | 2.47k | total_files += files; |
1887 | 2.47k | total_files_being_compacted += files_being_compacted[level]; |
1888 | 2.47k | if (comp_stats_[level].micros > 0 || comp_stats_[level].cpu_micros > 0 || |
1889 | 2.40k | files > 0) { |
1890 | 189 | compaction_stats_sum->Add(comp_stats_[level]); |
1891 | 189 | total_file_size += vstorage->NumLevelBytes(level); |
1892 | 189 | uint64_t input_bytes; |
1893 | 189 | if (level == 0) { |
1894 | 138 | input_bytes = curr_ingest; |
1895 | 138 | } else { |
1896 | 51 | input_bytes = comp_stats_[level].bytes_read_non_output_levels + |
1897 | 51 | comp_stats_[level].bytes_read_blob; |
1898 | 51 | } |
1899 | 189 | double w_amp = |
1900 | 189 | (input_bytes == 0) |
1901 | 189 | ? 0.0 |
1902 | 189 | : static_cast<double>(comp_stats_[level].bytes_written + |
1903 | 70 | comp_stats_[level].bytes_written_blob) / |
1904 | 70 | input_bytes; |
1905 | 189 | std::map<LevelStatType, double> level_stats; |
1906 | 189 | PrepareLevelStats(&level_stats, files, files_being_compacted[level], |
1907 | 189 | static_cast<double>(vstorage->NumLevelBytes(level)), |
1908 | 189 | compaction_score[level], w_amp, comp_stats_[level]); |
1909 | 189 | (*levels_stats)[level] = level_stats; |
1910 | 189 | } |
1911 | 2.47k | } |
1912 | | // Cumulative summary |
1913 | 354 | double w_amp = (0 == curr_ingest) |
1914 | 354 | ? 0.0 |
1915 | 354 | : (compaction_stats_sum->bytes_written + |
1916 | 70 | compaction_stats_sum->bytes_written_blob) / |
1917 | 70 | static_cast<double>(curr_ingest); |
1918 | | // Stats summary across levels |
1919 | 354 | std::map<LevelStatType, double> sum_stats; |
1920 | 354 | PrepareLevelStats(&sum_stats, total_files, total_files_being_compacted, |
1921 | 354 | total_file_size, 0, w_amp, *compaction_stats_sum); |
1922 | 354 | (*levels_stats)[-1] = sum_stats; // -1 is for the Sum level |
1923 | 354 | } |
1924 | | |
1925 | | void InternalStats::DumpCFMapStatsByPriority( |
1926 | 354 | std::map<int, std::map<LevelStatType, double>>* priorities_stats) { |
1927 | 2.12k | for (size_t priority = 0; priority < comp_stats_by_pri_.size(); priority++) { |
1928 | 1.77k | if (comp_stats_by_pri_[priority].micros > 0) { |
1929 | 70 | std::map<LevelStatType, double> priority_stats; |
1930 | 70 | PrepareLevelStats(&priority_stats, 0 /* num_files */, |
1931 | 70 | 0 /* being_compacted */, 0 /* total_file_size */, |
1932 | 70 | 0 /* compaction_score */, 0 /* w_amp */, |
1933 | 70 | comp_stats_by_pri_[priority]); |
1934 | 70 | (*priorities_stats)[static_cast<int>(priority)] = priority_stats; |
1935 | 70 | } |
1936 | 1.77k | } |
1937 | 354 | } |
1938 | | |
1939 | | void InternalStats::DumpCFMapStatsWriteStall( |
1940 | 354 | std::map<std::string, std::string>* value) { |
1941 | 354 | uint64_t total_delays = 0; |
1942 | 354 | uint64_t total_stops = 0; |
1943 | 354 | constexpr uint32_t max_cf_scope_write_stall_cause = |
1944 | 354 | static_cast<uint32_t>(WriteStallCause::kCFScopeWriteStallCauseEnumMax); |
1945 | | |
1946 | 354 | for (uint32_t i = |
1947 | 354 | max_cf_scope_write_stall_cause - kNumCFScopeWriteStallCauses; |
1948 | 1.41k | i < max_cf_scope_write_stall_cause; ++i) { |
1949 | 1.06k | for (uint32_t j = 0; |
1950 | 3.18k | j < static_cast<uint32_t>(WriteStallCondition::kNormal); ++j) { |
1951 | 2.12k | WriteStallCause cause = static_cast<WriteStallCause>(i); |
1952 | 2.12k | WriteStallCondition condition = static_cast<WriteStallCondition>(j); |
1953 | 2.12k | InternalStats::InternalCFStatsType internal_cf_stat = |
1954 | 2.12k | InternalCFStat(cause, condition); |
1955 | | |
1956 | 2.12k | if (internal_cf_stat == InternalStats::INTERNAL_CF_STATS_ENUM_MAX) { |
1957 | 0 | continue; |
1958 | 0 | } |
1959 | | |
1960 | 2.12k | std::string name = |
1961 | 2.12k | WriteStallStatsMapKeys::CauseConditionCount(cause, condition); |
1962 | 2.12k | uint64_t stat = |
1963 | 2.12k | cf_stats_count_[static_cast<std::size_t>(internal_cf_stat)]; |
1964 | 2.12k | (*value)[name] = std::to_string(stat); |
1965 | | |
1966 | 2.12k | if (condition == WriteStallCondition::kDelayed) { |
1967 | 1.06k | total_delays += stat; |
1968 | 1.06k | } else if (condition == WriteStallCondition::kStopped) { |
1969 | 1.06k | total_stops += stat; |
1970 | 1.06k | } |
1971 | 2.12k | } |
1972 | 1.06k | } |
1973 | | |
1974 | 354 | (*value)[WriteStallStatsMapKeys:: |
1975 | 354 | CFL0FileCountLimitDelaysWithOngoingCompaction()] = |
1976 | 354 | std::to_string( |
1977 | 354 | cf_stats_count_[L0_FILE_COUNT_LIMIT_DELAYS_WITH_ONGOING_COMPACTION]); |
1978 | 354 | (*value)[WriteStallStatsMapKeys:: |
1979 | 354 | CFL0FileCountLimitStopsWithOngoingCompaction()] = |
1980 | 354 | std::to_string( |
1981 | 354 | cf_stats_count_[L0_FILE_COUNT_LIMIT_STOPS_WITH_ONGOING_COMPACTION]); |
1982 | | |
1983 | 354 | (*value)[WriteStallStatsMapKeys::TotalStops()] = std::to_string(total_stops); |
1984 | 354 | (*value)[WriteStallStatsMapKeys::TotalDelays()] = |
1985 | 354 | std::to_string(total_delays); |
1986 | 354 | } |
1987 | | |
1988 | | void InternalStats::DumpCFStatsWriteStall(std::string* value, |
1989 | 354 | uint64_t* total_stall_count) { |
1990 | 354 | assert(value); |
1991 | | |
1992 | 354 | std::map<std::string, std::string> write_stall_stats_map; |
1993 | 354 | DumpCFMapStatsWriteStall(&write_stall_stats_map); |
1994 | | |
1995 | 354 | std::ostringstream str; |
1996 | 354 | str << "Write Stall (count): "; |
1997 | | |
1998 | 354 | for (auto write_stall_stats_map_iter = write_stall_stats_map.begin(); |
1999 | 3.89k | write_stall_stats_map_iter != write_stall_stats_map.end(); |
2000 | 3.54k | write_stall_stats_map_iter++) { |
2001 | 3.54k | const auto& name_and_stat = *write_stall_stats_map_iter; |
2002 | 3.54k | str << name_and_stat.first << ": " << name_and_stat.second; |
2003 | 3.54k | if (std::next(write_stall_stats_map_iter) == write_stall_stats_map.end()) { |
2004 | 354 | str << "\n"; |
2005 | 3.18k | } else { |
2006 | 3.18k | str << ", "; |
2007 | 3.18k | } |
2008 | 3.54k | } |
2009 | | |
2010 | 354 | if (total_stall_count) { |
2011 | 354 | *total_stall_count = |
2012 | 354 | ParseUint64( |
2013 | 354 | write_stall_stats_map[WriteStallStatsMapKeys::TotalStops()]) + |
2014 | 354 | ParseUint64( |
2015 | 354 | write_stall_stats_map[WriteStallStatsMapKeys::TotalDelays()]); |
2016 | 354 | if (*total_stall_count > 0) { |
2017 | 0 | str << "interval: " << *total_stall_count - cf_stats_snapshot_.stall_count |
2018 | 0 | << " total count\n"; |
2019 | 0 | } |
2020 | 354 | } |
2021 | 354 | *value = str.str(); |
2022 | 354 | } |
2023 | | |
2024 | 0 | void InternalStats::DumpCFStats(std::string* value) { |
2025 | 0 | DumpCFStatsNoFileHistogram(/*is_periodic=*/false, value); |
2026 | 0 | DumpCFFileHistogram(value); |
2027 | 0 | } |
2028 | | |
2029 | | void InternalStats::DumpCFStatsNoFileHistogram(bool is_periodic, |
2030 | 354 | std::string* value) { |
2031 | 354 | char buf[2000]; |
2032 | | // Per-ColumnFamily stats |
2033 | 354 | PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName(), "Level"); |
2034 | 354 | value->append(buf); |
2035 | | |
2036 | | // Print stats for each level |
2037 | 354 | const VersionStorageInfo* vstorage = cfd_->current()->storage_info(); |
2038 | 354 | std::map<int, std::map<LevelStatType, double>> levels_stats; |
2039 | 354 | CompactionStats compaction_stats_sum; |
2040 | 354 | DumpCFMapStats(vstorage, &levels_stats, &compaction_stats_sum); |
2041 | 2.83k | for (int l = 0; l < number_levels_; ++l) { |
2042 | 2.47k | if (levels_stats.find(l) != levels_stats.end()) { |
2043 | 189 | PrintLevelStats(buf, sizeof(buf), "L" + std::to_string(l), |
2044 | 189 | levels_stats[l]); |
2045 | 189 | value->append(buf); |
2046 | 189 | } |
2047 | 2.47k | } |
2048 | | |
2049 | | // Print sum of level stats |
2050 | 354 | PrintLevelStats(buf, sizeof(buf), "Sum", levels_stats[-1]); |
2051 | 354 | value->append(buf); |
2052 | | |
2053 | 354 | uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED]; |
2054 | 354 | uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE]; |
2055 | 354 | uint64_t ingest_files_addfile = cf_stats_value_[INGESTED_NUM_FILES_TOTAL]; |
2056 | 354 | uint64_t ingest_l0_files_addfile = |
2057 | 354 | cf_stats_value_[INGESTED_LEVEL0_NUM_FILES_TOTAL]; |
2058 | 354 | uint64_t ingest_keys_addfile = cf_stats_value_[INGESTED_NUM_KEYS_TOTAL]; |
2059 | | // Interval summary |
2060 | 354 | uint64_t interval_flush_ingest = |
2061 | 354 | flush_ingest - cf_stats_snapshot_.ingest_bytes_flush; |
2062 | 354 | uint64_t interval_add_file_inget = |
2063 | 354 | add_file_ingest - cf_stats_snapshot_.ingest_bytes_addfile; |
2064 | 354 | uint64_t interval_ingest = |
2065 | 354 | interval_flush_ingest + interval_add_file_inget + 1; |
2066 | 354 | CompactionStats interval_stats(compaction_stats_sum); |
2067 | 354 | interval_stats.Subtract(cf_stats_snapshot_.comp_stats); |
2068 | 354 | double w_amp = |
2069 | 354 | (interval_stats.bytes_written + interval_stats.bytes_written_blob) / |
2070 | 354 | static_cast<double>(interval_ingest); |
2071 | 354 | PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0, w_amp, interval_stats); |
2072 | 354 | value->append(buf); |
2073 | | |
2074 | 354 | PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName(), "Priority"); |
2075 | 354 | value->append(buf); |
2076 | 354 | std::map<int, std::map<LevelStatType, double>> priorities_stats; |
2077 | 354 | DumpCFMapStatsByPriority(&priorities_stats); |
2078 | 2.12k | for (size_t priority = 0; priority < comp_stats_by_pri_.size(); ++priority) { |
2079 | 1.77k | if (priorities_stats.find(static_cast<int>(priority)) != |
2080 | 1.77k | priorities_stats.end()) { |
2081 | 70 | PrintLevelStats( |
2082 | 70 | buf, sizeof(buf), |
2083 | 70 | Env::PriorityToString(static_cast<Env::Priority>(priority)), |
2084 | 70 | priorities_stats[static_cast<int>(priority)]); |
2085 | 70 | value->append(buf); |
2086 | 70 | } |
2087 | 1.77k | } |
2088 | | |
2089 | 354 | const auto blob_st = vstorage->GetBlobStats(); |
2090 | | |
2091 | 354 | snprintf(buf, sizeof(buf), |
2092 | 354 | "\nBlob file count: %" ROCKSDB_PRIszt |
2093 | 354 | ", total size: %.1f GB, garbage size: %.1f GB, space amp: %.1f\n\n", |
2094 | 354 | vstorage->GetBlobFiles().size(), blob_st.total_file_size / kGB, |
2095 | 354 | blob_st.total_garbage_size / kGB, blob_st.space_amp); |
2096 | 354 | value->append(buf); |
2097 | | |
2098 | 354 | uint64_t now_micros = clock_->NowMicros(); |
2099 | 354 | double seconds_up = (now_micros - started_at_) / kMicrosInSec; |
2100 | 354 | double interval_seconds_up = seconds_up - cf_stats_snapshot_.seconds_up; |
2101 | 354 | snprintf(buf, sizeof(buf), "Uptime(secs): %.1f total, %.1f interval\n", |
2102 | 354 | seconds_up, interval_seconds_up); |
2103 | 354 | value->append(buf); |
2104 | 354 | snprintf(buf, sizeof(buf), "Flush(GB): cumulative %.3f, interval %.3f\n", |
2105 | 354 | flush_ingest / kGB, interval_flush_ingest / kGB); |
2106 | 354 | value->append(buf); |
2107 | 354 | snprintf(buf, sizeof(buf), "AddFile(GB): cumulative %.3f, interval %.3f\n", |
2108 | 354 | add_file_ingest / kGB, interval_add_file_inget / kGB); |
2109 | 354 | value->append(buf); |
2110 | | |
2111 | 354 | uint64_t interval_ingest_files_addfile = |
2112 | 354 | ingest_files_addfile - cf_stats_snapshot_.ingest_files_addfile; |
2113 | 354 | snprintf(buf, sizeof(buf), |
2114 | 354 | "AddFile(Total Files): cumulative %" PRIu64 ", interval %" PRIu64 |
2115 | 354 | "\n", |
2116 | 354 | ingest_files_addfile, interval_ingest_files_addfile); |
2117 | 354 | value->append(buf); |
2118 | | |
2119 | 354 | uint64_t interval_ingest_l0_files_addfile = |
2120 | 354 | ingest_l0_files_addfile - cf_stats_snapshot_.ingest_l0_files_addfile; |
2121 | 354 | snprintf(buf, sizeof(buf), |
2122 | 354 | "AddFile(L0 Files): cumulative %" PRIu64 ", interval %" PRIu64 "\n", |
2123 | 354 | ingest_l0_files_addfile, interval_ingest_l0_files_addfile); |
2124 | 354 | value->append(buf); |
2125 | | |
2126 | 354 | uint64_t interval_ingest_keys_addfile = |
2127 | 354 | ingest_keys_addfile - cf_stats_snapshot_.ingest_keys_addfile; |
2128 | 354 | snprintf(buf, sizeof(buf), |
2129 | 354 | "AddFile(Keys): cumulative %" PRIu64 ", interval %" PRIu64 "\n", |
2130 | 354 | ingest_keys_addfile, interval_ingest_keys_addfile); |
2131 | 354 | value->append(buf); |
2132 | | |
2133 | | // Compact |
2134 | 354 | uint64_t compact_bytes_read = 0; |
2135 | 354 | uint64_t compact_bytes_write = 0; |
2136 | 354 | uint64_t compact_micros = 0; |
2137 | 2.83k | for (int level = 0; level < number_levels_; level++) { |
2138 | 2.47k | compact_bytes_read += comp_stats_[level].bytes_read_output_level + |
2139 | 2.47k | comp_stats_[level].bytes_read_non_output_levels + |
2140 | 2.47k | comp_stats_[level].bytes_read_blob; |
2141 | 2.47k | compact_bytes_write += comp_stats_[level].bytes_written + |
2142 | 2.47k | comp_stats_[level].bytes_written_blob; |
2143 | 2.47k | compact_micros += comp_stats_[level].micros; |
2144 | 2.47k | } |
2145 | | |
2146 | 354 | snprintf(buf, sizeof(buf), |
2147 | 354 | "Cumulative compaction: %.2f GB write, %.2f MB/s write, " |
2148 | 354 | "%.2f GB read, %.2f MB/s read, %.1f seconds\n", |
2149 | 354 | compact_bytes_write / kGB, |
2150 | 354 | compact_bytes_write / kMB / std::max(seconds_up, 0.001), |
2151 | 354 | compact_bytes_read / kGB, |
2152 | 354 | compact_bytes_read / kMB / std::max(seconds_up, 0.001), |
2153 | 354 | compact_micros / kMicrosInSec); |
2154 | 354 | value->append(buf); |
2155 | | |
2156 | | // Compaction interval |
2157 | 354 | uint64_t interval_compact_bytes_write = |
2158 | 354 | compact_bytes_write - cf_stats_snapshot_.compact_bytes_write; |
2159 | 354 | uint64_t interval_compact_bytes_read = |
2160 | 354 | compact_bytes_read - cf_stats_snapshot_.compact_bytes_read; |
2161 | 354 | uint64_t interval_compact_micros = |
2162 | 354 | compact_micros - cf_stats_snapshot_.compact_micros; |
2163 | | |
2164 | 354 | snprintf( |
2165 | 354 | buf, sizeof(buf), |
2166 | 354 | "Interval compaction: %.2f GB write, %.2f MB/s write, " |
2167 | 354 | "%.2f GB read, %.2f MB/s read, %.1f seconds\n", |
2168 | 354 | interval_compact_bytes_write / kGB, |
2169 | 354 | interval_compact_bytes_write / kMB / std::max(interval_seconds_up, 0.001), |
2170 | 354 | interval_compact_bytes_read / kGB, |
2171 | 354 | interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001), |
2172 | 354 | interval_compact_micros / kMicrosInSec); |
2173 | 354 | value->append(buf); |
2174 | | |
2175 | 354 | snprintf(buf, sizeof(buf), |
2176 | 354 | "Estimated pending compaction bytes: %" PRIu64 "\n", |
2177 | 354 | vstorage->estimated_compaction_needed_bytes()); |
2178 | 354 | value->append(buf); |
2179 | | |
2180 | 354 | if (is_periodic) { |
2181 | 354 | cf_stats_snapshot_.compact_bytes_write = compact_bytes_write; |
2182 | 354 | cf_stats_snapshot_.compact_bytes_read = compact_bytes_read; |
2183 | 354 | cf_stats_snapshot_.compact_micros = compact_micros; |
2184 | 354 | } |
2185 | | |
2186 | 354 | std::string write_stall_stats; |
2187 | 354 | uint64_t total_stall_count; |
2188 | 354 | DumpCFStatsWriteStall(&write_stall_stats, &total_stall_count); |
2189 | 354 | value->append(write_stall_stats); |
2190 | | |
2191 | 354 | if (is_periodic) { |
2192 | 354 | cf_stats_snapshot_.seconds_up = seconds_up; |
2193 | 354 | cf_stats_snapshot_.ingest_bytes_flush = flush_ingest; |
2194 | 354 | cf_stats_snapshot_.ingest_bytes_addfile = add_file_ingest; |
2195 | 354 | cf_stats_snapshot_.ingest_files_addfile = ingest_files_addfile; |
2196 | 354 | cf_stats_snapshot_.ingest_l0_files_addfile = ingest_l0_files_addfile; |
2197 | 354 | cf_stats_snapshot_.ingest_keys_addfile = ingest_keys_addfile; |
2198 | 354 | cf_stats_snapshot_.comp_stats = compaction_stats_sum; |
2199 | 354 | cf_stats_snapshot_.stall_count = total_stall_count; |
2200 | 354 | } |
2201 | | |
2202 | | // Do not gather cache entry stats during CFStats because DB |
2203 | | // mutex is held. Only dump last cached collection (rely on DB |
2204 | | // periodic stats dump to update) |
2205 | 354 | if (cache_entry_stats_collector_) { |
2206 | 354 | CacheEntryRoleStats stats; |
2207 | | // thread safe |
2208 | 354 | cache_entry_stats_collector_->GetStats(&stats); |
2209 | | |
2210 | 354 | constexpr uint64_t kDayInMicros = uint64_t{86400} * 1000000U; |
2211 | | |
2212 | | // Skip if stats are extremely old (> 1 day, incl not yet populated) |
2213 | 354 | if (now_micros - stats.last_end_time_micros_ < kDayInMicros) { |
2214 | 354 | value->append(stats.ToString(clock_)); |
2215 | 354 | } |
2216 | 354 | } |
2217 | 354 | } |
2218 | | |
2219 | 354 | void InternalStats::DumpCFFileHistogram(std::string* value) { |
2220 | 354 | assert(value); |
2221 | 354 | assert(cfd_); |
2222 | | |
2223 | 354 | std::ostringstream oss; |
2224 | 354 | oss << "\n** File Read Latency Histogram By Level [" << cfd_->GetName() |
2225 | 354 | << "] **\n"; |
2226 | | |
2227 | 2.83k | for (int level = 0; level < number_levels_; level++) { |
2228 | 2.47k | if (!file_read_latency_[level].Empty()) { |
2229 | 0 | oss << "** Level " << level << " read latency histogram (micros):\n" |
2230 | 0 | << file_read_latency_[level].ToString() << '\n'; |
2231 | 0 | } |
2232 | 2.47k | } |
2233 | | |
2234 | 354 | if (!blob_file_read_latency_.Empty()) { |
2235 | 0 | oss << "** Blob file read latency histogram (micros):\n" |
2236 | 0 | << blob_file_read_latency_.ToString() << '\n'; |
2237 | 0 | } |
2238 | | |
2239 | 354 | value->append(oss.str()); |
2240 | 354 | } |
2241 | | |
2242 | | namespace { |
2243 | | |
2244 | | class SumPropertyAggregator : public IntPropertyAggregator { |
2245 | | public: |
2246 | 0 | SumPropertyAggregator() : aggregated_value_(0) {} |
2247 | | ~SumPropertyAggregator() override = default; |
2248 | | |
2249 | 0 | void Add(ColumnFamilyData* cfd, uint64_t value) override { |
2250 | 0 | (void)cfd; |
2251 | 0 | aggregated_value_ += value; |
2252 | 0 | } |
2253 | | |
2254 | 0 | uint64_t Aggregate() const override { return aggregated_value_; } |
2255 | | |
2256 | | private: |
2257 | | uint64_t aggregated_value_; |
2258 | | }; |
2259 | | |
2260 | | // A block cache may be shared by multiple column families. |
2261 | | // BlockCachePropertyAggregator ensures that the same cache is only added once. |
2262 | | class BlockCachePropertyAggregator : public IntPropertyAggregator { |
2263 | | public: |
2264 | 0 | BlockCachePropertyAggregator() = default; |
2265 | 0 | ~BlockCachePropertyAggregator() override = default; |
2266 | | |
2267 | 0 | void Add(ColumnFamilyData* cfd, uint64_t value) override { |
2268 | 0 | auto* table_factory = cfd->GetCurrentMutableCFOptions().table_factory.get(); |
2269 | 0 | assert(table_factory != nullptr); |
2270 | 0 | Cache* cache = |
2271 | 0 | table_factory->GetOptions<Cache>(TableFactory::kBlockCacheOpts()); |
2272 | 0 | if (cache != nullptr) { |
2273 | 0 | block_cache_properties_.emplace(cache, value); |
2274 | 0 | } |
2275 | 0 | } |
2276 | | |
2277 | 0 | uint64_t Aggregate() const override { |
2278 | 0 | uint64_t sum = 0; |
2279 | 0 | for (const auto& p : block_cache_properties_) { |
2280 | 0 | sum += p.second; |
2281 | 0 | } |
2282 | 0 | return sum; |
2283 | 0 | } |
2284 | | |
2285 | | private: |
2286 | | std::unordered_map<Cache*, uint64_t> block_cache_properties_; |
2287 | | }; |
2288 | | |
2289 | | } // anonymous namespace |
2290 | | |
2291 | | std::unique_ptr<IntPropertyAggregator> CreateIntPropertyAggregator( |
2292 | 0 | const Slice& property) { |
2293 | 0 | if (property == DB::Properties::kBlockCacheCapacity || |
2294 | 0 | property == DB::Properties::kBlockCacheUsage || |
2295 | 0 | property == DB::Properties::kBlockCachePinnedUsage) { |
2296 | 0 | return std::make_unique<BlockCachePropertyAggregator>(); |
2297 | 0 | } else { |
2298 | 0 | return std::make_unique<SumPropertyAggregator>(); |
2299 | 0 | } |
2300 | 0 | } |
2301 | | |
2302 | | } // namespace ROCKSDB_NAMESPACE |