Coverage Report

Created: 2026-04-10 07:52

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/rocksdb/db/version_edit.cc
Line
Count
Source
1
//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
2
//  This source code is licensed under both the GPLv2 (found in the
3
//  COPYING file in the root directory) and Apache 2.0 License
4
//  (found in the LICENSE.Apache file in the root directory).
5
//
6
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7
// Use of this source code is governed by a BSD-style license that can be
8
// found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10
#include "db/version_edit.h"
11
12
#include "db/blob/blob_index.h"
13
#include "db/version_set.h"
14
#include "db/wide/wide_column_serialization.h"
15
#include "logging/event_logger.h"
16
#include "rocksdb/slice.h"
17
#include "table/unique_id_impl.h"
18
#include "test_util/sync_point.h"
19
#include "util/coding.h"
20
#include "util/string_util.h"
21
22
namespace ROCKSDB_NAMESPACE {
23
24
namespace {}  // anonymous namespace
25
26
// After copying, we rely on the caller to ensure no double releases. Fragile,
27
// but keeps copying cheap.
28
PinnedTableReader& PinnedTableReader::operator=(
29
828k
    const PinnedTableReader& other) {
30
828k
  TableReader* r = other.reader_.load(std::memory_order_acquire);
31
  // Only read handle_ when reader_ is non-null. Pin() writes handle_ before
32
  // reader_ (with release), so a non-null reader_ guarantees handle_ is stable.
33
  // If reader_ is null, Pin() may be in progress — avoid reading handle_.
34
828k
  handle_ = (r != nullptr) ? other.handle_ : nullptr;
35
828k
  reader_.store(r, std::memory_order_release);
36
828k
  return *this;
37
828k
}
38
39
90.7k
Cache::Handle* PinnedTableReader::GetCacheHandle() const {
40
90.7k
  (void)reader_.load(std::memory_order_acquire);
41
90.7k
  return handle_;
42
90.7k
}
43
44
90.4k
void PinnedTableReader::Pin(Cache::Handle* handle, TableReader* reader) {
45
90.4k
  assert(handle != nullptr);
46
90.4k
  assert(reader != nullptr);
47
90.4k
  handle_ = handle;
48
90.4k
  reader_.store(reader, std::memory_order_release);
49
90.4k
}
50
51
0
void PinnedTableReader::Release(Cache* cache) {
52
0
  (void)reader_.load(std::memory_order_acquire);
53
0
  if (handle_ != nullptr) {
54
0
    cache->Release(handle_);
55
0
    handle_ = nullptr;
56
0
    reader_.store(nullptr, std::memory_order_relaxed);
57
0
  }
58
0
}
59
60
815k
uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) {
61
815k
  assert(number <= kFileNumberMask);
62
815k
  return number | (path_id * (kFileNumberMask + 1));
63
815k
}
64
65
Status FileMetaData::UpdateBoundaries(const Slice& key, const Slice& value,
66
                                      SequenceNumber seqno,
67
38.8k
                                      ValueType value_type) {
68
  // Helper: update oldest_blob_file_number from a single BlobIndex.
69
38.8k
  auto update_oldest_blob = [&](const BlobIndex& blob_index) -> Status {
70
0
    if (!blob_index.IsInlined() && !blob_index.HasTTL()) {
71
0
      if (blob_index.file_number() == kInvalidBlobFileNumber) {
72
0
        return Status::Corruption("Invalid blob file number");
73
0
      }
74
75
0
      if (oldest_blob_file_number == kInvalidBlobFileNumber ||
76
0
          oldest_blob_file_number > blob_index.file_number()) {
77
0
        oldest_blob_file_number = blob_index.file_number();
78
0
      }
79
0
    }
80
0
    return Status::OK();
81
0
  };
82
83
38.8k
  if (value_type == kTypeBlobIndex) {
84
0
    BlobIndex blob_index;
85
0
    if (Status s = blob_index.DecodeFrom(value); !s.ok()) {
86
0
      return s;
87
0
    }
88
89
0
    if (Status s = update_oldest_blob(blob_index); !s.ok()) {
90
0
      return s;
91
0
    }
92
38.8k
  } else if (value_type == kTypeWideColumnEntity) {
93
0
    if (Status s = WideColumnSerialization::ForEachBlobFileNumber(
94
0
            value, update_oldest_blob);
95
0
        !s.ok()) {
96
0
      return s;
97
0
    }
98
0
  }
99
100
38.8k
  if (smallest.size() == 0) {
101
17.6k
    smallest.DecodeFrom(key);
102
17.6k
  }
103
38.8k
  largest.DecodeFrom(key);
104
38.8k
  fd.smallest_seqno = std::min(fd.smallest_seqno, seqno);
105
38.8k
  fd.largest_seqno = std::max(fd.largest_seqno, seqno);
106
107
38.8k
  return Status::OK();
108
38.8k
}
109
110
340k
void VersionEdit::Clear() { *this = VersionEdit(); }
111
112
bool VersionEdit::EncodeTo(std::string* dst,
113
407k
                           std::optional<size_t> ts_sz) const {
114
407k
  assert(!IsNoManifestWriteDummy());
115
407k
  if (has_db_id_) {
116
49.6k
    PutVarint32(dst, kDbId);
117
49.6k
    PutLengthPrefixedSlice(dst, db_id_);
118
49.6k
  }
119
407k
  if (has_comparator_) {
120
72.6k
    assert(has_persist_user_defined_timestamps_);
121
72.6k
    PutVarint32(dst, kComparator);
122
72.6k
    PutLengthPrefixedSlice(dst, comparator_);
123
72.6k
  }
124
407k
  if (has_log_number_) {
125
131k
    PutVarint32Varint64(dst, kLogNumber, log_number_);
126
131k
  }
127
407k
  if (has_prev_log_number_) {
128
170k
    PutVarint32Varint64(dst, kPrevLogNumber, prev_log_number_);
129
170k
  }
130
407k
  if (has_next_file_number_) {
131
208k
    PutVarint32Varint64(dst, kNextFileNumber, next_file_number_);
132
208k
  }
133
407k
  if (has_max_column_family_) {
134
36.9k
    PutVarint32(dst, kMaxColumnFamily, max_column_family_);
135
36.9k
  }
136
407k
  if (has_min_log_number_to_keep_) {
137
64.5k
    PutVarint32Varint64(dst, kMinLogNumberToKeep, min_log_number_to_keep_);
138
64.5k
  }
139
407k
  if (has_last_sequence_) {
140
265k
    PutVarint32Varint64(dst, kLastSequence, last_sequence_);
141
265k
  }
142
407k
  for (size_t i = 0; i < compact_cursors_.size(); i++) {
143
0
    if (compact_cursors_[i].second.Valid()) {
144
0
      PutVarint32(dst, kCompactCursor);
145
0
      PutVarint32(dst, compact_cursors_[i].first);  // level
146
0
      PutLengthPrefixedSlice(dst, compact_cursors_[i].second.Encode());
147
0
    }
148
0
  }
149
407k
  for (const auto& deleted : deleted_files_) {
150
11.0k
    PutVarint32Varint32Varint64(dst, kDeletedFile, deleted.first /* level */,
151
11.0k
                                deleted.second /* file number */);
152
11.0k
  }
153
154
407k
  bool min_log_num_written = false;
155
156
407k
  assert(new_files_.empty() || ts_sz.has_value());
157
498k
  for (size_t i = 0; i < new_files_.size(); i++) {
158
90.7k
    const FileMetaData& f = new_files_[i].second;
159
90.7k
    if (!f.smallest.Valid() || !f.largest.Valid() ||
160
90.7k
        f.epoch_number == kUnknownEpochNumber) {
161
0
      return false;
162
0
    }
163
90.7k
    EncodeToNewFile4(f, new_files_[i].first, ts_sz.value(),
164
90.7k
                     has_min_log_number_to_keep_, min_log_number_to_keep_,
165
90.7k
                     min_log_num_written, dst);
166
90.7k
  }
167
168
407k
  for (const auto& blob_file_addition : blob_file_additions_) {
169
0
    PutVarint32(dst, kBlobFileAddition);
170
0
    blob_file_addition.EncodeTo(dst);
171
0
  }
172
173
407k
  for (const auto& blob_file_garbage : blob_file_garbages_) {
174
0
    PutVarint32(dst, kBlobFileGarbage);
175
0
    blob_file_garbage.EncodeTo(dst);
176
0
  }
177
178
407k
  for (const auto& wal_addition : wal_additions_) {
179
0
    PutVarint32(dst, kWalAddition2);
180
0
    std::string encoded;
181
0
    wal_addition.EncodeTo(&encoded);
182
0
    PutLengthPrefixedSlice(dst, encoded);
183
0
  }
184
185
407k
  if (!wal_deletion_.IsEmpty()) {
186
27.9k
    PutVarint32(dst, kWalDeletion2);
187
27.9k
    std::string encoded;
188
27.9k
    wal_deletion_.EncodeTo(&encoded);
189
27.9k
    PutLengthPrefixedSlice(dst, encoded);
190
27.9k
  }
191
192
  // 0 is default and does not need to be explicitly written
193
407k
  if (column_family_ != 0) {
194
76.1k
    PutVarint32(dst, kColumnFamily, column_family_);
195
76.1k
  }
196
197
407k
  if (is_column_family_add_) {
198
30.4k
    PutVarint32(dst, kColumnFamilyAdd);
199
30.4k
    PutLengthPrefixedSlice(dst, Slice(column_family_name_));
200
30.4k
  }
201
202
407k
  if (is_column_family_drop_) {
203
15.2k
    PutVarint32(dst, kColumnFamilyDrop);
204
15.2k
  }
205
206
407k
  if (is_in_atomic_group_) {
207
0
    PutVarint32(dst, kInAtomicGroup);
208
0
    PutVarint32(dst, remaining_entries_);
209
0
  }
210
211
407k
  if (HasFullHistoryTsLow()) {
212
0
    PutVarint32(dst, kFullHistoryTsLow);
213
0
    PutLengthPrefixedSlice(dst, full_history_ts_low_);
214
0
  }
215
216
407k
  if (HasPersistUserDefinedTimestamps()) {
217
    // persist_user_defined_timestamps flag should be logged in the same
218
    // VersionEdit as the user comparator name.
219
72.6k
    assert(has_comparator_);
220
72.6k
    PutVarint32(dst, kPersistUserDefinedTimestamps);
221
72.6k
    char p = static_cast<char>(persist_user_defined_timestamps_);
222
72.6k
    PutLengthPrefixedSlice(dst, Slice(&p, 1));
223
72.6k
  }
224
225
407k
  if (HasSubcompactionProgress()) {
226
0
    PutVarint32(dst, kSubcompactionProgress);
227
0
    std::string progress_data;
228
0
    subcompaction_progress_.EncodeTo(&progress_data);
229
0
    PutLengthPrefixedSlice(dst, progress_data);
230
0
  }
231
232
407k
  return true;
233
407k
}
234
235
void VersionEdit::EncodeToNewFile4(const FileMetaData& f, int level,
236
                                   size_t ts_sz,
237
                                   bool has_min_log_number_to_keep,
238
                                   uint64_t min_log_number_to_keep,
239
                                   bool& min_log_num_written,
240
90.7k
                                   std::string* dst) {
241
90.7k
  PutVarint32(dst, kNewFile4);
242
90.7k
  PutVarint32Varint64(dst, level, f.fd.GetNumber());
243
90.7k
  PutVarint64(dst, f.fd.GetFileSize());
244
90.7k
  EncodeFileBoundaries(dst, f, ts_sz);
245
90.7k
  PutVarint64Varint64(dst, f.fd.smallest_seqno, f.fd.largest_seqno);
246
  // Customized fields' format:
247
  // +-----------------------------+
248
  // | 1st field's tag (varint32)  |
249
  // +-----------------------------+
250
  // | 1st field's size (varint32) |
251
  // +-----------------------------+
252
  // |    bytes for 1st field      |
253
  // |  (based on size decoded)    |
254
  // +-----------------------------+
255
  // |                             |
256
  // |          ......             |
257
  // |                             |
258
  // +-----------------------------+
259
  // | last field's size (varint32)|
260
  // +-----------------------------+
261
  // |    bytes for last field     |
262
  // |  (based on size decoded)    |
263
  // +-----------------------------+
264
  // | terminating tag (varint32)  |
265
  // +-----------------------------+
266
  //
267
  // Customized encoding for fields:
268
  //   tag kPathId: 1 byte as path_id
269
  //   tag kNeedCompaction:
270
  //        now only can take one char value 1 indicating need-compaction
271
  //
272
90.7k
  PutVarint32(dst, NewFileCustomTag::kOldestAncesterTime);
273
90.7k
  std::string varint_oldest_ancester_time;
274
90.7k
  PutVarint64(&varint_oldest_ancester_time, f.oldest_ancester_time);
275
90.7k
  TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintOldestAncesterTime",
276
90.7k
                           &varint_oldest_ancester_time);
277
90.7k
  PutLengthPrefixedSlice(dst, Slice(varint_oldest_ancester_time));
278
279
90.7k
  PutVarint32(dst, NewFileCustomTag::kFileCreationTime);
280
90.7k
  std::string varint_file_creation_time;
281
90.7k
  PutVarint64(&varint_file_creation_time, f.file_creation_time);
282
90.7k
  TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintFileCreationTime",
283
90.7k
                           &varint_file_creation_time);
284
90.7k
  PutLengthPrefixedSlice(dst, Slice(varint_file_creation_time));
285
286
90.7k
  PutVarint32(dst, NewFileCustomTag::kEpochNumber);
287
90.7k
  std::string varint_epoch_number;
288
90.7k
  PutVarint64(&varint_epoch_number, f.epoch_number);
289
90.7k
  PutLengthPrefixedSlice(dst, Slice(varint_epoch_number));
290
291
90.7k
  if (f.file_checksum_func_name != kUnknownFileChecksumFuncName) {
292
0
    PutVarint32(dst, NewFileCustomTag::kFileChecksum);
293
0
    PutLengthPrefixedSlice(dst, Slice(f.file_checksum));
294
295
0
    PutVarint32(dst, NewFileCustomTag::kFileChecksumFuncName);
296
0
    PutLengthPrefixedSlice(dst, Slice(f.file_checksum_func_name));
297
0
  }
298
299
90.7k
  if (f.fd.GetPathId() != 0) {
300
0
    PutVarint32(dst, NewFileCustomTag::kPathId);
301
0
    char p = static_cast<char>(f.fd.GetPathId());
302
0
    PutLengthPrefixedSlice(dst, Slice(&p, 1));
303
0
  }
304
90.7k
  if (f.temperature != Temperature::kUnknown) {
305
0
    PutVarint32(dst, NewFileCustomTag::kTemperature);
306
0
    char p = static_cast<char>(f.temperature);
307
0
    PutLengthPrefixedSlice(dst, Slice(&p, 1));
308
0
  }
309
90.7k
  if (f.marked_for_compaction) {
310
0
    PutVarint32(dst, NewFileCustomTag::kNeedCompaction);
311
0
    char p = static_cast<char>(1);
312
0
    PutLengthPrefixedSlice(dst, Slice(&p, 1));
313
0
  }
314
90.7k
  if (has_min_log_number_to_keep && !min_log_num_written) {
315
23.3k
    PutVarint32(dst, NewFileCustomTag::kMinLogNumberToKeepHack);
316
23.3k
    std::string varint_log_number;
317
23.3k
    PutFixed64(&varint_log_number, min_log_number_to_keep);
318
23.3k
    PutLengthPrefixedSlice(dst, Slice(varint_log_number));
319
23.3k
    min_log_num_written = true;
320
23.3k
  }
321
90.7k
  if (f.oldest_blob_file_number != kInvalidBlobFileNumber) {
322
0
    PutVarint32(dst, NewFileCustomTag::kOldestBlobFileNumber);
323
0
    std::string oldest_blob_file_number;
324
0
    PutVarint64(&oldest_blob_file_number, f.oldest_blob_file_number);
325
0
    PutLengthPrefixedSlice(dst, Slice(oldest_blob_file_number));
326
0
  }
327
90.7k
  UniqueId64x2 unique_id = f.unique_id;
328
90.7k
  TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:UniqueId", &unique_id);
329
90.7k
  if (unique_id != kNullUniqueId64x2) {
330
90.7k
    PutVarint32(dst, NewFileCustomTag::kUniqueId);
331
90.7k
    std::string unique_id_str = EncodeUniqueIdBytes(&unique_id);
332
90.7k
    PutLengthPrefixedSlice(dst, Slice(unique_id_str));
333
90.7k
  }
334
90.7k
  if (f.compensated_range_deletion_size) {
335
0
    PutVarint32(dst, NewFileCustomTag::kCompensatedRangeDeletionSize);
336
0
    std::string compensated_range_deletion_size;
337
0
    PutVarint64(&compensated_range_deletion_size,
338
0
                f.compensated_range_deletion_size);
339
0
    PutLengthPrefixedSlice(dst, Slice(compensated_range_deletion_size));
340
0
  }
341
90.7k
  if (f.tail_size) {
342
90.7k
    PutVarint32(dst, NewFileCustomTag::kTailSize);
343
90.7k
    std::string varint_tail_size;
344
90.7k
    PutVarint64(&varint_tail_size, f.tail_size);
345
90.7k
    PutLengthPrefixedSlice(dst, Slice(varint_tail_size));
346
90.7k
  }
347
90.7k
  if (!f.user_defined_timestamps_persisted) {
348
    // The default value for the flag is true, it's only explicitly persisted
349
    // when it's false. We are putting 0 as the value here to signal false
350
    // (i.e. UDTS not persisted).
351
0
    PutVarint32(dst, NewFileCustomTag::kUserDefinedTimestampsPersisted);
352
0
    char p = static_cast<char>(0);
353
0
    PutLengthPrefixedSlice(dst, Slice(&p, 1));
354
0
  }
355
  // Encode min/max timestamp if they are non-empty
356
90.7k
  if (!f.min_timestamp.empty()) {
357
0
    PutVarint32(dst, NewFileCustomTag::kMinTimestamp);
358
0
    PutLengthPrefixedSlice(dst, Slice(f.min_timestamp));
359
0
  }
360
90.7k
  if (!f.max_timestamp.empty()) {
361
0
    PutVarint32(dst, NewFileCustomTag::kMaxTimestamp);
362
0
    PutLengthPrefixedSlice(dst, Slice(f.max_timestamp));
363
0
  }
364
90.7k
  TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:NewFile4:CustomizeFields",
365
90.7k
                           dst);
366
367
90.7k
  PutVarint32(dst, NewFileCustomTag::kTerminate);
368
90.7k
}
369
151k
static bool GetInternalKey(Slice* input, InternalKey* dst) {
370
151k
  Slice str;
371
151k
  if (GetLengthPrefixedSlice(input, &str)) {
372
151k
    dst->DecodeFrom(str);
373
151k
    return dst->Valid();
374
151k
  } else {
375
0
    return false;
376
0
  }
377
151k
}
378
379
83.6k
bool VersionEdit::GetLevel(Slice* input, int* level, int& max_level) {
380
83.6k
  uint32_t v = 0;
381
83.6k
  if (GetVarint32(input, &v)) {
382
83.6k
    *level = v;
383
83.6k
    if (max_level < *level) {
384
11.5k
      max_level = *level;
385
11.5k
    }
386
83.6k
    return true;
387
83.6k
  } else {
388
0
    return false;
389
0
  }
390
83.6k
}
391
392
const char* VersionEdit::DecodeNewFile4From(Slice* input, int& max_level,
393
                                            uint64_t& min_log_number_to_keep,
394
                                            bool& has_min_log_number_to_keep,
395
                                            NewFiles& new_files,
396
75.5k
                                            FileMetaData& f) {
397
75.5k
  int level = 0;
398
75.5k
  uint64_t number = 0;
399
75.5k
  uint32_t path_id = 0;
400
75.5k
  uint64_t file_size = 0;
401
75.5k
  SequenceNumber smallest_seqno = 0;
402
75.5k
  SequenceNumber largest_seqno = kMaxSequenceNumber;
403
75.5k
  if (GetLevel(input, &level, max_level) && GetVarint64(input, &number) &&
404
75.5k
      GetVarint64(input, &file_size) && GetInternalKey(input, &f.smallest) &&
405
75.5k
      GetInternalKey(input, &f.largest) &&
406
75.5k
      GetVarint64(input, &smallest_seqno) &&
407
75.5k
      GetVarint64(input, &largest_seqno)) {
408
    // See comments in VersionEdit::EncodeTo() for format of customized fields
409
474k
    while (true) {
410
474k
      uint32_t custom_tag = 0;
411
474k
      Slice field;
412
474k
      if (!GetVarint32(input, &custom_tag)) {
413
0
        return "new-file4 custom field";
414
0
      }
415
474k
      if (custom_tag == kTerminate) {
416
75.5k
        break;
417
75.5k
      }
418
398k
      if (!GetLengthPrefixedSlice(input, &field)) {
419
0
        return "new-file4 custom field length prefixed slice error";
420
0
      }
421
398k
      switch (custom_tag) {
422
0
        case kPathId:
423
0
          if (field.size() != 1) {
424
0
            return "path_id field wrong size";
425
0
          }
426
0
          path_id = field[0];
427
0
          if (path_id > 3) {
428
0
            return "path_id wrong vaue";
429
0
          }
430
0
          break;
431
75.5k
        case kOldestAncesterTime:
432
75.5k
          if (!GetVarint64(&field, &f.oldest_ancester_time)) {
433
0
            return "invalid oldest ancester time";
434
0
          }
435
75.5k
          break;
436
75.5k
        case kFileCreationTime:
437
75.5k
          if (!GetVarint64(&field, &f.file_creation_time)) {
438
0
            return "invalid file creation time";
439
0
          }
440
75.5k
          break;
441
75.5k
        case kEpochNumber:
442
75.5k
          if (!GetVarint64(&field, &f.epoch_number)) {
443
0
            return "invalid epoch number";
444
0
          }
445
75.5k
          break;
446
75.5k
        case kFileChecksum:
447
0
          f.file_checksum = field.ToString();
448
0
          break;
449
0
        case kFileChecksumFuncName:
450
0
          f.file_checksum_func_name = field.ToString();
451
0
          break;
452
0
        case kNeedCompaction:
453
0
          if (field.size() != 1) {
454
0
            return "need_compaction field wrong size";
455
0
          }
456
0
          f.marked_for_compaction = (field[0] == 1);
457
0
          break;
458
20.9k
        case kMinLogNumberToKeepHack:
459
          // This is a hack to encode kMinLogNumberToKeep in a
460
          // forward-compatible fashion.
461
20.9k
          if (!GetFixed64(&field, &min_log_number_to_keep)) {
462
0
            return "deleted log number malformatted";
463
0
          }
464
20.9k
          has_min_log_number_to_keep = true;
465
20.9k
          break;
466
0
        case kOldestBlobFileNumber:
467
0
          if (!GetVarint64(&field, &f.oldest_blob_file_number)) {
468
0
            return "invalid oldest blob file number";
469
0
          }
470
0
          break;
471
0
        case kTemperature:
472
0
          if (field.size() != 1) {
473
0
            return "temperature field wrong size";
474
0
          } else {
475
0
            Temperature casted_field = static_cast<Temperature>(field[0]);
476
0
            if (casted_field < Temperature::kLastTemperature) {
477
0
              f.temperature = casted_field;
478
0
            }
479
0
          }
480
0
          break;
481
75.5k
        case kUniqueId:
482
75.5k
          if (!DecodeUniqueIdBytes(field.ToString(), &f.unique_id).ok()) {
483
0
            f.unique_id = kNullUniqueId64x2;
484
0
            return "invalid unique id";
485
0
          }
486
75.5k
          break;
487
75.5k
        case kCompensatedRangeDeletionSize:
488
0
          if (!GetVarint64(&field, &f.compensated_range_deletion_size)) {
489
0
            return "Invalid compensated range deletion size";
490
0
          }
491
0
          break;
492
75.5k
        case kTailSize:
493
75.5k
          if (!GetVarint64(&field, &f.tail_size)) {
494
0
            return "invalid tail start offset";
495
0
          }
496
75.5k
          break;
497
75.5k
        case kUserDefinedTimestampsPersisted:
498
0
          if (field.size() != 1) {
499
0
            return "user-defined timestamps persisted field wrong size";
500
0
          }
501
0
          f.user_defined_timestamps_persisted = (field[0] == 1);
502
0
          break;
503
0
        case kMinTimestamp:
504
0
          f.min_timestamp = field.ToString();
505
0
          break;
506
0
        case kMaxTimestamp:
507
0
          f.max_timestamp = field.ToString();
508
0
          break;
509
0
        default:
510
0
          if ((custom_tag & kCustomTagNonSafeIgnoreMask) != 0) {
511
            // Should not proceed if cannot understand it
512
0
            return "new-file4 custom field not supported";
513
0
          }
514
0
          break;
515
398k
      }
516
398k
    }
517
75.5k
  } else {
518
0
    return "new-file4 entry";
519
0
  }
520
75.5k
  f.fd =
521
75.5k
      FileDescriptor(number, path_id, file_size, smallest_seqno, largest_seqno);
522
75.5k
  new_files.emplace_back(level, f);
523
75.5k
  return nullptr;
524
75.5k
}
525
526
void VersionEdit::EncodeFileBoundaries(std::string* dst,
527
90.7k
                                       const FileMetaData& meta, size_t ts_sz) {
528
90.7k
  if (ts_sz == 0 || meta.user_defined_timestamps_persisted) {
529
90.7k
    PutLengthPrefixedSlice(dst, meta.smallest.Encode());
530
90.7k
    PutLengthPrefixedSlice(dst, meta.largest.Encode());
531
90.7k
    return;
532
90.7k
  }
533
0
  std::string smallest_buf;
534
0
  std::string largest_buf;
535
0
  StripTimestampFromInternalKey(&smallest_buf, meta.smallest.Encode(), ts_sz);
536
0
  StripTimestampFromInternalKey(&largest_buf, meta.largest.Encode(), ts_sz);
537
0
  PutLengthPrefixedSlice(dst, smallest_buf);
538
0
  PutLengthPrefixedSlice(dst, largest_buf);
539
0
}
540
541
340k
Status VersionEdit::DecodeFrom(const Slice& src) {
542
340k
  Clear();
543
#ifndef NDEBUG
544
  bool ignore_ignorable_tags = false;
545
  TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:IgnoreIgnorableTags",
546
                           &ignore_ignorable_tags);
547
#endif
548
340k
  Slice input = src;
549
340k
  const char* msg = nullptr;
550
340k
  uint32_t tag = 0;
551
552
  // Temporary storage for parsing
553
340k
  int level = 0;
554
340k
  FileMetaData f;
555
340k
  Slice str;
556
340k
  InternalKey key;
557
1.46M
  while (msg == nullptr && GetVarint32(&input, &tag)) {
558
#ifndef NDEBUG
559
    if (ignore_ignorable_tags && tag > kTagSafeIgnoreMask) {
560
      tag = kTagSafeIgnoreMask;
561
    }
562
#endif
563
1.12M
    switch (tag) {
564
42.2k
      case kDbId:
565
42.2k
        if (GetLengthPrefixedSlice(&input, &str)) {
566
42.2k
          db_id_ = str.ToString();
567
42.2k
          has_db_id_ = true;
568
42.2k
        } else {
569
0
          msg = "db id";
570
0
        }
571
42.2k
        break;
572
63.6k
      case kComparator:
573
63.6k
        if (GetLengthPrefixedSlice(&input, &str)) {
574
63.6k
          comparator_ = str.ToString();
575
63.6k
          has_comparator_ = true;
576
63.6k
        } else {
577
0
          msg = "comparator name";
578
0
        }
579
63.6k
        break;
580
581
113k
      case kLogNumber:
582
113k
        if (GetVarint64(&input, &log_number_)) {
583
113k
          has_log_number_ = true;
584
113k
        } else {
585
0
          msg = "log number";
586
0
        }
587
113k
        break;
588
589
137k
      case kPrevLogNumber:
590
137k
        if (GetVarint64(&input, &prev_log_number_)) {
591
137k
          has_prev_log_number_ = true;
592
137k
        } else {
593
0
          msg = "previous log number";
594
0
        }
595
137k
        break;
596
597
173k
      case kNextFileNumber:
598
173k
        if (GetVarint64(&input, &next_file_number_)) {
599
173k
          has_next_file_number_ = true;
600
173k
        } else {
601
0
          msg = "next file number";
602
0
        }
603
173k
        break;
604
605
33.0k
      case kMaxColumnFamily:
606
33.0k
        if (GetVarint32(&input, &max_column_family_)) {
607
33.0k
          has_max_column_family_ = true;
608
33.0k
        } else {
609
0
          msg = "max column family";
610
0
        }
611
33.0k
        break;
612
613
53.8k
      case kMinLogNumberToKeep:
614
53.8k
        if (GetVarint64(&input, &min_log_number_to_keep_)) {
615
53.8k
          has_min_log_number_to_keep_ = true;
616
53.8k
        } else {
617
0
          msg = "min log number to kee";
618
0
        }
619
53.8k
        break;
620
621
222k
      case kLastSequence:
622
222k
        if (GetVarint64(&input, &last_sequence_)) {
623
222k
          has_last_sequence_ = true;
624
222k
        } else {
625
0
          msg = "last sequence number";
626
0
        }
627
222k
        break;
628
629
0
      case kCompactCursor:
630
0
        if (GetLevel(&input, &level, max_level_) &&
631
0
            GetInternalKey(&input, &key)) {
632
          // Here we re-use the output format of compact pointer in LevelDB
633
          // to persist compact_cursors_
634
0
          compact_cursors_.push_back(std::make_pair(level, key));
635
0
        } else {
636
0
          if (!msg) {
637
0
            msg = "compaction cursor";
638
0
          }
639
0
        }
640
0
        break;
641
642
8.04k
      case kDeletedFile: {
643
8.04k
        uint64_t number = 0;
644
8.04k
        if (GetLevel(&input, &level, max_level_) &&
645
8.04k
            GetVarint64(&input, &number)) {
646
8.04k
          deleted_files_.insert(std::make_pair(level, number));
647
8.04k
        } else {
648
0
          if (!msg) {
649
0
            msg = "deleted file";
650
0
          }
651
0
        }
652
8.04k
        break;
653
0
      }
654
655
0
      case kNewFile: {
656
0
        uint64_t number = 0;
657
0
        uint64_t file_size = 0;
658
0
        if (GetLevel(&input, &level, max_level_) &&
659
0
            GetVarint64(&input, &number) && GetVarint64(&input, &file_size) &&
660
0
            GetInternalKey(&input, &f.smallest) &&
661
0
            GetInternalKey(&input, &f.largest)) {
662
0
          f.fd = FileDescriptor(number, 0, file_size);
663
0
          new_files_.push_back(std::make_pair(level, f));
664
0
        } else {
665
0
          if (!msg) {
666
0
            msg = "new-file entry";
667
0
          }
668
0
        }
669
0
        break;
670
0
      }
671
0
      case kNewFile2: {
672
0
        uint64_t number = 0;
673
0
        uint64_t file_size = 0;
674
0
        SequenceNumber smallest_seqno = 0;
675
0
        SequenceNumber largest_seqno = kMaxSequenceNumber;
676
0
        if (GetLevel(&input, &level, max_level_) &&
677
0
            GetVarint64(&input, &number) && GetVarint64(&input, &file_size) &&
678
0
            GetInternalKey(&input, &f.smallest) &&
679
0
            GetInternalKey(&input, &f.largest) &&
680
0
            GetVarint64(&input, &smallest_seqno) &&
681
0
            GetVarint64(&input, &largest_seqno)) {
682
0
          f.fd = FileDescriptor(number, 0, file_size, smallest_seqno,
683
0
                                largest_seqno);
684
0
          new_files_.push_back(std::make_pair(level, f));
685
0
        } else {
686
0
          if (!msg) {
687
0
            msg = "new-file2 entry";
688
0
          }
689
0
        }
690
0
        break;
691
0
      }
692
693
0
      case kNewFile3: {
694
0
        uint64_t number = 0;
695
0
        uint32_t path_id = 0;
696
0
        uint64_t file_size = 0;
697
0
        SequenceNumber smallest_seqno = 0;
698
0
        SequenceNumber largest_seqno = kMaxSequenceNumber;
699
0
        if (GetLevel(&input, &level, max_level_) &&
700
0
            GetVarint64(&input, &number) && GetVarint32(&input, &path_id) &&
701
0
            GetVarint64(&input, &file_size) &&
702
0
            GetInternalKey(&input, &f.smallest) &&
703
0
            GetInternalKey(&input, &f.largest) &&
704
0
            GetVarint64(&input, &smallest_seqno) &&
705
0
            GetVarint64(&input, &largest_seqno)) {
706
0
          f.fd = FileDescriptor(number, path_id, file_size, smallest_seqno,
707
0
                                largest_seqno);
708
0
          new_files_.push_back(std::make_pair(level, f));
709
0
        } else {
710
0
          if (!msg) {
711
0
            msg = "new-file3 entry";
712
0
          }
713
0
        }
714
0
        break;
715
0
      }
716
717
75.5k
      case kNewFile4: {
718
75.5k
        FileMetaData ignored_file;
719
75.5k
        msg = DecodeNewFile4From(&input, max_level_, min_log_number_to_keep_,
720
75.5k
                                 has_min_log_number_to_keep_, new_files_,
721
75.5k
                                 ignored_file);
722
75.5k
        break;
723
0
      }
724
725
0
      case kBlobFileAddition:
726
0
      case kBlobFileAddition_DEPRECATED: {
727
0
        BlobFileAddition blob_file_addition;
728
0
        const Status s = blob_file_addition.DecodeFrom(&input);
729
0
        if (!s.ok()) {
730
0
          return s;
731
0
        }
732
733
0
        AddBlobFile(std::move(blob_file_addition));
734
0
        break;
735
0
      }
736
737
0
      case kBlobFileGarbage:
738
0
      case kBlobFileGarbage_DEPRECATED: {
739
0
        BlobFileGarbage blob_file_garbage;
740
0
        const Status s = blob_file_garbage.DecodeFrom(&input);
741
0
        if (!s.ok()) {
742
0
          return s;
743
0
        }
744
745
0
        AddBlobFileGarbage(std::move(blob_file_garbage));
746
0
        break;
747
0
      }
748
749
0
      case kWalAddition: {
750
0
        WalAddition wal_addition;
751
0
        const Status s = wal_addition.DecodeFrom(&input);
752
0
        if (!s.ok()) {
753
0
          return s;
754
0
        }
755
756
0
        wal_additions_.emplace_back(std::move(wal_addition));
757
0
        break;
758
0
      }
759
760
0
      case kWalAddition2: {
761
0
        Slice encoded;
762
0
        if (!GetLengthPrefixedSlice(&input, &encoded)) {
763
0
          msg = "WalAddition not prefixed by length";
764
0
          break;
765
0
        }
766
767
0
        WalAddition wal_addition;
768
0
        const Status s = wal_addition.DecodeFrom(&encoded);
769
0
        if (!s.ok()) {
770
0
          return s;
771
0
        }
772
773
0
        wal_additions_.emplace_back(std::move(wal_addition));
774
0
        break;
775
0
      }
776
777
0
      case kWalDeletion: {
778
0
        WalDeletion wal_deletion;
779
0
        const Status s = wal_deletion.DecodeFrom(&input);
780
0
        if (!s.ok()) {
781
0
          return s;
782
0
        }
783
784
0
        wal_deletion_ = std::move(wal_deletion);
785
0
        break;
786
0
      }
787
788
25.1k
      case kWalDeletion2: {
789
25.1k
        Slice encoded;
790
25.1k
        if (!GetLengthPrefixedSlice(&input, &encoded)) {
791
0
          msg = "WalDeletion not prefixed by length";
792
0
          break;
793
0
        }
794
795
25.1k
        WalDeletion wal_deletion;
796
25.1k
        const Status s = wal_deletion.DecodeFrom(&encoded);
797
25.1k
        if (!s.ok()) {
798
0
          return s;
799
0
        }
800
801
25.1k
        wal_deletion_ = std::move(wal_deletion);
802
25.1k
        break;
803
25.1k
      }
804
805
69.9k
      case kColumnFamily:
806
69.9k
        if (!GetVarint32(&input, &column_family_)) {
807
0
          if (!msg) {
808
0
            msg = "set column family id";
809
0
          }
810
0
        }
811
69.9k
        break;
812
813
28.8k
      case kColumnFamilyAdd:
814
28.8k
        if (GetLengthPrefixedSlice(&input, &str)) {
815
28.8k
          is_column_family_add_ = true;
816
28.8k
          column_family_name_ = str.ToString();
817
28.8k
        } else {
818
0
          if (!msg) {
819
0
            msg = "column family add";
820
0
          }
821
0
        }
822
28.8k
        break;
823
824
13.6k
      case kColumnFamilyDrop:
825
13.6k
        is_column_family_drop_ = true;
826
13.6k
        break;
827
828
0
      case kInAtomicGroup:
829
0
        is_in_atomic_group_ = true;
830
0
        if (!GetVarint32(&input, &remaining_entries_)) {
831
0
          if (!msg) {
832
0
            msg = "remaining entries";
833
0
          }
834
0
        }
835
0
        break;
836
837
0
      case kFullHistoryTsLow:
838
0
        if (!GetLengthPrefixedSlice(&input, &str)) {
839
0
          msg = "full_history_ts_low";
840
0
        } else if (str.empty()) {
841
0
          msg = "full_history_ts_low: empty";
842
0
        } else {
843
0
          full_history_ts_low_.assign(str.data(), str.size());
844
0
        }
845
0
        break;
846
847
63.6k
      case kPersistUserDefinedTimestamps:
848
63.6k
        if (!GetLengthPrefixedSlice(&input, &str)) {
849
0
          msg = "persist_user_defined_timestamps";
850
63.6k
        } else if (str.size() != 1) {
851
0
          msg = "persist_user_defined_timestamps field wrong size";
852
63.6k
        } else {
853
63.6k
          persist_user_defined_timestamps_ = (str[0] == 1);
854
63.6k
          has_persist_user_defined_timestamps_ = true;
855
63.6k
        }
856
63.6k
        break;
857
858
0
      case kSubcompactionProgress: {
859
0
        Slice encoded;
860
0
        if (!GetLengthPrefixedSlice(&input, &encoded)) {
861
0
          msg = "SubcompactionProgress not prefixed by length";
862
0
          break;
863
0
        }
864
865
0
        SubcompactionProgress progress;
866
0
        Status s = progress.DecodeFrom(&encoded);
867
0
        if (!s.ok()) {
868
0
          return s;
869
0
        }
870
871
0
        SetSubcompactionProgress(progress);
872
0
        break;
873
0
      }
874
875
0
      default:
876
0
        if (tag & kTagSafeIgnoreMask) {
877
          // Tag from future which can be safely ignored.
878
          // The next field must be the length of the entry.
879
0
          uint32_t field_len;
880
0
          if (!GetVarint32(&input, &field_len) ||
881
0
              static_cast<size_t>(field_len) > input.size()) {
882
0
            if (!msg) {
883
0
              msg = "safely ignoreable tag length error";
884
0
            }
885
0
          } else {
886
0
            input.remove_prefix(static_cast<size_t>(field_len));
887
0
          }
888
0
        } else {
889
0
          msg = "unknown tag";
890
0
        }
891
0
        break;
892
1.12M
    }
893
1.12M
  }
894
895
340k
  if (msg == nullptr && !input.empty()) {
896
0
    msg = "invalid tag";
897
0
  }
898
899
340k
  Status result;
900
340k
  if (msg != nullptr) {
901
0
    result = Status::Corruption("VersionEdit", msg);
902
0
  }
903
340k
  return result;
904
340k
}
905
906
0
std::string VersionEdit::DebugString(bool hex_key) const {
907
0
  std::string r;
908
0
  r.append("VersionEdit {");
909
0
  if (has_db_id_) {
910
0
    r.append("\n  DB ID: ");
911
0
    r.append(db_id_);
912
0
  }
913
0
  if (has_comparator_) {
914
0
    r.append("\n  Comparator: ");
915
0
    r.append(comparator_);
916
0
  }
917
0
  if (has_persist_user_defined_timestamps_) {
918
0
    r.append("\n  PersistUserDefinedTimestamps: ");
919
0
    r.append(persist_user_defined_timestamps_ ? "true" : "false");
920
0
  }
921
0
  if (has_log_number_) {
922
0
    r.append("\n  LogNumber: ");
923
0
    AppendNumberTo(&r, log_number_);
924
0
  }
925
0
  if (has_prev_log_number_) {
926
0
    r.append("\n  PrevLogNumber: ");
927
0
    AppendNumberTo(&r, prev_log_number_);
928
0
  }
929
0
  if (has_next_file_number_) {
930
0
    r.append("\n  NextFileNumber: ");
931
0
    AppendNumberTo(&r, next_file_number_);
932
0
  }
933
0
  if (has_max_column_family_) {
934
0
    r.append("\n  MaxColumnFamily: ");
935
0
    AppendNumberTo(&r, max_column_family_);
936
0
  }
937
0
  if (has_min_log_number_to_keep_) {
938
0
    r.append("\n  MinLogNumberToKeep: ");
939
0
    AppendNumberTo(&r, min_log_number_to_keep_);
940
0
  }
941
0
  if (has_last_sequence_) {
942
0
    r.append("\n  LastSeq: ");
943
0
    AppendNumberTo(&r, last_sequence_);
944
0
  }
945
0
  for (const auto& level_and_compact_cursor : compact_cursors_) {
946
0
    r.append("\n  CompactCursor: ");
947
0
    AppendNumberTo(&r, level_and_compact_cursor.first);
948
0
    r.append(" ");
949
0
    r.append(level_and_compact_cursor.second.DebugString(hex_key));
950
0
  }
951
0
  for (const auto& deleted_file : deleted_files_) {
952
0
    r.append("\n  DeleteFile: ");
953
0
    AppendNumberTo(&r, deleted_file.first);
954
0
    r.append(" ");
955
0
    AppendNumberTo(&r, deleted_file.second);
956
0
  }
957
0
  for (size_t i = 0; i < new_files_.size(); i++) {
958
0
    const FileMetaData& f = new_files_[i].second;
959
0
    r.append("\n  AddFile: ");
960
0
    AppendNumberTo(&r, new_files_[i].first);
961
0
    r.append(" ");
962
0
    AppendNumberTo(&r, f.fd.GetNumber());
963
0
    r.append(" ");
964
0
    AppendNumberTo(&r, f.fd.GetFileSize());
965
0
    r.append(" ");
966
0
    r.append(f.smallest.DebugString(hex_key));
967
0
    r.append(" .. ");
968
0
    r.append(f.largest.DebugString(hex_key));
969
0
    if (f.oldest_blob_file_number != kInvalidBlobFileNumber) {
970
0
      r.append(" blob_file:");
971
0
      AppendNumberTo(&r, f.oldest_blob_file_number);
972
0
    }
973
0
    r.append(" oldest_ancester_time:");
974
0
    AppendNumberTo(&r, f.oldest_ancester_time);
975
0
    r.append(" file_creation_time:");
976
0
    AppendNumberTo(&r, f.file_creation_time);
977
0
    r.append(" epoch_number:");
978
0
    AppendNumberTo(&r, f.epoch_number);
979
0
    r.append(" file_checksum:");
980
0
    r.append(Slice(f.file_checksum).ToString(true));
981
0
    r.append(" file_checksum_func_name: ");
982
0
    r.append(f.file_checksum_func_name);
983
0
    if (f.temperature != Temperature::kUnknown) {
984
0
      r.append(" temperature: ");
985
      // Maybe change to human readable format whenthe feature becomes
986
      // permanent
987
0
      r.append(std::to_string(static_cast<int>(f.temperature)));
988
0
    }
989
0
    if (f.unique_id != kNullUniqueId64x2) {
990
0
      r.append(" unique_id(internal): ");
991
0
      UniqueId64x2 id = f.unique_id;
992
0
      r.append(InternalUniqueIdToHumanString(&id));
993
0
      r.append(" public_unique_id: ");
994
0
      InternalUniqueIdToExternal(&id);
995
0
      r.append(UniqueIdToHumanString(EncodeUniqueIdBytes(&id)));
996
0
    }
997
0
    r.append(" tail size: ");
998
0
    AppendNumberTo(&r, f.tail_size);
999
0
    r.append(" User-defined timestamps persisted: ");
1000
0
    r.append(f.user_defined_timestamps_persisted ? "true" : "false");
1001
0
  }
1002
1003
0
  for (const auto& blob_file_addition : blob_file_additions_) {
1004
0
    r.append("\n  BlobFileAddition: ");
1005
0
    r.append(blob_file_addition.DebugString());
1006
0
  }
1007
1008
0
  for (const auto& blob_file_garbage : blob_file_garbages_) {
1009
0
    r.append("\n  BlobFileGarbage: ");
1010
0
    r.append(blob_file_garbage.DebugString());
1011
0
  }
1012
1013
0
  for (const auto& wal_addition : wal_additions_) {
1014
0
    r.append("\n  WalAddition: ");
1015
0
    r.append(wal_addition.DebugString());
1016
0
  }
1017
1018
0
  if (!wal_deletion_.IsEmpty()) {
1019
0
    r.append("\n  WalDeletion: ");
1020
0
    r.append(wal_deletion_.DebugString());
1021
0
  }
1022
1023
0
  r.append("\n  ColumnFamily: ");
1024
0
  AppendNumberTo(&r, column_family_);
1025
0
  if (is_column_family_add_) {
1026
0
    r.append("\n  ColumnFamilyAdd: ");
1027
0
    r.append(column_family_name_);
1028
0
  }
1029
0
  if (is_column_family_drop_) {
1030
0
    r.append("\n  ColumnFamilyDrop");
1031
0
  }
1032
0
  if (is_in_atomic_group_) {
1033
0
    r.append("\n  AtomicGroup: ");
1034
0
    AppendNumberTo(&r, remaining_entries_);
1035
0
    r.append(" entries remains");
1036
0
  }
1037
0
  if (HasFullHistoryTsLow()) {
1038
0
    r.append("\n FullHistoryTsLow: ");
1039
0
    r.append(Slice(full_history_ts_low_).ToString(hex_key));
1040
0
  }
1041
0
  if (HasSubcompactionProgress()) {
1042
0
    r.append("\n SubcompactionProgress: ");
1043
0
    r.append(subcompaction_progress_.ToString());
1044
0
  }
1045
0
  r.append("\n}\n");
1046
0
  return r;
1047
0
}
1048
1049
0
std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
1050
0
  JSONWriter jw;
1051
0
  jw << "EditNumber" << edit_num;
1052
1053
0
  if (has_db_id_) {
1054
0
    jw << "DB ID" << db_id_;
1055
0
  }
1056
0
  if (has_comparator_) {
1057
0
    jw << "Comparator" << comparator_;
1058
0
  }
1059
0
  if (has_log_number_) {
1060
0
    jw << "LogNumber" << log_number_;
1061
0
  }
1062
0
  if (has_prev_log_number_) {
1063
0
    jw << "PrevLogNumber" << prev_log_number_;
1064
0
  }
1065
0
  if (has_next_file_number_) {
1066
0
    jw << "NextFileNumber" << next_file_number_;
1067
0
  }
1068
0
  if (has_max_column_family_) {
1069
0
    jw << "MaxColumnFamily" << max_column_family_;
1070
0
  }
1071
0
  if (has_min_log_number_to_keep_) {
1072
0
    jw << "MinLogNumberToKeep" << min_log_number_to_keep_;
1073
0
  }
1074
0
  if (has_last_sequence_) {
1075
0
    jw << "LastSeq" << last_sequence_;
1076
0
  }
1077
1078
0
  if (!deleted_files_.empty()) {
1079
0
    jw << "DeletedFiles";
1080
0
    jw.StartArray();
1081
1082
0
    for (const auto& deleted_file : deleted_files_) {
1083
0
      jw.StartArrayedObject();
1084
0
      jw << "Level" << deleted_file.first;
1085
0
      jw << "FileNumber" << deleted_file.second;
1086
0
      jw.EndArrayedObject();
1087
0
    }
1088
1089
0
    jw.EndArray();
1090
0
  }
1091
1092
0
  if (!new_files_.empty()) {
1093
0
    jw << "AddedFiles";
1094
0
    jw.StartArray();
1095
1096
0
    for (size_t i = 0; i < new_files_.size(); i++) {
1097
0
      jw.StartArrayedObject();
1098
0
      jw << "Level" << new_files_[i].first;
1099
0
      const FileMetaData& f = new_files_[i].second;
1100
0
      jw << "FileNumber" << f.fd.GetNumber();
1101
0
      jw << "FileSize" << f.fd.GetFileSize();
1102
0
      jw << "SmallestIKey" << f.smallest.DebugString(hex_key);
1103
0
      jw << "LargestIKey" << f.largest.DebugString(hex_key);
1104
0
      jw << "OldestAncesterTime" << f.oldest_ancester_time;
1105
0
      jw << "FileCreationTime" << f.file_creation_time;
1106
0
      jw << "EpochNumber" << f.epoch_number;
1107
0
      jw << "FileChecksum" << Slice(f.file_checksum).ToString(true);
1108
0
      jw << "FileChecksumFuncName" << f.file_checksum_func_name;
1109
0
      if (f.temperature != Temperature::kUnknown) {
1110
0
        jw << "temperature" << std::to_string(static_cast<int>(f.temperature));
1111
0
      }
1112
0
      if (f.oldest_blob_file_number != kInvalidBlobFileNumber) {
1113
0
        jw << "OldestBlobFile" << f.oldest_blob_file_number;
1114
0
      }
1115
0
      if (f.temperature != Temperature::kUnknown) {
1116
        // Maybe change to human readable format whenthe feature becomes
1117
        // permanent
1118
0
        jw << "Temperature" << static_cast<int>(f.temperature);
1119
0
      }
1120
0
      jw << "TailSize" << f.tail_size;
1121
0
      jw << "UserDefinedTimestampsPersisted"
1122
0
         << f.user_defined_timestamps_persisted;
1123
0
      jw.EndArrayedObject();
1124
0
    }
1125
1126
0
    jw.EndArray();
1127
0
  }
1128
1129
0
  if (!blob_file_additions_.empty()) {
1130
0
    jw << "BlobFileAdditions";
1131
1132
0
    jw.StartArray();
1133
1134
0
    for (const auto& blob_file_addition : blob_file_additions_) {
1135
0
      jw.StartArrayedObject();
1136
0
      jw << blob_file_addition;
1137
0
      jw.EndArrayedObject();
1138
0
    }
1139
1140
0
    jw.EndArray();
1141
0
  }
1142
1143
0
  if (!blob_file_garbages_.empty()) {
1144
0
    jw << "BlobFileGarbages";
1145
1146
0
    jw.StartArray();
1147
1148
0
    for (const auto& blob_file_garbage : blob_file_garbages_) {
1149
0
      jw.StartArrayedObject();
1150
0
      jw << blob_file_garbage;
1151
0
      jw.EndArrayedObject();
1152
0
    }
1153
1154
0
    jw.EndArray();
1155
0
  }
1156
1157
0
  if (!wal_additions_.empty()) {
1158
0
    jw << "WalAdditions";
1159
1160
0
    jw.StartArray();
1161
1162
0
    for (const auto& wal_addition : wal_additions_) {
1163
0
      jw.StartArrayedObject();
1164
0
      jw << wal_addition;
1165
0
      jw.EndArrayedObject();
1166
0
    }
1167
1168
0
    jw.EndArray();
1169
0
  }
1170
1171
0
  if (!wal_deletion_.IsEmpty()) {
1172
0
    jw << "WalDeletion";
1173
0
    jw.StartObject();
1174
0
    jw << wal_deletion_;
1175
0
    jw.EndObject();
1176
0
  }
1177
1178
0
  jw << "ColumnFamily" << column_family_;
1179
1180
0
  if (is_column_family_add_) {
1181
0
    jw << "ColumnFamilyAdd" << column_family_name_;
1182
0
  }
1183
0
  if (is_column_family_drop_) {
1184
0
    jw << "ColumnFamilyDrop" << column_family_name_;
1185
0
  }
1186
0
  if (is_in_atomic_group_) {
1187
0
    jw << "AtomicGroup" << remaining_entries_;
1188
0
  }
1189
1190
0
  if (HasFullHistoryTsLow()) {
1191
0
    jw << "FullHistoryTsLow" << Slice(full_history_ts_low_).ToString(hex_key);
1192
0
  }
1193
1194
0
  if (HasSubcompactionProgress()) {
1195
0
    jw << "SubcompactionProgress" << subcompaction_progress_.ToString();
1196
0
  }
1197
1198
0
  jw.EndObject();
1199
1200
0
  return jw.Get();
1201
0
}
1202
1203
0
void SubcompactionProgressPerLevel::EncodeTo(std::string* dst) const {
1204
0
  if (num_processed_output_records_ > 0) {
1205
0
    PutVarint32(
1206
0
        dst,
1207
0
        SubcompactionProgressPerLevelCustomTag::kNumProcessedOutputRecords);
1208
0
    std::string varint_records;
1209
0
    PutVarint64(&varint_records, num_processed_output_records_);
1210
0
    PutLengthPrefixedSlice(dst, varint_records);
1211
0
  }
1212
1213
0
  if (!output_files_.empty()) {
1214
0
    PutVarint32(dst, SubcompactionProgressPerLevelCustomTag::kOutputFilesDelta);
1215
0
    std::string files_data;
1216
0
    EncodeOutputFiles(&files_data);
1217
0
    PutLengthPrefixedSlice(dst, files_data);
1218
0
  }
1219
1220
0
  PutVarint32(dst, SubcompactionProgressPerLevelCustomTag::
1221
0
                       kSubcompactionProgressPerLevelTerminate);
1222
0
}
1223
1224
0
Status SubcompactionProgressPerLevel::DecodeFrom(Slice* input) {
1225
0
  Clear();
1226
1227
0
  while (true) {
1228
0
    uint32_t tag = 0;
1229
0
    if (!GetVarint32(input, &tag)) {
1230
0
      return Status::Corruption("SubcompactionProgressPerLevel", "tag error");
1231
0
    }
1232
1233
0
    if (tag == SubcompactionProgressPerLevelCustomTag::
1234
0
                   kSubcompactionProgressPerLevelTerminate) {
1235
0
      break;
1236
0
    }
1237
1238
0
    Slice field;
1239
0
    if (!GetLengthPrefixedSlice(input, &field)) {
1240
0
      return Status::Corruption("SubcompactionProgressPerLevel",
1241
0
                                "field length prefixed slice error");
1242
0
    }
1243
1244
0
    switch (tag) {
1245
0
      case SubcompactionProgressPerLevelCustomTag::kNumProcessedOutputRecords: {
1246
0
        if (!GetVarint64(&field, &num_processed_output_records_)) {
1247
0
          return Status::Corruption("SubcompactionProgressPerLevel",
1248
0
                                    "invalid num_processed_output_records_");
1249
0
        }
1250
0
        break;
1251
0
      }
1252
1253
0
      case SubcompactionProgressPerLevelCustomTag::kOutputFilesDelta: {
1254
0
        Status s = DecodeOutputFiles(&field, output_files_);
1255
0
        if (!s.ok()) {
1256
0
          return s;
1257
0
        }
1258
0
        break;
1259
0
      }
1260
1261
0
      default:
1262
        // Forward compatibility: Handle unknown tags
1263
0
        if ((tag & SubcompactionProgressPerLevelCustomTag::
1264
0
                       kSubcompactionProgressPerLevelCustomTagSafeIgnoreMask) !=
1265
0
            0) {
1266
0
          break;
1267
0
        } else {
1268
0
          return Status::NotSupported("SubcompactionProgress",
1269
0
                                      "unsupported critical custom field");
1270
0
        }
1271
0
    }
1272
0
  }
1273
1274
0
  return Status::OK();
1275
0
}
1276
1277
0
void SubcompactionProgressPerLevel::EncodeOutputFiles(std::string* dst) const {
1278
0
  size_t new_files_count =
1279
0
      output_files_.size() > last_persisted_output_files_count_
1280
0
          ? output_files_.size() - last_persisted_output_files_count_
1281
0
          : 0;
1282
1283
0
  assert(new_files_count > 0);
1284
1285
0
  PutVarint32(dst, static_cast<uint32_t>(new_files_count));
1286
1287
0
  for (size_t i = last_persisted_output_files_count_; i < output_files_.size();
1288
0
       ++i) {
1289
0
    std::string file_dst;
1290
0
    bool ignored_min_log_written = false;
1291
1292
0
    VersionEdit::EncodeToNewFile4(
1293
0
        output_files_[i], -1 /* level */, 0 /* ts_sz */,
1294
0
        false /* has_min_log_number_to_keep */, 0 /* min_log_number_to_keep */,
1295
0
        ignored_min_log_written, &file_dst);
1296
1297
0
    PutLengthPrefixedSlice(dst, file_dst);
1298
0
  }
1299
0
}
1300
1301
Status SubcompactionProgressPerLevel::DecodeOutputFiles(
1302
0
    Slice* input, autovector<FileMetaData>& output_files) {
1303
0
  uint32_t new_file_count = 0;
1304
0
  if (!GetVarint32(input, &new_file_count)) {
1305
0
    return Status::Corruption("SubcompactionProgressPerLevel",
1306
0
                              "new output file count");
1307
0
  }
1308
1309
0
  assert(output_files.size() == 0);
1310
1311
0
  output_files.reserve(new_file_count);
1312
1313
0
  for (uint32_t i = 0; i < new_file_count; ++i) {
1314
0
    Slice file_input;
1315
0
    if (!GetLengthPrefixedSlice(input, &file_input)) {
1316
0
      return Status::Corruption("SubcompactionProgressPerLevel",
1317
0
                                "output file metadata");
1318
0
    }
1319
1320
0
    uint32_t tag = 0;
1321
0
    if (!GetVarint32(&file_input, &tag) || tag != kNewFile4) {
1322
0
      return Status::Corruption("SubcompactionProgressPerLevel",
1323
0
                                "expected kNewFile4 tag");
1324
0
    }
1325
1326
0
    int ignored_max_level = -1;
1327
0
    uint64_t ignored_min_log_number_to_keep = 0;
1328
0
    bool ignored_has_min_log_number_to_keep = false;
1329
0
    VersionEdit::NewFiles ignored_new_files;
1330
0
    FileMetaData file;
1331
1332
0
    const char* err = VersionEdit::DecodeNewFile4From(
1333
0
        &file_input, ignored_max_level, ignored_min_log_number_to_keep,
1334
0
        ignored_has_min_log_number_to_keep, ignored_new_files, file);
1335
1336
0
    if (err != nullptr) {
1337
0
      return Status::Corruption("SubcompactionProgressPerLevel", err);
1338
0
    }
1339
1340
0
    output_files.push_back(std::move(file));
1341
0
  }
1342
1343
0
  return Status::OK();
1344
0
}
1345
1346
0
void SubcompactionProgress::EncodeTo(std::string* dst) const {
1347
0
  if (!next_internal_key_to_compact.empty()) {
1348
0
    PutVarint32(dst, SubcompactionProgressCustomTag::kNextInternalKeyToCompact);
1349
0
    PutLengthPrefixedSlice(dst, next_internal_key_to_compact);
1350
0
  }
1351
1352
0
  PutVarint32(dst, SubcompactionProgressCustomTag::kNumProcessedInputRecords);
1353
0
  std::string varint_records;
1354
0
  PutVarint64(&varint_records, num_processed_input_records);
1355
0
  PutLengthPrefixedSlice(dst, varint_records);
1356
1357
0
  if (output_level_progress.GetOutputFiles().size() >
1358
0
      output_level_progress.GetLastPersistedOutputFilesCount()) {
1359
0
    PutVarint32(dst, SubcompactionProgressCustomTag::kOutputLevelProgress);
1360
0
    std::string level_progress_data;
1361
0
    output_level_progress.EncodeTo(&level_progress_data);
1362
0
    PutLengthPrefixedSlice(dst, level_progress_data);
1363
0
  }
1364
1365
0
  if (proximal_output_level_progress.GetOutputFiles().size() >
1366
0
      proximal_output_level_progress.GetLastPersistedOutputFilesCount()) {
1367
0
    PutVarint32(dst,
1368
0
                SubcompactionProgressCustomTag::kProximalOutputLevelProgress);
1369
0
    std::string level_progress_data;
1370
0
    proximal_output_level_progress.EncodeTo(&level_progress_data);
1371
0
    PutLengthPrefixedSlice(dst, level_progress_data);
1372
0
  }
1373
0
  PutVarint32(dst,
1374
0
              SubcompactionProgressCustomTag::kSubcompactionProgressTerminate);
1375
0
}
1376
1377
0
Status SubcompactionProgress::DecodeFrom(Slice* input) {
1378
0
  Clear();
1379
1380
0
  while (true) {
1381
0
    uint32_t custom_tag = 0;
1382
0
    if (!GetVarint32(input, &custom_tag)) {
1383
0
      return Status::Corruption("SubcompactionProgress",
1384
0
                                "custom field tag error");
1385
0
    }
1386
1387
0
    if (custom_tag ==
1388
0
        SubcompactionProgressCustomTag::kSubcompactionProgressTerminate) {
1389
0
      break;
1390
0
    }
1391
1392
0
    Slice field;
1393
0
    if (!GetLengthPrefixedSlice(input, &field)) {
1394
0
      return Status::Corruption("SubcompactionProgress",
1395
0
                                "custom field length prefixed slice error");
1396
0
    }
1397
1398
0
    switch (custom_tag) {
1399
0
      case SubcompactionProgressCustomTag::kNextInternalKeyToCompact:
1400
0
        next_internal_key_to_compact = field.ToString();
1401
0
        break;
1402
1403
0
      case SubcompactionProgressCustomTag::kNumProcessedInputRecords:
1404
0
        if (!GetVarint64(&field, &num_processed_input_records)) {
1405
0
          return Status::Corruption("SubcompactionProgress",
1406
0
                                    "invalid num_processed_input_records");
1407
0
        }
1408
0
        break;
1409
1410
0
      case SubcompactionProgressCustomTag::kOutputLevelProgress: {
1411
0
        Status s = output_level_progress.DecodeFrom(&field);
1412
0
        if (!s.ok()) {
1413
0
          return s;
1414
0
        }
1415
0
        break;
1416
0
      }
1417
1418
0
      case SubcompactionProgressCustomTag::kProximalOutputLevelProgress: {
1419
0
        Status s = proximal_output_level_progress.DecodeFrom(&field);
1420
0
        if (!s.ok()) {
1421
0
          return s;
1422
0
        }
1423
0
        break;
1424
0
      }
1425
1426
0
      default:
1427
0
        if ((custom_tag & SubcompactionProgressCustomTag::
1428
0
                              kSubcompactionProgressCustomTagSafeIgnoreMask) !=
1429
0
            0) {
1430
0
          break;
1431
0
        } else {
1432
0
          return Status::NotSupported("SubcompactionProgress",
1433
0
                                      "unsupported critical custom field");
1434
0
        }
1435
0
    }
1436
0
  }
1437
1438
0
  return Status::OK();
1439
0
}
1440
1441
0
bool SubcompactionProgressBuilder::ProcessVersionEdit(const VersionEdit& edit) {
1442
0
  if (!edit.HasSubcompactionProgress()) {
1443
0
    return false;
1444
0
  }
1445
1446
0
  const SubcompactionProgress& progress = edit.GetSubcompactionProgress();
1447
1448
0
  MergeDeltaProgress(progress);
1449
1450
0
  has_subcompaction_progress_ = true;
1451
1452
0
  return true;
1453
0
}
1454
1455
void SubcompactionProgressBuilder::MergeDeltaProgress(
1456
0
    const SubcompactionProgress& delta_progress) {
1457
0
  accumulated_subcompaction_progress_.next_internal_key_to_compact =
1458
0
      delta_progress.next_internal_key_to_compact;
1459
1460
0
  accumulated_subcompaction_progress_.num_processed_input_records =
1461
0
      delta_progress.num_processed_input_records;
1462
1463
0
  MaybeMergeDeltaProgressPerLevel(
1464
0
      accumulated_subcompaction_progress_.output_level_progress,
1465
0
      delta_progress.output_level_progress);
1466
1467
0
  MaybeMergeDeltaProgressPerLevel(
1468
0
      accumulated_subcompaction_progress_.proximal_output_level_progress,
1469
0
      delta_progress.proximal_output_level_progress);
1470
0
}
1471
1472
void SubcompactionProgressBuilder::MaybeMergeDeltaProgressPerLevel(
1473
    SubcompactionProgressPerLevel& accumulated_level_progress,
1474
0
    const SubcompactionProgressPerLevel& delta_level_progress) {
1475
0
  const auto& delta_files = delta_level_progress.GetOutputFiles();
1476
0
  if (delta_files.empty()) {
1477
0
    return;
1478
0
  }
1479
0
  for (const FileMetaData& file : delta_files) {
1480
0
    accumulated_level_progress.AddToOutputFiles(file);  // Stored as copy
1481
0
  }
1482
1483
0
  accumulated_level_progress.SetNumProcessedOutputRecords(
1484
0
      delta_level_progress.GetNumProcessedOutputRecords());
1485
0
}
1486
1487
0
void SubcompactionProgressBuilder::Clear() {
1488
0
  accumulated_subcompaction_progress_.Clear();
1489
0
  has_subcompaction_progress_ = false;
1490
0
}
1491
}  // namespace ROCKSDB_NAMESPACE