Coverage Report

Created: 2026-08-13 07:08

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libgit2/deps/reftable/writer.c
Line
Count
Source
1
/*
2
 * Copyright 2020 Google LLC
3
 *
4
 * Use of this source code is governed by a BSD-style
5
 * license that can be found in the LICENSE file or at
6
 * https://developers.google.com/open-source/licenses/bsd
7
 */
8
9
#include "writer.h"
10
11
#include "system.h"
12
13
#include "block.h"
14
#include "constants.h"
15
#include "record.h"
16
#include "tree.h"
17
#include "reftable-error.h"
18
19
/* finishes a block, and writes it to storage */
20
static int writer_flush_block(struct reftable_writer *w);
21
22
/* deallocates memory related to the index */
23
static void writer_clear_index(struct reftable_writer *w);
24
25
/* finishes writing a 'r' (refs) or 'g' (reflogs) section */
26
static int writer_finish_public_section(struct reftable_writer *w);
27
28
static struct reftable_block_stats *
29
writer_reftable_block_stats(struct reftable_writer *w, uint8_t typ)
30
0
{
31
0
  switch (typ) {
32
0
  case 'r':
33
0
    return &w->stats.ref_stats;
34
0
  case 'o':
35
0
    return &w->stats.obj_stats;
36
0
  case 'i':
37
0
    return &w->stats.idx_stats;
38
0
  case 'g':
39
0
    return &w->stats.log_stats;
40
0
  }
41
0
  abort();
42
0
  return NULL;
43
0
}
44
45
/* write data, queuing the padding for the next write. Returns negative for
46
 * error. */
47
static int padded_write(struct reftable_writer *w, uint8_t *data, size_t len,
48
      int padding)
49
0
{
50
0
  int n = 0;
51
0
  if (w->pending_padding > 0) {
52
0
    uint8_t *zeroed;
53
0
    int n;
54
55
0
    zeroed = reftable_calloc(w->pending_padding, sizeof(*zeroed));
56
0
    if (!zeroed)
57
0
      return -1;
58
59
0
    n = w->write(w->write_arg, zeroed, w->pending_padding);
60
0
    if (n < 0) {
61
0
      reftable_free(zeroed);
62
0
      return n;
63
0
    }
64
65
0
    w->pending_padding = 0;
66
0
    reftable_free(zeroed);
67
0
  }
68
69
0
  w->pending_padding = padding;
70
0
  n = w->write(w->write_arg, data, len);
71
0
  if (n < 0)
72
0
    return n;
73
0
  n += padding;
74
0
  return 0;
75
0
}
76
77
static void options_set_defaults(struct reftable_write_options *opts)
78
0
{
79
0
  if (opts->restart_interval == 0) {
80
0
    opts->restart_interval = 16;
81
0
  }
82
83
0
  if (opts->block_size == 0) {
84
0
    opts->block_size = DEFAULT_BLOCK_SIZE;
85
0
  }
86
0
}
87
88
static int writer_version(struct reftable_writer *w)
89
0
{
90
0
  return (w->hash_id == 0 || w->hash_id == REFTABLE_HASH_SHA1) ?
91
0
           1 :
92
0
           2;
93
0
}
94
95
static int writer_write_header(struct reftable_writer *w, uint8_t *dest)
96
0
{
97
0
  memcpy(dest, "REFT", 4);
98
99
0
  dest[4] = writer_version(w);
100
101
0
  reftable_put_be24(dest + 5, w->opts.block_size);
102
0
  reftable_put_be64(dest + 8, w->min_update_index);
103
0
  reftable_put_be64(dest + 16, w->max_update_index);
104
0
  if (writer_version(w) == 2) {
105
0
    uint32_t hash_id;
106
107
0
    switch (w->hash_id) {
108
0
    case REFTABLE_HASH_SHA1:
109
0
      hash_id = REFTABLE_FORMAT_ID_SHA1;
110
0
      break;
111
0
    case REFTABLE_HASH_SHA256:
112
0
      hash_id = REFTABLE_FORMAT_ID_SHA256;
113
0
      break;
114
0
    default:
115
0
      return -1;
116
0
    }
117
118
0
    reftable_put_be32(dest + 24, hash_id);
119
0
  }
120
121
0
  return header_size(writer_version(w));
122
0
}
123
124
static int writer_reinit_block_writer(struct reftable_writer *w, uint8_t typ)
125
0
{
126
0
  int block_start = 0, ret;
127
128
0
  if (w->next == 0)
129
0
    block_start = header_size(writer_version(w));
130
131
0
  reftable_buf_reset(&w->last_key);
132
0
  ret = block_writer_init(&w->block_writer_data, typ, w->block,
133
0
        w->opts.block_size, block_start,
134
0
        hash_size(w->hash_id));
135
0
  if (ret < 0)
136
0
    return ret;
137
138
0
  w->block_writer = &w->block_writer_data;
139
0
  w->block_writer->restart_interval = w->opts.restart_interval;
140
141
0
  return 0;
142
0
}
143
144
int reftable_writer_new(struct reftable_writer **out,
145
      ssize_t (*writer_func)(void *, const void *, size_t),
146
      int (*flush_func)(void *),
147
      void *writer_arg,
148
      enum reftable_hash hash_id,
149
      const struct reftable_write_options *_opts)
150
0
{
151
0
  struct reftable_write_options opts = {0};
152
0
  struct reftable_writer *wp;
153
154
0
  if (_opts)
155
0
    opts = *_opts;
156
0
  options_set_defaults(&opts);
157
0
  if (opts.block_size >= (1 << 24))
158
0
    return REFTABLE_API_ERROR;
159
160
0
  if (!hash_id)
161
0
    hash_id = REFTABLE_HASH_SHA1;
162
163
0
  wp = reftable_calloc(1, sizeof(*wp));
164
0
  if (!wp)
165
0
    return REFTABLE_OUT_OF_MEMORY_ERROR;
166
167
0
  reftable_buf_init(&wp->block_writer_data.last_key);
168
0
  reftable_buf_init(&wp->last_key);
169
0
  reftable_buf_init(&wp->scratch);
170
0
  REFTABLE_CALLOC_ARRAY(wp->block, opts.block_size);
171
0
  if (!wp->block) {
172
0
    reftable_free(wp);
173
0
    return REFTABLE_OUT_OF_MEMORY_ERROR;
174
0
  }
175
0
  wp->write = writer_func;
176
0
  wp->write_arg = writer_arg;
177
0
  wp->opts = opts;
178
0
  wp->hash_id = hash_id;
179
0
  wp->flush = flush_func;
180
0
  writer_reinit_block_writer(wp, REFTABLE_BLOCK_TYPE_REF);
181
182
0
  *out = wp;
183
184
0
  return 0;
185
0
}
186
187
int reftable_writer_set_limits(struct reftable_writer *w, uint64_t min,
188
        uint64_t max)
189
0
{
190
  /*
191
    * Set the min/max update index limits for the reftable writer.
192
    * This must be called before adding any records, since:
193
    * - The 'next' field gets set after writing the first block.
194
    * - The 'last_key' field updates with each new record (but resets
195
    *   after sections).
196
    * Returns REFTABLE_API_ERROR if called after writing has begun.
197
   */
198
0
  if (w->next || w->last_key.len)
199
0
    return REFTABLE_API_ERROR;
200
201
0
  w->min_update_index = min;
202
0
  w->max_update_index = max;
203
204
0
  return 0;
205
0
}
206
207
static void writer_release(struct reftable_writer *w)
208
0
{
209
0
  if (w) {
210
0
    reftable_free(w->block);
211
0
    w->block = NULL;
212
0
    block_writer_release(&w->block_writer_data);
213
0
    w->block_writer = NULL;
214
0
    writer_clear_index(w);
215
0
    reftable_buf_release(&w->last_key);
216
0
    reftable_buf_release(&w->scratch);
217
0
  }
218
0
}
219
220
void reftable_writer_free(struct reftable_writer *w)
221
0
{
222
0
  writer_release(w);
223
0
  reftable_free(w);
224
0
}
225
226
struct obj_index_tree_node {
227
  struct reftable_buf hash;
228
  uint64_t *offsets;
229
  size_t offset_len;
230
  size_t offset_cap;
231
};
232
233
#define OBJ_INDEX_TREE_NODE_INIT    \
234
0
  {                           \
235
0
    .hash = REFTABLE_BUF_INIT \
236
0
  }
237
238
static int obj_index_tree_node_compare(const void *a, const void *b)
239
0
{
240
0
  return reftable_buf_cmp(&((const struct obj_index_tree_node *)a)->hash,
241
0
        &((const struct obj_index_tree_node *)b)->hash);
242
0
}
243
244
static int writer_index_hash(struct reftable_writer *w, struct reftable_buf *hash)
245
0
{
246
0
  uint64_t off = w->next;
247
0
  struct obj_index_tree_node want = { .hash = *hash };
248
0
  struct obj_index_tree_node *key;
249
0
  struct tree_node *node;
250
251
0
  node = tree_search(w->obj_index_tree, &want, &obj_index_tree_node_compare);
252
0
  if (!node) {
253
0
    struct obj_index_tree_node empty = OBJ_INDEX_TREE_NODE_INIT;
254
0
    int err;
255
256
0
    key = reftable_malloc(sizeof(*key));
257
0
    if (!key)
258
0
      return REFTABLE_OUT_OF_MEMORY_ERROR;
259
260
0
    *key = empty;
261
262
0
    reftable_buf_reset(&key->hash);
263
0
    err = reftable_buf_add(&key->hash, hash->buf, hash->len);
264
0
    if (err < 0) {
265
0
      reftable_free(key);
266
0
      return err;
267
0
    }
268
0
    tree_insert(&w->obj_index_tree, key,
269
0
          &obj_index_tree_node_compare);
270
0
  } else {
271
0
    key = node->key;
272
0
  }
273
274
0
  if (key->offset_len > 0 && key->offsets[key->offset_len - 1] == off)
275
0
    return 0;
276
277
0
  REFTABLE_ALLOC_GROW_OR_NULL(key->offsets, key->offset_len + 1,
278
0
            key->offset_cap);
279
0
  if (!key->offsets)
280
0
    return REFTABLE_OUT_OF_MEMORY_ERROR;
281
0
  key->offsets[key->offset_len++] = off;
282
283
0
  return 0;
284
0
}
285
286
static int writer_add_record(struct reftable_writer *w,
287
           struct reftable_record *rec)
288
0
{
289
0
  int err;
290
291
0
  err = reftable_record_key(rec, &w->scratch);
292
0
  if (err < 0)
293
0
    goto done;
294
295
0
  if (reftable_buf_cmp(&w->last_key, &w->scratch) >= 0) {
296
0
    err = REFTABLE_API_ERROR;
297
0
    goto done;
298
0
  }
299
300
0
  reftable_buf_reset(&w->last_key);
301
0
  err = reftable_buf_add(&w->last_key, w->scratch.buf, w->scratch.len);
302
0
  if (err < 0)
303
0
    goto done;
304
305
0
  if (!w->block_writer) {
306
0
    err = writer_reinit_block_writer(w, reftable_record_type(rec));
307
0
    if (err < 0)
308
0
      goto done;
309
0
  }
310
311
0
  if (block_writer_type(w->block_writer) != reftable_record_type(rec))
312
0
    return REFTABLE_API_ERROR;
313
314
  /*
315
   * Try to add the record to the writer. If this succeeds then we're
316
   * done. Otherwise the block writer may have hit the block size limit
317
   * and needs to be flushed.
318
   */
319
0
  err = block_writer_add(w->block_writer, rec);
320
0
  if (err == 0)
321
0
    goto done;
322
323
0
  if (err != REFTABLE_ENTRY_TOO_BIG_ERROR)
324
0
    goto done;
325
  /*
326
   * The current block is full, so we need to flush and reinitialize the
327
   * writer to start writing the next block.
328
   */
329
0
  err = writer_flush_block(w);
330
0
  if (err < 0)
331
0
    goto done;
332
0
  err = writer_reinit_block_writer(w, reftable_record_type(rec));
333
0
  if (err < 0)
334
0
    goto done;
335
336
  /*
337
   * Try to add the record to the writer again. If this still fails then
338
   * the record does not fit into the block size.
339
   */
340
0
  err = block_writer_add(w->block_writer, rec);
341
0
  if (err)
342
0
    goto done;
343
344
0
done:
345
0
  return err;
346
0
}
347
348
int reftable_writer_add_ref(struct reftable_writer *w,
349
          struct reftable_ref_record *ref)
350
0
{
351
0
  struct reftable_record rec = {
352
0
    .type = REFTABLE_BLOCK_TYPE_REF,
353
0
    .u = {
354
0
      .ref = *ref
355
0
    },
356
0
  };
357
0
  int err;
358
359
0
  if (!ref->refname ||
360
0
      ref->update_index < w->min_update_index ||
361
0
      ref->update_index > w->max_update_index)
362
0
    return REFTABLE_API_ERROR;
363
364
0
  rec.u.ref.update_index -= w->min_update_index;
365
366
0
  err = writer_add_record(w, &rec);
367
0
  if (err < 0)
368
0
    goto out;
369
370
0
  if (!w->opts.skip_index_objects && reftable_ref_record_val1(ref)) {
371
0
    reftable_buf_reset(&w->scratch);
372
0
    err = reftable_buf_add(&w->scratch, (char *)reftable_ref_record_val1(ref),
373
0
               hash_size(w->hash_id));
374
0
    if (err < 0)
375
0
      goto out;
376
377
0
    err = writer_index_hash(w, &w->scratch);
378
0
    if (err < 0)
379
0
      goto out;
380
0
  }
381
382
0
  if (!w->opts.skip_index_objects && reftable_ref_record_val2(ref)) {
383
0
    reftable_buf_reset(&w->scratch);
384
0
    err = reftable_buf_add(&w->scratch, reftable_ref_record_val2(ref),
385
0
               hash_size(w->hash_id));
386
0
    if (err < 0)
387
0
      goto out;
388
389
0
    err = writer_index_hash(w, &w->scratch);
390
0
    if (err < 0)
391
0
      goto out;
392
0
  }
393
394
0
  err = 0;
395
396
0
out:
397
0
  return err;
398
0
}
399
400
int reftable_writer_add_refs(struct reftable_writer *w,
401
           struct reftable_ref_record *refs, size_t n)
402
0
{
403
0
  int err = 0;
404
405
0
  if (n)
406
0
    qsort(refs, n, sizeof(*refs), reftable_ref_record_compare_name);
407
408
0
  for (size_t i = 0; err == 0 && i < n; i++)
409
0
    err = reftable_writer_add_ref(w, &refs[i]);
410
411
0
  return err;
412
0
}
413
414
static int reftable_writer_add_log_verbatim(struct reftable_writer *w,
415
              struct reftable_log_record *log)
416
0
{
417
0
  struct reftable_record rec = {
418
0
    .type = REFTABLE_BLOCK_TYPE_LOG,
419
0
    .u = {
420
0
      .log = *log,
421
0
    },
422
0
  };
423
0
  if (w->block_writer &&
424
0
      block_writer_type(w->block_writer) == REFTABLE_BLOCK_TYPE_REF) {
425
0
    int err = writer_finish_public_section(w);
426
0
    if (err < 0)
427
0
      return err;
428
0
  }
429
430
0
  w->next -= w->pending_padding;
431
0
  w->pending_padding = 0;
432
0
  return writer_add_record(w, &rec);
433
0
}
434
435
int reftable_writer_add_log(struct reftable_writer *w,
436
          struct reftable_log_record *log)
437
0
{
438
0
  char *input_log_message = NULL;
439
0
  struct reftable_buf cleaned_message = REFTABLE_BUF_INIT;
440
0
  int err = 0;
441
442
0
  if (log->value_type == REFTABLE_LOG_DELETION)
443
0
    return reftable_writer_add_log_verbatim(w, log);
444
445
  /*
446
   * Verify only the upper limit of the update_index. Each reflog entry
447
   * is tied to a specific update_index. Entries in the reflog can be
448
   * replaced by adding a new entry with the same update_index,
449
   * effectively canceling the old one.
450
   *
451
   * Consequently, reflog updates may include update_index values lower
452
   * than the writer's min_update_index.
453
   */
454
0
  if (log->update_index > w->max_update_index)
455
0
    return REFTABLE_API_ERROR;
456
457
0
  if (!log->refname)
458
0
    return REFTABLE_API_ERROR;
459
460
0
  input_log_message = log->value.update.message;
461
0
  if (!w->opts.exact_log_message && log->value.update.message) {
462
0
    err = reftable_buf_addstr(&cleaned_message, log->value.update.message);
463
0
    if (err < 0)
464
0
      goto done;
465
466
0
    while (cleaned_message.len &&
467
0
           cleaned_message.buf[cleaned_message.len - 1] == '\n') {
468
0
      err = reftable_buf_setlen(&cleaned_message,
469
0
              cleaned_message.len - 1);
470
0
      if (err < 0)
471
0
        goto done;
472
0
    }
473
0
    if (strchr(cleaned_message.buf, '\n')) {
474
      /* multiple lines not allowed. */
475
0
      err = REFTABLE_API_ERROR;
476
0
      goto done;
477
0
    }
478
479
0
    err = reftable_buf_addstr(&cleaned_message, "\n");
480
0
    if (err < 0)
481
0
      goto done;
482
483
0
    log->value.update.message = cleaned_message.buf;
484
0
  }
485
486
0
  err = reftable_writer_add_log_verbatim(w, log);
487
0
  log->value.update.message = input_log_message;
488
0
done:
489
0
  reftable_buf_release(&cleaned_message);
490
0
  return err;
491
0
}
492
493
int reftable_writer_add_logs(struct reftable_writer *w,
494
           struct reftable_log_record *logs, size_t n)
495
0
{
496
0
  int err = 0;
497
498
0
  if (n)
499
0
    qsort(logs, n, sizeof(*logs), reftable_log_record_compare_key);
500
501
0
  for (size_t i = 0; err == 0 && i < n; i++)
502
0
    err = reftable_writer_add_log(w, &logs[i]);
503
504
0
  return err;
505
0
}
506
507
static int writer_finish_section(struct reftable_writer *w)
508
0
{
509
0
  struct reftable_block_stats *bstats = NULL;
510
0
  uint8_t typ = block_writer_type(w->block_writer);
511
0
  uint64_t index_start = 0;
512
0
  int max_level = 0;
513
0
  size_t threshold = w->opts.unpadded ? 1 : 3;
514
0
  int before_blocks = w->stats.idx_stats.blocks;
515
0
  int err;
516
517
0
  err = writer_flush_block(w);
518
0
  if (err < 0)
519
0
    return err;
520
521
  /*
522
   * When the section we are about to index has a lot of blocks then the
523
   * index itself may span across multiple blocks, as well. This would
524
   * require a linear scan over index blocks only to find the desired
525
   * indexed block, which is inefficient. Instead, we write a multi-level
526
   * index where index records of level N+1 will refer to index blocks of
527
   * level N. This isn't constant time, either, but at least logarithmic.
528
   *
529
   * This loop handles writing this multi-level index. Note that we write
530
   * the lowest-level index pointing to the indexed blocks first. We then
531
   * continue writing additional index levels until the current level has
532
   * less blocks than the threshold so that the highest level will be at
533
   * the end of the index section.
534
   *
535
   * Readers are thus required to start reading the index section from
536
   * its end, which is why we set `index_start` to the beginning of the
537
   * last index section.
538
   */
539
0
  while (w->index_len > threshold) {
540
0
    struct reftable_index_record *idx = NULL;
541
0
    size_t i, idx_len;
542
543
0
    max_level++;
544
0
    index_start = w->next;
545
0
    err = writer_reinit_block_writer(w, REFTABLE_BLOCK_TYPE_INDEX);
546
0
    if (err < 0)
547
0
      return err;
548
549
0
    idx = w->index;
550
0
    idx_len = w->index_len;
551
552
0
    w->index = NULL;
553
0
    w->index_len = 0;
554
0
    w->index_cap = 0;
555
0
    for (i = 0; i < idx_len; i++) {
556
0
      struct reftable_record rec = {
557
0
        .type = REFTABLE_BLOCK_TYPE_INDEX,
558
0
        .u = {
559
0
          .idx = idx[i],
560
0
        },
561
0
      };
562
563
0
      err = writer_add_record(w, &rec);
564
0
      if (err < 0)
565
0
        return err;
566
0
    }
567
568
0
    err = writer_flush_block(w);
569
0
    if (err < 0)
570
0
      return err;
571
572
0
    for (i = 0; i < idx_len; i++)
573
0
      reftable_buf_release(&idx[i].last_key);
574
0
    reftable_free(idx);
575
0
  }
576
577
  /*
578
   * The index may still contain a number of index blocks lower than the
579
   * threshold. Clear it so that these entries don't leak into the next
580
   * index section.
581
   */
582
0
  writer_clear_index(w);
583
584
0
  bstats = writer_reftable_block_stats(w, typ);
585
0
  bstats->index_blocks = w->stats.idx_stats.blocks - before_blocks;
586
0
  bstats->index_offset = index_start;
587
0
  bstats->max_index_level = max_level;
588
589
  /* Reinit lastKey, as the next section can start with any key. */
590
0
  reftable_buf_reset(&w->last_key);
591
592
0
  return 0;
593
0
}
594
595
struct common_prefix_arg {
596
  struct reftable_buf *last;
597
  size_t max;
598
};
599
600
static void update_common(void *void_arg, void *key)
601
0
{
602
0
  struct common_prefix_arg *arg = void_arg;
603
0
  struct obj_index_tree_node *entry = key;
604
0
  if (arg->last) {
605
0
    size_t n = common_prefix_size(&entry->hash, arg->last);
606
0
    if (n > arg->max)
607
0
      arg->max = n;
608
0
  }
609
0
  arg->last = &entry->hash;
610
0
}
611
612
struct write_record_arg {
613
  struct reftable_writer *w;
614
  int err;
615
};
616
617
static void write_object_record(void *void_arg, void *key)
618
0
{
619
0
  struct write_record_arg *arg = void_arg;
620
0
  struct obj_index_tree_node *entry = key;
621
0
  struct reftable_record
622
0
    rec = { .type = REFTABLE_BLOCK_TYPE_OBJ,
623
0
      .u.obj = {
624
0
        .hash_prefix = (uint8_t *)entry->hash.buf,
625
0
        .hash_prefix_len = arg->w->stats.object_id_len,
626
0
        .offsets = entry->offsets,
627
0
        .offset_len = entry->offset_len,
628
0
      } };
629
0
  if (arg->err < 0)
630
0
    goto done;
631
632
  /*
633
   * Try to add the record to the writer. If this succeeds then we're
634
   * done. Otherwise the block writer may have hit the block size limit
635
   * and needs to be flushed.
636
   */
637
0
  arg->err = block_writer_add(arg->w->block_writer, &rec);
638
0
  if (arg->err == 0)
639
0
    goto done;
640
641
0
  if (arg->err != REFTABLE_ENTRY_TOO_BIG_ERROR)
642
0
    goto done;
643
644
  /*
645
   * The current block is full, so we need to flush and reinitialize the
646
   * writer to start writing the next block.
647
   */
648
0
  arg->err = writer_flush_block(arg->w);
649
0
  if (arg->err < 0)
650
0
    goto done;
651
652
0
  arg->err = writer_reinit_block_writer(arg->w, REFTABLE_BLOCK_TYPE_OBJ);
653
0
  if (arg->err < 0)
654
0
    goto done;
655
656
  /*
657
   * If this still fails then we may need to reset record's offset
658
   * length to reduce the data size to be written.
659
   */
660
0
  arg->err = block_writer_add(arg->w->block_writer, &rec);
661
0
  if (arg->err == 0)
662
0
    goto done;
663
664
0
  if (arg->err != REFTABLE_ENTRY_TOO_BIG_ERROR)
665
0
    goto done;
666
667
0
  rec.u.obj.offset_len = 0;
668
0
  arg->err = block_writer_add(arg->w->block_writer, &rec);
669
670
  /* Should be able to write into a fresh block. */
671
0
  assert(arg->err == 0);
672
673
0
done:;
674
0
}
675
676
static void object_record_free(void *void_arg REFTABLE_UNUSED, void *key)
677
0
{
678
0
  struct obj_index_tree_node *entry = key;
679
680
0
  REFTABLE_FREE_AND_NULL(entry->offsets);
681
0
  reftable_buf_release(&entry->hash);
682
0
  reftable_free(entry);
683
0
}
684
685
static int writer_dump_object_index(struct reftable_writer *w)
686
0
{
687
0
  struct write_record_arg closure = { .w = w };
688
0
  struct common_prefix_arg common = {
689
0
    .max = 1,   /* obj_id_len should be >= 2. */
690
0
  };
691
0
  int err;
692
693
0
  if (w->obj_index_tree)
694
0
    infix_walk(w->obj_index_tree, &update_common, &common);
695
0
  w->stats.object_id_len = common.max + 1;
696
697
0
  err = writer_reinit_block_writer(w, REFTABLE_BLOCK_TYPE_OBJ);
698
0
  if (err < 0)
699
0
    return err;
700
701
0
  if (w->obj_index_tree)
702
0
    infix_walk(w->obj_index_tree, &write_object_record, &closure);
703
704
0
  if (closure.err < 0)
705
0
    return closure.err;
706
0
  return writer_finish_section(w);
707
0
}
708
709
static int writer_finish_public_section(struct reftable_writer *w)
710
0
{
711
0
  uint8_t typ = 0;
712
0
  int err = 0;
713
714
0
  if (!w->block_writer)
715
0
    return 0;
716
717
0
  typ = block_writer_type(w->block_writer);
718
0
  err = writer_finish_section(w);
719
0
  if (err < 0)
720
0
    return err;
721
0
  if (typ == REFTABLE_BLOCK_TYPE_REF && !w->opts.skip_index_objects &&
722
0
      w->stats.ref_stats.index_blocks > 0) {
723
0
    err = writer_dump_object_index(w);
724
0
    if (err < 0)
725
0
      return err;
726
0
  }
727
728
0
  if (w->obj_index_tree) {
729
0
    infix_walk(w->obj_index_tree, &object_record_free, NULL);
730
0
    tree_free(w->obj_index_tree);
731
0
    w->obj_index_tree = NULL;
732
0
  }
733
734
0
  w->block_writer = NULL;
735
0
  return 0;
736
0
}
737
738
int reftable_writer_close(struct reftable_writer *w)
739
0
{
740
0
  uint8_t footer[72];
741
0
  uint8_t *p = footer;
742
0
  int err = writer_finish_public_section(w);
743
0
  int empty_table = w->next == 0;
744
0
  if (err != 0)
745
0
    goto done;
746
0
  w->pending_padding = 0;
747
0
  if (empty_table) {
748
    /* Empty tables need a header anyway. */
749
0
    uint8_t header[28];
750
0
    int n = writer_write_header(w, header);
751
0
    err = padded_write(w, header, n, 0);
752
0
    if (err < 0)
753
0
      goto done;
754
0
  }
755
756
0
  p += writer_write_header(w, footer);
757
0
  reftable_put_be64(p, w->stats.ref_stats.index_offset);
758
0
  p += 8;
759
0
  reftable_put_be64(p, (w->stats.obj_stats.offset) << 5 | w->stats.object_id_len);
760
0
  p += 8;
761
0
  reftable_put_be64(p, w->stats.obj_stats.index_offset);
762
0
  p += 8;
763
764
0
  reftable_put_be64(p, w->stats.log_stats.offset);
765
0
  p += 8;
766
0
  reftable_put_be64(p, w->stats.log_stats.index_offset);
767
0
  p += 8;
768
769
0
  reftable_put_be32(p, crc32(0, footer, p - footer));
770
0
  p += 4;
771
772
0
  err = w->flush(w->write_arg);
773
0
  if (err < 0) {
774
0
    err = REFTABLE_IO_ERROR;
775
0
    goto done;
776
0
  }
777
778
0
  err = padded_write(w, footer, footer_size(writer_version(w)), 0);
779
0
  if (err < 0)
780
0
    goto done;
781
782
0
  if (empty_table) {
783
0
    err = REFTABLE_EMPTY_TABLE_ERROR;
784
0
    goto done;
785
0
  }
786
787
0
done:
788
0
  writer_release(w);
789
0
  return err;
790
0
}
791
792
static void writer_clear_index(struct reftable_writer *w)
793
0
{
794
0
  for (size_t i = 0; w->index && i < w->index_len; i++)
795
0
    reftable_buf_release(&w->index[i].last_key);
796
0
  REFTABLE_FREE_AND_NULL(w->index);
797
0
  w->index_len = 0;
798
0
  w->index_cap = 0;
799
0
}
800
801
static int writer_flush_nonempty_block(struct reftable_writer *w)
802
0
{
803
0
  struct reftable_index_record index_record = {
804
0
    .last_key = REFTABLE_BUF_INIT,
805
0
  };
806
0
  uint8_t typ = block_writer_type(w->block_writer);
807
0
  struct reftable_block_stats *bstats;
808
0
  int raw_bytes, padding = 0, err;
809
0
  uint64_t block_typ_off;
810
811
  /*
812
   * Finish the current block. This will cause the block writer to emit
813
   * restart points and potentially compress records in case we are
814
   * writing a log block.
815
   *
816
   * Note that this is still happening in memory.
817
   */
818
0
  raw_bytes = block_writer_finish(w->block_writer);
819
0
  if (raw_bytes < 0)
820
0
    return raw_bytes;
821
822
  /*
823
   * By default, all records except for log records are padded to the
824
   * block size.
825
   */
826
0
  if (!w->opts.unpadded && typ != REFTABLE_BLOCK_TYPE_LOG)
827
0
    padding = w->opts.block_size - raw_bytes;
828
829
0
  bstats = writer_reftable_block_stats(w, typ);
830
0
  block_typ_off = (bstats->blocks == 0) ? w->next : 0;
831
0
  if (block_typ_off > 0)
832
0
    bstats->offset = block_typ_off;
833
0
  bstats->entries += w->block_writer->entries;
834
0
  bstats->restarts += w->block_writer->restart_len;
835
0
  bstats->blocks++;
836
0
  w->stats.blocks++;
837
838
  /*
839
   * If this is the first block we're writing to the table then we need
840
   * to also write the reftable header.
841
   */
842
0
  if (!w->next)
843
0
    writer_write_header(w, w->block);
844
845
0
  err = padded_write(w, w->block, raw_bytes, padding);
846
0
  if (err < 0)
847
0
    return err;
848
849
  /*
850
   * Add an index record for every block that we're writing. If we end up
851
   * having more than a threshold of index records we will end up writing
852
   * an index section in `writer_finish_section()`. Each index record
853
   * contains the last record key of the block it is indexing as well as
854
   * the offset of that block.
855
   *
856
   * Note that this also applies when flushing index blocks, in which
857
   * case we will end up with a multi-level index.
858
   */
859
0
  REFTABLE_ALLOC_GROW_OR_NULL(w->index, w->index_len + 1, w->index_cap);
860
0
  if (!w->index)
861
0
    return REFTABLE_OUT_OF_MEMORY_ERROR;
862
863
0
  index_record.offset = w->next;
864
0
  reftable_buf_reset(&index_record.last_key);
865
0
  err = reftable_buf_add(&index_record.last_key, w->block_writer->last_key.buf,
866
0
             w->block_writer->last_key.len);
867
0
  if (err < 0)
868
0
    return err;
869
0
  w->index[w->index_len] = index_record;
870
0
  w->index_len++;
871
872
0
  w->next += padding + raw_bytes;
873
0
  w->block_writer = NULL;
874
875
0
  return 0;
876
0
}
877
878
static int writer_flush_block(struct reftable_writer *w)
879
0
{
880
0
  if (!w->block_writer)
881
0
    return 0;
882
0
  if (w->block_writer->entries == 0)
883
0
    return 0;
884
0
  return writer_flush_nonempty_block(w);
885
0
}
886
887
const struct reftable_stats *reftable_writer_stats(struct reftable_writer *w)
888
0
{
889
0
  return &w->stats;
890
0
}