Coverage Report

Created: 2026-10-17 07:15

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/git/reftable/writer.c
Line
Count
Source
1
/*
2
 * Copyright 2020 Google LLC
3
 *
4
 * Use of this source code is governed by a BSD-style
5
 * license that can be found in the LICENSE file or at
6
 * https://developers.google.com/open-source/licenses/bsd
7
 */
8
9
#include "writer.h"
10
11
#include "system.h"
12
13
#include "block.h"
14
#include "constants.h"
15
#include "record.h"
16
#include "tree.h"
17
#include "reftable-error.h"
18
19
/* finishes a block, and writes it to storage */
20
static int writer_flush_block(struct reftable_writer *w);
21
22
/* deallocates memory related to the index */
23
static void writer_clear_index(struct reftable_writer *w);
24
25
/* finishes writing a 'r' (refs) or 'g' (reflogs) section */
26
static int writer_finish_public_section(struct reftable_writer *w);
27
28
static struct reftable_block_stats *
29
writer_reftable_block_stats(struct reftable_writer *w, uint8_t typ)
30
0
{
31
0
  switch (typ) {
32
0
  case 'r':
33
0
    return &w->stats.ref_stats;
34
0
  case 'o':
35
0
    return &w->stats.obj_stats;
36
0
  case 'i':
37
0
    return &w->stats.idx_stats;
38
0
  case 'g':
39
0
    return &w->stats.log_stats;
40
0
  }
41
0
  abort();
42
0
  return NULL;
43
0
}
44
45
/* write data, queuing the padding for the next write. Returns negative for
46
 * error. */
47
static int padded_write(struct reftable_writer *w, uint8_t *data, size_t len,
48
      int padding)
49
0
{
50
0
  int n = 0;
51
0
  if (w->pending_padding > 0) {
52
0
    uint8_t *zeroed;
53
0
    int n;
54
55
0
    zeroed = reftable_calloc(w->pending_padding, sizeof(*zeroed));
56
0
    if (!zeroed)
57
0
      return -1;
58
59
0
    n = w->write(w->write_arg, zeroed, w->pending_padding);
60
0
    if (n < 0) {
61
0
      reftable_free(zeroed);
62
0
      return n;
63
0
    }
64
65
0
    w->pending_padding = 0;
66
0
    reftable_free(zeroed);
67
0
  }
68
69
0
  w->pending_padding = padding;
70
0
  n = w->write(w->write_arg, data, len);
71
0
  if (n < 0)
72
0
    return n;
73
0
  n += padding;
74
0
  return 0;
75
0
}
76
77
static void options_set_defaults(struct reftable_write_options *opts)
78
0
{
79
0
  if (opts->restart_interval == 0) {
80
0
    opts->restart_interval = 16;
81
0
  }
82
83
0
  if (opts->block_size == 0) {
84
0
    opts->block_size = DEFAULT_BLOCK_SIZE;
85
0
  }
86
0
}
87
88
static int writer_version(struct reftable_writer *w)
89
0
{
90
0
  return (w->hash_id == 0 || w->hash_id == REFTABLE_HASH_SHA1) ?
91
0
           1 :
92
0
           2;
93
0
}
94
95
static int writer_write_header(struct reftable_writer *w, uint8_t *dest)
96
0
{
97
0
  memcpy(dest, "REFT", 4);
98
99
0
  dest[4] = writer_version(w);
100
101
0
  reftable_put_be24(dest + 5, w->opts.block_size);
102
0
  reftable_put_be64(dest + 8, w->min_update_index);
103
0
  reftable_put_be64(dest + 16, w->max_update_index);
104
0
  if (writer_version(w) == 2) {
105
0
    uint32_t hash_id;
106
107
0
    switch (w->hash_id) {
108
0
    case REFTABLE_HASH_SHA1:
109
0
      hash_id = REFTABLE_FORMAT_ID_SHA1;
110
0
      break;
111
0
    case REFTABLE_HASH_SHA256:
112
0
      hash_id = REFTABLE_FORMAT_ID_SHA256;
113
0
      break;
114
0
    default:
115
0
      return -1;
116
0
    }
117
118
0
    reftable_put_be32(dest + 24, hash_id);
119
0
  }
120
121
0
  return header_size(writer_version(w));
122
0
}
123
124
static int writer_reinit_block_writer(struct reftable_writer *w, uint8_t typ)
125
0
{
126
0
  int block_start = 0, ret;
127
128
0
  if (w->next == 0)
129
0
    block_start = header_size(writer_version(w));
130
131
0
  reftable_buf_reset(&w->last_key);
132
0
  ret = block_writer_init(&w->block_writer_data, typ, w->block,
133
0
        w->opts.block_size, block_start,
134
0
        hash_size(w->hash_id));
135
0
  if (ret < 0)
136
0
    return ret;
137
138
0
  w->block_writer = &w->block_writer_data;
139
0
  w->block_writer->restart_interval = w->opts.restart_interval;
140
141
0
  return 0;
142
0
}
143
144
int reftable_writer_new(struct reftable_writer **out,
145
      ssize_t (*writer_func)(void *, const void *, size_t),
146
      int (*flush_func)(void *),
147
      void *writer_arg,
148
      enum reftable_hash hash_id,
149
      const struct reftable_write_options *_opts)
150
0
{
151
0
  struct reftable_write_options opts = {0};
152
0
  struct reftable_writer *wp;
153
0
  int err;
154
155
0
  if (_opts)
156
0
    opts = *_opts;
157
0
  options_set_defaults(&opts);
158
0
  if (opts.block_size >= (1 << 24))
159
0
    return REFTABLE_API_ERROR;
160
161
0
  if (!hash_id)
162
0
    hash_id = REFTABLE_HASH_SHA1;
163
164
0
  wp = reftable_calloc(1, sizeof(*wp));
165
0
  if (!wp)
166
0
    return REFTABLE_OUT_OF_MEMORY_ERROR;
167
168
0
  reftable_buf_init(&wp->block_writer_data.last_key);
169
0
  reftable_buf_init(&wp->last_key);
170
0
  reftable_buf_init(&wp->scratch);
171
0
  REFTABLE_CALLOC_ARRAY(wp->block, opts.block_size);
172
0
  if (!wp->block) {
173
0
    reftable_free(wp);
174
0
    return REFTABLE_OUT_OF_MEMORY_ERROR;
175
0
  }
176
0
  wp->write = writer_func;
177
0
  wp->write_arg = writer_arg;
178
0
  wp->opts = opts;
179
0
  wp->hash_id = hash_id;
180
0
  wp->flush = flush_func;
181
0
  err = writer_reinit_block_writer(wp, REFTABLE_BLOCK_TYPE_REF);
182
0
  if (err < 0) {
183
0
    reftable_free(wp->block);
184
0
    reftable_free(wp);
185
0
    return err;
186
0
  }
187
188
0
  *out = wp;
189
190
0
  return 0;
191
0
}
192
193
int reftable_writer_set_limits(struct reftable_writer *w, uint64_t min,
194
        uint64_t max)
195
0
{
196
  /*
197
    * Set the min/max update index limits for the reftable writer.
198
    * This must be called before adding any records, since:
199
    * - The 'next' field gets set after writing the first block.
200
    * - The 'last_key' field updates with each new record (but resets
201
    *   after sections).
202
    * Returns REFTABLE_API_ERROR if called after writing has begun.
203
   */
204
0
  if (w->next || w->last_key.len)
205
0
    return REFTABLE_API_ERROR;
206
207
0
  w->min_update_index = min;
208
0
  w->max_update_index = max;
209
210
0
  return 0;
211
0
}
212
213
static void writer_release(struct reftable_writer *w)
214
0
{
215
0
  if (w) {
216
0
    reftable_free(w->block);
217
0
    w->block = NULL;
218
0
    block_writer_release(&w->block_writer_data);
219
0
    w->block_writer = NULL;
220
0
    writer_clear_index(w);
221
0
    reftable_buf_release(&w->last_key);
222
0
    reftable_buf_release(&w->scratch);
223
0
  }
224
0
}
225
226
void reftable_writer_free(struct reftable_writer *w)
227
0
{
228
0
  writer_release(w);
229
0
  reftable_free(w);
230
0
}
231
232
struct obj_index_tree_node {
233
  struct reftable_buf hash;
234
  uint64_t *offsets;
235
  size_t offset_len;
236
  size_t offset_cap;
237
};
238
239
#define OBJ_INDEX_TREE_NODE_INIT    \
240
0
  {                           \
241
0
    .hash = REFTABLE_BUF_INIT \
242
0
  }
243
244
static int obj_index_tree_node_compare(const void *a, const void *b)
245
0
{
246
0
  return reftable_buf_cmp(&((const struct obj_index_tree_node *)a)->hash,
247
0
        &((const struct obj_index_tree_node *)b)->hash);
248
0
}
249
250
static int writer_index_hash(struct reftable_writer *w, struct reftable_buf *hash)
251
0
{
252
0
  uint64_t off = w->next;
253
0
  struct obj_index_tree_node want = { .hash = *hash };
254
0
  struct obj_index_tree_node *key;
255
0
  struct tree_node *node;
256
257
0
  node = tree_search(w->obj_index_tree, &want, &obj_index_tree_node_compare);
258
0
  if (!node) {
259
0
    struct obj_index_tree_node empty = OBJ_INDEX_TREE_NODE_INIT;
260
0
    int err;
261
262
0
    key = reftable_malloc(sizeof(*key));
263
0
    if (!key)
264
0
      return REFTABLE_OUT_OF_MEMORY_ERROR;
265
266
0
    *key = empty;
267
268
0
    reftable_buf_reset(&key->hash);
269
0
    err = reftable_buf_add(&key->hash, hash->buf, hash->len);
270
0
    if (err < 0) {
271
0
      reftable_free(key);
272
0
      return err;
273
0
    }
274
0
    tree_insert(&w->obj_index_tree, key,
275
0
          &obj_index_tree_node_compare);
276
0
  } else {
277
0
    key = node->key;
278
0
  }
279
280
0
  if (key->offset_len > 0 && key->offsets[key->offset_len - 1] == off)
281
0
    return 0;
282
283
0
  REFTABLE_ALLOC_GROW_OR_NULL(key->offsets, key->offset_len + 1,
284
0
            key->offset_cap);
285
0
  if (!key->offsets)
286
0
    return REFTABLE_OUT_OF_MEMORY_ERROR;
287
0
  key->offsets[key->offset_len++] = off;
288
289
0
  return 0;
290
0
}
291
292
static int writer_add_record(struct reftable_writer *w,
293
           struct reftable_record *rec)
294
0
{
295
0
  int err;
296
297
0
  err = reftable_record_key(rec, &w->scratch);
298
0
  if (err < 0)
299
0
    goto done;
300
301
0
  if (reftable_buf_cmp(&w->last_key, &w->scratch) >= 0) {
302
0
    err = REFTABLE_API_ERROR;
303
0
    goto done;
304
0
  }
305
306
0
  reftable_buf_reset(&w->last_key);
307
0
  err = reftable_buf_add(&w->last_key, w->scratch.buf, w->scratch.len);
308
0
  if (err < 0)
309
0
    goto done;
310
311
0
  if (!w->block_writer) {
312
0
    err = writer_reinit_block_writer(w, reftable_record_type(rec));
313
0
    if (err < 0)
314
0
      goto done;
315
0
  }
316
317
0
  if (block_writer_type(w->block_writer) != reftable_record_type(rec))
318
0
    return REFTABLE_API_ERROR;
319
320
  /*
321
   * Try to add the record to the writer. If this succeeds then we're
322
   * done. Otherwise the block writer may have hit the block size limit
323
   * and needs to be flushed.
324
   */
325
0
  err = block_writer_add(w->block_writer, rec);
326
0
  if (err == 0)
327
0
    goto done;
328
329
0
  if (err != REFTABLE_ENTRY_TOO_BIG_ERROR)
330
0
    goto done;
331
  /*
332
   * The current block is full, so we need to flush and reinitialize the
333
   * writer to start writing the next block.
334
   */
335
0
  err = writer_flush_block(w);
336
0
  if (err < 0)
337
0
    goto done;
338
0
  err = writer_reinit_block_writer(w, reftable_record_type(rec));
339
0
  if (err < 0)
340
0
    goto done;
341
342
  /*
343
   * Try to add the record to the writer again. If this still fails then
344
   * the record does not fit into the block size.
345
   */
346
0
  err = block_writer_add(w->block_writer, rec);
347
0
  if (err)
348
0
    goto done;
349
350
0
done:
351
0
  return err;
352
0
}
353
354
int reftable_writer_add_ref(struct reftable_writer *w,
355
          struct reftable_ref_record *ref)
356
0
{
357
0
  struct reftable_record rec = {
358
0
    .type = REFTABLE_BLOCK_TYPE_REF,
359
0
    .u = {
360
0
      .ref = *ref
361
0
    },
362
0
  };
363
0
  int err;
364
365
0
  if (!ref->refname ||
366
0
      ref->update_index < w->min_update_index ||
367
0
      ref->update_index > w->max_update_index)
368
0
    return REFTABLE_API_ERROR;
369
370
0
  rec.u.ref.update_index -= w->min_update_index;
371
372
0
  err = writer_add_record(w, &rec);
373
0
  if (err < 0)
374
0
    goto out;
375
376
0
  if (!w->opts.skip_index_objects && reftable_ref_record_val1(ref)) {
377
0
    reftable_buf_reset(&w->scratch);
378
0
    err = reftable_buf_add(&w->scratch, (char *)reftable_ref_record_val1(ref),
379
0
               hash_size(w->hash_id));
380
0
    if (err < 0)
381
0
      goto out;
382
383
0
    err = writer_index_hash(w, &w->scratch);
384
0
    if (err < 0)
385
0
      goto out;
386
0
  }
387
388
0
  if (!w->opts.skip_index_objects && reftable_ref_record_val2(ref)) {
389
0
    reftable_buf_reset(&w->scratch);
390
0
    err = reftable_buf_add(&w->scratch, reftable_ref_record_val2(ref),
391
0
               hash_size(w->hash_id));
392
0
    if (err < 0)
393
0
      goto out;
394
395
0
    err = writer_index_hash(w, &w->scratch);
396
0
    if (err < 0)
397
0
      goto out;
398
0
  }
399
400
0
  err = 0;
401
402
0
out:
403
0
  return err;
404
0
}
405
406
int reftable_writer_add_refs(struct reftable_writer *w,
407
           struct reftable_ref_record *refs, size_t n)
408
0
{
409
0
  int err = 0;
410
411
0
  if (n)
412
0
    qsort(refs, n, sizeof(*refs), reftable_ref_record_compare_name);
413
414
0
  for (size_t i = 0; err == 0 && i < n; i++)
415
0
    err = reftable_writer_add_ref(w, &refs[i]);
416
417
0
  return err;
418
0
}
419
420
static int reftable_writer_add_log_verbatim(struct reftable_writer *w,
421
              struct reftable_log_record *log)
422
0
{
423
0
  struct reftable_record rec = {
424
0
    .type = REFTABLE_BLOCK_TYPE_LOG,
425
0
    .u = {
426
0
      .log = *log,
427
0
    },
428
0
  };
429
0
  if (w->block_writer &&
430
0
      block_writer_type(w->block_writer) == REFTABLE_BLOCK_TYPE_REF) {
431
0
    int err = writer_finish_public_section(w);
432
0
    if (err < 0)
433
0
      return err;
434
0
  }
435
436
0
  w->next -= w->pending_padding;
437
0
  w->pending_padding = 0;
438
0
  return writer_add_record(w, &rec);
439
0
}
440
441
int reftable_writer_add_log(struct reftable_writer *w,
442
          struct reftable_log_record *log)
443
0
{
444
0
  char *input_log_message = NULL;
445
0
  struct reftable_buf cleaned_message = REFTABLE_BUF_INIT;
446
0
  int err = 0;
447
448
0
  if (log->value_type == REFTABLE_LOG_DELETION)
449
0
    return reftable_writer_add_log_verbatim(w, log);
450
451
  /*
452
   * Verify only the upper limit of the update_index. Each reflog entry
453
   * is tied to a specific update_index. Entries in the reflog can be
454
   * replaced by adding a new entry with the same update_index,
455
   * effectively canceling the old one.
456
   *
457
   * Consequently, reflog updates may include update_index values lower
458
   * than the writer's min_update_index.
459
   */
460
0
  if (log->update_index > w->max_update_index)
461
0
    return REFTABLE_API_ERROR;
462
463
0
  if (!log->refname)
464
0
    return REFTABLE_API_ERROR;
465
466
0
  input_log_message = log->value.update.message;
467
0
  if (!w->opts.exact_log_message && log->value.update.message) {
468
0
    err = reftable_buf_addstr(&cleaned_message, log->value.update.message);
469
0
    if (err < 0)
470
0
      goto done;
471
472
0
    while (cleaned_message.len &&
473
0
           cleaned_message.buf[cleaned_message.len - 1] == '\n') {
474
0
      err = reftable_buf_setlen(&cleaned_message,
475
0
              cleaned_message.len - 1);
476
0
      if (err < 0)
477
0
        goto done;
478
0
    }
479
0
    if (strchr(cleaned_message.buf, '\n')) {
480
      /* multiple lines not allowed. */
481
0
      err = REFTABLE_API_ERROR;
482
0
      goto done;
483
0
    }
484
485
0
    err = reftable_buf_addstr(&cleaned_message, "\n");
486
0
    if (err < 0)
487
0
      goto done;
488
489
0
    log->value.update.message = cleaned_message.buf;
490
0
  }
491
492
0
  err = reftable_writer_add_log_verbatim(w, log);
493
0
  log->value.update.message = input_log_message;
494
0
done:
495
0
  reftable_buf_release(&cleaned_message);
496
0
  return err;
497
0
}
498
499
int reftable_writer_add_logs(struct reftable_writer *w,
500
           struct reftable_log_record *logs, size_t n)
501
0
{
502
0
  int err = 0;
503
504
0
  if (n)
505
0
    qsort(logs, n, sizeof(*logs), reftable_log_record_compare_key);
506
507
0
  for (size_t i = 0; err == 0 && i < n; i++)
508
0
    err = reftable_writer_add_log(w, &logs[i]);
509
510
0
  return err;
511
0
}
512
513
static int writer_finish_section(struct reftable_writer *w)
514
0
{
515
0
  struct reftable_block_stats *bstats = NULL;
516
0
  uint8_t typ = block_writer_type(w->block_writer);
517
0
  uint64_t index_start = 0;
518
0
  int max_level = 0;
519
0
  size_t threshold = w->opts.unpadded ? 1 : 3;
520
0
  int before_blocks = w->stats.idx_stats.blocks;
521
0
  int err;
522
523
0
  err = writer_flush_block(w);
524
0
  if (err < 0)
525
0
    return err;
526
527
  /*
528
   * When the section we are about to index has a lot of blocks then the
529
   * index itself may span across multiple blocks, as well. This would
530
   * require a linear scan over index blocks only to find the desired
531
   * indexed block, which is inefficient. Instead, we write a multi-level
532
   * index where index records of level N+1 will refer to index blocks of
533
   * level N. This isn't constant time, either, but at least logarithmic.
534
   *
535
   * This loop handles writing this multi-level index. Note that we write
536
   * the lowest-level index pointing to the indexed blocks first. We then
537
   * continue writing additional index levels until the current level has
538
   * less blocks than the threshold so that the highest level will be at
539
   * the end of the index section.
540
   *
541
   * Readers are thus required to start reading the index section from
542
   * its end, which is why we set `index_start` to the beginning of the
543
   * last index section.
544
   */
545
0
  while (w->index_len > threshold) {
546
0
    struct reftable_index_record *idx = NULL;
547
0
    size_t i, idx_len;
548
549
0
    max_level++;
550
0
    index_start = w->next;
551
0
    err = writer_reinit_block_writer(w, REFTABLE_BLOCK_TYPE_INDEX);
552
0
    if (err < 0)
553
0
      return err;
554
555
0
    idx = w->index;
556
0
    idx_len = w->index_len;
557
558
0
    w->index = NULL;
559
0
    w->index_len = 0;
560
0
    w->index_cap = 0;
561
0
    for (i = 0; i < idx_len; i++) {
562
0
      struct reftable_record rec = {
563
0
        .type = REFTABLE_BLOCK_TYPE_INDEX,
564
0
        .u = {
565
0
          .idx = idx[i],
566
0
        },
567
0
      };
568
569
0
      err = writer_add_record(w, &rec);
570
0
      if (err < 0)
571
0
        return err;
572
0
    }
573
574
0
    err = writer_flush_block(w);
575
0
    if (err < 0)
576
0
      return err;
577
578
0
    for (i = 0; i < idx_len; i++)
579
0
      reftable_buf_release(&idx[i].last_key);
580
0
    reftable_free(idx);
581
0
  }
582
583
  /*
584
   * The index may still contain a number of index blocks lower than the
585
   * threshold. Clear it so that these entries don't leak into the next
586
   * index section.
587
   */
588
0
  writer_clear_index(w);
589
590
0
  bstats = writer_reftable_block_stats(w, typ);
591
0
  bstats->index_blocks = w->stats.idx_stats.blocks - before_blocks;
592
0
  bstats->index_offset = index_start;
593
0
  bstats->max_index_level = max_level;
594
595
  /* Reinit lastKey, as the next section can start with any key. */
596
0
  reftable_buf_reset(&w->last_key);
597
598
0
  return 0;
599
0
}
600
601
struct common_prefix_arg {
602
  struct reftable_buf *last;
603
  size_t max;
604
};
605
606
static void update_common(void *void_arg, void *key)
607
0
{
608
0
  struct common_prefix_arg *arg = void_arg;
609
0
  struct obj_index_tree_node *entry = key;
610
0
  if (arg->last) {
611
0
    size_t n = common_prefix_size(&entry->hash, arg->last);
612
0
    if (n > arg->max)
613
0
      arg->max = n;
614
0
  }
615
0
  arg->last = &entry->hash;
616
0
}
617
618
struct write_record_arg {
619
  struct reftable_writer *w;
620
  int err;
621
};
622
623
static void write_object_record(void *void_arg, void *key)
624
0
{
625
0
  struct write_record_arg *arg = void_arg;
626
0
  struct obj_index_tree_node *entry = key;
627
0
  struct reftable_record
628
0
    rec = { .type = REFTABLE_BLOCK_TYPE_OBJ,
629
0
      .u.obj = {
630
0
        .hash_prefix = (uint8_t *)entry->hash.buf,
631
0
        .hash_prefix_len = arg->w->stats.object_id_len,
632
0
        .offsets = entry->offsets,
633
0
        .offset_len = entry->offset_len,
634
0
      } };
635
0
  if (arg->err < 0)
636
0
    goto done;
637
638
  /*
639
   * Try to add the record to the writer. If this succeeds then we're
640
   * done. Otherwise the block writer may have hit the block size limit
641
   * and needs to be flushed.
642
   */
643
0
  arg->err = block_writer_add(arg->w->block_writer, &rec);
644
0
  if (arg->err == 0)
645
0
    goto done;
646
647
0
  if (arg->err != REFTABLE_ENTRY_TOO_BIG_ERROR)
648
0
    goto done;
649
650
  /*
651
   * The current block is full, so we need to flush and reinitialize the
652
   * writer to start writing the next block.
653
   */
654
0
  arg->err = writer_flush_block(arg->w);
655
0
  if (arg->err < 0)
656
0
    goto done;
657
658
0
  arg->err = writer_reinit_block_writer(arg->w, REFTABLE_BLOCK_TYPE_OBJ);
659
0
  if (arg->err < 0)
660
0
    goto done;
661
662
  /*
663
   * If this still fails then we may need to reset record's offset
664
   * length to reduce the data size to be written.
665
   */
666
0
  arg->err = block_writer_add(arg->w->block_writer, &rec);
667
0
  if (arg->err == 0)
668
0
    goto done;
669
670
0
  if (arg->err != REFTABLE_ENTRY_TOO_BIG_ERROR)
671
0
    goto done;
672
673
0
  rec.u.obj.offset_len = 0;
674
0
  arg->err = block_writer_add(arg->w->block_writer, &rec);
675
676
  /* Should be able to write into a fresh block. */
677
0
  assert(arg->err == 0);
678
679
0
done:;
680
0
}
681
682
static void object_record_free(void *void_arg REFTABLE_UNUSED, void *key)
683
0
{
684
0
  struct obj_index_tree_node *entry = key;
685
686
0
  REFTABLE_FREE_AND_NULL(entry->offsets);
687
0
  reftable_buf_release(&entry->hash);
688
0
  reftable_free(entry);
689
0
}
690
691
static int writer_dump_object_index(struct reftable_writer *w)
692
0
{
693
0
  struct write_record_arg closure = { .w = w };
694
0
  struct common_prefix_arg common = {
695
0
    .max = 1,   /* obj_id_len should be >= 2. */
696
0
  };
697
0
  int err;
698
699
0
  if (w->obj_index_tree)
700
0
    infix_walk(w->obj_index_tree, &update_common, &common);
701
0
  w->stats.object_id_len = common.max + 1;
702
703
0
  err = writer_reinit_block_writer(w, REFTABLE_BLOCK_TYPE_OBJ);
704
0
  if (err < 0)
705
0
    return err;
706
707
0
  if (w->obj_index_tree)
708
0
    infix_walk(w->obj_index_tree, &write_object_record, &closure);
709
710
0
  if (closure.err < 0)
711
0
    return closure.err;
712
0
  return writer_finish_section(w);
713
0
}
714
715
static int writer_finish_public_section(struct reftable_writer *w)
716
0
{
717
0
  uint8_t typ = 0;
718
0
  int err = 0;
719
720
0
  if (!w->block_writer)
721
0
    return 0;
722
723
0
  typ = block_writer_type(w->block_writer);
724
0
  err = writer_finish_section(w);
725
0
  if (err < 0)
726
0
    return err;
727
0
  if (typ == REFTABLE_BLOCK_TYPE_REF && !w->opts.skip_index_objects &&
728
0
      w->stats.ref_stats.index_blocks > 0) {
729
0
    err = writer_dump_object_index(w);
730
0
    if (err < 0)
731
0
      return err;
732
0
  }
733
734
0
  if (w->obj_index_tree) {
735
0
    infix_walk(w->obj_index_tree, &object_record_free, NULL);
736
0
    tree_free(w->obj_index_tree);
737
0
    w->obj_index_tree = NULL;
738
0
  }
739
740
0
  w->block_writer = NULL;
741
0
  return 0;
742
0
}
743
744
int reftable_writer_close(struct reftable_writer *w)
745
0
{
746
0
  uint8_t footer[72];
747
0
  uint8_t *p = footer;
748
0
  int err = writer_finish_public_section(w);
749
0
  int empty_table = w->next == 0;
750
0
  if (err != 0)
751
0
    goto done;
752
0
  w->pending_padding = 0;
753
0
  if (empty_table) {
754
    /* Empty tables need a header anyway. */
755
0
    uint8_t header[28];
756
0
    int n = writer_write_header(w, header);
757
0
    err = padded_write(w, header, n, 0);
758
0
    if (err < 0)
759
0
      goto done;
760
0
  }
761
762
0
  p += writer_write_header(w, footer);
763
0
  reftable_put_be64(p, w->stats.ref_stats.index_offset);
764
0
  p += 8;
765
0
  reftable_put_be64(p, (w->stats.obj_stats.offset) << 5 | w->stats.object_id_len);
766
0
  p += 8;
767
0
  reftable_put_be64(p, w->stats.obj_stats.index_offset);
768
0
  p += 8;
769
770
0
  reftable_put_be64(p, w->stats.log_stats.offset);
771
0
  p += 8;
772
0
  reftable_put_be64(p, w->stats.log_stats.index_offset);
773
0
  p += 8;
774
775
0
  reftable_put_be32(p, crc32(0, footer, p - footer));
776
0
  p += 4;
777
778
0
  err = w->flush(w->write_arg);
779
0
  if (err < 0) {
780
0
    err = REFTABLE_IO_ERROR;
781
0
    goto done;
782
0
  }
783
784
0
  err = padded_write(w, footer, footer_size(writer_version(w)), 0);
785
0
  if (err < 0)
786
0
    goto done;
787
788
0
  if (empty_table) {
789
0
    err = REFTABLE_EMPTY_TABLE_ERROR;
790
0
    goto done;
791
0
  }
792
793
0
done:
794
0
  writer_release(w);
795
0
  return err;
796
0
}
797
798
static void writer_clear_index(struct reftable_writer *w)
799
0
{
800
0
  for (size_t i = 0; w->index && i < w->index_len; i++)
801
0
    reftable_buf_release(&w->index[i].last_key);
802
0
  REFTABLE_FREE_AND_NULL(w->index);
803
0
  w->index_len = 0;
804
0
  w->index_cap = 0;
805
0
}
806
807
static int writer_flush_nonempty_block(struct reftable_writer *w)
808
0
{
809
0
  struct reftable_index_record index_record = {
810
0
    .last_key = REFTABLE_BUF_INIT,
811
0
  };
812
0
  uint8_t typ = block_writer_type(w->block_writer);
813
0
  struct reftable_block_stats *bstats;
814
0
  int raw_bytes, padding = 0, err;
815
0
  uint64_t block_typ_off;
816
817
  /*
818
   * Finish the current block. This will cause the block writer to emit
819
   * restart points and potentially compress records in case we are
820
   * writing a log block.
821
   *
822
   * Note that this is still happening in memory.
823
   */
824
0
  raw_bytes = block_writer_finish(w->block_writer);
825
0
  if (raw_bytes < 0)
826
0
    return raw_bytes;
827
828
  /*
829
   * By default, all records except for log records are padded to the
830
   * block size.
831
   */
832
0
  if (!w->opts.unpadded && typ != REFTABLE_BLOCK_TYPE_LOG)
833
0
    padding = w->opts.block_size - raw_bytes;
834
835
0
  bstats = writer_reftable_block_stats(w, typ);
836
0
  block_typ_off = (bstats->blocks == 0) ? w->next : 0;
837
0
  if (block_typ_off > 0)
838
0
    bstats->offset = block_typ_off;
839
0
  bstats->entries += w->block_writer->entries;
840
0
  bstats->restarts += w->block_writer->restart_len;
841
0
  bstats->blocks++;
842
0
  w->stats.blocks++;
843
844
  /*
845
   * If this is the first block we're writing to the table then we need
846
   * to also write the reftable header.
847
   */
848
0
  if (!w->next)
849
0
    writer_write_header(w, w->block);
850
851
0
  err = padded_write(w, w->block, raw_bytes, padding);
852
0
  if (err < 0)
853
0
    return err;
854
855
  /*
856
   * Add an index record for every block that we're writing. If we end up
857
   * having more than a threshold of index records we will end up writing
858
   * an index section in `writer_finish_section()`. Each index record
859
   * contains the last record key of the block it is indexing as well as
860
   * the offset of that block.
861
   *
862
   * Note that this also applies when flushing index blocks, in which
863
   * case we will end up with a multi-level index.
864
   */
865
0
  REFTABLE_ALLOC_GROW_OR_NULL(w->index, w->index_len + 1, w->index_cap);
866
0
  if (!w->index)
867
0
    return REFTABLE_OUT_OF_MEMORY_ERROR;
868
869
0
  index_record.offset = w->next;
870
0
  reftable_buf_reset(&index_record.last_key);
871
0
  err = reftable_buf_add(&index_record.last_key, w->block_writer->last_key.buf,
872
0
             w->block_writer->last_key.len);
873
0
  if (err < 0)
874
0
    return err;
875
0
  w->index[w->index_len] = index_record;
876
0
  w->index_len++;
877
878
0
  w->next += padding + raw_bytes;
879
0
  w->block_writer = NULL;
880
881
0
  return 0;
882
0
}
883
884
static int writer_flush_block(struct reftable_writer *w)
885
0
{
886
0
  if (!w->block_writer)
887
0
    return 0;
888
0
  if (w->block_writer->entries == 0)
889
0
    return 0;
890
0
  return writer_flush_nonempty_block(w);
891
0
}
892
893
const struct reftable_stats *reftable_writer_stats(struct reftable_writer *w)
894
0
{
895
0
  return &w->stats;
896
0
}