/src/git/odb/source-files.c

Source
#include "git-compat-util.h"
#include "abspath.h"
#include "blob.h"
#include "chdir-notify.h"
#include "config.h"
#include "gettext.h"
#include "hex.h"
#include "lockfile.h"
#include "object-file.h"
#include "odb.h"
#include "odb/source.h"
#include "odb/source-files.h"
#include "odb/source-loose.h"
#include "pack-objects.h"
#include "packfile.h"
#include "path.h"
#include "promisor-remote.h"
#include "quote.h"
#include "repack.h"
#include "run-command.h"
#include "strbuf.h"
#include "string-list.h"
#include "strmap.h"
#include "strvec.h"
#include "tree.h"
#include "write-or-die.h"

struct odb_files_dir *odb_files_dir_new(struct object_database *odb,
          const char *path, bool local)
{
  struct odb_files_dir *dir;

  CALLOC_ARRAY(dir, 1);
  dir->abspath = absolute_pathdup(path);
  dir->local = local;
  dir->loose = odb_source_loose_new(odb, path);
  dir->packed = odb_source_packed_new(odb, path, local);

  return dir;
}

void odb_files_dir_free(struct odb_files_dir *dir)
{
  if (!dir)
    return;
  odb_source_free(&dir->loose->base);
  odb_source_free(&dir->packed->base);
  free(dir->abspath);
  free(dir);
}

static void odb_source_files_reparent(const char *old_cwd,
              const char *new_cwd,
              void *cb_data)
{
  struct odb_source_files *files = cb_data;
  char *path = reparent_relative_path(old_cwd, new_cwd,
              files->base.path);

  free(files->base.path);
  files->base.path = path;
}

static void odb_source_files_free(struct odb_source *source)
{
  struct odb_source_files *files = odb_source_files_downcast(source);

  chdir_notify_unregister(odb_source_files_reparent, files);

  while (files->dirs) {
    struct odb_files_dir *next = files->dirs->next;
    odb_files_dir_free(files->dirs);
    files->dirs = next;
  }
  strset_clear(&files->dir_paths);
  stat_validity_clear(&files->alternates_validity);

  odb_source_release(&files->base);
  free(files);
}

static void odb_source_files_close(struct odb_source *source)
{
  struct odb_source_files *files = odb_source_files_downcast(source);

  for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
    odb_source_close(&dir->loose->base);
    odb_source_close(&dir->packed->base);
  }
}

static int odb_source_files_create_on_disk(struct odb_source *source,
             const struct odb_create_on_disk_options *opts)
{
  struct lock_file alternates_lock = LOCK_INIT;
  struct strbuf path = STRBUF_INIT;
  struct strset seen = STRSET_INIT;
  struct strbuf line = STRBUF_INIT;
  int ret;

  safe_create_dir(source->odb->repo, source->path, 1);

  strbuf_addf(&path, "%s/pack", source->path);
  safe_create_dir(source->odb->repo, path.buf, 1);

  strbuf_reset(&path);
  strbuf_addf(&path, "%s/info", source->path);
  safe_create_dir(source->odb->repo, path.buf, 1);

  if (opts->alternates && opts->alternates->nr) {
    FILE *alternates, *orig;

    strbuf_reset(&path);
    strbuf_addf(&path, "%s/info/alternates", source->path);

    repo_hold_lock_file_for_update(source->odb->repo, &alternates_lock,
                 path.buf, LOCK_DIE_ON_ERROR);

    alternates = fdopen_lock_file(&alternates_lock, "w");
    if (!alternates) {
      ret = error_errno(_("unable to fdopen alternates lockfile"));
      goto out;
    }

    /*
     * The alternates file may already exist, e.g. when it has been
     * seeded from a template directory. Read any preexisting
     * entries so that we don't end up writing duplicates.
     */
    orig = fopen(path.buf, "r");
    if (orig) {
      while (strbuf_getline(&line, orig) != EOF) {
        strset_add(&seen, line.buf);
        fprintf(alternates, "%s\n", line.buf);
      }

      if (ferror(orig)) {
        ret = error_errno(_("unable to read alternates file"));
        fclose(orig);
        goto out;
      }

      fclose(orig);
    } else if (errno != ENOENT) {
      ret = error_errno(_("unable to read alternates file"));
      goto out;
    }

    for (size_t i = 0; i < opts->alternates->nr; i++) {
      const char *alternate = opts->alternates->v[i];
      if (!strset_add(&seen, alternate))
        continue;
      fprintf(alternates, "%s\n", alternate);
    }

    if (ferror(alternates)) {
      ret = error_errno(_("unable to write alternates file"));
      goto out;
    }

    if (commit_lock_file(&alternates_lock)) {
      ret = error_errno(_("unable to commit alternates file"));
      goto out;
    }
  }

  /* Reprepare the object database to activate alternates. */
  odb_reprepare(source->odb);

  ret = 0;

out:
  rollback_lock_file(&alternates_lock);
  strbuf_release(&line);
  strbuf_release(&path);
  strset_clear(&seen);
  return ret;
}

/*
 * Return non-zero iff the path is usable as an alternate object directory.
 */
static bool odb_files_dir_is_usable(struct odb_source_files *files,
            const char *path)
{
  struct strbuf buf = STRBUF_INIT;
  bool usable = false;

  /*
   * Prevent the common mistake of listing the same thing twice,
   * or object directory itself.
   *
   * NEEDSWORK: we're using "core.ignoreCase" to deduplicate
   * alternates that _may_ be the same. This requires quite a bit
   * of boilerplate for dubious benefit:
   *
   *   - Duplicating alternates should really only lead to
   *     regressed performance.
   *
   *   - We don't properly resolve symlinks or mointpoints, so we
   *     may still end up duplicating alternates.
   *
   *   - The value may be lying, in which case we might
   *     deduplicate alternates that are in fact not mapping to the
   *     same directory.
   *
   * We should investigate whether we can remove this whole
   * mechanism outright.
   */
  if (strset_empty(&files->dir_paths)) {
    files->dir_paths_icase = 0;
    repo_config_get_bool(files->base.odb->repo, "core.ignorecase",
             &files->dir_paths_icase);

    strbuf_realpath(&buf, files->base.path, 1);
    if (files->dir_paths_icase)
      strbuf_tolower(&buf);
    strset_add(&files->dir_paths, buf.buf);

    for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
      strbuf_reset(&buf);
      strbuf_addstr(&buf, dir->abspath);
      if (files->dir_paths_icase)
        strbuf_tolower(&buf);
      strset_add(&files->dir_paths, buf.buf);
    }
  }

  strbuf_reset(&buf);
  strbuf_addstr(&buf, path);
  if (files->dir_paths_icase)
    strbuf_tolower(&buf);

  if (strset_contains(&files->dir_paths, buf.buf))
    goto out;

  /* Detect cases where alternate disappeared */
  if (!is_directory(path)) {
    error(_("object directory %s does not exist; "
      "check .git/objects/info/alternates"),
          path);
    goto out;
  }

  usable = true;

out:
  strbuf_release(&buf);
  return usable;
}

static void parse_alternates(const char *string,
           int sep,
           const char *relative_base,
           struct strvec *out)
{
  struct strbuf pathbuf = STRBUF_INIT;
  struct strbuf buf = STRBUF_INIT;

  if (!string || !*string)
    return;

  while (*string) {
    const char *end;

    strbuf_reset(&buf);
    strbuf_reset(&pathbuf);

    if (*string == '#') {
      /* comment; consume up to next separator */
      end = strchrnul(string, sep);
    } else if (*string == '"' && !unquote_c_style(&buf, string, &end)) {
      /*
       * quoted path; unquote_c_style has copied the
       * data for us and set "end". Broken quoting (e.g.,
       * an entry that doesn't end with a quote) falls
       * back to the unquoted case below.
       */
    } else {
      /* normal, unquoted path */
      end = strchrnul(string, sep);
      strbuf_add(&buf, string, end - string);
    }

    if (*end)
      end++;
    string = end;

    if (!buf.len)
      continue;

    if (!is_absolute_path(buf.buf) && relative_base) {
      strbuf_realpath(&pathbuf, relative_base, 1);
      strbuf_addch(&pathbuf, '/');
    }
    strbuf_addbuf(&pathbuf, &buf);

    strbuf_reset(&buf);
    if (!strbuf_realpath(&buf, pathbuf.buf, 0)) {
      error(_("unable to normalize alternate object path: %s"),
            pathbuf.buf);
      continue;
    }

    /*
     * The trailing slash after the directory name is given by
     * this function at the end. Remove duplicates.
     */
    while (buf.len && buf.buf[buf.len - 1] == '/')
      strbuf_setlen(&buf, buf.len - 1);

    strvec_push(out, buf.buf);
  }

  strbuf_release(&pathbuf);
  strbuf_release(&buf);
}

static void read_alternates(const char *object_dir, struct strvec *out,
          struct stat_validity *validity)
{
  struct strbuf buf = STRBUF_INIT;
  char *path;

  path = xstrfmt("%s/info/alternates", object_dir);
  if (!validity || !stat_validity_check(validity, path)) {
    bool successful = false;
    int fd;

    fd = open(path, O_RDONLY);
    if (fd < 0) {
      warn_on_fopen_errors(path);
    } else {
      if (validity)
        stat_validity_update(validity, fd);
      if (strbuf_read(&buf, fd, 1024) < 0) {
        warning_errno(_("unable to read alternates file '%s'"),
                path);
      } else {
        parse_alternates(buf.buf, '\n', object_dir, out);
        successful = true;
      }
    }

    if (validity && !successful)
      stat_validity_clear(validity);
    if (fd >= 0)
      close(fd);
  }

  strbuf_release(&buf);
  free(path);
}

static void add_alternate_recursively(struct odb_source_files *files,
              const char *path,
              int depth)
{
  struct odb_files_dir *alternate;
  struct strvec alternates = STRVEC_INIT;

  if (!odb_files_dir_is_usable(files, path))
    goto out;

  alternate = odb_files_dir_new(files->base.odb, path, false);

  /* add the alternate entry */
  *files->dirs_tail = alternate;
  files->dirs_tail = &(alternate->next);

  if (files->dir_paths_icase) {
    struct strbuf path = STRBUF_INIT;

    strbuf_addstr(&path, alternate->abspath);
    strbuf_tolower(&path);
    strset_add(&files->dir_paths, path.buf);

    strbuf_release(&path);
  } else {
    strset_add(&files->dir_paths, alternate->abspath);
  }

  /* recursively add alternates */
  read_alternates(alternate->abspath, &alternates, NULL);
  if (alternates.nr && depth + 1 > 5) {
    error(_("%s: ignoring alternate object stores, nesting too deep"),
          path);
  } else {
    for (size_t i = 0; i < alternates.nr; i++)
      add_alternate_recursively(files, alternates.v[i], depth + 1);
  }

 out:
  strvec_clear(&alternates);
}

static void prepare_alternates(struct odb_source_files *files,
             const char *alternate_db)
{
  struct strvec alternates = STRVEC_INIT;

  parse_alternates(alternate_db, PATH_SEP, NULL, &alternates);
  read_alternates(files->base.path, &alternates,
      &files->alternates_validity);

  for (size_t i = 0; i < alternates.nr; i++)
    add_alternate_recursively(files, alternates.v[i], 0);

  strvec_clear(&alternates);
}

static void odb_source_files_prepare(struct odb_source *source,
             enum odb_prepare_flags flags)
{
  struct odb_source_files *files = odb_source_files_downcast(source);

  /*
   * Reprepare alternates, in case the alternates file was modified
   * during the course of this process. This only _adds_ directories to
   * the linked list, so existing directories will continue to exist
   * for the lifetime of the process.
   */
  if (flags & ODB_PREPARE_FLUSH_CACHES)
    prepare_alternates(files, NULL);

  for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
    odb_source_prepare(&dir->loose->base, flags);
    odb_source_prepare(&dir->packed->base, flags);
  }
}

static enum odb_read_status odb_source_files_read_object_info(struct odb_source *source,
                    const struct object_id *oid,
                    struct object_info *oi,
                    enum object_info_flags flags,
                    struct strbuf *errmsg)
{
  struct odb_source_files *files = odb_source_files_downcast(source);
  struct odb_files_dir **tail = NULL;
  enum odb_read_status status = ODB_READ_NOT_FOUND;
  enum odb_read_status ret;
  bool retry = false;

retry:
  /*
   * Reading an object may fail even though the object exists, for
   * example because it is corrupt. Report this failure to the caller in
   * case none of the directories was able to read the object, and
   * prefer the first such error in case multiple reads have failed.
   */
  for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
    if (retry)
      odb_source_prepare(&dir->packed->base, ODB_PREPARE_FLUSH_CACHES);

    ret = odb_source_read_object_info(&dir->packed->base, oid, oi, flags,
              status == ODB_READ_NOT_FOUND ? errmsg : NULL);
    if (!ret)
      return 0;
    if (ret != ODB_READ_NOT_FOUND && status == ODB_READ_NOT_FOUND)
      status = ret;
  }

  /*
   * When we have reprepared alternates then we only re-scan directories
   * that were newly added.
   */
  for (struct odb_files_dir *dir = tail ? *tail : files->dirs; dir; dir = dir->next) {
    ret = odb_source_read_object_info(&dir->loose->base, oid, oi, flags,
              status == ODB_READ_NOT_FOUND ? errmsg : NULL);
    if (!ret)
      return 0;
    if (ret != ODB_READ_NOT_FOUND && status == ODB_READ_NOT_FOUND)
      status = ret;
  }

  if (!retry && !(flags & OBJECT_INFO_QUICK)) {
    tail = files->dirs_tail;
    prepare_alternates(files, NULL);
    retry = true;
    goto retry;
  }

  return status;
}

static int odb_source_files_read_object_stream(struct odb_stream **out,
                 struct odb_source *source,
                 const struct object_id *oid)
{
  struct odb_source_files *files = odb_source_files_downcast(source);

  for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next)
    if (!odb_source_read_object_stream(out, &dir->packed->base, oid))
      return 0;
  for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next)
    if (!odb_source_read_object_stream(out, &dir->loose->base, oid))
      return 0;

  return -1;
}

static int odb_source_files_for_each_object(struct odb_source *source,
              const struct object_info *request,
              odb_for_each_object_cb cb,
              void *cb_data,
              const struct odb_for_each_object_options *opts)
{
  struct odb_source_files *files = odb_source_files_downcast(source);
  int ret;

  for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
    if (opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY && !dir->local)
      continue;

    if (!(opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY)) {
      ret = odb_source_for_each_object(&dir->loose->base, request, cb, cb_data, opts);
      if (ret)
        return ret;
    }

    ret = odb_source_for_each_object(&dir->packed->base, request, cb, cb_data, opts);
    if (ret)
      return ret;
  }

  return 0;
}

static int odb_source_files_count_objects(struct odb_source *source,
            enum odb_count_objects_flags flags,
            unsigned long *out)
{
  struct odb_source_files *files = odb_source_files_downcast(source);
  unsigned long count = 0;
  int ret;

  for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
    unsigned long dir_count;

    ret = odb_source_count_objects(&dir->packed->base, flags, &dir_count);
    if (ret < 0)
      goto out;
    count += dir_count;

    if (!(flags & ODB_COUNT_OBJECTS_APPROXIMATE)) {
      ret = odb_source_count_objects(&dir->loose->base, flags, &dir_count);
      if (ret < 0)
        goto out;
      count += dir_count;
    }
  }

  *out = count;
  ret = 0;

out:
  return ret;
}

static int odb_source_files_find_abbrev_len(struct odb_source *source,
              const struct object_id *oid,
              unsigned min_len,
              unsigned *out)
{
  struct odb_source_files *files = odb_source_files_downcast(source);
  unsigned len = min_len;
  int ret = 0;

  for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
    ret = odb_source_find_abbrev_len(&dir->packed->base, oid, len, &len);
    if (ret < 0)
      goto out;

    ret = odb_source_find_abbrev_len(&dir->loose->base, oid, len, &len);
    if (ret < 0)
      goto out;
  }

  *out = len;
  ret = 0;

out:
  return ret;
}

static int odb_source_files_freshen_object(struct odb_source *source,
             const struct object_id *oid,
             const time_t *mtime)
{
  struct odb_source_files *files = odb_source_files_downcast(source);

  for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next)
    if (odb_source_freshen_object(&dir->packed->base, oid, mtime) ||
        odb_source_freshen_object(&dir->loose->base, oid, mtime))
      return 1;

  return 0;
}

static int odb_source_files_write_object(struct odb_source *source,
           const void *buf, size_t len,
           enum object_type type,
           const struct object_id *oid,
           const struct object_id *compat_oid,
           const time_t *mtime,
           enum odb_write_object_flags flags)
{
  struct odb_source_files *files = odb_source_files_downcast(source);
  return odb_source_write_object(&files->dirs->loose->base, buf, len, type,
               oid, compat_oid, mtime, flags);
}

static int odb_source_files_write_object_stream(struct odb_source *source,
            struct odb_stream *stream,
            struct object_id *oid)
{
  struct odb_source_files *files = odb_source_files_downcast(source);
  return odb_source_write_object_stream(&files->dirs->loose->base, stream, oid);
}

static int odb_source_files_begin_transaction(struct odb_source *source,
                struct odb_transaction **out,
                enum odb_transaction_flags flags)
{
  return odb_transaction_files_begin(source, out, flags);
}

static int too_many_loose_objects(struct odb_source_files *files, int limit)
{
  unsigned long loose_count;

  if (limit <= 0)
    return 0;

  if (odb_source_count_objects(&files->dirs->loose->base, ODB_COUNT_OBJECTS_APPROXIMATE,
             &loose_count) < 0)
    return 0;

  /*
   * This is weird, but stems from legacy behaviour: the GC auto
   * threshold was always essentially interpreted as if it was rounded up
   * to the next multiple 256 of, so we retain this behaviour for now.
   */
  return loose_count > (DIV_ROUND_UP(((unsigned long) limit), 256) * 256);
}

static struct packed_git *find_base_packs(struct odb_source_files *files,
            struct string_list *packs,
            unsigned long limit)
{
  struct packfile_list_entry *e;
  struct packed_git *base = NULL;

  for (e = packfile_store_get_packs(files->dirs->packed); e; e = e->next) {
    if (e->pack->is_cruft)
      continue;
    if (limit) {
      if ((uintmax_t) e->pack->pack_size >= limit)
        string_list_append(packs, e->pack->pack_name);
    } else if (!base || base->pack_size < e->pack->pack_size) {
      base = e->pack;
    }
  }

  if (base)
    string_list_append(packs, base->pack_name);

  return base;
}

static int too_many_packs(struct odb_source_files *files, int gc_auto_pack_limit)
{
  struct packfile_list_entry *e;
  int cnt = 0;

  if (gc_auto_pack_limit <= 0)
    return 0;

  for (e = packfile_store_get_packs(files->dirs->packed); e; e = e->next) {
    if (e->pack->pack_keep)
      continue;
    /*
     * Perhaps check the size of the pack and count only
     * very small ones here?
     */
    cnt++;
  }
  return gc_auto_pack_limit < cnt;
}

static uint64_t total_ram(void)
{
#if defined(HAVE_SYSINFO)
  struct sysinfo si;

  if (!sysinfo(&si)) {
    uint64_t total = si.totalram;

    if (si.mem_unit > 1)
      total *= (uint64_t)si.mem_unit;
    return total;
  }
#elif defined(HAVE_BSD_SYSCTL) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM) || defined(HW_PHYSMEM64))
  uint64_t physical_memory;
  int mib[2];
  size_t length;

  mib[0] = CTL_HW;
# if defined(HW_MEMSIZE)
  mib[1] = HW_MEMSIZE;
# elif defined(HW_PHYSMEM64)
  mib[1] = HW_PHYSMEM64;
# else
  mib[1] = HW_PHYSMEM;
# endif
  length = sizeof(physical_memory);
  if (!sysctl(mib, 2, &physical_memory, &length, NULL, 0)) {
    if (length == 4) {
      uint32_t mem;

      if (!sysctl(mib, 2, &mem, &length, NULL, 0))
        physical_memory = mem;
    }
    return physical_memory;
  }
#elif defined(GIT_WINDOWS_NATIVE)
  MEMORYSTATUSEX memInfo;

  memInfo.dwLength = sizeof(MEMORYSTATUSEX);
  if (GlobalMemoryStatusEx(&memInfo))
    return memInfo.ullTotalPhys;
#endif
  return 0;
}

static uint64_t estimate_repack_memory(struct odb_source_files *files,
               struct packed_git *pack)
{
  unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE;
  unsigned long delta_base_cache_limit = DEFAULT_DELTA_BASE_CACHE_LIMIT;
  unsigned long nr_objects;
  size_t os_cache, heap;

  if (odb_source_count_objects(&files->base, ODB_COUNT_OBJECTS_APPROXIMATE,
             &nr_objects) < 0)
    return 0;

  if (!pack || !nr_objects)
    return 0;

  repo_config_get_ulong(files->base.odb->repo, "pack.deltacachesize",
            &max_delta_cache_size);
  repo_config_get_ulong(files->base.odb->repo, "core.deltabasecachelimit",
            &delta_base_cache_limit);

  /*
   * First we have to scan through at least one pack.
   * Assume enough room in OS file cache to keep the entire pack
   * or we may accidentally evict data of other processes from
   * the cache.
   */
  os_cache = pack->pack_size + pack->index_size;
  /* then pack-objects needs lots more for book keeping */
  heap = sizeof(struct object_entry) * nr_objects;
  /*
   * internal rev-list --all --objects takes up some memory too,
   * let's say half of it is for blobs
   */
  heap += sizeof(struct blob) * nr_objects / 2;
  /*
   * and the other half is for trees (commits and tags are
   * usually insignificant)
   */
  heap += sizeof(struct tree) * nr_objects / 2;
  /* and then obj_hash[], underestimated in fact */
  heap += sizeof(struct object *) * nr_objects;
  /* revindex is used also */
  heap += (sizeof(off_t) + sizeof(uint32_t)) * nr_objects;
  /*
   * read_sha1_file() (either at delta calculation phase, or
   * writing phase) also fills up the delta base cache
   */
  heap += delta_base_cache_limit;
  /* and of course pack-objects has its own delta cache */
  heap += max_delta_cache_size;

  return os_cache + heap;
}

static int keep_one_pack(struct string_list_item *item, void *data)
{
  struct strvec *args = data;
  strvec_pushf(args, "--keep-pack=%s", basename(item->string));
  return 0;
}

static void add_repack_all_option(struct repository *repo,
          const struct odb_optimize_options *opts,
          struct string_list *keep_pack,
          struct strvec *args)
{
  char *repack_filter = NULL;
  char *repack_filter_to = NULL;

  repo_config_get_string(repo, "gc.repackfilter", &repack_filter);
  repo_config_get_string(repo, "gc.repackfilterto", &repack_filter_to);

  if (opts->prune_expire && !strcmp(opts->prune_expire, "now") &&
      !(opts->cruft_packs && opts->expire_to))
    strvec_push(args, "-a");
  else if (opts->cruft_packs) {
    strvec_push(args, "--cruft");
    if (opts->prune_expire)
      strvec_pushf(args, "--cruft-expiration=%s", opts->prune_expire);
    if (opts->max_cruft_size)
      strvec_pushf(args, "--max-cruft-size=%lu",
             opts->max_cruft_size);
    if (opts->expire_to)
      strvec_pushf(args, "--expire-to=%s", opts->expire_to);
  } else {
    strvec_push(args, "-A");
    if (opts->prune_expire)
      strvec_pushf(args, "--unpack-unreachable=%s", opts->prune_expire);
  }

  if (keep_pack)
    for_each_string_list(keep_pack, keep_one_pack, args);

  if (repack_filter && *repack_filter)
    strvec_pushf(args, "--filter=%s", repack_filter);
  if (repack_filter_to && *repack_filter_to)
    strvec_pushf(args, "--filter-to=%s", repack_filter_to);

  free(repack_filter);
  free(repack_filter_to);
}

static void add_repack_incremental_option(struct strvec *args)
{
  strvec_push(args, "--no-write-bitmap-index");
}

bool odb_source_files_optimize_required(struct odb_source *source,
          const struct odb_optimize_options *opts)
{
  struct odb_source_files *files = odb_source_files_downcast(source);
  struct repository *repo = source->odb->repo;

  switch (opts->strategy) {
  case ODB_OPTIMIZE_INCREMENTAL: {
    int gc_auto_threshold = 6700;
    int gc_auto_pack_limit = 50;

    repo_config_get_int(repo, "gc.auto", &gc_auto_threshold);
    repo_config_get_int(repo, "gc.autopacklimit", &gc_auto_pack_limit);

    /*
     * Setting gc.auto to 0 or negative can disable the
     * automatic gc.
     */
    if (gc_auto_threshold <= 0)
      return false;
    if (!too_many_packs(files, gc_auto_pack_limit) &&
        !too_many_loose_objects(files, gc_auto_threshold))
      return false;

    return true;
  }
  case ODB_OPTIMIZE_GEOMETRIC: {
    struct pack_geometry geometry = {
      .split_factor = 2,
    };
    struct pack_objects_args po_args = {
      .local = 1,
    };
    struct existing_packs existing_packs = EXISTING_PACKS_INIT;
    struct string_list kept_packs = STRING_LIST_INIT_DUP;
    int auto_value = 6700;
    bool ret;

    repo_config_get_int(repo, "maintenance.geometric-repack.auto",
            &auto_value);
    if (!auto_value)
      return false;
    if (auto_value < 0)
      return true;

    repo_config_get_int(repo, "maintenance.geometric-repack.splitFactor",
            &geometry.split_factor);

    existing_packs.repo = repo;
    existing_packs_collect(&existing_packs, &kept_packs);
    pack_geometry_init(&geometry, &existing_packs, &po_args);
    pack_geometry_split(&geometry);

    /*
     * When we'd merge at least two packs with one another we always
     * perform the repack.
     */
    if (geometry.split) {
      ret = true;
      goto out;
    }

    /*
     * Otherwise, we estimate the number of loose objects to determine
     * whether we want to create a new packfile or not.
     */
    if (too_many_loose_objects(files, auto_value)) {
      ret = true;
      goto out;
    }

    ret = false;

  out:
    existing_packs_release(&existing_packs);
    pack_geometry_release(&geometry);
    return ret;
  }
  default:
    BUG("unknown maintenance strategy '%d'", opts->strategy);
  }
}

int odb_source_files_optimize(struct odb_source *source,
            const struct odb_optimize_options *opts)
{
  struct odb_source_files *files = odb_source_files_downcast(source);
  struct repository *repo = source->odb->repo;
  struct child_process repack_cmd = CHILD_PROCESS_INIT;
  unsigned long big_pack_threshold = 0;
  int gc_auto_threshold = 6700;
  int gc_auto_pack_limit = 50;
  int ret;

  repo_config_get_int(repo, "gc.auto", &gc_auto_threshold);
  repo_config_get_int(repo, "gc.autopacklimit", &gc_auto_pack_limit);
  repo_config_get_ulong(repo, "gc.bigpackthreshold", &big_pack_threshold);

  if (repo->repository_format_precious_objects)
    return 0;

  repack_cmd.git_cmd = 1;
  repack_cmd.odb_to_close = repo->objects;

  strvec_pushl(&repack_cmd.args, "repack", "-d", "-l", NULL);
  if (opts->flags & ODB_OPTIMIZE_NO_REUSE_DELTAS)
    strvec_push(&repack_cmd.args, "-f");
  if (opts->depth > 0)
    strvec_pushf(&repack_cmd.args, "--depth=%d", opts->depth);
  if (opts->window > 0)
    strvec_pushf(&repack_cmd.args, "--window=%d", opts->window);
  if (!(opts->flags & ODB_OPTIMIZE_VERBOSE))
    strvec_push(&repack_cmd.args, "-q");

  /*
   * There's three cases we need to consider:
   *
   *   - If we're invoked without `--auto` we'll need to perform a full
   *     repack.
   *
   *   - If we're invoked with `--auto` and there's too many packs, then
   *     we perform a full repack, as well.
   *
   *   - Otherwise we perform an incremental repack.
   */
  switch (opts->strategy) {
  case ODB_OPTIMIZE_INCREMENTAL:
    if (!(opts->flags & ODB_OPTIMIZE_AUTO)) {
      struct string_list keep_pack = STRING_LIST_INIT_NODUP;

      if (opts->keep_largest_pack != -1) {
        if (opts->keep_largest_pack)
          find_base_packs(files, &keep_pack, 0);
      } else if (big_pack_threshold) {
        find_base_packs(files, &keep_pack, big_pack_threshold);
      }

      add_repack_all_option(repo, opts, &keep_pack, &repack_cmd.args);
      string_list_clear(&keep_pack, 0);
    } else {
      if (too_many_packs(files, gc_auto_pack_limit)) {
        struct string_list keep_pack = STRING_LIST_INIT_NODUP;

        if (big_pack_threshold) {
          find_base_packs(files, &keep_pack, big_pack_threshold);
          if (keep_pack.nr >= (unsigned long) gc_auto_pack_limit) {
            string_list_clear(&keep_pack, 0);
            find_base_packs(files, &keep_pack, 0);
          }
        } else {
          struct packed_git *p = find_base_packs(files, &keep_pack, 0);
          uint64_t mem_have, mem_want;

          mem_have = total_ram();
          mem_want = estimate_repack_memory(files, p);

          /*
           * Only allow 1/2 of memory for pack-objects, leave
           * the rest for the OS and other processes in the
           * system.
           */
          if (!mem_have || mem_want < mem_have / 2)
            string_list_clear(&keep_pack, 0);
        }

        add_repack_all_option(repo, opts, &keep_pack, &repack_cmd.args);
        string_list_clear(&keep_pack, 0);
      } else {
        add_repack_incremental_option(&repack_cmd.args);
      }
    }

    break;
  case ODB_OPTIMIZE_GEOMETRIC: {
    struct pack_geometry geometry = {
      .split_factor = 2,
    };
    struct pack_objects_args po_args = {
      .local = 1,
    };
    struct existing_packs existing_packs = EXISTING_PACKS_INIT;
    struct string_list kept_packs = STRING_LIST_INIT_DUP;

    repo_config_get_int(repo, "maintenance.geometric-repack.splitFactor",
            &geometry.split_factor);

    existing_packs.repo = repo;
    existing_packs_collect(&existing_packs, &kept_packs);
    pack_geometry_init(&geometry, &existing_packs, &po_args);
    pack_geometry_split(&geometry);

    if (geometry.split < geometry.pack_nr) {
      strvec_pushf(&repack_cmd.args, "--geometric=%d",
             geometry.split_factor);
    } else {
      add_repack_all_option(repo, opts, NULL, &repack_cmd.args);
    }
    if (repo->settings.core_multi_pack_index)
      strvec_push(&repack_cmd.args, "--write-midx");

    existing_packs_release(&existing_packs);
    pack_geometry_release(&geometry);
    break;
  }
  default:
    die("unknown maintenance strategy '%d'", opts->strategy);
  }

  if (run_command(&repack_cmd)) {
    ret = error("failed to run %s", repack_cmd.args.v[0]);
    goto out;
  }

  /* Geometric repacking uses cruft packs, so we don't have to prune separately. */
  if (opts->strategy != ODB_OPTIMIZE_GEOMETRIC && opts->prune_expire) {
    struct child_process prune_cmd = CHILD_PROCESS_INIT;

    strvec_pushl(&prune_cmd.args, "prune", "--expire", NULL);
    /* run `git prune` even if using cruft packs */
    strvec_push(&prune_cmd.args, opts->prune_expire);
    if (!(opts->flags & ODB_OPTIMIZE_VERBOSE))
      strvec_push(&prune_cmd.args, "--no-progress");
    if (repo_has_promisor_remote(repo))
      strvec_push(&prune_cmd.args,
            "--exclude-promisor-objects");
    prune_cmd.git_cmd = 1;

    if (run_command(&prune_cmd)) {
      ret = error("failed to run %s", prune_cmd.args.v[0]);
      goto out;
    }
  }

  if (opts->flags & ODB_OPTIMIZE_AUTO && too_many_loose_objects(files, gc_auto_threshold))
    warning(_("There are too many unreachable loose objects; "
      "run 'git prune' to remove them."));

  ret = 0;

out:
  return ret;
}

struct odb_pack_generator_files {
  struct odb_pack_generator base;
  struct child_process cp;
};

static int odb_pack_generator_files_finish(struct odb_pack_generator *_generator)
{
  struct odb_pack_generator_files *generator =
    (struct odb_pack_generator_files *)_generator;
  int ret;

  ret = finish_command(&generator->cp);
  free(generator);

  if (ret) {
    /*
     * On failure, pack-objects is expected to have written a
     * useful error message to its standard error stream already.
     * Death by signal is worth mentioning, though, with the
     * exception of SIGPIPE: that is a normal occurrence when the
     * consumer of the pack hangs up.
     */
    if (ret > 128 && ret - 128 == SIGPIPE)
      return -1;
    if (ret > 128)
      error(_("pack-objects died of signal %d"), ret - 128);
    return -1;
  }

  return 0;
}

static int odb_source_files_generate_pack(struct odb_source *source UNUSED,
            struct odb_pack_generator **out,
            const struct odb_generate_pack_options *opts)
{
  struct odb_pack_generator_files *generator;
  struct child_process *cp;
  FILE *in;

  CALLOC_ARRAY(generator, 1);
  child_process_init(&generator->cp);
  cp = &generator->cp;

  /*
   * The hook is expected to spawn "$hook git pack-objects <args...>"
   * and to behave like git-pack-objects(1) would have. This can for
   * example be used to serve precomputed packfiles.
   */
  if (opts->pack_objects_hook) {
    strvec_push(&cp->args, opts->pack_objects_hook);
    strvec_push(&cp->args, "git");
    cp->use_shell = 1;
  } else {
    cp->git_cmd = 1;
  }

  /*
   * The caller-provided shallow boundary overrides any shallow state
   * that the repository itself may have, so the shallow file needs to
   * be neutralized.
   */
  if (opts->shallows.nr) {
    strvec_push(&cp->args, "--shallow-file");
    strvec_push(&cp->args, "");
  }
  strvec_push(&cp->args, "pack-objects");
  strvec_push(&cp->args, "--revs");
  strvec_push(&cp->args, "--stdout");
  if (opts->thin)
    strvec_push(&cp->args, "--thin");
  if (opts->shallow)
    strvec_push(&cp->args, "--shallow");
  if (opts->ofs_delta)
    strvec_push(&cp->args, "--delta-base-offset");
  if (opts->include_tag)
    strvec_push(&cp->args, "--include-tag");
  if (opts->missing_allow_promisor)
    strvec_push(&cp->args, "--missing=allow-promisor");
  if (opts->disable_bitmaps)
    strvec_push(&cp->args, "--no-use-bitmap-index");
  switch (opts->progress) {
  case ODB_GENERATE_PACK_PROGRESS_NONE:
    strvec_push(&cp->args, "--quiet");
    break;
  case ODB_GENERATE_PACK_PROGRESS_STANDARD:
    strvec_push(&cp->args, "--progress");
    break;
  case ODB_GENERATE_PACK_PROGRESS_VERBOSE:
    strvec_push(&cp->args, "--all-progress");
    break;
  default:
    BUG("unknown progress option %d", opts->progress);
  }
  if (opts->filter_spec)
    strvec_pushf(&cp->args, "--filter=%s", opts->filter_spec);
  if (opts->uri_protocols)
    for (size_t i = 0; i < opts->uri_protocols->nr; i++)
      strvec_pushf(&cp->args, "--uri-protocol=%s",
             opts->uri_protocols->items[i].string);

  cp->in = -1;
  cp->out = opts->pack_fd;
  cp->err = opts->progress_fd;
  cp->clean_on_exit = 1;

  if (start_command(cp)) {
    free(generator);
    return error(_("could not spawn pack-objects"));
  }

  /*
   * Feed the objects to pack-objects. This is safe to do synchronously
   * because pack-objects consumes all of its standard input before it
   * starts to generate the pack.
   */
  in = xfdopen(cp->in, "w");
  for (size_t i = 0; i < opts->shallows.nr; i++)
    fprintf(in, "--shallow %s\n", oid_to_hex(&opts->shallows.oid[i]));
  for (size_t i = 0; i < opts->wants.nr; i++)
    fprintf(in, "%s\n", oid_to_hex(&opts->wants.oid[i]));
  fprintf(in, "--not\n");
  for (size_t i = 0; i < opts->haves.nr; i++)
    fprintf(in, "%s\n", oid_to_hex(&opts->haves.oid[i]));
  fprintf(in, "\n");
  fflush(in);
  if (ferror(in)) {
    error(_("error writing to pack-objects"));
    fclose(in);
    if (opts->pack_fd < 0)
      close(cp->out);
    if (opts->progress_fd < 0)
      close(cp->err);
    finish_command(cp);
    free(generator);
    return -1;
  }
  fclose(in);

  generator->base.out = opts->pack_fd < 0 ? cp->out : -1;
  generator->base.err = opts->progress_fd < 0 ? cp->err : -1;
  generator->base.finish = odb_pack_generator_files_finish;

  *out = &generator->base;
  return 0;
}

static int odb_source_files_fsck(struct odb_source *source,
         struct odb_fsck_options *opts)
{
  struct odb_source_files *files = odb_source_files_downcast(source);
  int ret = 0;

  for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) {
    if (!(opts->flags & ODB_FSCK_FULL) && !dir->local)
      continue;

    ret |= odb_source_fsck(&dir->loose->base, opts);
    ret |= odb_source_fsck(&dir->packed->base, opts);
  }

  return ret;
}

struct odb_files_dir *odb_source_files_find_dir(struct object_database *odb, const char *obj_dir)
{
  struct odb_source_files *files = odb_source_files_downcast(odb->source);
  char *obj_dir_real = real_pathdup(obj_dir, 1);
  struct strbuf odb_path_real = STRBUF_INIT;
  struct odb_files_dir *dir;

  for (dir = files->dirs; dir; dir = dir->next) {
    strbuf_realpath(&odb_path_real, dir->abspath, 1);
    if (!strcmp(obj_dir_real, odb_path_real.buf))
      break;
  }

  free(obj_dir_real);
  strbuf_release(&odb_path_real);
  return dir;
}

struct odb_source_files *odb_source_files_new(struct object_database *odb,
                const char *payload,
                enum odb_new_flags flags)
{
  struct odb_source_files *files;
  char *object_dir = NULL;
  char *alternates = NULL;

  if (flags & ODB_NEW_HONOR_ENV) {
    object_dir = xstrdup_or_null(getenv(DB_ENVIRONMENT));
    alternates = xstrdup_or_null(getenv(ALTERNATE_DB_ENVIRONMENT));
  }
  if (!object_dir && payload) {
    if (is_absolute_path(payload))
      object_dir = xstrdup(payload);
    else
      object_dir = xstrfmt("%s/%s", odb->repo->commondir, payload);
  }
  if (!object_dir)
    object_dir = xstrfmt("%s/objects", odb->repo->commondir);

  CALLOC_ARRAY(files, 1);
  odb_source_init(&files->base, odb, ODB_SOURCE_FILES, object_dir);

  files->dirs = odb_files_dir_new(odb, object_dir, true);
  files->dirs_tail = &files->dirs->next;
  strset_init(&files->dir_paths);

  files->base.free = odb_source_files_free;
  files->base.close = odb_source_files_close;
  files->base.create_on_disk = odb_source_files_create_on_disk;
  files->base.prepare = odb_source_files_prepare;
  files->base.fsck = odb_source_files_fsck;
  files->base.read_object_info = odb_source_files_read_object_info;
  files->base.read_object_stream = odb_source_files_read_object_stream;
  files->base.for_each_object = odb_source_files_for_each_object;
  files->base.count_objects = odb_source_files_count_objects;
  files->base.find_abbrev_len = odb_source_files_find_abbrev_len;
  files->base.freshen_object = odb_source_files_freshen_object;
  files->base.write_object = odb_source_files_write_object;
  files->base.write_object_stream = odb_source_files_write_object_stream;
  files->base.begin_transaction = odb_source_files_begin_transaction;
  files->base.optimize = odb_source_files_optimize;
  files->base.optimize_required = odb_source_files_optimize_required;
  files->base.generate_pack = odb_source_files_generate_pack;

  /*
   * Ideally, we would only ever store absolute paths in the source. This
   * is not (yet) possible though because we access and assume relative
   * paths in the primary ODB source in some user-facing functionality.
   */
  if (!is_absolute_path(object_dir))
    chdir_notify_register(odb_source_files_reparent, files);

  prepare_alternates(files, alternates);

  free(object_dir);
  free(alternates);
  return files;
}

Coverage Report

Created: 2026-11-12 06:03