/src/libxmlb/src/xb-builder.c

Source
/*
 * Copyright 2018 Richard Hughes <richard@hughsie.com>
 *
 * SPDX-License-Identifier: LGPL-2.1-or-later
 */

#define G_LOG_DOMAIN "XbSilo"

#include "config.h"

#include <gio/gio.h>
#include <string.h>

#include "xb-builder-fixup-private.h"
#include "xb-builder-node-private.h"
#include "xb-builder-source-private.h"
#include "xb-builder.h"
#include "xb-opcode-private.h"
#include "xb-silo-private.h"
#include "xb-string-private.h"
#include "xb-version.h"

#define XB_BUILDER_MAX_DEPTH 100

typedef struct {
  GPtrArray *sources; /* of XbBuilderSource */
  GPtrArray *nodes;   /* of XbBuilderNode */
  GPtrArray *fixups;  /* of XbBuilderFixup */
  GPtrArray *locales; /* of str */
  XbSiloProfileFlags profile_flags;
  GString *guid;
} XbBuilderPrivate;

G_DEFINE_TYPE_WITH_PRIVATE(XbBuilder, xb_builder, G_TYPE_OBJECT)
#define GET_PRIVATE(o) (xb_builder_get_instance_private(o))

typedef struct {
  XbSilo *silo;
  XbBuilderNode *root;  /* transfer full */
  XbBuilderNode *current; /* transfer none */
  XbBuilderCompileFlags compile_flags;
  XbBuilderSourceFlags source_flags;
  GHashTable *strtab_hash;
  GByteArray *strtab;
  GPtrArray *locales;
  gboolean elem_closed;
  guint depth;
  GError *error;
} XbBuilderCompileHelper;

static guint32
xb_builder_compile_add_to_strtab(XbBuilderCompileHelper *helper, const gchar *str)
{
  gpointer val;
  guint32 idx;
  gsize str_len;

  /* already exists */
  if (g_hash_table_lookup_extended(helper->strtab_hash, str, NULL, &val))
    return GPOINTER_TO_UINT(val);

  /* check for integer truncation */
  if (helper->strtab->len > G_MAXINT32) {
    g_set_error(&helper->error,
          G_IO_ERROR,
          G_IO_ERROR_INVALID_DATA,
          "string table too large");
    return XB_SILO_UNSET;
  }

  /* check if adding this string would exceed the limit */
  str_len = strlen(str) + 1;
  if (helper->strtab->len + str_len > G_MAXUINT32) {
    g_set_error(&helper->error,
          G_IO_ERROR,
          G_IO_ERROR_INVALID_DATA,
          "string table too large");
    return XB_SILO_UNSET;
  }

  /* new */
  idx = helper->strtab->len;
  g_byte_array_append(helper->strtab, (const guint8 *)str, str_len);
  g_hash_table_insert(helper->strtab_hash, g_strdup(str), GUINT_TO_POINTER(idx));
  return idx;
}

static gint
xb_builder_get_locale_priority(XbBuilderCompileHelper *helper, const gchar *locale)
{
  for (guint i = 0; i < helper->locales->len; i++) {
    const gchar *locale_tmp = g_ptr_array_index(helper->locales, i);
    if (g_strcmp0(locale_tmp, locale) == 0)
      return helper->locales->len - i;
  }
  return -1;
}

static void
xb_builder_compile_start_element_cb(GMarkupParseContext *context,
            const gchar *element_name,
            const gchar **attr_names,
            const gchar **attr_values,
            gpointer user_data,
            GError **error)
{
  XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;
  g_autoptr(XbBuilderNode) bn = xb_builder_node_new(element_name);

  /* check recursion depth to prevent stack exhaustion */
  if (helper->depth >= XB_BUILDER_MAX_DEPTH) {
    g_set_error(error,
          G_IO_ERROR,
          G_IO_ERROR_INVALID_DATA,
          "nesting deeper than %u levels not supported",
          (guint)XB_BUILDER_MAX_DEPTH);
    return;
  }
  helper->depth++;

  /* parent node is being ignored */
  if (helper->current != NULL &&
      xb_builder_node_has_flag(helper->current, XB_BUILDER_NODE_FLAG_IGNORE))
    xb_builder_node_add_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE);

  /* check if we should ignore the locale */
  if (!xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE) &&
      helper->compile_flags & XB_BUILDER_COMPILE_FLAG_NATIVE_LANGS) {
    const gchar *xml_lang = NULL;
    for (guint i = 0; attr_names[i] != NULL; i++) {
      if (g_strcmp0(attr_names[i], "xml:lang") == 0) {
        xml_lang = attr_values[i];
        break;
      }
    }
    if (xml_lang == NULL) {
      if (helper->current != NULL) {
        gint prio = xb_builder_node_get_priority(helper->current);
        xb_builder_node_set_priority(bn, prio);
      }
    } else {
      gint prio = xb_builder_get_locale_priority(helper, xml_lang);
      if (prio < 0)
        xb_builder_node_add_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE);
      xb_builder_node_set_priority(bn, prio);
    }
  }

  /* add attributes */
  if (!xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE)) {
    for (guint i = 0; attr_names[i] != NULL; i++)
      xb_builder_node_set_attr(bn, attr_names[i], attr_values[i]);
  }

  /* add to tree */
  xb_builder_node_add_child(helper->current, bn);
  helper->current = bn;

  helper->elem_closed = FALSE;
}

static void
xb_builder_compile_end_element_cb(GMarkupParseContext *context,
          const gchar *element_name,
          gpointer user_data,
          GError **error)
{
  XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;
  g_autoptr(XbBuilderNode) parent = xb_builder_node_get_parent(helper->current);
  if (parent == NULL) {
    g_set_error_literal(error,
            G_IO_ERROR,
            G_IO_ERROR_INVALID_DATA,
            "Mismatched XML; no parent");
    return;
  }
  helper->current = parent;
  helper->depth--;
  helper->elem_closed = TRUE;
}

static void
xb_builder_compile_text_cb(GMarkupParseContext *context,
         const gchar *text,
         gsize text_len,
         gpointer user_data,
         GError **error)
{
  XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;
  XbBuilderNode *bn = helper->current;
  XbBuilderNode *bc = xb_builder_node_get_last_child(bn);

  /* unimportant */
  if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
    return;

  /* repair text unless we know it's valid */
  if (helper->source_flags & XB_BUILDER_SOURCE_FLAG_LITERAL_TEXT)
    xb_builder_node_add_flag(bn, XB_BUILDER_NODE_FLAG_LITERAL_TEXT);

  /* text or tail */
  if (!helper->elem_closed) {
    if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_HAS_TEXT)) {
      g_autoptr(GString) str = g_string_new(xb_builder_node_get_text(bn));
      g_string_append_len(str, text, text_len);
      xb_builder_node_set_text(bn, str->str, str->len);
    } else {
      xb_builder_node_set_text(bn, text, text_len);
    }
    return;
  }

  /* does this node have a child */
  if (bc != NULL) {
    xb_builder_node_set_tail(bc, text, text_len);
    return;
  }

  /* always set a tail, even if already set */
  xb_builder_node_set_tail(bn, text, text_len);
}

/**
 * xb_builder_import_source:
 * @self: a #XbSilo
 * @source: a #XbBuilderSource
 *
 * Adds a #XbBuilderSource to the #XbBuilder.
 *
 * Since: 0.1.0
 **/
void
xb_builder_import_source(XbBuilder *self, XbBuilderSource *source)
{
  XbBuilderPrivate *priv = GET_PRIVATE(self);
  g_autofree gchar *guid = NULL;

  g_return_if_fail(XB_IS_BUILDER(self));
  g_return_if_fail(XB_IS_BUILDER_SOURCE(source));

  /* get latest GUID */
  guid = xb_builder_source_get_guid(source);
  xb_builder_append_guid(self, guid);
  g_ptr_array_add(priv->sources, g_object_ref(source));
}

static gboolean
xb_builder_compile_source(XbBuilderCompileHelper *helper,
        XbBuilderSource *source,
        XbBuilderNode *root,
        GCancellable *cancellable,
        GError **error)
{
  GPtrArray *children;
  XbBuilderNode *info;
  gsize chunk_size = 32 * 1024;
  gssize len;
  g_autofree gchar *data = NULL;
  g_autofree gchar *guid = xb_builder_source_get_guid(source);
  g_autoptr(GPtrArray) children_copy = NULL;
  g_autoptr(GInputStream) istream = NULL;
  g_autoptr(GMarkupParseContext) ctx = NULL;
  g_autoptr(GTimer) timer = xb_silo_start_profile(helper->silo);
  g_autoptr(XbBuilderNode) root_tmp = xb_builder_node_new(NULL);
  const GMarkupParser parser = {xb_builder_compile_start_element_cb,
              xb_builder_compile_end_element_cb,
              xb_builder_compile_text_cb,
              NULL,
              NULL};

  /* add the source to a fake root in case it fails during processing */
  helper->current = root_tmp;
  helper->source_flags = xb_builder_source_get_flags(source);

  /* decompress */
  istream = xb_builder_source_get_istream(source, cancellable, error);
  if (istream == NULL)
    return FALSE;

  /* parse */
  ctx = g_markup_parse_context_new(&parser,
           G_MARKUP_PREFIX_ERROR_POSITION |
               G_MARKUP_TREAT_CDATA_AS_TEXT,
           helper,
           NULL);
  data = g_malloc(chunk_size);
  while ((len = g_input_stream_read(istream, data, chunk_size, cancellable, error)) > 0) {
    if (!g_markup_parse_context_parse(ctx, data, len, error))
      return FALSE;
  }
  if (len < 0)
    return FALSE;

  /* more opening than closing */
  if (root_tmp != helper->current) {
    g_set_error_literal(error, G_IO_ERROR, G_IO_ERROR_INVALID_DATA, "Mismatched XML");
    return FALSE;
  }

  /* run any node functions */
  if (!xb_builder_source_fixup(source, root_tmp, error))
    return FALSE;

  /* check to see if the root was ignored */
  if (xb_builder_node_has_flag(root_tmp, XB_BUILDER_NODE_FLAG_IGNORE)) {
    g_set_error_literal(error,
            G_IO_ERROR,
            G_IO_ERROR_INVALID_DATA,
            "root node cannot be ignored");
    return FALSE;
  }

  /* a single root with no siblings was required */
  if (helper->compile_flags & XB_BUILDER_COMPILE_FLAG_SINGLE_ROOT) {
    if (xb_builder_node_get_children(root_tmp)->len > 1) {
      g_set_error_literal(error,
              G_IO_ERROR,
              G_IO_ERROR_INVALID_DATA,
              "A root node without siblings was required");
      return FALSE;
    }
  }

  /* this is something we can query with later */
  info = xb_builder_source_get_info(source);
  if (info != NULL) {
    children = xb_builder_node_get_children(helper->current);
    for (guint i = 0; i < children->len; i++) {
      XbBuilderNode *bn = g_ptr_array_index(children, i);
      if (!xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
        xb_builder_node_add_child(bn, info);
    }
  }

  /* add the children to the main document */
  children = xb_builder_node_get_children(root_tmp);
  children_copy = g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref);
  for (guint i = 0; i < children->len; i++) {
    XbBuilderNode *bn = g_ptr_array_index(children, i);
    g_ptr_array_add(children_copy, g_object_ref(bn));
  }
  for (guint i = 0; i < children_copy->len; i++) {
    XbBuilderNode *bn = g_ptr_array_index(children_copy, i);
    xb_builder_node_unlink(bn);
    xb_builder_node_add_child(root, bn);
  }

  /* success */
  xb_silo_add_profile(helper->silo, timer, "compile %s", guid);
  return TRUE;
}

static gboolean
xb_builder_strtab_element_names_cb(XbBuilderNode *bn, gpointer user_data)
{
  XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;
  const gchar *tmp;
  guint32 strtab_idx;

  /* root node */
  if (xb_builder_node_get_element(bn) == NULL)
    return FALSE;
  if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
    return FALSE;
  tmp = xb_builder_node_get_element(bn);
  strtab_idx = xb_builder_compile_add_to_strtab(helper, tmp);
  if (strtab_idx == XB_SILO_UNSET)
    return TRUE;
  xb_builder_node_set_element_idx(bn, strtab_idx);
  return FALSE;
}

static gboolean
xb_builder_strtab_attr_name_cb(XbBuilderNode *bn, gpointer user_data)
{
  GPtrArray *attrs;
  XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;

  /* root node */
  if (xb_builder_node_get_element(bn) == NULL)
    return FALSE;
  if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
    return FALSE;
  attrs = xb_builder_node_get_attrs(bn);
  for (guint i = 0; attrs != NULL && i < attrs->len; i++) {
    XbBuilderNodeAttr *attr = g_ptr_array_index(attrs, i);
    attr->name_idx = xb_builder_compile_add_to_strtab(helper, attr->name);
    if (attr->name_idx == XB_SILO_UNSET)
      return TRUE;
  }
  return FALSE;
}

static gboolean
xb_builder_strtab_attr_value_cb(XbBuilderNode *bn, gpointer user_data)
{
  GPtrArray *attrs;
  XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;

  /* root node */
  if (xb_builder_node_get_element(bn) == NULL)
    return FALSE;
  if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
    return FALSE;
  attrs = xb_builder_node_get_attrs(bn);
  for (guint i = 0; attrs != NULL && i < attrs->len; i++) {
    XbBuilderNodeAttr *attr = g_ptr_array_index(attrs, i);
    attr->value_idx = xb_builder_compile_add_to_strtab(helper, attr->value);
    if (attr->value_idx == XB_SILO_UNSET)
      return TRUE;
  }
  return FALSE;
}

static gboolean
xb_builder_strtab_text_cb(XbBuilderNode *bn, gpointer user_data)
{
  XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;
  const gchar *tmp;
  guint32 strtab_idx;

  /* root node */
  if (xb_builder_node_get_element(bn) == NULL)
    return FALSE;
  if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
    return FALSE;
  if (xb_builder_node_get_text(bn) != NULL) {
    tmp = xb_builder_node_get_text(bn);
    strtab_idx = xb_builder_compile_add_to_strtab(helper, tmp);
    if (strtab_idx == XB_SILO_UNSET)
      return TRUE;
    xb_builder_node_set_text_idx(bn, strtab_idx);
  }
  if (xb_builder_node_get_tail(bn) != NULL) {
    tmp = xb_builder_node_get_tail(bn);
    strtab_idx = xb_builder_compile_add_to_strtab(helper, tmp);
    if (strtab_idx == XB_SILO_UNSET)
      return TRUE;
    xb_builder_node_set_tail_idx(bn, strtab_idx);
  }
  return FALSE;
}

static gboolean
xb_builder_strtab_tokens_cb(XbBuilderNode *bn, gpointer user_data)
{
  XbBuilderCompileHelper *helper = (XbBuilderCompileHelper *)user_data;
  GPtrArray *tokens = xb_builder_node_get_tokens(bn);
  guint32 strtab_idx;

  /* root node */
  if (xb_builder_node_get_element(bn) == NULL)
    return FALSE;
  if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
    return FALSE;
  if (tokens == NULL)
    return FALSE;
  for (guint i = 0; i < MIN(tokens->len, XB_OPCODE_TOKEN_MAX); i++) {
    const gchar *tmp = g_ptr_array_index(tokens, i);
    if (tmp == NULL)
      continue;
    strtab_idx = xb_builder_compile_add_to_strtab(helper, tmp);
    if (strtab_idx == XB_SILO_UNSET)
      return TRUE;
    xb_builder_node_add_token_idx(bn, strtab_idx);
  }
  return FALSE;
}

static gboolean
xb_builder_xml_lang_prio_cb(XbBuilderNode *bn, gpointer user_data)
{
  GPtrArray *nodes_to_destroy = (GPtrArray *)user_data;
  gint prio_best = 0;
  g_autoptr(GPtrArray) nodes = g_ptr_array_new();
  GPtrArray *siblings;
  g_autoptr(XbBuilderNode) parent = xb_builder_node_get_parent(bn);

  /* root node */
  if (xb_builder_node_get_element(bn) == NULL)
    return FALSE;

  /* already ignored */
  if (xb_builder_node_get_priority(bn) == -2)
    return FALSE;

  /* get all the siblings with the same name */
  siblings = xb_builder_node_get_children(parent);
  for (guint i = 0; i < siblings->len; i++) {
    XbBuilderNode *bn2 = g_ptr_array_index(siblings, i);
    if (g_strcmp0(xb_builder_node_get_element(bn), xb_builder_node_get_element(bn2)) ==
        0)
      g_ptr_array_add(nodes, bn2);
  }

  /* only one thing, so bail early */
  if (nodes->len == 1)
    return FALSE;

  /* find the best locale */
  for (guint i = 0; i < nodes->len; i++) {
    XbBuilderNode *bn2 = g_ptr_array_index(nodes, i);
    if (xb_builder_node_get_priority(bn2) > prio_best)
      prio_best = xb_builder_node_get_priority(bn2);
  }

  /* add any nodes not as good as the bext locale to the kill list */
  for (guint i = 0; i < nodes->len; i++) {
    XbBuilderNode *bn2 = g_ptr_array_index(nodes, i);
    if (xb_builder_node_get_priority(bn2) < prio_best)
      g_ptr_array_add(nodes_to_destroy, g_object_ref(bn2));

    /* never visit this node again */
    xb_builder_node_set_priority(bn2, -2);
  }

  return FALSE;
}

static gboolean
xb_builder_nodetab_size_cb(XbBuilderNode *bn, gpointer user_data)
{
  guint32 *sz = (guint32 *)user_data;

  /* root node */
  if (xb_builder_node_get_element(bn) == NULL)
    return FALSE;
  if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
    return FALSE;
  *sz += xb_builder_node_size(bn) + 1; /* +1 for the sentinel */
  return FALSE;
}

typedef struct {
  GByteArray *buf;
} XbBuilderNodetabHelper;

static void
xb_builder_nodetab_write_sentinel(XbBuilderNodetabHelper *helper)
{
  XbSiloNode sn = {
      .flags = XB_SILO_NODE_FLAG_NONE,
      .attr_count = 0,
  };
  //  g_debug ("SENT @%u", (guint) helper->buf->len);
  g_byte_array_append(helper->buf, (const guint8 *)&sn, xb_silo_node_get_size(&sn));
}

#define XB_BUILDER_ATTR_MAX ((1 << 6) - 1)

static gboolean
xb_builder_nodetab_write_node(XbBuilderNodetabHelper *helper, XbBuilderNode *bn, GError **error)
{
  GPtrArray *attrs = xb_builder_node_get_attrs(bn);
  GArray *token_idxs = xb_builder_node_get_token_idxs(bn);
  XbSiloNode sn = {
      .flags = XB_SILO_NODE_FLAG_IS_ELEMENT,
      .attr_count = (attrs != NULL) ? attrs->len : 0,
      .element_name = xb_builder_node_get_element_idx(bn),
      .next = 0x0,
      .parent = 0x0,
      .text = xb_builder_node_get_text_idx(bn),
      .tail = xb_builder_node_get_tail_idx(bn),
      .token_count = 0,
  };

  /* sanity check */
  if (attrs != NULL && attrs->len > XB_BUILDER_ATTR_MAX) {
    g_set_error(error,
          G_IO_ERROR,
          G_IO_ERROR_NOT_SUPPORTED,
          "too many attributes: %u",
          attrs->len);
    return FALSE;
  }

  /* add tokens */
  if (token_idxs != NULL)
    sn.flags |= XB_SILO_NODE_FLAG_IS_TOKENIZED;

  /* if the node had no children and the text is just whitespace then
   * remove it even in literal mode */
  if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_LITERAL_TEXT)) {
    if (xb_string_isspace(xb_builder_node_get_text(bn), -1))
      sn.text = XB_SILO_UNSET;
    if (xb_string_isspace(xb_builder_node_get_tail(bn), -1))
      sn.tail = XB_SILO_UNSET;
  }

  /* save this so we can set up the ->next pointers correctly */
  xb_builder_node_set_offset(bn, helper->buf->len);

  //  g_debug ("NODE @%u (%s)", (guint) helper->buf->len, xb_builder_node_get_element
  //(bn));

  /* there is no point adding more tokens than we can match */
  if (token_idxs != NULL)
    sn.token_count = MIN(token_idxs->len, XB_OPCODE_TOKEN_MAX);

  /* add to the buf */
  g_byte_array_append(helper->buf, (const guint8 *)&sn, sizeof(sn));

  /* add to the buf */
  for (guint i = 0; attrs != NULL && i < attrs->len; i++) {
    XbBuilderNodeAttr *ba = g_ptr_array_index(attrs, i);
    XbSiloNodeAttr attr = {
        .attr_name = ba->name_idx,
        .attr_value = ba->value_idx,
    };
    g_byte_array_append(helper->buf, (const guint8 *)&attr, sizeof(attr));
  }

  /* add tokens */
  for (guint i = 0; i < sn.token_count; i++) {
    guint32 idx = g_array_index(token_idxs, guint32, i);
    g_byte_array_append(helper->buf, (const guint8 *)&idx, sizeof(idx));
  }

  /* success */
  return TRUE;
}

static gboolean
xb_builder_nodetab_write(XbBuilderNodetabHelper *helper, XbBuilderNode *bn, GError **error)
{
  GPtrArray *children;

  /* ignore this */
  if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
    return TRUE;

  /* element */
  if (xb_builder_node_get_element(bn) != NULL) {
    if (!xb_builder_nodetab_write_node(helper, bn, error))
      return FALSE;
  }

  /* children */
  children = xb_builder_node_get_children(bn);
  for (guint i = 0; i < children->len; i++) {
    XbBuilderNode *bc = g_ptr_array_index(children, i);
    if (!xb_builder_nodetab_write(helper, bc, error))
      return FALSE;
  }

  /* sentinel */
  if (xb_builder_node_get_element(bn) != NULL)
    xb_builder_nodetab_write_sentinel(helper);

  /* success */
  return TRUE;
}

static XbSiloNode *
xb_builder_get_node(GByteArray *str, guint32 off)
{
  return (XbSiloNode *)(str->data + off);
}

static gboolean
xb_builder_nodetab_fix_cb(XbBuilderNode *bn, gpointer user_data)
{
  GPtrArray *siblings;
  XbBuilderNodetabHelper *helper = (XbBuilderNodetabHelper *)user_data;
  XbSiloNode *sn;
  gboolean found = FALSE;
  g_autoptr(XbBuilderNode) parent = xb_builder_node_get_parent(bn);

  /* root node */
  if (xb_builder_node_get_element(bn) == NULL)
    return FALSE;
  if (xb_builder_node_has_flag(bn, XB_BUILDER_NODE_FLAG_IGNORE))
    return FALSE;

  /* get the position in the buffer */
  sn = xb_builder_get_node(helper->buf, xb_builder_node_get_offset(bn));
  if (sn == NULL)
    return FALSE;

  /* set the parent if the node has one */
  if (xb_builder_node_get_element(parent) != NULL)
    sn->parent = xb_builder_node_get_offset(parent);

  /* set ->next if the node has one */
  siblings = xb_builder_node_get_children(parent);
  for (guint i = 0; i < siblings->len; i++) {
    XbBuilderNode *bn2 = g_ptr_array_index(siblings, i);
    if (bn2 == bn) {
      found = TRUE;
      continue;
    }
    if (!found)
      continue;
    if (!xb_builder_node_has_flag(bn2, XB_BUILDER_NODE_FLAG_IGNORE)) {
      sn->next = xb_builder_node_get_offset(bn2);
      break;
    }
  }

  return FALSE;
}

static void
xb_builder_compile_helper_free(XbBuilderCompileHelper *helper)
{
  g_hash_table_unref(helper->strtab_hash);
  g_byte_array_unref(helper->strtab);
  g_clear_error(&helper->error);
  g_clear_object(&helper->silo);
  g_object_unref(helper->root);
  g_free(helper);
}

G_DEFINE_AUTOPTR_CLEANUP_FUNC(XbBuilderCompileHelper, xb_builder_compile_helper_free)

static gchar *
xb_builder_generate_guid(XbBuilder *self)
{
  XbBuilderPrivate *priv = GET_PRIVATE(self);
  XbGuid guid = {0x0};
  if (priv->guid->len > 0) {
    xb_guid_compute_for_data(&guid, (const guint8 *)priv->guid->str, priv->guid->len);
  }
  return xb_guid_to_string(&guid);
}

/**
 * xb_builder_import_node:
 * @self: a #XbSilo
 * @bn: a #XbBuilderNode
 *
 * Adds a node tree to the builder.
 *
 * If you are manually adding dynamic data sourced from a non-static source then you MUST use
 * xb_builder_append_guid() with the appropriate GUID value, e.g. the file name and mtime.
 *
 * If no appropriate value is available, the caller can use something like:
 *
 *     g_autofree gchar *tmp = xb_builder_node_export(bn, XB_NODE_EXPORT_FLAG_NONE, NULL);
 *     xb_builder_append_guid(builder, tmp);
 *
 * Failure to include an appropriate GUID value would allow an out-of-data silo to be used.
 *
 * Since: 0.1.0
 **/
void
xb_builder_import_node(XbBuilder *self, XbBuilderNode *bn)
{
  XbBuilderPrivate *priv = GET_PRIVATE(self);
  g_return_if_fail(XB_IS_BUILDER(self));
  g_return_if_fail(XB_IS_BUILDER_NODE(bn));
  g_ptr_array_add(priv->nodes, g_object_ref(bn));
}

/**
 * xb_builder_add_locale:
 * @self: a #XbSilo
 * @locale: a locale, e.g. "en_US"
 *
 * Adds a locale to the builder. Locales added first will be prioritised over
 * locales added later.
 *
 * Since: 0.1.0
 **/
void
xb_builder_add_locale(XbBuilder *self, const gchar *locale)
{
  XbBuilderPrivate *priv = GET_PRIVATE(self);

  g_return_if_fail(XB_IS_BUILDER(self));
  g_return_if_fail(locale != NULL);

  if (g_str_has_suffix(locale, ".UTF-8"))
    return;
  for (guint i = 0; i < priv->locales->len; i++) {
    const gchar *locale_tmp = g_ptr_array_index(priv->locales, i);
    if (g_strcmp0(locale_tmp, locale) == 0)
      return;
  }
  g_ptr_array_add(priv->locales, g_strdup(locale));

  /* if the user changes LANG, the blob is no longer valid */
  xb_builder_append_guid(self, locale);
}

static gboolean
xb_builder_watch_source(XbBuilder *self,
      XbBuilderSource *source,
      XbSilo *silo,
      GCancellable *cancellable,
      GError **error)
{
  GFile *file = xb_builder_source_get_file(source);
  g_autoptr(GFile) watched_file = NULL;
  if (file == NULL)
    return TRUE;
  if ((xb_builder_source_get_flags(source) &
       (XB_BUILDER_SOURCE_FLAG_WATCH_FILE | XB_BUILDER_SOURCE_FLAG_WATCH_DIRECTORY)) == 0)
    return TRUE;

  if (xb_builder_source_get_flags(source) & XB_BUILDER_SOURCE_FLAG_WATCH_DIRECTORY)
    watched_file = g_file_get_parent(file);
  else
    watched_file = g_object_ref(file);

  return xb_silo_watch_file(silo, watched_file, cancellable, error);
}

static gboolean
xb_builder_watch_sources(XbBuilder *self, XbSilo *silo, GCancellable *cancellable, GError **error)
{
  XbBuilderPrivate *priv = GET_PRIVATE(self);
  for (guint i = 0; i < priv->sources->len; i++) {
    XbBuilderSource *source = g_ptr_array_index(priv->sources, i);
    if (!xb_builder_watch_source(self, source, silo, cancellable, error))
      return FALSE;
  }
  return TRUE;
}

/**
 * xb_builder_compile:
 * @self: a #XbSilo
 * @flags: some #XbBuilderCompileFlags, e.g. %XB_BUILDER_SOURCE_FLAG_LITERAL_TEXT
 * @cancellable: a #GCancellable, or %NULL
 * @error: the #GError, or %NULL
 *
 * Compiles a #XbSilo.
 *
 * Returns: (transfer full): a #XbSilo, or %NULL for error
 *
 * Since: 0.1.0
 **/
XbSilo *
xb_builder_compile(XbBuilder *self,
       XbBuilderCompileFlags flags,
       GCancellable *cancellable,
       GError **error)
{
  XbBuilderPrivate *priv = GET_PRIVATE(self);
  guint32 nodetabsz = sizeof(XbSiloHeader);
  g_autoptr(GByteArray) buf = NULL;
  g_autoptr(GBytes) blob = NULL;
  XbSiloHeader *hdrptr;
  XbSiloHeader hdr = {
      .magic = XB_SILO_MAGIC_BYTES,
      .version = XB_SILO_VERSION,
      .strtab = 0,
      .strtab_ntags = 0,
      .padding = {0x0},
      .guid = {0x0},
      .filesz = 0x0,
  };
  XbBuilderNodetabHelper nodetab_helper = {
      .buf = NULL,
  };
  g_autoptr(GPtrArray) nodes_to_destroy =
      g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref);
  g_autoptr(GTimer) timer = NULL;
  g_autoptr(XbBuilderCompileHelper) helper = NULL;

  g_return_val_if_fail(XB_IS_BUILDER(self), NULL);
  g_return_val_if_fail(cancellable == NULL || G_IS_CANCELLABLE(cancellable), NULL);
  g_return_val_if_fail(error == NULL || *error == NULL, NULL);

  /* this is inferred */
  if (flags & XB_BUILDER_COMPILE_FLAG_SINGLE_LANG)
    flags |= XB_BUILDER_COMPILE_FLAG_NATIVE_LANGS;

  /* the builder needs to know the locales */
  if (priv->locales->len == 0 && (flags & XB_BUILDER_COMPILE_FLAG_NATIVE_LANGS)) {
    g_set_error_literal(error,
            G_IO_ERROR,
            G_IO_ERROR_INVALID_DATA,
            "No locales set and using NATIVE_LANGS");
    return NULL;
  }

  /* create helper used for compiling */
  helper = g_new0(XbBuilderCompileHelper, 1);
  helper->compile_flags = flags;
  helper->root = xb_builder_node_new(NULL);
  helper->silo = xb_silo_new();
  helper->locales = priv->locales;
  helper->strtab = g_byte_array_new();
  helper->strtab_hash = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
  helper->elem_closed = FALSE;

  /* for profiling */
  xb_silo_set_profile_flags(helper->silo, priv->profile_flags);
  timer = xb_silo_start_profile(helper->silo);

  /* build node tree */
  for (guint i = 0; i < priv->sources->len; i++) {
    XbBuilderSource *source = g_ptr_array_index(priv->sources, i);
    const gchar *prefix = xb_builder_source_get_prefix(source);
    g_autofree gchar *source_guid = xb_builder_source_get_guid(source);
    g_autoptr(XbBuilderNode) root = NULL;
    g_autoptr(GError) error_local = NULL;

    /* find, or create the prefix */
    if (prefix != NULL) {
      root = xb_builder_node_get_child(helper->root, prefix, NULL);
      if (root == NULL)
        root = xb_builder_node_insert(helper->root, prefix, NULL);
    } else {
      /* don't allow damaged XML files to ruin all the next ones */
      root = g_object_ref(helper->root);
    }

    /* watch the source */
    if (!xb_builder_watch_source(self, source, helper->silo, cancellable, error))
      return NULL;

    if (priv->profile_flags & XB_SILO_PROFILE_FLAG_DEBUG)
      g_debug("compiling %s…", source_guid);
    if (!xb_builder_compile_source(helper, source, root, cancellable, &error_local)) {
      if (flags & XB_BUILDER_COMPILE_FLAG_IGNORE_INVALID) {
        g_debug("ignoring invalid file %s: %s",
          source_guid,
          error_local->message);
        continue;
      }
      g_propagate_prefixed_error(error,
               g_steal_pointer(&error_local),
               "failed to compile %s: ",
               source_guid);
      return NULL;
    }
  }

  /* run any node functions */
  for (guint i = 0; i < priv->fixups->len; i++) {
    XbBuilderFixup *fixup = g_ptr_array_index(priv->fixups, i);
    if (!xb_builder_fixup_node(fixup, helper->root, error))
      return NULL;
  }

  /* only include the highest priority translation */
  if (flags & XB_BUILDER_COMPILE_FLAG_SINGLE_LANG) {
    xb_builder_node_traverse(helper->root,
           G_PRE_ORDER,
           G_TRAVERSE_ALL,
           -1,
           xb_builder_xml_lang_prio_cb,
           nodes_to_destroy);
    for (guint i = 0; i < nodes_to_destroy->len; i++) {
      XbBuilderNode *bn = g_ptr_array_index(nodes_to_destroy, i);
      xb_builder_node_unlink(bn);
    }
    xb_silo_add_profile(helper->silo, timer, "filter single-lang");
  }

  /* add any manually build nodes */
  for (guint i = 0; i < priv->nodes->len; i++) {
    XbBuilderNode *bn = g_ptr_array_index(priv->nodes, i);
    xb_builder_node_add_child(helper->root, bn);
  }

  /* get the size of the nodetab */
  xb_builder_node_traverse(helper->root,
         G_PRE_ORDER,
         G_TRAVERSE_ALL,
         -1,
         xb_builder_nodetab_size_cb,
         &nodetabsz);
  buf = g_byte_array_sized_new(nodetabsz);
  xb_silo_add_profile(helper->silo, timer, "get size nodetab");

  /* add everything to the strtab */
  xb_builder_node_traverse(helper->root,
         G_PRE_ORDER,
         G_TRAVERSE_ALL,
         -1,
         xb_builder_strtab_element_names_cb,
         helper);
  if (helper->error != NULL) {
    g_propagate_error(error, g_steal_pointer(&helper->error));
    return NULL;
  }
  hdr.strtab_ntags = g_hash_table_size(helper->strtab_hash);
  xb_silo_add_profile(helper->silo, timer, "adding strtab element");
  xb_builder_node_traverse(helper->root,
         G_PRE_ORDER,
         G_TRAVERSE_ALL,
         -1,
         xb_builder_strtab_attr_name_cb,
         helper);
  if (helper->error != NULL) {
    g_propagate_error(error, g_steal_pointer(&helper->error));
    return NULL;
  }
  xb_silo_add_profile(helper->silo, timer, "adding strtab attr name");
  xb_builder_node_traverse(helper->root,
         G_PRE_ORDER,
         G_TRAVERSE_ALL,
         -1,
         xb_builder_strtab_attr_value_cb,
         helper);
  if (helper->error != NULL) {
    g_propagate_error(error, g_steal_pointer(&helper->error));
    return NULL;
  }
  xb_silo_add_profile(helper->silo, timer, "adding strtab attr value");
  xb_builder_node_traverse(helper->root,
         G_PRE_ORDER,
         G_TRAVERSE_ALL,
         -1,
         xb_builder_strtab_text_cb,
         helper);
  if (helper->error != NULL) {
    g_propagate_error(error, g_steal_pointer(&helper->error));
    return NULL;
  }
  xb_silo_add_profile(helper->silo, timer, "adding strtab text");
  xb_builder_node_traverse(helper->root,
         G_PRE_ORDER,
         G_TRAVERSE_ALL,
         -1,
         xb_builder_strtab_tokens_cb,
         helper);
  if (helper->error != NULL) {
    g_propagate_error(error, g_steal_pointer(&helper->error));
    return NULL;
  }
  xb_silo_add_profile(helper->silo, timer, "adding strtab tokens");

  /* add the initial header */
  hdr.strtab = nodetabsz;
  if (priv->guid->len > 0) {
    XbGuid guid_tmp;
    xb_guid_compute_for_data(&guid_tmp,
           (const guint8 *)priv->guid->str,
           priv->guid->len);
    memcpy(&hdr.guid, &guid_tmp, sizeof(guid_tmp));
  }
  g_byte_array_append(buf, (const guint8 *)&hdr, sizeof(hdr));

  /* write nodes to the nodetab */
  nodetab_helper.buf = buf;
  if (!xb_builder_nodetab_write(&nodetab_helper, helper->root, error))
    return NULL;
  xb_silo_add_profile(helper->silo, timer, "writing nodetab");

  /* set all the ->next and ->parent offsets */
  xb_builder_node_traverse(helper->root,
         G_PRE_ORDER,
         G_TRAVERSE_ALL,
         -1,
         xb_builder_nodetab_fix_cb,
         &nodetab_helper);
  xb_silo_add_profile(helper->silo, timer, "fixing ->parent and ->next");

  /* append the string table */
  g_byte_array_append(buf, (const guint8 *)helper->strtab->data, helper->strtab->len);
  xb_silo_add_profile(helper->silo, timer, "appending strtab");

  /* update the file size */
  hdrptr = (XbSiloHeader *)buf->data;
  hdrptr->filesz = buf->len;

  /* create data */
  blob = g_byte_array_free_to_bytes(g_steal_pointer(&buf));
  if (!xb_silo_load_from_bytes(helper->silo, blob, XB_SILO_LOAD_FLAG_NONE, error))
    return NULL;

  /* success */
  return g_steal_pointer(&helper->silo);
}

/**
 * xb_builder_ensure:
 * @self: a #XbSilo
 * @file: a #GFile
 * @flags: some #XbBuilderCompileFlags, e.g. %XB_BUILDER_COMPILE_FLAG_IGNORE_INVALID
 * @cancellable: a #GCancellable, or %NULL
 * @error: the #GError, or %NULL
 *
 * Ensures @file is up to date, and returns a compiled #XbSilo.
 *
 * If @silo is being used by a query (e.g. in another thread) then all node
 * data is immediately invalid.
 *
 * The returned #XbSilo will use the thread-default main context at the time of
 * calling this function for its future signal emissions.
 *
 * Returns: (transfer full): a #XbSilo, or %NULL for error
 *
 * Since: 0.1.0
 **/
XbSilo *
xb_builder_ensure(XbBuilder *self,
      GFile *file,
      XbBuilderCompileFlags flags,
      GCancellable *cancellable,
      GError **error)
{
  XbBuilderPrivate *priv = GET_PRIVATE(self);
  XbSiloLoadFlags load_flags = XB_SILO_LOAD_FLAG_NONE;
  g_autofree gchar *fn = NULL;
  g_autoptr(XbSilo) silo_tmp = xb_silo_new();
  g_autoptr(XbSilo) silo = NULL;
  g_autoptr(GError) error_local = NULL;

  g_return_val_if_fail(XB_IS_BUILDER(self), NULL);
  g_return_val_if_fail(G_IS_FILE(file), NULL);
  g_return_val_if_fail(cancellable == NULL || G_IS_CANCELLABLE(cancellable), NULL);
  g_return_val_if_fail(error == NULL || *error == NULL, NULL);

  /* watch the blob, so propagate flags */
  if (flags & XB_BUILDER_COMPILE_FLAG_WATCH_BLOB) {
    load_flags |= XB_SILO_LOAD_FLAG_WATCH_BLOB;
    if (!xb_silo_watch_file(silo_tmp, file, cancellable, error))
      return NULL;
  }

  /* ensure all the sources are watched */
  if (!xb_builder_watch_sources(self, silo_tmp, cancellable, error))
    return NULL;

  /* profile new silo if needed */
  xb_silo_set_profile_flags(silo_tmp, priv->profile_flags);

  /* load the file and peek at the GUIDs */
  fn = g_file_get_path(file);
  g_debug("attempting to load %s", fn);
  if (!xb_silo_load_from_file(silo_tmp,
            file,
            XB_SILO_LOAD_FLAG_NONE,
            cancellable,
            &error_local)) {
    g_debug("failed to load silo: %s", error_local->message);
  } else {
    g_autofree gchar *guid = xb_builder_generate_guid(self);
    if (priv->profile_flags & XB_SILO_PROFILE_FLAG_DEBUG)
      g_debug("GUID string: %s", priv->guid->str);
    g_debug("file: %s, current:%s", xb_silo_get_guid(silo_tmp), guid);

    /* no compile required */
    if (g_strcmp0(xb_silo_get_guid(silo_tmp), guid) == 0 ||
        (flags & XB_BUILDER_COMPILE_FLAG_IGNORE_GUID) > 0) {
      g_debug("loading silo with existing file contents");
      return g_steal_pointer(&silo_tmp);
    }
  }

  /* fallback to just creating a new file */
  silo = xb_builder_compile(self, flags, cancellable, error);
  if (silo == NULL)
    return NULL;

  /* this might seem unnecessary, but windows cannot do _chsize() (introduced in GLib commit
   * https://gitlab.gnome.org/GNOME/glib/-/commit/3f705ffa1230757b910a06a705104d4b0fee2c05)
   * on a mmap'd file -- so manually tear that down before writing the new file */
  g_clear_object(&silo_tmp);
  if (!xb_silo_save_to_file(silo, file, NULL, error))
    return NULL;

  /* load from a file to re-mmap it */
  if (!xb_silo_load_from_file(silo, file, load_flags, cancellable, error))
    return NULL;

  /* ensure all the sources are watched on the reloaded silo */
  if (!xb_builder_watch_sources(self, silo, cancellable, error))
    return NULL;

  /* success */
  return g_steal_pointer(&silo);
}

/**
 * xb_builder_append_guid:
 * @self: a #XbSilo
 * @guid: any text, typcically a filename or GUID
 *
 * Adds the GUID to the internal correctness hash.
 *
 * Since: 0.1.0
 **/
void
xb_builder_append_guid(XbBuilder *self, const gchar *guid)
{
  XbBuilderPrivate *priv = GET_PRIVATE(self);
  if (priv->guid->len > 0)
    g_string_append(priv->guid, "&");
  g_string_append(priv->guid, guid);
}

/**
 * xb_builder_set_profile_flags:
 * @self: a #XbBuilder
 * @profile_flags: some #XbSiloProfileFlags, e.g. %XB_SILO_PROFILE_FLAG_DEBUG
 *
 * Enables or disables the collection of profiling data.
 *
 * Since: 0.1.1
 **/
void
xb_builder_set_profile_flags(XbBuilder *self, XbSiloProfileFlags profile_flags)
{
  XbBuilderPrivate *priv = GET_PRIVATE(self);
  g_return_if_fail(XB_IS_BUILDER(self));
  priv->profile_flags = profile_flags;
}

/**
 * xb_builder_add_fixup:
 * @self: a #XbBuilder
 * @fixup: a #XbBuilderFixup
 *
 * Adds a function that will get run on every #XbBuilderNode compile creates
 * for the silo. This is run after all the #XbBuilderSource fixups have been
 * run.
 *
 * Since: 0.1.3
 **/
void
xb_builder_add_fixup(XbBuilder *self, XbBuilderFixup *fixup)
{
  XbBuilderPrivate *priv = GET_PRIVATE(self);
  g_autofree gchar *guid = NULL;

  g_return_if_fail(XB_IS_BUILDER(self));
  g_return_if_fail(XB_IS_BUILDER_FIXUP(fixup));

  /* append function IDs */
  guid = xb_builder_fixup_get_guid(fixup);
  xb_builder_append_guid(self, guid);
  g_ptr_array_add(priv->fixups, g_object_ref(fixup));
}

static void
xb_builder_finalize(GObject *obj)
{
  XbBuilder *self = XB_BUILDER(obj);
  XbBuilderPrivate *priv = GET_PRIVATE(self);

  g_ptr_array_unref(priv->sources);
  g_ptr_array_unref(priv->nodes);
  g_ptr_array_unref(priv->locales);
  g_ptr_array_unref(priv->fixups);
  g_string_free(priv->guid, TRUE);

  G_OBJECT_CLASS(xb_builder_parent_class)->finalize(obj);
}

static void
xb_builder_class_init(XbBuilderClass *klass)
{
  GObjectClass *object_class = G_OBJECT_CLASS(klass);
  object_class->finalize = xb_builder_finalize;
}

static void
xb_builder_init(XbBuilder *self)
{
  XbBuilderPrivate *priv = GET_PRIVATE(self);
  priv->sources = g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref);
  priv->nodes = g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref);
  priv->fixups = g_ptr_array_new_with_free_func((GDestroyNotify)g_object_unref);
  priv->locales = g_ptr_array_new_with_free_func(g_free);
  priv->guid = g_string_new(xb_version_string());
}

/**
 * xb_builder_new:
 *
 * Creates a new builder.
 *
 * The #XbSilo returned by the methods of this #XbBuilder will use the
 * thread-default main context at the time of calling this function for its
 * future signal emissions.
 *
 * Returns: a new #XbBuilder
 *
 * Since: 0.1.0
 **/
XbBuilder *
xb_builder_new(void)
{
  return g_object_new(XB_TYPE_BUILDER, NULL);
}

Coverage Report

Created: 2026-04-28 06:49