/src/wget2/libwget/xml.c

Source
/*
 * Copyright (c) 2012 Tim Ruehsen
 * Copyright (c) 2015-2026 Free Software Foundation, Inc.
 *
 * This file is part of libwget.
 *
 * Libwget is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Libwget is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with libwget.  If not, see <https://www.gnu.org/licenses/>.
 *
 *
 * xml parsing routines
 *
 * Changelog
 * 22.06.2012  Tim Ruehsen  created, but needs definitely a rewrite
 *
 * This derives from an old source code that I wrote in 2001.
 * It is short, fast and has a low memory print, BUT it is a hack.
 * It has to be replaced by e.g. libxml2 or something better.
 *
 * HTML parsing is (very) different from XML parsing, see here:
 * https://html.spec.whatwg.org/multipage/syntax.html
 * It is a PITA and should be handled by a specialized, external library !
 *
 */

#include <config.h>

#include <unistd.h>
#include <stdio.h>
#include <string.h>
#include <fcntl.h>
#include <sys/stat.h>
#ifdef HAVE_MMAP
#include <sys/mman.h>
#endif

#include <wget.h>
#include "private.h"

#define MAX_XML_DEPTH 1024

typedef struct {
  const char
    *buf, //!< pointer to original start of buffer (0-terminated)
    *p, //!< pointer next char in buffer
    *token; //!< token buffer
  int
    hints; //!< XML_HINT...
  size_t
    token_size, //!< size of token buffer
    token_len; //!< used bytes of token buffer (not counting terminating 0 byte)
  void
    *user_ctx; //!< user context (not needed if we were using nested functions)
  wget_xml_callback
    *callback; //!< callback function for tokens
} xml_context;

/* \cond _hide_internal_symbols */
#define ascii_isspace(c) (c == ' ' || (c >= 9 && c <=  13))

// working only for consecutive alphabets, e.g. EBCDIC would not work
#define ascii_isalpha(c) ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'))
/* \endcond */

// append a char to token buffer

static const char *getToken(xml_context *context)
{
  int c;
  const char *p;

  // skip leading whitespace
  while ((c = *context->p) && ascii_isspace(c))
    context->p++;
  if (!c) return NULL; // eof
  context->token = context->p++;

//  info_printf("a c=%c\n", c);

  if (ascii_isalpha(c) || c == '_') {
    while ((c = *context->p) && !ascii_isspace(c) && c != '>' && c != '=')
      context->p++;
    if (!c) return NULL; // syntax error

    context->token_len = context->p - context->token;
    return context->token;
  }

  if (c == '/') {
    if (!(c = *context->p)) return NULL; // syntax error
    context->p++;
    if (c == '>') {
      context->token_len = 2;
      return context->token;
    } else return NULL; // syntax error
  }

  if (c == '\"' || c == '\'') { // read in quoted value
    int quote = c;

    context->token = context->p;

    if (!(p = strchr(context->p, quote)))
      return NULL;
    context->p = p + 1;

    context->token_len = context->p - context->token - 1;
    return context->token;
  }

  if (c == '<') { // fetch specials, e.g. start of comments '<!--'
    if (!(c = *context->p)) return NULL; // syntax error
    context->p++;
    if (c == '?' || c == '/') {
      context->token_len = 2;
      return context->token;
    }

    if (c == '!') {
      // left: <!--, <![CDATA[ and <!WHATEVER
      if (!(c = *context->p)) return NULL; // syntax error
      if (c == '-') {
        context->p++;
        if (!(c = *context->p)) return NULL; // syntax error
        context->p++;
        if (c == '-') {
          context->token_len = 4;
          return context->token;
        } else {
          context->p -= 2;
          context->token_len = 2;
          return context->token;
        }
      } else {
        context->token_len = 2;
        return context->token;
      }
    } else {
      context->p--;
      context->token_len = 1;
      return context->token;
    }
  }

  if (c == '>' || c == '=') {
    context->token_len = 1;
    return context->token;
  }

  if (c == '-') { // fetch specials, e.g. end of comments '-->'
    if (!(c = *context->p)) return NULL; // syntax error
    if (c != '-') {
      c = '-';  //???
    } else {
      context->p++;
      if (!(c = *context->p)) return NULL; // syntax error
      context->p++;
      if (c != '>') {
        context->p -= 2;
        c = '-';
      } else {
        context->token_len = 3;
        return context->token;
      }
    }
  }

  if (c == '?') { // fetch specials, e.g. '?>'
    if (!(c = *context->p)) return NULL; // syntax error
    if (c != '>') {
      // c = '?';
    } else {
      context->p++;
      context->token_len = 2;
      return context->token;
    }
  }

  while ((c = *context->p) && !ascii_isspace(c))
    context->p++;

  if (c) {
    debug_printf("getToken =%.*s\n", (int)(context->p - context->token), context->token);
    context->token_len = context->p - context->token;
    return context->token;
  }

  return NULL;
}

static const char *getHTMLValue(xml_context *context)
{
  int c;
  const char *p;

  // skip leading whitespace
  while ((c = *context->p) && ascii_isspace(c))
    context->p++;
  if (!c) return NULL; // eof
  context->token = context->p++;

  // Check for and read in quoted value.
  if (c == '\"' || c == '\'' || c == '`') {
    int quote = c;

    context->token = context->p;

    if (!(p = strchr(context->p, quote)))
      return NULL;
    context->p = p + 1;

    context->token_len = context->p - context->token - 1;
    return context->token;
  }

  // Read in unquoted value.
  while ((c = *context->p) && !ascii_isspace(c) && c != '<' && c != '>' && !(c == '/' && *context->p == '>'))
    context->p++;
  if (c) {
    debug_printf("getHTMLValue =%.*s\n", (int)(context->p - context->token), context->token);
    context->token_len = context->p - context->token;
    return context->token;
  }

  return NULL;
}

static int getValue(xml_context *context)
{
  int c;

  context->token_len = 0;
  context->token = context->p;

  // remove leading spaces
  while ((c = *context->p) && ascii_isspace(c))
    context->p++;
  if (!c) return EOF;

  if (c == '=') {
    context->p++;

    if (context->hints&XML_HINT_HTML) {
      if (!getHTMLValue(context))
        return EOF; // syntax error
      else
        return 1; // token valid
    }

    if (!getToken(context))
      return EOF; // syntax error
    else
      return 1; // token valid
  }

  // attribute without value
  context->token = context->p;
  return 1;
}

// special HTML <script> content parsing
// see https://html.spec.whatwg.org/multipage/scripting.html#the-script-element
// see https://html.spec.whatwg.org/multipage/scripting.html#restrictions-for-contents-of-script-elements

static const char *getScriptContent(xml_context *context)
{
  int comment = 0, length_valid = 0;
  const char *p;

  for (p = context->token = context->p; *p; p++) {
    if (comment) {
      if (*p == '-' && !strncmp(p, "-->", 3)) {
        p += 3 - 1;
        comment = 0;
      }
    } else {
      if (*p == '<' && !strncmp(p, "<!--", 4)) {
        p += 4 - 1;
        comment = 1;
      } else if (*p == '<' && !wget_strncasecmp_ascii(p, "</script", 8)) {
        context->token_len = p - context->token;
        length_valid = 1;
        for (p += 8; ascii_isspace(*p); p++);
        if (*p == '>') {
          p++;
          break; // found end of <script>
        } else if (!*p)
          break; // end of input
      }
    }
  }
  context->p = p;

  if (!length_valid)
    context->token_len = p - context->token;

  if (!*p && !context->token_len)
    return NULL;

  if (context->callback)
    context->callback(context->user_ctx, XML_FLG_CONTENT | XML_FLG_END, "script", NULL, context->token, context->token_len, context->token - context->buf);

  return context->token;
}

// special HTML <style> content parsing
// see https://html.spec.whatwg.org/multipage/semantics.html#the-style-element
static const char *getStyleContent(xml_context *context)
{
  int comment = 0, length_valid = 0;
  const char *p;

  for (p = context->token = context->p; *p; p++) {
    if (comment) {
      if (p[0] == '*' && p[1] == '/') {
        p += 2 - 1;
        comment = 0;
      }
    } else {
      if (p[0] == '/' && p[1] == '*') {
        p += 2 - 1;
        comment = 1;
      } else if (*p == '<' && !wget_strncasecmp_ascii(p, "</style", 7)) {
        context->token_len = p - context->token;
        length_valid = 1;
        for (p += 7; ascii_isspace(*p); p++);
        if (*p == '>') {
          p++;
          break; // found end of <style>
        } else if (!*p)
          break; // end of input
      }
    }
  }
  context->p = p;

  if (!length_valid)
    context->token_len = p - context->token;

  if (!*p && !context->token_len)
    return NULL;

  if (context->callback)
    context->callback(context->user_ctx, XML_FLG_CONTENT | XML_FLG_END, "style", NULL, context->token, context->token_len, context->token - context->buf);

  return context->token;
}

static const char *getUnparsed(xml_context *context, int flags, const char *end, size_t len, const char *directory)
{
  int c;

  if (len == 1) {
    for (context->token = context->p; (c = *context->p) && c != *end; context->p++);
  } else {
    for (context->token = context->p; (c = *context->p); context->p++) {
      if (c == *end && context->p[1] == end[1] && (len == 2 || context->p[2] == end[2])) {
        break;
      }
    }
  }

  context->token_len = context->p - context->token;
  if (c) context->p += len;

  if (!c && !context->token_len)
    return NULL;
/*
  if (context->token && context->token_len && context->hints & XML_HINT_REMOVE_EMPTY_CONTENT) {
    int notempty = 0;
    char *p;

    for (p = context->token; *p; p++) {
      if (!ascii_isspace(*p)) {
        notempty = 1;
        break;
      }
    }

    if (notempty) {
      if (context->callback)
        context->callback(context->user_ctx, flags, directory, NULL, context->token, context->token_len, context->token - context->buf);
    } else {
      // ignore empty content
      context->token_len = 0;
      context->token[0] = 0;
    }
  } else {
*/
  if (context->callback)
    context->callback(context->user_ctx, flags, directory, NULL, context->token, context->token_len, context->token - context->buf);

//  }

  return context->token;
}

static const char *getComment(xml_context *context)
{
  return getUnparsed(context, XML_FLG_COMMENT, "-->", 3, NULL);
}

static const char *getProcessing(xml_context *context)
{
  return getUnparsed(context, XML_FLG_PROCESSING, "?>", 2, NULL);
}

static const char *getSpecial(xml_context *context)
{
  return getUnparsed(context, XML_FLG_SPECIAL, ">", 1, NULL);
}

static const char *getContent(xml_context *context, const char *directory)
{
  int c;

  for (context->token = context->p; (c = *context->p) && c != '<'; context->p++);

  context->token_len = context->p - context->token;

  if (!c && !context->token_len)
    return NULL;

  // debug_printf("content=%.*s\n", (int)context->token_len, context->token);
  if (context->callback && context->token_len)
    context->callback(context->user_ctx, XML_FLG_CONTENT, directory, NULL, context->token, context->token_len, context->token - context->buf);

  return context->token;
}

static int parseXML(const char *dir, xml_context *context, int depth)
{
  const char *tok;
  char directory[256] = "";
  size_t pos = 0;

  if (!(context->hints & XML_HINT_HTML)) {
    pos = wget_strlcpy(directory, dir, sizeof(directory));
    if (pos >= sizeof(directory)) pos = sizeof(directory) - 1;
  }

  do {
    getContent(context, directory);
    if (context->token_len)
      debug_printf("%s='%.*s'\n", directory, (int)context->token_len, context->token);

    if (!(tok = getToken(context))) return WGET_E_SUCCESS;  //eof
    // debug_printf("A Token '%.*s' len=%zu tok='%s'\n", (int)context->token_len, context->token, context->token_len, tok);

    if (context->token_len == 1 && *tok == '<') {
      // get element name and add it to directory
      int flags = XML_FLG_BEGIN;

      if (!(tok = getToken(context))) return WGET_E_XML_PARSE_ERR; // syntax error

      // debug_printf("A2 Token '%.*s'\n", (int)context->token_len, context->token);

      if (!(context->hints & XML_HINT_HTML)) {
        if (!pos || directory[pos - 1] != '/')
          wget_snprintf(&directory[pos], sizeof(directory) - pos, "/%.*s", (int)context->token_len, tok);
        else
          wget_snprintf(&directory[pos], sizeof(directory) - pos, "%.*s", (int)context->token_len, tok);
      } else {
        // wget_snprintf(directory, sizeof(directory), "%.*s", (int)context->token_len, tok);
        size_t dirlen = context->token_len >= sizeof(directory) ? sizeof(directory) - 1 : context->token_len;

        memcpy(directory, tok, dirlen);
        directory[dirlen] = 0;
      }

      while ((tok = getToken(context))) {
        // debug_printf("C Token %.*s %zu %p %p dir=%s tok=%s\n", (int)context->token_len, context->token, context->token_len, context->token, context->p, directory, tok);
        if (context->token_len == 2 && !strncmp(tok, "/>", 2)) {
          if (context->callback)
            context->callback(context->user_ctx, flags | XML_FLG_END, directory, NULL, NULL, 0, 0);
          break; // stay in this level
        } else if (context->token_len == 1 && *tok == '>') {
          if (context->callback)
            context->callback(context->user_ctx, flags | XML_FLG_CLOSE, directory, NULL, NULL, 0, 0);
          if (context->hints & XML_HINT_HTML) {
            if (!wget_strcasecmp_ascii(directory, "script")) {
              // special HTML <script> content parsing
              // see https://html.spec.whatwg.org/multipage/scripting.html#the-script-element
              // 4.3.1.2 Restrictions for contents of script elements
              debug_printf("*** need special <script> handling\n");
              getScriptContent(context);
              if (context->token_len)
                debug_printf("%s=%.*s\n", directory, (int)context->token_len, context->token);
            }
            else if (!wget_strcasecmp_ascii(directory, "style")) {
              getStyleContent(context);
              if (context->token_len)
                debug_printf("%s=%.*s\n", directory, (int)context->token_len, context->token);
            }
          } else {
            if (depth >= MAX_XML_DEPTH) return WGET_E_XML_MAX_DEPTH;

            int ret = parseXML(directory, context, depth + 1);
            if (ret != WGET_E_SUCCESS)
              return ret;
          }
          break;
        } else {
          char attribute[256];
          size_t attrlen = context->token_len >= sizeof(attribute) ? sizeof(attribute) - 1 : context->token_len;

          memcpy(attribute, tok, attrlen);
          attribute[attrlen] = 0;

          if (getValue(context) == EOF) return WGET_E_XML_PARSE_ERR; // syntax error

          if (context->token_len) {
            debug_printf("%s/@%s=%.*s\n", directory, attribute, (int)context->token_len, context->token);
            if (context->callback)
              context->callback(context->user_ctx, flags | XML_FLG_ATTRIBUTE, directory, attribute, context->token, context->token_len, context->token - context->buf);
          } else {
            debug_printf("%s/@%s\n", directory, attribute);
            if (context->callback)
              context->callback(context->user_ctx, flags | XML_FLG_ATTRIBUTE, directory, attribute, NULL, 0, 0);
          }
          flags = 0;
        }
      }
      directory[pos] = 0;
    } else if (context->token_len == 2) {
      if (!strncmp(tok, "</", 2)) {
        // ascend one level
        // cleanup - get name and '>'
        if (!(tok = getToken(context))) return WGET_E_XML_PARSE_ERR;
        // debug_printf("X Token %s\n",tok);
        if (context->callback) {
          if (!(context->hints & XML_HINT_HTML))
            context->callback(context->user_ctx, XML_FLG_END, directory, NULL, NULL, 0, 0);
          else {
            char tmp[128], *tag = tmp; // we need to \0 terminate tok
            if (context->token_len >= sizeof(tmp))
              tag = wget_malloc(context->token_len + 1);
            if (tag) {
              memcpy(tag, tok, context->token_len);
              tag[context->token_len] = 0;
              context->callback(context->user_ctx, XML_FLG_END, tag, NULL, NULL, 0, 0);
              if (tag != tmp)
                xfree(tag);
            }
          }
        }
        if (!(tok = getToken(context))) return WGET_E_XML_PARSE_ERR;
        // debug_printf("Y Token %s\n",tok);
        if (!(context->hints & XML_HINT_HTML))
          return WGET_E_SUCCESS;
        else
          continue;
      } else if (!strncmp(tok, "<?", 2)) { // special info - ignore
        getProcessing(context);
        debug_printf("%s=<?%.*s?>\n", directory, (int)context->token_len, context->token);
        continue;
      } else if (!strncmp(tok, "<!", 2)) {
        getSpecial(context);
        debug_printf("%s=<!%.*s>\n", directory, (int)context->token_len, context->token);
      }
    } else if (context->token_len == 4 && !strncmp(tok, "<!--", 4)) { // comment - ignore
      getComment(context);
      debug_printf("%s=<!--%.*s-->\n", directory, (int)context->token_len, context->token);
      continue;
    }
  } while (tok);
  return WGET_E_SUCCESS;
}

/**
 * \file
 * \brief XML parsing functions
 * \defgroup libwget-xml XML parsing functions
 * @{
 */

/**
 * \param[in] buf Zero-terminated XML or HTML input data
 * \param[in] callback Function called for each token scan result
 * \param[in] user_ctx User-defined context variable, handed to \p callback
 * \param[in] hints Flags to influence parsing
 *
 * This function scans the XML input from \p buf and calls \p callback for each token
 * found. \p user_ctx is a user-defined context variable and given to each call of \p callback.
 *
 * \p hints may be 0 or any combination of %XML_HINT_REMOVE_EMPTY_CONTENT and %XML_HINT_HTML.
 *
 * %XML_HINT_REMOVE_EMPTY_CONTENT reduces the number of calls to \p callback by ignoring
 * empty content and superfluous spaces.
 *
 * %XML_HINT_HTML turns on HTML scanning.
 */
int wget_xml_parse_buffer(
  const char *buf,
  wget_xml_callback *callback,
  void *user_ctx,
  int hints)
{
  xml_context context;

  context.token = NULL;
  context.token_size = 0;
  context.token_len = 0;
  context.buf = buf;
  context.p = buf;
  context.user_ctx = user_ctx;
  context.callback = callback;
  context.hints = hints;

  return parseXML ("/", &context, 0);
}

/**
 * \param[in] buf Zero-terminated HTML input data
 * \param[in] callback Function called for each token scan result
 * \param[in] user_ctx User-defined context variable, handed to \p callback
 * \param[in] hints Flags to influence parsing
 *
 * Convenience function that calls wget_xml_parse_buffer() with HTML parsing turned on.
 */
void wget_html_parse_buffer(
  const char *buf,
  wget_xml_callback *callback,
  void *user_ctx,
  int hints)
{
  wget_xml_parse_buffer(buf, callback, user_ctx, hints | XML_HINT_HTML);
}

/**
 * \param[in] fname Name of XML or HTML input file
 * \param[in] callback Function called for each token scan result
 * \param[in] user_ctx User-defined context variable, handed to \p callback
 * \param[in] hints Flags to influence parsing
 *
 * Convenience function that calls wget_xml_parse_buffer() with the file content.
 *
 * If \p fname is `-`, the data is read from stdin.
 */
void wget_xml_parse_file(
  const char *fname,
  wget_xml_callback *callback,
  void *user_ctx,
  int hints)
{
  if (strcmp(fname,"-")) {
    int fd;

    if ((fd = open(fname, O_RDONLY|O_BINARY)) != -1) {
      struct stat st;
      if (fstat(fd, &st) == 0) {
#ifdef HAVE_MMAP
        size_t nread = st.st_size;
        char *buf = mmap(NULL, nread + 1, PROT_READ|PROT_WRITE, MAP_PRIVATE, fd, 0);
#else
        char *buf = wget_malloc(st.st_size + 1);
        if (!buf)
          error_printf(_("Failed to allocate %zu bytes for XML parse buffer\n"), st.st_size + 1);
        size_t nread = buf ? read(fd, buf, st.st_size) : -1;
#endif

        if (nread > 0) {
          buf[nread] = 0; // PROT_WRITE allows this write, MAP_PRIVATE prevents changes in underlying file system
          wget_xml_parse_buffer(buf, callback, user_ctx, hints);
        }

#ifdef HAVE_MMAP
        munmap(buf, nread);
#else
        xfree(buf);
#endif
      }
      close(fd);
    } else
      error_printf(_("Failed to open %s\n"), fname);
  } else {
    // read data from STDIN.
    // maybe should use yy_scan_bytes instead of buffering into memory.
    char tmp[4096];
    ssize_t nbytes;
    wget_buffer buf;

    wget_buffer_init(&buf, NULL, 4096);

    while ((nbytes = read(STDIN_FILENO, tmp, sizeof(tmp))) > 0) {
      wget_buffer_memcat(&buf, tmp, nbytes);
    }

    if (buf.length)
      wget_xml_parse_buffer(buf.data, callback, user_ctx, hints);

    wget_buffer_deinit(&buf);
  }
}

/**
 * \param[in] fname Name of XML or HTML input file
 * \param[in] callback Function called for each token scan result
 * \param[in] user_ctx User-defined context variable, handed to \p callback
 * \param[in] hints Flags to influence parsing
 *
 * Convenience function that calls wget_xml_parse_file() with HTML parsing turned on.
 *
 * If \p fname is `-`, the data is read from stdin.
 */
void wget_html_parse_file(
  const char *fname,
  wget_xml_callback *callback,
  void *user_ctx,
  int hints)
{
  wget_xml_parse_file(fname, callback, user_ctx, hints | XML_HINT_HTML);
}

/**
 * \param[in] src A string
 * \return A pointer to \p src, after the XML entities have been converted
 *
 * Decode XML entities from \p src.
 *
 * **The transformation is done inline**, so `src` will be modified after this function returns.
 * If no XML entities have been found, \p src is left untouched.
 *
 * Only a small subset of available XML entities is currently recognized.
 */
char *wget_xml_decode_entities_inline(char *src)
{
  char *ret = NULL;
  unsigned char *s = (unsigned char *)src; // just a helper to avoid casting a lot
  unsigned char *d = s;

  while (*s) {
    if (*s == '&') {
      // entities are case sensitive (RFC1866, 3.2.3)
      if (s[1] == '#') {
        if (s[2] == 'x')
          *d = (unsigned char) strtol((char *) s + 3, (char **) &s, 16);
        else
          *d = (unsigned char) strtol((char *) s + 2, (char **) &s, 10);
        if (*d == ' ') *d = '+'; // hack
        d++;
        if (*s == ';') s++;
        ret = src;
        continue;
      } else if (!strncmp((char *) s + 1, "amp;", 4)) {
        *d++ = '&';
        s += 5;
        ret = src;
        continue;
      } else if (!strncmp((char *) s + 1, "gt;", 3)) {
        *d++ = '>';
        s += 4;
        ret = src;
        continue;
      } else if (!strncmp((char *) s + 1, "lt;", 3)) {
        *d++ = '<';
        s += 4;
        ret = src;
        continue;
      } else if (!strncmp((char *) s + 1, "quot;", 5)) {
        *d++ = '\"';
        s += 6;
        ret = src;
        continue;
      } else if (!strncmp((char *) s + 1, "apos;", 5)) {
        *d++ = '\'';
        s += 6;
        ret = src;
        continue;
      }
    }

    *d++ = *s++;
  }
  *d = 0;

  return ret;
}


/** @} */

Coverage Report

Created: 2026-08-14 06:46