/src/php-src/ext/standard/html.c

Source
/*
   +----------------------------------------------------------------------+
   | Copyright © The PHP Group and Contributors.                          |
   +----------------------------------------------------------------------+
   | This source file is subject to the Modified BSD License that is      |
   | bundled with this package in the file LICENSE, and is available      |
   | through the World Wide Web at <https://www.php.net/license/>.        |
   |                                                                      |
   | SPDX-License-Identifier: BSD-3-Clause                                |
   +----------------------------------------------------------------------+
   | Authors: Rasmus Lerdorf <rasmus@php.net>                             |
   |          Jaakko Hyvätti <jaakko.hyvatti@iki.fi>                      |
   |          Wez Furlong    <wez@thebrainroom.com>                       |
   |          Gustavo Lopes  <cataphract@php.net>                         |
   +----------------------------------------------------------------------+
*/

/*
 * HTML entity resources:
 *
 * http://www.unicode.org/Public/MAPPINGS/OBSOLETE/UNI2SGML.TXT
 *
 * XHTML 1.0 DTD
 * http://www.w3.org/TR/2002/REC-xhtml1-20020801/dtds.html#h-A2
 *
 * From HTML 4.01 strict DTD:
 * http://www.w3.org/TR/html4/HTMLlat1.ent
 * http://www.w3.org/TR/html4/HTMLsymbol.ent
 * http://www.w3.org/TR/html4/HTMLspecial.ent
 *
 * HTML 5:
 * http://dev.w3.org/html5/spec/Overview.html#named-character-references
 */

#include "php.h"
#ifdef PHP_WIN32
#include "config.w32.h"
#else
#include <php_config.h>
#endif
#include "php_standard.h"
#include "SAPI.h"
#include <locale.h>

#include <zend_hash.h>
#include "zend_bitset.h"
#include "zend_simd.h"
#include "html_tables.h"

/* Macro for disabling flag of translation of non-basic entities where this isn't supported.
 * Not appropriate for html_entity_decode/htmlspecialchars_decode */
#define LIMIT_ALL(all, doctype, charset) do { \
  (all) = (all) && !CHARSET_PARTIAL_SUPPORT((charset)) && ((doctype) != ENT_HTML_DOC_XML1); \
} while (0)

#define MB_FAILURE(pos, advance) do { \
  *cursor = pos + (advance); \
  *status = FAILURE; \
  return 0; \
} while (0)

#define CHECK_LEN(pos, chars_need) ((str_len - (pos)) >= (chars_need))

/* valid as single byte character or leading byte */
#define utf8_lead(c)  ((c) < 0x80 || ((c) >= 0xC2 && (c) <= 0xF4))
/* whether it's actually valid depends on other stuff;
 * this macro cannot check for non-shortest forms, surrogates or
 * code points above 0x10FFFF */
#define utf8_trail(c) ((c) >= 0x80 && (c) <= 0xBF)

#define gb2312_lead(c) ((c) != 0x8E && (c) != 0x8F && (c) != 0xA0 && (c) != 0xFF)
#define gb2312_trail(c) ((c) >= 0xA1 && (c) <= 0xFE)

#define sjis_lead(c) ((c) != 0x80 && (c) != 0xA0 && (c) < 0xFD)
#define sjis_trail(c) ((c) >= 0x40  && (c) != 0x7F && (c) < 0xFD)

/* {{{ get_default_charset */
static char *get_default_charset(void) {
  if (PG(internal_encoding) && PG(internal_encoding)[0]) {
    return PG(internal_encoding);
  } else if (SG(default_charset) && SG(default_charset)[0] ) {
    return SG(default_charset);
  }
  return NULL;
}
/* }}} */

/* UTF-8 case of get_next_char(), for a cursor before the end of str and a status set to SUCCESS */
static zend_always_inline unsigned int get_next_char_utf8(
    const unsigned char *str,
    size_t str_len,
    size_t *cursor,
    zend_result *status)
{
  size_t pos = *cursor;
  unsigned int this_char = 0;

  /* We'll follow strategy 2. from section 3.6.1 of UTR #36:
   * "In a reported illegal byte sequence, do not include any
   *  non-initial byte that encodes a valid character or is a leading
   *  byte for a valid sequence." */
  unsigned char c;
  c = str[pos];
  if (c < 0x80) {
    this_char = c;
    pos++;
  } else if (c < 0xc2) {
    MB_FAILURE(pos, 1);
  } else if (c < 0xe0) {
    if (!CHECK_LEN(pos, 2))
      MB_FAILURE(pos, 1);

    if (!utf8_trail(str[pos + 1])) {
      MB_FAILURE(pos, utf8_lead(str[pos + 1]) ? 1 : 2);
    }
    this_char = ((c & 0x1f) << 6) | (str[pos + 1] & 0x3f);
    if (this_char < 0x80) { /* non-shortest form */
      MB_FAILURE(pos, 2);
    }
    pos += 2;
  } else if (c < 0xf0) {
    size_t avail = str_len - pos;

    if (avail < 3 ||
        !utf8_trail(str[pos + 1]) || !utf8_trail(str[pos + 2])) {
      if (avail < 2 || utf8_lead(str[pos + 1]))
        MB_FAILURE(pos, 1);
      else if (avail < 3 || utf8_lead(str[pos + 2]))
        MB_FAILURE(pos, 2);
      else
        MB_FAILURE(pos, 3);
    }

    this_char = ((c & 0x0f) << 12) | ((str[pos + 1] & 0x3f) << 6) | (str[pos + 2] & 0x3f);
    if (this_char < 0x800) { /* non-shortest form */
      MB_FAILURE(pos, 3);
    } else if (this_char >= 0xd800 && this_char <= 0xdfff) { /* surrogate */
      MB_FAILURE(pos, 3);
    }
    pos += 3;
  } else if (c < 0xf5) {
    size_t avail = str_len - pos;

    if (avail < 4 ||
        !utf8_trail(str[pos + 1]) || !utf8_trail(str[pos + 2]) ||
        !utf8_trail(str[pos + 3])) {
      if (avail < 2 || utf8_lead(str[pos + 1]))
        MB_FAILURE(pos, 1);
      else if (avail < 3 || utf8_lead(str[pos + 2]))
        MB_FAILURE(pos, 2);
      else if (avail < 4 || utf8_lead(str[pos + 3]))
        MB_FAILURE(pos, 3);
      else
        MB_FAILURE(pos, 4);
    }

    this_char = ((c & 0x07) << 18) | ((str[pos + 1] & 0x3f) << 12) | ((str[pos + 2] & 0x3f) << 6) | (str[pos + 3] & 0x3f);
    if (this_char < 0x10000 || this_char > 0x10FFFF) { /* non-shortest form or outside range */
      MB_FAILURE(pos, 4);
    }
    pos += 4;
  } else {
    MB_FAILURE(pos, 1);
  }

  *cursor = pos;
  return this_char;
}

/* {{{ get_next_char */
static inline unsigned int get_next_char(
    enum entity_charset charset,
    const unsigned char *str,
    size_t str_len,
    size_t *cursor,
    zend_result *status)
{
  size_t pos = *cursor;
  unsigned int this_char = 0;

  *status = SUCCESS;
  assert(pos <= str_len);

  if (!CHECK_LEN(pos, 1))
    MB_FAILURE(pos, 1);

  switch (charset) {
  case cs_utf_8:
    return get_next_char_utf8(str, str_len, cursor, status);

  case cs_big5:
    /* reference http://demo.icu-project.org/icu-bin/convexp?conv=big5 */
    {
      unsigned char c = str[pos];
      if (c >= 0x81 && c <= 0xFE) {
        unsigned char next;
        if (!CHECK_LEN(pos, 2))
          MB_FAILURE(pos, 1);

        next = str[pos + 1];

        if ((next >= 0x40 && next <= 0x7E) ||
            (next >= 0xA1 && next <= 0xFE)) {
          this_char = (c << 8) | next;
        } else {
          MB_FAILURE(pos, 1);
        }
        pos += 2;
      } else {
        this_char = c;
        pos += 1;
      }
    }
    break;

  case cs_big5hkscs:
    {
      unsigned char c = str[pos];
      if (c >= 0x81 && c <= 0xFE) {
        unsigned char next;
        if (!CHECK_LEN(pos, 2))
          MB_FAILURE(pos, 1);

        next = str[pos + 1];

        if ((next >= 0x40 && next <= 0x7E) ||
            (next >= 0xA1 && next <= 0xFE)) {
          this_char = (c << 8) | next;
        } else if (next != 0x80 && next != 0xFF) {
          MB_FAILURE(pos, 1);
        } else {
          MB_FAILURE(pos, 2);
        }
        pos += 2;
      } else {
        this_char = c;
        pos += 1;
      }
    }
    break;

  case cs_gb2312: /* EUC-CN */
    {
      unsigned char c = str[pos];
      if (c >= 0xA1 && c <= 0xFE) {
        unsigned char next;
        if (!CHECK_LEN(pos, 2))
          MB_FAILURE(pos, 1);

        next = str[pos + 1];

        if (gb2312_trail(next)) {
          this_char = (c << 8) | next;
        } else if (gb2312_lead(next)) {
          MB_FAILURE(pos, 1);
        } else {
          MB_FAILURE(pos, 2);
        }
        pos += 2;
      } else if (gb2312_lead(c)) {
        this_char = c;
        pos += 1;
      } else {
        MB_FAILURE(pos, 1);
      }
    }
    break;

  case cs_sjis:
    {
      unsigned char c = str[pos];
      if ((c >= 0x81 && c <= 0x9F) || (c >= 0xE0 && c <= 0xFC)) {
        unsigned char next;
        if (!CHECK_LEN(pos, 2))
          MB_FAILURE(pos, 1);

        next = str[pos + 1];

        if (sjis_trail(next)) {
          this_char = (c << 8) | next;
        } else if (sjis_lead(next)) {
          MB_FAILURE(pos, 1);
        } else {
          MB_FAILURE(pos, 2);
        }
        pos += 2;
      } else if (c < 0x80 || (c >= 0xA1 && c <= 0xDF)) {
        this_char = c;
        pos += 1;
      } else {
        MB_FAILURE(pos, 1);
      }
    }
    break;

  case cs_eucjp:
    {
      unsigned char c = str[pos];

      if (c >= 0xA1 && c <= 0xFE) {
        unsigned next;
        if (!CHECK_LEN(pos, 2))
          MB_FAILURE(pos, 1);
        next = str[pos + 1];

        if (next >= 0xA1 && next <= 0xFE) {
          /* this a jis kanji char */
          this_char = (c << 8) | next;
        } else {
          MB_FAILURE(pos, (next != 0xA0 && next != 0xFF) ? 1 : 2);
        }
        pos += 2;
      } else if (c == 0x8E) {
        unsigned next;
        if (!CHECK_LEN(pos, 2))
          MB_FAILURE(pos, 1);

        next = str[pos + 1];
        if (next >= 0xA1 && next <= 0xDF) {
          /* JIS X 0201 kana */
          this_char = (c << 8) | next;
        } else {
          MB_FAILURE(pos, (next != 0xA0 && next != 0xFF) ? 1 : 2);
        }
        pos += 2;
      } else if (c == 0x8F) {
        size_t avail = str_len - pos;

        if (avail < 3 || !(str[pos + 1] >= 0xA1 && str[pos + 1] <= 0xFE) ||
            !(str[pos + 2] >= 0xA1 && str[pos + 2] <= 0xFE)) {
          if (avail < 2 || (str[pos + 1] != 0xA0 && str[pos + 1] != 0xFF))
            MB_FAILURE(pos, 1);
          else if (avail < 3 || (str[pos + 2] != 0xA0 && str[pos + 2] != 0xFF))
            MB_FAILURE(pos, 2);
          else
            MB_FAILURE(pos, 3);
        } else {
          /* JIS X 0212 hojo-kanji */
          this_char = (c << 16) | (str[pos + 1] << 8) | str[pos + 2];
        }
        pos += 3;
      } else if (c != 0xA0 && c != 0xFF) {
        /* character encoded in 1 code unit */
        this_char = c;
        pos += 1;
      } else {
        MB_FAILURE(pos, 1);
      }
    }
    break;
  default:
    /* single-byte charsets */
    this_char = str[pos++];
    break;
  }

  *cursor = pos;
  return this_char;
}
/* }}} */

/* {{{ php_next_utf8_char
 * Public interface for get_next_char used with UTF-8 */
PHPAPI unsigned int php_next_utf8_char(
    const unsigned char *str,
    size_t str_len,
    size_t *cursor,
    zend_result *status)
{
  return get_next_char(cs_utf_8, str, str_len, cursor, status);
}
/* }}} */

/* Returns whether a charset name is "utf-8", in any case */
static zend_always_inline bool charset_is_utf8(const char *charset_hint)
{
  return zend_tolower_ascii(charset_hint[0]) == 'u' && zend_tolower_ascii(charset_hint[1]) == 't'
    && zend_tolower_ascii(charset_hint[2]) == 'f' && charset_hint[3] == '-' && charset_hint[4] == '8'
    && charset_hint[5] == '\0';
}

/* Returns the charset identifier of a non-empty charset name, or UTF-8 if it is not supported. */
static zend_never_inline enum entity_charset lookup_charset(const char *charset_hint, bool quiet)
{
  /* the codesets of the charset map are lowercase, so lowercase the hint once */
  char lowercase_hint[LONGEST_CODESET_LENGTH];
  size_t len;

  for (len = 0; charset_hint[len]; len++) {
    if (len == LONGEST_CODESET_LENGTH) {
      /* longer than any codeset */
      goto not_supported;
    }
    lowercase_hint[len] = zend_tolower_ascii(charset_hint[len]);
  }

  /* now walk the charset map and look for the codeset (UTF-8 is resolved by the caller) */
  for (size_t i = 0; i < sizeof(charset_map)/sizeof(charset_map[0]); i++) {
    if (charset_map[i].codeset_len == len && charset_map[i].codeset[0] == lowercase_hint[0]
        && memcmp(lowercase_hint, charset_map[i].codeset, len) == 0) {
      return charset_map[i].charset;
    }
  }

not_supported:
  if (!quiet) {
    php_error_docref(NULL, E_WARNING, "Charset \"%s\" is not supported, assuming UTF-8",
        charset_hint);
  }

  return cs_utf_8;
}

/* {{{ entity_charset determine_charset
 * Returns the charset identifier based on an explicitly provided charset,
 * the internal_encoding and default_charset ini settings, or UTF-8 by default. */
static enum entity_charset determine_charset(const char *charset_hint, bool quiet)
{
  if (!charset_hint || !*charset_hint) {
    charset_hint = get_default_charset();
    if (!charset_hint) {
      return cs_utf_8;
    }
  }

  /* UTF-8 is the most used charset, and it is not in the charset map */
  if (charset_is_utf8(charset_hint)) {
    return cs_utf_8;
  }

  return lookup_charset(charset_hint, quiet);
}
/* }}} */

/* {{{ php_utf32_utf8 */
static inline size_t php_utf32_utf8(unsigned char *buf, unsigned k)
{
  size_t retval = 0;

  /* assert(0x0 <= k <= 0x10FFFF); */

  if (k < 0x80) {
    buf[0] = k;
    retval = 1;
  } else if (k < 0x800) {
    buf[0] = 0xc0 | (k >> 6);
    buf[1] = 0x80 | (k & 0x3f);
    retval = 2;
  } else if (k < 0x10000) {
    buf[0] = 0xe0 | (k >> 12);
    buf[1] = 0x80 | ((k >> 6) & 0x3f);
    buf[2] = 0x80 | (k & 0x3f);
    retval = 3;
  } else {
    buf[0] = 0xf0 | (k >> 18);
    buf[1] = 0x80 | ((k >> 12) & 0x3f);
    buf[2] = 0x80 | ((k >> 6) & 0x3f);
    buf[3] = 0x80 | (k & 0x3f);
    retval = 4;
  }
  /* UTF-8 has been restricted to max 4 bytes since RFC 3629 */

  return retval;
}
/* }}} */

/* {{{ unimap_bsearc_cmp
 * Binary search of unicode code points in unicode <--> charset mapping.
 * Returns the code point in the target charset (whose mapping table was given) or 0 if
 * the unicode code point is not in the table.
 */
static inline unsigned char unimap_bsearch(const uni_to_enc *table, unsigned code_key_a, size_t num)
{
  const uni_to_enc *l = table,
           *h = &table[num-1],
           *m;
  unsigned short code_key;

  /* we have no mappings outside the BMP */
  if (code_key_a > 0xFFFFU)
    return 0;

  code_key = (unsigned short) code_key_a;

  while (l <= h) {
    m = l + (h - l) / 2;
    if (code_key < m->un_code_point)
      h = m - 1;
    else if (code_key > m->un_code_point)
      l = m + 1;
    else
      return m->cs_code;
  }
  return 0;
}
/* }}} */

/* {{{ map_from_unicode */
static inline zend_result map_from_unicode(unsigned code, enum entity_charset charset, unsigned *res)
{
  unsigned char found;
  const uni_to_enc *table;
  size_t table_size;

  switch (charset) {
  case cs_8859_1:
    /* identity mapping of code points to unicode */
    if (code > 0xFF) {
      return FAILURE;
    }
    *res = code;
    break;

  case cs_8859_5:
    if (code <= 0xA0 || code == 0xAD /* soft hyphen */) {
      *res = code;
    } else if (code == 0x2116) {
      *res = 0xF0; /* numero sign */
    } else if (code == 0xA7) {
      *res = 0xFD; /* section sign */
    } else if (code >= 0x0401 && code <= 0x045F) {
      if (code == 0x040D || code == 0x0450 || code == 0x045D)
        return FAILURE;
      *res = code - 0x360;
    } else {
      return FAILURE;
    }
    break;

  case cs_8859_15:
    if (code < 0xA4 || (code > 0xBE && code <= 0xFF)) {
      *res = code;
    } else { /* between A4 and 0xBE */
      found = unimap_bsearch(unimap_iso885915,
        code, sizeof(unimap_iso885915) / sizeof(*unimap_iso885915));
      if (found)
        *res = found;
      else
        return FAILURE;
    }
    break;

  case cs_cp1252:
    if (code <= 0x7F || (code >= 0xA0 && code <= 0xFF)) {
      *res = code;
    } else {
      found = unimap_bsearch(unimap_win1252,
        code, sizeof(unimap_win1252) / sizeof(*unimap_win1252));
      if (found)
        *res = found;
      else
        return FAILURE;
    }
    break;

  case cs_macroman:
    if (code == 0x7F)
      return FAILURE;
    table = unimap_macroman;
    table_size = sizeof(unimap_macroman) / sizeof(*unimap_macroman);
    goto table_over_7F;
  case cs_cp1251:
    table = unimap_win1251;
    table_size = sizeof(unimap_win1251) / sizeof(*unimap_win1251);
    goto table_over_7F;
  case cs_koi8r:
    table = unimap_koi8r;
    table_size = sizeof(unimap_koi8r) / sizeof(*unimap_koi8r);
    goto table_over_7F;
  case cs_cp866:
    table = unimap_cp866;
    table_size = sizeof(unimap_cp866) / sizeof(*unimap_cp866);

table_over_7F:
    if (code <= 0x7F) {
      *res = code;
    } else {
      found = unimap_bsearch(table, code, table_size);
      if (found)
        *res = found;
      else
        return FAILURE;
    }
    break;

  /* from here on, only map the possible characters in the ASCII range.
   * to improve support here, it's a matter of building the unicode mappings.
   * See <http://www.unicode.org/Public/6.0.0/ucd/Unihan.zip> */
  case cs_sjis:
  case cs_eucjp:
    /* we interpret 0x5C as the Yen symbol. This is not universal.
     * See <http://www.w3.org/Submission/japanese-xml/#ambiguity_of_yen> */
    if (code >= 0x20 && code <= 0x7D) {
      if (code == 0x5C)
        return FAILURE;
      *res = code;
    } else {
      return FAILURE;
    }
    break;

  case cs_big5:
  case cs_big5hkscs:
  case cs_gb2312:
    if (code >= 0x20 && code <= 0x7D) {
      *res = code;
    } else {
      return FAILURE;
    }
    break;

  default:
    return FAILURE;
  }

  return SUCCESS;
}
/* }}} */

/* {{{ */
static inline void map_to_unicode(unsigned code, const enc_to_uni *table, unsigned *res)
{
  /* only single byte encodings are currently supported; assumed code <= 0xFF */
  *res = table->inner[ENT_ENC_TO_UNI_STAGE1(code)]->uni_cp[ENT_ENC_TO_UNI_STAGE2(code)];
}
/* }}} */

/* {{{ unicode_cp_is_allowed */
static inline int unicode_cp_is_allowed(unsigned uni_cp, int document_type)
{
  /* XML 1.0        HTML 4.01     HTML 5
   * 0x09..0x0A     0x09..0x0A      0x09..0x0A
   * 0x0D         0x0D        0x0C..0x0D
   * 0x0020..0xD7FF   0x20..0x7E      0x20..0x7E
   *            0x00A0..0xD7FF    0x00A0..0xD7FF
   * 0xE000..0xFFFD   0xE000..0x10FFFF  0xE000..0xFDCF
   * 0x010000..0x10FFFF           0xFDF0..0x10FFFF (*)
   *
   * (*) exclude code points where ((code & 0xFFFF) >= 0xFFFE)
   *
   * References:
   * XML 1.0:   <http://www.w3.org/TR/REC-xml/#charsets>
   * HTML 4.01: <http://www.w3.org/TR/1999/PR-html40-19990824/sgml/sgmldecl.html>
   * HTML 5:    <http://dev.w3.org/html5/spec/Overview.html#preprocessing-the-input-stream>
   *
   * Not sure this is the relevant part for HTML 5, though. I opted to
   * disallow the characters that would result in a parse error when
   * preprocessing of the input stream. See also section 8.1.3.
   *
   * It's unclear if XHTML 1.0 allows C1 characters. I'll opt to apply to
   * XHTML 1.0 the same rules as for XML 1.0.
   * See <http://cmsmcq.com/2007/C1.xml>.
   */

  switch (document_type) {
  case ENT_HTML_DOC_HTML401:
    return (uni_cp >= 0x20 && uni_cp <= 0x7E) ||
      (uni_cp == 0x0A || uni_cp == 0x09 || uni_cp == 0x0D) ||
      (uni_cp >= 0xA0 && uni_cp <= 0xD7FF) ||
      (uni_cp >= 0xE000 && uni_cp <= 0x10FFFF);
  case ENT_HTML_DOC_HTML5:
    return (uni_cp >= 0x20 && uni_cp <= 0x7E) ||
      (uni_cp >= 0x09 && uni_cp <= 0x0D && uni_cp != 0x0B) || /* form feed U+0C allowed */
      (uni_cp >= 0xA0 && uni_cp <= 0xD7FF) ||
      (uni_cp >= 0xE000 && uni_cp <= 0x10FFFF &&
        ((uni_cp & 0xFFFF) < 0xFFFE) && /* last two of each plane (nonchars) disallowed */
        (uni_cp < 0xFDD0 || uni_cp > 0xFDEF)); /* U+FDD0-U+FDEF (nonchars) disallowed */
  case ENT_HTML_DOC_XHTML:
  case ENT_HTML_DOC_XML1:
    return (uni_cp >= 0x20 && uni_cp <= 0xD7FF) ||
      (uni_cp == 0x0A || uni_cp == 0x09 || uni_cp == 0x0D) ||
      (uni_cp >= 0xE000 && uni_cp <= 0x10FFFF && uni_cp != 0xFFFE && uni_cp != 0xFFFF);
  default:
    return 1;
  }
}
/* }}} */

/* {{{ unicode_cp_is_allowed */
static inline int numeric_entity_is_allowed(unsigned uni_cp, int document_type)
{
  /* less restrictive than unicode_cp_is_allowed */
  switch (document_type) {
  case ENT_HTML_DOC_HTML401:
    /* all non-SGML characters (those marked with UNUSED in DESCSET) should be
     * representable with numeric entities */
    return uni_cp <= 0x10FFFF;
  case ENT_HTML_DOC_HTML5:
    /* 8.1.4. The numeric character reference forms described above are allowed to
     * reference any Unicode code point other than U+0000, U+000D, permanently
     * undefined Unicode characters (noncharacters), and control characters other
     * than space characters (U+0009, U+000A, U+000C and U+000D) */
    /* seems to allow surrogate characters, then */
    return (uni_cp >= 0x20 && uni_cp <= 0x7E) ||
      (uni_cp >= 0x09 && uni_cp <= 0x0C && uni_cp != 0x0B) || /* form feed U+0C allowed, but not U+0D */
      (uni_cp >= 0xA0 && uni_cp <= 0x10FFFF &&
        ((uni_cp & 0xFFFF) < 0xFFFE) && /* last two of each plane (nonchars) disallowed */
        (uni_cp < 0xFDD0 || uni_cp > 0xFDEF)); /* U+FDD0-U+FDEF (nonchars) disallowed */
  case ENT_HTML_DOC_XHTML:
  case ENT_HTML_DOC_XML1:
    /* OTOH, XML 1.0 requires "character references to match the production for Char
     * See <http://www.w3.org/TR/REC-xml/#NT-CharRef> */
    return unicode_cp_is_allowed(uni_cp, document_type);
  default:
    return 1;
  }
}
/* }}} */

/* {{{ process_numeric_entity
 * Auxiliary function to traverse_for_entities.
 * On input, *buf should point to the first character after # and on output, it's the last
 * byte read, no matter if there was success or insuccess.
 */
static inline zend_result process_numeric_entity(const char **buf, unsigned *code_point)
{
  zend_long code_l;
  int hexadecimal = (**buf == 'x' || **buf == 'X'); /* TODO: XML apparently disallows "X" */
  char *endptr;

  if (hexadecimal)
    (*buf)++;

  /* strtol allows whitespace and other stuff in the beginning
    * we're not interested */
  if ((hexadecimal && !isxdigit((unsigned char)**buf)) ||
      (!hexadecimal && !isdigit((unsigned char)**buf))) {
    return FAILURE;
  }

  code_l = ZEND_STRTOL(*buf, &endptr, hexadecimal ? 16 : 10);
  /* we're guaranteed there were valid digits, so *endptr > buf */
  *buf = endptr;

  if (**buf != ';')
    return FAILURE;

  /* many more are invalid, but that depends on whether it's HTML
   * (and which version) or XML. */
  if (code_l > Z_L(0x10FFFF))
    return FAILURE;

  if (code_point != NULL)
    *code_point = (unsigned)code_l;

  return SUCCESS;
}
/* }}} */

/* {{{ process_named_entity */
static inline zend_result process_named_entity_html(const char **buf, const char **start, size_t *length)
{
  *start = *buf;

  /* "&" is represented by a 0x26 in all supported encodings. That means
   * the byte after represents a character or is the leading byte of a
   * sequence of 8-bit code units. If in the ranges below, it represents
   * necessarily an alpha character because none of the supported encodings
   * has an overlap with ASCII in the leading byte (only on the second one) */
  while ((**buf >= 'a' && **buf <= 'z') ||
      (**buf >= 'A' && **buf <= 'Z') ||
      (**buf >= '0' && **buf <= '9')) {
    (*buf)++;
  }

  if (**buf != ';')
    return FAILURE;

  /* cast to size_t OK as the quantity is always non-negative */
  *length = *buf - *start;

  if (*length == 0)
    return FAILURE;

  return SUCCESS;
}
/* }}} */

/* {{{ resolve_named_entity_html */
static zend_result resolve_named_entity_html(const char *start, size_t length, const entity_ht *ht, unsigned *uni_cp1, unsigned *uni_cp2)
{
  const entity_cp_map *s;
  zend_ulong hash = zend_inline_hash_func(start, length);

  s = ht->buckets[hash % ht->num_elems];
  while (s->entity) {
    if (s->entity_len == length) {
      if (memcmp(start, s->entity, length) == 0) {
        *uni_cp1 = s->codepoint1;
        *uni_cp2 = s->codepoint2;
        return SUCCESS;
      }
    }
    s++;
  }
  return FAILURE;
}
/* }}} */

static inline size_t write_octet_sequence(unsigned char *buf, enum entity_charset charset, unsigned code) {
  /* code is not necessarily a unicode code point */
  switch (charset) {
  case cs_utf_8:
    return php_utf32_utf8(buf, code);

  case cs_8859_1:
  case cs_cp1252:
  case cs_8859_15:
  case cs_koi8r:
  case cs_cp1251:
  case cs_8859_5:
  case cs_cp866:
  case cs_macroman:
    /* single byte stuff */
    *buf = code;
    return 1;

  case cs_big5:
  case cs_big5hkscs:
  case cs_sjis:
  case cs_gb2312:
    /* we don't have complete unicode mappings for these yet in entity_decode,
     * and we opt to pass through the octet sequences for these in htmlentities
     * instead of converting to an int and then converting back. */
#if 0
    return php_mb2_int_to_char(buf, code);
#else
    ZEND_ASSERT(code <= 0xFFU);
    *buf = code;
    return 1;
#endif

  case cs_eucjp:
#if 0 /* idem */
    return php_mb2_int_to_char(buf, code);
#else
    ZEND_ASSERT(code <= 0xFFU);
    *buf = code;
    return 1;
#endif

  default:
    assert(0);
    return 0;
  }
}

/* [0-9A-Za-z], independent of the locale */
static zend_always_inline bool is_entity_name_byte(unsigned char c)
{
  return (unsigned char)((c | 0x20) - 'a') < 26 || (unsigned char)(c - '0') < 10;
}

/* {{{ traverse_for_entities
 * Auxiliary function to php_unescape_html_entities().
 * - The argument "all" determines if all numeric entities are decode or only those
 *   that correspond to quotes (depending on quote_style).
 */
/* maximum expansion (factor 1.2) for HTML 5 with &nGt; and &nLt; */
/* +2 is 1 because of rest (probably unnecessary), 1 because of terminating 0 */
#define TRAVERSE_FOR_ENTITIES_EXPAND_SIZE(oldlen) ((oldlen) + (oldlen) / 5 + 2)
static void traverse_for_entities(
  const zend_string *input,
  zend_string *output, /* should have allocated TRAVERSE_FOR_ENTITIES_EXPAND_SIZE(olden) */
  const int all,
  const int flags,
  const entity_ht *inv_map,
  const enum entity_charset charset)
{
  const char *current_ptr = ZSTR_VAL(input);
  const char *input_end   = current_ptr + ZSTR_LEN(input); /* terminator address */
  char *output_ptr    = ZSTR_VAL(output);
  const int doctype    = flags & ENT_HTML_DOC_TYPE_MASK;

  while (current_ptr < input_end) {
    /* Runs of '&' would call memchr() for every byte */
    const char *ampersand_ptr = *current_ptr == '&' ? current_ptr : memchr(current_ptr, '&', input_end - current_ptr);
    if (!ampersand_ptr) {
      const size_t tail_len = input_end - current_ptr;
      if (tail_len > 0) {
        memcpy(output_ptr, current_ptr, tail_len);
        output_ptr += tail_len;
      }
      break;
    }

    /* Copy everything up to the found '&' */
    const size_t chunk_len = ampersand_ptr - current_ptr;
    if (chunk_len > 0) {
      memcpy(output_ptr, current_ptr, chunk_len);
      output_ptr += chunk_len;
    }

    /* Now current_ptr points to the '&' character. */
    current_ptr = ampersand_ptr;

    /* If there are less than 4 bytes remaining, there isn't enough for an entity - 
     * copy '&' as a normal character. */
    if (input_end - current_ptr < 4) {
      const size_t remaining = input_end - current_ptr;
      memcpy(output_ptr, current_ptr, remaining);
      output_ptr += remaining;
      break;
    }

    unsigned code = 0, code2 = 0;
    const char *entity_end_ptr = NULL;

    if (current_ptr[1] == '#') {
      /* Processing numeric entity */
      const char *num_start = current_ptr + 2;
      entity_end_ptr = num_start;
      if (process_numeric_entity(&entity_end_ptr, &code) == FAILURE) {
        goto invalid_incomplete_entity;
      }
      if (!all && (code > 63U || stage3_table_be_apos_00000[code].data.ent.entity == NULL)) {
        /* If we're in htmlspecialchars_decode, we're only decoding entities
         * that represent &, <, >, " and '. Is this one of them? */
        goto invalid_incomplete_entity;
      } else if (!unicode_cp_is_allowed(code, doctype) ||
             (doctype == ENT_HTML_DOC_HTML5 && code == 0x0D)) {
        /* are we allowed to decode this entity in this document type?
         * HTML 5 is the only that has a character that cannot be used in
         * a numeric entity but is allowed literally (U+000D). The
         * unoptimized version would be ... || !numeric_entity_is_allowed(code) */
        goto invalid_incomplete_entity;
      }
    } else {
      /* Processing named entity */
      const char *name_start = current_ptr + 1;
      const char *semi_colon_ptr = name_start;
      const char *const name_limit = name_start + MIN(LONGEST_ENTITY_LENGTH, input_end - name_start - 1);
      /* A name is letters and digits only, so the first other byte must be the ';' */
      while (semi_colon_ptr < name_limit && is_entity_name_byte(*semi_colon_ptr)) {
        semi_colon_ptr++;
      }
      if (*semi_colon_ptr != ';') {
        goto invalid_incomplete_entity;
      } else {
        const size_t name_len = semi_colon_ptr - name_start;
        if (name_len == 0) {
          goto invalid_incomplete_entity;
        } else {
          if (resolve_named_entity_html(name_start, name_len, inv_map, &code, &code2) == FAILURE) {
            if (doctype == ENT_HTML_DOC_XHTML && name_len == 4 &&
              name_start[0] == 'a' && name_start[1] == 'p' &&
              name_start[2] == 'o' && name_start[3] == 's')
            {
              /* uses html4 inv_map, which doesn't include apos;. This is a
               * hack to support it */
              code = (unsigned)'\'';
            } else {
              goto invalid_incomplete_entity;
            }
          }
          entity_end_ptr = semi_colon_ptr;
        }
      }
    }

    /* At this stage the entity_end_ptr should be always set. */
    ZEND_ASSERT(entity_end_ptr != NULL);

    /* Check if quotes are allowed for entities representing ' or " */
    if ((code == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
      (code == '"'  && !(flags & ENT_HTML_QUOTE_DOUBLE)))
    {
      goto invalid_complete_entity;
    }

    /* UTF-8 doesn't need mapping (ISO-8859-1 doesn't either, but
     * the call is needed to ensure the codepoint <= U+00FF)  */
    if (charset != cs_utf_8) {
      /* replace unicode code point */
      if (map_from_unicode(code, charset, &code) == FAILURE || code2 != 0) {
        goto invalid_complete_entity;
      }
    }

    /* Write the parsed entity into the output buffer */
    output_ptr += write_octet_sequence((unsigned char*)output_ptr, charset, code);
    if (code2) {
      output_ptr += write_octet_sequence((unsigned char*)output_ptr, charset, code2);
    }
    /* Move current_ptr past the semicolon */
    current_ptr = entity_end_ptr + 1;
    continue;

invalid_incomplete_entity:
    /* If the entity is invalid at parse stage or entity_end_ptr was never found, copy '&' as normal */
    *output_ptr++ = *current_ptr++;
    continue;

invalid_complete_entity:
    /* If the entity became invalid after we found entity_end_ptr */
    if (entity_end_ptr) {
      const size_t len = entity_end_ptr - current_ptr;
      memcpy(output_ptr, current_ptr, len);
      output_ptr += len;
      current_ptr = entity_end_ptr;
    } else {
      *output_ptr++ = *current_ptr++;
    }
    continue;
  }

  *output_ptr = '\0';
  ZSTR_LEN(output) = (size_t)(output_ptr - ZSTR_VAL(output));
}
/* }}} */

/* {{{ unescape_inverse_map */
static const entity_ht *unescape_inverse_map(int all, int flags)
{
  int document_type = flags & ENT_HTML_DOC_TYPE_MASK;

  if (all) {
    switch (document_type) {
    case ENT_HTML_DOC_HTML401:
    case ENT_HTML_DOC_XHTML: /* but watch out for &apos;...*/
      return &ent_ht_html4;
    case ENT_HTML_DOC_HTML5:
      return &ent_ht_html5;
    default:
      return &ent_ht_be_apos;
    }
  } else {
    switch (document_type) {
    case ENT_HTML_DOC_HTML401:
      return &ent_ht_be_noapos;
    default:
      return &ent_ht_be_apos;
    }
  }
}
/* }}} */

/* {{{ determine_entity_table
 * Entity table to use. Note that entity tables are defined in terms of
 * unicode code points */
static entity_table_opt determine_entity_table(int all, int doctype)
{
  entity_table_opt retval = {0};

  assert(!(doctype == ENT_HTML_DOC_XML1 && all));

  if (all) {
    retval.ms_table = (doctype == ENT_HTML_DOC_HTML5) ?
      entity_ms_table_html5 : entity_ms_table_html4;
  } else {
    retval.table = (doctype == ENT_HTML_DOC_HTML401) ?
      stage3_table_be_noapos_00000 : stage3_table_be_apos_00000;
  }
  return retval;
}
/* }}} */

/* {{{ php_unescape_html_entities
 * The parameter "all" should be true to decode all possible entities, false to decode
 * only the basic ones, i.e., those in basic_entities_ex + the numeric entities
 * that correspond to quotes.
 */
PHPAPI zend_string *php_unescape_html_entities(zend_string *str, int all, int flags, const char *hint_charset)
{
  zend_string *ret;
  enum entity_charset charset;
  const entity_ht *inverse_map;
  size_t new_size;

  if (!memchr(ZSTR_VAL(str), '&', ZSTR_LEN(str))) {
    return zend_string_copy(str);
  }

  if (all) {
    charset = determine_charset(hint_charset, /* quiet */ false);
  } else {
    charset = cs_8859_1; /* charset shouldn't matter, use ISO-8859-1 for performance */
  }

  /* don't use LIMIT_ALL! */

  new_size = TRAVERSE_FOR_ENTITIES_EXPAND_SIZE(ZSTR_LEN(str));
  if (ZSTR_LEN(str) > new_size) {
    /* overflow, refuse to do anything */
    return zend_string_copy(str);
  }

  ret = zend_string_alloc(new_size, 0);

  inverse_map = unescape_inverse_map(all, flags);

  /* replace numeric entities */
  traverse_for_entities(str, ret, all, flags, inverse_map, charset);

  return ret;
}
/* }}} */

PHPAPI zend_string *php_escape_html_entities(const unsigned char *old, size_t oldlen, int all, int flags, const char *hint_charset)
{
  return php_escape_html_entities_ex(old, oldlen, all, flags, hint_charset, true, /* quiet */ false);
}

/* {{{ find_entity_for_char */
static inline void find_entity_for_char(
  unsigned int k,
  enum entity_charset charset,
  const entity_stage1_row *table,
  const unsigned char **entity,
  size_t *entity_len,
  const unsigned char *old,
  size_t oldlen,
  size_t *cursor)
{
  unsigned stage1_idx = ENT_STAGE1_INDEX(k);
  const entity_stage3_row *c;

  if (stage1_idx > 0x1D) {
    *entity     = NULL;
    *entity_len = 0;
    return;
  }

  c = &table[stage1_idx][ENT_STAGE2_INDEX(k)][ENT_STAGE3_INDEX(k)];

  if (!c->ambiguous) {
    *entity     = (const unsigned char *)c->data.ent.entity;
    *entity_len = c->data.ent.entity_len;
  } else {
    /* peek at next char */
    size_t cursor_before = *cursor;
    zend_result status = SUCCESS;
    unsigned next_char;

    if (!(*cursor < oldlen))
      goto no_suitable_2nd;

    next_char = get_next_char(charset, old, oldlen, cursor, &status);

    if (status == FAILURE)
      goto no_suitable_2nd;

    {
      const entity_multicodepoint_row *s, *e;

      s = &c->data.multicodepoint_table[1];
      e = s - 1 + c->data.multicodepoint_table[0].leading_entry.size;
      /* we could do a binary search but it's not worth it since we have
       * at most two entries... */
      for ( ; s <= e; s++) {
        if (s->normal_entry.second_cp == next_char) {
          *entity     = (const unsigned char *) s->normal_entry.entity;
          *entity_len = s->normal_entry.entity_len;
          return;
        }
      }
    }
no_suitable_2nd:
    *cursor = cursor_before;
    *entity = (const unsigned char *)
      c->data.multicodepoint_table[0].leading_entry.default_entity;
    *entity_len = c->data.multicodepoint_table[0].leading_entry.default_entity_len;
  }
}
/* }}} */

/* {{{ find_entity_for_char_basic */
static inline void find_entity_for_char_basic(
  unsigned int k,
  const entity_stage3_row *table,
  const unsigned char **entity,
  size_t *entity_len)
{
  if (k >= 64U) {
    *entity     = NULL;
    *entity_len = 0;
    return;
  }

  *entity     = (const unsigned char *) table[k].data.ent.entity;
  *entity_len = table[k].data.ent.entity_len;
}
/* }}} */

/* Character classes used by html_verbatim_prefix_len(). The quote classes are the flags
 * that make the quotes encoded. */
#define HTML_CC_SQUOTE    (1<<0) /* ' */
#define HTML_CC_DQUOTE    (1<<1) /* " */
#define HTML_CC_BASIC     (1<<2) /* &, < and >, always encoded */
#define HTML_CC_NON_ASCII (1<<3) /* bytes 0x80-0xFF */

static_assert(ENT_HTML_QUOTE_SINGLE == HTML_CC_SQUOTE && ENT_HTML_QUOTE_DOUBLE == HTML_CC_DQUOTE, "the quote classes must match the quote flags");

#define N HTML_CC_NON_ASCII
static const unsigned char html_char_class[256] = {
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, HTML_CC_DQUOTE, 0, 0, 0, HTML_CC_BASIC, HTML_CC_SQUOTE, 0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, HTML_CC_BASIC, 0, HTML_CC_BASIC, 0,
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
  N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
  N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
  N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
  N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
  N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
  N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
  N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
};
#undef N

/* Moves *pos after the UTF-8 character at *pos, or returns false if it is invalid */
static zend_always_inline bool html_skip_utf8_char(const unsigned char *str, size_t str_len, size_t *pos)
{
  size_t cursor = *pos;
  zend_result status = SUCCESS;

  get_next_char_utf8(str, str_len, &cursor, &status);
  if (status == FAILURE) {
    return false;
  }
  *pos = cursor;
  return true;
}

#ifdef XSSE2
/* Returns the bit mask of the bytes of in that html_verbatim_prefix_len() stops at. '<' and '>'
 * differ only by bit 1, and '&' and '\'' only by bit 0, which apos_mask clears when '\'' is
 * encoded. quot is '"', or '&' when '"' is not encoded, and high is 0x80, or 0 in single-byte
 * charsets. */
static zend_always_inline int html_stop_mask(__m128i in, __m128i apos_mask, __m128i quot, __m128i high)
{
  __m128i m = _mm_cmpeq_epi8(_mm_and_si128(in, _mm_set1_epi8(~2)), _mm_set1_epi8('<'));
  m = _mm_or_si128(m, _mm_cmpeq_epi8(_mm_and_si128(in, apos_mask), _mm_set1_epi8('&')));
  m = _mm_or_si128(m, _mm_or_si128(_mm_cmpeq_epi8(in, quot), _mm_and_si128(in, high)));
  return _mm_movemask_epi8(m);
}
#endif

/* Returns the length of the longest prefix of old that escape_html_entities_from() copies
 * unchanged when only the basic entities are encoded and disallowed characters are not
 * substituted: the prefix contains no character to encode and no invalid multi-byte
 * sequence, and ends on a character boundary. */
static zend_always_inline size_t html_verbatim_prefix_len(const unsigned char *old, size_t oldlen, int flags, enum entity_charset charset)
{
  /* In all supported charsets, bytes below 0x80 are single-byte characters. Bytes above
   * 0x7F are characters of their own in single-byte charsets. In UTF-8 they are validated
   * below, and the prefix stops at the first of them in the other multi-byte charsets. */
  const unsigned char stop = HTML_CC_BASIC
    | (flags & (ENT_HTML_QUOTE_SINGLE | ENT_HTML_QUOTE_DOUBLE))
    | (CHARSET_SINGLE_BYTE(charset) ? 0 : HTML_CC_NON_ASCII);
  size_t pos = 0;

#ifdef XSSE2
  if (oldlen >= sizeof(__m128i)) {
    /* Scan blocks of 16 bytes, the last one ending at the end of old */
    const __m128i apos_mask = (flags & ENT_HTML_QUOTE_SINGLE) ? _mm_set1_epi8(~1) : _mm_set1_epi8(~0);
    const __m128i quot = (flags & ENT_HTML_QUOTE_DOUBLE) ? _mm_set1_epi8('"') : _mm_set1_epi8('&');
    const __m128i high = (stop & HTML_CC_NON_ASCII) ? _mm_set1_epi8((char) 0x80) : _mm_setzero_si128();
    const size_t last = oldlen - sizeof(__m128i);

    while (1) {
      int mask;

      if (pos < last) {
        mask = html_stop_mask(_mm_loadu_si128((const __m128i *) (old + pos)), apos_mask, quot, high);
        if (!mask) {
          pos += sizeof(__m128i);
          continue;
        }
      } else {
        /* the last block, whose bytes before pos are already scanned */
        mask = html_stop_mask(_mm_loadu_si128((const __m128i *) (old + last)), apos_mask, quot, high) >> (pos - last);
        if (!mask) {
          return oldlen;
        }
      }
      pos += zend_ulong_ntz(mask);
      if (old[pos] < 0x80 || charset != cs_utf_8 || !html_skip_utf8_char(old, oldlen, &pos)) {
        return pos;
      }
    }
  }
#endif

  while (pos < oldlen && !(html_char_class[old[pos]] & stop)) {
    pos++;
  }
  while (pos < oldlen && old[pos] >= 0x80 && charset == cs_utf_8 && html_skip_utf8_char(old, oldlen, &pos)) {
    while (pos < oldlen && !(html_char_class[old[pos]] & stop)) {
      pos++;
    }
  }

  return pos;
}

/* Encodes old, whose first cursor bytes are known to be copied unchanged */
static zend_string *escape_html_entities_from(const unsigned char *old, size_t oldlen, size_t cursor, int all, int flags, enum entity_charset charset, bool double_encode)
{
  size_t maxlen, len;
  zend_string *replaced;
  int doctype = flags & ENT_HTML_DOC_TYPE_MASK;
  entity_table_opt entity_table;
  const enc_to_uni *to_uni_table = NULL;
  const entity_ht *inv_map = NULL; /* used for !double_encode */
  /* only used if flags includes ENT_HTML_IGNORE_ERRORS or ENT_HTML_SUBSTITUTE_DISALLOWED_CHARS */
  const unsigned char *replacement = NULL;
  size_t replacement_len = 0;

  entity_table = determine_entity_table(all, doctype);
  if (all && !CHARSET_UNICODE_COMPAT(charset)) {
    to_uni_table = enc_to_uni_index[charset];
  }

  if (!double_encode) {
    /* first arg is 1 because we want to identify valid named entities
     * even if we are only encoding the basic ones */
    inv_map = unescape_inverse_map(1, flags);
  }

  if (flags & (ENT_HTML_SUBSTITUTE_ERRORS | ENT_HTML_SUBSTITUTE_DISALLOWED_CHARS)) {
    if (charset == cs_utf_8) {
      replacement = (const unsigned char*)"\xEF\xBF\xBD";
      replacement_len = sizeof("\xEF\xBF\xBD") - 1;
    } else {
      replacement = (const unsigned char*)"&#xFFFD;";
      replacement_len = sizeof("&#xFFFD;") - 1;
    }
  }

  /* initial estimate */
  if (oldlen < 64) {
    maxlen = 128;
  } else {
    maxlen = zend_safe_addmult(oldlen, 2, 0, "html_entities");
  }

  replaced = zend_string_falloc(maxlen, /* persistent */ false, /* fallible */ true);
  if (!replaced) {
    return ZSTR_EMPTY_ALLOC();
  }
  memcpy(ZSTR_VAL(replaced), old, cursor);
  len = cursor;
  while (cursor < oldlen) {
    const unsigned char *mbsequence = NULL;
    size_t mbseqlen         = 0,
           cursor_before      = cursor;
    zend_result status        = SUCCESS;
    unsigned int this_char      = get_next_char(charset, old, oldlen, &cursor, &status);

    /* guarantee we have at least 40 bytes to write.
     * In HTML5, entities may take up to 33 bytes */
    if (len > maxlen - 40) { /* maxlen can never be smaller than 128 */
      zend_string *new_replaced = zend_string_safe_frealloc(replaced, maxlen, 1, 128, /* persistent */ false, /* fallible */ true);
      if (!new_replaced) {
        zend_string_release(replaced);
        return ZSTR_EMPTY_ALLOC();
      }
      replaced = new_replaced;
      maxlen += 128;
    }

    if (status == FAILURE) {
      /* invalid MB sequence */
      if (flags & ENT_HTML_IGNORE_ERRORS) {
        continue;
      } else if (flags & ENT_HTML_SUBSTITUTE_ERRORS) {
        memcpy(&ZSTR_VAL(replaced)[len], replacement, replacement_len);
        len += replacement_len;
        continue;
      } else {
        zend_string_efree(replaced);
        return ZSTR_EMPTY_ALLOC();
      }
    } else { /* SUCCESS */
      mbsequence = &old[cursor_before];
      mbseqlen = cursor - cursor_before;
    }

    if (this_char != '&') { /* no entity on this position */
      const unsigned char *rep  = NULL;
      size_t        rep_len = 0;

      if (((this_char == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
          (this_char == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE))))
        goto pass_char_through;

      if (all) { /* false that CHARSET_PARTIAL_SUPPORT(charset) */
        if (to_uni_table != NULL) {
          /* !CHARSET_UNICODE_COMPAT therefore not UTF-8; since UTF-8
           * is the only multibyte encoding with !CHARSET_PARTIAL_SUPPORT,
           * we're using a single byte encoding */
          map_to_unicode(this_char, to_uni_table, &this_char);
          if (this_char == 0xFFFF) /* no mapping; pass through */
            goto pass_char_through;
        }
        /* the cursor may advance */
        find_entity_for_char(this_char, charset, entity_table.ms_table, &rep,
          &rep_len, old, oldlen, &cursor);
      } else {
        find_entity_for_char_basic(this_char, entity_table.table, &rep, &rep_len);
      }

      if (rep != NULL) {
        ZSTR_VAL(replaced)[len++] = '&';
        memcpy(&ZSTR_VAL(replaced)[len], rep, rep_len);
        len += rep_len;
        ZSTR_VAL(replaced)[len++] = ';';
      } else {
        /* we did not find an entity for this char.
         * check for its validity, if its valid pass it unchanged */
        if (flags & ENT_HTML_SUBSTITUTE_DISALLOWED_CHARS) {
          if (CHARSET_UNICODE_COMPAT(charset)) {
            if (!unicode_cp_is_allowed(this_char, doctype)) {
              mbsequence = replacement;
              mbseqlen = replacement_len;
            }
          } else if (to_uni_table) {
            if (!all) /* otherwise we already did this */
              map_to_unicode(this_char, to_uni_table, &this_char);
            if (!unicode_cp_is_allowed(this_char, doctype)) {
              mbsequence = replacement;
              mbseqlen = replacement_len;
            }
          } else {
            /* not a unicode code point, unless, coincidentally, it's in
             * the 0x20..0x7D range (except 0x5C in sjis). We know nothing
             * about other code points, because we have no tables. Since
             * Unicode code points in that range are not disallowed in any
             * document type, we could do nothing. However, conversion
             * tables frequently map 0x00-0x1F to the respective C0 code
             * points. Let's play it safe and admit that's the case */
            if (this_char <= 0x7D &&
                !unicode_cp_is_allowed(this_char, doctype)) {
              mbsequence = replacement;
              mbseqlen = replacement_len;
            }
          }
        }
pass_char_through:
        if (mbseqlen > 1) {
          memcpy(ZSTR_VAL(replaced) + len, mbsequence, mbseqlen);
          len += mbseqlen;
        } else {
          ZSTR_VAL(replaced)[len++] = mbsequence[0];
        }
      }
    } else { /* this_char == '&' */
      if (double_encode) {
encode_amp:
        memcpy(&ZSTR_VAL(replaced)[len], "&amp;", sizeof("&amp;") - 1);
        len += sizeof("&amp;") - 1;
      } else { /* no double encode */
        /* check if entity is valid */
        size_t ent_len; /* not counting & or ; */
        /* peek at next char */
        if (old[cursor] == '#') { /* numeric entity */
          unsigned code_point;
          int valid;
          char *pos = (char*)&old[cursor+1];
          valid = process_numeric_entity((const char **)&pos, &code_point);
          if (valid == FAILURE)
            goto encode_amp;
          if (flags & ENT_HTML_SUBSTITUTE_DISALLOWED_CHARS) {
            if (!numeric_entity_is_allowed(code_point, doctype))
              goto encode_amp;
          }
          ent_len = pos - (char*)&old[cursor];
        } else { /* named entity */
          /* check for vality of named entity */
          const char *start = (const char *) &old[cursor],
                 *next = start;
          unsigned   dummy1, dummy2;

          if (process_named_entity_html(&next, &start, &ent_len) == FAILURE)
            goto encode_amp;
          if (resolve_named_entity_html(start, ent_len, inv_map, &dummy1, &dummy2) == FAILURE) {
            if (!(doctype == ENT_HTML_DOC_XHTML && ent_len == 4 && start[0] == 'a'
                  && start[1] == 'p' && start[2] == 'o' && start[3] == 's')) {
              /* uses html4 inv_map, which doesn't include apos;. This is a
               * hack to support it */
              goto encode_amp;
            }
          }
        }
        /* checks passed; copy entity to result */
        /* entity size is unbounded, we may need more memory */
        /* at this point maxlen - len >= 40 */
        if (maxlen - len < ent_len + 2 /* & and ; */) {
          /* ent_len < oldlen, which is certainly <= SIZE_MAX/2 */
          replaced = zend_string_safe_realloc(replaced, maxlen, 1, ent_len + 128, 0);
          maxlen += ent_len + 128;
        }
        ZSTR_VAL(replaced)[len++] = '&';
        memcpy(&ZSTR_VAL(replaced)[len], &old[cursor], ent_len);
        len += ent_len;
        ZSTR_VAL(replaced)[len++] = ';';
        cursor += ent_len + 1;
      }
    }
  }
  ZSTR_VAL(replaced)[len] = '\0';
  ZSTR_LEN(replaced) = len;

  return replaced;
}

/* {{{ php_escape_html_entities */
static zend_always_inline zend_string *escape_html_entities(const unsigned char *old, size_t oldlen, zend_string *old_str, int all, int flags, const char *hint_charset, bool double_encode, bool quiet)
{
  enum entity_charset charset = determine_charset(hint_charset, quiet);
  size_t verbatim_len = 0;

  if (all) { /* replace with all named entities */
    if (!quiet && CHARSET_PARTIAL_SUPPORT(charset)) {
      php_error_docref(NULL, E_NOTICE, "Only basic entities "
        "substitution is supported for multi-byte encodings other than UTF-8; "
        "functionality is equivalent to htmlspecialchars");
    }
    LIMIT_ALL(all, flags & ENT_HTML_DOC_TYPE_MASK, charset);
  }

  if (!all && !(flags & ENT_HTML_SUBSTITUTE_DISALLOWED_CHARS)) {
    verbatim_len = html_verbatim_prefix_len(old, oldlen, flags, charset);
    if (verbatim_len == oldlen) {
      /* nothing to encode */
      return old_str ? zend_string_copy(old_str) : zend_string_init((const char *) old, oldlen, 0);
    }
  }

  return escape_html_entities_from(old, oldlen, verbatim_len, all, flags, charset, double_encode);
}

PHPAPI zend_string *php_escape_html_entities_ex(const unsigned char *old, size_t oldlen, int all, int flags, const char *hint_charset, bool double_encode, bool quiet)
{
  return escape_html_entities(old, oldlen, NULL, all, flags, hint_charset, double_encode, quiet);
}
/* }}} */

/* {{{ php_html_entities */
static void php_html_entities(INTERNAL_FUNCTION_PARAMETERS, int all)
{
  zend_string *str, *hint_charset = NULL;
  zend_long flags = ENT_QUOTES|ENT_SUBSTITUTE;
  zend_string *replaced;
  bool double_encode = 1;

  ZEND_PARSE_PARAMETERS_START(1, 4)
    Z_PARAM_STR(str)
    Z_PARAM_OPTIONAL
    Z_PARAM_LONG(flags)
    Z_PARAM_STR_OR_NULL(hint_charset)
    Z_PARAM_BOOL(double_encode);
  ZEND_PARSE_PARAMETERS_END();

  if (ZSTR_LEN(str) == 0) {
    RETURN_EMPTY_STRING();
  }
  replaced = escape_html_entities(
    (unsigned char*)ZSTR_VAL(str), ZSTR_LEN(str), str, all, (int) flags,
    hint_charset ? ZSTR_VAL(hint_charset) : NULL, double_encode, /* quiet */ 0);
  RETVAL_STR(replaced);
}
/* }}} */

/* {{{ Convert special characters to HTML entities */
PHP_FUNCTION(htmlspecialchars)
{
  php_html_entities(INTERNAL_FUNCTION_PARAM_PASSTHRU, 0);
}
/* }}} */

/* {{{ Convert special HTML entities back to characters */
PHP_FUNCTION(htmlspecialchars_decode)
{
  zend_string *str;
  zend_long quote_style = ENT_QUOTES|ENT_SUBSTITUTE;
  zend_string *replaced;

  ZEND_PARSE_PARAMETERS_START(1, 2)
    Z_PARAM_STR(str)
    Z_PARAM_OPTIONAL
    Z_PARAM_LONG(quote_style)
  ZEND_PARSE_PARAMETERS_END();

  replaced = php_unescape_html_entities(str, 0 /*!all*/, (int)quote_style, NULL);
  RETURN_STR(replaced);
}
/* }}} */

/* {{{ Convert all HTML entities to their applicable characters */
PHP_FUNCTION(html_entity_decode)
{
  zend_string *str, *hint_charset = NULL;
  zend_long quote_style = ENT_QUOTES|ENT_SUBSTITUTE;
  zend_string *replaced;

  ZEND_PARSE_PARAMETERS_START(1, 3)
    Z_PARAM_STR(str)
    Z_PARAM_OPTIONAL
    Z_PARAM_LONG(quote_style)
    Z_PARAM_STR_OR_NULL(hint_charset)
  ZEND_PARSE_PARAMETERS_END();

  replaced = php_unescape_html_entities(
    str, 1 /*all*/, (int)quote_style, hint_charset ? ZSTR_VAL(hint_charset) : NULL);
  RETURN_STR(replaced);
}
/* }}} */


/* {{{ Convert all applicable characters to HTML entities */
PHP_FUNCTION(htmlentities)
{
  php_html_entities(INTERNAL_FUNCTION_PARAM_PASSTHRU, 1);
}
/* }}} */

/* {{{ write_s3row_data */
static inline void write_s3row_data(
  const entity_stage3_row *r,
  unsigned orig_cp,
  enum entity_charset charset,
  zval *arr)
{
  char key[9] = ""; /* two unicode code points in UTF-8 */
  char entity[LONGEST_ENTITY_LENGTH + 2] = {'&'};
  size_t written_k1;

  written_k1 = write_octet_sequence((unsigned char*)key, charset, orig_cp);

  if (!r->ambiguous) {
    size_t l = r->data.ent.entity_len;
    memcpy(&entity[1], r->data.ent.entity, l);
    entity[l + 1] = ';';
    add_assoc_stringl_ex(arr, key, written_k1, entity, l + 2);
  } else {
    unsigned i,
           num_entries;
    const entity_multicodepoint_row *mcpr = r->data.multicodepoint_table;

    if (mcpr[0].leading_entry.default_entity != NULL) {
      size_t l = mcpr[0].leading_entry.default_entity_len;
      memcpy(&entity[1], mcpr[0].leading_entry.default_entity, l);
      entity[l + 1] = ';';
      add_assoc_stringl_ex(arr, key, written_k1, entity, l + 2);
    }
    num_entries = mcpr[0].leading_entry.size;
    for (i = 1; i <= num_entries; i++) {
      size_t   l,
             written_k2;
      unsigned uni_cp,
           spe_cp;

      uni_cp = mcpr[i].normal_entry.second_cp;
      l = mcpr[i].normal_entry.entity_len;

      if (!CHARSET_UNICODE_COMPAT(charset)) {
        if (map_from_unicode(uni_cp, charset, &spe_cp) == FAILURE)
          continue; /* non representable in this charset */
      } else {
        spe_cp = uni_cp;
      }

      written_k2 = write_octet_sequence((unsigned char*)&key[written_k1], charset, spe_cp);
      memcpy(&entity[1], mcpr[i].normal_entry.entity, l);
      entity[l + 1] = ';';
      add_assoc_stringl_ex(arr, key, written_k1 + written_k2, entity, l + 2);
    }
  }
}
/* }}} */

/* {{{ Returns the internal translation table used by htmlspecialchars and htmlentities */
PHP_FUNCTION(get_html_translation_table)
{
  zend_long all = PHP_HTML_SPECIALCHARS,
     flags = ENT_QUOTES|ENT_SUBSTITUTE;
  int doctype;
  entity_table_opt entity_table;
  const enc_to_uni *to_uni_table = NULL;
  char *charset_hint = NULL;
  size_t charset_hint_len;
  enum entity_charset charset;

  /* in this function we have to jump through some loops because we're
   * getting the translated table from data structures that are optimized for
   * random access, not traversal */

  ZEND_PARSE_PARAMETERS_START(0, 3)
    Z_PARAM_OPTIONAL
    Z_PARAM_LONG(all)
    Z_PARAM_LONG(flags)
    Z_PARAM_STRING(charset_hint, charset_hint_len)
  ZEND_PARSE_PARAMETERS_END();

  charset = determine_charset(charset_hint, /* quiet */ 0);
  doctype = flags & ENT_HTML_DOC_TYPE_MASK;
  LIMIT_ALL(all, doctype, charset);

  array_init(return_value);

  entity_table = determine_entity_table((int)all, doctype);
  if (all && !CHARSET_UNICODE_COMPAT(charset)) {
    to_uni_table = enc_to_uni_index[charset];
  }

  if (all) { /* PHP_HTML_ENTITIES (actually, any non-zero value for 1st param) */
    const entity_stage1_row *ms_table = entity_table.ms_table;

    if (CHARSET_UNICODE_COMPAT(charset)) {
      unsigned i, j, k,
           max_i, max_j, max_k;
      /* no mapping to unicode required */
      if (CHARSET_SINGLE_BYTE(charset)) { /* ISO-8859-1 */
        max_i = 1; max_j = 4; max_k = 64;
      } else {
        max_i = 0x1E; max_j = 64; max_k = 64;
      }

      for (i = 0; i < max_i; i++) {
        if (ms_table[i] == empty_stage2_table)
          continue;
        for (j = 0; j < max_j; j++) {
          if (ms_table[i][j] == empty_stage3_table)
            continue;
          for (k = 0; k < max_k; k++) {
            const entity_stage3_row *r = &ms_table[i][j][k];
            unsigned code;

            if (r->data.ent.entity == NULL)
              continue;

            code = ENT_CODE_POINT_FROM_STAGES(i, j, k);
            if (((code == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
                (code == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE))))
              continue;
            write_s3row_data(r, code, charset, return_value);
          }
        }
      }
    } else {
      /* we have to iterate through the set of code points for this
       * encoding and map them to unicode code points */
      unsigned i;
      for (i = 0; i <= 0xFF; i++) {
        const entity_stage3_row *r;
        unsigned uni_cp;

        /* can be done before mapping, they're invariant */
        if (((i == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
            (i == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE))))
          continue;

        map_to_unicode(i, to_uni_table, &uni_cp);
        r = &ms_table[ENT_STAGE1_INDEX(uni_cp)][ENT_STAGE2_INDEX(uni_cp)][ENT_STAGE3_INDEX(uni_cp)];
        if (r->data.ent.entity == NULL)
          continue;

        write_s3row_data(r, i, charset, return_value);
      }
    }
  } else {
    /* we could use sizeof(stage3_table_be_apos_00000) as well */
    unsigned    j,
            numelems = sizeof(stage3_table_be_noapos_00000) /
              sizeof(*stage3_table_be_noapos_00000);

    for (j = 0; j < numelems; j++) {
      const entity_stage3_row *r = &entity_table.table[j];
      if (r->data.ent.entity == NULL)
        continue;

      if (((j == '\'' && !(flags & ENT_HTML_QUOTE_SINGLE)) ||
          (j == '"' && !(flags & ENT_HTML_QUOTE_DOUBLE))))
        continue;

      /* charset is indifferent, used cs_8859_1 for efficiency */
      write_s3row_data(r, j, cs_8859_1, return_value);
    }
  }
}
/* }}} */

Coverage Report

Created: 2026-10-28 06:20