/work/include/simdutf/scalar/utf8.h

Source
#ifndef SIMDUTF_UTF8_H
#define SIMDUTF_UTF8_H

#include <cstring>

namespace simdutf {
namespace scalar {
namespace {
namespace utf8 {

// credit: based on code from Google Fuchsia (Apache Licensed)
template <class BytePtr>
simdutf_constexpr23 simdutf_warn_unused bool validate(BytePtr data,
                                                      size_t len) noexcept {
  static_assert(
      std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value,
      "dereferencing the data pointer must result in a uint8_t");
  uint64_t pos = 0;
  uint32_t code_point = 0;
  while (pos < len) {
    uint64_t next_pos;
#if SIMDUTF_CPLUSPLUS23
    if !consteval
#endif
    { // check if the next 16 bytes are ascii.
      next_pos = pos + 16;
      if (next_pos <= len) { // if it is safe to read 16 more bytes, check
                             // that they are ascii
        uint64_t v1{};
        std::memcpy(&v1, data + pos, sizeof(uint64_t));
        uint64_t v2{};
        std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
        uint64_t v{v1 | v2};
        if ((v & 0x8080808080808080) == 0) {
          pos = next_pos;
          continue;
        }
      }
    }

    unsigned char byte = data[pos];

    while (byte < 0b10000000) {
      if (++pos == len) {
        return true;
      }
      byte = data[pos];
    }

    if ((byte & 0b11100000) == 0b11000000) {
      next_pos = pos + 2;
      if (next_pos > len) {
        return false;
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return false;
      }
      // range check
      code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
      if (code_point < 0x80) {
        return false;
      }
    } else if ((byte & 0b11110000) == 0b11100000) {
      next_pos = pos + 3;
      if (next_pos > len) {
        return false;
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return false;
      }
      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
        return false;
      }
      // range check
      code_point = (byte & 0b00001111) << 12 |
                   (data[pos + 1] & 0b00111111) << 6 |
                   (data[pos + 2] & 0b00111111);
      if ((code_point < 0x800) ||
          (0xd7ff < code_point && code_point < 0xe000)) {
        return false;
      }
    } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
      next_pos = pos + 4;
      if (next_pos > len) {
        return false;
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return false;
      }
      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
        return false;
      }
      if ((data[pos + 3] & 0b11000000) != 0b10000000) {
        return false;
      }
      // range check
      code_point =
          (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 |
          (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111);
      if (code_point <= 0xffff || 0x10ffff < code_point) {
        return false;
      }
    } else {
      // we may have a continuation
      return false;
    }
    pos = next_pos;
  }
  return true;
}

simdutf_really_inline simdutf_warn_unused bool validate(const char *buf,
                                                        size_t len) noexcept {
  return validate(reinterpret_cast<const uint8_t *>(buf), len);
}

template <class BytePtr>
simdutf_constexpr23 simdutf_warn_unused result
validate_with_errors(BytePtr data, size_t len) noexcept {
  static_assert(
      std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value,
      "dereferencing the data pointer must result in a uint8_t");
  size_t pos = 0;
  uint32_t code_point = 0;
  while (pos < len) {
    // check of the next 16 bytes are ascii.
    size_t next_pos = pos + 16;
    if (next_pos <=
        len) { // if it is safe to read 16 more bytes, check that they are ascii
      uint64_t v1;
      std::memcpy(&v1, data + pos, sizeof(uint64_t));
      uint64_t v2;
      std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
      uint64_t v{v1 | v2};
      if ((v & 0x8080808080808080) == 0) {
        pos = next_pos;
        continue;
      }
    }
    unsigned char byte = data[pos];

    while (byte < 0b10000000) {
      if (++pos == len) {
        return result(error_code::SUCCESS, len);
      }
      byte = data[pos];
    }

    if ((byte & 0b11100000) == 0b11000000) {
      next_pos = pos + 2;
      if (next_pos > len) {
        return result(error_code::TOO_SHORT, pos);
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return result(error_code::TOO_SHORT, pos);
      }
      // range check
      code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
      if (code_point < 0x80) {
        return result(error_code::OVERLONG, pos);
      }
    } else if ((byte & 0b11110000) == 0b11100000) {
      next_pos = pos + 3;
      if (next_pos > len) {
        return result(error_code::TOO_SHORT, pos);
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return result(error_code::TOO_SHORT, pos);
      }
      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
        return result(error_code::TOO_SHORT, pos);
      }
      // range check
      code_point = (byte & 0b00001111) << 12 |
                   (data[pos + 1] & 0b00111111) << 6 |
                   (data[pos + 2] & 0b00111111);
      if (code_point < 0x800) {
        return result(error_code::OVERLONG, pos);
      }
      if (0xd7ff < code_point && code_point < 0xe000) {
        return result(error_code::SURROGATE, pos);
      }
    } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
      next_pos = pos + 4;
      if (next_pos > len) {
        return result(error_code::TOO_SHORT, pos);
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return result(error_code::TOO_SHORT, pos);
      }
      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
        return result(error_code::TOO_SHORT, pos);
      }
      if ((data[pos + 3] & 0b11000000) != 0b10000000) {
        return result(error_code::TOO_SHORT, pos);
      }
      // range check
      code_point =
          (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 |
          (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111);
      if (code_point <= 0xffff) {
        return result(error_code::OVERLONG, pos);
      }
      if (0x10ffff < code_point) {
        return result(error_code::TOO_LARGE, pos);
      }
    } else {
      // we either have too many continuation bytes or an invalid leading byte
      if ((byte & 0b11000000) == 0b10000000) {
        return result(error_code::TOO_LONG, pos);
      } else {
        return result(error_code::HEADER_BITS, pos);
      }
    }
    pos = next_pos;
  }
  return result(error_code::SUCCESS, len);
}

simdutf_really_inline simdutf_warn_unused result
validate_with_errors(const char *buf, size_t len) noexcept {
  return validate_with_errors(reinterpret_cast<const uint8_t *>(buf), len);
}

// Finds the previous leading byte starting backward from buf and validates with
// errors from there Used to pinpoint the location of an error when an invalid
// chunk is detected We assume that the stream starts with a leading byte, and
// to check that it is the case, we ask that you pass a pointer to the start of
// the stream (start). Note that the resulting count is underflowed if an error
// is encountered in the rewinded segment.
inline simdutf_warn_unused result rewind_and_validate_with_errors(
    const char *start, const char *buf, size_t len) noexcept {
  // First check that we start with a leading byte
  if ((*start & 0b11000000) == 0b10000000) {
    return result(error_code::TOO_LONG, 0);
  }
  size_t extra_len{0};
  // A leading byte cannot be further than 4 bytes away
  for (int i = 0; i < 5; i++) {
    unsigned char byte = *buf;
    if ((byte & 0b11000000) != 0b10000000) {
      break;
    } else {
      buf--;
      extra_len++;
    }
  }

  result res = validate_with_errors(buf, len + extra_len);
  res.count -= extra_len; // Might underflow
  return res;
}

// credit: based on code from Google Fuchsia (Apache Licensed)
template <class BytePtr>
simdutf_constexpr23 simdutf_warn_unused utf8_result
validate_with_counts(BytePtr data, size_t len) noexcept {
  static_assert(
      std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value,
      "dereferencing the data pointer must result in a uint8_t");
  size_t pos = 0;
  uint32_t code_point = 0;
  size_t continuations = 0;
  size_t four_byte = 0;
  while (pos < len) {
    // std::memcpy is not usable during constant evaluation, so the block-wise
    // ASCII fast path is restricted to run time, like in validate() above.
#if SIMDUTF_CPLUSPLUS23
    if !consteval
#endif
    { // check if the next 16 bytes are ascii.
      const size_t next_ascii_pos = pos + 16;
      if (next_ascii_pos <= len) { // if it is safe to read 16 more bytes, check
                                   // that they are ascii
        uint64_t v1{};
        std::memcpy(&v1, data + pos, sizeof(uint64_t));
        uint64_t v2{};
        std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
        uint64_t v{v1 | v2};
        if ((v & 0x8080808080808080) == 0) {
          pos = next_ascii_pos;
          continue;
        }
      }
    }

    size_t next_pos;
    unsigned char byte = data[pos];

    while (byte < 0b10000000) {
      if (++pos == len) {
        return utf8_result(error_code::SUCCESS, pos, continuations, four_byte);
      }
      byte = data[pos];
    }

    if ((byte & 0b11100000) == 0b11000000) {
      next_pos = pos + 2;
      if (next_pos > len) {
        return utf8_result(error_code::TOO_SHORT, pos, continuations,
                           four_byte);
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return utf8_result(error_code::TOO_SHORT, pos, continuations,
                           four_byte);
      }
      // range check
      code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
      if (code_point < 0x80) {
        return utf8_result(error_code::OVERLONG, pos, continuations, four_byte);
      }
      continuations += 1;
    } else if ((byte & 0b11110000) == 0b11100000) {
      next_pos = pos + 3;
      if (next_pos > len) {
        return utf8_result(error_code::TOO_SHORT, pos, continuations,
                           four_byte);
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return utf8_result(error_code::TOO_SHORT, pos, continuations,
                           four_byte);
      }
      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
        return utf8_result(error_code::TOO_SHORT, pos, continuations,
                           four_byte);
      }
      // range check
      code_point = (byte & 0b00001111) << 12 |
                   (data[pos + 1] & 0b00111111) << 6 |
                   (data[pos + 2] & 0b00111111);
      if (code_point < 0x800) {
        return utf8_result(error_code::OVERLONG, pos, continuations, four_byte);
      } else if (0xd7ff < code_point && code_point < 0xe000) {
        return utf8_result(error_code::SURROGATE, pos, continuations,
                           four_byte);
      }
      continuations += 2;
    } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
      next_pos = pos + 4;
      if (next_pos > len) {
        return utf8_result(error_code::TOO_SHORT, pos, continuations,
                           four_byte);
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return utf8_result(error_code::TOO_SHORT, pos, continuations,
                           four_byte);
      }
      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
        return utf8_result(error_code::TOO_SHORT, pos, continuations,
                           four_byte);
      }
      if ((data[pos + 3] & 0b11000000) != 0b10000000) {
        return utf8_result(error_code::TOO_SHORT, pos, continuations,
                           four_byte);
      }
      // range check
      code_point =
          (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 |
          (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111);
      if (code_point <= 0xffff) {
        return utf8_result(error_code::OVERLONG, pos, continuations, four_byte);
      } else if (0x10ffff < code_point) {
        return utf8_result(error_code::TOO_LARGE, pos, continuations,
                           four_byte);
      }
      continuations += 3;
      four_byte += 1;
    } else {
      // Continuation byte or invalid header byte
      if ((byte & 0b11000000) == 0b10000000) {
        return utf8_result(error_code::TOO_LONG, pos, continuations, four_byte);
      } else {
        return utf8_result(error_code::HEADER_BITS, pos, continuations,
                           four_byte);
      }
    }
    pos = next_pos;
  }
  return utf8_result(error_code::SUCCESS, pos, continuations, four_byte);
}

simdutf_really_inline simdutf_warn_unused utf8_result
validate_with_counts(const char *buf, size_t len) noexcept {
  return validate_with_counts(reinterpret_cast<const uint8_t *>(buf), len);
}

// Finds the previous leading byte starting backward from buf and validates from
// there on out while counting continuation bytes and 4-byte leads. Used to
// pinpoint the location of an error when an invalid chunk is detected. We check
// that the stream starts with a leading byte with the passed pointer to the
// start of the stream (start).
//
// All three fields of the result are relative to buf. The bytes we rewind over
// sit in a block the caller has already counted, so their contribution is
// subtracted back out at the end. When the error lies inside the rewound
// region, those three subtractions underflow. That is intended, not an
// oversight: the caller adds the offset of buf and the counts of the preceding
// blocks, which include the rewound bytes, and the unsigned wraparound cancels
// exactly, leaving the correct absolute values.
inline simdutf_warn_unused utf8_result rewind_and_validate_with_counts(
    const char *start, const char *buf, size_t len) noexcept {
  // First check that we start with a leading byte
  if ((*start & 0b11000000) == 0b10000000) {
    return utf8_result(error_code::TOO_LONG, 0, 0, 0);
  }
  // If this is the start of the buffer, we don't need to rewind to check the
  // previous chunk.
  if (start == buf) {
    return validate_with_counts(buf, len);
  }
  // As this isn't the start of the buffer, the error might be in the previous
  // chunk. So we backtrack and also need to avoid double counting.
  size_t extra_len{0};
  size_t extra_continuations{0};
  size_t extra_four_bytes{0};
  // Four steps are enough. Everything before buf has already validated, so at
  // most three continuation bytes can precede it, from a four-byte sequence
  // ending exactly on the boundary; the fourth step then lands on its leading
  // byte. Note that the sibling rewind_and_validate_with_errors needs five
  // steps only because it tests the byte before stepping back, whereas we step
  // back first. Callers pass a block boundary at least 64 bytes into the
  // stream, so stepping back four bytes stays in bounds.
  for (int i = 0; i < 4; i++) {
    // We rewind at least one byte to find the actual previous leading byte.
    // We do so to backtrack into the last chunk in case the utf8 error happened
    // there already.
    buf--;
    unsigned char byte = *buf;
    extra_len++;

    if (byte >= 0b10000000) {
      if (byte >= 0b11110000) {
        extra_four_bytes += 1;
      } else if ((int8_t)byte < -65 + 1) {
        extra_continuations += 1;
      }
    }
    if ((byte & 0b11000000) != 0b10000000) {
      break;
    }
  }

  utf8_result res = validate_with_counts(buf, len + extra_len);
  res.input_count -= extra_len;
  res.continuation_count -= extra_continuations;
  res.four_byte_count -= extra_four_bytes;
  return res;
}

template <typename InputPtr>
#if SIMDUTF_CPLUSPLUS20
  requires simdutf::detail::indexes_into_byte_like<InputPtr>
#endif
simdutf_constexpr23 size_t count_code_points(InputPtr data, size_t len) {
  size_t counter{0};
  for (size_t i = 0; i < len; i++) {
    // -65 is 0b10111111, anything larger in two-complement's should start a new
    // code point.
    if (int8_t(data[i]) > -65) {
      counter++;
    }
  }
  return counter;
}

template <typename InputPtr>
#if SIMDUTF_CPLUSPLUS20
  requires simdutf::detail::indexes_into_byte_like<InputPtr>
#endif
simdutf_constexpr23 size_t utf16_length_from_utf8(InputPtr data, size_t len) {
  size_t counter{0};
  for (size_t i = 0; i < len; i++) {
    if (int8_t(data[i]) > -65) {
      counter++;
    }
    if (uint8_t(data[i]) >= 240) {
      counter++;
    }
  }
  return counter;
}

template <typename InputPtr>
#if SIMDUTF_CPLUSPLUS20
  requires simdutf::detail::indexes_into_byte_like<InputPtr>
#endif
simdutf_warn_unused simdutf_constexpr23 size_t
trim_partial_utf8(InputPtr input, size_t length) {
  if (length < 3) {
    switch (length) {
    case 2:
      if (uint8_t(input[length - 1]) >= 0xc0) {
        return length - 1;
      } // 2-, 3- and 4-byte characters with only 1 byte left
      if (uint8_t(input[length - 2]) >= 0xe0) {
        return length - 2;
      } // 3- and 4-byte characters with only 2 bytes left
      return length;
    case 1:
      if (uint8_t(input[length - 1]) >= 0xc0) {
        return length - 1;
      } // 2-, 3- and 4-byte characters with only 1 byte left
      return length;
    case 0:
      return length;
    }
  }
  if (uint8_t(input[length - 1]) >= 0xc0) {
    return length - 1;
  } // 2-, 3- and 4-byte characters with only 1 byte left
  if (uint8_t(input[length - 2]) >= 0xe0) {
    return length - 2;
  } // 3- and 4-byte characters with only 1 byte left
  if (uint8_t(input[length - 3]) >= 0xf0) {
    return length - 3;
  } // 4-byte characters with only 3 bytes left
  return length;
}

} // namespace utf8
} // unnamed namespace
} // namespace scalar
} // namespace simdutf

#endif

Coverage Report

Created: 2026-10-17 07:16

Line	Count	Source
1		#ifndef SIMDUTF_UTF8_H
2		#define SIMDUTF_UTF8_H
3
4		#include <cstring>
5
6		namespace simdutf {
7		namespace scalar {
8		namespace {
9		namespace utf8 {
10
11		// credit: based on code from Google Fuchsia (Apache Licensed)
12		template <class BytePtr>
13		simdutf_constexpr23 simdutf_warn_unused bool validate(BytePtr data,
14	0	size_t len) noexcept {
15	0	static_assert(
16	0	std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value,
17	0	"dereferencing the data pointer must result in a uint8_t");
18	0	uint64_t pos = 0;
19	0	uint32_t code_point = 0;
20	0	while (pos < len) {
21	0	uint64_t next_pos;
22	0	#if SIMDUTF_CPLUSPLUS23
23	0	if !consteval
24	0	#endif
25	0	{ // check if the next 16 bytes are ascii.
26	0	next_pos = pos + 16;
27	0	if (next_pos <= len) { // if it is safe to read 16 more bytes, check
28	0	// that they are ascii
29	0	uint64_t v1{};
30	0	std::memcpy(&v1, data + pos, sizeof(uint64_t));
31	0	uint64_t v2{};
32	0	std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
33	0	uint64_t v{v1 \| v2};
34	0	if ((v & 0x8080808080808080) == 0) {
35	0	pos = next_pos;
36	0	continue;
37	0	}
38	0	}
39	0	}
40	0
41	0	unsigned char byte = data[pos];
42	0
43	0	while (byte < 0b10000000) {
44	0	if (++pos == len) {
45	0	return true;
46	0	}
47	0	byte = data[pos];
48	0	}
49	0
50	0	if ((byte & 0b11100000) == 0b11000000) {
51	0	next_pos = pos + 2;
52	0	if (next_pos > len) {
53	0	return false;
54	0	}
55	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
56	0	return false;
57	0	}
58	0	// range check
59	0	code_point = (byte & 0b00011111) << 6 \| (data[pos + 1] & 0b00111111);
60	0	if (code_point < 0x80) {
61	0	return false;
62	0	}
63	0	} else if ((byte & 0b11110000) == 0b11100000) {
64	0	next_pos = pos + 3;
65	0	if (next_pos > len) {
66	0	return false;
67	0	}
68	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
69	0	return false;
70	0	}
71	0	if ((data[pos + 2] & 0b11000000) != 0b10000000) {
72	0	return false;
73	0	}
74	0	// range check
75	0	code_point = (byte & 0b00001111) << 12 \|
76	0	(data[pos + 1] & 0b00111111) << 6 \|
77	0	(data[pos + 2] & 0b00111111);
78	0	if ((code_point < 0x800) \|\|
79	0	(0xd7ff < code_point && code_point < 0xe000)) {
80	0	return false;
81	0	}
82	0	} else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
83	0	next_pos = pos + 4;
84	0	if (next_pos > len) {
85	0	return false;
86	0	}
87	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
88	0	return false;
89	0	}
90	0	if ((data[pos + 2] & 0b11000000) != 0b10000000) {
91	0	return false;
92	0	}
93	0	if ((data[pos + 3] & 0b11000000) != 0b10000000) {
94	0	return false;
95	0	}
96	0	// range check
97	0	code_point =
98	0	(byte & 0b00000111) << 18 \| (data[pos + 1] & 0b00111111) << 12 \|
99	0	(data[pos + 2] & 0b00111111) << 6 \| (data[pos + 3] & 0b00111111);
100	0	if (code_point <= 0xffff \|\| 0x10ffff < code_point) {
101	0	return false;
102	0	}
103	0	} else {
104	0	// we may have a continuation
105	0	return false;
106	0	}
107	0	pos = next_pos;
108	0	}
109	0	return true;
110	0	}
111
112		simdutf_really_inline simdutf_warn_unused bool validate(const char *buf,
113	0	size_t len) noexcept {
114	0	return validate(reinterpret_cast<const uint8_t *>(buf), len);
115	0	}
116
117		template <class BytePtr>
118		simdutf_constexpr23 simdutf_warn_unused result
119	0	validate_with_errors(BytePtr data, size_t len) noexcept {
120	0	static_assert(
121	0	std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value,
122	0	"dereferencing the data pointer must result in a uint8_t");
123	0	size_t pos = 0;
124	0	uint32_t code_point = 0;
125	0	while (pos < len) {
126	0	// check of the next 16 bytes are ascii.
127	0	size_t next_pos = pos + 16;
128	0	if (next_pos <=
129	0	len) { // if it is safe to read 16 more bytes, check that they are ascii
130	0	uint64_t v1;
131	0	std::memcpy(&v1, data + pos, sizeof(uint64_t));
132	0	uint64_t v2;
133	0	std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
134	0	uint64_t v{v1 \| v2};
135	0	if ((v & 0x8080808080808080) == 0) {
136	0	pos = next_pos;
137	0	continue;
138	0	}
139	0	}
140	0	unsigned char byte = data[pos];
141	0
142	0	while (byte < 0b10000000) {
143	0	if (++pos == len) {
144	0	return result(error_code::SUCCESS, len);
145	0	}
146	0	byte = data[pos];
147	0	}
148	0
149	0	if ((byte & 0b11100000) == 0b11000000) {
150	0	next_pos = pos + 2;
151	0	if (next_pos > len) {
152	0	return result(error_code::TOO_SHORT, pos);
153	0	}
154	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
155	0	return result(error_code::TOO_SHORT, pos);
156	0	}
157	0	// range check
158	0	code_point = (byte & 0b00011111) << 6 \| (data[pos + 1] & 0b00111111);
159	0	if (code_point < 0x80) {
160	0	return result(error_code::OVERLONG, pos);
161	0	}
162	0	} else if ((byte & 0b11110000) == 0b11100000) {
163	0	next_pos = pos + 3;
164	0	if (next_pos > len) {
165	0	return result(error_code::TOO_SHORT, pos);
166	0	}
167	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
168	0	return result(error_code::TOO_SHORT, pos);
169	0	}
170	0	if ((data[pos + 2] & 0b11000000) != 0b10000000) {
171	0	return result(error_code::TOO_SHORT, pos);
172	0	}
173	0	// range check
174	0	code_point = (byte & 0b00001111) << 12 \|
175	0	(data[pos + 1] & 0b00111111) << 6 \|
176	0	(data[pos + 2] & 0b00111111);
177	0	if (code_point < 0x800) {
178	0	return result(error_code::OVERLONG, pos);
179	0	}
180	0	if (0xd7ff < code_point && code_point < 0xe000) {
181	0	return result(error_code::SURROGATE, pos);
182	0	}
183	0	} else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
184	0	next_pos = pos + 4;
185	0	if (next_pos > len) {
186	0	return result(error_code::TOO_SHORT, pos);
187	0	}
188	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
189	0	return result(error_code::TOO_SHORT, pos);
190	0	}
191	0	if ((data[pos + 2] & 0b11000000) != 0b10000000) {
192	0	return result(error_code::TOO_SHORT, pos);
193	0	}
194	0	if ((data[pos + 3] & 0b11000000) != 0b10000000) {
195	0	return result(error_code::TOO_SHORT, pos);
196	0	}
197	0	// range check
198	0	code_point =
199	0	(byte & 0b00000111) << 18 \| (data[pos + 1] & 0b00111111) << 12 \|
200	0	(data[pos + 2] & 0b00111111) << 6 \| (data[pos + 3] & 0b00111111);
201	0	if (code_point <= 0xffff) {
202	0	return result(error_code::OVERLONG, pos);
203	0	}
204	0	if (0x10ffff < code_point) {
205	0	return result(error_code::TOO_LARGE, pos);
206	0	}
207	0	} else {
208	0	// we either have too many continuation bytes or an invalid leading byte
209	0	if ((byte & 0b11000000) == 0b10000000) {
210	0	return result(error_code::TOO_LONG, pos);
211	0	} else {
212	0	return result(error_code::HEADER_BITS, pos);
213	0	}
214	0	}
215	0	pos = next_pos;
216	0	}
217	0	return result(error_code::SUCCESS, len);
218	0	}
219
220		simdutf_really_inline simdutf_warn_unused result
221	0	validate_with_errors(const char *buf, size_t len) noexcept {
222	0	return validate_with_errors(reinterpret_cast<const uint8_t *>(buf), len);
223	0	}
224
225		// Finds the previous leading byte starting backward from buf and validates with
226		// errors from there Used to pinpoint the location of an error when an invalid
227		// chunk is detected We assume that the stream starts with a leading byte, and
228		// to check that it is the case, we ask that you pass a pointer to the start of
229		// the stream (start). Note that the resulting count is underflowed if an error
230		// is encountered in the rewinded segment.
231		inline simdutf_warn_unused result rewind_and_validate_with_errors(
232	0	const char start, const char buf, size_t len) noexcept {
233	0	// First check that we start with a leading byte
234	0	if ((*start & 0b11000000) == 0b10000000) {
235	0	return result(error_code::TOO_LONG, 0);
236	0	}
237	0	size_t extra_len{0};
238	0	// A leading byte cannot be further than 4 bytes away
239	0	for (int i = 0; i < 5; i++) {
240	0	unsigned char byte = *buf;
241	0	if ((byte & 0b11000000) != 0b10000000) {
242	0	break;
243	0	} else {
244	0	buf--;
245	0	extra_len++;
246	0	}
247	0	}
248	0
249	0	result res = validate_with_errors(buf, len + extra_len);
250	0	res.count -= extra_len; // Might underflow
251	0	return res;
252	0	}
253
254		// credit: based on code from Google Fuchsia (Apache Licensed)
255		template <class BytePtr>
256		simdutf_constexpr23 simdutf_warn_unused utf8_result
257	0	validate_with_counts(BytePtr data, size_t len) noexcept {
258	0	static_assert(
259	0	std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value,
260	0	"dereferencing the data pointer must result in a uint8_t");
261	0	size_t pos = 0;
262	0	uint32_t code_point = 0;
263	0	size_t continuations = 0;
264	0	size_t four_byte = 0;
265	0	while (pos < len) {
266	0	// std::memcpy is not usable during constant evaluation, so the block-wise
267	0	// ASCII fast path is restricted to run time, like in validate() above.
268	0	#if SIMDUTF_CPLUSPLUS23
269	0	if !consteval
270	0	#endif
271	0	{ // check if the next 16 bytes are ascii.
272	0	const size_t next_ascii_pos = pos + 16;
273	0	if (next_ascii_pos <= len) { // if it is safe to read 16 more bytes, check
274	0	// that they are ascii
275	0	uint64_t v1{};
276	0	std::memcpy(&v1, data + pos, sizeof(uint64_t));
277	0	uint64_t v2{};
278	0	std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
279	0	uint64_t v{v1 \| v2};
280	0	if ((v & 0x8080808080808080) == 0) {
281	0	pos = next_ascii_pos;
282	0	continue;
283	0	}
284	0	}
285	0	}
286	0
287	0	size_t next_pos;
288	0	unsigned char byte = data[pos];
289	0
290	0	while (byte < 0b10000000) {
291	0	if (++pos == len) {
292	0	return utf8_result(error_code::SUCCESS, pos, continuations, four_byte);
293	0	}
294	0	byte = data[pos];
295	0	}
296	0
297	0	if ((byte & 0b11100000) == 0b11000000) {
298	0	next_pos = pos + 2;
299	0	if (next_pos > len) {
300	0	return utf8_result(error_code::TOO_SHORT, pos, continuations,
301	0	four_byte);
302	0	}
303	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
304	0	return utf8_result(error_code::TOO_SHORT, pos, continuations,
305	0	four_byte);
306	0	}
307	0	// range check
308	0	code_point = (byte & 0b00011111) << 6 \| (data[pos + 1] & 0b00111111);
309	0	if (code_point < 0x80) {
310	0	return utf8_result(error_code::OVERLONG, pos, continuations, four_byte);
311	0	}
312	0	continuations += 1;
313	0	} else if ((byte & 0b11110000) == 0b11100000) {
314	0	next_pos = pos + 3;
315	0	if (next_pos > len) {
316	0	return utf8_result(error_code::TOO_SHORT, pos, continuations,
317	0	four_byte);
318	0	}
319	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
320	0	return utf8_result(error_code::TOO_SHORT, pos, continuations,
321	0	four_byte);
322	0	}
323	0	if ((data[pos + 2] & 0b11000000) != 0b10000000) {
324	0	return utf8_result(error_code::TOO_SHORT, pos, continuations,
325	0	four_byte);
326	0	}
327	0	// range check
328	0	code_point = (byte & 0b00001111) << 12 \|
329	0	(data[pos + 1] & 0b00111111) << 6 \|
330	0	(data[pos + 2] & 0b00111111);
331	0	if (code_point < 0x800) {
332	0	return utf8_result(error_code::OVERLONG, pos, continuations, four_byte);
333	0	} else if (0xd7ff < code_point && code_point < 0xe000) {
334	0	return utf8_result(error_code::SURROGATE, pos, continuations,
335	0	four_byte);
336	0	}
337	0	continuations += 2;
338	0	} else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
339	0	next_pos = pos + 4;
340	0	if (next_pos > len) {
341	0	return utf8_result(error_code::TOO_SHORT, pos, continuations,
342	0	four_byte);
343	0	}
344	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
345	0	return utf8_result(error_code::TOO_SHORT, pos, continuations,
346	0	four_byte);
347	0	}
348	0	if ((data[pos + 2] & 0b11000000) != 0b10000000) {
349	0	return utf8_result(error_code::TOO_SHORT, pos, continuations,
350	0	four_byte);
351	0	}
352	0	if ((data[pos + 3] & 0b11000000) != 0b10000000) {
353	0	return utf8_result(error_code::TOO_SHORT, pos, continuations,
354	0	four_byte);
355	0	}
356	0	// range check
357	0	code_point =
358	0	(byte & 0b00000111) << 18 \| (data[pos + 1] & 0b00111111) << 12 \|
359	0	(data[pos + 2] & 0b00111111) << 6 \| (data[pos + 3] & 0b00111111);
360	0	if (code_point <= 0xffff) {
361	0	return utf8_result(error_code::OVERLONG, pos, continuations, four_byte);
362	0	} else if (0x10ffff < code_point) {
363	0	return utf8_result(error_code::TOO_LARGE, pos, continuations,
364	0	four_byte);
365	0	}
366	0	continuations += 3;
367	0	four_byte += 1;
368	0	} else {
369	0	// Continuation byte or invalid header byte
370	0	if ((byte & 0b11000000) == 0b10000000) {
371	0	return utf8_result(error_code::TOO_LONG, pos, continuations, four_byte);
372	0	} else {
373	0	return utf8_result(error_code::HEADER_BITS, pos, continuations,
374	0	four_byte);
375	0	}
376	0	}
377	0	pos = next_pos;
378	0	}
379	0	return utf8_result(error_code::SUCCESS, pos, continuations, four_byte);
380	0	}
381
382		simdutf_really_inline simdutf_warn_unused utf8_result
383	0	validate_with_counts(const char *buf, size_t len) noexcept {
384	0	return validate_with_counts(reinterpret_cast<const uint8_t *>(buf), len);
385	0	}
386
387		// Finds the previous leading byte starting backward from buf and validates from
388		// there on out while counting continuation bytes and 4-byte leads. Used to
389		// pinpoint the location of an error when an invalid chunk is detected. We check
390		// that the stream starts with a leading byte with the passed pointer to the
391		// start of the stream (start).
392		//
393		// All three fields of the result are relative to buf. The bytes we rewind over
394		// sit in a block the caller has already counted, so their contribution is
395		// subtracted back out at the end. When the error lies inside the rewound
396		// region, those three subtractions underflow. That is intended, not an
397		// oversight: the caller adds the offset of buf and the counts of the preceding
398		// blocks, which include the rewound bytes, and the unsigned wraparound cancels
399		// exactly, leaving the correct absolute values.
400		inline simdutf_warn_unused utf8_result rewind_and_validate_with_counts(
401	0	const char start, const char buf, size_t len) noexcept {
402	0	// First check that we start with a leading byte
403	0	if ((*start & 0b11000000) == 0b10000000) {
404	0	return utf8_result(error_code::TOO_LONG, 0, 0, 0);
405	0	}
406	0	// If this is the start of the buffer, we don't need to rewind to check the
407	0	// previous chunk.
408	0	if (start == buf) {
409	0	return validate_with_counts(buf, len);
410	0	}
411	0	// As this isn't the start of the buffer, the error might be in the previous
412	0	// chunk. So we backtrack and also need to avoid double counting.
413	0	size_t extra_len{0};
414	0	size_t extra_continuations{0};
415	0	size_t extra_four_bytes{0};
416	0	// Four steps are enough. Everything before buf has already validated, so at
417	0	// most three continuation bytes can precede it, from a four-byte sequence
418	0	// ending exactly on the boundary; the fourth step then lands on its leading
419	0	// byte. Note that the sibling rewind_and_validate_with_errors needs five
420	0	// steps only because it tests the byte before stepping back, whereas we step
421	0	// back first. Callers pass a block boundary at least 64 bytes into the
422	0	// stream, so stepping back four bytes stays in bounds.
423	0	for (int i = 0; i < 4; i++) {
424	0	// We rewind at least one byte to find the actual previous leading byte.
425	0	// We do so to backtrack into the last chunk in case the utf8 error happened
426	0	// there already.
427	0	buf--;
428	0	unsigned char byte = *buf;
429	0	extra_len++;
430	0
431	0	if (byte >= 0b10000000) {
432	0	if (byte >= 0b11110000) {
433	0	extra_four_bytes += 1;
434	0	} else if ((int8_t)byte < -65 + 1) {
435	0	extra_continuations += 1;
436	0	}
437	0	}
438	0	if ((byte & 0b11000000) != 0b10000000) {
439	0	break;
440	0	}
441	0	}
442	0
443	0	utf8_result res = validate_with_counts(buf, len + extra_len);
444	0	res.input_count -= extra_len;
445	0	res.continuation_count -= extra_continuations;
446	0	res.four_byte_count -= extra_four_bytes;
447	0	return res;
448	0	}
449
450		template <typename InputPtr>
451		#if SIMDUTF_CPLUSPLUS20
452		requires simdutf::detail::indexes_into_byte_like<InputPtr>
453		#endif
454		simdutf_constexpr23 size_t count_code_points(InputPtr data, size_t len) {
455		size_t counter{0};
456		for (size_t i = 0; i < len; i++) {
457		// -65 is 0b10111111, anything larger in two-complement's should start a new
458		// code point.
459		if (int8_t(data[i]) > -65) {
460		counter++;
461		}
462		}
463		return counter;
464		}
465
466		template <typename InputPtr>
467		#if SIMDUTF_CPLUSPLUS20
468		requires simdutf::detail::indexes_into_byte_like<InputPtr>
469		#endif
470		simdutf_constexpr23 size_t utf16_length_from_utf8(InputPtr data, size_t len) {
471		size_t counter{0};
472		for (size_t i = 0; i < len; i++) {
473		if (int8_t(data[i]) > -65) {
474		counter++;
475		}
476		if (uint8_t(data[i]) >= 240) {
477		counter++;
478		}
479		}
480		return counter;
481		}
482
483		template <typename InputPtr>
484		#if SIMDUTF_CPLUSPLUS20
485		requires simdutf::detail::indexes_into_byte_like<InputPtr>
486		#endif
487		simdutf_warn_unused simdutf_constexpr23 size_t
488		trim_partial_utf8(InputPtr input, size_t length) {
489		if (length < 3) {
490		switch (length) {
491		case 2:
492		if (uint8_t(input[length - 1]) >= 0xc0) {
493		return length - 1;
494		} // 2-, 3- and 4-byte characters with only 1 byte left
495		if (uint8_t(input[length - 2]) >= 0xe0) {
496		return length - 2;
497		} // 3- and 4-byte characters with only 2 bytes left
498		return length;
499		case 1:
500		if (uint8_t(input[length - 1]) >= 0xc0) {
501		return length - 1;
502		} // 2-, 3- and 4-byte characters with only 1 byte left
503		return length;
504		case 0:
505		return length;
506		}
507		}
508		if (uint8_t(input[length - 1]) >= 0xc0) {
509		return length - 1;
510		} // 2-, 3- and 4-byte characters with only 1 byte left
511		if (uint8_t(input[length - 2]) >= 0xe0) {
512		return length - 2;
513		} // 3- and 4-byte characters with only 1 byte left
514		if (uint8_t(input[length - 3]) >= 0xf0) {
515		return length - 3;
516		} // 4-byte characters with only 3 bytes left
517		return length;
518		}
519
520		} // namespace utf8
521		} // unnamed namespace
522		} // namespace scalar
523		} // namespace simdutf
524
525		#endif