/work/include/simdutf/scalar/utf8.h

Source
#ifndef SIMDUTF_UTF8_H
#define SIMDUTF_UTF8_H

#include <cstring>

namespace simdutf {
namespace scalar {
namespace {
namespace utf8 {

// credit: based on code from Google Fuchsia (Apache Licensed)
template <class BytePtr>
simdutf_constexpr23 simdutf_warn_unused bool validate(BytePtr data,
                                                      size_t len) noexcept {
  static_assert(
      std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value,
      "dereferencing the data pointer must result in a uint8_t");
  uint64_t pos = 0;
  uint32_t code_point = 0;
  while (pos < len) {
    uint64_t next_pos;
#if SIMDUTF_CPLUSPLUS23
    if !consteval
#endif
    { // check if the next 16 bytes are ascii.
      next_pos = pos + 16;
      if (next_pos <= len) { // if it is safe to read 16 more bytes, check
                             // that they are ascii
        uint64_t v1{};
        std::memcpy(&v1, data + pos, sizeof(uint64_t));
        uint64_t v2{};
        std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
        uint64_t v{v1 | v2};
        if ((v & 0x8080808080808080) == 0) {
          pos = next_pos;
          continue;
        }
      }
    }

    unsigned char byte = data[pos];

    while (byte < 0b10000000) {
      if (++pos == len) {
        return true;
      }
      byte = data[pos];
    }

    if ((byte & 0b11100000) == 0b11000000) {
      next_pos = pos + 2;
      if (next_pos > len) {
        return false;
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return false;
      }
      // range check
      code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
      if (code_point < 0x80) {
        return false;
      }
    } else if ((byte & 0b11110000) == 0b11100000) {
      next_pos = pos + 3;
      if (next_pos > len) {
        return false;
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return false;
      }
      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
        return false;
      }
      // range check
      code_point = (byte & 0b00001111) << 12 |
                   (data[pos + 1] & 0b00111111) << 6 |
                   (data[pos + 2] & 0b00111111);
      if ((code_point < 0x800) ||
          (0xd7ff < code_point && code_point < 0xe000)) {
        return false;
      }
    } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
      next_pos = pos + 4;
      if (next_pos > len) {
        return false;
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return false;
      }
      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
        return false;
      }
      if ((data[pos + 3] & 0b11000000) != 0b10000000) {
        return false;
      }
      // range check
      code_point =
          (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 |
          (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111);
      if (code_point <= 0xffff || 0x10ffff < code_point) {
        return false;
      }
    } else {
      // we may have a continuation
      return false;
    }
    pos = next_pos;
  }
  return true;
}

simdutf_really_inline simdutf_warn_unused bool validate(const char *buf,
                                                        size_t len) noexcept {
  return validate(reinterpret_cast<const uint8_t *>(buf), len);
}

template <class BytePtr>
simdutf_constexpr23 simdutf_warn_unused result
validate_with_errors(BytePtr data, size_t len) noexcept {
  static_assert(
      std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value,
      "dereferencing the data pointer must result in a uint8_t");
  size_t pos = 0;
  uint32_t code_point = 0;
  while (pos < len) {
    // check of the next 16 bytes are ascii.
    size_t next_pos = pos + 16;
    if (next_pos <=
        len) { // if it is safe to read 16 more bytes, check that they are ascii
      uint64_t v1;
      std::memcpy(&v1, data + pos, sizeof(uint64_t));
      uint64_t v2;
      std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
      uint64_t v{v1 | v2};
      if ((v & 0x8080808080808080) == 0) {
        pos = next_pos;
        continue;
      }
    }
    unsigned char byte = data[pos];

    while (byte < 0b10000000) {
      if (++pos == len) {
        return result(error_code::SUCCESS, len);
      }
      byte = data[pos];
    }

    if ((byte & 0b11100000) == 0b11000000) {
      next_pos = pos + 2;
      if (next_pos > len) {
        return result(error_code::TOO_SHORT, pos);
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return result(error_code::TOO_SHORT, pos);
      }
      // range check
      code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
      if (code_point < 0x80) {
        return result(error_code::OVERLONG, pos);
      }
    } else if ((byte & 0b11110000) == 0b11100000) {
      next_pos = pos + 3;
      if (next_pos > len) {
        return result(error_code::TOO_SHORT, pos);
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return result(error_code::TOO_SHORT, pos);
      }
      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
        return result(error_code::TOO_SHORT, pos);
      }
      // range check
      code_point = (byte & 0b00001111) << 12 |
                   (data[pos + 1] & 0b00111111) << 6 |
                   (data[pos + 2] & 0b00111111);
      if (code_point < 0x800) {
        return result(error_code::OVERLONG, pos);
      }
      if (0xd7ff < code_point && code_point < 0xe000) {
        return result(error_code::SURROGATE, pos);
      }
    } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
      next_pos = pos + 4;
      if (next_pos > len) {
        return result(error_code::TOO_SHORT, pos);
      }
      if ((data[pos + 1] & 0b11000000) != 0b10000000) {
        return result(error_code::TOO_SHORT, pos);
      }
      if ((data[pos + 2] & 0b11000000) != 0b10000000) {
        return result(error_code::TOO_SHORT, pos);
      }
      if ((data[pos + 3] & 0b11000000) != 0b10000000) {
        return result(error_code::TOO_SHORT, pos);
      }
      // range check
      code_point =
          (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 |
          (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111);
      if (code_point <= 0xffff) {
        return result(error_code::OVERLONG, pos);
      }
      if (0x10ffff < code_point) {
        return result(error_code::TOO_LARGE, pos);
      }
    } else {
      // we either have too many continuation bytes or an invalid leading byte
      if ((byte & 0b11000000) == 0b10000000) {
        return result(error_code::TOO_LONG, pos);
      } else {
        return result(error_code::HEADER_BITS, pos);
      }
    }
    pos = next_pos;
  }
  return result(error_code::SUCCESS, len);
}

simdutf_really_inline simdutf_warn_unused result
validate_with_errors(const char *buf, size_t len) noexcept {
  return validate_with_errors(reinterpret_cast<const uint8_t *>(buf), len);
}

// Finds the previous leading byte starting backward from buf and validates with
// errors from there Used to pinpoint the location of an error when an invalid
// chunk is detected We assume that the stream starts with a leading byte, and
// to check that it is the case, we ask that you pass a pointer to the start of
// the stream (start). Note that the resulting count is underflowed if an error
// is encountered in the rewinded segment.
inline simdutf_warn_unused result rewind_and_validate_with_errors(
    const char *start, const char *buf, size_t len) noexcept {
  // First check that we start with a leading byte
  if ((*start & 0b11000000) == 0b10000000) {
    return result(error_code::TOO_LONG, 0);
  }
  size_t extra_len{0};
  // A leading byte cannot be further than 4 bytes away
  for (int i = 0; i < 5; i++) {
    unsigned char byte = *buf;
    if ((byte & 0b11000000) != 0b10000000) {
      break;
    } else {
      buf--;
      extra_len++;
    }
  }

  result res = validate_with_errors(buf, len + extra_len);
  res.count -= extra_len; // Might underflow
  return res;
}

template <typename InputPtr>
#if SIMDUTF_CPLUSPLUS20
  requires simdutf::detail::indexes_into_byte_like<InputPtr>
#endif
simdutf_constexpr23 size_t count_code_points(InputPtr data, size_t len) {
  size_t counter{0};
  for (size_t i = 0; i < len; i++) {
    // -65 is 0b10111111, anything larger in two-complement's should start a new
    // code point.
    if (int8_t(data[i]) > -65) {
      counter++;
    }
  }
  return counter;
}

template <typename InputPtr>
#if SIMDUTF_CPLUSPLUS20
  requires simdutf::detail::indexes_into_byte_like<InputPtr>
#endif
simdutf_constexpr23 size_t utf16_length_from_utf8(InputPtr data, size_t len) {
  size_t counter{0};
  for (size_t i = 0; i < len; i++) {
    if (int8_t(data[i]) > -65) {
      counter++;
    }
    if (uint8_t(data[i]) >= 240) {
      counter++;
    }
  }
  return counter;
}

template <typename InputPtr>
#if SIMDUTF_CPLUSPLUS20
  requires simdutf::detail::indexes_into_byte_like<InputPtr>
#endif
simdutf_warn_unused simdutf_constexpr23 size_t
trim_partial_utf8(InputPtr input, size_t length) {
  if (length < 3) {
    switch (length) {
    case 2:
      if (uint8_t(input[length - 1]) >= 0xc0) {
        return length - 1;
      } // 2-, 3- and 4-byte characters with only 1 byte left
      if (uint8_t(input[length - 2]) >= 0xe0) {
        return length - 2;
      } // 3- and 4-byte characters with only 2 bytes left
      return length;
    case 1:
      if (uint8_t(input[length - 1]) >= 0xc0) {
        return length - 1;
      } // 2-, 3- and 4-byte characters with only 1 byte left
      return length;
    case 0:
      return length;
    }
  }
  if (uint8_t(input[length - 1]) >= 0xc0) {
    return length - 1;
  } // 2-, 3- and 4-byte characters with only 1 byte left
  if (uint8_t(input[length - 2]) >= 0xe0) {
    return length - 2;
  } // 3- and 4-byte characters with only 1 byte left
  if (uint8_t(input[length - 3]) >= 0xf0) {
    return length - 3;
  } // 4-byte characters with only 3 bytes left
  return length;
}

} // namespace utf8
} // unnamed namespace
} // namespace scalar
} // namespace simdutf

#endif

Coverage Report

Created: 2026-07-25 06:54

Line	Count	Source
1		#ifndef SIMDUTF_UTF8_H
2		#define SIMDUTF_UTF8_H
3
4		#include <cstring>
5
6		namespace simdutf {
7		namespace scalar {
8		namespace {
9		namespace utf8 {
10
11		// credit: based on code from Google Fuchsia (Apache Licensed)
12		template <class BytePtr>
13		simdutf_constexpr23 simdutf_warn_unused bool validate(BytePtr data,
14	0	size_t len) noexcept {
15	0	static_assert(
16	0	std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value,
17	0	"dereferencing the data pointer must result in a uint8_t");
18	0	uint64_t pos = 0;
19	0	uint32_t code_point = 0;
20	0	while (pos < len) {
21	0	uint64_t next_pos;
22	0	#if SIMDUTF_CPLUSPLUS23
23	0	if !consteval
24	0	#endif
25	0	{ // check if the next 16 bytes are ascii.
26	0	next_pos = pos + 16;
27	0	if (next_pos <= len) { // if it is safe to read 16 more bytes, check
28	0	// that they are ascii
29	0	uint64_t v1{};
30	0	std::memcpy(&v1, data + pos, sizeof(uint64_t));
31	0	uint64_t v2{};
32	0	std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
33	0	uint64_t v{v1 \| v2};
34	0	if ((v & 0x8080808080808080) == 0) {
35	0	pos = next_pos;
36	0	continue;
37	0	}
38	0	}
39	0	}
40	0
41	0	unsigned char byte = data[pos];
42	0
43	0	while (byte < 0b10000000) {
44	0	if (++pos == len) {
45	0	return true;
46	0	}
47	0	byte = data[pos];
48	0	}
49	0
50	0	if ((byte & 0b11100000) == 0b11000000) {
51	0	next_pos = pos + 2;
52	0	if (next_pos > len) {
53	0	return false;
54	0	}
55	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
56	0	return false;
57	0	}
58	0	// range check
59	0	code_point = (byte & 0b00011111) << 6 \| (data[pos + 1] & 0b00111111);
60	0	if (code_point < 0x80) {
61	0	return false;
62	0	}
63	0	} else if ((byte & 0b11110000) == 0b11100000) {
64	0	next_pos = pos + 3;
65	0	if (next_pos > len) {
66	0	return false;
67	0	}
68	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
69	0	return false;
70	0	}
71	0	if ((data[pos + 2] & 0b11000000) != 0b10000000) {
72	0	return false;
73	0	}
74	0	// range check
75	0	code_point = (byte & 0b00001111) << 12 \|
76	0	(data[pos + 1] & 0b00111111) << 6 \|
77	0	(data[pos + 2] & 0b00111111);
78	0	if ((code_point < 0x800) \|\|
79	0	(0xd7ff < code_point && code_point < 0xe000)) {
80	0	return false;
81	0	}
82	0	} else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
83	0	next_pos = pos + 4;
84	0	if (next_pos > len) {
85	0	return false;
86	0	}
87	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
88	0	return false;
89	0	}
90	0	if ((data[pos + 2] & 0b11000000) != 0b10000000) {
91	0	return false;
92	0	}
93	0	if ((data[pos + 3] & 0b11000000) != 0b10000000) {
94	0	return false;
95	0	}
96	0	// range check
97	0	code_point =
98	0	(byte & 0b00000111) << 18 \| (data[pos + 1] & 0b00111111) << 12 \|
99	0	(data[pos + 2] & 0b00111111) << 6 \| (data[pos + 3] & 0b00111111);
100	0	if (code_point <= 0xffff \|\| 0x10ffff < code_point) {
101	0	return false;
102	0	}
103	0	} else {
104	0	// we may have a continuation
105	0	return false;
106	0	}
107	0	pos = next_pos;
108	0	}
109	0	return true;
110	0	}
111
112		simdutf_really_inline simdutf_warn_unused bool validate(const char *buf,
113	0	size_t len) noexcept {
114	0	return validate(reinterpret_cast<const uint8_t *>(buf), len);
115	0	}
116
117		template <class BytePtr>
118		simdutf_constexpr23 simdutf_warn_unused result
119	0	validate_with_errors(BytePtr data, size_t len) noexcept {
120	0	static_assert(
121	0	std::is_same<typename std::decay<decltype(*data)>::type, uint8_t>::value,
122	0	"dereferencing the data pointer must result in a uint8_t");
123	0	size_t pos = 0;
124	0	uint32_t code_point = 0;
125	0	while (pos < len) {
126	0	// check of the next 16 bytes are ascii.
127	0	size_t next_pos = pos + 16;
128	0	if (next_pos <=
129	0	len) { // if it is safe to read 16 more bytes, check that they are ascii
130	0	uint64_t v1;
131	0	std::memcpy(&v1, data + pos, sizeof(uint64_t));
132	0	uint64_t v2;
133	0	std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
134	0	uint64_t v{v1 \| v2};
135	0	if ((v & 0x8080808080808080) == 0) {
136	0	pos = next_pos;
137	0	continue;
138	0	}
139	0	}
140	0	unsigned char byte = data[pos];
141	0
142	0	while (byte < 0b10000000) {
143	0	if (++pos == len) {
144	0	return result(error_code::SUCCESS, len);
145	0	}
146	0	byte = data[pos];
147	0	}
148	0
149	0	if ((byte & 0b11100000) == 0b11000000) {
150	0	next_pos = pos + 2;
151	0	if (next_pos > len) {
152	0	return result(error_code::TOO_SHORT, pos);
153	0	}
154	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
155	0	return result(error_code::TOO_SHORT, pos);
156	0	}
157	0	// range check
158	0	code_point = (byte & 0b00011111) << 6 \| (data[pos + 1] & 0b00111111);
159	0	if (code_point < 0x80) {
160	0	return result(error_code::OVERLONG, pos);
161	0	}
162	0	} else if ((byte & 0b11110000) == 0b11100000) {
163	0	next_pos = pos + 3;
164	0	if (next_pos > len) {
165	0	return result(error_code::TOO_SHORT, pos);
166	0	}
167	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
168	0	return result(error_code::TOO_SHORT, pos);
169	0	}
170	0	if ((data[pos + 2] & 0b11000000) != 0b10000000) {
171	0	return result(error_code::TOO_SHORT, pos);
172	0	}
173	0	// range check
174	0	code_point = (byte & 0b00001111) << 12 \|
175	0	(data[pos + 1] & 0b00111111) << 6 \|
176	0	(data[pos + 2] & 0b00111111);
177	0	if (code_point < 0x800) {
178	0	return result(error_code::OVERLONG, pos);
179	0	}
180	0	if (0xd7ff < code_point && code_point < 0xe000) {
181	0	return result(error_code::SURROGATE, pos);
182	0	}
183	0	} else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
184	0	next_pos = pos + 4;
185	0	if (next_pos > len) {
186	0	return result(error_code::TOO_SHORT, pos);
187	0	}
188	0	if ((data[pos + 1] & 0b11000000) != 0b10000000) {
189	0	return result(error_code::TOO_SHORT, pos);
190	0	}
191	0	if ((data[pos + 2] & 0b11000000) != 0b10000000) {
192	0	return result(error_code::TOO_SHORT, pos);
193	0	}
194	0	if ((data[pos + 3] & 0b11000000) != 0b10000000) {
195	0	return result(error_code::TOO_SHORT, pos);
196	0	}
197	0	// range check
198	0	code_point =
199	0	(byte & 0b00000111) << 18 \| (data[pos + 1] & 0b00111111) << 12 \|
200	0	(data[pos + 2] & 0b00111111) << 6 \| (data[pos + 3] & 0b00111111);
201	0	if (code_point <= 0xffff) {
202	0	return result(error_code::OVERLONG, pos);
203	0	}
204	0	if (0x10ffff < code_point) {
205	0	return result(error_code::TOO_LARGE, pos);
206	0	}
207	0	} else {
208	0	// we either have too many continuation bytes or an invalid leading byte
209	0	if ((byte & 0b11000000) == 0b10000000) {
210	0	return result(error_code::TOO_LONG, pos);
211	0	} else {
212	0	return result(error_code::HEADER_BITS, pos);
213	0	}
214	0	}
215	0	pos = next_pos;
216	0	}
217	0	return result(error_code::SUCCESS, len);
218	0	}
219
220		simdutf_really_inline simdutf_warn_unused result
221	0	validate_with_errors(const char *buf, size_t len) noexcept {
222	0	return validate_with_errors(reinterpret_cast<const uint8_t *>(buf), len);
223	0	}
224
225		// Finds the previous leading byte starting backward from buf and validates with
226		// errors from there Used to pinpoint the location of an error when an invalid
227		// chunk is detected We assume that the stream starts with a leading byte, and
228		// to check that it is the case, we ask that you pass a pointer to the start of
229		// the stream (start). Note that the resulting count is underflowed if an error
230		// is encountered in the rewinded segment.
231		inline simdutf_warn_unused result rewind_and_validate_with_errors(
232	0	const char start, const char buf, size_t len) noexcept {
233	0	// First check that we start with a leading byte
234	0	if ((*start & 0b11000000) == 0b10000000) {
235	0	return result(error_code::TOO_LONG, 0);
236	0	}
237	0	size_t extra_len{0};
238	0	// A leading byte cannot be further than 4 bytes away
239	0	for (int i = 0; i < 5; i++) {
240	0	unsigned char byte = *buf;
241	0	if ((byte & 0b11000000) != 0b10000000) {
242	0	break;
243	0	} else {
244	0	buf--;
245	0	extra_len++;
246	0	}
247	0	}
248	0
249	0	result res = validate_with_errors(buf, len + extra_len);
250	0	res.count -= extra_len; // Might underflow
251	0	return res;
252	0	}
253
254		template <typename InputPtr>
255		#if SIMDUTF_CPLUSPLUS20
256		requires simdutf::detail::indexes_into_byte_like<InputPtr>
257		#endif
258		simdutf_constexpr23 size_t count_code_points(InputPtr data, size_t len) {
259		size_t counter{0};
260		for (size_t i = 0; i < len; i++) {
261		// -65 is 0b10111111, anything larger in two-complement's should start a new
262		// code point.
263		if (int8_t(data[i]) > -65) {
264		counter++;
265		}
266		}
267		return counter;
268		}
269
270		template <typename InputPtr>
271		#if SIMDUTF_CPLUSPLUS20
272		requires simdutf::detail::indexes_into_byte_like<InputPtr>
273		#endif
274		simdutf_constexpr23 size_t utf16_length_from_utf8(InputPtr data, size_t len) {
275		size_t counter{0};
276		for (size_t i = 0; i < len; i++) {
277		if (int8_t(data[i]) > -65) {
278		counter++;
279		}
280		if (uint8_t(data[i]) >= 240) {
281		counter++;
282		}
283		}
284		return counter;
285		}
286
287		template <typename InputPtr>
288		#if SIMDUTF_CPLUSPLUS20
289		requires simdutf::detail::indexes_into_byte_like<InputPtr>
290		#endif
291		simdutf_warn_unused simdutf_constexpr23 size_t
292		trim_partial_utf8(InputPtr input, size_t length) {
293		if (length < 3) {
294		switch (length) {
295		case 2:
296		if (uint8_t(input[length - 1]) >= 0xc0) {
297		return length - 1;
298		} // 2-, 3- and 4-byte characters with only 1 byte left
299		if (uint8_t(input[length - 2]) >= 0xe0) {
300		return length - 2;
301		} // 3- and 4-byte characters with only 2 bytes left
302		return length;
303		case 1:
304		if (uint8_t(input[length - 1]) >= 0xc0) {
305		return length - 1;
306		} // 2-, 3- and 4-byte characters with only 1 byte left
307		return length;
308		case 0:
309		return length;
310		}
311		}
312		if (uint8_t(input[length - 1]) >= 0xc0) {
313		return length - 1;
314		} // 2-, 3- and 4-byte characters with only 1 byte left
315		if (uint8_t(input[length - 2]) >= 0xe0) {
316		return length - 2;
317		} // 3- and 4-byte characters with only 1 byte left
318		if (uint8_t(input[length - 3]) >= 0xf0) {
319		return length - 3;
320		} // 4-byte characters with only 3 bytes left
321		return length;
322		}
323
324		} // namespace utf8
325		} // unnamed namespace
326		} // namespace scalar
327		} // namespace simdutf
328
329		#endif