/src/simdutf/include/simdutf/scalar/utf16.h

Source
#ifndef SIMDUTF_UTF16_H
#define SIMDUTF_UTF16_H

namespace simdutf {
namespace scalar {
namespace utf16 {

template <endianness big_endian>
simdutf_warn_unused simdutf_constexpr23 bool
validate_as_ascii(const char16_t *data, size_t len) noexcept {
  for (size_t pos = 0; pos < len; pos++) {
    char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]);
    if (word >= 0x80) {
      return false;
    }
  }
  return true;
}

template <endianness big_endian>
inline simdutf_warn_unused simdutf_constexpr23 bool
validate(const char16_t *data, size_t len) noexcept {
  uint64_t pos = 0;
  while (pos < len) {
    char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]);
    if ((word & 0xF800) == 0xD800) {
      if (pos + 1 >= len) {
        return false;
      }
      char16_t diff = char16_t(word - 0xD800);
      if (diff > 0x3FF) {
        return false;
      }
      char16_t next_word = !match_system(big_endian)
                               ? u16_swap_bytes(data[pos + 1])
                               : data[pos + 1];
      char16_t diff2 = char16_t(next_word - 0xDC00);
      if (diff2 > 0x3FF) {
        return false;
      }
      pos += 2;
    } else {
      pos++;
    }
  }
  return true;
}

template <endianness big_endian>
inline simdutf_warn_unused simdutf_constexpr23 result
validate_with_errors(const char16_t *data, size_t len) noexcept {
  size_t pos = 0;
  while (pos < len) {
    char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]);
    if ((word & 0xF800) == 0xD800) {
      if (pos + 1 >= len) {
        return result(error_code::SURROGATE, pos);
      }
      char16_t diff = char16_t(word - 0xD800);
      if (diff > 0x3FF) {
        return result(error_code::SURROGATE, pos);
      }
      char16_t next_word = !match_system(big_endian)
                               ? u16_swap_bytes(data[pos + 1])
                               : data[pos + 1];
      char16_t diff2 = uint16_t(next_word - 0xDC00);
      if (diff2 > 0x3FF) {
        return result(error_code::SURROGATE, pos);
      }
      pos += 2;
    } else {
      pos++;
    }
  }
  return result(error_code::SUCCESS, pos);
}

template <endianness big_endian>
simdutf_constexpr23 size_t count_code_points(const char16_t *p, size_t len) {
  // We are not BOM aware.
  size_t counter{0};
  for (size_t i = 0; i < len; i++) {
    char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]);
    counter += ((word & 0xFC00) != 0xDC00);
  }
  return counter;
}

template <endianness big_endian>
simdutf_constexpr23 size_t utf8_length_from_utf16(const char16_t *p,
                                                  size_t len) {
  // We are not BOM aware.
  size_t counter{0};
  for (size_t i = 0; i < len; i++) {
    char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]);
    counter++; // ASCII
    counter += static_cast<size_t>(
        word >
        0x7F); // non-ASCII is at least 2 bytes, surrogates are 2*2 == 4 bytes
    counter += static_cast<size_t>((word > 0x7FF && word <= 0xD7FF) ||
                                   (word >= 0xE000)); // three-byte
  }
  return counter;
}

template <endianness big_endian>
simdutf_constexpr23 size_t utf32_length_from_utf16(const char16_t *p,
                                                   size_t len) {
  // We are not BOM aware.
  size_t counter{0};
  for (size_t i = 0; i < len; i++) {
    char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]);
    counter += ((word & 0xFC00) != 0xDC00);
  }
  return counter;
}

simdutf_really_inline simdutf_constexpr23 void
change_endianness_utf16(const char16_t *input, size_t size, char16_t *output) {
  for (size_t i = 0; i < size; i++) {
    *output++ = char16_t(input[i] >> 8 | input[i] << 8);
  }
}

template <endianness big_endian>
simdutf_warn_unused simdutf_constexpr23 size_t
trim_partial_utf16(const char16_t *input, size_t length) {
  if (length == 0) {
    return 0;
  }
  uint16_t last_word = uint16_t(input[length - 1]);
  last_word = scalar::utf16::swap_if_needed<big_endian>(last_word);
  length -= ((last_word & 0xFC00) == 0xD800);
  return length;
}

template <endianness big_endian> constexpr bool is_high_surrogate(char16_t c) {
  c = scalar::utf16::swap_if_needed<big_endian>(c);
  return (0xd800 <= c && c <= 0xdbff);
}

template <endianness big_endian> constexpr bool is_low_surrogate(char16_t c) {
  c = scalar::utf16::swap_if_needed<big_endian>(c);
  return (0xdc00 <= c && c <= 0xdfff);
}

simdutf_unused simdutf_really_inline constexpr bool high_surrogate(char16_t c) {
  return (0xd800 <= c && c <= 0xdbff);
}

template <endianness big_endian>
simdutf_constexpr23 result
utf8_length_from_utf16_with_replacement(const char16_t *p, size_t len) {
  bool any_surrogates = false;
  // We are not BOM aware.
  size_t counter{0};
  for (size_t i = 0; i < len; i++) {
    if (is_high_surrogate<big_endian>(p[i])) {
      any_surrogates = true;
      // surrogate pair
      if (i + 1 < len && is_low_surrogate<big_endian>(p[i + 1])) {
        counter += 4;
        i++; // skip low surrogate
      } else {
        counter += 3; // unpaired high surrogate replaced by U+FFFD
      }
      continue;
    } else if (is_low_surrogate<big_endian>(p[i])) {
      any_surrogates = true;
      counter += 3; // unpaired low surrogate replaced by U+FFFD
      continue;
    }
    char16_t word = !match_system(big_endian) ? u16_swap_bytes(p[i]) : p[i];
    counter++; // at least 1 byte
    counter +=
        static_cast<size_t>(word > 0x7F); // non-ASCII is at least 2 bytes
    counter += static_cast<size_t>(word > 0x7FF); // three-byte
  }
  return {any_surrogates ? error_code::SURROGATE : error_code::SUCCESS,
          counter};
}

// variable templates are a C++14 extension
template <endianness big_endian> constexpr char16_t replacement() {
  return !match_system(big_endian) ? scalar::u16_swap_bytes(0xfffd) : 0xfffd;
}

template <endianness big_endian>
simdutf_constexpr23 void to_well_formed_utf16(const char16_t *input, size_t len,
                                              char16_t *output) {
  const char16_t replacement = utf16::replacement<big_endian>();
  bool high_surrogate_prev = false, high_surrogate, low_surrogate;
  size_t i = 0;
  for (; i < len; i++) {
    char16_t c = input[i];
    high_surrogate = is_high_surrogate<big_endian>(c);
    low_surrogate = is_low_surrogate<big_endian>(c);
    if (high_surrogate_prev && !low_surrogate) {
      output[i - 1] = replacement;
    }

    if (!high_surrogate_prev && low_surrogate) {
      output[i] = replacement;
    } else {
      output[i] = input[i];
    }
    high_surrogate_prev = high_surrogate;
  }

  /* string may not end with high surrogate */
  if (high_surrogate_prev) {
    output[i - 1] = replacement;
  }
}

} // namespace utf16
} // namespace scalar
} // namespace simdutf

#endif

Coverage Report

Created: 2026-09-01 06:49

Line	Count	Source
1		#ifndef SIMDUTF_UTF16_H
2		#define SIMDUTF_UTF16_H
3
4		namespace simdutf {
5		namespace scalar {
6		namespace utf16 {
7
8		template <endianness big_endian>
9		simdutf_warn_unused simdutf_constexpr23 bool
10	0	validate_as_ascii(const char16_t *data, size_t len) noexcept {
11	0	for (size_t pos = 0; pos < len; pos++) {
12	0	char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]);
13	0	if (word >= 0x80) {
14	0	return false;
15	0	}
16	0	}
17	0	return true;
18	0	} Unexecuted instantiation: bool simdutf::scalar::utf16::validate_as_ascii<(simdutf::endianness)0>(char16_t const, unsigned long) Unexecuted instantiation: bool simdutf::scalar::utf16::validate_as_ascii<(simdutf::endianness)1>(char16_t const, unsigned long)
19
20		template <endianness big_endian>
21		inline simdutf_warn_unused simdutf_constexpr23 bool
22	0	validate(const char16_t *data, size_t len) noexcept {
23	0	uint64_t pos = 0;
24	0	while (pos < len) {
25	0	char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]);
26	0	if ((word & 0xF800) == 0xD800) {
27	0	if (pos + 1 >= len) {
28	0	return false;
29	0	}
30	0	char16_t diff = char16_t(word - 0xD800);
31	0	if (diff > 0x3FF) {
32	0	return false;
33	0	}
34	0	char16_t next_word = !match_system(big_endian)
35	0	? u16_swap_bytes(data[pos + 1])
36	0	: data[pos + 1];
37	0	char16_t diff2 = char16_t(next_word - 0xDC00);
38	0	if (diff2 > 0x3FF) {
39	0	return false;
40	0	}
41	0	pos += 2;
42	0	} else {
43	0	pos++;
44	0	}
45	0	}
46	0	return true;
47	0	} Unexecuted instantiation: bool simdutf::scalar::utf16::validate<(simdutf::endianness)0>(char16_t const, unsigned long) Unexecuted instantiation: bool simdutf::scalar::utf16::validate<(simdutf::endianness)1>(char16_t const, unsigned long)
48
49		template <endianness big_endian>
50		inline simdutf_warn_unused simdutf_constexpr23 result
51	0	validate_with_errors(const char16_t *data, size_t len) noexcept {
52	0	size_t pos = 0;
53	0	while (pos < len) {
54	0	char16_t word = scalar::utf16::swap_if_needed<big_endian>(data[pos]);
55	0	if ((word & 0xF800) == 0xD800) {
56	0	if (pos + 1 >= len) {
57	0	return result(error_code::SURROGATE, pos);
58	0	}
59	0	char16_t diff = char16_t(word - 0xD800);
60	0	if (diff > 0x3FF) {
61	0	return result(error_code::SURROGATE, pos);
62	0	}
63	0	char16_t next_word = !match_system(big_endian)
64	0	? u16_swap_bytes(data[pos + 1])
65	0	: data[pos + 1];
66	0	char16_t diff2 = uint16_t(next_word - 0xDC00);
67	0	if (diff2 > 0x3FF) {
68	0	return result(error_code::SURROGATE, pos);
69	0	}
70	0	pos += 2;
71	0	} else {
72	0	pos++;
73	0	}
74	0	}
75	0	return result(error_code::SUCCESS, pos);
76	0	} Unexecuted instantiation: simdutf::result simdutf::scalar::utf16::validate_with_errors<(simdutf::endianness)0>(char16_t const, unsigned long) Unexecuted instantiation: simdutf::result simdutf::scalar::utf16::validate_with_errors<(simdutf::endianness)1>(char16_t const, unsigned long)
77
78		template <endianness big_endian>
79	0	simdutf_constexpr23 size_t count_code_points(const char16_t *p, size_t len) {
80		// We are not BOM aware.
81	0	size_t counter{0};
82	0	for (size_t i = 0; i < len; i++) {
83	0	char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]);
84	0	counter += ((word & 0xFC00) != 0xDC00);
85	0	}
86	0	return counter;
87	0	} Unexecuted instantiation: unsigned long simdutf::scalar::utf16::count_code_points<(simdutf::endianness)0>(char16_t const, unsigned long) Unexecuted instantiation: unsigned long simdutf::scalar::utf16::count_code_points<(simdutf::endianness)1>(char16_t const, unsigned long)
88
89		template <endianness big_endian>
90		simdutf_constexpr23 size_t utf8_length_from_utf16(const char16_t *p,
91	0	size_t len) {
92		// We are not BOM aware.
93	0	size_t counter{0};
94	0	for (size_t i = 0; i < len; i++) {
95	0	char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]);
96	0	counter++; // ASCII
97	0	counter += static_cast<size_t>(
98	0	word >
99	0	0x7F); // non-ASCII is at least 2 bytes, surrogates are 2*2 == 4 bytes
100	0	counter += static_cast<size_t>((word > 0x7FF && word <= 0xD7FF) \|\|
101	0	(word >= 0xE000)); // three-byte
102	0	}
103	0	return counter;
104	0	} Unexecuted instantiation: unsigned long simdutf::scalar::utf16::utf8_length_from_utf16<(simdutf::endianness)0>(char16_t const, unsigned long) Unexecuted instantiation: unsigned long simdutf::scalar::utf16::utf8_length_from_utf16<(simdutf::endianness)1>(char16_t const, unsigned long)
105
106		template <endianness big_endian>
107		simdutf_constexpr23 size_t utf32_length_from_utf16(const char16_t *p,
108	0	size_t len) {
109		// We are not BOM aware.
110	0	size_t counter{0};
111	0	for (size_t i = 0; i < len; i++) {
112	0	char16_t word = scalar::utf16::swap_if_needed<big_endian>(p[i]);
113	0	counter += ((word & 0xFC00) != 0xDC00);
114	0	}
115	0	return counter;
116	0	} Unexecuted instantiation: unsigned long simdutf::scalar::utf16::utf32_length_from_utf16<(simdutf::endianness)0>(char16_t const, unsigned long) Unexecuted instantiation: unsigned long simdutf::scalar::utf16::utf32_length_from_utf16<(simdutf::endianness)1>(char16_t const, unsigned long)
117
118		simdutf_really_inline simdutf_constexpr23 void
119	0	change_endianness_utf16(const char16_t input, size_t size, char16_t output) {
120	0	for (size_t i = 0; i < size; i++) {
121	0	*output++ = char16_t(input[i] >> 8 \| input[i] << 8);
122	0	}
123	0	}
124
125		template <endianness big_endian>
126		simdutf_warn_unused simdutf_constexpr23 size_t
127	0	trim_partial_utf16(const char16_t *input, size_t length) {
128	0	if (length == 0) {
129	0	return 0;
130	0	}
131	0	uint16_t last_word = uint16_t(input[length - 1]);
132	0	last_word = scalar::utf16::swap_if_needed<big_endian>(last_word);
133	0	length -= ((last_word & 0xFC00) == 0xD800);
134	0	return length;
135	0	} Unexecuted instantiation: unsigned long simdutf::scalar::utf16::trim_partial_utf16<(simdutf::endianness)1>(char16_t const, unsigned long) Unexecuted instantiation: unsigned long simdutf::scalar::utf16::trim_partial_utf16<(simdutf::endianness)0>(char16_t const, unsigned long)
136
137	0	template <endianness big_endian> constexpr bool is_high_surrogate(char16_t c) {
138	0	c = scalar::utf16::swap_if_needed<big_endian>(c);
139	0	return (0xd800 <= c && c <= 0xdbff);
140	0	} Unexecuted instantiation: bool simdutf::scalar::utf16::is_high_surrogate<(simdutf::endianness)0>(char16_t) Unexecuted instantiation: bool simdutf::scalar::utf16::is_high_surrogate<(simdutf::endianness)1>(char16_t)
141
142	0	template <endianness big_endian> constexpr bool is_low_surrogate(char16_t c) {
143	0	c = scalar::utf16::swap_if_needed<big_endian>(c);
144	0	return (0xdc00 <= c && c <= 0xdfff);
145	0	} Unexecuted instantiation: bool simdutf::scalar::utf16::is_low_surrogate<(simdutf::endianness)0>(char16_t) Unexecuted instantiation: bool simdutf::scalar::utf16::is_low_surrogate<(simdutf::endianness)1>(char16_t)
146
147	0	simdutf_unused simdutf_really_inline constexpr bool high_surrogate(char16_t c) {
148	0	return (0xd800 <= c && c <= 0xdbff);
149	0	}
150
151		template <endianness big_endian>
152		simdutf_constexpr23 result
153	0	utf8_length_from_utf16_with_replacement(const char16_t *p, size_t len) {
154	0	bool any_surrogates = false;
155		// We are not BOM aware.
156	0	size_t counter{0};
157	0	for (size_t i = 0; i < len; i++) {
158	0	if (is_high_surrogate<big_endian>(p[i])) {
159	0	any_surrogates = true;
160		// surrogate pair
161	0	if (i + 1 < len && is_low_surrogate<big_endian>(p[i + 1])) {
162	0	counter += 4;
163	0	i++; // skip low surrogate
164	0	} else {
165	0	counter += 3; // unpaired high surrogate replaced by U+FFFD
166	0	}
167	0	continue;
168	0	} else if (is_low_surrogate<big_endian>(p[i])) {
169	0	any_surrogates = true;
170	0	counter += 3; // unpaired low surrogate replaced by U+FFFD
171	0	continue;
172	0	}
173	0	char16_t word = !match_system(big_endian) ? u16_swap_bytes(p[i]) : p[i];
174	0	counter++; // at least 1 byte
175	0	counter +=
176	0	static_cast<size_t>(word > 0x7F); // non-ASCII is at least 2 bytes
177	0	counter += static_cast<size_t>(word > 0x7FF); // three-byte
178	0	}
179	0	return {any_surrogates ? error_code::SURROGATE : error_code::SUCCESS,
180	0	counter};
181	0	} Unexecuted instantiation: simdutf::result simdutf::scalar::utf16::utf8_length_from_utf16_with_replacement<(simdutf::endianness)0>(char16_t const, unsigned long) Unexecuted instantiation: simdutf::result simdutf::scalar::utf16::utf8_length_from_utf16_with_replacement<(simdutf::endianness)1>(char16_t const, unsigned long)
182
183		// variable templates are a C++14 extension
184	0	template <endianness big_endian> constexpr char16_t replacement() {
185	0	return !match_system(big_endian) ? scalar::u16_swap_bytes(0xfffd) : 0xfffd;
186	0	} Unexecuted instantiation: char16_t simdutf::scalar::utf16::replacement<(simdutf::endianness)0>() Unexecuted instantiation: char16_t simdutf::scalar::utf16::replacement<(simdutf::endianness)1>()
187
188		template <endianness big_endian>
189		simdutf_constexpr23 void to_well_formed_utf16(const char16_t *input, size_t len,
190	0	char16_t *output) {
191	0	const char16_t replacement = utf16::replacement<big_endian>();
192	0	bool high_surrogate_prev = false, high_surrogate, low_surrogate;
193	0	size_t i = 0;
194	0	for (; i < len; i++) {
195	0	char16_t c = input[i];
196	0	high_surrogate = is_high_surrogate<big_endian>(c);
197	0	low_surrogate = is_low_surrogate<big_endian>(c);
198	0	if (high_surrogate_prev && !low_surrogate) {
199	0	output[i - 1] = replacement;
200	0	}
201
202	0	if (!high_surrogate_prev && low_surrogate) {
203	0	output[i] = replacement;
204	0	} else {
205	0	output[i] = input[i];
206	0	}
207	0	high_surrogate_prev = high_surrogate;
208	0	}
209
210		/* string may not end with high surrogate */
211	0	if (high_surrogate_prev) {
212	0	output[i - 1] = replacement;
213	0	}
214	0	} Unexecuted instantiation: void simdutf::scalar::utf16::to_well_formed_utf16<(simdutf::endianness)0>(char16_t const, unsigned long, char16_t) Unexecuted instantiation: void simdutf::scalar::utf16::to_well_formed_utf16<(simdutf::endianness)1>(char16_t const, unsigned long, char16_t)
215
216		} // namespace utf16
217		} // namespace scalar
218		} // namespace simdutf
219
220		#endif