/src/serenity/AK/Utf16View.cpp
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2021-2023, Tim Flynn <trflynn89@serenityos.org> |
3 | | * |
4 | | * SPDX-License-Identifier: BSD-2-Clause |
5 | | */ |
6 | | |
7 | | #include <AK/CharacterTypes.h> |
8 | | #include <AK/Concepts.h> |
9 | | #include <AK/StringBuilder.h> |
10 | | #include <AK/StringView.h> |
11 | | #include <AK/Utf16View.h> |
12 | | #include <AK/Utf32View.h> |
13 | | #include <AK/Utf8View.h> |
14 | | |
15 | | namespace AK { |
16 | | |
17 | | static constexpr u16 high_surrogate_min = 0xd800; |
18 | | static constexpr u16 high_surrogate_max = 0xdbff; |
19 | | static constexpr u16 low_surrogate_min = 0xdc00; |
20 | | static constexpr u16 low_surrogate_max = 0xdfff; |
21 | | static constexpr u32 replacement_code_point = 0xfffd; |
22 | | static constexpr u32 first_supplementary_plane_code_point = 0x10000; |
23 | | |
24 | | template<OneOf<Utf8View, Utf32View> UtfViewType> |
25 | | static ErrorOr<Utf16Data> to_utf16_impl(UtfViewType const& view) |
26 | 0 | { |
27 | 0 | Utf16Data utf16_data; |
28 | 0 | TRY(utf16_data.try_ensure_capacity(view.length())); |
29 | |
|
30 | 0 | for (auto code_point : view) |
31 | 0 | TRY(code_point_to_utf16(utf16_data, code_point)); |
32 | |
|
33 | 0 | return utf16_data; |
34 | 0 | } Unexecuted instantiation: Utf16View.cpp:_ZN2AKL13to_utf16_implITkNS_8Concepts5OneOfINS_8Utf8ViewENS_9Utf32ViewEEES3_EENS_7ErrorOrINS_6VectorItLm1EEENS_5ErrorEEERKT_ Unexecuted instantiation: Utf16View.cpp:_ZN2AKL13to_utf16_implITkNS_8Concepts5OneOfINS_8Utf8ViewENS_9Utf32ViewEEES4_EENS_7ErrorOrINS_6VectorItLm1EEENS_5ErrorEEERKT_ |
35 | | |
36 | | ErrorOr<Utf16Data> utf8_to_utf16(StringView utf8_view) |
37 | 0 | { |
38 | 0 | return to_utf16_impl(Utf8View { utf8_view }); |
39 | 0 | } |
40 | | |
41 | | ErrorOr<Utf16Data> utf8_to_utf16(Utf8View const& utf8_view) |
42 | 0 | { |
43 | 0 | return to_utf16_impl(utf8_view); |
44 | 0 | } |
45 | | |
46 | | ErrorOr<Utf16Data> utf32_to_utf16(Utf32View const& utf32_view) |
47 | 0 | { |
48 | 0 | return to_utf16_impl(utf32_view); |
49 | 0 | } |
50 | | |
51 | | ErrorOr<void> code_point_to_utf16(Utf16Data& string, u32 code_point) |
52 | 0 | { |
53 | 0 | VERIFY(is_unicode(code_point)); |
54 | | |
55 | 0 | if (code_point < first_supplementary_plane_code_point) { |
56 | 0 | TRY(string.try_append(static_cast<u16>(code_point))); |
57 | 0 | } else { |
58 | 0 | code_point -= first_supplementary_plane_code_point; |
59 | 0 | TRY(string.try_append(static_cast<u16>(high_surrogate_min | (code_point >> 10)))); |
60 | 0 | TRY(string.try_append(static_cast<u16>(low_surrogate_min | (code_point & 0x3ff)))); |
61 | 0 | } |
62 | |
|
63 | 0 | return {}; |
64 | 0 | } |
65 | | |
66 | | size_t utf16_code_unit_length_from_utf8(StringView string) |
67 | 0 | { |
68 | | // FIXME: This is inefficient! |
69 | 0 | auto utf16_data = MUST(AK::utf8_to_utf16(string)); |
70 | 0 | return Utf16View { utf16_data }.length_in_code_units(); |
71 | 0 | } |
72 | | |
73 | | bool Utf16View::is_high_surrogate(u16 code_unit) |
74 | 3.76M | { |
75 | 3.76M | return (code_unit >= high_surrogate_min) && (code_unit <= high_surrogate_max); |
76 | 3.76M | } |
77 | | |
78 | | bool Utf16View::is_low_surrogate(u16 code_unit) |
79 | 158k | { |
80 | 158k | return (code_unit >= low_surrogate_min) && (code_unit <= low_surrogate_max); |
81 | 158k | } |
82 | | |
83 | | u32 Utf16View::decode_surrogate_pair(u16 high_surrogate, u16 low_surrogate) |
84 | 17.1k | { |
85 | 17.1k | VERIFY(is_high_surrogate(high_surrogate)); |
86 | 17.1k | VERIFY(is_low_surrogate(low_surrogate)); |
87 | | |
88 | 17.1k | return ((high_surrogate - high_surrogate_min) << 10) + (low_surrogate - low_surrogate_min) + first_supplementary_plane_code_point; |
89 | 17.1k | } |
90 | | |
91 | | ErrorOr<ByteString> Utf16View::to_byte_string(AllowInvalidCodeUnits allow_invalid_code_units) const |
92 | 0 | { |
93 | 0 | return TRY(to_utf8(allow_invalid_code_units)).to_byte_string(); |
94 | 0 | } |
95 | | |
96 | | ErrorOr<String> Utf16View::to_utf8(AllowInvalidCodeUnits allow_invalid_code_units) const |
97 | 0 | { |
98 | 0 | StringBuilder builder; |
99 | |
|
100 | 0 | if (allow_invalid_code_units == AllowInvalidCodeUnits::Yes) { |
101 | 0 | for (auto const* ptr = begin_ptr(); ptr < end_ptr(); ++ptr) { |
102 | 0 | if (is_high_surrogate(*ptr)) { |
103 | 0 | auto const* next = ptr + 1; |
104 | |
|
105 | 0 | if ((next < end_ptr()) && is_low_surrogate(*next)) { |
106 | 0 | auto code_point = decode_surrogate_pair(*ptr, *next); |
107 | 0 | TRY(builder.try_append_code_point(code_point)); |
108 | 0 | ++ptr; |
109 | 0 | continue; |
110 | 0 | } |
111 | 0 | } |
112 | | |
113 | 0 | TRY(builder.try_append_code_point(static_cast<u32>(*ptr))); |
114 | 0 | } |
115 | 0 | return builder.to_string_without_validation(); |
116 | 0 | } |
117 | | |
118 | 0 | for (auto code_point : *this) |
119 | 0 | TRY(builder.try_append_code_point(code_point)); |
120 | |
|
121 | 0 | return builder.to_string(); |
122 | 0 | } |
123 | | |
124 | | size_t Utf16View::length_in_code_points() const |
125 | 0 | { |
126 | 0 | if (!m_length_in_code_points.has_value()) |
127 | 0 | m_length_in_code_points = calculate_length_in_code_points(); |
128 | 0 | return *m_length_in_code_points; |
129 | 0 | } |
130 | | |
131 | | u16 Utf16View::code_unit_at(size_t index) const |
132 | 0 | { |
133 | 0 | VERIFY(index < length_in_code_units()); |
134 | 0 | return m_code_units[index]; |
135 | 0 | } |
136 | | |
137 | | u32 Utf16View::code_point_at(size_t index) const |
138 | 0 | { |
139 | 0 | VERIFY(index < length_in_code_units()); |
140 | | |
141 | 0 | u32 code_point = code_unit_at(index); |
142 | 0 | if (!is_high_surrogate(code_point) && !is_low_surrogate(code_point)) |
143 | 0 | return code_point; |
144 | 0 | if (is_low_surrogate(code_point) || (index + 1 == length_in_code_units())) |
145 | 0 | return code_point; |
146 | | |
147 | 0 | auto second = code_unit_at(index + 1); |
148 | 0 | if (!is_low_surrogate(second)) |
149 | 0 | return code_point; |
150 | | |
151 | 0 | return decode_surrogate_pair(code_point, second); |
152 | 0 | } |
153 | | |
154 | | size_t Utf16View::code_point_offset_of(size_t code_unit_offset) const |
155 | 0 | { |
156 | 0 | size_t code_point_offset = 0; |
157 | |
|
158 | 0 | for (auto it = begin(); it != end(); ++it) { |
159 | 0 | if (code_unit_offset == 0) |
160 | 0 | return code_point_offset; |
161 | | |
162 | 0 | code_unit_offset -= it.length_in_code_units(); |
163 | 0 | ++code_point_offset; |
164 | 0 | } |
165 | | |
166 | 0 | return code_point_offset; |
167 | 0 | } |
168 | | |
169 | | size_t Utf16View::code_unit_offset_of(size_t code_point_offset) const |
170 | 0 | { |
171 | 0 | size_t code_unit_offset = 0; |
172 | |
|
173 | 0 | for (auto it = begin(); it != end(); ++it) { |
174 | 0 | if (code_point_offset == 0) |
175 | 0 | return code_unit_offset; |
176 | | |
177 | 0 | code_unit_offset += it.length_in_code_units(); |
178 | 0 | --code_point_offset; |
179 | 0 | } |
180 | | |
181 | 0 | return code_unit_offset; |
182 | 0 | } |
183 | | |
184 | | size_t Utf16View::code_unit_offset_of(Utf16CodePointIterator const& it) const |
185 | 0 | { |
186 | 0 | VERIFY(it.m_ptr >= begin_ptr()); |
187 | 0 | VERIFY(it.m_ptr <= end_ptr()); |
188 | | |
189 | 0 | return it.m_ptr - begin_ptr(); |
190 | 0 | } |
191 | | |
192 | | Utf16View Utf16View::substring_view(size_t code_unit_offset, size_t code_unit_length) const |
193 | 0 | { |
194 | 0 | VERIFY(!Checked<size_t>::addition_would_overflow(code_unit_offset, code_unit_length)); |
195 | 0 | VERIFY(code_unit_offset + code_unit_length <= length_in_code_units()); |
196 | | |
197 | 0 | return Utf16View { m_code_units.slice(code_unit_offset, code_unit_length) }; |
198 | 0 | } |
199 | | |
200 | | Utf16View Utf16View::unicode_substring_view(size_t code_point_offset, size_t code_point_length) const |
201 | 0 | { |
202 | 0 | if (code_point_length == 0) |
203 | 0 | return {}; |
204 | | |
205 | 0 | auto code_unit_offset_of = [&](Utf16CodePointIterator const& it) { return it.m_ptr - begin_ptr(); }; |
206 | 0 | size_t code_point_index = 0; |
207 | 0 | size_t code_unit_offset = 0; |
208 | |
|
209 | 0 | for (auto it = begin(); it != end(); ++it) { |
210 | 0 | if (code_point_index == code_point_offset) |
211 | 0 | code_unit_offset = code_unit_offset_of(it); |
212 | |
|
213 | 0 | if (code_point_index == (code_point_offset + code_point_length - 1)) { |
214 | 0 | size_t code_unit_length = code_unit_offset_of(++it) - code_unit_offset; |
215 | 0 | return substring_view(code_unit_offset, code_unit_length); |
216 | 0 | } |
217 | | |
218 | 0 | ++code_point_index; |
219 | 0 | } |
220 | | |
221 | 0 | VERIFY_NOT_REACHED(); |
222 | 0 | } |
223 | | |
224 | | bool Utf16View::starts_with(Utf16View const& needle) const |
225 | 0 | { |
226 | 0 | if (needle.is_empty()) |
227 | 0 | return true; |
228 | 0 | if (is_empty()) |
229 | 0 | return false; |
230 | 0 | if (needle.length_in_code_units() > length_in_code_units()) |
231 | 0 | return false; |
232 | 0 | if (begin_ptr() == needle.begin_ptr()) |
233 | 0 | return true; |
234 | | |
235 | 0 | for (auto this_it = begin(), needle_it = needle.begin(); needle_it != needle.end(); ++needle_it, ++this_it) { |
236 | 0 | if (*this_it != *needle_it) |
237 | 0 | return false; |
238 | 0 | } |
239 | | |
240 | 0 | return true; |
241 | 0 | } |
242 | | |
243 | | bool Utf16View::validate(size_t& valid_code_units) const |
244 | 0 | { |
245 | 0 | valid_code_units = 0; |
246 | |
|
247 | 0 | for (auto const* ptr = begin_ptr(); ptr < end_ptr(); ++ptr) { |
248 | 0 | if (is_high_surrogate(*ptr)) { |
249 | 0 | if ((++ptr >= end_ptr()) || !is_low_surrogate(*ptr)) |
250 | 0 | return false; |
251 | 0 | ++valid_code_units; |
252 | 0 | } else if (is_low_surrogate(*ptr)) { |
253 | 0 | return false; |
254 | 0 | } |
255 | | |
256 | 0 | ++valid_code_units; |
257 | 0 | } |
258 | | |
259 | 0 | return true; |
260 | 0 | } |
261 | | |
262 | | size_t Utf16View::calculate_length_in_code_points() const |
263 | 0 | { |
264 | 0 | size_t code_points = 0; |
265 | 0 | for ([[maybe_unused]] auto code_point : *this) |
266 | 0 | ++code_points; |
267 | 0 | return code_points; |
268 | 0 | } |
269 | | |
270 | | bool Utf16View::equals_ignoring_case(Utf16View const& other) const |
271 | 0 | { |
272 | 0 | if (length_in_code_units() == 0) |
273 | 0 | return other.length_in_code_units() == 0; |
274 | 0 | if (length_in_code_units() != other.length_in_code_units()) |
275 | 0 | return false; |
276 | | |
277 | 0 | for (size_t i = 0; i < length_in_code_units(); ++i) { |
278 | | // FIXME: Handle non-ASCII case insensitive comparisons. |
279 | 0 | if (to_ascii_lowercase(m_code_units[i]) != to_ascii_lowercase(other.m_code_units[i])) |
280 | 0 | return false; |
281 | 0 | } |
282 | | |
283 | 0 | return true; |
284 | 0 | } |
285 | | |
286 | | Utf16CodePointIterator& Utf16CodePointIterator::operator++() |
287 | 0 | { |
288 | 0 | size_t code_units = length_in_code_units(); |
289 | |
|
290 | 0 | if (code_units > m_remaining_code_units) { |
291 | | // If there aren't enough code units remaining, skip to the end. |
292 | 0 | m_ptr += m_remaining_code_units; |
293 | 0 | m_remaining_code_units = 0; |
294 | 0 | } else { |
295 | 0 | m_ptr += code_units; |
296 | 0 | m_remaining_code_units -= code_units; |
297 | 0 | } |
298 | |
|
299 | 0 | return *this; |
300 | 0 | } |
301 | | |
302 | | u32 Utf16CodePointIterator::operator*() const |
303 | 0 | { |
304 | 0 | VERIFY(m_remaining_code_units > 0); |
305 | | |
306 | | // rfc2781, 2.2 Decoding UTF-16 |
307 | | // 1) If W1 < 0xD800 or W1 > 0xDFFF, the character value U is the value |
308 | | // of W1. Terminate. |
309 | | // 2) Determine if W1 is between 0xD800 and 0xDBFF. If not, the sequence |
310 | | // is in error and no valid character can be obtained using W1. |
311 | | // Terminate. |
312 | | // 3) If there is no W2 (that is, the sequence ends with W1), or if W2 |
313 | | // is not between 0xDC00 and 0xDFFF, the sequence is in error. |
314 | | // Terminate. |
315 | | // 4) Construct a 20-bit unsigned integer U', taking the 10 low-order |
316 | | // bits of W1 as its 10 high-order bits and the 10 low-order bits of |
317 | | // W2 as its 10 low-order bits. |
318 | | // 5) Add 0x10000 to U' to obtain the character value U. Terminate. |
319 | | |
320 | 0 | if (Utf16View::is_high_surrogate(*m_ptr)) { |
321 | 0 | if ((m_remaining_code_units > 1) && Utf16View::is_low_surrogate(*(m_ptr + 1))) |
322 | 0 | return Utf16View::decode_surrogate_pair(*m_ptr, *(m_ptr + 1)); |
323 | 0 | return replacement_code_point; |
324 | 0 | } else if (Utf16View::is_low_surrogate(*m_ptr)) { |
325 | 0 | return replacement_code_point; |
326 | 0 | } |
327 | | |
328 | 0 | return static_cast<u32>(*m_ptr); |
329 | 0 | } |
330 | | |
331 | | size_t Utf16CodePointIterator::length_in_code_units() const |
332 | 0 | { |
333 | 0 | VERIFY(m_remaining_code_units > 0); |
334 | | |
335 | 0 | if (Utf16View::is_high_surrogate(*m_ptr)) { |
336 | 0 | if ((m_remaining_code_units > 1) && Utf16View::is_low_surrogate(*(m_ptr + 1))) |
337 | 0 | return 2; |
338 | 0 | } |
339 | | |
340 | | // If this return is reached, either the encoded code point is a valid single code unit, or that |
341 | | // code point is invalid (e.g. began with a low surrogate, or a low surrogate did not follow a |
342 | | // high surrogate). In the latter case, a single replacement code unit will be used. |
343 | 0 | return 1; |
344 | 0 | } |
345 | | |
346 | | } |