/src/serenity/AK/Utf8View.h
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2019-2020, Sergey Bugaev <bugaevc@serenityos.org> |
3 | | * Copyright (c) 2021, Max Wipfli <mail@maxwipfli.ch> |
4 | | * |
5 | | * SPDX-License-Identifier: BSD-2-Clause |
6 | | */ |
7 | | |
8 | | #pragma once |
9 | | |
10 | | #include <AK/Format.h> |
11 | | #include <AK/Function.h> |
12 | | #include <AK/StringView.h> |
13 | | #include <AK/Types.h> |
14 | | |
15 | | #ifndef KERNEL |
16 | | # include <AK/ByteString.h> |
17 | | #endif |
18 | | |
19 | | namespace AK { |
20 | | |
21 | | class Utf8View; |
22 | | |
23 | | class Utf8CodePointIterator { |
24 | | friend class Utf8View; |
25 | | |
26 | | public: |
27 | 0 | Utf8CodePointIterator() = default; |
28 | | ~Utf8CodePointIterator() = default; |
29 | | |
30 | 100M | bool operator==(Utf8CodePointIterator const&) const = default; |
31 | 99.8M | bool operator!=(Utf8CodePointIterator const&) const = default; |
32 | | Utf8CodePointIterator& operator++(); |
33 | | u32 operator*() const; |
34 | | // NOTE: This returns {} if the peek is at or past EOF. |
35 | | Optional<u32> peek(size_t offset = 0) const; |
36 | | |
37 | | ssize_t operator-(Utf8CodePointIterator const& other) const |
38 | 6.29k | { |
39 | 6.29k | return m_ptr - other.m_ptr; |
40 | 6.29k | } |
41 | | |
42 | 0 | u8 const* ptr() const { return m_ptr; } |
43 | | |
44 | | // Note : These methods return the information about the underlying UTF-8 bytes. |
45 | | // If the UTF-8 string encoding is not valid at the iterator's position, then the underlying bytes might be different from the |
46 | | // decoded character's re-encoded bytes (which will be an `0xFFFD REPLACEMENT CHARACTER` with an UTF-8 length of three bytes). |
47 | | // If your code relies on the decoded character being equivalent to the re-encoded character, use the `UTF8View::validate()` |
48 | | // method on the view prior to using its iterator. |
49 | | size_t underlying_code_point_length_in_bytes() const; |
50 | | ReadonlyBytes underlying_code_point_bytes() const; |
51 | 508M | bool done() const { return m_length == 0; } |
52 | | |
53 | | private: |
54 | | Utf8CodePointIterator(u8 const* ptr, size_t length) |
55 | 70.7M | : m_ptr(ptr) |
56 | 70.7M | , m_length(length) |
57 | 70.7M | { |
58 | 70.7M | } |
59 | | |
60 | | u8 const* m_ptr { nullptr }; |
61 | | size_t m_length { 0 }; |
62 | | }; |
63 | | |
64 | | class Utf8View { |
65 | | public: |
66 | | using Iterator = Utf8CodePointIterator; |
67 | | |
68 | 0 | Utf8View() = default; |
69 | | |
70 | | explicit constexpr Utf8View(StringView string) |
71 | 86.3M | : m_string(string) |
72 | 86.3M | { |
73 | 86.3M | } |
74 | | |
75 | | #ifndef KERNEL |
76 | | explicit Utf8View(ByteString& string) |
77 | 18.2M | : m_string(string.view()) |
78 | 18.2M | { |
79 | 18.2M | } |
80 | | |
81 | | explicit Utf8View(ByteString&&) = delete; |
82 | | #endif |
83 | | |
84 | | enum class AllowSurrogates { |
85 | | Yes, |
86 | | No, |
87 | | }; |
88 | | |
89 | | ~Utf8View() = default; |
90 | | |
91 | 3.83k | StringView as_string() const { return m_string; } |
92 | | |
93 | 45.1M | Utf8CodePointIterator begin() const { return { begin_ptr(), m_string.length() }; } |
94 | 25.6M | Utf8CodePointIterator end() const { return { end_ptr(), 0 }; } |
95 | | Utf8CodePointIterator iterator_at_byte_offset(size_t) const; |
96 | | |
97 | | Utf8CodePointIterator iterator_at_byte_offset_without_validation(size_t) const; |
98 | | |
99 | 1.35k | unsigned char const* bytes() const { return begin_ptr(); } |
100 | 638k | size_t byte_length() const { return m_string.length(); } |
101 | | size_t byte_offset_of(Utf8CodePointIterator const&) const; |
102 | | size_t byte_offset_of(size_t code_point_offset) const; |
103 | | |
104 | 4.15k | Utf8View substring_view(size_t byte_offset, size_t byte_length) const { return Utf8View { m_string.substring_view(byte_offset, byte_length) }; } |
105 | 3.99k | Utf8View substring_view(size_t byte_offset) const { return substring_view(byte_offset, byte_length() - byte_offset); } |
106 | | Utf8View unicode_substring_view(size_t code_point_offset, size_t code_point_length) const; |
107 | 0 | Utf8View unicode_substring_view(size_t code_point_offset) const { return unicode_substring_view(code_point_offset, length() - code_point_offset); } |
108 | | |
109 | 874k | bool is_empty() const { return m_string.is_empty(); } |
110 | 0 | bool is_null() const { return m_string.is_null(); } |
111 | | bool starts_with(Utf8View const&) const; |
112 | | bool contains(u32) const; |
113 | | |
114 | | Utf8View trim(Utf8View const& characters, TrimMode mode = TrimMode::Both) const; |
115 | | |
116 | | size_t iterator_offset(Utf8CodePointIterator const& it) const |
117 | 6.41k | { |
118 | 6.41k | return byte_offset_of(it); |
119 | 6.41k | } |
120 | | |
121 | | size_t length() const |
122 | 0 | { |
123 | 0 | if (!m_have_length) { |
124 | 0 | m_length = calculate_length(); |
125 | 0 | m_have_length = true; |
126 | 0 | } |
127 | 0 | return m_length; |
128 | 0 | } |
129 | | |
130 | | constexpr bool validate(AllowSurrogates surrogates = AllowSurrogates::Yes) const |
131 | 51.2M | { |
132 | 51.2M | size_t valid_bytes = 0; |
133 | 51.2M | return validate(valid_bytes, surrogates); |
134 | 51.2M | } |
135 | | |
136 | | constexpr bool validate(size_t& valid_bytes, AllowSurrogates surrogates = AllowSurrogates::Yes) const |
137 | 51.2M | { |
138 | 51.2M | valid_bytes = 0; |
139 | | |
140 | 2.75G | for (auto it = m_string.begin(); it != m_string.end(); ++it) { |
141 | 2.70G | auto [byte_length, code_point, is_valid] = decode_leading_byte(static_cast<u8>(*it)); |
142 | 2.70G | if (!is_valid) |
143 | 170k | return false; |
144 | | |
145 | 2.72G | for (size_t i = 1; i < byte_length; ++i) { |
146 | 17.8M | if (++it == m_string.end()) |
147 | 11.3k | return false; |
148 | | |
149 | 17.8M | auto [code_point_bits, is_valid] = decode_continuation_byte(static_cast<u8>(*it)); |
150 | 17.8M | if (!is_valid) |
151 | 73.9k | return false; |
152 | | |
153 | 17.7M | code_point <<= 6; |
154 | 17.7M | code_point |= code_point_bits; |
155 | 17.7M | } |
156 | | |
157 | 2.70G | if (!is_valid_code_point(code_point, byte_length, surrogates)) |
158 | 2.55k | return false; |
159 | | |
160 | 2.70G | valid_bytes += byte_length; |
161 | 2.70G | } |
162 | | |
163 | 50.9M | return true; |
164 | 51.2M | } |
165 | | |
166 | | template<typename Callback> |
167 | | auto for_each_split_view(Function<bool(u32)> splitter, SplitBehavior split_behavior, Callback callback) const |
168 | | { |
169 | | bool keep_empty = has_flag(split_behavior, SplitBehavior::KeepEmpty); |
170 | | bool keep_trailing_separator = has_flag(split_behavior, SplitBehavior::KeepTrailingSeparator); |
171 | | |
172 | | auto start_offset = 0u; |
173 | | auto offset = 0u; |
174 | | |
175 | | auto run_callback = [&]() { |
176 | | auto length = offset - start_offset; |
177 | | |
178 | | if (length == 0 && !keep_empty) |
179 | | return; |
180 | | |
181 | | auto substring = unicode_substring_view(start_offset, length); |
182 | | |
183 | | // Reject splitter-only entries if we're not keeping empty results |
184 | | if (keep_trailing_separator && !keep_empty && length == 1 && splitter(*substring.begin())) |
185 | | return; |
186 | | |
187 | | callback(substring); |
188 | | }; |
189 | | |
190 | | auto iterator = begin(); |
191 | | while (iterator != end()) { |
192 | | if (splitter(*iterator)) { |
193 | | if (keep_trailing_separator) |
194 | | ++offset; |
195 | | |
196 | | run_callback(); |
197 | | |
198 | | if (!keep_trailing_separator) |
199 | | ++offset; |
200 | | |
201 | | start_offset = offset; |
202 | | ++iterator; |
203 | | continue; |
204 | | } |
205 | | |
206 | | ++offset; |
207 | | ++iterator; |
208 | | } |
209 | | run_callback(); |
210 | | } |
211 | | |
212 | | private: |
213 | | friend class Utf8CodePointIterator; |
214 | | |
215 | 72.6M | u8 const* begin_ptr() const { return reinterpret_cast<u8 const*>(m_string.characters_without_null_termination()); } |
216 | 26.2M | u8 const* end_ptr() const { return begin_ptr() + m_string.length(); } |
217 | | size_t calculate_length() const; |
218 | | |
219 | | struct Utf8EncodedByteData { |
220 | | size_t byte_length { 0 }; |
221 | | u8 encoding_bits { 0 }; |
222 | | u8 encoding_mask { 0 }; |
223 | | u32 first_code_point { 0 }; |
224 | | u32 last_code_point { 0 }; |
225 | | }; |
226 | | |
227 | | static constexpr Array<Utf8EncodedByteData, 4> utf8_encoded_byte_data { { |
228 | | { 1, 0b0000'0000, 0b1000'0000, 0x0000, 0x007F }, |
229 | | { 2, 0b1100'0000, 0b1110'0000, 0x0080, 0x07FF }, |
230 | | { 3, 0b1110'0000, 0b1111'0000, 0x0800, 0xFFFF }, |
231 | | { 4, 0b1111'0000, 0b1111'1000, 0x10000, 0x10FFFF }, |
232 | | } }; |
233 | | |
234 | | struct LeadingByte { |
235 | | size_t byte_length { 0 }; |
236 | | u32 code_point_bits { 0 }; |
237 | | bool is_valid { false }; |
238 | | }; |
239 | | |
240 | | static constexpr LeadingByte decode_leading_byte(u8 byte) |
241 | 3.09G | { |
242 | 3.85G | for (auto const& data : utf8_encoded_byte_data) { |
243 | 3.85G | if ((byte & data.encoding_mask) != data.encoding_bits) |
244 | 938M | continue; |
245 | | |
246 | 2.91G | byte &= ~data.encoding_mask; |
247 | 2.91G | return { data.byte_length, byte, true }; |
248 | 3.85G | } |
249 | | |
250 | 179M | return { .is_valid = false }; |
251 | 3.09G | } |
252 | | |
253 | | struct ContinuationByte { |
254 | | u32 code_point_bits { 0 }; |
255 | | bool is_valid { false }; |
256 | | }; |
257 | | |
258 | | static constexpr ContinuationByte decode_continuation_byte(u8 byte) |
259 | 17.8M | { |
260 | 17.8M | constexpr u8 continuation_byte_encoding_bits = 0b1000'0000; |
261 | 17.8M | constexpr u8 continuation_byte_encoding_mask = 0b1100'0000; |
262 | | |
263 | 17.8M | if ((byte & continuation_byte_encoding_mask) == continuation_byte_encoding_bits) { |
264 | 17.7M | byte &= ~continuation_byte_encoding_mask; |
265 | 17.7M | return { byte, true }; |
266 | 17.7M | } |
267 | | |
268 | 73.9k | return { .is_valid = false }; |
269 | 17.8M | } |
270 | | |
271 | | static constexpr bool is_valid_code_point(u32 code_point, size_t byte_length, AllowSurrogates surrogates = AllowSurrogates::Yes) |
272 | 2.70G | { |
273 | 2.70G | if (surrogates == AllowSurrogates::No && byte_length == 3 && code_point >= 0xD800 && code_point <= 0xDFFF) |
274 | 12 | return false; |
275 | 2.72G | for (auto const& data : utf8_encoded_byte_data) { |
276 | 2.72G | if (code_point >= data.first_code_point && code_point <= data.last_code_point) |
277 | 2.70G | return byte_length == data.byte_length; |
278 | 2.72G | } |
279 | | |
280 | 1.02k | return false; |
281 | 2.70G | } |
282 | | |
283 | | StringView m_string; |
284 | | mutable size_t m_length { 0 }; |
285 | | mutable bool m_have_length { false }; |
286 | | }; |
287 | | |
288 | | #ifndef KERNEL |
289 | | class DeprecatedStringCodePointIterator { |
290 | | public: |
291 | | Optional<u32> next() |
292 | 0 | { |
293 | 0 | if (m_it.done()) |
294 | 0 | return {}; |
295 | 0 | auto value = *m_it; |
296 | 0 | ++m_it; |
297 | 0 | return value; |
298 | 0 | } |
299 | | |
300 | | [[nodiscard]] Optional<u32> peek() const |
301 | 0 | { |
302 | 0 | if (m_it.done()) |
303 | 0 | return {}; |
304 | 0 | return *m_it; |
305 | 0 | } |
306 | | |
307 | | [[nodiscard]] size_t byte_offset() const |
308 | 0 | { |
309 | 0 | return Utf8View(m_string).byte_offset_of(m_it); |
310 | 0 | } |
311 | | |
312 | | DeprecatedStringCodePointIterator(ByteString string) |
313 | 0 | : m_string(move(string)) |
314 | 0 | , m_it(Utf8View(m_string).begin()) |
315 | 0 | { |
316 | 0 | } |
317 | | |
318 | | private: |
319 | | ByteString m_string; |
320 | | Utf8CodePointIterator m_it; |
321 | | }; |
322 | | #endif |
323 | | |
324 | | template<> |
325 | | struct Formatter<Utf8View> : Formatter<StringView> { |
326 | | ErrorOr<void> format(FormatBuilder&, Utf8View const&); |
327 | | }; |
328 | | |
329 | | } |
330 | | |
331 | | #if USING_AK_GLOBALLY |
332 | | # ifndef KERNEL |
333 | | using AK::DeprecatedStringCodePointIterator; |
334 | | # endif |
335 | | using AK::Utf8CodePointIterator; |
336 | | using AK::Utf8View; |
337 | | #endif |