Coverage Report

Created: 2025-12-18 07:52

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/serenity/AK/Utf8View.h
Line
Count
Source
1
/*
2
 * Copyright (c) 2019-2020, Sergey Bugaev <bugaevc@serenityos.org>
3
 * Copyright (c) 2021, Max Wipfli <mail@maxwipfli.ch>
4
 *
5
 * SPDX-License-Identifier: BSD-2-Clause
6
 */
7
8
#pragma once
9
10
#include <AK/Format.h>
11
#include <AK/Function.h>
12
#include <AK/StringView.h>
13
#include <AK/Types.h>
14
15
#ifndef KERNEL
16
#    include <AK/ByteString.h>
17
#endif
18
19
namespace AK {
20
21
class Utf8View;
22
23
class Utf8CodePointIterator {
24
    friend class Utf8View;
25
26
public:
27
0
    Utf8CodePointIterator() = default;
28
    ~Utf8CodePointIterator() = default;
29
30
100M
    bool operator==(Utf8CodePointIterator const&) const = default;
31
99.8M
    bool operator!=(Utf8CodePointIterator const&) const = default;
32
    Utf8CodePointIterator& operator++();
33
    u32 operator*() const;
34
    // NOTE: This returns {} if the peek is at or past EOF.
35
    Optional<u32> peek(size_t offset = 0) const;
36
37
    ssize_t operator-(Utf8CodePointIterator const& other) const
38
6.29k
    {
39
6.29k
        return m_ptr - other.m_ptr;
40
6.29k
    }
41
42
0
    u8 const* ptr() const { return m_ptr; }
43
44
    // Note : These methods return the information about the underlying UTF-8 bytes.
45
    // If the UTF-8 string encoding is not valid at the iterator's position, then the underlying bytes might be different from the
46
    // decoded character's re-encoded bytes (which will be an `0xFFFD REPLACEMENT CHARACTER` with an UTF-8 length of three bytes).
47
    // If your code relies on the decoded character being equivalent to the re-encoded character, use the `UTF8View::validate()`
48
    // method on the view prior to using its iterator.
49
    size_t underlying_code_point_length_in_bytes() const;
50
    ReadonlyBytes underlying_code_point_bytes() const;
51
508M
    bool done() const { return m_length == 0; }
52
53
private:
54
    Utf8CodePointIterator(u8 const* ptr, size_t length)
55
70.7M
        : m_ptr(ptr)
56
70.7M
        , m_length(length)
57
70.7M
    {
58
70.7M
    }
59
60
    u8 const* m_ptr { nullptr };
61
    size_t m_length { 0 };
62
};
63
64
class Utf8View {
65
public:
66
    using Iterator = Utf8CodePointIterator;
67
68
0
    Utf8View() = default;
69
70
    explicit constexpr Utf8View(StringView string)
71
86.3M
        : m_string(string)
72
86.3M
    {
73
86.3M
    }
74
75
#ifndef KERNEL
76
    explicit Utf8View(ByteString& string)
77
18.2M
        : m_string(string.view())
78
18.2M
    {
79
18.2M
    }
80
81
    explicit Utf8View(ByteString&&) = delete;
82
#endif
83
84
    enum class AllowSurrogates {
85
        Yes,
86
        No,
87
    };
88
89
    ~Utf8View() = default;
90
91
3.83k
    StringView as_string() const { return m_string; }
92
93
45.1M
    Utf8CodePointIterator begin() const { return { begin_ptr(), m_string.length() }; }
94
25.6M
    Utf8CodePointIterator end() const { return { end_ptr(), 0 }; }
95
    Utf8CodePointIterator iterator_at_byte_offset(size_t) const;
96
97
    Utf8CodePointIterator iterator_at_byte_offset_without_validation(size_t) const;
98
99
1.35k
    unsigned char const* bytes() const { return begin_ptr(); }
100
638k
    size_t byte_length() const { return m_string.length(); }
101
    size_t byte_offset_of(Utf8CodePointIterator const&) const;
102
    size_t byte_offset_of(size_t code_point_offset) const;
103
104
4.15k
    Utf8View substring_view(size_t byte_offset, size_t byte_length) const { return Utf8View { m_string.substring_view(byte_offset, byte_length) }; }
105
3.99k
    Utf8View substring_view(size_t byte_offset) const { return substring_view(byte_offset, byte_length() - byte_offset); }
106
    Utf8View unicode_substring_view(size_t code_point_offset, size_t code_point_length) const;
107
0
    Utf8View unicode_substring_view(size_t code_point_offset) const { return unicode_substring_view(code_point_offset, length() - code_point_offset); }
108
109
874k
    bool is_empty() const { return m_string.is_empty(); }
110
0
    bool is_null() const { return m_string.is_null(); }
111
    bool starts_with(Utf8View const&) const;
112
    bool contains(u32) const;
113
114
    Utf8View trim(Utf8View const& characters, TrimMode mode = TrimMode::Both) const;
115
116
    size_t iterator_offset(Utf8CodePointIterator const& it) const
117
6.41k
    {
118
6.41k
        return byte_offset_of(it);
119
6.41k
    }
120
121
    size_t length() const
122
0
    {
123
0
        if (!m_have_length) {
124
0
            m_length = calculate_length();
125
0
            m_have_length = true;
126
0
        }
127
0
        return m_length;
128
0
    }
129
130
    constexpr bool validate(AllowSurrogates surrogates = AllowSurrogates::Yes) const
131
51.2M
    {
132
51.2M
        size_t valid_bytes = 0;
133
51.2M
        return validate(valid_bytes, surrogates);
134
51.2M
    }
135
136
    constexpr bool validate(size_t& valid_bytes, AllowSurrogates surrogates = AllowSurrogates::Yes) const
137
51.2M
    {
138
51.2M
        valid_bytes = 0;
139
140
2.75G
        for (auto it = m_string.begin(); it != m_string.end(); ++it) {
141
2.70G
            auto [byte_length, code_point, is_valid] = decode_leading_byte(static_cast<u8>(*it));
142
2.70G
            if (!is_valid)
143
170k
                return false;
144
145
2.72G
            for (size_t i = 1; i < byte_length; ++i) {
146
17.8M
                if (++it == m_string.end())
147
11.3k
                    return false;
148
149
17.8M
                auto [code_point_bits, is_valid] = decode_continuation_byte(static_cast<u8>(*it));
150
17.8M
                if (!is_valid)
151
73.9k
                    return false;
152
153
17.7M
                code_point <<= 6;
154
17.7M
                code_point |= code_point_bits;
155
17.7M
            }
156
157
2.70G
            if (!is_valid_code_point(code_point, byte_length, surrogates))
158
2.55k
                return false;
159
160
2.70G
            valid_bytes += byte_length;
161
2.70G
        }
162
163
50.9M
        return true;
164
51.2M
    }
165
166
    template<typename Callback>
167
    auto for_each_split_view(Function<bool(u32)> splitter, SplitBehavior split_behavior, Callback callback) const
168
    {
169
        bool keep_empty = has_flag(split_behavior, SplitBehavior::KeepEmpty);
170
        bool keep_trailing_separator = has_flag(split_behavior, SplitBehavior::KeepTrailingSeparator);
171
172
        auto start_offset = 0u;
173
        auto offset = 0u;
174
175
        auto run_callback = [&]() {
176
            auto length = offset - start_offset;
177
178
            if (length == 0 && !keep_empty)
179
                return;
180
181
            auto substring = unicode_substring_view(start_offset, length);
182
183
            // Reject splitter-only entries if we're not keeping empty results
184
            if (keep_trailing_separator && !keep_empty && length == 1 && splitter(*substring.begin()))
185
                return;
186
187
            callback(substring);
188
        };
189
190
        auto iterator = begin();
191
        while (iterator != end()) {
192
            if (splitter(*iterator)) {
193
                if (keep_trailing_separator)
194
                    ++offset;
195
196
                run_callback();
197
198
                if (!keep_trailing_separator)
199
                    ++offset;
200
201
                start_offset = offset;
202
                ++iterator;
203
                continue;
204
            }
205
206
            ++offset;
207
            ++iterator;
208
        }
209
        run_callback();
210
    }
211
212
private:
213
    friend class Utf8CodePointIterator;
214
215
72.6M
    u8 const* begin_ptr() const { return reinterpret_cast<u8 const*>(m_string.characters_without_null_termination()); }
216
26.2M
    u8 const* end_ptr() const { return begin_ptr() + m_string.length(); }
217
    size_t calculate_length() const;
218
219
    struct Utf8EncodedByteData {
220
        size_t byte_length { 0 };
221
        u8 encoding_bits { 0 };
222
        u8 encoding_mask { 0 };
223
        u32 first_code_point { 0 };
224
        u32 last_code_point { 0 };
225
    };
226
227
    static constexpr Array<Utf8EncodedByteData, 4> utf8_encoded_byte_data { {
228
        { 1, 0b0000'0000, 0b1000'0000, 0x0000, 0x007F },
229
        { 2, 0b1100'0000, 0b1110'0000, 0x0080, 0x07FF },
230
        { 3, 0b1110'0000, 0b1111'0000, 0x0800, 0xFFFF },
231
        { 4, 0b1111'0000, 0b1111'1000, 0x10000, 0x10FFFF },
232
    } };
233
234
    struct LeadingByte {
235
        size_t byte_length { 0 };
236
        u32 code_point_bits { 0 };
237
        bool is_valid { false };
238
    };
239
240
    static constexpr LeadingByte decode_leading_byte(u8 byte)
241
3.09G
    {
242
3.85G
        for (auto const& data : utf8_encoded_byte_data) {
243
3.85G
            if ((byte & data.encoding_mask) != data.encoding_bits)
244
938M
                continue;
245
246
2.91G
            byte &= ~data.encoding_mask;
247
2.91G
            return { data.byte_length, byte, true };
248
3.85G
        }
249
250
179M
        return { .is_valid = false };
251
3.09G
    }
252
253
    struct ContinuationByte {
254
        u32 code_point_bits { 0 };
255
        bool is_valid { false };
256
    };
257
258
    static constexpr ContinuationByte decode_continuation_byte(u8 byte)
259
17.8M
    {
260
17.8M
        constexpr u8 continuation_byte_encoding_bits = 0b1000'0000;
261
17.8M
        constexpr u8 continuation_byte_encoding_mask = 0b1100'0000;
262
263
17.8M
        if ((byte & continuation_byte_encoding_mask) == continuation_byte_encoding_bits) {
264
17.7M
            byte &= ~continuation_byte_encoding_mask;
265
17.7M
            return { byte, true };
266
17.7M
        }
267
268
73.9k
        return { .is_valid = false };
269
17.8M
    }
270
271
    static constexpr bool is_valid_code_point(u32 code_point, size_t byte_length, AllowSurrogates surrogates = AllowSurrogates::Yes)
272
2.70G
    {
273
2.70G
        if (surrogates == AllowSurrogates::No && byte_length == 3 && code_point >= 0xD800 && code_point <= 0xDFFF)
274
12
            return false;
275
2.72G
        for (auto const& data : utf8_encoded_byte_data) {
276
2.72G
            if (code_point >= data.first_code_point && code_point <= data.last_code_point)
277
2.70G
                return byte_length == data.byte_length;
278
2.72G
        }
279
280
1.02k
        return false;
281
2.70G
    }
282
283
    StringView m_string;
284
    mutable size_t m_length { 0 };
285
    mutable bool m_have_length { false };
286
};
287
288
#ifndef KERNEL
289
class DeprecatedStringCodePointIterator {
290
public:
291
    Optional<u32> next()
292
0
    {
293
0
        if (m_it.done())
294
0
            return {};
295
0
        auto value = *m_it;
296
0
        ++m_it;
297
0
        return value;
298
0
    }
299
300
    [[nodiscard]] Optional<u32> peek() const
301
0
    {
302
0
        if (m_it.done())
303
0
            return {};
304
0
        return *m_it;
305
0
    }
306
307
    [[nodiscard]] size_t byte_offset() const
308
0
    {
309
0
        return Utf8View(m_string).byte_offset_of(m_it);
310
0
    }
311
312
    DeprecatedStringCodePointIterator(ByteString string)
313
0
        : m_string(move(string))
314
0
        , m_it(Utf8View(m_string).begin())
315
0
    {
316
0
    }
317
318
private:
319
    ByteString m_string;
320
    Utf8CodePointIterator m_it;
321
};
322
#endif
323
324
template<>
325
struct Formatter<Utf8View> : Formatter<StringView> {
326
    ErrorOr<void> format(FormatBuilder&, Utf8View const&);
327
};
328
329
}
330
331
#if USING_AK_GLOBALLY
332
#    ifndef KERNEL
333
using AK::DeprecatedStringCodePointIterator;
334
#    endif
335
using AK::Utf8CodePointIterator;
336
using AK::Utf8View;
337
#endif