Coverage Report

Created: 2026-09-26 06:49

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/serenity/AK/Utf16View.cpp
Line
Count
Source
1
/*
2
 * Copyright (c) 2021-2023, Tim Flynn <trflynn89@serenityos.org>
3
 *
4
 * SPDX-License-Identifier: BSD-2-Clause
5
 */
6
7
#include <AK/CharacterTypes.h>
8
#include <AK/Concepts.h>
9
#include <AK/StringBuilder.h>
10
#include <AK/StringView.h>
11
#include <AK/Utf16View.h>
12
#include <AK/Utf32View.h>
13
#include <AK/Utf8View.h>
14
15
namespace AK {
16
17
static constexpr u16 high_surrogate_min = 0xd800;
18
static constexpr u16 high_surrogate_max = 0xdbff;
19
static constexpr u16 low_surrogate_min = 0xdc00;
20
static constexpr u16 low_surrogate_max = 0xdfff;
21
static constexpr u32 replacement_code_point = 0xfffd;
22
static constexpr u32 first_supplementary_plane_code_point = 0x10000;
23
24
template<OneOf<Utf8View, Utf32View> UtfViewType>
25
static ErrorOr<Utf16Data> to_utf16_impl(UtfViewType const& view)
26
0
{
27
0
    Utf16Data utf16_data;
28
0
    TRY(utf16_data.try_ensure_capacity(view.length()));
29
30
0
    for (auto code_point : view)
31
0
        TRY(code_point_to_utf16(utf16_data, code_point));
32
33
0
    return utf16_data;
34
0
}
Unexecuted instantiation: Utf16View.cpp:_ZN2AKL13to_utf16_implITkNS_8Concepts5OneOfINS_8Utf8ViewENS_9Utf32ViewEEES3_EENS_7ErrorOrINS_6VectorItLm1EEENS_5ErrorEEERKT_
Unexecuted instantiation: Utf16View.cpp:_ZN2AKL13to_utf16_implITkNS_8Concepts5OneOfINS_8Utf8ViewENS_9Utf32ViewEEES4_EENS_7ErrorOrINS_6VectorItLm1EEENS_5ErrorEEERKT_
35
36
ErrorOr<Utf16Data> utf8_to_utf16(StringView utf8_view)
37
0
{
38
0
    return to_utf16_impl(Utf8View { utf8_view });
39
0
}
40
41
ErrorOr<Utf16Data> utf8_to_utf16(Utf8View const& utf8_view)
42
0
{
43
0
    return to_utf16_impl(utf8_view);
44
0
}
45
46
ErrorOr<Utf16Data> utf32_to_utf16(Utf32View const& utf32_view)
47
0
{
48
0
    return to_utf16_impl(utf32_view);
49
0
}
50
51
ErrorOr<void> code_point_to_utf16(Utf16Data& string, u32 code_point)
52
0
{
53
0
    VERIFY(is_unicode(code_point));
54
55
0
    if (code_point < first_supplementary_plane_code_point) {
56
0
        TRY(string.try_append(static_cast<u16>(code_point)));
57
0
    } else {
58
0
        code_point -= first_supplementary_plane_code_point;
59
0
        TRY(string.try_append(static_cast<u16>(high_surrogate_min | (code_point >> 10))));
60
0
        TRY(string.try_append(static_cast<u16>(low_surrogate_min | (code_point & 0x3ff))));
61
0
    }
62
63
0
    return {};
64
0
}
65
66
size_t utf16_code_unit_length_from_utf8(StringView string)
67
0
{
68
    // FIXME: This is inefficient!
69
0
    auto utf16_data = MUST(AK::utf8_to_utf16(string));
70
0
    return Utf16View { utf16_data }.length_in_code_units();
71
0
}
72
73
bool Utf16View::is_high_surrogate(u16 code_unit)
74
3.76M
{
75
3.76M
    return (code_unit >= high_surrogate_min) && (code_unit <= high_surrogate_max);
76
3.76M
}
77
78
bool Utf16View::is_low_surrogate(u16 code_unit)
79
158k
{
80
158k
    return (code_unit >= low_surrogate_min) && (code_unit <= low_surrogate_max);
81
158k
}
82
83
u32 Utf16View::decode_surrogate_pair(u16 high_surrogate, u16 low_surrogate)
84
17.1k
{
85
17.1k
    VERIFY(is_high_surrogate(high_surrogate));
86
17.1k
    VERIFY(is_low_surrogate(low_surrogate));
87
88
17.1k
    return ((high_surrogate - high_surrogate_min) << 10) + (low_surrogate - low_surrogate_min) + first_supplementary_plane_code_point;
89
17.1k
}
90
91
ErrorOr<ByteString> Utf16View::to_byte_string(AllowInvalidCodeUnits allow_invalid_code_units) const
92
0
{
93
0
    return TRY(to_utf8(allow_invalid_code_units)).to_byte_string();
94
0
}
95
96
ErrorOr<String> Utf16View::to_utf8(AllowInvalidCodeUnits allow_invalid_code_units) const
97
0
{
98
0
    StringBuilder builder;
99
100
0
    if (allow_invalid_code_units == AllowInvalidCodeUnits::Yes) {
101
0
        for (auto const* ptr = begin_ptr(); ptr < end_ptr(); ++ptr) {
102
0
            if (is_high_surrogate(*ptr)) {
103
0
                auto const* next = ptr + 1;
104
105
0
                if ((next < end_ptr()) && is_low_surrogate(*next)) {
106
0
                    auto code_point = decode_surrogate_pair(*ptr, *next);
107
0
                    TRY(builder.try_append_code_point(code_point));
108
0
                    ++ptr;
109
0
                    continue;
110
0
                }
111
0
            }
112
113
0
            TRY(builder.try_append_code_point(static_cast<u32>(*ptr)));
114
0
        }
115
0
        return builder.to_string_without_validation();
116
0
    }
117
118
0
    for (auto code_point : *this)
119
0
        TRY(builder.try_append_code_point(code_point));
120
121
0
    return builder.to_string();
122
0
}
123
124
size_t Utf16View::length_in_code_points() const
125
0
{
126
0
    if (!m_length_in_code_points.has_value())
127
0
        m_length_in_code_points = calculate_length_in_code_points();
128
0
    return *m_length_in_code_points;
129
0
}
130
131
u16 Utf16View::code_unit_at(size_t index) const
132
0
{
133
0
    VERIFY(index < length_in_code_units());
134
0
    return m_code_units[index];
135
0
}
136
137
u32 Utf16View::code_point_at(size_t index) const
138
0
{
139
0
    VERIFY(index < length_in_code_units());
140
141
0
    u32 code_point = code_unit_at(index);
142
0
    if (!is_high_surrogate(code_point) && !is_low_surrogate(code_point))
143
0
        return code_point;
144
0
    if (is_low_surrogate(code_point) || (index + 1 == length_in_code_units()))
145
0
        return code_point;
146
147
0
    auto second = code_unit_at(index + 1);
148
0
    if (!is_low_surrogate(second))
149
0
        return code_point;
150
151
0
    return decode_surrogate_pair(code_point, second);
152
0
}
153
154
size_t Utf16View::code_point_offset_of(size_t code_unit_offset) const
155
0
{
156
0
    size_t code_point_offset = 0;
157
158
0
    for (auto it = begin(); it != end(); ++it) {
159
0
        if (code_unit_offset == 0)
160
0
            return code_point_offset;
161
162
0
        code_unit_offset -= it.length_in_code_units();
163
0
        ++code_point_offset;
164
0
    }
165
166
0
    return code_point_offset;
167
0
}
168
169
size_t Utf16View::code_unit_offset_of(size_t code_point_offset) const
170
0
{
171
0
    size_t code_unit_offset = 0;
172
173
0
    for (auto it = begin(); it != end(); ++it) {
174
0
        if (code_point_offset == 0)
175
0
            return code_unit_offset;
176
177
0
        code_unit_offset += it.length_in_code_units();
178
0
        --code_point_offset;
179
0
    }
180
181
0
    return code_unit_offset;
182
0
}
183
184
size_t Utf16View::code_unit_offset_of(Utf16CodePointIterator const& it) const
185
0
{
186
0
    VERIFY(it.m_ptr >= begin_ptr());
187
0
    VERIFY(it.m_ptr <= end_ptr());
188
189
0
    return it.m_ptr - begin_ptr();
190
0
}
191
192
Utf16View Utf16View::substring_view(size_t code_unit_offset, size_t code_unit_length) const
193
0
{
194
0
    VERIFY(!Checked<size_t>::addition_would_overflow(code_unit_offset, code_unit_length));
195
0
    VERIFY(code_unit_offset + code_unit_length <= length_in_code_units());
196
197
0
    return Utf16View { m_code_units.slice(code_unit_offset, code_unit_length) };
198
0
}
199
200
Utf16View Utf16View::unicode_substring_view(size_t code_point_offset, size_t code_point_length) const
201
0
{
202
0
    if (code_point_length == 0)
203
0
        return {};
204
205
0
    auto code_unit_offset_of = [&](Utf16CodePointIterator const& it) { return it.m_ptr - begin_ptr(); };
206
0
    size_t code_point_index = 0;
207
0
    size_t code_unit_offset = 0;
208
209
0
    for (auto it = begin(); it != end(); ++it) {
210
0
        if (code_point_index == code_point_offset)
211
0
            code_unit_offset = code_unit_offset_of(it);
212
213
0
        if (code_point_index == (code_point_offset + code_point_length - 1)) {
214
0
            size_t code_unit_length = code_unit_offset_of(++it) - code_unit_offset;
215
0
            return substring_view(code_unit_offset, code_unit_length);
216
0
        }
217
218
0
        ++code_point_index;
219
0
    }
220
221
0
    VERIFY_NOT_REACHED();
222
0
}
223
224
bool Utf16View::starts_with(Utf16View const& needle) const
225
0
{
226
0
    if (needle.is_empty())
227
0
        return true;
228
0
    if (is_empty())
229
0
        return false;
230
0
    if (needle.length_in_code_units() > length_in_code_units())
231
0
        return false;
232
0
    if (begin_ptr() == needle.begin_ptr())
233
0
        return true;
234
235
0
    for (auto this_it = begin(), needle_it = needle.begin(); needle_it != needle.end(); ++needle_it, ++this_it) {
236
0
        if (*this_it != *needle_it)
237
0
            return false;
238
0
    }
239
240
0
    return true;
241
0
}
242
243
bool Utf16View::validate(size_t& valid_code_units) const
244
0
{
245
0
    valid_code_units = 0;
246
247
0
    for (auto const* ptr = begin_ptr(); ptr < end_ptr(); ++ptr) {
248
0
        if (is_high_surrogate(*ptr)) {
249
0
            if ((++ptr >= end_ptr()) || !is_low_surrogate(*ptr))
250
0
                return false;
251
0
            ++valid_code_units;
252
0
        } else if (is_low_surrogate(*ptr)) {
253
0
            return false;
254
0
        }
255
256
0
        ++valid_code_units;
257
0
    }
258
259
0
    return true;
260
0
}
261
262
size_t Utf16View::calculate_length_in_code_points() const
263
0
{
264
0
    size_t code_points = 0;
265
0
    for ([[maybe_unused]] auto code_point : *this)
266
0
        ++code_points;
267
0
    return code_points;
268
0
}
269
270
bool Utf16View::equals_ignoring_case(Utf16View const& other) const
271
0
{
272
0
    if (length_in_code_units() == 0)
273
0
        return other.length_in_code_units() == 0;
274
0
    if (length_in_code_units() != other.length_in_code_units())
275
0
        return false;
276
277
0
    for (size_t i = 0; i < length_in_code_units(); ++i) {
278
        // FIXME: Handle non-ASCII case insensitive comparisons.
279
0
        if (to_ascii_lowercase(m_code_units[i]) != to_ascii_lowercase(other.m_code_units[i]))
280
0
            return false;
281
0
    }
282
283
0
    return true;
284
0
}
285
286
Utf16CodePointIterator& Utf16CodePointIterator::operator++()
287
0
{
288
0
    size_t code_units = length_in_code_units();
289
290
0
    if (code_units > m_remaining_code_units) {
291
        // If there aren't enough code units remaining, skip to the end.
292
0
        m_ptr += m_remaining_code_units;
293
0
        m_remaining_code_units = 0;
294
0
    } else {
295
0
        m_ptr += code_units;
296
0
        m_remaining_code_units -= code_units;
297
0
    }
298
299
0
    return *this;
300
0
}
301
302
u32 Utf16CodePointIterator::operator*() const
303
0
{
304
0
    VERIFY(m_remaining_code_units > 0);
305
306
    // rfc2781, 2.2 Decoding UTF-16
307
    // 1) If W1 < 0xD800 or W1 > 0xDFFF, the character value U is the value
308
    //    of W1. Terminate.
309
    // 2) Determine if W1 is between 0xD800 and 0xDBFF. If not, the sequence
310
    //    is in error and no valid character can be obtained using W1.
311
    //    Terminate.
312
    // 3) If there is no W2 (that is, the sequence ends with W1), or if W2
313
    //    is not between 0xDC00 and 0xDFFF, the sequence is in error.
314
    //    Terminate.
315
    // 4) Construct a 20-bit unsigned integer U', taking the 10 low-order
316
    //    bits of W1 as its 10 high-order bits and the 10 low-order bits of
317
    //    W2 as its 10 low-order bits.
318
    // 5) Add 0x10000 to U' to obtain the character value U. Terminate.
319
320
0
    if (Utf16View::is_high_surrogate(*m_ptr)) {
321
0
        if ((m_remaining_code_units > 1) && Utf16View::is_low_surrogate(*(m_ptr + 1)))
322
0
            return Utf16View::decode_surrogate_pair(*m_ptr, *(m_ptr + 1));
323
0
        return replacement_code_point;
324
0
    } else if (Utf16View::is_low_surrogate(*m_ptr)) {
325
0
        return replacement_code_point;
326
0
    }
327
328
0
    return static_cast<u32>(*m_ptr);
329
0
}
330
331
size_t Utf16CodePointIterator::length_in_code_units() const
332
0
{
333
0
    VERIFY(m_remaining_code_units > 0);
334
335
0
    if (Utf16View::is_high_surrogate(*m_ptr)) {
336
0
        if ((m_remaining_code_units > 1) && Utf16View::is_low_surrogate(*(m_ptr + 1)))
337
0
            return 2;
338
0
    }
339
340
    // If this return is reached, either the encoded code point is a valid single code unit, or that
341
    // code point is invalid (e.g. began with a low surrogate, or a low surrogate did not follow a
342
    // high surrogate). In the latter case, a single replacement code unit will be used.
343
0
    return 1;
344
0
}
345
346
}