/src/simdutf/include/simdutf/scalar/utf32.h
Line | Count | Source |
1 | | #ifndef SIMDUTF_UTF32_H |
2 | | #define SIMDUTF_UTF32_H |
3 | | |
4 | | namespace simdutf { |
5 | | namespace scalar { |
6 | | namespace utf32 { |
7 | | |
8 | | template <typename InputPtr> |
9 | | #if SIMDUTF_CPLUSPLUS20 |
10 | | requires simdutf::detail::indexes_into_uint32<InputPtr> |
11 | | #endif |
12 | | simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, |
13 | 15.8k | size_t len) noexcept { |
14 | 15.8k | uint64_t pos = 0; |
15 | 160k | for (; pos < len; pos++) { |
16 | 144k | uint32_t word = data[pos]; |
17 | 144k | if (word > 0x10FFFF || (word >= 0xD800 && word <= 0xDFFF)) { |
18 | 0 | return false; |
19 | 0 | } |
20 | 144k | } |
21 | 15.8k | return true; |
22 | 15.8k | } |
23 | | |
24 | | simdutf_warn_unused simdutf_really_inline bool validate(const char32_t *buf, |
25 | 15.8k | size_t len) noexcept { |
26 | 15.8k | return validate(reinterpret_cast<const uint32_t *>(buf), len); |
27 | 15.8k | } |
28 | | |
29 | | template <typename InputPtr> |
30 | | #if SIMDUTF_CPLUSPLUS20 |
31 | | requires simdutf::detail::indexes_into_uint32<InputPtr> |
32 | | #endif |
33 | | simdutf_warn_unused simdutf_constexpr23 result |
34 | 0 | validate_with_errors(InputPtr data, size_t len) noexcept { |
35 | 0 | size_t pos = 0; |
36 | 0 | for (; pos < len; pos++) { |
37 | 0 | uint32_t word = data[pos]; |
38 | 0 | if (word > 0x10FFFF) { |
39 | 0 | return result(error_code::TOO_LARGE, pos); |
40 | 0 | } |
41 | 0 | if (word >= 0xD800 && word <= 0xDFFF) { |
42 | 0 | return result(error_code::SURROGATE, pos); |
43 | 0 | } |
44 | 0 | } |
45 | 0 | return result(error_code::SUCCESS, pos); |
46 | 0 | } |
47 | | |
48 | | simdutf_warn_unused simdutf_really_inline result |
49 | 0 | validate_with_errors(const char32_t *buf, size_t len) noexcept { |
50 | 0 | return validate_with_errors(reinterpret_cast<const uint32_t *>(buf), len); |
51 | 0 | } |
52 | | |
53 | | inline simdutf_constexpr23 size_t utf8_length_from_utf32(const char32_t *p, |
54 | 16.7k | size_t len) { |
55 | | // We are not BOM aware. |
56 | 16.7k | size_t counter{0}; |
57 | 56.9k | for (size_t i = 0; i < len; i++) { |
58 | | // credit: @ttsugriy for the vectorizable approach |
59 | 40.1k | counter++; // ASCII |
60 | 40.1k | counter += static_cast<size_t>(p[i] > 0x7F); // two-byte |
61 | 40.1k | counter += static_cast<size_t>(p[i] > 0x7FF); // three-byte |
62 | 40.1k | counter += static_cast<size_t>(p[i] > 0xFFFF); // four-bytes |
63 | 40.1k | } |
64 | 16.7k | return counter; |
65 | 16.7k | } |
66 | | |
67 | | inline simdutf_warn_unused simdutf_constexpr23 size_t |
68 | 0 | utf16_length_from_utf32(const char32_t *p, size_t len) { |
69 | | // We are not BOM aware. |
70 | 0 | size_t counter{0}; |
71 | 0 | for (size_t i = 0; i < len; i++) { |
72 | 0 | counter++; // non-surrogate word |
73 | 0 | counter += static_cast<size_t>(p[i] > 0xFFFF); // surrogate pair |
74 | 0 | } |
75 | 0 | return counter; |
76 | 0 | } |
77 | | |
78 | | } // namespace utf32 |
79 | | } // namespace scalar |
80 | | } // namespace simdutf |
81 | | |
82 | | #endif |