/src/simdutf/include/simdutf/scalar/utf32.h
Line | Count | Source |
1 | | #ifndef SIMDUTF_UTF32_H |
2 | | #define SIMDUTF_UTF32_H |
3 | | |
4 | | namespace simdutf { |
5 | | namespace scalar { |
6 | | namespace utf32 { |
7 | | |
8 | | template <typename InputPtr> |
9 | | #if SIMDUTF_CPLUSPLUS20 |
10 | | requires simdutf::detail::indexes_into_uint32<InputPtr> |
11 | | #endif |
12 | | simdutf_warn_unused simdutf_constexpr23 bool validate(InputPtr data, |
13 | 5.77k | size_t len) noexcept { |
14 | 5.77k | uint64_t pos = 0; |
15 | 583k | for (; pos < len; pos++) { |
16 | 580k | uint32_t word = data[pos]; |
17 | 580k | if (word > 0x10FFFF || (word >= 0xD800 && word <= 0xDFFF)) { |
18 | 3.25k | return false; |
19 | 3.25k | } |
20 | 580k | } |
21 | 2.52k | return true; |
22 | 5.77k | } |
23 | | |
24 | | simdutf_warn_unused simdutf_really_inline bool validate(const char32_t *buf, |
25 | 5.77k | size_t len) noexcept { |
26 | 5.77k | return validate(reinterpret_cast<const uint32_t *>(buf), len); |
27 | 5.77k | } |
28 | | |
29 | | template <typename InputPtr> |
30 | | #if SIMDUTF_CPLUSPLUS20 |
31 | | requires simdutf::detail::indexes_into_uint32<InputPtr> |
32 | | #endif |
33 | | simdutf_warn_unused simdutf_constexpr23 result |
34 | 8.01k | validate_with_errors(InputPtr data, size_t len) noexcept { |
35 | 8.01k | size_t pos = 0; |
36 | 589k | for (; pos < len; pos++) { |
37 | 587k | uint32_t word = data[pos]; |
38 | 587k | if (word > 0x10FFFF) { |
39 | 4.56k | return result(error_code::TOO_LARGE, pos); |
40 | 4.56k | } |
41 | 582k | if (word >= 0xD800 && word <= 0xDFFF) { |
42 | 924 | return result(error_code::SURROGATE, pos); |
43 | 924 | } |
44 | 582k | } |
45 | 2.52k | return result(error_code::SUCCESS, pos); |
46 | 8.01k | } |
47 | | |
48 | | simdutf_warn_unused simdutf_really_inline result |
49 | 8.01k | validate_with_errors(const char32_t *buf, size_t len) noexcept { |
50 | 8.01k | return validate_with_errors(reinterpret_cast<const uint32_t *>(buf), len); |
51 | 8.01k | } |
52 | | |
53 | | inline simdutf_constexpr23 size_t utf8_length_from_utf32(const char32_t *p, |
54 | 2.67k | size_t len) { |
55 | | // We are not BOM aware. |
56 | 2.67k | size_t counter{0}; |
57 | 2.40M | for (size_t i = 0; i < len; i++) { |
58 | | // credit: @ttsugriy for the vectorizable approach |
59 | 2.40M | counter++; // ASCII |
60 | 2.40M | counter += static_cast<size_t>(p[i] > 0x7F); // two-byte |
61 | 2.40M | counter += static_cast<size_t>(p[i] > 0x7FF); // three-byte |
62 | 2.40M | counter += static_cast<size_t>(p[i] > 0xFFFF); // four-bytes |
63 | 2.40M | } |
64 | 2.67k | return counter; |
65 | 2.67k | } |
66 | | |
67 | | inline simdutf_warn_unused simdutf_constexpr23 size_t |
68 | 4.37k | utf16_length_from_utf32(const char32_t *p, size_t len) { |
69 | | // We are not BOM aware. |
70 | 4.37k | size_t counter{0}; |
71 | 3.87M | for (size_t i = 0; i < len; i++) { |
72 | 3.86M | counter++; // non-surrogate word |
73 | 3.86M | counter += static_cast<size_t>(p[i] > 0xFFFF); // surrogate pair |
74 | 3.86M | } |
75 | 4.37k | return counter; |
76 | 4.37k | } |
77 | | |
78 | | } // namespace utf32 |
79 | | } // namespace scalar |
80 | | } // namespace simdutf |
81 | | |
82 | | #endif |