/src/simdutf/src/generic/utf8.h
Line | Count | Source |
1 | | namespace simdutf { |
2 | | namespace SIMDUTF_IMPLEMENTATION { |
3 | | namespace { |
4 | | namespace utf8 { |
5 | | |
6 | | using namespace simd; |
7 | | |
8 | 0 | simdutf_really_inline size_t count_code_points(const char *in, size_t size) { |
9 | 0 | size_t pos = 0; |
10 | 0 | size_t count = 0; |
11 | 0 | for (; pos + 64 <= size; pos += 64) { |
12 | 0 | simd8x64<int8_t> input(reinterpret_cast<const int8_t *>(in + pos)); |
13 | 0 | uint64_t utf8_continuation_mask = input.gt(-65); |
14 | 0 | count += count_ones(utf8_continuation_mask); |
15 | 0 | } |
16 | 0 | return count + scalar::utf8::count_code_points(in + pos, size - pos); |
17 | 0 | } Unexecuted instantiation: simdutf.cpp:simdutf::haswell::(anonymous namespace)::utf8::count_code_points(char const*, unsigned long) Unexecuted instantiation: simdutf.cpp:simdutf::westmere::(anonymous namespace)::utf8::count_code_points(char const*, unsigned long) |
18 | | |
19 | | #ifdef SIMDUTF_SIMD_HAS_BYTEMASK |
20 | | simdutf_unused simdutf_really_inline size_t |
21 | 0 | count_code_points_bytemask(const char *in, size_t size) { |
22 | 0 | using vector_i8 = simd8<int8_t>; |
23 | 0 | using vector_u8 = simd8<uint8_t>; |
24 | 0 | using vector_u64 = simd64<uint64_t>; |
25 | |
|
26 | 0 | constexpr size_t N = vector_i8::SIZE; |
27 | 0 | constexpr size_t max_iterations = 255 / 4; |
28 | |
|
29 | 0 | size_t pos = 0; |
30 | 0 | size_t count = 0; |
31 | |
|
32 | 0 | auto counters = vector_u64::zero(); |
33 | 0 | auto local = vector_u8::zero(); |
34 | 0 | size_t iterations = 0; |
35 | 0 | for (; pos + 4 * N <= size; pos += 4 * N) { |
36 | 0 | const auto input0 = |
37 | 0 | simd8<int8_t>::load(reinterpret_cast<const int8_t *>(in + pos + 0 * N)); |
38 | 0 | const auto input1 = |
39 | 0 | simd8<int8_t>::load(reinterpret_cast<const int8_t *>(in + pos + 1 * N)); |
40 | 0 | const auto input2 = |
41 | 0 | simd8<int8_t>::load(reinterpret_cast<const int8_t *>(in + pos + 2 * N)); |
42 | 0 | const auto input3 = |
43 | 0 | simd8<int8_t>::load(reinterpret_cast<const int8_t *>(in + pos + 3 * N)); |
44 | 0 | const auto mask0 = input0 > int8_t(-65); |
45 | 0 | const auto mask1 = input1 > int8_t(-65); |
46 | 0 | const auto mask2 = input2 > int8_t(-65); |
47 | 0 | const auto mask3 = input3 > int8_t(-65); |
48 | |
|
49 | 0 | local -= vector_u8(mask0); |
50 | 0 | local -= vector_u8(mask1); |
51 | 0 | local -= vector_u8(mask2); |
52 | 0 | local -= vector_u8(mask3); |
53 | |
|
54 | 0 | iterations += 1; |
55 | 0 | if (iterations == max_iterations) { |
56 | 0 | counters += sum_8bytes(local); |
57 | 0 | local = vector_u8::zero(); |
58 | 0 | iterations = 0; |
59 | 0 | } |
60 | 0 | } |
61 | |
|
62 | 0 | if (iterations > 0) { |
63 | 0 | count += local.sum_bytes(); |
64 | 0 | } |
65 | |
|
66 | 0 | count += counters.sum(); |
67 | |
|
68 | 0 | return count + scalar::utf8::count_code_points(in + pos, size - pos); |
69 | 0 | } Unexecuted instantiation: simdutf.cpp:simdutf::haswell::(anonymous namespace)::utf8::count_code_points_bytemask(char const*, unsigned long) Unexecuted instantiation: simdutf.cpp:simdutf::westmere::(anonymous namespace)::utf8::count_code_points_bytemask(char const*, unsigned long) |
70 | | #endif // SIMDUTF_SIMD_HAS_BYTEMASK |
71 | | |
72 | | simdutf_really_inline size_t utf16_length_from_utf8(const char *in, |
73 | 0 | size_t size) { |
74 | 0 | size_t pos = 0; |
75 | 0 | size_t count = 0; |
76 | 0 | // This algorithm could no doubt be improved! |
77 | 0 | for (; pos + 64 <= size; pos += 64) { |
78 | 0 | simd8x64<int8_t> input(reinterpret_cast<const int8_t *>(in + pos)); |
79 | 0 | uint64_t utf8_continuation_mask = input.lt(-65 + 1); |
80 | 0 | // We count one word for anything that is not a continuation (so |
81 | 0 | // leading bytes). |
82 | 0 | count += 64 - count_ones(utf8_continuation_mask); |
83 | 0 | int64_t utf8_4byte = input.gteq_unsigned(240); |
84 | 0 | count += count_ones(utf8_4byte); |
85 | 0 | } |
86 | 0 | return count + scalar::utf8::utf16_length_from_utf8(in + pos, size - pos); |
87 | 0 | } Unexecuted instantiation: simdutf.cpp:simdutf::haswell::(anonymous namespace)::utf8::utf16_length_from_utf8(char const*, unsigned long) Unexecuted instantiation: simdutf.cpp:simdutf::westmere::(anonymous namespace)::utf8::utf16_length_from_utf8(char const*, unsigned long) |
88 | | |
89 | | } // namespace utf8 |
90 | | } // unnamed namespace |
91 | | } // namespace SIMDUTF_IMPLEMENTATION |
92 | | } // namespace simdutf |