/src/duckdb/third_party/brotli/enc/utf8_util.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | /* Copyright 2013 Google Inc. All Rights Reserved. |
2 | | |
3 | | Distributed under MIT license. |
4 | | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT |
5 | | */ |
6 | | |
7 | | /* Heuristics for deciding about the UTF8-ness of strings. */ |
8 | | |
9 | | #include "utf8_util.h" |
10 | | |
11 | | #include <brotli/types.h> |
12 | | |
13 | | using namespace duckdb_brotli; |
14 | | |
15 | | static size_t BrotliParseAsUTF8( |
16 | 0 | int* symbol, const uint8_t* input, size_t size) { |
17 | | /* ASCII */ |
18 | 0 | if ((input[0] & 0x80) == 0) { |
19 | 0 | *symbol = input[0]; |
20 | 0 | if (*symbol > 0) { |
21 | 0 | return 1; |
22 | 0 | } |
23 | 0 | } |
24 | | /* 2-byte UTF8 */ |
25 | 0 | if (size > 1u && |
26 | 0 | (input[0] & 0xE0) == 0xC0 && |
27 | 0 | (input[1] & 0xC0) == 0x80) { |
28 | 0 | *symbol = (((input[0] & 0x1F) << 6) | |
29 | 0 | (input[1] & 0x3F)); |
30 | 0 | if (*symbol > 0x7F) { |
31 | 0 | return 2; |
32 | 0 | } |
33 | 0 | } |
34 | | /* 3-byte UFT8 */ |
35 | 0 | if (size > 2u && |
36 | 0 | (input[0] & 0xF0) == 0xE0 && |
37 | 0 | (input[1] & 0xC0) == 0x80 && |
38 | 0 | (input[2] & 0xC0) == 0x80) { |
39 | 0 | *symbol = (((input[0] & 0x0F) << 12) | |
40 | 0 | ((input[1] & 0x3F) << 6) | |
41 | 0 | (input[2] & 0x3F)); |
42 | 0 | if (*symbol > 0x7FF) { |
43 | 0 | return 3; |
44 | 0 | } |
45 | 0 | } |
46 | | /* 4-byte UFT8 */ |
47 | 0 | if (size > 3u && |
48 | 0 | (input[0] & 0xF8) == 0xF0 && |
49 | 0 | (input[1] & 0xC0) == 0x80 && |
50 | 0 | (input[2] & 0xC0) == 0x80 && |
51 | 0 | (input[3] & 0xC0) == 0x80) { |
52 | 0 | *symbol = (((input[0] & 0x07) << 18) | |
53 | 0 | ((input[1] & 0x3F) << 12) | |
54 | 0 | ((input[2] & 0x3F) << 6) | |
55 | 0 | (input[3] & 0x3F)); |
56 | 0 | if (*symbol > 0xFFFF && *symbol <= 0x10FFFF) { |
57 | 0 | return 4; |
58 | 0 | } |
59 | 0 | } |
60 | | /* Not UTF8, emit a special symbol above the UTF8-code space */ |
61 | 0 | *symbol = 0x110000 | input[0]; |
62 | 0 | return 1; |
63 | 0 | } |
64 | | |
65 | | /* Returns 1 if at least min_fraction of the data is UTF8-encoded.*/ |
66 | | BROTLI_BOOL duckdb_brotli::BrotliIsMostlyUTF8( |
67 | | const uint8_t* data, const size_t pos, const size_t mask, |
68 | 0 | const size_t length, const double min_fraction) { |
69 | 0 | size_t size_utf8 = 0; |
70 | 0 | size_t i = 0; |
71 | 0 | while (i < length) { |
72 | 0 | int symbol; |
73 | 0 | size_t bytes_read = |
74 | 0 | BrotliParseAsUTF8(&symbol, &data[(pos + i) & mask], length - i); |
75 | 0 | i += bytes_read; |
76 | 0 | if (symbol < 0x110000) size_utf8 += bytes_read; |
77 | 0 | } |
78 | 0 | return TO_BROTLI_BOOL((double)size_utf8 > min_fraction * (double)length); |
79 | 0 | } |
80 | | |
81 | | |