/src/simdutf/fuzz/misc.cpp
Line | Count | Source |
1 | | #include <cstddef> |
2 | | #include <cstdint> |
3 | | #include <ranges> |
4 | | #include <cstdlib> |
5 | | |
6 | | #include "helpers/common.h" |
7 | | #include "simdutf.h" |
8 | | |
9 | 898 | void autodetect(std::span<const char> chardata) { |
10 | 898 | std::vector<simdutf::encoding_type> results; |
11 | 898 | const auto implementations = get_supported_implementations(); |
12 | 2.69k | for (const simdutf::implementation* impl : implementations) { |
13 | 2.69k | results.push_back( |
14 | 2.69k | impl->autodetect_encoding(chardata.data(), chardata.size())); |
15 | 2.69k | } |
16 | 1.79k | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
17 | 898 | if (std::ranges::adjacent_find(results, neq) != results.end()) { |
18 | 0 | std::cerr << "output differs between implementations\n"; |
19 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
20 | 0 | std::cerr << "implementation " << implementations[i] << " gave " |
21 | 0 | << results.at(i) << '\n'; |
22 | 0 | } |
23 | 0 | std::abort(); |
24 | 0 | } |
25 | 898 | } |
26 | | |
27 | 650 | void detect(std::span<const char> chardata) { |
28 | 650 | std::vector<int> results; |
29 | 650 | const auto implementations = get_supported_implementations(); |
30 | 1.95k | for (const simdutf::implementation* impl : implementations) { |
31 | 1.95k | results.push_back(impl->detect_encodings(chardata.data(), chardata.size())); |
32 | 1.95k | } |
33 | 1.30k | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
34 | 650 | if (std::ranges::adjacent_find(results, neq) != results.end()) { |
35 | 0 | std::cerr << "in detect_encodings(const char*, std::size_t):\n"; |
36 | 0 | std::cerr << "output differs between implementations\n"; |
37 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
38 | 0 | std::cerr << "implementation " << implementations[i]->name() << " gave " |
39 | 0 | << results.at(i) << '\n'; |
40 | 0 | } |
41 | 0 | std::cerr << " std::vector<unsigned char> data{"; |
42 | 0 | for (unsigned char x : chardata) { |
43 | 0 | std::cerr << +x << ", "; |
44 | 0 | }; |
45 | 0 | std::cerr << "};\n"; |
46 | 0 | std::abort(); |
47 | 0 | } |
48 | 650 | } |
49 | | |
50 | 201 | void validate_ascii(std::span<const char> chardata) { |
51 | | // use int, not bool to avoid vector<bool> |
52 | 201 | std::vector<int> results; |
53 | 201 | const auto implementations = get_supported_implementations(); |
54 | 603 | for (const simdutf::implementation* impl : implementations) { |
55 | 603 | results.push_back(+impl->validate_ascii(chardata.data(), chardata.size())); |
56 | 603 | } |
57 | 402 | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
58 | 201 | if (std::ranges::adjacent_find(results, neq) != results.end()) { |
59 | 0 | std::cerr << "in validate_ascii(const char*, std::size_t):\n"; |
60 | 0 | std::cerr << "output differs between implementations\n"; |
61 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
62 | 0 | std::cerr << "implementation " << implementations[i]->name() << " gave " |
63 | 0 | << results.at(i) << '\n'; |
64 | 0 | } |
65 | 0 | std::cerr << " std::vector<unsigned char> data{"; |
66 | 0 | for (unsigned char x : chardata) { |
67 | 0 | std::cerr << +x << ", "; |
68 | 0 | }; |
69 | 0 | std::cerr << "};\n"; |
70 | 0 | std::abort(); |
71 | 0 | } |
72 | 201 | } |
73 | | |
74 | 285 | void validate_ascii_with_err(std::span<const char> chardata) { |
75 | | // use int, not bool to avoid vector<bool> |
76 | 285 | std::vector<simdutf::result> results; |
77 | 285 | const auto implementations = get_supported_implementations(); |
78 | 855 | for (const simdutf::implementation* impl : implementations) { |
79 | 855 | results.push_back( |
80 | 855 | impl->validate_ascii_with_errors(chardata.data(), chardata.size())); |
81 | 855 | } |
82 | 570 | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
83 | 285 | if (std::ranges::adjacent_find(results, neq) != results.end()) { |
84 | 0 | std::cerr << "in validate_ascii(const char*, std::size_t):\n"; |
85 | 0 | std::cerr << "output differs between implementations\n"; |
86 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
87 | 0 | std::cerr << "implementation " << implementations[i]->name() << " gave " |
88 | 0 | << results.at(i) << '\n'; |
89 | 0 | } |
90 | 0 | std::cerr << " std::vector<unsigned char> data{"; |
91 | 0 | for (unsigned char x : chardata) { |
92 | 0 | std::cerr << +x << ", "; |
93 | 0 | }; |
94 | 0 | std::cerr << "};\n"; |
95 | 0 | std::abort(); |
96 | 0 | } |
97 | 285 | } |
98 | | |
99 | 115 | void utf16_endianness(std::span<const char16_t> data) { |
100 | 115 | std::vector<std::string> results; |
101 | 115 | const auto implementations = get_supported_implementations(); |
102 | 345 | for (const simdutf::implementation* impl : implementations) { |
103 | 345 | std::vector<char16_t> out(data.size()); |
104 | 345 | impl->change_endianness_utf16(data.data(), data.size(), out.data()); |
105 | 345 | results.push_back(FNV1A_hash::as_str(out)); |
106 | 345 | } |
107 | 230 | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
108 | 115 | if (std::ranges::adjacent_find(results, neq) != results.end()) { |
109 | 0 | std::cerr << "in utf16_endianness(const char*, std::size_t):\n"; |
110 | 0 | std::cerr << "output differs between implementations\n"; |
111 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
112 | 0 | std::cerr << "implementation " << implementations[i]->name() << " gave " |
113 | 0 | << results.at(i) << '\n'; |
114 | 0 | } |
115 | 0 | std::cerr << " std::vector<char16_t> data{"; |
116 | 0 | for (int x : data) { |
117 | 0 | std::cerr << +x << ", "; |
118 | 0 | }; |
119 | 0 | std::cerr << "};\n"; |
120 | 0 | std::abort(); |
121 | 0 | } |
122 | 115 | } |
123 | | |
124 | | // Checks that validate_utf16le_as_ascii and validate_utf16be_as_ascii agree |
125 | | // across all implementations, and that a true result implies valid UTF-16. |
126 | 206 | void validate_utf16_as_ascii(std::span<const char16_t> data) { |
127 | 206 | const auto implementations = get_supported_implementations(); |
128 | | // use int, not bool to avoid vector<bool> |
129 | 206 | std::vector<int> le_results, be_results; |
130 | 206 | le_results.reserve(implementations.size()); |
131 | 206 | be_results.reserve(implementations.size()); |
132 | 618 | for (const simdutf::implementation* impl : implementations) { |
133 | 618 | le_results.push_back( |
134 | 618 | +impl->validate_utf16le_as_ascii(data.data(), data.size())); |
135 | 618 | be_results.push_back( |
136 | 618 | +impl->validate_utf16be_as_ascii(data.data(), data.size())); |
137 | 618 | } |
138 | 824 | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
139 | 206 | if (std::ranges::adjacent_find(le_results, neq) != le_results.end()) { |
140 | 0 | std::cerr << "validate_utf16le_as_ascii: output differs between " |
141 | 0 | "implementations\n"; |
142 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
143 | 0 | std::cerr << " " << implementations[i]->name() << " gave " |
144 | 0 | << le_results[i] << '\n'; |
145 | 0 | } |
146 | 0 | std::abort(); |
147 | 0 | } |
148 | 206 | if (std::ranges::adjacent_find(be_results, neq) != be_results.end()) { |
149 | 0 | std::cerr << "validate_utf16be_as_ascii: output differs between " |
150 | 0 | "implementations\n"; |
151 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
152 | 0 | std::cerr << " " << implementations[i]->name() << " gave " |
153 | 0 | << be_results[i] << '\n'; |
154 | 0 | } |
155 | 0 | std::abort(); |
156 | 0 | } |
157 | | // If LE validates as ASCII, it must also validate as UTF-16LE (ASCII is a |
158 | | // subset). |
159 | 206 | if (le_results[0]) { |
160 | 81 | for (const simdutf::implementation* impl : implementations) { |
161 | 81 | if (!impl->validate_utf16le(data.data(), data.size())) { |
162 | 0 | std::cerr << "validate_utf16le_as_ascii returned true but " |
163 | 0 | "validate_utf16le returned false" |
164 | 0 | << " impl=" << impl->name() << "\n"; |
165 | 0 | std::abort(); |
166 | 0 | } |
167 | 81 | } |
168 | 27 | } |
169 | | // Same for BE. |
170 | 206 | if (be_results[0]) { |
171 | 81 | for (const simdutf::implementation* impl : implementations) { |
172 | 81 | if (!impl->validate_utf16be(data.data(), data.size())) { |
173 | 0 | std::cerr << "validate_utf16be_as_ascii returned true but " |
174 | 0 | "validate_utf16be returned false" |
175 | 0 | << " impl=" << impl->name() << "\n"; |
176 | 0 | std::abort(); |
177 | 0 | } |
178 | 81 | } |
179 | 27 | } |
180 | 206 | } |
181 | | |
182 | | // Checks that to_well_formed_utf16le / to_well_formed_utf16be: |
183 | | // 1. All implementations agree on the output. |
184 | | // 2. The output is always valid UTF-16LE / UTF-16BE. |
185 | | // 3. When the input is already valid UTF-16, the output equals the input. |
186 | 623 | void to_well_formed_utf16(std::span<const char16_t> data) { |
187 | 623 | const auto implementations = get_supported_implementations(); |
188 | 623 | if (implementations.empty()) { |
189 | 0 | return; |
190 | 0 | } |
191 | | |
192 | | // Check LE variant |
193 | 623 | { |
194 | 623 | std::vector<std::vector<char16_t>> le_outputs; |
195 | 623 | le_outputs.reserve(implementations.size()); |
196 | 1.86k | for (const simdutf::implementation* impl : implementations) { |
197 | 1.86k | std::vector<char16_t> out(data.size()); |
198 | 1.86k | impl->to_well_formed_utf16le(data.data(), data.size(), out.data()); |
199 | 1.86k | le_outputs.push_back(std::move(out)); |
200 | 1.86k | } |
201 | 1.24k | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
202 | 623 | if (std::ranges::adjacent_find(le_outputs, neq) != le_outputs.end()) { |
203 | 0 | std::cerr |
204 | 0 | << "to_well_formed_utf16le: outputs differ between implementations\n"; |
205 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
206 | 0 | std::cerr << " " << implementations[i]->name() |
207 | 0 | << ": hash=" << FNV1A_hash::as_str(le_outputs[i]) << "\n"; |
208 | 0 | } |
209 | 0 | std::abort(); |
210 | 0 | } |
211 | | // Output must be valid UTF-16LE. |
212 | 2.49k | for (std::size_t i = 0; i < implementations.size(); ++i) { |
213 | 1.86k | if (!implementations[i]->validate_utf16le(le_outputs[i].data(), |
214 | 1.86k | le_outputs[i].size())) { |
215 | 0 | std::cerr << "to_well_formed_utf16le: output is not valid UTF-16LE" |
216 | 0 | << " impl=" << implementations[i]->name() << "\n"; |
217 | 0 | std::abort(); |
218 | 0 | } |
219 | 1.86k | } |
220 | | // If input was already valid UTF-16LE, output must equal input. |
221 | 623 | if (implementations[0]->validate_utf16le(data.data(), data.size())) { |
222 | 203 | if (!std::ranges::equal(le_outputs[0], data)) { |
223 | 0 | std::cerr << "to_well_formed_utf16le: valid input was modified\n"; |
224 | 0 | std::abort(); |
225 | 0 | } |
226 | 203 | } |
227 | 623 | } |
228 | | |
229 | | // Check BE variant |
230 | 623 | { |
231 | 623 | std::vector<std::vector<char16_t>> be_outputs; |
232 | 623 | be_outputs.reserve(implementations.size()); |
233 | 1.86k | for (const simdutf::implementation* impl : implementations) { |
234 | 1.86k | std::vector<char16_t> out(data.size()); |
235 | 1.86k | impl->to_well_formed_utf16be(data.data(), data.size(), out.data()); |
236 | 1.86k | be_outputs.push_back(std::move(out)); |
237 | 1.86k | } |
238 | 1.24k | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
239 | 623 | if (std::ranges::adjacent_find(be_outputs, neq) != be_outputs.end()) { |
240 | 0 | std::cerr |
241 | 0 | << "to_well_formed_utf16be: outputs differ between implementations\n"; |
242 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
243 | 0 | std::cerr << " " << implementations[i]->name() |
244 | 0 | << ": hash=" << FNV1A_hash::as_str(be_outputs[i]) << "\n"; |
245 | 0 | } |
246 | 0 | std::abort(); |
247 | 0 | } |
248 | | // Output must be valid UTF-16BE. |
249 | 2.49k | for (std::size_t i = 0; i < implementations.size(); ++i) { |
250 | 1.86k | if (!implementations[i]->validate_utf16be(be_outputs[i].data(), |
251 | 1.86k | be_outputs[i].size())) { |
252 | 0 | std::cerr << "to_well_formed_utf16be: output is not valid UTF-16BE" |
253 | 0 | << " impl=" << implementations[i]->name() << "\n"; |
254 | 0 | std::abort(); |
255 | 0 | } |
256 | 1.86k | } |
257 | | // If input was already valid UTF-16BE, output must equal input. |
258 | 623 | if (implementations[0]->validate_utf16be(data.data(), data.size())) { |
259 | 203 | if (!std::ranges::equal(be_outputs[0], data)) { |
260 | 0 | std::cerr << "to_well_formed_utf16be: valid input was modified\n"; |
261 | 0 | std::abort(); |
262 | 0 | } |
263 | 203 | } |
264 | 623 | } |
265 | 623 | } |
266 | | |
267 | | void convert_latin1_to_utf8_safe(std::span<const char> chardata, |
268 | 248 | const std::size_t outputsize) { |
269 | | // convert with a limited output buffer |
270 | 248 | std::vector<char> limited_output(outputsize); |
271 | 248 | const auto limited_ret = simdutf::convert_latin1_to_utf8_safe( |
272 | 248 | chardata.data(), chardata.size(), limited_output.data(), outputsize); |
273 | | |
274 | | // convert with a sufficiently large output buffer |
275 | 248 | std::vector<char> large_output(2 * chardata.size()); |
276 | 248 | const auto large_ret = simdutf::convert_latin1_to_utf8( |
277 | 248 | chardata.data(), chardata.size(), large_output.data()); |
278 | | |
279 | 248 | if (large_ret != 0) { |
280 | | // conversion was possible with a large buffer. |
281 | 224 | if (large_ret <= outputsize) { |
282 | | // the limited buffer was large enough, ensure we got the same result |
283 | 106 | assert(limited_ret == large_ret); |
284 | 106 | assert(std::ranges::equal(limited_output | std::views::take(large_ret), |
285 | 106 | large_output | std::views::take(large_ret))); |
286 | 118 | } else { |
287 | | // the number of written bytes for a limited buffer must not exceed what |
288 | | // the large buffer got. |
289 | 118 | assert(limited_ret <= large_ret); |
290 | | // the written data should be equal |
291 | 118 | assert(std::ranges::equal(limited_output | std::views::take(limited_ret), |
292 | 118 | large_output | std::views::take(limited_ret))); |
293 | 118 | } |
294 | 224 | } else { |
295 | | // conversion with a big buffer failed - is there anything we can check or |
296 | | // assert for the limited buffer? I don't think so. |
297 | 24 | } |
298 | 248 | } |
299 | | |
300 | 3.38k | extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { |
301 | | // pick one of the functions, based on the fuzz data. |
302 | | // the first byte is which action to take. step forward |
303 | | // several bytes so the input is aligned. |
304 | 3.38k | if (size < 4) { |
305 | 2 | return 0; |
306 | 2 | } |
307 | 3.38k | constexpr auto Ncases = 11u; |
308 | 3.38k | constexpr auto actionmask = std::bit_ceil(Ncases) - 1; |
309 | 3.38k | const auto action = data[0] & actionmask; |
310 | | |
311 | 3.38k | const std::uint16_t u16 = data[1] + (data[2] << 8); |
312 | | |
313 | 3.38k | data += 4; |
314 | 3.38k | size -= 4; |
315 | | |
316 | 3.38k | const std::span<const char> chardata{(const char*)data, size}; |
317 | 3.38k | const std::span<const char16_t> u16data{(const char16_t*)data, |
318 | 3.38k | size / sizeof(char16_t)}; |
319 | | |
320 | 3.38k | switch (action) { |
321 | 898 | case 0: |
322 | 898 | autodetect(chardata); |
323 | 898 | break; |
324 | 650 | case 1: |
325 | 650 | detect(chardata); |
326 | 650 | break; |
327 | 201 | case 2: |
328 | 201 | validate_ascii(chardata); |
329 | 201 | break; |
330 | 285 | case 3: |
331 | 285 | validate_ascii_with_err(chardata); |
332 | 285 | break; |
333 | 115 | case 4: |
334 | 115 | utf16_endianness(u16data); |
335 | 115 | break; |
336 | 45 | case 5: { |
337 | 45 | [[maybe_unused]] auto ret = |
338 | 45 | simdutf::trim_partial_utf16le(u16data.data(), u16data.size()); |
339 | 45 | assert(ret == u16data.size() || ret + 1 == u16data.size()); |
340 | 45 | } break; |
341 | 45 | case 6: { |
342 | 19 | [[maybe_unused]] auto ret = |
343 | 19 | simdutf::trim_partial_utf16be(u16data.data(), u16data.size()); |
344 | 19 | assert(ret == u16data.size() || ret + 1 == u16data.size()); |
345 | 19 | } break; |
346 | 95 | case 7: { |
347 | 95 | [[maybe_unused]] const std::size_t N = chardata.size(); |
348 | 95 | [[maybe_unused]] const auto ret = |
349 | 95 | simdutf::trim_partial_utf8(chardata.data(), chardata.size()); |
350 | 95 | if ((ret + 3 < N) || (ret > N)) { |
351 | 0 | std::cerr << "ret=" << ret << " N=" << N << '\n'; |
352 | 0 | std::abort(); |
353 | 0 | } |
354 | 95 | } break; |
355 | 248 | case 8: |
356 | 248 | convert_latin1_to_utf8_safe(chardata, u16); |
357 | 248 | break; |
358 | 206 | case 9: |
359 | 206 | validate_utf16_as_ascii(u16data); |
360 | 206 | break; |
361 | 623 | case 10: |
362 | 623 | to_well_formed_utf16(u16data); |
363 | 623 | break; |
364 | 3.38k | } |
365 | 3.38k | return 0; |
366 | 3.38k | } |