/src/simdutf/fuzz/misc.cpp
Line | Count | Source |
1 | | #include <cstddef> |
2 | | #include <cstdint> |
3 | | #include <ranges> |
4 | | #include <cstdlib> |
5 | | |
6 | | #include "helpers/common.h" |
7 | | #include "simdutf.h" |
8 | | |
9 | 718 | void autodetect(std::span<const char> chardata) { |
10 | 718 | std::vector<simdutf::encoding_type> results; |
11 | 718 | const auto implementations = get_supported_implementations(); |
12 | 2.15k | for (const simdutf::implementation* impl : implementations) { |
13 | 2.15k | results.push_back( |
14 | 2.15k | impl->autodetect_encoding(chardata.data(), chardata.size())); |
15 | 2.15k | } |
16 | 1.43k | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
17 | 718 | if (std::ranges::adjacent_find(results, neq) != results.end()) { |
18 | 0 | std::cerr << "output differs between implementations\n"; |
19 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
20 | 0 | std::cerr << "implementation " << implementations[i] << " gave " |
21 | 0 | << results.at(i) << '\n'; |
22 | 0 | } |
23 | 0 | std::abort(); |
24 | 0 | } |
25 | 718 | } |
26 | | |
27 | 636 | void detect(std::span<const char> chardata) { |
28 | 636 | std::vector<int> results; |
29 | 636 | const auto implementations = get_supported_implementations(); |
30 | 1.90k | for (const simdutf::implementation* impl : implementations) { |
31 | 1.90k | results.push_back(impl->detect_encodings(chardata.data(), chardata.size())); |
32 | 1.90k | } |
33 | 1.27k | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
34 | 636 | if (std::ranges::adjacent_find(results, neq) != results.end()) { |
35 | 0 | std::cerr << "in detect_encodings(const char*, std::size_t):\n"; |
36 | 0 | std::cerr << "output differs between implementations\n"; |
37 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
38 | 0 | std::cerr << "implementation " << implementations[i]->name() << " gave " |
39 | 0 | << results.at(i) << '\n'; |
40 | 0 | } |
41 | 0 | std::cerr << " std::vector<unsigned char> data{"; |
42 | 0 | for (unsigned char x : chardata) { |
43 | 0 | std::cerr << +x << ", "; |
44 | 0 | }; |
45 | 0 | std::cerr << "};\n"; |
46 | 0 | std::abort(); |
47 | 0 | } |
48 | 636 | } |
49 | | |
50 | 190 | void validate_ascii(std::span<const char> chardata) { |
51 | | // use int, not bool to avoid vector<bool> |
52 | 190 | std::vector<int> results; |
53 | 190 | const auto implementations = get_supported_implementations(); |
54 | 570 | for (const simdutf::implementation* impl : implementations) { |
55 | 570 | results.push_back(+impl->validate_ascii(chardata.data(), chardata.size())); |
56 | 570 | } |
57 | 380 | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
58 | 190 | if (std::ranges::adjacent_find(results, neq) != results.end()) { |
59 | 0 | std::cerr << "in validate_ascii(const char*, std::size_t):\n"; |
60 | 0 | std::cerr << "output differs between implementations\n"; |
61 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
62 | 0 | std::cerr << "implementation " << implementations[i]->name() << " gave " |
63 | 0 | << results.at(i) << '\n'; |
64 | 0 | } |
65 | 0 | std::cerr << " std::vector<unsigned char> data{"; |
66 | 0 | for (unsigned char x : chardata) { |
67 | 0 | std::cerr << +x << ", "; |
68 | 0 | }; |
69 | 0 | std::cerr << "};\n"; |
70 | 0 | std::abort(); |
71 | 0 | } |
72 | 190 | } |
73 | | |
74 | 274 | void validate_ascii_with_err(std::span<const char> chardata) { |
75 | | // use int, not bool to avoid vector<bool> |
76 | 274 | std::vector<simdutf::result> results; |
77 | 274 | const auto implementations = get_supported_implementations(); |
78 | 822 | for (const simdutf::implementation* impl : implementations) { |
79 | 822 | results.push_back( |
80 | 822 | impl->validate_ascii_with_errors(chardata.data(), chardata.size())); |
81 | 822 | } |
82 | 548 | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
83 | 274 | if (std::ranges::adjacent_find(results, neq) != results.end()) { |
84 | 0 | std::cerr << "in validate_ascii(const char*, std::size_t):\n"; |
85 | 0 | std::cerr << "output differs between implementations\n"; |
86 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
87 | 0 | std::cerr << "implementation " << implementations[i]->name() << " gave " |
88 | 0 | << results.at(i) << '\n'; |
89 | 0 | } |
90 | 0 | std::cerr << " std::vector<unsigned char> data{"; |
91 | 0 | for (unsigned char x : chardata) { |
92 | 0 | std::cerr << +x << ", "; |
93 | 0 | }; |
94 | 0 | std::cerr << "};\n"; |
95 | 0 | std::abort(); |
96 | 0 | } |
97 | 274 | } |
98 | | |
99 | 125 | void utf16_endianness(std::span<const char16_t> data) { |
100 | 125 | std::vector<std::string> results; |
101 | 125 | const auto implementations = get_supported_implementations(); |
102 | 375 | for (const simdutf::implementation* impl : implementations) { |
103 | 375 | std::vector<char16_t> out(data.size()); |
104 | 375 | impl->change_endianness_utf16(data.data(), data.size(), out.data()); |
105 | 375 | results.push_back(FNV1A_hash::as_str(out)); |
106 | 375 | } |
107 | 250 | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
108 | 125 | if (std::ranges::adjacent_find(results, neq) != results.end()) { |
109 | 0 | std::cerr << "in utf16_endianness(const char*, std::size_t):\n"; |
110 | 0 | std::cerr << "output differs between implementations\n"; |
111 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
112 | 0 | std::cerr << "implementation " << implementations[i]->name() << " gave " |
113 | 0 | << results.at(i) << '\n'; |
114 | 0 | } |
115 | 0 | std::cerr << " std::vector<char16_t> data{"; |
116 | 0 | for (int x : data) { |
117 | 0 | std::cerr << +x << ", "; |
118 | 0 | }; |
119 | 0 | std::cerr << "};\n"; |
120 | 0 | std::abort(); |
121 | 0 | } |
122 | 125 | } |
123 | | |
124 | | // Checks that validate_utf16le_as_ascii and validate_utf16be_as_ascii agree |
125 | | // across all implementations, and that a true result implies valid UTF-16. |
126 | 230 | void validate_utf16_as_ascii(std::span<const char16_t> data) { |
127 | 230 | const auto implementations = get_supported_implementations(); |
128 | | // use int, not bool to avoid vector<bool> |
129 | 230 | std::vector<int> le_results, be_results; |
130 | 230 | le_results.reserve(implementations.size()); |
131 | 230 | be_results.reserve(implementations.size()); |
132 | 690 | for (const simdutf::implementation* impl : implementations) { |
133 | 690 | le_results.push_back( |
134 | 690 | +impl->validate_utf16le_as_ascii(data.data(), data.size())); |
135 | 690 | be_results.push_back( |
136 | 690 | +impl->validate_utf16be_as_ascii(data.data(), data.size())); |
137 | 690 | } |
138 | 920 | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
139 | 230 | if (std::ranges::adjacent_find(le_results, neq) != le_results.end()) { |
140 | 0 | std::cerr << "validate_utf16le_as_ascii: output differs between " |
141 | 0 | "implementations\n"; |
142 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
143 | 0 | std::cerr << " " << implementations[i]->name() << " gave " |
144 | 0 | << le_results[i] << '\n'; |
145 | 0 | } |
146 | 0 | std::abort(); |
147 | 0 | } |
148 | 230 | if (std::ranges::adjacent_find(be_results, neq) != be_results.end()) { |
149 | 0 | std::cerr << "validate_utf16be_as_ascii: output differs between " |
150 | 0 | "implementations\n"; |
151 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
152 | 0 | std::cerr << " " << implementations[i]->name() << " gave " |
153 | 0 | << be_results[i] << '\n'; |
154 | 0 | } |
155 | 0 | std::abort(); |
156 | 0 | } |
157 | | // If LE validates as ASCII, it must also validate as UTF-16LE (ASCII is a |
158 | | // subset). |
159 | 230 | if (le_results[0]) { |
160 | 63 | for (const simdutf::implementation* impl : implementations) { |
161 | 63 | if (!impl->validate_utf16le(data.data(), data.size())) { |
162 | 0 | std::cerr << "validate_utf16le_as_ascii returned true but " |
163 | 0 | "validate_utf16le returned false" |
164 | 0 | << " impl=" << impl->name() << "\n"; |
165 | 0 | std::abort(); |
166 | 0 | } |
167 | 63 | } |
168 | 21 | } |
169 | | // Same for BE. |
170 | 230 | if (be_results[0]) { |
171 | 78 | for (const simdutf::implementation* impl : implementations) { |
172 | 78 | if (!impl->validate_utf16be(data.data(), data.size())) { |
173 | 0 | std::cerr << "validate_utf16be_as_ascii returned true but " |
174 | 0 | "validate_utf16be returned false" |
175 | 0 | << " impl=" << impl->name() << "\n"; |
176 | 0 | std::abort(); |
177 | 0 | } |
178 | 78 | } |
179 | 26 | } |
180 | 230 | } |
181 | | |
182 | | // Checks that to_well_formed_utf16le / to_well_formed_utf16be: |
183 | | // 1. All implementations agree on the output. |
184 | | // 2. The output is always valid UTF-16LE / UTF-16BE. |
185 | | // 3. When the input is already valid UTF-16, the output equals the input. |
186 | 719 | void to_well_formed_utf16(std::span<const char16_t> data) { |
187 | 719 | const auto implementations = get_supported_implementations(); |
188 | 719 | if (implementations.empty()) { |
189 | 0 | return; |
190 | 0 | } |
191 | | |
192 | | // Check LE variant |
193 | 719 | { |
194 | 719 | std::vector<std::vector<char16_t>> le_outputs; |
195 | 719 | le_outputs.reserve(implementations.size()); |
196 | 2.15k | for (const simdutf::implementation* impl : implementations) { |
197 | 2.15k | std::vector<char16_t> out(data.size()); |
198 | 2.15k | impl->to_well_formed_utf16le(data.data(), data.size(), out.data()); |
199 | 2.15k | le_outputs.push_back(std::move(out)); |
200 | 2.15k | } |
201 | 1.43k | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
202 | 719 | if (std::ranges::adjacent_find(le_outputs, neq) != le_outputs.end()) { |
203 | 0 | std::cerr |
204 | 0 | << "to_well_formed_utf16le: outputs differ between implementations\n"; |
205 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
206 | 0 | std::cerr << " " << implementations[i]->name() |
207 | 0 | << ": hash=" << FNV1A_hash::as_str(le_outputs[i]) << "\n"; |
208 | 0 | } |
209 | 0 | std::abort(); |
210 | 0 | } |
211 | | // Output must be valid UTF-16LE. |
212 | 2.87k | for (std::size_t i = 0; i < implementations.size(); ++i) { |
213 | 2.15k | if (!implementations[i]->validate_utf16le(le_outputs[i].data(), |
214 | 2.15k | le_outputs[i].size())) { |
215 | 0 | std::cerr << "to_well_formed_utf16le: output is not valid UTF-16LE" |
216 | 0 | << " impl=" << implementations[i]->name() << "\n"; |
217 | 0 | std::abort(); |
218 | 0 | } |
219 | 2.15k | } |
220 | | // If input was already valid UTF-16LE, output must equal input. |
221 | 719 | if (implementations[0]->validate_utf16le(data.data(), data.size())) { |
222 | 232 | if (!std::ranges::equal(le_outputs[0], data)) { |
223 | 0 | std::cerr << "to_well_formed_utf16le: valid input was modified\n"; |
224 | 0 | std::abort(); |
225 | 0 | } |
226 | 232 | } |
227 | 719 | } |
228 | | |
229 | | // Check BE variant |
230 | 719 | { |
231 | 719 | std::vector<std::vector<char16_t>> be_outputs; |
232 | 719 | be_outputs.reserve(implementations.size()); |
233 | 2.15k | for (const simdutf::implementation* impl : implementations) { |
234 | 2.15k | std::vector<char16_t> out(data.size()); |
235 | 2.15k | impl->to_well_formed_utf16be(data.data(), data.size(), out.data()); |
236 | 2.15k | be_outputs.push_back(std::move(out)); |
237 | 2.15k | } |
238 | 1.43k | auto neq = [](const auto& a, const auto& b) { return a != b; }; |
239 | 719 | if (std::ranges::adjacent_find(be_outputs, neq) != be_outputs.end()) { |
240 | 0 | std::cerr |
241 | 0 | << "to_well_formed_utf16be: outputs differ between implementations\n"; |
242 | 0 | for (std::size_t i = 0; i < implementations.size(); ++i) { |
243 | 0 | std::cerr << " " << implementations[i]->name() |
244 | 0 | << ": hash=" << FNV1A_hash::as_str(be_outputs[i]) << "\n"; |
245 | 0 | } |
246 | 0 | std::abort(); |
247 | 0 | } |
248 | | // Output must be valid UTF-16BE. |
249 | 2.87k | for (std::size_t i = 0; i < implementations.size(); ++i) { |
250 | 2.15k | if (!implementations[i]->validate_utf16be(be_outputs[i].data(), |
251 | 2.15k | be_outputs[i].size())) { |
252 | 0 | std::cerr << "to_well_formed_utf16be: output is not valid UTF-16BE" |
253 | 0 | << " impl=" << implementations[i]->name() << "\n"; |
254 | 0 | std::abort(); |
255 | 0 | } |
256 | 2.15k | } |
257 | | // If input was already valid UTF-16BE, output must equal input. |
258 | 719 | if (implementations[0]->validate_utf16be(data.data(), data.size())) { |
259 | 224 | if (!std::ranges::equal(be_outputs[0], data)) { |
260 | 0 | std::cerr << "to_well_formed_utf16be: valid input was modified\n"; |
261 | 0 | std::abort(); |
262 | 0 | } |
263 | 224 | } |
264 | 719 | } |
265 | 719 | } |
266 | | |
267 | | void convert_latin1_to_utf8_safe(std::span<const char> chardata, |
268 | 270 | const std::size_t outputsize) { |
269 | | // convert with a limited output buffer |
270 | 270 | std::vector<char> limited_output(outputsize); |
271 | 270 | const auto limited_ret = simdutf::convert_latin1_to_utf8_safe( |
272 | 270 | chardata.data(), chardata.size(), limited_output.data(), outputsize); |
273 | | |
274 | | // convert with a sufficiently large output buffer |
275 | 270 | std::vector<char> large_output(2 * chardata.size()); |
276 | 270 | const auto large_ret = simdutf::convert_latin1_to_utf8( |
277 | 270 | chardata.data(), chardata.size(), large_output.data()); |
278 | | |
279 | 270 | if (large_ret != 0) { |
280 | | // conversion was possible with a large buffer. |
281 | 245 | if (large_ret <= outputsize) { |
282 | | // the limited buffer was large enough, ensure we got the same result |
283 | 111 | assert(limited_ret == large_ret); |
284 | 111 | assert(std::ranges::equal(limited_output | std::views::take(large_ret), |
285 | 111 | large_output | std::views::take(large_ret))); |
286 | 134 | } else { |
287 | | // the number of written bytes for a limited buffer must not exceed what |
288 | | // the large buffer got. |
289 | 134 | assert(limited_ret <= large_ret); |
290 | | // the written data should be equal |
291 | 134 | assert(std::ranges::equal(limited_output | std::views::take(limited_ret), |
292 | 134 | large_output | std::views::take(limited_ret))); |
293 | 134 | } |
294 | 245 | } else { |
295 | | // conversion with a big buffer failed - is there anything we can check or |
296 | | // assert for the limited buffer? I don't think so. |
297 | 25 | } |
298 | 270 | } |
299 | | |
300 | 3.35k | extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { |
301 | | // pick one of the functions, based on the fuzz data. |
302 | | // the first byte is which action to take. step forward |
303 | | // several bytes so the input is aligned. |
304 | 3.35k | if (size < 4) { |
305 | 2 | return 0; |
306 | 2 | } |
307 | 3.35k | constexpr auto Ncases = 11u; |
308 | 3.35k | constexpr auto actionmask = std::bit_ceil(Ncases) - 1; |
309 | 3.35k | const auto action = data[0] & actionmask; |
310 | | |
311 | 3.35k | const std::uint16_t u16 = data[1] + (data[2] << 8); |
312 | | |
313 | 3.35k | data += 4; |
314 | 3.35k | size -= 4; |
315 | | |
316 | 3.35k | const std::span<const char> chardata{(const char*)data, size}; |
317 | 3.35k | const std::span<const char16_t> u16data{(const char16_t*)data, |
318 | 3.35k | size / sizeof(char16_t)}; |
319 | | |
320 | 3.35k | switch (action) { |
321 | 718 | case 0: |
322 | 718 | autodetect(chardata); |
323 | 718 | break; |
324 | 636 | case 1: |
325 | 636 | detect(chardata); |
326 | 636 | break; |
327 | 190 | case 2: |
328 | 190 | validate_ascii(chardata); |
329 | 190 | break; |
330 | 274 | case 3: |
331 | 274 | validate_ascii_with_err(chardata); |
332 | 274 | break; |
333 | 125 | case 4: |
334 | 125 | utf16_endianness(u16data); |
335 | 125 | break; |
336 | 46 | case 5: { |
337 | 46 | [[maybe_unused]] auto ret = |
338 | 46 | simdutf::trim_partial_utf16le(u16data.data(), u16data.size()); |
339 | 46 | assert(ret == u16data.size() || ret + 1 == u16data.size()); |
340 | 46 | } break; |
341 | 47 | case 6: { |
342 | 47 | [[maybe_unused]] auto ret = |
343 | 47 | simdutf::trim_partial_utf16be(u16data.data(), u16data.size()); |
344 | 47 | assert(ret == u16data.size() || ret + 1 == u16data.size()); |
345 | 47 | } break; |
346 | 101 | case 7: { |
347 | 101 | [[maybe_unused]] const std::size_t N = chardata.size(); |
348 | 101 | [[maybe_unused]] const auto ret = |
349 | 101 | simdutf::trim_partial_utf8(chardata.data(), chardata.size()); |
350 | 101 | if ((ret + 3 < N) || (ret > N)) { |
351 | 0 | std::cerr << "ret=" << ret << " N=" << N << '\n'; |
352 | 0 | std::abort(); |
353 | 0 | } |
354 | 101 | } break; |
355 | 270 | case 8: |
356 | 270 | convert_latin1_to_utf8_safe(chardata, u16); |
357 | 270 | break; |
358 | 230 | case 9: |
359 | 230 | validate_utf16_as_ascii(u16data); |
360 | 230 | break; |
361 | 719 | case 10: |
362 | 719 | to_well_formed_utf16(u16data); |
363 | 719 | break; |
364 | 3.35k | } |
365 | 3.35k | return 0; |
366 | 3.35k | } |