Coverage Report

Created: 2026-10-28 07:02

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/simdutf/fuzz/misc.cpp
Line
Count
Source
1
#include <cstddef>
2
#include <cstdint>
3
#include <ranges>
4
#include <cstdlib>
5
6
#include "helpers/common.h"
7
#include "simdutf.h"
8
9
898
void autodetect(std::span<const char> chardata) {
10
898
  std::vector<simdutf::encoding_type> results;
11
898
  const auto implementations = get_supported_implementations();
12
2.69k
  for (const simdutf::implementation* impl : implementations) {
13
2.69k
    results.push_back(
14
2.69k
        impl->autodetect_encoding(chardata.data(), chardata.size()));
15
2.69k
  }
16
1.79k
  auto neq = [](const auto& a, const auto& b) { return a != b; };
17
898
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
18
0
    std::cerr << "output differs between implementations\n";
19
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
20
0
      std::cerr << "implementation " << implementations[i] << " gave "
21
0
                << results.at(i) << '\n';
22
0
    }
23
0
    std::abort();
24
0
  }
25
898
}
26
27
650
void detect(std::span<const char> chardata) {
28
650
  std::vector<int> results;
29
650
  const auto implementations = get_supported_implementations();
30
1.95k
  for (const simdutf::implementation* impl : implementations) {
31
1.95k
    results.push_back(impl->detect_encodings(chardata.data(), chardata.size()));
32
1.95k
  }
33
1.30k
  auto neq = [](const auto& a, const auto& b) { return a != b; };
34
650
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
35
0
    std::cerr << "in detect_encodings(const char*, std::size_t):\n";
36
0
    std::cerr << "output differs between implementations\n";
37
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
38
0
      std::cerr << "implementation " << implementations[i]->name() << " gave "
39
0
                << results.at(i) << '\n';
40
0
    }
41
0
    std::cerr << " std::vector<unsigned char> data{";
42
0
    for (unsigned char x : chardata) {
43
0
      std::cerr << +x << ", ";
44
0
    };
45
0
    std::cerr << "};\n";
46
0
    std::abort();
47
0
  }
48
650
}
49
50
201
void validate_ascii(std::span<const char> chardata) {
51
  // use int, not bool to avoid vector<bool>
52
201
  std::vector<int> results;
53
201
  const auto implementations = get_supported_implementations();
54
603
  for (const simdutf::implementation* impl : implementations) {
55
603
    results.push_back(+impl->validate_ascii(chardata.data(), chardata.size()));
56
603
  }
57
402
  auto neq = [](const auto& a, const auto& b) { return a != b; };
58
201
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
59
0
    std::cerr << "in validate_ascii(const char*, std::size_t):\n";
60
0
    std::cerr << "output differs between implementations\n";
61
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
62
0
      std::cerr << "implementation " << implementations[i]->name() << " gave "
63
0
                << results.at(i) << '\n';
64
0
    }
65
0
    std::cerr << " std::vector<unsigned char> data{";
66
0
    for (unsigned char x : chardata) {
67
0
      std::cerr << +x << ", ";
68
0
    };
69
0
    std::cerr << "};\n";
70
0
    std::abort();
71
0
  }
72
201
}
73
74
285
void validate_ascii_with_err(std::span<const char> chardata) {
75
  // use int, not bool to avoid vector<bool>
76
285
  std::vector<simdutf::result> results;
77
285
  const auto implementations = get_supported_implementations();
78
855
  for (const simdutf::implementation* impl : implementations) {
79
855
    results.push_back(
80
855
        impl->validate_ascii_with_errors(chardata.data(), chardata.size()));
81
855
  }
82
570
  auto neq = [](const auto& a, const auto& b) { return a != b; };
83
285
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
84
0
    std::cerr << "in validate_ascii(const char*, std::size_t):\n";
85
0
    std::cerr << "output differs between implementations\n";
86
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
87
0
      std::cerr << "implementation " << implementations[i]->name() << " gave "
88
0
                << results.at(i) << '\n';
89
0
    }
90
0
    std::cerr << " std::vector<unsigned char> data{";
91
0
    for (unsigned char x : chardata) {
92
0
      std::cerr << +x << ", ";
93
0
    };
94
0
    std::cerr << "};\n";
95
0
    std::abort();
96
0
  }
97
285
}
98
99
115
void utf16_endianness(std::span<const char16_t> data) {
100
115
  std::vector<std::string> results;
101
115
  const auto implementations = get_supported_implementations();
102
345
  for (const simdutf::implementation* impl : implementations) {
103
345
    std::vector<char16_t> out(data.size());
104
345
    impl->change_endianness_utf16(data.data(), data.size(), out.data());
105
345
    results.push_back(FNV1A_hash::as_str(out));
106
345
  }
107
230
  auto neq = [](const auto& a, const auto& b) { return a != b; };
108
115
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
109
0
    std::cerr << "in utf16_endianness(const char*, std::size_t):\n";
110
0
    std::cerr << "output differs between implementations\n";
111
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
112
0
      std::cerr << "implementation " << implementations[i]->name() << " gave "
113
0
                << results.at(i) << '\n';
114
0
    }
115
0
    std::cerr << " std::vector<char16_t> data{";
116
0
    for (int x : data) {
117
0
      std::cerr << +x << ", ";
118
0
    };
119
0
    std::cerr << "};\n";
120
0
    std::abort();
121
0
  }
122
115
}
123
124
// Checks that validate_utf16le_as_ascii and validate_utf16be_as_ascii agree
125
// across all implementations, and that a true result implies valid UTF-16.
126
206
void validate_utf16_as_ascii(std::span<const char16_t> data) {
127
206
  const auto implementations = get_supported_implementations();
128
  // use int, not bool to avoid vector<bool>
129
206
  std::vector<int> le_results, be_results;
130
206
  le_results.reserve(implementations.size());
131
206
  be_results.reserve(implementations.size());
132
618
  for (const simdutf::implementation* impl : implementations) {
133
618
    le_results.push_back(
134
618
        +impl->validate_utf16le_as_ascii(data.data(), data.size()));
135
618
    be_results.push_back(
136
618
        +impl->validate_utf16be_as_ascii(data.data(), data.size()));
137
618
  }
138
824
  auto neq = [](const auto& a, const auto& b) { return a != b; };
139
206
  if (std::ranges::adjacent_find(le_results, neq) != le_results.end()) {
140
0
    std::cerr << "validate_utf16le_as_ascii: output differs between "
141
0
                 "implementations\n";
142
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
143
0
      std::cerr << "  " << implementations[i]->name() << " gave "
144
0
                << le_results[i] << '\n';
145
0
    }
146
0
    std::abort();
147
0
  }
148
206
  if (std::ranges::adjacent_find(be_results, neq) != be_results.end()) {
149
0
    std::cerr << "validate_utf16be_as_ascii: output differs between "
150
0
                 "implementations\n";
151
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
152
0
      std::cerr << "  " << implementations[i]->name() << " gave "
153
0
                << be_results[i] << '\n';
154
0
    }
155
0
    std::abort();
156
0
  }
157
  // If LE validates as ASCII, it must also validate as UTF-16LE (ASCII is a
158
  // subset).
159
206
  if (le_results[0]) {
160
81
    for (const simdutf::implementation* impl : implementations) {
161
81
      if (!impl->validate_utf16le(data.data(), data.size())) {
162
0
        std::cerr << "validate_utf16le_as_ascii returned true but "
163
0
                     "validate_utf16le returned false"
164
0
                  << " impl=" << impl->name() << "\n";
165
0
        std::abort();
166
0
      }
167
81
    }
168
27
  }
169
  // Same for BE.
170
206
  if (be_results[0]) {
171
81
    for (const simdutf::implementation* impl : implementations) {
172
81
      if (!impl->validate_utf16be(data.data(), data.size())) {
173
0
        std::cerr << "validate_utf16be_as_ascii returned true but "
174
0
                     "validate_utf16be returned false"
175
0
                  << " impl=" << impl->name() << "\n";
176
0
        std::abort();
177
0
      }
178
81
    }
179
27
  }
180
206
}
181
182
// Checks that to_well_formed_utf16le / to_well_formed_utf16be:
183
// 1. All implementations agree on the output.
184
// 2. The output is always valid UTF-16LE / UTF-16BE.
185
// 3. When the input is already valid UTF-16, the output equals the input.
186
623
void to_well_formed_utf16(std::span<const char16_t> data) {
187
623
  const auto implementations = get_supported_implementations();
188
623
  if (implementations.empty()) {
189
0
    return;
190
0
  }
191
192
  // Check LE variant
193
623
  {
194
623
    std::vector<std::vector<char16_t>> le_outputs;
195
623
    le_outputs.reserve(implementations.size());
196
1.86k
    for (const simdutf::implementation* impl : implementations) {
197
1.86k
      std::vector<char16_t> out(data.size());
198
1.86k
      impl->to_well_formed_utf16le(data.data(), data.size(), out.data());
199
1.86k
      le_outputs.push_back(std::move(out));
200
1.86k
    }
201
1.24k
    auto neq = [](const auto& a, const auto& b) { return a != b; };
202
623
    if (std::ranges::adjacent_find(le_outputs, neq) != le_outputs.end()) {
203
0
      std::cerr
204
0
          << "to_well_formed_utf16le: outputs differ between implementations\n";
205
0
      for (std::size_t i = 0; i < implementations.size(); ++i) {
206
0
        std::cerr << "  " << implementations[i]->name()
207
0
                  << ": hash=" << FNV1A_hash::as_str(le_outputs[i]) << "\n";
208
0
      }
209
0
      std::abort();
210
0
    }
211
    // Output must be valid UTF-16LE.
212
2.49k
    for (std::size_t i = 0; i < implementations.size(); ++i) {
213
1.86k
      if (!implementations[i]->validate_utf16le(le_outputs[i].data(),
214
1.86k
                                                le_outputs[i].size())) {
215
0
        std::cerr << "to_well_formed_utf16le: output is not valid UTF-16LE"
216
0
                  << " impl=" << implementations[i]->name() << "\n";
217
0
        std::abort();
218
0
      }
219
1.86k
    }
220
    // If input was already valid UTF-16LE, output must equal input.
221
623
    if (implementations[0]->validate_utf16le(data.data(), data.size())) {
222
203
      if (!std::ranges::equal(le_outputs[0], data)) {
223
0
        std::cerr << "to_well_formed_utf16le: valid input was modified\n";
224
0
        std::abort();
225
0
      }
226
203
    }
227
623
  }
228
229
  // Check BE variant
230
623
  {
231
623
    std::vector<std::vector<char16_t>> be_outputs;
232
623
    be_outputs.reserve(implementations.size());
233
1.86k
    for (const simdutf::implementation* impl : implementations) {
234
1.86k
      std::vector<char16_t> out(data.size());
235
1.86k
      impl->to_well_formed_utf16be(data.data(), data.size(), out.data());
236
1.86k
      be_outputs.push_back(std::move(out));
237
1.86k
    }
238
1.24k
    auto neq = [](const auto& a, const auto& b) { return a != b; };
239
623
    if (std::ranges::adjacent_find(be_outputs, neq) != be_outputs.end()) {
240
0
      std::cerr
241
0
          << "to_well_formed_utf16be: outputs differ between implementations\n";
242
0
      for (std::size_t i = 0; i < implementations.size(); ++i) {
243
0
        std::cerr << "  " << implementations[i]->name()
244
0
                  << ": hash=" << FNV1A_hash::as_str(be_outputs[i]) << "\n";
245
0
      }
246
0
      std::abort();
247
0
    }
248
    // Output must be valid UTF-16BE.
249
2.49k
    for (std::size_t i = 0; i < implementations.size(); ++i) {
250
1.86k
      if (!implementations[i]->validate_utf16be(be_outputs[i].data(),
251
1.86k
                                                be_outputs[i].size())) {
252
0
        std::cerr << "to_well_formed_utf16be: output is not valid UTF-16BE"
253
0
                  << " impl=" << implementations[i]->name() << "\n";
254
0
        std::abort();
255
0
      }
256
1.86k
    }
257
    // If input was already valid UTF-16BE, output must equal input.
258
623
    if (implementations[0]->validate_utf16be(data.data(), data.size())) {
259
203
      if (!std::ranges::equal(be_outputs[0], data)) {
260
0
        std::cerr << "to_well_formed_utf16be: valid input was modified\n";
261
0
        std::abort();
262
0
      }
263
203
    }
264
623
  }
265
623
}
266
267
void convert_latin1_to_utf8_safe(std::span<const char> chardata,
268
248
                                 const std::size_t outputsize) {
269
  // convert with a limited output buffer
270
248
  std::vector<char> limited_output(outputsize);
271
248
  const auto limited_ret = simdutf::convert_latin1_to_utf8_safe(
272
248
      chardata.data(), chardata.size(), limited_output.data(), outputsize);
273
274
  // convert with a sufficiently large output buffer
275
248
  std::vector<char> large_output(2 * chardata.size());
276
248
  const auto large_ret = simdutf::convert_latin1_to_utf8(
277
248
      chardata.data(), chardata.size(), large_output.data());
278
279
248
  if (large_ret != 0) {
280
    // conversion was possible with a large buffer.
281
224
    if (large_ret <= outputsize) {
282
      // the limited buffer was large enough, ensure we got the same result
283
106
      assert(limited_ret == large_ret);
284
106
      assert(std::ranges::equal(limited_output | std::views::take(large_ret),
285
106
                                large_output | std::views::take(large_ret)));
286
118
    } else {
287
      // the number of written bytes for a limited buffer must not exceed what
288
      // the large buffer got.
289
118
      assert(limited_ret <= large_ret);
290
      // the written data should be equal
291
118
      assert(std::ranges::equal(limited_output | std::views::take(limited_ret),
292
118
                                large_output | std::views::take(limited_ret)));
293
118
    }
294
224
  } else {
295
    // conversion with a big buffer failed - is there anything we can check or
296
    // assert for the limited buffer? I don't think so.
297
24
  }
298
248
}
299
300
3.38k
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
301
  // pick one of the functions, based on the fuzz data.
302
  // the first byte is which action to take. step forward
303
  // several bytes so the input is aligned.
304
3.38k
  if (size < 4) {
305
2
    return 0;
306
2
  }
307
3.38k
  constexpr auto Ncases = 11u;
308
3.38k
  constexpr auto actionmask = std::bit_ceil(Ncases) - 1;
309
3.38k
  const auto action = data[0] & actionmask;
310
311
3.38k
  const std::uint16_t u16 = data[1] + (data[2] << 8);
312
313
3.38k
  data += 4;
314
3.38k
  size -= 4;
315
316
3.38k
  const std::span<const char> chardata{(const char*)data, size};
317
3.38k
  const std::span<const char16_t> u16data{(const char16_t*)data,
318
3.38k
                                          size / sizeof(char16_t)};
319
320
3.38k
  switch (action) {
321
898
  case 0:
322
898
    autodetect(chardata);
323
898
    break;
324
650
  case 1:
325
650
    detect(chardata);
326
650
    break;
327
201
  case 2:
328
201
    validate_ascii(chardata);
329
201
    break;
330
285
  case 3:
331
285
    validate_ascii_with_err(chardata);
332
285
    break;
333
115
  case 4:
334
115
    utf16_endianness(u16data);
335
115
    break;
336
45
  case 5: {
337
45
    [[maybe_unused]] auto ret =
338
45
        simdutf::trim_partial_utf16le(u16data.data(), u16data.size());
339
45
    assert(ret == u16data.size() || ret + 1 == u16data.size());
340
45
  } break;
341
45
  case 6: {
342
19
    [[maybe_unused]] auto ret =
343
19
        simdutf::trim_partial_utf16be(u16data.data(), u16data.size());
344
19
    assert(ret == u16data.size() || ret + 1 == u16data.size());
345
19
  } break;
346
95
  case 7: {
347
95
    [[maybe_unused]] const std::size_t N = chardata.size();
348
95
    [[maybe_unused]] const auto ret =
349
95
        simdutf::trim_partial_utf8(chardata.data(), chardata.size());
350
95
    if ((ret + 3 < N) || (ret > N)) {
351
0
      std::cerr << "ret=" << ret << " N=" << N << '\n';
352
0
      std::abort();
353
0
    }
354
95
  } break;
355
248
  case 8:
356
248
    convert_latin1_to_utf8_safe(chardata, u16);
357
248
    break;
358
206
  case 9:
359
206
    validate_utf16_as_ascii(u16data);
360
206
    break;
361
623
  case 10:
362
623
    to_well_formed_utf16(u16data);
363
623
    break;
364
3.38k
  }
365
3.38k
  return 0;
366
3.38k
}