Coverage Report

Created: 2026-08-13 06:24

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/simdutf/fuzz/misc.cpp
Line
Count
Source
1
#include <cstddef>
2
#include <cstdint>
3
#include <ranges>
4
#include <cstdlib>
5
6
#include "helpers/common.h"
7
#include "simdutf.h"
8
9
718
void autodetect(std::span<const char> chardata) {
10
718
  std::vector<simdutf::encoding_type> results;
11
718
  const auto implementations = get_supported_implementations();
12
2.15k
  for (const simdutf::implementation* impl : implementations) {
13
2.15k
    results.push_back(
14
2.15k
        impl->autodetect_encoding(chardata.data(), chardata.size()));
15
2.15k
  }
16
1.43k
  auto neq = [](const auto& a, const auto& b) { return a != b; };
17
718
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
18
0
    std::cerr << "output differs between implementations\n";
19
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
20
0
      std::cerr << "implementation " << implementations[i] << " gave "
21
0
                << results.at(i) << '\n';
22
0
    }
23
0
    std::abort();
24
0
  }
25
718
}
26
27
636
void detect(std::span<const char> chardata) {
28
636
  std::vector<int> results;
29
636
  const auto implementations = get_supported_implementations();
30
1.90k
  for (const simdutf::implementation* impl : implementations) {
31
1.90k
    results.push_back(impl->detect_encodings(chardata.data(), chardata.size()));
32
1.90k
  }
33
1.27k
  auto neq = [](const auto& a, const auto& b) { return a != b; };
34
636
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
35
0
    std::cerr << "in detect_encodings(const char*, std::size_t):\n";
36
0
    std::cerr << "output differs between implementations\n";
37
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
38
0
      std::cerr << "implementation " << implementations[i]->name() << " gave "
39
0
                << results.at(i) << '\n';
40
0
    }
41
0
    std::cerr << " std::vector<unsigned char> data{";
42
0
    for (unsigned char x : chardata) {
43
0
      std::cerr << +x << ", ";
44
0
    };
45
0
    std::cerr << "};\n";
46
0
    std::abort();
47
0
  }
48
636
}
49
50
190
void validate_ascii(std::span<const char> chardata) {
51
  // use int, not bool to avoid vector<bool>
52
190
  std::vector<int> results;
53
190
  const auto implementations = get_supported_implementations();
54
570
  for (const simdutf::implementation* impl : implementations) {
55
570
    results.push_back(+impl->validate_ascii(chardata.data(), chardata.size()));
56
570
  }
57
380
  auto neq = [](const auto& a, const auto& b) { return a != b; };
58
190
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
59
0
    std::cerr << "in validate_ascii(const char*, std::size_t):\n";
60
0
    std::cerr << "output differs between implementations\n";
61
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
62
0
      std::cerr << "implementation " << implementations[i]->name() << " gave "
63
0
                << results.at(i) << '\n';
64
0
    }
65
0
    std::cerr << " std::vector<unsigned char> data{";
66
0
    for (unsigned char x : chardata) {
67
0
      std::cerr << +x << ", ";
68
0
    };
69
0
    std::cerr << "};\n";
70
0
    std::abort();
71
0
  }
72
190
}
73
74
274
void validate_ascii_with_err(std::span<const char> chardata) {
75
  // use int, not bool to avoid vector<bool>
76
274
  std::vector<simdutf::result> results;
77
274
  const auto implementations = get_supported_implementations();
78
822
  for (const simdutf::implementation* impl : implementations) {
79
822
    results.push_back(
80
822
        impl->validate_ascii_with_errors(chardata.data(), chardata.size()));
81
822
  }
82
548
  auto neq = [](const auto& a, const auto& b) { return a != b; };
83
274
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
84
0
    std::cerr << "in validate_ascii(const char*, std::size_t):\n";
85
0
    std::cerr << "output differs between implementations\n";
86
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
87
0
      std::cerr << "implementation " << implementations[i]->name() << " gave "
88
0
                << results.at(i) << '\n';
89
0
    }
90
0
    std::cerr << " std::vector<unsigned char> data{";
91
0
    for (unsigned char x : chardata) {
92
0
      std::cerr << +x << ", ";
93
0
    };
94
0
    std::cerr << "};\n";
95
0
    std::abort();
96
0
  }
97
274
}
98
99
125
void utf16_endianness(std::span<const char16_t> data) {
100
125
  std::vector<std::string> results;
101
125
  const auto implementations = get_supported_implementations();
102
375
  for (const simdutf::implementation* impl : implementations) {
103
375
    std::vector<char16_t> out(data.size());
104
375
    impl->change_endianness_utf16(data.data(), data.size(), out.data());
105
375
    results.push_back(FNV1A_hash::as_str(out));
106
375
  }
107
250
  auto neq = [](const auto& a, const auto& b) { return a != b; };
108
125
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
109
0
    std::cerr << "in utf16_endianness(const char*, std::size_t):\n";
110
0
    std::cerr << "output differs between implementations\n";
111
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
112
0
      std::cerr << "implementation " << implementations[i]->name() << " gave "
113
0
                << results.at(i) << '\n';
114
0
    }
115
0
    std::cerr << " std::vector<char16_t> data{";
116
0
    for (int x : data) {
117
0
      std::cerr << +x << ", ";
118
0
    };
119
0
    std::cerr << "};\n";
120
0
    std::abort();
121
0
  }
122
125
}
123
124
// Checks that validate_utf16le_as_ascii and validate_utf16be_as_ascii agree
125
// across all implementations, and that a true result implies valid UTF-16.
126
230
void validate_utf16_as_ascii(std::span<const char16_t> data) {
127
230
  const auto implementations = get_supported_implementations();
128
  // use int, not bool to avoid vector<bool>
129
230
  std::vector<int> le_results, be_results;
130
230
  le_results.reserve(implementations.size());
131
230
  be_results.reserve(implementations.size());
132
690
  for (const simdutf::implementation* impl : implementations) {
133
690
    le_results.push_back(
134
690
        +impl->validate_utf16le_as_ascii(data.data(), data.size()));
135
690
    be_results.push_back(
136
690
        +impl->validate_utf16be_as_ascii(data.data(), data.size()));
137
690
  }
138
920
  auto neq = [](const auto& a, const auto& b) { return a != b; };
139
230
  if (std::ranges::adjacent_find(le_results, neq) != le_results.end()) {
140
0
    std::cerr << "validate_utf16le_as_ascii: output differs between "
141
0
                 "implementations\n";
142
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
143
0
      std::cerr << "  " << implementations[i]->name() << " gave "
144
0
                << le_results[i] << '\n';
145
0
    }
146
0
    std::abort();
147
0
  }
148
230
  if (std::ranges::adjacent_find(be_results, neq) != be_results.end()) {
149
0
    std::cerr << "validate_utf16be_as_ascii: output differs between "
150
0
                 "implementations\n";
151
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
152
0
      std::cerr << "  " << implementations[i]->name() << " gave "
153
0
                << be_results[i] << '\n';
154
0
    }
155
0
    std::abort();
156
0
  }
157
  // If LE validates as ASCII, it must also validate as UTF-16LE (ASCII is a
158
  // subset).
159
230
  if (le_results[0]) {
160
63
    for (const simdutf::implementation* impl : implementations) {
161
63
      if (!impl->validate_utf16le(data.data(), data.size())) {
162
0
        std::cerr << "validate_utf16le_as_ascii returned true but "
163
0
                     "validate_utf16le returned false"
164
0
                  << " impl=" << impl->name() << "\n";
165
0
        std::abort();
166
0
      }
167
63
    }
168
21
  }
169
  // Same for BE.
170
230
  if (be_results[0]) {
171
78
    for (const simdutf::implementation* impl : implementations) {
172
78
      if (!impl->validate_utf16be(data.data(), data.size())) {
173
0
        std::cerr << "validate_utf16be_as_ascii returned true but "
174
0
                     "validate_utf16be returned false"
175
0
                  << " impl=" << impl->name() << "\n";
176
0
        std::abort();
177
0
      }
178
78
    }
179
26
  }
180
230
}
181
182
// Checks that to_well_formed_utf16le / to_well_formed_utf16be:
183
// 1. All implementations agree on the output.
184
// 2. The output is always valid UTF-16LE / UTF-16BE.
185
// 3. When the input is already valid UTF-16, the output equals the input.
186
719
void to_well_formed_utf16(std::span<const char16_t> data) {
187
719
  const auto implementations = get_supported_implementations();
188
719
  if (implementations.empty()) {
189
0
    return;
190
0
  }
191
192
  // Check LE variant
193
719
  {
194
719
    std::vector<std::vector<char16_t>> le_outputs;
195
719
    le_outputs.reserve(implementations.size());
196
2.15k
    for (const simdutf::implementation* impl : implementations) {
197
2.15k
      std::vector<char16_t> out(data.size());
198
2.15k
      impl->to_well_formed_utf16le(data.data(), data.size(), out.data());
199
2.15k
      le_outputs.push_back(std::move(out));
200
2.15k
    }
201
1.43k
    auto neq = [](const auto& a, const auto& b) { return a != b; };
202
719
    if (std::ranges::adjacent_find(le_outputs, neq) != le_outputs.end()) {
203
0
      std::cerr
204
0
          << "to_well_formed_utf16le: outputs differ between implementations\n";
205
0
      for (std::size_t i = 0; i < implementations.size(); ++i) {
206
0
        std::cerr << "  " << implementations[i]->name()
207
0
                  << ": hash=" << FNV1A_hash::as_str(le_outputs[i]) << "\n";
208
0
      }
209
0
      std::abort();
210
0
    }
211
    // Output must be valid UTF-16LE.
212
2.87k
    for (std::size_t i = 0; i < implementations.size(); ++i) {
213
2.15k
      if (!implementations[i]->validate_utf16le(le_outputs[i].data(),
214
2.15k
                                                le_outputs[i].size())) {
215
0
        std::cerr << "to_well_formed_utf16le: output is not valid UTF-16LE"
216
0
                  << " impl=" << implementations[i]->name() << "\n";
217
0
        std::abort();
218
0
      }
219
2.15k
    }
220
    // If input was already valid UTF-16LE, output must equal input.
221
719
    if (implementations[0]->validate_utf16le(data.data(), data.size())) {
222
232
      if (!std::ranges::equal(le_outputs[0], data)) {
223
0
        std::cerr << "to_well_formed_utf16le: valid input was modified\n";
224
0
        std::abort();
225
0
      }
226
232
    }
227
719
  }
228
229
  // Check BE variant
230
719
  {
231
719
    std::vector<std::vector<char16_t>> be_outputs;
232
719
    be_outputs.reserve(implementations.size());
233
2.15k
    for (const simdutf::implementation* impl : implementations) {
234
2.15k
      std::vector<char16_t> out(data.size());
235
2.15k
      impl->to_well_formed_utf16be(data.data(), data.size(), out.data());
236
2.15k
      be_outputs.push_back(std::move(out));
237
2.15k
    }
238
1.43k
    auto neq = [](const auto& a, const auto& b) { return a != b; };
239
719
    if (std::ranges::adjacent_find(be_outputs, neq) != be_outputs.end()) {
240
0
      std::cerr
241
0
          << "to_well_formed_utf16be: outputs differ between implementations\n";
242
0
      for (std::size_t i = 0; i < implementations.size(); ++i) {
243
0
        std::cerr << "  " << implementations[i]->name()
244
0
                  << ": hash=" << FNV1A_hash::as_str(be_outputs[i]) << "\n";
245
0
      }
246
0
      std::abort();
247
0
    }
248
    // Output must be valid UTF-16BE.
249
2.87k
    for (std::size_t i = 0; i < implementations.size(); ++i) {
250
2.15k
      if (!implementations[i]->validate_utf16be(be_outputs[i].data(),
251
2.15k
                                                be_outputs[i].size())) {
252
0
        std::cerr << "to_well_formed_utf16be: output is not valid UTF-16BE"
253
0
                  << " impl=" << implementations[i]->name() << "\n";
254
0
        std::abort();
255
0
      }
256
2.15k
    }
257
    // If input was already valid UTF-16BE, output must equal input.
258
719
    if (implementations[0]->validate_utf16be(data.data(), data.size())) {
259
224
      if (!std::ranges::equal(be_outputs[0], data)) {
260
0
        std::cerr << "to_well_formed_utf16be: valid input was modified\n";
261
0
        std::abort();
262
0
      }
263
224
    }
264
719
  }
265
719
}
266
267
void convert_latin1_to_utf8_safe(std::span<const char> chardata,
268
270
                                 const std::size_t outputsize) {
269
  // convert with a limited output buffer
270
270
  std::vector<char> limited_output(outputsize);
271
270
  const auto limited_ret = simdutf::convert_latin1_to_utf8_safe(
272
270
      chardata.data(), chardata.size(), limited_output.data(), outputsize);
273
274
  // convert with a sufficiently large output buffer
275
270
  std::vector<char> large_output(2 * chardata.size());
276
270
  const auto large_ret = simdutf::convert_latin1_to_utf8(
277
270
      chardata.data(), chardata.size(), large_output.data());
278
279
270
  if (large_ret != 0) {
280
    // conversion was possible with a large buffer.
281
245
    if (large_ret <= outputsize) {
282
      // the limited buffer was large enough, ensure we got the same result
283
111
      assert(limited_ret == large_ret);
284
111
      assert(std::ranges::equal(limited_output | std::views::take(large_ret),
285
111
                                large_output | std::views::take(large_ret)));
286
134
    } else {
287
      // the number of written bytes for a limited buffer must not exceed what
288
      // the large buffer got.
289
134
      assert(limited_ret <= large_ret);
290
      // the written data should be equal
291
134
      assert(std::ranges::equal(limited_output | std::views::take(limited_ret),
292
134
                                large_output | std::views::take(limited_ret)));
293
134
    }
294
245
  } else {
295
    // conversion with a big buffer failed - is there anything we can check or
296
    // assert for the limited buffer? I don't think so.
297
25
  }
298
270
}
299
300
3.35k
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
301
  // pick one of the functions, based on the fuzz data.
302
  // the first byte is which action to take. step forward
303
  // several bytes so the input is aligned.
304
3.35k
  if (size < 4) {
305
2
    return 0;
306
2
  }
307
3.35k
  constexpr auto Ncases = 11u;
308
3.35k
  constexpr auto actionmask = std::bit_ceil(Ncases) - 1;
309
3.35k
  const auto action = data[0] & actionmask;
310
311
3.35k
  const std::uint16_t u16 = data[1] + (data[2] << 8);
312
313
3.35k
  data += 4;
314
3.35k
  size -= 4;
315
316
3.35k
  const std::span<const char> chardata{(const char*)data, size};
317
3.35k
  const std::span<const char16_t> u16data{(const char16_t*)data,
318
3.35k
                                          size / sizeof(char16_t)};
319
320
3.35k
  switch (action) {
321
718
  case 0:
322
718
    autodetect(chardata);
323
718
    break;
324
636
  case 1:
325
636
    detect(chardata);
326
636
    break;
327
190
  case 2:
328
190
    validate_ascii(chardata);
329
190
    break;
330
274
  case 3:
331
274
    validate_ascii_with_err(chardata);
332
274
    break;
333
125
  case 4:
334
125
    utf16_endianness(u16data);
335
125
    break;
336
46
  case 5: {
337
46
    [[maybe_unused]] auto ret =
338
46
        simdutf::trim_partial_utf16le(u16data.data(), u16data.size());
339
46
    assert(ret == u16data.size() || ret + 1 == u16data.size());
340
46
  } break;
341
47
  case 6: {
342
47
    [[maybe_unused]] auto ret =
343
47
        simdutf::trim_partial_utf16be(u16data.data(), u16data.size());
344
47
    assert(ret == u16data.size() || ret + 1 == u16data.size());
345
47
  } break;
346
101
  case 7: {
347
101
    [[maybe_unused]] const std::size_t N = chardata.size();
348
101
    [[maybe_unused]] const auto ret =
349
101
        simdutf::trim_partial_utf8(chardata.data(), chardata.size());
350
101
    if ((ret + 3 < N) || (ret > N)) {
351
0
      std::cerr << "ret=" << ret << " N=" << N << '\n';
352
0
      std::abort();
353
0
    }
354
101
  } break;
355
270
  case 8:
356
270
    convert_latin1_to_utf8_safe(chardata, u16);
357
270
    break;
358
230
  case 9:
359
230
    validate_utf16_as_ascii(u16data);
360
230
    break;
361
719
  case 10:
362
719
    to_well_formed_utf16(u16data);
363
719
    break;
364
3.35k
  }
365
3.35k
  return 0;
366
3.35k
}