Coverage Report

Created: 2026-07-16 06:41

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/simdutf/fuzz/misc.cpp
Line
Count
Source
1
#include <cstddef>
2
#include <cstdint>
3
#include <ranges>
4
5
#include "helpers/common.h"
6
#include "simdutf.h"
7
8
693
void autodetect(std::span<const char> chardata) {
9
693
  std::vector<simdutf::encoding_type> results;
10
693
  const auto implementations = get_supported_implementations();
11
2.07k
  for (const simdutf::implementation* impl : implementations) {
12
2.07k
    results.push_back(
13
2.07k
        impl->autodetect_encoding(chardata.data(), chardata.size()));
14
2.07k
  }
15
1.38k
  auto neq = [](const auto& a, const auto& b) { return a != b; };
16
693
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
17
0
    std::cerr << "output differs between implementations\n";
18
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
19
0
      std::cerr << "implementation " << implementations[i] << " gave "
20
0
                << results.at(i) << '\n';
21
0
    }
22
0
    std::abort();
23
0
  }
24
693
}
25
26
663
void detect(std::span<const char> chardata) {
27
663
  std::vector<int> results;
28
663
  const auto implementations = get_supported_implementations();
29
1.98k
  for (const simdutf::implementation* impl : implementations) {
30
1.98k
    results.push_back(impl->detect_encodings(chardata.data(), chardata.size()));
31
1.98k
  }
32
1.32k
  auto neq = [](const auto& a, const auto& b) { return a != b; };
33
663
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
34
0
    std::cerr << "in detect_encodings(const char*, std::size_t):\n";
35
0
    std::cerr << "output differs between implementations\n";
36
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
37
0
      std::cerr << "implementation " << implementations[i]->name() << " gave "
38
0
                << results.at(i) << '\n';
39
0
    }
40
0
    std::cerr << " std::vector<unsigned char> data{";
41
0
    for (unsigned char x : chardata) {
42
0
      std::cerr << +x << ", ";
43
0
    };
44
0
    std::cerr << "};\n";
45
0
    std::abort();
46
0
  }
47
663
}
48
49
208
void validate_ascii(std::span<const char> chardata) {
50
  // use int, not bool to avoid vector<bool>
51
208
  std::vector<int> results;
52
208
  const auto implementations = get_supported_implementations();
53
624
  for (const simdutf::implementation* impl : implementations) {
54
624
    results.push_back(+impl->validate_ascii(chardata.data(), chardata.size()));
55
624
  }
56
416
  auto neq = [](const auto& a, const auto& b) { return a != b; };
57
208
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
58
0
    std::cerr << "in validate_ascii(const char*, std::size_t):\n";
59
0
    std::cerr << "output differs between implementations\n";
60
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
61
0
      std::cerr << "implementation " << implementations[i]->name() << " gave "
62
0
                << results.at(i) << '\n';
63
0
    }
64
0
    std::cerr << " std::vector<unsigned char> data{";
65
0
    for (unsigned char x : chardata) {
66
0
      std::cerr << +x << ", ";
67
0
    };
68
0
    std::cerr << "};\n";
69
0
    std::abort();
70
0
  }
71
208
}
72
73
287
void validate_ascii_with_err(std::span<const char> chardata) {
74
  // use int, not bool to avoid vector<bool>
75
287
  std::vector<simdutf::result> results;
76
287
  const auto implementations = get_supported_implementations();
77
861
  for (const simdutf::implementation* impl : implementations) {
78
861
    results.push_back(
79
861
        impl->validate_ascii_with_errors(chardata.data(), chardata.size()));
80
861
  }
81
574
  auto neq = [](const auto& a, const auto& b) { return a != b; };
82
287
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
83
0
    std::cerr << "in validate_ascii(const char*, std::size_t):\n";
84
0
    std::cerr << "output differs between implementations\n";
85
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
86
0
      std::cerr << "implementation " << implementations[i]->name() << " gave "
87
0
                << results.at(i) << '\n';
88
0
    }
89
0
    std::cerr << " std::vector<unsigned char> data{";
90
0
    for (unsigned char x : chardata) {
91
0
      std::cerr << +x << ", ";
92
0
    };
93
0
    std::cerr << "};\n";
94
0
    std::abort();
95
0
  }
96
287
}
97
98
133
void utf16_endianness(std::span<const char16_t> data) {
99
133
  std::vector<std::string> results;
100
133
  const auto implementations = get_supported_implementations();
101
399
  for (const simdutf::implementation* impl : implementations) {
102
399
    std::vector<char16_t> out(data.size());
103
399
    impl->change_endianness_utf16(data.data(), data.size(), out.data());
104
399
    results.push_back(FNV1A_hash::as_str(out));
105
399
  }
106
266
  auto neq = [](const auto& a, const auto& b) { return a != b; };
107
133
  if (std::ranges::adjacent_find(results, neq) != results.end()) {
108
0
    std::cerr << "in utf16_endianness(const char*, std::size_t):\n";
109
0
    std::cerr << "output differs between implementations\n";
110
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
111
0
      std::cerr << "implementation " << implementations[i]->name() << " gave "
112
0
                << results.at(i) << '\n';
113
0
    }
114
0
    std::cerr << " std::vector<char16_t> data{";
115
0
    for (int x : data) {
116
0
      std::cerr << +x << ", ";
117
0
    };
118
0
    std::cerr << "};\n";
119
0
    std::abort();
120
0
  }
121
133
}
122
123
// Checks that validate_utf16le_as_ascii and validate_utf16be_as_ascii agree
124
// across all implementations, and that a true result implies valid UTF-16.
125
230
void validate_utf16_as_ascii(std::span<const char16_t> data) {
126
230
  const auto implementations = get_supported_implementations();
127
  // use int, not bool to avoid vector<bool>
128
230
  std::vector<int> le_results, be_results;
129
230
  le_results.reserve(implementations.size());
130
230
  be_results.reserve(implementations.size());
131
690
  for (const simdutf::implementation* impl : implementations) {
132
690
    le_results.push_back(
133
690
        +impl->validate_utf16le_as_ascii(data.data(), data.size()));
134
690
    be_results.push_back(
135
690
        +impl->validate_utf16be_as_ascii(data.data(), data.size()));
136
690
  }
137
920
  auto neq = [](const auto& a, const auto& b) { return a != b; };
138
230
  if (std::ranges::adjacent_find(le_results, neq) != le_results.end()) {
139
0
    std::cerr << "validate_utf16le_as_ascii: output differs between "
140
0
                 "implementations\n";
141
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
142
0
      std::cerr << "  " << implementations[i]->name() << " gave "
143
0
                << le_results[i] << '\n';
144
0
    }
145
0
    std::abort();
146
0
  }
147
230
  if (std::ranges::adjacent_find(be_results, neq) != be_results.end()) {
148
0
    std::cerr << "validate_utf16be_as_ascii: output differs between "
149
0
                 "implementations\n";
150
0
    for (std::size_t i = 0; i < implementations.size(); ++i) {
151
0
      std::cerr << "  " << implementations[i]->name() << " gave "
152
0
                << be_results[i] << '\n';
153
0
    }
154
0
    std::abort();
155
0
  }
156
  // If LE validates as ASCII, it must also validate as UTF-16LE (ASCII is a
157
  // subset).
158
230
  if (le_results[0]) {
159
66
    for (const simdutf::implementation* impl : implementations) {
160
66
      if (!impl->validate_utf16le(data.data(), data.size())) {
161
0
        std::cerr << "validate_utf16le_as_ascii returned true but "
162
0
                     "validate_utf16le returned false"
163
0
                  << " impl=" << impl->name() << "\n";
164
0
        std::abort();
165
0
      }
166
66
    }
167
22
  }
168
  // Same for BE.
169
230
  if (be_results[0]) {
170
72
    for (const simdutf::implementation* impl : implementations) {
171
72
      if (!impl->validate_utf16be(data.data(), data.size())) {
172
0
        std::cerr << "validate_utf16be_as_ascii returned true but "
173
0
                     "validate_utf16be returned false"
174
0
                  << " impl=" << impl->name() << "\n";
175
0
        std::abort();
176
0
      }
177
72
    }
178
24
  }
179
230
}
180
181
// Checks that to_well_formed_utf16le / to_well_formed_utf16be:
182
// 1. All implementations agree on the output.
183
// 2. The output is always valid UTF-16LE / UTF-16BE.
184
// 3. When the input is already valid UTF-16, the output equals the input.
185
710
void to_well_formed_utf16(std::span<const char16_t> data) {
186
710
  const auto implementations = get_supported_implementations();
187
710
  if (implementations.empty()) {
188
0
    return;
189
0
  }
190
191
  // Check LE variant
192
710
  {
193
710
    std::vector<std::vector<char16_t>> le_outputs;
194
710
    le_outputs.reserve(implementations.size());
195
2.13k
    for (const simdutf::implementation* impl : implementations) {
196
2.13k
      std::vector<char16_t> out(data.size());
197
2.13k
      impl->to_well_formed_utf16le(data.data(), data.size(), out.data());
198
2.13k
      le_outputs.push_back(std::move(out));
199
2.13k
    }
200
1.42k
    auto neq = [](const auto& a, const auto& b) { return a != b; };
201
710
    if (std::ranges::adjacent_find(le_outputs, neq) != le_outputs.end()) {
202
0
      std::cerr
203
0
          << "to_well_formed_utf16le: outputs differ between implementations\n";
204
0
      for (std::size_t i = 0; i < implementations.size(); ++i) {
205
0
        std::cerr << "  " << implementations[i]->name()
206
0
                  << ": hash=" << FNV1A_hash::as_str(le_outputs[i]) << "\n";
207
0
      }
208
0
      std::abort();
209
0
    }
210
    // Output must be valid UTF-16LE.
211
2.84k
    for (std::size_t i = 0; i < implementations.size(); ++i) {
212
2.13k
      if (!implementations[i]->validate_utf16le(le_outputs[i].data(),
213
2.13k
                                                le_outputs[i].size())) {
214
0
        std::cerr << "to_well_formed_utf16le: output is not valid UTF-16LE"
215
0
                  << " impl=" << implementations[i]->name() << "\n";
216
0
        std::abort();
217
0
      }
218
2.13k
    }
219
    // If input was already valid UTF-16LE, output must equal input.
220
710
    if (implementations[0]->validate_utf16le(data.data(), data.size())) {
221
221
      if (!std::ranges::equal(le_outputs[0], data)) {
222
0
        std::cerr << "to_well_formed_utf16le: valid input was modified\n";
223
0
        std::abort();
224
0
      }
225
221
    }
226
710
  }
227
228
  // Check BE variant
229
710
  {
230
710
    std::vector<std::vector<char16_t>> be_outputs;
231
710
    be_outputs.reserve(implementations.size());
232
2.13k
    for (const simdutf::implementation* impl : implementations) {
233
2.13k
      std::vector<char16_t> out(data.size());
234
2.13k
      impl->to_well_formed_utf16be(data.data(), data.size(), out.data());
235
2.13k
      be_outputs.push_back(std::move(out));
236
2.13k
    }
237
1.42k
    auto neq = [](const auto& a, const auto& b) { return a != b; };
238
710
    if (std::ranges::adjacent_find(be_outputs, neq) != be_outputs.end()) {
239
0
      std::cerr
240
0
          << "to_well_formed_utf16be: outputs differ between implementations\n";
241
0
      for (std::size_t i = 0; i < implementations.size(); ++i) {
242
0
        std::cerr << "  " << implementations[i]->name()
243
0
                  << ": hash=" << FNV1A_hash::as_str(be_outputs[i]) << "\n";
244
0
      }
245
0
      std::abort();
246
0
    }
247
    // Output must be valid UTF-16BE.
248
2.84k
    for (std::size_t i = 0; i < implementations.size(); ++i) {
249
2.13k
      if (!implementations[i]->validate_utf16be(be_outputs[i].data(),
250
2.13k
                                                be_outputs[i].size())) {
251
0
        std::cerr << "to_well_formed_utf16be: output is not valid UTF-16BE"
252
0
                  << " impl=" << implementations[i]->name() << "\n";
253
0
        std::abort();
254
0
      }
255
2.13k
    }
256
    // If input was already valid UTF-16BE, output must equal input.
257
710
    if (implementations[0]->validate_utf16be(data.data(), data.size())) {
258
235
      if (!std::ranges::equal(be_outputs[0], data)) {
259
0
        std::cerr << "to_well_formed_utf16be: valid input was modified\n";
260
0
        std::abort();
261
0
      }
262
235
    }
263
710
  }
264
710
}
265
266
void convert_latin1_to_utf8_safe(std::span<const char> chardata,
267
295
                                 const std::size_t outputsize) {
268
  // convert with a limited output buffer
269
295
  std::vector<char> limited_output(outputsize);
270
295
  const auto limited_ret = simdutf::convert_latin1_to_utf8_safe(
271
295
      chardata.data(), chardata.size(), limited_output.data(), outputsize);
272
273
  // convert with a sufficiently large output buffer
274
295
  std::vector<char> large_output(2 * chardata.size());
275
295
  const auto large_ret = simdutf::convert_latin1_to_utf8(
276
295
      chardata.data(), chardata.size(), large_output.data());
277
278
295
  if (large_ret != 0) {
279
    // conversion was possible with a large buffer.
280
264
    if (large_ret <= outputsize) {
281
      // the limited buffer was large enough, ensure we got the same result
282
130
      assert(limited_ret == large_ret);
283
130
      assert(std::ranges::equal(limited_output | std::views::take(large_ret),
284
130
                                large_output | std::views::take(large_ret)));
285
134
    } else {
286
      // the number of written bytes for a limited buffer must not exceed what
287
      // the large buffer got.
288
134
      assert(limited_ret <= large_ret);
289
      // the written data should be equal
290
134
      assert(std::ranges::equal(limited_output | std::views::take(limited_ret),
291
134
                                large_output | std::views::take(limited_ret)));
292
134
    }
293
264
  } else {
294
    // conversion with a big buffer failed - is there anything we can check or
295
    // assert for the limited buffer? I don't think so.
296
31
  }
297
295
}
298
299
3.37k
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
300
  // pick one of the functions, based on the fuzz data.
301
  // the first byte is which action to take. step forward
302
  // several bytes so the input is aligned.
303
3.37k
  if (size < 4) {
304
2
    return 0;
305
2
  }
306
3.37k
  constexpr auto Ncases = 11u;
307
3.37k
  constexpr auto actionmask = std::bit_ceil(Ncases) - 1;
308
3.37k
  const auto action = data[0] & actionmask;
309
310
3.37k
  const std::uint16_t u16 = data[1] + (data[2] << 8);
311
312
3.37k
  data += 4;
313
3.37k
  size -= 4;
314
315
3.37k
  const std::span<const char> chardata{(const char*)data, size};
316
3.37k
  const std::span<const char16_t> u16data{(const char16_t*)data,
317
3.37k
                                          size / sizeof(char16_t)};
318
319
3.37k
  switch (action) {
320
693
  case 0:
321
693
    autodetect(chardata);
322
693
    break;
323
663
  case 1:
324
663
    detect(chardata);
325
663
    break;
326
208
  case 2:
327
208
    validate_ascii(chardata);
328
208
    break;
329
287
  case 3:
330
287
    validate_ascii_with_err(chardata);
331
287
    break;
332
133
  case 4:
333
133
    utf16_endianness(u16data);
334
133
    break;
335
28
  case 5: {
336
28
    [[maybe_unused]] auto ret =
337
28
        simdutf::trim_partial_utf16le(u16data.data(), u16data.size());
338
28
    assert(ret == u16data.size() || ret + 1 == u16data.size());
339
28
  } break;
340
41
  case 6: {
341
41
    [[maybe_unused]] auto ret =
342
41
        simdutf::trim_partial_utf16be(u16data.data(), u16data.size());
343
41
    assert(ret == u16data.size() || ret + 1 == u16data.size());
344
41
  } break;
345
86
  case 7: {
346
86
    [[maybe_unused]] const std::size_t N = chardata.size();
347
86
    [[maybe_unused]] const auto ret =
348
86
        simdutf::trim_partial_utf8(chardata.data(), chardata.size());
349
86
    if ((ret + 3 < N) || (ret > N)) {
350
0
      std::cerr << "ret=" << ret << " N=" << N << '\n';
351
0
      std::abort();
352
0
    }
353
86
  } break;
354
295
  case 8:
355
295
    convert_latin1_to_utf8_safe(chardata, u16);
356
295
    break;
357
230
  case 9:
358
230
    validate_utf16_as_ascii(u16data);
359
230
    break;
360
710
  case 10:
361
710
    to_well_formed_utf16(u16data);
362
710
    break;
363
3.37k
  }
364
3.37k
  return 0;
365
3.37k
}