/src/simdutf/src/haswell/avx2_utf16fix.cpp

Source
/*
 * Process one block of 16 characters.  If in_place is false,
 * copy the block from in to out.  If there is a sequencing
 * error in the block, overwrite the illsequenced characters
 * with the replacement character.  This function reads one
 * character before the beginning of the buffer as a lookback.
 * If that character is illsequenced, it too is overwritten.
 */
template <endianness big_endian, bool in_place>
void utf16fix_block(char16_t *out, const char16_t *in) {
  auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t {
    return scalar::utf16::swap_if_needed<big_endian>(x);
  };
  const char16_t replacement = scalar::utf16::replacement<big_endian>();
  __m256i lookback, block, lb_masked, block_masked, lb_is_high, block_is_low;
  __m256i illseq, lb_illseq, block_illseq, lb_illseq_shifted;

  lookback = _mm256_loadu_si256((const __m256i *)(in - 1));
  block = _mm256_loadu_si256((const __m256i *)in);
  lb_masked =
      _mm256_and_si256(lookback, _mm256_set1_epi16(swap_if_needed(0xfc00u)));
  block_masked =
      _mm256_and_si256(block, _mm256_set1_epi16(swap_if_needed(0xfc00u)));
  lb_is_high =
      _mm256_cmpeq_epi16(lb_masked, _mm256_set1_epi16(swap_if_needed(0xd800u)));
  block_is_low = _mm256_cmpeq_epi16(block_masked,
                                    _mm256_set1_epi16(swap_if_needed(0xdc00u)));

  illseq = _mm256_xor_si256(lb_is_high, block_is_low);
  if (!_mm256_testz_si256(illseq, illseq)) {
    int lb;

    /* compute the cause of the illegal sequencing */
    lb_illseq = _mm256_andnot_si256(block_is_low, lb_is_high);
#if SIMDUTF_GCC9OROLDER
    // Old GCC versions are missing _mm256_zextsi128_si256, so we emulate it.
    __m128i tmp_legacygcc =
        _mm_bslli_si128(_mm256_extracti128_si256(lb_illseq, 1), 14);
    __m256i tmp_legacygcc256 =
        _mm256_set_m128i(_mm_setzero_si128(), tmp_legacygcc);
    lb_illseq_shifted =
        _mm256_or_si256(_mm256_bsrli_epi128(lb_illseq, 2), tmp_legacygcc256);
#else
    lb_illseq_shifted =
        _mm256_or_si256(_mm256_bsrli_epi128(lb_illseq, 2),
                        _mm256_zextsi128_si256(_mm_bslli_si128(
                            _mm256_extracti128_si256(lb_illseq, 1), 14)));
#endif // SIMDUTF_GCC9OROLDER
    block_illseq = _mm256_or_si256(
        _mm256_andnot_si256(lb_is_high, block_is_low), lb_illseq_shifted);

    /* fix illegal sequencing in the lookback */
#if SIMDUTF_GCC10 || SIMDUTF_GCC9OROLDER
    // GCC 10 is missing important intrinsics.
    lb = _mm_cvtsi128_si32(_mm256_extractf128_si256(lb_illseq, 0));
#else
    lb = _mm256_cvtsi256_si32(lb_illseq);
#endif
    lb = (lb & replacement) | (~lb & out[-1]);
    out[-1] = char16_t(lb);

    /* fix illegal sequencing in the main block */
    block =
        _mm256_blendv_epi8(block, _mm256_set1_epi16(replacement), block_illseq);
    _mm256_storeu_si256((__m256i *)out, block);
  } else if (!in_place) {
    _mm256_storeu_si256((__m256i *)out, block);
  }
}

template <endianness big_endian, bool in_place>
void utf16fix_block_sse(char16_t *out, const char16_t *in) {
  auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t {
    return scalar::utf16::swap_if_needed<big_endian>(x);
  };
  const char16_t replacement = scalar::utf16::replacement<big_endian>();
  __m128i lookback, block, lb_masked, block_masked, lb_is_high, block_is_low;
  __m128i illseq, lb_illseq, block_illseq;

  lookback = _mm_loadu_si128((const __m128i *)(in - 1));
  block = _mm_loadu_si128((const __m128i *)in);
  lb_masked = _mm_and_si128(lookback, _mm_set1_epi16(swap_if_needed(0xfc00U)));
  block_masked = _mm_and_si128(block, _mm_set1_epi16(swap_if_needed(0xfc00U)));
  lb_is_high =
      _mm_cmpeq_epi16(lb_masked, _mm_set1_epi16(swap_if_needed(0xd800U)));
  block_is_low =
      _mm_cmpeq_epi16(block_masked, _mm_set1_epi16(swap_if_needed(0xdc00U)));

  illseq = _mm_xor_si128(lb_is_high, block_is_low);
  if (_mm_movemask_epi8(illseq) != 0) {
    /* compute the cause of the illegal sequencing */
    lb_illseq = _mm_andnot_si128(block_is_low, lb_is_high);
    block_illseq = _mm_or_si128(_mm_andnot_si128(lb_is_high, block_is_low),
                                _mm_bsrli_si128(lb_illseq, 2));
    /* fix illegal sequencing in the lookback */
    int lb = _mm_cvtsi128_si32(lb_illseq);
    lb = (lb & replacement) | (~lb & out[-1]);
    out[-1] = char16_t(lb);
    /* fix illegal sequencing in the main block */
    block =
        _mm_or_si128(_mm_andnot_si128(block_illseq, block),
                     _mm_and_si128(block_illseq, _mm_set1_epi16(replacement)));
    _mm_storeu_si128((__m128i *)out, block);
  } else if (!in_place) {
    _mm_storeu_si128((__m128i *)out, block);
  }
}

template <endianness big_endian>
void utf16fix_sse(const char16_t *in, size_t n, char16_t *out) {
  const char16_t replacement = scalar::utf16::replacement<big_endian>();
  size_t i;

  if (n < 9) {
    scalar::utf16::to_well_formed_utf16<big_endian>(in, n, out);
    return;
  }

  out[0] =
      scalar::utf16::is_low_surrogate<big_endian>(in[0]) ? replacement : in[0];

  /* duplicate code to have the compiler specialise utf16fix_block() */
  if (in == out) {
    for (i = 1; i + 8 < n; i += 8) {
      utf16fix_block_sse<big_endian, true>(out + i, in + i);
    }

    utf16fix_block_sse<big_endian, true>(out + n - 8, in + n - 8);
  } else {
    for (i = 1; i + 8 < n; i += 8) {
      utf16fix_block_sse<big_endian, false>(out + i, in + i);
    }

    utf16fix_block_sse<big_endian, false>(out + n - 8, in + n - 8);
  }

  out[n - 1] = scalar::utf16::is_high_surrogate<big_endian>(out[n - 1])
                   ? replacement
                   : out[n - 1];
}

template <endianness big_endian>
void utf16fix_avx(const char16_t *in, size_t n, char16_t *out) {
  const char16_t replacement = scalar::utf16::replacement<big_endian>();
  size_t i;

  if (n < 17) {
    utf16fix_sse<big_endian>(in, n, out);
    return;
  }

  out[0] =
      scalar::utf16::is_low_surrogate<big_endian>(in[0]) ? replacement : in[0];

  /* duplicate code to have the compiler specialise utf16fix_block() */
  if (in == out) {
    for (i = 1; i + 16 < n; i += 16) {
      utf16fix_block<big_endian, true>(out + i, in + i);
    }

    utf16fix_block<big_endian, true>(out + n - 16, in + n - 16);
  } else {
    for (i = 1; i + 16 < n; i += 16) {
      utf16fix_block<big_endian, false>(out + i, in + i);
    }

    utf16fix_block<big_endian, false>(out + n - 16, in + n - 16);
  }

  out[n - 1] = scalar::utf16::is_high_surrogate<big_endian>(out[n - 1])
                   ? replacement
                   : out[n - 1];
}

Line	Count	Source
1		/*
2		* Process one block of 16 characters. If in_place is false,
3		* copy the block from in to out. If there is a sequencing
4		* error in the block, overwrite the illsequenced characters
5		* with the replacement character. This function reads one
6		* character before the beginning of the buffer as a lookback.
7		* If that character is illsequenced, it too is overwritten.
8		*/
9		template <endianness big_endian, bool in_place>
10	0	void utf16fix_block(char16_t out, const char16_t in) {
11	0	auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t {
12	0	return scalar::utf16::swap_if_needed<big_endian>(x);
13	0	}; Unexecuted instantiation: simdutf.cpp:simdutf::haswell::(anonymous namespace)::utf16fix_block<(simdutf::endianness)0, true>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::haswell::(anonymous namespace)::utf16fix_block<(simdutf::endianness)0, false>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::haswell::(anonymous namespace)::utf16fix_block<(simdutf::endianness)1, true>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::haswell::(anonymous namespace)::utf16fix_block<(simdutf::endianness)1, false>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const
14	0	const char16_t replacement = scalar::utf16::replacement<big_endian>();
15	0	__m256i lookback, block, lb_masked, block_masked, lb_is_high, block_is_low;
16	0	__m256i illseq, lb_illseq, block_illseq, lb_illseq_shifted;
17
18	0	lookback = _mm256_loadu_si256((const __m256i *)(in - 1));
19	0	block = _mm256_loadu_si256((const __m256i *)in);
20	0	lb_masked =
21	0	_mm256_and_si256(lookback, _mm256_set1_epi16(swap_if_needed(0xfc00u)));
22	0	block_masked =
23	0	_mm256_and_si256(block, _mm256_set1_epi16(swap_if_needed(0xfc00u)));
24	0	lb_is_high =
25	0	_mm256_cmpeq_epi16(lb_masked, _mm256_set1_epi16(swap_if_needed(0xd800u)));
26	0	block_is_low = _mm256_cmpeq_epi16(block_masked,
27	0	_mm256_set1_epi16(swap_if_needed(0xdc00u)));
28
29	0	illseq = _mm256_xor_si256(lb_is_high, block_is_low);
30	0	if (!_mm256_testz_si256(illseq, illseq)) {
31	0	int lb;
32
33		/* compute the cause of the illegal sequencing */
34	0	lb_illseq = _mm256_andnot_si256(block_is_low, lb_is_high);
35		#if SIMDUTF_GCC9OROLDER
36		// Old GCC versions are missing _mm256_zextsi128_si256, so we emulate it.
37		__m128i tmp_legacygcc =
38		_mm_bslli_si128(_mm256_extracti128_si256(lb_illseq, 1), 14);
39		__m256i tmp_legacygcc256 =
40		_mm256_set_m128i(_mm_setzero_si128(), tmp_legacygcc);
41		lb_illseq_shifted =
42		_mm256_or_si256(_mm256_bsrli_epi128(lb_illseq, 2), tmp_legacygcc256);
43		#else
44	0	lb_illseq_shifted =
45	0	_mm256_or_si256(_mm256_bsrli_epi128(lb_illseq, 2),
46	0	_mm256_zextsi128_si256(_mm_bslli_si128(
47	0	_mm256_extracti128_si256(lb_illseq, 1), 14)));
48	0	#endif // SIMDUTF_GCC9OROLDER
49	0	block_illseq = _mm256_or_si256(
50	0	_mm256_andnot_si256(lb_is_high, block_is_low), lb_illseq_shifted);
51
52		/* fix illegal sequencing in the lookback */
53		#if SIMDUTF_GCC10 \|\| SIMDUTF_GCC9OROLDER
54		// GCC 10 is missing important intrinsics.
55		lb = _mm_cvtsi128_si32(_mm256_extractf128_si256(lb_illseq, 0));
56		#else
57	0	lb = _mm256_cvtsi256_si32(lb_illseq);
58	0	#endif
59	0	lb = (lb & replacement) \| (~lb & out[-1]);
60	0	out[-1] = char16_t(lb);
61
62		/* fix illegal sequencing in the main block */
63	0	block =
64	0	_mm256_blendv_epi8(block, _mm256_set1_epi16(replacement), block_illseq);
65	0	_mm256_storeu_si256((__m256i *)out, block);
66	0	} else if (!in_place) {
67	0	_mm256_storeu_si256((__m256i *)out, block);
68	0	}
69	0	} Unexecuted instantiation: simdutf.cpp:void simdutf::haswell::(anonymous namespace)::utf16fix_block<(simdutf::endianness)0, true>(char16_t, char16_t const) Unexecuted instantiation: simdutf.cpp:void simdutf::haswell::(anonymous namespace)::utf16fix_block<(simdutf::endianness)0, false>(char16_t, char16_t const) Unexecuted instantiation: simdutf.cpp:void simdutf::haswell::(anonymous namespace)::utf16fix_block<(simdutf::endianness)1, true>(char16_t, char16_t const) Unexecuted instantiation: simdutf.cpp:void simdutf::haswell::(anonymous namespace)::utf16fix_block<(simdutf::endianness)1, false>(char16_t, char16_t const)
70
71		template <endianness big_endian, bool in_place>
72	0	void utf16fix_block_sse(char16_t out, const char16_t in) {
73	0	auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t {
74	0	return scalar::utf16::swap_if_needed<big_endian>(x);
75	0	}; Unexecuted instantiation: simdutf.cpp:simdutf::haswell::(anonymous namespace)::utf16fix_block_sse<(simdutf::endianness)0, true>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::haswell::(anonymous namespace)::utf16fix_block_sse<(simdutf::endianness)0, false>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::haswell::(anonymous namespace)::utf16fix_block_sse<(simdutf::endianness)1, true>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::haswell::(anonymous namespace)::utf16fix_block_sse<(simdutf::endianness)1, false>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const
76	0	const char16_t replacement = scalar::utf16::replacement<big_endian>();
77	0	__m128i lookback, block, lb_masked, block_masked, lb_is_high, block_is_low;
78	0	__m128i illseq, lb_illseq, block_illseq;
79
80	0	lookback = _mm_loadu_si128((const __m128i *)(in - 1));
81	0	block = _mm_loadu_si128((const __m128i *)in);
82	0	lb_masked = _mm_and_si128(lookback, _mm_set1_epi16(swap_if_needed(0xfc00U)));
83	0	block_masked = _mm_and_si128(block, _mm_set1_epi16(swap_if_needed(0xfc00U)));
84	0	lb_is_high =
85	0	_mm_cmpeq_epi16(lb_masked, _mm_set1_epi16(swap_if_needed(0xd800U)));
86	0	block_is_low =
87	0	_mm_cmpeq_epi16(block_masked, _mm_set1_epi16(swap_if_needed(0xdc00U)));
88
89	0	illseq = _mm_xor_si128(lb_is_high, block_is_low);
90	0	if (_mm_movemask_epi8(illseq) != 0) {
91		/* compute the cause of the illegal sequencing */
92	0	lb_illseq = _mm_andnot_si128(block_is_low, lb_is_high);
93	0	block_illseq = _mm_or_si128(_mm_andnot_si128(lb_is_high, block_is_low),
94	0	_mm_bsrli_si128(lb_illseq, 2));
95		/* fix illegal sequencing in the lookback */
96	0	int lb = _mm_cvtsi128_si32(lb_illseq);
97	0	lb = (lb & replacement) \| (~lb & out[-1]);
98	0	out[-1] = char16_t(lb);
99		/* fix illegal sequencing in the main block */
100	0	block =
101	0	_mm_or_si128(_mm_andnot_si128(block_illseq, block),
102	0	_mm_and_si128(block_illseq, _mm_set1_epi16(replacement)));
103	0	_mm_storeu_si128((__m128i *)out, block);
104	0	} else if (!in_place) {
105	0	_mm_storeu_si128((__m128i *)out, block);
106	0	}
107	0	} Unexecuted instantiation: simdutf.cpp:void simdutf::haswell::(anonymous namespace)::utf16fix_block_sse<(simdutf::endianness)0, true>(char16_t, char16_t const) Unexecuted instantiation: simdutf.cpp:void simdutf::haswell::(anonymous namespace)::utf16fix_block_sse<(simdutf::endianness)0, false>(char16_t, char16_t const) Unexecuted instantiation: simdutf.cpp:void simdutf::haswell::(anonymous namespace)::utf16fix_block_sse<(simdutf::endianness)1, true>(char16_t, char16_t const) Unexecuted instantiation: simdutf.cpp:void simdutf::haswell::(anonymous namespace)::utf16fix_block_sse<(simdutf::endianness)1, false>(char16_t, char16_t const)
108
109		template <endianness big_endian>
110	0	void utf16fix_sse(const char16_t in, size_t n, char16_t out) {
111	0	const char16_t replacement = scalar::utf16::replacement<big_endian>();
112	0	size_t i;
113
114	0	if (n < 9) {
115	0	scalar::utf16::to_well_formed_utf16<big_endian>(in, n, out);
116	0	return;
117	0	}
118
119	0	out[0] =
120	0	scalar::utf16::is_low_surrogate<big_endian>(in[0]) ? replacement : in[0];
121
122		/* duplicate code to have the compiler specialise utf16fix_block() */
123	0	if (in == out) {
124	0	for (i = 1; i + 8 < n; i += 8) {
125	0	utf16fix_block_sse<big_endian, true>(out + i, in + i);
126	0	}
127
128	0	utf16fix_block_sse<big_endian, true>(out + n - 8, in + n - 8);
129	0	} else {
130	0	for (i = 1; i + 8 < n; i += 8) {
131	0	utf16fix_block_sse<big_endian, false>(out + i, in + i);
132	0	}
133
134	0	utf16fix_block_sse<big_endian, false>(out + n - 8, in + n - 8);
135	0	}
136
137	0	out[n - 1] = scalar::utf16::is_high_surrogate<big_endian>(out[n - 1])
138	0	? replacement
139	0	: out[n - 1];
140	0	} Unexecuted instantiation: simdutf.cpp:void simdutf::haswell::(anonymous namespace)::utf16fix_sse<(simdutf::endianness)0>(char16_t const, unsigned long, char16_t) Unexecuted instantiation: simdutf.cpp:void simdutf::haswell::(anonymous namespace)::utf16fix_sse<(simdutf::endianness)1>(char16_t const, unsigned long, char16_t)
141
142		template <endianness big_endian>
143	0	void utf16fix_avx(const char16_t in, size_t n, char16_t out) {
144	0	const char16_t replacement = scalar::utf16::replacement<big_endian>();
145	0	size_t i;
146
147	0	if (n < 17) {
148	0	utf16fix_sse<big_endian>(in, n, out);
149	0	return;
150	0	}
151
152	0	out[0] =
153	0	scalar::utf16::is_low_surrogate<big_endian>(in[0]) ? replacement : in[0];
154
155		/* duplicate code to have the compiler specialise utf16fix_block() */
156	0	if (in == out) {
157	0	for (i = 1; i + 16 < n; i += 16) {
158	0	utf16fix_block<big_endian, true>(out + i, in + i);
159	0	}
160
161	0	utf16fix_block<big_endian, true>(out + n - 16, in + n - 16);
162	0	} else {
163	0	for (i = 1; i + 16 < n; i += 16) {
164	0	utf16fix_block<big_endian, false>(out + i, in + i);
165	0	}
166
167	0	utf16fix_block<big_endian, false>(out + n - 16, in + n - 16);
168	0	}
169
170	0	out[n - 1] = scalar::utf16::is_high_surrogate<big_endian>(out[n - 1])
171	0	? replacement
172	0	: out[n - 1];
173	0	} Unexecuted instantiation: simdutf.cpp:void simdutf::haswell::(anonymous namespace)::utf16fix_avx<(simdutf::endianness)0>(char16_t const, unsigned long, char16_t) Unexecuted instantiation: simdutf.cpp:void simdutf::haswell::(anonymous namespace)::utf16fix_avx<(simdutf::endianness)1>(char16_t const, unsigned long, char16_t)

Coverage Report

Created: 2026-07-25 06:54