/src/simdutf/src/icelake/icelake_utf16fix.cpp

Source
#include <immintrin.h>

/*
 * Process one block of 32 characters.  If in_place is false,
 * copy the block from in to out.  If there is a sequencing
 * error in the block, overwrite the illsequenced characters
 * with the replacement character.  This function reads one
 * character before the beginning of the buffer as a lookback.
 * If that character is illsequenced, it too is overwritten.
 */
template <endianness big_endian, bool in_place>
simdutf_really_inline void utf16fix_block(char16_t *out, const char16_t *in) {
  const char16_t replacement = scalar::utf16::replacement<big_endian>();
  __m512i lookback, block, lb_masked, block_masked;
  __mmask32 lb_is_high, block_is_low, illseq;
  auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t {
    return scalar::utf16::swap_if_needed<big_endian>(x);
  };

  lookback = _mm512_loadu_si512((const __m512i *)(in - 1));
  block = _mm512_loadu_si512((const __m512i *)in);
  lb_masked =
      _mm512_and_epi32(lookback, _mm512_set1_epi16(swap_if_needed(0xfc00U)));
  block_masked =
      _mm512_and_epi32(block, _mm512_set1_epi16(swap_if_needed(0xfc00U)));

  lb_is_high = _mm512_cmpeq_epi16_mask(
      lb_masked, _mm512_set1_epi16(swap_if_needed(0xd800U)));
  block_is_low = _mm512_cmpeq_epi16_mask(
      block_masked, _mm512_set1_epi16(swap_if_needed(0xdc00U)));
  illseq = _kxor_mask32(lb_is_high, block_is_low);
  if (!_ktestz_mask32_u8(illseq, illseq)) {
    __mmask32 lb_illseq, block_illseq;

    /* compute the cause of the illegal sequencing */
    lb_illseq = _kandn_mask32(block_is_low, lb_is_high);
    block_illseq = _kor_mask32(_kandn_mask32(lb_is_high, block_is_low),
                               _kshiftri_mask32(lb_illseq, 1));

    /* fix illegal sequencing in the lookback */
    lb_illseq = _kand_mask32(lb_illseq, _cvtu32_mask32(1));
    _mm512_mask_storeu_epi16(out - 1, lb_illseq,
                             _mm512_set1_epi16(replacement));

    /* fix illegal sequencing in the main block */
    if (in_place) {
      _mm512_mask_storeu_epi16(out, block_illseq,
                               _mm512_set1_epi16(replacement));
    } else {
      _mm512_storeu_epi32(
          out, _mm512_mask_blend_epi16(block_illseq, block,
                                       _mm512_set1_epi16(replacement)));
    }
  } else if (!in_place) {
    _mm512_storeu_si512((__m512i *)out, block);
  }
}

/*
 * Special case for inputs of 0--32 bytes.  Works for both in-place and
 * out-of-place operation.
 */
template <endianness big_endian>
void utf16fix_short(const char16_t *in, size_t n, char16_t *out) {
  const char16_t replacement = scalar::utf16::replacement<big_endian>();
  __m512i lookback, block, lb_masked, block_masked;
  __mmask32 lb_is_high, block_is_low, illseq;
  uint32_t mask = 0xFFFFFFFF >> (32 - n);
  auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t {
    return scalar::utf16::swap_if_needed<big_endian>(x);
  };
  lookback = _mm512_maskz_loadu_epi16(_cvtmask32_u32(mask << 1),
                                      (const uint16_t *)(in - 1));
  block = _mm512_maskz_loadu_epi16(_cvtmask32_u32(mask), (const uint16_t *)in);
  lb_masked =
      _mm512_and_epi32(lookback, _mm512_set1_epi16(swap_if_needed(0xfc00u)));
  block_masked =
      _mm512_and_epi32(block, _mm512_set1_epi16(swap_if_needed(0xfc00u)));

  lb_is_high = _mm512_cmpeq_epi16_mask(
      lb_masked, _mm512_set1_epi16(swap_if_needed(0xd800u)));
  block_is_low = _mm512_cmpeq_epi16_mask(
      block_masked, _mm512_set1_epi16(swap_if_needed(0xdc00u)));
  illseq = _kxor_mask32(lb_is_high, block_is_low);
  if (!_ktestz_mask32_u8(illseq, illseq)) {
    __mmask32 lb_illseq, block_illseq;

    /* compute the cause of the illegal sequencing */
    lb_illseq = _kandn_mask32(block_is_low, lb_is_high);
    block_illseq = _kor_mask32(_kandn_mask32(lb_is_high, block_is_low),
                               _kshiftri_mask32(lb_illseq, 1));

    /* fix illegal sequencing in the main block */
    _mm512_mask_storeu_epi16(
        (uint16_t *)out, _cvtmask32_u32(mask),
        _mm512_mask_blend_epi16(block_illseq, block,
                                _mm512_set1_epi16(replacement)));
  } else {
    _mm512_mask_storeu_epi16((uint16_t *)out, _cvtmask32_u32(mask), block);
  }
  out[n - 1] = scalar::utf16::is_high_surrogate<big_endian>(out[n - 1])
                   ? replacement
                   : out[n - 1];
}

template <endianness big_endian>
void utf16fix_avx512(const char16_t *in, size_t n, char16_t *out) {
  const char16_t replacement = scalar::utf16::replacement<big_endian>();
  size_t i;

  if (n == 0)
    return;
  else if (n < 33) {
    utf16fix_short<big_endian>(in, n, out);
    return;
  }
  out[0] =
      scalar::utf16::is_low_surrogate<big_endian>(in[0]) ? replacement : in[0];

  /* duplicate code to have the compiler specialise utf16fix_block() */
  if (in == out) {
    for (i = 1; i + 32 < n; i += 32) {
      utf16fix_block<big_endian, true>(out + i, in + i);
    }

    utf16fix_block<big_endian, true>(out + n - 32, in + n - 32);
  } else {
    for (i = 1; i + 32 < n; i += 32) {
      utf16fix_block<big_endian, false>(out + i, in + i);
    }

    utf16fix_block<big_endian, false>(out + n - 32, in + n - 32);
  }

  out[n - 1] = scalar::utf16::is_high_surrogate<big_endian>(out[n - 1])
                   ? replacement
                   : out[n - 1];
}

Line	Count	Source
1		#include <immintrin.h>
2
3		/*
4		* Process one block of 32 characters. If in_place is false,
5		* copy the block from in to out. If there is a sequencing
6		* error in the block, overwrite the illsequenced characters
7		* with the replacement character. This function reads one
8		* character before the beginning of the buffer as a lookback.
9		* If that character is illsequenced, it too is overwritten.
10		*/
11		template <endianness big_endian, bool in_place>
12	0	simdutf_really_inline void utf16fix_block(char16_t out, const char16_t in) {
13	0	const char16_t replacement = scalar::utf16::replacement<big_endian>();
14	0	__m512i lookback, block, lb_masked, block_masked;
15	0	__mmask32 lb_is_high, block_is_low, illseq;
16	0	auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t {
17	0	return scalar::utf16::swap_if_needed<big_endian>(x);
18	0	}; Unexecuted instantiation: simdutf.cpp:simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)0, true>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)0, false>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)1, true>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)1, false>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const
19
20	0	lookback = _mm512_loadu_si512((const __m512i *)(in - 1));
21	0	block = _mm512_loadu_si512((const __m512i *)in);
22	0	lb_masked =
23	0	_mm512_and_epi32(lookback, _mm512_set1_epi16(swap_if_needed(0xfc00U)));
24	0	block_masked =
25	0	_mm512_and_epi32(block, _mm512_set1_epi16(swap_if_needed(0xfc00U)));
26
27	0	lb_is_high = _mm512_cmpeq_epi16_mask(
28	0	lb_masked, _mm512_set1_epi16(swap_if_needed(0xd800U)));
29	0	block_is_low = _mm512_cmpeq_epi16_mask(
30	0	block_masked, _mm512_set1_epi16(swap_if_needed(0xdc00U)));
31	0	illseq = _kxor_mask32(lb_is_high, block_is_low);
32	0	if (!_ktestz_mask32_u8(illseq, illseq)) {
33	0	__mmask32 lb_illseq, block_illseq;
34
35		/* compute the cause of the illegal sequencing */
36	0	lb_illseq = _kandn_mask32(block_is_low, lb_is_high);
37	0	block_illseq = _kor_mask32(_kandn_mask32(lb_is_high, block_is_low),
38	0	_kshiftri_mask32(lb_illseq, 1));
39
40		/* fix illegal sequencing in the lookback */
41	0	lb_illseq = _kand_mask32(lb_illseq, _cvtu32_mask32(1));
42	0	_mm512_mask_storeu_epi16(out - 1, lb_illseq,
43	0	_mm512_set1_epi16(replacement));
44
45		/* fix illegal sequencing in the main block */
46	0	if (in_place) {
47	0	_mm512_mask_storeu_epi16(out, block_illseq,
48	0	_mm512_set1_epi16(replacement));
49	0	} else {
50	0	_mm512_storeu_epi32(
51	0	out, _mm512_mask_blend_epi16(block_illseq, block,
52	0	_mm512_set1_epi16(replacement)));
53	0	}
54	0	} else if (!in_place) {
55	0	_mm512_storeu_si512((__m512i *)out, block);
56	0	}
57	0	} Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)0, true>(char16_t, char16_t const) Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)0, false>(char16_t, char16_t const) Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)1, true>(char16_t, char16_t const) Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)1, false>(char16_t, char16_t const)
58
59		/*
60		* Special case for inputs of 0--32 bytes. Works for both in-place and
61		* out-of-place operation.
62		*/
63		template <endianness big_endian>
64	0	void utf16fix_short(const char16_t in, size_t n, char16_t out) {
65	0	const char16_t replacement = scalar::utf16::replacement<big_endian>();
66	0	__m512i lookback, block, lb_masked, block_masked;
67	0	__mmask32 lb_is_high, block_is_low, illseq;
68	0	uint32_t mask = 0xFFFFFFFF >> (32 - n);
69	0	auto swap_if_needed = [](uint16_t x) constexpr -> uint16_t {
70	0	return scalar::utf16::swap_if_needed<big_endian>(x);
71	0	}; Unexecuted instantiation: simdutf.cpp:simdutf::icelake::(anonymous namespace)::utf16fix_short<(simdutf::endianness)0>(char16_t const, unsigned long, char16_t)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::icelake::(anonymous namespace)::utf16fix_short<(simdutf::endianness)1>(char16_t const, unsigned long, char16_t)::{lambda(unsigned short)#1}::operator()(unsigned short) const
72	0	lookback = _mm512_maskz_loadu_epi16(_cvtmask32_u32(mask << 1),
73	0	(const uint16_t *)(in - 1));
74	0	block = _mm512_maskz_loadu_epi16(_cvtmask32_u32(mask), (const uint16_t *)in);
75	0	lb_masked =
76	0	_mm512_and_epi32(lookback, _mm512_set1_epi16(swap_if_needed(0xfc00u)));
77	0	block_masked =
78	0	_mm512_and_epi32(block, _mm512_set1_epi16(swap_if_needed(0xfc00u)));
79
80	0	lb_is_high = _mm512_cmpeq_epi16_mask(
81	0	lb_masked, _mm512_set1_epi16(swap_if_needed(0xd800u)));
82	0	block_is_low = _mm512_cmpeq_epi16_mask(
83	0	block_masked, _mm512_set1_epi16(swap_if_needed(0xdc00u)));
84	0	illseq = _kxor_mask32(lb_is_high, block_is_low);
85	0	if (!_ktestz_mask32_u8(illseq, illseq)) {
86	0	__mmask32 lb_illseq, block_illseq;
87
88		/* compute the cause of the illegal sequencing */
89	0	lb_illseq = _kandn_mask32(block_is_low, lb_is_high);
90	0	block_illseq = _kor_mask32(_kandn_mask32(lb_is_high, block_is_low),
91	0	_kshiftri_mask32(lb_illseq, 1));
92
93		/* fix illegal sequencing in the main block */
94	0	_mm512_mask_storeu_epi16(
95	0	(uint16_t *)out, _cvtmask32_u32(mask),
96	0	_mm512_mask_blend_epi16(block_illseq, block,
97	0	_mm512_set1_epi16(replacement)));
98	0	} else {
99	0	_mm512_mask_storeu_epi16((uint16_t *)out, _cvtmask32_u32(mask), block);
100	0	}
101	0	out[n - 1] = scalar::utf16::is_high_surrogate<big_endian>(out[n - 1])
102	0	? replacement
103	0	: out[n - 1];
104	0	} Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_short<(simdutf::endianness)0>(char16_t const, unsigned long, char16_t) Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_short<(simdutf::endianness)1>(char16_t const, unsigned long, char16_t)
105
106		template <endianness big_endian>
107	0	void utf16fix_avx512(const char16_t in, size_t n, char16_t out) {
108	0	const char16_t replacement = scalar::utf16::replacement<big_endian>();
109	0	size_t i;
110
111	0	if (n == 0)
112	0	return;
113	0	else if (n < 33) {
114	0	utf16fix_short<big_endian>(in, n, out);
115	0	return;
116	0	}
117	0	out[0] =
118	0	scalar::utf16::is_low_surrogate<big_endian>(in[0]) ? replacement : in[0];
119
120		/* duplicate code to have the compiler specialise utf16fix_block() */
121	0	if (in == out) {
122	0	for (i = 1; i + 32 < n; i += 32) {
123	0	utf16fix_block<big_endian, true>(out + i, in + i);
124	0	}
125
126	0	utf16fix_block<big_endian, true>(out + n - 32, in + n - 32);
127	0	} else {
128	0	for (i = 1; i + 32 < n; i += 32) {
129	0	utf16fix_block<big_endian, false>(out + i, in + i);
130	0	}
131
132	0	utf16fix_block<big_endian, false>(out + n - 32, in + n - 32);
133	0	}
134
135	0	out[n - 1] = scalar::utf16::is_high_surrogate<big_endian>(out[n - 1])
136	0	? replacement
137	0	: out[n - 1];
138	0	} Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_avx512<(simdutf::endianness)0>(char16_t const, unsigned long, char16_t) Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_avx512<(simdutf::endianness)1>(char16_t const, unsigned long, char16_t)

Coverage Report

Created: 2026-09-03 07:11