/src/simdutf/src/icelake/icelake_utf16fix.cpp

Source (jump to first uncovered line)
/*
 * Process one block of 32 characters.  If in_place is false,
 * copy the block from in to out.  If there is a sequencing
 * error in the block, overwrite the illsequenced characters
 * with the replacement character.  This function reads one
 * character before the beginning of the buffer as a lookback.
 * If that character is illsequenced, it too is overwritten.
 */
template <endianness big_endian, bool in_place>
simdutf_really_inline void utf16fix_block(char16_t *out, const char16_t *in) {
  const char16_t replacement = scalar::utf16::replacement<big_endian>();
  auto swap_if_needed = [](uint16_t c) -> uint16_t {
    return !simdutf::match_system(big_endian) ? scalar::u16_swap_bytes(c) : c;
  };

  __m512i lookback, block, lb_masked, block_masked;
  __mmask32 lb_is_high, block_is_low, illseq;

  lookback = _mm512_loadu_si512((const __m512i *)(in - 1));
  block = _mm512_loadu_si512((const __m512i *)in);
  lb_masked =
      _mm512_and_epi32(lookback, _mm512_set1_epi16(swap_if_needed(0xfc00U)));
  block_masked =
      _mm512_and_epi32(block, _mm512_set1_epi16(swap_if_needed(0xfc00U)));

  lb_is_high = _mm512_cmpeq_epi16_mask(
      lb_masked, _mm512_set1_epi16(swap_if_needed(0xd800U)));
  block_is_low = _mm512_cmpeq_epi16_mask(
      block_masked, _mm512_set1_epi16(swap_if_needed(0xdc00U)));
  illseq = _kxor_mask32(lb_is_high, block_is_low);
  if (!_ktestz_mask32_u8(illseq, illseq)) {
    __mmask32 lb_illseq, block_illseq;

    /* compute the cause of the illegal sequencing */
    lb_illseq = _kandn_mask32(block_is_low, lb_is_high);
    block_illseq = _kor_mask32(_kandn_mask32(lb_is_high, block_is_low),
                               _kshiftri_mask32(lb_illseq, 1));

    /* fix illegal sequencing in the lookback */
    lb_illseq = _kand_mask32(lb_illseq, _cvtu32_mask32(1));
    _mm512_mask_storeu_epi16(out - 1, lb_illseq,
                             _mm512_set1_epi16(replacement));

    /* fix illegal sequencing in the main block */
    if (in_place) {
      _mm512_mask_storeu_epi16(out, block_illseq,
                               _mm512_set1_epi16(replacement));
    } else {
      _mm512_storeu_epi32(
          out, _mm512_mask_blend_epi16(block_illseq, block,
                                       _mm512_set1_epi16(replacement)));
    }
  } else if (!in_place) {
    _mm512_storeu_si512((__m512i *)out, block);
  }
}

/*
 * Special case for inputs of 0--32 bytes.  Works for both in-place and
 * out-of-place operation.
 */
template <endianness big_endian>
void utf16fix_runt(const char16_t *in, size_t n, char16_t *out) {
  const char16_t replacement = scalar::utf16::replacement<big_endian>();
  auto swap_if_needed = [](uint16_t c) -> uint16_t {
    return !simdutf::match_system(big_endian) ? scalar::u16_swap_bytes(c) : c;
  };
  __m512i lookback, block, lb_masked, block_masked;
  __mmask32 lb_is_high, block_is_low, illseq;
  uint32_t mask = 0xFFFFFFFF >> (32 - n);
  lookback = _mm512_maskz_loadu_epi16(_cvtmask32_u32(mask << 1),
                                      (const uint16_t *)(in - 1));
  block = _mm512_maskz_loadu_epi16(_cvtmask32_u32(mask), (const uint16_t *)in);
  lb_masked =
      _mm512_and_epi32(lookback, _mm512_set1_epi16(swap_if_needed(0xfc00u)));
  block_masked =
      _mm512_and_epi32(block, _mm512_set1_epi16(swap_if_needed(0xfc00u)));

  lb_is_high = _mm512_cmpeq_epi16_mask(
      lb_masked, _mm512_set1_epi16(swap_if_needed(0xd800u)));
  block_is_low = _mm512_cmpeq_epi16_mask(
      block_masked, _mm512_set1_epi16(swap_if_needed(0xdc00u)));
  illseq = _kxor_mask32(lb_is_high, block_is_low);
  if (!_ktestz_mask32_u8(illseq, illseq)) {
    __mmask32 lb_illseq, block_illseq;

    /* compute the cause of the illegal sequencing */
    lb_illseq = _kandn_mask32(block_is_low, lb_is_high);
    block_illseq = _kor_mask32(_kandn_mask32(lb_is_high, block_is_low),
                               _kshiftri_mask32(lb_illseq, 1));

    /* fix illegal sequencing in the main block */
    _mm512_mask_storeu_epi16(
        (uint16_t *)out, _cvtmask32_u32(mask),
        _mm512_mask_blend_epi16(block_illseq, block,
                                _mm512_set1_epi16(replacement)));
  } else {
    _mm512_mask_storeu_epi16((uint16_t *)out, _cvtmask32_u32(mask), block);
  }
  out[n - 1] = scalar::utf16::is_high_surrogate<big_endian>(out[n - 1])
                   ? replacement
                   : out[n - 1];
}

template <endianness big_endian>
void utf16fix_avx512(const char16_t *in, size_t n, char16_t *out) {
  const char16_t replacement = scalar::utf16::replacement<big_endian>();
  size_t i;

  if (n == 0)
    return;
  else if (n < 33) {
    utf16fix_runt<big_endian>(in, n, out);
    return;
  }
  out[0] =
      scalar::utf16::is_low_surrogate<big_endian>(in[0]) ? replacement : in[0];

  /* duplicate code to have the compiler specialise utf16fix_block() */
  if (in == out) {
    for (i = 1; i + 32 < n; i += 32) {
      utf16fix_block<big_endian, true>(out + i, in + i);
    }

    utf16fix_block<big_endian, true>(out + n - 32, in + n - 32);
  } else {
    for (i = 1; i + 32 < n; i += 32) {
      utf16fix_block<big_endian, false>(out + i, in + i);
    }

    utf16fix_block<big_endian, false>(out + n - 32, in + n - 32);
  }

  out[n - 1] = scalar::utf16::is_high_surrogate<big_endian>(out[n - 1])
                   ? replacement
                   : out[n - 1];
}

Line	Count	Source (jump to first uncovered line)
1		/*
2		* Process one block of 32 characters. If in_place is false,
3		* copy the block from in to out. If there is a sequencing
4		* error in the block, overwrite the illsequenced characters
5		* with the replacement character. This function reads one
6		* character before the beginning of the buffer as a lookback.
7		* If that character is illsequenced, it too is overwritten.
8		*/
9		template <endianness big_endian, bool in_place>
10	0	simdutf_really_inline void utf16fix_block(char16_t out, const char16_t in) {
11	0	const char16_t replacement = scalar::utf16::replacement<big_endian>();
12	0	auto swap_if_needed = [](uint16_t c) -> uint16_t {
13	0	return !simdutf::match_system(big_endian) ? scalar::u16_swap_bytes(c) : c;
14	0	}; Unexecuted instantiation: simdutf.cpp:simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)0, true>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)0, false>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)1, true>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)1, false>(char16_t, char16_t const)::{lambda(unsigned short)#1}::operator()(unsigned short) const
15
16	0	__m512i lookback, block, lb_masked, block_masked;
17	0	__mmask32 lb_is_high, block_is_low, illseq;
18
19	0	lookback = _mm512_loadu_si512((const __m512i *)(in - 1));
20	0	block = _mm512_loadu_si512((const __m512i *)in);
21	0	lb_masked =
22	0	_mm512_and_epi32(lookback, _mm512_set1_epi16(swap_if_needed(0xfc00U)));
23	0	block_masked =
24	0	_mm512_and_epi32(block, _mm512_set1_epi16(swap_if_needed(0xfc00U)));
25
26	0	lb_is_high = _mm512_cmpeq_epi16_mask(
27	0	lb_masked, _mm512_set1_epi16(swap_if_needed(0xd800U)));
28	0	block_is_low = _mm512_cmpeq_epi16_mask(
29	0	block_masked, _mm512_set1_epi16(swap_if_needed(0xdc00U)));
30	0	illseq = _kxor_mask32(lb_is_high, block_is_low);
31	0	if (!_ktestz_mask32_u8(illseq, illseq)) {
32	0	__mmask32 lb_illseq, block_illseq;
33
34		/* compute the cause of the illegal sequencing */
35	0	lb_illseq = _kandn_mask32(block_is_low, lb_is_high);
36	0	block_illseq = _kor_mask32(_kandn_mask32(lb_is_high, block_is_low),
37	0	_kshiftri_mask32(lb_illseq, 1));
38
39		/* fix illegal sequencing in the lookback */
40	0	lb_illseq = _kand_mask32(lb_illseq, _cvtu32_mask32(1));
41	0	_mm512_mask_storeu_epi16(out - 1, lb_illseq,
42	0	_mm512_set1_epi16(replacement));
43
44		/* fix illegal sequencing in the main block */
45	0	if (in_place) {
46	0	_mm512_mask_storeu_epi16(out, block_illseq,
47	0	_mm512_set1_epi16(replacement));
48	0	} else {
49	0	_mm512_storeu_epi32(
50	0	out, _mm512_mask_blend_epi16(block_illseq, block,
51	0	_mm512_set1_epi16(replacement)));
52	0	}
53	0	} else if (!in_place) {
54	0	_mm512_storeu_si512((__m512i *)out, block);
55	0	}
56	0	} Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)0, true>(char16_t, char16_t const) Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)0, false>(char16_t, char16_t const) Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)1, true>(char16_t, char16_t const) Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_block<(simdutf::endianness)1, false>(char16_t, char16_t const)
57
58		/*
59		* Special case for inputs of 0--32 bytes. Works for both in-place and
60		* out-of-place operation.
61		*/
62		template <endianness big_endian>
63	0	void utf16fix_runt(const char16_t in, size_t n, char16_t out) {
64	0	const char16_t replacement = scalar::utf16::replacement<big_endian>();
65	0	auto swap_if_needed = [](uint16_t c) -> uint16_t {
66	0	return !simdutf::match_system(big_endian) ? scalar::u16_swap_bytes(c) : c;
67	0	}; Unexecuted instantiation: simdutf.cpp:simdutf::icelake::(anonymous namespace)::utf16fix_runt<(simdutf::endianness)0>(char16_t const, unsigned long, char16_t)::{lambda(unsigned short)#1}::operator()(unsigned short) const Unexecuted instantiation: simdutf.cpp:simdutf::icelake::(anonymous namespace)::utf16fix_runt<(simdutf::endianness)1>(char16_t const, unsigned long, char16_t)::{lambda(unsigned short)#1}::operator()(unsigned short) const
68	0	__m512i lookback, block, lb_masked, block_masked;
69	0	__mmask32 lb_is_high, block_is_low, illseq;
70	0	uint32_t mask = 0xFFFFFFFF >> (32 - n);
71	0	lookback = _mm512_maskz_loadu_epi16(_cvtmask32_u32(mask << 1),
72	0	(const uint16_t *)(in - 1));
73	0	block = _mm512_maskz_loadu_epi16(_cvtmask32_u32(mask), (const uint16_t *)in);
74	0	lb_masked =
75	0	_mm512_and_epi32(lookback, _mm512_set1_epi16(swap_if_needed(0xfc00u)));
76	0	block_masked =
77	0	_mm512_and_epi32(block, _mm512_set1_epi16(swap_if_needed(0xfc00u)));
78
79	0	lb_is_high = _mm512_cmpeq_epi16_mask(
80	0	lb_masked, _mm512_set1_epi16(swap_if_needed(0xd800u)));
81	0	block_is_low = _mm512_cmpeq_epi16_mask(
82	0	block_masked, _mm512_set1_epi16(swap_if_needed(0xdc00u)));
83	0	illseq = _kxor_mask32(lb_is_high, block_is_low);
84	0	if (!_ktestz_mask32_u8(illseq, illseq)) {
85	0	__mmask32 lb_illseq, block_illseq;
86
87		/* compute the cause of the illegal sequencing */
88	0	lb_illseq = _kandn_mask32(block_is_low, lb_is_high);
89	0	block_illseq = _kor_mask32(_kandn_mask32(lb_is_high, block_is_low),
90	0	_kshiftri_mask32(lb_illseq, 1));
91
92		/* fix illegal sequencing in the main block */
93	0	_mm512_mask_storeu_epi16(
94	0	(uint16_t *)out, _cvtmask32_u32(mask),
95	0	_mm512_mask_blend_epi16(block_illseq, block,
96	0	_mm512_set1_epi16(replacement)));
97	0	} else {
98	0	_mm512_mask_storeu_epi16((uint16_t *)out, _cvtmask32_u32(mask), block);
99	0	}
100	0	out[n - 1] = scalar::utf16::is_high_surrogate<big_endian>(out[n - 1])
101	0	? replacement
102	0	: out[n - 1];
103	0	} Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_runt<(simdutf::endianness)0>(char16_t const, unsigned long, char16_t) Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_runt<(simdutf::endianness)1>(char16_t const, unsigned long, char16_t)
104
105		template <endianness big_endian>
106	0	void utf16fix_avx512(const char16_t in, size_t n, char16_t out) {
107	0	const char16_t replacement = scalar::utf16::replacement<big_endian>();
108	0	size_t i;
109
110	0	if (n == 0)
111	0	return;
112	0	else if (n < 33) {
113	0	utf16fix_runt<big_endian>(in, n, out);
114	0	return;
115	0	}
116	0	out[0] =
117	0	scalar::utf16::is_low_surrogate<big_endian>(in[0]) ? replacement : in[0];
118
119		/* duplicate code to have the compiler specialise utf16fix_block() */
120	0	if (in == out) {
121	0	for (i = 1; i + 32 < n; i += 32) {
122	0	utf16fix_block<big_endian, true>(out + i, in + i);
123	0	}
124
125	0	utf16fix_block<big_endian, true>(out + n - 32, in + n - 32);
126	0	} else {
127	0	for (i = 1; i + 32 < n; i += 32) {
128	0	utf16fix_block<big_endian, false>(out + i, in + i);
129	0	}
130
131	0	utf16fix_block<big_endian, false>(out + n - 32, in + n - 32);
132	0	}
133
134	0	out[n - 1] = scalar::utf16::is_high_surrogate<big_endian>(out[n - 1])
135	0	? replacement
136	0	: out[n - 1];
137	0	} Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_avx512<(simdutf::endianness)0>(char16_t const, unsigned long, char16_t) Unexecuted instantiation: simdutf.cpp:void simdutf::icelake::(anonymous namespace)::utf16fix_avx512<(simdutf::endianness)1>(char16_t const, unsigned long, char16_t)

Coverage Report

Created: 2025-07-23 06:28