/rust/registry/src/index.crates.io-6f17d22bba15001f/simdutf8-0.1.4/src/implementation/algorithm.rs

Source (jump to first uncovered line)
/// Macros requires newtypes in scope:
/// `SimdU8Value` - implementation of SIMD primitives
/// `SimdInput` - which  holds 64 bytes of SIMD input
/// `TempSimdChunk` - correctly aligned `TempSimdChunk`, either `TempSimdChunkA16` or `TempSimdChunkA32`

macro_rules! algorithm_simd {
    ($feat:expr) => {
        use crate::{basic, compat};

        impl Utf8CheckAlgorithm<SimdU8Value> {
            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn default() -> Self {
                Self {
                    prev: SimdU8Value::splat0(),
                    incomplete: SimdU8Value::splat0(),
                    error: SimdU8Value::splat0(),
                }
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn check_incomplete_pending(&mut self) {
                self.error = self.error.or(self.incomplete);
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn is_incomplete(input: SimdU8Value) -> SimdU8Value {
                input.saturating_sub(SimdU8Value::from_32_cut_off_leading(
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0xff,
                    0b1111_0000 - 1,
                    0b1110_0000 - 1,
                    0b1100_0000 - 1,
                ))
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            #[allow(clippy::too_many_lines)]
            unsafe fn check_special_cases(input: SimdU8Value, prev1: SimdU8Value) -> SimdU8Value {
                const TOO_SHORT: u8 = 1 << 0;
                const TOO_LONG: u8 = 1 << 1;
                const OVERLONG_3: u8 = 1 << 2;
                const SURROGATE: u8 = 1 << 4;
                const OVERLONG_2: u8 = 1 << 5;
                const TWO_CONTS: u8 = 1 << 7;
                const TOO_LARGE: u8 = 1 << 3;
                const TOO_LARGE_1000: u8 = 1 << 6;
                const OVERLONG_4: u8 = 1 << 6;
                const CARRY: u8 = TOO_SHORT | TOO_LONG | TWO_CONTS;

                let byte_1_high = prev1.shr4().lookup_16(
                    TOO_LONG,
                    TOO_LONG,
                    TOO_LONG,
                    TOO_LONG,
                    TOO_LONG,
                    TOO_LONG,
                    TOO_LONG,
                    TOO_LONG,
                    TWO_CONTS,
                    TWO_CONTS,
                    TWO_CONTS,
                    TWO_CONTS,
                    TOO_SHORT | OVERLONG_2,
                    TOO_SHORT,
                    TOO_SHORT | OVERLONG_3 | SURROGATE,
                    TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4,
                );

                let byte_1_low = prev1.and(SimdU8Value::splat(0x0F)).lookup_16(
                    CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4,
                    CARRY | OVERLONG_2,
                    CARRY,
                    CARRY,
                    CARRY | TOO_LARGE,
                    CARRY | TOO_LARGE | TOO_LARGE_1000,
                    CARRY | TOO_LARGE | TOO_LARGE_1000,
                    CARRY | TOO_LARGE | TOO_LARGE_1000,
                    CARRY | TOO_LARGE | TOO_LARGE_1000,
                    CARRY | TOO_LARGE | TOO_LARGE_1000,
                    CARRY | TOO_LARGE | TOO_LARGE_1000,
                    CARRY | TOO_LARGE | TOO_LARGE_1000,
                    CARRY | TOO_LARGE | TOO_LARGE_1000,
                    CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE,
                    CARRY | TOO_LARGE | TOO_LARGE_1000,
                    CARRY | TOO_LARGE | TOO_LARGE_1000,
                );

                let byte_2_high = input.shr4().lookup_16(
                    TOO_SHORT,
                    TOO_SHORT,
                    TOO_SHORT,
                    TOO_SHORT,
                    TOO_SHORT,
                    TOO_SHORT,
                    TOO_SHORT,
                    TOO_SHORT,
                    TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4,
                    TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE,
                    TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE,
                    TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE,
                    TOO_SHORT,
                    TOO_SHORT,
                    TOO_SHORT,
                    TOO_SHORT,
                );

                byte_1_high.and(byte_1_low).and(byte_2_high)
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn check_multibyte_lengths(
                input: SimdU8Value,
                prev: SimdU8Value,
                special_cases: SimdU8Value,
            ) -> SimdU8Value {
                let prev2 = input.prev2(prev);
                let prev3 = input.prev3(prev);
                let must23 = Self::must_be_2_3_continuation(prev2, prev3);
                let must23_80 = must23.and(SimdU8Value::splat(0x80));
                must23_80.xor(special_cases)
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn has_error(&self) -> bool {
                self.error.any_bit_set()
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn check_bytes(&mut self, input: SimdU8Value) {
                let prev1 = input.prev1(self.prev);
                let sc = Self::check_special_cases(input, prev1);
                self.error = self
                    .error
                    .or(Self::check_multibyte_lengths(input, self.prev, sc));
                self.prev = input;
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn check_utf8(&mut self, input: SimdInput) {
                if input.is_ascii() {
                    self.check_incomplete_pending();
                } else {
                    self.check_block(input);
                }
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            #[allow(unconditional_panic)] // does not panic because len is checked
            #[allow(const_err)] // the same, but for Rust 1.38.0
            unsafe fn check_block(&mut self, input: SimdInput) {
                // WORKAROUND
                // necessary because the for loop is not unrolled on ARM64
                if input.vals.len() == 2 {
                    self.check_bytes(input.vals[0]);
                    self.check_bytes(input.vals[1]);
                    self.incomplete = Self::is_incomplete(input.vals[1]);
                } else if input.vals.len() == 4 {
                    self.check_bytes(input.vals[0]);
                    self.check_bytes(input.vals[1]);
                    self.check_bytes(input.vals[2]);
                    self.check_bytes(input.vals[3]);
                    self.incomplete = Self::is_incomplete(input.vals[3]);
                } else {
                    panic!("Unsupported number of chunks");
                }
            }
        }

        /// Validation implementation for CPUs supporting the SIMD extension (see module).
        ///
        /// # Errors
        /// Returns the zero-sized [`basic::Utf8Error`] on failure.
        ///
        /// # Safety
        /// This function is inherently unsafe because it is compiled with SIMD extensions
        /// enabled. Make sure that the CPU supports it before calling.
        ///
        #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
        #[inline]
        pub unsafe fn validate_utf8_basic(
            input: &[u8],
        ) -> core::result::Result<(), basic::Utf8Error> {
            use crate::implementation::helpers::SIMD_CHUNK_SIZE;
            let len = input.len();
            let mut algorithm = Utf8CheckAlgorithm::<SimdU8Value>::default();
            let mut idx: usize = 0;
            let iter_lim = len - (len % SIMD_CHUNK_SIZE);

            while idx < iter_lim {
                let simd_input = SimdInput::new(input.get_unchecked(idx as usize..));
                idx += SIMD_CHUNK_SIZE;
                if !simd_input.is_ascii() {
                    algorithm.check_block(simd_input);
                    break;
                }
            }

            while idx < iter_lim {
                if PREFETCH {
                    simd_prefetch(input.as_ptr().add(idx + SIMD_CHUNK_SIZE * 2));
                }
                let input = SimdInput::new(input.get_unchecked(idx as usize..));
                algorithm.check_utf8(input);
                idx += SIMD_CHUNK_SIZE;
            }

            if idx < len {
                let mut tmpbuf = TempSimdChunk::new();
                crate::implementation::helpers::memcpy_unaligned_nonoverlapping_inline_opt_lt_64(
                    input.as_ptr().add(idx),
                    tmpbuf.0.as_mut_ptr(),
                    len - idx,
                );
                let simd_input = SimdInput::new(&tmpbuf.0);
                algorithm.check_utf8(simd_input);
            }
            algorithm.check_incomplete_pending();
            if algorithm.has_error() {
                Err(basic::Utf8Error {})
            } else {
                Ok(())
            }
        }

        /// Validation implementation for CPUs supporting the SIMD extension (see module).
        ///
        /// # Errors
        /// Returns [`compat::Utf8Error`] with detailed error information on failure.
        ///
        /// # Safety
        /// This function is inherently unsafe because it is compiled with SIMD extensions
        /// enabled. Make sure that the CPU supports it before calling.
        ///
        #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
        #[inline]
        pub unsafe fn validate_utf8_compat(
            input: &[u8],
        ) -> core::result::Result<(), compat::Utf8Error> {
            validate_utf8_compat_simd0(input)
                .map_err(|idx| crate::implementation::helpers::get_compat_error(input, idx))
        }

        #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
        #[inline]
        unsafe fn validate_utf8_compat_simd0(input: &[u8]) -> core::result::Result<(), usize> {
            use crate::implementation::helpers::SIMD_CHUNK_SIZE;
            let len = input.len();
            let mut algorithm = Utf8CheckAlgorithm::<SimdU8Value>::default();
            let mut idx: usize = 0;
            let mut only_ascii = true;
            let iter_lim = len - (len % SIMD_CHUNK_SIZE);

            'outer: loop {
                if only_ascii {
                    while idx < iter_lim {
                        let simd_input = SimdInput::new(input.get_unchecked(idx as usize..));
                        if !simd_input.is_ascii() {
                            algorithm.check_block(simd_input);
                            if algorithm.has_error() {
                                return Err(idx);
                            } else {
                                only_ascii = false;
                                idx += SIMD_CHUNK_SIZE;
                                continue 'outer;
                            }
                        }
                        idx += SIMD_CHUNK_SIZE;
                    }
                } else {
                    while idx < iter_lim {
                        if PREFETCH {
                            simd_prefetch(input.as_ptr().add(idx + SIMD_CHUNK_SIZE * 2));
                        }
                        let simd_input = SimdInput::new(input.get_unchecked(idx as usize..));
                        if simd_input.is_ascii() {
                            algorithm.check_incomplete_pending();
                            if algorithm.has_error() {
                                return Err(idx);
                            } else {
                                // we are in pure ASCII territory again
                                only_ascii = true;
                                idx += SIMD_CHUNK_SIZE;
                                continue 'outer;
                            }
                        } else {
                            algorithm.check_block(simd_input);
                            if algorithm.has_error() {
                                return Err(idx);
                            }
                        }
                        idx += SIMD_CHUNK_SIZE;
                    }
                }
                break;
            }
            if idx < len {
                let mut tmpbuf = TempSimdChunk::new();
                crate::implementation::helpers::memcpy_unaligned_nonoverlapping_inline_opt_lt_64(
                    input.as_ptr().add(idx),
                    tmpbuf.0.as_mut_ptr(),
                    len - idx,
                );
                let simd_input = SimdInput::new(&tmpbuf.0);

                algorithm.check_utf8(simd_input);
            }
            algorithm.check_incomplete_pending();
            if algorithm.has_error() {
                Err(idx)
            } else {
                Ok(())
            }
        }

        /// Low-level implementation of the [`basic::imp::Utf8Validator`] trait.
        ///
        /// This is implementation requires CPU SIMD features specified by the module it resides in.
        /// It is undefined behavior to call it if the required CPU features are not
        /// available.
        #[cfg(feature = "public_imp")]
        pub struct Utf8ValidatorImp {
            algorithm: Utf8CheckAlgorithm<SimdU8Value>,
            incomplete_data: [u8; 64],
            incomplete_len: usize,
        }

        #[cfg(feature = "public_imp")]
        impl Utf8ValidatorImp {
            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn update_from_incomplete_data(&mut self) {
                let simd_input = SimdInput::new(&self.incomplete_data);
                self.algorithm.check_utf8(simd_input);
                self.incomplete_len = 0;
            }
        }

        #[cfg(feature = "public_imp")]
        impl basic::imp::Utf8Validator for Utf8ValidatorImp {
            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            #[must_use]
            unsafe fn new() -> Self {
                Self {
                    algorithm: Utf8CheckAlgorithm::<SimdU8Value>::default(),
                    incomplete_data: [0; 64],
                    incomplete_len: 0,
                }
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn update(&mut self, mut input: &[u8]) {
                use crate::implementation::helpers::SIMD_CHUNK_SIZE;
                if input.is_empty() {
                    return;
                }
                if self.incomplete_len != 0 {
                    let to_copy =
                        core::cmp::min(SIMD_CHUNK_SIZE - self.incomplete_len, input.len());
                    self.incomplete_data
                        .as_mut_ptr()
                        .add(self.incomplete_len)
                        .copy_from_nonoverlapping(input.as_ptr(), to_copy);
                    if self.incomplete_len + to_copy == SIMD_CHUNK_SIZE {
                        self.update_from_incomplete_data();
                        input = &input[to_copy..];
                    } else {
                        self.incomplete_len += to_copy;
                        return;
                    }
                }
                let len = input.len();
                let mut idx: usize = 0;
                let iter_lim = len - (len % SIMD_CHUNK_SIZE);
                while idx < iter_lim {
                    let input = SimdInput::new(input.get_unchecked(idx as usize..));
                    self.algorithm.check_utf8(input);
                    idx += SIMD_CHUNK_SIZE;
                }
                if idx < len {
                    let to_copy = len - idx;
                    self.incomplete_data
                        .as_mut_ptr()
                        .copy_from_nonoverlapping(input.as_ptr().add(idx), to_copy);
                    self.incomplete_len = to_copy;
                }
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn finalize(mut self) -> core::result::Result<(), basic::Utf8Error> {
                if self.incomplete_len != 0 {
                    for i in &mut self.incomplete_data[self.incomplete_len..] {
                        *i = 0;
                    }
                    self.update_from_incomplete_data();
                }
                self.algorithm.check_incomplete_pending();
                if self.algorithm.has_error() {
                    Err(basic::Utf8Error {})
                } else {
                    Ok(())
                }
            }
        }

        /// Low-level implementation of the [`basic::imp::ChunkedUtf8Validator`] trait.
        ///
        /// This is implementation requires CPU SIMD features specified by the module it resides in.
        /// It is undefined behavior to call it if the required CPU features are not
        /// available.
        #[cfg(feature = "public_imp")]
        pub struct ChunkedUtf8ValidatorImp {
            algorithm: Utf8CheckAlgorithm<SimdU8Value>,
        }

        #[cfg(feature = "public_imp")]
        impl basic::imp::ChunkedUtf8Validator for ChunkedUtf8ValidatorImp {
            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            #[must_use]
            unsafe fn new() -> Self {
                Self {
                    algorithm: Utf8CheckAlgorithm::<SimdU8Value>::default(),
                }
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn update_from_chunks(&mut self, input: &[u8]) {
                use crate::implementation::helpers::SIMD_CHUNK_SIZE;

                assert!(
                    input.len() % SIMD_CHUNK_SIZE == 0,
                    "Input size must be a multiple of 64."
                );
                for chunk in input.chunks_exact(SIMD_CHUNK_SIZE) {
                    let input = SimdInput::new(chunk);
                    self.algorithm.check_utf8(input);
                }
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn finalize(
                mut self,
                remaining_input: core::option::Option<&[u8]>,
            ) -> core::result::Result<(), basic::Utf8Error> {
                use crate::implementation::helpers::SIMD_CHUNK_SIZE;

                if let Some(mut remaining_input) = remaining_input {
                    if !remaining_input.is_empty() {
                        let len = remaining_input.len();
                        let chunks_lim = len - (len % SIMD_CHUNK_SIZE);
                        if chunks_lim > 0 {
                            self.update_from_chunks(&remaining_input[..chunks_lim]);
                        }
                        let rem = len - chunks_lim;
                        if rem > 0 {
                            remaining_input = &remaining_input[chunks_lim..];
                            let mut tmpbuf = TempSimdChunk::new();
                            tmpbuf.0.as_mut_ptr().copy_from_nonoverlapping(
                                remaining_input.as_ptr(),
                                remaining_input.len(),
                            );
                            let simd_input = SimdInput::new(&tmpbuf.0);
                            self.algorithm.check_utf8(simd_input);
                        }
                    }
                }
                self.algorithm.check_incomplete_pending();
                if self.algorithm.has_error() {
                    Err(basic::Utf8Error {})
                } else {
                    Ok(())
                }
            }
        }
    };
}

macro_rules! simd_input_128_bit {
    ($feat:expr) => {
        #[repr(C)]
        struct SimdInput {
            vals: [SimdU8Value; 4],
        }

        impl SimdInput {
            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            #[allow(clippy::cast_ptr_alignment)]
            unsafe fn new(ptr: &[u8]) -> Self {
                Self {
                    vals: [
                        SimdU8Value::load_from(ptr.as_ptr()),
                        SimdU8Value::load_from(ptr.as_ptr().add(16)),
                        SimdU8Value::load_from(ptr.as_ptr().add(32)),
                        SimdU8Value::load_from(ptr.as_ptr().add(48)),
                    ],
                }
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn is_ascii(&self) -> bool {
                let r1 = self.vals[0].or(self.vals[1]);
                let r2 = self.vals[2].or(self.vals[3]);
                let r = r1.or(r2);
                r.is_ascii()
            }
        }
    };
}

macro_rules! simd_input_256_bit {
    ($feat:expr) => {
        #[repr(C)]
        struct SimdInput {
            vals: [SimdU8Value; 2],
        }

        impl SimdInput {
            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            #[allow(clippy::cast_ptr_alignment)]
            unsafe fn new(ptr: &[u8]) -> Self {
                Self {
                    vals: [
                        SimdU8Value::load_from(ptr.as_ptr()),
                        SimdU8Value::load_from(ptr.as_ptr().add(32)),
                    ],
                }
            }

            #[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
            #[inline]
            unsafe fn is_ascii(&self) -> bool {
                self.vals[0].or(self.vals[1]).is_ascii()
            }
        }
    };
}

Coverage Report

Created: 2024-10-16 07:58

Line	Count	Source (jump to first uncovered line)
1		/// Macros requires newtypes in scope:
2		/// `SimdU8Value` - implementation of SIMD primitives
3		/// `SimdInput` - which holds 64 bytes of SIMD input
4		/// `TempSimdChunk` - correctly aligned `TempSimdChunk`, either `TempSimdChunkA16` or `TempSimdChunkA32`
5
6		macro_rules! algorithm_simd {
7		($feat:expr) => {
8		use crate::{basic, compat};
9
10		impl Utf8CheckAlgorithm<SimdU8Value> {
11		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
12		#[inline]
13	0	unsafe fn default() -> Self {
14	0	Self {
15	0	prev: SimdU8Value::splat0(),
16	0	incomplete: SimdU8Value::splat0(),
17	0	error: SimdU8Value::splat0(),
18	0	}
19	0	} Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m256i>>>::default Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m128i>>>::default
20
21		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
22		#[inline]
23	0	unsafe fn check_incomplete_pending(&mut self) {
24	0	self.error = self.error.or(self.incomplete);
25	0	} Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m256i>>>::check_incomplete_pending Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m128i>>>::check_incomplete_pending
26
27		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
28		#[inline]
29	0	unsafe fn is_incomplete(input: SimdU8Value) -> SimdU8Value {
30	0	input.saturating_sub(SimdU8Value::from_32_cut_off_leading(
31	0	0xff,
32	0	0xff,
33	0	0xff,
34	0	0xff,
35	0	0xff,
36	0	0xff,
37	0	0xff,
38	0	0xff,
39	0	0xff,
40	0	0xff,
41	0	0xff,
42	0	0xff,
43	0	0xff,
44	0	0xff,
45	0	0xff,
46	0	0xff,
47	0	0xff,
48	0	0xff,
49	0	0xff,
50	0	0xff,
51	0	0xff,
52	0	0xff,
53	0	0xff,
54	0	0xff,
55	0	0xff,
56	0	0xff,
57	0	0xff,
58	0	0xff,
59	0	0xff,
60	0	0b1111_0000 - 1,
61	0	0b1110_0000 - 1,
62	0	0b1100_0000 - 1,
63	0	))
64	0	} Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m256i>>>::is_incomplete Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m128i>>>::is_incomplete
65
66		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
67		#[inline]
68		#[allow(clippy::too_many_lines)]
69	0	unsafe fn check_special_cases(input: SimdU8Value, prev1: SimdU8Value) -> SimdU8Value {
70	0	const TOO_SHORT: u8 = 1 << 0;
71	0	const TOO_LONG: u8 = 1 << 1;
72	0	const OVERLONG_3: u8 = 1 << 2;
73	0	const SURROGATE: u8 = 1 << 4;
74	0	const OVERLONG_2: u8 = 1 << 5;
75	0	const TWO_CONTS: u8 = 1 << 7;
76	0	const TOO_LARGE: u8 = 1 << 3;
77	0	const TOO_LARGE_1000: u8 = 1 << 6;
78	0	const OVERLONG_4: u8 = 1 << 6;
79	0	const CARRY: u8 = TOO_SHORT \| TOO_LONG \| TWO_CONTS;
80	0
81	0	let byte_1_high = prev1.shr4().lookup_16(
82	0	TOO_LONG,
83	0	TOO_LONG,
84	0	TOO_LONG,
85	0	TOO_LONG,
86	0	TOO_LONG,
87	0	TOO_LONG,
88	0	TOO_LONG,
89	0	TOO_LONG,
90	0	TWO_CONTS,
91	0	TWO_CONTS,
92	0	TWO_CONTS,
93	0	TWO_CONTS,
94	0	TOO_SHORT \| OVERLONG_2,
95	0	TOO_SHORT,
96	0	TOO_SHORT \| OVERLONG_3 \| SURROGATE,
97	0	TOO_SHORT \| TOO_LARGE \| TOO_LARGE_1000 \| OVERLONG_4,
98	0	);
99	0
100	0	let byte_1_low = prev1.and(SimdU8Value::splat(0x0F)).lookup_16(
101	0	CARRY \| OVERLONG_3 \| OVERLONG_2 \| OVERLONG_4,
102	0	CARRY \| OVERLONG_2,
103	0	CARRY,
104	0	CARRY,
105	0	CARRY \| TOO_LARGE,
106	0	CARRY \| TOO_LARGE \| TOO_LARGE_1000,
107	0	CARRY \| TOO_LARGE \| TOO_LARGE_1000,
108	0	CARRY \| TOO_LARGE \| TOO_LARGE_1000,
109	0	CARRY \| TOO_LARGE \| TOO_LARGE_1000,
110	0	CARRY \| TOO_LARGE \| TOO_LARGE_1000,
111	0	CARRY \| TOO_LARGE \| TOO_LARGE_1000,
112	0	CARRY \| TOO_LARGE \| TOO_LARGE_1000,
113	0	CARRY \| TOO_LARGE \| TOO_LARGE_1000,
114	0	CARRY \| TOO_LARGE \| TOO_LARGE_1000 \| SURROGATE,
115	0	CARRY \| TOO_LARGE \| TOO_LARGE_1000,
116	0	CARRY \| TOO_LARGE \| TOO_LARGE_1000,
117	0	);
118	0
119	0	let byte_2_high = input.shr4().lookup_16(
120	0	TOO_SHORT,
121	0	TOO_SHORT,
122	0	TOO_SHORT,
123	0	TOO_SHORT,
124	0	TOO_SHORT,
125	0	TOO_SHORT,
126	0	TOO_SHORT,
127	0	TOO_SHORT,
128	0	TOO_LONG \| OVERLONG_2 \| TWO_CONTS \| OVERLONG_3 \| TOO_LARGE_1000 \| OVERLONG_4,
129	0	TOO_LONG \| OVERLONG_2 \| TWO_CONTS \| OVERLONG_3 \| TOO_LARGE,
130	0	TOO_LONG \| OVERLONG_2 \| TWO_CONTS \| SURROGATE \| TOO_LARGE,
131	0	TOO_LONG \| OVERLONG_2 \| TWO_CONTS \| SURROGATE \| TOO_LARGE,
132	0	TOO_SHORT,
133	0	TOO_SHORT,
134	0	TOO_SHORT,
135	0	TOO_SHORT,
136	0	);
137	0
138	0	byte_1_high.and(byte_1_low).and(byte_2_high)
139	0	} Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m256i>>>::check_special_cases Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m128i>>>::check_special_cases
140
141		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
142		#[inline]
143	0	unsafe fn check_multibyte_lengths(
144	0	input: SimdU8Value,
145	0	prev: SimdU8Value,
146	0	special_cases: SimdU8Value,
147	0	) -> SimdU8Value {
148	0	let prev2 = input.prev2(prev);
149	0	let prev3 = input.prev3(prev);
150	0	let must23 = Self::must_be_2_3_continuation(prev2, prev3);
151	0	let must23_80 = must23.and(SimdU8Value::splat(0x80));
152	0	must23_80.xor(special_cases)
153	0	} Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m256i>>>::check_multibyte_lengths Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m128i>>>::check_multibyte_lengths
154
155		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
156		#[inline]
157	0	unsafe fn has_error(&self) -> bool {
158	0	self.error.any_bit_set()
159	0	} Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m256i>>>::has_error Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m128i>>>::has_error
160
161		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
162		#[inline]
163	0	unsafe fn check_bytes(&mut self, input: SimdU8Value) {
164	0	let prev1 = input.prev1(self.prev);
165	0	let sc = Self::check_special_cases(input, prev1);
166	0	self.error = self
167	0	.error
168	0	.or(Self::check_multibyte_lengths(input, self.prev, sc));
169	0	self.prev = input;
170	0	} Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m256i>>>::check_bytes Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m128i>>>::check_bytes
171
172		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
173		#[inline]
174	0	unsafe fn check_utf8(&mut self, input: SimdInput) {
175	0	if input.is_ascii() {
176	0	self.check_incomplete_pending();
177	0	} else {
178	0	self.check_block(input);
179	0	}
180	0	} Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m256i>>>::check_utf8 Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m128i>>>::check_utf8
181
182		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
183		#[inline]
184		#[allow(unconditional_panic)] // does not panic because len is checked
185		#[allow(const_err)] // the same, but for Rust 1.38.0
186	0	unsafe fn check_block(&mut self, input: SimdInput) {
187	0	// WORKAROUND
188	0	// necessary because the for loop is not unrolled on ARM64
189	0	if input.vals.len() == 2 {
190	0	self.check_bytes(input.vals[0]);
191	0	self.check_bytes(input.vals[1]);
192	0	self.incomplete = Self::is_incomplete(input.vals[1]);
193	0	} else if input.vals.len() == 4 {
194	0	self.check_bytes(input.vals[0]);
195	0	self.check_bytes(input.vals[1]);
196	0	self.check_bytes(input.vals[2]);
197	0	self.check_bytes(input.vals[3]);
198	0	self.incomplete = Self::is_incomplete(input.vals[3]);
199	0	} else {
200	0	panic!("Unsupported number of chunks");
201		}
202	0	} Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m256i>>>::check_block Unexecuted instantiation: <simdutf8::implementation::helpers::Utf8CheckAlgorithm<simdutf8::implementation::helpers::SimdU8Value<core::core_arch::x86::__m128i>>>::check_block
203		}
204
205		/// Validation implementation for CPUs supporting the SIMD extension (see module).
206		///
207		/// # Errors
208		/// Returns the zero-sized [`basic::Utf8Error`] on failure.
209		///
210		/// # Safety
211		/// This function is inherently unsafe because it is compiled with SIMD extensions
212		/// enabled. Make sure that the CPU supports it before calling.
213		///
214		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
215		#[inline]
216	0	pub unsafe fn validate_utf8_basic(
217	0	input: &[u8],
218	0	) -> core::result::Result<(), basic::Utf8Error> {
219	0	use crate::implementation::helpers::SIMD_CHUNK_SIZE;
220	0	let len = input.len();
221	0	let mut algorithm = Utf8CheckAlgorithm::<SimdU8Value>::default();
222	0	let mut idx: usize = 0;
223	0	let iter_lim = len - (len % SIMD_CHUNK_SIZE);
224
225	0	while idx < iter_lim {
226	0	let simd_input = SimdInput::new(input.get_unchecked(idx as usize..));
227	0	idx += SIMD_CHUNK_SIZE;
228	0	if !simd_input.is_ascii() {
229	0	algorithm.check_block(simd_input);
230	0	break;
231	0	}
232		}
233
234	0	while idx < iter_lim {
235	0	if PREFETCH {
236	0	simd_prefetch(input.as_ptr().add(idx + SIMD_CHUNK_SIZE * 2));
237	0	}
238	0	let input = SimdInput::new(input.get_unchecked(idx as usize..));
239	0	algorithm.check_utf8(input);
240	0	idx += SIMD_CHUNK_SIZE;
241		}
242
243	0	if idx < len {
244	0	let mut tmpbuf = TempSimdChunk::new();
245	0	crate::implementation::helpers::memcpy_unaligned_nonoverlapping_inline_opt_lt_64(
246	0	input.as_ptr().add(idx),
247	0	tmpbuf.0.as_mut_ptr(),
248	0	len - idx,
249	0	);
250	0	let simd_input = SimdInput::new(&tmpbuf.0);
251	0	algorithm.check_utf8(simd_input);
252	0	}
253	0	algorithm.check_incomplete_pending();
254	0	if algorithm.has_error() {
255	0	Err(basic::Utf8Error {})
256		} else {
257	0	Ok(())
258		}
259	0	} Unexecuted instantiation: simdutf8::implementation::x86::avx2::validate_utf8_basic Unexecuted instantiation: simdutf8::implementation::x86::sse42::validate_utf8_basic
260
261		/// Validation implementation for CPUs supporting the SIMD extension (see module).
262		///
263		/// # Errors
264		/// Returns [`compat::Utf8Error`] with detailed error information on failure.
265		///
266		/// # Safety
267		/// This function is inherently unsafe because it is compiled with SIMD extensions
268		/// enabled. Make sure that the CPU supports it before calling.
269		///
270		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
271		#[inline]
272	0	pub unsafe fn validate_utf8_compat(
273	0	input: &[u8],
274	0	) -> core::result::Result<(), compat::Utf8Error> {
275	0	validate_utf8_compat_simd0(input)
276	0	.map_err(\|idx\| crate::implementation::helpers::get_compat_error(input, idx)) Unexecuted instantiation: simdutf8::implementation::x86::avx2::validate_utf8_compat::{closure#0} Unexecuted instantiation: simdutf8::implementation::x86::sse42::validate_utf8_compat::{closure#0}
277	0	} Unexecuted instantiation: simdutf8::implementation::x86::avx2::validate_utf8_compat Unexecuted instantiation: simdutf8::implementation::x86::sse42::validate_utf8_compat
278
279		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
280		#[inline]
281	0	unsafe fn validate_utf8_compat_simd0(input: &[u8]) -> core::result::Result<(), usize> {
282	0	use crate::implementation::helpers::SIMD_CHUNK_SIZE;
283	0	let len = input.len();
284	0	let mut algorithm = Utf8CheckAlgorithm::<SimdU8Value>::default();
285	0	let mut idx: usize = 0;
286	0	let mut only_ascii = true;
287	0	let iter_lim = len - (len % SIMD_CHUNK_SIZE);
288
289	0	'outer: loop {
290	0	if only_ascii {
291	0	while idx < iter_lim {
292	0	let simd_input = SimdInput::new(input.get_unchecked(idx as usize..));
293	0	if !simd_input.is_ascii() {
294	0	algorithm.check_block(simd_input);
295	0	if algorithm.has_error() {
296	0	return Err(idx);
297		} else {
298	0	only_ascii = false;
299	0	idx += SIMD_CHUNK_SIZE;
300	0	continue 'outer;
301		}
302	0	}
303	0	idx += SIMD_CHUNK_SIZE;
304		}
305		} else {
306	0	while idx < iter_lim {
307	0	if PREFETCH {
308	0	simd_prefetch(input.as_ptr().add(idx + SIMD_CHUNK_SIZE * 2));
309	0	}
310	0	let simd_input = SimdInput::new(input.get_unchecked(idx as usize..));
311	0	if simd_input.is_ascii() {
312	0	algorithm.check_incomplete_pending();
313	0	if algorithm.has_error() {
314	0	return Err(idx);
315		} else {
316		// we are in pure ASCII territory again
317	0	only_ascii = true;
318	0	idx += SIMD_CHUNK_SIZE;
319	0	continue 'outer;
320		}
321		} else {
322	0	algorithm.check_block(simd_input);
323	0	if algorithm.has_error() {
324	0	return Err(idx);
325	0	}
326	0	}
327	0	idx += SIMD_CHUNK_SIZE;
328		}
329		}
330	0	break;
331	0	}
332	0	if idx < len {
333	0	let mut tmpbuf = TempSimdChunk::new();
334	0	crate::implementation::helpers::memcpy_unaligned_nonoverlapping_inline_opt_lt_64(
335	0	input.as_ptr().add(idx),
336	0	tmpbuf.0.as_mut_ptr(),
337	0	len - idx,
338	0	);
339	0	let simd_input = SimdInput::new(&tmpbuf.0);
340	0
341	0	algorithm.check_utf8(simd_input);
342	0	}
343	0	algorithm.check_incomplete_pending();
344	0	if algorithm.has_error() {
345	0	Err(idx)
346		} else {
347	0	Ok(())
348		}
349	0	} Unexecuted instantiation: simdutf8::implementation::x86::avx2::validate_utf8_compat_simd0 Unexecuted instantiation: simdutf8::implementation::x86::sse42::validate_utf8_compat_simd0
350
351		/// Low-level implementation of the [`basic::imp::Utf8Validator`] trait.
352		///
353		/// This is implementation requires CPU SIMD features specified by the module it resides in.
354		/// It is undefined behavior to call it if the required CPU features are not
355		/// available.
356		#[cfg(feature = "public_imp")]
357		pub struct Utf8ValidatorImp {
358		algorithm: Utf8CheckAlgorithm<SimdU8Value>,
359		incomplete_data: [u8; 64],
360		incomplete_len: usize,
361		}
362
363		#[cfg(feature = "public_imp")]
364		impl Utf8ValidatorImp {
365		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
366		#[inline]
367		unsafe fn update_from_incomplete_data(&mut self) {
368		let simd_input = SimdInput::new(&self.incomplete_data);
369		self.algorithm.check_utf8(simd_input);
370		self.incomplete_len = 0;
371		}
372		}
373
374		#[cfg(feature = "public_imp")]
375		impl basic::imp::Utf8Validator for Utf8ValidatorImp {
376		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
377		#[inline]
378		#[must_use]
379		unsafe fn new() -> Self {
380		Self {
381		algorithm: Utf8CheckAlgorithm::<SimdU8Value>::default(),
382		incomplete_data: [0; 64],
383		incomplete_len: 0,
384		}
385		}
386
387		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
388		#[inline]
389		unsafe fn update(&mut self, mut input: &[u8]) {
390		use crate::implementation::helpers::SIMD_CHUNK_SIZE;
391		if input.is_empty() {
392		return;
393		}
394		if self.incomplete_len != 0 {
395		let to_copy =
396		core::cmp::min(SIMD_CHUNK_SIZE - self.incomplete_len, input.len());
397		self.incomplete_data
398		.as_mut_ptr()
399		.add(self.incomplete_len)
400		.copy_from_nonoverlapping(input.as_ptr(), to_copy);
401		if self.incomplete_len + to_copy == SIMD_CHUNK_SIZE {
402		self.update_from_incomplete_data();
403		input = &input[to_copy..];
404		} else {
405		self.incomplete_len += to_copy;
406		return;
407		}
408		}
409		let len = input.len();
410		let mut idx: usize = 0;
411		let iter_lim = len - (len % SIMD_CHUNK_SIZE);
412		while idx < iter_lim {
413		let input = SimdInput::new(input.get_unchecked(idx as usize..));
414		self.algorithm.check_utf8(input);
415		idx += SIMD_CHUNK_SIZE;
416		}
417		if idx < len {
418		let to_copy = len - idx;
419		self.incomplete_data
420		.as_mut_ptr()
421		.copy_from_nonoverlapping(input.as_ptr().add(idx), to_copy);
422		self.incomplete_len = to_copy;
423		}
424		}
425
426		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
427		#[inline]
428		unsafe fn finalize(mut self) -> core::result::Result<(), basic::Utf8Error> {
429		if self.incomplete_len != 0 {
430		for i in &mut self.incomplete_data[self.incomplete_len..] {
431		*i = 0;
432		}
433		self.update_from_incomplete_data();
434		}
435		self.algorithm.check_incomplete_pending();
436		if self.algorithm.has_error() {
437		Err(basic::Utf8Error {})
438		} else {
439		Ok(())
440		}
441		}
442		}
443
444		/// Low-level implementation of the [`basic::imp::ChunkedUtf8Validator`] trait.
445		///
446		/// This is implementation requires CPU SIMD features specified by the module it resides in.
447		/// It is undefined behavior to call it if the required CPU features are not
448		/// available.
449		#[cfg(feature = "public_imp")]
450		pub struct ChunkedUtf8ValidatorImp {
451		algorithm: Utf8CheckAlgorithm<SimdU8Value>,
452		}
453
454		#[cfg(feature = "public_imp")]
455		impl basic::imp::ChunkedUtf8Validator for ChunkedUtf8ValidatorImp {
456		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
457		#[inline]
458		#[must_use]
459		unsafe fn new() -> Self {
460		Self {
461		algorithm: Utf8CheckAlgorithm::<SimdU8Value>::default(),
462		}
463		}
464
465		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
466		#[inline]
467		unsafe fn update_from_chunks(&mut self, input: &[u8]) {
468		use crate::implementation::helpers::SIMD_CHUNK_SIZE;
469
470		assert!(
471		input.len() % SIMD_CHUNK_SIZE == 0,
472		"Input size must be a multiple of 64."
473		);
474		for chunk in input.chunks_exact(SIMD_CHUNK_SIZE) {
475		let input = SimdInput::new(chunk);
476		self.algorithm.check_utf8(input);
477		}
478		}
479
480		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
481		#[inline]
482		unsafe fn finalize(
483		mut self,
484		remaining_input: core::option::Option<&[u8]>,
485		) -> core::result::Result<(), basic::Utf8Error> {
486		use crate::implementation::helpers::SIMD_CHUNK_SIZE;
487
488		if let Some(mut remaining_input) = remaining_input {
489		if !remaining_input.is_empty() {
490		let len = remaining_input.len();
491		let chunks_lim = len - (len % SIMD_CHUNK_SIZE);
492		if chunks_lim > 0 {
493		self.update_from_chunks(&remaining_input[..chunks_lim]);
494		}
495		let rem = len - chunks_lim;
496		if rem > 0 {
497		remaining_input = &remaining_input[chunks_lim..];
498		let mut tmpbuf = TempSimdChunk::new();
499		tmpbuf.0.as_mut_ptr().copy_from_nonoverlapping(
500		remaining_input.as_ptr(),
501		remaining_input.len(),
502		);
503		let simd_input = SimdInput::new(&tmpbuf.0);
504		self.algorithm.check_utf8(simd_input);
505		}
506		}
507		}
508		self.algorithm.check_incomplete_pending();
509		if self.algorithm.has_error() {
510		Err(basic::Utf8Error {})
511		} else {
512		Ok(())
513		}
514		}
515		}
516		};
517		}
518
519		macro_rules! simd_input_128_bit {
520		($feat:expr) => {
521		#[repr(C)]
522		struct SimdInput {
523		vals: [SimdU8Value; 4],
524		}
525
526		impl SimdInput {
527		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
528		#[inline]
529		#[allow(clippy::cast_ptr_alignment)]
530	0	unsafe fn new(ptr: &[u8]) -> Self {
531	0	Self {
532	0	vals: [
533	0	SimdU8Value::load_from(ptr.as_ptr()),
534	0	SimdU8Value::load_from(ptr.as_ptr().add(16)),
535	0	SimdU8Value::load_from(ptr.as_ptr().add(32)),
536	0	SimdU8Value::load_from(ptr.as_ptr().add(48)),
537	0	],
538	0	}
539	0	}
540
541		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
542		#[inline]
543	0	unsafe fn is_ascii(&self) -> bool {
544	0	let r1 = self.vals[0].or(self.vals[1]);
545	0	let r2 = self.vals[2].or(self.vals[3]);
546	0	let r = r1.or(r2);
547	0	r.is_ascii()
548	0	}
549		}
550		};
551		}
552
553		macro_rules! simd_input_256_bit {
554		($feat:expr) => {
555		#[repr(C)]
556		struct SimdInput {
557		vals: [SimdU8Value; 2],
558		}
559
560		impl SimdInput {
561		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
562		#[inline]
563		#[allow(clippy::cast_ptr_alignment)]
564	0	unsafe fn new(ptr: &[u8]) -> Self {
565	0	Self {
566	0	vals: [
567	0	SimdU8Value::load_from(ptr.as_ptr()),
568	0	SimdU8Value::load_from(ptr.as_ptr().add(32)),
569	0	],
570	0	}
571	0	}
572
573		#[cfg_attr(not(target_arch="aarch64"), target_feature(enable = $feat))]
574		#[inline]
575	0	unsafe fn is_ascii(&self) -> bool {
576	0	self.vals[0].or(self.vals[1]).is_ascii()
577	0	}
578		}
579		};
580		}