Coverage Report

Created: 2026-10-28 07:08

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/rust/registry/src/index.crates.io-1949cf8c6b5b557f/escape-simd-0.1.0/src/simd/avx2.rs
Line
Count
Source
1
#[cfg(target_arch = "x86")]
2
use std::arch::x86::*;
3
#[cfg(target_arch = "x86_64")]
4
use std::arch::x86_64::*;
5
6
use std::ops::{BitAnd, BitOr, BitOrAssign};
7
8
use super::{Mask, Simd, traits::BitMask, util::escape_unchecked};
9
10
#[cfg(any(target_os = "linux", target_os = "macos"))]
11
use super::util::check_cross_page;
12
#[cfg(feature = "html")]
13
use super::util::{HTML_HI_NIBBLE_TAB, HTML_LO_NIBBLE_TAB};
14
15
const LANES: usize = 32;
16
const CHUNK: usize = LANES * 4;
17
18
#[derive(Debug, Clone, Copy)]
19
#[repr(transparent)]
20
pub struct Simd256u(__m256i);
21
22
#[derive(Debug, Clone, Copy)]
23
#[repr(transparent)]
24
pub struct Mask256(__m256i);
25
26
impl Mask for Mask256 {
27
    type BitMask = u32;
28
    type Element = u8;
29
30
    #[inline(always)]
31
0
    fn bitmask(self) -> Self::BitMask {
32
0
        unsafe { _mm256_movemask_epi8(self.0) as u32 }
33
0
    }
34
}
35
36
impl BitAnd<Mask256> for Mask256 {
37
    type Output = Self;
38
39
    #[inline(always)]
40
0
    fn bitand(self, rhs: Mask256) -> Self::Output {
41
0
        unsafe { Mask256(_mm256_and_si256(self.0, rhs.0)) }
42
0
    }
43
}
44
45
impl BitOr<Mask256> for Mask256 {
46
    type Output = Self;
47
48
    #[inline(always)]
49
0
    fn bitor(self, rhs: Mask256) -> Self::Output {
50
0
        unsafe { Mask256(_mm256_or_si256(self.0, rhs.0)) }
51
0
    }
52
}
53
54
impl BitOrAssign<Mask256> for Mask256 {
55
    #[inline(always)]
56
0
    fn bitor_assign(&mut self, rhs: Mask256) {
57
0
        unsafe { self.0 = _mm256_or_si256(self.0, rhs.0) }
58
0
    }
59
}
60
61
impl Simd for Simd256u {
62
    const LANES: usize = LANES;
63
    type Mask = Mask256;
64
    type Element = u8;
65
66
    #[inline(always)]
67
0
    unsafe fn loadu(ptr: *const u8) -> Self {
68
0
        unsafe { Simd256u(_mm256_loadu_si256(ptr as *const __m256i)) }
69
0
    }
70
71
    #[inline(always)]
72
0
    unsafe fn storeu(&self, ptr: *mut u8) {
73
0
        unsafe { _mm256_storeu_si256(ptr as *mut __m256i, self.0) }
74
0
    }
75
76
    #[inline(always)]
77
0
    fn eq(&self, rhs: &Self) -> Self::Mask {
78
        unsafe {
79
0
            let eq = _mm256_cmpeq_epi8(self.0, rhs.0);
80
0
            Mask256(eq)
81
        }
82
0
    }
83
84
    #[inline(always)]
85
0
    fn splat(ch: u8) -> Self {
86
0
        unsafe { Simd256u(_mm256_set1_epi8(ch as i8)) }
87
0
    }
88
89
    #[inline(always)]
90
0
    fn le(&self, rhs: &Self) -> Self::Mask {
91
        unsafe {
92
0
            let max = _mm256_max_epu8(self.0, rhs.0);
93
0
            let eq = _mm256_cmpeq_epi8(max, rhs.0);
94
0
            Mask256(eq)
95
        }
96
0
    }
97
}
98
99
/// simdjson-style two-shuffle HTML classifier: look up the byte's low and high
100
/// nibbles in the shared tables and test whether they share a bit. Indices are
101
/// masked to 0..=15, so `_mm256_shuffle_epi8` never sees the high bit set and
102
/// cannot zero out a lane spuriously.
103
#[cfg(feature = "html")]
104
#[inline(always)]
105
fn escaped_mask_html(v: Simd256u) -> u32 {
106
    unsafe {
107
        let nibble = _mm256_set1_epi8(0x0f);
108
        let lo_idx = _mm256_and_si256(v.0, nibble);
109
        // No byte-wise shift: shift 16-bit lanes right and mask off the
110
        // bleeding bits to recover the high nibble of each byte.
111
        let hi_idx = _mm256_and_si256(_mm256_srli_epi16::<4>(v.0), nibble);
112
        let lo_tab = _mm256_broadcastsi128_si256(_mm_loadu_si128(
113
            HTML_LO_NIBBLE_TAB.as_ptr() as *const __m128i
114
        ));
115
        let hi_tab = _mm256_broadcastsi128_si256(_mm_loadu_si128(
116
            HTML_HI_NIBBLE_TAB.as_ptr() as *const __m128i
117
        ));
118
        let lo = _mm256_shuffle_epi8(lo_tab, lo_idx);
119
        let hi = _mm256_shuffle_epi8(hi_tab, hi_idx);
120
        let and = _mm256_and_si256(lo, hi);
121
        // `and != 0` marks an escape; invert the cheaper `and == 0` compare.
122
        let non_escape = _mm256_cmpeq_epi8(and, _mm256_setzero_si256());
123
        !(_mm256_movemask_epi8(non_escape) as u32)
124
    }
125
}
126
127
#[inline(always)]
128
0
fn escaped_mask<const HTML: bool>(v: Simd256u) -> u32 {
129
    #[cfg(feature = "html")]
130
    if HTML {
131
        return escaped_mask_html(v);
132
    }
133
0
    let x1f = Simd256u::splat(0x1f); // 0x00 ~ 0x20
134
0
    let blash = Simd256u::splat(b'\\');
135
0
    let quote = Simd256u::splat(b'"');
136
0
    let v = v.le(&x1f) | v.eq(&blash) | v.eq(&quote);
137
0
    v.bitmask()
138
0
}
139
140
#[target_feature(enable = "avx2")]
141
0
pub unsafe fn format_string<const HTML: bool>(value: &str, dst: &mut [u8]) -> usize {
142
    unsafe {
143
0
        let slice = value.as_bytes();
144
0
        let mut sptr = slice.as_ptr();
145
0
        let mut dptr = dst.as_mut_ptr();
146
0
        let dstart = dptr;
147
0
        let mut nb: usize = slice.len();
148
149
0
        if !HTML {
150
0
            *dptr = b'"';
151
0
            dptr = dptr.add(1);
152
0
        }
153
154
        // Process CHUNK (4 * LANES = 128 bytes) at a time
155
0
        while nb >= CHUNK {
156
            // Load 4 SIMD vectors
157
0
            let v1 = Simd256u::loadu(sptr);
158
0
            let v2 = Simd256u::loadu(sptr.add(LANES));
159
0
            let v3 = Simd256u::loadu(sptr.add(LANES * 2));
160
0
            let v4 = Simd256u::loadu(sptr.add(LANES * 3));
161
162
            // Check all 4 masks
163
0
            let mask1 = escaped_mask::<HTML>(v1);
164
0
            let mask2 = escaped_mask::<HTML>(v2);
165
0
            let mask3 = escaped_mask::<HTML>(v3);
166
0
            let mask4 = escaped_mask::<HTML>(v4);
167
168
            // Fast path: if all vectors are clean, write the entire chunk
169
0
            if mask1.all_zero() && mask2.all_zero() && mask3.all_zero() && mask4.all_zero() {
170
0
                v1.storeu(dptr);
171
0
                v2.storeu(dptr.add(LANES));
172
0
                v3.storeu(dptr.add(LANES * 2));
173
0
                v4.storeu(dptr.add(LANES * 3));
174
0
                nb -= CHUNK;
175
0
                dptr = dptr.add(CHUNK);
176
0
                sptr = sptr.add(CHUNK);
177
0
            } else {
178
                // Slow path: handle escape character
179
                // Process v1
180
0
                v1.storeu(dptr);
181
0
                if !mask1.all_zero() {
182
0
                    let cn = mask1.first_offset();
183
0
                    nb -= cn;
184
0
                    dptr = dptr.add(cn);
185
0
                    sptr = sptr.add(cn);
186
0
                    escape_unchecked::<HTML>(&mut sptr, &mut nb, &mut dptr);
187
0
                    continue;
188
0
                }
189
0
                nb -= LANES;
190
0
                dptr = dptr.add(LANES);
191
0
                sptr = sptr.add(LANES);
192
193
                // Process v2
194
0
                v2.storeu(dptr);
195
0
                if !mask2.all_zero() {
196
0
                    let cn = mask2.first_offset();
197
0
                    nb -= cn;
198
0
                    dptr = dptr.add(cn);
199
0
                    sptr = sptr.add(cn);
200
0
                    escape_unchecked::<HTML>(&mut sptr, &mut nb, &mut dptr);
201
0
                    continue;
202
0
                }
203
0
                nb -= LANES;
204
0
                dptr = dptr.add(LANES);
205
0
                sptr = sptr.add(LANES);
206
207
                // Process v3
208
0
                v3.storeu(dptr);
209
0
                if !mask3.all_zero() {
210
0
                    let cn = mask3.first_offset();
211
0
                    nb -= cn;
212
0
                    dptr = dptr.add(cn);
213
0
                    sptr = sptr.add(cn);
214
0
                    escape_unchecked::<HTML>(&mut sptr, &mut nb, &mut dptr);
215
0
                    continue;
216
0
                }
217
0
                nb -= LANES;
218
0
                dptr = dptr.add(LANES);
219
0
                sptr = sptr.add(LANES);
220
221
                // Process v4
222
0
                v4.storeu(dptr);
223
0
                if !mask4.all_zero() {
224
0
                    let cn = mask4.first_offset();
225
0
                    nb -= cn;
226
0
                    dptr = dptr.add(cn);
227
0
                    sptr = sptr.add(cn);
228
0
                    escape_unchecked::<HTML>(&mut sptr, &mut nb, &mut dptr);
229
0
                    continue;
230
0
                }
231
0
                nb -= LANES;
232
0
                dptr = dptr.add(LANES);
233
0
                sptr = sptr.add(LANES);
234
            }
235
        }
236
237
        // Process remaining LANES bytes at a time
238
0
        while nb >= LANES {
239
0
            let v = Simd256u::loadu(sptr);
240
0
            v.storeu(dptr);
241
0
            let mask = escaped_mask::<HTML>(v);
242
243
0
            if mask.all_zero() {
244
0
                nb -= LANES;
245
0
                dptr = dptr.add(LANES);
246
0
                sptr = sptr.add(LANES);
247
0
            } else {
248
0
                let cn = mask.first_offset();
249
0
                nb -= cn;
250
0
                dptr = dptr.add(cn);
251
0
                sptr = sptr.add(cn);
252
0
                escape_unchecked::<HTML>(&mut sptr, &mut nb, &mut dptr);
253
0
            }
254
        }
255
256
        // Handle remaining bytes
257
0
        let mut placeholder: [u8; LANES] = [0; LANES];
258
0
        while nb > 0 {
259
0
            let v = {
260
                #[cfg(not(any(target_os = "linux", target_os = "macos")))]
261
                {
262
                    std::ptr::copy_nonoverlapping(sptr, placeholder[..].as_mut_ptr(), nb);
263
                    Simd256u::loadu(placeholder[..].as_ptr())
264
                }
265
                #[cfg(any(target_os = "linux", target_os = "macos"))]
266
                {
267
0
                    if check_cross_page(sptr, LANES) {
268
0
                        std::ptr::copy_nonoverlapping(sptr, placeholder[..].as_mut_ptr(), nb);
269
0
                        Simd256u::loadu(placeholder[..].as_ptr())
270
                    } else {
271
                        #[cfg(any(debug_assertions, miri, feature = "asan"))]
272
                        {
273
                            std::ptr::copy_nonoverlapping(sptr, placeholder[..].as_mut_ptr(), nb);
274
                            Simd256u::loadu(placeholder[..].as_ptr())
275
                        }
276
                        #[cfg(not(any(debug_assertions, miri)))]
277
                        {
278
0
                            Simd256u::loadu(sptr)
279
                        }
280
                    }
281
                }
282
            };
283
284
0
            v.storeu(dptr);
285
0
            let mask = escaped_mask::<HTML>(v).clear_high_bits(LANES - nb);
286
287
0
            if mask.all_zero() {
288
0
                dptr = dptr.add(nb);
289
0
                break;
290
0
            } else {
291
0
                let cn = mask.first_offset();
292
0
                nb -= cn;
293
0
                dptr = dptr.add(cn);
294
0
                sptr = sptr.add(cn);
295
0
                escape_unchecked::<HTML>(&mut sptr, &mut nb, &mut dptr);
296
0
            }
297
        }
298
299
0
        if !HTML {
300
0
            *dptr = b'"';
301
0
            dptr = dptr.add(1);
302
0
        }
303
0
        dptr as usize - dstart as usize
304
    }
305
0
}