/rust/registry/src/index.crates.io-1949cf8c6b5b557f/escape-simd-0.1.0/src/simd/avx2.rs
Line | Count | Source |
1 | | #[cfg(target_arch = "x86")] |
2 | | use std::arch::x86::*; |
3 | | #[cfg(target_arch = "x86_64")] |
4 | | use std::arch::x86_64::*; |
5 | | |
6 | | use std::ops::{BitAnd, BitOr, BitOrAssign}; |
7 | | |
8 | | use super::{Mask, Simd, traits::BitMask, util::escape_unchecked}; |
9 | | |
10 | | #[cfg(any(target_os = "linux", target_os = "macos"))] |
11 | | use super::util::check_cross_page; |
12 | | #[cfg(feature = "html")] |
13 | | use super::util::{HTML_HI_NIBBLE_TAB, HTML_LO_NIBBLE_TAB}; |
14 | | |
15 | | const LANES: usize = 32; |
16 | | const CHUNK: usize = LANES * 4; |
17 | | |
18 | | #[derive(Debug, Clone, Copy)] |
19 | | #[repr(transparent)] |
20 | | pub struct Simd256u(__m256i); |
21 | | |
22 | | #[derive(Debug, Clone, Copy)] |
23 | | #[repr(transparent)] |
24 | | pub struct Mask256(__m256i); |
25 | | |
26 | | impl Mask for Mask256 { |
27 | | type BitMask = u32; |
28 | | type Element = u8; |
29 | | |
30 | | #[inline(always)] |
31 | 0 | fn bitmask(self) -> Self::BitMask { |
32 | 0 | unsafe { _mm256_movemask_epi8(self.0) as u32 } |
33 | 0 | } |
34 | | } |
35 | | |
36 | | impl BitAnd<Mask256> for Mask256 { |
37 | | type Output = Self; |
38 | | |
39 | | #[inline(always)] |
40 | 0 | fn bitand(self, rhs: Mask256) -> Self::Output { |
41 | 0 | unsafe { Mask256(_mm256_and_si256(self.0, rhs.0)) } |
42 | 0 | } |
43 | | } |
44 | | |
45 | | impl BitOr<Mask256> for Mask256 { |
46 | | type Output = Self; |
47 | | |
48 | | #[inline(always)] |
49 | 0 | fn bitor(self, rhs: Mask256) -> Self::Output { |
50 | 0 | unsafe { Mask256(_mm256_or_si256(self.0, rhs.0)) } |
51 | 0 | } |
52 | | } |
53 | | |
54 | | impl BitOrAssign<Mask256> for Mask256 { |
55 | | #[inline(always)] |
56 | 0 | fn bitor_assign(&mut self, rhs: Mask256) { |
57 | 0 | unsafe { self.0 = _mm256_or_si256(self.0, rhs.0) } |
58 | 0 | } |
59 | | } |
60 | | |
61 | | impl Simd for Simd256u { |
62 | | const LANES: usize = LANES; |
63 | | type Mask = Mask256; |
64 | | type Element = u8; |
65 | | |
66 | | #[inline(always)] |
67 | 0 | unsafe fn loadu(ptr: *const u8) -> Self { |
68 | 0 | unsafe { Simd256u(_mm256_loadu_si256(ptr as *const __m256i)) } |
69 | 0 | } |
70 | | |
71 | | #[inline(always)] |
72 | 0 | unsafe fn storeu(&self, ptr: *mut u8) { |
73 | 0 | unsafe { _mm256_storeu_si256(ptr as *mut __m256i, self.0) } |
74 | 0 | } |
75 | | |
76 | | #[inline(always)] |
77 | 0 | fn eq(&self, rhs: &Self) -> Self::Mask { |
78 | | unsafe { |
79 | 0 | let eq = _mm256_cmpeq_epi8(self.0, rhs.0); |
80 | 0 | Mask256(eq) |
81 | | } |
82 | 0 | } |
83 | | |
84 | | #[inline(always)] |
85 | 0 | fn splat(ch: u8) -> Self { |
86 | 0 | unsafe { Simd256u(_mm256_set1_epi8(ch as i8)) } |
87 | 0 | } |
88 | | |
89 | | #[inline(always)] |
90 | 0 | fn le(&self, rhs: &Self) -> Self::Mask { |
91 | | unsafe { |
92 | 0 | let max = _mm256_max_epu8(self.0, rhs.0); |
93 | 0 | let eq = _mm256_cmpeq_epi8(max, rhs.0); |
94 | 0 | Mask256(eq) |
95 | | } |
96 | 0 | } |
97 | | } |
98 | | |
99 | | /// simdjson-style two-shuffle HTML classifier: look up the byte's low and high |
100 | | /// nibbles in the shared tables and test whether they share a bit. Indices are |
101 | | /// masked to 0..=15, so `_mm256_shuffle_epi8` never sees the high bit set and |
102 | | /// cannot zero out a lane spuriously. |
103 | | #[cfg(feature = "html")] |
104 | | #[inline(always)] |
105 | | fn escaped_mask_html(v: Simd256u) -> u32 { |
106 | | unsafe { |
107 | | let nibble = _mm256_set1_epi8(0x0f); |
108 | | let lo_idx = _mm256_and_si256(v.0, nibble); |
109 | | // No byte-wise shift: shift 16-bit lanes right and mask off the |
110 | | // bleeding bits to recover the high nibble of each byte. |
111 | | let hi_idx = _mm256_and_si256(_mm256_srli_epi16::<4>(v.0), nibble); |
112 | | let lo_tab = _mm256_broadcastsi128_si256(_mm_loadu_si128( |
113 | | HTML_LO_NIBBLE_TAB.as_ptr() as *const __m128i |
114 | | )); |
115 | | let hi_tab = _mm256_broadcastsi128_si256(_mm_loadu_si128( |
116 | | HTML_HI_NIBBLE_TAB.as_ptr() as *const __m128i |
117 | | )); |
118 | | let lo = _mm256_shuffle_epi8(lo_tab, lo_idx); |
119 | | let hi = _mm256_shuffle_epi8(hi_tab, hi_idx); |
120 | | let and = _mm256_and_si256(lo, hi); |
121 | | // `and != 0` marks an escape; invert the cheaper `and == 0` compare. |
122 | | let non_escape = _mm256_cmpeq_epi8(and, _mm256_setzero_si256()); |
123 | | !(_mm256_movemask_epi8(non_escape) as u32) |
124 | | } |
125 | | } |
126 | | |
127 | | #[inline(always)] |
128 | 0 | fn escaped_mask<const HTML: bool>(v: Simd256u) -> u32 { |
129 | | #[cfg(feature = "html")] |
130 | | if HTML { |
131 | | return escaped_mask_html(v); |
132 | | } |
133 | 0 | let x1f = Simd256u::splat(0x1f); // 0x00 ~ 0x20 |
134 | 0 | let blash = Simd256u::splat(b'\\'); |
135 | 0 | let quote = Simd256u::splat(b'"'); |
136 | 0 | let v = v.le(&x1f) | v.eq(&blash) | v.eq("e); |
137 | 0 | v.bitmask() |
138 | 0 | } |
139 | | |
140 | | #[target_feature(enable = "avx2")] |
141 | 0 | pub unsafe fn format_string<const HTML: bool>(value: &str, dst: &mut [u8]) -> usize { |
142 | | unsafe { |
143 | 0 | let slice = value.as_bytes(); |
144 | 0 | let mut sptr = slice.as_ptr(); |
145 | 0 | let mut dptr = dst.as_mut_ptr(); |
146 | 0 | let dstart = dptr; |
147 | 0 | let mut nb: usize = slice.len(); |
148 | | |
149 | 0 | if !HTML { |
150 | 0 | *dptr = b'"'; |
151 | 0 | dptr = dptr.add(1); |
152 | 0 | } |
153 | | |
154 | | // Process CHUNK (4 * LANES = 128 bytes) at a time |
155 | 0 | while nb >= CHUNK { |
156 | | // Load 4 SIMD vectors |
157 | 0 | let v1 = Simd256u::loadu(sptr); |
158 | 0 | let v2 = Simd256u::loadu(sptr.add(LANES)); |
159 | 0 | let v3 = Simd256u::loadu(sptr.add(LANES * 2)); |
160 | 0 | let v4 = Simd256u::loadu(sptr.add(LANES * 3)); |
161 | | |
162 | | // Check all 4 masks |
163 | 0 | let mask1 = escaped_mask::<HTML>(v1); |
164 | 0 | let mask2 = escaped_mask::<HTML>(v2); |
165 | 0 | let mask3 = escaped_mask::<HTML>(v3); |
166 | 0 | let mask4 = escaped_mask::<HTML>(v4); |
167 | | |
168 | | // Fast path: if all vectors are clean, write the entire chunk |
169 | 0 | if mask1.all_zero() && mask2.all_zero() && mask3.all_zero() && mask4.all_zero() { |
170 | 0 | v1.storeu(dptr); |
171 | 0 | v2.storeu(dptr.add(LANES)); |
172 | 0 | v3.storeu(dptr.add(LANES * 2)); |
173 | 0 | v4.storeu(dptr.add(LANES * 3)); |
174 | 0 | nb -= CHUNK; |
175 | 0 | dptr = dptr.add(CHUNK); |
176 | 0 | sptr = sptr.add(CHUNK); |
177 | 0 | } else { |
178 | | // Slow path: handle escape character |
179 | | // Process v1 |
180 | 0 | v1.storeu(dptr); |
181 | 0 | if !mask1.all_zero() { |
182 | 0 | let cn = mask1.first_offset(); |
183 | 0 | nb -= cn; |
184 | 0 | dptr = dptr.add(cn); |
185 | 0 | sptr = sptr.add(cn); |
186 | 0 | escape_unchecked::<HTML>(&mut sptr, &mut nb, &mut dptr); |
187 | 0 | continue; |
188 | 0 | } |
189 | 0 | nb -= LANES; |
190 | 0 | dptr = dptr.add(LANES); |
191 | 0 | sptr = sptr.add(LANES); |
192 | | |
193 | | // Process v2 |
194 | 0 | v2.storeu(dptr); |
195 | 0 | if !mask2.all_zero() { |
196 | 0 | let cn = mask2.first_offset(); |
197 | 0 | nb -= cn; |
198 | 0 | dptr = dptr.add(cn); |
199 | 0 | sptr = sptr.add(cn); |
200 | 0 | escape_unchecked::<HTML>(&mut sptr, &mut nb, &mut dptr); |
201 | 0 | continue; |
202 | 0 | } |
203 | 0 | nb -= LANES; |
204 | 0 | dptr = dptr.add(LANES); |
205 | 0 | sptr = sptr.add(LANES); |
206 | | |
207 | | // Process v3 |
208 | 0 | v3.storeu(dptr); |
209 | 0 | if !mask3.all_zero() { |
210 | 0 | let cn = mask3.first_offset(); |
211 | 0 | nb -= cn; |
212 | 0 | dptr = dptr.add(cn); |
213 | 0 | sptr = sptr.add(cn); |
214 | 0 | escape_unchecked::<HTML>(&mut sptr, &mut nb, &mut dptr); |
215 | 0 | continue; |
216 | 0 | } |
217 | 0 | nb -= LANES; |
218 | 0 | dptr = dptr.add(LANES); |
219 | 0 | sptr = sptr.add(LANES); |
220 | | |
221 | | // Process v4 |
222 | 0 | v4.storeu(dptr); |
223 | 0 | if !mask4.all_zero() { |
224 | 0 | let cn = mask4.first_offset(); |
225 | 0 | nb -= cn; |
226 | 0 | dptr = dptr.add(cn); |
227 | 0 | sptr = sptr.add(cn); |
228 | 0 | escape_unchecked::<HTML>(&mut sptr, &mut nb, &mut dptr); |
229 | 0 | continue; |
230 | 0 | } |
231 | 0 | nb -= LANES; |
232 | 0 | dptr = dptr.add(LANES); |
233 | 0 | sptr = sptr.add(LANES); |
234 | | } |
235 | | } |
236 | | |
237 | | // Process remaining LANES bytes at a time |
238 | 0 | while nb >= LANES { |
239 | 0 | let v = Simd256u::loadu(sptr); |
240 | 0 | v.storeu(dptr); |
241 | 0 | let mask = escaped_mask::<HTML>(v); |
242 | | |
243 | 0 | if mask.all_zero() { |
244 | 0 | nb -= LANES; |
245 | 0 | dptr = dptr.add(LANES); |
246 | 0 | sptr = sptr.add(LANES); |
247 | 0 | } else { |
248 | 0 | let cn = mask.first_offset(); |
249 | 0 | nb -= cn; |
250 | 0 | dptr = dptr.add(cn); |
251 | 0 | sptr = sptr.add(cn); |
252 | 0 | escape_unchecked::<HTML>(&mut sptr, &mut nb, &mut dptr); |
253 | 0 | } |
254 | | } |
255 | | |
256 | | // Handle remaining bytes |
257 | 0 | let mut placeholder: [u8; LANES] = [0; LANES]; |
258 | 0 | while nb > 0 { |
259 | 0 | let v = { |
260 | | #[cfg(not(any(target_os = "linux", target_os = "macos")))] |
261 | | { |
262 | | std::ptr::copy_nonoverlapping(sptr, placeholder[..].as_mut_ptr(), nb); |
263 | | Simd256u::loadu(placeholder[..].as_ptr()) |
264 | | } |
265 | | #[cfg(any(target_os = "linux", target_os = "macos"))] |
266 | | { |
267 | 0 | if check_cross_page(sptr, LANES) { |
268 | 0 | std::ptr::copy_nonoverlapping(sptr, placeholder[..].as_mut_ptr(), nb); |
269 | 0 | Simd256u::loadu(placeholder[..].as_ptr()) |
270 | | } else { |
271 | | #[cfg(any(debug_assertions, miri, feature = "asan"))] |
272 | | { |
273 | | std::ptr::copy_nonoverlapping(sptr, placeholder[..].as_mut_ptr(), nb); |
274 | | Simd256u::loadu(placeholder[..].as_ptr()) |
275 | | } |
276 | | #[cfg(not(any(debug_assertions, miri)))] |
277 | | { |
278 | 0 | Simd256u::loadu(sptr) |
279 | | } |
280 | | } |
281 | | } |
282 | | }; |
283 | | |
284 | 0 | v.storeu(dptr); |
285 | 0 | let mask = escaped_mask::<HTML>(v).clear_high_bits(LANES - nb); |
286 | | |
287 | 0 | if mask.all_zero() { |
288 | 0 | dptr = dptr.add(nb); |
289 | 0 | break; |
290 | 0 | } else { |
291 | 0 | let cn = mask.first_offset(); |
292 | 0 | nb -= cn; |
293 | 0 | dptr = dptr.add(cn); |
294 | 0 | sptr = sptr.add(cn); |
295 | 0 | escape_unchecked::<HTML>(&mut sptr, &mut nb, &mut dptr); |
296 | 0 | } |
297 | | } |
298 | | |
299 | 0 | if !HTML { |
300 | 0 | *dptr = b'"'; |
301 | 0 | dptr = dptr.add(1); |
302 | 0 | } |
303 | 0 | dptr as usize - dstart as usize |
304 | | } |
305 | 0 | } |