/rust/registry/src/index.crates.io-1949cf8c6b5b557f/base64-simd-0.8.0/src/encode.rs
Line | Count | Source |
1 | | use crate::{Config, Kind}; |
2 | | use crate::{STANDARD_CHARSET, URL_SAFE_CHARSET}; |
3 | | |
4 | | use vsimd::isa::{NEON, SSE2, WASM128}; |
5 | | use vsimd::tools::{read, write}; |
6 | | use vsimd::vector::{V128, V256}; |
7 | | use vsimd::{matches_isa, POD}; |
8 | | use vsimd::{Scalable, SIMD128, SIMD256}; |
9 | | |
10 | | #[inline(always)] |
11 | 0 | pub(crate) const fn encoded_length_unchecked(len: usize, config: Config) -> usize { |
12 | 0 | let extra = len % 3; |
13 | 0 | if extra == 0 { |
14 | 0 | len / 3 * 4 |
15 | 0 | } else if config.extra.padding() { |
16 | 0 | len / 3 * 4 + 4 |
17 | | } else { |
18 | 0 | len / 3 * 4 + extra + 1 |
19 | | } |
20 | 0 | } |
21 | | |
22 | | #[inline(always)] |
23 | 0 | unsafe fn encode_bits24(src: *const u8, dst: *mut u8, charset: *const u8) { |
24 | 0 | let x = u32::from_be_bytes([0, read(src, 0), read(src, 1), read(src, 2)]); |
25 | 0 | let mut i = 0; |
26 | 0 | while i < 4 { |
27 | 0 | let bits = (x >> (18 - i * 6)) & 0x3f; |
28 | 0 | let y = read(charset, bits as usize); |
29 | 0 | write(dst, i, y); |
30 | 0 | i += 1; |
31 | 0 | } |
32 | 0 | } |
33 | | |
34 | | #[inline(always)] |
35 | 0 | unsafe fn encode_bits48(src: *const u8, dst: *mut u8, charset: *const u8) { |
36 | 0 | let x = u64::from_be_bytes(src.cast::<[u8; 8]>().read()); |
37 | 0 | let mut i = 0; |
38 | 0 | while i < 8 { |
39 | 0 | let bits = (x >> (58 - i * 6)) & 0x3f; |
40 | 0 | let y = read(charset, bits as usize); |
41 | 0 | write(dst, i, y); |
42 | 0 | i += 1; |
43 | 0 | } |
44 | 0 | } |
45 | | |
46 | | #[inline(always)] |
47 | 0 | unsafe fn encode_extra(extra: usize, src: *const u8, dst: *mut u8, charset: *const u8, padding: bool) { |
48 | 0 | match extra { |
49 | 0 | 0 => {} |
50 | | 1 => { |
51 | 0 | let x = read(src, 0); |
52 | 0 | let y1 = read(charset, (x >> 2) as usize); |
53 | 0 | let y2 = read(charset, ((x << 6) >> 2) as usize); |
54 | 0 | write(dst, 0, y1); |
55 | 0 | write(dst, 1, y2); |
56 | 0 | if padding { |
57 | 0 | write(dst, 2, b'='); |
58 | 0 | write(dst, 3, b'='); |
59 | 0 | } |
60 | | } |
61 | | 2 => { |
62 | 0 | let x1 = read(src, 0); |
63 | 0 | let x2 = read(src, 1); |
64 | 0 | let y1 = read(charset, (x1 >> 2) as usize); |
65 | 0 | let y2 = read(charset, (((x1 << 6) >> 2) | (x2 >> 4)) as usize); |
66 | 0 | let y3 = read(charset, ((x2 << 4) >> 2) as usize); |
67 | 0 | write(dst, 0, y1); |
68 | 0 | write(dst, 1, y2); |
69 | 0 | write(dst, 2, y3); |
70 | 0 | if padding { |
71 | 0 | write(dst, 3, b'='); |
72 | 0 | } |
73 | | } |
74 | 0 | _ => core::hint::unreachable_unchecked(), |
75 | | } |
76 | 0 | } |
77 | | |
78 | | #[inline] |
79 | 0 | pub(crate) unsafe fn encode_fallback(mut src: *const u8, mut len: usize, mut dst: *mut u8, config: Config) { |
80 | 0 | let kind = config.kind; |
81 | 0 | let padding = config.extra.padding(); |
82 | | |
83 | 0 | let charset = match kind { |
84 | 0 | Kind::Standard => STANDARD_CHARSET.as_ptr(), |
85 | 0 | Kind::UrlSafe => URL_SAFE_CHARSET.as_ptr(), |
86 | | }; |
87 | | |
88 | | const L: usize = 4; |
89 | 0 | while len >= L * 6 + 2 { |
90 | 0 | let mut i = 0; |
91 | 0 | while i < L { |
92 | 0 | encode_bits48(src, dst, charset); |
93 | 0 | src = src.add(6); |
94 | 0 | dst = dst.add(8); |
95 | 0 | i += 1; |
96 | 0 | } |
97 | 0 | len -= L * 6; |
98 | | } |
99 | | |
100 | 0 | while len >= 6 + 2 { |
101 | 0 | encode_bits48(src, dst, charset); |
102 | 0 | src = src.add(6); |
103 | 0 | dst = dst.add(8); |
104 | 0 | len -= 6; |
105 | 0 | } |
106 | | |
107 | 0 | let end = src.add(len / 3 * 3); |
108 | 0 | while src < end { |
109 | 0 | encode_bits24(src, dst, charset); |
110 | 0 | src = src.add(3); |
111 | 0 | dst = dst.add(4); |
112 | 0 | } |
113 | 0 | len %= 3; |
114 | | |
115 | 0 | encode_extra(len, src, dst, charset, padding); |
116 | 0 | } |
117 | | |
118 | | #[inline(always)] |
119 | 0 | pub(crate) unsafe fn encode_simd<S: SIMD256>( |
120 | 0 | s: S, |
121 | 0 | mut src: *const u8, |
122 | 0 | mut len: usize, |
123 | 0 | mut dst: *mut u8, |
124 | 0 | config: Config, |
125 | 0 | ) { |
126 | 0 | let kind = config.kind; |
127 | | |
128 | 0 | if len >= (6 + 24 + 4) { |
129 | 0 | let (charset, shift_lut) = match kind { |
130 | 0 | Kind::Standard => (STANDARD_CHARSET.as_ptr(), STANDARD_ENCODING_SHIFT_X2), |
131 | 0 | Kind::UrlSafe => (URL_SAFE_CHARSET.as_ptr(), URL_SAFE_ENCODING_SHIFT_X2), |
132 | | }; |
133 | | |
134 | 0 | for _ in 0..2 { |
135 | 0 | encode_bits24(src, dst, charset); |
136 | 0 | src = src.add(3); |
137 | 0 | dst = dst.add(4); |
138 | 0 | len -= 3; |
139 | 0 | } |
140 | | |
141 | 0 | while len >= (24 + 4) { |
142 | 0 | let x = s.v256_load_unaligned(src.sub(4)); |
143 | 0 | let y = encode_bytes24(s, x, shift_lut); |
144 | 0 | s.v256_store_unaligned(dst, y); |
145 | 0 | src = src.add(24); |
146 | 0 | dst = dst.add(32); |
147 | 0 | len -= 24; |
148 | 0 | } |
149 | 0 | } |
150 | | |
151 | 0 | if len >= 12 + 4 { |
152 | 0 | let shift_lut = match kind { |
153 | 0 | Kind::Standard => STANDARD_ENCODING_SHIFT, |
154 | 0 | Kind::UrlSafe => URL_SAFE_ENCODING_SHIFT, |
155 | | }; |
156 | | |
157 | 0 | let x = s.v128_load_unaligned(src); |
158 | 0 | let y = encode_bytes12(s, x, shift_lut); |
159 | 0 | s.v128_store_unaligned(dst, y); |
160 | 0 | src = src.add(12); |
161 | 0 | dst = dst.add(16); |
162 | 0 | len -= 12; |
163 | 0 | } |
164 | | |
165 | 0 | encode_fallback(src, len, dst, config); |
166 | 0 | } Unexecuted instantiation: base64_simd::encode::encode_simd::<vsimd::isa::AVX2> Unexecuted instantiation: base64_simd::encode::encode_simd::<vsimd::isa::SSSE3> |
167 | | |
168 | | const SPLIT_SHUFFLE: V256 = V256::from_bytes([ |
169 | | 0x05, 0x04, 0x06, 0x05, 0x08, 0x07, 0x09, 0x08, // |
170 | | 0x0b, 0x0a, 0x0c, 0x0b, 0x0e, 0x0d, 0x0f, 0x0e, // |
171 | | 0x01, 0x00, 0x02, 0x01, 0x04, 0x03, 0x05, 0x04, // |
172 | | 0x07, 0x06, 0x08, 0x07, 0x0a, 0x09, 0x0b, 0x0a, // |
173 | | ]); |
174 | | |
175 | | #[inline(always)] |
176 | 0 | fn split_bits_x2<S: SIMD256>(s: S, x: V256) -> V256 { |
177 | | // x: {????|AAAB|BBCC|CDDD|EEEF|FFGG|GHHH|????} |
178 | | |
179 | 0 | let x0 = s.u8x16x2_swizzle(x, SPLIT_SHUFFLE); |
180 | | // x0: {bbbbcccc|aaaaaabb|ccdddddd|bbbbcccc} x8 (1021) |
181 | | |
182 | 0 | if matches_isa!(S, SSE2) { |
183 | 0 | let m1 = s.u32x8_splat(u32::from_le_bytes([0x00, 0xfc, 0xc0, 0x0f])); |
184 | 0 | let x1 = s.v256_and(x0, m1); |
185 | | // x1: {00000000|aaaaaa00|cc000000|0000cccc} x8 |
186 | | |
187 | 0 | let m2 = s.u32x8_splat(u32::from_le_bytes([0xf0, 0x03, 0x3f, 0x00])); |
188 | 0 | let x2 = s.v256_and(x0, m2); |
189 | | // x2: {bbbb0000|000000bb|00dddddd|00000000} x8 |
190 | | |
191 | 0 | let m3 = s.u32x8_splat(u32::from_le_bytes([0x40, 0x00, 0x00, 0x04])); |
192 | 0 | let x3 = s.u16x16_mul_hi(x1, m3); |
193 | | // x3: {00aaaaaa|00000000|00cccccc|00000000} x8 |
194 | | |
195 | 0 | let m4 = s.u32x8_splat(u32::from_le_bytes([0x10, 0x00, 0x00, 0x01])); |
196 | 0 | let x4 = s.i16x16_mul_lo(x2, m4); |
197 | | // x4: {00000000|00bbbbbb|00000000|00dddddd} x8 |
198 | | |
199 | 0 | return s.v256_or(x3, x4); |
200 | | // {00aaaaaa|00bbbbbb|00cccccc|00dddddd} x8 |
201 | 0 | } |
202 | | |
203 | 0 | if matches_isa!(S, NEON | WASM128) { |
204 | 0 | let m1 = s.u32x8_splat(u32::from_le_bytes([0x00, 0xfc, 0x00, 0x00])); |
205 | 0 | let x1 = s.u16x16_shr::<10>(s.v256_and(x0, m1)); |
206 | | // x1: {00aaaaaa|000000000|00000000|00000000} x8 |
207 | | |
208 | 0 | let m2 = s.u32x8_splat(u32::from_le_bytes([0xf0, 0x03, 0x00, 0x00])); |
209 | 0 | let x2 = s.u16x16_shl::<4>(s.v256_and(x0, m2)); |
210 | | // x2: {00000000|00bbbbbb|00000000|00000000} x8 |
211 | | |
212 | 0 | let m3 = s.u32x8_splat(u32::from_le_bytes([0x00, 0x00, 0xc0, 0x0f])); |
213 | 0 | let x3 = s.u16x16_shr::<6>(s.v256_and(x0, m3)); |
214 | | // x3: {00000000|00000000|00cccccc|00000000} x8 |
215 | | |
216 | 0 | let m4 = s.u32x8_splat(u32::from_le_bytes([0x00, 0x00, 0x3f, 0x00])); |
217 | 0 | let x4 = s.u16x16_shl::<8>(s.v256_and(x0, m4)); |
218 | | // x4: {00000000|00000000|00000000|00dddddd} x8 |
219 | | |
220 | 0 | return s.v256_or(s.v256_or(x1, x2), s.v256_or(x3, x4)); |
221 | | // {00aaaaaa|00bbbbbb|00cccccc|00dddddd} x8 |
222 | 0 | } |
223 | | |
224 | 0 | unreachable!() |
225 | 0 | } Unexecuted instantiation: base64_simd::encode::split_bits_x2::<vsimd::isa::AVX2> Unexecuted instantiation: base64_simd::encode::split_bits_x2::<vsimd::isa::SSSE3> |
226 | | |
227 | | #[inline(always)] |
228 | 0 | fn split_bits_x1<S: SIMD128>(s: S, x: V128) -> V128 { |
229 | | // x: {AAAB|BBCC|CDDD|????} |
230 | | |
231 | | const SHUFFLE: V128 = SPLIT_SHUFFLE.to_v128x2().1; |
232 | 0 | let x0 = s.u8x16_swizzle(x, SHUFFLE); |
233 | | // x0: {bbbbcccc|aaaaaabb|ccdddddd|bbbbcccc} x8 (1021) |
234 | | |
235 | 0 | if matches_isa!(S, SSE2) { |
236 | 0 | let m1 = s.u32x4_splat(u32::from_le_bytes([0x00, 0xfc, 0xc0, 0x0f])); |
237 | 0 | let x1 = s.v128_and(x0, m1); |
238 | | |
239 | 0 | let m2 = s.u32x4_splat(u32::from_le_bytes([0xf0, 0x03, 0x3f, 0x00])); |
240 | 0 | let x2 = s.v128_and(x0, m2); |
241 | | |
242 | 0 | let m3 = s.u32x4_splat(u32::from_le_bytes([0x40, 0x00, 0x00, 0x04])); |
243 | 0 | let x3 = s.u16x8_mul_hi(x1, m3); |
244 | | |
245 | 0 | let m4 = s.u32x4_splat(u32::from_le_bytes([0x10, 0x00, 0x00, 0x01])); |
246 | 0 | let x4 = s.i16x8_mul_lo(x2, m4); |
247 | | |
248 | 0 | return s.v128_or(x3, x4); |
249 | 0 | } |
250 | | |
251 | 0 | if matches_isa!(S, NEON | WASM128) { |
252 | 0 | let m1 = s.u32x4_splat(u32::from_le_bytes([0x00, 0xfc, 0x00, 0x00])); |
253 | 0 | let x1 = s.u16x8_shr::<10>(s.v128_and(x0, m1)); |
254 | | |
255 | 0 | let m2 = s.u32x4_splat(u32::from_le_bytes([0xf0, 0x03, 0x00, 0x00])); |
256 | 0 | let x2 = s.u16x8_shl::<4>(s.v128_and(x0, m2)); |
257 | | |
258 | 0 | let m3 = s.u32x4_splat(u32::from_le_bytes([0x00, 0x00, 0xc0, 0x0f])); |
259 | 0 | let x3 = s.u16x8_shr::<6>(s.v128_and(x0, m3)); |
260 | | |
261 | 0 | let m4 = s.u32x4_splat(u32::from_le_bytes([0x00, 0x00, 0x3f, 0x00])); |
262 | 0 | let x4 = s.u16x8_shl::<8>(s.v128_and(x0, m4)); |
263 | | |
264 | 0 | return s.v128_or(s.v128_or(x1, x2), s.v128_or(x3, x4)); |
265 | 0 | } |
266 | | |
267 | 0 | unreachable!() |
268 | 0 | } Unexecuted instantiation: base64_simd::encode::split_bits_x1::<vsimd::isa::AVX2> Unexecuted instantiation: base64_simd::encode::split_bits_x1::<vsimd::isa::SSSE3> |
269 | | |
270 | | #[inline] |
271 | | const fn encoding_shift(charset: &'static [u8; 64]) -> V128 { |
272 | | // 0~25 'A' [13] |
273 | | // 26~51 'a' [0] |
274 | | // 52~61 '0' [1~10] |
275 | | // 62 c62 [11] |
276 | | // 63 c63 [12] |
277 | | |
278 | | let mut lut = [0x80; 16]; |
279 | | lut[13] = b'A'; |
280 | | lut[0] = b'a' - 26; |
281 | | let mut i = 1; |
282 | | while i <= 10 { |
283 | | lut[i] = b'0'.wrapping_sub(52); |
284 | | i += 1; |
285 | | } |
286 | | lut[11] = charset[62].wrapping_sub(62); |
287 | | lut[12] = charset[63].wrapping_sub(63); |
288 | | V128::from_bytes(lut) |
289 | | } |
290 | | |
291 | | const STANDARD_ENCODING_SHIFT: V128 = encoding_shift(STANDARD_CHARSET); |
292 | | const URL_SAFE_ENCODING_SHIFT: V128 = encoding_shift(URL_SAFE_CHARSET); |
293 | | |
294 | | const STANDARD_ENCODING_SHIFT_X2: V256 = STANDARD_ENCODING_SHIFT.x2(); |
295 | | const URL_SAFE_ENCODING_SHIFT_X2: V256 = URL_SAFE_ENCODING_SHIFT.x2(); |
296 | | |
297 | | #[inline(always)] |
298 | 0 | fn encode_values<S: Scalable<V>, V: POD>(s: S, x: V, shift_lut: V) -> V { |
299 | | // x: {00aaaaaa|00bbbbbb|00cccccc|00dddddd} xn |
300 | | |
301 | 0 | let x1 = s.u8xn_sub_sat(x, s.u8xn_splat(51)); |
302 | | // 0~25 => 0 |
303 | | // 26~51 => 0 |
304 | | // 52~61 => 1~10 |
305 | | // 62 => 11 |
306 | | // 63 => 12 |
307 | | |
308 | 0 | let x2 = s.i8xn_lt(x, s.u8xn_splat(26)); |
309 | 0 | let x3 = s.and(x2, s.u8xn_splat(13)); |
310 | 0 | let x4 = s.or(x1, x3); |
311 | | // 0~25 => 0xff => 13 |
312 | | // 26~51 => 0 => 0 |
313 | | // 52~61 => 0 => 1~10 |
314 | | // 62 => 0 => 11 |
315 | | // 63 => 0 => 12 |
316 | | |
317 | 0 | let shift = s.u8x16xn_swizzle(shift_lut, x4); |
318 | 0 | s.u8xn_add(x, shift) |
319 | | // {{ascii}} xn |
320 | 0 | } Unexecuted instantiation: base64_simd::encode::encode_values::<vsimd::isa::AVX2, vsimd::vector::V128> Unexecuted instantiation: base64_simd::encode::encode_values::<vsimd::isa::AVX2, vsimd::vector::V256> Unexecuted instantiation: base64_simd::encode::encode_values::<vsimd::isa::SSSE3, vsimd::vector::V128> Unexecuted instantiation: base64_simd::encode::encode_values::<vsimd::isa::SSSE3, vsimd::vector::V256> |
321 | | |
322 | | #[inline(always)] |
323 | 0 | fn encode_bytes24<S: SIMD256>(s: S, x: V256, shift_lut: V256) -> V256 { |
324 | | // x: {????|AAAB|BBCC|CDDD|EEEF|FFGG|GHHH|????} |
325 | | |
326 | 0 | let values = split_bits_x2(s, x); |
327 | | // values: {00aaaaaa|00bbbbbb|00cccccc|00dddddd} x8 |
328 | | |
329 | 0 | encode_values(s, values, shift_lut) |
330 | | // {{ascii}} x32 |
331 | 0 | } Unexecuted instantiation: base64_simd::encode::encode_bytes24::<vsimd::isa::AVX2> Unexecuted instantiation: base64_simd::encode::encode_bytes24::<vsimd::isa::SSSE3> |
332 | | |
333 | | #[inline(always)] |
334 | 0 | fn encode_bytes12<S: SIMD256>(s: S, x: V128, shift_lut: V128) -> V128 { |
335 | | // x: {AAAB|BBCC|CDDD|????} |
336 | | |
337 | 0 | let values = split_bits_x1(s, x); |
338 | | // values: {00aaaaaa|00bbbbbb|00cccccc|00dddddd} x4 |
339 | | |
340 | 0 | encode_values(s, values, shift_lut) |
341 | | // {{ascii}} x16 |
342 | 0 | } Unexecuted instantiation: base64_simd::encode::encode_bytes12::<vsimd::isa::AVX2> Unexecuted instantiation: base64_simd::encode::encode_bytes12::<vsimd::isa::SSSE3> |