/rust/registry/src/index.crates.io-1949cf8c6b5b557f/zlib-rs-0.6.4/src/crc32/braid.rs
Line | Count | Source |
1 | | // Several implementations of CRC-32: |
2 | | // * A naive byte-granularity approach |
3 | | // * A word-sized approach that processes a usize word at a time |
4 | | // * A "braid" implementation that processes a block of N words |
5 | | // at a time, based on the algorithm in section 4.11 from |
6 | | // https://github.com/zlib-ng/zlib-ng/blob/develop/doc/crc-doc.1.0.pdf. |
7 | | |
8 | | // The binary encoding of the CRC-32 polynomial. |
9 | | // We are assuming little-endianness so we process the input |
10 | | // LSB-first. We need to use the "reversed" value from e.g |
11 | | // https://en.wikipedia.org/wiki/Cyclic_redundancy_check#Polynomial_representations. |
12 | | pub(crate) const CRC32_LSB_POLY: usize = 0xedb8_8320usize; |
13 | | |
14 | | const W: usize = core::mem::size_of::<usize>(); |
15 | | |
16 | | // The logic assumes that W >= sizeof(u32). |
17 | | // In Rust, this is generally true. |
18 | | const _: () = assert!(W >= core::mem::size_of::<u32>()); |
19 | | |
20 | | // Pre-computed tables for the CRC32 algorithm. |
21 | | // CRC32_BYTE_TABLE corresponds to MulByXPowD from the paper. |
22 | | static CRC32_BYTE_TABLE: [[u32; 256]; 1] = build_crc32_table::<256, 1, 1>(); |
23 | | // CRC32_WORD_TABLE is MulWordByXpowD. |
24 | | static CRC32_WORD_TABLE: [[u32; 256]; W] = build_crc32_table::<256, W, 1>(); |
25 | | |
26 | | // FIXME: make const fn when msrv allows. |
27 | 0 | pub(crate) fn get_crc_table() -> &'static [u32; 256] { |
28 | 0 | &CRC32_BYTE_TABLE[0] |
29 | 0 | } |
30 | | |
31 | | // Work-around for not being able to define generic consts or statics |
32 | | // Crc32BraidTable::<N>::TABLE is the generic table for any braid size N. |
33 | | struct Crc32BraidTable<const N: usize>; |
34 | | |
35 | | impl<const N: usize> Crc32BraidTable<N> { |
36 | | const TABLE: [[u32; 256]; W] = build_crc32_table::<256, W, N>(); |
37 | | } |
38 | | |
39 | | // Build the CRC32 tables using a more efficient and simpler approach |
40 | | // than the combination of Multiply and XpowN (which implement polynomial |
41 | | // multiplication and exponentiation, respectively) from the paper, |
42 | | // but with identical results. This function is const, so it should be |
43 | | // fully evaluated at compile time. |
44 | 0 | const fn build_crc32_table<const A: usize, const W: usize, const N: usize>() -> [[u32; A]; W] { |
45 | 0 | let mut arr = [[0u32; A]; W]; |
46 | 0 | let mut i = 0; |
47 | 0 | while i < W { |
48 | 0 | let mut j = 0; |
49 | 0 | while j < A { |
50 | 0 | let mut c = j; |
51 | 0 | let mut k = 0; |
52 | 0 | while k < 8 * (W * N - i) { |
53 | 0 | if c & 1 != 0 { |
54 | 0 | c = CRC32_LSB_POLY ^ (c >> 1); |
55 | 0 | } else { |
56 | 0 | c >>= 1; |
57 | 0 | } |
58 | 0 | k += 1; |
59 | | } |
60 | 0 | arr[i][j] = c as u32; |
61 | 0 | j += 1; |
62 | | } |
63 | 0 | i += 1; |
64 | | } |
65 | 0 | arr |
66 | 0 | } |
67 | | |
68 | 0 | fn crc32_naive_inner(data: &[u8], start: u32) -> u32 { |
69 | 0 | data.iter().fold(start, |crc, val| { |
70 | 0 | let crc32_lsb = crc.to_le_bytes()[0]; |
71 | 0 | CRC32_BYTE_TABLE[0][usize::from(crc32_lsb ^ *val)] ^ (crc >> 8) |
72 | 0 | }) |
73 | 0 | } |
74 | | |
75 | 0 | fn crc32_words_inner(words: &[usize], start: u32, per_word_crcs: &[u32]) -> u32 { |
76 | 0 | words.iter().enumerate().fold(start, |crc, (i, word)| { |
77 | 0 | let value = word.to_le() ^ (crc ^ per_word_crcs.get(i).unwrap_or(&0)) as usize; |
78 | 0 | value |
79 | 0 | .to_le_bytes() |
80 | 0 | .into_iter() |
81 | 0 | .zip(CRC32_WORD_TABLE) |
82 | 0 | .fold(0u32, |crc, (b, tab)| crc ^ tab[usize::from(b)]) |
83 | 0 | }) |
84 | 0 | } |
85 | | |
86 | 0 | pub fn crc32_braid<const N: usize>(start: u32, data: &[u8]) -> u32 { |
87 | | // Get a word-aligned sub-slice of the input data |
88 | | // SAFETY: it is safe to transmute a slice of u8 into a usize. |
89 | 0 | let (prefix, words, suffix) = unsafe { data.align_to::<usize>() }; |
90 | 0 | let crc = !start; |
91 | 0 | let crc = crc32_naive_inner(prefix, crc); |
92 | | |
93 | 0 | let mut crcs = [0u32; N]; |
94 | 0 | crcs[0] = crc; |
95 | | |
96 | | // TODO: this would normally use words.chunks_exact(N), but |
97 | | // we need to pass the last full block to crc32_words_inner |
98 | | // because we accumulate partial crcs in the array and we |
99 | | // need to roll those into the final value. The last call to |
100 | | // crc32_words_inner does that for us with its per_word_crcs |
101 | | // argument. |
102 | 0 | let blocks = words.len() / N; |
103 | 0 | let blocks = blocks.saturating_sub(1); |
104 | 0 | for i in 0..blocks { |
105 | | // Load the next N words. |
106 | 0 | let mut buffer: [usize; N] = |
107 | 0 | core::array::from_fn(|j| usize::to_le(words[i * N + j]) ^ (crcs[j] as usize)); |
108 | | |
109 | 0 | crcs.fill(0); |
110 | 0 | for j in 0..W { |
111 | 0 | braid_core(&mut crcs, &mut buffer, j); |
112 | 0 | } |
113 | | } |
114 | | |
115 | 0 | let crc = core::mem::take(&mut crcs[0]); |
116 | 0 | let crc = crc32_words_inner(&words[blocks * N..], crc, &crcs); |
117 | 0 | let crc = crc32_naive_inner(suffix, crc); |
118 | 0 | !crc |
119 | 0 | } |
120 | | |
121 | | // A workaround for https://github.com/trifectatechfoundation/zlib-rs/issues/407. |
122 | | // |
123 | | // We're seeing misoptimization with rust versions that use LLVM 20, earlier LLVMs are fine, and |
124 | | // LLVM 21 similarly appears to do fine. The offending feature is `+avx512vl`, the |
125 | | // "Vector Length Extension", which extends some instructions operating on 512-bit operands with |
126 | | // variants that support 256-bit or 128-bit operands. |
127 | | // |
128 | | // The avx512vl target feature only became stable in 1.89.0: before that, we can't detect it |
129 | | // statically. Therefore we use avx2 as a proxy, it is implied by avx512vl. |
130 | | #[cfg_attr(all(target_arch = "x86_64", target_feature = "avx2"), inline(never))] |
131 | | #[cfg_attr(not(target_arch = "x86_64"), inline(always))] |
132 | 0 | fn braid_core<const N: usize>(crcs: &mut [u32; N], buffer: &mut [usize; N], j: usize) { |
133 | 0 | for k in 0..N { |
134 | 0 | crcs[k] ^= Crc32BraidTable::<N>::TABLE[j][buffer[k] & 0xff]; |
135 | 0 | buffer[k] >>= 8; |
136 | 0 | } |
137 | 0 | } |
138 | | |
139 | | #[cfg(test)] |
140 | | mod test { |
141 | | use super::*; |
142 | | |
143 | | fn crc32_naive(data: &[u8], start: u32) -> u32 { |
144 | | let crc = !start; |
145 | | let crc = crc32_naive_inner(data, crc); |
146 | | !crc |
147 | | } |
148 | | |
149 | | fn crc32_words(data: &[u8], start: u32) -> u32 { |
150 | | // Get a word-aligned sub-slice of the input data |
151 | | let (prefix, words, suffix) = unsafe { data.align_to::<usize>() }; |
152 | | let crc = !start; |
153 | | let crc = crc32_naive_inner(prefix, crc); |
154 | | let crc = crc32_words_inner(words, crc, &[]); |
155 | | let crc = crc32_naive_inner(suffix, crc); |
156 | | !crc |
157 | | } |
158 | | |
159 | | #[test] |
160 | | fn empty_is_identity() { |
161 | | assert_eq!(crc32_naive(&[], 32), 32); |
162 | | } |
163 | | |
164 | | #[test] |
165 | | fn words_endianness() { |
166 | | let v = [0, 0, 0, 0, 0, 16, 0, 1]; |
167 | | let start = 1534327806; |
168 | | |
169 | | let mut h = crc32fast::Hasher::new_with_initial(start); |
170 | | h.update(&v[..]); |
171 | | assert_eq!(crc32_words(&v[..], start), h.finalize()); |
172 | | } |
173 | | |
174 | | #[test] |
175 | | fn crc32_naive_inner_endianness_and_alignment() { |
176 | | assert_eq!(crc32_naive_inner(&[0, 1], 0), 1996959894); |
177 | | |
178 | | let v: Vec<_> = (0..1024).map(|i| i as u8).collect(); |
179 | | let start = 0; |
180 | | |
181 | | // test alignment |
182 | | for i in 0..8 { |
183 | | let mut h = crc32fast::Hasher::new_with_initial(start); |
184 | | h.update(&v[i..]); |
185 | | assert_eq!(crc32_braid::<5>(start, &v[i..]), h.finalize()); |
186 | | } |
187 | | } |
188 | | |
189 | | quickcheck::quickcheck! { |
190 | | fn naive_is_crc32fast(v: Vec<u8>, start: u32) -> bool { |
191 | | let mut h = crc32fast::Hasher::new_with_initial(start); |
192 | | h.update(&v[..]); |
193 | | crc32_naive(&v[..], start) == h.finalize() |
194 | | } |
195 | | |
196 | | fn words_is_crc32fast(v: Vec<u8>, start: u32) -> bool { |
197 | | let mut h = crc32fast::Hasher::new_with_initial(start); |
198 | | h.update(&v[..]); |
199 | | crc32_words(&v[..], start) == h.finalize() |
200 | | } |
201 | | |
202 | | #[cfg_attr(miri, ignore)] |
203 | | fn braid_4_is_crc32fast(v: Vec<u8>, start: u32) -> bool { |
204 | | let mut h = crc32fast::Hasher::new_with_initial(start); |
205 | | h.update(&v[..]); |
206 | | crc32_braid::<4>(start, &v[..]) == h.finalize() |
207 | | } |
208 | | |
209 | | #[cfg_attr(miri, ignore)] |
210 | | fn braid_5_is_crc32fast(v: Vec<u8>, start: u32) -> bool { |
211 | | let mut h = crc32fast::Hasher::new_with_initial(start); |
212 | | h.update(&v[..]); |
213 | | crc32_braid::<5>(start, &v[..]) == h.finalize() |
214 | | } |
215 | | |
216 | | #[cfg_attr(miri, ignore)] |
217 | | fn braid_6_is_crc32fast(v: Vec<u8>, start: u32) -> bool { |
218 | | let mut h = crc32fast::Hasher::new_with_initial(start); |
219 | | h.update(&v[..]); |
220 | | crc32_braid::<6>(start, &v[..]) == h.finalize() |
221 | | } |
222 | | } |
223 | | } |