Coverage Report

Created: 2026-07-16 07:16

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/rust/registry/src/index.crates.io-1949cf8c6b5b557f/simd-adler32-0.3.10/src/imp/avx2.rs
Line
Count
Source
1
use super::Adler32Imp;
2
3
/// Resolves update implementation if CPU supports avx2 instructions.
4
37.4k
pub fn get_imp() -> Option<Adler32Imp> {
5
37.4k
  get_imp_inner()
6
37.4k
}
7
8
#[inline]
9
#[cfg(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64")))]
10
37.4k
fn get_imp_inner() -> Option<Adler32Imp> {
11
37.4k
  if std::is_x86_feature_detected!("avx2") {
12
37.4k
    Some(imp::update)
13
  } else {
14
0
    None
15
  }
16
37.4k
}
17
18
#[inline]
19
#[cfg(all(
20
  target_feature = "avx2",
21
  not(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64")))
22
))]
23
fn get_imp_inner() -> Option<Adler32Imp> {
24
  Some(imp::update)
25
}
26
27
#[inline]
28
#[cfg(all(
29
  not(target_feature = "avx2"),
30
  not(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64")))
31
))]
32
fn get_imp_inner() -> Option<Adler32Imp> {
33
  None
34
}
35
36
#[cfg(all(
37
  any(target_arch = "x86", target_arch = "x86_64"),
38
  any(feature = "std", target_feature = "avx2")
39
))]
40
mod imp {
41
  const MOD: u32 = 65521;
42
  const NMAX: usize = 5552;
43
  const BLOCK_SIZE: usize = 32;
44
  const CHUNK_SIZE: usize = NMAX / BLOCK_SIZE * BLOCK_SIZE;
45
46
  #[cfg(target_arch = "x86")]
47
  use core::arch::x86::*;
48
  #[cfg(target_arch = "x86_64")]
49
  use core::arch::x86_64::*;
50
51
212k
  pub fn update(a: u16, b: u16, data: &[u8]) -> (u16, u16) {
52
212k
    unsafe { update_imp(a, b, data) }
53
212k
  }
54
55
  #[inline]
56
  #[target_feature(enable = "avx2")]
57
212k
  unsafe fn update_imp(a: u16, b: u16, data: &[u8]) -> (u16, u16) {
58
212k
    let mut a = a as u32;
59
212k
    let mut b = b as u32;
60
61
212k
    let chunks = data.chunks_exact(CHUNK_SIZE);
62
212k
    let remainder = chunks.remainder();
63
1.23M
    for chunk in chunks {
64
1.01M
      update_chunk_block(&mut a, &mut b, chunk);
65
1.01M
    }
66
67
212k
    update_block(&mut a, &mut b, remainder);
68
69
212k
    (a as u16, b as u16)
70
212k
  }
71
72
  #[inline]
73
1.01M
  unsafe fn update_chunk_block(a: &mut u32, b: &mut u32, chunk: &[u8]) {
74
1.01M
    debug_assert_eq!(
75
0
      chunk.len(),
76
      CHUNK_SIZE,
77
0
      "Unexpected chunk size (expected {}, got {})",
78
      CHUNK_SIZE,
79
0
      chunk.len()
80
    );
81
82
1.01M
    reduce_add_blocks(a, b, chunk);
83
84
1.01M
    *a %= MOD;
85
1.01M
    *b %= MOD;
86
1.01M
  }
87
88
  #[inline]
89
212k
  unsafe fn update_block(a: &mut u32, b: &mut u32, chunk: &[u8]) {
90
212k
    debug_assert!(
91
0
      chunk.len() <= CHUNK_SIZE,
92
0
      "Unexpected chunk size (expected <= {}, got {})",
93
      CHUNK_SIZE,
94
0
      chunk.len()
95
    );
96
97
212k
    for byte in reduce_add_blocks(a, b, chunk) {
98
118k
      *a += *byte as u32;
99
118k
      *b += *a;
100
118k
    }
101
102
212k
    *a %= MOD;
103
212k
    *b %= MOD;
104
212k
  }
105
106
  #[inline(always)]
107
1.23M
  unsafe fn reduce_add_blocks<'a>(a: &mut u32, b: &mut u32, chunk: &'a [u8]) -> &'a [u8] {
108
1.23M
    if chunk.len() < BLOCK_SIZE {
109
521
      return chunk;
110
1.23M
    }
111
112
1.23M
    let blocks = chunk.chunks_exact(BLOCK_SIZE);
113
1.23M
    let blocks_remainder = blocks.remainder();
114
115
1.23M
    let one_v = _mm256_set1_epi16(1);
116
1.23M
    let zero_v = _mm256_setzero_si256();
117
1.23M
    let weights = get_weights();
118
119
1.23M
    let mut p_v = _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, (*a * blocks.len() as u32) as _);
120
1.23M
    let mut a_v = _mm256_setzero_si256();
121
1.23M
    let mut b_v = _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, *b as _);
122
123
210M
    for block in blocks {
124
209M
      let block_ptr = block.as_ptr() as *const _;
125
209M
      let block = _mm256_loadu_si256(block_ptr);
126
209M
127
209M
      p_v = _mm256_add_epi32(p_v, a_v);
128
209M
129
209M
      a_v = _mm256_add_epi32(a_v, _mm256_sad_epu8(block, zero_v));
130
209M
      let mad = _mm256_maddubs_epi16(block, weights);
131
209M
      b_v = _mm256_add_epi32(b_v, _mm256_madd_epi16(mad, one_v));
132
209M
    }
133
134
1.23M
    b_v = _mm256_add_epi32(b_v, _mm256_slli_epi32(p_v, 5));
135
136
1.23M
    *a += reduce_add(a_v);
137
1.23M
    *b = reduce_add(b_v);
138
139
1.23M
    blocks_remainder
140
1.23M
  }
141
142
  #[inline(always)]
143
2.46M
  unsafe fn reduce_add(v: __m256i) -> u32 {
144
2.46M
    let sum = _mm_add_epi32(_mm256_castsi256_si128(v), _mm256_extracti128_si256(v, 1));
145
2.46M
    let hi = _mm_unpackhi_epi64(sum, sum);
146
147
2.46M
    let sum = _mm_add_epi32(hi, sum);
148
2.46M
    let hi = _mm_shuffle_epi32(sum, crate::imp::_MM_SHUFFLE(2, 3, 0, 1));
149
150
2.46M
    let sum = _mm_add_epi32(sum, hi);
151
152
2.46M
    _mm_cvtsi128_si32(sum) as _
153
2.46M
  }
154
155
  #[inline(always)]
156
1.23M
  unsafe fn get_weights() -> __m256i {
157
1.23M
    _mm256_set_epi8(
158
      1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
159
      24, 25, 26, 27, 28, 29, 30, 31, 32,
160
    )
161
1.23M
  }
162
}
163
164
#[cfg(test)]
165
mod tests {
166
  use rand::{Rng, SeedableRng, rngs::SmallRng};
167
168
  #[test]
169
  fn zeroes() {
170
    assert_sum_eq(&[]);
171
    assert_sum_eq(&[0]);
172
    assert_sum_eq(&[0, 0]);
173
    assert_sum_eq(&[0; 100]);
174
    assert_sum_eq(&[0; 1024]);
175
    assert_sum_eq(&[0; 1024 - 5]);
176
    #[cfg(not(miri))]
177
    assert_sum_eq(&[0; 1024 * 1024]);
178
  }
179
180
  #[test]
181
  fn ones() {
182
    assert_sum_eq(&[]);
183
    assert_sum_eq(&[1]);
184
    assert_sum_eq(&[1, 1]);
185
    assert_sum_eq(&[1; 100]);
186
    assert_sum_eq(&[1; 1024]);
187
    assert_sum_eq(&[1; 1024 - 5]); // non-power-of-2 to test remainder handling
188
    #[cfg(not(miri))]
189
    assert_sum_eq(&[1; 1024 * 1024]);
190
  }
191
192
  #[test]
193
  fn random() {
194
    if super::get_imp().is_none() { return; } // don't do any work if we're not on this target
195
    let mut random = [0; 1024 * 10];
196
    SmallRng::from_entropy().fill(&mut random[..]);
197
198
    assert_sum_eq(&random[..1]);
199
    assert_sum_eq(&random[..100]);
200
    assert_sum_eq(&random[..1024]);
201
    assert_sum_eq(&random[..1024 - 5]); // non-power-of-2 to test remainder handling
202
    assert_sum_eq(&random[..1024 * 10]);
203
  }
204
205
  /// Example calculation from https://en.wikipedia.org/wiki/Adler-32.
206
  #[test]
207
  fn wiki() {
208
    assert_sum_eq(b"Wikipedia");
209
  }
210
211
  fn assert_sum_eq(data: &[u8]) {
212
    if let Some(update) = super::get_imp() {
213
      let (a, b) = update(1, 0, data);
214
      let left = u32::from(b) << 16 | u32::from(a);
215
      let right = adler2::adler32_slice(data);
216
217
      assert_eq!(left, right, "len({})", data.len());
218
    }
219
  }
220
}