/rust/registry/src/index.crates.io-1949cf8c6b5b557f/chacha20-0.10.1/src/backends/avx2.rs

Source
//! AVX2 backend.

#![allow(unsafe_op_in_unsafe_fn, reason = "needs triage")]
#![allow(clippy::cast_possible_truncation, reason = "needs triage")]
#![allow(clippy::cast_possible_wrap, reason = "needs triage")]
#![allow(clippy::cast_sign_loss, reason = "needs triage")]
#![allow(clippy::undocumented_unsafe_blocks, reason = "TODO")]

use crate::{Rounds, Variant};
use core::marker::PhantomData;

#[cfg(feature = "rng")]
use crate::ChaChaCore;

#[cfg(feature = "cipher")]
use crate::{STATE_WORDS, chacha::Block};

#[cfg(feature = "cipher")]
use cipher::{
    BlockSizeUser, ParBlocks, ParBlocksSizeUser, StreamCipherBackend, StreamCipherClosure,
    consts::{U4, U64},
};

#[cfg(target_arch = "x86")]
use core::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use core::arch::x86_64::*;

/// Number of blocks processed in parallel.
const PAR_BLOCKS: usize = 4;
/// Number of `__m256i` to store parallel blocks.
const N: usize = PAR_BLOCKS / 2;

#[inline]
#[target_feature(enable = "avx2")]
#[cfg(feature = "cipher")]
#[cfg_attr(chacha20_backend = "avx512", expect(unused))]
pub(crate) unsafe fn inner<R, F, V>(state: &mut [u32; STATE_WORDS], f: F)
where
    R: Rounds,
    F: StreamCipherClosure<BlockSize = U64>,
    V: Variant,
{
    let state_ptr = state.as_ptr().cast::<__m128i>();
    let v = [
        _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(0))),
        _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(1))),
        _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(2))),
    ];
    let mut c = _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(3)));
    c = match size_of::<V::Counter>() {
        4 => _mm256_add_epi32(c, _mm256_set_epi32(0, 0, 0, 1, 0, 0, 0, 0)),
        8 => _mm256_add_epi64(c, _mm256_set_epi64x(0, 1, 0, 0)),
        _ => unreachable!(),
    };
    let mut ctr = [c; N];
    for i in 0..N {
        ctr[i] = c;
        c = match size_of::<V::Counter>() {
            4 => _mm256_add_epi32(c, _mm256_set_epi32(0, 0, 0, 2, 0, 0, 0, 2)),
            8 => _mm256_add_epi64(c, _mm256_set_epi64x(0, 2, 0, 2)),
            _ => unreachable!(),
        };
    }
    let mut backend = Backend::<R, V> {
        v,
        ctr,
        _pd: PhantomData,
    };

    f.call(&mut backend);

    state[12] = _mm256_extract_epi32(backend.ctr[0], 0) as u32;
    match size_of::<V::Counter>() {
        4 => {}
        8 => state[13] = _mm256_extract_epi32(backend.ctr[0], 1) as u32,
        _ => unreachable!(),
    }
}

#[inline]
#[target_feature(enable = "avx2")]
#[cfg(feature = "rng")]
pub(crate) unsafe fn rng_inner<R, V>(core: &mut ChaChaCore<R, V>, buffer: &mut [u32; 64])
where
    R: Rounds,
    V: Variant,
{
    let state_ptr = core.state.as_ptr().cast::<__m128i>();
    let v = [
        _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(0))),
        _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(1))),
        _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(2))),
    ];
    let mut c = _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(3)));
    c = _mm256_add_epi64(c, _mm256_set_epi64x(0, 1, 0, 0));
    let mut ctr = [c; N];
    for i in 0..N {
        ctr[i] = c;
        c = _mm256_add_epi64(c, _mm256_set_epi64x(0, 2, 0, 2));
    }
    let mut backend = Backend::<R, V> {
        v,
        ctr,
        _pd: PhantomData,
    };

    backend.rng_gen_par_ks_blocks(buffer);

    core.state[12] = _mm256_extract_epi32(backend.ctr[0], 0) as u32;
    core.state[13] = _mm256_extract_epi32(backend.ctr[0], 1) as u32;
}

struct Backend<R: Rounds, V: Variant> {
    v: [__m256i; 3],
    ctr: [__m256i; N],
    _pd: PhantomData<(R, V)>,
}

#[cfg(feature = "cipher")]
impl<R: Rounds, V: Variant> BlockSizeUser for Backend<R, V> {
    type BlockSize = U64;
}

#[cfg(feature = "cipher")]
impl<R: Rounds, V: Variant> ParBlocksSizeUser for Backend<R, V> {
    type ParBlocksSize = U4;
}

#[cfg(feature = "cipher")]
impl<R: Rounds, V: Variant> StreamCipherBackend for Backend<R, V> {
    #[inline(always)]
    fn gen_ks_block(&mut self, block: &mut Block) {
        unsafe {
            let res = rounds::<R>(&self.v, &self.ctr);
            for c in self.ctr.iter_mut() {
                *c = match size_of::<V::Counter>() {
                    4 => _mm256_add_epi32(*c, _mm256_set_epi32(0, 0, 0, 1, 0, 0, 0, 1)),
                    8 => _mm256_add_epi64(*c, _mm256_set_epi64x(0, 1, 0, 1)),
                    _ => unreachable!(),
                };
            }

            let res0: [__m128i; 8] = core::mem::transmute(res[0]);

            let block_ptr = block.as_mut_ptr().cast::<__m128i>();
            for i in 0..4 {
                _mm_storeu_si128(block_ptr.add(i), res0[2 * i]);
            }
        }
    }

    #[inline(always)]
    fn gen_par_ks_blocks(&mut self, blocks: &mut ParBlocks<Self>) {
        unsafe {
            let vs = rounds::<R>(&self.v, &self.ctr);

            let pb = PAR_BLOCKS as i32;
            for c in self.ctr.iter_mut() {
                *c = match size_of::<V::Counter>() {
                    4 => _mm256_add_epi32(*c, _mm256_set_epi32(0, 0, 0, pb, 0, 0, 0, pb)),
                    8 => {
                        _mm256_add_epi64(*c, _mm256_set_epi64x(0, i64::from(pb), 0, i64::from(pb)))
                    }
                    _ => unreachable!(),
                }
            }

            let mut block_ptr = blocks.as_mut_ptr().cast::<__m128i>();
            for v in vs {
                let t: [__m128i; 8] = core::mem::transmute(v);
                for i in 0..4 {
                    _mm_storeu_si128(block_ptr.add(i), t[2 * i]);
                    _mm_storeu_si128(block_ptr.add(4 + i), t[2 * i + 1]);
                }
                block_ptr = block_ptr.add(8);
            }
        }
    }
}

#[cfg(feature = "rng")]
impl<R: Rounds, V: Variant> Backend<R, V> {
    #[inline(always)]
    fn rng_gen_par_ks_blocks(&mut self, blocks: &mut [u32; 64]) {
        unsafe {
            let vs = rounds::<R>(&self.v, &self.ctr);

            let pb = PAR_BLOCKS as i32;
            for c in self.ctr.iter_mut() {
                *c = _mm256_add_epi64(*c, _mm256_set_epi64x(0, i64::from(pb), 0, i64::from(pb)));
            }

            let mut block_ptr = blocks.as_mut_ptr().cast::<__m128i>();
            for v in vs {
                let t: [__m128i; 8] = core::mem::transmute(v);
                for i in 0..4 {
                    _mm_storeu_si128(block_ptr.add(i), t[2 * i]);
                    _mm_storeu_si128(block_ptr.add(4 + i), t[2 * i + 1]);
                }
                block_ptr = block_ptr.add(8);
            }
        }
    }
}

#[inline]
#[target_feature(enable = "avx2")]
unsafe fn rounds<R: Rounds>(v: &[__m256i; 3], c: &[__m256i; N]) -> [[__m256i; 4]; N] {
    let mut vs: [[__m256i; 4]; N] = [[_mm256_setzero_si256(); 4]; N];
    for i in 0..N {
        vs[i] = [v[0], v[1], v[2], c[i]];
    }
    for _ in 0..R::COUNT {
        double_quarter_round(&mut vs);
    }

    for i in 0..N {
        for j in 0..3 {
            vs[i][j] = _mm256_add_epi32(vs[i][j], v[j]);
        }
        vs[i][3] = _mm256_add_epi32(vs[i][3], c[i]);
    }

    vs
}

#[inline]
#[target_feature(enable = "avx2")]
unsafe fn double_quarter_round(v: &mut [[__m256i; 4]; N]) {
    add_xor_rot(v);
    rows_to_cols(v);
    add_xor_rot(v);
    cols_to_rows(v);
}

/// The goal of this function is to transform the state words from:
/// ```text
/// [a0, a1, a2, a3]    [ 0,  1,  2,  3]
/// [b0, b1, b2, b3] == [ 4,  5,  6,  7]
/// [c0, c1, c2, c3]    [ 8,  9, 10, 11]
/// [d0, d1, d2, d3]    [12, 13, 14, 15]
/// ```
///
/// to:
/// ```text
/// [a0, a1, a2, a3]    [ 0,  1,  2,  3]
/// [b1, b2, b3, b0] == [ 5,  6,  7,  4]
/// [c2, c3, c0, c1]    [10, 11,  8,  9]
/// [d3, d0, d1, d2]    [15, 12, 13, 14]
/// ```
///
/// so that we can apply [`add_xor_rot`] to the resulting columns, and have it compute the
/// "diagonal rounds" (as defined in RFC 7539) in parallel. In practice, this shuffle is
/// non-optimal: the last state word to be altered in `add_xor_rot` is `b`, so the shuffle
/// blocks on the result of `b` being calculated.
///
/// We can optimize this by observing that the four quarter rounds in `add_xor_rot` are
/// data-independent: they only access a single column of the state, and thus the order of
/// the columns does not matter. We therefore instead shuffle the other three state words,
/// to obtain the following equivalent layout:
/// ```text
/// [a3, a0, a1, a2]    [ 3,  0,  1,  2]
/// [b0, b1, b2, b3] == [ 4,  5,  6,  7]
/// [c1, c2, c3, c0]    [ 9, 10, 11,  8]
/// [d2, d3, d0, d1]    [14, 15, 12, 13]
/// ```
///
/// See https://github.com/sneves/blake2-avx2/pull/4 for additional details. The earliest
/// known occurrence of this optimization is in floodyberry's SSE4 ChaCha code from 2014:
/// - https://github.com/floodyberry/chacha-opt/blob/0ab65cb99f5016633b652edebaf3691ceb4ff753/chacha_blocks_ssse3-64.S#L639-L643
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn rows_to_cols(vs: &mut [[__m256i; 4]; N]) {
    // c >>>= 32; d >>>= 64; a >>>= 96;
    for [a, _, c, d] in vs {
        *c = _mm256_shuffle_epi32(*c, 0b_00_11_10_01); // _MM_SHUFFLE(0, 3, 2, 1)
        *d = _mm256_shuffle_epi32(*d, 0b_01_00_11_10); // _MM_SHUFFLE(1, 0, 3, 2)
        *a = _mm256_shuffle_epi32(*a, 0b_10_01_00_11); // _MM_SHUFFLE(2, 1, 0, 3)
    }
}

/// The goal of this function is to transform the state words from:
/// ```text
/// [a3, a0, a1, a2]    [ 3,  0,  1,  2]
/// [b0, b1, b2, b3] == [ 4,  5,  6,  7]
/// [c1, c2, c3, c0]    [ 9, 10, 11,  8]
/// [d2, d3, d0, d1]    [14, 15, 12, 13]
/// ```
///
/// to:
/// ```text
/// [a0, a1, a2, a3]    [ 0,  1,  2,  3]
/// [b0, b1, b2, b3] == [ 4,  5,  6,  7]
/// [c0, c1, c2, c3]    [ 8,  9, 10, 11]
/// [d0, d1, d2, d3]    [12, 13, 14, 15]
/// ```
///
/// reversing the transformation of [`rows_to_cols`].
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn cols_to_rows(vs: &mut [[__m256i; 4]; N]) {
    // c <<<= 32; d <<<= 64; a <<<= 96;
    for [a, _, c, d] in vs {
        *c = _mm256_shuffle_epi32(*c, 0b_10_01_00_11); // _MM_SHUFFLE(2, 1, 0, 3)
        *d = _mm256_shuffle_epi32(*d, 0b_01_00_11_10); // _MM_SHUFFLE(1, 0, 3, 2)
        *a = _mm256_shuffle_epi32(*a, 0b_00_11_10_01); // _MM_SHUFFLE(0, 3, 2, 1)
    }
}

#[inline]
#[target_feature(enable = "avx2")]
unsafe fn add_xor_rot(vs: &mut [[__m256i; 4]; N]) {
    let rol16_mask = _mm256_set_epi64x(
        0x0d0c_0f0e_0908_0b0a,
        0x0504_0706_0100_0302,
        0x0d0c_0f0e_0908_0b0a,
        0x0504_0706_0100_0302,
    );
    let rol8_mask = _mm256_set_epi64x(
        0x0e0d_0c0f_0a09_080b,
        0x0605_0407_0201_0003,
        0x0e0d_0c0f_0a09_080b,
        0x0605_0407_0201_0003,
    );

    // a += b; d ^= a; d <<<= (16, 16, 16, 16);
    for [a, b, _, d] in vs.iter_mut() {
        *a = _mm256_add_epi32(*a, *b);
        *d = _mm256_xor_si256(*d, *a);
        *d = _mm256_shuffle_epi8(*d, rol16_mask);
    }

    // c += d; b ^= c; b <<<= (12, 12, 12, 12);
    for [_, b, c, d] in vs.iter_mut() {
        *c = _mm256_add_epi32(*c, *d);
        *b = _mm256_xor_si256(*b, *c);
        *b = _mm256_xor_si256(_mm256_slli_epi32(*b, 12), _mm256_srli_epi32(*b, 20));
    }

    // a += b; d ^= a; d <<<= (8, 8, 8, 8);
    for [a, b, _, d] in vs.iter_mut() {
        *a = _mm256_add_epi32(*a, *b);
        *d = _mm256_xor_si256(*d, *a);
        *d = _mm256_shuffle_epi8(*d, rol8_mask);
    }

    // c += d; b ^= c; b <<<= (7, 7, 7, 7);
    for [_, b, c, d] in vs.iter_mut() {
        *c = _mm256_add_epi32(*c, *d);
        *b = _mm256_xor_si256(*b, *c);
        *b = _mm256_xor_si256(_mm256_slli_epi32(*b, 7), _mm256_srli_epi32(*b, 25));
    }
}

Coverage Report

Created: 2026-07-25 06:45

Line	Count	Source
1		//! AVX2 backend.
2
3		#![allow(unsafe_op_in_unsafe_fn, reason = "needs triage")]
4		#![allow(clippy::cast_possible_truncation, reason = "needs triage")]
5		#![allow(clippy::cast_possible_wrap, reason = "needs triage")]
6		#![allow(clippy::cast_sign_loss, reason = "needs triage")]
7		#![allow(clippy::undocumented_unsafe_blocks, reason = "TODO")]
8
9		use crate::{Rounds, Variant};
10		use core::marker::PhantomData;
11
12		#[cfg(feature = "rng")]
13		use crate::ChaChaCore;
14
15		#[cfg(feature = "cipher")]
16		use crate::{STATE_WORDS, chacha::Block};
17
18		#[cfg(feature = "cipher")]
19		use cipher::{
20		BlockSizeUser, ParBlocks, ParBlocksSizeUser, StreamCipherBackend, StreamCipherClosure,
21		consts::{U4, U64},
22		};
23
24		#[cfg(target_arch = "x86")]
25		use core::arch::x86::*;
26		#[cfg(target_arch = "x86_64")]
27		use core::arch::x86_64::*;
28
29		/// Number of blocks processed in parallel.
30		const PAR_BLOCKS: usize = 4;
31		/// Number of `__m256i` to store parallel blocks.
32		const N: usize = PAR_BLOCKS / 2;
33
34		#[inline]
35		#[target_feature(enable = "avx2")]
36		#[cfg(feature = "cipher")]
37		#[cfg_attr(chacha20_backend = "avx512", expect(unused))]
38		pub(crate) unsafe fn inner<R, F, V>(state: &mut [u32; STATE_WORDS], f: F)
39		where
40		R: Rounds,
41		F: StreamCipherClosure<BlockSize = U64>,
42		V: Variant,
43		{
44		let state_ptr = state.as_ptr().cast::<__m128i>();
45		let v = [
46		_mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(0))),
47		_mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(1))),
48		_mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(2))),
49		];
50		let mut c = _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(3)));
51		c = match size_of::<V::Counter>() {
52		4 => _mm256_add_epi32(c, _mm256_set_epi32(0, 0, 0, 1, 0, 0, 0, 0)),
53		8 => _mm256_add_epi64(c, _mm256_set_epi64x(0, 1, 0, 0)),
54		_ => unreachable!(),
55		};
56		let mut ctr = [c; N];
57		for i in 0..N {
58		ctr[i] = c;
59		c = match size_of::<V::Counter>() {
60		4 => _mm256_add_epi32(c, _mm256_set_epi32(0, 0, 0, 2, 0, 0, 0, 2)),
61		8 => _mm256_add_epi64(c, _mm256_set_epi64x(0, 2, 0, 2)),
62		_ => unreachable!(),
63		};
64		}
65		let mut backend = Backend::<R, V> {
66		v,
67		ctr,
68		_pd: PhantomData,
69		};
70
71		f.call(&mut backend);
72
73		state[12] = _mm256_extract_epi32(backend.ctr[0], 0) as u32;
74		match size_of::<V::Counter>() {
75		4 => {}
76		8 => state[13] = _mm256_extract_epi32(backend.ctr[0], 1) as u32,
77		_ => unreachable!(),
78		}
79		}
80
81		#[inline]
82		#[target_feature(enable = "avx2")]
83		#[cfg(feature = "rng")]
84	0	pub(crate) unsafe fn rng_inner<R, V>(core: &mut ChaChaCore<R, V>, buffer: &mut [u32; 64])
85	0	where
86	0	R: Rounds,
87	0	V: Variant,
88		{
89	0	let state_ptr = core.state.as_ptr().cast::<__m128i>();
90	0	let v = [
91	0	_mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(0))),
92	0	_mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(1))),
93	0	_mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(2))),
94	0	];
95	0	let mut c = _mm256_broadcastsi128_si256(_mm_loadu_si128(state_ptr.add(3)));
96	0	c = _mm256_add_epi64(c, _mm256_set_epi64x(0, 1, 0, 0));
97	0	let mut ctr = [c; N];
98	0	for i in 0..N {
99	0	ctr[i] = c;
100	0	c = _mm256_add_epi64(c, _mm256_set_epi64x(0, 2, 0, 2));
101	0	}
102	0	let mut backend = Backend::<R, V> {
103	0	v,
104	0	ctr,
105	0	_pd: PhantomData,
106	0	};
107
108	0	backend.rng_gen_par_ks_blocks(buffer);
109
110	0	core.state[12] = _mm256_extract_epi32(backend.ctr[0], 0) as u32;
111	0	core.state[13] = _mm256_extract_epi32(backend.ctr[0], 1) as u32;
112	0	} Unexecuted instantiation: chacha20::backends::avx2::rng_inner::<chacha20::R12, chacha20::variants::Legacy> Unexecuted instantiation: chacha20::backends::avx2::rng_inner::<_, _>
113
114		struct Backend<R: Rounds, V: Variant> {
115		v: [__m256i; 3],
116		ctr: [__m256i; N],
117		_pd: PhantomData<(R, V)>,
118		}
119
120		#[cfg(feature = "cipher")]
121		impl<R: Rounds, V: Variant> BlockSizeUser for Backend<R, V> {
122		type BlockSize = U64;
123		}
124
125		#[cfg(feature = "cipher")]
126		impl<R: Rounds, V: Variant> ParBlocksSizeUser for Backend<R, V> {
127		type ParBlocksSize = U4;
128		}
129
130		#[cfg(feature = "cipher")]
131		impl<R: Rounds, V: Variant> StreamCipherBackend for Backend<R, V> {
132		#[inline(always)]
133		fn gen_ks_block(&mut self, block: &mut Block) {
134		unsafe {
135		let res = rounds::<R>(&self.v, &self.ctr);
136		for c in self.ctr.iter_mut() {
137		*c = match size_of::<V::Counter>() {
138		4 => _mm256_add_epi32(*c, _mm256_set_epi32(0, 0, 0, 1, 0, 0, 0, 1)),
139		8 => _mm256_add_epi64(*c, _mm256_set_epi64x(0, 1, 0, 1)),
140		_ => unreachable!(),
141		};
142		}
143
144		let res0: [__m128i; 8] = core::mem::transmute(res[0]);
145
146		let block_ptr = block.as_mut_ptr().cast::<__m128i>();
147		for i in 0..4 {
148		_mm_storeu_si128(block_ptr.add(i), res0[2 * i]);
149		}
150		}
151		}
152
153		#[inline(always)]
154		fn gen_par_ks_blocks(&mut self, blocks: &mut ParBlocks<Self>) {
155		unsafe {
156		let vs = rounds::<R>(&self.v, &self.ctr);
157
158		let pb = PAR_BLOCKS as i32;
159		for c in self.ctr.iter_mut() {
160		*c = match size_of::<V::Counter>() {
161		4 => _mm256_add_epi32(*c, _mm256_set_epi32(0, 0, 0, pb, 0, 0, 0, pb)),
162		8 => {
163		_mm256_add_epi64(*c, _mm256_set_epi64x(0, i64::from(pb), 0, i64::from(pb)))
164		}
165		_ => unreachable!(),
166		}
167		}
168
169		let mut block_ptr = blocks.as_mut_ptr().cast::<__m128i>();
170		for v in vs {
171		let t: [__m128i; 8] = core::mem::transmute(v);
172		for i in 0..4 {
173		_mm_storeu_si128(block_ptr.add(i), t[2 * i]);
174		_mm_storeu_si128(block_ptr.add(4 + i), t[2 * i + 1]);
175		}
176		block_ptr = block_ptr.add(8);
177		}
178		}
179		}
180		}
181
182		#[cfg(feature = "rng")]
183		impl<R: Rounds, V: Variant> Backend<R, V> {
184		#[inline(always)]
185	0	fn rng_gen_par_ks_blocks(&mut self, blocks: &mut [u32; 64]) {
186		unsafe {
187	0	let vs = rounds::<R>(&self.v, &self.ctr);
188
189	0	let pb = PAR_BLOCKS as i32;
190	0	for c in self.ctr.iter_mut() {
191	0	c = _mm256_add_epi64(c, _mm256_set_epi64x(0, i64::from(pb), 0, i64::from(pb)));
192	0	}
193
194	0	let mut block_ptr = blocks.as_mut_ptr().cast::<__m128i>();
195	0	for v in vs {
196	0	let t: [__m128i; 8] = core::mem::transmute(v);
197	0	for i in 0..4 {
198	0	_mm_storeu_si128(block_ptr.add(i), t[2 * i]);
199	0	_mm_storeu_si128(block_ptr.add(4 + i), t[2 * i + 1]);
200	0	}
201	0	block_ptr = block_ptr.add(8);
202		}
203		}
204	0	} Unexecuted instantiation: <chacha20::backends::avx2::Backend<chacha20::R12, chacha20::variants::Legacy>>::rng_gen_par_ks_blocks Unexecuted instantiation: <chacha20::backends::avx2::Backend<_, _>>::rng_gen_par_ks_blocks
205		}
206
207		#[inline]
208		#[target_feature(enable = "avx2")]
209	0	unsafe fn rounds<R: Rounds>(v: &[__m256i; 3], c: &[__m256i; N]) -> [[__m256i; 4]; N] {
210	0	let mut vs: [[__m256i; 4]; N] = [[_mm256_setzero_si256(); 4]; N];
211	0	for i in 0..N {
212	0	vs[i] = [v[0], v[1], v[2], c[i]];
213	0	}
214	0	for _ in 0..R::COUNT {
215	0	double_quarter_round(&mut vs);
216	0	}
217
218	0	for i in 0..N {
219	0	for j in 0..3 {
220	0	vs[i][j] = _mm256_add_epi32(vs[i][j], v[j]);
221	0	}
222	0	vs[i][3] = _mm256_add_epi32(vs[i][3], c[i]);
223		}
224
225	0	vs
226	0	} Unexecuted instantiation: chacha20::backends::avx2::rounds::<chacha20::R12> Unexecuted instantiation: chacha20::backends::avx2::rounds::<_>
227
228		#[inline]
229		#[target_feature(enable = "avx2")]
230	0	unsafe fn double_quarter_round(v: &mut [[__m256i; 4]; N]) {
231	0	add_xor_rot(v);
232	0	rows_to_cols(v);
233	0	add_xor_rot(v);
234	0	cols_to_rows(v);
235	0	} Unexecuted instantiation: chacha20::backends::avx2::double_quarter_round Unexecuted instantiation: chacha20::backends::avx2::double_quarter_round
236
237		/// The goal of this function is to transform the state words from:
238		/// ```text
239		/// [a0, a1, a2, a3] [ 0, 1, 2, 3]
240		/// [b0, b1, b2, b3] == [ 4, 5, 6, 7]
241		/// [c0, c1, c2, c3] [ 8, 9, 10, 11]
242		/// [d0, d1, d2, d3] [12, 13, 14, 15]
243		/// ```
244		///
245		/// to:
246		/// ```text
247		/// [a0, a1, a2, a3] [ 0, 1, 2, 3]
248		/// [b1, b2, b3, b0] == [ 5, 6, 7, 4]
249		/// [c2, c3, c0, c1] [10, 11, 8, 9]
250		/// [d3, d0, d1, d2] [15, 12, 13, 14]
251		/// ```
252		///
253		/// so that we can apply [`add_xor_rot`] to the resulting columns, and have it compute the
254		/// "diagonal rounds" (as defined in RFC 7539) in parallel. In practice, this shuffle is
255		/// non-optimal: the last state word to be altered in `add_xor_rot` is `b`, so the shuffle
256		/// blocks on the result of `b` being calculated.
257		///
258		/// We can optimize this by observing that the four quarter rounds in `add_xor_rot` are
259		/// data-independent: they only access a single column of the state, and thus the order of
260		/// the columns does not matter. We therefore instead shuffle the other three state words,
261		/// to obtain the following equivalent layout:
262		/// ```text
263		/// [a3, a0, a1, a2] [ 3, 0, 1, 2]
264		/// [b0, b1, b2, b3] == [ 4, 5, 6, 7]
265		/// [c1, c2, c3, c0] [ 9, 10, 11, 8]
266		/// [d2, d3, d0, d1] [14, 15, 12, 13]
267		/// ```
268		///
269		/// See https://github.com/sneves/blake2-avx2/pull/4 for additional details. The earliest
270		/// known occurrence of this optimization is in floodyberry's SSE4 ChaCha code from 2014:
271		/// - https://github.com/floodyberry/chacha-opt/blob/0ab65cb99f5016633b652edebaf3691ceb4ff753/chacha_blocks_ssse3-64.S#L639-L643
272		#[inline]
273		#[target_feature(enable = "avx2")]
274	0	unsafe fn rows_to_cols(vs: &mut [[__m256i; 4]; N]) {
275		// c >>>= 32; d >>>= 64; a >>>= 96;
276	0	for [a, _, c, d] in vs {
277	0	c = _mm256_shuffle_epi32(c, 0b_00_11_10_01); // _MM_SHUFFLE(0, 3, 2, 1)
278	0	d = _mm256_shuffle_epi32(d, 0b_01_00_11_10); // _MM_SHUFFLE(1, 0, 3, 2)
279	0	a = _mm256_shuffle_epi32(a, 0b_10_01_00_11); // _MM_SHUFFLE(2, 1, 0, 3)
280	0	}
281	0	} Unexecuted instantiation: chacha20::backends::avx2::rows_to_cols Unexecuted instantiation: chacha20::backends::avx2::rows_to_cols
282
283		/// The goal of this function is to transform the state words from:
284		/// ```text
285		/// [a3, a0, a1, a2] [ 3, 0, 1, 2]
286		/// [b0, b1, b2, b3] == [ 4, 5, 6, 7]
287		/// [c1, c2, c3, c0] [ 9, 10, 11, 8]
288		/// [d2, d3, d0, d1] [14, 15, 12, 13]
289		/// ```
290		///
291		/// to:
292		/// ```text
293		/// [a0, a1, a2, a3] [ 0, 1, 2, 3]
294		/// [b0, b1, b2, b3] == [ 4, 5, 6, 7]
295		/// [c0, c1, c2, c3] [ 8, 9, 10, 11]
296		/// [d0, d1, d2, d3] [12, 13, 14, 15]
297		/// ```
298		///
299		/// reversing the transformation of [`rows_to_cols`].
300		#[inline]
301		#[target_feature(enable = "avx2")]
302	0	unsafe fn cols_to_rows(vs: &mut [[__m256i; 4]; N]) {
303		// c <<<= 32; d <<<= 64; a <<<= 96;
304	0	for [a, _, c, d] in vs {
305	0	c = _mm256_shuffle_epi32(c, 0b_10_01_00_11); // _MM_SHUFFLE(2, 1, 0, 3)
306	0	d = _mm256_shuffle_epi32(d, 0b_01_00_11_10); // _MM_SHUFFLE(1, 0, 3, 2)
307	0	a = _mm256_shuffle_epi32(a, 0b_00_11_10_01); // _MM_SHUFFLE(0, 3, 2, 1)
308	0	}
309	0	} Unexecuted instantiation: chacha20::backends::avx2::cols_to_rows Unexecuted instantiation: chacha20::backends::avx2::cols_to_rows
310
311		#[inline]
312		#[target_feature(enable = "avx2")]
313	0	unsafe fn add_xor_rot(vs: &mut [[__m256i; 4]; N]) {
314	0	let rol16_mask = _mm256_set_epi64x(
315		0x0d0c_0f0e_0908_0b0a,
316		0x0504_0706_0100_0302,
317		0x0d0c_0f0e_0908_0b0a,
318		0x0504_0706_0100_0302,
319		);
320	0	let rol8_mask = _mm256_set_epi64x(
321		0x0e0d_0c0f_0a09_080b,
322		0x0605_0407_0201_0003,
323		0x0e0d_0c0f_0a09_080b,
324		0x0605_0407_0201_0003,
325		);
326
327		// a += b; d ^= a; d <<<= (16, 16, 16, 16);
328	0	for [a, b, _, d] in vs.iter_mut() {
329	0	a = _mm256_add_epi32(a, *b);
330	0	d = _mm256_xor_si256(d, *a);
331	0	d = _mm256_shuffle_epi8(d, rol16_mask);
332	0	}
333
334		// c += d; b ^= c; b <<<= (12, 12, 12, 12);
335	0	for [_, b, c, d] in vs.iter_mut() {
336	0	c = _mm256_add_epi32(c, *d);
337	0	b = _mm256_xor_si256(b, *c);
338	0	b = _mm256_xor_si256(_mm256_slli_epi32(b, 12), _mm256_srli_epi32(*b, 20));
339	0	}
340
341		// a += b; d ^= a; d <<<= (8, 8, 8, 8);
342	0	for [a, b, _, d] in vs.iter_mut() {
343	0	a = _mm256_add_epi32(a, *b);
344	0	d = _mm256_xor_si256(d, *a);
345	0	d = _mm256_shuffle_epi8(d, rol8_mask);
346	0	}
347
348		// c += d; b ^= c; b <<<= (7, 7, 7, 7);
349	0	for [_, b, c, d] in vs.iter_mut() {
350	0	c = _mm256_add_epi32(c, *d);
351	0	b = _mm256_xor_si256(b, *c);
352	0	b = _mm256_xor_si256(_mm256_slli_epi32(b, 7), _mm256_srli_epi32(*b, 25));
353	0	}
354	0	} Unexecuted instantiation: chacha20::backends::avx2::add_xor_rot Unexecuted instantiation: chacha20::backends::avx2::add_xor_rot