Line data Source code
1 :
2 : // Source originally from https://github.com/BLAKE3-team/BLAKE3
3 : // From commit: c0ea395cf91d242f078c23d5f8d87eb9dd5f7b78
4 :
5 : #include "fd_blake3.h"
6 : #include "fd_blake3_private.h"
7 : #include "../../util/simd/fd_sse.h"
8 :
9 : #define _mm_shuffle_ps2(a, b, c) \
10 151856 : (_mm_castps_si128( \
11 151856 : _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c))))
12 :
13 132874 : #define vu_rot16 vb_exch_adj_pair
14 :
15 : static inline __attribute__((always_inline)) vu_t
16 132874 : vu_rot12( vu_t x ) {
17 132874 : return vu_xor( vu_shr( x, 12 ), vu_shl( x, 32-12 ) );
18 132874 : }
19 :
20 : static inline __attribute__((always_inline)) vu_t
21 132916 : vu_rot8( vu_t x ) {
22 132916 : vb_t const mask = vb( 1,2,3,0, 5,6,7,4, 9,10,11,8, 13,14,15,12 );
23 132916 : return _mm_shuffle_epi8( x, mask );
24 132916 : }
25 :
26 : static inline __attribute__((always_inline)) vu_t
27 132930 : vu_rot7( vu_t x ) {
28 132930 : return vu_xor( vu_shr( x, 7 ), vu_shl( x, 32-7 ) );
29 132930 : }
30 :
31 : static inline __attribute__((always_inline)) void
32 : g1( vu_t * row0,
33 : vu_t * row1,
34 : vu_t * row2,
35 : vu_t * row3,
36 132874 : vu_t m ) {
37 132874 : *row0 = vu_add(vu_add(*row0, m), *row1);
38 132874 : *row3 = vu_xor(*row3, *row0);
39 132874 : *row3 = vu_rot16(*row3);
40 132874 : *row2 = vu_add(*row2, *row3);
41 132874 : *row1 = vu_xor(*row1, *row2);
42 132874 : *row1 = vu_rot12(*row1);
43 132874 : }
44 :
45 : static inline __attribute__((always_inline)) void
46 : g2( vu_t * row0,
47 : vu_t * row1,
48 : vu_t * row2,
49 : vu_t * row3,
50 132916 : vu_t m ) {
51 132916 : *row0 = vu_add(vu_add(*row0, m), *row1);
52 132916 : *row3 = vu_xor(*row3, *row0);
53 132916 : *row3 = vu_rot8(*row3);
54 132916 : *row2 = vu_add(*row2, *row3);
55 132916 : *row1 = vu_xor(*row1, *row2);
56 132916 : *row1 = vu_rot7(*row1);
57 132916 : }
58 :
59 : // Note the optimization here of leaving row1 as the unrotated row, rather than
60 : // row0. All the message loads below are adjusted to compensate for this. See
61 : // discussion at https://github.com/sneves/blake2-avx2/pull/4
62 : static inline __attribute__((always_inline)) void
63 66437 : diagonalize(vu_t *row0, vu_t *row2, vu_t *row3) {
64 66437 : *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3));
65 66437 : *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
66 66437 : *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1));
67 66437 : }
68 :
69 : static inline __attribute__((always_inline)) void
70 66451 : undiagonalize(vu_t *row0, vu_t *row2, vu_t *row3) {
71 66451 : *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1));
72 66451 : *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
73 66451 : *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3));
74 66451 : }
75 :
76 : static inline __attribute__((always_inline)) void
77 : compress_pre( vu_t rows[4],
78 : uint const cv[ static 8 ],
79 : uchar const block[ static FD_BLAKE3_BLOCK_SZ ],
80 : uint block_len,
81 : ulong ctr,
82 9491 : uint flags ) {
83 9491 : rows[0] = vu_ld( cv );
84 9491 : rows[1] = vu_ld( cv+4 );
85 9491 : rows[2] = vu( FD_BLAKE3_IV[0], FD_BLAKE3_IV[1], FD_BLAKE3_IV[2], FD_BLAKE3_IV[3] );
86 9491 : rows[3] = vu( (uint)(ctr&UINT_MAX), (uint)(ctr>>32),
87 9491 : block_len, flags );
88 :
89 9491 : vu_t m0 = vb_ldu( block ); vu_t m1 = vb_ldu( block+16 );
90 9491 : vu_t m2 = vb_ldu( block+32 ); vu_t m3 = vb_ldu( block+48 );
91 :
92 9491 : vu_t t0, t1, t2, t3, tt;
93 :
94 : // Round 1. The first round permutes the message words from the original
95 : // input order, into the groups that get mixed in parallel.
96 9491 : t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0
97 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
98 9491 : t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1
99 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
100 9491 : diagonalize(&rows[0], &rows[2], &rows[3]);
101 9491 : t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8
102 9491 : t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14
103 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
104 9491 : t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9
105 9491 : t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15
106 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
107 9491 : undiagonalize(&rows[0], &rows[2], &rows[3]);
108 9491 : m0 = t0;
109 9491 : m1 = t1;
110 9491 : m2 = t2;
111 9491 : m3 = t3;
112 :
113 : // Round 2. This round and all following rounds apply a fixed permutation
114 : // to the message words from the round before.
115 9491 : t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
116 9491 : t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
117 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
118 9491 : t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
119 9491 : tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
120 9491 : t1 = _mm_blend_epi16(tt, t1, 0xCC);
121 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
122 9491 : diagonalize(&rows[0], &rows[2], &rows[3]);
123 9491 : t2 = _mm_unpacklo_epi64(m3, m1);
124 9491 : tt = _mm_blend_epi16(t2, m2, 0xC0);
125 9491 : t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
126 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
127 9491 : t3 = _mm_unpackhi_epi32(m1, m3);
128 9491 : tt = _mm_unpacklo_epi32(m2, t3);
129 9491 : t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
130 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
131 9491 : undiagonalize(&rows[0], &rows[2], &rows[3]);
132 9491 : m0 = t0;
133 9491 : m1 = t1;
134 9491 : m2 = t2;
135 9491 : m3 = t3;
136 :
137 : // Round 3
138 9491 : t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
139 9491 : t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
140 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
141 9491 : t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
142 9491 : tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
143 9491 : t1 = _mm_blend_epi16(tt, t1, 0xCC);
144 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
145 9491 : diagonalize(&rows[0], &rows[2], &rows[3]);
146 9491 : t2 = _mm_unpacklo_epi64(m3, m1);
147 9491 : tt = _mm_blend_epi16(t2, m2, 0xC0);
148 9491 : t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
149 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
150 9491 : t3 = _mm_unpackhi_epi32(m1, m3);
151 9491 : tt = _mm_unpacklo_epi32(m2, t3);
152 9491 : t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
153 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
154 9491 : undiagonalize(&rows[0], &rows[2], &rows[3]);
155 9491 : m0 = t0;
156 9491 : m1 = t1;
157 9491 : m2 = t2;
158 9491 : m3 = t3;
159 :
160 : // Round 4
161 9491 : t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
162 9491 : t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
163 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
164 9491 : t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
165 9491 : tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
166 9491 : t1 = _mm_blend_epi16(tt, t1, 0xCC);
167 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
168 9491 : diagonalize(&rows[0], &rows[2], &rows[3]);
169 9491 : t2 = _mm_unpacklo_epi64(m3, m1);
170 9491 : tt = _mm_blend_epi16(t2, m2, 0xC0);
171 9491 : t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
172 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
173 9491 : t3 = _mm_unpackhi_epi32(m1, m3);
174 9491 : tt = _mm_unpacklo_epi32(m2, t3);
175 9491 : t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
176 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
177 9491 : undiagonalize(&rows[0], &rows[2], &rows[3]);
178 9491 : m0 = t0;
179 9491 : m1 = t1;
180 9491 : m2 = t2;
181 9491 : m3 = t3;
182 :
183 : // Round 5
184 9491 : t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
185 9491 : t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
186 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
187 9491 : t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
188 9491 : tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
189 9491 : t1 = _mm_blend_epi16(tt, t1, 0xCC);
190 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
191 9491 : diagonalize(&rows[0], &rows[2], &rows[3]);
192 9491 : t2 = _mm_unpacklo_epi64(m3, m1);
193 9491 : tt = _mm_blend_epi16(t2, m2, 0xC0);
194 9491 : t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
195 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
196 9491 : t3 = _mm_unpackhi_epi32(m1, m3);
197 9491 : tt = _mm_unpacklo_epi32(m2, t3);
198 9491 : t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
199 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
200 9491 : undiagonalize(&rows[0], &rows[2], &rows[3]);
201 9491 : m0 = t0;
202 9491 : m1 = t1;
203 9491 : m2 = t2;
204 9491 : m3 = t3;
205 :
206 : // Round 6
207 9491 : t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
208 9491 : t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
209 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
210 9491 : t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
211 9491 : tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
212 9491 : t1 = _mm_blend_epi16(tt, t1, 0xCC);
213 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
214 9491 : diagonalize(&rows[0], &rows[2], &rows[3]);
215 9491 : t2 = _mm_unpacklo_epi64(m3, m1);
216 9491 : tt = _mm_blend_epi16(t2, m2, 0xC0);
217 9491 : t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
218 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
219 9491 : t3 = _mm_unpackhi_epi32(m1, m3);
220 9491 : tt = _mm_unpacklo_epi32(m2, t3);
221 9491 : t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
222 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
223 9491 : undiagonalize(&rows[0], &rows[2], &rows[3]);
224 9491 : m0 = t0;
225 9491 : m1 = t1;
226 9491 : m2 = t2;
227 9491 : m3 = t3;
228 :
229 : // Round 7
230 9491 : t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
231 9491 : t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
232 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
233 9491 : t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
234 9491 : tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
235 9491 : t1 = _mm_blend_epi16(tt, t1, 0xCC);
236 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
237 9491 : diagonalize(&rows[0], &rows[2], &rows[3]);
238 9491 : t2 = _mm_unpacklo_epi64(m3, m1);
239 9491 : tt = _mm_blend_epi16(t2, m2, 0xC0);
240 9491 : t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
241 9491 : g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
242 9491 : t3 = _mm_unpackhi_epi32(m1, m3);
243 9491 : tt = _mm_unpacklo_epi32(m2, t3);
244 9491 : t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
245 9491 : g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
246 9491 : undiagonalize(&rows[0], &rows[2], &rows[3]);
247 9491 : }
248 :
249 : void
250 : fd_blake3_sse_compress1( uchar * restrict out,
251 : uchar const * restrict msg,
252 : uint msg_sz,
253 : ulong counter,
254 : uint const flags,
255 : uchar * restrict out_chain,
256 6888 : uchar const * restrict in_chain ) {
257 6888 : FD_BLAKE3_TRACE(( "fd_blake3_sse_compress1(out=%p,msg=%p,sz=%u,counter=%lu,flags=%02x)",
258 6888 : (void *)out, (void *)msg, msg_sz, counter, flags ));
259 6888 : FD_DCHECK_CRIT( msg_sz<=FD_BLAKE3_CHUNK_SZ, "internal error" );
260 :
261 6888 : uint cv[8] = { FD_BLAKE3_IV[0], FD_BLAKE3_IV[1], FD_BLAKE3_IV[2], FD_BLAKE3_IV[3],
262 6888 : FD_BLAKE3_IV[4], FD_BLAKE3_IV[5], FD_BLAKE3_IV[6], FD_BLAKE3_IV[7] };
263 6888 : if( FD_UNLIKELY( in_chain ) ) {
264 0 : memcpy( cv, in_chain, FD_BLAKE3_OUTCHAIN_SZ );
265 0 : }
266 6888 : vu_t rows[4];
267 :
268 6888 : uint flag_mask = ~fd_uint_if( flags&FD_BLAKE3_FLAG_PARENT,
269 6888 : FD_BLAKE3_FLAG_CHUNK_START|FD_BLAKE3_FLAG_CHUNK_END,
270 6888 : 0U );
271 :
272 6888 : uint block_flags = flags | (flag_mask & FD_BLAKE3_FLAG_CHUNK_START);
273 6888 : if( FD_UNLIKELY( in_chain && !(flags&FD_BLAKE3_FLAG_CHUNK_START) ) ) {
274 0 : block_flags &= ~FD_BLAKE3_FLAG_CHUNK_START;
275 0 : }
276 13482 : do {
277 13482 : uint block_sz = fd_uint_min( msg_sz, FD_BLAKE3_BLOCK_SZ );
278 13482 : block_flags |= FD_BLAKE3_FLAG_CHUNK_END;
279 13482 : block_flags &= (flag_mask & ~fd_uint_if( msg_sz<=FD_BLAKE3_BLOCK_SZ, 0, (FD_BLAKE3_FLAG_CHUNK_END|FD_BLAKE3_FLAG_ROOT) ) );
280 :
281 13482 : uchar tail[ FD_BLAKE3_BLOCK_SZ ] __attribute__((aligned(16)));
282 13482 : uchar const * restrict block;
283 13482 : if( FD_LIKELY( msg_sz>=FD_BLAKE3_BLOCK_SZ ) ) {
284 8137 : block = msg;
285 8137 : } else {
286 5345 : vb_st( tail, vu_zero() );
287 5345 : vb_st( tail+16, vu_zero() );
288 5345 : vb_st( tail+32, vu_zero() );
289 5345 : vb_st( tail+48, vu_zero() );
290 5345 : fd_memcpy( tail, msg, msg_sz );
291 5345 : block = tail;
292 5345 : }
293 :
294 13482 : if( FD_UNLIKELY( out_chain && (block_flags & FD_BLAKE3_FLAG_CHUNK_END) ) ) {
295 : /* FIXME better document and polish the transition from the compress
296 : part to the expand part. */
297 3988 : fd_memcpy( out, block, FD_BLAKE3_BLOCK_SZ ); /* FIXME DOCUMENT OVERLOADING OF OUT ARGUMENT */
298 3988 : fd_memcpy( out_chain, cv, FD_BLAKE3_OUTCHAIN_SZ );
299 3988 : FD_BLAKE3_TRACE(( "fd_blake3_sse_compress1: done (XOF mode)" ));
300 3988 : return;
301 3988 : }
302 :
303 9494 : FD_BLAKE3_TRACE(( "fd_blake3_sse_compress1: sz=%u counter=%u flags=%x", block_sz, (uint)counter, block_flags ));
304 9494 : compress_pre( rows, cv, block, block_sz, counter, block_flags );
305 9494 : if( FD_UNLIKELY( in_chain ) ) {
306 : /* FIXME UGLY */
307 0 : vu_stu( out+32, vu_xor( vu_ld( cv ), rows[2] ) );
308 0 : vu_stu( out+48, vu_xor( vu_ld( cv+4 ), rows[3] ) );
309 0 : }
310 9494 : vu_st( cv, vu_xor( rows[0], rows[2] ) );
311 9494 : vu_st( cv+4, vu_xor( rows[1], rows[3] ) );
312 9494 : msg += FD_BLAKE3_BLOCK_SZ;
313 9494 : msg_sz -= block_sz;
314 9494 : block_flags = flags;
315 9494 : } while( (int)msg_sz>0 );
316 :
317 2900 : vu_stu( out, vu_ld( cv ) );
318 2900 : vu_stu( out+16, vu_ld( cv+4 ) );
319 :
320 2900 : FD_BLAKE3_TRACE(( "fd_blake3_sse_compress1: done" ));
321 2900 : }
|
