Line data Source code
1 : #include "fd_sha256.h"
2 : #include "fd_sha256_constants.h"
3 :
4 : #if FD_HAS_SHANI
5 : /* For the optimized repeated hash */
6 : #include "../../util/simd/fd_sse.h"
7 : #endif
8 :
9 : ulong
10 23457 : fd_sha256_align( void ) {
11 23457 : return FD_SHA256_ALIGN;
12 23457 : }
13 :
14 : ulong
15 11727 : fd_sha256_footprint( void ) {
16 11727 : return FD_SHA256_FOOTPRINT;
17 11727 : }
18 :
19 : void *
20 11735 : fd_sha256_new( void * shmem ) {
21 11735 : fd_sha256_t * sha = (fd_sha256_t *)shmem;
22 :
23 11735 : if( FD_UNLIKELY( !shmem ) ) {
24 0 : FD_LOG_WARNING(( "NULL shmem" ));
25 0 : return NULL;
26 0 : }
27 :
28 11735 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_sha256_align() ) ) ) {
29 0 : FD_LOG_WARNING(( "misaligned shmem" ));
30 0 : return NULL;
31 0 : }
32 :
33 11735 : ulong footprint = fd_sha256_footprint();
34 :
35 11735 : fd_memset( sha, 0, footprint );
36 :
37 11735 : FD_COMPILER_MFENCE();
38 11735 : FD_VOLATILE( sha->magic ) = FD_SHA256_MAGIC;
39 11735 : FD_COMPILER_MFENCE();
40 :
41 11735 : return (void *)sha;
42 11735 : }
43 :
44 : fd_sha256_t *
45 11730 : fd_sha256_join( void * shsha ) {
46 :
47 11730 : if( FD_UNLIKELY( !shsha ) ) {
48 0 : FD_LOG_WARNING(( "NULL shsha" ));
49 0 : return NULL;
50 0 : }
51 :
52 11730 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shsha, fd_sha256_align() ) ) ) {
53 0 : FD_LOG_WARNING(( "misaligned shsha" ));
54 0 : return NULL;
55 0 : }
56 :
57 11730 : fd_sha256_t * sha = (fd_sha256_t *)shsha;
58 :
59 11730 : if( FD_UNLIKELY( sha->magic!=FD_SHA256_MAGIC ) ) {
60 0 : FD_LOG_WARNING(( "bad magic" ));
61 0 : return NULL;
62 0 : }
63 :
64 11730 : return sha;
65 11730 : }
66 :
67 : void *
68 0 : fd_sha256_leave( fd_sha256_t * sha ) {
69 :
70 0 : if( FD_UNLIKELY( !sha ) ) {
71 0 : FD_LOG_WARNING(( "NULL sha" ));
72 0 : return NULL;
73 0 : }
74 :
75 0 : return (void *)sha;
76 0 : }
77 :
78 : void *
79 0 : fd_sha256_delete( void * shsha ) {
80 :
81 0 : if( FD_UNLIKELY( !shsha ) ) {
82 0 : FD_LOG_WARNING(( "NULL shsha" ));
83 0 : return NULL;
84 0 : }
85 :
86 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shsha, fd_sha256_align() ) ) ) {
87 0 : FD_LOG_WARNING(( "misaligned shsha" ));
88 0 : return NULL;
89 0 : }
90 :
91 0 : fd_sha256_t * sha = (fd_sha256_t *)shsha;
92 :
93 0 : if( FD_UNLIKELY( sha->magic!=FD_SHA256_MAGIC ) ) {
94 0 : FD_LOG_WARNING(( "bad magic" ));
95 0 : return NULL;
96 0 : }
97 :
98 0 : FD_COMPILER_MFENCE();
99 0 : FD_VOLATILE( sha->magic ) = 0UL;
100 0 : FD_COMPILER_MFENCE();
101 :
102 0 : return (void *)sha;
103 0 : }
104 :
105 : #ifndef FD_SHA256_CORE_IMPL
106 : #if FD_HAS_ARM_SHA256
107 : #define FD_SHA256_CORE_IMPL 2
108 : #elif FD_HAS_SHANI
109 : #define FD_SHA256_CORE_IMPL 1
110 : #elif FD_HAS_RISCV_SHA256
111 : #define FD_SHA256_CORE_IMPL 3
112 : #else
113 : #define FD_SHA256_CORE_IMPL 0
114 : #endif
115 : #endif
116 :
117 : #if FD_SHA256_CORE_IMPL==0
118 :
119 : /* The implementation below was derived from OpenSSL's SHA-256
120 : implementation (Apache-2.0 licensed). See in particular:
121 :
122 : https://github.com/openssl/openssl/blob/master/crypto/sha/sha256.c
123 :
124 : (link valid circa 2022-Dec). It has been made more strict with more
125 : extensive implementation documentation, has been simplified and has
126 : been streamlined specifically for use inside Firedancer base machine
127 : model (no machine specific capabilities required).
128 :
129 : In particular, fd_sha256_core_ref is based on OpenSSL's
130 : OPENSSL_SMALL_FOOTPRINT SHA-256 implementation (Apache licensed).
131 : This should work anywhere but it is not the highest performance
132 : implementation possible.
133 :
134 : It is also straightforward to replace these implementations with HPC
135 : implementations that target specific machine capabilities without
136 : requiring any changes to caller code. */
137 :
138 : static void
139 : fd_sha256_core_ref( uint * state,
140 : uchar const * block,
141 : ulong block_cnt ) {
142 :
143 :
144 : # define ROTATE fd_uint_rotate_left
145 : # define Sigma0(x) (ROTATE((x),30) ^ ROTATE((x),19) ^ ROTATE((x),10))
146 : # define Sigma1(x) (ROTATE((x),26) ^ ROTATE((x),21) ^ ROTATE((x),7))
147 : # define sigma0(x) (ROTATE((x),25) ^ ROTATE((x),14) ^ ((x)>>3))
148 : # define sigma1(x) (ROTATE((x),15) ^ ROTATE((x),13) ^ ((x)>>10))
149 : # define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z)))
150 : # define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
151 :
152 : uchar const * W = block;
153 : do {
154 : uint a = state[0];
155 : uint b = state[1];
156 : uint c = state[2];
157 : uint d = state[3];
158 : uint e = state[4];
159 : uint f = state[5];
160 : uint g = state[6];
161 : uint h = state[7];
162 :
163 : uint X[16];
164 :
165 : ulong i;
166 : for( i=0UL; i<16UL; i++ ) {
167 : X[i] = fd_uint_bswap( FD_LOAD( uint, W + i*sizeof(uint) ) );
168 : uint T1 = X[i] + h + Sigma1(e) + Ch(e, f, g) + fd_sha256_K[i];
169 : uint T2 = Sigma0(a) + Maj(a, b, c);
170 : h = g;
171 : g = f;
172 : f = e;
173 : e = d + T1;
174 : d = c;
175 : c = b;
176 : b = a;
177 : a = T1 + T2;
178 : }
179 : for( ; i<64UL; i++ ) {
180 : uint s0 = X[(i + 1UL) & 0x0fUL];
181 : uint s1 = X[(i + 14UL) & 0x0fUL];
182 : s0 = sigma0(s0);
183 : s1 = sigma1(s1);
184 : X[i & 0xfUL] += s0 + s1 + X[(i + 9UL) & 0xfUL];
185 : uint T1 = X[i & 0xfUL ] + h + Sigma1(e) + Ch(e, f, g) + fd_sha256_K[i];
186 : uint T2 = Sigma0(a) + Maj(a, b, c);
187 : h = g;
188 : g = f;
189 : f = e;
190 : e = d + T1;
191 : d = c;
192 : c = b;
193 : b = a;
194 : a = T1 + T2;
195 : }
196 :
197 : state[0] += a;
198 : state[1] += b;
199 : state[2] += c;
200 : state[3] += d;
201 : state[4] += e;
202 : state[5] += f;
203 : state[6] += g;
204 : state[7] += h;
205 :
206 : W += 16UL*sizeof(uint);
207 : } while( --block_cnt );
208 :
209 : # undef ROTATE
210 : # undef Sigma0
211 : # undef Sigma1
212 : # undef sigma0
213 : # undef sigma1
214 : # undef Ch
215 : # undef Maj
216 :
217 : }
218 :
219 : #define fd_sha256_core fd_sha256_core_ref
220 :
221 : #elif FD_SHA256_CORE_IMPL==1
222 :
223 : /* _mm_sha256rnds2_epu32 does two rounds, one from the first uint in
224 : wk and one from the second. Since wk stores four rounds worth of
225 : message schedule values, it makes sense for the macro to do four
226 : rounds at a time. We need to permute wk in between so that the
227 : second call to the intrinsic will use the other values. */
228 913184 : #define FOUR_ROUNDS_STATE( stateFEBA, stateHGDC, wk ) do { \
229 913184 : vu_t __wk = (wk); \
230 913184 : vu_t temp_state = stateFEBA; \
231 913184 : stateFEBA = _mm_sha256rnds2_epu32( stateHGDC, stateFEBA, __wk ); \
232 913184 : stateHGDC = temp_state; \
233 913184 : \
234 913184 : temp_state = stateFEBA; \
235 913184 : stateFEBA = _mm_sha256rnds2_epu32( stateHGDC, stateFEBA, vu_permute( __wk, 2,3,0,1 ) );\
236 913184 : stateHGDC = temp_state; \
237 913184 : } while( 0 )
238 :
239 913184 : #define FOUR_ROUNDS( wk ) FOUR_ROUNDS_STATE( stateFEBA, stateHGDC, wk )
240 :
241 :
242 : /* For completeness, here's the documentation for _mm_sha256msg1_epu32
243 : and _mm_sha256msg2_epu32 in a slightly reformatted way, where all
244 : values are uints, and "-" indicates a don't-care value:
245 :
246 : _mm_sha256msg1_epu32( (w[j ], w[j+1], w[j+1], w[j+3]),
247 : (w[j+4], -, -, - ) )
248 : = ( w[j ]+s0( w[j+1] ), w[j+1]+s0( w[j+2] ),
249 : w[j+2]+s0( w[j+3] ), w[j+3]+s0( w[j+4] ) ).
250 :
251 :
252 : _mm_sha256msg2_epu32( (v[j ], v[j+1], v[j+1], v[j+3]),
253 : (-, -, w[j-2], w[j-1]) )
254 : sets w[j ] = v[j ] + s1( w[j-2] ) and
255 : w[j+1] = v[j+1] + s1( w[j-1] ), and then returns
256 :
257 : ( v[j ]+s1( w[j-2] ), v[j+1]+s1( w[j-1] ),
258 : v[j+2]+s1( w[j ] ), v[j+3]+s1( w[j+1] ) ) */
259 :
260 :
261 : /* w[i] for i>= 16 is w[i-16] + s0(w[i-15]) + w[i-7] + s1(w[i-2])
262 : Since our vector size is 4 uints, it's only s1 that is a little
263 : problematic, because it references items in the same vector.
264 : Thankfully, the msg2 intrinsic takes care of the complexity, but we
265 : need to execute it last.
266 :
267 : We get w[i-16] and s0(s[i-15]) using the msg1 intrinsic, setting j =
268 : i-16. For example, to compute w1013, we pass in w0003 and w0407.
269 : Then we can get w[i-7] by using the alignr instruction on
270 : (w[i-8], w[i-7], w[i-6], w[i-5]) and (w[i-4], w[i-3], w[i-2], w[i-1])
271 : to concatenate them and shift by one uint. Continuing with the
272 : example of w1013, we need w080b and w0c0f. We then put
273 : v[i] = w[i-16] + s0(w[i-15]) + w[i-7],
274 : and invoke the msg2 intrinsic with j=i, which gives w[i], as desired.
275 : Each invocation of NEXT_W computes 4 values of w. */
276 :
277 684888 : #define NEXT_W( w_minus_16, w_minus_12, w_minus_8, w_minus_4 ) (__extension__({ \
278 684888 : vu_t __w_i_16_s0_i_15 = _mm_sha256msg1_epu32( w_minus_16, w_minus_12 ); \
279 684888 : vu_t __w_i_7 = _mm_alignr_epi8( w_minus_4, w_minus_8, 4 ); \
280 684888 : _mm_sha256msg2_epu32( vu_add( __w_i_7, __w_i_16_s0_i_15 ), w_minus_4 ); \
281 684888 : }))
282 :
283 : /* Zen 5's sha256rnds2 has an RTP of 2, while Zen 4's has an RTP of 1. We can
284 : win some performance by moving the schedule updates earlier in the loop,
285 : which improves the speed by around 1M hashes/s for the repeated hashing. */
286 : #ifdef __znver5__
287 : #define FULL_ROUNDS() do { \
288 : vu_t w1013 = NEXT_W( w0003, w0407, w080b, w0c0f ); FOUR_ROUNDS( vu_add( w0003, vu_ld( fd_sha256_K+ 0UL ) ) ); \
289 : vu_t w1417 = NEXT_W( w0407, w080b, w0c0f, w1013 ); FOUR_ROUNDS( vu_add( w0407, vu_ld( fd_sha256_K+ 4UL ) ) ); \
290 : vu_t w181b = NEXT_W( w080b, w0c0f, w1013, w1417 ); FOUR_ROUNDS( vu_add( w080b, vu_ld( fd_sha256_K+ 8UL ) ) ); \
291 : vu_t w1c1f = NEXT_W( w0c0f, w1013, w1417, w181b ); FOUR_ROUNDS( vu_add( w0c0f, vu_ld( fd_sha256_K+12UL ) ) ); \
292 : vu_t w2023 = NEXT_W( w1013, w1417, w181b, w1c1f ); FOUR_ROUNDS( vu_add( w1013, vu_ld( fd_sha256_K+16UL ) ) ); \
293 : vu_t w2427 = NEXT_W( w1417, w181b, w1c1f, w2023 ); FOUR_ROUNDS( vu_add( w1417, vu_ld( fd_sha256_K+20UL ) ) ); \
294 : vu_t w282b = NEXT_W( w181b, w1c1f, w2023, w2427 ); FOUR_ROUNDS( vu_add( w181b, vu_ld( fd_sha256_K+24UL ) ) ); \
295 : vu_t w2c2f = NEXT_W( w1c1f, w2023, w2427, w282b ); FOUR_ROUNDS( vu_add( w1c1f, vu_ld( fd_sha256_K+28UL ) ) ); \
296 : vu_t w3033 = NEXT_W( w2023, w2427, w282b, w2c2f ); FOUR_ROUNDS( vu_add( w2023, vu_ld( fd_sha256_K+32UL ) ) ); \
297 : vu_t w3437 = NEXT_W( w2427, w282b, w2c2f, w3033 ); FOUR_ROUNDS( vu_add( w2427, vu_ld( fd_sha256_K+36UL ) ) ); \
298 : vu_t w383b = NEXT_W( w282b, w2c2f, w3033, w3437 ); FOUR_ROUNDS( vu_add( w282b, vu_ld( fd_sha256_K+40UL ) ) ); \
299 : vu_t w3c3f = NEXT_W( w2c2f, w3033, w3437, w383b ); FOUR_ROUNDS( vu_add( w2c2f, vu_ld( fd_sha256_K+44UL ) ) ); \
300 : /* */ FOUR_ROUNDS( vu_add( w3033, vu_ld( fd_sha256_K+48UL ) ) ); \
301 : /* */ FOUR_ROUNDS( vu_add( w3437, vu_ld( fd_sha256_K+52UL ) ) ); \
302 : /* */ FOUR_ROUNDS( vu_add( w383b, vu_ld( fd_sha256_K+56UL ) ) ); \
303 : /* */ FOUR_ROUNDS( vu_add( w3c3f, vu_ld( fd_sha256_K+60UL ) ) ); \
304 : } while ( 0 )
305 : #else
306 57074 : #define FULL_ROUNDS() do { \
307 57074 : /* */ FOUR_ROUNDS( vu_add( w0003, vu_ld( fd_sha256_K+ 0UL ) ) ); \
308 57074 : /* */ FOUR_ROUNDS( vu_add( w0407, vu_ld( fd_sha256_K+ 4UL ) ) ); \
309 57074 : /* */ FOUR_ROUNDS( vu_add( w080b, vu_ld( fd_sha256_K+ 8UL ) ) ); \
310 57074 : /* */ FOUR_ROUNDS( vu_add( w0c0f, vu_ld( fd_sha256_K+12UL ) ) ); \
311 57074 : vu_t w1013 = NEXT_W( w0003, w0407, w080b, w0c0f ); FOUR_ROUNDS( vu_add( w1013, vu_ld( fd_sha256_K+16UL ) ) ); \
312 57074 : vu_t w1417 = NEXT_W( w0407, w080b, w0c0f, w1013 ); FOUR_ROUNDS( vu_add( w1417, vu_ld( fd_sha256_K+20UL ) ) ); \
313 57074 : vu_t w181b = NEXT_W( w080b, w0c0f, w1013, w1417 ); FOUR_ROUNDS( vu_add( w181b, vu_ld( fd_sha256_K+24UL ) ) ); \
314 57074 : vu_t w1c1f = NEXT_W( w0c0f, w1013, w1417, w181b ); FOUR_ROUNDS( vu_add( w1c1f, vu_ld( fd_sha256_K+28UL ) ) ); \
315 57074 : vu_t w2023 = NEXT_W( w1013, w1417, w181b, w1c1f ); FOUR_ROUNDS( vu_add( w2023, vu_ld( fd_sha256_K+32UL ) ) ); \
316 57074 : vu_t w2427 = NEXT_W( w1417, w181b, w1c1f, w2023 ); FOUR_ROUNDS( vu_add( w2427, vu_ld( fd_sha256_K+36UL ) ) ); \
317 57074 : vu_t w282b = NEXT_W( w181b, w1c1f, w2023, w2427 ); FOUR_ROUNDS( vu_add( w282b, vu_ld( fd_sha256_K+40UL ) ) ); \
318 57074 : vu_t w2c2f = NEXT_W( w1c1f, w2023, w2427, w282b ); FOUR_ROUNDS( vu_add( w2c2f, vu_ld( fd_sha256_K+44UL ) ) ); \
319 57074 : vu_t w3033 = NEXT_W( w2023, w2427, w282b, w2c2f ); FOUR_ROUNDS( vu_add( w3033, vu_ld( fd_sha256_K+48UL ) ) ); \
320 57074 : vu_t w3437 = NEXT_W( w2427, w282b, w2c2f, w3033 ); FOUR_ROUNDS( vu_add( w3437, vu_ld( fd_sha256_K+52UL ) ) ); \
321 57074 : vu_t w383b = NEXT_W( w282b, w2c2f, w3033, w3437 ); FOUR_ROUNDS( vu_add( w383b, vu_ld( fd_sha256_K+56UL ) ) ); \
322 57074 : vu_t w3c3f = NEXT_W( w2c2f, w3033, w3437, w383b ); FOUR_ROUNDS( vu_add( w3c3f, vu_ld( fd_sha256_K+60UL ) ) ); \
323 57074 : } while ( 0 )
324 : #endif
325 :
326 :
327 : void
328 : fd_sha256_core_shaext( uint * state, /* 64-byte aligned, 8 entries */
329 : uchar const * block, /* ideally 128-byte aligned (but not required), 64*block_cnt in size */
330 32980 : ulong block_cnt ) {/* positive */
331 32980 : vu_t stateABCD = vu_ld( state );
332 32980 : vu_t stateEFGH = vu_ld( state+4UL );
333 :
334 32980 : vu_t baseFEBA = vu_permute2( stateEFGH, stateABCD, 1, 0, 1, 0 );
335 32980 : vu_t baseHGDC = vu_permute2( stateEFGH, stateABCD, 3, 2, 3, 2 );
336 :
337 90054 : for( ulong b=0UL; b<block_cnt; b++ ) {
338 57074 : vu_t stateFEBA = baseFEBA;
339 57074 : vu_t stateHGDC = baseHGDC;
340 :
341 57074 : vu_t w0003 = vu_bswap( vu_ldu( block+64UL*b ) );
342 57074 : vu_t w0407 = vu_bswap( vu_ldu( block+64UL*b+16UL ) );
343 57074 : vu_t w080b = vu_bswap( vu_ldu( block+64UL*b+32UL ) );
344 57074 : vu_t w0c0f = vu_bswap( vu_ldu( block+64UL*b+48UL ) );
345 :
346 57074 : FULL_ROUNDS();
347 :
348 57074 : baseFEBA = vu_add( baseFEBA, stateFEBA );
349 57074 : baseHGDC = vu_add( baseHGDC, stateHGDC );
350 :
351 57074 : }
352 :
353 32980 : stateABCD = vu_permute2( baseFEBA, baseHGDC, 3, 2, 3, 2 );
354 32980 : stateEFGH = vu_permute2( baseFEBA, baseHGDC, 1, 0, 1, 0 );
355 32980 : vu_st( state, stateABCD );
356 32980 : vu_st( state+4UL, stateEFGH );
357 32980 : }
358 :
359 32996 : #define fd_sha256_core fd_sha256_core_shaext
360 :
361 : #elif FD_SHA256_CORE_IMPL==2
362 :
363 : void
364 : fd_sha256_core_arm( uint * state,
365 : uchar const * block,
366 : ulong block_cnt );
367 :
368 : #define fd_sha256_core fd_sha256_core_arm
369 :
370 : #elif FD_SHA256_CORE_IMPL==3
371 :
372 : void
373 : fd_sha256_core_riscv( uint * state,
374 : uchar const * block,
375 : ulong block_cnt );
376 :
377 : #define fd_sha256_core fd_sha256_core_riscv
378 :
379 : #else
380 : #error "Unsupported FD_SHA256_CORE_IMPL"
381 : #endif
382 :
383 : fd_sha256_t *
384 12517 : fd_sha256_init( fd_sha256_t * sha ) {
385 12517 : sha->state[0] = FD_SHA256_INITIAL_A;
386 12517 : sha->state[1] = FD_SHA256_INITIAL_B;
387 12517 : sha->state[2] = FD_SHA256_INITIAL_C;
388 12517 : sha->state[3] = FD_SHA256_INITIAL_D;
389 12517 : sha->state[4] = FD_SHA256_INITIAL_E;
390 12517 : sha->state[5] = FD_SHA256_INITIAL_F;
391 12517 : sha->state[6] = FD_SHA256_INITIAL_G;
392 12517 : sha->state[7] = FD_SHA256_INITIAL_H;
393 12517 : sha->buf_used = 0UL;
394 12517 : sha->bit_cnt = 0UL;
395 12517 : return sha;
396 12517 : }
397 :
398 : fd_sha256_t *
399 : fd_sha256_append( fd_sha256_t * sha,
400 : void const * _data,
401 29790 : ulong sz ) {
402 :
403 : /* If no data to append, we are done */
404 :
405 29790 : if( FD_UNLIKELY( !sz ) ) return sha; /* optimize for non-trivial append */
406 :
407 : /* Unpack inputs */
408 :
409 29675 : uint * state = sha->state;
410 29675 : uchar * buf = sha->buf;
411 29675 : ulong buf_used = sha->buf_used;
412 29675 : ulong bit_cnt = sha->bit_cnt;
413 :
414 29675 : uchar const * data = (uchar const *)_data;
415 :
416 : /* Update bit_cnt */
417 : /* FIXME: could accumulate bytes here and do bit conversion in append */
418 : /* FIXME: Overflow handling if more than 2^64 bits (unlikely) */
419 :
420 29675 : sha->bit_cnt = bit_cnt + (sz<<3);
421 :
422 : /* Handle buffered bytes from previous appends */
423 :
424 29675 : if( FD_UNLIKELY( buf_used ) ) { /* optimized for well aligned use of append */
425 :
426 : /* If the append isn't large enough to complete the current block,
427 : buffer these bytes too and return */
428 :
429 16976 : ulong buf_rem = FD_SHA256_PRIVATE_BUF_MAX - buf_used; /* In (0,FD_SHA256_PRIVATE_BUF_MAX) */
430 16976 : if( FD_UNLIKELY( sz < buf_rem ) ) { /* optimize for large append */
431 8879 : fd_memcpy( buf + buf_used, data, sz );
432 8879 : sha->buf_used = buf_used + sz;
433 8879 : return sha;
434 8879 : }
435 :
436 : /* Otherwise, buffer enough leading bytes of data to complete the
437 : block, update the hash and then continue processing any remaining
438 : bytes of data. */
439 :
440 8097 : fd_memcpy( buf + buf_used, data, buf_rem );
441 8097 : data += buf_rem;
442 8097 : sz -= buf_rem;
443 :
444 8097 : fd_sha256_core( state, buf, 1UL );
445 8097 : sha->buf_used = 0UL;
446 8097 : }
447 :
448 : /* Append the bulk of the data */
449 :
450 20796 : ulong block_cnt = sz >> FD_SHA256_PRIVATE_LG_BUF_MAX;
451 20796 : if( FD_LIKELY( block_cnt ) ) fd_sha256_core( state, data, block_cnt ); /* optimized for large append */
452 :
453 : /* Buffer any leftover bytes */
454 :
455 20796 : buf_used = sz & (FD_SHA256_PRIVATE_BUF_MAX-1UL); /* In [0,FD_SHA256_PRIVATE_BUF_MAX) */
456 20796 : if( FD_UNLIKELY( buf_used ) ) { /* optimized for well aligned use of append */
457 20421 : fd_memcpy( buf, data + (block_cnt << FD_SHA256_PRIVATE_LG_BUF_MAX), buf_used );
458 20421 : sha->buf_used = buf_used; /* In (0,FD_SHA256_PRIVATE_BUF_MAX) */
459 20421 : }
460 :
461 20796 : return sha;
462 29675 : }
463 :
464 : void *
465 : fd_sha256_fini( fd_sha256_t * sha,
466 12503 : void * _hash ) {
467 :
468 : /* Unpack inputs */
469 :
470 12503 : uint * state = sha->state;
471 12503 : uchar * buf = sha->buf;
472 12503 : ulong buf_used = sha->buf_used; /* In [0,FD_SHA256_PRIVATE_BUF_MAX) */
473 12503 : ulong bit_cnt = sha->bit_cnt;
474 :
475 : /* Append the terminating message byte */
476 :
477 12503 : buf[ buf_used ] = (uchar)0x80;
478 12503 : buf_used++;
479 :
480 : /* If there isn't enough room to save the message length in bits at
481 : the end of the in progress block, clear the rest of the in progress
482 : block, update the hash and start a new block. */
483 :
484 12503 : if( FD_UNLIKELY( buf_used > (FD_SHA256_PRIVATE_BUF_MAX-8UL) ) ) { /* optimize for well aligned use of append */
485 421 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-buf_used );
486 421 : fd_sha256_core( state, buf, 1UL );
487 421 : buf_used = 0UL;
488 421 : }
489 :
490 : /* Clear in progress block up to last 64-bits, append the message
491 : size in bytes in the last 64-bits of the in progress block and
492 : update the hash to finalize it. */
493 :
494 12503 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-8UL-buf_used );
495 12503 : FD_STORE( ulong, buf+FD_SHA256_PRIVATE_BUF_MAX-8UL, fd_ulong_bswap( bit_cnt ) );
496 12503 : fd_sha256_core( state, buf, 1UL );
497 :
498 : /* Unpack the result into md (annoying bswaps here) */
499 :
500 12503 : state[0] = fd_uint_bswap( state[0] );
501 12503 : state[1] = fd_uint_bswap( state[1] );
502 12503 : state[2] = fd_uint_bswap( state[2] );
503 12503 : state[3] = fd_uint_bswap( state[3] );
504 12503 : state[4] = fd_uint_bswap( state[4] );
505 12503 : state[5] = fd_uint_bswap( state[5] );
506 12503 : state[6] = fd_uint_bswap( state[6] );
507 12503 : state[7] = fd_uint_bswap( state[7] );
508 12503 : return memcpy( _hash, state, 32 );
509 12503 : }
510 :
511 : void *
512 : fd_sha256_hash( void const * _data,
513 : ulong sz,
514 7817 : void * _hash ) {
515 7817 : uchar const * data = (uchar const *)_data;
516 :
517 : /* This is just the above streamlined to eliminate all the overheads
518 : to support incremental hashing. */
519 :
520 7817 : uchar buf[ FD_SHA256_PRIVATE_BUF_MAX ] __attribute__((aligned(128)));
521 7817 : uint state[8] __attribute__((aligned(32)));
522 :
523 7817 : state[0] = FD_SHA256_INITIAL_A;
524 7817 : state[1] = FD_SHA256_INITIAL_B;
525 7817 : state[2] = FD_SHA256_INITIAL_C;
526 7817 : state[3] = FD_SHA256_INITIAL_D;
527 7817 : state[4] = FD_SHA256_INITIAL_E;
528 7817 : state[5] = FD_SHA256_INITIAL_F;
529 7817 : state[6] = FD_SHA256_INITIAL_G;
530 7817 : state[7] = FD_SHA256_INITIAL_H;
531 :
532 7817 : ulong block_cnt = sz >> FD_SHA256_PRIVATE_LG_BUF_MAX;
533 7817 : if( FD_LIKELY( block_cnt ) ) fd_sha256_core( state, data, block_cnt );
534 :
535 7817 : ulong buf_used = sz & (FD_SHA256_PRIVATE_BUF_MAX-1UL);
536 7817 : if( FD_UNLIKELY( buf_used ) ) fd_memcpy( buf, data + (block_cnt << FD_SHA256_PRIVATE_LG_BUF_MAX), buf_used );
537 7817 : buf[ buf_used ] = (uchar)0x80;
538 7817 : buf_used++;
539 :
540 7817 : if( FD_UNLIKELY( buf_used > (FD_SHA256_PRIVATE_BUF_MAX-8UL) ) ) {
541 0 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-buf_used );
542 0 : fd_sha256_core( state, buf, 1UL );
543 0 : buf_used = 0UL;
544 0 : }
545 :
546 7817 : ulong bit_cnt = sz << 3;
547 7817 : fd_memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-8UL-buf_used );
548 7817 : FD_STORE( ulong, buf+FD_SHA256_PRIVATE_BUF_MAX-8UL, fd_ulong_bswap( bit_cnt ) );
549 7817 : fd_sha256_core( state, buf, 1UL );
550 :
551 7817 : state[0] = fd_uint_bswap( state[0] );
552 7817 : state[1] = fd_uint_bswap( state[1] );
553 7817 : state[2] = fd_uint_bswap( state[2] );
554 7817 : state[3] = fd_uint_bswap( state[3] );
555 7817 : state[4] = fd_uint_bswap( state[4] );
556 7817 : state[5] = fd_uint_bswap( state[5] );
557 7817 : state[6] = fd_uint_bswap( state[6] );
558 7817 : state[7] = fd_uint_bswap( state[7] );
559 7817 : return memcpy( _hash, state, 32 );
560 7817 : }
561 :
562 :
563 :
564 : #if FD_SHA256_CORE_IMPL==2
565 : void
566 : fd_sha256_hash_32_repeated_arm( uchar const * data,
567 : uchar * hash,
568 : ulong cnt );
569 : #elif FD_SHA256_CORE_IMPL==3
570 : void
571 : fd_sha256_hash_32_repeated_riscv( uchar const * data,
572 : uchar * hash,
573 : ulong cnt );
574 : #endif
575 :
576 : void *
577 : fd_sha256_hash_32_repeated( void const * _data,
578 : void * _hash,
579 0 : ulong cnt ) {
580 : #if FD_SHA256_CORE_IMPL==2
581 : fd_sha256_hash_32_repeated_arm( _data, _hash, cnt );
582 : return _hash;
583 : #elif FD_SHA256_CORE_IMPL==3
584 : fd_sha256_hash_32_repeated_riscv( _data, _hash, cnt );
585 : return _hash;
586 : #elif FD_SHA256_CORE_IMPL==1
587 :
588 0 : uchar const * data = (uchar const *)_data;
589 0 : uchar * hash = (uchar *)_hash;
590 :
591 0 : vu_t w0003 = vu_bswap( vu_ldu( data ) );
592 0 : vu_t w0407 = vu_bswap( vu_ldu( data+16UL ) );
593 0 : vb_t const w080b = vb( 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00,
594 0 : 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 );
595 0 : vb_t const w0c0f = vb( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
596 0 : 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00 ); /* 32 bytes */
597 :
598 0 : vu_t const initialFEBA = vu( FD_SHA256_INITIAL_F, FD_SHA256_INITIAL_E, FD_SHA256_INITIAL_B, FD_SHA256_INITIAL_A );
599 0 : vu_t const initialHGDC = vu( FD_SHA256_INITIAL_H, FD_SHA256_INITIAL_G, FD_SHA256_INITIAL_D, FD_SHA256_INITIAL_C );
600 :
601 0 : for( ulong iter=0UL; iter<cnt; iter++ ) {
602 0 : vu_t stateFEBA = initialFEBA;
603 0 : vu_t stateHGDC = initialHGDC;
604 :
605 0 : FULL_ROUNDS();
606 :
607 0 : stateFEBA = vu_add( stateFEBA, initialFEBA );
608 0 : stateHGDC = vu_add( stateHGDC, initialHGDC );
609 :
610 0 : vu_t stateABCD = vu_permute2( stateFEBA, stateHGDC, 3, 2, 3, 2 );
611 0 : vu_t stateEFGH = vu_permute2( stateFEBA, stateHGDC, 1, 0, 1, 0 );
612 :
613 0 : w0003 = stateABCD;
614 0 : w0407 = stateEFGH;
615 0 : }
616 0 : vu_stu( hash, vu_bswap( w0003 ) );
617 0 : vu_stu( hash+16UL, vu_bswap( w0407 ) );
618 :
619 : #else
620 :
621 : uchar const * data = (uchar const *)_data;
622 : uchar * hash = (uchar *)_hash;
623 :
624 : uchar buf[ FD_SHA256_PRIVATE_BUF_MAX ] __attribute__((aligned(128)));
625 :
626 : /* Prepare padding once */
627 : ulong buf_used = 32UL;
628 : memcpy( buf, data, 32UL );
629 : buf[ buf_used ] = (uchar)0x80;
630 : buf_used++;
631 :
632 : ulong bit_cnt = 32UL << 3;
633 : memset( buf + buf_used, 0, FD_SHA256_PRIVATE_BUF_MAX-8UL-buf_used );
634 : FD_STORE( ulong, buf+FD_SHA256_PRIVATE_BUF_MAX-8UL, fd_ulong_bswap( bit_cnt ) );
635 :
636 : /* This is just the above streamlined to eliminate all the overheads
637 : to support incremental hashing. */
638 : for( ulong iter=0UL; iter<cnt; iter++ ) {
639 :
640 : uint state[8] __attribute__((aligned(32)));
641 :
642 : state[0] = FD_SHA256_INITIAL_A;
643 : state[1] = FD_SHA256_INITIAL_B;
644 : state[2] = FD_SHA256_INITIAL_C;
645 : state[3] = FD_SHA256_INITIAL_D;
646 : state[4] = FD_SHA256_INITIAL_E;
647 : state[5] = FD_SHA256_INITIAL_F;
648 : state[6] = FD_SHA256_INITIAL_G;
649 : state[7] = FD_SHA256_INITIAL_H;
650 :
651 : fd_sha256_core( state, buf, 1UL );
652 :
653 : state[0] = fd_uint_bswap( state[0] );
654 : state[1] = fd_uint_bswap( state[1] );
655 : state[2] = fd_uint_bswap( state[2] );
656 : state[3] = fd_uint_bswap( state[3] );
657 : state[4] = fd_uint_bswap( state[4] );
658 : state[5] = fd_uint_bswap( state[5] );
659 : state[6] = fd_uint_bswap( state[6] );
660 : state[7] = fd_uint_bswap( state[7] );
661 : memcpy( buf, state, 32UL );
662 : }
663 : memcpy( hash, buf, 32UL );
664 : #endif
665 0 : return _hash;
666 0 : }
667 :
668 : #undef fd_sha256_core
669 :
670 : #if FD_SHA256_CORE_IMPL==1
671 :
672 : /* A single repeated hash chain is bound by the sha256rnds2 dependency
673 : chain. The kernels below run 2-5 independent chains, issuing each
674 : step for every chain in turn so their rounds overlap. Every chain
675 : has its own named registers (suffix _<lane>). Zen 4 M hashes/s by
676 : chain count: 1: 31.1, 2: 58.9, 3: 74.8, 4: 80.1, 5: 79.6 (6+ spill
677 : heavily and get slower). */
678 :
679 0 : #define EACH2( M, ... ) M( 0, __VA_ARGS__ ) M( 1, __VA_ARGS__ )
680 0 : #define EACH3( M, ... ) EACH2( M, __VA_ARGS__ ) M( 2, __VA_ARGS__ )
681 0 : #define EACH4( M, ... ) EACH3( M, __VA_ARGS__ ) M( 3, __VA_ARGS__ )
682 0 : #define EACH5( M, ... ) EACH4( M, __VA_ARGS__ ) M( 4, __VA_ARGS__ )
683 :
684 : #define X_LOAD( l, in ) \
685 0 : vu_t w0003_##l = vu_bswap( vu_ldu( (in)+32UL*(l) ) ); \
686 0 : vu_t w0407_##l = vu_bswap( vu_ldu( (in)+32UL*(l)+16UL ) );
687 : #define X_INIT( l, _ ) \
688 0 : vu_t feba_##l = initialFEBA; vu_t hgdc_##l = initialHGDC; \
689 0 : vu_t w080b_##l = w080b; vu_t w0c0f_##l = w0c0f;
690 0 : #define X_R( l, w, k ) FOUR_ROUNDS_STATE( feba_##l, hgdc_##l, vu_add( w##_##l, vu_ld( fd_sha256_K+(k) ) ) );
691 0 : #define X_W( l, d, a, b, c, e ) vu_t d##_##l = NEXT_W( a##_##l, b##_##l, c##_##l, e##_##l );
692 : #define X_FINI( l, _ ) \
693 0 : feba_##l = vu_add( feba_##l, initialFEBA ); hgdc_##l = vu_add( hgdc_##l, initialHGDC ); \
694 0 : w0003_##l = vu_permute2( feba_##l, hgdc_##l, 3, 2, 3, 2 ); \
695 0 : w0407_##l = vu_permute2( feba_##l, hgdc_##l, 1, 0, 1, 0 );
696 : #define X_STORE( l, out ) \
697 0 : vu_stu( (out)+32UL*(l), vu_bswap( w0003_##l ) ); \
698 0 : vu_stu( (out)+32UL*(l)+16UL, vu_bswap( w0407_##l ) );
699 :
700 0 : #define REPEATED_MULTI( EACH ) do { \
701 0 : vu_t const w080b = vu( 0x80000000U, 0U, 0U, 0U ); \
702 0 : vu_t const w0c0f = vu( 0U, 0U, 0U, 256U ); /* 32 bytes */ \
703 0 : vu_t const initialFEBA = vu( FD_SHA256_INITIAL_F, FD_SHA256_INITIAL_E, FD_SHA256_INITIAL_B, FD_SHA256_INITIAL_A ); \
704 0 : vu_t const initialHGDC = vu( FD_SHA256_INITIAL_H, FD_SHA256_INITIAL_G, FD_SHA256_INITIAL_D, FD_SHA256_INITIAL_C ); \
705 0 : EACH( X_LOAD, in ) \
706 0 : for( ulong iter=0UL; iter<cnt; iter++ ) { \
707 0 : EACH( X_INIT, _ ) \
708 0 : /* */ EACH( X_R, w0003, 0 ); \
709 0 : /* */ EACH( X_R, w0407, 4 ); \
710 0 : /* */ EACH( X_R, w080b, 8 ); \
711 0 : /* */ EACH( X_R, w0c0f, 12 ); \
712 0 : EACH( X_W, w1013, w0003, w0407, w080b, w0c0f ); EACH( X_R, w1013, 16 ); \
713 0 : EACH( X_W, w1417, w0407, w080b, w0c0f, w1013 ); EACH( X_R, w1417, 20 ); \
714 0 : EACH( X_W, w181b, w080b, w0c0f, w1013, w1417 ); EACH( X_R, w181b, 24 ); \
715 0 : EACH( X_W, w1c1f, w0c0f, w1013, w1417, w181b ); EACH( X_R, w1c1f, 28 ); \
716 0 : EACH( X_W, w2023, w1013, w1417, w181b, w1c1f ); EACH( X_R, w2023, 32 ); \
717 0 : EACH( X_W, w2427, w1417, w181b, w1c1f, w2023 ); EACH( X_R, w2427, 36 ); \
718 0 : EACH( X_W, w282b, w181b, w1c1f, w2023, w2427 ); EACH( X_R, w282b, 40 ); \
719 0 : EACH( X_W, w2c2f, w1c1f, w2023, w2427, w282b ); EACH( X_R, w2c2f, 44 ); \
720 0 : EACH( X_W, w3033, w2023, w2427, w282b, w2c2f ); EACH( X_R, w3033, 48 ); \
721 0 : EACH( X_W, w3437, w2427, w282b, w2c2f, w3033 ); EACH( X_R, w3437, 52 ); \
722 0 : EACH( X_W, w383b, w282b, w2c2f, w3033, w3437 ); EACH( X_R, w383b, 56 ); \
723 0 : EACH( X_W, w3c3f, w2c2f, w3033, w3437, w383b ); EACH( X_R, w3c3f, 60 ); \
724 0 : EACH( X_FINI, _ ) \
725 0 : } \
726 0 : EACH( X_STORE, out ) \
727 0 : } while( 0 )
728 :
729 0 : static void fd_sha256_hash_32_repeated_x2( uchar const * in, uchar * out, ulong cnt ) { REPEATED_MULTI( EACH2 ); }
730 0 : static void fd_sha256_hash_32_repeated_x3( uchar const * in, uchar * out, ulong cnt ) { REPEATED_MULTI( EACH3 ); }
731 0 : static void fd_sha256_hash_32_repeated_x4( uchar const * in, uchar * out, ulong cnt ) { REPEATED_MULTI( EACH4 ); }
732 0 : static void fd_sha256_hash_32_repeated_x5( uchar const * in, uchar * out, ulong cnt ) { REPEATED_MULTI( EACH5 ); }
733 :
734 : #undef REPEATED_MULTI
735 : #undef X_STORE
736 : #undef X_FINI
737 : #undef X_W
738 : #undef X_R
739 : #undef X_INIT
740 : #undef X_LOAD
741 : #undef EACH5
742 : #undef EACH4
743 : #undef EACH3
744 : #undef EACH2
745 :
746 : static void
747 : fd_sha256_hash_32_repeated_shani( uchar const * in,
748 : uchar * out,
749 : ulong cnt,
750 0 : ulong lane_cnt ) {
751 : /* Split into the fewest groups of at most 5 chains, of near equal
752 : size (e.g. 4+3 beats 5+2). */
753 0 : ulong grp_cnt = (lane_cnt+4UL)/5UL;
754 0 : while( lane_cnt ) {
755 0 : ulong k = (lane_cnt+grp_cnt-1UL)/grp_cnt;
756 0 : switch( k ) {
757 0 : case 5UL: fd_sha256_hash_32_repeated_x5( in, out, cnt ); break;
758 0 : case 4UL: fd_sha256_hash_32_repeated_x4( in, out, cnt ); break;
759 0 : case 3UL: fd_sha256_hash_32_repeated_x3( in, out, cnt ); break;
760 0 : case 2UL: fd_sha256_hash_32_repeated_x2( in, out, cnt ); break;
761 0 : default: fd_sha256_hash_32_repeated ( in, out, cnt ); break;
762 0 : }
763 0 : in += 32UL*k; out += 32UL*k; lane_cnt -= k; grp_cnt--;
764 0 : }
765 0 : }
766 :
767 : #undef NEXT_W
768 : #undef FOUR_ROUNDS
769 : #undef FOUR_ROUNDS_STATE
770 : #undef FULL_ROUNDS
771 :
772 : #endif
773 :
774 : #if FD_SHA256_CORE_IMPL==2
775 : void
776 : fd_sha256_hash_32_repeated_batch_arm( uchar const * hash_in,
777 : uchar * hash_out,
778 : ulong cnt,
779 : ulong batch_cnt );
780 : #endif
781 :
782 : #if FD_HAS_AVX512
783 : void
784 : fd_sha256_hash_32_repeated_batch_avx512( uchar const * hash_in,
785 : uchar * hash_out,
786 : ulong cnt,
787 : ulong batch_cnt );
788 : #endif
789 :
790 : /* x86 lane policy. AVX512_LANE_MIN is the batch width at which the
791 : 16 lane AVX-512 kernel beats running the lanes as interleaved SHA-NI
792 : chains (or serially without SHA-NI). On Zen 4 SHA-NI wins at every
793 : width (16 lanes: 80 vs 66 M hashes/s). */
794 :
795 : #if FD_SHA256_CORE_IMPL==1 || (FD_SHA256_CORE_IMPL==0 && FD_HAS_AVX512)
796 : #define X86_BATCH 1
797 : #if !FD_HAS_AVX512
798 : #define AVX512_LANE_MIN (ULONG_MAX)
799 : #elif FD_SHA256_CORE_IMPL==1 && defined(__znver4__)
800 : #define AVX512_LANE_MIN (ULONG_MAX)
801 : #elif defined(__znver5__)
802 : #define AVX512_LANE_MIN (6UL) /* Zen 5 has high AVX-512 throughput */
803 : #else
804 0 : #define AVX512_LANE_MIN (8UL) /* Baseline 1 IPC AVX-512 needs more batching to win against SHA-NI */
805 : #endif
806 : #endif
807 :
808 : void
809 : fd_sha256_hash_32_repeated_batch( void const * _hash_in,
810 : void * _hash_out,
811 : ulong cnt,
812 0 : ulong batch_cnt ) {
813 0 : uchar const * hash_in = (uchar const *)_hash_in;
814 0 : uchar * hash_out = (uchar *)_hash_out;
815 0 : if( FD_UNLIKELY( batch_cnt>fd_sha256_simd_lane_max() ) ) FD_LOG_CRIT(( "batch_cnt %lu exceeds fd_sha256_simd_lane_max %lu", batch_cnt, fd_sha256_simd_lane_max() ));
816 : #if FD_SHA256_CORE_IMPL==2
817 : fd_sha256_hash_32_repeated_batch_arm( hash_in, hash_out, cnt, batch_cnt );
818 : return;
819 : #elif defined(X86_BATCH)
820 : # if FD_HAS_AVX512
821 0 : if( batch_cnt>=AVX512_LANE_MIN ) {
822 0 : fd_sha256_hash_32_repeated_batch_avx512( hash_in, hash_out, cnt, batch_cnt );
823 0 : return;
824 0 : }
825 0 : # endif
826 0 : # if FD_SHA256_CORE_IMPL==1
827 0 : fd_sha256_hash_32_repeated_shani( hash_in, hash_out, cnt, batch_cnt );
828 0 : return;
829 0 : # endif
830 0 : #endif
831 0 : for( ulong i=0UL; i<batch_cnt; i++ ) {
832 0 : fd_sha256_hash_32_repeated( hash_in+32*i, hash_out+32*i, cnt );
833 0 : }
834 0 : }
835 :
836 : #ifdef X86_BATCH
837 :
838 : ulong
839 0 : fd_sha256_simd_lane_min( void ) {
840 0 : # if FD_SHA256_CORE_IMPL==1
841 0 : return fd_ulong_min( 2UL, AVX512_LANE_MIN );
842 : # else
843 : return AVX512_LANE_MIN;
844 : # endif
845 0 : }
846 :
847 0 : ulong fd_sha256_simd_lane_max( void ) { return 16UL; }
848 :
849 : ulong
850 0 : fd_sha256_simd_iter_cost_q8( ulong lane_cnt ) {
851 0 : if( lane_cnt>=AVX512_LANE_MIN ) return 1357UL; /* 5.3x on Zen 5: 16 lanes at 96.5 M hashes/s vs 31.7 M hashes/s single lane with SHA-NI */
852 0 : # if FD_SHA256_CORE_IMPL==1
853 : /* Zen 4, per group of k interleaved chains: 256*k*rate(1)/rate(k) */
854 0 : static ushort const group_cost_q8[6] = { 0, 256, 273, 322, 398, 495 };
855 0 : ulong cost = 0UL;
856 0 : ulong grp_cnt = (lane_cnt+4UL)/5UL;
857 0 : while( lane_cnt ) {
858 0 : ulong k = (lane_cnt+grp_cnt-1UL)/grp_cnt;
859 0 : cost += group_cost_q8[ k ]; lane_cnt -= k; grp_cnt--;
860 0 : }
861 0 : return fd_ulong_max( cost, 256UL );
862 : # else
863 : return fd_ulong_max( lane_cnt, 1UL )*256UL;
864 : # endif
865 0 : }
866 :
867 : #undef AVX512_LANE_MIN
868 : #undef X86_BATCH
869 :
870 : #else
871 :
872 : __attribute__((weak)) ulong fd_sha256_simd_lane_min( void ) { return ULONG_MAX; }
873 : __attribute__((weak)) ulong fd_sha256_simd_lane_max( void ) { return 1UL; }
874 : __attribute__((weak)) ulong fd_sha256_simd_iter_cost_q8( ulong lane_cnt ) { return fd_ulong_max( lane_cnt, 1UL )*256UL; }
875 :
876 : #endif
|
