Line data Source code
1 : #ifndef HEADER_fd_src_ballet_blake3_fd_blake3_private_h
2 : #define HEADER_fd_src_ballet_blake3_fd_blake3_private_h
3 :
4 : #include "fd_blake3.h"
5 : #if FD_HAS_AVX
6 : #include "../../util/simd/fd_avx.h"
7 : #endif
8 :
9 : /* Set FD_BLAKE3_TRACING to 1 to dump out a high-level trace of BLAKE3
10 : operations to the debug log. This is useful during debugging or
11 : development. */
12 : #define FD_BLAKE3_TRACING 0
13 :
14 : #if FD_BLAKE3_TRACING
15 : #define FD_BLAKE3_TRACE( ... ) FD_LOG_DEBUG( __VA_ARGS__ )
16 : #else
17 144480 : #define FD_BLAKE3_TRACE( ... ) (void)0
18 : #endif
19 :
20 : /* Protocol constants *************************************************/
21 :
22 : static const uchar FD_BLAKE3_MSG_SCHEDULE[7][16] = {
23 : { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
24 : { 2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8 },
25 : { 3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1 },
26 : { 10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6 },
27 : { 12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4 },
28 : { 9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7 },
29 : { 11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13 },
30 : };
31 :
32 : static const uint FD_BLAKE3_IV[8] = {
33 : 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL,
34 : 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL
35 : };
36 :
37 29224 : #define FD_BLAKE3_FLAG_CHUNK_START (1u<<0) /* 1 */
38 69945 : #define FD_BLAKE3_FLAG_CHUNK_END (1u<<1) /* 2 */
39 178232 : #define FD_BLAKE3_FLAG_PARENT (1u<<2) /* 4 */
40 155214 : #define FD_BLAKE3_FLAG_ROOT (1u<<3) /* 8 */
41 :
42 : /* Possible flag combinations:
43 : 0x1: first block of a chunk with at least 2 blocks
44 : 0x2: last block of a chunk, tree that has at least 1 parent
45 : 0x3: last chunk (<=64 bytes), input >1024 bytes
46 : 0x4: non-root parent node
47 : 0xa: last block of the only chunk, input_sz>64 input_sz<=1024
48 : 0xb: only block, input_sz<=64
49 : 0xc: root parent node */
50 :
51 : /* Scheduler **********************************************************/
52 :
53 : union __attribute__((aligned(32))) fd_blake3_op {
54 :
55 : struct {
56 : uchar const * msg;
57 : uchar * out;
58 :
59 : ulong counter;
60 : union {
61 : struct {
62 : ushort off;
63 : ushort sz;
64 : };
65 : uint off_sz;
66 : };
67 : uchar flags;
68 : };
69 :
70 : };
71 :
72 : typedef union fd_blake3_op fd_blake3_op_t;
73 :
74 : /* Compression function ***********************************************/
75 :
76 : FD_PROTOTYPES_BEGIN
77 :
78 : void
79 : fd_blake3_fini_xof_compress( fd_blake3_t * sha,
80 : uchar * root_msg,
81 : uchar * root_cv_pre );
82 :
83 : void
84 : fd_blake3_ref_compress1( uchar * restrict out, /* align==1 len==32 */
85 : uchar const * restrict msg, /* align==1 len==64 */
86 : uint msg_sz,
87 : ulong counter,
88 : uint flags,
89 : uchar * restrict out_chain, /* optional, 32 byte output chaining value of last block */
90 : uchar const * restrict in_chain ); /* optional, 32 byte input chaining value of first block (default IV) */
91 :
92 : #if FD_HAS_SSE
93 :
94 : void
95 : fd_blake3_sse_compress1( uchar * restrict out, /* align==1 len==32 */
96 : uchar const * restrict msg, /* align==1 len==64 */
97 : uint msg_sz,
98 : ulong counter,
99 : uint flags,
100 : uchar * restrict out_chain,
101 : uchar const * restrict in_chain );
102 :
103 : #endif /* FD_HAS_SSE */
104 :
105 : #if FD_HAS_SVE2
106 :
107 : void
108 : fd_blake3_sve2_compress4( ulong batch_cnt,
109 : void const * restrict batch_data,
110 : uint const * restrict batch_sz,
111 : ulong const * restrict ctr_vec,
112 : uint const * restrict batch_flags,
113 : void * const * restrict batch_hash,
114 : uint out_sz,
115 : void const * restrict batch_cv );
116 :
117 : void
118 : fd_blake3_sve2_compress4_fast( uchar const * restrict batch_data,
119 : uchar * restrict batch_hash,
120 : ulong counter,
121 : uchar flags );
122 :
123 : /* fd_blake3_sve2_xof4 is the 4 lane analog of fd_blake3_avx512_xof16:
124 : expands 4 consecutive 64 byte XOF output blocks (counters
125 : ctr0..ctr0+3) of root block root_msg (block_sz bytes, zero padded)
126 : with input chaining value root_cv. */
127 :
128 : void
129 : fd_blake3_sve2_xof4( uchar const * restrict root_msg, /* len==block_sz */
130 : uchar const * restrict root_cv, /* len==32 */
131 : ulong ctr0,
132 : uint block_sz,
133 : uint flags,
134 : uchar * restrict out ); /* len==4*64 */
135 :
136 : #endif /* FD_HAS_SVE2 */
137 :
138 : #if FD_HAS_AVX
139 :
140 : /* BLAKE3 AVX cores
141 :
142 : compress8 compresses one to eight tree nodes. batch_cnt is the
143 : number of nodes to process. For each node in the batch with index i,
144 : - _batch_data[i] points to the input data of the node (message bytes
145 : for leaf nodes, a pair of output chaining values for branch nodes)
146 : - batch_sz[i] is the input byte count of the node, from which the
147 : 'len' value of each of the node's blocks is derived
148 : - ctr_vec[i] is the 'counter' value of the node
149 : - batch_flags[i] is the 'flag' value of the node
150 : - cv is optional. If set, cv[i] is the 'chaining value' of the first
151 : block of the node. This is useful for XOF.
152 :
153 : compress8 has three different output modes:
154 : - "LtHash in-place": If lthash is set, each node's output is expanded
155 : (XOF) to 2048 bytes and interpreted as an 'LtHash' value (i.e.
156 : a vector of 1024 uint16). These vectors are then added together
157 : and the result is written to lthash. The root flag MUST be set for
158 : all batch_flags inputs, otherwise this function will read OOB.
159 : - "Simple": Otherwise, _batch_hash[i] must be 32-byte aligned and is
160 : populated with the 32-byte output chaining value. (If node i is a
161 : root node, this is 'the BLAKE3 hash', i.e. the first 32 bytes of the
162 : XOF stream).
163 :
164 : These modes are all packed into the same function because the
165 : alternatives are worse (either worse code footprint due to duplicated
166 : core, or worse throughput due to high penalty passing vector regs
167 : between functions in SysV ABI).
168 :
169 : compress8_fast does a subset of what compress8 can, but is ~10-20%
170 : faster. */
171 :
172 : void
173 : fd_blake3_avx_compress8( ulong batch_cnt,
174 : void const * restrict _batch_data, /* align==32 len==8 */
175 : uint const * restrict batch_sz, /* align==4 len==8 */
176 : ulong const * restrict ctr_vec, /* len==8 */
177 : uint const * restrict batch_flags, /* align==4 len==8 */
178 : void * const * restrict _batch_hash, /* align==32 len==batch_cnt */
179 : ushort * restrict lthash, /* align==32 byte_sz=2048 */
180 : uint out_sz, /* 32 or 64 */
181 : void const * restrict batch_cv ); /* align==8 len==8 ele_align==32 optional */
182 :
183 : void
184 : fd_blake3_avx_compress8_fast( uchar const * restrict batch_data, /* align==32 len==8*64 */
185 : uchar * restrict batch_hash, /* align==32 len==8*32 */
186 : ulong counter,
187 : uchar flags );
188 :
189 : #endif /* FD_HAS_AVX */
190 :
191 : #if FD_HAS_AVX512
192 :
193 : /* fd_blake3_avx512_compress16{,fast} are analogous to the avx APIs
194 : above. The only difference is larger alignment assumptions and that
195 : these process up to sixteen elements. */
196 :
197 : void
198 : fd_blake3_avx512_compress16( ulong batch_cnt,
199 : void const * restrict _batch_data, /* align=64 len=16 ele_align=1 */
200 : uint const * restrict batch_sz, /* align= 4 len=16 */
201 : ulong const * restrict ctr_vec, /* align= 8 len=16 */
202 : uint const * restrict batch_flags, /* align= 4 len=16 */
203 : void * const * restrict _batch_hash, /* align=64 len=16 */
204 : ushort * restrict lthash, /* align=64 byte_sz=2048 */
205 : uint out_sz, /* 32 or 64 */
206 : void const * restrict batch_cv ); /* align= 8 len=16 ele_align=16 optional */
207 :
208 : void
209 : fd_blake3_avx512_compress16_fast( uchar const * restrict batch_data, /* align==32 len==16*64 */
210 : uchar * restrict batch_hash, /* align==32 len==16*32 */
211 : ulong counter,
212 : uchar flags );
213 :
214 : /* fd_blake3_avx512_xof16 expands 16 consecutive 64 byte XOF output
215 : blocks (counters ctr0..ctr0+15) of the root block root_msg (64 bytes,
216 : zero padded past block_sz) with input chaining value root_cv.
217 : Equivalent to fd_blake3_avx512_compress16 with out_sz=64, all lanes
218 : sharing root_msg/root_cv, minus the per lane message transposes. */
219 :
220 : void
221 : fd_blake3_avx512_xof16( uchar const * restrict root_msg, /* len==64 */
222 : uchar const * restrict root_cv, /* len==32 */
223 : ulong ctr0,
224 : uint block_sz,
225 : uint flags,
226 : uchar * restrict out ); /* len==16*64 */
227 :
228 : #endif /* FD_HAS_AVX512 */
229 :
230 : FD_PROTOTYPES_END
231 :
232 : #endif /* HEADER_fd_src_ballet_blake3_fd_blake3_private_h */
|
