Coverage Report

Created: 2026-07-25 06:58

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/zlib-ng/arch/generic/crc32_chorba_c.c
Line
Count
Source
1
#include "zbuild.h"
2
#include "arch_functions.h"
3
4
#ifdef CRC32_CHORBA_FALLBACK
5
6
#if defined(__EMSCRIPTEN__)
7
#  include "zutil_p.h"
8
#endif
9
#include "zmemory.h"
10
#include "crc32_chorba_p.h"
11
#include "crc32_braid_tbl.h"
12
13
/* Implement Chorba algorithm from https://arxiv.org/abs/2412.16398 */
14
0
#define bitbuffer_size_bytes (16 * 1024 * sizeof(chorba_word_t))
15
0
#define bitbuffer_size_zwords (bitbuffer_size_bytes / sizeof(chorba_word_t))
16
0
#define bitbuffer_size_qwords (bitbuffer_size_bytes / sizeof(uint64_t))
17
18
#if defined(HAVE_MAY_ALIAS)
19
    typedef chorba_word_t __attribute__ ((__may_alias__)) chorba_worda_t;
20
    typedef uint64_t __attribute__ ((__may_alias__)) uint64a_t;
21
#else
22
    typedef chorba_word_t chorba_worda_t;
23
    typedef uint64_t uint64a_t;
24
#endif
25
26
#define NEXT_ROUND(invec, a, b, c, d) \
27
0
    do { \
28
0
        a = ((invec) << 17) ^ ((invec) << 55); \
29
0
        b = ((invec) >> 47) ^ ((invec) >> 9) ^ ((invec) << 19); \
30
0
        c = ((invec) >> 45) ^ ((invec) << 44); \
31
0
        d = (invec) >> 20; \
32
0
    } while (0)
33
34
#define ACCUM_ROUND(n1, n2, n3, n4, n5) \
35
0
    do { \
36
0
        n1 = n5 ^ (a3 ^ b2 ^ c1); \
37
0
        n2 = a4 ^ b3 ^ c2 ^ d1; \
38
0
        n3 = b4 ^ c3 ^ d2; \
39
0
        n4 = c4 ^ d3; \
40
0
        n5 = d4; \
41
0
    } while (0)
42
43
/**
44
 * Implements the Chorba algorithm for CRC32 computation (https://arxiv.org/abs/2412.16398).
45
 *
46
 * This implementation processes data in three phases:
47
 * 1. Initial pass: Zeros out bitbuffer
48
 * 2. Intermediate pass: Processes half the values
49
 * 3. Main pass: Processes remaining data
50
 *
51
 * @param crc Initial CRC value
52
 * @param input Input data buffer
53
 * @param len Length of input data
54
 * @return Computed CRC32 value
55
 *
56
 * @note Requires minimum input size of 118960 + 512 bytes
57
 * @note Uses 128KB temporary buffer
58
 */
59
0
Z_INTERNAL uint32_t crc32_chorba_118960_nondestructive(uint32_t crc, const uint8_t *buf, size_t len) {
60
#if defined(__EMSCRIPTEN__)
61
    chorba_worda_t *bitbuffer = (chorba_worda_t*)zng_alloc(bitbuffer_size_bytes);
62
#else
63
0
    ALIGNED_(16) chorba_worda_t bitbuffer[bitbuffer_size_zwords];
64
0
#endif
65
0
    const uint8_t *bitbuffer_bytes = (const uint8_t*)bitbuffer;
66
0
    uint64a_t *bitbuffer_qwords = (uint64a_t*)bitbuffer;
67
    /* The calling function ensured that this is aligned correctly */
68
0
    const chorba_worda_t* input = (const chorba_worda_t*)buf;
69
0
    const uint64a_t* input_qwords = (const uint64a_t*)buf;
70
71
0
    size_t i = 0;
72
73
0
    chorba_word_t next1 = CHORBA_WORD_FROM_LE(~crc);
74
75
0
    chorba_word_t next2 = 0;
76
0
    chorba_word_t next3 = 0;
77
0
    chorba_word_t next4 = 0;
78
0
    chorba_word_t next5 = 0;
79
0
    chorba_word_t next6 = 0;
80
0
    chorba_word_t next7 = 0;
81
0
    chorba_word_t next8 = 0;
82
0
    chorba_word_t next9 = 0;
83
0
    chorba_word_t next10 = 0;
84
0
    chorba_word_t next11 = 0;
85
0
    chorba_word_t next12 = 0;
86
0
    chorba_word_t next13 = 0;
87
0
    chorba_word_t next14 = 0;
88
0
    chorba_word_t next15 = 0;
89
0
    chorba_word_t next16 = 0;
90
0
    chorba_word_t next17 = 0;
91
0
    chorba_word_t next18 = 0;
92
0
    chorba_word_t next19 = 0;
93
0
    chorba_word_t next20 = 0;
94
0
    chorba_word_t next21 = 0;
95
0
    chorba_word_t next22 = 0;
96
0
    crc = 0;
97
98
    // do a first pass to zero out bitbuffer
99
0
    for (; i < (14848 * sizeof(chorba_word_t)); i += (32 * sizeof(chorba_word_t))) {
100
0
        chorba_word_t in1, in2, in3, in4, in5, in6, in7, in8;
101
0
        chorba_word_t in9, in10, in11, in12, in13, in14, in15, in16;
102
0
        chorba_word_t in17, in18, in19, in20, in21, in22, in23, in24;
103
0
        chorba_word_t in25, in26, in27, in28, in29, in30, in31, in32;
104
0
        int out_offset1 = ((i / sizeof(chorba_word_t)) + 14848) % bitbuffer_size_zwords;
105
0
        int out_offset2 = ((i / sizeof(chorba_word_t)) + 14880) % bitbuffer_size_zwords;
106
107
0
        in1 = input[i / sizeof(chorba_word_t) + 0] ^ next1;
108
0
        in2 = input[i / sizeof(chorba_word_t) + 1] ^ next2;
109
0
        in3 = input[i / sizeof(chorba_word_t) + 2] ^ next3;
110
0
        in4 = input[i / sizeof(chorba_word_t) + 3] ^ next4;
111
0
        in5 = input[i / sizeof(chorba_word_t) + 4] ^ next5;
112
0
        in6 = input[i / sizeof(chorba_word_t) + 5] ^ next6;
113
0
        in7 = input[i / sizeof(chorba_word_t) + 6] ^ next7;
114
0
        in8 = input[i / sizeof(chorba_word_t) + 7] ^ next8 ^ in1;
115
0
        in9 = input[i / sizeof(chorba_word_t) + 8] ^ next9 ^ in2;
116
0
        in10 = input[i / sizeof(chorba_word_t) + 9] ^ next10 ^ in3;
117
0
        in11 = input[i / sizeof(chorba_word_t) + 10] ^ next11 ^ in4;
118
0
        in12 = input[i / sizeof(chorba_word_t) + 11] ^ next12 ^ in1 ^ in5;
119
0
        in13 = input[i / sizeof(chorba_word_t) + 12] ^ next13 ^ in2 ^ in6;
120
0
        in14 = input[i / sizeof(chorba_word_t) + 13] ^ next14 ^ in3 ^ in7;
121
0
        in15 = input[i / sizeof(chorba_word_t) + 14] ^ next15 ^ in4 ^ in8;
122
0
        in16 = input[i / sizeof(chorba_word_t) + 15] ^ next16 ^ in5 ^ in9;
123
0
        in17 = input[i / sizeof(chorba_word_t) + 16] ^ next17 ^ in6 ^ in10;
124
0
        in18 = input[i / sizeof(chorba_word_t) + 17] ^ next18 ^ in7 ^ in11;
125
0
        in19 = input[i / sizeof(chorba_word_t) + 18] ^ next19 ^ in8 ^ in12;
126
0
        in20 = input[i / sizeof(chorba_word_t) + 19] ^ next20 ^ in9 ^ in13;
127
0
        in21 = input[i / sizeof(chorba_word_t) + 20] ^ next21 ^ in10 ^ in14;
128
0
        in22 = input[i / sizeof(chorba_word_t) + 21] ^ next22 ^ in11 ^ in15;
129
0
        in23 = input[i / sizeof(chorba_word_t) + 22] ^ in1 ^ in12 ^ in16;
130
0
        in24 = input[i / sizeof(chorba_word_t) + 23] ^ in2 ^ in13 ^ in17;
131
0
        in25 = input[i / sizeof(chorba_word_t) + 24] ^ in3 ^ in14 ^ in18;
132
0
        in26 = input[i / sizeof(chorba_word_t) + 25] ^ in4 ^ in15 ^ in19;
133
0
        in27 = input[i / sizeof(chorba_word_t) + 26] ^ in5 ^ in16 ^ in20;
134
0
        in28 = input[i / sizeof(chorba_word_t) + 27] ^ in6 ^ in17 ^ in21;
135
0
        in29 = input[i / sizeof(chorba_word_t) + 28] ^ in7 ^ in18 ^ in22;
136
0
        in30 = input[i / sizeof(chorba_word_t) + 29] ^ in8 ^ in19 ^ in23;
137
0
        in31 = input[i / sizeof(chorba_word_t) + 30] ^ in9 ^ in20 ^ in24;
138
0
        in32 = input[i / sizeof(chorba_word_t) + 31] ^ in10 ^ in21 ^ in25;
139
140
0
        next1 = in11 ^ in22 ^ in26;
141
0
        next2 = in12 ^ in23 ^ in27;
142
0
        next3 = in13 ^ in24 ^ in28;
143
0
        next4 = in14 ^ in25 ^ in29;
144
0
        next5 = in15 ^ in26 ^ in30;
145
0
        next6 = in16 ^ in27 ^ in31;
146
0
        next7 = in17 ^ in28 ^ in32;
147
0
        next8 = in18 ^ in29;
148
0
        next9 = in19 ^ in30;
149
0
        next10 = in20 ^ in31;
150
0
        next11 = in21 ^ in32;
151
0
        next12 = in22;
152
0
        next13 = in23;
153
0
        next14 = in24;
154
0
        next15 = in25;
155
0
        next16 = in26;
156
0
        next17 = in27;
157
0
        next18 = in28;
158
0
        next19 = in29;
159
0
        next20 = in30;
160
0
        next21 = in31;
161
0
        next22 = in32;
162
163
0
        bitbuffer[out_offset1 + 22] = in1;
164
0
        bitbuffer[out_offset1 + 23] = in2;
165
0
        bitbuffer[out_offset1 + 24] = in3;
166
0
        bitbuffer[out_offset1 + 25] = in4;
167
0
        bitbuffer[out_offset1 + 26] = in5;
168
0
        bitbuffer[out_offset1 + 27] = in6;
169
0
        bitbuffer[out_offset1 + 28] = in7;
170
0
        bitbuffer[out_offset1 + 29] = in8;
171
0
        bitbuffer[out_offset1 + 30] = in9;
172
0
        bitbuffer[out_offset1 + 31] = in10;
173
0
        bitbuffer[out_offset2 + 0] = in11;
174
0
        bitbuffer[out_offset2 + 1] = in12;
175
0
        bitbuffer[out_offset2 + 2] = in13;
176
0
        bitbuffer[out_offset2 + 3] = in14;
177
0
        bitbuffer[out_offset2 + 4] = in15;
178
0
        bitbuffer[out_offset2 + 5] = in16;
179
0
        bitbuffer[out_offset2 + 6] = in17;
180
0
        bitbuffer[out_offset2 + 7] = in18;
181
0
        bitbuffer[out_offset2 + 8] = in19;
182
0
        bitbuffer[out_offset2 + 9] = in20;
183
0
        bitbuffer[out_offset2 + 10] = in21;
184
0
        bitbuffer[out_offset2 + 11] = in22;
185
0
        bitbuffer[out_offset2 + 12] = in23;
186
0
        bitbuffer[out_offset2 + 13] = in24;
187
0
        bitbuffer[out_offset2 + 14] = in25;
188
0
        bitbuffer[out_offset2 + 15] = in26;
189
0
        bitbuffer[out_offset2 + 16] = in27;
190
0
        bitbuffer[out_offset2 + 17] = in28;
191
0
        bitbuffer[out_offset2 + 18] = in29;
192
0
        bitbuffer[out_offset2 + 19] = in30;
193
0
        bitbuffer[out_offset2 + 20] = in31;
194
0
        bitbuffer[out_offset2 + 21] = in32;
195
0
    }
196
197
    // one intermediate pass where we pull half the values
198
0
    for (; i < (14880 * sizeof(chorba_word_t)); i += (32 * sizeof(chorba_word_t))) {
199
0
        chorba_word_t in1, in2, in3, in4, in5, in6, in7, in8;
200
0
        chorba_word_t in9, in10, in11, in12, in13, in14, in15, in16;
201
0
        chorba_word_t in17, in18, in19, in20, in21, in22, in23, in24;
202
0
        chorba_word_t in25, in26, in27, in28, in29, in30, in31, in32;
203
0
        int in_offset = (i / sizeof(chorba_word_t)) % bitbuffer_size_zwords;
204
0
        int out_offset1 = ((i / sizeof(chorba_word_t)) + 14848) % bitbuffer_size_zwords;
205
0
        int out_offset2 = ((i / sizeof(chorba_word_t)) + 14880) % bitbuffer_size_zwords;
206
207
0
        in1 = input[i / sizeof(chorba_word_t) + 0] ^ next1;
208
0
        in2 = input[i / sizeof(chorba_word_t) + 1] ^ next2;
209
0
        in3 = input[i / sizeof(chorba_word_t) + 2] ^ next3;
210
0
        in4 = input[i / sizeof(chorba_word_t) + 3] ^ next4;
211
0
        in5 = input[i / sizeof(chorba_word_t) + 4] ^ next5;
212
0
        in6 = input[i / sizeof(chorba_word_t) + 5] ^ next6;
213
0
        in7 = input[i / sizeof(chorba_word_t) + 6] ^ next7;
214
0
        in8 = input[i / sizeof(chorba_word_t) + 7] ^ next8 ^ in1;
215
0
        in9 = input[i / sizeof(chorba_word_t) + 8] ^ next9 ^ in2;
216
0
        in10 = input[i / sizeof(chorba_word_t) + 9] ^ next10 ^ in3;
217
0
        in11 = input[i / sizeof(chorba_word_t) + 10] ^ next11 ^ in4;
218
0
        in12 = input[i / sizeof(chorba_word_t) + 11] ^ next12 ^ in1 ^ in5;
219
0
        in13 = input[i / sizeof(chorba_word_t) + 12] ^ next13 ^ in2 ^ in6;
220
0
        in14 = input[i / sizeof(chorba_word_t) + 13] ^ next14 ^ in3 ^ in7;
221
0
        in15 = input[i / sizeof(chorba_word_t) + 14] ^ next15 ^ in4 ^ in8;
222
0
        in16 = input[i / sizeof(chorba_word_t) + 15] ^ next16 ^ in5 ^ in9;
223
0
        in17 = input[i / sizeof(chorba_word_t) + 16] ^ next17 ^ in6 ^ in10;
224
0
        in18 = input[i / sizeof(chorba_word_t) + 17] ^ next18 ^ in7 ^ in11;
225
0
        in19 = input[i / sizeof(chorba_word_t) + 18] ^ next19 ^ in8 ^ in12;
226
0
        in20 = input[i / sizeof(chorba_word_t) + 19] ^ next20 ^ in9 ^ in13;
227
0
        in21 = input[i / sizeof(chorba_word_t) + 20] ^ next21 ^ in10 ^ in14;
228
0
        in22 = input[i / sizeof(chorba_word_t) + 21] ^ next22 ^ in11 ^ in15;
229
0
        in23 = input[i / sizeof(chorba_word_t) + 22] ^ in1 ^ in12 ^ in16 ^ bitbuffer[in_offset + 22];
230
0
        in24 = input[i / sizeof(chorba_word_t) + 23] ^ in2 ^ in13 ^ in17 ^ bitbuffer[in_offset + 23];
231
0
        in25 = input[i / sizeof(chorba_word_t) + 24] ^ in3 ^ in14 ^ in18 ^ bitbuffer[in_offset + 24];
232
0
        in26 = input[i / sizeof(chorba_word_t) + 25] ^ in4 ^ in15 ^ in19 ^ bitbuffer[in_offset + 25];
233
0
        in27 = input[i / sizeof(chorba_word_t) + 26] ^ in5 ^ in16 ^ in20 ^ bitbuffer[in_offset + 26];
234
0
        in28 = input[i / sizeof(chorba_word_t) + 27] ^ in6 ^ in17 ^ in21 ^ bitbuffer[in_offset + 27];
235
0
        in29 = input[i / sizeof(chorba_word_t) + 28] ^ in7 ^ in18 ^ in22 ^ bitbuffer[in_offset + 28];
236
0
        in30 = input[i / sizeof(chorba_word_t) + 29] ^ in8 ^ in19 ^ in23 ^ bitbuffer[in_offset + 29];
237
0
        in31 = input[i / sizeof(chorba_word_t) + 30] ^ in9 ^ in20 ^ in24 ^ bitbuffer[in_offset + 30];
238
0
        in32 = input[i / sizeof(chorba_word_t) + 31] ^ in10 ^ in21 ^ in25 ^ bitbuffer[in_offset + 31];
239
240
0
        next1 = in11 ^ in22 ^ in26;
241
0
        next2 = in12 ^ in23 ^ in27;
242
0
        next3 = in13 ^ in24 ^ in28;
243
0
        next4 = in14 ^ in25 ^ in29;
244
0
        next5 = in15 ^ in26 ^ in30;
245
0
        next6 = in16 ^ in27 ^ in31;
246
0
        next7 = in17 ^ in28 ^ in32;
247
0
        next8 = in18 ^ in29;
248
0
        next9 = in19 ^ in30;
249
0
        next10 = in20 ^ in31;
250
0
        next11 = in21 ^ in32;
251
0
        next12 = in22;
252
0
        next13 = in23;
253
0
        next14 = in24;
254
0
        next15 = in25;
255
0
        next16 = in26;
256
0
        next17 = in27;
257
0
        next18 = in28;
258
0
        next19 = in29;
259
0
        next20 = in30;
260
0
        next21 = in31;
261
0
        next22 = in32;
262
263
0
        bitbuffer[out_offset1 + 22] = in1;
264
0
        bitbuffer[out_offset1 + 23] = in2;
265
0
        bitbuffer[out_offset1 + 24] = in3;
266
0
        bitbuffer[out_offset1 + 25] = in4;
267
0
        bitbuffer[out_offset1 + 26] = in5;
268
0
        bitbuffer[out_offset1 + 27] = in6;
269
0
        bitbuffer[out_offset1 + 28] = in7;
270
0
        bitbuffer[out_offset1 + 29] = in8;
271
0
        bitbuffer[out_offset1 + 30] = in9;
272
0
        bitbuffer[out_offset1 + 31] = in10;
273
0
        bitbuffer[out_offset2 + 0] = in11;
274
0
        bitbuffer[out_offset2 + 1] = in12;
275
0
        bitbuffer[out_offset2 + 2] = in13;
276
0
        bitbuffer[out_offset2 + 3] = in14;
277
0
        bitbuffer[out_offset2 + 4] = in15;
278
0
        bitbuffer[out_offset2 + 5] = in16;
279
0
        bitbuffer[out_offset2 + 6] = in17;
280
0
        bitbuffer[out_offset2 + 7] = in18;
281
0
        bitbuffer[out_offset2 + 8] = in19;
282
0
        bitbuffer[out_offset2 + 9] = in20;
283
0
        bitbuffer[out_offset2 + 10] = in21;
284
0
        bitbuffer[out_offset2 + 11] = in22;
285
0
        bitbuffer[out_offset2 + 12] = in23;
286
0
        bitbuffer[out_offset2 + 13] = in24;
287
0
        bitbuffer[out_offset2 + 14] = in25;
288
0
        bitbuffer[out_offset2 + 15] = in26;
289
0
        bitbuffer[out_offset2 + 16] = in27;
290
0
        bitbuffer[out_offset2 + 17] = in28;
291
0
        bitbuffer[out_offset2 + 18] = in29;
292
0
        bitbuffer[out_offset2 + 19] = in30;
293
0
        bitbuffer[out_offset2 + 20] = in31;
294
0
        bitbuffer[out_offset2 + 21] = in32;
295
0
    }
296
297
0
    for (; (i + (14870 + 64) * sizeof(chorba_word_t)) < len; i += (32 * sizeof(chorba_word_t))) {
298
0
        chorba_word_t in1, in2, in3, in4, in5, in6, in7, in8;
299
0
        chorba_word_t in9, in10, in11, in12, in13, in14, in15, in16;
300
0
        chorba_word_t in17, in18, in19, in20, in21, in22, in23, in24;
301
0
        chorba_word_t in25, in26, in27, in28, in29, in30, in31, in32;
302
0
        int in_offset = (i / sizeof(chorba_word_t)) % bitbuffer_size_zwords;
303
0
        int out_offset1 = ((i / sizeof(chorba_word_t)) + 14848) % bitbuffer_size_zwords;
304
0
        int out_offset2 = ((i / sizeof(chorba_word_t)) + 14880) % bitbuffer_size_zwords;
305
306
0
        in1 = input[i / sizeof(chorba_word_t) + 0] ^ next1 ^ bitbuffer[in_offset + 0];
307
0
        in2 = input[i / sizeof(chorba_word_t) + 1] ^ next2 ^ bitbuffer[in_offset + 1];
308
0
        in3 = input[i / sizeof(chorba_word_t) + 2] ^ next3 ^ bitbuffer[in_offset + 2];
309
0
        in4 = input[i / sizeof(chorba_word_t) + 3] ^ next4 ^ bitbuffer[in_offset + 3];
310
0
        in5 = input[i / sizeof(chorba_word_t) + 4] ^ next5 ^ bitbuffer[in_offset + 4];
311
0
        in6 = input[i / sizeof(chorba_word_t) + 5] ^ next6 ^ bitbuffer[in_offset + 5];
312
0
        in7 = input[i / sizeof(chorba_word_t) + 6] ^ next7 ^ bitbuffer[in_offset + 6];
313
0
        in8 = input[i / sizeof(chorba_word_t) + 7] ^ next8 ^ in1 ^ bitbuffer[in_offset + 7];
314
0
        in9 = input[i / sizeof(chorba_word_t) + 8] ^ next9 ^ in2 ^ bitbuffer[in_offset + 8];
315
0
        in10 = input[i / sizeof(chorba_word_t) + 9] ^ next10 ^ in3 ^ bitbuffer[in_offset + 9];
316
0
        in11 = input[i / sizeof(chorba_word_t) + 10] ^ next11 ^ in4 ^ bitbuffer[in_offset + 10];
317
0
        in12 = input[i / sizeof(chorba_word_t) + 11] ^ next12 ^ in1 ^ in5 ^ bitbuffer[in_offset + 11];
318
0
        in13 = input[i / sizeof(chorba_word_t) + 12] ^ next13 ^ in2 ^ in6 ^ bitbuffer[in_offset + 12];
319
0
        in14 = input[i / sizeof(chorba_word_t) + 13] ^ next14 ^ in3 ^ in7 ^ bitbuffer[in_offset + 13];
320
0
        in15 = input[i / sizeof(chorba_word_t) + 14] ^ next15 ^ in4 ^ in8 ^ bitbuffer[in_offset + 14];
321
0
        in16 = input[i / sizeof(chorba_word_t) + 15] ^ next16 ^ in5 ^ in9 ^ bitbuffer[in_offset + 15];
322
0
        in17 = input[i / sizeof(chorba_word_t) + 16] ^ next17 ^ in6 ^ in10 ^ bitbuffer[in_offset + 16];
323
0
        in18 = input[i / sizeof(chorba_word_t) + 17] ^ next18 ^ in7 ^ in11 ^ bitbuffer[in_offset + 17];
324
0
        in19 = input[i / sizeof(chorba_word_t) + 18] ^ next19 ^ in8 ^ in12 ^ bitbuffer[in_offset + 18];
325
0
        in20 = input[i / sizeof(chorba_word_t) + 19] ^ next20 ^ in9 ^ in13 ^ bitbuffer[in_offset + 19];
326
0
        in21 = input[i / sizeof(chorba_word_t) + 20] ^ next21 ^ in10 ^ in14 ^ bitbuffer[in_offset + 20];
327
0
        in22 = input[i / sizeof(chorba_word_t) + 21] ^ next22 ^ in11 ^ in15 ^ bitbuffer[in_offset + 21];
328
0
        in23 = input[i / sizeof(chorba_word_t) + 22] ^ in1 ^ in12 ^ in16 ^ bitbuffer[in_offset + 22];
329
0
        in24 = input[i / sizeof(chorba_word_t) + 23] ^ in2 ^ in13 ^ in17 ^ bitbuffer[in_offset + 23];
330
0
        in25 = input[i / sizeof(chorba_word_t) + 24] ^ in3 ^ in14 ^ in18 ^ bitbuffer[in_offset + 24];
331
0
        in26 = input[i / sizeof(chorba_word_t) + 25] ^ in4 ^ in15 ^ in19 ^ bitbuffer[in_offset + 25];
332
0
        in27 = input[i / sizeof(chorba_word_t) + 26] ^ in5 ^ in16 ^ in20 ^ bitbuffer[in_offset + 26];
333
0
        in28 = input[i / sizeof(chorba_word_t) + 27] ^ in6 ^ in17 ^ in21 ^ bitbuffer[in_offset + 27];
334
0
        in29 = input[i / sizeof(chorba_word_t) + 28] ^ in7 ^ in18 ^ in22 ^ bitbuffer[in_offset + 28];
335
0
        in30 = input[i / sizeof(chorba_word_t) + 29] ^ in8 ^ in19 ^ in23 ^ bitbuffer[in_offset + 29];
336
0
        in31 = input[i / sizeof(chorba_word_t) + 30] ^ in9 ^ in20 ^ in24 ^ bitbuffer[in_offset + 30];
337
0
        in32 = input[i / sizeof(chorba_word_t) + 31] ^ in10 ^ in21 ^ in25 ^ bitbuffer[in_offset + 31];
338
339
0
        next1 = in11 ^ in22 ^ in26;
340
0
        next2 = in12 ^ in23 ^ in27;
341
0
        next3 = in13 ^ in24 ^ in28;
342
0
        next4 = in14 ^ in25 ^ in29;
343
0
        next5 = in15 ^ in26 ^ in30;
344
0
        next6 = in16 ^ in27 ^ in31;
345
0
        next7 = in17 ^ in28 ^ in32;
346
0
        next8 = in18 ^ in29;
347
0
        next9 = in19 ^ in30;
348
0
        next10 = in20 ^ in31;
349
0
        next11 = in21 ^ in32;
350
0
        next12 = in22;
351
0
        next13 = in23;
352
0
        next14 = in24;
353
0
        next15 = in25;
354
0
        next16 = in26;
355
0
        next17 = in27;
356
0
        next18 = in28;
357
0
        next19 = in29;
358
0
        next20 = in30;
359
0
        next21 = in31;
360
0
        next22 = in32;
361
362
0
        bitbuffer[out_offset1 + 22] = in1;
363
0
        bitbuffer[out_offset1 + 23] = in2;
364
0
        bitbuffer[out_offset1 + 24] = in3;
365
0
        bitbuffer[out_offset1 + 25] = in4;
366
0
        bitbuffer[out_offset1 + 26] = in5;
367
0
        bitbuffer[out_offset1 + 27] = in6;
368
0
        bitbuffer[out_offset1 + 28] = in7;
369
0
        bitbuffer[out_offset1 + 29] = in8;
370
0
        bitbuffer[out_offset1 + 30] = in9;
371
0
        bitbuffer[out_offset1 + 31] = in10;
372
0
        bitbuffer[out_offset2 + 0] = in11;
373
0
        bitbuffer[out_offset2 + 1] = in12;
374
0
        bitbuffer[out_offset2 + 2] = in13;
375
0
        bitbuffer[out_offset2 + 3] = in14;
376
0
        bitbuffer[out_offset2 + 4] = in15;
377
0
        bitbuffer[out_offset2 + 5] = in16;
378
0
        bitbuffer[out_offset2 + 6] = in17;
379
0
        bitbuffer[out_offset2 + 7] = in18;
380
0
        bitbuffer[out_offset2 + 8] = in19;
381
0
        bitbuffer[out_offset2 + 9] = in20;
382
0
        bitbuffer[out_offset2 + 10] = in21;
383
0
        bitbuffer[out_offset2 + 11] = in22;
384
0
        bitbuffer[out_offset2 + 12] = in23;
385
0
        bitbuffer[out_offset2 + 13] = in24;
386
0
        bitbuffer[out_offset2 + 14] = in25;
387
0
        bitbuffer[out_offset2 + 15] = in26;
388
0
        bitbuffer[out_offset2 + 16] = in27;
389
0
        bitbuffer[out_offset2 + 17] = in28;
390
0
        bitbuffer[out_offset2 + 18] = in29;
391
0
        bitbuffer[out_offset2 + 19] = in30;
392
0
        bitbuffer[out_offset2 + 20] = in31;
393
0
        bitbuffer[out_offset2 + 21] = in32;
394
0
    }
395
396
0
    bitbuffer[(i / sizeof(chorba_word_t) + 0) % bitbuffer_size_zwords] ^= next1;
397
0
    bitbuffer[(i / sizeof(chorba_word_t) + 1) % bitbuffer_size_zwords] ^= next2;
398
0
    bitbuffer[(i / sizeof(chorba_word_t) + 2) % bitbuffer_size_zwords] ^= next3;
399
0
    bitbuffer[(i / sizeof(chorba_word_t) + 3) % bitbuffer_size_zwords] ^= next4;
400
0
    bitbuffer[(i / sizeof(chorba_word_t) + 4) % bitbuffer_size_zwords] ^= next5;
401
0
    bitbuffer[(i / sizeof(chorba_word_t) + 5) % bitbuffer_size_zwords] ^= next6;
402
0
    bitbuffer[(i / sizeof(chorba_word_t) + 6) % bitbuffer_size_zwords] ^= next7;
403
0
    bitbuffer[(i / sizeof(chorba_word_t) + 7) % bitbuffer_size_zwords] ^= next8;
404
0
    bitbuffer[(i / sizeof(chorba_word_t) + 8) % bitbuffer_size_zwords] ^= next9;
405
0
    bitbuffer[(i / sizeof(chorba_word_t) + 9) % bitbuffer_size_zwords] ^= next10;
406
0
    bitbuffer[(i / sizeof(chorba_word_t) + 10) % bitbuffer_size_zwords] ^= next11;
407
0
    bitbuffer[(i / sizeof(chorba_word_t) + 11) % bitbuffer_size_zwords] ^= next12;
408
0
    bitbuffer[(i / sizeof(chorba_word_t) + 12) % bitbuffer_size_zwords] ^= next13;
409
0
    bitbuffer[(i / sizeof(chorba_word_t) + 13) % bitbuffer_size_zwords] ^= next14;
410
0
    bitbuffer[(i / sizeof(chorba_word_t) + 14) % bitbuffer_size_zwords] ^= next15;
411
0
    bitbuffer[(i / sizeof(chorba_word_t) + 15) % bitbuffer_size_zwords] ^= next16;
412
0
    bitbuffer[(i / sizeof(chorba_word_t) + 16) % bitbuffer_size_zwords] ^= next17;
413
0
    bitbuffer[(i / sizeof(chorba_word_t) + 17) % bitbuffer_size_zwords] ^= next18;
414
0
    bitbuffer[(i / sizeof(chorba_word_t) + 18) % bitbuffer_size_zwords] ^= next19;
415
0
    bitbuffer[(i / sizeof(chorba_word_t) + 19) % bitbuffer_size_zwords] ^= next20;
416
0
    bitbuffer[(i / sizeof(chorba_word_t) + 20) % bitbuffer_size_zwords] ^= next21;
417
0
    bitbuffer[(i / sizeof(chorba_word_t) + 21) % bitbuffer_size_zwords] ^= next22;
418
419
0
    for (int j = 14870; j < 14870 + 64; j++) {
420
0
        bitbuffer[(j + (i / sizeof(chorba_word_t))) % bitbuffer_size_zwords] = 0;
421
0
    }
422
423
0
    uint64_t next1_64 = 0;
424
0
    uint64_t next2_64 = 0;
425
0
    uint64_t next3_64 = 0;
426
0
    uint64_t next4_64 = 0;
427
0
    uint64_t next5_64 = 0;
428
0
    uint64_t final[9] = {0};
429
430
0
    for (; (i + 72 < len); i += 32) {
431
0
        uint64_t in1;
432
0
        uint64_t in2;
433
0
        uint64_t in3;
434
0
        uint64_t in4;
435
0
        uint64_t a1, a2, a3, a4;
436
0
        uint64_t b1, b2, b3, b4;
437
0
        uint64_t c1, c2, c3, c4;
438
0
        uint64_t d1, d2, d3, d4;
439
440
441
0
        in1 = input_qwords[i / sizeof(uint64_t)] ^ bitbuffer_qwords[(i / sizeof(uint64_t)) % bitbuffer_size_qwords];
442
0
        in2 = input_qwords[i / sizeof(uint64_t) + 1] ^ bitbuffer_qwords[(i / sizeof(uint64_t) + 1) % bitbuffer_size_qwords];
443
0
        in1 = Z_U64_FROM_LE(in1) ^ next1_64;
444
0
        in2 = Z_U64_FROM_LE(in2) ^ next2_64;
445
446
0
        NEXT_ROUND(in1, a1, a2, a3, a4);
447
448
0
        NEXT_ROUND(in2, b1, b2, b3, b4);
449
450
0
        in3 = input_qwords[i / sizeof(uint64_t) + 2] ^ bitbuffer_qwords[(i / sizeof(uint64_t) + 2) % bitbuffer_size_qwords];
451
0
        in4 = input_qwords[i / sizeof(uint64_t) + 3] ^ bitbuffer_qwords[(i / sizeof(uint64_t) + 3) % bitbuffer_size_qwords];
452
0
        in3 = Z_U64_FROM_LE(in3) ^ next3_64 ^ a1;
453
0
        in4 = Z_U64_FROM_LE(in4) ^ next4_64 ^ a2 ^ b1;
454
455
0
        NEXT_ROUND(in3, c1, c2, c3, c4);
456
457
0
        NEXT_ROUND(in4, d1, d2, d3, d4);
458
459
0
        ACCUM_ROUND(next1_64, next2_64, next3_64, next4_64, next5_64);
460
0
    }
461
462
0
    memcpy(final, input_qwords + (i / sizeof(uint64_t)), len-i);
463
0
    final[0] ^= Z_U64_TO_LE(next1_64);
464
0
    final[1] ^= Z_U64_TO_LE(next2_64);
465
0
    final[2] ^= Z_U64_TO_LE(next3_64);
466
0
    final[3] ^= Z_U64_TO_LE(next4_64);
467
0
    final[4] ^= Z_U64_TO_LE(next5_64);
468
469
0
    uint8_t *final_bytes = (uint8_t*)final;
470
471
0
    for (size_t j = 0; j < (len-i); j++) {
472
0
        crc = crc_table[(crc ^ final_bytes[j] ^ bitbuffer_bytes[(j+i) % bitbuffer_size_bytes]) & 0xff] ^ (crc >> 8);
473
0
    }
474
475
#if defined(__EMSCRIPTEN__)
476
    zng_free(bitbuffer);
477
#endif
478
0
    return ~crc;
479
0
}
480
481
#  if CHORBA_W == 8
482
/* Implement Chorba algorithm from https://arxiv.org/abs/2412.16398 */
483
0
Z_INTERNAL uint32_t crc32_chorba_32768_nondestructive(uint32_t crc, const uint8_t *buf, size_t len) {
484
    /* The calling function ensured that this is aligned correctly */
485
0
    const uint64_t* input = (const uint64_t*)buf;
486
0
    uint64_t bitbuffer[32768 / sizeof(uint64_t)];
487
0
    const uint8_t *bitbuffer_bytes = (const uint8_t*)bitbuffer;
488
0
    memset(bitbuffer, 0, 32768);
489
0
    bitbuffer[0] = Z_U64_TO_LE(~crc);
490
491
0
    crc = 0;
492
493
0
    size_t i = 0;
494
495
0
    for(; i + 300*8+64 < len; i += 64) {
496
0
        uint64_t in1, in2, in3, in4;
497
0
        uint64_t in5, in6, in7, in8;
498
0
        size_t in_offset = (i/8);
499
500
0
        in1 = input[i / sizeof(uint64_t) + 0] ^ bitbuffer[in_offset + 0];
501
0
        in2 = input[i / sizeof(uint64_t) + 1] ^ bitbuffer[in_offset + 1];
502
0
        in3 = input[i / sizeof(uint64_t) + 2] ^ bitbuffer[in_offset + 2];
503
0
        in4 = input[i / sizeof(uint64_t) + 3] ^ bitbuffer[in_offset + 3];
504
0
        in5 = input[i / sizeof(uint64_t) + 4] ^ bitbuffer[in_offset + 4];
505
0
        in6 = input[i / sizeof(uint64_t) + 5] ^ bitbuffer[in_offset + 5];
506
0
        in7 = input[i / sizeof(uint64_t) + 6] ^ bitbuffer[in_offset + 6];
507
0
        in8 = input[i / sizeof(uint64_t) + 7] ^ bitbuffer[in_offset + 7];
508
509
        // [0, 145, 183, 211]
510
511
0
        bitbuffer[(i/8 + 0 + 145)] ^= in1;
512
0
        bitbuffer[(i/8 + 1 + 145)] ^= in2;
513
0
        bitbuffer[(i/8 + 2 + 145)] ^= in3;
514
0
        bitbuffer[(i/8 + 3 + 145)] ^= in4;
515
0
        bitbuffer[(i/8 + 4 + 145)] ^= in5;
516
0
        bitbuffer[(i/8 + 5 + 145)] ^= in6;
517
0
        bitbuffer[(i/8 + 6 + 145)] ^= in7;
518
0
        bitbuffer[(i/8 + 7 + 145)] ^= in8;
519
520
0
        bitbuffer[(i/8 + 0 + 183)] ^= in1;
521
0
        bitbuffer[(i/8 + 1 + 183)] ^= in2;
522
0
        bitbuffer[(i/8 + 2 + 183)] ^= in3;
523
0
        bitbuffer[(i/8 + 3 + 183)] ^= in4;
524
0
        bitbuffer[(i/8 + 4 + 183)] ^= in5;
525
0
        bitbuffer[(i/8 + 5 + 183)] ^= in6;
526
0
        bitbuffer[(i/8 + 6 + 183)] ^= in7;
527
0
        bitbuffer[(i/8 + 7 + 183)] ^= in8;
528
529
0
        bitbuffer[(i/8 + 0 + 211)] ^= in1;
530
0
        bitbuffer[(i/8 + 1 + 211)] ^= in2;
531
0
        bitbuffer[(i/8 + 2 + 211)] ^= in3;
532
0
        bitbuffer[(i/8 + 3 + 211)] ^= in4;
533
0
        bitbuffer[(i/8 + 4 + 211)] ^= in5;
534
0
        bitbuffer[(i/8 + 5 + 211)] ^= in6;
535
0
        bitbuffer[(i/8 + 6 + 211)] ^= in7;
536
0
        bitbuffer[(i/8 + 7 + 211)] ^= in8;
537
538
0
        bitbuffer[(i/8 + 0 + 300)] = in1;
539
0
        bitbuffer[(i/8 + 1 + 300)] = in2;
540
0
        bitbuffer[(i/8 + 2 + 300)] = in3;
541
0
        bitbuffer[(i/8 + 3 + 300)] = in4;
542
0
        bitbuffer[(i/8 + 4 + 300)] = in5;
543
0
        bitbuffer[(i/8 + 5 + 300)] = in6;
544
0
        bitbuffer[(i/8 + 6 + 300)] = in7;
545
0
        bitbuffer[(i/8 + 7 + 300)] = in8;
546
0
    }
547
548
0
    uint64_t next1_64 = 0;
549
0
    uint64_t next2_64 = 0;
550
0
    uint64_t next3_64 = 0;
551
0
    uint64_t next4_64 = 0;
552
0
    uint64_t next5_64 = 0;
553
0
    uint64_t final[9] = {0};
554
555
0
    for (; (i + 72 < len); i += 32) {
556
0
        uint64_t in1;
557
0
        uint64_t in2;
558
0
        uint64_t in3;
559
0
        uint64_t in4;
560
0
        uint64_t a1, a2, a3, a4;
561
0
        uint64_t b1, b2, b3, b4;
562
0
        uint64_t c1, c2, c3, c4;
563
0
        uint64_t d1, d2, d3, d4;
564
565
566
0
        in1 = input[i / sizeof(uint64_t)] ^ bitbuffer[(i / sizeof(uint64_t))];
567
0
        in2 = input[(i + 8) / sizeof(uint64_t)] ^ bitbuffer[(i / sizeof(uint64_t) + 1)];
568
0
        in1 = Z_U64_FROM_LE(in1) ^ next1_64;
569
0
        in2 = Z_U64_FROM_LE(in2) ^ next2_64;
570
571
0
        NEXT_ROUND(in1, a1, a2, a3, a4);
572
573
0
        NEXT_ROUND(in2, b1, b2, b3, b4);
574
575
0
        in3 = input[(i + 16) / sizeof(uint64_t)] ^ bitbuffer[(i / sizeof(uint64_t) + 2)];
576
0
        in4 = input[(i + 24) / sizeof(uint64_t)] ^ bitbuffer[(i / sizeof(uint64_t) + 3)];
577
0
        in3 = Z_U64_FROM_LE(in3) ^ next3_64 ^ a1;
578
0
        in4 = Z_U64_FROM_LE(in4) ^ next4_64 ^ a2 ^ b1;
579
580
0
        NEXT_ROUND(in3, c1, c2, c3, c4);
581
582
0
        NEXT_ROUND(in4, d1, d2, d3, d4);
583
584
0
        ACCUM_ROUND(next1_64, next2_64, next3_64, next4_64, next5_64);
585
586
0
    }
587
588
0
    memcpy(final, input+(i / sizeof(uint64_t)), len-i);
589
0
    final[0] ^= Z_U64_TO_LE(next1_64);
590
0
    final[1] ^= Z_U64_TO_LE(next2_64);
591
0
    final[2] ^= Z_U64_TO_LE(next3_64);
592
0
    final[3] ^= Z_U64_TO_LE(next4_64);
593
0
    final[4] ^= Z_U64_TO_LE(next5_64);
594
595
0
    uint8_t *final_bytes = (uint8_t*)final;
596
597
0
    for (size_t j = 0; j < (len-i); j++) {
598
0
        crc = crc_table[(crc ^ final_bytes[j] ^ bitbuffer_bytes[(j+i)]) & 0xff] ^ (crc >> 8);
599
0
    }
600
601
0
    return ~crc;
602
0
}
603
604
/* Implement Chorba algorithm from https://arxiv.org/abs/2412.16398 */
605
0
Z_INTERNAL uint32_t crc32_chorba_small_nondestructive(uint32_t crc, const uint8_t *buf, size_t len) {
606
    /* The calling function ensured that this is aligned correctly */
607
0
    const uint64_t* input = (const uint64_t*)buf;
608
0
    uint64_t final[9] = {0};
609
0
    uint64_t next1 = ~crc;
610
0
    crc = 0;
611
0
    uint64_t next2 = 0;
612
0
    uint64_t next3 = 0;
613
0
    uint64_t next4 = 0;
614
0
    uint64_t next5 = 0;
615
616
0
    size_t i = 0;
617
618
    /* This is weird, doing for vs while drops 10% off the exec time */
619
0
    for (; (i + 256 + 40 + 32 + 32) < len; i += 32) {
620
0
        uint64_t in1;
621
0
        uint64_t in2;
622
0
        uint64_t in3;
623
0
        uint64_t in4;
624
0
        uint64_t a1, a2, a3, a4;
625
0
        uint64_t b1, b2, b3, b4;
626
0
        uint64_t c1, c2, c3, c4;
627
0
        uint64_t d1, d2, d3, d4;
628
629
630
0
        uint64_t chorba1 = Z_U64_FROM_LE(input[i / sizeof(uint64_t)]) ^ next1;
631
0
        uint64_t chorba2 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 1]) ^ next2;
632
0
        uint64_t chorba3 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 2]) ^ next3;
633
0
        uint64_t chorba4 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 3]) ^ next4;
634
0
        uint64_t chorba5 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 4]) ^ next5;
635
0
        uint64_t chorba6 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 5]);
636
0
        uint64_t chorba7 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 6]) ^ chorba1;
637
0
        uint64_t chorba8 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 7]) ^ chorba2;
638
639
0
        i += 8 * 8;
640
641
        /* 0-3 */
642
0
        in1 = Z_U64_FROM_LE(input[i / sizeof(uint64_t)]) ^ chorba3;
643
0
        in2 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 1]) ^ chorba4 ^ chorba1;
644
645
0
        NEXT_ROUND(in1, a1, a2, a3, a4);
646
647
0
        NEXT_ROUND(in2, b1, b2, b3, b4);
648
649
0
        in3 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 2]) ^ a1 ^ chorba5 ^ chorba2 ^ chorba1;
650
0
        in4 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 3]) ^ a2 ^ b1 ^ chorba6 ^ chorba3 ^ chorba2;
651
652
0
        NEXT_ROUND(in3, c1, c2, c3, c4);
653
654
0
        NEXT_ROUND(in4, d1, d2, d3, d4);
655
656
        /* chorba5 already consumed next5, clear it so ACCUM_ROUND
657
           does not xor the stale value into next1 */
658
0
        next5 = 0;
659
0
        ACCUM_ROUND(next1, next2, next3, next4, next5);
660
661
0
        i += 32;
662
663
        /* 4-7 */
664
0
        in1 = Z_U64_FROM_LE(input[i / sizeof(uint64_t)]) ^ next1 ^ chorba7 ^ chorba4 ^ chorba3;
665
0
        in2 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 1]) ^ next2 ^ chorba8 ^ chorba5 ^ chorba4;
666
667
0
        NEXT_ROUND(in1, a1, a2, a3, a4);
668
669
0
        NEXT_ROUND(in2, b1, b2, b3, b4);
670
671
0
        in3 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 2]) ^ next3 ^ a1 ^ chorba6 ^ chorba5;
672
0
        in4 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 3]) ^ next4 ^ a2 ^ b1 ^ chorba7 ^ chorba6;
673
674
0
        NEXT_ROUND(in3, c1, c2, c3, c4);
675
676
0
        NEXT_ROUND(in4, d1, d2, d3, d4);
677
678
0
        ACCUM_ROUND(next1, next2, next3, next4, next5);
679
680
0
        i += 32;
681
682
        /* 8-11 */
683
0
        in1 = Z_U64_FROM_LE(input[i / sizeof(uint64_t)]) ^ next1 ^ chorba8 ^ chorba7 ^ chorba1;
684
0
        in2 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 1]) ^ next2 ^ chorba8 ^ chorba2;
685
686
0
        NEXT_ROUND(in1, a1, a2, a3, a4);
687
688
0
        NEXT_ROUND(in2, b1, b2, b3, b4);
689
690
0
        in3 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 2]) ^ next3 ^ a1 ^ chorba3;
691
0
        in4 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 3]) ^ next4 ^ a2 ^ b1 ^ chorba4;
692
693
0
        NEXT_ROUND(in3, c1, c2, c3, c4);
694
695
0
        NEXT_ROUND(in4, d1, d2, d3, d4);
696
697
0
        ACCUM_ROUND(next1, next2, next3, next4, next5);
698
699
0
        i += 32;
700
701
        /* 12-15 */
702
0
        in1 = Z_U64_FROM_LE(input[i / sizeof(uint64_t)]) ^ next1 ^ chorba5 ^ chorba1;
703
0
        in2 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 1]) ^ next2 ^ chorba6 ^ chorba2 ^ chorba1;
704
705
0
        NEXT_ROUND(in1, a1, a2, a3, a4);
706
707
0
        NEXT_ROUND(in2, b1, b2, b3, b4);
708
709
0
        in3 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 2]) ^ next3 ^ a1 ^ chorba7 ^ chorba3 ^ chorba2 ^ chorba1;
710
0
        in4 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 3]) ^ next4 ^ a2 ^ b1 ^ chorba8 ^ chorba4 ^ chorba3 ^ chorba2;
711
712
0
        NEXT_ROUND(in3, c1, c2, c3, c4);
713
714
0
        NEXT_ROUND(in4, d1, d2, d3, d4);
715
716
0
        ACCUM_ROUND(next1, next2, next3, next4, next5);
717
718
0
        i += 32;
719
720
        /* 16-19 */
721
0
        in1 = Z_U64_FROM_LE(input[i / sizeof(uint64_t)]) ^ next1 ^ chorba5 ^ chorba4 ^ chorba3 ^ chorba1;
722
0
        in2 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 1]) ^ next2 ^ chorba6 ^ chorba5 ^ chorba4 ^ chorba1 ^ chorba2;
723
724
0
        NEXT_ROUND(in1, a1, a2, a3, a4);
725
726
0
        NEXT_ROUND(in2, b1, b2, b3, b4);
727
728
0
        in3 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 2]) ^ next3 ^ a1 ^ chorba7 ^ chorba6 ^ chorba5 ^ chorba2 ^ chorba3;
729
0
        in4 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 3]) ^ next4 ^ a2 ^ b1 ^ chorba8 ^ chorba7 ^ chorba6 ^ chorba3 ^ chorba4 ^ chorba1;
730
731
0
        NEXT_ROUND(in3, c1, c2, c3, c4);
732
733
0
        NEXT_ROUND(in4, d1, d2, d3, d4);
734
735
0
        ACCUM_ROUND(next1, next2, next3, next4, next5);
736
737
0
        i += 32;
738
739
        /* 20-23 */
740
0
        in1 = Z_U64_FROM_LE(input[i / sizeof(uint64_t)]) ^ next1 ^ chorba8 ^ chorba7 ^ chorba4 ^ chorba5 ^ chorba2 ^ chorba1;
741
0
        in2 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 1]) ^ next2 ^ chorba8 ^ chorba5 ^ chorba6 ^ chorba3 ^ chorba2;
742
743
0
        NEXT_ROUND(in1, a1, a2, a3, a4);
744
745
0
        NEXT_ROUND(in2, b1, b2, b3, b4);
746
747
0
        in3 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 2]) ^ next3 ^ a1 ^ chorba7 ^ chorba6 ^ chorba4 ^ chorba3 ^ chorba1;
748
0
        in4 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 3]) ^ next4 ^ a2 ^ b1 ^ chorba8 ^ chorba7 ^ chorba5 ^ chorba4 ^ chorba2 ^ chorba1;
749
750
0
        NEXT_ROUND(in3, c1, c2, c3, c4);
751
752
0
        NEXT_ROUND(in4, d1, d2, d3, d4);
753
754
0
        ACCUM_ROUND(next1, next2, next3, next4, next5);
755
756
0
        i += 32;
757
758
        /* 24-27 */
759
0
        in1 = Z_U64_FROM_LE(input[i / sizeof(uint64_t)]) ^ next1 ^ chorba8 ^ chorba6 ^ chorba5 ^ chorba3 ^ chorba2 ^ chorba1;
760
0
        in2 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 1]) ^ next2 ^ chorba7 ^ chorba6 ^ chorba4 ^ chorba3 ^ chorba2;
761
762
0
        NEXT_ROUND(in1, a1, a2, a3, a4);
763
764
0
        NEXT_ROUND(in2, b1, b2, b3, b4);
765
766
0
        in3 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 2]) ^ next3 ^ a1 ^ chorba8 ^ chorba7 ^ chorba5 ^ chorba4 ^ chorba3;
767
0
        in4 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 3]) ^ next4 ^ a2 ^ b1 ^ chorba8 ^ chorba6 ^ chorba5 ^ chorba4;
768
769
0
        NEXT_ROUND(in3, c1, c2, c3, c4);
770
771
0
        NEXT_ROUND(in4, d1, d2, d3, d4);
772
773
0
        ACCUM_ROUND(next1, next2, next3, next4, next5);
774
775
0
        i += 32;
776
777
        /* 28-31 */
778
0
        in1 = Z_U64_FROM_LE(input[i / sizeof(uint64_t)]) ^ next1 ^ chorba7 ^ chorba6 ^ chorba5;
779
0
        in2 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 1]) ^ next2 ^ chorba8 ^ chorba7 ^ chorba6;
780
781
0
        NEXT_ROUND(in1, a1, a2, a3, a4);
782
783
0
        NEXT_ROUND(in2, b1, b2, b3, b4);
784
785
0
        in3 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 2]) ^ next3 ^ a1 ^ chorba8 ^ chorba7;
786
0
        in4 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 3]) ^ next4 ^ a2 ^ b1 ^ chorba8;
787
788
0
        NEXT_ROUND(in3, c1, c2, c3, c4);
789
790
0
        NEXT_ROUND(in4, d1, d2, d3, d4);
791
792
0
        ACCUM_ROUND(next1, next2, next3, next4, next5);
793
0
    }
794
795
0
    for (; (i + 40 + 32) < len; i += 32) {
796
0
        uint64_t in1;
797
0
        uint64_t in2;
798
0
        uint64_t in3;
799
0
        uint64_t in4;
800
0
        uint64_t a1, a2, a3, a4;
801
0
        uint64_t b1, b2, b3, b4;
802
0
        uint64_t c1, c2, c3, c4;
803
0
        uint64_t d1, d2, d3, d4;
804
805
806
0
        in1 = Z_U64_FROM_LE(input[i / sizeof(uint64_t)]) ^ next1;
807
0
        in2 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 1]) ^ next2;
808
809
0
        NEXT_ROUND(in1, a1, a2, a3, a4);
810
811
0
        NEXT_ROUND(in2, b1, b2, b3, b4);
812
813
0
        in3 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 2]) ^ next3 ^ a1;
814
0
        in4 = Z_U64_FROM_LE(input[i / sizeof(uint64_t) + 3]) ^ next4 ^ a2 ^ b1;
815
816
0
        NEXT_ROUND(in3, c1, c2, c3, c4);
817
818
0
        NEXT_ROUND(in4, d1, d2, d3, d4);
819
820
0
        ACCUM_ROUND(next1, next2, next3, next4, next5);
821
0
    }
822
823
0
    memcpy(final, input+(i / sizeof(uint64_t)), len-i);
824
0
    final[0] ^= Z_U64_TO_LE(next1);
825
0
    final[1] ^= Z_U64_TO_LE(next2);
826
0
    final[2] ^= Z_U64_TO_LE(next3);
827
0
    final[3] ^= Z_U64_TO_LE(next4);
828
0
    final[4] ^= Z_U64_TO_LE(next5);
829
830
0
    return crc32_braid(~crc, (uint8_t*)final, len-i);
831
0
}
832
833
#else // CHORBA_W == 8
834
835
Z_INTERNAL uint32_t crc32_chorba_small_nondestructive_32bit(uint32_t crc, const uint8_t *buf, size_t len) {
836
    /* The calling function ensured that this is aligned correctly */
837
    const uint32_t* input = (const uint32_t*)buf;
838
    uint32_t final[20] = {0};
839
840
    uint32_t next1 = ~crc;
841
    crc = 0;
842
    uint32_t next2 = 0;
843
    uint32_t next3 = 0;
844
    uint32_t next4 = 0;
845
    uint32_t next5 = 0;
846
    uint32_t next6 = 0;
847
    uint32_t next7 = 0;
848
    uint32_t next8 = 0;
849
    uint32_t next9 = 0;
850
    uint32_t next10 = 0;
851
852
    size_t i = 0;
853
    for (; i + 80 < len; i += 40) {
854
        uint32_t in1;
855
        uint32_t in2;
856
        uint32_t in3;
857
        uint32_t in4;
858
        uint32_t in5;
859
        uint32_t in6;
860
        uint32_t in7;
861
        uint32_t in8;
862
        uint32_t in9;
863
        uint32_t in10;
864
865
        uint32_t a1, a2, a3, a4, a6, a7;
866
        uint32_t b1, b2, b3, b4, b6, b7;
867
        uint32_t c1, c2, c3, c4, c6, c7;
868
        uint32_t d1, d2, d3, d4, d6, d7;
869
        uint32_t e1, e2, e3, e4, e6, e7;
870
        uint32_t f1, f2, f3, f4, f6, f7;
871
        uint32_t g1, g2, g3, g4, g6, g7;
872
        uint32_t h1, h2, h3, h4, h6, h7;
873
        uint32_t i1, i2, i3, i4, i6, i7;
874
        uint32_t j1, j2, j3, j4, j6, j7;
875
876
        uint32_t out1;
877
        uint32_t out2;
878
        uint32_t out3;
879
        uint32_t out4;
880
        uint32_t out5;
881
        uint32_t out6;
882
        uint32_t out7;
883
        uint32_t out8;
884
        uint32_t out9;
885
        uint32_t out10;
886
887
        in1 = Z_U32_FROM_LE(input[i/sizeof(uint32_t) + 0]) ^ next1;
888
        in2 = Z_U32_FROM_LE(input[i/sizeof(uint32_t) + 1]) ^ next2;
889
        in3 = Z_U32_FROM_LE(input[i/sizeof(uint32_t) + 2]) ^ next3;
890
        in4 = Z_U32_FROM_LE(input[i/sizeof(uint32_t) + 3]) ^ next4;
891
892
        a1 = (in1 << 17);
893
        a2 = (in1 >> 15) ^ (in1 << 23);
894
        a3 = (in1 >> 9) ^ (in1 << 19);
895
        a4 = (in1 >> 13);
896
        a6 = (in1 << 12);
897
        a7 = (in1 >> 20);
898
899
        b1 = (in2 << 17);
900
        b2 = (in2 >> 15) ^ (in2 << 23);
901
        b3 = (in2 >> 9) ^ (in2 << 19);
902
        b4 = (in2 >> 13);
903
        b6 = (in2 << 12);
904
        b7 = (in2 >> 20);
905
906
        c1 = (in3 << 17);
907
        c2 = (in3 >> 15) ^ (in3 << 23);
908
        c3 = (in3 >> 9) ^ (in3 << 19);
909
        c4 = (in3 >> 13);
910
        c6 = (in3 << 12);
911
        c7 = (in3 >> 20);
912
913
        d1 = (in4 << 17);
914
        d2 = (in4 >> 15) ^ (in4 << 23);
915
        d3 = (in4 >> 9) ^ (in4 << 19);
916
        d4 = (in4 >> 13);
917
        d6 = (in4 << 12);
918
        d7 = (in4 >> 20);
919
920
        in5 = Z_U32_FROM_LE(input[i/sizeof(uint32_t) + 4]) ^ next5 ^ a1;
921
        in6 = Z_U32_FROM_LE(input[i/sizeof(uint32_t) + 5]) ^ next6 ^ a2 ^ b1;
922
        in7 = Z_U32_FROM_LE(input[i/sizeof(uint32_t) + 6]) ^ next7 ^ a3 ^ b2 ^ c1;
923
        in8 = Z_U32_FROM_LE(input[i/sizeof(uint32_t) + 7]) ^ next8 ^ a4 ^ b3 ^ c2 ^ d1;
924
925
        e1 = (in5 << 17);
926
        e2 = (in5 >> 15) ^ (in5 << 23);
927
        e3 = (in5 >> 9) ^ (in5 << 19);
928
        e4 = (in5 >> 13);
929
        e6 = (in5 << 12);
930
        e7 = (in5 >> 20);
931
932
        f1 = (in6 << 17);
933
        f2 = (in6 >> 15) ^ (in6 << 23);
934
        f3 = (in6 >> 9) ^ (in6 << 19);
935
        f4 = (in6 >> 13);
936
        f6 = (in6 << 12);
937
        f7 = (in6 >> 20);
938
939
        g1 = (in7 << 17);
940
        g2 = (in7 >> 15) ^ (in7 << 23);
941
        g3 = (in7 >> 9) ^ (in7 << 19);
942
        g4 = (in7 >> 13);
943
        g6 = (in7 << 12);
944
        g7 = (in7 >> 20);
945
946
        h1 = (in8 << 17);
947
        h2 = (in8 >> 15) ^ (in8 << 23);
948
        h3 = (in8 >> 9) ^ (in8 << 19);
949
        h4 = (in8 >> 13);
950
        h6 = (in8 << 12);
951
        h7 = (in8 >> 20);
952
953
        in9 = Z_U32_FROM_LE(input[i/sizeof(uint32_t) + 8]) ^ next9 ^ b4 ^ c3 ^ d2 ^ e1;
954
        in10 = Z_U32_FROM_LE(input[i/sizeof(uint32_t) + 9]) ^ next10 ^ a6 ^ c4 ^ d3 ^ e2 ^ f1;
955
956
        i1 = (in9 << 17);
957
        i2 = (in9 >> 15) ^ (in9 << 23);
958
        i3 = (in9 >> 9) ^ (in9 << 19);
959
        i4 = (in9 >> 13);
960
        i6 = (in9 << 12);
961
        i7 = (in9 >> 20);
962
963
        j1 = (in10 << 17);
964
        j2 = (in10 >> 15) ^ (in10 << 23);
965
        j3 = (in10 >> 9) ^ (in10 << 19);
966
        j4 = (in10 >> 13);
967
        j6 = (in10 << 12);
968
        j7 = (in10 >> 20);
969
970
        out1 = a7 ^ b6 ^ d4 ^ e3 ^ f2 ^ g1;
971
        out2 = b7 ^ c6 ^ e4 ^ f3 ^ g2 ^ h1;
972
        out3 = c7 ^ d6 ^ f4 ^ g3 ^ h2 ^ i1;
973
        out4 = d7 ^ e6 ^ g4 ^ h3 ^ i2 ^ j1;
974
        out5 = e7 ^ f6 ^ h4 ^ i3 ^ j2;
975
        out6 = f7 ^ g6 ^ i4 ^ j3;
976
        out7 = g7 ^ h6 ^ j4;
977
        out8 = h7 ^ i6;
978
        out9 = i7 ^ j6;
979
        out10 = j7;
980
981
        next1 = out1;
982
        next2 = out2;
983
        next3 = out3;
984
        next4 = out4;
985
        next5 = out5;
986
        next6 = out6;
987
        next7 = out7;
988
        next8 = out8;
989
        next9 = out9;
990
        next10 = out10;
991
992
    }
993
994
    memcpy(final, input+(i/sizeof(uint32_t)), len-i);
995
    final[0] ^= Z_U32_TO_LE(next1);
996
    final[1] ^= Z_U32_TO_LE(next2);
997
    final[2] ^= Z_U32_TO_LE(next3);
998
    final[3] ^= Z_U32_TO_LE(next4);
999
    final[4] ^= Z_U32_TO_LE(next5);
1000
    final[5] ^= Z_U32_TO_LE(next6);
1001
    final[6] ^= Z_U32_TO_LE(next7);
1002
    final[7] ^= Z_U32_TO_LE(next8);
1003
    final[8] ^= Z_U32_TO_LE(next9);
1004
    final[9] ^= Z_U32_TO_LE(next10);
1005
1006
    return crc32_braid(~crc, (uint8_t*)final, len-i);
1007
}
1008
#endif // CHORBA_W == 8
1009
1010
0
Z_INTERNAL uint32_t crc32_chorba(uint32_t crc, const uint8_t *buf, size_t len) {
1011
0
    uintptr_t align_diff = ALIGN_DIFF(buf, 8);
1012
0
    if (len <= align_diff + CHORBA_SMALL_THRESHOLD)
1013
0
        return crc32_braid(crc, buf, len);
1014
1015
0
    if (align_diff) {
1016
0
        crc = crc32_braid(crc, buf, align_diff);
1017
0
        len -= align_diff;
1018
0
        buf += align_diff;
1019
0
    }
1020
0
    if (len > CHORBA_LARGE_THRESHOLD)
1021
0
        return crc32_chorba_118960_nondestructive(crc, buf, len);
1022
0
#if CHORBA_W == 8
1023
0
    if (len > CHORBA_MEDIUM_LOWER_THRESHOLD && len <= CHORBA_MEDIUM_UPPER_THRESHOLD)
1024
0
        return crc32_chorba_32768_nondestructive(crc, buf, len);
1025
0
    return crc32_chorba_small_nondestructive(crc, buf, len);
1026
#else
1027
    return crc32_chorba_small_nondestructive_32bit(crc, buf, len);
1028
#endif
1029
0
}
1030
1031
0
uint32_t crc32_copy_chorba(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len) {
1032
0
    crc = crc32_chorba(crc, src, len);
1033
0
    memcpy(dst, src, len);
1034
0
    return crc;
1035
0
}
1036
1037
#endif /* CRC32_CHORBA_FALLBACK */