/src/zlib-ng/arch/x86/crc32_chorba_sse41.c

Source
#include "zbuild.h"
#include "arch_functions.h"

#if defined(X86_SSE41) && defined(CRC32_CHORBA_SSE_FALLBACK)

#include "crc32_chorba_p.h"
#include "crc32_p.h"
#include <emmintrin.h>
#include <smmintrin.h>
#include "arch/x86/x86_intrins.h"

#define READ_NEXT(in, off, a, b) \
    do { \
        a = _mm_load_si128((__m128i*)(in + off / sizeof(uint64_t))); \
        b = _mm_load_si128((__m128i*)(in + off / sizeof(uint64_t) + 2)); \
    } while (0)

#define NEXT_ROUND(invec, a, b, c, d) \
    do { \
        a = _mm_xor_si128(_mm_slli_epi64(invec, 17), _mm_slli_epi64(invec, 55)); \
        b = _mm_xor_si128(_mm_xor_si128(_mm_srli_epi64(invec, 47), _mm_srli_epi64(invec, 9)), _mm_slli_epi64(invec, 19)); \
        c = _mm_xor_si128(_mm_srli_epi64(invec, 45), _mm_slli_epi64(invec, 44)); \
        d  = _mm_srli_epi64(invec, 20); \
    } while (0)

#define REALIGN_CHORBA(in0, in1, in2, in3, out0, out1, out2, out3, out4, shift) \
    do { \
        out0 = _mm_slli_si128(in0, shift); \
        out1 = _mm_alignr_epi8(in1, in0, shift); \
        out2 = _mm_alignr_epi8(in2, in1, shift); \
        out3 = _mm_alignr_epi8(in3, in2, shift); \
        out4 = _mm_srli_si128(in3, shift); \
    } while (0)

#define STORE4(out0, out1, out2, out3, out) \
    do { \
        _mm_store_si128(out++, out0); \
        _mm_store_si128(out++, out1); \
        _mm_store_si128(out++, out2); \
        _mm_store_si128(out++, out3); \
    } while (0)

#define READ4(out0, out1, out2, out3, in) \
    do { \
        out0 = _mm_load_si128(in++); \
        out1 = _mm_load_si128(in++); \
        out2 = _mm_load_si128(in++); \
        out3 = _mm_load_si128(in++); \
    } while (0)

/* This is intentionally shifted one down to compensate for the deferred store from
 * the last iteration */
#define READ4_WITHXOR(out0, out1, out2, out3, xor0, xor1, xor2, xor3, in) \
    do { \
        out0 = _mm_xor_si128(in[1], xor0); \
        out1 = _mm_xor_si128(in[2], xor1); \
        out2 = _mm_xor_si128(in[3], xor2); \
        out3 = _mm_xor_si128(in[4], xor3); \
    } while (0)

/* Stores into a lane that hasn't been written yet, the last vector carries into the next round */
#define STORE_LANE(carry, out) \
    do { \
        __m128i first = _mm_xor_si128(carry, in_1); \
        STORE4(first, in23, in45, in67, out); \
        carry = in8_; \
    } while (0)

/* Same, but XORs into what earlier rounds already stored in the lane */
#define STORE_LANE_XOR(carry, out) \
    do { \
        __m128i first = _mm_xor_si128(carry, in_1), x1, x2, x3; \
        READ4_WITHXOR(x1, x2, x3, carry, in23, in45, in67, in8_, out); \
        STORE4(first, x1, x2, x3, out); \
    } while (0)

Z_FORCEINLINE static uint32_t crc32_chorba_32768_nondestructive_sse41(uint32_t crc, const uint8_t *buf, size_t len) {
    /* The calling function ensured that this is aligned correctly */
    const uint64_t* input = (const uint64_t*)buf;
    ALIGNED_(16) uint64_t bitbuffer[32768 / sizeof(uint64_t)];
    __m128i *bitbuffer_v = (__m128i*)bitbuffer;
    __m128i z = _mm_setzero_si128();

    __m128i *bitbuf128 = &bitbuffer_v[64];
    __m128i *bitbuf144 = &bitbuffer_v[72];
    __m128i *bitbuf182 = &bitbuffer_v[91];
    __m128i *bitbuf210 = &bitbuffer_v[105];
    __m128i *bitbuf300 = &bitbuffer_v[150];
    __m128i *bitbuf0 = bitbuf128;
    __m128i *inptr = (__m128i*)input;

    /* We only need to zero out the bytes between the 128'th value and the 144th
     * that are actually read */
    __m128i *z_cursor = bitbuf128;
    for (size_t i = 0; i < 2; ++i) {
        STORE4(z, z, z, z, z_cursor);
    }

    /* We only need to zero out the bytes between the 144'th value and the 182nd that
     * are actually read */
    z_cursor = bitbuf144 + 8;
    for (size_t i = 0; i < 11; ++i) {
        _mm_store_si128(z_cursor++, z);
    }

    /* We only need to zero out the bytes between the 182nd value and the 210th that
     * are actually read. */
    z_cursor = bitbuf182;
    for (size_t i = 0; i < 4; ++i) {
        STORE4(z, z, z, z, z_cursor);
    }

    /* We need to mix this in */
    __m128i init_crc = _mm_cvtsi64_si128(~crc);
    crc = 0;

    size_t i = 0;

    /* Previous iteration runs carried over */
    __m128i buf144 = z;
    __m128i buf182 = z;
    __m128i buf210 = z;

    for (; i + 300*8+64 < len && i < 22 * 8; i += 64) {
        __m128i in12, in34, in56, in78,
                in_1, in23, in45, in67, in8_;

        READ4(in12, in34, in56, in78, inptr);

        if (i == 0) {
            in12 = _mm_xor_si128(in12, init_crc);
        }

        REALIGN_CHORBA(in12, in34, in56, in78,
                       in_1, in23, in45, in67, in8_, 8);

        STORE_LANE(buf144, bitbuf144);
        STORE_LANE(buf182, bitbuf182);
        STORE_LANE(buf210, bitbuf210);
        STORE4(in12, in34, in56, in78, bitbuf300);
    }

    for (; i + 300*8+64 < len && i < 32 * 8; i += 64) {
        __m128i in12, in34, in56, in78,
                in_1, in23, in45, in67, in8_;
        READ4(in12, in34, in56, in78, inptr);

        REALIGN_CHORBA(in12, in34, in56, in78,
                       in_1, in23, in45, in67, in8_, 8);

        STORE_LANE(buf144, bitbuf144);
        STORE_LANE_XOR(buf182, bitbuf182);
        STORE_LANE(buf210, bitbuf210);
        STORE4(in12, in34, in56, in78, bitbuf300);
    }

    for (; i + 300*8+64 < len && i < 84 * 8; i += 64) {
        __m128i in12, in34, in56, in78,
                in_1, in23, in45, in67, in8_;
        READ4(in12, in34, in56, in78, inptr);

        REALIGN_CHORBA(in12, in34, in56, in78,
                       in_1, in23, in45, in67, in8_, 8);

        STORE_LANE_XOR(buf144, bitbuf144);
        STORE_LANE_XOR(buf182, bitbuf182);
        STORE_LANE(buf210, bitbuf210);
        STORE4(in12, in34, in56, in78, bitbuf300);
    }

    for (; i + 300*8+64 < len; i += 64) {
        __m128i in12, in34, in56, in78,
                in_1, in23, in45, in67, in8_;

        if (i < 128 * 8) {
            READ4(in12, in34, in56, in78, inptr);
        } else {
            in12 = _mm_xor_si128(_mm_load_si128(inptr++), _mm_load_si128(bitbuf0++));
            in34 = _mm_xor_si128(_mm_load_si128(inptr++), _mm_load_si128(bitbuf0++));
            in56 = _mm_xor_si128(_mm_load_si128(inptr++), _mm_load_si128(bitbuf0++));
            in78 = _mm_xor_si128(_mm_load_si128(inptr++), _mm_load_si128(bitbuf0++));
        }

        // [0, 145, 183, 211]

        /* Pre Penryn CPUs the unpack should be faster */
        REALIGN_CHORBA(in12, in34, in56, in78,
                       in_1, in23, in45, in67, in8_, 8);

        STORE_LANE_XOR(buf144, bitbuf144);
        STORE_LANE_XOR(buf182, bitbuf182);

        __m128i n, o, p;
        __m128i m = _mm_xor_si128(buf210, in_1);

        /* Couldn't tell you why but despite knowing that this is always false,
         * removing this branch with GCC makes things significantly slower. Some
         * loop bodies must be being joined or something */
        if (i < 84 * 8) {
            n = in23;
            o = in45;
            p = in67;
            buf210 = in8_;
        } else {
            READ4_WITHXOR(n, o, p, buf210, in23, in45, in67, in8_, bitbuf210);
        }

        STORE4(m, n, o, p, bitbuf210);
        STORE4(in12, in34, in56, in78, bitbuf300);
    }

    /* Second half of stores bubbled out */
    _mm_store_si128(bitbuf144, buf144);
    _mm_store_si128(bitbuf182, buf182);
    _mm_store_si128(bitbuf210, buf210);

    /* We also have to zero out the tail */
    size_t left_to_z = len - (300*8 + i);
    __m128i *bitbuf_tail = (__m128i*)(bitbuffer + 300 + i/8);
    while (left_to_z >= 64) {
       STORE4(z, z, z, z, bitbuf_tail);
       left_to_z -= 64;
    }

    while (left_to_z >= 16) {
       _mm_store_si128(bitbuf_tail++, z);
       left_to_z -= 16;
    }

    uint8_t *tail_bytes = (uint8_t*)bitbuf_tail;
    while (left_to_z--) {
       *tail_bytes++ = 0;
    }

    ALIGNED_(16) uint64_t final[9] = {0};
    __m128i next12, next34, next56;
    next12 = z;
    next34 = z;
    next56 = z;

    for (; (i + 72 < len); i += 32) {
        __m128i in1in2, in3in4;
        __m128i in1in2_, in3in4_;
        __m128i ab1, ab2, ab3, ab4;
        __m128i cd1, cd2, cd3, cd4;

        READ_NEXT(input, i, in1in2, in3in4);
        READ_NEXT(bitbuffer, i, in1in2_, in3in4_);

        in1in2 = _mm_xor_si128(_mm_xor_si128(in1in2, in1in2_), next12);
        in3in4 = _mm_xor_si128(in3in4, in3in4_);

        NEXT_ROUND(in1in2, ab1, ab2, ab3, ab4);

        __m128i a2_ = _mm_slli_si128(ab2, 8);
        __m128i ab1_next34 = _mm_xor_si128(next34, ab1);
        in3in4 = _mm_xor_si128(in3in4, ab1_next34);
        in3in4 = _mm_xor_si128(a2_, in3in4);
        NEXT_ROUND(in3in4, cd1, cd2, cd3, cd4);

        __m128i b2c2 = _mm_alignr_epi8(cd2, ab2, 8);
        __m128i a4_ = _mm_slli_si128(ab4, 8);
        a4_ = _mm_xor_si128(b2c2, a4_);
        next12 = _mm_xor_si128(ab3, a4_);
        next12 = _mm_xor_si128(next12, cd1);

        __m128i d2_ = _mm_srli_si128(cd2, 8);
        __m128i b4c4 = _mm_alignr_epi8(cd4, ab4, 8);
        next12 = _mm_xor_si128(next12, next56);
        next34 = _mm_xor_si128(cd3, _mm_xor_si128(b4c4, d2_));
        next56 = _mm_srli_si128(cd4, 8);
    }

    memcpy(final, input+(i / sizeof(uint64_t)), len-i);
    __m128i *final128 = (__m128i*)final;
    _mm_store_si128(final128, _mm_xor_si128(_mm_load_si128(final128), next12));
    ++final128;
    _mm_store_si128(final128, _mm_xor_si128(_mm_load_si128(final128), next34));
    ++final128;
    _mm_store_si128(final128, _mm_xor_si128(_mm_load_si128(final128), next56));

    /* The buffer still owes the last words */
    for (size_t j = 0; j < (len - i + sizeof(uint64_t) - 1) / sizeof(uint64_t); j++) {
        final[j] ^= bitbuffer[i / sizeof(uint64_t) + j];
    }

    /* At most 72 bytes */
    return ~crc32_copy_small(crc, NULL, (uint8_t*)final, len-i, 72, 0);
}

Z_INTERNAL uint32_t crc32_chorba_sse41(uint32_t crc, const uint8_t *buf, size_t len) {
    uintptr_t align_diff = ALIGN_DIFF(buf, 16);
    /* At most 87 bytes */
    if (len <= align_diff + CHORBA_SMALL_THRESHOLD_64BIT)
        return ~crc32_copy_small(~crc, NULL, buf, len, 15 + CHORBA_SMALL_THRESHOLD_64BIT, 0);

    if (align_diff) {
        /* At most 15 bytes */
        crc = ~crc32_copy_small(~crc, NULL, buf, align_diff, 15, 0);
        len -= align_diff;
        buf += align_diff;
    }
#ifdef CRC32_CHORBA_FALLBACK
    if (len > CHORBA_LARGE_THRESHOLD)
        return crc32_chorba_118960_nondestructive(crc, buf, len);
#endif
    if (len > CHORBA_MEDIUM_LOWER_THRESHOLD && len <= CHORBA_MEDIUM_UPPER_THRESHOLD)
        return crc32_chorba_32768_nondestructive_sse41(crc, buf, len);
    return chorba_small_nondestructive_sse2(crc, buf, len);
}

Z_INTERNAL uint32_t crc32_copy_chorba_sse41(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len) {
    crc = crc32_chorba_sse41(crc, src, len);
    memcpy(dst, src, len);
    return crc;
}
#endif

Coverage Report

Created: 2026-11-12 06:21

Line	Count	Source
1		#include "zbuild.h"
2		#include "arch_functions.h"
3
4		#if defined(X86_SSE41) && defined(CRC32_CHORBA_SSE_FALLBACK)
5
6		#include "crc32_chorba_p.h"
7		#include "crc32_p.h"
8		#include <emmintrin.h>
9		#include <smmintrin.h>
10		#include "arch/x86/x86_intrins.h"
11
12		#define READ_NEXT(in, off, a, b) \
13	0	do { \
14	0	a = _mm_load_si128((__m128i*)(in + off / sizeof(uint64_t))); \
15	0	b = _mm_load_si128((__m128i*)(in + off / sizeof(uint64_t) + 2)); \
16	0	} while (0)
17
18		#define NEXT_ROUND(invec, a, b, c, d) \
19	0	do { \
20	0	a = _mm_xor_si128(_mm_slli_epi64(invec, 17), _mm_slli_epi64(invec, 55)); \
21	0	b = _mm_xor_si128(_mm_xor_si128(_mm_srli_epi64(invec, 47), _mm_srli_epi64(invec, 9)), _mm_slli_epi64(invec, 19)); \
22	0	c = _mm_xor_si128(_mm_srli_epi64(invec, 45), _mm_slli_epi64(invec, 44)); \
23	0	d = _mm_srli_epi64(invec, 20); \
24	0	} while (0)
25
26		#define REALIGN_CHORBA(in0, in1, in2, in3, out0, out1, out2, out3, out4, shift) \
27	0	do { \
28	0	out0 = _mm_slli_si128(in0, shift); \
29	0	out1 = _mm_alignr_epi8(in1, in0, shift); \
30	0	out2 = _mm_alignr_epi8(in2, in1, shift); \
31	0	out3 = _mm_alignr_epi8(in3, in2, shift); \
32	0	out4 = _mm_srli_si128(in3, shift); \
33	0	} while (0)
34
35		#define STORE4(out0, out1, out2, out3, out) \
36	0	do { \
37	0	_mm_store_si128(out++, out0); \
38	0	_mm_store_si128(out++, out1); \
39	0	_mm_store_si128(out++, out2); \
40	0	_mm_store_si128(out++, out3); \
41	0	} while (0)
42
43		#define READ4(out0, out1, out2, out3, in) \
44	0	do { \
45	0	out0 = _mm_load_si128(in++); \
46	0	out1 = _mm_load_si128(in++); \
47	0	out2 = _mm_load_si128(in++); \
48	0	out3 = _mm_load_si128(in++); \
49	0	} while (0)
50
51		/* This is intentionally shifted one down to compensate for the deferred store from
52		* the last iteration */
53		#define READ4_WITHXOR(out0, out1, out2, out3, xor0, xor1, xor2, xor3, in) \
54	0	do { \
55	0	out0 = _mm_xor_si128(in[1], xor0); \
56	0	out1 = _mm_xor_si128(in[2], xor1); \
57	0	out2 = _mm_xor_si128(in[3], xor2); \
58	0	out3 = _mm_xor_si128(in[4], xor3); \
59	0	} while (0)
60
61		/* Stores into a lane that hasn't been written yet, the last vector carries into the next round */
62		#define STORE_LANE(carry, out) \
63	0	do { \
64	0	__m128i first = _mm_xor_si128(carry, in_1); \
65	0	STORE4(first, in23, in45, in67, out); \
66	0	carry = in8_; \
67	0	} while (0)
68
69		/* Same, but XORs into what earlier rounds already stored in the lane */
70		#define STORE_LANE_XOR(carry, out) \
71	0	do { \
72	0	__m128i first = _mm_xor_si128(carry, in_1), x1, x2, x3; \
73	0	READ4_WITHXOR(x1, x2, x3, carry, in23, in45, in67, in8_, out); \
74	0	STORE4(first, x1, x2, x3, out); \
75	0	} while (0)
76
77	0	Z_FORCEINLINE static uint32_t crc32_chorba_32768_nondestructive_sse41(uint32_t crc, const uint8_t *buf, size_t len) {
78		/* The calling function ensured that this is aligned correctly */
79	0	const uint64_t* input = (const uint64_t*)buf;
80	0	ALIGNED_(16) uint64_t bitbuffer[32768 / sizeof(uint64_t)];
81	0	__m128i bitbuffer_v = (__m128i)bitbuffer;
82	0	__m128i z = _mm_setzero_si128();
83
84	0	__m128i *bitbuf128 = &bitbuffer_v[64];
85	0	__m128i *bitbuf144 = &bitbuffer_v[72];
86	0	__m128i *bitbuf182 = &bitbuffer_v[91];
87	0	__m128i *bitbuf210 = &bitbuffer_v[105];
88	0	__m128i *bitbuf300 = &bitbuffer_v[150];
89	0	__m128i *bitbuf0 = bitbuf128;
90	0	__m128i inptr = (__m128i)input;
91
92		/* We only need to zero out the bytes between the 128'th value and the 144th
93		* that are actually read */
94	0	__m128i *z_cursor = bitbuf128;
95	0	for (size_t i = 0; i < 2; ++i) {
96	0	STORE4(z, z, z, z, z_cursor);
97	0	}
98
99		/* We only need to zero out the bytes between the 144'th value and the 182nd that
100		* are actually read */
101	0	z_cursor = bitbuf144 + 8;
102	0	for (size_t i = 0; i < 11; ++i) {
103	0	_mm_store_si128(z_cursor++, z);
104	0	}
105
106		/* We only need to zero out the bytes between the 182nd value and the 210th that
107		* are actually read. */
108	0	z_cursor = bitbuf182;
109	0	for (size_t i = 0; i < 4; ++i) {
110	0	STORE4(z, z, z, z, z_cursor);
111	0	}
112
113		/* We need to mix this in */
114	0	__m128i init_crc = _mm_cvtsi64_si128(~crc);
115	0	crc = 0;
116
117	0	size_t i = 0;
118
119		/* Previous iteration runs carried over */
120	0	__m128i buf144 = z;
121	0	__m128i buf182 = z;
122	0	__m128i buf210 = z;
123
124	0	for (; i + 3008+64 < len && i < 22 8; i += 64) {
125	0	__m128i in12, in34, in56, in78,
126	0	in_1, in23, in45, in67, in8_;
127
128	0	READ4(in12, in34, in56, in78, inptr);
129
130	0	if (i == 0) {
131	0	in12 = _mm_xor_si128(in12, init_crc);
132	0	}
133
134	0	REALIGN_CHORBA(in12, in34, in56, in78,
135	0	in_1, in23, in45, in67, in8_, 8);
136
137	0	STORE_LANE(buf144, bitbuf144);
138	0	STORE_LANE(buf182, bitbuf182);
139	0	STORE_LANE(buf210, bitbuf210);
140	0	STORE4(in12, in34, in56, in78, bitbuf300);
141	0	}
142
143	0	for (; i + 3008+64 < len && i < 32 8; i += 64) {
144	0	__m128i in12, in34, in56, in78,
145	0	in_1, in23, in45, in67, in8_;
146	0	READ4(in12, in34, in56, in78, inptr);
147
148	0	REALIGN_CHORBA(in12, in34, in56, in78,
149	0	in_1, in23, in45, in67, in8_, 8);
150
151	0	STORE_LANE(buf144, bitbuf144);
152	0	STORE_LANE_XOR(buf182, bitbuf182);
153	0	STORE_LANE(buf210, bitbuf210);
154	0	STORE4(in12, in34, in56, in78, bitbuf300);
155	0	}
156
157	0	for (; i + 3008+64 < len && i < 84 8; i += 64) {
158	0	__m128i in12, in34, in56, in78,
159	0	in_1, in23, in45, in67, in8_;
160	0	READ4(in12, in34, in56, in78, inptr);
161
162	0	REALIGN_CHORBA(in12, in34, in56, in78,
163	0	in_1, in23, in45, in67, in8_, 8);
164
165	0	STORE_LANE_XOR(buf144, bitbuf144);
166	0	STORE_LANE_XOR(buf182, bitbuf182);
167	0	STORE_LANE(buf210, bitbuf210);
168	0	STORE4(in12, in34, in56, in78, bitbuf300);
169	0	}
170
171	0	for (; i + 300*8+64 < len; i += 64) {
172	0	__m128i in12, in34, in56, in78,
173	0	in_1, in23, in45, in67, in8_;
174
175	0	if (i < 128 * 8) {
176	0	READ4(in12, in34, in56, in78, inptr);
177	0	} else {
178	0	in12 = _mm_xor_si128(_mm_load_si128(inptr++), _mm_load_si128(bitbuf0++));
179	0	in34 = _mm_xor_si128(_mm_load_si128(inptr++), _mm_load_si128(bitbuf0++));
180	0	in56 = _mm_xor_si128(_mm_load_si128(inptr++), _mm_load_si128(bitbuf0++));
181	0	in78 = _mm_xor_si128(_mm_load_si128(inptr++), _mm_load_si128(bitbuf0++));
182	0	}
183
184		// [0, 145, 183, 211]
185
186		/* Pre Penryn CPUs the unpack should be faster */
187	0	REALIGN_CHORBA(in12, in34, in56, in78,
188	0	in_1, in23, in45, in67, in8_, 8);
189
190	0	STORE_LANE_XOR(buf144, bitbuf144);
191	0	STORE_LANE_XOR(buf182, bitbuf182);
192
193	0	__m128i n, o, p;
194	0	__m128i m = _mm_xor_si128(buf210, in_1);
195
196		/* Couldn't tell you why but despite knowing that this is always false,
197		* removing this branch with GCC makes things significantly slower. Some
198		* loop bodies must be being joined or something */
199	0	if (i < 84 * 8) {
200	0	n = in23;
201	0	o = in45;
202	0	p = in67;
203	0	buf210 = in8_;
204	0	} else {
205	0	READ4_WITHXOR(n, o, p, buf210, in23, in45, in67, in8_, bitbuf210);
206	0	}
207
208	0	STORE4(m, n, o, p, bitbuf210);
209	0	STORE4(in12, in34, in56, in78, bitbuf300);
210	0	}
211
212		/* Second half of stores bubbled out */
213	0	_mm_store_si128(bitbuf144, buf144);
214	0	_mm_store_si128(bitbuf182, buf182);
215	0	_mm_store_si128(bitbuf210, buf210);
216
217		/* We also have to zero out the tail */
218	0	size_t left_to_z = len - (300*8 + i);
219	0	__m128i bitbuf_tail = (__m128i)(bitbuffer + 300 + i/8);
220	0	while (left_to_z >= 64) {
221	0	STORE4(z, z, z, z, bitbuf_tail);
222	0	left_to_z -= 64;
223	0	}
224
225	0	while (left_to_z >= 16) {
226	0	_mm_store_si128(bitbuf_tail++, z);
227	0	left_to_z -= 16;
228	0	}
229
230	0	uint8_t tail_bytes = (uint8_t)bitbuf_tail;
231	0	while (left_to_z--) {
232	0	*tail_bytes++ = 0;
233	0	}
234
235	0	ALIGNED_(16) uint64_t final[9] = {0};
236	0	__m128i next12, next34, next56;
237	0	next12 = z;
238	0	next34 = z;
239	0	next56 = z;
240
241	0	for (; (i + 72 < len); i += 32) {
242	0	__m128i in1in2, in3in4;
243	0	__m128i in1in2_, in3in4_;
244	0	__m128i ab1, ab2, ab3, ab4;
245	0	__m128i cd1, cd2, cd3, cd4;
246
247	0	READ_NEXT(input, i, in1in2, in3in4);
248	0	READ_NEXT(bitbuffer, i, in1in2_, in3in4_);
249
250	0	in1in2 = _mm_xor_si128(_mm_xor_si128(in1in2, in1in2_), next12);
251	0	in3in4 = _mm_xor_si128(in3in4, in3in4_);
252
253	0	NEXT_ROUND(in1in2, ab1, ab2, ab3, ab4);
254
255	0	__m128i a2_ = _mm_slli_si128(ab2, 8);
256	0	__m128i ab1_next34 = _mm_xor_si128(next34, ab1);
257	0	in3in4 = _mm_xor_si128(in3in4, ab1_next34);
258	0	in3in4 = _mm_xor_si128(a2_, in3in4);
259	0	NEXT_ROUND(in3in4, cd1, cd2, cd3, cd4);
260
261	0	__m128i b2c2 = _mm_alignr_epi8(cd2, ab2, 8);
262	0	__m128i a4_ = _mm_slli_si128(ab4, 8);
263	0	a4_ = _mm_xor_si128(b2c2, a4_);
264	0	next12 = _mm_xor_si128(ab3, a4_);
265	0	next12 = _mm_xor_si128(next12, cd1);
266
267	0	__m128i d2_ = _mm_srli_si128(cd2, 8);
268	0	__m128i b4c4 = _mm_alignr_epi8(cd4, ab4, 8);
269	0	next12 = _mm_xor_si128(next12, next56);
270	0	next34 = _mm_xor_si128(cd3, _mm_xor_si128(b4c4, d2_));
271	0	next56 = _mm_srli_si128(cd4, 8);
272	0	}
273
274	0	memcpy(final, input+(i / sizeof(uint64_t)), len-i);
275	0	__m128i final128 = (__m128i)final;
276	0	_mm_store_si128(final128, _mm_xor_si128(_mm_load_si128(final128), next12));
277	0	++final128;
278	0	_mm_store_si128(final128, _mm_xor_si128(_mm_load_si128(final128), next34));
279	0	++final128;
280	0	_mm_store_si128(final128, _mm_xor_si128(_mm_load_si128(final128), next56));
281
282		/* The buffer still owes the last words */
283	0	for (size_t j = 0; j < (len - i + sizeof(uint64_t) - 1) / sizeof(uint64_t); j++) {
284	0	final[j] ^= bitbuffer[i / sizeof(uint64_t) + j];
285	0	}
286
287		/* At most 72 bytes */
288	0	return ~crc32_copy_small(crc, NULL, (uint8_t*)final, len-i, 72, 0);
289	0	}
290
291	0	Z_INTERNAL uint32_t crc32_chorba_sse41(uint32_t crc, const uint8_t *buf, size_t len) {
292	0	uintptr_t align_diff = ALIGN_DIFF(buf, 16);
293		/* At most 87 bytes */
294	0	if (len <= align_diff + CHORBA_SMALL_THRESHOLD_64BIT)
295	0	return ~crc32_copy_small(~crc, NULL, buf, len, 15 + CHORBA_SMALL_THRESHOLD_64BIT, 0);
296
297	0	if (align_diff) {
298		/* At most 15 bytes */
299	0	crc = ~crc32_copy_small(~crc, NULL, buf, align_diff, 15, 0);
300	0	len -= align_diff;
301	0	buf += align_diff;
302	0	}
303	0	#ifdef CRC32_CHORBA_FALLBACK
304	0	if (len > CHORBA_LARGE_THRESHOLD)
305	0	return crc32_chorba_118960_nondestructive(crc, buf, len);
306	0	#endif
307	0	if (len > CHORBA_MEDIUM_LOWER_THRESHOLD && len <= CHORBA_MEDIUM_UPPER_THRESHOLD)
308	0	return crc32_chorba_32768_nondestructive_sse41(crc, buf, len);
309	0	return chorba_small_nondestructive_sse2(crc, buf, len);
310	0	}
311
312	0	Z_INTERNAL uint32_t crc32_copy_chorba_sse41(uint32_t crc, uint8_t dst, const uint8_t src, size_t len) {
313	0	crc = crc32_chorba_sse41(crc, src, len);
314	0	memcpy(dst, src, len);
315	0	return crc;
316	0	}
317		#endif