zng_adler32_z: 17| 3.51M|uint32_t Z_EXPORT PREFIX(adler32_z)(uint32_t adler, const unsigned char *buf, size_t len) { 18| 3.51M| if (buf == NULL) ------------------ | Branch (18:9): [True: 0, False: 3.51M] ------------------ 19| 0| return ADLER32_INITIAL_VALUE; ------------------ | | 65| 0|#define ADLER32_INITIAL_VALUE 1 /* initial adler-32 hash value */ ------------------ 20| 3.51M| return FUNCTABLE_CALL(adler32)(adler, buf, len); ------------------ | | 45| 3.51M|# define FUNCTABLE_CALL(name) functable.name ------------------ 21| 3.51M|} zng_adler32: 32| 624|uint32_t Z_EXPORT PREFIX(adler32)(uint32_t adler, const unsigned char *buf, uint32_t len) { 33| 624| if (buf == NULL) ------------------ | Branch (33:9): [True: 624, False: 0] ------------------ 34| 624| return ADLER32_INITIAL_VALUE; ------------------ | | 65| 624|#define ADLER32_INITIAL_VALUE 1 /* initial adler-32 hash value */ ------------------ 35| 0| return FUNCTABLE_CALL(adler32)(adler, buf, len); ------------------ | | 45| 0|# define FUNCTABLE_CALL(name) functable.name ------------------ 36| 624|} zng_adler32_combine: 74| 1.24k|uint32_t Z_EXPORT PREFIX4(adler32_combine)(uint32_t adler1, uint32_t adler2, z_off64_t len2) { 75| 1.24k| return adler32_combine_(adler1, adler2, len2); 76| 1.24k|} adler32.c:adler32_combine_: 40| 1.24k|static uint32_t adler32_combine_(uint32_t adler1, uint32_t adler2, z_off64_t len2) { 41| 1.24k| uint32_t sum1; 42| 1.24k| uint32_t sum2; 43| 1.24k| unsigned rem; 44| | 45| | /* for negative len, return invalid adler32 as a clue for debugging */ 46| 1.24k| if (len2 < 0) ------------------ | Branch (46:9): [True: 0, False: 1.24k] ------------------ 47| 0| return 0xffffffff; 48| | 49| | /* the derivation of this formula is left as an exercise for the reader */ 50| 1.24k| len2 %= BASE; /* assumes len2 >= 0 */ ------------------ | | 13| 1.24k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 51| 1.24k| rem = (unsigned)len2; 52| 1.24k| sum1 = adler1 & 0xffff; 53| 1.24k| sum2 = rem * sum1; 54| 1.24k| sum2 %= BASE; ------------------ | | 13| 1.24k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 55| 1.24k| sum1 += (adler2 & 0xffff) + BASE - 1; ------------------ | | 13| 1.24k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 56| 1.24k| sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; ------------------ | | 13| 1.24k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 57| 1.24k| if (sum1 >= BASE) sum1 -= BASE; ------------------ | | 13| 1.24k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ if (sum1 >= BASE) sum1 -= BASE; ------------------ | | 13| 1.24k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ | Branch (57:9): [True: 1.24k, False: 2] ------------------ 58| 1.24k| if (sum1 >= BASE) sum1 -= BASE; ------------------ | | 13| 1.24k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ if (sum1 >= BASE) sum1 -= BASE; ------------------ | | 13| 260|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ | Branch (58:9): [True: 260, False: 988] ------------------ 59| 1.24k| if (sum2 >= (BASE << 1)) sum2 -= (BASE << 1); ------------------ | | 13| 1.24k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ if (sum2 >= (BASE << 1)) sum2 -= (BASE << 1); ------------------ | | 13| 736|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ | Branch (59:9): [True: 736, False: 512] ------------------ 60| 1.24k| if (sum2 >= BASE) sum2 -= BASE; ------------------ | | 13| 1.24k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ if (sum2 >= BASE) sum2 -= BASE; ------------------ | | 13| 694|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ | Branch (60:9): [True: 694, False: 554] ------------------ 61| 1.24k| return sum1 | (sum2 << 16); 62| 1.24k|} adler32_ssse3.c:adler32_copy_tail: 134| 539k| uint32_t sum2, const int REBASE, const int MAX_LEN, const int COPY) { 135| 539k| if (len) { ------------------ | Branch (135:9): [True: 449k, False: 90.0k] ------------------ 136| 449k| Z_UNUSED(MAX_LEN); ------------------ | | 154| 449k|#define Z_UNUSED(var) (void)(var) ------------------ 137| | /* Process using packed 64-bit arithmetic when source is aligned */ 138| 554k| while (len >= 8 && ((uintptr_t)buf & 7) == 0) { ------------------ | Branch (138:16): [True: 432k, False: 122k] | Branch (138:28): [True: 105k, False: 327k] ------------------ 139| 105k| size_t chunk = MIN(ALIGN_DOWN(len, (size_t)8), (size_t)ADLER32_SWAR_MAX_BYTES); ------------------ | | 148| 105k|#define MIN(a, b) ((a) > (b) ? (b) : (a)) | | ------------------ | | | Branch (148:20): [True: 0, False: 105k] | | ------------------ ------------------ 140| 105k| adler32_swar(&adler, dst, buf, chunk, &sum2, COPY); 141| 105k| buf += chunk; 142| 105k| if (COPY) ------------------ | Branch (142:17): [True: 0, False: 105k] ------------------ 143| 0| dst += chunk; 144| 105k| len -= chunk; 145| 105k| } 146| | /* DO4 loop avoids GCC x86 register pressure from hoisted DO8/DO16 loads. */ 147| 1.52M| while (len >= 4) { ------------------ | Branch (147:16): [True: 1.07M, False: 449k] ------------------ 148| 1.07M| if (COPY) { ------------------ | Branch (148:17): [True: 0, False: 1.07M] ------------------ 149| 0| memcpy(dst, buf, 4); 150| 0| dst += 4; 151| 0| } 152| 1.07M| len -= 4; 153| 1.07M| ADLER_DO4(adler, sum2, buf, 0); ------------------ | | 21| 1.07M|#define ADLER_DO4(sum1, sum2, buf, i) {ADLER_DO2(sum1, sum2, buf, i); ADLER_DO2(sum1, sum2, buf, i+2);} | | ------------------ | | | | 20| 1.07M|#define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 1.07M|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | | | #define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 1.07M|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | ------------------ | | #define ADLER_DO4(sum1, sum2, buf, i) {ADLER_DO2(sum1, sum2, buf, i); ADLER_DO2(sum1, sum2, buf, i+2);} | | ------------------ | | | | 20| 1.07M|#define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 1.07M|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | | | #define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 1.07M|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | ------------------ ------------------ 154| 1.07M| buf += 4; 155| 1.07M| } 156| 449k| if (len & 2) { ------------------ | Branch (156:13): [True: 369k, False: 79.5k] ------------------ 157| 369k| if (COPY) { ------------------ | Branch (157:17): [True: 0, False: 369k] ------------------ 158| 0| memcpy(dst, buf, 2); 159| 0| dst += 2; 160| 0| } 161| 369k| ADLER_DO2(adler, sum2, buf, 0); ------------------ | | 20| 369k|#define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | ------------------ | | | | 19| 369k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | ------------------ | | #define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | ------------------ | | | | 19| 369k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | ------------------ ------------------ 162| 369k| buf += 2; 163| 369k| } 164| 449k| if (len & 1) { ------------------ | Branch (164:13): [True: 132k, False: 316k] ------------------ 165| 132k| if (COPY) ------------------ | Branch (165:17): [True: 0, False: 132k] ------------------ 166| 0| *dst = *buf; 167| 132k| ADLER_DO1(adler, sum2, buf, 0); ------------------ | | 19| 132k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} ------------------ 168| 132k| } 169| 449k| } 170| 539k| if (REBASE) { ------------------ | Branch (170:9): [True: 449k, False: 90.0k] ------------------ 171| 449k| adler %= BASE; ------------------ | | 13| 449k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 172| 449k| sum2 %= BASE; ------------------ | | 13| 449k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 173| 449k| } 174| | /* D = B * 65536 + A, see: https://en.wikipedia.org/wiki/Adler-32. */ 175| 539k| return adler | (sum2 << 16); 176| 539k|} adler32_ssse3.c:adler32_swar: 66| 105k| uint32_t *sum2, const int COPY) { 67| 105k| uint64_t sum_even = 0, sum_odd = 0, prefix_even = 0, prefix_odd = 0; 68| | 69| 105k| *sum2 += *adler * (uint32_t)len; 70| | 71| 105k| const uint64_t *src64 = (const uint64_t *)buf; 72| | 73| 105k| while (len >= 16) { ------------------ | Branch (73:12): [True: 0, False: 105k] ------------------ 74| 0| uint64_t v0 = src64[0]; 75| 0| uint64_t v1 = src64[1]; 76| 0| if (COPY) { ------------------ | Branch (76:13): [True: 0, False: 0] ------------------ 77| 0| memcpy(dst, &v0, sizeof(v0)); 78| 0| memcpy(dst + 8, &v1, sizeof(v1)); 79| 0| dst += 16; 80| 0| } 81| | 82| 0| prefix_even += sum_even; 83| 0| prefix_odd += sum_odd; 84| 0| sum_even += v0 & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 85| 0| sum_odd += (v0 >> 8) & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 86| | 87| 0| prefix_even += sum_even; 88| 0| prefix_odd += sum_odd; 89| 0| sum_even += v1 & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 90| 0| sum_odd += (v1 >> 8) & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 91| | 92| 0| src64 += 2; 93| 0| len -= 16; 94| 0| } 95| | 96| | /* Handle remaining 8 bytes if present */ 97| 105k| if (len >= 8) { ------------------ | Branch (97:9): [True: 105k, False: 0] ------------------ 98| 105k| uint64_t v = *src64; 99| 105k| if (COPY) ------------------ | Branch (99:13): [True: 0, False: 105k] ------------------ 100| 0| memcpy(dst, &v, sizeof(v)); 101| | 102| 105k| prefix_even += sum_even; 103| 105k| prefix_odd += sum_odd; 104| 105k| sum_even += v & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 105k|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 105| 105k| sum_odd += (v >> 8) & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 105k|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 106| 105k| } 107| | 108| | /* Horizontal sum of 4x16-bit lanes for s1 */ 109| 105k| *adler += (uint32_t)(((sum_even + sum_odd) * ADLER32_SWAR_HSUM) >> 48); ------------------ | | 63| 105k|#define ADLER32_SWAR_HSUM 0x1000100010001ULL ------------------ 110| | 111| | /* Widen prefix sums to 32-bit pairs and horizontal sum for s2 */ 112| 105k| uint64_t pe_lo = prefix_even & 0xFFFF0000FFFFULL; 113| 105k| uint64_t pe_hi = (prefix_even >> 16) & 0xFFFF0000FFFFULL; 114| 105k| uint64_t po_lo = prefix_odd & 0xFFFF0000FFFFULL; 115| 105k| uint64_t po_hi = (prefix_odd >> 16) & 0xFFFF0000FFFFULL; 116| | 117| 105k| *sum2 += (uint32_t)(((pe_lo + po_lo + pe_hi + po_hi) * 0x800000008ULL) >> 32); 118| | 119| | /* Positional weights [8,7,6,5,4,3,2,1] per 8-byte group for s2. 120| | * On big-endian the even mask captures odd-index memory bytes (b1,b3,b5,b7) 121| | * so HSUM (+1 per odd-index byte) must be applied to sum_even, not sum_odd. */ 122| 105k|#if BYTE_ORDER == LITTLE_ENDIAN 123| 105k| *sum2 += 2 * (uint32_t)((sum_even * 0x4000300020001ULL) >> 48) 124| 105k| + (uint32_t)((sum_odd * ADLER32_SWAR_HSUM) >> 48) ------------------ | | 63| 105k|#define ADLER32_SWAR_HSUM 0x1000100010001ULL ------------------ 125| 105k| + 2 * (uint32_t)((sum_odd * 0x3000200010000ULL) >> 48); 126| |#else 127| | *sum2 += 2 * (uint32_t)((sum_even * 0x0000100020003ULL) >> 48) 128| | + (uint32_t)((sum_even * ADLER32_SWAR_HSUM) >> 48) 129| | + 2 * (uint32_t)((sum_odd * 0x1000200030004ULL) >> 48); 130| |#endif 131| 105k|} adler32_avx2.c:adler32_copy_tail: 134| 2.91M| uint32_t sum2, const int REBASE, const int MAX_LEN, const int COPY) { 135| 2.91M| if (len) { ------------------ | Branch (135:9): [True: 2.91M, False: 227] ------------------ 136| 2.91M| Z_UNUSED(MAX_LEN); ------------------ | | 154| 2.91M|#define Z_UNUSED(var) (void)(var) ------------------ 137| | /* Process using packed 64-bit arithmetic when source is aligned */ 138| 2.92M| while (len >= 8 && ((uintptr_t)buf & 7) == 0) { ------------------ | Branch (138:16): [True: 51.2k, False: 2.87M] | Branch (138:28): [True: 13.2k, False: 38.0k] ------------------ 139| 13.2k| size_t chunk = MIN(ALIGN_DOWN(len, (size_t)8), (size_t)ADLER32_SWAR_MAX_BYTES); ------------------ | | 148| 13.2k|#define MIN(a, b) ((a) > (b) ? (b) : (a)) | | ------------------ | | | Branch (148:20): [True: 0, False: 13.2k] | | ------------------ ------------------ 140| 13.2k| adler32_swar(&adler, dst, buf, chunk, &sum2, COPY); 141| 13.2k| buf += chunk; 142| 13.2k| if (COPY) ------------------ | Branch (142:17): [True: 0, False: 13.2k] ------------------ 143| 0| dst += chunk; 144| 13.2k| len -= chunk; 145| 13.2k| } 146| | /* DO4 loop avoids GCC x86 register pressure from hoisted DO8/DO16 loads. */ 147| 3.01M| while (len >= 4) { ------------------ | Branch (147:16): [True: 101k, False: 2.91M] ------------------ 148| 101k| if (COPY) { ------------------ | Branch (148:17): [True: 0, False: 101k] ------------------ 149| 0| memcpy(dst, buf, 4); 150| 0| dst += 4; 151| 0| } 152| 101k| len -= 4; 153| 101k| ADLER_DO4(adler, sum2, buf, 0); ------------------ | | 21| 101k|#define ADLER_DO4(sum1, sum2, buf, i) {ADLER_DO2(sum1, sum2, buf, i); ADLER_DO2(sum1, sum2, buf, i+2);} | | ------------------ | | | | 20| 101k|#define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 101k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | | | #define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 101k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | ------------------ | | #define ADLER_DO4(sum1, sum2, buf, i) {ADLER_DO2(sum1, sum2, buf, i); ADLER_DO2(sum1, sum2, buf, i+2);} | | ------------------ | | | | 20| 101k|#define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 101k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | | | #define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 101k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | ------------------ ------------------ 154| 101k| buf += 4; 155| 101k| } 156| 2.91M| if (len & 2) { ------------------ | Branch (156:13): [True: 1.09M, False: 1.81M] ------------------ 157| 1.09M| if (COPY) { ------------------ | Branch (157:17): [True: 0, False: 1.09M] ------------------ 158| 0| memcpy(dst, buf, 2); 159| 0| dst += 2; 160| 0| } 161| 1.09M| ADLER_DO2(adler, sum2, buf, 0); ------------------ | | 20| 1.09M|#define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | ------------------ | | | | 19| 1.09M|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | ------------------ | | #define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | ------------------ | | | | 19| 1.09M|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | ------------------ ------------------ 162| 1.09M| buf += 2; 163| 1.09M| } 164| 2.91M| if (len & 1) { ------------------ | Branch (164:13): [True: 1.83M, False: 1.07M] ------------------ 165| 1.83M| if (COPY) ------------------ | Branch (165:17): [True: 0, False: 1.83M] ------------------ 166| 0| *dst = *buf; 167| 1.83M| ADLER_DO1(adler, sum2, buf, 0); ------------------ | | 19| 1.83M|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} ------------------ 168| 1.83M| } 169| 2.91M| } 170| 2.91M| if (REBASE) { ------------------ | Branch (170:9): [True: 2.91M, False: 0] ------------------ 171| 2.91M| adler %= BASE; ------------------ | | 13| 2.91M|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 172| 2.91M| sum2 %= BASE; ------------------ | | 13| 2.91M|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 173| 2.91M| } 174| | /* D = B * 65536 + A, see: https://en.wikipedia.org/wiki/Adler-32. */ 175| 2.91M| return adler | (sum2 << 16); 176| 2.91M|} adler32_avx2.c:adler32_swar: 66| 13.2k| uint32_t *sum2, const int COPY) { 67| 13.2k| uint64_t sum_even = 0, sum_odd = 0, prefix_even = 0, prefix_odd = 0; 68| | 69| 13.2k| *sum2 += *adler * (uint32_t)len; 70| | 71| 13.2k| const uint64_t *src64 = (const uint64_t *)buf; 72| | 73| 13.2k| while (len >= 16) { ------------------ | Branch (73:12): [True: 0, False: 13.2k] ------------------ 74| 0| uint64_t v0 = src64[0]; 75| 0| uint64_t v1 = src64[1]; 76| 0| if (COPY) { ------------------ | Branch (76:13): [True: 0, False: 0] ------------------ 77| 0| memcpy(dst, &v0, sizeof(v0)); 78| 0| memcpy(dst + 8, &v1, sizeof(v1)); 79| 0| dst += 16; 80| 0| } 81| | 82| 0| prefix_even += sum_even; 83| 0| prefix_odd += sum_odd; 84| 0| sum_even += v0 & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 85| 0| sum_odd += (v0 >> 8) & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 86| | 87| 0| prefix_even += sum_even; 88| 0| prefix_odd += sum_odd; 89| 0| sum_even += v1 & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 90| 0| sum_odd += (v1 >> 8) & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 91| | 92| 0| src64 += 2; 93| 0| len -= 16; 94| 0| } 95| | 96| | /* Handle remaining 8 bytes if present */ 97| 13.2k| if (len >= 8) { ------------------ | Branch (97:9): [True: 13.2k, False: 0] ------------------ 98| 13.2k| uint64_t v = *src64; 99| 13.2k| if (COPY) ------------------ | Branch (99:13): [True: 0, False: 13.2k] ------------------ 100| 0| memcpy(dst, &v, sizeof(v)); 101| | 102| 13.2k| prefix_even += sum_even; 103| 13.2k| prefix_odd += sum_odd; 104| 13.2k| sum_even += v & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 13.2k|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 105| 13.2k| sum_odd += (v >> 8) & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 13.2k|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 106| 13.2k| } 107| | 108| | /* Horizontal sum of 4x16-bit lanes for s1 */ 109| 13.2k| *adler += (uint32_t)(((sum_even + sum_odd) * ADLER32_SWAR_HSUM) >> 48); ------------------ | | 63| 13.2k|#define ADLER32_SWAR_HSUM 0x1000100010001ULL ------------------ 110| | 111| | /* Widen prefix sums to 32-bit pairs and horizontal sum for s2 */ 112| 13.2k| uint64_t pe_lo = prefix_even & 0xFFFF0000FFFFULL; 113| 13.2k| uint64_t pe_hi = (prefix_even >> 16) & 0xFFFF0000FFFFULL; 114| 13.2k| uint64_t po_lo = prefix_odd & 0xFFFF0000FFFFULL; 115| 13.2k| uint64_t po_hi = (prefix_odd >> 16) & 0xFFFF0000FFFFULL; 116| | 117| 13.2k| *sum2 += (uint32_t)(((pe_lo + po_lo + pe_hi + po_hi) * 0x800000008ULL) >> 32); 118| | 119| | /* Positional weights [8,7,6,5,4,3,2,1] per 8-byte group for s2. 120| | * On big-endian the even mask captures odd-index memory bytes (b1,b3,b5,b7) 121| | * so HSUM (+1 per odd-index byte) must be applied to sum_even, not sum_odd. */ 122| 13.2k|#if BYTE_ORDER == LITTLE_ENDIAN 123| 13.2k| *sum2 += 2 * (uint32_t)((sum_even * 0x4000300020001ULL) >> 48) 124| 13.2k| + (uint32_t)((sum_odd * ADLER32_SWAR_HSUM) >> 48) ------------------ | | 63| 13.2k|#define ADLER32_SWAR_HSUM 0x1000100010001ULL ------------------ 125| 13.2k| + 2 * (uint32_t)((sum_odd * 0x3000200010000ULL) >> 48); 126| |#else 127| | *sum2 += 2 * (uint32_t)((sum_even * 0x0000100020003ULL) >> 48) 128| | + (uint32_t)((sum_even * ADLER32_SWAR_HSUM) >> 48) 129| | + 2 * (uint32_t)((sum_odd * 0x1000200030004ULL) >> 48); 130| |#endif 131| 13.2k|} adler32_avx2: 164| 3.51M|Z_INTERNAL uint32_t adler32_avx2(uint32_t adler, const uint8_t *src, size_t len) { 165| | return adler32_copy_impl(adler, NULL, src, len, 0); 166| 3.51M|} adler32_avx2.c:adler32_copy_impl: 21| 3.51M|Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len, const int COPY) { 22| 3.51M| uint32_t adler0, adler1; 23| 3.51M| adler1 = (adler >> 16) & 0xffff; 24| 3.51M| adler0 = adler & 0xffff; 25| | 26| 3.70M|rem_peel: 27| 3.70M| if (len < 16) { ------------------ | Branch (27:9): [True: 2.91M, False: 796k] ------------------ 28| 2.91M| return adler32_copy_tail(adler0, dst, src, len, adler1, 1, 15, COPY); 29| 2.91M| } else if (len < 32) { ------------------ | Branch (29:16): [True: 539k, False: 256k] ------------------ 30| 539k| if (COPY) { ------------------ | Branch (30:13): [True: 0, False: 539k] ------------------ 31| 0| return adler32_copy_sse42(adler, dst, src, len); 32| 539k| } else { 33| 539k| return adler32_ssse3(adler, src, len); 34| 539k| } 35| 539k| } 36| | 37| 256k| __m256i vs1, vs2, vs2_0; 38| | 39| 256k| const __m256i dot2v = _mm256_setr_epi8(64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 40| 256k| 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33); 41| 256k| const __m256i dot2v_0 = _mm256_setr_epi8(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 42| 256k| 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); 43| 256k| const __m256i dot3v = _mm256_set1_epi16(1); 44| 256k| const __m256i zero = _mm256_setzero_si256(); 45| | 46| 521k| while (len >= 32) { ------------------ | Branch (46:12): [True: 264k, False: 256k] ------------------ 47| 264k| vs1 = _mm256_zextsi128_si256(_mm_cvtsi32_si128(adler0)); 48| 264k| vs2 = _mm256_zextsi128_si256(_mm_cvtsi32_si128(adler1)); 49| 264k| __m256i vs1_0 = vs1; 50| 264k| __m256i vs3 = _mm256_setzero_si256(); 51| 264k| vs2_0 = vs3; 52| | 53| 264k| size_t k = ALIGN_DOWN(MIN(len, NMAX), 32); ------------------ | | 293| 529k| ((value) & ~((align) - 1)) | | ------------------ | | | Branch (293:7): [True: 7.92k, False: 256k] | | ------------------ ------------------ 54| 264k| len -= k; 55| | 56| 1.25M| while (k >= 64) { ------------------ | Branch (56:16): [True: 995k, False: 264k] ------------------ 57| 995k| __m256i vbuf = _mm256_loadu_si256((__m256i*)src); 58| 995k| __m256i vbuf_0 = _mm256_loadu_si256((__m256i*)(src + 32)); 59| 995k| src += 64; 60| 995k| k -= 64; 61| | 62| 995k| __m256i vs1_sad = _mm256_sad_epu8(vbuf, zero); 63| 995k| __m256i vs1_sad2 = _mm256_sad_epu8(vbuf_0, zero); 64| | 65| 995k| if (COPY) { ------------------ | Branch (65:17): [True: 0, False: 995k] ------------------ 66| 0| _mm256_storeu_si256((__m256i*)dst, vbuf); 67| 0| _mm256_storeu_si256((__m256i*)(dst + 32), vbuf_0); 68| 0| dst += 64; 69| 0| } 70| | 71| 995k| vs1 = _mm256_add_epi32(vs1, vs1_sad); 72| 995k| vs3 = _mm256_add_epi32(vs3, vs1_0); 73| 995k| __m256i v_short_sum2 = _mm256_maddubs_epi16(vbuf, dot2v); // sum 32 uint8s to 16 shorts 74| 995k| __m256i v_short_sum2_0 = _mm256_maddubs_epi16(vbuf_0, dot2v_0); // sum 32 uint8s to 16 shorts 75| 995k| __m256i vsum2 = _mm256_madd_epi16(v_short_sum2, dot3v); // sum 16 shorts to 8 uint32s 76| 995k| __m256i vsum2_0 = _mm256_madd_epi16(v_short_sum2_0, dot3v); // sum 16 shorts to 8 uint32s 77| 995k| vs1 = _mm256_add_epi32(vs1_sad2, vs1); 78| 995k| vs2 = _mm256_add_epi32(vsum2, vs2); 79| 995k| vs2_0 = _mm256_add_epi32(vsum2_0, vs2_0); 80| 995k| vs1_0 = vs1; 81| 995k| } 82| | 83| 264k| vs2 = _mm256_add_epi32(vs2_0, vs2); 84| 264k| vs3 = _mm256_slli_epi32(vs3, 6); 85| 264k| vs2 = _mm256_add_epi32(vs3, vs2); 86| 264k| vs3 = _mm256_setzero_si256(); 87| | 88| 453k| while (k >= 32) { ------------------ | Branch (88:16): [True: 189k, False: 264k] ------------------ 89| | /* 90| | vs1 = adler + sum(c[i]) 91| | vs2 = sum2 + 32 vs1 + sum( (32-i+1) c[i] ) 92| | */ 93| 189k| __m256i vbuf = _mm256_loadu_si256((__m256i*)src); 94| 189k| src += 32; 95| 189k| k -= 32; 96| | 97| 189k| __m256i vs1_sad = _mm256_sad_epu8(vbuf, zero); // Sum of abs diff, resulting in 2 x int32's 98| | 99| 189k| if (COPY) { ------------------ | Branch (99:17): [True: 0, False: 189k] ------------------ 100| 0| _mm256_storeu_si256((__m256i*)dst, vbuf); 101| 0| dst += 32; 102| 0| } 103| | 104| 189k| vs1 = _mm256_add_epi32(vs1, vs1_sad); 105| 189k| vs3 = _mm256_add_epi32(vs3, vs1_0); 106| 189k| __m256i v_short_sum2 = _mm256_maddubs_epi16(vbuf, dot2v_0); // sum 32 uint8s to 16 shorts 107| 189k| __m256i vsum2 = _mm256_madd_epi16(v_short_sum2, dot3v); // sum 16 shorts to 8 uint32s 108| 189k| vs2 = _mm256_add_epi32(vsum2, vs2); 109| 189k| vs1_0 = vs1; 110| 189k| } 111| | 112| | /* Defer the multiplication with 32 to outside of the loop */ 113| 264k| vs3 = _mm256_slli_epi32(vs3, 5); 114| 264k| vs2 = _mm256_add_epi32(vs2, vs3); 115| | 116| | /* The compiler is generating the following sequence for this integer modulus 117| | * when done the scalar way, in GPRs: 118| | 119| | adler = (s1_unpack[0] % BASE) + (s1_unpack[1] % BASE) + (s1_unpack[2] % BASE) + (s1_unpack[3] % BASE) + 120| | (s1_unpack[4] % BASE) + (s1_unpack[5] % BASE) + (s1_unpack[6] % BASE) + (s1_unpack[7] % BASE); 121| | 122| | mov $0x80078071,%edi // move magic constant into 32 bit register %edi 123| | ... 124| | vmovd %xmm1,%esi // move vector lane 0 to 32 bit register %esi 125| | mov %rsi,%rax // zero-extend this value to 64 bit precision in %rax 126| | imul %rdi,%rsi // do a signed multiplication with magic constant and vector element 127| | shr $0x2f,%rsi // shift right by 47 128| | imul $0xfff1,%esi,%esi // do a signed multiplication with value truncated to 32 bits with 0xfff1 129| | sub %esi,%eax // subtract lower 32 bits of original vector value from modified one above 130| | ... 131| | // repeats for each element with vpextract instructions 132| | 133| | This is tricky with AVX2 for a number of reasons: 134| | 1.) There's no 64 bit multiplication instruction, but there is a sequence to get there 135| | 2.) There's ways to extend vectors to 64 bit precision, but no simple way to truncate 136| | back down to 32 bit precision later (there is in AVX512) 137| | 3.) Full width integer multiplications aren't cheap 138| | 139| | We can, however, do a relatively cheap sequence for horizontal sums. 140| | Then, we simply do the integer modulus on the resulting 64 bit GPR, on a scalar value. It was 141| | previously thought that casting to 64 bit precision was needed prior to the horizontal sum, but 142| | that is simply not the case, as NMAX is defined as the maximum number of scalar sums that can be 143| | performed on the maximum possible inputs before overflow 144| | */ 145| | 146| | 147| | /* In AVX2-land, this trip through GPRs will probably be unavoidable, as there's no cheap and easy 148| | * conversion from 64 bit integer to 32 bit (needed for the inexpensive modulus with a constant). 149| | * This casting to 32 bit is cheap through GPRs (just register aliasing). See above for exactly 150| | * what the compiler is doing to avoid integer divisions. */ 151| 264k| adler0 = partial_hsum256(vs1) % BASE; ------------------ | | 13| 264k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 152| 264k| adler1 = hsum256(vs2) % BASE; ------------------ | | 13| 264k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 153| 264k| } 154| | 155| 256k| adler = adler0 | (adler1 << 16); 156| | 157| 256k| if (len) { ------------------ | Branch (157:9): [True: 192k, False: 64.0k] ------------------ 158| 192k| goto rem_peel; 159| 192k| } 160| | 161| 64.0k| return adler; 162| 256k|} adler32_avx2.c:partial_hsum256: 20| 264k|static inline uint32_t partial_hsum256(__m256i x) { 21| | /* We need a permutation vector to extract every other integer. The 22| | * rest are going to be zeros */ 23| 264k| const __m256i perm_vec = _mm256_setr_epi32(0, 2, 4, 6, 1, 1, 1, 1); 24| 264k| __m256i non_zero = _mm256_permutevar8x32_epi32(x, perm_vec); 25| 264k| __m128i non_zero_sse = _mm256_castsi256_si128(non_zero); 26| 264k| __m128i sum2 = _mm_add_epi32(non_zero_sse,_mm_unpackhi_epi64(non_zero_sse, non_zero_sse)); 27| | __m128i sum3 = _mm_add_epi32(sum2, _mm_shuffle_epi32(sum2, 1)); 28| 264k| return (uint32_t)_mm_cvtsi128_si32(sum3); 29| 264k|} adler32_avx2.c:hsum256: 12| 264k|static inline uint32_t hsum256(__m256i x) { 13| 264k| __m128i sum1 = _mm_add_epi32(_mm256_extracti128_si256(x, 1), 14| 264k| _mm256_castsi256_si128(x)); 15| 264k| __m128i sum2 = _mm_add_epi32(sum1, _mm_unpackhi_epi64(sum1, sum1)); 16| | __m128i sum3 = _mm_add_epi32(sum2, _mm_shuffle_epi32(sum2, 1)); 17| 264k| return (uint32_t)_mm_cvtsi128_si32(sum3); 18| 264k|} adler32_ssse3: 17| 539k|Z_INTERNAL uint32_t adler32_ssse3(uint32_t adler, const uint8_t *buf, size_t len) { 18| | /* split Adler-32 into component sums */ 19| 539k| uint32_t sum2 = (adler >> 16) & 0xffff; 20| 539k| adler &= 0xffff; 21| | 22| | /* in case user likes doing a byte at a time, keep it fast */ 23| 539k| if (UNLIKELY(len == 1)) ------------------ | | 258| 539k|# define UNLIKELY(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (258:33): [True: 0, False: 539k] | | ------------------ ------------------ 24| 0| return adler32_copy_tail(adler, NULL, buf, 1, sum2, 1, 1, 0); 25| | 26| | /* in case short lengths are provided, keep it somewhat fast */ 27| 539k| if (UNLIKELY(len < 16)) ------------------ | | 258| 539k|# define UNLIKELY(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (258:33): [True: 0, False: 539k] | | ------------------ ------------------ 28| 0| return adler32_copy_tail(adler, NULL, buf, len, sum2, 1, 15, 0); 29| | 30| 539k| const __m128i dot2v = _mm_setr_epi8(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17); 31| 539k| const __m128i dot2v_0 = _mm_setr_epi8(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); 32| 539k| const __m128i dot3v = _mm_set1_epi16(1); 33| 539k| const __m128i zero = _mm_setzero_si128(); 34| | 35| 539k| __m128i vbuf, vs1_0, vs3, vs1, vs2, vs2_0, v_sad_sum1, v_short_sum2, v_short_sum2_0, 36| 539k| vbuf_0, v_sad_sum2, vsum2, vsum2_0; 37| | 38| | /* If our buffer is unaligned (likely), make the determination whether 39| | * or not there's enough of a buffer to consume to make the scalar, aligning 40| | * additions worthwhile or if it's worth it to just eat the cost of an unaligned 41| | * load. This is a pretty simple test, just test if len < 32 */ 42| 539k| size_t n = NMAX; ------------------ | | 14| 539k|#define NMAX 5552 ------------------ 43| 539k| size_t k = 0; 44| | 45| 539k| if (len < 32) { ------------------ | Branch (45:9): [True: 539k, False: 0] ------------------ 46| | /* Let's eat the cost of this one unaligned load so that 47| | * we don't completely skip over the vectorization. Doing 48| | * 16 bytes at a time unaligned is better than 16 + <= 15 49| | * sums */ 50| 539k| vbuf = _mm_loadu_si128((__m128i*)buf); 51| 539k| len -= 16; 52| 539k| buf += 16; 53| 539k| vs1 = _mm_cvtsi32_si128(adler); 54| 539k| vs2 = _mm_cvtsi32_si128(sum2); 55| 539k| vs3 = _mm_setzero_si128(); 56| 539k| vs1_0 = vs1; 57| 539k| goto unaligned_jmp; 58| 539k| } 59| | 60| 0| size_t align_diff = MIN(ALIGN_DIFF(buf, 16), len); ------------------ | | 148| 0|#define MIN(a, b) ((a) > (b) ? (b) : (a)) | | ------------------ | | | Branch (148:20): [True: 0, False: 0] | | ------------------ ------------------ 61| 0| if (align_diff) { ------------------ | Branch (61:9): [True: 0, False: 0] ------------------ 62| 0| adler32_copy_align(&adler, NULL, buf, align_diff, &sum2, 15, 0); 63| | 64| 0| buf += align_diff; 65| 0| len -= align_diff; 66| 0| n -= align_diff; 67| 0| } 68| | 69| 539k| while (len >= 16) { ------------------ | Branch (69:12): [True: 0, False: 539k] ------------------ 70| 0| vs1 = _mm_cvtsi32_si128(adler); 71| 0| vs2 = _mm_cvtsi32_si128(sum2); 72| 0| vs3 = _mm_setzero_si128(); 73| 0| vs2_0 = _mm_setzero_si128(); 74| 0| vs1_0 = vs1; 75| | 76| 0| k = ALIGN_DOWN(MIN(len, n), 16); ------------------ | | 293| 0| ((value) & ~((align) - 1)) | | ------------------ | | | Branch (293:7): [True: 0, False: 0] | | ------------------ ------------------ 77| 0| len -= k; 78| | 79| 0| while (k >= 32) { ------------------ | Branch (79:16): [True: 0, False: 0] ------------------ 80| | /* 81| | vs1 = adler + sum(c[i]) 82| | vs2 = sum2 + 16 vs1 + sum( (16-i+1) c[i] ) 83| | */ 84| 0| vbuf = _mm_load_si128((__m128i*)buf); 85| 0| vbuf_0 = _mm_load_si128((__m128i*)(buf + 16)); 86| 0| buf += 32; 87| 0| k -= 32; 88| | 89| 0| v_sad_sum1 = _mm_sad_epu8(vbuf, zero); 90| 0| v_sad_sum2 = _mm_sad_epu8(vbuf_0, zero); 91| 0| vs1 = _mm_add_epi32(v_sad_sum1, vs1); 92| 0| vs3 = _mm_add_epi32(vs1_0, vs3); 93| | 94| 0| vs1 = _mm_add_epi32(v_sad_sum2, vs1); 95| 0| v_short_sum2 = _mm_maddubs_epi16(vbuf, dot2v); 96| 0| vsum2 = _mm_madd_epi16(v_short_sum2, dot3v); 97| 0| v_short_sum2_0 = _mm_maddubs_epi16(vbuf_0, dot2v_0); 98| 0| vs2 = _mm_add_epi32(vsum2, vs2); 99| 0| vsum2_0 = _mm_madd_epi16(v_short_sum2_0, dot3v); 100| 0| vs2_0 = _mm_add_epi32(vsum2_0, vs2_0); 101| 0| vs1_0 = vs1; 102| 0| } 103| | 104| 0| vs2 = _mm_add_epi32(vs2_0, vs2); 105| 0| vs3 = _mm_slli_epi32(vs3, 5); 106| 0| vs2 = _mm_add_epi32(vs3, vs2); 107| 0| vs3 = _mm_setzero_si128(); 108| | 109| 539k| while (k >= 16) { ------------------ | Branch (109:16): [True: 0, False: 539k] ------------------ 110| | /* 111| | vs1 = adler + sum(c[i]) 112| | vs2 = sum2 + 16 vs1 + sum( (16-i+1) c[i] ) 113| | */ 114| 0| vbuf = _mm_load_si128((__m128i*)buf); 115| 0| buf += 16; 116| 0| k -= 16; 117| | 118| 539k|unaligned_jmp: 119| 539k| v_sad_sum1 = _mm_sad_epu8(vbuf, zero); 120| 539k| vs1 = _mm_add_epi32(v_sad_sum1, vs1); 121| 539k| vs3 = _mm_add_epi32(vs1_0, vs3); 122| 539k| v_short_sum2 = _mm_maddubs_epi16(vbuf, dot2v_0); 123| 539k| vsum2 = _mm_madd_epi16(v_short_sum2, dot3v); 124| 539k| vs2 = _mm_add_epi32(vsum2, vs2); 125| 539k| vs1_0 = vs1; 126| 539k| } 127| | 128| 539k| vs3 = _mm_slli_epi32(vs3, 4); 129| 539k| vs2 = _mm_add_epi32(vs2, vs3); 130| | 131| | /* We don't actually need to do a full horizontal sum, since psadbw is actually doing 132| | * a partial reduction sum implicitly and only summing to integers in vector positions 133| | * 0 and 2. This saves us some contention on the shuffle port(s) */ 134| 539k| adler = partial_hsum(vs1) % BASE; ------------------ | | 13| 539k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 135| 539k| sum2 = hsum(vs2) % BASE; ------------------ | | 13| 539k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 136| 539k| n = NMAX; ------------------ | | 14| 539k|#define NMAX 5552 ------------------ 137| 539k| } 138| | 139| | /* Process tail (len < 16). */ 140| 539k| return adler32_copy_tail(adler, NULL, buf, len, sum2, len != 0, 15, 0); 141| 0|} adler32_ssse3.c:partial_hsum: 14| 539k|static inline uint32_t partial_hsum(__m128i x) { 15| | __m128i second_int = _mm_srli_si128(x, 8); 16| 539k| __m128i sum = _mm_add_epi32(x, second_int); 17| 539k| return _mm_cvtsi128_si32(sum); 18| 539k|} adler32_ssse3.c:hsum: 20| 539k|static inline uint32_t hsum(__m128i x) { 21| 539k| __m128i sum1 = _mm_unpackhi_epi64(x, x); 22| 539k| __m128i sum2 = _mm_add_epi32(x, sum1); 23| | __m128i sum3 = _mm_shuffle_epi32(sum2, 0x01); 24| 539k| __m128i sum4 = _mm_add_epi32(sum2, sum3); 25| 539k| return _mm_cvtsi128_si32(sum4); 26| 539k|} crc32_pclmulqdq: 24| 7.02M|Z_INTERNAL uint32_t crc32_pclmulqdq(uint32_t crc, const uint8_t *buf, size_t len) { 25| | return crc32_copy_impl(crc, NULL, buf, len, 0); 26| 7.02M|} crc32_pclmulqdq.c:crc32_copy_impl: 539| 7.02M| const int COPY) { 540| 7.02M| size_t copy_len = len; 541| 7.02M| if (len >= 16) { ------------------ | Branch (541:9): [True: 1.33M, False: 5.69M] ------------------ 542| | /* Calculate 16-byte alignment offset */ 543| 1.33M| uintptr_t align_diff = ALIGN_DIFF(src, 16); ------------------ | | 285| 1.33M| (((uintptr_t)(align) - ((uintptr_t)(ptr) & ((align) - 1))) & ((align) - 1)) ------------------ 544| | 545| | /* If total length is less than (alignment bytes + 16), use the faster small method. 546| | * Handles both initially small buffers and cases where alignment would leave < 16 bytes */ 547| 1.33M| copy_len = len < align_diff + 16 ? len : align_diff; ------------------ | Branch (547:20): [True: 24.8k, False: 1.30M] ------------------ 548| 1.33M| } 549| | 550| 7.02M| if (copy_len > 0) { ------------------ | Branch (550:9): [True: 6.60M, False: 427k] ------------------ 551| 6.60M| crc = ~crc32_copy_small(~crc, dst, src, copy_len, 31, COPY); 552| 6.60M| src += copy_len; 553| 6.60M| len -= copy_len; 554| 6.60M| if (COPY) { ------------------ | Branch (554:13): [True: 0, False: 6.60M] ------------------ 555| 0| dst += copy_len; 556| 0| } 557| 6.60M| } 558| | 559| 7.02M| if (len == 0) ------------------ | Branch (559:9): [True: 5.71M, False: 1.30M] ------------------ 560| 5.71M| return crc; 561| | 562| 1.30M| const __m128i xmm_fold4 = _mm_set_epi32(0x00000001, 0x54442bd4, 0x00000001, 0xc6e41596); 563| | 564| 1.30M| __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3; 565| 1.30M| __m128i xmm_crc0 = _mm_cvtsi32_si128(0x9db42487); 566| 1.30M| __m128i xmm_crc1 = _mm_setzero_si128(); 567| 1.30M| __m128i xmm_crc2 = _mm_setzero_si128(); 568| 1.30M| __m128i xmm_crc3 = _mm_setzero_si128(); 569| | 570| 1.30M| if (crc != 0) { ------------------ | Branch (570:9): [True: 1.09M, False: 214k] ------------------ 571| | // Process the first 16 bytes and handle initial CRC 572| 1.09M| len -= 16; 573| 1.09M| xmm_t0 = _mm_load_si128((__m128i *)src); 574| 1.09M| src += 16; 575| | 576| 1.09M| fold_state_1(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4); 577| 1.09M| if (COPY) { ------------------ | Branch (577:13): [True: 0, False: 1.09M] ------------------ 578| 0| _mm_storeu_si128((__m128i *)dst, xmm_t0); 579| 0| dst += 16; 580| 0| } 581| 1.09M| xmm_crc3 = z128_xor3_epi64(xmm_crc3, xmm_t0, _mm_cvtsi32_si128(crc)); ------------------ | | 62| 1.09M|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 582| 1.09M| } 583| | 584| |#if defined(X86_VPCLMULQDQ) && defined(__AVX512F__) 585| | /* 512-bit VPCLMULQDQ path requires AVX-512F */ 586| | if (len >= 256) 587| | fold_block_16(&src, &dst, &len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, COPY); 588| |#elif defined(X86_VPCLMULQDQ) 589| | /* 256-bit VPCLMULQDQ path (doesn't require AVX-512F) */ 590| | if (len >= 128) 591| | fold_block_8(&src, &dst, &len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4, COPY); 592| |#else 593| | /* Chorba algorithm for PCLMULQDQ path (when VPCLMULQDQ not available) */ 594| 1.30M| if (len >= 512 + 64 + (16 * 8)) ------------------ | Branch (594:9): [True: 203, False: 1.30M] ------------------ 595| 203| fold_block_chorba(&src, &dst, &len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4, COPY); 596| 1.30M|#endif /* X86_VPCLMULQDQ */ 597| | 598| 2.55M| while (len >= 64) { ------------------ | Branch (598:12): [True: 1.25M, False: 1.30M] ------------------ 599| 1.25M| len -= 64; 600| 1.25M| xmm_t0 = _mm_load_si128((__m128i *)src); 601| 1.25M| xmm_t1 = _mm_load_si128((__m128i *)src + 1); 602| 1.25M| xmm_t2 = _mm_load_si128((__m128i *)src + 2); 603| 1.25M| xmm_t3 = _mm_load_si128((__m128i *)src + 3); 604| 1.25M| src += 64; 605| | 606| 1.25M| fold_state_4(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4); 607| 1.25M| if (COPY) { ------------------ | Branch (607:13): [True: 0, False: 1.25M] ------------------ 608| 0| _mm_storeu_si128((__m128i *)dst, xmm_t0); 609| 0| _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); 610| 0| _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); 611| 0| _mm_storeu_si128((__m128i *)dst + 3, xmm_t3); 612| 0| dst += 64; 613| 0| } 614| | 615| 1.25M| xmm_crc0 = _mm_xor_si128(xmm_crc0, xmm_t0); 616| 1.25M| xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t1); 617| 1.25M| xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t2); 618| 1.25M| xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t3); 619| 1.25M| } 620| | 621| | /* 622| | * len = num bytes left - 64 623| | */ 624| 1.30M| if (len >= 48) { ------------------ | Branch (624:9): [True: 79.9k, False: 1.22M] ------------------ 625| 79.9k| len -= 48; 626| | 627| 79.9k| xmm_t0 = _mm_load_si128((__m128i *)src); 628| 79.9k| xmm_t1 = _mm_load_si128((__m128i *)src + 1); 629| 79.9k| xmm_t2 = _mm_load_si128((__m128i *)src + 2); 630| 79.9k| src += 48; 631| | 632| 79.9k| fold_state_3(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4); 633| 79.9k| if (COPY) { ------------------ | Branch (633:13): [True: 0, False: 79.9k] ------------------ 634| 0| _mm_storeu_si128((__m128i *)dst, xmm_t0); 635| 0| _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); 636| 0| _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); 637| 0| dst += 48; 638| 0| } 639| | 640| 79.9k| xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t0); 641| 79.9k| xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t1); 642| 79.9k| xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t2); 643| 1.22M| } else if (len >= 32) { ------------------ | Branch (643:16): [True: 187k, False: 1.04M] ------------------ 644| 187k| len -= 32; 645| | 646| 187k| xmm_t0 = _mm_load_si128((__m128i *)src); 647| 187k| xmm_t1 = _mm_load_si128((__m128i *)src + 1); 648| 187k| src += 32; 649| | 650| 187k| fold_state_2(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4); 651| 187k| if (COPY) { ------------------ | Branch (651:13): [True: 0, False: 187k] ------------------ 652| 0| _mm_storeu_si128((__m128i *)dst, xmm_t0); 653| 0| _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); 654| 0| dst += 32; 655| 0| } 656| | 657| 187k| xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t0); 658| 187k| xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t1); 659| 1.04M| } else if (len >= 16) { ------------------ | Branch (659:16): [True: 224k, False: 816k] ------------------ 660| 224k| len -= 16; 661| 224k| xmm_t0 = _mm_load_si128((__m128i *)src); 662| 224k| src += 16; 663| | 664| 224k| fold_state_1(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4); 665| 224k| if (COPY) { ------------------ | Branch (665:13): [True: 0, False: 224k] ------------------ 666| 0| _mm_storeu_si128((__m128i *)dst, xmm_t0); 667| 0| dst += 16; 668| 0| } 669| | 670| 224k| xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t0); 671| 224k| } 672| | 673| 1.30M| const __m128i k12 = _mm_set_epi32(0x00000001, 0x751997d0, 0x00000000, 0xccaa009e); 674| 1.30M| const __m128i barrett_k = _mm_set_epi32(0x00000001, 0xdb710640, 0xb4e5b025, 0xf7011641); 675| | 676| | /* Fold 4x128-bit into a single 128-bit value using k1/k2 constants */ 677| 1.30M| __m128i x_low0 = _mm_clmulepi64_si128(xmm_crc0, k12, 0x01); 678| 1.30M| __m128i x_high0 = _mm_clmulepi64_si128(xmm_crc0, k12, 0x10); 679| 1.30M| xmm_crc1 = z128_xor3_epi64(xmm_crc1, x_low0, x_high0); ------------------ | | 62| 1.30M|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 680| | 681| 1.30M| __m128i x_low1 = _mm_clmulepi64_si128(xmm_crc1, k12, 0x01); 682| 1.30M| __m128i x_high1 = _mm_clmulepi64_si128(xmm_crc1, k12, 0x10); 683| 1.30M| xmm_crc2 = z128_xor3_epi64(xmm_crc2, x_low1, x_high1); ------------------ | | 62| 1.30M|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 684| | 685| 1.30M| __m128i x_low2 = _mm_clmulepi64_si128(xmm_crc2, k12, 0x01); 686| 1.30M| __m128i x_high2 = _mm_clmulepi64_si128(xmm_crc2, k12, 0x10); 687| 1.30M| xmm_crc3 = z128_xor3_epi64(xmm_crc3, x_low2, x_high2); ------------------ | | 62| 1.30M|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 688| | 689| | /* Fold remaining bytes into the 128-bit state */ 690| 1.30M| if (len) { ------------------ | Branch (690:9): [True: 881k, False: 426k] ------------------ 691| 881k| const __m128i xmm_mask3 = _mm_set1_epi32((int32_t)0x80808080); 692| 881k| const __m128i xmm_seq = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); 693| | 694| | /* Create masks to shift bytes for partial input */ 695| 881k| __m128i xmm_shl = _mm_add_epi8(xmm_seq, _mm_set1_epi8((char)len - 16)); 696| 881k| __m128i xmm_shr = _mm_xor_si128(xmm_shl, xmm_mask3); 697| | 698| | /* Shift out bytes from crc3 to make space for new data */ 699| 881k| __m128i xmm_overflow = _mm_shuffle_epi8(xmm_crc3, xmm_shl); 700| 881k| xmm_crc3 = _mm_shuffle_epi8(xmm_crc3, xmm_shr); 701| | 702| | /* Insert the partial input into crc3 */ 703| |#if defined(__AVX512BW__) && defined(__AVX512VL__) 704| | __mmask16 k = (1 << len) - 1; 705| | __m128i xmm_crc_part = _mm_maskz_loadu_epi8(k, src); 706| | if (COPY) { 707| | _mm_mask_storeu_epi8(dst, k, xmm_crc_part); 708| | } 709| |#else 710| 881k| __m128i xmm_crc_part = _mm_setzero_si128(); 711| 881k| memcpy(&xmm_crc_part, src, len); 712| 881k| if (COPY) { ------------------ | Branch (712:13): [True: 0, False: 881k] ------------------ 713| 0| memcpy(dst, src, len); 714| 0| } 715| 881k|#endif 716| 881k| __m128i part_aligned = _mm_shuffle_epi8(xmm_crc_part, xmm_shl); 717| 881k| xmm_crc3 = _mm_xor_si128(xmm_crc3, part_aligned); 718| | 719| | /* Fold the bytes that were shifted out back into crc3 */ 720| 881k| __m128i ovf_low = _mm_clmulepi64_si128(xmm_overflow, k12, 0x01); 721| 881k| __m128i ovf_high = _mm_clmulepi64_si128(xmm_overflow, k12, 0x10); 722| 881k| xmm_crc3 = z128_xor3_epi64(xmm_crc3, ovf_low, ovf_high); ------------------ | | 62| 881k|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 723| 881k| } 724| | 725| | /* Reduce 128-bits to 32-bits using two-stage Barrett reduction */ 726| 1.30M| __m128i x_tmp0 = _mm_clmulepi64_si128(xmm_crc3, barrett_k, 0x00); 727| 1.30M| __m128i x_tmp1 = _mm_clmulepi64_si128(x_tmp0, barrett_k, 0x10); 728| | 729| 1.30M| x_tmp1 = _mm_blend_epi16(x_tmp1, _mm_setzero_si128(), 0xcf); 730| 1.30M| x_tmp0 = _mm_xor_si128(x_tmp1, xmm_crc3); 731| | 732| 1.30M| __m128i x_res_a = _mm_clmulepi64_si128(x_tmp0, barrett_k, 0x01); 733| 1.30M| __m128i x_res_b = _mm_clmulepi64_si128(x_res_a, barrett_k, 0x10); 734| | 735| 1.30M| crc = ((uint32_t)_mm_extract_epi32(x_res_b, 2)); 736| | 737| 1.30M| return ~crc; 738| 7.02M|} crc32_pclmulqdq.c:fold_state_1: 66| 1.31M| const __m128i xmm_fold4) { 67| 1.31M| __m128i x_low = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); 68| 1.31M| __m128i x_high = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x10); 69| | 70| 1.31M| *xmm_crc0 = *xmm_crc1; 71| 1.31M| *xmm_crc1 = *xmm_crc2; 72| 1.31M| *xmm_crc2 = *xmm_crc3; 73| 1.31M| *xmm_crc3 = _mm_xor_si128(x_low, x_high); 74| 1.31M|} crc32_pclmulqdq.c:fold_block_chorba: 347| 203| const __m128i xmm_fold4, const int COPY) { 348| 203| __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3; 349| | 350| 203|#ifndef __AVX512VL__ 351| 203| if (!COPY) ------------------ | Branch (351:9): [True: 203, False: 0] ------------------ 352| 203| return; 353| 0|#endif 354| 0| while (*len >= 512 + 64 + (16 * 8)) { ------------------ | Branch (354:12): [True: 0, False: 0] ------------------ 355| 0| __m128i chorba8 = _mm_load_si128((__m128i *)*src); 356| 0| __m128i chorba7 = _mm_load_si128((__m128i *)*src + 1); 357| 0| __m128i chorba6 = _mm_load_si128((__m128i *)*src + 2); 358| 0| __m128i chorba5 = _mm_load_si128((__m128i *)*src + 3); 359| 0| __m128i chorba4 = _mm_load_si128((__m128i *)*src + 4); 360| 0| __m128i chorba3 = _mm_load_si128((__m128i *)*src + 5); 361| 0| __m128i chorba2 = _mm_load_si128((__m128i *)*src + 6); 362| 0| __m128i chorba1 = _mm_load_si128((__m128i *)*src + 7); 363| 0| if (COPY) { ------------------ | Branch (363:13): [True: 0, False: 0] ------------------ 364| 0| _mm_storeu_si128((__m128i *)*dst, chorba8); 365| 0| _mm_storeu_si128((__m128i *)*dst + 1, chorba7); 366| 0| _mm_storeu_si128((__m128i *)*dst + 2, chorba6); 367| 0| _mm_storeu_si128((__m128i *)*dst + 3, chorba5); 368| 0| _mm_storeu_si128((__m128i *)*dst + 4, chorba4); 369| 0| _mm_storeu_si128((__m128i *)*dst + 5, chorba3); 370| 0| _mm_storeu_si128((__m128i *)*dst + 6, chorba2); 371| 0| _mm_storeu_si128((__m128i *)*dst + 7, chorba1); 372| 0| *dst += 16 * 8; 373| 0| } 374| | 375| 0| chorba2 = _mm_xor_si128(chorba2, chorba8); 376| 0| chorba1 = _mm_xor_si128(chorba1, chorba7); 377| 0| *src += 16 * 8; 378| 0| *len -= 16 * 8; 379| | 380| 0| xmm_t0 = _mm_load_si128((__m128i *)*src); 381| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 1); 382| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 2); 383| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 3); 384| | 385| 0| fold_state_12(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3); 386| 0| if (COPY) { ------------------ | Branch (386:13): [True: 0, False: 0] ------------------ 387| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 388| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 389| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 390| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 391| 0| *dst += 64; 392| 0| } 393| | 394| 0| *xmm_crc0 = z128_xor3_epi64(xmm_t0, chorba6, *xmm_crc0); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 395| 0| *xmm_crc1 = _mm_xor_si128(z128_xor3_epi64(xmm_t1, chorba5, chorba8), *xmm_crc1); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 396| 0| *xmm_crc2 = z128_xor3_epi64(z128_xor3_epi64(xmm_t2, chorba4, chorba8), chorba7, *xmm_crc2); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 397| 0| *xmm_crc3 = z128_xor3_epi64(z128_xor3_epi64(xmm_t3, chorba3, chorba7), chorba6, *xmm_crc3); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 398| | 399| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 4); 400| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 5); 401| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 6); 402| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 7); 403| | 404| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 405| 0| if (COPY) { ------------------ | Branch (405:13): [True: 0, False: 0] ------------------ 406| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 407| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 408| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 409| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 410| 0| *dst += 64; 411| 0| } 412| | 413| 0| *xmm_crc0 = z128_xor3_epi64(z128_xor3_epi64(xmm_t0, chorba2, chorba6), chorba5, *xmm_crc0); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 414| 0| *xmm_crc1 = z128_xor3_epi64(z128_xor3_epi64(xmm_t1, chorba1, chorba4), chorba5, *xmm_crc1); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 415| 0| *xmm_crc2 = _mm_xor_si128(z128_xor3_epi64(xmm_t2, chorba3, chorba4), *xmm_crc2); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 416| 0| *xmm_crc3 = _mm_xor_si128(z128_xor3_epi64(xmm_t3, chorba2, chorba3), *xmm_crc3); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 417| | 418| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 8); 419| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 9); 420| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 10); 421| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 11); 422| | 423| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 424| 0| if (COPY) { ------------------ | Branch (424:13): [True: 0, False: 0] ------------------ 425| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 426| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 427| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 428| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 429| 0| *dst += 64; 430| 0| } 431| | 432| 0| *xmm_crc0 = z128_xor3_epi64(z128_xor3_epi64(xmm_t0, chorba1, chorba2), chorba8, *xmm_crc0); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 433| 0| *xmm_crc1 = _mm_xor_si128(z128_xor3_epi64(xmm_t1, chorba1, chorba7), *xmm_crc1); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 434| 0| *xmm_crc2 = z128_xor3_epi64(xmm_t2, chorba6, *xmm_crc2); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 435| 0| *xmm_crc3 = z128_xor3_epi64(xmm_t3, chorba5, *xmm_crc3); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 436| | 437| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 12); 438| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 13); 439| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 14); 440| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 15); 441| | 442| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 443| 0| if (COPY) { ------------------ | Branch (443:13): [True: 0, False: 0] ------------------ 444| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 445| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 446| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 447| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 448| 0| *dst += 64; 449| 0| } 450| | 451| 0| *xmm_crc0 = _mm_xor_si128(z128_xor3_epi64(xmm_t0, chorba4, chorba8), *xmm_crc0); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 452| 0| *xmm_crc1 = z128_xor3_epi64(z128_xor3_epi64(xmm_t1, chorba3, chorba8), chorba7, *xmm_crc1); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 453| 0| *xmm_crc2 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(xmm_t2, chorba2, chorba8), chorba7, chorba6), *xmm_crc2); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 454| 0| *xmm_crc3 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(xmm_t3, chorba1, chorba7), chorba6, chorba5), *xmm_crc3); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 455| | 456| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 16); 457| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 17); 458| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 18); 459| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 19); 460| | 461| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 462| 0| if (COPY) { ------------------ | Branch (462:13): [True: 0, False: 0] ------------------ 463| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 464| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 465| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 466| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 467| 0| *dst += 64; 468| 0| } 469| | 470| 0| *xmm_crc0 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(xmm_t0, chorba4, chorba8), chorba6, chorba5), *xmm_crc0); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 471| 0| *xmm_crc1 = z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t1, chorba3, chorba4), chorba8, chorba7), chorba5, *xmm_crc1); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 472| 0| *xmm_crc2 = z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t2, chorba2, chorba3), chorba4, chorba7), chorba6, *xmm_crc2); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 473| 0| *xmm_crc3 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t3, chorba1, chorba2), chorba3, chorba8), chorba6, chorba5), *xmm_crc3); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 474| | 475| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 20); 476| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 21); 477| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 22); 478| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 23); 479| | 480| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 481| 0| if (COPY) { ------------------ | Branch (481:13): [True: 0, False: 0] ------------------ 482| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 483| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 484| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 485| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 486| 0| *dst += 64; 487| 0| } 488| | 489| 0| *xmm_crc0 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t0, chorba1, chorba2), chorba4, chorba8), chorba7, chorba5), *xmm_crc0); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 490| 0| *xmm_crc1 = z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t1, chorba1, chorba3), chorba4, chorba7), chorba6, *xmm_crc1); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 491| 0| *xmm_crc2 = z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t2, chorba2, chorba3), chorba8, chorba6), chorba5, *xmm_crc2); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 492| 0| *xmm_crc3 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t3, chorba1, chorba2), chorba4, chorba8), chorba7, chorba5), *xmm_crc3); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 493| | 494| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 24); 495| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 25); 496| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 26); 497| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 27); 498| | 499| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 500| 0| if (COPY) { ------------------ | Branch (500:13): [True: 0, False: 0] ------------------ 501| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 502| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 503| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 504| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 505| 0| *dst += 64; 506| 0| } 507| | 508| 0| *xmm_crc0 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t0, chorba1, chorba3), chorba4, chorba8), chorba7, chorba6), *xmm_crc0); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 509| 0| *xmm_crc1 = z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t1, chorba2, chorba3), chorba7, chorba6), chorba5, *xmm_crc1); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 510| 0| *xmm_crc2 = z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t2, chorba1, chorba2), chorba4, chorba6), chorba5, *xmm_crc2); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 511| 0| *xmm_crc3 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(xmm_t3, chorba1, chorba3), chorba4, chorba5), *xmm_crc3); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 512| | 513| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 28); 514| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 29); 515| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 30); 516| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 31); 517| | 518| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 519| 0| if (COPY) { ------------------ | Branch (519:13): [True: 0, False: 0] ------------------ 520| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 521| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 522| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 523| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 524| 0| *dst += 64; 525| 0| } 526| | 527| 0| *xmm_crc0 = z128_xor3_epi64(z128_xor3_epi64(xmm_t0, chorba2, chorba3), chorba4, *xmm_crc0); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 528| 0| *xmm_crc1 = z128_xor3_epi64(z128_xor3_epi64(xmm_t1, chorba1, chorba2), chorba3, *xmm_crc1); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 529| 0| *xmm_crc2 = _mm_xor_si128(z128_xor3_epi64(xmm_t2, chorba1, chorba2), *xmm_crc2); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 530| 0| *xmm_crc3 = z128_xor3_epi64(xmm_t3, chorba1, *xmm_crc3); ------------------ | | 62| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 531| | 532| 0| *len -= 512; 533| 0| *src += 512; 534| 0| } 535| 0|} crc32_pclmulqdq.c:fold_state_4: 105| 1.25M| const __m128i xmm_fold4) { 106| 1.25M| __m128i x_low0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); 107| 1.25M| __m128i x_high0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x10); 108| 1.25M| __m128i x_low1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); 109| 1.25M| __m128i x_high1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10); 110| 1.25M| __m128i x_low2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); 111| 1.25M| __m128i x_high2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); 112| 1.25M| __m128i x_low3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x01); 113| 1.25M| __m128i x_high3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); 114| | 115| 1.25M| *xmm_crc0 = _mm_xor_si128(x_low0, x_high0); 116| 1.25M| *xmm_crc1 = _mm_xor_si128(x_low1, x_high1); 117| 1.25M| *xmm_crc2 = _mm_xor_si128(x_low2, x_high2); 118| 1.25M| *xmm_crc3 = _mm_xor_si128(x_low3, x_high3); 119| 1.25M|} crc32_pclmulqdq.c:fold_state_3: 90| 79.9k| const __m128i xmm_fold4) { 91| 79.9k| __m128i x_low0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); 92| 79.9k| __m128i x_high0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x10); 93| 79.9k| __m128i x_low1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); 94| 79.9k| __m128i x_high1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10); 95| 79.9k| __m128i x_low2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); 96| 79.9k| __m128i x_high2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); 97| | 98| 79.9k| *xmm_crc0 = *xmm_crc3; 99| 79.9k| *xmm_crc1 = _mm_xor_si128(x_low0, x_high0); 100| 79.9k| *xmm_crc2 = _mm_xor_si128(x_low1, x_high1); 101| 79.9k| *xmm_crc3 = _mm_xor_si128(x_low2, x_high2); 102| 79.9k|} crc32_pclmulqdq.c:fold_state_2: 77| 187k| const __m128i xmm_fold4) { 78| 187k| __m128i x_low0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); 79| 187k| __m128i x_high0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x10); 80| 187k| __m128i x_low1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); 81| 187k| __m128i x_high1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10); 82| | 83| 187k| *xmm_crc0 = *xmm_crc2; 84| 187k| *xmm_crc1 = *xmm_crc3; 85| 187k| *xmm_crc2 = _mm_xor_si128(x_low0, x_high0); 86| 187k| *xmm_crc3 = _mm_xor_si128(x_low1, x_high1); 87| 187k|} x86_check_features: 78| 2|void Z_INTERNAL x86_check_features(struct x86_cpu_features *features) { 79| 2| unsigned eax, ebx, ecx, edx; 80| 2| unsigned maxbasic; 81| | 82| 2| cpuid(0, &maxbasic, &ebx, &ecx, &edx); 83| 2| cpuid(1 /*CPU_PROCINFO_AND_FEATUREBITS*/, &eax, &ebx, &ecx, &edx); 84| | 85| 2| features->has_sse2 = edx & 0x4000000; 86| 2| features->has_ssse3 = ecx & 0x200; 87| 2| features->has_sse41 = ecx & 0x80000; 88| 2| features->has_sse42 = ecx & 0x100000; 89| 2| features->has_pclmulqdq = ecx & 0x2; 90| | 91| 2| if (ecx & 0x08000000) { ------------------ | Branch (91:9): [True: 2, False: 0] ------------------ 92| 2| uint64_t xfeature = xgetbv(0); 93| | 94| 2| features->has_os_save_ymm = ((xfeature & 0x06) == 0x06); 95| 2| features->has_os_save_zmm = ((xfeature & 0xe6) == 0xe6); 96| 2| } 97| | 98| 2| if (maxbasic >= 7) { ------------------ | Branch (98:9): [True: 2, False: 0] ------------------ 99| | // Reference: https://software.intel.com/sites/default/files/article/405250/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family.pdf 100| 2| cpuidex(7, 0, &eax, &ebx, &ecx, &edx); 101| | 102| | // check BMI2 bit 103| 2| features->has_bmi2 = ebx & 0x100; 104| | 105| | // check AVX2 bit if the OS supports saving YMM registers 106| 2| if (features->has_os_save_ymm) { ------------------ | Branch (106:13): [True: 2, False: 0] ------------------ 107| 2| features->has_avx2 = ebx & 0x20; 108| 2| features->has_vpclmulqdq = ecx & 0x400; 109| 2| } 110| | 111| | // check AVX512 bits if the OS supports saving ZMM registers 112| 2| if (features->has_os_save_zmm) { ------------------ | Branch (112:13): [True: 0, False: 2] ------------------ 113| 0| features->has_avx512f = ebx & 0x00010000; 114| 0| if (features->has_avx512f) { ------------------ | Branch (114:17): [True: 0, False: 0] ------------------ 115| | // According to the Intel Software Developer's Manual, AVX512F must be enabled too in order to enable 116| | // AVX512(DQ,BW,VL). 117| 0| features->has_avx512dq = ebx & 0x00020000; 118| 0| features->has_avx512bw = ebx & 0x40000000; 119| 0| features->has_avx512vl = ebx & 0x80000000; 120| 0| } 121| 0| features->has_avx512_common = features->has_avx512f && features->has_avx512dq && features->has_avx512bw \ ------------------ | Branch (121:43): [True: 0, False: 0] | Branch (121:68): [True: 0, False: 0] | Branch (121:94): [True: 0, False: 0] ------------------ 122| 0| && features->has_avx512vl && features->has_bmi2; ------------------ | Branch (122:18): [True: 0, False: 0] | Branch (122:44): [True: 0, False: 0] ------------------ 123| 0| features->has_avx512vnni = ecx & 0x800; 124| 0| } 125| 2| } 126| 2|} x86_features.c:cpuid: 29| 4|static inline void cpuid(int info, unsigned* eax, unsigned* ebx, unsigned* ecx, unsigned* edx) { 30| |#if defined(HAVE_CPUID_MS) 31| | unsigned int registers[4]; 32| | __cpuid((int *)registers, info); 33| | 34| | *eax = registers[0]; 35| | *ebx = registers[1]; 36| | *ecx = registers[2]; 37| | *edx = registers[3]; 38| |#elif defined(HAVE_CPUID_GNU) 39| | *eax = *ebx = *ecx = *edx = 0; 40| 4| __cpuid(info, *eax, *ebx, *ecx, *edx); 41| |#else 42| | /* When using this fallback, the faster SSE/AVX code is disabled */ 43| | *eax = *ebx = *ecx = *edx = 0; 44| |#endif 45| 4|} x86_features.c:xgetbv: 65| 2|static inline uint64_t xgetbv(unsigned int xcr) { 66| 2|#if defined(_MSC_VER) || defined(X86_HAVE_XSAVE_INTRIN) 67| 2| return _xgetbv(xcr); 68| |#elif defined(__GNUC__) 69| | uint32_t eax, edx; 70| | __asm__ ( ".byte 0x0f, 0x01, 0xd0" : "=a"(eax), "=d"(edx) : "c"(xcr)); 71| | return (uint64_t)(edx) << 32 | eax; 72| |#else 73| | /* When using this fallback, some of the faster code is disabled */ 74| | return 0; 75| |#endif 76| 2|} x86_features.c:cpuidex: 47| 2|static inline void cpuidex(int info, int subinfo, unsigned* eax, unsigned* ebx, unsigned* ecx, unsigned* edx) { 48| |#if defined(HAVE_CPUID_MS) 49| | unsigned int registers[4]; 50| | __cpuidex((int *)registers, info, subinfo); 51| | 52| | *eax = registers[0]; 53| | *ebx = registers[1]; 54| | *ecx = registers[2]; 55| | *edx = registers[3]; 56| |#elif defined(HAVE_CPUID_GNU) 57| | *eax = *ebx = *ecx = *edx = 0; 58| 2| __cpuid_count(info, subinfo, *eax, *ebx, *ecx, *edx); 59| |#else 60| | /* When using this fallback, the faster SSE/AVX code is disabled */ 61| | *eax = *ebx = *ecx = *edx = 0; 62| |#endif 63| 2|} cpu_check_features: 10| 2|Z_INTERNAL void cpu_check_features(struct cpu_features *features) { 11| 2| memset(features, 0, sizeof(struct cpu_features)); 12| 2|#if defined(X86_FEATURES) 13| 2| x86_check_features(&features->x86); 14| |#elif defined(ARM_FEATURES) 15| | arm_check_features(&features->arm); 16| |#elif defined(PPC_FEATURES) || defined(POWER_FEATURES) 17| | power_check_features(&features->power); 18| |#elif defined(S390_FEATURES) 19| | s390_check_features(&features->s390); 20| |#elif defined(RISCV_FEATURES) 21| | riscv_check_features(&features->riscv); 22| |#elif defined(LOONGARCH_FEATURES) 23| | loongarch_check_features(&features->loongarch); 24| |#endif 25| 2|} zng_crc32_z: 36| 7.02M|uint32_t Z_EXPORT PREFIX(crc32_z)(uint32_t crc, const unsigned char *buf, size_t len) { 37| 7.02M| if (buf == NULL) ------------------ | Branch (37:9): [True: 0, False: 7.02M] ------------------ 38| 0| return CRC32_INITIAL_VALUE; ------------------ | | 66| 0|#define CRC32_INITIAL_VALUE 0 /* initial crc-32 hash value */ ------------------ 39| 7.02M| return FUNCTABLE_CALL(crc32)(crc, buf, len); ------------------ | | 45| 7.02M|# define FUNCTABLE_CALL(name) functable.name ------------------ 40| 7.02M|} zng_crc32: 50| 624|uint32_t Z_EXPORT PREFIX(crc32)(uint32_t crc, const unsigned char *buf, uint32_t len) { 51| 624| if (buf == NULL) ------------------ | Branch (51:9): [True: 624, False: 0] ------------------ 52| 624| return CRC32_INITIAL_VALUE; ------------------ | | 66| 624|#define CRC32_INITIAL_VALUE 0 /* initial crc-32 hash value */ ------------------ 53| 0| return FUNCTABLE_CALL(crc32)(crc, buf, len); ------------------ | | 45| 0|# define FUNCTABLE_CALL(name) functable.name ------------------ 54| 624|} zng_crc32_combine: 45| 1.87k|uint32_t Z_EXPORT PREFIX4(crc32_combine)(uint32_t crc1, uint32_t crc2, z_off64_t len2) { 46| 1.87k| return crc32_combine_(crc1, crc2, len2); 47| 1.87k|} zng_crc32_combine_gen: 48| 1.24k|uint32_t Z_EXPORT PREFIX(crc32_combine_gen)(z_off64_t len2) { 49| 1.24k| return crc32_combine_gen_(len2); 50| 1.24k|} zng_crc32_combine_op: 51| 3.51M|uint32_t Z_EXPORT PREFIX(crc32_combine_op)(uint32_t crc1, uint32_t crc2, const uint32_t op) { 52| 3.51M| return crc32_combine_op_(crc1, crc2, op); 53| 3.51M|} crc32_braid_comb.c:crc32_combine_: 16| 1.87k|static uint32_t crc32_combine_(uint32_t crc1, uint32_t crc2, z_off64_t len2) { 17| 1.87k| return multmodp(x2nmodp(len2, 3), crc1) ^ crc2; 18| 1.87k|} crc32_braid_comb.c:crc32_combine_gen_: 19| 1.24k|static uint32_t crc32_combine_gen_(z_off64_t len2) { 20| 1.24k| return x2nmodp(len2, 3); 21| 1.24k|} crc32_braid_comb.c:crc32_combine_op_: 22| 3.51M|static uint32_t crc32_combine_op_(uint32_t crc1, uint32_t crc2, const uint32_t op) { 23| 3.51M| return multmodp(op, crc1) ^ crc2; 24| 3.51M|} crc32_braid_comb.c:multmodp: 8| 3.52M|static uint32_t multmodp(uint32_t a, uint32_t b) { 9| 3.52M| uint32_t m, p; 10| | 11| 3.52M| m = (uint32_t)1 << 31; 12| 3.52M| p = 0; 13| 55.5M| for (;;) { 14| 55.5M| if (a & m) { ------------------ | Branch (14:13): [True: 13.8M, False: 41.6M] ------------------ 15| 13.8M| p ^= b; 16| 13.8M| if ((a & (m - 1)) == 0) ------------------ | Branch (16:17): [True: 3.52M, False: 10.3M] ------------------ 17| 3.52M| break; 18| 13.8M| } 19| 52.0M| m >>= 1; 20| 52.0M| b = b & 1 ? (b >> 1) ^ POLY : b >> 1; ------------------ | | 34| 25.9M|#define POLY 0xedb88320 /* p(x) reflected, with x^32 implied */ ------------------ | Branch (20:13): [True: 25.9M, False: 26.0M] ------------------ 21| 52.0M| } 22| 3.52M| return p; 23| 3.52M|} crc32_braid_comb.c:x2nmodp: 29| 3.12k|static uint32_t x2nmodp(z_off64_t n, unsigned k) { 30| 3.12k| uint32_t p; 31| | 32| 3.12k| p = (uint32_t)1 << 31; /* x^0 == 1 */ 33| 28.1k| while (n) { ------------------ | Branch (33:12): [True: 25.0k, False: 3.12k] ------------------ 34| 25.0k| if (n & 1) ------------------ | Branch (34:13): [True: 12.9k, False: 12.0k] ------------------ 35| 12.9k| p = multmodp(x2n_table[k & 31], p); 36| 25.0k| n >>= 1; 37| 25.0k| k++; 38| 25.0k| } 39| 3.12k| return p; 40| 3.12k|} crc32_pclmulqdq.c:crc32_copy_small: 19| 6.60M| const int MAX_LEN, const int COPY) { 20| 6.60M| if (MAX_LEN >= 8) { ------------------ | Branch (20:9): [True: 6.60M, False: 0] ------------------ 21| 7.16M| while (len >= 8) { ------------------ | Branch (21:16): [True: 565k, False: 6.60M] ------------------ 22| 565k| if (COPY) { ------------------ | Branch (22:17): [True: 0, False: 565k] ------------------ 23| 0| memcpy(dst, buf, 8); 24| 0| dst += 8; 25| 0| } 26| 565k| CRC_DO8(crc, buf, 0); ------------------ | | 16| 565k|#define CRC_DO8(c, buf, i) {CRC_DO4(c, buf, i); CRC_DO4(c, buf, i+4);} | | ------------------ | | | | 15| 565k|#define CRC_DO4(c, buf, i) {CRC_DO2(c, buf, i); CRC_DO2(c, buf, i+2);} | | | | ------------------ | | | | | | 14| 565k|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 565k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 565k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | ------------------ | | | | #define CRC_DO4(c, buf, i) {CRC_DO2(c, buf, i); CRC_DO2(c, buf, i+2);} | | | | ------------------ | | | | | | 14| 565k|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 565k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 565k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | ------------------ | | ------------------ | | #define CRC_DO8(c, buf, i) {CRC_DO4(c, buf, i); CRC_DO4(c, buf, i+4);} | | ------------------ | | | | 15| 565k|#define CRC_DO4(c, buf, i) {CRC_DO2(c, buf, i); CRC_DO2(c, buf, i+2);} | | | | ------------------ | | | | | | 14| 565k|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 565k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 565k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | ------------------ | | | | #define CRC_DO4(c, buf, i) {CRC_DO2(c, buf, i); CRC_DO2(c, buf, i+2);} | | | | ------------------ | | | | | | 14| 565k|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 565k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 565k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | ------------------ | | ------------------ ------------------ 27| 565k| buf += 8; 28| 565k| len -= 8; 29| 565k| } 30| 6.60M| } 31| 6.60M| if (len & 4) { ------------------ | Branch (31:9): [True: 499k, False: 6.10M] ------------------ 32| 499k| if (COPY) { ------------------ | Branch (32:13): [True: 0, False: 499k] ------------------ 33| 0| memcpy(dst, buf, 4); 34| 0| dst += 4; 35| 0| } 36| 499k| CRC_DO4(crc, buf, 0); ------------------ | | 15| 499k|#define CRC_DO4(c, buf, i) {CRC_DO2(c, buf, i); CRC_DO2(c, buf, i+2);} | | ------------------ | | | | 14| 499k|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | ------------------ | | | | | | 13| 499k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | ------------------ | | | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | ------------------ | | | | | | 13| 499k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | ------------------ | | ------------------ | | #define CRC_DO4(c, buf, i) {CRC_DO2(c, buf, i); CRC_DO2(c, buf, i+2);} | | ------------------ | | | | 14| 499k|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | ------------------ | | | | | | 13| 499k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | ------------------ | | | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | ------------------ | | | | | | 13| 499k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | ------------------ | | ------------------ ------------------ 37| 499k| buf += 4; 38| 499k| } 39| 6.60M| if (len & 2) { ------------------ | Branch (39:9): [True: 2.54M, False: 4.05M] ------------------ 40| 2.54M| if (COPY) { ------------------ | Branch (40:13): [True: 0, False: 2.54M] ------------------ 41| 0| memcpy(dst, buf, 2); 42| 0| dst += 2; 43| 0| } 44| 2.54M| CRC_DO2(crc, buf, 0); ------------------ | | 14| 2.54M|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | ------------------ | | | | 13| 2.54M|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | ------------------ | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | ------------------ | | | | 13| 2.54M|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | ------------------ ------------------ 45| 2.54M| buf += 2; 46| 2.54M| } 47| 6.60M| if (len & 1) { ------------------ | Branch (47:9): [True: 3.77M, False: 2.82M] ------------------ 48| 3.77M| if (COPY) ------------------ | Branch (48:13): [True: 0, False: 3.77M] ------------------ 49| 0| *dst = *buf; 50| 3.77M| CRC_DO1(crc, buf, 0); ------------------ | | 13| 3.77M|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) ------------------ 51| 3.77M| } 52| | 53| 6.60M| return crc; 54| 6.60M|} functable.c:functable_constructor: 441| 2|static void __attribute__((constructor)) functable_constructor(void) { 442| 2| FUNCTABLE_INIT_ABORT; ------------------ | | 57| 2| if (init_functable()) { \ | | ------------------ | | | Branch (57:9): [True: 0, False: 2] | | ------------------ | | 58| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 59| 0| abort(); \ | | 60| 2| }; ------------------ 443| 2|} functable.c:init_functable: 70| 2|static int init_functable(void) { 71| 2| struct functable_s ft; 72| 2| struct cpu_features cf; 73| | 74| 2| memset(&ft, 0, sizeof(struct functable_s)); 75| 2| cpu_check_features(&cf); 76| 2| ft.force_init = &force_init_empty; 77| | 78| | // Only use necessary generic functions when no suitable simd versions are available. 79| 2|#ifdef ADLER32_FALLBACK 80| 2| ft.adler32 = &adler32_c; 81| 2| ft.adler32_copy = &adler32_copy_c; 82| 2|#endif 83| |#ifdef CHUNKSET_FALLBACK 84| | ft.chunkmemset_safe = &chunkmemset_safe_c; 85| | ft.inflate_fast = &inflate_fast_c; 86| |#endif 87| |#ifdef COMPARE256_FALLBACK 88| | ft.compare256 = &compare256_c; 89| | ft.longest_match = &longest_match_c; 90| | ft.longest_match_roll = &longest_match_roll_c; 91| |#endif 92| 2|#ifdef CRC32_BRAID_FALLBACK 93| 2| ft.crc32 = &crc32_braid; 94| 2| ft.crc32_copy = &crc32_copy_braid; 95| 2|#endif 96| |#ifdef SLIDE_HASH_FALLBACK 97| | ft.slide_hash = &slide_hash_c; 98| |#endif 99| | 100| | // Select arch-optimized functions 101| 2|#ifdef WITH_OPTIM 102| | 103| | // Chorba generic C fallback 104| 2|#ifdef CRC32_CHORBA_FALLBACK 105| 2| ft.crc32 = &crc32_chorba; 106| 2| ft.crc32_copy = &crc32_copy_chorba; 107| 2|#endif 108| | 109| | // X86 - SSE2 110| 2|#ifdef X86_SSE2 111| |# ifndef X86_SSE2_NATIVE 112| | if (cf.x86.has_sse2) 113| |# endif 114| 2| { 115| 2|# ifndef X86_AVX2_NATIVE 116| 2| ft.chunkmemset_safe = &chunkmemset_safe_sse2; 117| 2| ft.compare256 = &compare256_sse2; 118| 2| ft.inflate_fast = &inflate_fast_sse2; 119| 2| ft.longest_match = &longest_match_sse2; 120| 2| ft.longest_match_roll = &longest_match_roll_sse2; 121| 2| ft.slide_hash = &slide_hash_sse2; 122| 2|# endif 123| 2|# if defined(CRC32_CHORBA_SSE_FALLBACK) && !defined(X86_SSE41_NATIVE) && !defined(X86_PCLMULQDQ_NATIVE) 124| 2| ft.crc32 = &crc32_chorba_sse2; 125| 2| ft.crc32_copy = &crc32_copy_chorba_sse2; 126| 2|# endif 127| 2| } 128| 2|#endif 129| | // X86 - SSSE3 130| 2|#ifdef X86_SSSE3 131| 2|# ifndef X86_SSSE3_NATIVE 132| 2| if (cf.x86.has_ssse3) ------------------ | Branch (132:9): [True: 2, False: 0] ------------------ 133| 2|# endif 134| 2| { 135| 2| ft.adler32 = &adler32_ssse3; 136| 2| ft.adler32_copy = &adler32_copy_ssse3; 137| 2|# ifndef X86_AVX2_NATIVE 138| 2| ft.chunkmemset_safe = &chunkmemset_safe_ssse3; 139| 2| ft.inflate_fast = &inflate_fast_ssse3; 140| 2|# endif 141| 2| } 142| 2|#endif 143| | 144| | // X86 - SSE4.1 145| 2|#if defined(X86_SSE41) && !defined(X86_PCLMULQDQ_NATIVE) 146| 2|# ifndef X86_SSE41_NATIVE 147| 2| if (cf.x86.has_sse41) ------------------ | Branch (147:9): [True: 2, False: 0] ------------------ 148| 2|# endif 149| 2| { 150| 2|# ifdef CRC32_CHORBA_SSE_FALLBACK 151| 2| ft.crc32 = &crc32_chorba_sse41; 152| 2| ft.crc32_copy = &crc32_copy_chorba_sse41; 153| 2|# endif 154| 2| } 155| 2|#endif 156| | 157| | // X86 - SSE4.2 158| 2|#if defined(X86_SSE42) && !defined(X86_AVX512_NATIVE) 159| 2|# ifndef X86_SSE42_NATIVE 160| 2| if (cf.x86.has_sse42) ------------------ | Branch (160:9): [True: 2, False: 0] ------------------ 161| 2|# endif 162| 2| { 163| 2| ft.adler32_copy = &adler32_copy_sse42; 164| 2| } 165| 2|#endif 166| | // X86 - PCLMUL 167| 2|#if defined(X86_PCLMULQDQ_CRC) && !defined(X86_VPCLMULQDQ_NATIVE) 168| 2|# ifndef X86_PCLMULQDQ_NATIVE 169| 2| if (cf.x86.has_pclmulqdq) ------------------ | Branch (169:9): [True: 2, False: 0] ------------------ 170| 2|# endif 171| 2| { 172| 2| ft.crc32 = &crc32_pclmulqdq; 173| 2| ft.crc32_copy = &crc32_copy_pclmulqdq; 174| 2| } 175| 2|#endif 176| | // X86 - AVX2 177| 2|#ifdef X86_AVX2 178| | /* BMI2 support is all but implicit with AVX2 but let's sanity check this just in case. Enabling BMI2 allows for 179| | * flagless shifts, resulting in fewer flag stalls for the pipeline, and allows us to set destination registers 180| | * for the shift results as an operand, eliminating several register-register moves when the original value needs 181| | * to remain intact. They also allow for a count operand that isn't the CL register, avoiding contention there */ 182| 2|# ifndef X86_AVX2_NATIVE 183| 2| if (cf.x86.has_avx2 && cf.x86.has_bmi2) ------------------ | Branch (183:9): [True: 2, False: 0] | Branch (183:28): [True: 2, False: 0] ------------------ 184| 2|# endif 185| 2| { 186| 2|# ifndef X86_AVX512_NATIVE 187| 2| ft.adler32 = &adler32_avx2; 188| 2| ft.adler32_copy = &adler32_copy_avx2; 189| 2| ft.chunkmemset_safe = &chunkmemset_safe_avx2; 190| 2| ft.compare256 = &compare256_avx2; 191| 2| ft.inflate_fast = &inflate_fast_avx2; 192| 2| ft.longest_match = &longest_match_avx2; 193| 2| ft.longest_match_roll = &longest_match_roll_avx2; 194| 2|# endif 195| 2| ft.slide_hash = &slide_hash_avx2; 196| 2| } 197| 2|#endif 198| | // X86 - AVX512 (F,DQ,BW,Vl) 199| 2|#ifdef X86_AVX512 200| 2|# ifndef X86_AVX512_NATIVE 201| 2| if (cf.x86.has_avx512_common) ------------------ | Branch (201:9): [True: 0, False: 2] ------------------ 202| 0|# endif 203| 0| { 204| 0|# ifndef X86_AVX512VNNI_NATIVE 205| 0| ft.adler32 = &adler32_avx512; 206| 0| ft.adler32_copy = &adler32_copy_avx512; 207| 0|# endif 208| 0| ft.chunkmemset_safe = &chunkmemset_safe_avx512; 209| 0| ft.compare256 = &compare256_avx512; 210| 0| ft.inflate_fast = &inflate_fast_avx512; 211| 0| ft.longest_match = &longest_match_avx512; 212| 0| ft.longest_match_roll = &longest_match_roll_avx512; 213| 0| } 214| 2|#endif 215| 2|#ifdef X86_AVX512VNNI 216| 2|# ifndef X86_AVX512VNNI_NATIVE 217| 2| if (cf.x86.has_avx512vnni) ------------------ | Branch (217:9): [True: 0, False: 2] ------------------ 218| 0|# endif 219| 0| { 220| 0| ft.adler32 = &adler32_avx512_vnni; 221| 0| ft.adler32_copy = &adler32_copy_avx512_vnni; 222| 0| } 223| 2|#endif 224| | // X86 - VPCLMULQDQ (AVX2) 225| 2|#ifdef X86_VPCLMULQDQ_AVX2 226| 2|# ifndef X86_VPCLMULQDQ_AVX2_NATIVE 227| 2| if (cf.x86.has_pclmulqdq && cf.x86.has_avx2 && cf.x86.has_vpclmulqdq) ------------------ | Branch (227:9): [True: 2, False: 0] | Branch (227:33): [True: 2, False: 0] | Branch (227:52): [True: 0, False: 2] ------------------ 228| 0|# endif 229| 0| { 230| 0| ft.crc32 = &crc32_vpclmulqdq_avx2; 231| 0| ft.crc32_copy = &crc32_copy_vpclmulqdq_avx2; 232| 0| } 233| 2|#endif 234| | // X86 - VPCLMULQDQ (AVX-512) 235| 2|#ifdef X86_VPCLMULQDQ_AVX512 236| 2|# ifndef X86_VPCLMULQDQ_AVX512_NATIVE 237| 2| if (cf.x86.has_pclmulqdq && cf.x86.has_avx512_common && cf.x86.has_vpclmulqdq) ------------------ | Branch (237:9): [True: 2, False: 0] | Branch (237:33): [True: 0, False: 2] | Branch (237:61): [True: 0, False: 0] ------------------ 238| 0|# endif 239| 0| { 240| 0| ft.crc32 = &crc32_vpclmulqdq_avx512; 241| 0| ft.crc32_copy = &crc32_copy_vpclmulqdq_avx512; 242| 0| } 243| 2|#endif 244| | 245| | 246| | // ARM - SIMD 247| |#if defined(ARM_SIMD) && !defined(ARM_NEON_NATIVE) 248| |# ifndef ARM_SIMD_NATIVE 249| | if (cf.arm.has_simd) 250| |# endif 251| | { 252| | ft.slide_hash = &slide_hash_armv6; 253| | } 254| |#endif 255| | // ARM - NEON 256| |#ifdef ARM_NEON 257| |# ifndef ARM_NEON_NATIVE 258| | if (cf.arm.has_neon) 259| |# endif 260| | { 261| | ft.adler32 = &adler32_neon; 262| | ft.adler32_copy = &adler32_copy_neon; 263| | ft.chunkmemset_safe = &chunkmemset_safe_neon; 264| | ft.compare256 = &compare256_neon; 265| | ft.inflate_fast = &inflate_fast_neon; 266| | ft.longest_match = &longest_match_neon; 267| | ft.longest_match_roll = &longest_match_roll_neon; 268| | ft.slide_hash = &slide_hash_neon; 269| | } 270| |#endif 271| | // ARM - CRC32 272| |#if defined(ARM_CRC32) && !defined(ARM_PMULL_EOR3_NATIVE) 273| |# ifndef ARM_CRC32_NATIVE 274| | if (cf.arm.has_crc32) 275| |# endif 276| | { 277| | ft.crc32 = &crc32_armv8; 278| | ft.crc32_copy = &crc32_copy_armv8; 279| | } 280| |#endif 281| | // ARM - PMULL EOR3 282| |#ifdef ARM_PMULL_EOR3 283| |# ifndef ARM_PMULL_EOR3_NATIVE 284| | if (cf.arm.has_crc32 && cf.arm.has_pmull && cf.arm.has_eor3 && cf.arm.has_fast_pmull) 285| |# endif 286| | { 287| | ft.crc32 = &crc32_armv8_pmull_eor3; 288| | ft.crc32_copy = &crc32_copy_armv8_pmull_eor3; 289| | } 290| |#endif 291| | 292| | // Power - VMX 293| |#ifdef PPC_VMX 294| |# ifndef PPC_VMX_NATIVE 295| | if (cf.power.has_altivec) 296| |# endif 297| | { 298| | ft.adler32 = &adler32_vmx; 299| | ft.adler32_copy = &adler32_copy_vmx; 300| | ft.slide_hash = &slide_hash_vmx; 301| | } 302| |#endif 303| | // Power8 - VSX 304| |#ifdef POWER8_VSX 305| |# ifndef POWER8_VSX_NATIVE 306| | if (cf.power.has_arch_2_07) 307| |# endif 308| | { 309| | ft.adler32 = &adler32_power8; 310| | ft.adler32_copy = &adler32_copy_power8; 311| | ft.chunkmemset_safe = &chunkmemset_safe_power8; 312| | ft.inflate_fast = &inflate_fast_power8; 313| | ft.slide_hash = &slide_hash_power8; 314| | } 315| |#endif 316| |#ifdef POWER8_VSX_CRC32 317| |# ifndef POWER8_VSX_CRC32_NATIVE 318| | if (cf.power.has_arch_2_07) 319| |# endif 320| | { 321| | ft.crc32 = &crc32_power8; 322| | ft.crc32_copy = &crc32_copy_power8; 323| | } 324| |#endif 325| | // Power9 326| |#ifdef POWER9 327| |# ifndef POWER9_NATIVE 328| | if (cf.power.has_arch_3_00) 329| |# endif 330| | { 331| | ft.compare256 = &compare256_power9; 332| | ft.longest_match = &longest_match_power9; 333| | ft.longest_match_roll = &longest_match_roll_power9; 334| | } 335| |#endif 336| | 337| | 338| | // RISCV - RVV 339| |#ifdef RISCV_RVV 340| |# ifndef RISCV_RVV_NATIVE 341| | if (cf.riscv.has_rvv) 342| |# endif 343| | { 344| | ft.adler32 = &adler32_rvv; 345| | ft.adler32_copy = &adler32_copy_rvv; 346| | ft.chunkmemset_safe = &chunkmemset_safe_rvv; 347| | ft.compare256 = &compare256_rvv; 348| | ft.inflate_fast = &inflate_fast_rvv; 349| | ft.longest_match = &longest_match_rvv; 350| | ft.longest_match_roll = &longest_match_roll_rvv; 351| | ft.slide_hash = &slide_hash_rvv; 352| | } 353| |#endif 354| | 355| | // RISCV - ZBC 356| |#ifdef RISCV_CRC32_ZBC 357| |# ifndef RISCV_ZBC_NATIVE 358| | if (cf.riscv.has_zbc) 359| |# endif 360| | { 361| | ft.crc32 = &crc32_riscv64_zbc; 362| | ft.crc32_copy = &crc32_copy_riscv64_zbc; 363| | } 364| |#endif 365| | 366| | // S390 367| |#ifdef S390_VX 368| |# ifndef S390_VX_NATIVE 369| | if (cf.s390.has_vx) 370| |# endif 371| | { 372| | ft.crc32 = &crc32_s390_vx; 373| | ft.crc32_copy = &crc32_copy_s390_vx; 374| | ft.slide_hash = &slide_hash_vx; 375| | } 376| |#endif 377| | 378| | // LOONGARCH 379| |#ifdef LOONGARCH_CRC 380| |# ifndef LOONGARCH_CRC_NATIVE 381| | if (cf.loongarch.has_crc) 382| |# endif 383| | { 384| | ft.crc32 = &crc32_loongarch64; 385| | ft.crc32_copy = &crc32_copy_loongarch64; 386| | } 387| |#endif 388| |#if defined(LOONGARCH_LSX) && !defined(LOONGARCH_LASX_NATIVE) 389| |# ifndef LOONGARCH_LSX_NATIVE 390| | if (cf.loongarch.has_lsx) 391| |# endif 392| | { 393| | ft.adler32 = &adler32_lsx; 394| | ft.adler32_copy = &adler32_copy_lsx; 395| | ft.chunkmemset_safe = &chunkmemset_safe_lsx; 396| | ft.compare256 = &compare256_lsx; 397| | ft.inflate_fast = &inflate_fast_lsx; 398| | ft.longest_match = &longest_match_lsx; 399| | ft.longest_match_roll = &longest_match_roll_lsx; 400| | ft.slide_hash = &slide_hash_lsx; 401| | } 402| |#endif 403| |#ifdef LOONGARCH_LASX 404| |# ifndef LOONGARCH_LASX_NATIVE 405| | if (cf.loongarch.has_lasx) 406| |# endif 407| | { 408| | ft.adler32 = &adler32_lasx; 409| | ft.adler32_copy = &adler32_copy_lasx; 410| | ft.chunkmemset_safe = &chunkmemset_safe_lasx; 411| | ft.compare256 = &compare256_lasx; 412| | ft.inflate_fast = &inflate_fast_lasx; 413| | ft.longest_match = &longest_match_lasx; 414| | ft.longest_match_roll = &longest_match_roll_lasx; 415| | ft.slide_hash = &slide_hash_lasx; 416| | } 417| |#endif 418| | 419| 2|#endif // WITH_OPTIM 420| | 421| | // Assign function pointers individually for atomic operation 422| 2| FUNCTABLE_ASSIGN(ft, force_init); ------------------ | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) ------------------ 423| 2| FUNCTABLE_VERIFY_ASSIGN(ft, adler32); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 424| 2| FUNCTABLE_VERIFY_ASSIGN(ft, adler32_copy); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 425| 2| FUNCTABLE_VERIFY_ASSIGN(ft, chunkmemset_safe); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 426| 2| FUNCTABLE_VERIFY_ASSIGN(ft, compare256); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 427| 2| FUNCTABLE_VERIFY_ASSIGN(ft, crc32); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 428| 2| FUNCTABLE_VERIFY_ASSIGN(ft, crc32_copy); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 429| 2| FUNCTABLE_VERIFY_ASSIGN(ft, inflate_fast); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 430| 2| FUNCTABLE_VERIFY_ASSIGN(ft, longest_match); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 431| 2| FUNCTABLE_VERIFY_ASSIGN(ft, longest_match_roll); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 432| 2| FUNCTABLE_VERIFY_ASSIGN(ft, slide_hash); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 433| | 434| | // Memory barrier for weak memory order CPUs 435| 2| FUNCTABLE_BARRIER(); ------------------ | | 21| 2|# define FUNCTABLE_BARRIER() __atomic_thread_fence(__ATOMIC_SEQ_CST) ------------------ 436| | 437| 2| return Z_OK; ------------------ | | 180| 2|#define Z_OK 0 ------------------ 438| 2|} LLVMFuzzerTestOneInput: 9| 624|int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataLen) { 10| 624| uint32_t crc0 = PREFIX(crc32)(0L, NULL, 0); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 11| 624| uint32_t crc1 = crc0; 12| 624| uint32_t crc2 = crc0; 13| 624| uint32_t adler0 = PREFIX(adler32)(0L, NULL, 0); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 14| 624| uint32_t adler1 = adler0; 15| 624| uint32_t adler2 = adler0; 16| 624| uint32_t combine1, combine2; 17| | /* Checksum with a buffer of size equal to the first byte in the input. */ 18| 624| uint32_t buffSize = data[0]; 19| 624| uint32_t offset = 0; 20| 624| uint32_t op; 21| | 22| | /* Discard inputs larger than 1Mb. */ 23| 624| static size_t kMaxSize = 1024 * 1024; 24| 624| if (dataLen < 1 || dataLen > kMaxSize) ------------------ | Branch (24:9): [True: 0, False: 624] | Branch (24:24): [True: 0, False: 624] ------------------ 25| 0| return 0; 26| | 27| | /* Make sure the buffer has at least a byte. */ 28| 624| if (buffSize == 0) ------------------ | Branch (28:9): [True: 44, False: 580] ------------------ 29| 44| ++buffSize; 30| | 31| | /* CRC32 */ 32| 624| op = PREFIX(crc32_combine_gen)(buffSize); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 33| 3.51M| for (offset = 0; offset + buffSize <= dataLen; offset += buffSize) { ------------------ | Branch (33:22): [True: 3.51M, False: 624] ------------------ 34| 3.51M| uint32_t crc3 = PREFIX(crc32_z)(crc0, data + offset, buffSize); ------------------ | | 121| 3.51M|# define PREFIX(x) zng_ ## x ------------------ 35| 3.51M| uint32_t crc4 = PREFIX(crc32_combine_op)(crc1, crc3, op); ------------------ | | 121| 3.51M|# define PREFIX(x) zng_ ## x ------------------ 36| 3.51M| crc1 = PREFIX(crc32_z)(crc1, data + offset, buffSize); ------------------ | | 121| 3.51M|# define PREFIX(x) zng_ ## x ------------------ 37| 3.51M| assert(crc1 == crc4); 38| 3.51M| Z_UNUSED(crc1); ------------------ | | 154| 3.51M|#define Z_UNUSED(var) (void)(var) ------------------ 39| 3.51M| Z_UNUSED(crc4); ------------------ | | 154| 3.51M|#define Z_UNUSED(var) (void)(var) ------------------ 40| 3.51M| } 41| 624| crc1 = PREFIX(crc32_z)(crc1, data + offset, dataLen % buffSize); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 42| | 43| 624| crc2 = PREFIX(crc32_z)(crc2, data, dataLen); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 44| | 45| 624| assert(crc1 == crc2); 46| 624| Z_UNUSED(crc1); ------------------ | | 154| 624|#define Z_UNUSED(var) (void)(var) ------------------ 47| 624| Z_UNUSED(crc2); ------------------ | | 154| 624|#define Z_UNUSED(var) (void)(var) ------------------ 48| 624| combine1 = PREFIX(crc32_combine)(crc1, crc2, (z_off_t)dataLen); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 49| 624| combine2 = PREFIX(crc32_combine)(crc1, crc1, (z_off_t)dataLen); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 50| 624| assert(combine1 == combine2); 51| | 52| | /* Fast CRC32 combine. */ 53| 624| op = PREFIX(crc32_combine_gen)((z_off_t)dataLen); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 54| 624| combine1 = PREFIX(crc32_combine_op)(crc1, crc2, op); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 55| 624| combine2 = PREFIX(crc32_combine_op)(crc2, crc1, op); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 56| 624| assert(combine1 == combine2); 57| 624| combine1 = PREFIX(crc32_combine)(crc1, crc2, (z_off_t)dataLen); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 58| 624| combine2 = PREFIX(crc32_combine_op)(crc2, crc1, op); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 59| 624| assert(combine1 == combine2); 60| | 61| | /* Adler32 */ 62| 3.51M| for (offset = 0; offset + buffSize <= dataLen; offset += buffSize) ------------------ | Branch (62:22): [True: 3.51M, False: 624] ------------------ 63| 3.51M| adler1 = PREFIX(adler32_z)(adler1, data + offset, buffSize); ------------------ | | 121| 3.51M|# define PREFIX(x) zng_ ## x ------------------ 64| 624| adler1 = PREFIX(adler32_z)(adler1, data + offset, dataLen % buffSize); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 65| | 66| 624| adler2 = PREFIX(adler32_z)(adler2, data, dataLen); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 67| | 68| 624| assert(adler1 == adler2); 69| 624| Z_UNUSED(adler1); ------------------ | | 154| 624|#define Z_UNUSED(var) (void)(var) ------------------ 70| 624| Z_UNUSED(adler2); ------------------ | | 154| 624|#define Z_UNUSED(var) (void)(var) ------------------ 71| 624| combine1 = PREFIX(adler32_combine)(adler1, adler2, (z_off_t)dataLen); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 72| 624| combine2 = PREFIX(adler32_combine)(adler1, adler1, (z_off_t)dataLen); ------------------ | | 121| 624|# define PREFIX(x) zng_ ## x ------------------ 73| 624| assert(combine1 == combine2); 74| 624| Z_UNUSED(combine1); ------------------ | | 154| 624|#define Z_UNUSED(var) (void)(var) ------------------ 75| 624| Z_UNUSED(combine2); ------------------ | | 154| 624|#define Z_UNUSED(var) (void)(var) ------------------ 76| | 77| | /* This function must return 0. */ 78| 624| return 0; 79| 624|}