zng_adler32_z: 17| 1.56M|uint32_t Z_EXPORT PREFIX(adler32_z)(uint32_t adler, const unsigned char *buf, size_t len) { 18| 1.56M| if (buf == NULL) ------------------ | Branch (18:9): [True: 0, False: 1.56M] ------------------ 19| 0| return ADLER32_INITIAL_VALUE; ------------------ | | 70| 0|#define ADLER32_INITIAL_VALUE 1 /* initial adler-32 hash value */ ------------------ 20| 1.56M| return FUNCTABLE_CALL(adler32)(adler, buf, len); ------------------ | | 46| 1.56M|# define FUNCTABLE_CALL(name) functable.name ------------------ 21| 1.56M|} zng_adler32: 32| 604|uint32_t Z_EXPORT PREFIX(adler32)(uint32_t adler, const unsigned char *buf, uint32_t len) { 33| 604| if (buf == NULL) ------------------ | Branch (33:9): [True: 604, False: 0] ------------------ 34| 604| return ADLER32_INITIAL_VALUE; ------------------ | | 70| 604|#define ADLER32_INITIAL_VALUE 1 /* initial adler-32 hash value */ ------------------ 35| 0| return FUNCTABLE_CALL(adler32)(adler, buf, len); ------------------ | | 46| 0|# define FUNCTABLE_CALL(name) functable.name ------------------ 36| 604|} zng_adler32_combine: 74| 1.20k|uint32_t Z_EXPORT PREFIX4(adler32_combine)(uint32_t adler1, uint32_t adler2, z_off64_t len2) { 75| 1.20k| return adler32_combine_(adler1, adler2, len2); 76| 1.20k|} adler32.c:adler32_combine_: 40| 1.20k|static uint32_t adler32_combine_(uint32_t adler1, uint32_t adler2, z_off64_t len2) { 41| 1.20k| uint32_t sum1; 42| 1.20k| uint32_t sum2; 43| 1.20k| unsigned rem; 44| | 45| | /* for negative len, return invalid adler32 as a clue for debugging */ 46| 1.20k| if (len2 < 0) ------------------ | Branch (46:9): [True: 0, False: 1.20k] ------------------ 47| 0| return 0xffffffff; 48| | 49| | /* the derivation of this formula is left as an exercise for the reader */ 50| 1.20k| len2 %= BASE; /* assumes len2 >= 0 */ ------------------ | | 13| 1.20k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 51| 1.20k| rem = (unsigned)len2; 52| 1.20k| sum1 = adler1 & 0xffff; 53| 1.20k| sum2 = rem * sum1; 54| 1.20k| sum2 %= BASE; ------------------ | | 13| 1.20k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 55| 1.20k| sum1 += (adler2 & 0xffff) + BASE - 1; ------------------ | | 13| 1.20k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 56| 1.20k| sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; ------------------ | | 13| 1.20k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 57| 1.20k| if (sum1 >= BASE) sum1 -= BASE; ------------------ | | 13| 1.20k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ if (sum1 >= BASE) sum1 -= BASE; ------------------ | | 13| 1.20k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ | Branch (57:9): [True: 1.20k, False: 6] ------------------ 58| 1.20k| if (sum1 >= BASE) sum1 -= BASE; ------------------ | | 13| 1.20k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ if (sum1 >= BASE) sum1 -= BASE; ------------------ | | 13| 276|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ | Branch (58:9): [True: 276, False: 932] ------------------ 59| 1.20k| if (sum2 >= (BASE << 1)) sum2 -= (BASE << 1); ------------------ | | 13| 1.20k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ if (sum2 >= (BASE << 1)) sum2 -= (BASE << 1); ------------------ | | 13| 728|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ | Branch (59:9): [True: 728, False: 480] ------------------ 60| 1.20k| if (sum2 >= BASE) sum2 -= BASE; ------------------ | | 13| 1.20k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ if (sum2 >= BASE) sum2 -= BASE; ------------------ | | 13| 686|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ | Branch (60:9): [True: 686, False: 522] ------------------ 61| 1.20k| return sum1 | (sum2 << 16); 62| 1.20k|} adler32_ssse3.c:adler32_copy_tail: 134| 375k| uint32_t sum2, const int REBASE, const int MAX_LEN, const int COPY) { 135| 375k| if (len) { ------------------ | Branch (135:9): [True: 286k, False: 88.5k] ------------------ 136| 286k| Z_UNUSED(MAX_LEN); ------------------ | | 154| 286k|#define Z_UNUSED(var) (void)(var) ------------------ 137| | /* Process using packed 64-bit arithmetic when source is aligned */ 138| 328k| while (len >= 8 && ((uintptr_t)buf & 7) == 0) { ------------------ | Branch (138:16): [True: 242k, False: 85.5k] | Branch (138:28): [True: 41.2k, False: 201k] ------------------ 139| 41.2k| size_t chunk = MIN(ALIGN_DOWN(len, (size_t)8), (size_t)ADLER32_SWAR_MAX_BYTES); ------------------ | | 148| 41.2k|#define MIN(a, b) ((a) > (b) ? (b) : (a)) | | ------------------ | | | Branch (148:20): [True: 0, False: 41.2k] | | ------------------ ------------------ 140| 41.2k| adler32_swar(&adler, dst, buf, chunk, &sum2, COPY); 141| 41.2k| buf += chunk; 142| 41.2k| if (COPY) ------------------ | Branch (142:17): [True: 0, False: 41.2k] ------------------ 143| 0| dst += chunk; 144| 41.2k| len -= chunk; 145| 41.2k| } 146| | /* DO4 loop avoids GCC x86 register pressure from hoisted DO8/DO16 loads. */ 147| 915k| while (len >= 4) { ------------------ | Branch (147:16): [True: 628k, False: 286k] ------------------ 148| 628k| if (COPY) { ------------------ | Branch (148:17): [True: 0, False: 628k] ------------------ 149| 0| memcpy(dst, buf, 4); 150| 0| dst += 4; 151| 0| } 152| 628k| len -= 4; 153| 628k| ADLER_DO4(adler, sum2, buf, 0); ------------------ | | 21| 628k|#define ADLER_DO4(sum1, sum2, buf, i) {ADLER_DO2(sum1, sum2, buf, i); ADLER_DO2(sum1, sum2, buf, i+2);} | | ------------------ | | | | 20| 628k|#define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 628k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | | | #define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 628k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | ------------------ | | #define ADLER_DO4(sum1, sum2, buf, i) {ADLER_DO2(sum1, sum2, buf, i); ADLER_DO2(sum1, sum2, buf, i+2);} | | ------------------ | | | | 20| 628k|#define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 628k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | | | #define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 628k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | ------------------ ------------------ 154| 628k| buf += 4; 155| 628k| } 156| 286k| if (len & 2) { ------------------ | Branch (156:13): [True: 204k, False: 82.1k] ------------------ 157| 204k| if (COPY) { ------------------ | Branch (157:17): [True: 0, False: 204k] ------------------ 158| 0| memcpy(dst, buf, 2); 159| 0| dst += 2; 160| 0| } 161| 204k| ADLER_DO2(adler, sum2, buf, 0); ------------------ | | 20| 204k|#define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | ------------------ | | | | 19| 204k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | ------------------ | | #define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | ------------------ | | | | 19| 204k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | ------------------ ------------------ 162| 204k| buf += 2; 163| 204k| } 164| 286k| if (len & 1) { ------------------ | Branch (164:13): [True: 231k, False: 55.8k] ------------------ 165| 231k| if (COPY) ------------------ | Branch (165:17): [True: 0, False: 231k] ------------------ 166| 0| *dst = *buf; 167| 231k| ADLER_DO1(adler, sum2, buf, 0); ------------------ | | 19| 231k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} ------------------ 168| 231k| } 169| 286k| } 170| 375k| if (REBASE) { ------------------ | Branch (170:9): [True: 286k, False: 88.5k] ------------------ 171| 286k| adler %= BASE; ------------------ | | 13| 286k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 172| 286k| sum2 %= BASE; ------------------ | | 13| 286k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 173| 286k| } 174| | /* D = B * 65536 + A, see: https://en.wikipedia.org/wiki/Adler-32. */ 175| 375k| return adler | (sum2 << 16); 176| 375k|} adler32_ssse3.c:adler32_swar: 66| 41.2k| uint32_t *sum2, const int COPY) { 67| 41.2k| uint64_t sum_even = 0, sum_odd = 0, prefix_even = 0, prefix_odd = 0; 68| | 69| 41.2k| *sum2 += *adler * (uint32_t)len; 70| | 71| 41.2k| const uint64_t *src64 = (const uint64_t *)buf; 72| | 73| 41.2k| while (len >= 16) { ------------------ | Branch (73:12): [True: 0, False: 41.2k] ------------------ 74| 0| uint64_t v0 = src64[0]; 75| 0| uint64_t v1 = src64[1]; 76| 0| if (COPY) { ------------------ | Branch (76:13): [True: 0, False: 0] ------------------ 77| 0| memcpy(dst, &v0, sizeof(v0)); 78| 0| memcpy(dst + 8, &v1, sizeof(v1)); 79| 0| dst += 16; 80| 0| } 81| | 82| 0| prefix_even += sum_even; 83| 0| prefix_odd += sum_odd; 84| 0| sum_even += v0 & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 85| 0| sum_odd += (v0 >> 8) & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 86| | 87| 0| prefix_even += sum_even; 88| 0| prefix_odd += sum_odd; 89| 0| sum_even += v1 & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 90| 0| sum_odd += (v1 >> 8) & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 91| | 92| 0| src64 += 2; 93| 0| len -= 16; 94| 0| } 95| | 96| | /* Handle remaining 8 bytes if present */ 97| 41.2k| if (len >= 8) { ------------------ | Branch (97:9): [True: 41.2k, False: 0] ------------------ 98| 41.2k| uint64_t v = *src64; 99| 41.2k| if (COPY) ------------------ | Branch (99:13): [True: 0, False: 41.2k] ------------------ 100| 0| memcpy(dst, &v, sizeof(v)); 101| | 102| 41.2k| prefix_even += sum_even; 103| 41.2k| prefix_odd += sum_odd; 104| 41.2k| sum_even += v & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 41.2k|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 105| 41.2k| sum_odd += (v >> 8) & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 41.2k|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 106| 41.2k| } 107| | 108| | /* Horizontal sum of 4x16-bit lanes for s1 */ 109| 41.2k| *adler += (uint32_t)(((sum_even + sum_odd) * ADLER32_SWAR_HSUM) >> 48); ------------------ | | 63| 41.2k|#define ADLER32_SWAR_HSUM 0x1000100010001ULL ------------------ 110| | 111| | /* Widen prefix sums to 32-bit pairs and horizontal sum for s2 */ 112| 41.2k| uint64_t pe_lo = prefix_even & 0xFFFF0000FFFFULL; 113| 41.2k| uint64_t pe_hi = (prefix_even >> 16) & 0xFFFF0000FFFFULL; 114| 41.2k| uint64_t po_lo = prefix_odd & 0xFFFF0000FFFFULL; 115| 41.2k| uint64_t po_hi = (prefix_odd >> 16) & 0xFFFF0000FFFFULL; 116| | 117| 41.2k| *sum2 += (uint32_t)(((pe_lo + po_lo + pe_hi + po_hi) * 0x800000008ULL) >> 32); 118| | 119| | /* Positional weights [8,7,6,5,4,3,2,1] per 8-byte group for s2. 120| | * On big-endian the even mask captures odd-index memory bytes (b1,b3,b5,b7) 121| | * so HSUM (+1 per odd-index byte) must be applied to sum_even, not sum_odd. */ 122| 41.2k|#if BYTE_ORDER == LITTLE_ENDIAN 123| 41.2k| *sum2 += 2 * (uint32_t)((sum_even * 0x4000300020001ULL) >> 48) 124| 41.2k| + (uint32_t)((sum_odd * ADLER32_SWAR_HSUM) >> 48) ------------------ | | 63| 41.2k|#define ADLER32_SWAR_HSUM 0x1000100010001ULL ------------------ 125| 41.2k| + 2 * (uint32_t)((sum_odd * 0x3000200010000ULL) >> 48); 126| |#else 127| | *sum2 += 2 * (uint32_t)((sum_even * 0x0000100020003ULL) >> 48) 128| | + (uint32_t)((sum_even * ADLER32_SWAR_HSUM) >> 48) 129| | + 2 * (uint32_t)((sum_odd * 0x1000200030004ULL) >> 48); 130| |#endif 131| 41.2k|} adler32_avx2.c:adler32_copy_tail: 134| 1.14M| uint32_t sum2, const int REBASE, const int MAX_LEN, const int COPY) { 135| 1.14M| if (len) { ------------------ | Branch (135:9): [True: 1.14M, False: 222] ------------------ 136| 1.14M| Z_UNUSED(MAX_LEN); ------------------ | | 154| 1.14M|#define Z_UNUSED(var) (void)(var) ------------------ 137| | /* Process using packed 64-bit arithmetic when source is aligned */ 138| 1.15M| while (len >= 8 && ((uintptr_t)buf & 7) == 0) { ------------------ | Branch (138:16): [True: 116k, False: 1.04M] | Branch (138:28): [True: 16.5k, False: 99.9k] ------------------ 139| 16.5k| size_t chunk = MIN(ALIGN_DOWN(len, (size_t)8), (size_t)ADLER32_SWAR_MAX_BYTES); ------------------ | | 148| 16.5k|#define MIN(a, b) ((a) > (b) ? (b) : (a)) | | ------------------ | | | Branch (148:20): [True: 0, False: 16.5k] | | ------------------ ------------------ 140| 16.5k| adler32_swar(&adler, dst, buf, chunk, &sum2, COPY); 141| 16.5k| buf += chunk; 142| 16.5k| if (COPY) ------------------ | Branch (142:17): [True: 0, False: 16.5k] ------------------ 143| 0| dst += chunk; 144| 16.5k| len -= chunk; 145| 16.5k| } 146| | /* DO4 loop avoids GCC x86 register pressure from hoisted DO8/DO16 loads. */ 147| 1.42M| while (len >= 4) { ------------------ | Branch (147:16): [True: 278k, False: 1.14M] ------------------ 148| 278k| if (COPY) { ------------------ | Branch (148:17): [True: 0, False: 278k] ------------------ 149| 0| memcpy(dst, buf, 4); 150| 0| dst += 4; 151| 0| } 152| 278k| len -= 4; 153| 278k| ADLER_DO4(adler, sum2, buf, 0); ------------------ | | 21| 278k|#define ADLER_DO4(sum1, sum2, buf, i) {ADLER_DO2(sum1, sum2, buf, i); ADLER_DO2(sum1, sum2, buf, i+2);} | | ------------------ | | | | 20| 278k|#define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 278k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | | | #define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 278k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | ------------------ | | #define ADLER_DO4(sum1, sum2, buf, i) {ADLER_DO2(sum1, sum2, buf, i); ADLER_DO2(sum1, sum2, buf, i+2);} | | ------------------ | | | | 20| 278k|#define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 278k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | | | #define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | | | ------------------ | | | | | | 19| 278k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | | | ------------------ | | ------------------ ------------------ 154| 278k| buf += 4; 155| 278k| } 156| 1.14M| if (len & 2) { ------------------ | Branch (156:13): [True: 222k, False: 918k] ------------------ 157| 222k| if (COPY) { ------------------ | Branch (157:17): [True: 0, False: 222k] ------------------ 158| 0| memcpy(dst, buf, 2); 159| 0| dst += 2; 160| 0| } 161| 222k| ADLER_DO2(adler, sum2, buf, 0); ------------------ | | 20| 222k|#define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | ------------------ | | | | 19| 222k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | ------------------ | | #define ADLER_DO2(sum1, sum2, buf, i) {ADLER_DO1(sum1, sum2, buf, i); ADLER_DO1(sum1, sum2, buf, i+1);} | | ------------------ | | | | 19| 222k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} | | ------------------ ------------------ 162| 222k| buf += 2; 163| 222k| } 164| 1.14M| if (len & 1) { ------------------ | Branch (164:13): [True: 996k, False: 145k] ------------------ 165| 996k| if (COPY) ------------------ | Branch (165:17): [True: 0, False: 996k] ------------------ 166| 0| *dst = *buf; 167| 996k| ADLER_DO1(adler, sum2, buf, 0); ------------------ | | 19| 996k|#define ADLER_DO1(sum1, sum2, buf, i) {(sum1) += buf[(i)]; (sum2) += (sum1);} ------------------ 168| 996k| } 169| 1.14M| } 170| 1.14M| if (REBASE) { ------------------ | Branch (170:9): [True: 1.14M, False: 0] ------------------ 171| 1.14M| adler %= BASE; ------------------ | | 13| 1.14M|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 172| 1.14M| sum2 %= BASE; ------------------ | | 13| 1.14M|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 173| 1.14M| } 174| | /* D = B * 65536 + A, see: https://en.wikipedia.org/wiki/Adler-32. */ 175| 1.14M| return adler | (sum2 << 16); 176| 1.14M|} adler32_avx2.c:adler32_swar: 66| 16.5k| uint32_t *sum2, const int COPY) { 67| 16.5k| uint64_t sum_even = 0, sum_odd = 0, prefix_even = 0, prefix_odd = 0; 68| | 69| 16.5k| *sum2 += *adler * (uint32_t)len; 70| | 71| 16.5k| const uint64_t *src64 = (const uint64_t *)buf; 72| | 73| 16.5k| while (len >= 16) { ------------------ | Branch (73:12): [True: 0, False: 16.5k] ------------------ 74| 0| uint64_t v0 = src64[0]; 75| 0| uint64_t v1 = src64[1]; 76| 0| if (COPY) { ------------------ | Branch (76:13): [True: 0, False: 0] ------------------ 77| 0| memcpy(dst, &v0, sizeof(v0)); 78| 0| memcpy(dst + 8, &v1, sizeof(v1)); 79| 0| dst += 16; 80| 0| } 81| | 82| 0| prefix_even += sum_even; 83| 0| prefix_odd += sum_odd; 84| 0| sum_even += v0 & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 85| 0| sum_odd += (v0 >> 8) & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 86| | 87| 0| prefix_even += sum_even; 88| 0| prefix_odd += sum_odd; 89| 0| sum_even += v1 & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 90| 0| sum_odd += (v1 >> 8) & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 0|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 91| | 92| 0| src64 += 2; 93| 0| len -= 16; 94| 0| } 95| | 96| | /* Handle remaining 8 bytes if present */ 97| 16.5k| if (len >= 8) { ------------------ | Branch (97:9): [True: 16.5k, False: 0] ------------------ 98| 16.5k| uint64_t v = *src64; 99| 16.5k| if (COPY) ------------------ | Branch (99:13): [True: 0, False: 16.5k] ------------------ 100| 0| memcpy(dst, &v, sizeof(v)); 101| | 102| 16.5k| prefix_even += sum_even; 103| 16.5k| prefix_odd += sum_odd; 104| 16.5k| sum_even += v & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 16.5k|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 105| 16.5k| sum_odd += (v >> 8) & ADLER32_SWAR_EVEN_MASK; ------------------ | | 62| 16.5k|#define ADLER32_SWAR_EVEN_MASK 0x00FF00FF00FF00FFULL ------------------ 106| 16.5k| } 107| | 108| | /* Horizontal sum of 4x16-bit lanes for s1 */ 109| 16.5k| *adler += (uint32_t)(((sum_even + sum_odd) * ADLER32_SWAR_HSUM) >> 48); ------------------ | | 63| 16.5k|#define ADLER32_SWAR_HSUM 0x1000100010001ULL ------------------ 110| | 111| | /* Widen prefix sums to 32-bit pairs and horizontal sum for s2 */ 112| 16.5k| uint64_t pe_lo = prefix_even & 0xFFFF0000FFFFULL; 113| 16.5k| uint64_t pe_hi = (prefix_even >> 16) & 0xFFFF0000FFFFULL; 114| 16.5k| uint64_t po_lo = prefix_odd & 0xFFFF0000FFFFULL; 115| 16.5k| uint64_t po_hi = (prefix_odd >> 16) & 0xFFFF0000FFFFULL; 116| | 117| 16.5k| *sum2 += (uint32_t)(((pe_lo + po_lo + pe_hi + po_hi) * 0x800000008ULL) >> 32); 118| | 119| | /* Positional weights [8,7,6,5,4,3,2,1] per 8-byte group for s2. 120| | * On big-endian the even mask captures odd-index memory bytes (b1,b3,b5,b7) 121| | * so HSUM (+1 per odd-index byte) must be applied to sum_even, not sum_odd. */ 122| 16.5k|#if BYTE_ORDER == LITTLE_ENDIAN 123| 16.5k| *sum2 += 2 * (uint32_t)((sum_even * 0x4000300020001ULL) >> 48) 124| 16.5k| + (uint32_t)((sum_odd * ADLER32_SWAR_HSUM) >> 48) ------------------ | | 63| 16.5k|#define ADLER32_SWAR_HSUM 0x1000100010001ULL ------------------ 125| 16.5k| + 2 * (uint32_t)((sum_odd * 0x3000200010000ULL) >> 48); 126| |#else 127| | *sum2 += 2 * (uint32_t)((sum_even * 0x0000100020003ULL) >> 48) 128| | + (uint32_t)((sum_even * ADLER32_SWAR_HSUM) >> 48) 129| | + 2 * (uint32_t)((sum_odd * 0x1000200030004ULL) >> 48); 130| |#endif 131| 16.5k|} adler32_avx2: 164| 1.56M|Z_INTERNAL uint32_t adler32_avx2(uint32_t adler, const uint8_t *src, size_t len) { 165| | return adler32_copy_impl(adler, NULL, src, len, 0); 166| 1.56M|} adler32_avx2.c:adler32_copy_impl: 21| 1.56M|Z_FORCEINLINE static uint32_t adler32_copy_impl(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len, const int COPY) { 22| 1.56M| uint32_t adler0, adler1; 23| 1.56M| adler1 = (adler >> 16) & 0xffff; 24| 1.56M| adler0 = adler & 0xffff; 25| | 26| 1.84M|rem_peel: 27| 1.84M| if (len < 16) { ------------------ | Branch (27:9): [True: 1.14M, False: 701k] ------------------ 28| 1.14M| return adler32_copy_tail(adler0, dst, src, len, adler1, 1, 15, COPY); 29| 1.14M| } else if (len < 32) { ------------------ | Branch (29:16): [True: 375k, False: 325k] ------------------ 30| 375k| if (COPY) { ------------------ | Branch (30:13): [True: 0, False: 375k] ------------------ 31| 0| return adler32_copy_sse42(adler, dst, src, len); 32| 375k| } else { 33| 375k| return adler32_ssse3(adler, src, len); 34| 375k| } 35| 375k| } 36| | 37| 325k| __m256i vs1, vs2, vs2_0; 38| | 39| 325k| const __m256i dot2v = _mm256_setr_epi8(64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 40| 325k| 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33); 41| 325k| const __m256i dot2v_0 = _mm256_setr_epi8(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 42| 325k| 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); 43| 325k| const __m256i dot3v = _mm256_set1_epi16(1); 44| 325k| const __m256i zero = _mm256_setzero_si256(); 45| | 46| 659k| while (len >= 32) { ------------------ | Branch (46:12): [True: 333k, False: 325k] ------------------ 47| 333k| vs1 = _mm256_zextsi128_si256(_mm_cvtsi32_si128(adler0)); 48| 333k| vs2 = _mm256_zextsi128_si256(_mm_cvtsi32_si128(adler1)); 49| 333k| __m256i vs1_0 = vs1; 50| 333k| __m256i vs3 = _mm256_setzero_si256(); 51| 333k| vs2_0 = vs3; 52| | 53| 333k| size_t k = ALIGN_DOWN(MIN(len, NMAX), 32); ------------------ | | 293| 667k| ((value) & ~((align) - 1)) | | ------------------ | | | Branch (293:7): [True: 8.01k, False: 325k] | | ------------------ ------------------ 54| 333k| len -= k; 55| | 56| 1.40M| while (k >= 64) { ------------------ | Branch (56:16): [True: 1.06M, False: 333k] ------------------ 57| 1.06M| __m256i vbuf = _mm256_loadu_si256((__m256i*)src); 58| 1.06M| __m256i vbuf_0 = _mm256_loadu_si256((__m256i*)(src + 32)); 59| 1.06M| src += 64; 60| 1.06M| k -= 64; 61| | 62| 1.06M| __m256i vs1_sad = _mm256_sad_epu8(vbuf, zero); 63| 1.06M| __m256i vs1_sad2 = _mm256_sad_epu8(vbuf_0, zero); 64| | 65| 1.06M| if (COPY) { ------------------ | Branch (65:17): [True: 0, False: 1.06M] ------------------ 66| 0| _mm256_storeu_si256((__m256i*)dst, vbuf); 67| 0| _mm256_storeu_si256((__m256i*)(dst + 32), vbuf_0); 68| 0| dst += 64; 69| 0| } 70| | 71| 1.06M| vs1 = _mm256_add_epi32(vs1, vs1_sad); 72| 1.06M| vs3 = _mm256_add_epi32(vs3, vs1_0); 73| 1.06M| __m256i v_short_sum2 = _mm256_maddubs_epi16(vbuf, dot2v); // sum 32 uint8s to 16 shorts 74| 1.06M| __m256i v_short_sum2_0 = _mm256_maddubs_epi16(vbuf_0, dot2v_0); // sum 32 uint8s to 16 shorts 75| 1.06M| __m256i vsum2 = _mm256_madd_epi16(v_short_sum2, dot3v); // sum 16 shorts to 8 uint32s 76| 1.06M| __m256i vsum2_0 = _mm256_madd_epi16(v_short_sum2_0, dot3v); // sum 16 shorts to 8 uint32s 77| 1.06M| vs1 = _mm256_add_epi32(vs1_sad2, vs1); 78| 1.06M| vs2 = _mm256_add_epi32(vsum2, vs2); 79| 1.06M| vs2_0 = _mm256_add_epi32(vsum2_0, vs2_0); 80| 1.06M| vs1_0 = vs1; 81| 1.06M| } 82| | 83| 333k| vs2 = _mm256_add_epi32(vs2_0, vs2); 84| 333k| vs3 = _mm256_slli_epi32(vs3, 6); 85| 333k| vs2 = _mm256_add_epi32(vs3, vs2); 86| 333k| vs3 = _mm256_setzero_si256(); 87| | 88| 625k| while (k >= 32) { ------------------ | Branch (88:16): [True: 291k, False: 333k] ------------------ 89| | /* 90| | vs1 = adler + sum(c[i]) 91| | vs2 = sum2 + 32 vs1 + sum( (32-i+1) c[i] ) 92| | */ 93| 291k| __m256i vbuf = _mm256_loadu_si256((__m256i*)src); 94| 291k| src += 32; 95| 291k| k -= 32; 96| | 97| 291k| __m256i vs1_sad = _mm256_sad_epu8(vbuf, zero); // Sum of abs diff, resulting in 2 x int32's 98| | 99| 291k| if (COPY) { ------------------ | Branch (99:17): [True: 0, False: 291k] ------------------ 100| 0| _mm256_storeu_si256((__m256i*)dst, vbuf); 101| 0| dst += 32; 102| 0| } 103| | 104| 291k| vs1 = _mm256_add_epi32(vs1, vs1_sad); 105| 291k| vs3 = _mm256_add_epi32(vs3, vs1_0); 106| 291k| __m256i v_short_sum2 = _mm256_maddubs_epi16(vbuf, dot2v_0); // sum 32 uint8s to 16 shorts 107| 291k| __m256i vsum2 = _mm256_madd_epi16(v_short_sum2, dot3v); // sum 16 shorts to 8 uint32s 108| 291k| vs2 = _mm256_add_epi32(vsum2, vs2); 109| 291k| vs1_0 = vs1; 110| 291k| } 111| | 112| | /* Defer the multiplication with 32 to outside of the loop */ 113| 333k| vs3 = _mm256_slli_epi32(vs3, 5); 114| 333k| vs2 = _mm256_add_epi32(vs2, vs3); 115| | 116| | /* The compiler is generating the following sequence for this integer modulus 117| | * when done the scalar way, in GPRs: 118| | 119| | adler = (s1_unpack[0] % BASE) + (s1_unpack[1] % BASE) + (s1_unpack[2] % BASE) + (s1_unpack[3] % BASE) + 120| | (s1_unpack[4] % BASE) + (s1_unpack[5] % BASE) + (s1_unpack[6] % BASE) + (s1_unpack[7] % BASE); 121| | 122| | mov $0x80078071,%edi // move magic constant into 32 bit register %edi 123| | ... 124| | vmovd %xmm1,%esi // move vector lane 0 to 32 bit register %esi 125| | mov %rsi,%rax // zero-extend this value to 64 bit precision in %rax 126| | imul %rdi,%rsi // do a signed multiplication with magic constant and vector element 127| | shr $0x2f,%rsi // shift right by 47 128| | imul $0xfff1,%esi,%esi // do a signed multiplication with value truncated to 32 bits with 0xfff1 129| | sub %esi,%eax // subtract lower 32 bits of original vector value from modified one above 130| | ... 131| | // repeats for each element with vpextract instructions 132| | 133| | This is tricky with AVX2 for a number of reasons: 134| | 1.) There's no 64 bit multiplication instruction, but there is a sequence to get there 135| | 2.) There's ways to extend vectors to 64 bit precision, but no simple way to truncate 136| | back down to 32 bit precision later (there is in AVX512) 137| | 3.) Full width integer multiplications aren't cheap 138| | 139| | We can, however, do a relatively cheap sequence for horizontal sums. 140| | Then, we simply do the integer modulus on the resulting 64 bit GPR, on a scalar value. It was 141| | previously thought that casting to 64 bit precision was needed prior to the horizontal sum, but 142| | that is simply not the case, as NMAX is defined as the maximum number of scalar sums that can be 143| | performed on the maximum possible inputs before overflow 144| | */ 145| | 146| | 147| | /* In AVX2-land, this trip through GPRs will probably be unavoidable, as there's no cheap and easy 148| | * conversion from 64 bit integer to 32 bit (needed for the inexpensive modulus with a constant). 149| | * This casting to 32 bit is cheap through GPRs (just register aliasing). See above for exactly 150| | * what the compiler is doing to avoid integer divisions. */ 151| 333k| adler0 = partial_hsum256(vs1) % BASE; ------------------ | | 13| 333k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 152| 333k| adler1 = hsum256(vs2) % BASE; ------------------ | | 13| 333k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 153| 333k| } 154| | 155| 325k| adler = adler0 | (adler1 << 16); 156| | 157| 325k| if (len) { ------------------ | Branch (157:9): [True: 283k, False: 42.6k] ------------------ 158| 283k| goto rem_peel; 159| 283k| } 160| | 161| 42.6k| return adler; 162| 325k|} adler32_avx2.c:partial_hsum256: 20| 333k|static inline uint32_t partial_hsum256(__m256i x) { 21| | /* We need a permutation vector to extract every other integer. The 22| | * rest are going to be zeros */ 23| 333k| const __m256i perm_vec = _mm256_setr_epi32(0, 2, 4, 6, 1, 1, 1, 1); 24| 333k| __m256i non_zero = _mm256_permutevar8x32_epi32(x, perm_vec); 25| 333k| __m128i non_zero_sse = _mm256_castsi256_si128(non_zero); 26| 333k| __m128i sum2 = _mm_add_epi32(non_zero_sse,_mm_unpackhi_epi64(non_zero_sse, non_zero_sse)); 27| | __m128i sum3 = _mm_add_epi32(sum2, _mm_shuffle_epi32(sum2, 1)); 28| 333k| return (uint32_t)_mm_cvtsi128_si32(sum3); 29| 333k|} adler32_avx2.c:hsum256: 12| 333k|static inline uint32_t hsum256(__m256i x) { 13| 333k| __m128i sum1 = _mm_add_epi32(_mm256_extracti128_si256(x, 1), 14| 333k| _mm256_castsi256_si128(x)); 15| 333k| __m128i sum2 = _mm_add_epi32(sum1, _mm_unpackhi_epi64(sum1, sum1)); 16| | __m128i sum3 = _mm_add_epi32(sum2, _mm_shuffle_epi32(sum2, 1)); 17| 333k| return (uint32_t)_mm_cvtsi128_si32(sum3); 18| 333k|} adler32_ssse3: 17| 375k|Z_INTERNAL uint32_t adler32_ssse3(uint32_t adler, const uint8_t *buf, size_t len) { 18| | /* split Adler-32 into component sums */ 19| 375k| uint32_t sum2 = (adler >> 16) & 0xffff; 20| 375k| adler &= 0xffff; 21| | 22| | /* in case user likes doing a byte at a time, keep it fast */ 23| 375k| if (UNLIKELY(len == 1)) ------------------ | | 258| 375k|# define UNLIKELY(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (258:33): [True: 0, False: 375k] | | ------------------ ------------------ 24| 0| return adler32_copy_tail(adler, NULL, buf, 1, sum2, 1, 1, 0); 25| | 26| | /* in case short lengths are provided, keep it somewhat fast */ 27| 375k| if (UNLIKELY(len < 16)) ------------------ | | 258| 375k|# define UNLIKELY(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (258:33): [True: 0, False: 375k] | | ------------------ ------------------ 28| 0| return adler32_copy_tail(adler, NULL, buf, len, sum2, 1, 15, 0); 29| | 30| 375k| const __m128i dot2v = _mm_setr_epi8(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17); 31| 375k| const __m128i dot2v_0 = _mm_setr_epi8(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1); 32| 375k| const __m128i dot3v = _mm_set1_epi16(1); 33| 375k| const __m128i zero = _mm_setzero_si128(); 34| | 35| 375k| __m128i vbuf, vs1_0, vs3, vs1, vs2, vs2_0, v_sad_sum1, v_short_sum2, v_short_sum2_0, 36| 375k| vbuf_0, v_sad_sum2, vsum2, vsum2_0; 37| | 38| | /* If our buffer is unaligned (likely), make the determination whether 39| | * or not there's enough of a buffer to consume to make the scalar, aligning 40| | * additions worthwhile or if it's worth it to just eat the cost of an unaligned 41| | * load. This is a pretty simple test, just test if len < 32 */ 42| 375k| size_t n = NMAX; ------------------ | | 14| 375k|#define NMAX 5552 ------------------ 43| 375k| size_t k = 0; 44| | 45| 375k| if (len < 32) { ------------------ | Branch (45:9): [True: 375k, False: 0] ------------------ 46| | /* Let's eat the cost of this one unaligned load so that 47| | * we don't completely skip over the vectorization. Doing 48| | * 16 bytes at a time unaligned is better than 16 + <= 15 49| | * sums */ 50| 375k| vbuf = _mm_loadu_si128((__m128i*)buf); 51| 375k| len -= 16; 52| 375k| buf += 16; 53| 375k| vs1 = _mm_cvtsi32_si128(adler); 54| 375k| vs2 = _mm_cvtsi32_si128(sum2); 55| 375k| vs3 = _mm_setzero_si128(); 56| 375k| vs1_0 = vs1; 57| 375k| goto unaligned_jmp; 58| 375k| } 59| | 60| 0| { 61| 0| size_t align_diff = MIN(ALIGN_DIFF(buf, 16), len); ------------------ | | 148| 0|#define MIN(a, b) ((a) > (b) ? (b) : (a)) | | ------------------ | | | Branch (148:20): [True: 0, False: 0] | | ------------------ ------------------ 62| 0| if (align_diff) { ------------------ | Branch (62:13): [True: 0, False: 0] ------------------ 63| 0| adler32_copy_align(&adler, NULL, buf, align_diff, &sum2, 15, 0); 64| | 65| 0| buf += align_diff; 66| 0| len -= align_diff; 67| 0| n -= align_diff; 68| 0| } 69| 0| } 70| | 71| 375k| while (len >= 16) { ------------------ | Branch (71:12): [True: 0, False: 375k] ------------------ 72| 0| vs1 = _mm_cvtsi32_si128(adler); 73| 0| vs2 = _mm_cvtsi32_si128(sum2); 74| 0| vs3 = _mm_setzero_si128(); 75| 0| vs2_0 = _mm_setzero_si128(); 76| 0| vs1_0 = vs1; 77| | 78| 0| k = ALIGN_DOWN(MIN(len, n), 16); ------------------ | | 293| 0| ((value) & ~((align) - 1)) | | ------------------ | | | Branch (293:7): [True: 0, False: 0] | | ------------------ ------------------ 79| 0| len -= k; 80| | 81| 0| while (k >= 32) { ------------------ | Branch (81:16): [True: 0, False: 0] ------------------ 82| | /* 83| | vs1 = adler + sum(c[i]) 84| | vs2 = sum2 + 16 vs1 + sum( (16-i+1) c[i] ) 85| | */ 86| 0| vbuf = _mm_load_si128((__m128i*)buf); 87| 0| vbuf_0 = _mm_load_si128((__m128i*)(buf + 16)); 88| 0| buf += 32; 89| 0| k -= 32; 90| | 91| 0| v_sad_sum1 = _mm_sad_epu8(vbuf, zero); 92| 0| v_sad_sum2 = _mm_sad_epu8(vbuf_0, zero); 93| 0| vs1 = _mm_add_epi32(v_sad_sum1, vs1); 94| 0| vs3 = _mm_add_epi32(vs1_0, vs3); 95| | 96| 0| vs1 = _mm_add_epi32(v_sad_sum2, vs1); 97| 0| v_short_sum2 = _mm_maddubs_epi16(vbuf, dot2v); 98| 0| vsum2 = _mm_madd_epi16(v_short_sum2, dot3v); 99| 0| v_short_sum2_0 = _mm_maddubs_epi16(vbuf_0, dot2v_0); 100| 0| vs2 = _mm_add_epi32(vsum2, vs2); 101| 0| vsum2_0 = _mm_madd_epi16(v_short_sum2_0, dot3v); 102| 0| vs2_0 = _mm_add_epi32(vsum2_0, vs2_0); 103| 0| vs1_0 = vs1; 104| 0| } 105| | 106| 0| vs2 = _mm_add_epi32(vs2_0, vs2); 107| 0| vs3 = _mm_slli_epi32(vs3, 5); 108| 0| vs2 = _mm_add_epi32(vs3, vs2); 109| 0| vs3 = _mm_setzero_si128(); 110| | 111| 375k| while (k >= 16) { ------------------ | Branch (111:16): [True: 0, False: 375k] ------------------ 112| | /* 113| | vs1 = adler + sum(c[i]) 114| | vs2 = sum2 + 16 vs1 + sum( (16-i+1) c[i] ) 115| | */ 116| 0| vbuf = _mm_load_si128((__m128i*)buf); 117| 0| buf += 16; 118| 0| k -= 16; 119| | 120| 375k|unaligned_jmp: 121| 375k| v_sad_sum1 = _mm_sad_epu8(vbuf, zero); 122| 375k| vs1 = _mm_add_epi32(v_sad_sum1, vs1); 123| 375k| vs3 = _mm_add_epi32(vs1_0, vs3); 124| 375k| v_short_sum2 = _mm_maddubs_epi16(vbuf, dot2v_0); 125| 375k| vsum2 = _mm_madd_epi16(v_short_sum2, dot3v); 126| 375k| vs2 = _mm_add_epi32(vsum2, vs2); 127| 375k| vs1_0 = vs1; 128| 375k| } 129| | 130| 375k| vs3 = _mm_slli_epi32(vs3, 4); 131| 375k| vs2 = _mm_add_epi32(vs2, vs3); 132| | 133| | /* We don't actually need to do a full horizontal sum, since psadbw is actually doing 134| | * a partial reduction sum implicitly and only summing to integers in vector positions 135| | * 0 and 2. This saves us some contention on the shuffle port(s) */ 136| 375k| adler = partial_hsum(vs1) % BASE; ------------------ | | 13| 375k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 137| 375k| sum2 = hsum(vs2) % BASE; ------------------ | | 13| 375k|#define BASE 65521U /* largest prime smaller than 65536 */ ------------------ 138| 375k| n = NMAX; ------------------ | | 14| 375k|#define NMAX 5552 ------------------ 139| 375k| } 140| | 141| | /* Process tail (len < 16). */ 142| 375k| return adler32_copy_tail(adler, NULL, buf, len, sum2, len != 0, 15, 0); 143| 0|} adler32_ssse3.c:partial_hsum: 14| 375k|static inline uint32_t partial_hsum(__m128i x) { 15| | __m128i second_int = _mm_srli_si128(x, 8); 16| 375k| __m128i sum = _mm_add_epi32(x, second_int); 17| 375k| return _mm_cvtsi128_si32(sum); 18| 375k|} adler32_ssse3.c:hsum: 20| 375k|static inline uint32_t hsum(__m128i x) { 21| 375k| __m128i sum1 = _mm_unpackhi_epi64(x, x); 22| 375k| __m128i sum2 = _mm_add_epi32(x, sum1); 23| | __m128i sum3 = _mm_shuffle_epi32(sum2, 0x01); 24| 375k| __m128i sum4 = _mm_add_epi32(sum2, sum3); 25| 375k| return _mm_cvtsi128_si32(sum4); 26| 375k|} crc32_pclmulqdq: 24| 3.11M|Z_INTERNAL uint32_t crc32_pclmulqdq(uint32_t crc, const uint8_t *buf, size_t len) { 25| | return crc32_copy_impl(crc, NULL, buf, len, 0); 26| 3.11M|} crc32_pclmulqdq.c:crc32_copy_impl: 538| 3.11M| const int COPY) { 539| 3.11M| size_t copy_len = len; 540| 3.11M| if (len >= 16) { ------------------ | Branch (540:9): [True: 1.02M, False: 2.09M] ------------------ 541| | /* Calculate 16-byte alignment offset */ 542| 1.02M| uintptr_t align_diff = ALIGN_DIFF(src, 16); ------------------ | | 285| 1.02M| (((uintptr_t)(align) - ((uintptr_t)(ptr) & ((align) - 1))) & ((align) - 1)) ------------------ 543| | 544| | /* If total length is less than (alignment bytes + 16), use the faster small method. 545| | * Handles both initially small buffers and cases where alignment would leave < 16 bytes */ 546| 1.02M| copy_len = len < align_diff + 16 ? len : align_diff; ------------------ | Branch (546:20): [True: 79.8k, False: 948k] ------------------ 547| 1.02M| } 548| | 549| 3.11M| if (copy_len > 0) { ------------------ | Branch (549:9): [True: 2.79M, False: 323k] ------------------ 550| 2.79M| crc = ~crc32_copy_small(~crc, dst, src, copy_len, 31, COPY); 551| 2.79M| src += copy_len; 552| 2.79M| len -= copy_len; 553| 2.79M| if (COPY) { ------------------ | Branch (553:13): [True: 0, False: 2.79M] ------------------ 554| 0| dst += copy_len; 555| 0| } 556| 2.79M| } 557| | 558| 3.11M| if (len == 0) ------------------ | Branch (558:9): [True: 2.17M, False: 948k] ------------------ 559| 2.17M| return crc; 560| | 561| 948k| const __m128i xmm_fold4 = _mm_set_epi32(0x00000001, 0x54442bd4, 0x00000001, 0xc6e41596); 562| | 563| 948k| __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3; 564| 948k| __m128i xmm_crc0 = _mm_cvtsi32_si128(0x9db42487); 565| 948k| __m128i xmm_crc1 = _mm_setzero_si128(); 566| 948k| __m128i xmm_crc2 = _mm_setzero_si128(); 567| 948k| __m128i xmm_crc3 = _mm_setzero_si128(); 568| | 569| 948k| if (crc != 0) { ------------------ | Branch (569:9): [True: 786k, False: 162k] ------------------ 570| | // Process the first 16 bytes and handle initial CRC 571| 786k| len -= 16; 572| 786k| xmm_t0 = _mm_load_si128((__m128i *)src); 573| 786k| src += 16; 574| | 575| 786k| fold_state_1(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4); 576| 786k| if (COPY) { ------------------ | Branch (576:13): [True: 0, False: 786k] ------------------ 577| 0| _mm_storeu_si128((__m128i *)dst, xmm_t0); 578| 0| dst += 16; 579| 0| } 580| 786k| xmm_crc3 = z128_xor3_epi64(xmm_crc3, xmm_t0, _mm_cvtsi32_si128(crc)); ------------------ | | 61| 786k|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 581| 786k| } 582| | 583| |#if defined(X86_VPCLMULQDQ) && defined(__AVX512F__) 584| | /* 512-bit VPCLMULQDQ path requires AVX-512F */ 585| | if (len >= 256) 586| | fold_block_16(&src, &dst, &len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, COPY); 587| |#elif defined(X86_VPCLMULQDQ) 588| | /* 256-bit VPCLMULQDQ path (doesn't require AVX-512F) */ 589| | if (len >= 128) 590| | fold_block_8(&src, &dst, &len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4, COPY); 591| |#else 592| | /* Chorba algorithm for PCLMULQDQ path (when VPCLMULQDQ not available) */ 593| 948k| if (len >= 512 + 64 + (16 * 8)) ------------------ | Branch (593:9): [True: 201, False: 948k] ------------------ 594| 201| fold_block_chorba(&src, &dst, &len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4, COPY); 595| 948k|#endif /* X86_VPCLMULQDQ */ 596| | 597| 2.34M| while (len >= 64) { ------------------ | Branch (597:12): [True: 1.39M, False: 948k] ------------------ 598| 1.39M| len -= 64; 599| 1.39M| xmm_t0 = _mm_load_si128((__m128i *)src); 600| 1.39M| xmm_t1 = _mm_load_si128((__m128i *)src + 1); 601| 1.39M| xmm_t2 = _mm_load_si128((__m128i *)src + 2); 602| 1.39M| xmm_t3 = _mm_load_si128((__m128i *)src + 3); 603| 1.39M| src += 64; 604| | 605| 1.39M| fold_state_4(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4); 606| 1.39M| if (COPY) { ------------------ | Branch (606:13): [True: 0, False: 1.39M] ------------------ 607| 0| _mm_storeu_si128((__m128i *)dst, xmm_t0); 608| 0| _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); 609| 0| _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); 610| 0| _mm_storeu_si128((__m128i *)dst + 3, xmm_t3); 611| 0| dst += 64; 612| 0| } 613| | 614| 1.39M| xmm_crc0 = _mm_xor_si128(xmm_crc0, xmm_t0); 615| 1.39M| xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t1); 616| 1.39M| xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t2); 617| 1.39M| xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t3); 618| 1.39M| } 619| | 620| | /* 621| | * len = num bytes left - 64 622| | */ 623| 948k| if (len >= 48) { ------------------ | Branch (623:9): [True: 117k, False: 831k] ------------------ 624| 117k| len -= 48; 625| | 626| 117k| xmm_t0 = _mm_load_si128((__m128i *)src); 627| 117k| xmm_t1 = _mm_load_si128((__m128i *)src + 1); 628| 117k| xmm_t2 = _mm_load_si128((__m128i *)src + 2); 629| 117k| src += 48; 630| | 631| 117k| fold_state_3(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4); 632| 117k| if (COPY) { ------------------ | Branch (632:13): [True: 0, False: 117k] ------------------ 633| 0| _mm_storeu_si128((__m128i *)dst, xmm_t0); 634| 0| _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); 635| 0| _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); 636| 0| dst += 48; 637| 0| } 638| | 639| 117k| xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t0); 640| 117k| xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t1); 641| 117k| xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t2); 642| 831k| } else if (len >= 32) { ------------------ | Branch (642:16): [True: 299k, False: 531k] ------------------ 643| 299k| len -= 32; 644| | 645| 299k| xmm_t0 = _mm_load_si128((__m128i *)src); 646| 299k| xmm_t1 = _mm_load_si128((__m128i *)src + 1); 647| 299k| src += 32; 648| | 649| 299k| fold_state_2(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4); 650| 299k| if (COPY) { ------------------ | Branch (650:13): [True: 0, False: 299k] ------------------ 651| 0| _mm_storeu_si128((__m128i *)dst, xmm_t0); 652| 0| _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); 653| 0| dst += 32; 654| 0| } 655| | 656| 299k| xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t0); 657| 299k| xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t1); 658| 531k| } else if (len >= 16) { ------------------ | Branch (658:16): [True: 219k, False: 312k] ------------------ 659| 219k| len -= 16; 660| 219k| xmm_t0 = _mm_load_si128((__m128i *)src); 661| 219k| src += 16; 662| | 663| 219k| fold_state_1(&xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, xmm_fold4); 664| 219k| if (COPY) { ------------------ | Branch (664:13): [True: 0, False: 219k] ------------------ 665| 0| _mm_storeu_si128((__m128i *)dst, xmm_t0); 666| 0| dst += 16; 667| 0| } 668| | 669| 219k| xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t0); 670| 219k| } 671| | 672| 948k| const __m128i k12 = _mm_set_epi32(0x00000001, 0x751997d0, 0x00000000, 0xccaa009e); 673| 948k| const __m128i barrett_k = _mm_set_epi32(0x00000001, 0xdb710640, 0xb4e5b025, 0xf7011641); 674| | 675| | /* Fold 4x128-bit into a single 128-bit value using k1/k2 constants */ 676| 948k| __m128i x_low0 = _mm_clmulepi64_si128(xmm_crc0, k12, 0x01); 677| 948k| __m128i x_high0 = _mm_clmulepi64_si128(xmm_crc0, k12, 0x10); 678| 948k| xmm_crc1 = z128_xor3_epi64(xmm_crc1, x_low0, x_high0); ------------------ | | 61| 948k|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 679| | 680| 948k| __m128i x_low1 = _mm_clmulepi64_si128(xmm_crc1, k12, 0x01); 681| 948k| __m128i x_high1 = _mm_clmulepi64_si128(xmm_crc1, k12, 0x10); 682| 948k| xmm_crc2 = z128_xor3_epi64(xmm_crc2, x_low1, x_high1); ------------------ | | 61| 948k|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 683| | 684| 948k| __m128i x_low2 = _mm_clmulepi64_si128(xmm_crc2, k12, 0x01); 685| 948k| __m128i x_high2 = _mm_clmulepi64_si128(xmm_crc2, k12, 0x10); 686| 948k| xmm_crc3 = z128_xor3_epi64(xmm_crc3, x_low2, x_high2); ------------------ | | 61| 948k|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 687| | 688| | /* Fold remaining bytes into the 128-bit state */ 689| 948k| if (len) { ------------------ | Branch (689:9): [True: 626k, False: 322k] ------------------ 690| 626k| const __m128i xmm_mask3 = _mm_set1_epi32((int32_t)0x80808080); 691| 626k| const __m128i xmm_seq = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); 692| | 693| | /* Create masks to shift bytes for partial input */ 694| 626k| __m128i xmm_shl = _mm_add_epi8(xmm_seq, _mm_set1_epi8((char)len - 16)); 695| 626k| __m128i xmm_shr = _mm_xor_si128(xmm_shl, xmm_mask3); 696| | 697| | /* Shift out bytes from crc3 to make space for new data */ 698| 626k| __m128i xmm_overflow = _mm_shuffle_epi8(xmm_crc3, xmm_shl); 699| 626k| xmm_crc3 = _mm_shuffle_epi8(xmm_crc3, xmm_shr); 700| | 701| | /* Insert the partial input into crc3 */ 702| |#if defined(__AVX512BW__) && defined(__AVX512VL__) 703| | __mmask16 k = (1 << len) - 1; 704| | __m128i xmm_crc_part = _mm_maskz_loadu_epi8(k, src); 705| | if (COPY) { 706| | _mm_mask_storeu_epi8(dst, k, xmm_crc_part); 707| | } 708| |#else 709| 626k| __m128i xmm_crc_part = _mm_setzero_si128(); 710| 626k| memcpy(&xmm_crc_part, src, len); 711| 626k| if (COPY) { ------------------ | Branch (711:13): [True: 0, False: 626k] ------------------ 712| 0| memcpy(dst, src, len); 713| 0| } 714| 626k|#endif 715| 626k| __m128i part_aligned = _mm_shuffle_epi8(xmm_crc_part, xmm_shl); 716| 626k| xmm_crc3 = _mm_xor_si128(xmm_crc3, part_aligned); 717| | 718| | /* Fold the bytes that were shifted out back into crc3 */ 719| 626k| __m128i ovf_low = _mm_clmulepi64_si128(xmm_overflow, k12, 0x01); 720| 626k| __m128i ovf_high = _mm_clmulepi64_si128(xmm_overflow, k12, 0x10); 721| 626k| xmm_crc3 = z128_xor3_epi64(xmm_crc3, ovf_low, ovf_high); ------------------ | | 61| 626k|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 722| 626k| } 723| | 724| | /* Reduce 128-bits to 32-bits using two-stage Barrett reduction */ 725| 948k| __m128i x_tmp0 = _mm_clmulepi64_si128(xmm_crc3, barrett_k, 0x00); 726| 948k| __m128i x_tmp1 = _mm_clmulepi64_si128(x_tmp0, barrett_k, 0x10); 727| | 728| 948k| x_tmp1 = _mm_blend_epi16(x_tmp1, _mm_setzero_si128(), 0xcf); 729| 948k| x_tmp0 = _mm_xor_si128(x_tmp1, xmm_crc3); 730| | 731| 948k| __m128i x_res_a = _mm_clmulepi64_si128(x_tmp0, barrett_k, 0x01); 732| 948k| __m128i x_res_b = _mm_clmulepi64_si128(x_res_a, barrett_k, 0x10); 733| | 734| 948k| crc = ((uint32_t)_mm_extract_epi32(x_res_b, 2)); 735| | 736| 948k| return ~crc; 737| 3.11M|} crc32_pclmulqdq.c:fold_state_1: 65| 1.00M| const __m128i xmm_fold4) { 66| 1.00M| __m128i x_low = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); 67| 1.00M| __m128i x_high = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x10); 68| | 69| 1.00M| *xmm_crc0 = *xmm_crc1; 70| 1.00M| *xmm_crc1 = *xmm_crc2; 71| 1.00M| *xmm_crc2 = *xmm_crc3; 72| 1.00M| *xmm_crc3 = _mm_xor_si128(x_low, x_high); 73| 1.00M|} crc32_pclmulqdq.c:fold_block_chorba: 346| 201| const __m128i xmm_fold4, const int COPY) { 347| 201| __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3; 348| | 349| 201|#ifndef __AVX512VL__ 350| 201| if (!COPY) ------------------ | Branch (350:9): [True: 201, False: 0] ------------------ 351| 201| return; 352| 0|#endif 353| 0| while (*len >= 512 + 64 + (16 * 8)) { ------------------ | Branch (353:12): [True: 0, False: 0] ------------------ 354| 0| __m128i chorba8 = _mm_load_si128((__m128i *)*src); 355| 0| __m128i chorba7 = _mm_load_si128((__m128i *)*src + 1); 356| 0| __m128i chorba6 = _mm_load_si128((__m128i *)*src + 2); 357| 0| __m128i chorba5 = _mm_load_si128((__m128i *)*src + 3); 358| 0| __m128i chorba4 = _mm_load_si128((__m128i *)*src + 4); 359| 0| __m128i chorba3 = _mm_load_si128((__m128i *)*src + 5); 360| 0| __m128i chorba2 = _mm_load_si128((__m128i *)*src + 6); 361| 0| __m128i chorba1 = _mm_load_si128((__m128i *)*src + 7); 362| 0| if (COPY) { ------------------ | Branch (362:13): [True: 0, False: 0] ------------------ 363| 0| _mm_storeu_si128((__m128i *)*dst, chorba8); 364| 0| _mm_storeu_si128((__m128i *)*dst + 1, chorba7); 365| 0| _mm_storeu_si128((__m128i *)*dst + 2, chorba6); 366| 0| _mm_storeu_si128((__m128i *)*dst + 3, chorba5); 367| 0| _mm_storeu_si128((__m128i *)*dst + 4, chorba4); 368| 0| _mm_storeu_si128((__m128i *)*dst + 5, chorba3); 369| 0| _mm_storeu_si128((__m128i *)*dst + 6, chorba2); 370| 0| _mm_storeu_si128((__m128i *)*dst + 7, chorba1); 371| 0| *dst += 16 * 8; 372| 0| } 373| | 374| 0| chorba2 = _mm_xor_si128(chorba2, chorba8); 375| 0| chorba1 = _mm_xor_si128(chorba1, chorba7); 376| 0| *src += 16 * 8; 377| 0| *len -= 16 * 8; 378| | 379| 0| xmm_t0 = _mm_load_si128((__m128i *)*src); 380| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 1); 381| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 2); 382| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 3); 383| | 384| 0| fold_state_12(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3); 385| 0| if (COPY) { ------------------ | Branch (385:13): [True: 0, False: 0] ------------------ 386| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 387| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 388| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 389| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 390| 0| *dst += 64; 391| 0| } 392| | 393| 0| *xmm_crc0 = z128_xor3_epi64(xmm_t0, chorba6, *xmm_crc0); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 394| 0| *xmm_crc1 = _mm_xor_si128(z128_xor3_epi64(xmm_t1, chorba5, chorba8), *xmm_crc1); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 395| 0| *xmm_crc2 = z128_xor3_epi64(z128_xor3_epi64(xmm_t2, chorba4, chorba8), chorba7, *xmm_crc2); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 396| 0| *xmm_crc3 = z128_xor3_epi64(z128_xor3_epi64(xmm_t3, chorba3, chorba7), chorba6, *xmm_crc3); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 397| | 398| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 4); 399| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 5); 400| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 6); 401| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 7); 402| | 403| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 404| 0| if (COPY) { ------------------ | Branch (404:13): [True: 0, False: 0] ------------------ 405| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 406| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 407| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 408| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 409| 0| *dst += 64; 410| 0| } 411| | 412| 0| *xmm_crc0 = z128_xor3_epi64(z128_xor3_epi64(xmm_t0, chorba2, chorba6), chorba5, *xmm_crc0); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 413| 0| *xmm_crc1 = z128_xor3_epi64(z128_xor3_epi64(xmm_t1, chorba1, chorba4), chorba5, *xmm_crc1); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 414| 0| *xmm_crc2 = _mm_xor_si128(z128_xor3_epi64(xmm_t2, chorba3, chorba4), *xmm_crc2); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 415| 0| *xmm_crc3 = _mm_xor_si128(z128_xor3_epi64(xmm_t3, chorba2, chorba3), *xmm_crc3); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 416| | 417| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 8); 418| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 9); 419| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 10); 420| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 11); 421| | 422| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 423| 0| if (COPY) { ------------------ | Branch (423:13): [True: 0, False: 0] ------------------ 424| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 425| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 426| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 427| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 428| 0| *dst += 64; 429| 0| } 430| | 431| 0| *xmm_crc0 = z128_xor3_epi64(z128_xor3_epi64(xmm_t0, chorba1, chorba2), chorba8, *xmm_crc0); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 432| 0| *xmm_crc1 = _mm_xor_si128(z128_xor3_epi64(xmm_t1, chorba1, chorba7), *xmm_crc1); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 433| 0| *xmm_crc2 = z128_xor3_epi64(xmm_t2, chorba6, *xmm_crc2); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 434| 0| *xmm_crc3 = z128_xor3_epi64(xmm_t3, chorba5, *xmm_crc3); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 435| | 436| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 12); 437| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 13); 438| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 14); 439| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 15); 440| | 441| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 442| 0| if (COPY) { ------------------ | Branch (442:13): [True: 0, False: 0] ------------------ 443| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 444| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 445| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 446| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 447| 0| *dst += 64; 448| 0| } 449| | 450| 0| *xmm_crc0 = _mm_xor_si128(z128_xor3_epi64(xmm_t0, chorba4, chorba8), *xmm_crc0); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 451| 0| *xmm_crc1 = z128_xor3_epi64(z128_xor3_epi64(xmm_t1, chorba3, chorba8), chorba7, *xmm_crc1); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 452| 0| *xmm_crc2 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(xmm_t2, chorba2, chorba8), chorba7, chorba6), *xmm_crc2); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 453| 0| *xmm_crc3 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(xmm_t3, chorba1, chorba7), chorba6, chorba5), *xmm_crc3); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 454| | 455| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 16); 456| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 17); 457| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 18); 458| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 19); 459| | 460| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 461| 0| if (COPY) { ------------------ | Branch (461:13): [True: 0, False: 0] ------------------ 462| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 463| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 464| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 465| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 466| 0| *dst += 64; 467| 0| } 468| | 469| 0| *xmm_crc0 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(xmm_t0, chorba4, chorba8), chorba6, chorba5), *xmm_crc0); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 470| 0| *xmm_crc1 = z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t1, chorba3, chorba4), chorba8, chorba7), chorba5, *xmm_crc1); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 471| 0| *xmm_crc2 = z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t2, chorba2, chorba3), chorba4, chorba7), chorba6, *xmm_crc2); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 472| 0| *xmm_crc3 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t3, chorba1, chorba2), chorba3, chorba8), chorba6, chorba5), *xmm_crc3); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 473| | 474| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 20); 475| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 21); 476| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 22); 477| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 23); 478| | 479| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 480| 0| if (COPY) { ------------------ | Branch (480:13): [True: 0, False: 0] ------------------ 481| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 482| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 483| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 484| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 485| 0| *dst += 64; 486| 0| } 487| | 488| 0| *xmm_crc0 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t0, chorba1, chorba2), chorba4, chorba8), chorba7, chorba5), *xmm_crc0); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 489| 0| *xmm_crc1 = z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t1, chorba1, chorba3), chorba4, chorba7), chorba6, *xmm_crc1); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 490| 0| *xmm_crc2 = z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t2, chorba2, chorba3), chorba8, chorba6), chorba5, *xmm_crc2); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 491| 0| *xmm_crc3 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t3, chorba1, chorba2), chorba4, chorba8), chorba7, chorba5), *xmm_crc3); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 492| | 493| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 24); 494| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 25); 495| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 26); 496| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 27); 497| | 498| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 499| 0| if (COPY) { ------------------ | Branch (499:13): [True: 0, False: 0] ------------------ 500| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 501| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 502| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 503| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 504| 0| *dst += 64; 505| 0| } 506| | 507| 0| *xmm_crc0 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t0, chorba1, chorba3), chorba4, chorba8), chorba7, chorba6), *xmm_crc0); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 508| 0| *xmm_crc1 = z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t1, chorba2, chorba3), chorba7, chorba6), chorba5, *xmm_crc1); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 509| 0| *xmm_crc2 = z128_xor3_epi64(z128_xor3_epi64(z128_xor3_epi64(xmm_t2, chorba1, chorba2), chorba4, chorba6), chorba5, *xmm_crc2); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 510| 0| *xmm_crc3 = _mm_xor_si128(z128_xor3_epi64(z128_xor3_epi64(xmm_t3, chorba1, chorba3), chorba4, chorba5), *xmm_crc3); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 511| | 512| 0| xmm_t0 = _mm_load_si128((__m128i *)*src + 28); 513| 0| xmm_t1 = _mm_load_si128((__m128i *)*src + 29); 514| 0| xmm_t2 = _mm_load_si128((__m128i *)*src + 30); 515| 0| xmm_t3 = _mm_load_si128((__m128i *)*src + 31); 516| | 517| 0| fold_state_4(xmm_crc0, xmm_crc1, xmm_crc2, xmm_crc3, xmm_fold4); 518| 0| if (COPY) { ------------------ | Branch (518:13): [True: 0, False: 0] ------------------ 519| 0| _mm_storeu_si128((__m128i *)*dst, xmm_t0); 520| 0| _mm_storeu_si128((__m128i *)*dst + 1, xmm_t1); 521| 0| _mm_storeu_si128((__m128i *)*dst + 2, xmm_t2); 522| 0| _mm_storeu_si128((__m128i *)*dst + 3, xmm_t3); 523| 0| *dst += 64; 524| 0| } 525| | 526| 0| *xmm_crc0 = z128_xor3_epi64(z128_xor3_epi64(xmm_t0, chorba2, chorba3), chorba4, *xmm_crc0); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 527| 0| *xmm_crc1 = z128_xor3_epi64(z128_xor3_epi64(xmm_t1, chorba1, chorba2), chorba3, *xmm_crc1); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 528| 0| *xmm_crc2 = _mm_xor_si128(z128_xor3_epi64(xmm_t2, chorba1, chorba2), *xmm_crc2); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 529| 0| *xmm_crc3 = z128_xor3_epi64(xmm_t3, chorba1, *xmm_crc3); ------------------ | | 61| 0|# define z128_xor3_epi64(a, b, c) _mm_xor_si128(_mm_xor_si128(a, b), c) ------------------ 530| | 531| 0| *len -= 512; 532| 0| *src += 512; 533| 0| } 534| 0|} crc32_pclmulqdq.c:fold_state_4: 104| 1.39M| const __m128i xmm_fold4) { 105| 1.39M| __m128i x_low0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); 106| 1.39M| __m128i x_high0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x10); 107| 1.39M| __m128i x_low1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); 108| 1.39M| __m128i x_high1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10); 109| 1.39M| __m128i x_low2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); 110| 1.39M| __m128i x_high2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); 111| 1.39M| __m128i x_low3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x01); 112| 1.39M| __m128i x_high3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); 113| | 114| 1.39M| *xmm_crc0 = _mm_xor_si128(x_low0, x_high0); 115| 1.39M| *xmm_crc1 = _mm_xor_si128(x_low1, x_high1); 116| 1.39M| *xmm_crc2 = _mm_xor_si128(x_low2, x_high2); 117| 1.39M| *xmm_crc3 = _mm_xor_si128(x_low3, x_high3); 118| 1.39M|} crc32_pclmulqdq.c:fold_state_3: 89| 117k| const __m128i xmm_fold4) { 90| 117k| __m128i x_low0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); 91| 117k| __m128i x_high0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x10); 92| 117k| __m128i x_low1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); 93| 117k| __m128i x_high1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10); 94| 117k| __m128i x_low2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); 95| 117k| __m128i x_high2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); 96| | 97| 117k| *xmm_crc0 = *xmm_crc3; 98| 117k| *xmm_crc1 = _mm_xor_si128(x_low0, x_high0); 99| 117k| *xmm_crc2 = _mm_xor_si128(x_low1, x_high1); 100| 117k| *xmm_crc3 = _mm_xor_si128(x_low2, x_high2); 101| 117k|} crc32_pclmulqdq.c:fold_state_2: 76| 299k| const __m128i xmm_fold4) { 77| 299k| __m128i x_low0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); 78| 299k| __m128i x_high0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x10); 79| 299k| __m128i x_low1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); 80| 299k| __m128i x_high1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10); 81| | 82| 299k| *xmm_crc0 = *xmm_crc2; 83| 299k| *xmm_crc1 = *xmm_crc3; 84| 299k| *xmm_crc2 = _mm_xor_si128(x_low0, x_high0); 85| 299k| *xmm_crc3 = _mm_xor_si128(x_low1, x_high1); 86| 299k|} x86_check_features: 78| 2|void Z_INTERNAL x86_check_features(struct x86_cpu_features *features) { 79| 2| unsigned eax, ebx, ecx, edx; 80| 2| unsigned maxbasic; 81| | 82| 2| cpuid(0, &maxbasic, &ebx, &ecx, &edx); 83| 2| cpuid(1 /*CPU_PROCINFO_AND_FEATUREBITS*/, &eax, &ebx, &ecx, &edx); 84| | 85| 2| features->has_sse2 = edx & 0x4000000; 86| 2| features->has_ssse3 = ecx & 0x200; 87| 2| features->has_sse41 = ecx & 0x80000; 88| 2| features->has_sse42 = ecx & 0x100000; 89| 2| features->has_pclmulqdq = ecx & 0x2; 90| | 91| 2| if (ecx & 0x08000000) { ------------------ | Branch (91:9): [True: 2, False: 0] ------------------ 92| 2| uint64_t xfeature = xgetbv(0); 93| | 94| 2| features->has_os_save_ymm = ((xfeature & 0x06) == 0x06); 95| 2| features->has_os_save_zmm = ((xfeature & 0xe6) == 0xe6); 96| 2| } 97| | 98| 2| if (maxbasic >= 7) { ------------------ | Branch (98:9): [True: 2, False: 0] ------------------ 99| | // Reference: https://software.intel.com/sites/default/files/article/405250/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family.pdf 100| 2| cpuidex(7, 0, &eax, &ebx, &ecx, &edx); 101| | 102| | // check BMI2 bit 103| 2| features->has_bmi2 = ebx & 0x100; 104| | 105| | // check AVX2 bit if the OS supports saving YMM registers 106| 2| if (features->has_os_save_ymm) { ------------------ | Branch (106:13): [True: 2, False: 0] ------------------ 107| 2| features->has_avx2 = ebx & 0x20; 108| 2| features->has_vpclmulqdq = ecx & 0x400; 109| 2| } 110| | 111| | // check AVX512 bits if the OS supports saving ZMM registers 112| 2| if (features->has_os_save_zmm) { ------------------ | Branch (112:13): [True: 0, False: 2] ------------------ 113| 0| features->has_avx512f = ebx & 0x00010000; 114| 0| if (features->has_avx512f) { ------------------ | Branch (114:17): [True: 0, False: 0] ------------------ 115| | // According to the Intel Software Developer's Manual, AVX512F must be enabled too in order to enable 116| | // AVX512(DQ,BW,VL). 117| 0| features->has_avx512dq = ebx & 0x00020000; 118| 0| features->has_avx512bw = ebx & 0x40000000; 119| 0| features->has_avx512vl = ebx & 0x80000000; 120| 0| } 121| 0| features->has_avx512_common = features->has_avx512f && features->has_avx512dq && features->has_avx512bw \ ------------------ | Branch (121:43): [True: 0, False: 0] | Branch (121:68): [True: 0, False: 0] | Branch (121:94): [True: 0, False: 0] ------------------ 122| 0| && features->has_avx512vl && features->has_bmi2; ------------------ | Branch (122:18): [True: 0, False: 0] | Branch (122:44): [True: 0, False: 0] ------------------ 123| 0| features->has_avx512vnni = ecx & 0x800; 124| 0| } 125| | 126| 2| if (features->has_os_save_ymm) { ------------------ | Branch (126:13): [True: 2, False: 0] ------------------ 127| 2| cpuidex(7, 1, &eax, &ebx, &ecx, &edx); 128| 2| features->has_avx2vnni = eax & 0x10; 129| 2| } 130| | 131| 2| } 132| 2|} x86_features.c:cpuid: 29| 4|static inline void cpuid(int info, unsigned* eax, unsigned* ebx, unsigned* ecx, unsigned* edx) { 30| |#if defined(HAVE_CPUID_MS) 31| | unsigned int registers[4]; 32| | __cpuid((int *)registers, info); 33| | 34| | *eax = registers[0]; 35| | *ebx = registers[1]; 36| | *ecx = registers[2]; 37| | *edx = registers[3]; 38| |#elif defined(HAVE_CPUID_GNU) 39| | *eax = *ebx = *ecx = *edx = 0; 40| 4| __cpuid(info, *eax, *ebx, *ecx, *edx); 41| |#else 42| | /* When using this fallback, the faster SSE/AVX code is disabled */ 43| | *eax = *ebx = *ecx = *edx = 0; 44| |#endif 45| 4|} x86_features.c:xgetbv: 65| 2|static inline uint64_t xgetbv(unsigned int xcr) { 66| 2|#if defined(_MSC_VER) || defined(X86_HAVE_XSAVE_INTRIN) 67| 2| return _xgetbv(xcr); 68| |#elif defined(__GNUC__) 69| | uint32_t eax, edx; 70| | __asm__ ( ".byte 0x0f, 0x01, 0xd0" : "=a"(eax), "=d"(edx) : "c"(xcr)); 71| | return (uint64_t)(edx) << 32 | eax; 72| |#else 73| | /* When using this fallback, some of the faster code is disabled */ 74| | return 0; 75| |#endif 76| 2|} x86_features.c:cpuidex: 47| 4|static inline void cpuidex(int info, int subinfo, unsigned* eax, unsigned* ebx, unsigned* ecx, unsigned* edx) { 48| |#if defined(HAVE_CPUID_MS) 49| | unsigned int registers[4]; 50| | __cpuidex((int *)registers, info, subinfo); 51| | 52| | *eax = registers[0]; 53| | *ebx = registers[1]; 54| | *ecx = registers[2]; 55| | *edx = registers[3]; 56| |#elif defined(HAVE_CPUID_GNU) 57| | *eax = *ebx = *ecx = *edx = 0; 58| 4| __cpuid_count(info, subinfo, *eax, *ebx, *ecx, *edx); 59| |#else 60| | /* When using this fallback, the faster SSE/AVX code is disabled */ 61| | *eax = *ebx = *ecx = *edx = 0; 62| |#endif 63| 4|} cpu_check_features: 10| 2|Z_INTERNAL void cpu_check_features(struct cpu_features *features) { 11| 2| memset(features, 0, sizeof(struct cpu_features)); 12| 2|#if defined(X86_FEATURES) 13| 2| x86_check_features(&features->x86); 14| |#elif defined(ARM_FEATURES) 15| | arm_check_features(&features->arm); 16| |#elif defined(PPC_FEATURES) || defined(POWER_FEATURES) 17| | power_check_features(&features->power); 18| |#elif defined(S390_FEATURES) 19| | s390_check_features(&features->s390); 20| |#elif defined(RISCV_FEATURES) 21| | riscv_check_features(&features->riscv); 22| |#elif defined(LOONGARCH_FEATURES) 23| | loongarch_check_features(&features->loongarch); 24| |#endif 25| 2|} zng_crc32_z: 36| 3.11M|uint32_t Z_EXPORT PREFIX(crc32_z)(uint32_t crc, const unsigned char *buf, size_t len) { 37| 3.11M| if (buf == NULL) ------------------ | Branch (37:9): [True: 0, False: 3.11M] ------------------ 38| 0| return CRC32_INITIAL_VALUE; ------------------ | | 71| 0|#define CRC32_INITIAL_VALUE 0 /* initial crc-32 hash value */ ------------------ 39| 3.11M| return FUNCTABLE_CALL(crc32)(crc, buf, len); ------------------ | | 46| 3.11M|# define FUNCTABLE_CALL(name) functable.name ------------------ 40| 3.11M|} zng_crc32: 50| 604|uint32_t Z_EXPORT PREFIX(crc32)(uint32_t crc, const unsigned char *buf, uint32_t len) { 51| 604| if (buf == NULL) ------------------ | Branch (51:9): [True: 604, False: 0] ------------------ 52| 604| return CRC32_INITIAL_VALUE; ------------------ | | 71| 604|#define CRC32_INITIAL_VALUE 0 /* initial crc-32 hash value */ ------------------ 53| 0| return FUNCTABLE_CALL(crc32)(crc, buf, len); ------------------ | | 46| 0|# define FUNCTABLE_CALL(name) functable.name ------------------ 54| 604|} zng_crc32_combine: 48| 1.81k|uint32_t Z_EXPORT PREFIX4(crc32_combine)(uint32_t crc1, uint32_t crc2, z_off64_t len2) { 49| 1.81k| return crc32_combine_(crc1, crc2, len2); 50| 1.81k|} zng_crc32_combine_gen: 51| 1.20k|uint32_t Z_EXPORT PREFIX(crc32_combine_gen)(z_off64_t len2) { 52| 1.20k| return crc32_combine_gen_(len2); 53| 1.20k|} zng_crc32_combine_op: 54| 1.56M|uint32_t Z_EXPORT PREFIX(crc32_combine_op)(uint32_t crc1, uint32_t crc2, const uint32_t op) { 55| 1.56M| return crc32_combine_op_(crc1, crc2, op); 56| 1.56M|} crc32_braid_comb.c:crc32_combine_: 15| 1.81k|static uint32_t crc32_combine_(uint32_t crc1, uint32_t crc2, z_off64_t len2) { 16| 1.81k| if (len2 < 0) ------------------ | Branch (16:9): [True: 0, False: 1.81k] ------------------ 17| 0| return 0; 18| 1.81k| return multmodp(x2nmodp(len2, 3), crc1) ^ crc2; 19| 1.81k|} crc32_braid_comb.c:crc32_combine_gen_: 20| 1.20k|static uint32_t crc32_combine_gen_(z_off64_t len2) { 21| 1.20k| if (len2 < 0) ------------------ | Branch (21:9): [True: 0, False: 1.20k] ------------------ 22| 0| return 0; 23| 1.20k| return x2nmodp(len2, 3); 24| 1.20k|} crc32_braid_comb.c:crc32_combine_op_: 25| 1.56M|static uint32_t crc32_combine_op_(uint32_t crc1, uint32_t crc2, const uint32_t op) { 26| 1.56M| return multmodp(op, crc1) ^ crc2; 27| 1.56M|} crc32_braid_comb.c:multmodp: 8| 1.57M|static uint32_t multmodp(uint32_t a, uint32_t b) { 9| 1.57M| uint32_t m, p; 10| | 11| 1.57M| m = (uint32_t)1 << 31; 12| 1.57M| p = 0; 13| 27.2M| for (;;) { 14| 27.2M| if (a & m) { ------------------ | Branch (14:13): [True: 9.89M, False: 17.3M] ------------------ 15| 9.89M| p ^= b; 16| 9.89M| if ((a & (m - 1)) == 0) ------------------ | Branch (16:17): [True: 1.57M, False: 8.31M] ------------------ 17| 1.57M| break; 18| 9.89M| } 19| 25.7M| m >>= 1; 20| 25.7M| b = b & 1 ? (b >> 1) ^ POLY : b >> 1; ------------------ | | 34| 12.8M|#define POLY 0xedb88320 /* p(x) reflected, with x^32 implied */ ------------------ | Branch (20:13): [True: 12.8M, False: 12.9M] ------------------ 21| 25.7M| } 22| 1.57M| return p; 23| 1.57M|} crc32_braid_comb.c:x2nmodp: 29| 3.02k|static uint32_t x2nmodp(z_off64_t n, unsigned k) { 30| 3.02k| uint32_t p; 31| | 32| 3.02k| p = (uint32_t)1 << 31; /* x^0 == 1 */ 33| 27.7k| while (n) { ------------------ | Branch (33:12): [True: 24.7k, False: 3.02k] ------------------ 34| 24.7k| if (n & 1) ------------------ | Branch (34:13): [True: 12.7k, False: 12.0k] ------------------ 35| 12.7k| p = multmodp(x2n_table[k & 31], p); 36| 24.7k| n >>= 1; 37| 24.7k| k++; 38| 24.7k| } 39| 3.02k| return p; 40| 3.02k|} crc32_pclmulqdq.c:crc32_copy_small: 19| 2.79M| const int MAX_LEN, const int COPY) { 20| 2.79M| if (MAX_LEN >= 8) { ------------------ | Branch (20:9): [True: 2.79M, False: 0] ------------------ 21| 3.35M| while (len >= 8) { ------------------ | Branch (21:16): [True: 557k, False: 2.79M] ------------------ 22| 557k| if (COPY) { ------------------ | Branch (22:17): [True: 0, False: 557k] ------------------ 23| 0| memcpy(dst, buf, 8); 24| 0| dst += 8; 25| 0| } 26| 557k| CRC_DO8(crc, buf, 0); ------------------ | | 16| 557k|#define CRC_DO8(c, buf, i) {CRC_DO4(c, buf, i); CRC_DO4(c, buf, i+4);} | | ------------------ | | | | 15| 557k|#define CRC_DO4(c, buf, i) {CRC_DO2(c, buf, i); CRC_DO2(c, buf, i+2);} | | | | ------------------ | | | | | | 14| 557k|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 557k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 557k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | ------------------ | | | | #define CRC_DO4(c, buf, i) {CRC_DO2(c, buf, i); CRC_DO2(c, buf, i+2);} | | | | ------------------ | | | | | | 14| 557k|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 557k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 557k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | ------------------ | | ------------------ | | #define CRC_DO8(c, buf, i) {CRC_DO4(c, buf, i); CRC_DO4(c, buf, i+4);} | | ------------------ | | | | 15| 557k|#define CRC_DO4(c, buf, i) {CRC_DO2(c, buf, i); CRC_DO2(c, buf, i+2);} | | | | ------------------ | | | | | | 14| 557k|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 557k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 557k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | ------------------ | | | | #define CRC_DO4(c, buf, i) {CRC_DO2(c, buf, i); CRC_DO2(c, buf, i+2);} | | | | ------------------ | | | | | | 14| 557k|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 557k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | | | ------------------ | | | | | | | | 13| 557k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | | | ------------------ | | | | ------------------ | | ------------------ ------------------ 27| 557k| buf += 8; 28| 557k| len -= 8; 29| 557k| } 30| 2.79M| } 31| 2.79M| if (len & 4) { ------------------ | Branch (31:9): [True: 335k, False: 2.45M] ------------------ 32| 335k| if (COPY) { ------------------ | Branch (32:13): [True: 0, False: 335k] ------------------ 33| 0| memcpy(dst, buf, 4); 34| 0| dst += 4; 35| 0| } 36| 335k| CRC_DO4(crc, buf, 0); ------------------ | | 15| 335k|#define CRC_DO4(c, buf, i) {CRC_DO2(c, buf, i); CRC_DO2(c, buf, i+2);} | | ------------------ | | | | 14| 335k|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | ------------------ | | | | | | 13| 335k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | ------------------ | | | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | ------------------ | | | | | | 13| 335k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | ------------------ | | ------------------ | | #define CRC_DO4(c, buf, i) {CRC_DO2(c, buf, i); CRC_DO2(c, buf, i+2);} | | ------------------ | | | | 14| 335k|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | ------------------ | | | | | | 13| 335k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | ------------------ | | | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | | | ------------------ | | | | | | 13| 335k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | | | ------------------ | | ------------------ ------------------ 37| 335k| buf += 4; 38| 335k| } 39| 2.79M| if (len & 2) { ------------------ | Branch (39:9): [True: 587k, False: 2.20M] ------------------ 40| 587k| if (COPY) { ------------------ | Branch (40:13): [True: 0, False: 587k] ------------------ 41| 0| memcpy(dst, buf, 2); 42| 0| dst += 2; 43| 0| } 44| 587k| CRC_DO2(crc, buf, 0); ------------------ | | 14| 587k|#define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | ------------------ | | | | 13| 587k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | ------------------ | | #define CRC_DO2(c, buf, i) {CRC_DO1(c, buf, i); CRC_DO1(c, buf, i+1);} | | ------------------ | | | | 13| 587k|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) | | ------------------ ------------------ 45| 587k| buf += 2; 46| 587k| } 47| 2.79M| if (len & 1) { ------------------ | Branch (47:9): [True: 2.17M, False: 616k] ------------------ 48| 2.17M| if (COPY) ------------------ | Branch (48:13): [True: 0, False: 2.17M] ------------------ 49| 0| *dst = *buf; 50| 2.17M| CRC_DO1(crc, buf, 0); ------------------ | | 13| 2.17M|#define CRC_DO1(c, buf, i) c = crc_table[(c ^ buf[i]) & 0xff] ^ (c >> 8) ------------------ 51| 2.17M| } 52| | 53| 2.79M| return crc; 54| 2.79M|} functable.c:functable_constructor: 469| 2|static void __attribute__((constructor)) functable_constructor(void) { 470| 2| FUNCTABLE_INIT_ABORT; ------------------ | | 57| 2| if (init_functable()) { \ | | ------------------ | | | Branch (57:9): [True: 0, False: 2] | | ------------------ | | 58| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 59| 0| abort(); \ | | 60| 2| }; ------------------ 471| 2|} functable.c:init_functable: 70| 2|static int init_functable(void) { 71| 2| struct functable_s ft; 72| 2| struct cpu_features cf; 73| | 74| 2| memset(&ft, 0, sizeof(struct functable_s)); 75| 2| cpu_check_features(&cf); 76| 2| ft.force_init = &force_init_empty; 77| | 78| | // Only use necessary generic functions when no suitable simd versions are available. 79| 2|#ifdef ADLER32_FALLBACK 80| 2| ft.adler32 = &adler32_c; 81| 2| ft.adler32_copy = &adler32_copy_c; 82| 2|#endif 83| |#ifdef CHUNKSET_FALLBACK 84| | ft.chunkmemset_safe = &chunkmemset_safe_c; 85| | ft.inflate_fast = &inflate_fast_c; 86| |#endif 87| |#ifdef COMPARE256_FALLBACK 88| | ft.compare256 = &compare256_c; 89| | ft.longest_match = &longest_match_c; 90| | ft.longest_match_roll = &longest_match_roll_c; 91| |#endif 92| 2|#ifdef CRC32_BRAID_FALLBACK 93| 2| ft.crc32 = &crc32_braid; 94| 2| ft.crc32_copy = &crc32_copy_braid; 95| 2|#endif 96| |#ifdef SLIDE_HASH_FALLBACK 97| | ft.slide_hash = &slide_hash_c; 98| | ft.slide_hash_head = &slide_hash_head_c; 99| |#endif 100| | 101| | // Select arch-optimized functions 102| 2|#ifdef WITH_OPTIM 103| | 104| | // Chorba generic C fallback 105| 2|#ifdef CRC32_CHORBA_FALLBACK 106| 2| ft.crc32 = &crc32_chorba; 107| 2| ft.crc32_copy = &crc32_copy_chorba; 108| 2|#endif 109| | 110| | // X86 - SSE2 111| 2|#ifdef X86_SSE2 112| |# ifndef X86_SSE2_NATIVE 113| | if (cf.x86.has_sse2) 114| |# endif 115| 2| { 116| 2|# ifndef X86_AVX2_NATIVE 117| 2| ft.chunkmemset_safe = &chunkmemset_safe_sse2; 118| 2| ft.compare256 = &compare256_sse2; 119| 2| ft.inflate_fast = &inflate_fast_sse2; 120| 2| ft.longest_match = &longest_match_sse2; 121| 2| ft.longest_match_roll = &longest_match_roll_sse2; 122| 2| ft.slide_hash = &slide_hash_sse2; 123| 2| ft.slide_hash_head = &slide_hash_head_sse2; 124| 2|# endif 125| 2|# if defined(CRC32_CHORBA_SSE_FALLBACK) && !defined(X86_SSE41_NATIVE) && !defined(X86_PCLMULQDQ_NATIVE) 126| 2| ft.crc32 = &crc32_chorba_sse2; 127| 2| ft.crc32_copy = &crc32_copy_chorba_sse2; 128| 2|# endif 129| 2| } 130| 2|#endif 131| | // X86 - SSSE3 132| 2|#ifdef X86_SSSE3 133| 2|# ifndef X86_SSSE3_NATIVE 134| 2| if (cf.x86.has_ssse3) ------------------ | Branch (134:9): [True: 2, False: 0] ------------------ 135| 2|# endif 136| 2| { 137| 2| ft.adler32 = &adler32_ssse3; 138| 2| ft.adler32_copy = &adler32_copy_ssse3; 139| 2|# ifndef X86_AVX2_NATIVE 140| 2| ft.chunkmemset_safe = &chunkmemset_safe_ssse3; 141| 2| ft.inflate_fast = &inflate_fast_ssse3; 142| 2|# endif 143| 2| } 144| 2|#endif 145| | 146| | // X86 - SSE4.1 147| 2|#if defined(X86_SSE41) && !defined(X86_PCLMULQDQ_NATIVE) 148| 2|# ifndef X86_SSE41_NATIVE 149| 2| if (cf.x86.has_sse41) ------------------ | Branch (149:9): [True: 2, False: 0] ------------------ 150| 2|# endif 151| 2| { 152| 2|# ifdef CRC32_CHORBA_SSE_FALLBACK 153| 2| ft.crc32 = &crc32_chorba_sse41; 154| 2| ft.crc32_copy = &crc32_copy_chorba_sse41; 155| 2|# endif 156| 2| } 157| 2|#endif 158| | 159| | // X86 - SSE4.2 160| 2|#if defined(X86_SSE42) && !defined(X86_AVX512_NATIVE) 161| 2|# ifndef X86_SSE42_NATIVE 162| 2| if (cf.x86.has_sse42) ------------------ | Branch (162:9): [True: 2, False: 0] ------------------ 163| 2|# endif 164| 2| { 165| 2| ft.adler32_copy = &adler32_copy_sse42; 166| 2| } 167| 2|#endif 168| | // X86 - PCLMUL 169| 2|#if defined(X86_PCLMULQDQ_CRC) && !defined(X86_VPCLMULQDQ_NATIVE) 170| 2|# ifndef X86_PCLMULQDQ_NATIVE 171| 2| if (cf.x86.has_pclmulqdq) ------------------ | Branch (171:9): [True: 2, False: 0] ------------------ 172| 2|# endif 173| 2| { 174| 2| ft.crc32 = &crc32_pclmulqdq; 175| 2| ft.crc32_copy = &crc32_copy_pclmulqdq; 176| 2| } 177| 2|#endif 178| | // X86 - AVX2 179| 2|#ifdef X86_AVX2 180| | /* BMI2 support is all but implicit with AVX2 but let's sanity check this just in case. Enabling BMI2 allows for 181| | * flagless shifts, resulting in fewer flag stalls for the pipeline, and allows us to set destination registers 182| | * for the shift results as an operand, eliminating several register-register moves when the original value needs 183| | * to remain intact. They also allow for a count operand that isn't the CL register, avoiding contention there */ 184| 2|# ifndef X86_AVX2_NATIVE 185| 2| if (cf.x86.has_avx2 && cf.x86.has_bmi2) ------------------ | Branch (185:9): [True: 2, False: 0] | Branch (185:28): [True: 2, False: 0] ------------------ 186| 2|# endif 187| 2| { 188| 2|# ifndef X86_AVX512_NATIVE 189| 2| ft.adler32 = &adler32_avx2; 190| 2| ft.adler32_copy = &adler32_copy_avx2; 191| 2| ft.chunkmemset_safe = &chunkmemset_safe_avx2; 192| 2| ft.compare256 = &compare256_avx2; 193| 2| ft.inflate_fast = &inflate_fast_avx2; 194| 2| ft.longest_match = &longest_match_avx2; 195| 2| ft.longest_match_roll = &longest_match_roll_avx2; 196| 2|# endif 197| 2| ft.slide_hash = &slide_hash_avx2; 198| 2| ft.slide_hash_head = &slide_hash_head_avx2; 199| 2| } 200| 2|#endif 201| 2|#ifdef X86_AVX2VNNI 202| 2| if (cf.x86.has_avx2vnni) { ------------------ | Branch (202:9): [True: 0, False: 2] ------------------ 203| 0| ft.adler32 = &adler32_avx2_vnni; 204| | //ft.adler32_copy = &adler32_copy_avx2_vnni; 205| 0| } 206| 2|#endif 207| | // X86 - AVX512 (F,DQ,BW,Vl) 208| 2|#ifdef X86_AVX512 209| 2|# ifndef X86_AVX512_NATIVE 210| 2| if (cf.x86.has_avx512_common) ------------------ | Branch (210:9): [True: 0, False: 2] ------------------ 211| 0|# endif 212| 0| { 213| 0|# ifndef X86_AVX512VNNI_NATIVE 214| 0| ft.adler32 = &adler32_avx512; 215| 0| ft.adler32_copy = &adler32_copy_avx512; 216| 0|# endif 217| 0| ft.chunkmemset_safe = &chunkmemset_safe_avx512; 218| 0| ft.compare256 = &compare256_avx512; 219| 0| ft.inflate_fast = &inflate_fast_avx512; 220| 0| ft.longest_match = &longest_match_avx512; 221| 0| ft.longest_match_roll = &longest_match_roll_avx512; 222| 0| } 223| 2|#endif 224| 2|#ifdef X86_AVX512VNNI 225| 2|# ifndef X86_AVX512VNNI_NATIVE 226| 2| if (cf.x86.has_avx512vnni) ------------------ | Branch (226:9): [True: 0, False: 2] ------------------ 227| 0|# endif 228| 0| { 229| 0| ft.adler32 = &adler32_avx512_vnni; 230| 0| ft.adler32_copy = &adler32_copy_avx512_vnni; 231| 0| } 232| 2|#endif 233| | // X86 - VPCLMULQDQ (AVX2) 234| 2|#ifdef X86_VPCLMULQDQ_AVX2 235| 2|# ifndef X86_VPCLMULQDQ_AVX2_NATIVE 236| 2| if (cf.x86.has_pclmulqdq && cf.x86.has_avx2 && cf.x86.has_vpclmulqdq) ------------------ | Branch (236:9): [True: 2, False: 0] | Branch (236:33): [True: 2, False: 0] | Branch (236:52): [True: 0, False: 2] ------------------ 237| 0|# endif 238| 0| { 239| 0| ft.crc32 = &crc32_vpclmulqdq_avx2; 240| 0| ft.crc32_copy = &crc32_copy_vpclmulqdq_avx2; 241| 0| } 242| 2|#endif 243| | // X86 - VPCLMULQDQ (AVX-512) 244| 2|#ifdef X86_VPCLMULQDQ_AVX512 245| 2|# ifndef X86_VPCLMULQDQ_AVX512_NATIVE 246| 2| if (cf.x86.has_pclmulqdq && cf.x86.has_avx512_common && cf.x86.has_vpclmulqdq) ------------------ | Branch (246:9): [True: 2, False: 0] | Branch (246:33): [True: 0, False: 2] | Branch (246:61): [True: 0, False: 0] ------------------ 247| 0|# endif 248| 0| { 249| 0| ft.crc32 = &crc32_vpclmulqdq_avx512; 250| 0| ft.crc32_copy = &crc32_copy_vpclmulqdq_avx512; 251| 0| } 252| 2|#endif 253| | 254| | 255| | // ARM - SIMD 256| |#if defined(ARM_SIMD) && !defined(ARM_NEON_NATIVE) 257| |# ifndef ARM_SIMD_NATIVE 258| | if (cf.arm.has_simd) 259| |# endif 260| | { 261| | ft.slide_hash = &slide_hash_armv6; 262| | ft.slide_hash_head = &slide_hash_head_armv6; 263| | } 264| |#endif 265| | // ARM - NEON 266| |#ifdef ARM_NEON 267| |# ifndef ARM_NEON_NATIVE 268| | if (cf.arm.has_neon) 269| |# endif 270| | { 271| | ft.adler32 = &adler32_neon; 272| | ft.adler32_copy = &adler32_copy_neon; 273| | ft.chunkmemset_safe = &chunkmemset_safe_neon; 274| | ft.compare256 = &compare256_neon; 275| | ft.inflate_fast = &inflate_fast_neon; 276| | ft.longest_match = &longest_match_neon; 277| | ft.longest_match_roll = &longest_match_roll_neon; 278| | ft.slide_hash = &slide_hash_neon; 279| | ft.slide_hash_head = &slide_hash_head_neon; 280| | } 281| |#endif 282| | // ARM - NEON DotProd 283| |#ifdef ARM_NEON_DOTPROD 284| |# ifndef ARM_NEON_DOTPROD_NATIVE 285| | if (cf.arm.has_neon && cf.arm.has_dotprod) 286| |# endif 287| | { 288| | ft.adler32 = &adler32_neon_dotprod; 289| | ft.adler32_copy = &adler32_copy_neon_dotprod; 290| | } 291| |#endif 292| | // ARM - CRC32 293| |#if defined(ARM_CRC32) && !defined(ARM_PMULL_EOR3_NATIVE) 294| |# ifndef ARM_CRC32_NATIVE 295| | if (cf.arm.has_crc32) 296| |# endif 297| | { 298| | ft.crc32 = &crc32_armv8; 299| | ft.crc32_copy = &crc32_copy_armv8; 300| | } 301| |#endif 302| | // ARM - PMULL EOR3 303| |#ifdef ARM_PMULL_EOR3 304| |# ifndef ARM_PMULL_EOR3_NATIVE 305| | if (cf.arm.has_crc32 && cf.arm.has_pmull && cf.arm.has_eor3 && cf.arm.has_fast_pmull) 306| |# endif 307| | { 308| | ft.crc32 = &crc32_armv8_pmull_eor3; 309| | ft.crc32_copy = &crc32_copy_armv8_pmull_eor3; 310| | } 311| |#endif 312| | 313| | // Power - VMX 314| |#ifdef PPC_VMX 315| |# ifndef PPC_VMX_NATIVE 316| | if (cf.power.has_altivec) 317| |# endif 318| | { 319| | ft.adler32 = &adler32_vmx; 320| | ft.adler32_copy = &adler32_copy_vmx; 321| | ft.slide_hash = &slide_hash_vmx; 322| | ft.slide_hash_head = &slide_hash_head_vmx; 323| | } 324| |#endif 325| | // Power8 - VSX 326| |#ifdef POWER8_VSX 327| |# ifndef POWER8_VSX_NATIVE 328| | if (cf.power.has_arch_2_07) 329| |# endif 330| | { 331| | ft.adler32 = &adler32_power8; 332| | ft.adler32_copy = &adler32_copy_power8; 333| | ft.chunkmemset_safe = &chunkmemset_safe_power8; 334| | ft.inflate_fast = &inflate_fast_power8; 335| | ft.slide_hash = &slide_hash_power8; 336| | ft.slide_hash_head = &slide_hash_head_power8; 337| | } 338| |#endif 339| |#ifdef POWER8_VSX_CRC32 340| |# ifndef POWER8_VSX_CRC32_NATIVE 341| | if (cf.power.has_arch_2_07) 342| |# endif 343| | { 344| | ft.crc32 = &crc32_power8; 345| | ft.crc32_copy = &crc32_copy_power8; 346| | } 347| |#endif 348| | // Power9 349| |#ifdef POWER9 350| |# ifndef POWER9_NATIVE 351| | if (cf.power.has_arch_3_00) 352| |# endif 353| | { 354| | ft.compare256 = &compare256_power9; 355| | ft.longest_match = &longest_match_power9; 356| | ft.longest_match_roll = &longest_match_roll_power9; 357| | } 358| |#endif 359| | 360| | 361| | // RISCV - RVV 362| |#ifdef RISCV_RVV 363| |# ifndef RISCV_RVV_NATIVE 364| | if (cf.riscv.has_rvv) 365| |# endif 366| | { 367| | ft.adler32 = &adler32_rvv; 368| | ft.adler32_copy = &adler32_copy_rvv; 369| | ft.chunkmemset_safe = &chunkmemset_safe_rvv; 370| | ft.compare256 = &compare256_rvv; 371| | ft.inflate_fast = &inflate_fast_rvv; 372| | ft.longest_match = &longest_match_rvv; 373| | ft.longest_match_roll = &longest_match_roll_rvv; 374| | ft.slide_hash = &slide_hash_rvv; 375| | ft.slide_hash_head = &slide_hash_head_rvv; 376| | } 377| |#endif 378| | 379| | // RISCV - ZBC 380| |#ifdef RISCV_CRC32_ZBC 381| |# ifndef RISCV_ZBC_NATIVE 382| | if (cf.riscv.has_zbc) 383| |# endif 384| | { 385| | ft.crc32 = &crc32_riscv64_zbc; 386| | ft.crc32_copy = &crc32_copy_riscv64_zbc; 387| | } 388| |#endif 389| | 390| | // S390 391| |#ifdef S390_VX 392| |# ifndef S390_VX_NATIVE 393| | if (cf.s390.has_vx) 394| |# endif 395| | { 396| | ft.crc32 = &crc32_s390_vx; 397| | ft.crc32_copy = &crc32_copy_s390_vx; 398| | ft.slide_hash = &slide_hash_vx; 399| | ft.slide_hash_head = &slide_hash_head_vx; 400| | } 401| |#endif 402| | 403| | // LOONGARCH 404| |#ifdef LOONGARCH_CRC 405| |# ifndef LOONGARCH_CRC_NATIVE 406| | if (cf.loongarch.has_crc) 407| |# endif 408| | { 409| | ft.crc32 = &crc32_loongarch64; 410| | ft.crc32_copy = &crc32_copy_loongarch64; 411| | } 412| |#endif 413| |#if defined(LOONGARCH_LSX) && !defined(LOONGARCH_LASX_NATIVE) 414| |# ifndef LOONGARCH_LSX_NATIVE 415| | if (cf.loongarch.has_lsx) 416| |# endif 417| | { 418| | ft.adler32 = &adler32_lsx; 419| | ft.adler32_copy = &adler32_copy_lsx; 420| | ft.chunkmemset_safe = &chunkmemset_safe_lsx; 421| | ft.compare256 = &compare256_lsx; 422| | ft.inflate_fast = &inflate_fast_lsx; 423| | ft.longest_match = &longest_match_lsx; 424| | ft.longest_match_roll = &longest_match_roll_lsx; 425| | ft.slide_hash = &slide_hash_lsx; 426| | ft.slide_hash_head = &slide_hash_head_lsx; 427| | } 428| |#endif 429| |#ifdef LOONGARCH_LASX 430| |# ifndef LOONGARCH_LASX_NATIVE 431| | if (cf.loongarch.has_lasx) 432| |# endif 433| | { 434| | ft.adler32 = &adler32_lasx; 435| | ft.adler32_copy = &adler32_copy_lasx; 436| | ft.chunkmemset_safe = &chunkmemset_safe_lasx; 437| | ft.compare256 = &compare256_lasx; 438| | ft.inflate_fast = &inflate_fast_lasx; 439| | ft.longest_match = &longest_match_lasx; 440| | ft.longest_match_roll = &longest_match_roll_lasx; 441| | ft.slide_hash = &slide_hash_lasx; 442| | ft.slide_hash_head = &slide_hash_head_lasx; 443| | } 444| |#endif 445| | 446| 2|#endif // WITH_OPTIM 447| | 448| | // Assign function pointers individually for atomic operation 449| 2| FUNCTABLE_ASSIGN(ft, force_init); ------------------ | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) ------------------ 450| 2| FUNCTABLE_VERIFY_ASSIGN(ft, adler32); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 451| 2| FUNCTABLE_VERIFY_ASSIGN(ft, adler32_copy); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 452| 2| FUNCTABLE_VERIFY_ASSIGN(ft, chunkmemset_safe); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 453| 2| FUNCTABLE_VERIFY_ASSIGN(ft, compare256); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 454| 2| FUNCTABLE_VERIFY_ASSIGN(ft, crc32); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 455| 2| FUNCTABLE_VERIFY_ASSIGN(ft, crc32_copy); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 456| 2| FUNCTABLE_VERIFY_ASSIGN(ft, inflate_fast); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 457| 2| FUNCTABLE_VERIFY_ASSIGN(ft, longest_match); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 458| 2| FUNCTABLE_VERIFY_ASSIGN(ft, longest_match_roll); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 459| 2| FUNCTABLE_VERIFY_ASSIGN(ft, slide_hash); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 460| 2| FUNCTABLE_VERIFY_ASSIGN(ft, slide_hash_head); ------------------ | | 46| 2| if (!VAR.FUNC_NAME) { \ | | ------------------ | | | Branch (46:9): [True: 0, False: 2] | | ------------------ | | 47| 0| fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \ | | 48| 0| return 1; \ | | 49| 0| } \ | | 50| 2| FUNCTABLE_ASSIGN(VAR, FUNC_NAME); | | ------------------ | | | | 20| 2| __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST) | | ------------------ ------------------ 461| | 462| | // Memory barrier for weak memory order CPUs 463| 2| FUNCTABLE_BARRIER(); ------------------ | | 21| 2|# define FUNCTABLE_BARRIER() __atomic_thread_fence(__ATOMIC_SEQ_CST) ------------------ 464| | 465| 2| return Z_OK; ------------------ | | 180| 2|#define Z_OK 0 ------------------ 466| 2|} LLVMFuzzerTestOneInput: 9| 604|int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataLen) { 10| 604| uint32_t crc0 = PREFIX(crc32)(0L, NULL, 0); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 11| 604| uint32_t crc1 = crc0; 12| 604| uint32_t crc2 = crc0; 13| 604| uint32_t adler0 = PREFIX(adler32)(0L, NULL, 0); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 14| 604| uint32_t adler1 = adler0; 15| 604| uint32_t adler2 = adler0; 16| 604| uint32_t combine1, combine2; 17| | /* Checksum with a buffer of size equal to the first byte in the input. */ 18| 604| uint32_t buffSize = data[0]; 19| 604| uint32_t offset = 0; 20| 604| uint32_t op; 21| | 22| | /* Discard inputs larger than 1Mb. */ 23| 604| static size_t kMaxSize = 1024 * 1024; 24| 604| if (dataLen < 1 || dataLen > kMaxSize) ------------------ | Branch (24:9): [True: 0, False: 604] | Branch (24:24): [True: 0, False: 604] ------------------ 25| 0| return 0; 26| | 27| | /* Make sure the buffer has at least a byte. */ 28| 604| if (buffSize == 0) ------------------ | Branch (28:9): [True: 45, False: 559] ------------------ 29| 45| ++buffSize; 30| | 31| | /* CRC32 */ 32| 604| op = PREFIX(crc32_combine_gen)(buffSize); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 33| 1.55M| for (offset = 0; offset + buffSize <= dataLen; offset += buffSize) { ------------------ | Branch (33:22): [True: 1.55M, False: 604] ------------------ 34| 1.55M| uint32_t crc3 = PREFIX(crc32_z)(crc0, data + offset, buffSize); ------------------ | | 121| 1.55M|# define PREFIX(x) zng_ ## x ------------------ 35| 1.55M| uint32_t crc4 = PREFIX(crc32_combine_op)(crc1, crc3, op); ------------------ | | 121| 1.55M|# define PREFIX(x) zng_ ## x ------------------ 36| 1.55M| crc1 = PREFIX(crc32_z)(crc1, data + offset, buffSize); ------------------ | | 121| 1.55M|# define PREFIX(x) zng_ ## x ------------------ 37| 1.55M| assert(crc1 == crc4); 38| 1.55M| Z_UNUSED(crc1); ------------------ | | 154| 1.55M|#define Z_UNUSED(var) (void)(var) ------------------ 39| 1.55M| Z_UNUSED(crc4); ------------------ | | 154| 1.55M|#define Z_UNUSED(var) (void)(var) ------------------ 40| 1.55M| } 41| 604| crc1 = PREFIX(crc32_z)(crc1, data + offset, dataLen % buffSize); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 42| | 43| 604| crc2 = PREFIX(crc32_z)(crc2, data, dataLen); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 44| | 45| 604| assert(crc1 == crc2); 46| 604| Z_UNUSED(crc1); ------------------ | | 154| 604|#define Z_UNUSED(var) (void)(var) ------------------ 47| 604| Z_UNUSED(crc2); ------------------ | | 154| 604|#define Z_UNUSED(var) (void)(var) ------------------ 48| 604| combine1 = PREFIX(crc32_combine)(crc1, crc2, (z_off_t)dataLen); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 49| 604| combine2 = PREFIX(crc32_combine)(crc1, crc1, (z_off_t)dataLen); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 50| 604| assert(combine1 == combine2); 51| | 52| | /* Fast CRC32 combine. */ 53| 604| op = PREFIX(crc32_combine_gen)((z_off_t)dataLen); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 54| 604| combine1 = PREFIX(crc32_combine_op)(crc1, crc2, op); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 55| 604| combine2 = PREFIX(crc32_combine_op)(crc2, crc1, op); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 56| 604| assert(combine1 == combine2); 57| 604| combine1 = PREFIX(crc32_combine)(crc1, crc2, (z_off_t)dataLen); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 58| 604| combine2 = PREFIX(crc32_combine_op)(crc2, crc1, op); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 59| 604| assert(combine1 == combine2); 60| | 61| | /* Adler32 */ 62| 1.55M| for (offset = 0; offset + buffSize <= dataLen; offset += buffSize) ------------------ | Branch (62:22): [True: 1.55M, False: 604] ------------------ 63| 1.55M| adler1 = PREFIX(adler32_z)(adler1, data + offset, buffSize); ------------------ | | 121| 1.55M|# define PREFIX(x) zng_ ## x ------------------ 64| 604| adler1 = PREFIX(adler32_z)(adler1, data + offset, dataLen % buffSize); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 65| | 66| 604| adler2 = PREFIX(adler32_z)(adler2, data, dataLen); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 67| | 68| 604| assert(adler1 == adler2); 69| 604| Z_UNUSED(adler1); ------------------ | | 154| 604|#define Z_UNUSED(var) (void)(var) ------------------ 70| 604| Z_UNUSED(adler2); ------------------ | | 154| 604|#define Z_UNUSED(var) (void)(var) ------------------ 71| 604| combine1 = PREFIX(adler32_combine)(adler1, adler2, (z_off_t)dataLen); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 72| 604| combine2 = PREFIX(adler32_combine)(adler1, adler1, (z_off_t)dataLen); ------------------ | | 121| 604|# define PREFIX(x) zng_ ## x ------------------ 73| 604| assert(combine1 == combine2); 74| 604| Z_UNUSED(combine1); ------------------ | | 154| 604|#define Z_UNUSED(var) (void)(var) ------------------ 75| 604| Z_UNUSED(combine2); ------------------ | | 154| 604|#define Z_UNUSED(var) (void)(var) ------------------ 76| | 77| | /* This function must return 0. */ 78| 604| return 0; 79| 604|}