zxc_compress.c:rapidhash: 500| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash(const void *key, size_t len) RAPIDHASH_NOEXCEPT { 501| 1.63k| return rapidhash_withSeed(key, len, 0); 502| 1.63k|} zxc_compress.c:rapidhash_withSeed: 486| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_withSeed(const void *key, size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT { 487| 1.63k| return rapidhash_internal(key, len, seed, rapid_secret); 488| 1.63k|} zxc_compress.c:rapidhash_internal: 243| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_internal(const void *key, size_t len, uint64_t seed, const uint64_t* secret) RAPIDHASH_NOEXCEPT { 244| 1.63k| const uint8_t *p=(const uint8_t *)key; 245| 1.63k| seed ^= rapid_mix(seed ^ secret[2], secret[1]); 246| 1.63k| uint64_t a=0, b=0; 247| 1.63k| size_t i = len; 248| 1.63k| if (_likely_(len <= 16)) { ------------------ | | 106| 1.63k| # define _likely_(x) __builtin_expect(x,1) | | ------------------ | | | Branch (106:24): [True: 73, False: 1.56k] | | ------------------ ------------------ 249| 73| if (len >= 4) { ------------------ | Branch (249:9): [True: 52, False: 21] ------------------ 250| 52| seed ^= len; 251| 52| if (len >= 8) { ------------------ | Branch (251:11): [True: 46, False: 6] ------------------ 252| 46| const uint8_t* plast = p + len - 8; 253| 46| a = rapid_read64(p); 254| 46| b = rapid_read64(plast); 255| 46| } else { 256| 6| const uint8_t* plast = p + len - 4; 257| 6| a = rapid_read32(p); 258| 6| b = rapid_read32(plast); 259| 6| } 260| 52| } else if (len > 0) { ------------------ | Branch (260:16): [True: 21, False: 0] ------------------ 261| 21| a = (((uint64_t)p[0])<<45)|p[len-1]; 262| 21| b = p[len>>1]; 263| 21| } else 264| 0| a = b = 0; 265| 1.56k| } else { 266| 1.56k| if (len > 112) { ------------------ | Branch (266:9): [True: 1.24k, False: 316] ------------------ 267| 1.24k| uint64_t see1 = seed, see2 = seed; 268| 1.24k| uint64_t see3 = seed, see4 = seed; 269| 1.24k| uint64_t see5 = seed, see6 = seed; 270| 1.24k|#ifdef RAPIDHASH_COMPACT 271| 563k| do { 272| 563k| seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 273| 563k| see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 274| 563k| see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 275| 563k| see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 276| 563k| see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 277| 563k| see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 278| 563k| see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 279| 563k| p += 112; 280| 563k| i -= 112; 281| 563k| } while(i > 112); ------------------ | Branch (281:15): [True: 562k, False: 1.24k] ------------------ 282| |#else 283| | while (i > 224) { 284| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 285| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 286| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 287| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 288| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 289| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 290| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 291| | seed = rapid_mix(rapid_read64(p + 112) ^ secret[0], rapid_read64(p + 120) ^ seed); 292| | see1 = rapid_mix(rapid_read64(p + 128) ^ secret[1], rapid_read64(p + 136) ^ see1); 293| | see2 = rapid_mix(rapid_read64(p + 144) ^ secret[2], rapid_read64(p + 152) ^ see2); 294| | see3 = rapid_mix(rapid_read64(p + 160) ^ secret[3], rapid_read64(p + 168) ^ see3); 295| | see4 = rapid_mix(rapid_read64(p + 176) ^ secret[4], rapid_read64(p + 184) ^ see4); 296| | see5 = rapid_mix(rapid_read64(p + 192) ^ secret[5], rapid_read64(p + 200) ^ see5); 297| | see6 = rapid_mix(rapid_read64(p + 208) ^ secret[6], rapid_read64(p + 216) ^ see6); 298| | p += 224; 299| | i -= 224; 300| | } 301| | if (i > 112) { 302| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 303| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 304| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 305| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 306| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 307| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 308| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 309| | p += 112; 310| | i -= 112; 311| | } 312| |#endif 313| 1.24k| seed ^= see1; 314| 1.24k| see2 ^= see3; 315| 1.24k| see4 ^= see5; 316| 1.24k| seed ^= see6; 317| 1.24k| see2 ^= see4; 318| 1.24k| seed ^= see2; 319| 1.24k| } 320| 1.56k| if (i > 16) { ------------------ | Branch (320:9): [True: 1.36k, False: 202] ------------------ 321| 1.36k| seed = rapid_mix(rapid_read64(p) ^ secret[2], rapid_read64(p + 8) ^ seed); 322| 1.36k| if (i > 32) { ------------------ | Branch (322:11): [True: 1.14k, False: 218] ------------------ 323| 1.14k| seed = rapid_mix(rapid_read64(p + 16) ^ secret[2], rapid_read64(p + 24) ^ seed); 324| 1.14k| if (i > 48) { ------------------ | Branch (324:15): [True: 939, False: 204] ------------------ 325| 939| seed = rapid_mix(rapid_read64(p + 32) ^ secret[1], rapid_read64(p + 40) ^ seed); 326| 939| if (i > 64) { ------------------ | Branch (326:19): [True: 650, False: 289] ------------------ 327| 650| seed = rapid_mix(rapid_read64(p + 48) ^ secret[1], rapid_read64(p + 56) ^ seed); 328| 650| if (i > 80) { ------------------ | Branch (328:23): [True: 416, False: 234] ------------------ 329| 416| seed = rapid_mix(rapid_read64(p + 64) ^ secret[2], rapid_read64(p + 72) ^ seed); 330| 416| if (i > 96) { ------------------ | Branch (330:27): [True: 181, False: 235] ------------------ 331| 181| seed = rapid_mix(rapid_read64(p + 80) ^ secret[1], rapid_read64(p + 88) ^ seed); 332| 181| } 333| 416| } 334| 650| } 335| 939| } 336| 1.14k| } 337| 1.36k| } 338| 1.56k| a=rapid_read64(p+i-16) ^ i; b=rapid_read64(p+i-8); 339| 1.56k| } 340| 1.63k| a ^= secret[1]; 341| 1.63k| b ^= seed; 342| 1.63k| rapid_mum(&a, &b); 343| 1.63k| return rapid_mix(a ^ secret[7], b ^ secret[1] ^ i); 344| 1.63k|} zxc_compress.c:rapid_mix: 208| 3.95M| RAPIDHASH_INLINE_CONSTEXPR uint64_t rapid_mix(uint64_t A, uint64_t B) RAPIDHASH_NOEXCEPT { rapid_mum(&A,&B); return A^B; } zxc_compress.c:rapid_read64: 214| 7.90M| RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return v;} zxc_compress.c:rapid_read32: 215| 12| RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return v;} zxc_compress.c:rapid_mum: 156| 3.95M| RAPIDHASH_INLINE_CONSTEXPR void rapid_mum(uint64_t *A, uint64_t *B) RAPIDHASH_NOEXCEPT { 157| 3.95M| #if defined(__SIZEOF_INT128__) 158| 3.95M| __uint128_t r=*A; r*=*B; 159| | #ifdef RAPIDHASH_PROTECTED 160| | *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); 161| | #else 162| 3.95M| *A=(uint64_t)r; *B=(uint64_t)(r>>64); 163| 3.95M| #endif 164| | #elif defined(_MSC_VER) && (defined(_WIN64) || defined(_M_HYBRID_CHPE_ARM64)) 165| | #if defined(_M_X64) 166| | #ifdef RAPIDHASH_PROTECTED 167| | uint64_t a, b; 168| | a=_umul128(*A,*B,&b); 169| | *A^=a; *B^=b; 170| | #else 171| | *A=_umul128(*A,*B,B); 172| | #endif 173| | #else 174| | #ifdef RAPIDHASH_PROTECTED 175| | uint64_t a, b; 176| | b = __umulh(*A, *B); 177| | a = *A * *B; 178| | *A^=a; *B^=b; 179| | #else 180| | uint64_t c = __umulh(*A, *B); 181| | *A = *A * *B; 182| | *B = c; 183| | #endif 184| | #endif 185| | #else 186| | uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B; 187| | uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t>32)+(rm1>>32)+c; 191| | #ifdef RAPIDHASH_PROTECTED 192| | *A^=lo; *B^=hi; 193| | #else 194| | *A=lo; *B=hi; 195| | #endif 196| | #endif 197| 3.95M| } zxc_decompress.c:rapidhash: 500| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash(const void *key, size_t len) RAPIDHASH_NOEXCEPT { 501| 1.63k| return rapidhash_withSeed(key, len, 0); 502| 1.63k|} zxc_decompress.c:rapidhash_withSeed: 486| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_withSeed(const void *key, size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT { 487| 1.63k| return rapidhash_internal(key, len, seed, rapid_secret); 488| 1.63k|} zxc_decompress.c:rapidhash_internal: 243| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_internal(const void *key, size_t len, uint64_t seed, const uint64_t* secret) RAPIDHASH_NOEXCEPT { 244| 1.63k| const uint8_t *p=(const uint8_t *)key; 245| 1.63k| seed ^= rapid_mix(seed ^ secret[2], secret[1]); 246| 1.63k| uint64_t a=0, b=0; 247| 1.63k| size_t i = len; 248| 1.63k| if (_likely_(len <= 16)) { ------------------ | | 106| 1.63k| # define _likely_(x) __builtin_expect(x,1) | | ------------------ | | | Branch (106:24): [True: 73, False: 1.56k] | | ------------------ ------------------ 249| 73| if (len >= 4) { ------------------ | Branch (249:9): [True: 52, False: 21] ------------------ 250| 52| seed ^= len; 251| 52| if (len >= 8) { ------------------ | Branch (251:11): [True: 46, False: 6] ------------------ 252| 46| const uint8_t* plast = p + len - 8; 253| 46| a = rapid_read64(p); 254| 46| b = rapid_read64(plast); 255| 46| } else { 256| 6| const uint8_t* plast = p + len - 4; 257| 6| a = rapid_read32(p); 258| 6| b = rapid_read32(plast); 259| 6| } 260| 52| } else if (len > 0) { ------------------ | Branch (260:16): [True: 21, False: 0] ------------------ 261| 21| a = (((uint64_t)p[0])<<45)|p[len-1]; 262| 21| b = p[len>>1]; 263| 21| } else 264| 0| a = b = 0; 265| 1.56k| } else { 266| 1.56k| if (len > 112) { ------------------ | Branch (266:9): [True: 1.24k, False: 316] ------------------ 267| 1.24k| uint64_t see1 = seed, see2 = seed; 268| 1.24k| uint64_t see3 = seed, see4 = seed; 269| 1.24k| uint64_t see5 = seed, see6 = seed; 270| 1.24k|#ifdef RAPIDHASH_COMPACT 271| 563k| do { 272| 563k| seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 273| 563k| see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 274| 563k| see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 275| 563k| see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 276| 563k| see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 277| 563k| see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 278| 563k| see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 279| 563k| p += 112; 280| 563k| i -= 112; 281| 563k| } while(i > 112); ------------------ | Branch (281:15): [True: 562k, False: 1.24k] ------------------ 282| |#else 283| | while (i > 224) { 284| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 285| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 286| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 287| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 288| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 289| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 290| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 291| | seed = rapid_mix(rapid_read64(p + 112) ^ secret[0], rapid_read64(p + 120) ^ seed); 292| | see1 = rapid_mix(rapid_read64(p + 128) ^ secret[1], rapid_read64(p + 136) ^ see1); 293| | see2 = rapid_mix(rapid_read64(p + 144) ^ secret[2], rapid_read64(p + 152) ^ see2); 294| | see3 = rapid_mix(rapid_read64(p + 160) ^ secret[3], rapid_read64(p + 168) ^ see3); 295| | see4 = rapid_mix(rapid_read64(p + 176) ^ secret[4], rapid_read64(p + 184) ^ see4); 296| | see5 = rapid_mix(rapid_read64(p + 192) ^ secret[5], rapid_read64(p + 200) ^ see5); 297| | see6 = rapid_mix(rapid_read64(p + 208) ^ secret[6], rapid_read64(p + 216) ^ see6); 298| | p += 224; 299| | i -= 224; 300| | } 301| | if (i > 112) { 302| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 303| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 304| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 305| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 306| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 307| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 308| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 309| | p += 112; 310| | i -= 112; 311| | } 312| |#endif 313| 1.24k| seed ^= see1; 314| 1.24k| see2 ^= see3; 315| 1.24k| see4 ^= see5; 316| 1.24k| seed ^= see6; 317| 1.24k| see2 ^= see4; 318| 1.24k| seed ^= see2; 319| 1.24k| } 320| 1.56k| if (i > 16) { ------------------ | Branch (320:9): [True: 1.36k, False: 202] ------------------ 321| 1.36k| seed = rapid_mix(rapid_read64(p) ^ secret[2], rapid_read64(p + 8) ^ seed); 322| 1.36k| if (i > 32) { ------------------ | Branch (322:11): [True: 1.14k, False: 218] ------------------ 323| 1.14k| seed = rapid_mix(rapid_read64(p + 16) ^ secret[2], rapid_read64(p + 24) ^ seed); 324| 1.14k| if (i > 48) { ------------------ | Branch (324:15): [True: 939, False: 204] ------------------ 325| 939| seed = rapid_mix(rapid_read64(p + 32) ^ secret[1], rapid_read64(p + 40) ^ seed); 326| 939| if (i > 64) { ------------------ | Branch (326:19): [True: 650, False: 289] ------------------ 327| 650| seed = rapid_mix(rapid_read64(p + 48) ^ secret[1], rapid_read64(p + 56) ^ seed); 328| 650| if (i > 80) { ------------------ | Branch (328:23): [True: 416, False: 234] ------------------ 329| 416| seed = rapid_mix(rapid_read64(p + 64) ^ secret[2], rapid_read64(p + 72) ^ seed); 330| 416| if (i > 96) { ------------------ | Branch (330:27): [True: 181, False: 235] ------------------ 331| 181| seed = rapid_mix(rapid_read64(p + 80) ^ secret[1], rapid_read64(p + 88) ^ seed); 332| 181| } 333| 416| } 334| 650| } 335| 939| } 336| 1.14k| } 337| 1.36k| } 338| 1.56k| a=rapid_read64(p+i-16) ^ i; b=rapid_read64(p+i-8); 339| 1.56k| } 340| 1.63k| a ^= secret[1]; 341| 1.63k| b ^= seed; 342| 1.63k| rapid_mum(&a, &b); 343| 1.63k| return rapid_mix(a ^ secret[7], b ^ secret[1] ^ i); 344| 1.63k|} zxc_decompress.c:rapid_mix: 208| 3.95M| RAPIDHASH_INLINE_CONSTEXPR uint64_t rapid_mix(uint64_t A, uint64_t B) RAPIDHASH_NOEXCEPT { rapid_mum(&A,&B); return A^B; } zxc_decompress.c:rapid_read64: 214| 7.90M| RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return v;} zxc_decompress.c:rapid_read32: 215| 12| RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return v;} zxc_decompress.c:rapid_mum: 156| 3.95M| RAPIDHASH_INLINE_CONSTEXPR void rapid_mum(uint64_t *A, uint64_t *B) RAPIDHASH_NOEXCEPT { 157| 3.95M| #if defined(__SIZEOF_INT128__) 158| 3.95M| __uint128_t r=*A; r*=*B; 159| | #ifdef RAPIDHASH_PROTECTED 160| | *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); 161| | #else 162| 3.95M| *A=(uint64_t)r; *B=(uint64_t)(r>>64); 163| 3.95M| #endif 164| | #elif defined(_MSC_VER) && (defined(_WIN64) || defined(_M_HYBRID_CHPE_ARM64)) 165| | #if defined(_M_X64) 166| | #ifdef RAPIDHASH_PROTECTED 167| | uint64_t a, b; 168| | a=_umul128(*A,*B,&b); 169| | *A^=a; *B^=b; 170| | #else 171| | *A=_umul128(*A,*B,B); 172| | #endif 173| | #else 174| | #ifdef RAPIDHASH_PROTECTED 175| | uint64_t a, b; 176| | b = __umulh(*A, *B); 177| | a = *A * *B; 178| | *A^=a; *B^=b; 179| | #else 180| | uint64_t c = __umulh(*A, *B); 181| | *A = *A * *B; 182| | *B = c; 183| | #endif 184| | #endif 185| | #else 186| | uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B; 187| | uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t>32)+(rm1>>32)+c; 191| | #ifdef RAPIDHASH_PROTECTED 192| | *A^=lo; *B^=hi; 193| | #else 194| | *A=lo; *B=hi; 195| | #endif 196| | #endif 197| 3.95M| } zxc_aligned_malloc: 30| 9.21k|void* zxc_aligned_malloc(const size_t size, const size_t alignment) { 31| |#if defined(_WIN32) 32| | return _aligned_malloc(size, alignment); 33| |#else 34| 9.21k| void* ptr = NULL; 35| 9.21k| if (posix_memalign(&ptr, alignment, size) != 0) return NULL; ------------------ | Branch (35:9): [True: 0, False: 9.21k] ------------------ 36| 9.21k| return ptr; 37| 9.21k|#endif 38| 9.21k|} zxc_aligned_free: 43| 9.21k|void zxc_aligned_free(void* ptr) { 44| |#if defined(_WIN32) 45| | _aligned_free(ptr); 46| |#else 47| 9.21k| free(ptr); 48| 9.21k|#endif 49| 9.21k|} zxc_cctx_init_in_workspace: 277| 9.21k| const int defer_entropy_scratch) { 278| 9.21k| if (UNLIKELY(!ctx || !workspace || chunk_size == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 36.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | ------------------ ------------------ 279| | 280| 9.21k| const zxc_cctx_layout_t layout = 281| 9.21k| compute_cctx_layout(chunk_size, mode, level, dict_size, defer_entropy_scratch); 282| 9.21k| if (UNLIKELY(workspace_size < layout.total)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 283| | 284| 9.21k| ZXC_MEMSET(ctx, 0, sizeof(zxc_cctx_t)); ------------------ | | 172| 9.21k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 285| 9.21k| ctx->checksum_enabled = checksum_enabled; 286| 9.21k| ctx->chunk_size = chunk_size; 287| 9.21k| const uint32_t offset_bits = zxc_log2_u32((uint32_t)chunk_size); 288| 9.21k| ctx->offset_bits = offset_bits; 289| 9.21k| ctx->offset_mask = (uint32_t)((1ULL << offset_bits) - 1); 290| 9.21k| ctx->max_epoch = (uint32_t)(1ULL << (32 - offset_bits)); 291| | 292| | // memory_block stays NULL on the static-init path so zxc_cctx_free does 293| | // not try to free the caller's workspace. Sub-buffer pointers carry the 294| | // partition; ownership is implicit (the caller owns @p workspace). 295| 9.21k| uint8_t* const mem = (uint8_t*)workspace; 296| | 297| | // Dictionary concat scratch (both modes); init owns dict_size now so callers 298| | // no longer assign ctx->dict_size after init. 299| 9.21k| ctx->dict_size = dict_size; 300| 9.21k| if (dict_size > 0) { ------------------ | Branch (300:9): [True: 0, False: 9.21k] ------------------ 301| 0| ctx->dict_buffer = mem + layout.off_dict; 302| 0| ctx->dict_buffer_cap = layout.sz_dict; 303| 0| ctx->dict_huf = (zxc_dict_huf_state_t*)(void*)(mem + layout.off_dict_huf); 304| 0| } 305| | 306| 9.21k| if (mode == 0) { ------------------ | Branch (306:9): [True: 4.60k, False: 4.60k] ------------------ 307| 4.60k| ctx->work_buf = mem + layout.off_work; 308| 4.60k| ctx->work_buf_cap = chunk_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 354| 4.60k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 328| 4.60k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 309| 4.60k| ctx->lit_buffer = mem + layout.off_lit_dctx; 310| 4.60k| ctx->lit_buffer_cap = chunk_size + ZXC_PAD_SIZE; ------------------ | | 328| 4.60k|#define ZXC_PAD_SIZE 32 ------------------ 311| 4.60k| if (layout.sz_pivco_dctx) { ------------------ | Branch (311:13): [True: 0, False: 4.60k] ------------------ 312| 0| ctx->tok_buffer = mem + layout.off_tok_dctx; 313| 0| ctx->tok_buffer_cap = layout.sz_tok_dctx; 314| 0| ctx->pivco_scratch = mem + layout.off_pivco_dctx; 315| 0| ctx->pivco_scratch_cap = layout.sz_pivco_dctx; 316| 0| } 317| 4.60k| return ZXC_OK; 318| 4.60k| } 319| | 320| 4.60k| ctx->hash_table = (uint32_t*)(mem + layout.off_hash_pos); 321| 4.60k| ctx->hash_tags = mem + layout.off_hash_tags; 322| 4.60k| ctx->chain_table = (uint16_t*)(mem + layout.off_chain); 323| 4.60k| ctx->buf_sequences = (uint32_t*)(mem + layout.off_seq_union); 324| 4.60k| ctx->buf_offsets = (uint16_t*)(mem + layout.off_seq_union); 325| 4.60k| ctx->buf_tokens = mem + layout.off_seq_union + layout.max_seq * sizeof(uint16_t); 326| 4.60k| ctx->buf_extras = mem + layout.off_extras; 327| 4.60k| ctx->literals = mem + layout.off_lit_cctx; 328| 4.60k| if (layout.sz_opt) { ------------------ | Branch (328:9): [True: 0, False: 4.60k] ------------------ 329| 0| ctx->opt_scratch = mem + layout.off_opt; 330| 0| ctx->opt_scratch_cap = layout.sz_opt; 331| 0| } 332| | 333| 4.60k| ctx->compression_level = level; 334| 4.60k| ctx->epoch = 1; 335| | 336| 4.60k| ZXC_MEMSET(ctx->hash_table, 0, layout.sz_hash_pos); ------------------ | | 172| 4.60k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 337| 4.60k| ZXC_MEMSET(ctx->hash_tags, 0, layout.sz_hash_tags); ------------------ | | 172| 4.60k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 338| 4.60k| return ZXC_OK; 339| 9.21k|} zxc_cctx_init: 352| 9.21k| const int level, const int checksum_enabled, const size_t dict_size) { 353| 9.21k| if (UNLIKELY(chunk_size == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 354| | // Heap contexts defer the decode-side entropy scratch to the first entropy 355| | // section; static workspaces must have it already, they may never allocate. 356| 9.21k| const int defer_entropy = (mode == 0); 357| 9.21k| const size_t total = 358| 9.21k| compute_cctx_layout(chunk_size, mode, level, dict_size, defer_entropy).total; 359| 9.21k| if (UNLIKELY(total == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 360| | 361| 9.21k| uint8_t* const mem = (uint8_t*)ZXC_ALIGNED_MALLOC(total, ZXC_CACHE_LINE_SIZE); ------------------ | | 110| 9.21k|#define ZXC_ALIGNED_MALLOC(size, alignment) zxc_aligned_malloc(size, alignment) ------------------ 362| 9.21k| if (UNLIKELY(!mem)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 363| | 364| 9.21k| const int rc = zxc_cctx_init_in_workspace(ctx, mem, total, chunk_size, mode, level, 365| 9.21k| checksum_enabled, dict_size, defer_entropy); 366| 9.21k| if (UNLIKELY(rc != ZXC_OK)) { ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 367| | // LCOV_EXCL_START 368| 0| ZXC_ALIGNED_FREE(mem); ------------------ | | 116| 0|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 369| 0| return rc; 370| | // LCOV_EXCL_STOP 371| 0| } 372| | // Library-owned buffer: record the allocation so zxc_cctx_free frees it. 373| 9.21k| ctx->memory_block = mem; 374| 9.21k| return ZXC_OK; 375| 9.21k|} zxc_cctx_free: 410| 9.21k|void zxc_cctx_free(zxc_cctx_t* ctx) { 411| 9.21k| if (ctx->memory_block) { ------------------ | Branch (411:9): [True: 9.21k, False: 0] ------------------ 412| 9.21k| ZXC_ALIGNED_FREE(ctx->memory_block); ------------------ | | 116| 9.21k|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 413| 9.21k| ctx->memory_block = NULL; 414| 9.21k| } 415| 9.21k| if (ctx->entropy_block) { ------------------ | Branch (415:9): [True: 0, False: 9.21k] ------------------ 416| 0| ZXC_ALIGNED_FREE(ctx->entropy_block); ------------------ | | 116| 0|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 417| 0| ctx->entropy_block = NULL; 418| 0| } 419| | 420| 9.21k| ctx->lit_buffer = NULL; 421| 9.21k| ctx->hash_table = NULL; 422| 9.21k| ctx->hash_tags = NULL; 423| 9.21k| ctx->chain_table = NULL; 424| 9.21k| ctx->buf_sequences = NULL; 425| 9.21k| ctx->buf_tokens = NULL; 426| 9.21k| ctx->buf_offsets = NULL; 427| 9.21k| ctx->buf_extras = NULL; 428| 9.21k| ctx->literals = NULL; 429| 9.21k| ctx->work_buf = NULL; 430| 9.21k| ctx->tok_buffer = NULL; 431| 9.21k| ctx->pivco_scratch = NULL; 432| 9.21k| ctx->opt_scratch = NULL; 433| 9.21k| ctx->dict_buffer = NULL; 434| 9.21k| ctx->dict_huf = NULL; 435| | 436| 9.21k| ctx->epoch = 0; 437| 9.21k| ctx->lit_buffer_cap = 0; 438| 9.21k| ctx->work_buf_cap = 0; 439| 9.21k| ctx->tok_buffer_cap = 0; 440| 9.21k| ctx->pivco_scratch_cap = 0; 441| 9.21k| ctx->opt_scratch_cap = 0; 442| 9.21k| ctx->dict_buffer_cap = 0; 443| 9.21k| ctx->dict_size = 0; 444| 9.21k| ctx->dict_huf_tree_ok = 0; 445| | ctx->lit_freq_acc = NULL; 446| 9.21k|} zxc_write_file_header: 497| 4.60k| const int has_checksum, const uint32_t dict_id) { 498| 4.60k| if (UNLIKELY(dst_capacity < ZXC_FILE_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 499| | 500| 4.60k| zxc_store_le32(dst, ZXC_MAGIC_WORD); ------------------ | | 322| 4.60k|#define ZXC_MAGIC_WORD 0x9CB02EF5U ------------------ 501| 4.60k| dst[4] = ZXC_FILE_FORMAT_VERSION; ------------------ | | 325| 4.60k|#define ZXC_FILE_FORMAT_VERSION 8 ------------------ 502| | 503| | // Block size stored as log2 exponent (e.g. 18 = 256 KB) 504| 4.60k| dst[5] = (uint8_t)zxc_log2_u32((uint32_t)chunk_size); 505| | 506| 4.60k| uint8_t flags = has_checksum ? (ZXC_FILE_FLAG_HAS_CHECKSUM | ZXC_CHECKSUM_RAPIDHASH) : 0; ------------------ | | 370| 1.53k|#define ZXC_FILE_FLAG_HAS_CHECKSUM 0x80U ------------------ uint8_t flags = has_checksum ? (ZXC_FILE_FLAG_HAS_CHECKSUM | ZXC_CHECKSUM_RAPIDHASH) : 0; ------------------ | | 421| 1.53k|#define ZXC_CHECKSUM_RAPIDHASH 0 ------------------ | Branch (506:21): [True: 1.53k, False: 3.06k] ------------------ 507| 4.60k| if (dict_id != 0) flags |= ZXC_FILE_FLAG_HAS_DICTIONARY; ------------------ | | 372| 0|#define ZXC_FILE_FLAG_HAS_DICTIONARY 0x40U ------------------ | Branch (507:9): [True: 0, False: 4.60k] ------------------ 508| 4.60k| dst[6] = flags; 509| | 510| | // Bytes 7-13: Reserved / dict_id 511| 4.60k| ZXC_MEMSET(dst + 7, 0, 7); ------------------ | | 172| 4.60k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 512| 4.60k| if (dict_id != 0) zxc_store_le32(dst + 7, dict_id); ------------------ | Branch (512:9): [True: 0, False: 4.60k] ------------------ 513| | 514| | // Bytes 14-15: checksum (16-bit) 515| 4.60k| zxc_store_le16(dst + 14, 0); // Zero out before hashing 516| 4.60k| const uint16_t crc = zxc_hash16(dst); 517| 4.60k| zxc_store_le16(dst + 14, crc); 518| | 519| 4.60k| return ZXC_FILE_HEADER_SIZE; ------------------ | | 104| 4.60k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ 520| 4.60k|} zxc_read_file_header: 529| 9.10k| uint32_t* RESTRICT out_dict_id) { 530| 9.10k| if (UNLIKELY(src_size < ZXC_FILE_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 9.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.10k] | | ------------------ ------------------ 531| 9.10k| if (UNLIKELY(zxc_le32(src) != ZXC_MAGIC_WORD)) return ZXC_ERROR_BAD_MAGIC; ------------------ | | 151| 9.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 4.37k, False: 4.72k] | | ------------------ ------------------ 532| 4.72k| if (UNLIKELY(src[4] != ZXC_FILE_FORMAT_VERSION)) return ZXC_ERROR_BAD_VERSION; ------------------ | | 151| 4.72k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 123, False: 4.60k] | | ------------------ ------------------ 533| | 534| 4.60k| uint8_t temp[ZXC_FILE_HEADER_SIZE]; 535| 4.60k| ZXC_MEMCPY(temp, src, ZXC_FILE_HEADER_SIZE); ------------------ | | 167| 4.60k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 536| | // Zero out checksum bytes (14-15) before hash check 537| 4.60k| temp[14] = 0; 538| 4.60k| temp[15] = 0; 539| | // Header checksum (integrity), then the checksum-algorithm id in flags bits 0-3 540| | // (only 0 = RapidHash is defined). It is checked first via short-circuit. 541| 4.60k| if (UNLIKELY(zxc_le16(src + 14) != zxc_hash16(temp) || ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 542| 4.60k| (src[6] & ZXC_FILE_CHECKSUM_ALGO_MASK) != ZXC_CHECKSUM_RAPIDHASH)) 543| 0| return ZXC_ERROR_BAD_HEADER; 544| | 545| 4.60k| if (out_block_size) { ------------------ | Branch (545:9): [True: 4.60k, False: 0] ------------------ 546| 4.60k| const uint8_t code = src[5]; 547| 4.60k| if (UNLIKELY(code < ZXC_BLOCK_SIZE_MIN_LOG2 || code > ZXC_BLOCK_SIZE_MAX_LOG2)) ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 548| 0| return ZXC_ERROR_BAD_BLOCK_SIZE; 549| | // Exponent encoding: block_size = 2^code (4 KB - 2 MB) 550| 4.60k| *out_block_size = (size_t)1U << code; 551| 4.60k| } 552| 4.60k| if (out_has_checksum) *out_has_checksum = (src[6] & ZXC_FILE_FLAG_HAS_CHECKSUM) ? 1 : 0; ------------------ | | 370| 4.60k|#define ZXC_FILE_FLAG_HAS_CHECKSUM 0x80U ------------------ | Branch (552:9): [True: 4.60k, False: 0] | Branch (552:47): [True: 1.53k, False: 3.06k] ------------------ 553| 4.60k| if (out_dict_id) *out_dict_id = (src[6] & ZXC_FILE_FLAG_HAS_DICTIONARY) ? zxc_le32(src + 7) : 0; ------------------ | | 372| 0|#define ZXC_FILE_FLAG_HAS_DICTIONARY 0x40U ------------------ | Branch (553:9): [True: 0, False: 4.60k] | Branch (553:37): [True: 0, False: 0] ------------------ 554| | 555| 4.60k| return ZXC_OK; 556| 4.60k|} zxc_write_block_header: 562| 10.3k| const zxc_block_header_t* RESTRICT bh) { 563| 10.3k| if (UNLIKELY(dst_capacity < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 10.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 10.3k] | | ------------------ ------------------ 564| | 565| 10.3k| dst[0] = bh->block_type; 566| 10.3k| dst[1] = 0; // Flags not used currently 567| 10.3k| dst[2] = 0; // Reserved 568| 10.3k| zxc_store_le32(dst + 3, bh->comp_size); 569| 10.3k| dst[7] = 0; // Zero before hashing 570| 10.3k| dst[7] = zxc_hash8(dst); // Checksum at the end 571| | 572| 10.3k| return ZXC_BLOCK_HEADER_SIZE; ------------------ | | 402| 10.3k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 573| 10.3k|} zxc_read_block_header: 581| 11.3k| zxc_block_header_t* RESTRICT bh) { 582| 11.3k| if (UNLIKELY(src_size < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 11.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 11.3k] | | ------------------ ------------------ 583| | 584| 11.3k| uint8_t temp[ZXC_BLOCK_HEADER_SIZE]; 585| 11.3k| ZXC_MEMCPY(temp, src, ZXC_BLOCK_HEADER_SIZE); ------------------ | | 167| 11.3k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 586| 11.3k| temp[7] = 0; // Zero out checksum byte before hashing 587| 11.3k| if (UNLIKELY(src[7] != zxc_hash8(temp))) return ZXC_ERROR_BAD_HEADER; ------------------ | | 151| 11.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 3.14k, False: 8.18k] | | ------------------ ------------------ 588| | 589| 8.18k| bh->block_type = src[0]; 590| 8.18k| bh->block_flags = 0; // Flags not used currently 591| 8.18k| bh->reserved = src[2]; 592| 8.18k| bh->comp_size = zxc_le32(src + 3); 593| 8.18k| bh->header_crc = src[7]; 594| | 595| 8.18k| return ZXC_OK; 596| 11.3k|} zxc_write_file_footer: 602| 4.60k| const uint32_t global_hash, const int checksum_enabled) { 603| 4.60k| if (UNLIKELY(dst_capacity < ZXC_FILE_FOOTER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 604| | 605| 4.60k| zxc_store_le64(dst, src_size); 606| | 607| 4.60k| if (checksum_enabled) { ------------------ | Branch (607:9): [True: 1.53k, False: 3.06k] ------------------ 608| 1.53k| zxc_store_le32(dst + sizeof(uint64_t), global_hash); 609| 3.06k| } else { 610| 3.06k| ZXC_MEMSET(dst + sizeof(uint64_t), 0, sizeof(uint32_t)); ------------------ | | 172| 3.06k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 611| 3.06k| } 612| | 613| 4.60k| return ZXC_FILE_FOOTER_SIZE; ------------------ | | 106| 4.60k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 614| 4.60k|} zxc_write_ghi_header: 726| 4.90k| const zxc_gnr_header_t* RESTRICT gh) { 727| 4.90k| if (UNLIKELY(rem < ZXC_GHI_HEADER_BINARY_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 728| | 729| 4.90k| zxc_write_gnr_header(dst, gh); 730| 4.90k| return ZXC_GHI_HEADER_BINARY_SIZE; ------------------ | | 408| 4.90k|#define ZXC_GHI_HEADER_BINARY_SIZE 12 ------------------ 731| 4.90k|} zxc_read_ghi_header: 737| 4.10k| zxc_gnr_header_t* RESTRICT gh) { 738| 4.10k| if (UNLIKELY(len < ZXC_GHI_HEADER_BINARY_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 739| | 740| 4.10k| zxc_read_gnr_header(src, gh); 741| 4.10k| return ZXC_OK; 742| 4.10k|} zxc_compress_bound: 757| 4.60k|uint64_t zxc_compress_bound(const size_t input_size) { 758| | // Guard against uint64 overflow when summing per-block overhead 759| | // across very large inputs (input_size approaching SIZE_MAX). 760| 4.60k| if (UNLIKELY(input_size > (SIZE_MAX - (SIZE_MAX >> 8)))) return 0; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 761| 4.60k| uint64_t n = ((uint64_t)input_size + ZXC_BLOCK_SIZE_MIN - 1) / ZXC_BLOCK_SIZE_MIN; ------------------ | | 62| 4.60k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 4.60k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ uint64_t n = ((uint64_t)input_size + ZXC_BLOCK_SIZE_MIN - 1) / ZXC_BLOCK_SIZE_MIN; ------------------ | | 62| 4.60k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 4.60k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ 762| 4.60k| if (n == 0) n = 1; ------------------ | Branch (762:9): [True: 0, False: 4.60k] ------------------ 763| 4.60k| return ZXC_FILE_HEADER_SIZE + ------------------ | | 104| 4.60k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ 764| 4.60k| (n * (ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_CHECKSUM_SIZE + ZXC_BLOCK_FORMAT_OVERHEAD)) + ------------------ | | 402| 4.60k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ (n * (ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_CHECKSUM_SIZE + ZXC_BLOCK_FORMAT_OVERHEAD)) + ------------------ | | 404| 4.60k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ (n * (ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_CHECKSUM_SIZE + ZXC_BLOCK_FORMAT_OVERHEAD)) + ------------------ | | 418| 4.60k|#define ZXC_BLOCK_FORMAT_OVERHEAD 68 ------------------ 765| 4.60k| (uint64_t)input_size + ZXC_BLOCK_HEADER_SIZE + /* EOF block */ ------------------ | | 402| 4.60k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 766| 4.60k| ZXC_BLOCK_HEADER_SIZE + /* SEK block header (seekable) */ ------------------ | | 402| 4.60k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 767| 4.60k| (n * ZXC_SEEK_ENTRY_SIZE) + /* SEK entries: 4 bytes per block */ ------------------ | | 440| 4.60k|#define ZXC_SEEK_ENTRY_SIZE 4 ------------------ 768| 4.60k| ZXC_FILE_FOOTER_SIZE; ------------------ | | 106| 4.60k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 769| 4.60k|} zxc_compress_block_bound: 774| 9.51k|uint64_t zxc_compress_block_bound(const size_t input_size) { 775| | // Mirrors the Block API contract: outside [1, ZXC_BLOCK_SIZE_MAX] the call 776| | // would fail anyway, so the bound is undefined and 0 says "unusable". The cap 777| | // also makes the addition below trivially overflow-free. 778| 9.51k| if (UNLIKELY(input_size == 0 || input_size > ZXC_BLOCK_SIZE_MAX)) return 0; ------------------ | | 151| 19.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.51k] | | | Branch (151:42): [True: 0, False: 9.51k] | | | Branch (151:42): [True: 0, False: 9.51k] | | ------------------ ------------------ 779| | // Outer block header + payload (worst case: incompressible, raw bytes) 780| | // + inner format overhead + optional checksum. 781| 9.51k| return (uint64_t)ZXC_BLOCK_HEADER_SIZE + (uint64_t)input_size + ZXC_BLOCK_FORMAT_OVERHEAD + ------------------ | | 402| 9.51k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ return (uint64_t)ZXC_BLOCK_HEADER_SIZE + (uint64_t)input_size + ZXC_BLOCK_FORMAT_OVERHEAD + ------------------ | | 418| 9.51k|#define ZXC_BLOCK_FORMAT_OVERHEAD 68 ------------------ 782| 9.51k| ZXC_BLOCK_CHECKSUM_SIZE; ------------------ | | 404| 9.51k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ 783| 9.51k|} zxc_common.c:compute_cctx_layout: 158| 18.4k| const int defer_entropy_scratch) { 159| 18.4k| zxc_cctx_layout_t layout = {0}; 160| | 161| 18.4k| const size_t max_seq = zxc_cctx_max_seq(chunk_size); 162| | 163| 18.4k| if (mode == 0) { ------------------ | Branch (163:9): [True: 9.21k, False: 9.21k] ------------------ 164| | // Decompress: work_buf + lit_buffer, padded for wild-copy overshoot and 165| | // sized worst-case. lit_buffer is provisioned at every level - the decoder 166| | // cannot predict a block's literal encoding (RAW / RLE / HUFFMAN). 167| 9.21k| const size_t sz_work = chunk_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 354| 9.21k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 328| 9.21k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 168| 9.21k| const size_t sz_lit = chunk_size + ZXC_PAD_SIZE; ------------------ | | 328| 9.21k|#define ZXC_PAD_SIZE 32 ------------------ 169| | 170| 9.21k| layout.off_work = layout.total; 171| 9.21k| layout.total += ZXC_ALIGN_CL(sz_work); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 172| 9.21k| layout.off_lit_dctx = layout.total; 173| 9.21k| layout.total += ZXC_ALIGN_CL(sz_lit); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 174| | // Token-section decode scratch (level-7 GLO Huffman-codes the tokens) + 175| | // PivCo ping-pong scratch. enc_lit/enc_tok are unpredictable, so static 176| | // workspaces provision both up front (no-alloc contract); heap contexts 177| | // defer them to the first entropy section, sparing L1-5 archives ~1.2x 178| | // chunk_size. See zxc_cctx_alloc_entropy_scratch. 179| 9.21k| if (!defer_entropy_scratch) { ------------------ | Branch (179:13): [True: 0, False: 9.21k] ------------------ 180| 0| size_t sz_tok = 0; 181| 0| size_t sz_pivco = 0; 182| 0| zxc_dctx_entropy_sizes(chunk_size, &sz_tok, &sz_pivco); 183| 0| layout.sz_tok_dctx = sz_tok; 184| 0| layout.off_tok_dctx = layout.total; 185| 0| layout.total += ZXC_ALIGN_CL(layout.sz_tok_dctx); ------------------ | | 360| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 186| 0| layout.sz_pivco_dctx = sz_pivco; 187| 0| layout.off_pivco_dctx = layout.total; 188| 0| layout.total += ZXC_ALIGN_CL(layout.sz_pivco_dctx); ------------------ | | 360| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 189| 0| } 190| 9.21k| } else { 191| | // Compress: 6 partitions + optional opt_scratch at level >= ZXC_LEVEL_DENSITY. 192| 9.21k| const uint32_t offset_bits = zxc_log2_u32((uint32_t)chunk_size); 193| 9.21k| layout.max_seq = max_seq; 194| 9.21k| layout.sz_hash_pos = ZXC_LZ_HASH_SIZE * sizeof(uint32_t); ------------------ | | 499| 9.21k|#define ZXC_LZ_HASH_SIZE (1U << ZXC_LZ_HASH_BITS) | | ------------------ | | | | 493| 9.21k|#define ZXC_LZ_HASH_BITS 15 | | ------------------ ------------------ 195| 9.21k| layout.sz_hash_tags = ZXC_LZ_HASH_SIZE * sizeof(uint8_t); ------------------ | | 499| 9.21k|#define ZXC_LZ_HASH_SIZE (1U << ZXC_LZ_HASH_BITS) | | ------------------ | | | | 493| 9.21k|#define ZXC_LZ_HASH_BITS 15 | | ------------------ ------------------ 196| 9.21k| const size_t sz_chain = ZXC_LZ_WINDOW_SIZE * sizeof(uint16_t); ------------------ | | 501| 9.21k|#define ZXC_LZ_WINDOW_SIZE (1U << 16) ------------------ 197| | // buf_sequences (GHI, level <= ZXC_LEVEL_FAST) aliases buf_offsets + buf_tokens (GLO, 198| | // level >= ZXC_LEVEL_DEFAULT). Mutually exclusive per block; sized for the larger. 199| 9.21k| const size_t sz_seq_union = layout.max_seq * sizeof(uint32_t); 200| 9.21k| const size_t vbyte_len = (offset_bits + 6) / 7; 201| 9.21k| const size_t sz_extras = layout.max_seq * 2 * vbyte_len; 202| 9.21k| const size_t sz_lit = chunk_size + ZXC_PAD_SIZE; ------------------ | | 328| 9.21k|#define ZXC_PAD_SIZE 32 ------------------ 203| | 204| | // opt_scratch (level >= ZXC_LEVEL_DENSITY): the optimal parser's DP arrays, 205| | // reused transiently as package-merge scratch by the code-length builder, 206| | // so sized to the larger demand. Keep in sync with zxc_estimate_cctx_size() 207| | // and its consumer in zxc_compress.c. 208| 9.21k| if (level >= ZXC_LEVEL_DENSITY) { ------------------ | Branch (208:13): [True: 0, False: 9.21k] ------------------ 209| 0| size_t sz_dp; 210| 0| size_t sz_pl; 211| 0| size_t sz_po; 212| 0| size_t sz_bm; 213| 0| zxc_opt_dp_sizes(chunk_size, &sz_dp, &sz_pl, &sz_po, &sz_bm); 214| 0| const size_t dp_needed = sz_dp + sz_pl + sz_po + sz_bm; 215| 0| layout.sz_opt = 216| 0| (dp_needed > ZXC_HUF_BUILD_SCRATCH_SIZE) ? dp_needed : ZXC_HUF_BUILD_SCRATCH_SIZE; ------------------ | | 883| 0| ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 610| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 876| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 617| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 884| 0| sizeof(zxc_huf_pm_item_t) + \ | | 885| 0| 8U + (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * sizeof(int) + 8U + \ | | ------------------ | | | | 610| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | 886| 0| (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 610| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 876| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 617| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 887| 0| sizeof(zxc_huf_pm_frame_t)) ------------------ (dp_needed > ZXC_HUF_BUILD_SCRATCH_SIZE) ? dp_needed : ZXC_HUF_BUILD_SCRATCH_SIZE; ------------------ | | 883| 0| ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 610| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 876| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 617| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 884| 0| sizeof(zxc_huf_pm_item_t) + \ | | 885| 0| 8U + (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * sizeof(int) + 8U + \ | | ------------------ | | | | 610| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | 886| 0| (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 610| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 876| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 617| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 887| 0| sizeof(zxc_huf_pm_frame_t)) ------------------ | Branch (216:17): [True: 0, False: 0] ------------------ 217| 0| } 218| | 219| 9.21k| layout.off_hash_pos = layout.total; 220| 9.21k| layout.total += ZXC_ALIGN_CL(layout.sz_hash_pos); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 221| 9.21k| layout.off_hash_tags = layout.total; 222| 9.21k| layout.total += ZXC_ALIGN_CL(layout.sz_hash_tags); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 223| 9.21k| layout.off_chain = layout.total; 224| 9.21k| layout.total += ZXC_ALIGN_CL(sz_chain); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 225| 9.21k| layout.off_seq_union = layout.total; 226| 9.21k| layout.total += ZXC_ALIGN_CL(sz_seq_union); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 227| 9.21k| layout.off_extras = layout.total; 228| 9.21k| layout.total += ZXC_ALIGN_CL(sz_extras); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 229| 9.21k| layout.off_lit_cctx = layout.total; 230| 9.21k| layout.total += ZXC_ALIGN_CL(sz_lit); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 231| | // opt_scratch is appended last so it is absent for levels 1..5 (zero 232| | // waste on the common path) and only inflates the workspace at level 6. 233| 9.21k| if (layout.sz_opt) { ------------------ | Branch (233:13): [True: 0, False: 9.21k] ------------------ 234| 0| layout.off_opt = layout.total; 235| 0| layout.total += ZXC_ALIGN_CL(layout.sz_opt); ------------------ | | 360| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 236| 0| } 237| 9.21k| } 238| | 239| | // [dict | data] concat scratch (dict only). Compress chunk_size already 240| | // spans [dict | block]; decompress prepends dict to a (chunk + PAD) region. 241| 18.4k| if (dict_size > 0) { ------------------ | Branch (241:9): [True: 0, False: 18.4k] ------------------ 242| 0| layout.sz_dict = (mode == 1) ? (chunk_size + ZXC_DECOMPRESS_TAIL_PAD) ------------------ | | 354| 0|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 328| 0|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ | Branch (242:26): [True: 0, False: 0] ------------------ 243| 0| : (dict_size + chunk_size + ZXC_DECOMPRESS_TAIL_PAD); ------------------ | | 354| 0|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 328| 0|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 244| 0| layout.off_dict = layout.total; 245| 0| layout.total += ZXC_ALIGN_CL(layout.sz_dict); ------------------ | | 360| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 246| 0| layout.off_dict_huf = layout.total; 247| 0| layout.total += ZXC_ALIGN_CL(sizeof(zxc_dict_huf_state_t)); ------------------ | | 360| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 248| 0| } 249| 18.4k| return layout; 250| 18.4k|} zxc_common.c:zxc_cctx_max_seq: 111| 18.4k|static ZXC_ALWAYS_INLINE size_t zxc_cctx_max_seq(const size_t chunk_size) { 112| 18.4k| return chunk_size / ZXC_LZ_MIN_MATCH_LEN + 16; ------------------ | | 505| 18.4k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 113| 18.4k|} zxc_common.c:zxc_write_gnr_header: 623| 4.90k| const zxc_gnr_header_t* RESTRICT gh) { 624| 4.90k| zxc_store_le32(dst, gh->n_sequences); 625| 4.90k| zxc_store_le32(dst + 4, gh->n_literals); 626| | 627| 4.90k| dst[8] = gh->enc_lit; 628| 4.90k| dst[9] = gh->enc_tok; 629| 4.90k| dst[10] = gh->enc_mlen; 630| 4.90k| dst[11] = gh->enc_off; 631| 4.90k|} zxc_common.c:zxc_read_gnr_header: 640| 4.10k| zxc_gnr_header_t* RESTRICT gh) { 641| 4.10k| gh->n_sequences = zxc_le32(src); 642| 4.10k| gh->n_literals = zxc_le32(src + 4); 643| 4.10k| gh->enc_lit = src[8]; 644| 4.10k| gh->enc_tok = src[9]; 645| 4.10k| gh->enc_mlen = src[10]; 646| 4.10k| gh->enc_off = src[11]; 647| 4.10k|} zxc_compress_chunk_wrapper_default: 2023| 4.90k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 2024| 4.90k| if (UNLIKELY(dst_cap < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 2025| | 2026| 4.90k| const size_t dict_sz = ctx->dict_size; 2027| 4.90k| const size_t block_sz = src_sz - dict_sz; 2028| 4.90k| const uint8_t* block_data = chunk + dict_sz; 2029| 4.90k| size_t w = 0; 2030| 4.90k| int res = ZXC_OK; 2031| | 2032| 4.90k| if (ctx->compression_level <= 2) ------------------ | Branch (2032:9): [True: 4.90k, False: 0] ------------------ 2033| 4.90k| res = zxc_encode_block_ghi(ctx, chunk, src_sz, dst, dst_cap, &w); 2034| 0| else 2035| 0| res = zxc_encode_block_glo(ctx, chunk, src_sz, dst, dst_cap, &w); 2036| | 2037| | // Check expansion against block data size (excluding dict prefix). 2038| 4.90k| if (UNLIKELY(res != ZXC_OK || w >= block_sz)) { ------------------ | | 151| 9.81k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 802, False: 4.10k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 802, False: 4.10k] | | ------------------ ------------------ 2039| 802| res = zxc_encode_block_raw(block_data, block_sz, dst, dst_cap, &w); 2040| 802| if (UNLIKELY(res != ZXC_OK)) return res; ------------------ | | 151| 802|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 802] | | ------------------ ------------------ 2041| 802| } 2042| | 2043| 4.90k| if (ctx->checksum_enabled) { ------------------ | Branch (2043:9): [True: 1.63k, False: 3.27k] ------------------ 2044| | // Calculate checksum on the compressed payload (w currently excludes checksum) 2045| | // Header is at dst, data starts at dst + ZXC_BLOCK_HEADER_SIZE 2046| 1.63k| if (UNLIKELY(w < ZXC_BLOCK_HEADER_SIZE || w + ZXC_BLOCK_CHECKSUM_SIZE > dst_cap)) ------------------ | | 151| 3.27k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.63k] | | | Branch (151:42): [True: 0, False: 1.63k] | | | Branch (151:42): [True: 0, False: 1.63k] | | ------------------ ------------------ 2047| 0| return ZXC_ERROR_OVERFLOW; 2048| | 2049| 1.63k| uint32_t payload_sz = (uint32_t)(w - ZXC_BLOCK_HEADER_SIZE); ------------------ | | 402| 1.63k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2050| 1.63k| uint32_t crc = 2051| 1.63k| zxc_checksum(dst + ZXC_BLOCK_HEADER_SIZE, payload_sz, ZXC_CHECKSUM_RAPIDHASH); ------------------ | | 402| 1.63k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ zxc_checksum(dst + ZXC_BLOCK_HEADER_SIZE, payload_sz, ZXC_CHECKSUM_RAPIDHASH); ------------------ | | 421| 1.63k|#define ZXC_CHECKSUM_RAPIDHASH 0 ------------------ 2052| 1.63k| zxc_store_le32(dst + w, crc); 2053| 1.63k| w += ZXC_BLOCK_CHECKSUM_SIZE; ------------------ | | 404| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ 2054| 1.63k| } 2055| | 2056| 4.90k| return (int)w; 2057| 4.90k|} zxc_compress.c:zxc_encode_block_ghi: 1825| 4.90k| size_t* RESTRICT const out_sz) { 1826| 4.90k| if (UNLIKELY(dst_cap < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1827| | 1828| 4.90k| const int level = ctx->compression_level; 1829| 4.90k| const size_t dict_sz = ctx->dict_size; 1830| | 1831| 4.90k| zxc_lz77_params_t lzp = zxc_get_lz77_params(level); 1832| | // The floor is a candidate until the block itself clears it. 1833| 4.90k| if (lzp.min_offset > 1 && zxc_block_is_short_dist_bound(src + dict_sz, src_sz - dict_sz)) ------------------ | Branch (1833:9): [True: 4.90k, False: 0] | Branch (1833:31): [True: 4.03k, False: 870] ------------------ 1834| 4.03k| lzp.min_offset = 1; 1835| | 1836| 4.90k| ctx->epoch++; 1837| 4.90k| if (UNLIKELY(ctx->epoch >= ctx->max_epoch)) { ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1838| 0| ZXC_MEMSET(ctx->hash_table, 0, ZXC_LZ_HASH_SIZE * sizeof(uint32_t)); ------------------ | | 172| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 1839| 0| ZXC_MEMSET(ctx->hash_tags, 0, ZXC_LZ_HASH_SIZE * sizeof(uint8_t)); ------------------ | | 172| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 1840| 0| ctx->epoch = 1; 1841| 0| } 1842| 4.90k| const uint32_t offset_bits = ctx->offset_bits; 1843| 4.90k| const uint32_t offset_mask = ctx->offset_mask; 1844| 4.90k| const uint32_t epoch_mark = ctx->epoch << offset_bits; 1845| | 1846| 4.90k| if (dict_sz > 0) ------------------ | Branch (1846:9): [True: 0, False: 4.90k] ------------------ 1847| 0| zxc_lz_seed_dict(src, dict_sz, ctx->hash_table, ctx->hash_tags, ctx->chain_table, 1848| 0| epoch_mark, offset_mask, level); 1849| | 1850| 4.90k| const uint8_t* ip = src + dict_sz; 1851| 4.90k| const uint8_t* iend = src + src_sz; 1852| 4.90k| const uint8_t* anchor = ip; 1853| 4.90k| const uint8_t* search_limit = iend - ZXC_LZ_SEARCH_MARGIN; ------------------ | | 560| 4.90k|#define ZXC_LZ_SEARCH_MARGIN (sizeof(uint64_t)) ------------------ 1854| | 1855| 4.90k| uint32_t* const hash_table = ctx->hash_table; 1856| 4.90k| uint8_t* const hash_tags = ctx->hash_tags; 1857| 4.90k| uint8_t* const buf_extras = ctx->buf_extras; 1858| 4.90k| uint16_t* const chain_table = ctx->chain_table; 1859| 4.90k| uint8_t* const literals = ctx->literals; 1860| 4.90k| uint32_t* const buf_sequences = ctx->buf_sequences; 1861| | 1862| 4.90k| uint32_t seq_c = 0; 1863| 4.90k| size_t extras_c = 0; 1864| 4.90k| size_t lit_c = 0; 1865| | 1866| 8.29M| while (LIKELY(ip < search_limit)) { ------------------ | | 145| 8.29M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 8.28M, False: 4.90k] | | ------------------ ------------------ 1867| 8.28M| size_t dist = (size_t)(ip - anchor); 1868| 8.28M| size_t step = lzp.step_base + (dist >> lzp.step_shift); 1869| 8.28M| if (UNLIKELY(ip + step >= search_limit)) step = 1; ------------------ | | 151| 8.28M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.43M, False: 6.85M] | | ------------------ ------------------ 1870| | 1871| 8.28M| ZXC_PREFETCH_READ(ip + step * 4 + ZXC_CACHE_LINE_SIZE); ------------------ | | 162| 8.28M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 1872| | 1873| 8.28M| if (LIKELY(ip + step + sizeof(uint64_t) <= iend)) { ------------------ | | 145| 8.28M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 8.28M, False: 0] | | ------------------ ------------------ 1874| 8.28M| const uint64_t v_next = zxc_le64(ip + step); 1875| | // cppcheck-suppress unreadVariable 1876| 8.28M| const uint32_t h_next = zxc_hash_func(v_next, 0); 1877| 8.28M| ZXC_PREFETCH_READ(&hash_tags[h_next]); ------------------ | | 162| 8.28M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 1878| 8.28M| ZXC_PREFETCH_READ(&hash_table[h_next]); ------------------ | | 162| 8.28M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 1879| 8.28M| } 1880| | 1881| 8.28M| const zxc_match_t m = 1882| 8.28M| zxc_lz77_find_best_match(src, ip, iend, search_limit, anchor, hash_table, hash_tags, 1883| 8.28M| chain_table, epoch_mark, offset_mask, level, lzp, 1884| 8.28M| /*last_off=*/0U); 1885| | 1886| 8.28M| if (m.ref) { ------------------ | Branch (1886:13): [True: 2.62M, False: 5.66M] ------------------ 1887| 2.62M| ip -= m.backtrack; 1888| 2.62M| const uint32_t ll = (uint32_t)(ip - anchor); 1889| 2.62M| const uint32_t ml = m.len - ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 505| 2.62M|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1890| 2.62M| const uint32_t off = (uint32_t)(ip - m.ref); 1891| | 1892| 2.62M| if (ll > 0) zxc_flush_literals(literals, &lit_c, anchor, ll, iend); ------------------ | Branch (1892:17): [True: 1.08M, False: 1.53M] ------------------ 1893| | 1894| 2.62M| const uint32_t ll_write = (ll >= ZXC_SEQ_LL_MASK) ? 255U : ll; ------------------ | | 466| 2.62M|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 460| 2.62M|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ | Branch (1894:39): [True: 15.4k, False: 2.60M] ------------------ 1895| 2.62M| const uint32_t ml_write = (ml >= ZXC_SEQ_ML_MASK) ? 255U : ml; ------------------ | | 468| 2.62M|#define ZXC_SEQ_ML_MASK ((1U << ZXC_SEQ_ML_BITS) - 1) | | ------------------ | | | | 462| 2.62M|#define ZXC_SEQ_ML_BITS 8 | | ------------------ ------------------ | Branch (1895:39): [True: 88.8k, False: 2.53M] ------------------ 1896| 2.62M| const uint32_t seq_val = (ll_write << (ZXC_SEQ_ML_BITS + ZXC_SEQ_OFF_BITS)) | ------------------ | | 462| 2.62M|#define ZXC_SEQ_ML_BITS 8 ------------------ const uint32_t seq_val = (ll_write << (ZXC_SEQ_ML_BITS + ZXC_SEQ_OFF_BITS)) | ------------------ | | 464| 2.62M|#define ZXC_SEQ_OFF_BITS 16 ------------------ 1897| 2.62M| (ml_write << ZXC_SEQ_OFF_BITS) | ------------------ | | 464| 2.62M|#define ZXC_SEQ_OFF_BITS 16 ------------------ 1898| 2.62M| ((off - ZXC_LZ_OFFSET_BIAS) & ZXC_SEQ_OFF_MASK); ------------------ | | 530| 2.62M|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ ((off - ZXC_LZ_OFFSET_BIAS) & ZXC_SEQ_OFF_MASK); ------------------ | | 470| 2.62M|#define ZXC_SEQ_OFF_MASK ((1U << ZXC_SEQ_OFF_BITS) - 1) | | ------------------ | | | | 464| 2.62M|#define ZXC_SEQ_OFF_BITS 16 | | ------------------ ------------------ 1899| 2.62M| buf_sequences[seq_c] = seq_val; 1900| 2.62M| seq_c++; 1901| | 1902| 2.62M| if (UNLIKELY(!zxc_emit_extra(buf_extras, &extras_c, ll, ZXC_SEQ_LL_MASK) || ------------------ | | 151| 5.24M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 2.62M] | | | Branch (151:42): [True: 0, False: 2.62M] | | | Branch (151:42): [True: 0, False: 2.62M] | | ------------------ ------------------ 1903| 2.62M| !zxc_emit_extra(buf_extras, &extras_c, ml, ZXC_SEQ_ML_MASK))) 1904| 0| return ZXC_ERROR_OVERFLOW; 1905| | 1906| 2.62M| ip += m.len; 1907| 2.62M| anchor = ip; 1908| 5.66M| } else { 1909| 5.66M| ip += step; 1910| 5.66M| } 1911| 8.28M| } 1912| | 1913| 4.90k| const size_t last_lits = iend - anchor; 1914| 4.90k| if (last_lits > 0) { ------------------ | Branch (1914:9): [True: 4.07k, False: 833] ------------------ 1915| 4.07k| ZXC_MEMCPY(literals + lit_c, anchor, last_lits); ------------------ | | 167| 4.07k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1916| 4.07k| lit_c += last_lits; 1917| 4.07k| } 1918| | 1919| 4.90k| zxc_block_header_t bh = {.block_type = ZXC_BLOCK_GHI}; 1920| 4.90k| uint8_t* const p = dst + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 402| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1921| 4.90k| size_t rem = dst_cap - ZXC_BLOCK_HEADER_SIZE; ------------------ | | 402| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1922| | 1923| 4.90k| const zxc_gnr_header_t gh = {.n_sequences = seq_c, 1924| 4.90k| .n_literals = (uint32_t)lit_c, 1925| 4.90k| .enc_lit = ZXC_SECTION_ENCODING_RAW, 1926| 4.90k| .enc_tok = 0, 1927| 4.90k| .enc_mlen = 0, 1928| 4.90k| .enc_off = 0}; 1929| | 1930| 4.90k| const int ghs = zxc_write_ghi_header(p, rem, &gh); 1931| 4.90k| if (UNLIKELY(ghs < 0)) return ghs; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1932| | 1933| 4.90k| uint8_t* p_curr = p + ghs; 1934| 4.90k| rem -= ghs; 1935| | 1936| 4.90k| const size_t sz_lit = lit_c; 1937| 4.90k| const size_t sz_seq = seq_c * sizeof(uint32_t); 1938| 4.90k| const size_t sz_ext = extras_c; 1939| | 1940| | // See the GLO twin. GHI literals are always RAW, so the slack always 1941| | // matters here. 1942| 4.90k| const size_t behind_lit = sz_seq + sz_ext; 1943| 4.90k| const size_t pad = (behind_lit < ZXC_BLOCK_LIT_SLACK) ? ZXC_BLOCK_LIT_SLACK - behind_lit : 0; ------------------ | | 342| 4.90k|#define ZXC_BLOCK_LIT_SLACK 32 ------------------ const size_t pad = (behind_lit < ZXC_BLOCK_LIT_SLACK) ? ZXC_BLOCK_LIT_SLACK - behind_lit : 0; ------------------ | | 342| 1.69k|#define ZXC_BLOCK_LIT_SLACK 32 ------------------ | Branch (1943:24): [True: 1.69k, False: 3.21k] ------------------ 1944| | 1945| 4.90k| if (UNLIKELY(rem < sz_lit + sz_seq + sz_ext + pad)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1946| | 1947| 4.90k| ZXC_MEMCPY(p_curr, literals, lit_c); ------------------ | | 167| 4.90k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1948| 4.90k| p_curr += lit_c; 1949| | 1950| | // Write sequences in little-endian order 1951| |#ifdef ZXC_BIG_ENDIAN 1952| | for (uint32_t i = 0; i < seq_c; i++) { 1953| | zxc_store_le32(p_curr, buf_sequences[i]); 1954| | p_curr += sizeof(uint32_t); 1955| | } 1956| |#else 1957| 4.90k| ZXC_MEMCPY(p_curr, buf_sequences, sz_seq); ------------------ | | 167| 4.90k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1958| 4.90k| p_curr += sz_seq; 1959| 4.90k|#endif 1960| | 1961| | // --- WRITE EXTRAS --- 1962| 4.90k| ZXC_MEMCPY(p_curr, buf_extras, sz_ext); ------------------ | | 167| 4.90k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1963| 4.90k| p_curr += sz_ext; 1964| | 1965| 4.90k| ZXC_MEMSET(p_curr, 0, pad); ------------------ | | 172| 4.90k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 1966| 4.90k| p_curr += pad; 1967| | 1968| 4.90k| bh.comp_size = (uint32_t)(p_curr - (dst + ZXC_BLOCK_HEADER_SIZE)); ------------------ | | 402| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1969| 4.90k| const int hw = zxc_write_block_header(dst, dst_cap, &bh); 1970| 4.90k| if (UNLIKELY(hw < 0)) return hw; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1971| | 1972| | // Checksum will be appended by the wrapper 1973| 4.90k| *out_sz = ZXC_BLOCK_HEADER_SIZE + bh.comp_size; ------------------ | | 402| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1974| 4.90k| return ZXC_OK; 1975| 4.90k|} zxc_compress.c:zxc_hash_tag: 61| 8.28M|static ZXC_ALWAYS_INLINE uint8_t zxc_hash_tag(const uint32_t val4) { 62| 8.28M| return (uint8_t)(val4 ^ (val4 >> 16)); 63| 8.28M|} zxc_compress.c:zxc_epoch_pos: 72| 3.91M| const uint32_t epoch_mark) { 73| 3.91M| return ((head & ~offset_mask) == epoch_mark) ? (head & offset_mask) : 0; ------------------ | Branch (73:12): [True: 3.34M, False: 571k] ------------------ 74| 3.91M|} zxc_compress.c:zxc_chain_delta: 82| 8.28M|static ZXC_ALWAYS_INLINE uint16_t zxc_chain_delta(const uint32_t cur_pos, const uint32_t prev_idx) { 83| 8.28M| const uint32_t dist = cur_pos - prev_idx; 84| 8.28M| const uint32_t valid = -((int32_t)((prev_idx != 0) & (dist < ZXC_LZ_WINDOW_SIZE))); ------------------ | | 501| 8.28M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) ------------------ 85| 8.28M| return (uint16_t)(dist & valid); 86| 8.28M|} zxc_compress.c:zxc_hash_func: 45| 16.5M|static ZXC_ALWAYS_INLINE uint32_t zxc_hash_func(const uint64_t val, const int use_hash5) { 46| 16.5M| if (use_hash5) { ------------------ | Branch (46:9): [True: 0, False: 16.5M] ------------------ 47| 0| const uint64_t v5 = val & 0xFFFFFFFFFFULL; 48| 0| return (uint32_t)((v5 * ZXC_LZ_HASH_PRIME2) >> (64 - ZXC_LZ_HASH_BITS)); ------------------ | | 497| 0|#define ZXC_LZ_HASH_PRIME2 0x2545F4914F6CDD1DULL ------------------ return (uint32_t)((v5 * ZXC_LZ_HASH_PRIME2) >> (64 - ZXC_LZ_HASH_BITS)); ------------------ | | 493| 0|#define ZXC_LZ_HASH_BITS 15 ------------------ 49| 16.5M| } else { 50| 16.5M| const uint64_t v4 = val ^ (val >> 15); 51| 16.5M| return ((uint32_t)v4 * ZXC_LZ_HASH_PRIME1) >> (32 - ZXC_LZ_HASH_BITS); ------------------ | | 495| 16.5M|#define ZXC_LZ_HASH_PRIME1 0x2D35182DU ------------------ return ((uint32_t)v4 * ZXC_LZ_HASH_PRIME1) >> (32 - ZXC_LZ_HASH_BITS); ------------------ | | 493| 16.5M|#define ZXC_LZ_HASH_BITS 15 ------------------ 52| 16.5M| } 53| 16.5M|} zxc_compress.c:zxc_lz77_find_best_match: 305| 8.28M| const int level, const zxc_lz77_params_t p, const uint32_t last_off) { 306| 8.28M| const int use_hash5 = (level >= 3); 307| | // Sentinel length just below the minimum, so any valid match replaces it. 308| 8.28M| zxc_match_t best = (zxc_match_t){NULL, ZXC_LZ_MIN_MATCH_LEN - 1, 0}; ------------------ | | 505| 8.28M|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 309| | 310| 8.28M| uint64_t cur_val8 = zxc_le64(ip); 311| 8.28M| uint32_t cur_val = (uint32_t)cur_val8; 312| 8.28M| uint32_t h = zxc_hash_func(cur_val8, use_hash5); 313| | 314| | // 8-bit tag: XOR fold of first 4 bytes for fast rejection 315| 8.28M| const uint8_t cur_tag = zxc_hash_tag(cur_val); 316| | 317| 8.28M| const uint32_t cur_pos = (uint32_t)(ip - src); 318| | 319| | // Tag-first filter on fast levels. 320| 8.28M| const uint8_t stored_tag = hash_tags[h]; 321| 8.28M| uint32_t match_idx; 322| 8.28M| if (level <= ZXC_LEVEL_FAST && stored_tag != cur_tag) { ------------------ | Branch (322:9): [True: 8.28M, False: 0] | Branch (322:36): [True: 4.37M, False: 3.91M] ------------------ 323| 4.37M| match_idx = 0; 324| 4.37M| } else { 325| 3.91M| match_idx = zxc_epoch_pos(hash_table[h], offset_mask, epoch_mark); 326| 3.91M| } 327| | 328| | // skip_head still drives the chain walk on level >= 3 (advances past the 329| | // mismatched head without comparing). On level <= 2 it is always 0 here: 330| | // either match_idx == 0 (filter-skip) or stored_tag == cur_tag. 331| 8.28M| const int skip_head = (match_idx != 0) & (stored_tag != cur_tag); 332| | 333| | // Split table writes 334| 8.28M| hash_table[h] = epoch_mark | cur_pos; 335| 8.28M| hash_tags[h] = cur_tag; 336| | 337| 8.28M| chain_table[cur_pos & ZXC_LZ_WINDOW_MASK] = zxc_chain_delta(cur_pos, match_idx); ------------------ | | 503| 8.28M|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 501| 8.28M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 338| | 339| 8.28M| int attempts = p.search_depth; 340| | 341| | // Repeat-offset seed (level-6 parser passes last_off; others pass 0). 342| | // Probing the previous offset first often finds the longest match right 343| | // away, speeding up the chain walk. It only raises best.len, never lowers 344| | // it, so the result is unchanged - this is purely a speed optimization. 345| 8.28M| if (last_off != 0U && last_off >= p.min_offset && last_off <= (uint32_t)ZXC_LZ_MAX_DIST && ------------------ | | 532| 8.28M|#define ZXC_LZ_MAX_DIST (ZXC_LZ_WINDOW_SIZE - 1) | | ------------------ | | | | 501| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ | Branch (345:9): [True: 0, False: 8.28M] | Branch (345:27): [True: 0, False: 0] | Branch (345:55): [True: 0, False: 0] ------------------ 346| 0| last_off <= cur_pos) { ------------------ | Branch (346:9): [True: 0, False: 0] ------------------ 347| 0| const uint8_t* const rep_ref = src + (cur_pos - last_off); 348| 0| if (zxc_le32(rep_ref) == cur_val) { ------------------ | Branch (348:13): [True: 0, False: 0] ------------------ 349| 0| uint32_t mlen = sizeof(uint32_t); 350| 0| const uint8_t* const limit_8 = iend - sizeof(uint64_t); 351| 0| while (ip + mlen < limit_8) { ------------------ | Branch (351:20): [True: 0, False: 0] ------------------ 352| 0| const uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(rep_ref + mlen); 353| 0| if (diff == 0) { ------------------ | Branch (353:21): [True: 0, False: 0] ------------------ 354| 0| mlen += sizeof(uint64_t); 355| 0| } else { 356| 0| mlen += (uint32_t)(zxc_ctz64(diff) >> 3); 357| 0| goto _rep_done; 358| 0| } 359| 0| } 360| 0| while (ip + mlen < iend && rep_ref[mlen] == ip[mlen]) mlen++; ------------------ | Branch (360:20): [True: 0, False: 0] | Branch (360:40): [True: 0, False: 0] ------------------ 361| 0| _rep_done:; 362| 0| best.len = mlen; 363| 0| best.ref = rep_ref; 364| 0| if (UNLIKELY(best.len >= (uint32_t)p.sufficient_len || ip + best.len >= iend)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 365| 0| goto _finalize_match; 366| 0| } 367| 0| } 368| | 369| 8.28M| if (match_idx == 0) goto _finalize_match; ------------------ | Branch (369:9): [True: 4.94M, False: 3.34M] ------------------ 370| | 371| | // Optimization: If head tag doesn't match, advance immediately without loading the first 372| | // mismatch. 373| 3.34M| if (skip_head) { ------------------ | Branch (373:9): [True: 0, False: 3.34M] ------------------ 374| 0| const uint16_t delta = chain_table[match_idx & ZXC_LZ_WINDOW_MASK]; ------------------ | | 503| 0|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 501| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 375| 0| const uint32_t next_idx = match_idx - delta; 376| 0| match_idx = (delta != 0) ? next_idx : 0; ------------------ | Branch (376:21): [True: 0, False: 0] ------------------ 377| 0| attempts--; 378| 0| } 379| | 380| 9.63M| while (match_idx > 0) { ------------------ | Branch (380:12): [True: 9.26M, False: 362k] ------------------ 381| 9.26M| if (UNLIKELY(attempts-- < 0 || cur_pos - match_idx > ZXC_LZ_MAX_DIST)) break; ------------------ | | 151| 17.2M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.89M, False: 7.37M] | | | Branch (151:42): [True: 1.29M, False: 7.97M] | | | Branch (151:42): [True: 598k, False: 7.37M] | | ------------------ ------------------ 382| 7.37M| const uint8_t* ref = src + match_idx; 383| | 384| | // Load the next chain link early (before the compare) so its address 385| | // resolves while we prefetch. 386| 7.37M| const uint16_t delta = chain_table[match_idx & ZXC_LZ_WINDOW_MASK]; ------------------ | | 503| 7.37M|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 501| 7.37M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 387| 7.37M| const uint32_t next_idx = match_idx - delta; 388| 7.37M| ZXC_PREFETCH_READ(src + next_idx); ------------------ | | 162| 7.37M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 389| | 390| 7.37M| const uint32_t ref_val = zxc_le32(ref); 391| 7.37M| const int tag_match = (ref_val == cur_val); 392| | // Cheap gate: 4-byte tag match, then check the byte past the current 393| | // best (the && skips that load unless the tag already matched). 394| 7.37M| const int should_compare = 395| 7.37M| (cur_pos - match_idx) >= p.min_offset && tag_match && (ref[best.len] == ip[best.len]); ------------------ | Branch (395:13): [True: 6.84M, False: 529k] | Branch (395:54): [True: 6.62M, False: 216k] | Branch (395:67): [True: 3.72M, False: 2.90M] ------------------ 396| | 397| 7.37M| if (should_compare) { ------------------ | Branch (397:13): [True: 3.72M, False: 3.64M] ------------------ 398| 3.72M| uint32_t mlen = sizeof(uint32_t); // We already know the first 4 bytes match 399| | 400| | // Fast path: Scalar 64-bit comparison for short matches (=< 64 bytes) 401| | // Most matches are short, so this avoids SIMD overhead for common cases 402| 3.72M| const uint8_t* limit_8 = iend - sizeof(uint64_t); 403| 3.72M| const uint8_t* scalar_limit = ip + mlen + 64; 404| 3.72M| if (scalar_limit > limit_8) scalar_limit = limit_8; ------------------ | Branch (404:17): [True: 7.56k, False: 3.72M] ------------------ 405| | 406| 8.35M| while (ip + mlen < scalar_limit) { ------------------ | Branch (406:20): [True: 8.10M, False: 252k] ------------------ 407| 8.10M| uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(ref + mlen); 408| 8.10M| if (diff == 0) ------------------ | Branch (408:21): [True: 4.62M, False: 3.47M] ------------------ 409| 4.62M| mlen += sizeof(uint64_t); 410| 3.47M| else { 411| 3.47M| mlen += (zxc_ctz64(diff) >> 3); 412| 3.47M| goto _match_len_done; 413| 3.47M| } 414| 8.10M| } 415| | 416| | // Long match path: Use SIMD for matches exceeding 64 bytes 417| |#if defined(ZXC_USE_AVX512) 418| | const uint8_t* limit_64 = iend - 64; 419| | while (ip + mlen < limit_64) { 420| | const __m512i v_src = _mm512_loadu_si512((const void*)(ip + mlen)); 421| | const __m512i v_ref = _mm512_loadu_si512((const void*)(ref + mlen)); 422| | const __mmask64 mask = _mm512_cmpeq_epi8_mask(v_src, v_ref); 423| | if (mask == 0xFFFFFFFFFFFFFFFF) 424| | mlen += 64; 425| | else { 426| | mlen += (uint32_t)zxc_ctz64(~mask); 427| | goto _match_len_done; 428| | } 429| | } 430| |#elif defined(ZXC_USE_AVX2) 431| | const uint8_t* limit_32 = iend - 32; 432| | while (ip + mlen < limit_32) { 433| | const __m256i v_src = _mm256_loadu_si256((const __m256i*)(ip + mlen)); 434| | const __m256i v_ref = _mm256_loadu_si256((const __m256i*)(ref + mlen)); 435| | const __m256i v_cmp = _mm256_cmpeq_epi8(v_src, v_ref); 436| | const uint32_t mask = (uint32_t)_mm256_movemask_epi8(v_cmp); 437| | if (mask == 0xFFFFFFFF) 438| | mlen += 32; 439| | else { 440| | mlen += zxc_ctz32(~mask); 441| | goto _match_len_done; 442| | } 443| | } 444| |#elif defined(ZXC_USE_SSE2) 445| 252k| const uint8_t* limit_16 = iend - 16; 446| 11.2M| while (ip + mlen < limit_16) { ------------------ | Branch (446:20): [True: 11.2M, False: 2.09k] ------------------ 447| 11.2M| const __m128i v_src = _mm_loadu_si128((const __m128i*)(ip + mlen)); 448| 11.2M| const __m128i v_ref = _mm_loadu_si128((const __m128i*)(ref + mlen)); 449| 11.2M| const __m128i v_cmp = _mm_cmpeq_epi8(v_src, v_ref); 450| 11.2M| const uint32_t mask = (uint32_t)_mm_movemask_epi8(v_cmp); 451| 11.2M| if (mask == 0xFFFFU) ------------------ | Branch (451:21): [True: 11.0M, False: 250k] ------------------ 452| 11.0M| mlen += 16; 453| 250k| else { 454| | // mask != 0xFFFF => a differing byte exists in bits 0..15, 455| | // so the lowest set bit of ~mask lies in that range. 456| 250k| mlen += zxc_ctz32(~mask); 457| 250k| goto _match_len_done; 458| 250k| } 459| 11.2M| } 460| |#elif defined(ZXC_USE_NEON64) 461| | { 462| | const uint8_t* limit_32 = iend - 32; 463| | while (ip + mlen < limit_32) { 464| | const uint8x16_t s0 = vld1q_u8(ip + mlen); 465| | const uint8x16_t r0 = vld1q_u8(ref + mlen); 466| | const uint8x16_t c0 = vceqq_u8(s0, r0); 467| | const uint64_t m0 = vget_lane_u64( 468| | vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(c0), 4)), 0); 469| | if (UNLIKELY(m0 != ~(uint64_t)0)) { 470| | mlen += (uint32_t)(zxc_ctz64(~m0) >> 2); 471| | goto _match_len_done; 472| | } 473| | const uint8x16_t s1 = vld1q_u8(ip + mlen + 16); 474| | const uint8x16_t r1 = vld1q_u8(ref + mlen + 16); 475| | const uint8x16_t c1 = vceqq_u8(s1, r1); 476| | const uint64_t m1 = vget_lane_u64( 477| | vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(c1), 4)), 0); 478| | if (UNLIKELY(m1 != ~(uint64_t)0)) { 479| | mlen += 16 + (uint32_t)(zxc_ctz64(~m1) >> 2); 480| | goto _match_len_done; 481| | } 482| | mlen += 32; 483| | } 484| | if (ip + mlen < iend - 16) { 485| | const uint8x16_t v_src = vld1q_u8(ip + mlen); 486| | const uint8x16_t v_ref = vld1q_u8(ref + mlen); 487| | const uint8x16_t v_cmp = vceqq_u8(v_src, v_ref); 488| | const uint64_t mask = vget_lane_u64( 489| | vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(v_cmp), 4)), 0); 490| | if (LIKELY(mask == ~(uint64_t)0)) 491| | mlen += 16; 492| | else { 493| | mlen += (uint32_t)(zxc_ctz64(~mask) >> 2); 494| | goto _match_len_done; 495| | } 496| | } 497| | } 498| |#elif defined(ZXC_USE_NEON32) 499| | { 500| | const uint8_t* limit_16 = iend - 16; 501| | while (ip + mlen < limit_16) { 502| | const uint8x16_t v_src = vld1q_u8(ip + mlen); 503| | const uint8x16_t v_ref = vld1q_u8(ref + mlen); 504| | const uint8x16_t v_cmp = vceqq_u8(v_src, v_ref); 505| | uint8x8_t p1 = vpmin_u8(vget_low_u8(v_cmp), vget_high_u8(v_cmp)); 506| | uint8x8_t p2 = vpmin_u8(p1, p1); 507| | uint8x8_t p3 = vpmin_u8(p2, p2); 508| | uint8x8_t p4 = vpmin_u8(p3, p3); 509| | uint8_t min_val = vget_lane_u8(p4, 0); 510| | if (min_val == 0xFF) 511| | mlen += 16; 512| | else { 513| | uint8x16_t v_diff = vmvnq_u8(v_cmp); 514| | uint64_t lo = 515| | (uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 0) | 516| | ((uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 1) << 32); 517| | if (lo != 0) 518| | mlen += (zxc_ctz64(lo) >> 3); 519| | else 520| | mlen += 521| | 8 + (zxc_ctz64( 522| | (uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 2) | 523| | ((uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 3) 524| | << 32)) >> 525| | 3); 526| | goto _match_len_done; 527| | } 528| | } 529| | } 530| |#endif 531| 2.31k| while (ip + mlen < limit_8) { ------------------ | Branch (531:20): [True: 382, False: 1.93k] ------------------ 532| 382| const uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(ref + mlen); 533| 382| if (diff == 0) ------------------ | Branch (533:21): [True: 220, False: 162] ------------------ 534| 220| mlen += sizeof(uint64_t); 535| 162| else { 536| 162| mlen += (zxc_ctz64(diff) >> 3); 537| 162| goto _match_len_done; 538| 162| } 539| 382| } 540| 8.94k| while (ip + mlen < iend && ref[mlen] == ip[mlen]) mlen++; ------------------ | Branch (540:20): [True: 8.11k, False: 833] | Branch (540:40): [True: 7.01k, False: 1.09k] ------------------ 541| | 542| 3.72M| _match_len_done:; 543| 3.72M| const int better = (mlen > best.len); 544| 3.72M| best.len = better ? mlen : best.len; ------------------ | Branch (544:24): [True: 3.40M, False: 324k] ------------------ 545| 3.72M| best.ref = better ? ref : best.ref; ------------------ | Branch (545:24): [True: 3.40M, False: 324k] ------------------ 546| | 547| 3.72M| if (UNLIKELY(best.len >= (uint32_t)p.sufficient_len || ip + best.len >= iend)) break; ------------------ | | 151| 6.37M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.08M, False: 2.64M] | | | Branch (151:42): [True: 1.08M, False: 2.64M] | | | Branch (151:42): [True: 247, False: 2.64M] | | ------------------ ------------------ 548| 3.72M| } 549| | 550| 6.29M| match_idx = (delta != 0) ? next_idx : 0; ------------------ | Branch (550:21): [True: 5.92M, False: 362k] ------------------ 551| 6.29M| } 552| | 553| 8.28M|_finalize_match: 554| 8.28M| if (best.ref) { ------------------ | Branch (554:9): [True: 2.62M, False: 5.66M] ------------------ 555| | // Backtrack to extend match backwards 556| 2.62M| const uint8_t* b_ip = ip; 557| 2.62M| const uint8_t* b_ref = best.ref; 558| 5.03M| while (b_ip > anchor && b_ref > src && b_ip[-1] == b_ref[-1]) { ------------------ | Branch (558:16): [True: 3.49M, False: 1.53M] | Branch (558:33): [True: 3.49M, False: 751] | Branch (558:48): [True: 2.40M, False: 1.08M] ------------------ 559| 2.40M| b_ip--; 560| 2.40M| b_ref--; 561| 2.40M| best.len++; 562| 2.40M| best.backtrack++; 563| 2.40M| } 564| 2.62M| best.ref = b_ref; 565| 2.62M| } 566| | 567| 8.28M| if (p.use_lazy && best.ref && best.len < (uint32_t)p.lazy_len_threshold && ------------------ | Branch (567:9): [True: 0, False: 8.28M] | Branch (567:23): [True: 0, False: 0] | Branch (567:35): [True: 0, False: 0] ------------------ 568| 0| ip + 1 < search_limit) { ------------------ | Branch (568:9): [True: 0, False: 0] ------------------ 569| | // --- Lazy evaluation at ip+1 --- 570| 0| const uint64_t next_val8 = zxc_le64(ip + 1); 571| 0| const uint32_t next_val = (uint32_t)next_val8; 572| 0| const uint32_t h2 = zxc_hash_func(next_val8, use_hash5); 573| 0| const uint8_t next_stored_tag = hash_tags[h2]; 574| 0| const uint32_t next_head = hash_table[h2]; 575| 0| uint32_t next_idx = zxc_epoch_pos(next_head, offset_mask, epoch_mark); 576| 0| const uint8_t next_tag = zxc_hash_tag(next_val); 577| 0| const int skip_lazy_head = (next_idx > 0 && next_stored_tag != next_tag); ------------------ | Branch (577:37): [True: 0, False: 0] | Branch (577:53): [True: 0, False: 0] ------------------ 578| 0| uint32_t max_lazy2 = 0; 579| 0| int lazy_att = p.lazy_attempts; 580| 0| int is_lazy_first = 1; 581| | 582| 0| while (next_idx > 0) { ------------------ | Branch (582:16): [True: 0, False: 0] ------------------ 583| 0| if (UNLIKELY(lazy_att-- <= 0 || (uint32_t)(ip + 1 - src) - next_idx > ZXC_LZ_MAX_DIST)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 584| 0| break; 585| 0| const uint8_t* ref2 = src + next_idx; 586| | 587| | // Filtered here too: a lazy hit only cancels best.ref, so a match 588| | // we could never emit would drop a legal one for nothing. 589| 0| if (((uint32_t)(ip + 1 - src) - next_idx) >= p.min_offset && ------------------ | Branch (589:17): [True: 0, False: 0] ------------------ 590| 0| (!is_lazy_first || !skip_lazy_head) && zxc_le32(ref2) == next_val) { ------------------ | Branch (590:18): [True: 0, False: 0] | Branch (590:36): [True: 0, False: 0] | Branch (590:56): [True: 0, False: 0] ------------------ 591| 0| uint32_t l2 = sizeof(uint32_t); 592| 0| const uint8_t* limit = iend - sizeof(uint64_t); 593| | 594| 0| while (ip + 1 + l2 < limit) { ------------------ | Branch (594:24): [True: 0, False: 0] ------------------ 595| 0| const uint64_t v1 = zxc_le64(ip + 1 + l2); 596| 0| const uint64_t v2 = zxc_le64(ref2 + l2); 597| 0| if (v1 != v2) { ------------------ | Branch (597:25): [True: 0, False: 0] ------------------ 598| 0| l2 += (uint32_t)(zxc_ctz64(v1 ^ v2) >> 3); 599| 0| goto lazy2_done; 600| 0| } 601| 0| l2 += sizeof(uint64_t); 602| 0| } 603| 0| while (ip + 1 + l2 < iend && ref2[l2] == ip[1 + l2]) l2++; ------------------ | Branch (603:24): [True: 0, False: 0] | Branch (603:46): [True: 0, False: 0] ------------------ 604| 0| lazy2_done: 605| 0| max_lazy2 = l2 > max_lazy2 ? l2 : max_lazy2; ------------------ | Branch (605:29): [True: 0, False: 0] ------------------ 606| 0| } 607| | 608| 0| const uint16_t delta = chain_table[next_idx & ZXC_LZ_WINDOW_MASK]; ------------------ | | 503| 0|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 501| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 609| 0| if (UNLIKELY(delta == 0)) break; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 610| 0| next_idx -= delta; 611| 0| is_lazy_first = 0; 612| 0| } 613| | 614| | // --- Lazy evaluation at ip+2 (computed in parallel, no dependency on lazy 1) --- 615| 0| uint32_t max_lazy3 = 0; 616| 0| if (level >= ZXC_LEVEL_BALANCED && ip + 2 < search_limit) { ------------------ | Branch (616:13): [True: 0, False: 0] | Branch (616:44): [True: 0, False: 0] ------------------ 617| 0| const uint64_t val3_8 = zxc_le64(ip + 2); 618| 0| const uint32_t val3 = (uint32_t)val3_8; 619| 0| const uint32_t h3 = zxc_hash_func(val3_8, use_hash5); 620| 0| const uint8_t tag3 = hash_tags[h3]; 621| 0| const uint32_t head3 = hash_table[h3]; 622| 0| uint32_t idx3 = zxc_epoch_pos(head3, offset_mask, epoch_mark); 623| 0| const uint8_t cur_tag3 = zxc_hash_tag(val3); 624| 0| const int skip_head3 = (idx3 > 0 && tag3 != cur_tag3); ------------------ | Branch (624:37): [True: 0, False: 0] | Branch (624:49): [True: 0, False: 0] ------------------ 625| | 626| 0| int is_first3 = 1; 627| 0| lazy_att = p.lazy_attempts; 628| 0| while (idx3 > 0) { ------------------ | Branch (628:20): [True: 0, False: 0] ------------------ 629| 0| if (UNLIKELY(lazy_att-- <= 0 || (uint32_t)(ip + 2 - src) - idx3 > ZXC_LZ_MAX_DIST)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 630| 0| break; 631| | 632| 0| const uint8_t* ref3 = src + idx3; 633| 0| if (((uint32_t)(ip + 2 - src) - idx3) >= p.min_offset && ------------------ | Branch (633:21): [True: 0, False: 0] ------------------ 634| 0| (!is_first3 || !skip_head3) && zxc_le32(ref3) == val3) { ------------------ | Branch (634:22): [True: 0, False: 0] | Branch (634:36): [True: 0, False: 0] | Branch (634:52): [True: 0, False: 0] ------------------ 635| 0| uint32_t l3 = sizeof(uint32_t); 636| 0| const uint8_t* limit = iend - sizeof(uint64_t); 637| | 638| 0| while (ip + 2 + l3 < limit) { ------------------ | Branch (638:28): [True: 0, False: 0] ------------------ 639| 0| const uint64_t v1 = zxc_le64(ip + 2 + l3); 640| 0| const uint64_t v2 = zxc_le64(ref3 + l3); 641| 0| if (v1 != v2) { ------------------ | Branch (641:29): [True: 0, False: 0] ------------------ 642| 0| l3 += (uint32_t)(zxc_ctz64(v1 ^ v2) >> 3); 643| 0| goto lazy3_done; 644| 0| } 645| 0| l3 += sizeof(uint64_t); 646| 0| } 647| 0| while (ip + 2 + l3 < iend && ref3[l3] == ip[2 + l3]) l3++; ------------------ | Branch (647:28): [True: 0, False: 0] | Branch (647:50): [True: 0, False: 0] ------------------ 648| 0| lazy3_done: 649| 0| max_lazy3 = l3 > max_lazy3 ? l3 : max_lazy3; ------------------ | Branch (649:33): [True: 0, False: 0] ------------------ 650| 0| } 651| | 652| 0| const uint16_t delta = chain_table[idx3 & ZXC_LZ_WINDOW_MASK]; ------------------ | | 503| 0|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 501| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 653| 0| if (UNLIKELY(delta == 0)) break; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 654| 0| idx3 -= delta; 655| 0| is_first3 = 0; 656| 0| } 657| 0| } 658| | 659| | // Single decision: invalidate if either lazy position found a better match 660| 0| if (max_lazy2 > best.len + 1 || max_lazy3 > best.len + 2) best.ref = NULL; ------------------ | Branch (660:13): [True: 0, False: 0] | Branch (660:41): [True: 0, False: 0] ------------------ 661| 0| } 662| | 663| 8.28M| return best; 664| 8.28M|} zxc_compress.c:zxc_flush_literals: 249| 1.08M| const uint8_t* iend) { 250| 1.08M| if (LIKELY(anchor + ZXC_PAD_SIZE <= iend)) { ------------------ | | 145| 1.08M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 1.08M, False: 970] | | ------------------ ------------------ 251| 1.08M| zxc_copy32(literals + *lit_c, anchor); 252| 1.08M| if (UNLIKELY(ll > ZXC_PAD_SIZE)) ------------------ | | 151| 1.08M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 113k, False: 971k] | | ------------------ ------------------ 253| 113k| ZXC_MEMCPY(literals + *lit_c + ZXC_PAD_SIZE, anchor + ZXC_PAD_SIZE, ll - ZXC_PAD_SIZE); ------------------ | | 167| 113k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 254| 1.08M| } else { 255| 970| ZXC_MEMCPY(literals + *lit_c, anchor, ll); ------------------ | | 167| 970|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 256| 970| } 257| 1.08M| *lit_c += ll; 258| 1.08M|} zxc_compress.c:zxc_emit_extra: 233| 5.24M| const uint32_t val, const uint32_t mask) { 234| 5.24M| if (LIKELY(val < mask)) return 1; ------------------ | | 145| 5.24M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 5.14M, False: 104k] | | ------------------ ------------------ 235| 104k| const size_t n = zxc_write_varint(extras + *used, val - mask); 236| 104k| if (UNLIKELY(n == 0)) return 0; ------------------ | | 151| 104k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 104k] | | ------------------ ------------------ 237| 104k| *used += n; 238| 104k| return 1; 239| 104k|} zxc_compress.c:zxc_write_varint: 197| 104k|static ZXC_ALWAYS_INLINE size_t zxc_write_varint(uint8_t* RESTRICT dst, const uint32_t val) { 198| | // Values above ZXC_MAX_VARINT_VALUE are out-of-spec and the decoder would 199| | // reject them. Unreachable from the Block API, so this is defence in depth; 200| | // callers must treat a return of 0 as an encoding error. 201| 104k| if (UNLIKELY(val > ZXC_MAX_VARINT_VALUE)) { ------------------ | | 151| 104k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 104k] | | ------------------ ------------------ 202| 0| return 0; 203| 0| } 204| | 205| | // 1 byte: 0xxxxxxx (7 bits) = 2^7 = 128 206| 104k| if (LIKELY(val < (1U << 7))) { ------------------ | | 145| 104k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 30.1k, False: 74.1k] | | ------------------ ------------------ 207| 30.1k| dst[0] = (uint8_t)val; 208| 30.1k| return 1; 209| 30.1k| } 210| | 211| | // 2 bytes: 10xxxxxx xxxxxxxx (14 bits) = 2^14 = 16384 212| 74.1k| if (LIKELY(val < (1U << 14))) { ------------------ | | 145| 74.1k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 71.4k, False: 2.71k] | | ------------------ ------------------ 213| 71.4k| dst[0] = (uint8_t)(0x80 | (val & 0x3F)); 214| 71.4k| dst[1] = (uint8_t)(val >> 6); 215| 71.4k| return 2; 216| 71.4k| } 217| | 218| | // 3 bytes: 110xxxxx xxxxxxxx xxxxxxxx (21 bits) -> max emittable value, 219| | // matching ZXC_MAX_VARINT_VALUE = ZXC_BLOCK_SIZE_MAX - 1. 220| 2.71k| dst[0] = (uint8_t)(0xC0 | (val & 0x1F)); 221| 2.71k| dst[1] = (uint8_t)(val >> 5); 222| 2.71k| dst[2] = (uint8_t)(val >> 13); 223| 2.71k| return 3; 224| 74.1k|} zxc_compress.c:zxc_encode_block_raw: 1994| 802| size_t* RESTRICT const out_sz) { 1995| 802| if (UNLIKELY(dst_cap < ZXC_BLOCK_HEADER_SIZE + src_sz)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 802|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 802] | | ------------------ ------------------ 1996| | 1997| 802| zxc_block_header_t bh; 1998| 802| bh.block_type = ZXC_BLOCK_RAW; 1999| 802| bh.block_flags = 0; // Checksum flag moved to file header 2000| 802| bh.reserved = 0; 2001| 802| bh.comp_size = (uint32_t)src_sz; 2002| | 2003| 802| const int hw = zxc_write_block_header(dst, dst_cap, &bh); 2004| 802| if (UNLIKELY(hw < 0)) return hw; ------------------ | | 151| 802|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 802] | | ------------------ ------------------ 2005| | 2006| 802| ZXC_MEMCPY(dst + ZXC_BLOCK_HEADER_SIZE, src, src_sz); ------------------ | | 167| 802|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 2007| | 2008| | // Checksum will be appended by the wrapper 2009| 802| *out_sz = ZXC_BLOCK_HEADER_SIZE + src_sz; ------------------ | | 402| 802|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2010| 802| return ZXC_OK; 2011| 802|} zxc_decompress_chunk_wrapper_default: 1705| 4.90k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 1706| 4.90k| return zxc_decompress_chunk_wrapper_body(ctx, src, src_sz, dst, dst_cap, 0, 0); 1707| 4.90k|} zxc_decompress.c:zxc_decompress_chunk_wrapper_body: 1648| 4.90k| uint8_t* RESTRICT dst, const size_t dst_cap, const int has_dict, const int safe) { 1649| 4.90k| if (UNLIKELY(src_sz < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1650| | 1651| 4.90k| const uint8_t type = src[0]; 1652| 4.90k| const uint32_t comp_sz = zxc_le32(src + 3); 1653| 4.90k| const int has_crc = ctx->checksum_enabled; 1654| | 1655| | // Check bounds: Header + Body + Checksum(if any) 1656| 4.90k| const size_t expected_sz = 1657| 4.90k| (size_t)ZXC_BLOCK_HEADER_SIZE + comp_sz + (has_crc ? ZXC_BLOCK_CHECKSUM_SIZE : 0); ------------------ | | 402| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ (size_t)ZXC_BLOCK_HEADER_SIZE + comp_sz + (has_crc ? ZXC_BLOCK_CHECKSUM_SIZE : 0); ------------------ | | 404| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (1657:52): [True: 1.63k, False: 3.27k] ------------------ 1658| 4.90k| if (UNLIKELY(src_sz < expected_sz)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1659| | 1660| 4.90k| const uint8_t* data = src + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 402| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1661| | 1662| 4.90k| if (has_crc) { ------------------ | Branch (1662:9): [True: 1.63k, False: 3.27k] ------------------ 1663| 1.63k| const uint32_t stored = zxc_le32(data + comp_sz); 1664| 1.63k| const uint32_t calc = zxc_checksum(data, comp_sz, ZXC_CHECKSUM_RAPIDHASH); ------------------ | | 421| 1.63k|#define ZXC_CHECKSUM_RAPIDHASH 0 ------------------ 1665| 1.63k| if (UNLIKELY(stored != calc)) return ZXC_ERROR_BAD_CHECKSUM; ------------------ | | 151| 1.63k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.63k] | | ------------------ ------------------ 1666| 1.63k| } 1667| | 1668| 4.90k| int decoded_sz = ZXC_ERROR_BAD_BLOCK_TYPE; 1669| | 1670| 4.90k| switch (type) { 1671| 0| case ZXC_BLOCK_GLO: ------------------ | Branch (1671:9): [True: 0, False: 4.90k] ------------------ 1672| 0| decoded_sz = safe ? zxc_decode_block_glo_safe(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1672:26): [True: 0, False: 0] ------------------ 1673| 0| : has_dict ? zxc_decode_block_glo_dict(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1673:28): [True: 0, False: 0] ------------------ 1674| 0| : zxc_decode_block_glo(ctx, data, comp_sz, dst, dst_cap); 1675| 0| break; 1676| 4.10k| case ZXC_BLOCK_GHI: ------------------ | Branch (1676:9): [True: 4.10k, False: 802] ------------------ 1677| 4.10k| decoded_sz = safe ? zxc_decode_block_ghi_safe(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1677:26): [True: 0, False: 4.10k] ------------------ 1678| 4.10k| : has_dict ? zxc_decode_block_ghi_dict(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1678:28): [True: 0, False: 4.10k] ------------------ 1679| 4.10k| : zxc_decode_block_ghi(ctx, data, comp_sz, dst, dst_cap); 1680| 4.10k| break; 1681| 802| case ZXC_BLOCK_RAW: ------------------ | Branch (1681:9): [True: 802, False: 4.10k] ------------------ 1682| | // For RAW blocks, comp_sz == raw_sz (uncompressed data stored as-is) 1683| 802| if (UNLIKELY(comp_sz > dst_cap)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 802|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 802] | | ------------------ ------------------ 1684| 802| ZXC_MEMCPY(dst, data, comp_sz); ------------------ | | 167| 802|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1685| 802| decoded_sz = (int)comp_sz; 1686| 802| break; 1687| 0| case ZXC_BLOCK_EOF: ------------------ | Branch (1687:9): [True: 0, False: 4.90k] ------------------ 1688| | // EOF should be handled by the dispatcher, not here 1689| 0| return ZXC_ERROR_CORRUPT_DATA; 1690| 0| default: ------------------ | Branch (1690:9): [True: 0, False: 4.90k] ------------------ 1691| 0| return ZXC_ERROR_BAD_BLOCK_TYPE; 1692| 4.90k| } 1693| | 1694| 4.90k| return decoded_sz; 1695| 4.90k|} zxc_decompress.c:zxc_read_varint: 51| 104k|static ZXC_ALWAYS_INLINE uint32_t zxc_read_varint(const uint8_t** ptr, const uint8_t* end) { 52| 104k| const uint8_t* p = *ptr; 53| 104k| if (UNLIKELY(p >= end)) return 0; ------------------ | | 151| 104k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 104k] | | ------------------ ------------------ 54| | 55| 104k| const uint32_t b0 = p[0]; 56| | 57| | // 1 Byte: 0xxxxxxx (7 bits) -> val < 128 (2^7) 58| 104k| if (LIKELY(b0 < 0x80)) { ------------------ | | 145| 104k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 30.1k, False: 74.1k] | | ------------------ ------------------ 59| 30.1k| *ptr = p + 1; 60| 30.1k| return b0; 61| 30.1k| } 62| | 63| | // 2 Bytes: 10xxxxxx xxxxxxxx (14 bits) -> val < 16384 (2^14) 64| 74.1k| if (LIKELY(b0 < 0xC0)) { ------------------ | | 145| 74.1k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 71.4k, False: 2.70k] | | ------------------ ------------------ 65| 71.4k| if (UNLIKELY(p + 1 >= end)) { ------------------ | | 151| 71.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 71.4k] | | ------------------ ------------------ 66| 0| *ptr = end; 67| 0| return 0; 68| 0| } 69| 71.4k| *ptr = p + 2; 70| 71.4k| return (b0 & 0x3F) | ((uint32_t)p[1] << 6); 71| 71.4k| } 72| | 73| | // 3 Bytes: 110xxxxx xxxxxxxx xxxxxxxx (21 bits) -> val < 2^21. The longest 74| | // a legitimate varint can be: values are (ll - MASK) or (ml - MASK), always 75| | // strictly below block_size_max = 2^21. 76| 2.70k| if (LIKELY(b0 < 0xE0)) { ------------------ | | 145| 2.70k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 2.70k, False: 0] | | ------------------ ------------------ 77| 2.70k| if (UNLIKELY(p + 2 >= end)) { ------------------ | | 151| 2.70k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 2.70k] | | ------------------ ------------------ 78| 0| *ptr = end; 79| 0| return 0; 80| 0| } 81| 2.70k| *ptr = p + 3; 82| 2.70k| return (b0 & 0x1F) | ((uint32_t)p[1] << 5) | ((uint32_t)p[2] << 13); 83| 2.70k| } 84| | 85| | // extra encoding: out-of-spec for the current format, reject. 86| 0| *ptr = end; 87| 0| return 0; 88| 2.70k|} zxc_decompress.c:zxc_decode_copy_literals: 439| 2.45M| const uint64_t ll) { 440| 2.45M| zxc_copy32(dst, src); 441| 2.45M| if (UNLIKELY(ll > 32)) { ------------------ | | 151| 2.45M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 111k, False: 2.33M] | | ------------------ ------------------ 442| 111k| dst += 32; 443| 111k| src += 32; 444| 111k| size_t rem = ll - 32; 445| 4.49M| while (rem > 32) { ------------------ | Branch (445:16): [True: 4.38M, False: 111k] ------------------ 446| 4.38M| zxc_copy32(dst, src); 447| 4.38M| dst += 32; 448| 4.38M| src += 32; 449| 4.38M| rem -= 32; 450| 4.38M| } 451| 111k| zxc_copy32(dst, src); 452| 111k| } 453| 2.45M|} zxc_decompress.c:zxc_decode_fill_run: 364| 466| const uint64_t ml) { 365| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 366| | const __m256i v = _mm256_set1_epi8((char)byte); 367| | _mm256_storeu_si256((__m256i*)dst, v); 368| | if (UNLIKELY(ml > 32)) { 369| | uint8_t* out = dst + 32; 370| | size_t rem = ml - 32; 371| | while (rem > 32) { 372| | _mm256_storeu_si256((__m256i*)out, v); 373| | out += 32; 374| | rem -= 32; 375| | } 376| | _mm256_storeu_si256((__m256i*)out, v); 377| | } 378| |#elif defined(ZXC_USE_SSE2) 379| | const __m128i v = _mm_set1_epi8((char)byte); 380| 466| _mm_storeu_si128((__m128i*)dst, v); 381| 466| _mm_storeu_si128((__m128i*)(dst + 16), v); 382| 466| if (UNLIKELY(ml > 32)) { ------------------ | | 151| 466|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 278, False: 188] | | ------------------ ------------------ 383| 278| uint8_t* out = dst + 32; 384| 278| size_t rem = ml - 32; 385| 13.1k| while (rem > 32) { ------------------ | Branch (385:16): [True: 12.8k, False: 278] ------------------ 386| 12.8k| _mm_storeu_si128((__m128i*)out, v); 387| 12.8k| _mm_storeu_si128((__m128i*)(out + 16), v); 388| 12.8k| out += 32; 389| 12.8k| rem -= 32; 390| 12.8k| } 391| 278| _mm_storeu_si128((__m128i*)out, v); 392| 278| _mm_storeu_si128((__m128i*)(out + 16), v); 393| 278| } 394| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 395| | const uint8x16_t v = vdupq_n_u8(byte); 396| | vst1q_u8(dst, v); 397| | vst1q_u8(dst + 16, v); 398| | if (UNLIKELY(ml > 32)) { 399| | uint8_t* out = dst + 32; 400| | size_t rem = ml - 32; 401| | while (rem > 32) { 402| | vst1q_u8(out, v); 403| | vst1q_u8(out + 16, v); 404| | out += 32; 405| | rem -= 32; 406| | } 407| | vst1q_u8(out, v); 408| | vst1q_u8(out + 16, v); 409| | } 410| |#else 411| | ZXC_MEMSET(dst, byte, ml); 412| |#endif 413| 466|} zxc_decompress.c:zxc_decode_copy_match: 479| 2.45M| const uint64_t ml) { 480| 2.45M| const uint8_t* match_src = d_ptr - off; 481| 2.45M| if (LIKELY(off >= 32)) { ------------------ | | 145| 2.45M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 2.15M, False: 301k] | | ------------------ ------------------ 482| 2.15M| zxc_copy32(d_ptr, match_src); 483| 2.15M| if (UNLIKELY(ml > 32)) { ------------------ | | 151| 2.15M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 495k, False: 1.65M] | | ------------------ ------------------ 484| 495k| uint8_t* out = d_ptr + 32; 485| 495k| const uint8_t* ref = match_src + 32; 486| 495k| size_t rem = ml - 32; 487| 4.27M| while (rem > 32) { ------------------ | Branch (487:20): [True: 3.77M, False: 495k] ------------------ 488| 3.77M| zxc_copy32(out, ref); 489| 3.77M| out += 32; 490| 3.77M| ref += 32; 491| 3.77M| rem -= 32; 492| 3.77M| } 493| 495k| zxc_copy32(out, ref); 494| 495k| } 495| 2.15M| } else if (off == 1) { ------------------ | Branch (495:16): [True: 466, False: 300k] ------------------ 496| 466| zxc_decode_fill_run(d_ptr, match_src[0], ml); 497| 300k| } else { 498| 300k| zxc_decode_copy_overlap_run32(d_ptr, off, ml); 499| 300k| } 500| 2.45M|} zxc_decompress.c:zxc_decode_copy_overlap_run32: 198| 300k| const uint64_t ml) { 199| 300k| const size_t stride = zxc_overlap_strides32[off]; 200| 300k| size_t copied = 0; 201| |#if defined(ZXC_USE_NEON64) 202| | uint8x16x2_t tbl; 203| | tbl.val[0] = vld1q_u8(dst - off); 204| | tbl.val[1] = vld1q_u8(dst - off + 16); 205| | const uint8x16_t pat_lo = vqtbl2q_u8(tbl, vld1q_u8(zxc_overlap_masks32[off])); 206| | const uint8x16_t pat_hi = vqtbl2q_u8(tbl, vld1q_u8(zxc_overlap_masks32[off] + 16)); 207| | do { 208| | vst1q_u8(dst + copied, pat_lo); 209| | vst1q_u8(dst + copied + 16, pat_hi); 210| | copied += stride; 211| | } while (copied < ml); 212| | 213| |#elif defined(ZXC_USE_NEON32) 214| | // VTBL reaches 8 bytes per lookup from a 4-register (32-byte) table, so the 215| | // pattern takes four lookups instead of NEON64's two. 216| | uint8x8x4_t tbl; 217| | tbl.val[0] = vld1_u8(dst - off); 218| | tbl.val[1] = vld1_u8(dst - off + 8); 219| | tbl.val[2] = vld1_u8(dst - off + 16); 220| | tbl.val[3] = vld1_u8(dst - off + 24); 221| | const uint8_t* const m = zxc_overlap_masks32[off]; 222| | const uint8x8_t p0 = vtbl4_u8(tbl, vld1_u8(m)); 223| | const uint8x8_t p1 = vtbl4_u8(tbl, vld1_u8(m + 8)); 224| | const uint8x8_t p2 = vtbl4_u8(tbl, vld1_u8(m + 16)); 225| | const uint8x8_t p3 = vtbl4_u8(tbl, vld1_u8(m + 24)); 226| | do { 227| | vst1_u8(dst + copied, p0); 228| | vst1_u8(dst + copied + 8, p1); 229| | vst1_u8(dst + copied + 16, p2); 230| | vst1_u8(dst + copied + 24, p3); 231| | copied += stride; 232| | } while (copied < ml); 233| | 234| |#elif defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 235| | // pshufb only reaches 16 bytes per lane, so the 32-byte table is emulated: 236| | // shuffle both halves, then pick per byte on bit 4 of the index. 237| | const __m128i t0 = _mm_loadu_si128((const __m128i*)(dst - off)); 238| | const __m128i t1 = _mm_loadu_si128((const __m128i*)(dst - off + 16)); 239| | const __m128i sixteen = _mm_set1_epi8(16); 240| | const __m128i m_lo = _mm_load_si128((const __m128i*)zxc_overlap_masks32[off]); 241| | const __m128i m_hi = _mm_load_si128((const __m128i*)(zxc_overlap_masks32[off] + 16)); 242| | const __m128i pat_lo = 243| | _mm_blendv_epi8(_mm_shuffle_epi8(t0, m_lo), 244| | _mm_shuffle_epi8(t1, _mm_sub_epi8(m_lo, sixteen)), _mm_slli_epi16(m_lo, 3)); 245| | const __m128i pat_hi = 246| | _mm_blendv_epi8(_mm_shuffle_epi8(t0, m_hi), 247| | _mm_shuffle_epi8(t1, _mm_sub_epi8(m_hi, sixteen)), _mm_slli_epi16(m_hi, 3)); 248| | do { 249| | _mm_storeu_si128((__m128i*)(dst + copied), pat_lo); 250| | _mm_storeu_si128((__m128i*)(dst + copied + 16), pat_hi); 251| | copied += stride; 252| | } while (copied < ml); 253| | 254| |#else 255| | // SSE2 (no PSHUFB) and non-SIMD tiers: build the 32-byte pattern with a 256| | // wrap counter (no per-byte modulo), then store it as two 16-byte halves. 257| 300k| const uint8_t* src = dst - off; 258| 300k| uint8_t pat[32]; 259| 300k| uint32_t k = 0; 260| 9.92M| for (size_t i = 0; i < 32; i++) { ------------------ | Branch (260:24): [True: 9.62M, False: 300k] ------------------ 261| 9.62M| pat[i] = src[k]; 262| 9.62M| if (++k == off) k = 0; ------------------ | Branch (262:13): [True: 657k, False: 8.96M] ------------------ 263| 9.62M| } 264| 1.12M| do { 265| 1.12M| zxc_copy16(dst + copied, pat); 266| 1.12M| zxc_copy16(dst + copied + 16, pat + 16); 267| 1.12M| copied += stride; 268| 1.12M| } while (copied < ml); ------------------ | Branch (268:14): [True: 824k, False: 300k] ------------------ 269| 300k|#endif 270| 300k|} zxc_decompress.c:zxc_decode_copy_match_exact: 562| 172k| const size_t off, const size_t ml) { 563| 172k| if (off >= ml) { ------------------ | Branch (563:9): [True: 163k, False: 9.05k] ------------------ 564| 163k| ZXC_MEMCPY(d_ptr, match_src, ml); ------------------ | | 167| 163k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 565| 163k| } else if (ml < 16) { ------------------ | Branch (565:16): [True: 1.28k, False: 7.76k] ------------------ 566| 15.0k| for (size_t i = 0; i < ml; i++) d_ptr[i] = match_src[i]; ------------------ | Branch (566:28): [True: 13.8k, False: 1.28k] ------------------ 567| 7.76k| } else { 568| 7.76k| size_t n = off; 569| 7.76k| ZXC_MEMCPY(d_ptr, match_src, n); ------------------ | | 167| 7.76k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 570| 21.2k| while (n <= ml - n) { ------------------ | Branch (570:16): [True: 13.4k, False: 7.76k] ------------------ 571| 13.4k| ZXC_MEMCPY(d_ptr + n, d_ptr, n); ------------------ | | 167| 13.4k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 572| 13.4k| n <<= 1; 573| 13.4k| } 574| 7.76k| ZXC_MEMCPY(d_ptr + n, d_ptr, ml - n); ------------------ | | 167| 7.76k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 575| 7.76k| } 576| 172k|} zxc_decompress.c:zxc_decode_block_ghi_impl: 1235| 4.10k| const int has_dict) { 1236| 4.10k| zxc_gnr_header_t gh; 1237| | 1238| | // 0 when !has_dict (safe path) -> folds `d_floor` to `dst`. 1239| 4.10k| const size_t dict_size = has_dict ? ctx->dict_size : 0; ------------------ | Branch (1239:30): [True: 0, False: 4.10k] ------------------ 1240| | 1241| 4.10k| if (UNLIKELY(zxc_read_ghi_header(src, src_size, &gh) != ZXC_OK)) return ZXC_ERROR_BAD_HEADER; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 1242| 4.10k| if (UNLIKELY(gh.enc_lit != ZXC_SECTION_ENCODING_RAW || gh.enc_tok != 0)) ------------------ | | 151| 8.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | | Branch (151:42): [True: 0, False: 4.10k] | | | Branch (151:42): [True: 0, False: 4.10k] | | ------------------ ------------------ 1243| 0| return ZXC_ERROR_CORRUPT_DATA; 1244| | 1245| 4.10k| const uint8_t* const p_data = src + ZXC_GHI_HEADER_BINARY_SIZE; ------------------ | | 408| 4.10k|#define ZXC_GHI_HEADER_BINARY_SIZE 12 ------------------ 1246| 4.10k| const uint8_t* p_curr = p_data; 1247| | 1248| | // --- Stream Pointers & Validation --- 1249| | // GHI carries no section descriptors: literals are always RAW, the sequence 1250| | // stream is four bytes per sequence, and extras run to the payload end. 1251| 4.10k| const size_t sz_lit = gh.n_literals; 1252| 4.10k| const uint64_t sz_seqs = (uint64_t)gh.n_sequences * sizeof(uint32_t); 1253| | 1254| 4.10k| const size_t payload_avail = (size_t)(src + src_size - p_data); 1255| 4.10k| const uint64_t consumed = (uint64_t)sz_lit + sz_seqs; 1256| 4.10k| if (UNLIKELY(consumed > (uint64_t)payload_avail)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 1257| 4.10k| const size_t sz_exts = payload_avail - (size_t)consumed; /* slack padding included */ 1258| | 1259| | // GHI literals are always RAW, so the slack is always load-bearing here. 1260| | // No underflow: sz_lit <= consumed <= payload_avail. 1261| 4.10k| if (UNLIKELY(payload_avail - sz_lit < ZXC_BLOCK_LIT_SLACK)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 1262| | 1263| 4.10k| const uint8_t* l_ptr = p_curr; 1264| 4.10k| const uint8_t* l_end = l_ptr + sz_lit; 1265| 4.10k| p_curr += sz_lit; 1266| | 1267| 4.10k| const uint8_t* seq_ptr = p_curr; 1268| 4.10k| const uint8_t* extras_ptr = p_curr + (size_t)sz_seqs; 1269| 4.10k| const uint8_t* const extras_end = extras_ptr + sz_exts; 1270| | 1271| 4.10k| uint8_t* d_ptr = dst; 1272| 4.10k| const uint8_t* const d_end = dst + dst_capacity; 1273| | // Lowest address a match may reach: the dictionary prefix if any, else dst. 1274| 4.10k| const uint8_t* const d_floor = dst - dict_size; 1275| 4.10k| const uint8_t* const d_end_safe = d_end - (ZXC_PAD_SIZE * 4); // 128 ------------------ | | 328| 4.10k|#define ZXC_PAD_SIZE 32 ------------------ 1276| | // Safety margin for 4x unrolled loop: 4 * (ZXC_SEQ_LL_MASK LL + 1277| | // ZXC_SEQ_ML_MASK+ZXC_LZ_MIN_MATCH_LEN ML) + ZXC_PAD_SIZE Pad = 4 x (255 + 255 + 5) + 32 = 2092 1278| 4.10k| const uint8_t* const d_end_fast = d_end - ZXC_DECOMPRESS_TAIL_PAD; // 2112 ------------------ | | 354| 4.10k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 328| 4.10k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 1279| | 1280| | // Literal margin for the GHI loops: without a varint, ll <= 254 per sequence, 1281| | // so 4 * 254 = 1016. Past that only the cold varint path checks l_ptr. 1282| 4.10k| const size_t ghi_margin_4x = 4 * (ZXC_SEQ_LL_MASK - 1); // 1016 ------------------ | | 466| 4.10k|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 460| 4.10k|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ 1283| 4.10k| const size_t ghi_margin_1x = ZXC_SEQ_LL_MASK - 1; // 254 ------------------ | | 466| 4.10k|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 460| 4.10k|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ 1284| 4.10k| const uint8_t* const l_end_safe_4x = (sz_lit > ghi_margin_4x) ? l_end - ghi_margin_4x : l_ptr; ------------------ | Branch (1284:42): [True: 2.93k, False: 1.16k] ------------------ 1285| 4.10k| const uint8_t* const l_end_safe_1x = (sz_lit > ghi_margin_1x) ? l_end - ghi_margin_1x : l_ptr; ------------------ | Branch (1285:42): [True: 3.36k, False: 740] ------------------ 1286| | 1287| 4.10k| uint32_t n_seq = gh.n_sequences; 1288| | 1289| | // Same handover as GLO, but the GHI offset is inline in the sequence word and 1290| | // always 16 bits -- enc_off is not a width here, so the span is fixed. 1291| 4.10k| const size_t off_span = 1U << 16; 1292| 4.10k| const size_t span = (dict_size >= off_span) ? 0 : off_span - dict_size; ------------------ | Branch (1292:25): [True: 0, False: 4.10k] ------------------ 1293| 4.10k| const uint8_t* const d_bounds = (span >= dst_capacity) ? d_end : dst + span; ------------------ | Branch (1293:37): [True: 0, False: 4.10k] ------------------ 1294| | 1295| | // --- SAFE loop: validate offsets until d_bounds (4x unroll) --- 1296| 4.10k| if (safe) { ------------------ | Branch (1296:9): [True: 0, False: 4.10k] ------------------ 1297| | // SAFE variant: save per-batch state so an OVERFLOW can rollback and 1298| | // hand over to the 1x loop / Safe Path. Wild writes already committed 1299| | // are deterministically overwritten when the 1x loop replays. 1300| 0| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (1300:16): [True: 0, False: 0] | Branch (1300:30): [True: 0, False: 0] | Branch (1300:52): [True: 0, False: 0] | Branch (1300:77): [True: 0, False: 0] ------------------ 1301| 0| const uint8_t* const t_save = seq_ptr; 1302| 0| const uint8_t* const e_save = extras_ptr; 1303| 0| uint8_t* const d_save = d_ptr; 1304| 0| const uint8_t* const l_save = l_ptr; 1305| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, goto rollback_safe_4x, (void)0); ------------------ | | 735| 0| do { \ | | 736| 0| uint32_t s1 = zxc_le32(seq_ptr); \ | | 737| 0| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 738| 0| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 739| 0| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 740| 0| seq_ptr += 4 * sizeof(uint32_t); \ | | 741| 0| PREFETCH; \ | | 742| 0| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 0| do { \ | | | | 703| 0| uint64_t ll = (S) >> 24; \ | | | | 704| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 707| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 0| const uint64_t reserve = (RESERVE); \ | | | | 709| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 711| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 0| (size_t)(d_end - d_ptr))) \ | | | | 713| 0| ON_FAIL; \ | | | | 714| 0| } \ | | | | 715| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 716| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 718| 0| (size_t)(d_end - d_ptr))) \ | | | | 719| 0| ON_FAIL; \ | | | | 720| 0| } \ | | | | 721| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 0| l_ptr += ll; \ | | | | 725| 0| d_ptr += ll; \ | | | | 726| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1305| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 586| 0| do { \ | | | | | | | | 587| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 588| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 589| 0| d_ptr += ml; \ | | | | | | | | 590| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (590:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 0| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 743| 0| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 0| do { \ | | | | 703| 0| uint64_t ll = (S) >> 24; \ | | | | 704| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 707| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 0| const uint64_t reserve = (RESERVE); \ | | | | 709| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 711| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 0| (size_t)(d_end - d_ptr))) \ | | | | 713| 0| ON_FAIL; \ | | | | 714| 0| } \ | | | | 715| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 716| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 718| 0| (size_t)(d_end - d_ptr))) \ | | | | 719| 0| ON_FAIL; \ | | | | 720| 0| } \ | | | | 721| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 0| l_ptr += ll; \ | | | | 725| 0| d_ptr += ll; \ | | | | 726| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1305| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 586| 0| do { \ | | | | | | | | 587| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 588| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 589| 0| d_ptr += ml; \ | | | | | | | | 590| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (590:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 0| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 744| 0| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 0| do { \ | | | | 703| 0| uint64_t ll = (S) >> 24; \ | | | | 704| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 707| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 0| const uint64_t reserve = (RESERVE); \ | | | | 709| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 711| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 0| (size_t)(d_end - d_ptr))) \ | | | | 713| 0| ON_FAIL; \ | | | | 714| 0| } \ | | | | 715| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 716| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 718| 0| (size_t)(d_end - d_ptr))) \ | | | | 719| 0| ON_FAIL; \ | | | | 720| 0| } \ | | | | 721| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 0| l_ptr += ll; \ | | | | 725| 0| d_ptr += ll; \ | | | | 726| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1305| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 586| 0| do { \ | | | | | | | | 587| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 588| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 589| 0| d_ptr += ml; \ | | | | | | | | 590| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (590:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 0| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 745| 0| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 0| do { \ | | | | 703| 0| uint64_t ll = (S) >> 24; \ | | | | 704| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 707| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 0| const uint64_t reserve = (RESERVE); \ | | | | 709| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 711| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 0| (size_t)(d_end - d_ptr))) \ | | | | 713| 0| ON_FAIL; \ | | | | 714| 0| } \ | | | | 715| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 716| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 718| 0| (size_t)(d_end - d_ptr))) \ | | | | 719| 0| ON_FAIL; \ | | | | 720| 0| } \ | | | | 721| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 0| l_ptr += ll; \ | | | | 725| 0| d_ptr += ll; \ | | | | 726| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1305| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 586| 0| do { \ | | | | | | | | 587| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 588| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 589| 0| d_ptr += ml; \ | | | | | | | | 590| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (590:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 0| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 746| 0| n_seq -= 4; \ | | 747| 0| } while (0) | | ------------------ | | | Branch (747:14): [Folded, False: 0] | | ------------------ ------------------ 1306| 0| continue; 1307| | 1308| 0| rollback_safe_4x: 1309| 0| seq_ptr = t_save; 1310| 0| extras_ptr = e_save; 1311| 0| d_ptr = d_save; 1312| 0| l_ptr = l_save; 1313| 0| break; 1314| 0| } 1315| 4.10k| } else { 1316| 160k| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (1316:16): [True: 158k, False: 1.66k] | Branch (1316:30): [True: 158k, False: 0] | Branch (1316:52): [True: 157k, False: 1.58k] | Branch (1316:77): [True: 156k, False: 862] ------------------ 1317| 156k| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); ------------------ | | 735| 156k| do { \ | | 736| 156k| uint32_t s1 = zxc_le32(seq_ptr); \ | | 737| 156k| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 738| 156k| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 739| 156k| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 740| 156k| seq_ptr += 4 * sizeof(uint32_t); \ | | 741| 156k| PREFETCH; \ | | 742| 156k| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 156k| do { \ | | | | 703| 156k| uint64_t ll = (S) >> 24; \ | | | | 704| 156k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 156k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 156k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 156k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.93k, False: 154k] | | | | | | ------------------ | | | | ------------------ | | | | 707| 1.93k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 1.93k| const uint64_t reserve = (RESERVE); \ | | | | 709| 1.93k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 1.93k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.87k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.93k] | | | | | | | Branch (151:42): [True: 0, False: 1.93k] | | | | | | | Branch (151:42): [True: 0, False: 1.93k] | | | | | | ------------------ | | | | ------------------ | | | | 711| 1.93k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 1.93k| (size_t)(d_end - d_ptr))) \ | | | | 713| 1.93k| ON_FAIL; \ | | | | 714| 1.93k| } \ | | | | 715| 156k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 6.76k, False: 149k] | | | | | | ------------------ | | | | ------------------ | | | | 716| 6.76k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 6.76k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 6.76k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 6.76k] | | | | | | ------------------ | | | | ------------------ | | | | 718| 6.76k| (size_t)(d_end - d_ptr))) \ | | | | 719| 6.76k| ON_FAIL; \ | | | | 720| 6.76k| } \ | | | | 721| 156k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 156k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 156k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 156k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 156k| l_ptr += ll; \ | | | | 725| 156k| d_ptr += ll; \ | | | | 726| 156k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1317| 156k| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 586| 156k| do { \ | | | | | | | | 587| 156k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 156k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 588| 156k| COPY(d_ptr, off, ml); \ | | | | | | | | 589| 156k| d_ptr += ml; \ | | | | | | | | 590| 156k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (590:14): [Folded, False: 156k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 156k| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 156k] | | | | ------------------ | | ------------------ | | 743| 156k| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 156k| do { \ | | | | 703| 156k| uint64_t ll = (S) >> 24; \ | | | | 704| 156k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 156k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 156k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 156k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.56k, False: 154k] | | | | | | ------------------ | | | | ------------------ | | | | 707| 1.56k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 1.56k| const uint64_t reserve = (RESERVE); \ | | | | 709| 1.56k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 1.56k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.12k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.56k] | | | | | | | Branch (151:42): [True: 0, False: 1.56k] | | | | | | | Branch (151:42): [True: 0, False: 1.56k] | | | | | | ------------------ | | | | ------------------ | | | | 711| 1.56k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 1.56k| (size_t)(d_end - d_ptr))) \ | | | | 713| 1.56k| ON_FAIL; \ | | | | 714| 1.56k| } \ | | | | 715| 156k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 6.82k, False: 149k] | | | | | | ------------------ | | | | ------------------ | | | | 716| 6.82k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 6.82k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 6.82k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 6.82k] | | | | | | ------------------ | | | | ------------------ | | | | 718| 6.82k| (size_t)(d_end - d_ptr))) \ | | | | 719| 6.82k| ON_FAIL; \ | | | | 720| 6.82k| } \ | | | | 721| 156k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 156k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 156k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 156k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 156k| l_ptr += ll; \ | | | | 725| 156k| d_ptr += ll; \ | | | | 726| 156k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1317| 156k| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 586| 156k| do { \ | | | | | | | | 587| 156k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 156k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 588| 156k| COPY(d_ptr, off, ml); \ | | | | | | | | 589| 156k| d_ptr += ml; \ | | | | | | | | 590| 156k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (590:14): [Folded, False: 156k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 156k| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 156k] | | | | ------------------ | | ------------------ | | 744| 156k| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 156k| do { \ | | | | 703| 156k| uint64_t ll = (S) >> 24; \ | | | | 704| 156k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 156k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 156k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 156k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.48k, False: 154k] | | | | | | ------------------ | | | | ------------------ | | | | 707| 1.48k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 1.48k| const uint64_t reserve = (RESERVE); \ | | | | 709| 1.48k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 1.48k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 2.96k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.48k] | | | | | | | Branch (151:42): [True: 0, False: 1.48k] | | | | | | | Branch (151:42): [True: 0, False: 1.48k] | | | | | | ------------------ | | | | ------------------ | | | | 711| 1.48k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 1.48k| (size_t)(d_end - d_ptr))) \ | | | | 713| 1.48k| ON_FAIL; \ | | | | 714| 1.48k| } \ | | | | 715| 156k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 6.96k, False: 149k] | | | | | | ------------------ | | | | ------------------ | | | | 716| 6.96k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 6.96k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 6.96k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 6.96k] | | | | | | ------------------ | | | | ------------------ | | | | 718| 6.96k| (size_t)(d_end - d_ptr))) \ | | | | 719| 6.96k| ON_FAIL; \ | | | | 720| 6.96k| } \ | | | | 721| 156k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 156k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 156k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 156k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 156k| l_ptr += ll; \ | | | | 725| 156k| d_ptr += ll; \ | | | | 726| 156k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1317| 156k| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 586| 156k| do { \ | | | | | | | | 587| 156k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 156k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 588| 156k| COPY(d_ptr, off, ml); \ | | | | | | | | 589| 156k| d_ptr += ml; \ | | | | | | | | 590| 156k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (590:14): [Folded, False: 156k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 156k| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 156k] | | | | ------------------ | | ------------------ | | 745| 156k| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 156k| do { \ | | | | 703| 156k| uint64_t ll = (S) >> 24; \ | | | | 704| 156k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 156k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 156k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 156k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.52k, False: 154k] | | | | | | ------------------ | | | | ------------------ | | | | 707| 1.52k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 1.52k| const uint64_t reserve = (RESERVE); \ | | | | 709| 1.52k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 1.52k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.05k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.52k] | | | | | | | Branch (151:42): [True: 0, False: 1.52k] | | | | | | | Branch (151:42): [True: 0, False: 1.52k] | | | | | | ------------------ | | | | ------------------ | | | | 711| 1.52k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 1.52k| (size_t)(d_end - d_ptr))) \ | | | | 713| 1.52k| ON_FAIL; \ | | | | 714| 1.52k| } \ | | | | 715| 156k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 6.88k, False: 149k] | | | | | | ------------------ | | | | ------------------ | | | | 716| 6.88k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 6.88k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 6.88k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 6.88k] | | | | | | ------------------ | | | | ------------------ | | | | 718| 6.88k| (size_t)(d_end - d_ptr))) \ | | | | 719| 6.88k| ON_FAIL; \ | | | | 720| 6.88k| } \ | | | | 721| 156k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 156k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 156k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 156k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 156k| l_ptr += ll; \ | | | | 725| 156k| d_ptr += ll; \ | | | | 726| 156k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1317| 156k| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 586| 156k| do { \ | | | | | | | | 587| 156k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 156k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 588| 156k| COPY(d_ptr, off, ml); \ | | | | | | | | 589| 156k| d_ptr += ml; \ | | | | | | | | 590| 156k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (590:14): [Folded, False: 156k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 156k| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 156k] | | | | ------------------ | | ------------------ | | 746| 156k| n_seq -= 4; \ | | 747| 156k| } while (0) | | ------------------ | | | Branch (747:14): [Folded, False: 156k] | | ------------------ ------------------ 1318| 156k| } 1319| 4.10k| } 1320| | 1321| | // --- SAFE Loop tail: remaining sequences with offset validation (1x) --- 1322| 138k| while (n_seq > 0 && d_ptr < d_end_safe && d_ptr < d_bounds) { ------------------ | Branch (1322:12): [True: 136k, False: 2.93k] | Branch (1322:25): [True: 136k, False: 0] | Branch (1322:47): [True: 134k, False: 1.17k] ------------------ 1323| 134k| uint32_t seq = zxc_le32(seq_ptr); 1324| 134k| seq_ptr += sizeof(uint32_t); 1325| | 1326| 134k| uint64_t ll = seq >> 24; 1327| 134k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 134k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 669, False: 134k] | | ------------------ ------------------ 1328| | 1329| 134k| uint32_t m_bits = (seq >> 16) & 0xFF; 1330| 134k| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 505| 134k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1331| 134k| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 134k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 4.76k, False: 130k] | | ------------------ ------------------ 1332| | 1333| 134k| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 530| 134k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1334| | 1335| | // Strict bounds check: sequence must fit, AND wild copies must not overshoot 1336| | // Check both destination (d_ptr) and source literal stream (l_ptr) 1337| 134k| if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr) || ------------------ | | 151| 269k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 17.0k, False: 117k] | | | Branch (151:42): [True: 0, False: 134k] | | | Branch (151:42): [True: 17.0k, False: 117k] | | ------------------ ------------------ 1338| 134k| ll + ZXC_PAD_SIZE > (size_t)(l_end - l_ptr))) { 1339| | // Fallback to exact copy (slow but safe) 1340| 17.0k| if (UNLIKELY(d_ptr + ll > d_end || l_ptr + ll > l_end)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 34.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 17.0k] | | | Branch (151:42): [True: 0, False: 17.0k] | | | Branch (151:42): [True: 0, False: 17.0k] | | ------------------ ------------------ 1341| 17.0k| ZXC_MEMCPY(d_ptr, l_ptr, ll); ------------------ | | 167| 17.0k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1342| 17.0k| l_ptr += ll; 1343| 17.0k| d_ptr += ll; 1344| | 1345| 17.0k| if (UNLIKELY(d_ptr + ml > d_end)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 17.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 17.0k] | | ------------------ ------------------ 1346| 17.0k| if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET; ------------------ | | 151| 17.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 17.0k] | | ------------------ ------------------ 1347| 17.0k| const uint8_t* match_src = d_ptr - offset; 1348| | 1349| 17.0k| zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml); 1350| 17.0k| d_ptr += ml; 1351| 117k| } else { 1352| 117k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); 1353| 117k| l_ptr += ll; 1354| 117k| d_ptr += ll; 1355| 117k| DECODE_MATCH_SAFE(ml, offset, zxc_decode_copy_match); ------------------ | | 586| 117k| do { \ | | 587| 117k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | ------------------ | | | | 151| 117k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | ------------------ | | | | | Branch (151:21): [True: 0, False: 117k] | | | | ------------------ | | ------------------ | | 588| 117k| COPY(d_ptr, off, ml); \ | | 589| 117k| d_ptr += ml; \ | | 590| 117k| } while (0) | | ------------------ | | | Branch (590:14): [Folded, False: 117k] | | ------------------ ------------------ 1356| 117k| } 1357| 134k| n_seq--; 1358| 134k| } 1359| | 1360| | // --- FAST Loop: After threshold, check large margin to avoid individual bounds checks --- 1361| 4.10k| if (safe) { ------------------ | Branch (1361:9): [True: 0, False: 4.10k] ------------------ 1362| 0| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) { ------------------ | Branch (1362:16): [True: 0, False: 0] | Branch (1362:30): [True: 0, False: 0] | Branch (1362:52): [True: 0, False: 0] ------------------ 1363| 0| const uint8_t* const t_save = seq_ptr; 1364| 0| const uint8_t* const e_save = extras_ptr; 1365| 0| uint8_t* const d_save = d_ptr; 1366| 0| const uint8_t* const l_save = l_ptr; 1367| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, goto rollback_fast_4x, ------------------ | | 735| 0| do { \ | | 736| 0| uint32_t s1 = zxc_le32(seq_ptr); \ | | 737| 0| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 738| 0| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 739| 0| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 740| 0| seq_ptr += 4 * sizeof(uint32_t); \ | | 741| 0| PREFETCH; \ | | 742| 0| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 0| do { \ | | | | 703| 0| uint64_t ll = (S) >> 24; \ | | | | 704| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 707| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 0| const uint64_t reserve = (RESERVE); \ | | | | 709| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 711| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 0| (size_t)(d_end - d_ptr))) \ | | | | 713| 0| ON_FAIL; \ | | | | 714| 0| } \ | | | | 715| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 716| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 718| 0| (size_t)(d_end - d_ptr))) \ | | | | 719| 0| ON_FAIL; \ | | | | 720| 0| } \ | | | | 721| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 0| l_ptr += ll; \ | | | | 725| 0| d_ptr += ll; \ | | | | 726| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1367| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 595| 0| do { \ | | | | | | | | 596| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 597| 0| d_ptr += ml; \ | | | | | | | | 598| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (598:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 0| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 743| 0| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 0| do { \ | | | | 703| 0| uint64_t ll = (S) >> 24; \ | | | | 704| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 707| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 0| const uint64_t reserve = (RESERVE); \ | | | | 709| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 711| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 0| (size_t)(d_end - d_ptr))) \ | | | | 713| 0| ON_FAIL; \ | | | | 714| 0| } \ | | | | 715| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 716| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 718| 0| (size_t)(d_end - d_ptr))) \ | | | | 719| 0| ON_FAIL; \ | | | | 720| 0| } \ | | | | 721| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 0| l_ptr += ll; \ | | | | 725| 0| d_ptr += ll; \ | | | | 726| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1367| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 595| 0| do { \ | | | | | | | | 596| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 597| 0| d_ptr += ml; \ | | | | | | | | 598| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (598:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 0| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 744| 0| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 0| do { \ | | | | 703| 0| uint64_t ll = (S) >> 24; \ | | | | 704| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 707| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 0| const uint64_t reserve = (RESERVE); \ | | | | 709| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 711| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 0| (size_t)(d_end - d_ptr))) \ | | | | 713| 0| ON_FAIL; \ | | | | 714| 0| } \ | | | | 715| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 716| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 718| 0| (size_t)(d_end - d_ptr))) \ | | | | 719| 0| ON_FAIL; \ | | | | 720| 0| } \ | | | | 721| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 0| l_ptr += ll; \ | | | | 725| 0| d_ptr += ll; \ | | | | 726| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1367| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 595| 0| do { \ | | | | | | | | 596| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 597| 0| d_ptr += ml; \ | | | | | | | | 598| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (598:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 0| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 745| 0| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 0| do { \ | | | | 703| 0| uint64_t ll = (S) >> 24; \ | | | | 704| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 707| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 0| const uint64_t reserve = (RESERVE); \ | | | | 709| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 711| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 0| (size_t)(d_end - d_ptr))) \ | | | | 713| 0| ON_FAIL; \ | | | | 714| 0| } \ | | | | 715| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 716| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 718| 0| (size_t)(d_end - d_ptr))) \ | | | | 719| 0| ON_FAIL; \ | | | | 720| 0| } \ | | | | 721| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 0| l_ptr += ll; \ | | | | 725| 0| d_ptr += ll; \ | | | | 726| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1367| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 595| 0| do { \ | | | | | | | | 596| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 597| 0| d_ptr += ml; \ | | | | | | | | 598| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (598:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 0| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 746| 0| n_seq -= 4; \ | | 747| 0| } while (0) | | ------------------ | | | Branch (747:14): [Folded, False: 0] | | ------------------ ------------------ 1368| 0| ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE)); 1369| 0| continue; 1370| | 1371| 0| rollback_fast_4x: 1372| 0| seq_ptr = t_save; 1373| 0| extras_ptr = e_save; 1374| 0| d_ptr = d_save; 1375| 0| l_ptr = l_save; 1376| 0| break; 1377| 0| } 1378| 4.10k| } else { 1379| 392k| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) { ------------------ | Branch (1379:16): [True: 388k, False: 3.28k] | Branch (1379:30): [True: 388k, False: 0] | Branch (1379:52): [True: 387k, False: 827] ------------------ 1380| 387k| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, return ZXC_ERROR_OVERFLOW, ------------------ | | 735| 387k| do { \ | | 736| 387k| uint32_t s1 = zxc_le32(seq_ptr); \ | | 737| 387k| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 738| 387k| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 739| 387k| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 740| 387k| seq_ptr += 4 * sizeof(uint32_t); \ | | 741| 387k| PREFETCH; \ | | 742| 387k| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 387k| do { \ | | | | 703| 387k| uint64_t ll = (S) >> 24; \ | | | | 704| 387k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 387k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 387k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 387k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.90k, False: 386k] | | | | | | ------------------ | | | | ------------------ | | | | 707| 1.90k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 1.90k| const uint64_t reserve = (RESERVE); \ | | | | 709| 1.90k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 1.90k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.81k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.90k] | | | | | | | Branch (151:42): [True: 0, False: 1.90k] | | | | | | | Branch (151:42): [True: 0, False: 1.90k] | | | | | | ------------------ | | | | ------------------ | | | | 711| 1.90k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 1.90k| (size_t)(d_end - d_ptr))) \ | | | | 713| 1.90k| ON_FAIL; \ | | | | 714| 1.90k| } \ | | | | 715| 387k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 10.8k, False: 377k] | | | | | | ------------------ | | | | ------------------ | | | | 716| 10.8k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 10.8k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 10.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 10.8k] | | | | | | ------------------ | | | | ------------------ | | | | 718| 10.8k| (size_t)(d_end - d_ptr))) \ | | | | 719| 10.8k| ON_FAIL; \ | | | | 720| 10.8k| } \ | | | | 721| 387k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 387k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 387k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 387k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 387k| l_ptr += ll; \ | | | | 725| 387k| d_ptr += ll; \ | | | | 726| 387k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1380| 387k| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 595| 387k| do { \ | | | | | | | | 596| 387k| COPY(d_ptr, off, ml); \ | | | | | | | | 597| 387k| d_ptr += ml; \ | | | | | | | | 598| 387k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (598:14): [Folded, False: 387k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 387k| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 387k] | | | | ------------------ | | ------------------ | | 743| 387k| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 387k| do { \ | | | | 703| 387k| uint64_t ll = (S) >> 24; \ | | | | 704| 387k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 387k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 387k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 387k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.96k, False: 386k] | | | | | | ------------------ | | | | ------------------ | | | | 707| 1.96k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 1.96k| const uint64_t reserve = (RESERVE); \ | | | | 709| 1.96k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 1.96k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.93k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.96k] | | | | | | | Branch (151:42): [True: 0, False: 1.96k] | | | | | | | Branch (151:42): [True: 0, False: 1.96k] | | | | | | ------------------ | | | | ------------------ | | | | 711| 1.96k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 1.96k| (size_t)(d_end - d_ptr))) \ | | | | 713| 1.96k| ON_FAIL; \ | | | | 714| 1.96k| } \ | | | | 715| 387k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 10.8k, False: 377k] | | | | | | ------------------ | | | | ------------------ | | | | 716| 10.8k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 10.8k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 10.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 10.8k] | | | | | | ------------------ | | | | ------------------ | | | | 718| 10.8k| (size_t)(d_end - d_ptr))) \ | | | | 719| 10.8k| ON_FAIL; \ | | | | 720| 10.8k| } \ | | | | 721| 387k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 387k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 387k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 387k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 387k| l_ptr += ll; \ | | | | 725| 387k| d_ptr += ll; \ | | | | 726| 387k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1380| 387k| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 595| 387k| do { \ | | | | | | | | 596| 387k| COPY(d_ptr, off, ml); \ | | | | | | | | 597| 387k| d_ptr += ml; \ | | | | | | | | 598| 387k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (598:14): [Folded, False: 387k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 387k| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 387k] | | | | ------------------ | | ------------------ | | 744| 387k| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 387k| do { \ | | | | 703| 387k| uint64_t ll = (S) >> 24; \ | | | | 704| 387k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 387k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 387k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 387k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.91k, False: 386k] | | | | | | ------------------ | | | | ------------------ | | | | 707| 1.91k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 1.91k| const uint64_t reserve = (RESERVE); \ | | | | 709| 1.91k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 1.91k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.82k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.91k] | | | | | | | Branch (151:42): [True: 0, False: 1.91k] | | | | | | | Branch (151:42): [True: 0, False: 1.91k] | | | | | | ------------------ | | | | ------------------ | | | | 711| 1.91k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 1.91k| (size_t)(d_end - d_ptr))) \ | | | | 713| 1.91k| ON_FAIL; \ | | | | 714| 1.91k| } \ | | | | 715| 387k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 10.8k, False: 377k] | | | | | | ------------------ | | | | ------------------ | | | | 716| 10.8k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 10.8k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 10.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 10.8k] | | | | | | ------------------ | | | | ------------------ | | | | 718| 10.8k| (size_t)(d_end - d_ptr))) \ | | | | 719| 10.8k| ON_FAIL; \ | | | | 720| 10.8k| } \ | | | | 721| 387k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 387k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 387k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 387k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 387k| l_ptr += ll; \ | | | | 725| 387k| d_ptr += ll; \ | | | | 726| 387k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1380| 387k| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 595| 387k| do { \ | | | | | | | | 596| 387k| COPY(d_ptr, off, ml); \ | | | | | | | | 597| 387k| d_ptr += ml; \ | | | | | | | | 598| 387k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (598:14): [Folded, False: 387k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 387k| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 387k] | | | | ------------------ | | ------------------ | | 745| 387k| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 702| 387k| do { \ | | | | 703| 387k| uint64_t ll = (S) >> 24; \ | | | | 704| 387k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 705| 387k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 505| 387k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 706| 387k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 2.03k, False: 385k] | | | | | | ------------------ | | | | ------------------ | | | | 707| 2.03k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 708| 2.03k| const uint64_t reserve = (RESERVE); \ | | | | 709| 2.03k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 710| 2.03k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 4.07k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 2.03k] | | | | | | | Branch (151:42): [True: 0, False: 2.03k] | | | | | | | Branch (151:42): [True: 0, False: 2.03k] | | | | | | ------------------ | | | | ------------------ | | | | 711| 2.03k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 712| 2.03k| (size_t)(d_end - d_ptr))) \ | | | | 713| 2.03k| ON_FAIL; \ | | | | 714| 2.03k| } \ | | | | 715| 387k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 10.6k, False: 377k] | | | | | | ------------------ | | | | ------------------ | | | | 716| 10.6k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 717| 10.6k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 10.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 10.6k] | | | | | | ------------------ | | | | ------------------ | | | | 718| 10.6k| (size_t)(d_end - d_ptr))) \ | | | | 719| 10.6k| ON_FAIL; \ | | | | 720| 10.6k| } \ | | | | 721| 387k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 530| 387k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 722| 387k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 723| 387k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 724| 387k| l_ptr += ll; \ | | | | 725| 387k| d_ptr += ll; \ | | | | 726| 387k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1380| 387k| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 595| 387k| do { \ | | | | | | | | 596| 387k| COPY(d_ptr, off, ml); \ | | | | | | | | 597| 387k| d_ptr += ml; \ | | | | | | | | 598| 387k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (598:14): [Folded, False: 387k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 727| 387k| } while (0) | | | | ------------------ | | | | | Branch (727:14): [Folded, False: 387k] | | | | ------------------ | | ------------------ | | 746| 387k| n_seq -= 4; \ | | 747| 387k| } while (0) | | ------------------ | | | Branch (747:14): [Folded, False: 387k] | | ------------------ ------------------ 1381| 387k| ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE)); 1382| 387k| } 1383| 4.10k| } 1384| | 1385| | // --- Remaining 1 sequence (Fast Path) --- 1386| 160k| while (n_seq > 0 && d_ptr < d_end_safe && l_ptr < l_end_safe_1x) { ------------------ | Branch (1386:12): [True: 156k, False: 3.12k] | Branch (1386:25): [True: 156k, False: 0] | Branch (1386:47): [True: 156k, False: 878] ------------------ 1387| | // Save state for fallback 1388| 156k| const uint8_t* seq_save = seq_ptr; 1389| 156k| const uint8_t* ext_save = extras_ptr; 1390| | 1391| 156k| const uint32_t seq = zxc_le32(seq_ptr); 1392| 156k| seq_ptr += sizeof(uint32_t); 1393| | 1394| 156k| uint64_t ll = seq >> 24; 1395| 156k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { ------------------ | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 318, False: 155k] | | ------------------ ------------------ 1396| 318| ll += zxc_read_varint(&extras_ptr, extras_end); 1397| 318| if (UNLIKELY(l_ptr + ll > l_end)) { ------------------ | | 151| 318|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 318] | | ------------------ ------------------ 1398| 0| seq_ptr = seq_save; 1399| 0| extras_ptr = ext_save; 1400| 0| break; 1401| 0| } 1402| 318| } 1403| | 1404| 156k| uint32_t m_bits = (seq >> 16) & 0xFF; 1405| 156k| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 505| 156k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1406| 156k| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 6.86k, False: 149k] | | ------------------ ------------------ 1407| | 1408| | // Strict bounds checks (including wild copy overrun safety) 1409| 156k| if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr) || ------------------ | | 151| 312k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 100, False: 155k] | | | Branch (151:42): [True: 0, False: 156k] | | | Branch (151:42): [True: 100, False: 155k] | | ------------------ ------------------ 1410| 156k| ll + ZXC_PAD_SIZE > (size_t)(l_end - l_ptr))) { 1411| 100| seq_ptr = seq_save; 1412| 100| extras_ptr = ext_save; 1413| 100| break; 1414| 100| } 1415| 155k| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 530| 155k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1416| | 1417| 155k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); 1418| 155k| l_ptr += ll; 1419| 155k| d_ptr += ll; 1420| | 1421| 155k| if (UNLIKELY(d_ptr < d_bounds && (size_t)(d_ptr - d_floor) < offset)) ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 155k] | | | Branch (151:42): [True: 0, False: 155k] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1422| 0| return ZXC_ERROR_BAD_OFFSET; 1423| | 1424| | // The loop entry check guarantees ll + ml + ZXC_PAD_SIZE bytes of 1425| | // headroom, so the wild-copy ladder (incl. overlap/fill runs) is safe. 1426| 155k| zxc_decode_copy_match(d_ptr, offset, ml); 1427| 155k| d_ptr += ml; 1428| 155k| n_seq--; 1429| 155k| } 1430| | 1431| | // --- Safe Path for Remaining Sequences --- 1432| 159k| while (n_seq > 0) { ------------------ | Branch (1432:12): [True: 155k, False: 4.10k] ------------------ 1433| 155k| uint32_t seq = zxc_le32(seq_ptr); 1434| 155k| seq_ptr += sizeof(uint32_t); 1435| | 1436| 155k| uint64_t ll = seq >> 24; 1437| 155k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 97, False: 155k] | | ------------------ ------------------ 1438| | 1439| 155k| uint32_t m_bits = (seq >> 16) & 0xFF; 1440| 155k| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 505| 155k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1441| 155k| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 6.71k, False: 148k] | | ------------------ ------------------ 1442| 155k| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 530| 155k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1443| | 1444| 155k| if (UNLIKELY(ll + ml > (size_t)(d_end - d_ptr) || l_ptr + ll > l_end)) ------------------ | | 151| 310k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 155k] | | | Branch (151:42): [True: 0, False: 155k] | | | Branch (151:42): [True: 0, False: 155k] | | ------------------ ------------------ 1445| 0| return ZXC_ERROR_OVERFLOW; 1446| 155k| ZXC_MEMCPY(d_ptr, l_ptr, ll); ------------------ | | 167| 155k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1447| 155k| l_ptr += ll; 1448| 155k| d_ptr += ll; 1449| | 1450| 155k| if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET; ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 155k] | | ------------------ ------------------ 1451| 155k| const uint8_t* match_src = d_ptr - offset; 1452| | 1453| 155k| zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml); 1454| 155k| d_ptr += ml; 1455| 155k| n_seq--; 1456| 155k| } 1457| | 1458| | // --- Trailing Literals --- 1459| | // Copy remaining literals from source stream (literal exhaustion) 1460| 4.10k| if (UNLIKELY(l_ptr > l_end)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 1461| 4.10k| if (UNLIKELY(d_ptr > d_end)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 1462| | 1463| 4.10k| const size_t remaining_literals = (size_t)(l_end - l_ptr); 1464| 4.10k| if (UNLIKELY(remaining_literals > (size_t)(d_end - d_ptr))) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 1465| 4.10k| ZXC_MEMCPY(d_ptr, l_ptr, remaining_literals); ------------------ | | 167| 4.10k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1466| 4.10k| d_ptr += remaining_literals; 1467| | 1468| 4.10k| return (int)(d_ptr - dst); 1469| 4.10k|} zxc_decompress.c:zxc_decode_block_ghi: 1532| 4.10k| const size_t dst_capacity) { 1533| 4.10k| return zxc_decode_block_ghi_impl(ctx, src, src_size, dst, dst_capacity, 0, 0); 1534| 4.10k|} zxc_decompress_chunk_wrapper: 396| 4.90k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 397| | // dict_size is constant for a stream; this per-block branch (outside the decode 398| | // loop) routes to the dict variant only when a dictionary is active, so the 399| | // no-dict path runs the dict-free chunk wrapper (identical codegen to main). 400| 4.90k| const zxc_decompress_func_t func = ctx->dict_size ? ZXC_DISPATCH_LOAD(zxc_decompress_dict_ptr) ------------------ | | 298| 4.90k|#define ZXC_DISPATCH_LOAD(ptr) atomic_load_explicit(&(ptr), memory_order_acquire) ------------------ | Branch (400:40): [True: 0, False: 4.90k] ------------------ 401| 4.90k| : ZXC_DISPATCH_LOAD(zxc_decompress_ptr); ------------------ | | 298| 4.90k|#define ZXC_DISPATCH_LOAD(ptr) atomic_load_explicit(&(ptr), memory_order_acquire) ------------------ 402| 4.90k| if (UNLIKELY(!func)) return zxc_decompress_dispatch_init(ctx, src, src_sz, dst, dst_cap); ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1, False: 4.90k] | | ------------------ ------------------ 403| 4.90k| return func(ctx, src, src_sz, dst, dst_cap); 404| 4.90k|} zxc_compress_chunk_wrapper: 431| 4.90k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 432| 4.90k| const zxc_compress_func_t func = ZXC_DISPATCH_LOAD(zxc_compress_ptr); ------------------ | | 298| 4.90k|#define ZXC_DISPATCH_LOAD(ptr) atomic_load_explicit(&(ptr), memory_order_acquire) ------------------ 433| 4.90k| if (UNLIKELY(!func)) return zxc_compress_dispatch_init(ctx, src, src_sz, dst, dst_cap); ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1, False: 4.90k] | | ------------------ ------------------ 434| 4.90k| return func(ctx, src, src_sz, dst, dst_cap); 435| 4.90k|} zxc_create_cctx: 1115| 4.60k|zxc_cctx* zxc_create_cctx(const zxc_compress_opts_t* opts) { 1116| 4.60k| zxc_cctx* const cctx = (zxc_cctx*)ZXC_CALLOC(1, sizeof(zxc_cctx)); ------------------ | | 72| 4.60k|#define ZXC_CALLOC(nmemb, size) calloc(nmemb, size) ------------------ 1117| 4.60k| if (UNLIKELY(!cctx)) return NULL; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 1118| | 1119| | // Resolve and store sticky defaults. 1120| 4.60k| cctx->stored_level = ZXC_OPTS_LEVEL(opts, ZXC_LEVEL_DEFAULT); ------------------ | | 842| 4.60k|#define ZXC_OPTS_LEVEL(o, dflt) zxc_level_clamp(((o) && (o)->level > 0) ? (o)->level : (dflt)) | | ------------------ | | | Branch (842:50): [True: 4.60k, False: 0] | | | Branch (842:57): [True: 4.60k, False: 0] | | ------------------ ------------------ 1121| 4.60k| cctx->stored_block_size = ZXC_OPTS_BLOCK_SIZE(opts, ZXC_BLOCK_SIZE_DEFAULT); ------------------ | | 844| 4.60k|#define ZXC_OPTS_BLOCK_SIZE(o, dflt) (((o) && (o)->block_size > 0) ? (o)->block_size : (dflt)) | | ------------------ | | | Branch (844:40): [True: 4.60k, False: 0] | | | Branch (844:47): [True: 4.60k, False: 0] | | ------------------ ------------------ 1122| 4.60k| cctx->stored_checksum = opts ? opts->checksum_enabled : 0; ------------------ | Branch (1122:29): [True: 4.60k, False: 0] ------------------ 1123| | 1124| 4.60k| if (opts) { ------------------ | Branch (1124:9): [True: 4.60k, False: 0] ------------------ 1125| | // LCOV_EXCL_START 1126| 4.60k| if (UNLIKELY(!zxc_validate_block_size(cctx->stored_block_size) || ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 1127| 4.60k| zxc_cctx_init(&cctx->inner, cctx->stored_block_size, 1, cctx->stored_level, 1128| 4.60k| cctx->stored_checksum, 0) != ZXC_OK)) { 1129| 0| ZXC_FREE(cctx); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 1130| 0| return NULL; 1131| 0| } 1132| | // LCOV_EXCL_STOP 1133| 4.60k| cctx->last_block_size = cctx->stored_block_size; 1134| 4.60k| cctx->initialized = 1; 1135| 4.60k| } 1136| | 1137| 4.60k| return cctx; 1138| 4.60k|} zxc_free_cctx: 1147| 4.60k|void zxc_free_cctx(zxc_cctx* cctx) { 1148| 4.60k| if (UNLIKELY(!cctx)) return; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 1149| | // Static cctx: handle + inner buffers live inside the caller's workspace, 1150| | // which we do not own. Free is a no-op; the caller owns the workspace. 1151| 4.60k| if (cctx->owns_workspace) return; ------------------ | Branch (1151:9): [True: 0, False: 4.60k] ------------------ 1152| 4.60k| if (cctx->initialized) zxc_cctx_free(&cctx->inner); ------------------ | Branch (1152:9): [True: 4.60k, False: 0] ------------------ 1153| 4.60k| ZXC_FREE(cctx); ------------------ | | 84| 4.60k|#define ZXC_FREE(ptr) free(ptr) ------------------ 1154| 4.60k|} zxc_compress_block: 1439| 4.90k| const zxc_compress_opts_t* opts) { 1440| 4.90k| if (UNLIKELY(!cctx || !src || !dst || src_size == 0 || dst_capacity == 0)) ------------------ | | 151| 39.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | ------------------ ------------------ 1441| 0| return ZXC_ERROR_NULL_INPUT; 1442| | 1443| | // Block API processes a single format-conformant block: src_size must not 1444| | // exceed ZXC_BLOCK_SIZE_MAX. Callers with larger inputs should use the 1445| | // frame or streaming APIs which chunk transparently. 1446| 4.90k| if (UNLIKELY(src_size > ZXC_BLOCK_SIZE_MAX)) return ZXC_ERROR_BAD_BLOCK_SIZE; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1447| | 1448| 4.90k| const int checksum_enabled = opts ? opts->checksum_enabled : cctx->stored_checksum; ------------------ | Branch (1448:34): [True: 4.90k, False: 0] ------------------ 1449| 4.90k| const int level = ZXC_OPTS_LEVEL(opts, cctx->stored_level); ------------------ | | 842| 4.90k|#define ZXC_OPTS_LEVEL(o, dflt) zxc_level_clamp(((o) && (o)->level > 0) ? (o)->level : (dflt)) | | ------------------ | | | Branch (842:50): [True: 4.90k, False: 0] | | | Branch (842:57): [True: 4.90k, False: 0] | | ------------------ ------------------ 1450| | // For block API, block_size == src_size (the caller compresses one block at a time). 1451| 4.90k| const size_t block_size = ZXC_OPTS_BLOCK_SIZE(opts, cctx->stored_block_size); ------------------ | | 844| 4.90k|#define ZXC_OPTS_BLOCK_SIZE(o, dflt) (((o) && (o)->block_size > 0) ? (o)->block_size : (dflt)) | | ------------------ | | | Branch (844:40): [True: 4.90k, False: 0] | | | Branch (844:47): [True: 4.90k, False: 0] | | ------------------ ------------------ 1452| 4.90k| const size_t min_bs = zxc_block_size_ceil(src_size); 1453| | 1454| | // Always ensure internal buffers can hold src_size. 1455| | // When a dictionary is active, offset_bits must accommodate dict + block. 1456| 4.90k| const uint8_t* b_dict = opts ? (const uint8_t*)opts->dict : NULL; ------------------ | Branch (1456:29): [True: 4.90k, False: 0] ------------------ 1457| 4.90k| const size_t b_dict_size = ZXC_OPTS_DICT_SIZE(opts); ------------------ | | 837| 4.90k|#define ZXC_OPTS_DICT_SIZE(o) (((o) && (o)->dict) ? (o)->dict_size : (size_t)0) | | ------------------ | | | Branch (837:33): [True: 4.90k, False: 0] | | | Branch (837:40): [True: 0, False: 4.90k] | | ------------------ ------------------ 1458| 4.90k| const size_t base_block_size = (block_size > min_bs) ? block_size : min_bs; ------------------ | Branch (1458:36): [True: 4.22k, False: 687] ------------------ 1459| 4.90k| const size_t effective_block_size = 1460| 4.90k| b_dict_size > 0 ? zxc_block_size_ceil(b_dict_size + base_block_size) : base_block_size; ------------------ | Branch (1460:9): [True: 0, False: 4.90k] ------------------ 1461| | 1462| | // Static cctx: the workspace cannot grow, so reject anything forcing a 1463| | // re-partition - another block size, or a level raise into the 1464| | // optimal-parser tier it carries no opt_scratch for. Re-initing on the heap 1465| | // would break the no-allocation contract and leak: zxc_free_cctx is a no-op 1466| | // for static contexts. 1467| 4.90k| if (UNLIKELY(cctx->owns_workspace && effective_block_size != cctx->last_block_size)) ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1468| 0| return ZXC_ERROR_BAD_BLOCK_SIZE; 1469| 4.90k| if (UNLIKELY(cctx->owns_workspace && level >= ZXC_LEVEL_DENSITY && !cctx->inner.opt_scratch)) ------------------ | | 151| 9.81k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1470| 0| return ZXC_ERROR_BAD_LEVEL; 1471| | 1472| 4.90k| cctx->stored_level = level; 1473| 4.90k| cctx->stored_block_size = effective_block_size; 1474| 4.90k| cctx->stored_checksum = checksum_enabled; 1475| | 1476| | // Re-init when block_size changed, or when a per-call level raise into 1477| | // the optimal-parser tier requires the opt_scratch region that inits at 1478| | // level < ZXC_LEVEL_DENSITY do not allocate (using it NULL would crash). 1479| 4.90k| if (UNLIKELY(!cctx->initialized || cctx->last_block_size != effective_block_size || ------------------ | | 151| 24.5k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1480| 4.90k| (level >= ZXC_LEVEL_DENSITY && !cctx->inner.opt_scratch))) { 1481| 0| if (cctx->initialized) { ------------------ | Branch (1481:13): [True: 0, False: 0] ------------------ 1482| | // LCOV_EXCL_START 1483| 0| zxc_cctx_free(&cctx->inner); 1484| 0| cctx->initialized = 0; 1485| | // LCOV_EXCL_STOP 1486| 0| } 1487| | // LCOV_EXCL_START 1488| 0| if (UNLIKELY(zxc_cctx_init(&cctx->inner, effective_block_size, 1, level, checksum_enabled, ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1489| 0| b_dict_size) != ZXC_OK)) 1490| 0| return ZXC_ERROR_MEMORY; 1491| | // LCOV_EXCL_STOP 1492| 0| cctx->last_block_size = effective_block_size; 1493| 0| cctx->initialized = 1; 1494| 4.90k| } else { 1495| 4.90k| cctx->inner.compression_level = level; 1496| 4.90k| cctx->inner.checksum_enabled = checksum_enabled; 1497| 4.90k| } 1498| | 1499| 4.90k| cctx->inner.dict_size = b_dict_size; 1500| | 1501| 4.90k| int res; 1502| 4.90k| if (b_dict && b_dict_size > 0) { ------------------ | Branch (1502:9): [True: 0, False: 4.90k] | Branch (1502:19): [True: 0, False: 0] ------------------ 1503| | // [dict | block] assembled in the cctx-owned dict_buffer 1504| 0| uint8_t* const combined = cctx->inner.dict_buffer; 1505| 0| ZXC_MEMCPY(combined, b_dict, b_dict_size); ------------------ | | 167| 0|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1506| 0| ZXC_MEMCPY(combined + b_dict_size, src, src_size); ------------------ | | 167| 0|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1507| 0| res = zxc_compress_chunk_wrapper(&cctx->inner, combined, b_dict_size + src_size, 1508| 0| (uint8_t*)dst, dst_capacity); 1509| 4.90k| } else { 1510| 4.90k| res = zxc_compress_chunk_wrapper(&cctx->inner, (const uint8_t*)src, src_size, (uint8_t*)dst, 1511| 4.90k| dst_capacity); 1512| 4.90k| } 1513| 4.90k| if (UNLIKELY(res < 0)) return res; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1514| 4.90k| return (int64_t)res; 1515| 4.90k|} zxc_dispatch.c:zxc_decompress_dispatch_init: 364| 1| const size_t dst_cap) { 365| 1| const zxc_variant_set_t v = zxc_select_variants(); 366| 1| ZXC_DISPATCH_STORE(zxc_decompress_ptr, v.decompress); ------------------ | | 297| 1|#define ZXC_DISPATCH_STORE(ptr, val) atomic_store_explicit(&(ptr), (val), memory_order_release) ------------------ 367| 1| ZXC_DISPATCH_STORE(zxc_decompress_dict_ptr, v.decompress_dict); ------------------ | | 297| 1|#define ZXC_DISPATCH_STORE(ptr, val) atomic_store_explicit(&(ptr), (val), memory_order_release) ------------------ 368| 1| return (ctx->dict_size ? v.decompress_dict : v.decompress)(ctx, src, src_sz, dst, dst_cap); ------------------ | Branch (368:13): [True: 0, False: 1] ------------------ 369| 1|} zxc_dispatch.c:zxc_select_variants: 335| 2|static zxc_variant_set_t zxc_select_variants(void) { 336| 2| const zxc_cpu_feature_t cpu = zxc_detect_cpu_features(); 337| 2| (void)cpu; 338| | 339| |#ifndef ZXC_ONLY_DEFAULT 340| |#if defined(__x86_64__) || defined(_M_X64) 341| | if (cpu == ZXC_CPU_AVX512) ZXC_RETURN_VARIANT_SET(_avx512); 342| | if (cpu == ZXC_CPU_AVX2) ZXC_RETURN_VARIANT_SET(_avx2); 343| |#elif defined(__arm__) || defined(_M_ARM) 344| | // 32-bit ARM: the only arch with a real runtime NEON probe (getauxval). 345| | // cppcheck-suppress knownConditionTrueFalse 346| | if (cpu == ZXC_CPU_NEON) ZXC_RETURN_VARIANT_SET(_neon32); 347| |#endif 348| |#endif 349| 2| ZXC_RETURN_VARIANT_SET(_default); ------------------ | | 321| 2| do { \ | | 322| 2| const zxc_variant_set_t v_ = { \ | | 323| 2| zxc_decompress_chunk_wrapper##sfx, zxc_decompress_chunk_wrapper_dict##sfx, \ | | 324| 2| zxc_decompress_chunk_wrapper_safe##sfx, zxc_compress_chunk_wrapper##sfx}; \ | | 325| 2| return v_; \ | | 326| 2| } while (0) | | ------------------ | | | Branch (326:14): [Folded, False: 0] | | ------------------ ------------------ 350| 2|} zxc_dispatch.c:zxc_detect_cpu_features: 200| 2|static zxc_cpu_feature_t zxc_detect_cpu_features(void) { 201| 2|#ifdef ZXC_ONLY_DEFAULT 202| 2| return ZXC_CPU_GENERIC; 203| |#else 204| | zxc_cpu_feature_t features = ZXC_CPU_GENERIC; 205| | 206| |#if defined(__x86_64__) || defined(_M_X64) 207| | // AVX2/AVX512 need OS-enabled YMM/ZMM state: gate on OSXSAVE + XGETBV/XCR0, 208| | // not CPUID alone (else a VEX/EVEX op faults #UD when the OS hasn't enabled it). 209| | uint32_t regs[4]; 210| | int sse2 = 0; 211| | int avx2 = 0; 212| | int avx512 = 0; 213| | int bmi_lzcnt = 0; 214| | 215| | zxc_cpuid(0, 0, regs); 216| | const uint32_t max_leaf = regs[0]; 217| | 218| | zxc_cpuid(1, 0, regs); 219| | if (regs[3] & (1U << 26)) sse2 = 1; // CPUID.1:EDX[26] 220| | if ((regs[2] & (1U << 27)) && max_leaf >= 7) { // OSXSAVE, and leaf 7 is real 221| | const uint64_t xcr0 = zxc_xgetbv0(); 222| | if ((xcr0 & 0x6) == 0x6) { // SSE+YMM enabled 223| | zxc_cpuid(7, 0, regs); 224| | const int bmi1 = (regs[1] >> 3) & 1; // CPUID.7.0:EBX[3] 225| | const int bmi2 = (regs[1] >> 8) & 1; // CPUID.7.0:EBX[8] 226| | if (regs[1] & (1U << 5)) avx2 = 1; 227| | // AVX512 also needs XCR0[5..7] (opmask/ZMM) 228| | if ((regs[1] & (1U << 16)) && (regs[1] & (1U << 30)) && (regs[2] & (1U << 1)) && 229| | (regs[2] & (1U << 6)) && (xcr0 & 0xE0) == 0xE0) 230| | avx512 = 1; /* AVX512 tier = F+BW+VBMI+VBMI2, as the variant is built */ 231| | // The AVX2/AVX512 variants are compiled with BMI1/BMI2/LZCNT enabled, 232| | // so both gates must prove those bits too. LZCNT (ABM) lives in 233| | // CPUID.80000001H:ECX[5]; that leaf is architectural on x86-64. 234| | zxc_cpuid(0x80000001U, 0, regs); 235| | bmi_lzcnt = bmi1 && bmi2 && ((regs[2] >> 5) & 1); 236| | } 237| | } 238| | 239| | if (avx512 && bmi_lzcnt) { 240| | features = ZXC_CPU_AVX512; 241| | } else if (avx2 && bmi_lzcnt) { 242| | features = ZXC_CPU_AVX2; 243| | } else if (sse2) { 244| | features = ZXC_CPU_SSE2; 245| | } 246| | 247| |#elif defined(__aarch64__) || defined(_M_ARM64) 248| | // ARM64 usually guarantees NEON 249| | features = ZXC_CPU_NEON; 250| | 251| |#elif defined(__arm__) || defined(_M_ARM) 252| | // ARM32 Runtime detection for Linux 253| |#if defined(__linux__) 254| | const unsigned long hwcaps = getauxval(AT_HWCAP); 255| | if (hwcaps & HWCAP_NEON) { 256| | features = ZXC_CPU_NEON; 257| | } 258| |#else 259| |// Fallback for non-Linux: rely on compiler flags. 260| |// If compiled with -mfpu=neon, we assume target supports it. 261| |// Otherwise, safe default is GENERIC. 262| |#if defined(__ARM_NEON) 263| | features = ZXC_CPU_NEON; 264| |#endif 265| |#endif 266| |#endif 267| | 268| | return features; 269| |#endif 270| 2|} zxc_dispatch.c:zxc_compress_dispatch_init: 385| 1| const size_t dst_cap) { 386| 1| const zxc_variant_set_t v = zxc_select_variants(); 387| | ZXC_DISPATCH_STORE(zxc_compress_ptr, v.compress); ------------------ | | 297| 1|#define ZXC_DISPATCH_STORE(ptr, val) atomic_store_explicit(&(ptr), (val), memory_order_release) ------------------ 388| 1| return v.compress(ctx, src, src_sz, dst, dst_cap); 389| 1|} zxc_common.c:zxc_log2_u32: 913| 23.0k|static ZXC_ALWAYS_INLINE uint32_t zxc_log2_u32(const uint32_t v) { 914| |#ifdef _MSC_VER 915| | unsigned long index; 916| | return (v == 0) ? 0 : (_BitScanReverse(&index, v) ? index : 0); 917| |#else 918| 23.0k| return (v == 0) ? 0 : (uint32_t)(31 - __builtin_clz(v)); ------------------ | Branch (918:12): [True: 0, False: 23.0k] ------------------ 919| 23.0k|#endif 920| 23.0k|} zxc_common.c:zxc_store_le32: 1246| 26.2k|static ZXC_ALWAYS_INLINE void zxc_store_le32(void* p, const uint32_t v) { 1247| |#ifdef ZXC_BIG_ENDIAN 1248| | const uint32_t s = ZXC_BSWAP32(v); 1249| | ZXC_MEMCPY(p, &s, sizeof(s)); 1250| |#else 1251| 26.2k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 167| 26.2k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1252| 26.2k|#endif 1253| 26.2k|} zxc_common.c:zxc_store_le16: 1228| 9.21k|static ZXC_ALWAYS_INLINE void zxc_store_le16(void* p, const uint16_t v) { 1229| |#ifdef ZXC_BIG_ENDIAN 1230| | const uint16_t s = ZXC_BSWAP16(v); 1231| | ZXC_MEMCPY(p, &s, sizeof(s)); 1232| |#else 1233| 9.21k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 167| 9.21k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1234| 9.21k|#endif 1235| 9.21k|} zxc_common.c:zxc_hash16: 1298| 9.21k|static ZXC_ALWAYS_INLINE uint16_t zxc_hash16(const uint8_t* p) { 1299| 9.21k| const uint64_t v1 = zxc_le64(p); 1300| 9.21k| const uint64_t v2 = zxc_le64(p + 8); 1301| 9.21k| uint64_t h = v1 ^ v2 ^ ZXC_HASH_PRIME2; ------------------ | | 587| 9.21k|#define ZXC_HASH_PRIME2 0xD2D84A61D2D84A61ULL ------------------ 1302| 9.21k| h ^= h << 13; 1303| 9.21k| h ^= h >> 7; 1304| 9.21k| h ^= h << 17; 1305| 9.21k| const uint32_t res = (uint32_t)((h >> 32) ^ h); 1306| 9.21k| return (uint16_t)((res >> 16) ^ res); 1307| 9.21k|} zxc_common.c:zxc_le64: 1209| 40.0k|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1210| 40.0k| uint64_t v; 1211| 40.0k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 40.0k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1212| |#ifdef ZXC_BIG_ENDIAN 1213| | return ZXC_BSWAP64(v); 1214| |#else 1215| 40.0k| return v; 1216| 40.0k|#endif 1217| 40.0k|} zxc_common.c:zxc_le32: 1146| 25.5k|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1147| 25.5k| uint32_t v; 1148| 25.5k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 25.5k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1149| |#ifdef ZXC_BIG_ENDIAN 1150| | return ZXC_BSWAP32(v); 1151| |#else 1152| 25.5k| return v; 1153| 25.5k|#endif 1154| 25.5k|} zxc_common.c:zxc_le16: 1127| 4.60k|static ZXC_ALWAYS_INLINE uint16_t zxc_le16(const void* p) { 1128| 4.60k| uint16_t v; 1129| 4.60k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 4.60k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1130| |#ifdef ZXC_BIG_ENDIAN 1131| | return ZXC_BSWAP16(v); 1132| |#else 1133| 4.60k| return v; 1134| 4.60k|#endif 1135| 4.60k|} zxc_common.c:zxc_hash8: 1281| 21.6k|static ZXC_ALWAYS_INLINE uint8_t zxc_hash8(const uint8_t* p) { 1282| 21.6k| const uint64_t v = zxc_le64(p); 1283| 21.6k| uint64_t h = v ^ ZXC_HASH_PRIME1; ------------------ | | 585| 21.6k|#define ZXC_HASH_PRIME1 0x9E3779B97F4A7C15ULL ------------------ 1284| 21.6k| h ^= h << 13; 1285| 21.6k| h ^= h >> 7; 1286| 21.6k| h ^= h << 17; 1287| 21.6k| return (uint8_t)((h >> 32) ^ h); 1288| 21.6k|} zxc_common.c:zxc_store_le64: 1264| 4.60k|static ZXC_ALWAYS_INLINE void zxc_store_le64(void* p, const uint64_t v) { 1265| |#ifdef ZXC_BIG_ENDIAN 1266| | const uint64_t s = ZXC_BSWAP64(v); 1267| | ZXC_MEMCPY(p, &s, sizeof(s)); 1268| |#else 1269| 4.60k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 167| 4.60k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1270| 4.60k|#endif 1271| 4.60k|} zxc_compress.c:zxc_get_lz77_params: 1012| 4.90k|static ZXC_ALWAYS_INLINE zxc_lz77_params_t zxc_get_lz77_params(const int level) { 1013| | // The distance floor stops at level 5: the slow levels keep every distance. 1014| | // search_depth, sufficient_len, use_lazy, lazy_attempts, lazy_len_threshold, step_base, 1015| | // step_shift, min_offset 1016| 4.90k| static const zxc_lz77_params_t table[7] = { 1017| 4.90k| {3, 16, 0, 0, 0, 4, 4, ZXC_LZ_MINDIST}, // fallback ------------------ | | 540| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1018| 4.90k| {3, 16, 0, 0, 0, 4, 4, ZXC_LZ_MINDIST}, // level 1 ------------------ | | 540| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1019| 4.90k| {3, 18, 0, 0, 0, 3, 6, ZXC_LZ_MINDIST}, // level 2 ------------------ | | 540| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1020| 4.90k| {3, 16, 1, 4, 128, 1, 4, ZXC_LZ_MINDIST}, // level 3 ------------------ | | 540| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1021| 4.90k| {3, 18, 1, 4, 128, 1, 5, ZXC_LZ_MINDIST}, // level 4 ------------------ | | 540| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1022| 4.90k| {64, 256, 1, 16, 128, 1, 8, ZXC_LZ_MINDIST}, // level 5 ------------------ | | 540| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1023| 4.90k| {64, 256, 0, 0, 0, 1, 8, 1} // level 6 1024| 4.90k| }; 1025| 4.90k| return (level >= ZXC_LEVEL_ULTRA) ------------------ | Branch (1025:12): [True: 0, False: 4.90k] ------------------ 1026| 4.90k| ? (zxc_lz77_params_t){128, 256, 0, 0, 0, 1, 8, 1} 1027| 4.90k| : table[level < ZXC_LEVEL_FASTEST ? ZXC_LEVEL_FASTEST : level]; ------------------ | Branch (1027:24): [True: 0, False: 4.90k] ------------------ 1028| 4.90k|} zxc_compress.c:zxc_block_is_short_dist_bound: 1169| 4.90k| const size_t size) { 1170| 4.90k| const size_t d_max = ZXC_LZ_MINDIST; ------------------ | | 540| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1171| | // Too small to sample, and too small to gain: keep every distance. 1172| 4.90k| if (size < d_max + sizeof(uint32_t)) return 1; ------------------ | Branch (1172:9): [True: 377, False: 4.53k] ------------------ 1173| | 1174| 4.53k| size_t want = size / ZXC_LZ_MINDIST_PROBE_PER_KB; ------------------ | | 547| 4.53k|#define ZXC_LZ_MINDIST_PROBE_PER_KB 1024 ------------------ 1175| 4.53k| if (want < ZXC_LZ_MINDIST_PROBE_MIN) want = ZXC_LZ_MINDIST_PROBE_MIN; ------------------ | | 548| 4.53k|#define ZXC_LZ_MINDIST_PROBE_MIN 16 ------------------ if (want < ZXC_LZ_MINDIST_PROBE_MIN) want = ZXC_LZ_MINDIST_PROBE_MIN; ------------------ | | 548| 2.82k|#define ZXC_LZ_MINDIST_PROBE_MIN 16 ------------------ | Branch (1175:9): [True: 2.82k, False: 1.70k] ------------------ 1176| 4.53k| if (want > ZXC_LZ_MINDIST_PROBE_MAX) want = ZXC_LZ_MINDIST_PROBE_MAX; ------------------ | | 549| 4.53k|#define ZXC_LZ_MINDIST_PROBE_MAX 64 ------------------ if (want > ZXC_LZ_MINDIST_PROBE_MAX) want = ZXC_LZ_MINDIST_PROBE_MAX; ------------------ | | 549| 1.34k|#define ZXC_LZ_MINDIST_PROBE_MAX 64 ------------------ | Branch (1176:9): [True: 1.34k, False: 3.18k] ------------------ 1177| 4.53k| size_t stride = size / want; 1178| 4.53k| if (stride < d_max) stride = d_max; ------------------ | Branch (1178:9): [True: 840, False: 3.69k] ------------------ 1179| | 1180| 4.53k| const size_t planned = (size - sizeof(uint32_t) - d_max) / stride + 1; 1181| 4.53k| const size_t bar = (size_t)ZXC_LZ_MINDIST_MAX_SHORT_PCT * planned; ------------------ | | 557| 4.53k|#define ZXC_LZ_MINDIST_MAX_SHORT_PCT 20 ------------------ 1182| | 1183| 4.53k| size_t samples = 0; 1184| 4.53k| size_t hits = 0; 1185| 70.6k| for (size_t i = d_max; i + sizeof(uint32_t) <= size; i += stride) { ------------------ | Branch (1185:28): [True: 70.6k, False: 0] ------------------ 1186| 70.6k| const uint32_t cur = zxc_le32(blk + i); 1187| 70.6k| samples++; 1188| 1.50M| for (size_t d = 1; d < d_max; d++) { ------------------ | Branch (1188:28): [True: 1.45M, False: 44.7k] ------------------ 1189| 1.45M| if (zxc_le32(blk + i - d) == cur) { ------------------ | Branch (1189:17): [True: 25.9k, False: 1.43M] ------------------ 1190| 25.9k| hits++; 1191| 25.9k| break; 1192| 25.9k| } 1193| 1.45M| } 1194| 70.6k| if (hits * 100U > bar) return 1; // can only rise ------------------ | Branch (1194:13): [True: 3.66k, False: 67.0k] ------------------ 1195| 67.0k| if ((hits + (planned - samples)) * 100U <= bar) return 0; // out of reach ------------------ | Branch (1195:13): [True: 870, False: 66.1k] ------------------ 1196| 67.0k| } 1197| 0| return hits * 100U > bar; 1198| 4.53k|} zxc_compress.c:zxc_le32: 1146| 8.90M|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1147| 8.90M| uint32_t v; 1148| 8.90M| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 8.90M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1149| |#ifdef ZXC_BIG_ENDIAN 1150| | return ZXC_BSWAP32(v); 1151| |#else 1152| 8.90M| return v; 1153| 8.90M|#endif 1154| 8.90M|} zxc_compress.c:zxc_le64: 1209| 32.7M|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1210| 32.7M| uint64_t v; 1211| 32.7M| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 32.7M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1212| |#ifdef ZXC_BIG_ENDIAN 1213| | return ZXC_BSWAP64(v); 1214| |#else 1215| 32.7M| return v; 1216| 32.7M|#endif 1217| 32.7M|} zxc_compress.c:zxc_ctz64: 1390| 3.47M|static ZXC_ALWAYS_INLINE int zxc_ctz64(const uint64_t x) { 1391| 3.47M| if (x == 0) return 64; ------------------ | Branch (1391:9): [True: 0, False: 3.47M] ------------------ 1392| 3.47M|#if defined(__GNUC__) || defined(__clang__) 1393| 3.47M| return __builtin_ctzll(x); 1394| |#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) 1395| | unsigned long r; 1396| | _BitScanForward64(&r, x); 1397| | return (int)r; 1398| |#elif defined(_MSC_VER) 1399| | // Use two 32-bit scans to avoid fragile 64-bit De Bruijn multiplication. 1400| | unsigned long r; 1401| | const uint32_t lo = (uint32_t)x; 1402| | if (_BitScanForward(&r, lo)) return (int)r; 1403| | _BitScanForward(&r, (uint32_t)(x >> 32)); 1404| | return 32 + (int)r; 1405| |#else 1406| | // Fallback De Bruijn for non-GCC/non-MSVC compilers 1407| | static const int Debruijn64[64] = { 1408| | 0, 1, 48, 2, 57, 49, 28, 3, 61, 58, 50, 42, 38, 29, 17, 4, 62, 55, 59, 36, 53, 51, 1409| | 43, 22, 45, 39, 33, 30, 24, 18, 12, 5, 63, 47, 56, 27, 60, 41, 37, 16, 54, 35, 52, 21, 1410| | 44, 32, 23, 11, 46, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6}; 1411| | return Debruijn64[((x & (0ULL - x)) * 0x03F79D71B4CA8B09ULL) >> 58]; 1412| |#endif 1413| 3.47M|} zxc_compress.c:zxc_ctz32: 1364| 250k|static ZXC_ALWAYS_INLINE int zxc_ctz32(const uint32_t x) { 1365| 250k| if (x == 0) return 32; ------------------ | Branch (1365:9): [True: 0, False: 250k] ------------------ 1366| 250k|#if defined(__GNUC__) || defined(__clang__) 1367| 250k| return __builtin_ctz(x); 1368| |#elif defined(_MSC_VER) 1369| | unsigned long r; 1370| | _BitScanForward(&r, x); 1371| | return (int)r; 1372| |#else 1373| | // Fallback De Bruijn (32 bits) 1374| | static const int DeBruijn32[32] = {0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 1375| | 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 1376| | 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; 1377| | return DeBruijn32[((uint32_t)((x & (0U - x)) * 0x077CB531U)) >> 27]; 1378| |#endif 1379| 250k|} zxc_compress.c:zxc_copy32: 1337| 1.08M|static ZXC_ALWAYS_INLINE void zxc_copy32(void* dst, const void* src) { 1338| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 1339| | // AVX2/AVX512: Single 256-bit (32 byte) unaligned load/store 1340| | _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src)); 1341| |#elif defined(ZXC_USE_SSE2) 1342| | // SSE2: Two 128-bit (16 byte) unaligned load/stores (no 256-bit regs) 1343| 1.08M| _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); 1344| 1.08M| _mm_storeu_si128((__m128i*)((uint8_t*)dst + 16), 1345| 1.08M| _mm_loadu_si128((const __m128i*)((const uint8_t*)src + 16))); 1346| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1347| | // NEON: Two 128-bit (16 byte) unaligned load/stores 1348| | vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); 1349| | vst1q_u8((uint8_t*)dst + 16, vld1q_u8((const uint8_t*)src + 16)); 1350| |#else 1351| | ZXC_MEMCPY(dst, src, 32); 1352| |#endif 1353| 1.08M|} zxc_compress.c:zxc_checksum: 1447| 1.63k| const uint8_t hash_method) { 1448| 1.63k| (void)hash_method; /* single algorithm for now; extend when adding more */ 1449| 1.63k| const uint64_t hash = rapidhash(input, len); 1450| | 1451| | return (uint32_t)(hash ^ (hash >> (sizeof(uint32_t) * CHAR_BIT))); 1452| 1.63k|} zxc_compress.c:zxc_store_le32: 1246| 1.63k|static ZXC_ALWAYS_INLINE void zxc_store_le32(void* p, const uint32_t v) { 1247| |#ifdef ZXC_BIG_ENDIAN 1248| | const uint32_t s = ZXC_BSWAP32(v); 1249| | ZXC_MEMCPY(p, &s, sizeof(s)); 1250| |#else 1251| 1.63k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 167| 1.63k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1252| 1.63k|#endif 1253| 1.63k|} zxc_decompress.c:zxc_le32: 1146| 2.63M|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1147| 2.63M| uint32_t v; 1148| 2.63M| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 2.63M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1149| |#ifdef ZXC_BIG_ENDIAN 1150| | return ZXC_BSWAP32(v); 1151| |#else 1152| 2.63M| return v; 1153| 2.63M|#endif 1154| 2.63M|} zxc_decompress.c:zxc_checksum: 1447| 1.63k| const uint8_t hash_method) { 1448| 1.63k| (void)hash_method; /* single algorithm for now; extend when adding more */ 1449| 1.63k| const uint64_t hash = rapidhash(input, len); 1450| | 1451| | return (uint32_t)(hash ^ (hash >> (sizeof(uint32_t) * CHAR_BIT))); 1452| 1.63k|} zxc_decompress.c:zxc_copy32: 1337| 13.3M|static ZXC_ALWAYS_INLINE void zxc_copy32(void* dst, const void* src) { 1338| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 1339| | // AVX2/AVX512: Single 256-bit (32 byte) unaligned load/store 1340| | _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src)); 1341| |#elif defined(ZXC_USE_SSE2) 1342| | // SSE2: Two 128-bit (16 byte) unaligned load/stores (no 256-bit regs) 1343| 13.3M| _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); 1344| 13.3M| _mm_storeu_si128((__m128i*)((uint8_t*)dst + 16), 1345| 13.3M| _mm_loadu_si128((const __m128i*)((const uint8_t*)src + 16))); 1346| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1347| | // NEON: Two 128-bit (16 byte) unaligned load/stores 1348| | vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); 1349| | vst1q_u8((uint8_t*)dst + 16, vld1q_u8((const uint8_t*)src + 16)); 1350| |#else 1351| | ZXC_MEMCPY(dst, src, 32); 1352| |#endif 1353| 13.3M|} zxc_decompress.c:zxc_copy16: 1318| 2.25M|static ZXC_ALWAYS_INLINE void zxc_copy16(void* dst, const void* src) { 1319| 2.25M|#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) || defined(ZXC_USE_SSE2) 1320| | // x86 SSE2/AVX2/AVX512: Single 128-bit unaligned load/store 1321| 2.25M| _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); 1322| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1323| | vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); 1324| |#else 1325| | ZXC_MEMCPY(dst, src, 16); 1326| |#endif 1327| 2.25M|} zxc_dispatch.c:zxc_level_clamp: 828| 9.51k|static inline int zxc_level_clamp(const int level) { 829| 9.51k| return (level > ZXC_LEVEL_ULTRA) ? ZXC_LEVEL_ULTRA : level; ------------------ | Branch (829:12): [True: 0, False: 9.51k] ------------------ 830| 9.51k|} zxc_dispatch.c:zxc_validate_block_size: 946| 4.60k|static ZXC_ALWAYS_INLINE int zxc_validate_block_size(const size_t bs) { 947| 4.60k| return bs >= ZXC_BLOCK_SIZE_MIN && bs <= ZXC_BLOCK_SIZE_MAX && (bs & (bs - 1)) == 0; ------------------ | | 62| 9.21k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 4.60k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ return bs >= ZXC_BLOCK_SIZE_MIN && bs <= ZXC_BLOCK_SIZE_MAX && (bs & (bs - 1)) == 0; ------------------ | | 64| 9.21k|#define ZXC_BLOCK_SIZE_MAX (1U << ZXC_BLOCK_SIZE_MAX_LOG2) | | ------------------ | | | | 58| 4.60k|#define ZXC_BLOCK_SIZE_MAX_LOG2 21 | | ------------------ ------------------ | Branch (947:12): [True: 4.60k, False: 0] | Branch (947:40): [True: 4.60k, False: 0] | Branch (947:68): [True: 4.60k, False: 0] ------------------ 948| 4.60k|} zxc_dispatch.c:zxc_block_size_ceil: 927| 4.90k|static ZXC_ALWAYS_INLINE size_t zxc_block_size_ceil(const size_t v) { 928| 4.90k| uint64_t x = (uint64_t)v - 1; 929| 4.90k| x |= x >> 1; 930| 4.90k| x |= x >> 2; 931| 4.90k| x |= x >> 4; 932| 4.90k| x |= x >> 8; 933| 4.90k| x |= x >> 16; 934| 4.90k| x |= x >> 32; 935| 4.90k| x++; 936| 4.90k| const size_t bs = (size_t)x; 937| 4.90k| return (bs < ZXC_BLOCK_SIZE_MIN) ? ZXC_BLOCK_SIZE_MIN : bs; ------------------ | | 62| 4.90k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 4.90k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ return (bs < ZXC_BLOCK_SIZE_MIN) ? ZXC_BLOCK_SIZE_MIN : bs; ------------------ | | 62| 2.54k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 2.54k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ | Branch (937:12): [True: 2.54k, False: 2.36k] ------------------ 938| 4.90k|} zxc_pstream.c:zxc_level_clamp: 828| 4.60k|static inline int zxc_level_clamp(const int level) { 829| 4.60k| return (level > ZXC_LEVEL_ULTRA) ? ZXC_LEVEL_ULTRA : level; ------------------ | Branch (829:12): [True: 0, False: 4.60k] ------------------ 830| 4.60k|} zxc_pstream.c:zxc_le32: 1146| 4.29k|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1147| 4.29k| uint32_t v; 1148| 4.29k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 4.29k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1149| |#ifdef ZXC_BIG_ENDIAN 1150| | return ZXC_BSWAP32(v); 1151| |#else 1152| 4.29k| return v; 1153| 4.29k|#endif 1154| 4.29k|} zxc_pstream.c:zxc_hash_combine_rotate: 1484| 3.27k| const uint32_t block_hash) { 1485| 3.27k| return ((hash << 1) | (hash >> 31)) ^ block_hash; 1486| 3.27k|} zxc_pstream.c:zxc_le64: 1209| 3.12k|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1210| 3.12k| uint64_t v; 1211| 3.12k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 3.12k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1212| |#ifdef ZXC_BIG_ENDIAN 1213| | return ZXC_BSWAP64(v); 1214| |#else 1215| 3.12k| return v; 1216| 3.12k|#endif 1217| 3.12k|} zxc_cstream_create: 266| 4.60k|zxc_cstream* zxc_cstream_create(const zxc_compress_opts_t* opts) { 267| 4.60k| zxc_cstream* cs = (zxc_cstream*)ZXC_CALLOC(1, sizeof(*cs)); ------------------ | | 72| 4.60k|#define ZXC_CALLOC(nmemb, size) calloc(nmemb, size) ------------------ 268| 4.60k| if (UNLIKELY(!cs)) return NULL; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 269| | 270| 4.60k| if (opts) cs->opts = *opts; ------------------ | Branch (270:9): [True: 4.60k, False: 0] ------------------ 271| | 272| | // The push-stream format carries no dict_id, so a dictionary here would 273| | // produce archives that decode wrong (or not at all) elsewhere. 274| 4.60k| if (UNLIKELY(cs->opts.dict || cs->opts.dict_size || cs->opts.dict_huf)) { ------------------ | | 151| 18.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 275| 0| ZXC_FREE(cs); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 276| 0| return NULL; 277| 0| } 278| | 279| 4.60k| cs->opts.level = ZXC_OPTS_LEVEL(&cs->opts, ZXC_LEVEL_DEFAULT); ------------------ | | 842| 4.60k|#define ZXC_OPTS_LEVEL(o, dflt) zxc_level_clamp(((o) && (o)->level > 0) ? (o)->level : (dflt)) | | ------------------ | | | Branch (842:50): [True: 4.60k, False: 0] | | | Branch (842:57): [True: 4.60k, False: 0] | | ------------------ ------------------ 280| 4.60k| cs->opts.block_size = ZXC_OPTS_BLOCK_SIZE(&cs->opts, ZXC_BLOCK_SIZE_DEFAULT); ------------------ | | 844| 4.60k|#define ZXC_OPTS_BLOCK_SIZE(o, dflt) (((o) && (o)->block_size > 0) ? (o)->block_size : (dflt)) | | ------------------ | | | Branch (844:40): [True: 4.60k, False: 0] | | | Branch (844:47): [True: 0, False: 4.60k] | | ------------------ ------------------ 281| | // n_threads is ignored on this single-threaded path. 282| 4.60k| cs->opts.n_threads = 0; 283| 4.60k| cs->opts.progress_cb = NULL; 284| 4.60k| cs->opts.user_data = NULL; 285| 4.60k| cs->opts.seekable = 0; 286| 4.60k| cs->block_size = cs->opts.block_size; 287| | 288| 4.60k| cs->cctx = zxc_create_cctx(&cs->opts); 289| | // LCOV_EXCL_START 290| 4.60k| if (UNLIKELY(!cs->cctx)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 291| 0| ZXC_FREE(cs); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 292| 0| return NULL; 293| 0| } 294| 4.60k| cs->in_block = (uint8_t*)ZXC_MALLOC(cs->block_size); ------------------ | | 66| 4.60k|#define ZXC_MALLOC(size) malloc(size) ------------------ 295| 4.60k| if (UNLIKELY(!cs->in_block)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 296| 0| zxc_free_cctx(cs->cctx); 297| 0| ZXC_FREE(cs); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 298| 0| return NULL; 299| 0| } 300| | // LCOV_EXCL_STOP 301| | // Pre-size pending so the file header path never needs realloc. 302| 4.60k| cs->pending_cap = 303| 4.60k| ZXC_FILE_HEADER_SIZE > ZXC_FILE_FOOTER_SIZE ? ZXC_FILE_HEADER_SIZE : ZXC_FILE_FOOTER_SIZE; ------------------ | | 104| 4.60k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ ZXC_FILE_HEADER_SIZE > ZXC_FILE_FOOTER_SIZE ? ZXC_FILE_HEADER_SIZE : ZXC_FILE_FOOTER_SIZE; ------------------ | | 106| 4.60k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ ZXC_FILE_HEADER_SIZE > ZXC_FILE_FOOTER_SIZE ? ZXC_FILE_HEADER_SIZE : ZXC_FILE_FOOTER_SIZE; ------------------ | | 104| 4.60k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ ZXC_FILE_HEADER_SIZE > ZXC_FILE_FOOTER_SIZE ? ZXC_FILE_HEADER_SIZE : ZXC_FILE_FOOTER_SIZE; ------------------ | | 106| 4.60k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ | Branch (303:9): [True: 4.60k, Folded] ------------------ 304| 4.60k| cs->pending = (uint8_t*)ZXC_MALLOC(cs->pending_cap); ------------------ | | 66| 4.60k|#define ZXC_MALLOC(size) malloc(size) ------------------ 305| | // LCOV_EXCL_START 306| 4.60k| if (UNLIKELY(!cs->pending)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 307| 0| ZXC_FREE(cs->in_block); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 308| 0| zxc_free_cctx(cs->cctx); 309| 0| ZXC_FREE(cs); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 310| 0| return NULL; 311| 0| } 312| | // LCOV_EXCL_STOP 313| 4.60k| cs->state = CS_INIT; 314| 4.60k| return cs; 315| 4.60k|} zxc_cstream_free: 390| 4.60k|void zxc_cstream_free(zxc_cstream* cs) { 391| 4.60k| if (!cs) return; ------------------ | Branch (391:9): [True: 0, False: 4.60k] ------------------ 392| 4.60k| ZXC_FREE(cs->pending); ------------------ | | 84| 4.60k|#define ZXC_FREE(ptr) free(ptr) ------------------ 393| 4.60k| ZXC_FREE(cs->in_block); ------------------ | | 84| 4.60k|#define ZXC_FREE(ptr) free(ptr) ------------------ 394| 4.60k| zxc_free_cctx(cs->cctx); 395| 4.60k| ZXC_FREE(cs); ------------------ | | 84| 4.60k|#define ZXC_FREE(ptr) free(ptr) ------------------ 396| 4.60k|} zxc_cstream_compress: 429| 109M|int64_t zxc_cstream_compress(zxc_cstream* cs, zxc_outbuf_t* out, zxc_inbuf_t* in) { 430| 109M| if (UNLIKELY(!cs || !out || !in || in->pos > in->size || out->pos > out->size || ------------------ | | 151| 1.96G|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 109M, False: 84.1k] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 109M, False: 0] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | ------------------ ------------------ 431| 109M| (in->size > in->pos && !in->src) || (out->size > out->pos && !out->dst) || 432| 109M| cs->state == CS_DONE)) { 433| 0| return ZXC_ERROR_NULL_INPUT; 434| 0| } 435| 109M| if (UNLIKELY(cs->state == CS_ERRORED)) return cs->error_code; ------------------ | | 151| 109M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 109M] | | ------------------ ------------------ 436| | 437| 109M| for (;;) { 438| 109M| switch (cs->state) { ------------------ | Branch (438:17): [True: 109M, False: 0] ------------------ 439| 4.60k| case CS_INIT: { ------------------ | Branch (439:13): [True: 4.60k, False: 109M] ------------------ 440| 4.60k| const int rc = cs_stage_file_header(cs); 441| 4.60k| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 442| 4.60k| cs->state = CS_DRAIN_HEADER; 443| 4.60k| break; 444| 4.60k| } 445| | 446| 4.60k| case CS_DRAIN_HEADER: ------------------ | Branch (446:13): [True: 4.60k, False: 109M] ------------------ 447| 323k| case CS_DRAIN_BLOCK: { ------------------ | Branch (447:13): [True: 319k, False: 109M] ------------------ 448| 323k| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (448:21): [True: 318k, False: 4.91k] ------------------ 449| 4.91k| cs->state = CS_ACCUMULATE; 450| 4.91k| break; 451| 323k| } 452| | 453| 109M| case CS_ACCUMULATE: { ------------------ | Branch (453:13): [True: 109M, False: 328k] ------------------ 454| 109M| const size_t avail_in = in->size - in->pos; 455| 109M| if (cs->in_used == 0 && avail_in >= cs->block_size) { ------------------ | Branch (455:21): [True: 4.99k, False: 109M] | Branch (455:41): [True: 0, False: 4.99k] ------------------ 456| 0| const int rc = cs_compress_block_from(cs, (const uint8_t*)in->src + in->pos, 457| 0| cs->block_size); 458| 0| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 459| 0| in->pos += cs->block_size; 460| 0| cs->state = CS_DRAIN_BLOCK; 461| 0| break; 462| 0| } 463| | 464| 109M| const size_t room = cs->block_size - cs->in_used; 465| 109M| const size_t n = avail_in < room ? avail_in : room; ------------------ | Branch (465:34): [True: 109M, False: 313] ------------------ 466| 109M| if (n) { ------------------ | Branch (466:21): [True: 109M, False: 90] ------------------ 467| 109M| ZXC_MEMCPY(cs->in_block + cs->in_used, (const uint8_t*)in->src + in->pos, n); ------------------ | | 167| 109M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 468| 109M| in->pos += n; 469| 109M| cs->in_used += n; 470| 109M| } 471| | 472| 109M| if (cs->in_used == cs->block_size) { ------------------ | Branch (472:21): [True: 313, False: 109M] ------------------ 473| 313| const int rc = cs_compress_one_block(cs); 474| 313| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 313|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 313] | | ------------------ ------------------ 475| 313| cs->state = CS_DRAIN_BLOCK; 476| 313| break; 477| 313| } 478| | // Block not yet full either in is empty or we made no progress. 479| 109M| return 0; 480| 109M| } 481| | 482| 0| case CS_DRAIN_LAST: ------------------ | Branch (482:13): [True: 0, False: 109M] ------------------ 483| 0| case CS_DRAIN_EOF: ------------------ | Branch (483:13): [True: 0, False: 109M] ------------------ 484| 0| case CS_DRAIN_FOOTER: ------------------ | Branch (484:13): [True: 0, False: 109M] ------------------ 485| 0| case CS_DONE: ------------------ | Branch (485:13): [True: 0, False: 109M] ------------------ 486| 0| case CS_ERRORED: ------------------ | Branch (486:13): [True: 0, False: 109M] ------------------ 487| | // These states are owned by _end(). 488| 0| return ZXC_ERROR_NULL_INPUT; 489| 109M| } 490| 109M| } 491| 109M|} zxc_cstream_end: 502| 512k|int64_t zxc_cstream_end(zxc_cstream* cs, zxc_outbuf_t* out) { 503| 512k| if (UNLIKELY(!cs || !out || cs->state == CS_DONE)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 2.05M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 512k] | | | Branch (151:42): [True: 0, False: 512k] | | | Branch (151:42): [True: 0, False: 512k] | | | Branch (151:42): [True: 0, False: 512k] | | ------------------ ------------------ 504| 512k| if (UNLIKELY(cs->state == CS_ERRORED)) return cs->error_code; ------------------ | | 151| 512k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 512k] | | ------------------ ------------------ 505| | 506| 526k| for (;;) { 507| 526k| switch (cs->state) { ------------------ | Branch (507:17): [True: 526k, False: 0] ------------------ 508| 0| case CS_INIT: { ------------------ | Branch (508:13): [True: 0, False: 526k] ------------------ 509| | // _end before any input, still need to emit file header. 510| 0| const int rc = cs_stage_file_header(cs); 511| 0| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 512| 0| cs->state = CS_DRAIN_HEADER; 513| 0| break; 514| 0| } 515| | 516| 0| case CS_DRAIN_HEADER: ------------------ | Branch (516:13): [True: 0, False: 526k] ------------------ 517| 0| case CS_DRAIN_BLOCK: { ------------------ | Branch (517:13): [True: 0, False: 526k] ------------------ 518| 0| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (518:21): [True: 0, False: 0] ------------------ 519| 0| cs->state = CS_ACCUMULATE; 520| 0| break; 521| 0| } 522| | 523| 4.60k| case CS_ACCUMULATE: { ------------------ | Branch (523:13): [True: 4.60k, False: 521k] ------------------ 524| | // Compress the residual partial block (if any), then EOF + footer. 525| 4.60k| if (cs->in_used > 0) { ------------------ | Branch (525:21): [True: 4.59k, False: 10] ------------------ 526| 4.59k| const int rc = cs_compress_one_block(cs); 527| 4.59k| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 4.59k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.59k] | | ------------------ ------------------ 528| 4.59k| cs->state = CS_DRAIN_LAST; 529| 4.59k| break; 530| 4.59k| } 531| | // No residual data: go straight to EOF. 532| 10| { 533| 10| const int rc = cs_stage_eof(cs); 534| 10| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 10|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 10] | | ------------------ ------------------ 535| 10| cs->state = CS_DRAIN_EOF; 536| 10| break; 537| 10| } 538| 10| } 539| | 540| 512k| case CS_DRAIN_LAST: { ------------------ | Branch (540:13): [True: 512k, False: 14.1k] ------------------ 541| 512k| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (541:21): [True: 507k, False: 4.59k] ------------------ 542| | // After last data block -> EOF. 543| 4.59k| const int rc = cs_stage_eof(cs); 544| 4.59k| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 4.59k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.59k] | | ------------------ ------------------ 545| 4.59k| cs->state = CS_DRAIN_EOF; 546| 4.59k| break; 547| 4.59k| } 548| | 549| 4.71k| case CS_DRAIN_EOF: { ------------------ | Branch (549:13): [True: 4.71k, False: 521k] ------------------ 550| 4.71k| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (550:21): [True: 106, False: 4.60k] ------------------ 551| 4.60k| const int rc = cs_stage_footer(cs); 552| 4.60k| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 553| 4.60k| cs->state = CS_DRAIN_FOOTER; 554| 4.60k| break; 555| 4.60k| } 556| | 557| 4.80k| case CS_DRAIN_FOOTER: { ------------------ | Branch (557:13): [True: 4.80k, False: 521k] ------------------ 558| 4.80k| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (558:21): [True: 195, False: 4.60k] ------------------ 559| 4.60k| cs->state = CS_DONE; 560| 4.60k| return 0; 561| 4.80k| } 562| | 563| 0| case CS_DONE: ------------------ | Branch (563:13): [True: 0, False: 526k] ------------------ 564| 0| case CS_ERRORED: ------------------ | Branch (564:13): [True: 0, False: 526k] ------------------ 565| 0| return cs->state == CS_ERRORED ? cs->error_code : 0; // LCOV_EXCL_LINE ------------------ | Branch (565:24): [True: 0, False: 0] ------------------ 566| 526k| } 567| 526k| } 568| 512k|} zxc_dstream_create: 767| 9.21k|zxc_dstream* zxc_dstream_create(const zxc_decompress_opts_t* opts) { 768| 9.21k| zxc_dstream* ds = (zxc_dstream*)ZXC_CALLOC(1, sizeof(*ds)); ------------------ | | 72| 9.21k|#define ZXC_CALLOC(nmemb, size) calloc(nmemb, size) ------------------ 769| 9.21k| if (UNLIKELY(!ds)) return NULL; // LCOV_EXCL_LINE ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 770| 9.21k| if (opts) ds->opts = *opts; ------------------ | Branch (770:9): [True: 9.21k, False: 0] ------------------ 771| 9.21k| if (UNLIKELY(ds->opts.dict || ds->opts.dict_size || ds->opts.dict_huf)) { ------------------ | | 151| 36.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | ------------------ ------------------ 772| 0| ZXC_FREE(ds); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 773| 0| return NULL; 774| 0| } 775| 9.21k| ds->opts.n_threads = 0; 776| 9.21k| ds->opts.progress_cb = NULL; 777| 9.21k| ds->opts.user_data = NULL; 778| 9.21k| ds->state = DS_NEED_FILE_HEADER; 779| 9.21k| ds->scratch_need = ZXC_FILE_HEADER_SIZE; ------------------ | | 104| 9.21k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ 780| 9.21k| return ds; 781| 9.21k|} zxc_dstream_free: 788| 9.21k|void zxc_dstream_free(zxc_dstream* ds) { 789| 9.21k| if (UNLIKELY(!ds)) return; // LCOV_EXCL_LINE ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 790| 9.21k| ZXC_FREE(ds->payload); ------------------ | | 84| 9.21k|#define ZXC_FREE(ptr) free(ptr) ------------------ 791| 9.21k| ZXC_FREE(ds->decoded); ------------------ | | 84| 9.21k|#define ZXC_FREE(ptr) free(ptr) ------------------ 792| 9.21k| if (ds->inner_initialized) zxc_cctx_free(&ds->inner); ------------------ | Branch (792:9): [True: 4.60k, False: 4.60k] ------------------ 793| 9.21k| ZXC_FREE(ds); ------------------ | | 84| 9.21k|#define ZXC_FREE(ptr) free(ptr) ------------------ 794| 9.21k|} zxc_dstream_finished: 799| 3.12k|int zxc_dstream_finished(const zxc_dstream* ds) { return (ds && ds->state == DS_DONE) ? 1 : 0; } ------------------ | Branch (799:59): [True: 3.12k, False: 0] | Branch (799:65): [True: 3.12k, False: 4] ------------------ zxc_dstream_decompress: 958| 42.7M|int64_t zxc_dstream_decompress(zxc_dstream* ds, zxc_outbuf_t* out, zxc_inbuf_t* in) { 959| 42.7M| if (UNLIKELY(!ds || !out || !in || in->pos > in->size || out->pos > out->size || ------------------ | | 151| 683M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 42.7M, False: 0] | | | Branch (151:42): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 42.7M, False: 0] | | | Branch (151:42): [True: 0, False: 42.7M] | | ------------------ ------------------ 960| 42.7M| (in->size > in->pos && !in->src) || (out->size > out->pos && !out->dst))) { 961| 0| return ZXC_ERROR_NULL_INPUT; 962| 0| } 963| 42.7M| if (UNLIKELY(ds->state == DS_ERRORED)) return ds->error_code; ------------------ | | 151| 42.7M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 42.7M] | | ------------------ ------------------ 964| 42.7M| if (UNLIKELY(ds->state == DS_DONE)) return 0; ------------------ | | 151| 42.7M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 42.7M] | | ------------------ ------------------ 965| | 966| 42.7M| size_t produced = 0; 967| | 968| 42.7M| for (;;) { 969| 42.7M| switch (ds->state) { ------------------ | Branch (969:17): [True: 42.7M, False: 0] ------------------ 970| 47.9k| case DS_NEED_FILE_HEADER: { ------------------ | Branch (970:13): [True: 47.9k, False: 42.7M] ------------------ 971| 47.9k| const int rc = ds_handle_need_file_header(ds, in); 972| 47.9k| if (rc == 1) return (int64_t)produced; ------------------ | Branch (972:21): [True: 38.8k, False: 9.10k] ------------------ 973| 9.10k| if (rc < 0) return rc; ------------------ | Branch (973:21): [True: 4.50k, False: 4.60k] ------------------ 974| 4.60k| break; 975| 9.10k| } 976| | 977| 18.7k| case DS_NEED_BLOCK_HEADER: { ------------------ | Branch (977:13): [True: 18.7k, False: 42.7M] ------------------ 978| 18.7k| const int rc = ds_handle_need_block_header(ds, in); 979| 18.7k| if (rc == 1) return (int64_t)produced; ------------------ | Branch (979:21): [True: 10.6k, False: 8.15k] ------------------ 980| 8.15k| if (rc < 0) return rc; ------------------ | Branch (980:21): [True: 0, False: 8.15k] ------------------ 981| 8.15k| break; 982| 8.15k| } 983| | 984| 40.9M| case DS_NEED_BLOCK_PAYLOAD: { ------------------ | Branch (984:13): [True: 40.9M, False: 1.78M] ------------------ 985| 40.9M| if (!ds_pull(ds->payload, &ds->payload_used, ds->payload_need, in)) ------------------ | Branch (985:21): [True: 40.9M, False: 4.90k] ------------------ 986| 40.9M| return (int64_t)produced; 987| 4.90k| ds->state = DS_DECODE_BLOCK; 988| 4.90k| break; 989| 40.9M| } 990| | 991| 4.90k| case DS_DECODE_BLOCK: { ------------------ | Branch (991:13): [True: 4.90k, False: 42.7M] ------------------ 992| 4.90k| const int direct = (out->size - out->pos) >= ds->decoded_cap; 993| 4.90k| uint8_t* const ddst = direct ? (uint8_t*)out->dst + out->pos : ds->decoded; ------------------ | Branch (993:39): [True: 0, False: 4.90k] ------------------ 994| 4.90k| const int dsz = zxc_decompress_chunk_wrapper( 995| 4.90k| &ds->inner, ds->payload, ds->payload_used, ddst, ds->decoded_cap); 996| 4.90k| if (UNLIKELY(dsz < 0)) return ds_set_error(ds, dsz); ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 997| | 998| | // If file-level checksum verification is enabled, fold this 999| | // block's trailer into the rolling global hash (last 1000| | // ZXC_BLOCK_CHECKSUM_SIZE bytes of the *raw* block). 1001| 4.90k| if (ds->opts.checksum_enabled && ds->file_has_checksum && ------------------ | Branch (1001:21): [True: 1.63k, False: 3.27k] | Branch (1001:50): [True: 1.63k, False: 0] ------------------ 1002| 1.63k| ds->payload_used >= ZXC_BLOCK_CHECKSUM_SIZE) { ------------------ | | 404| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (1002:21): [True: 1.63k, False: 0] ------------------ 1003| 1.63k| const uint32_t bh = 1004| 1.63k| zxc_le32(ds->payload + ds->payload_used - ZXC_BLOCK_CHECKSUM_SIZE); ------------------ | | 404| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ 1005| 1.63k| ds->global_hash = zxc_hash_combine_rotate(ds->global_hash, bh); 1006| 1.63k| } 1007| | 1008| 4.90k| if (direct) { ------------------ | Branch (1008:21): [True: 0, False: 4.90k] ------------------ 1009| 0| out->pos += (size_t)dsz; 1010| 0| produced += (size_t)dsz; 1011| 0| ds->total_out += (size_t)dsz; 1012| 0| ds->decoded_size = 0; 1013| 0| ds->decoded_pos = 0; 1014| 0| ds->state = DS_NEED_BLOCK_HEADER; 1015| 0| ds->scratch_used = 0; 1016| 0| ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 402| 0|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1017| 0| break; 1018| 0| } 1019| | 1020| 4.90k| ds->decoded_size = (size_t)dsz; 1021| 4.90k| ds->decoded_pos = 0; 1022| 4.90k| ds->state = DS_EMIT_DECODED; 1023| 4.90k| break; 1024| 4.90k| } 1025| | 1026| 1.70M| case DS_EMIT_DECODED: { ------------------ | Branch (1026:13): [True: 1.70M, False: 41.0M] ------------------ 1027| 1.70M| const int done = ds_drain_decoded(ds, out, &produced); 1028| 1.70M| if (!done) return (int64_t)produced; ------------------ | Branch (1028:21): [True: 1.70M, False: 4.90k] ------------------ 1029| 4.90k| ds->state = DS_NEED_BLOCK_HEADER; 1030| 4.90k| ds->scratch_used = 0; 1031| 4.90k| ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 402| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1032| 4.90k| break; 1033| 1.70M| } 1034| | 1035| 3.25k| case DS_PEEK_TAIL: { ------------------ | Branch (1035:13): [True: 3.25k, False: 42.7M] ------------------ 1036| 3.25k| if (!ds_pull(ds->scratch, &ds->scratch_used, ds->scratch_need, in)) ------------------ | Branch (1036:21): [True: 80, False: 3.17k] ------------------ 1037| 80| return (int64_t)produced; 1038| | // Try to interpret as a block header (SEK). 1039| 3.17k| zxc_block_header_t peek; 1040| 3.17k| const int sek_rc = zxc_read_block_header(ds->scratch, ds->scratch_used, &peek); 1041| 3.17k| if (sek_rc == ZXC_OK && peek.block_type == (uint8_t)ZXC_BLOCK_SEK) { ------------------ | Branch (1041:21): [True: 28, False: 3.14k] | Branch (1041:41): [True: 4, False: 24] ------------------ 1042| | // SEK block: skip its payload (peek.comp_size bytes). 1043| 4| ds->sek_remaining = (size_t)peek.comp_size; 1044| 4| ds->state = DS_DRAIN_SEK_PAYLOAD; 1045| 4| break; 1046| 4| } 1047| | // Not SEK -> these 8 bytes are the first 8 of the 12-byte footer. 1048| 3.16k| ds->state = DS_NEED_FOOTER_REST; 1049| 3.16k| ds->scratch_need = ZXC_FILE_FOOTER_SIZE; /* keep first 8, want 4 more */ ------------------ | | 106| 3.16k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 1050| 3.16k| break; 1051| 3.17k| } 1052| | 1053| 4| case DS_DRAIN_SEK_PAYLOAD: { ------------------ | Branch (1053:13): [True: 4, False: 42.7M] ------------------ 1054| 4| const size_t avail = in->size - in->pos; 1055| 4| const size_t n = avail < ds->sek_remaining ? avail : ds->sek_remaining; ------------------ | Branch (1055:34): [True: 0, False: 4] ------------------ 1056| 4| in->pos += n; 1057| 4| ds->sek_remaining -= n; 1058| 4| if (ds->sek_remaining > 0) return (int64_t)produced; ------------------ | Branch (1058:21): [True: 0, False: 4] ------------------ 1059| 4| ds->state = DS_NEED_FOOTER_FULL; 1060| 4| ds->scratch_used = 0; 1061| 4| ds->scratch_need = ZXC_FILE_FOOTER_SIZE; ------------------ | | 106| 4|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 1062| 4| break; 1063| 4| } 1064| | 1065| 3.16k| case DS_NEED_FOOTER_REST: ------------------ | Branch (1065:13): [True: 3.16k, False: 42.7M] ------------------ 1066| 3.17k| case DS_NEED_FOOTER_FULL: { ------------------ | Branch (1066:13): [True: 4, False: 42.7M] ------------------ 1067| 3.17k| if (!ds_pull(ds->scratch, &ds->scratch_used, ds->scratch_need, in)) ------------------ | Branch (1067:21): [True: 45, False: 3.12k] ------------------ 1068| 45| return (int64_t)produced; 1069| 3.12k| ds->state = DS_VALIDATE_FOOTER; 1070| 3.12k| break; 1071| 3.17k| } 1072| | 1073| 3.12k| case DS_VALIDATE_FOOTER: { ------------------ | Branch (1073:13): [True: 3.12k, False: 42.7M] ------------------ 1074| 3.12k| const uint64_t declared = zxc_le64(ds->scratch); 1075| 3.12k| if (UNLIKELY(declared != ds->total_out)) ------------------ | | 151| 3.12k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 3.12k] | | ------------------ ------------------ 1076| 0| return ds_set_error(ds, ZXC_ERROR_CORRUPT_DATA); 1077| 3.12k| if (ds->opts.checksum_enabled && ds->file_has_checksum) { ------------------ | Branch (1077:21): [True: 1.02k, False: 2.10k] | Branch (1077:50): [True: 1.02k, False: 0] ------------------ 1078| 1.02k| const uint32_t fh = zxc_le32(ds->scratch + sizeof(uint64_t)); 1079| 1.02k| if (UNLIKELY(fh != ds->global_hash)) ------------------ | | 151| 1.02k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.02k] | | ------------------ ------------------ 1080| 0| return ds_set_error(ds, ZXC_ERROR_BAD_CHECKSUM); 1081| 1.02k| } 1082| 3.12k| ds->state = DS_DONE; 1083| 3.12k| return (int64_t)produced; 1084| 3.12k| } 1085| | 1086| 0| case DS_DONE: ------------------ | Branch (1086:13): [True: 0, False: 42.7M] ------------------ 1087| 0| case DS_ERRORED: ------------------ | Branch (1087:13): [True: 0, False: 42.7M] ------------------ 1088| 0| return ds->state == DS_ERRORED ? ds->error_code : (int64_t)produced; ------------------ | Branch (1088:24): [True: 0, False: 0] ------------------ 1089| 42.7M| } 1090| 42.7M| } 1091| 42.7M|} zxc_pstream.c:cs_stage_file_header: 323| 4.60k|static int cs_stage_file_header(zxc_cstream* cs) { 324| 4.60k| const int w = zxc_write_file_header(cs->pending, cs->pending_cap, cs->block_size, 325| 4.60k| cs->opts.checksum_enabled, 0); 326| 4.60k| if (UNLIKELY(w < 0)) return w; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 327| 4.60k| cs->pending_len = (size_t)w; 328| 4.60k| cs->pending_pos = 0; 329| 4.60k| return ZXC_OK; 330| 4.60k|} zxc_pstream.c:cs_drain_pending: 246| 845k|static int cs_drain_pending(zxc_cstream* cs, zxc_outbuf_t* out) { 247| 845k| zxc_outbuf_push(out, cs->pending, &cs->pending_pos, cs->pending_len); 248| 845k| return cs->pending_pos == cs->pending_len; 249| 845k|} zxc_pstream.c:zxc_outbuf_push: 233| 2.55M| const size_t len) { 234| 2.55M| const size_t avail_out = out->size - out->pos; 235| 2.55M| const size_t avail_src = len - *pos; 236| 2.55M| const size_t n = avail_out < avail_src ? avail_out : avail_src; ------------------ | Branch (236:22): [True: 2.52M, False: 23.6k] ------------------ 237| 2.55M| if (n) { ------------------ | Branch (237:9): [True: 2.55M, False: 71] ------------------ 238| 2.55M| ZXC_MEMCPY((uint8_t*)out->dst + out->pos, src + *pos, n); ------------------ | | 167| 2.55M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 239| 2.55M| out->pos += n; 240| 2.55M| *pos += n; 241| 2.55M| } 242| 2.55M| return n; 243| 2.55M|} zxc_pstream.c:cs_compress_block_from: 179| 4.90k|static int cs_compress_block_from(zxc_cstream* cs, const uint8_t* RESTRICT src, const size_t len) { 180| 4.90k| const uint64_t bound = zxc_compress_block_bound(len); 181| | // LCOV_EXCL_START 182| 4.90k| if (UNLIKELY(bound == 0 || bound > SIZE_MAX)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 9.81k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | ------------------ ------------------ 183| 4.90k| if (UNLIKELY(bound > cs->pending_cap)) { ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 4.60k, False: 303] | | ------------------ ------------------ 184| 4.60k| uint8_t* nb = (uint8_t*)ZXC_REALLOC(cs->pending, (size_t)bound); ------------------ | | 78| 4.60k|#define ZXC_REALLOC(ptr, size) realloc(ptr, size) ------------------ 185| 4.60k| if (UNLIKELY(!nb)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 186| 4.60k| cs->pending = nb; 187| 4.60k| cs->pending_cap = (size_t)bound; 188| 4.60k| } 189| | // LCOV_EXCL_STOP 190| 4.90k| const int64_t csize = 191| 4.90k| zxc_compress_block(cs->cctx, src, len, cs->pending, cs->pending_cap, &cs->opts); 192| 4.90k| if (UNLIKELY(csize < 0)) return (int)csize; // LCOV_EXCL_LINE ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 193| | 194| 4.90k| cs->pending_len = (size_t)csize; 195| 4.90k| cs->pending_pos = 0; 196| 4.90k| cs->total_in += len; 197| | 198| | // If checksums are on, the block trailer is the last ZXC_BLOCK_CHECKSUM_SIZE 199| | // bytes of pending; fold it into the rolling global hash. 200| 4.90k| if (cs->opts.checksum_enabled && cs->pending_len >= ZXC_BLOCK_CHECKSUM_SIZE) { ------------------ | | 404| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (200:9): [True: 1.63k, False: 3.27k] | Branch (200:38): [True: 1.63k, False: 0] ------------------ 201| 1.63k| const uint32_t bh = zxc_le32(cs->pending + cs->pending_len - ZXC_BLOCK_CHECKSUM_SIZE); ------------------ | | 404| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ 202| 1.63k| cs->global_hash = zxc_hash_combine_rotate(cs->global_hash, bh); 203| 1.63k| } 204| 4.90k| return ZXC_OK; 205| 4.90k|} zxc_pstream.c:cs_compress_one_block: 220| 4.90k|static int cs_compress_one_block(zxc_cstream* cs) { 221| 4.90k| const int rc = cs_compress_block_from(cs, cs->in_block, cs->in_used); 222| 4.90k| if (LIKELY(rc == ZXC_OK)) cs->in_used = 0; ------------------ | | 145| 4.90k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 4.90k, False: 0] | | ------------------ ------------------ 223| 4.90k| return rc; 224| 4.90k|} zxc_pstream.c:cs_stage_eof: 341| 4.60k|static int cs_stage_eof(zxc_cstream* cs) { 342| | // LCOV_EXCL_START 343| 4.60k| if (UNLIKELY(ZXC_BLOCK_HEADER_SIZE > cs->pending_cap)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 344| 0| uint8_t* nb = (uint8_t*)ZXC_REALLOC(cs->pending, ZXC_BLOCK_HEADER_SIZE); ------------------ | | 78| 0|#define ZXC_REALLOC(ptr, size) realloc(ptr, size) ------------------ 345| 0| if (UNLIKELY(!nb)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 346| 0| cs->pending = nb; 347| 0| cs->pending_cap = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 402| 0|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 348| 0| } 349| | // LCOV_EXCL_STOP 350| 4.60k| const zxc_block_header_t eof = { 351| 4.60k| .block_type = (uint8_t)ZXC_BLOCK_EOF, 352| 4.60k| .block_flags = 0, 353| 4.60k| .reserved = 0, 354| 4.60k| .header_crc = 0, 355| 4.60k| .comp_size = 0, 356| 4.60k| }; 357| 4.60k| const int w = zxc_write_block_header(cs->pending, cs->pending_cap, &eof); 358| 4.60k| if (UNLIKELY(w < 0)) return w; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 359| 4.60k| cs->pending_len = (size_t)w; 360| 4.60k| cs->pending_pos = 0; 361| 4.60k| return ZXC_OK; 362| 4.60k|} zxc_pstream.c:cs_stage_footer: 373| 4.60k|static int cs_stage_footer(zxc_cstream* cs) { 374| | // LCOV_EXCL_START 375| 4.60k| if (UNLIKELY(ZXC_FILE_FOOTER_SIZE > cs->pending_cap)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 376| 0| uint8_t* nb = (uint8_t*)ZXC_REALLOC(cs->pending, ZXC_FILE_FOOTER_SIZE); ------------------ | | 78| 0|#define ZXC_REALLOC(ptr, size) realloc(ptr, size) ------------------ 377| 0| if (UNLIKELY(!nb)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 378| 0| cs->pending = nb; 379| 0| cs->pending_cap = ZXC_FILE_FOOTER_SIZE; ------------------ | | 106| 0|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 380| 0| } 381| | // LCOV_EXCL_STOP 382| 4.60k| const int w = zxc_write_file_footer(cs->pending, cs->pending_cap, cs->total_in, cs->global_hash, 383| 4.60k| cs->opts.checksum_enabled); 384| 4.60k| if (UNLIKELY(w < 0)) return w; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 385| 4.60k| cs->pending_len = (size_t)w; 386| 4.60k| cs->pending_pos = 0; 387| 4.60k| return ZXC_OK; 388| 4.60k|} zxc_pstream.c:ds_handle_need_file_header: 850| 47.9k|static int ds_handle_need_file_header(zxc_dstream* ds, zxc_inbuf_t* in) { 851| 47.9k| if (!ds_pull(ds->scratch, &ds->scratch_used, ds->scratch_need, in)) return 1; ------------------ | Branch (851:9): [True: 38.8k, False: 9.10k] ------------------ 852| | 853| 9.10k| size_t bs = 0; 854| 9.10k| int has_csum = 0; 855| 9.10k| const int rc = zxc_read_file_header(ds->scratch, ds->scratch_used, &bs, &has_csum, NULL); 856| 9.10k| if (UNLIKELY(rc != ZXC_OK)) return ds_set_error(ds, rc); // LCOV_EXCL_LINE ------------------ | | 151| 9.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 4.50k, False: 4.60k] | | ------------------ ------------------ 857| 4.60k| ds->block_size = bs; 858| 4.60k| ds->file_has_checksum = has_csum; 859| | 860| | // Allocate payload + decoded buffers now that block_size is known. 861| 4.60k| const uint64_t pb = zxc_compress_block_bound(ds->block_size); 862| | // LCOV_EXCL_START 863| 4.60k| if (UNLIKELY(pb == 0 || pb > SIZE_MAX)) return ds_set_error(ds, ZXC_ERROR_OVERFLOW); ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 864| | // LCOV_EXCL_STOP 865| 4.60k| ds->payload_cap = (size_t)pb; 866| 4.60k| ds->payload = (uint8_t*)ZXC_MALLOC(ds->payload_cap); ------------------ | | 66| 4.60k|#define ZXC_MALLOC(size) malloc(size) ------------------ 867| | 868| 4.60k| ds->decoded_cap = ds->block_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 354| 4.60k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 328| 4.60k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 869| 4.60k| ds->decoded = (uint8_t*)ZXC_MALLOC(ds->decoded_cap); ------------------ | | 66| 4.60k|#define ZXC_MALLOC(size) malloc(size) ------------------ 870| | // LCOV_EXCL_START 871| 4.60k| if (UNLIKELY(!ds->payload || !ds->decoded)) return ds_set_error(ds, ZXC_ERROR_MEMORY); ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 872| | 873| 4.60k| if (UNLIKELY(zxc_cctx_init(&ds->inner, ds->block_size, 0, 0, ------------------ | | 151| 6.14k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 1.53k, False: 3.06k] | | | Branch (151:42): [True: 1.53k, False: 0] | | ------------------ ------------------ 874| 4.60k| ds->file_has_checksum && ds->opts.checksum_enabled, 0) != ZXC_OK)) { 875| 0| return ds_set_error(ds, ZXC_ERROR_MEMORY); 876| 0| } 877| | // LCOV_EXCL_STOP 878| 4.60k| ds->inner_initialized = 1; 879| | 880| 4.60k| ds->state = DS_NEED_BLOCK_HEADER; 881| 4.60k| ds->scratch_used = 0; 882| 4.60k| ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 402| 4.60k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 883| 4.60k| return 0; 884| 4.60k|} zxc_pstream.c:ds_handle_need_block_header: 902| 18.7k|static int ds_handle_need_block_header(zxc_dstream* ds, zxc_inbuf_t* in) { 903| 18.7k| if (!ds_pull(ds->scratch, &ds->scratch_used, ds->scratch_need, in)) return 1; ------------------ | Branch (903:9): [True: 10.6k, False: 8.15k] ------------------ 904| | 905| 8.15k| const int rc = zxc_read_block_header(ds->scratch, ds->scratch_used, &ds->cur_bh); 906| 8.15k| if (UNLIKELY(rc != ZXC_OK)) return ds_set_error(ds, rc); // LCOV_EXCL_LINE ------------------ | | 151| 8.15k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.15k] | | ------------------ ------------------ 907| | 908| 8.15k| if (ds->cur_bh.block_type == (uint8_t)ZXC_BLOCK_EOF) { ------------------ | Branch (908:9): [True: 3.25k, False: 4.90k] ------------------ 909| | // EOF block: comp_size must be 0; no payload, no checksum. 910| 3.25k| if (UNLIKELY(ds->cur_bh.comp_size != 0)) return ds_set_error(ds, ZXC_ERROR_BAD_BLOCK_SIZE); ------------------ | | 151| 3.25k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 3.25k] | | ------------------ ------------------ 911| 3.25k| ds->state = DS_PEEK_TAIL; 912| 3.25k| ds->scratch_used = 0; 913| 3.25k| ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; /* sniff */ ------------------ | | 402| 3.25k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 914| 3.25k| return 0; 915| 3.25k| } 916| | 917| | // Normal data block: read comp_size [+ ZXC_BLOCK_CHECKSUM_SIZE if file-level checksums]. 918| 4.90k| const uint64_t need = (uint64_t)ds->cur_bh.comp_size + 919| 4.90k| (ds->file_has_checksum ? (uint64_t)ZXC_BLOCK_CHECKSUM_SIZE : 0U); ------------------ | | 404| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (919:28): [True: 1.63k, False: 3.27k] ------------------ 920| 4.90k| if (UNLIKELY(need > ds->payload_cap)) return ds_set_error(ds, ZXC_ERROR_BAD_BLOCK_SIZE); ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 921| | 922| | // Feed the full block (header + payload + opt csum) to zxc_decompress_block, 923| | // so prefix with the 8-byte header we just parsed. 924| 4.90k| ZXC_MEMCPY(ds->payload, ds->scratch, ZXC_BLOCK_HEADER_SIZE); ------------------ | | 167| 4.90k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 925| 4.90k| ds->payload_used = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 402| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 926| 4.90k| ds->payload_need = (size_t)need + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 402| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 927| | // LCOV_EXCL_START 928| 4.90k| if (UNLIKELY(ds->payload_need > ds->payload_cap)) { ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 929| | // grow 930| 0| uint8_t* nb = (uint8_t*)ZXC_REALLOC(ds->payload, ds->payload_need); ------------------ | | 78| 0|#define ZXC_REALLOC(ptr, size) realloc(ptr, size) ------------------ 931| 0| if (UNLIKELY(!nb)) return ds_set_error(ds, ZXC_ERROR_MEMORY); ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 932| 0| ds->payload = nb; 933| 0| ds->payload_cap = ds->payload_need; 934| 0| } 935| | // LCOV_EXCL_STOP 936| 4.90k| ds->state = DS_NEED_BLOCK_PAYLOAD; 937| 4.90k| return 0; 938| 4.90k|} zxc_pstream.c:ds_pull: 746| 41.0M| zxc_inbuf_t* in) { 747| 41.0M| const size_t want = need - *used; 748| 41.0M| const size_t avail = in->size - in->pos; 749| 41.0M| const size_t n = want < avail ? want : avail; ------------------ | Branch (749:22): [True: 19.9k, False: 41.0M] ------------------ 750| 41.0M| if (n) { ------------------ | Branch (750:9): [True: 41.0M, False: 2.89k] ------------------ 751| 41.0M| ZXC_MEMCPY(dst + *used, (const uint8_t*)in->src + in->pos, n); ------------------ | | 167| 41.0M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 752| 41.0M| in->pos += n; 753| 41.0M| *used += n; 754| 41.0M| } 755| 41.0M| return *used == need; 756| 41.0M|} zxc_pstream.c:ds_set_error: 732| 4.50k|static int ds_set_error(zxc_dstream* ds, const int code) { 733| 4.50k| ds->error_code = code; 734| 4.50k| ds->state = DS_ERRORED; 735| 4.50k| return code; 736| 4.50k|} zxc_pstream.c:ds_drain_decoded: 827| 1.70M|static int ds_drain_decoded(zxc_dstream* ds, zxc_outbuf_t* out, size_t* produced) { 828| 1.70M| const size_t n = zxc_outbuf_push(out, ds->decoded, &ds->decoded_pos, ds->decoded_size); 829| 1.70M| ds->total_out += n; 830| 1.70M| if (produced) *produced += n; ------------------ | Branch (830:9): [True: 1.70M, False: 0] ------------------ 831| 1.70M| return ds->decoded_pos == ds->decoded_size; 832| 1.70M|} LLVMFuzzerTestOneInput: 104| 4.62k|int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { 105| 4.62k| if (size < 3) return 0; ------------------ | Branch (105:9): [True: 2, False: 4.62k] ------------------ 106| | 107| | /* Save raw input for the parser-fuzzing phase. */ 108| 4.62k| const uint8_t* const raw_data = data; 109| 4.62k| const size_t raw_size = size; 110| | 111| | /* Header bytes drive pstream-specific axes. data[0] is reserved (the 112| | * compression level is intentionally fixed to 1, see copts below); it 113| | * is still consumed so corpus byte offsets stay stable. */ 114| 4.62k| const int checksum_enabled = data[1] & 1; 115| 4.62k| const size_t in_chunk = derive_chunk_size(data[2]); 116| 4.62k| data += 3; 117| 4.62k| size -= 3; 118| | 119| 4.62k| if (size == 0 || size > FUZZ_PSTREAM_MAX_INPUT) return 0; ------------------ | | 41| 4.60k|#define FUZZ_PSTREAM_MAX_INPUT (2 << 20) /* 2 MiB */ ------------------ | Branch (119:9): [True: 17, False: 4.60k] | Branch (119:22): [True: 0, False: 4.60k] ------------------ 120| | 121| | /* Persistent buffers - reused across iterations to reduce allocator 122| | * pressure (same pattern as fuzz_seekable.c). */ 123| 4.60k| static uint8_t* comp_buf = NULL; 124| 4.60k| static size_t comp_cap = 0; 125| 4.60k| static uint8_t* round_buf = NULL; 126| 4.60k| static size_t round_cap = 0; 127| | 128| | /* ------------------------------------------------------------------ */ 129| | /* Phase 1: Push compression in chunks */ 130| | /* ------------------------------------------------------------------ */ 131| | /* Level forced to 1: pstream's state-machine surface (framing, drain, 132| | * resumption, sticky errors) is independent of compression strength, 133| | * and buffer-API fuzzers (roundtrip, decompress) already exercise the 134| | * level dimension. Cheaper compression = more iterations/sec. */ 135| 4.60k| zxc_compress_opts_t copts = { 136| 4.60k| .level = 1, 137| 4.60k| .checksum_enabled = checksum_enabled, 138| 4.60k| }; 139| 4.60k| zxc_cstream* cs = zxc_cstream_create(&copts); 140| 4.60k| if (!cs) return 0; ------------------ | Branch (140:9): [True: 0, False: 4.60k] ------------------ 141| | 142| | /* Sized so the compressor never runs out - bound is a safe upper limit. */ 143| | /* Use the public buffer-API bound function via a local extern declaration 144| | * to avoid pulling zxc_buffer.h (kept narrow on purpose). */ 145| 4.60k| extern uint64_t zxc_compress_bound(size_t input_size); 146| 4.60k| const uint64_t bound64 = zxc_compress_bound(size); 147| 4.60k| if (bound64 == 0 || bound64 > SIZE_MAX) { ------------------ | Branch (147:9): [True: 0, False: 4.60k] | Branch (147:25): [True: 0, False: 4.60k] ------------------ 148| 0| zxc_cstream_free(cs); 149| 0| return 0; 150| 0| } 151| 4.60k| const size_t bound = (size_t)bound64; 152| 4.60k| if (bound > comp_cap) { ------------------ | Branch (152:9): [True: 2.98k, False: 1.62k] ------------------ 153| 2.98k| void* nb = realloc(comp_buf, bound); 154| 2.98k| if (!nb) { ------------------ | Branch (154:13): [True: 0, False: 2.98k] ------------------ 155| 0| zxc_cstream_free(cs); 156| 0| return 0; 157| 0| } 158| 2.98k| comp_buf = (uint8_t*)nb; 159| 2.98k| comp_cap = bound; 160| 2.98k| } 161| | 162| 4.60k| size_t comp_len = 0; 163| 4.60k| size_t in_pos = 0; 164| | 165| | /* Feed input in `in_chunk`-sized slices. */ 166| 109M| while (in_pos < size) { ------------------ | Branch (166:12): [True: 109M, False: 4.60k] ------------------ 167| 109M| const size_t feed = (size - in_pos < in_chunk) ? (size - in_pos) : in_chunk; ------------------ | Branch (167:29): [True: 3.47k, False: 109M] ------------------ 168| 109M| zxc_inbuf_t in = {.src = data + in_pos, .size = feed, .pos = 0}; 169| | 170| | /* Drain repeatedly until this slice is fully consumed. */ 171| 109M| for (;;) { 172| 109M| size_t window = bound - comp_len; 173| 109M| if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 109M|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 109M|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ | Branch (173:17): [True: 109M, False: 15.2k] ------------------ 174| 109M| if (window == 0) { ------------------ | Branch (174:17): [True: 0, False: 109M] ------------------ 175| | /* Bound exhausted - shouldn't happen with zxc_compress_bound. */ 176| 0| zxc_cstream_free(cs); 177| 0| return 0; 178| 0| } 179| 109M| zxc_outbuf_t ob = {.dst = comp_buf + comp_len, .size = window, .pos = 0}; 180| 109M| const int64_t r = zxc_cstream_compress(cs, &ob, &in); 181| 109M| if (r < 0) { ------------------ | Branch (181:17): [True: 0, False: 109M] ------------------ 182| 0| zxc_cstream_free(cs); 183| 0| return 0; 184| 0| } 185| 109M| comp_len += ob.pos; 186| 109M| if (in.pos == in.size && r == 0) break; ------------------ | Branch (186:17): [True: 109M, False: 234k] | Branch (186:38): [True: 109M, False: 84.1k] ------------------ 187| 109M| } 188| 109M| in_pos += in.pos; 189| 109M| } 190| | 191| | /* Finalize: zxc_cstream_end may report >0 pending bytes; drain until 0. */ 192| 512k| for (;;) { 193| 512k| size_t window = bound - comp_len; 194| 512k| if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 512k|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 511k|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ | Branch (194:13): [True: 511k, False: 1.12k] ------------------ 195| 512k| if (window == 0) { ------------------ | Branch (195:13): [True: 0, False: 512k] ------------------ 196| 0| zxc_cstream_free(cs); 197| 0| return 0; 198| 0| } 199| 512k| zxc_outbuf_t ob = {.dst = comp_buf + comp_len, .size = window, .pos = 0}; 200| 512k| const int64_t pending = zxc_cstream_end(cs, &ob); 201| 512k| if (pending < 0) { ------------------ | Branch (201:13): [True: 0, False: 512k] ------------------ 202| 0| zxc_cstream_free(cs); 203| 0| return 0; 204| 0| } 205| 512k| comp_len += ob.pos; 206| 512k| if (pending == 0) break; ------------------ | Branch (206:13): [True: 4.60k, False: 508k] ------------------ 207| 512k| } 208| | 209| 4.60k| zxc_cstream_free(cs); 210| | 211| | /* ------------------------------------------------------------------ */ 212| | /* Phase 2: Push decompression with chunked feeding + roundtrip */ 213| | /* ------------------------------------------------------------------ */ 214| 4.60k| if (size > round_cap) { ------------------ | Branch (214:9): [True: 2.98k, False: 1.62k] ------------------ 215| 2.98k| void* nb = realloc(round_buf, size); 216| 2.98k| if (!nb) return 0; ------------------ | Branch (216:13): [True: 0, False: 2.98k] ------------------ 217| 2.98k| round_buf = (uint8_t*)nb; 218| 2.98k| round_cap = size; 219| 2.98k| } 220| | 221| 4.60k| const int64_t dsize = 222| 4.60k| drip_decompress(comp_buf, comp_len, round_buf, size, in_chunk, checksum_enabled); 223| 4.60k| if (dsize >= 0) { ------------------ | Branch (223:9): [True: 4.60k, False: 0] ------------------ 224| 4.60k| assert((size_t)dsize == size); ------------------ | Branch (224:9): [True: 0, False: 4.60k] | Branch (224:9): [True: 4.60k, False: 0] ------------------ 225| 4.60k| assert(memcmp(data, round_buf, size) == 0); ------------------ | Branch (225:9): [True: 0, False: 4.60k] | Branch (225:9): [True: 4.60k, False: 0] ------------------ 226| 4.60k| } 227| | 228| | /* ------------------------------------------------------------------ */ 229| | /* Phase 3: Parser fuzzing - feed raw input directly */ 230| | /* */ 231| | /* This is where malformed-frame and truncation bugs live. We don't */ 232| | /* care about the result, only that no UB / crash occurs. */ 233| | /* ------------------------------------------------------------------ */ 234| 4.60k| if (raw_size > 0) { ------------------ | Branch (234:9): [True: 4.60k, False: 0] ------------------ 235| | /* Output is discarded - a single small stack buffer suffices and 236| | * keeps the parser's resumption code (small ob.size) under fuzz. */ 237| 4.60k| uint8_t scratch[FUZZ_PSTREAM_OUT_CAP]; 238| 4.60k| const size_t parser_chunk = derive_chunk_size(raw_data[raw_size - 1]); 239| | 240| 4.60k| zxc_decompress_opts_t opts = {.checksum_enabled = checksum_enabled}; 241| 4.60k| zxc_dstream* ds = zxc_dstream_create(&opts); 242| 4.60k| if (ds) { ------------------ | Branch (242:13): [True: 4.60k, False: 0] ------------------ 243| 4.60k| size_t pos = 0; 244| 4.60k| int rounds = 0; 245| 26.9k| while (pos < raw_size && rounds < 1024) { ------------------ | Branch (245:20): [True: 26.8k, False: 105] | Branch (245:38): [True: 26.8k, False: 0] ------------------ 246| 26.8k| const size_t feed = 247| 26.8k| (raw_size - pos < parser_chunk) ? (raw_size - pos) : parser_chunk; ------------------ | Branch (247:21): [True: 515, False: 26.2k] ------------------ 248| 26.8k| zxc_inbuf_t in = {.src = raw_data + pos, .size = feed, .pos = 0}; 249| 26.8k| zxc_outbuf_t ob = {.dst = scratch, .size = sizeof scratch, .pos = 0}; 250| 26.8k| const size_t before_in = in.pos; 251| 26.8k| const size_t before_out = ob.pos; 252| 26.8k| const int64_t r = zxc_dstream_decompress(ds, &ob, &in); 253| 26.8k| pos += in.pos; 254| 26.8k| if (r < 0) break; ------------------ | Branch (254:21): [True: 4.50k, False: 22.3k] ------------------ 255| 22.3k| if (in.pos == before_in && ob.pos == before_out) break; ------------------ | Branch (255:21): [True: 0, False: 22.3k] | Branch (255:44): [True: 0, False: 0] ------------------ 256| 22.3k| rounds++; 257| 22.3k| } 258| 4.60k| zxc_dstream_free(ds); 259| 4.60k| } 260| 4.60k| } 261| | 262| 4.60k| return 0; 263| 4.60k|} fuzz_pstream.c:derive_chunk_size: 47| 9.22k|static size_t derive_chunk_size(uint8_t b) { 48| | /* Map [0..255] -> [1..512] with a non-linear distribution biased toward 49| | * small values (small chunks stress the state machine the most). */ 50| 9.22k| if (b == 0) return 1; ------------------ | Branch (50:9): [True: 2.18k, False: 7.04k] ------------------ 51| 7.04k| if (b < 64) return (size_t)b; /* 1..63 */ ------------------ | Branch (51:9): [True: 2.41k, False: 4.62k] ------------------ 52| 4.62k| if (b < 192) return (size_t)b * 2; /* 128..382 */ ------------------ | Branch (52:9): [True: 1.61k, False: 3.01k] ------------------ 53| 3.01k| return (size_t)b + 256; /* 448..511 */ 54| 4.62k|} fuzz_pstream.c:drip_decompress: 65| 4.60k| size_t in_chunk, int checksum_enabled) { 66| 4.60k| zxc_decompress_opts_t opts = {.checksum_enabled = checksum_enabled}; 67| 4.60k| zxc_dstream* ds = zxc_dstream_create(&opts); 68| 4.60k| if (!ds) return -1; ------------------ | Branch (68:9): [True: 0, False: 4.60k] ------------------ 69| | 70| 4.60k| size_t in_pos = 0; 71| 4.60k| size_t out_total = 0; 72| | 73| 42.7M| while (in_pos < csize || !zxc_dstream_finished(ds)) { ------------------ | Branch (73:12): [True: 42.7M, False: 3.12k] | Branch (73:30): [True: 4, False: 3.12k] ------------------ 74| 42.7M| if (out_total >= out_cap) { ------------------ | Branch (74:13): [True: 1.48k, False: 42.7M] ------------------ 75| | /* Caller's buffer full - stop without asserting. */ 76| 1.48k| zxc_dstream_free(ds); 77| 1.48k| return (int64_t)out_total; 78| 1.48k| } 79| 42.7M| const size_t feed = (csize - in_pos < in_chunk) ? (csize - in_pos) : in_chunk; ------------------ | Branch (79:29): [True: 858k, False: 41.8M] ------------------ 80| 42.7M| size_t window = out_cap - out_total; 81| 42.7M| if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 42.7M|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 42.6M|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ | Branch (81:13): [True: 42.6M, False: 30.8k] ------------------ 82| 42.7M| zxc_inbuf_t in = {.src = comp + in_pos, .size = feed, .pos = 0}; 83| 42.7M| zxc_outbuf_t ob = {.dst = out + out_total, .size = window, .pos = 0}; 84| | 85| 42.7M| const size_t before_in = in.pos; 86| 42.7M| const size_t before_out = ob.pos; 87| 42.7M| const int64_t r = zxc_dstream_decompress(ds, &ob, &in); 88| 42.7M| if (r < 0) { ------------------ | Branch (88:13): [True: 0, False: 42.7M] ------------------ 89| 0| zxc_dstream_free(ds); 90| 0| return r; 91| 0| } 92| | 93| 42.7M| out_total += ob.pos; 94| 42.7M| in_pos += in.pos; 95| | 96| | /* No progress and no more input: prevent infinite loop on truncation. */ 97| 42.7M| if (in.pos == before_in && ob.pos == before_out && in_pos == csize) break; ------------------ | Branch (97:13): [True: 1.69M, False: 41.0M] | Branch (97:36): [True: 0, False: 1.69M] | Branch (97:60): [True: 0, False: 0] ------------------ 98| 42.7M| } 99| | 100| 3.12k| zxc_dstream_free(ds); 101| 3.12k| return (int64_t)out_total; 102| 4.60k|}