zxc_compress.c:rapidhash: 500| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash(const void *key, size_t len) RAPIDHASH_NOEXCEPT { 501| 1.63k| return rapidhash_withSeed(key, len, 0); 502| 1.63k|} zxc_compress.c:rapidhash_withSeed: 486| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_withSeed(const void *key, size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT { 487| 1.63k| return rapidhash_internal(key, len, seed, rapid_secret); 488| 1.63k|} zxc_compress.c:rapidhash_internal: 243| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_internal(const void *key, size_t len, uint64_t seed, const uint64_t* secret) RAPIDHASH_NOEXCEPT { 244| 1.63k| const uint8_t *p=(const uint8_t *)key; 245| 1.63k| seed ^= rapid_mix(seed ^ secret[2], secret[1]); 246| 1.63k| uint64_t a=0, b=0; 247| 1.63k| size_t i = len; 248| 1.63k| if (_likely_(len <= 16)) { ------------------ | | 106| 1.63k| # define _likely_(x) __builtin_expect(x,1) | | ------------------ | | | Branch (106:24): [True: 73, False: 1.56k] | | ------------------ ------------------ 249| 73| if (len >= 4) { ------------------ | Branch (249:9): [True: 52, False: 21] ------------------ 250| 52| seed ^= len; 251| 52| if (len >= 8) { ------------------ | Branch (251:11): [True: 46, False: 6] ------------------ 252| 46| const uint8_t* plast = p + len - 8; 253| 46| a = rapid_read64(p); 254| 46| b = rapid_read64(plast); 255| 46| } else { 256| 6| const uint8_t* plast = p + len - 4; 257| 6| a = rapid_read32(p); 258| 6| b = rapid_read32(plast); 259| 6| } 260| 52| } else if (len > 0) { ------------------ | Branch (260:16): [True: 21, False: 0] ------------------ 261| 21| a = (((uint64_t)p[0])<<45)|p[len-1]; 262| 21| b = p[len>>1]; 263| 21| } else 264| 0| a = b = 0; 265| 1.56k| } else { 266| 1.56k| if (len > 112) { ------------------ | Branch (266:9): [True: 1.27k, False: 290] ------------------ 267| 1.27k| uint64_t see1 = seed, see2 = seed; 268| 1.27k| uint64_t see3 = seed, see4 = seed; 269| 1.27k| uint64_t see5 = seed, see6 = seed; 270| 1.27k|#ifdef RAPIDHASH_COMPACT 271| 562k| do { 272| 562k| seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 273| 562k| see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 274| 562k| see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 275| 562k| see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 276| 562k| see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 277| 562k| see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 278| 562k| see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 279| 562k| p += 112; 280| 562k| i -= 112; 281| 562k| } while(i > 112); ------------------ | Branch (281:15): [True: 560k, False: 1.27k] ------------------ 282| |#else 283| | while (i > 224) { 284| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 285| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 286| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 287| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 288| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 289| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 290| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 291| | seed = rapid_mix(rapid_read64(p + 112) ^ secret[0], rapid_read64(p + 120) ^ seed); 292| | see1 = rapid_mix(rapid_read64(p + 128) ^ secret[1], rapid_read64(p + 136) ^ see1); 293| | see2 = rapid_mix(rapid_read64(p + 144) ^ secret[2], rapid_read64(p + 152) ^ see2); 294| | see3 = rapid_mix(rapid_read64(p + 160) ^ secret[3], rapid_read64(p + 168) ^ see3); 295| | see4 = rapid_mix(rapid_read64(p + 176) ^ secret[4], rapid_read64(p + 184) ^ see4); 296| | see5 = rapid_mix(rapid_read64(p + 192) ^ secret[5], rapid_read64(p + 200) ^ see5); 297| | see6 = rapid_mix(rapid_read64(p + 208) ^ secret[6], rapid_read64(p + 216) ^ see6); 298| | p += 224; 299| | i -= 224; 300| | } 301| | if (i > 112) { 302| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 303| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 304| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 305| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 306| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 307| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 308| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 309| | p += 112; 310| | i -= 112; 311| | } 312| |#endif 313| 1.27k| seed ^= see1; 314| 1.27k| see2 ^= see3; 315| 1.27k| see4 ^= see5; 316| 1.27k| seed ^= see6; 317| 1.27k| see2 ^= see4; 318| 1.27k| seed ^= see2; 319| 1.27k| } 320| 1.56k| if (i > 16) { ------------------ | Branch (320:9): [True: 1.35k, False: 213] ------------------ 321| 1.35k| seed = rapid_mix(rapid_read64(p) ^ secret[2], rapid_read64(p + 8) ^ seed); 322| 1.35k| if (i > 32) { ------------------ | Branch (322:11): [True: 1.10k, False: 244] ------------------ 323| 1.10k| seed = rapid_mix(rapid_read64(p + 16) ^ secret[2], rapid_read64(p + 24) ^ seed); 324| 1.10k| if (i > 48) { ------------------ | Branch (324:15): [True: 932, False: 174] ------------------ 325| 932| seed = rapid_mix(rapid_read64(p + 32) ^ secret[1], rapid_read64(p + 40) ^ seed); 326| 932| if (i > 64) { ------------------ | Branch (326:19): [True: 693, False: 239] ------------------ 327| 693| seed = rapid_mix(rapid_read64(p + 48) ^ secret[1], rapid_read64(p + 56) ^ seed); 328| 693| if (i > 80) { ------------------ | Branch (328:23): [True: 460, False: 233] ------------------ 329| 460| seed = rapid_mix(rapid_read64(p + 64) ^ secret[2], rapid_read64(p + 72) ^ seed); 330| 460| if (i > 96) { ------------------ | Branch (330:27): [True: 212, False: 248] ------------------ 331| 212| seed = rapid_mix(rapid_read64(p + 80) ^ secret[1], rapid_read64(p + 88) ^ seed); 332| 212| } 333| 460| } 334| 693| } 335| 932| } 336| 1.10k| } 337| 1.35k| } 338| 1.56k| a=rapid_read64(p+i-16) ^ i; b=rapid_read64(p+i-8); 339| 1.56k| } 340| 1.63k| a ^= secret[1]; 341| 1.63k| b ^= seed; 342| 1.63k| rapid_mum(&a, &b); 343| 1.63k| return rapid_mix(a ^ secret[7], b ^ secret[1] ^ i); 344| 1.63k|} zxc_compress.c:rapid_mix: 208| 3.94M| RAPIDHASH_INLINE_CONSTEXPR uint64_t rapid_mix(uint64_t A, uint64_t B) RAPIDHASH_NOEXCEPT { rapid_mum(&A,&B); return A^B; } zxc_compress.c:rapid_read64: 214| 7.88M| RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return v;} zxc_compress.c:rapid_read32: 215| 12| RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return v;} zxc_compress.c:rapid_mum: 156| 3.94M| RAPIDHASH_INLINE_CONSTEXPR void rapid_mum(uint64_t *A, uint64_t *B) RAPIDHASH_NOEXCEPT { 157| 3.94M| #if defined(__SIZEOF_INT128__) 158| 3.94M| __uint128_t r=*A; r*=*B; 159| | #ifdef RAPIDHASH_PROTECTED 160| | *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); 161| | #else 162| 3.94M| *A=(uint64_t)r; *B=(uint64_t)(r>>64); 163| 3.94M| #endif 164| | #elif defined(_MSC_VER) && (defined(_WIN64) || defined(_M_HYBRID_CHPE_ARM64)) 165| | #if defined(_M_X64) 166| | #ifdef RAPIDHASH_PROTECTED 167| | uint64_t a, b; 168| | a=_umul128(*A,*B,&b); 169| | *A^=a; *B^=b; 170| | #else 171| | *A=_umul128(*A,*B,B); 172| | #endif 173| | #else 174| | #ifdef RAPIDHASH_PROTECTED 175| | uint64_t a, b; 176| | b = __umulh(*A, *B); 177| | a = *A * *B; 178| | *A^=a; *B^=b; 179| | #else 180| | uint64_t c = __umulh(*A, *B); 181| | *A = *A * *B; 182| | *B = c; 183| | #endif 184| | #endif 185| | #else 186| | uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B; 187| | uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t>32)+(rm1>>32)+c; 191| | #ifdef RAPIDHASH_PROTECTED 192| | *A^=lo; *B^=hi; 193| | #else 194| | *A=lo; *B=hi; 195| | #endif 196| | #endif 197| 3.94M| } zxc_decompress.c:rapidhash: 500| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash(const void *key, size_t len) RAPIDHASH_NOEXCEPT { 501| 1.63k| return rapidhash_withSeed(key, len, 0); 502| 1.63k|} zxc_decompress.c:rapidhash_withSeed: 486| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_withSeed(const void *key, size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT { 487| 1.63k| return rapidhash_internal(key, len, seed, rapid_secret); 488| 1.63k|} zxc_decompress.c:rapidhash_internal: 243| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_internal(const void *key, size_t len, uint64_t seed, const uint64_t* secret) RAPIDHASH_NOEXCEPT { 244| 1.63k| const uint8_t *p=(const uint8_t *)key; 245| 1.63k| seed ^= rapid_mix(seed ^ secret[2], secret[1]); 246| 1.63k| uint64_t a=0, b=0; 247| 1.63k| size_t i = len; 248| 1.63k| if (_likely_(len <= 16)) { ------------------ | | 106| 1.63k| # define _likely_(x) __builtin_expect(x,1) | | ------------------ | | | Branch (106:24): [True: 73, False: 1.56k] | | ------------------ ------------------ 249| 73| if (len >= 4) { ------------------ | Branch (249:9): [True: 52, False: 21] ------------------ 250| 52| seed ^= len; 251| 52| if (len >= 8) { ------------------ | Branch (251:11): [True: 46, False: 6] ------------------ 252| 46| const uint8_t* plast = p + len - 8; 253| 46| a = rapid_read64(p); 254| 46| b = rapid_read64(plast); 255| 46| } else { 256| 6| const uint8_t* plast = p + len - 4; 257| 6| a = rapid_read32(p); 258| 6| b = rapid_read32(plast); 259| 6| } 260| 52| } else if (len > 0) { ------------------ | Branch (260:16): [True: 21, False: 0] ------------------ 261| 21| a = (((uint64_t)p[0])<<45)|p[len-1]; 262| 21| b = p[len>>1]; 263| 21| } else 264| 0| a = b = 0; 265| 1.56k| } else { 266| 1.56k| if (len > 112) { ------------------ | Branch (266:9): [True: 1.27k, False: 290] ------------------ 267| 1.27k| uint64_t see1 = seed, see2 = seed; 268| 1.27k| uint64_t see3 = seed, see4 = seed; 269| 1.27k| uint64_t see5 = seed, see6 = seed; 270| 1.27k|#ifdef RAPIDHASH_COMPACT 271| 562k| do { 272| 562k| seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 273| 562k| see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 274| 562k| see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 275| 562k| see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 276| 562k| see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 277| 562k| see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 278| 562k| see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 279| 562k| p += 112; 280| 562k| i -= 112; 281| 562k| } while(i > 112); ------------------ | Branch (281:15): [True: 560k, False: 1.27k] ------------------ 282| |#else 283| | while (i > 224) { 284| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 285| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 286| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 287| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 288| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 289| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 290| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 291| | seed = rapid_mix(rapid_read64(p + 112) ^ secret[0], rapid_read64(p + 120) ^ seed); 292| | see1 = rapid_mix(rapid_read64(p + 128) ^ secret[1], rapid_read64(p + 136) ^ see1); 293| | see2 = rapid_mix(rapid_read64(p + 144) ^ secret[2], rapid_read64(p + 152) ^ see2); 294| | see3 = rapid_mix(rapid_read64(p + 160) ^ secret[3], rapid_read64(p + 168) ^ see3); 295| | see4 = rapid_mix(rapid_read64(p + 176) ^ secret[4], rapid_read64(p + 184) ^ see4); 296| | see5 = rapid_mix(rapid_read64(p + 192) ^ secret[5], rapid_read64(p + 200) ^ see5); 297| | see6 = rapid_mix(rapid_read64(p + 208) ^ secret[6], rapid_read64(p + 216) ^ see6); 298| | p += 224; 299| | i -= 224; 300| | } 301| | if (i > 112) { 302| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 303| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 304| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 305| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 306| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 307| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 308| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 309| | p += 112; 310| | i -= 112; 311| | } 312| |#endif 313| 1.27k| seed ^= see1; 314| 1.27k| see2 ^= see3; 315| 1.27k| see4 ^= see5; 316| 1.27k| seed ^= see6; 317| 1.27k| see2 ^= see4; 318| 1.27k| seed ^= see2; 319| 1.27k| } 320| 1.56k| if (i > 16) { ------------------ | Branch (320:9): [True: 1.35k, False: 213] ------------------ 321| 1.35k| seed = rapid_mix(rapid_read64(p) ^ secret[2], rapid_read64(p + 8) ^ seed); 322| 1.35k| if (i > 32) { ------------------ | Branch (322:11): [True: 1.10k, False: 244] ------------------ 323| 1.10k| seed = rapid_mix(rapid_read64(p + 16) ^ secret[2], rapid_read64(p + 24) ^ seed); 324| 1.10k| if (i > 48) { ------------------ | Branch (324:15): [True: 932, False: 174] ------------------ 325| 932| seed = rapid_mix(rapid_read64(p + 32) ^ secret[1], rapid_read64(p + 40) ^ seed); 326| 932| if (i > 64) { ------------------ | Branch (326:19): [True: 693, False: 239] ------------------ 327| 693| seed = rapid_mix(rapid_read64(p + 48) ^ secret[1], rapid_read64(p + 56) ^ seed); 328| 693| if (i > 80) { ------------------ | Branch (328:23): [True: 460, False: 233] ------------------ 329| 460| seed = rapid_mix(rapid_read64(p + 64) ^ secret[2], rapid_read64(p + 72) ^ seed); 330| 460| if (i > 96) { ------------------ | Branch (330:27): [True: 212, False: 248] ------------------ 331| 212| seed = rapid_mix(rapid_read64(p + 80) ^ secret[1], rapid_read64(p + 88) ^ seed); 332| 212| } 333| 460| } 334| 693| } 335| 932| } 336| 1.10k| } 337| 1.35k| } 338| 1.56k| a=rapid_read64(p+i-16) ^ i; b=rapid_read64(p+i-8); 339| 1.56k| } 340| 1.63k| a ^= secret[1]; 341| 1.63k| b ^= seed; 342| 1.63k| rapid_mum(&a, &b); 343| 1.63k| return rapid_mix(a ^ secret[7], b ^ secret[1] ^ i); 344| 1.63k|} zxc_decompress.c:rapid_mix: 208| 3.94M| RAPIDHASH_INLINE_CONSTEXPR uint64_t rapid_mix(uint64_t A, uint64_t B) RAPIDHASH_NOEXCEPT { rapid_mum(&A,&B); return A^B; } zxc_decompress.c:rapid_read64: 214| 7.88M| RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return v;} zxc_decompress.c:rapid_read32: 215| 12| RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return v;} zxc_decompress.c:rapid_mum: 156| 3.94M| RAPIDHASH_INLINE_CONSTEXPR void rapid_mum(uint64_t *A, uint64_t *B) RAPIDHASH_NOEXCEPT { 157| 3.94M| #if defined(__SIZEOF_INT128__) 158| 3.94M| __uint128_t r=*A; r*=*B; 159| | #ifdef RAPIDHASH_PROTECTED 160| | *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); 161| | #else 162| 3.94M| *A=(uint64_t)r; *B=(uint64_t)(r>>64); 163| 3.94M| #endif 164| | #elif defined(_MSC_VER) && (defined(_WIN64) || defined(_M_HYBRID_CHPE_ARM64)) 165| | #if defined(_M_X64) 166| | #ifdef RAPIDHASH_PROTECTED 167| | uint64_t a, b; 168| | a=_umul128(*A,*B,&b); 169| | *A^=a; *B^=b; 170| | #else 171| | *A=_umul128(*A,*B,B); 172| | #endif 173| | #else 174| | #ifdef RAPIDHASH_PROTECTED 175| | uint64_t a, b; 176| | b = __umulh(*A, *B); 177| | a = *A * *B; 178| | *A^=a; *B^=b; 179| | #else 180| | uint64_t c = __umulh(*A, *B); 181| | *A = *A * *B; 182| | *B = c; 183| | #endif 184| | #endif 185| | #else 186| | uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B; 187| | uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t>32)+(rm1>>32)+c; 191| | #ifdef RAPIDHASH_PROTECTED 192| | *A^=lo; *B^=hi; 193| | #else 194| | *A=lo; *B=hi; 195| | #endif 196| | #endif 197| 3.94M| } zxc_aligned_malloc: 36| 9.21k|void* zxc_aligned_malloc(const size_t size, const size_t alignment) { 37| |#if defined(_WIN32) 38| | return _aligned_malloc(size, alignment); 39| |#else 40| 9.21k| void* ptr = NULL; 41| 9.21k| if (posix_memalign(&ptr, alignment, size) != 0) return NULL; ------------------ | Branch (41:9): [True: 0, False: 9.21k] ------------------ 42| 9.21k| return ptr; 43| 9.21k|#endif 44| 9.21k|} zxc_aligned_free: 51| 9.21k|void zxc_aligned_free(void* ptr) { 52| |#if defined(_WIN32) 53| | _aligned_free(ptr); 54| |#else 55| 9.21k| free(ptr); 56| 9.21k|#endif 57| 9.21k|} zxc_cctx_init_in_workspace: 298| 9.21k| const int defer_entropy_scratch) { 299| 9.21k| if (UNLIKELY(!ctx || !workspace || chunk_size == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 36.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | ------------------ ------------------ 300| | 301| 9.21k| const zxc_cctx_layout_t layout = 302| 9.21k| compute_cctx_layout(chunk_size, mode, level, dict_size, defer_entropy_scratch); 303| 9.21k| if (UNLIKELY(workspace_size < layout.total)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 304| | 305| 9.21k| ZXC_MEMSET(ctx, 0, sizeof(zxc_cctx_t)); ------------------ | | 178| 9.21k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 306| 9.21k| ctx->checksum_enabled = checksum_enabled; 307| 9.21k| ctx->chunk_size = chunk_size; 308| 9.21k| const uint32_t offset_bits = zxc_log2_u32((uint32_t)chunk_size); 309| 9.21k| ctx->offset_bits = offset_bits; 310| 9.21k| ctx->offset_mask = (uint32_t)((1ULL << offset_bits) - 1); 311| 9.21k| ctx->max_epoch = (uint32_t)(1ULL << (32 - offset_bits)); 312| | 313| | /* memory_block stays NULL on the static-init path so zxc_cctx_free does 314| | * not try to free the caller's workspace. Sub-buffer pointers carry the 315| | * partition; ownership is implicit (the caller owns @p workspace). */ 316| 9.21k| uint8_t* const mem = (uint8_t*)workspace; 317| | 318| | /* Dictionary concat scratch (both modes); init owns dict_size now so callers 319| | * no longer assign ctx->dict_size after init. */ 320| 9.21k| ctx->dict_size = dict_size; 321| 9.21k| if (dict_size > 0) { ------------------ | Branch (321:9): [True: 0, False: 9.21k] ------------------ 322| 0| ctx->dict_buffer = mem + layout.off_dict; 323| 0| ctx->dict_buffer_cap = layout.sz_dict; 324| 0| ctx->dict_huf = (zxc_dict_huf_state_t*)(void*)(mem + layout.off_dict_huf); 325| 0| } 326| | 327| 9.21k| if (mode == 0) { ------------------ | Branch (327:9): [True: 4.60k, False: 4.60k] ------------------ 328| 4.60k| ctx->work_buf = mem + layout.off_work; 329| 4.60k| ctx->work_buf_cap = chunk_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 349| 4.60k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 4.60k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 330| 4.60k| ctx->lit_buffer = mem + layout.off_lit_dctx; 331| 4.60k| ctx->lit_buffer_cap = chunk_size + ZXC_PAD_SIZE; ------------------ | | 337| 4.60k|#define ZXC_PAD_SIZE 32 ------------------ 332| 4.60k| if (layout.sz_pivco_dctx) { ------------------ | Branch (332:13): [True: 0, False: 4.60k] ------------------ 333| 0| ctx->tok_buffer = mem + layout.off_tok_dctx; 334| 0| ctx->tok_buffer_cap = layout.sz_tok_dctx; 335| 0| ctx->pivco_scratch = mem + layout.off_pivco_dctx; 336| 0| ctx->pivco_scratch_cap = layout.sz_pivco_dctx; 337| 0| } 338| 4.60k| return ZXC_OK; 339| 4.60k| } 340| | 341| 4.60k| ctx->hash_table = (uint32_t*)(mem + layout.off_hash_pos); 342| 4.60k| ctx->hash_tags = mem + layout.off_hash_tags; 343| 4.60k| ctx->chain_table = (uint16_t*)(mem + layout.off_chain); 344| 4.60k| ctx->buf_sequences = (uint32_t*)(mem + layout.off_seq_union); 345| 4.60k| ctx->buf_offsets = (uint16_t*)(mem + layout.off_seq_union); 346| 4.60k| ctx->buf_tokens = mem + layout.off_seq_union + layout.max_seq * sizeof(uint16_t); 347| 4.60k| ctx->buf_extras = mem + layout.off_extras; 348| 4.60k| ctx->literals = mem + layout.off_lit_cctx; 349| 4.60k| if (layout.sz_opt) { ------------------ | Branch (349:9): [True: 0, False: 4.60k] ------------------ 350| 0| ctx->opt_scratch = mem + layout.off_opt; 351| 0| ctx->opt_scratch_cap = layout.sz_opt; 352| 0| } 353| | 354| 4.60k| ctx->compression_level = level; 355| 4.60k| ctx->epoch = 1; 356| | 357| 4.60k| ZXC_MEMSET(ctx->hash_table, 0, layout.sz_hash_pos); ------------------ | | 178| 4.60k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 358| 4.60k| ZXC_MEMSET(ctx->hash_tags, 0, layout.sz_hash_tags); ------------------ | | 178| 4.60k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 359| 4.60k| return ZXC_OK; 360| 9.21k|} zxc_cctx_init: 381| 9.21k| const int level, const int checksum_enabled, const size_t dict_size) { 382| 9.21k| if (UNLIKELY(chunk_size == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 383| | /* Heap contexts defer the decode-side entropy scratch to the first entropy 384| | * section; static workspaces must have it already, they may never allocate. */ 385| 9.21k| const int defer_entropy = (mode == 0); 386| 9.21k| const size_t total = 387| 9.21k| compute_cctx_layout(chunk_size, mode, level, dict_size, defer_entropy).total; 388| 9.21k| if (UNLIKELY(total == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 389| | 390| 9.21k| uint8_t* const mem = (uint8_t*)ZXC_ALIGNED_MALLOC(total, ZXC_CACHE_LINE_SIZE); ------------------ | | 110| 9.21k|#define ZXC_ALIGNED_MALLOC(size, alignment) zxc_aligned_malloc(size, alignment) ------------------ 391| 9.21k| if (UNLIKELY(!mem)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 392| | 393| 9.21k| const int rc = zxc_cctx_init_in_workspace(ctx, mem, total, chunk_size, mode, level, 394| 9.21k| checksum_enabled, dict_size, defer_entropy); 395| 9.21k| if (UNLIKELY(rc != ZXC_OK)) { ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 396| | // LCOV_EXCL_START 397| 0| ZXC_ALIGNED_FREE(mem); ------------------ | | 116| 0|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 398| 0| return rc; 399| | // LCOV_EXCL_STOP 400| 0| } 401| | /* Library-owned buffer: record the allocation so zxc_cctx_free frees it. */ 402| 9.21k| ctx->memory_block = mem; 403| 9.21k| return ZXC_OK; 404| 9.21k|} zxc_cctx_free: 441| 9.21k|void zxc_cctx_free(zxc_cctx_t* ctx) { 442| 9.21k| if (ctx->memory_block) { ------------------ | Branch (442:9): [True: 9.21k, False: 0] ------------------ 443| 9.21k| ZXC_ALIGNED_FREE(ctx->memory_block); ------------------ | | 116| 9.21k|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 444| 9.21k| ctx->memory_block = NULL; 445| 9.21k| } 446| 9.21k| if (ctx->entropy_block) { ------------------ | Branch (446:9): [True: 0, False: 9.21k] ------------------ 447| 0| ZXC_ALIGNED_FREE(ctx->entropy_block); ------------------ | | 116| 0|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 448| 0| ctx->entropy_block = NULL; 449| 0| } 450| | 451| 9.21k| ctx->lit_buffer = NULL; 452| 9.21k| ctx->hash_table = NULL; 453| 9.21k| ctx->hash_tags = NULL; 454| 9.21k| ctx->chain_table = NULL; 455| 9.21k| ctx->buf_sequences = NULL; 456| 9.21k| ctx->buf_tokens = NULL; 457| 9.21k| ctx->buf_offsets = NULL; 458| 9.21k| ctx->buf_extras = NULL; 459| 9.21k| ctx->literals = NULL; 460| 9.21k| ctx->work_buf = NULL; 461| 9.21k| ctx->tok_buffer = NULL; 462| 9.21k| ctx->pivco_scratch = NULL; 463| 9.21k| ctx->opt_scratch = NULL; 464| 9.21k| ctx->dict_buffer = NULL; 465| 9.21k| ctx->dict_huf = NULL; 466| | 467| 9.21k| ctx->epoch = 0; 468| 9.21k| ctx->lit_buffer_cap = 0; 469| 9.21k| ctx->work_buf_cap = 0; 470| 9.21k| ctx->tok_buffer_cap = 0; 471| 9.21k| ctx->pivco_scratch_cap = 0; 472| 9.21k| ctx->opt_scratch_cap = 0; 473| 9.21k| ctx->dict_buffer_cap = 0; 474| 9.21k| ctx->dict_size = 0; 475| 9.21k| ctx->dict_huf_tree_ok = 0; 476| | ctx->lit_freq_acc = NULL; 477| 9.21k|} zxc_write_file_header: 539| 4.60k| const int has_checksum, const uint32_t dict_id) { 540| 4.60k| if (UNLIKELY(dst_capacity < ZXC_FILE_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 541| | 542| 4.60k| zxc_store_le32(dst, ZXC_MAGIC_WORD); ------------------ | | 331| 4.60k|#define ZXC_MAGIC_WORD 0x9CB02EF5U ------------------ 543| 4.60k| dst[4] = ZXC_FILE_FORMAT_VERSION; ------------------ | | 334| 4.60k|#define ZXC_FILE_FORMAT_VERSION 7 ------------------ 544| | 545| | // Block size stored as log2 exponent (e.g. 18 = 256 KB) 546| 4.60k| dst[5] = (uint8_t)zxc_log2_u32((uint32_t)chunk_size); 547| | 548| 4.60k| uint8_t flags = has_checksum ? (ZXC_FILE_FLAG_HAS_CHECKSUM | ZXC_CHECKSUM_RAPIDHASH) : 0; ------------------ | | 365| 1.53k|#define ZXC_FILE_FLAG_HAS_CHECKSUM 0x80U ------------------ uint8_t flags = has_checksum ? (ZXC_FILE_FLAG_HAS_CHECKSUM | ZXC_CHECKSUM_RAPIDHASH) : 0; ------------------ | | 424| 1.53k|#define ZXC_CHECKSUM_RAPIDHASH 0 ------------------ | Branch (548:21): [True: 1.53k, False: 3.06k] ------------------ 549| 4.60k| if (dict_id != 0) flags |= ZXC_FILE_FLAG_HAS_DICTIONARY; ------------------ | | 367| 0|#define ZXC_FILE_FLAG_HAS_DICTIONARY 0x40U ------------------ | Branch (549:9): [True: 0, False: 4.60k] ------------------ 550| 4.60k| dst[6] = flags; 551| | 552| | // Bytes 7-13: Reserved / dict_id 553| 4.60k| ZXC_MEMSET(dst + 7, 0, 7); ------------------ | | 178| 4.60k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 554| 4.60k| if (dict_id != 0) zxc_store_le32(dst + 7, dict_id); ------------------ | Branch (554:9): [True: 0, False: 4.60k] ------------------ 555| | 556| | // Bytes 14-15: CRC (16-bit) 557| 4.60k| zxc_store_le16(dst + 14, 0); // Zero out before hashing 558| 4.60k| const uint16_t crc = zxc_hash16(dst); 559| 4.60k| zxc_store_le16(dst + 14, crc); 560| | 561| 4.60k| return ZXC_FILE_HEADER_SIZE; ------------------ | | 103| 4.60k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ 562| 4.60k|} zxc_read_file_header: 580| 9.10k| uint32_t* RESTRICT out_dict_id) { 581| 9.10k| if (UNLIKELY(src_size < ZXC_FILE_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 9.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.10k] | | ------------------ ------------------ 582| 9.10k| if (UNLIKELY(zxc_le32(src) != ZXC_MAGIC_WORD)) return ZXC_ERROR_BAD_MAGIC; ------------------ | | 151| 9.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 4.37k, False: 4.72k] | | ------------------ ------------------ 583| 4.72k| if (UNLIKELY(src[4] != ZXC_FILE_FORMAT_VERSION)) return ZXC_ERROR_BAD_VERSION; ------------------ | | 151| 4.72k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 86, False: 4.64k] | | ------------------ ------------------ 584| | 585| 4.64k| uint8_t temp[ZXC_FILE_HEADER_SIZE]; 586| 4.64k| ZXC_MEMCPY(temp, src, ZXC_FILE_HEADER_SIZE); ------------------ | | 173| 4.64k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 587| | // Zero out CRC bytes (14-15) before hash check 588| 4.64k| temp[14] = 0; 589| 4.64k| temp[15] = 0; 590| | // Header CRC16 (integrity), then the checksum-algorithm id in flags bits 0-3 591| | // (only 0 = RapidHash is defined). CRC is checked first via short-circuit. 592| 4.64k| if (UNLIKELY(zxc_le16(src + 14) != zxc_hash16(temp) || ------------------ | | 151| 9.25k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 36, False: 4.60k] | | | Branch (151:42): [True: 32, False: 4.61k] | | | Branch (151:42): [True: 4, False: 4.60k] | | ------------------ ------------------ 593| 4.64k| (src[6] & 0x0FU) != ZXC_CHECKSUM_RAPIDHASH)) 594| 36| return ZXC_ERROR_BAD_HEADER; 595| | 596| 4.60k| if (out_block_size) { ------------------ | Branch (596:9): [True: 4.60k, False: 0] ------------------ 597| 4.60k| const uint8_t code = src[5]; 598| 4.60k| if (UNLIKELY(code < ZXC_BLOCK_SIZE_MIN_LOG2 || code > ZXC_BLOCK_SIZE_MAX_LOG2)) ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1, False: 4.60k] | | | Branch (151:42): [True: 1, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 599| 1| return ZXC_ERROR_BAD_BLOCK_SIZE; 600| | // Exponent encoding: block_size = 2^code (4 KB - 2 MB) 601| 4.60k| *out_block_size = (size_t)1U << code; 602| 4.60k| } 603| 4.60k| if (out_has_checksum) *out_has_checksum = (src[6] & ZXC_FILE_FLAG_HAS_CHECKSUM) ? 1 : 0; ------------------ | | 365| 4.60k|#define ZXC_FILE_FLAG_HAS_CHECKSUM 0x80U ------------------ | Branch (603:9): [True: 4.60k, False: 0] | Branch (603:47): [True: 1.53k, False: 3.06k] ------------------ 604| 4.60k| if (out_dict_id) *out_dict_id = (src[6] & ZXC_FILE_FLAG_HAS_DICTIONARY) ? zxc_le32(src + 7) : 0; ------------------ | | 367| 0|#define ZXC_FILE_FLAG_HAS_DICTIONARY 0x40U ------------------ | Branch (604:9): [True: 0, False: 4.60k] | Branch (604:37): [True: 0, False: 0] ------------------ 605| | 606| 4.60k| return ZXC_OK; 607| 4.60k|} zxc_write_block_header: 619| 10.2k| const zxc_block_header_t* RESTRICT bh) { 620| 10.2k| if (UNLIKELY(dst_capacity < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 10.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 10.2k] | | ------------------ ------------------ 621| | 622| 10.2k| dst[0] = bh->block_type; 623| 10.2k| dst[1] = 0; // Flags not used currently 624| 10.2k| dst[2] = 0; // Reserved 625| 10.2k| zxc_store_le32(dst + 3, bh->comp_size); 626| 10.2k| dst[7] = 0; // Zero before hashing 627| 10.2k| dst[7] = zxc_hash8(dst); // Checksum at the end 628| | 629| 10.2k| return ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 10.2k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 630| 10.2k|} zxc_read_block_header: 643| 11.3k| zxc_block_header_t* RESTRICT bh) { 644| 11.3k| if (UNLIKELY(src_size < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 11.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 11.3k] | | ------------------ ------------------ 645| | 646| 11.3k| uint8_t temp[ZXC_BLOCK_HEADER_SIZE]; 647| 11.3k| ZXC_MEMCPY(temp, src, ZXC_BLOCK_HEADER_SIZE); ------------------ | | 173| 11.3k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 648| 11.3k| temp[7] = 0; // Zero out checksum byte before hashing 649| 11.3k| if (UNLIKELY(src[7] != zxc_hash8(temp))) return ZXC_ERROR_BAD_HEADER; ------------------ | | 151| 11.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 3.13k, False: 8.18k] | | ------------------ ------------------ 650| | 651| 8.18k| bh->block_type = src[0]; 652| 8.18k| bh->block_flags = 0; // Flags not used currently 653| 8.18k| bh->reserved = src[2]; 654| 8.18k| bh->comp_size = zxc_le32(src + 3); 655| 8.18k| bh->header_crc = src[7]; 656| | 657| 8.18k| return ZXC_OK; 658| 11.3k|} zxc_write_file_footer: 672| 4.60k| const uint32_t global_hash, const int checksum_enabled) { 673| 4.60k| if (UNLIKELY(dst_capacity < ZXC_FILE_FOOTER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 674| | 675| 4.60k| zxc_store_le64(dst, src_size); 676| | 677| 4.60k| if (checksum_enabled) { ------------------ | Branch (677:9): [True: 1.53k, False: 3.06k] ------------------ 678| 1.53k| zxc_store_le32(dst + sizeof(uint64_t), global_hash); 679| 3.06k| } else { 680| 3.06k| ZXC_MEMSET(dst + sizeof(uint64_t), 0, sizeof(uint32_t)); ------------------ | | 178| 3.06k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 681| 3.06k| } 682| | 683| 4.60k| return ZXC_FILE_FOOTER_SIZE; ------------------ | | 105| 4.60k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 684| 4.60k|} zxc_write_ghi_header_and_desc: 766| 4.90k| const zxc_section_desc_t desc[ZXC_GHI_SECTIONS]) { 767| 4.90k| const size_t needed = 768| 4.90k| ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 403| 4.90k|#define ZXC_GHI_HEADER_BINARY_SIZE 16 ------------------ ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 421| 4.90k|#define ZXC_GHI_SECTIONS 3 ------------------ ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 4.90k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 769| | 770| 4.90k| if (UNLIKELY(rem < needed)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 771| | 772| 4.90k| zxc_store_le32(dst, gh->n_sequences); 773| 4.90k| zxc_store_le32(dst + 4, gh->n_literals); 774| | 775| 4.90k| dst[8] = gh->enc_lit; 776| 4.90k| dst[9] = gh->enc_litlen; 777| 4.90k| dst[10] = gh->enc_mlen; 778| 4.90k| dst[11] = gh->enc_off; 779| | 780| 4.90k| zxc_store_le32(dst + 12, 0); 781| 4.90k| uint8_t* p = dst + ZXC_GHI_HEADER_BINARY_SIZE; ------------------ | | 403| 4.90k|#define ZXC_GHI_HEADER_BINARY_SIZE 16 ------------------ 782| | 783| 19.6k| for (int i = 0; i < ZXC_GHI_SECTIONS; i++) { ------------------ | | 421| 19.6k|#define ZXC_GHI_SECTIONS 3 ------------------ | Branch (783:21): [True: 14.7k, False: 4.90k] ------------------ 784| 14.7k| zxc_store_le64(p, desc[i].sizes); 785| 14.7k| p += ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 14.7k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 786| 14.7k| } 787| | 788| 4.90k| return (int)needed; 789| 4.90k|} zxc_read_ghi_header_and_desc: 802| 4.13k| zxc_section_desc_t desc[ZXC_GHI_SECTIONS]) { 803| 4.13k| const size_t needed = 804| 4.13k| ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 403| 4.13k|#define ZXC_GHI_HEADER_BINARY_SIZE 16 ------------------ ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 421| 4.13k|#define ZXC_GHI_SECTIONS 3 ------------------ ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 4.13k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 805| | 806| 4.13k| if (UNLIKELY(len < needed)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 4.13k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.13k] | | ------------------ ------------------ 807| | 808| 4.13k| gh->n_sequences = zxc_le32(src); 809| 4.13k| gh->n_literals = zxc_le32(src + 4); 810| 4.13k| gh->enc_lit = src[8]; 811| 4.13k| gh->enc_litlen = src[9]; 812| 4.13k| gh->enc_mlen = src[10]; 813| 4.13k| gh->enc_off = src[11]; 814| | 815| 4.13k| const uint8_t* p = src + ZXC_GHI_HEADER_BINARY_SIZE; ------------------ | | 403| 4.13k|#define ZXC_GHI_HEADER_BINARY_SIZE 16 ------------------ 816| | 817| 16.5k| for (int i = 0; i < ZXC_GHI_SECTIONS; i++) { ------------------ | | 421| 16.5k|#define ZXC_GHI_SECTIONS 3 ------------------ | Branch (817:21): [True: 12.4k, False: 4.13k] ------------------ 818| 12.4k| desc[i].sizes = zxc_le64(p); 819| 12.4k| p += ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 12.4k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 820| 12.4k| } 821| 4.13k| return ZXC_OK; 822| 4.13k|} zxc_compress_bound: 842| 4.60k|uint64_t zxc_compress_bound(const size_t input_size) { 843| | // Guard against uint64 overflow when summing per-block overhead 844| | // across very large inputs (input_size approaching SIZE_MAX). 845| 4.60k| if (UNLIKELY(input_size > (SIZE_MAX - (SIZE_MAX >> 8)))) return 0; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 846| 4.60k| uint64_t n = ((uint64_t)input_size + ZXC_BLOCK_SIZE_MIN - 1) / ZXC_BLOCK_SIZE_MIN; ------------------ | | 62| 4.60k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 4.60k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ uint64_t n = ((uint64_t)input_size + ZXC_BLOCK_SIZE_MIN - 1) / ZXC_BLOCK_SIZE_MIN; ------------------ | | 62| 4.60k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 4.60k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ 847| 4.60k| if (n == 0) n = 1; ------------------ | Branch (847:9): [True: 0, False: 4.60k] ------------------ 848| 4.60k| return ZXC_FILE_HEADER_SIZE + ------------------ | | 103| 4.60k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ 849| 4.60k| (n * (ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_CHECKSUM_SIZE + ZXC_BLOCK_FORMAT_OVERHEAD)) + ------------------ | | 397| 4.60k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ (n * (ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_CHECKSUM_SIZE + ZXC_BLOCK_FORMAT_OVERHEAD)) + ------------------ | | 399| 4.60k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ (n * (ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_CHECKSUM_SIZE + ZXC_BLOCK_FORMAT_OVERHEAD)) + ------------------ | | 412| 4.60k|#define ZXC_BLOCK_FORMAT_OVERHEAD 64 ------------------ 850| 4.60k| (uint64_t)input_size + ZXC_BLOCK_HEADER_SIZE + /* EOF block */ ------------------ | | 397| 4.60k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 851| 4.60k| ZXC_BLOCK_HEADER_SIZE + /* SEK block header (seekable) */ ------------------ | | 397| 4.60k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 852| 4.60k| (n * ZXC_SEEK_ENTRY_SIZE) + /* SEK entries: 4 bytes per block */ ------------------ | | 443| 4.60k|#define ZXC_SEEK_ENTRY_SIZE 4 ------------------ 853| 4.60k| ZXC_FILE_FOOTER_SIZE; ------------------ | | 105| 4.60k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 854| 4.60k|} zxc_compress_block_bound: 865| 9.51k|uint64_t zxc_compress_block_bound(const size_t input_size) { 866| | // Mirrors the Block API contract: outside [1, ZXC_BLOCK_SIZE_MAX] the call 867| | // would fail anyway, so the bound is undefined and 0 says "unusable". The cap 868| | // also makes the addition below trivially overflow-free. 869| 9.51k| if (UNLIKELY(input_size == 0 || input_size > ZXC_BLOCK_SIZE_MAX)) return 0; ------------------ | | 151| 19.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.51k] | | | Branch (151:42): [True: 0, False: 9.51k] | | | Branch (151:42): [True: 0, False: 9.51k] | | ------------------ ------------------ 870| | // Outer block header + payload (worst case: incompressible, raw bytes) 871| | // + inner format overhead + optional checksum. 872| 9.51k| return (uint64_t)ZXC_BLOCK_HEADER_SIZE + (uint64_t)input_size + ZXC_BLOCK_FORMAT_OVERHEAD + ------------------ | | 397| 9.51k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ return (uint64_t)ZXC_BLOCK_HEADER_SIZE + (uint64_t)input_size + ZXC_BLOCK_FORMAT_OVERHEAD + ------------------ | | 412| 9.51k|#define ZXC_BLOCK_FORMAT_OVERHEAD 64 ------------------ 873| 9.51k| ZXC_BLOCK_CHECKSUM_SIZE; ------------------ | | 399| 9.51k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ 874| 9.51k|} zxc_common.c:compute_cctx_layout: 163| 18.4k| const int defer_entropy_scratch) { 164| 18.4k| zxc_cctx_layout_t layout = {0}; 165| | 166| 18.4k| const size_t max_seq = zxc_cctx_max_seq(chunk_size); 167| | 168| 18.4k| if (mode == 0) { ------------------ | Branch (168:9): [True: 9.21k, False: 9.21k] ------------------ 169| | /* Decompress: work_buf + lit_buffer, padded for wild-copy overshoot and 170| | * sized worst-case. lit_buffer is provisioned at every level - the decoder 171| | * cannot predict a block's literal encoding (RAW / RLE / HUFFMAN). */ 172| 9.21k| const size_t sz_work = chunk_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 349| 9.21k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 9.21k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 173| 9.21k| const size_t sz_lit = chunk_size + ZXC_PAD_SIZE; ------------------ | | 337| 9.21k|#define ZXC_PAD_SIZE 32 ------------------ 174| | 175| 9.21k| layout.off_work = layout.total; 176| 9.21k| layout.total += ZXC_ALIGN_CL(sz_work); ------------------ | | 355| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 177| 9.21k| layout.off_lit_dctx = layout.total; 178| 9.21k| layout.total += ZXC_ALIGN_CL(sz_lit); ------------------ | | 355| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 179| | /* Token-section decode scratch (level-7 GLO Huffman-codes the tokens) + 180| | * PivCo ping-pong scratch. enc_lit/enc_litlen are unpredictable, so static 181| | * workspaces provision both up front (no-alloc contract); heap contexts 182| | * defer them to the first entropy section, sparing L1-5 archives ~1.2x 183| | * chunk_size. See zxc_cctx_alloc_entropy_scratch. */ 184| 9.21k| if (!defer_entropy_scratch) { ------------------ | Branch (184:13): [True: 0, False: 9.21k] ------------------ 185| 0| size_t sz_tok = 0; 186| 0| size_t sz_pivco = 0; 187| 0| zxc_dctx_entropy_sizes(chunk_size, &sz_tok, &sz_pivco); 188| 0| layout.sz_tok_dctx = sz_tok; 189| 0| layout.off_tok_dctx = layout.total; 190| 0| layout.total += ZXC_ALIGN_CL(layout.sz_tok_dctx); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 191| 0| layout.sz_pivco_dctx = sz_pivco; 192| 0| layout.off_pivco_dctx = layout.total; 193| 0| layout.total += ZXC_ALIGN_CL(layout.sz_pivco_dctx); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 194| 0| } 195| 9.21k| } else { 196| | /* Compress: 6 partitions + optional opt_scratch at level >= ZXC_LEVEL_DENSITY. */ 197| 9.21k| const uint32_t offset_bits = zxc_log2_u32((uint32_t)chunk_size); 198| 9.21k| layout.max_seq = max_seq; 199| 9.21k| layout.sz_hash_pos = ZXC_LZ_HASH_SIZE * sizeof(uint32_t); ------------------ | | 502| 9.21k|#define ZXC_LZ_HASH_SIZE (1U << ZXC_LZ_HASH_BITS) | | ------------------ | | | | 496| 9.21k|#define ZXC_LZ_HASH_BITS 15 | | ------------------ ------------------ 200| 9.21k| layout.sz_hash_tags = ZXC_LZ_HASH_SIZE * sizeof(uint8_t); ------------------ | | 502| 9.21k|#define ZXC_LZ_HASH_SIZE (1U << ZXC_LZ_HASH_BITS) | | ------------------ | | | | 496| 9.21k|#define ZXC_LZ_HASH_BITS 15 | | ------------------ ------------------ 201| 9.21k| const size_t sz_chain = ZXC_LZ_WINDOW_SIZE * sizeof(uint16_t); ------------------ | | 504| 9.21k|#define ZXC_LZ_WINDOW_SIZE (1U << 16) ------------------ 202| | /* buf_sequences (GHI, level <= ZXC_LEVEL_FAST) aliases buf_offsets + buf_tokens (GLO, 203| | * level >= ZXC_LEVEL_DEFAULT). Mutually exclusive per block; sized for the larger. */ 204| 9.21k| const size_t sz_seq_union = layout.max_seq * sizeof(uint32_t); 205| 9.21k| const size_t vbyte_len = (offset_bits + 6) / 7; 206| 9.21k| const size_t sz_extras = layout.max_seq * 2 * vbyte_len; 207| 9.21k| const size_t sz_lit = chunk_size + ZXC_PAD_SIZE; ------------------ | | 337| 9.21k|#define ZXC_PAD_SIZE 32 ------------------ 208| | 209| | /* opt_scratch (level >= ZXC_LEVEL_DENSITY): the optimal parser's DP arrays, 210| | * reused transiently as package-merge scratch by the code-length builder, 211| | * so sized to the larger demand. Keep in sync with zxc_estimate_cctx_size() 212| | * and its consumer in zxc_compress.c. */ 213| 9.21k| if (level >= ZXC_LEVEL_DENSITY) { ------------------ | Branch (213:13): [True: 0, False: 9.21k] ------------------ 214| 0| const size_t sz_dp = ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint32_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 215| 0| const size_t sz_pl = ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint16_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 216| 0| const size_t sz_po = ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint16_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 217| 0| const size_t n_bm_words = ZXC_BITMAP_WORDS(chunk_size + 1); ------------------ | | 362| 0|#define ZXC_BITMAP_WORDS(n_bits) (((n_bits) + 63) / 64) ------------------ 218| 0| const size_t sz_bm = ZXC_ALIGN_CL(n_bm_words * sizeof(uint64_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 219| 0| const size_t dp_needed = sz_dp + sz_pl + sz_po + sz_bm; 220| 0| layout.sz_opt = 221| 0| (dp_needed > ZXC_HUF_BUILD_SCRATCH_SIZE) ? dp_needed : ZXC_HUF_BUILD_SCRATCH_SIZE; ------------------ | | 841| 0| ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 834| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 589| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 842| 0| sizeof(zxc_huf_pm_item_t) + \ | | 843| 0| 8U + (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * sizeof(int) + 8U + \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | 844| 0| (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 834| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 589| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 845| 0| sizeof(zxc_huf_pm_frame_t)) ------------------ (dp_needed > ZXC_HUF_BUILD_SCRATCH_SIZE) ? dp_needed : ZXC_HUF_BUILD_SCRATCH_SIZE; ------------------ | | 841| 0| ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 834| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 589| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 842| 0| sizeof(zxc_huf_pm_item_t) + \ | | 843| 0| 8U + (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * sizeof(int) + 8U + \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | 844| 0| (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 834| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 589| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 845| 0| sizeof(zxc_huf_pm_frame_t)) ------------------ | Branch (221:17): [True: 0, False: 0] ------------------ 222| 0| } 223| | 224| 9.21k| layout.off_hash_pos = layout.total; 225| 9.21k| layout.total += ZXC_ALIGN_CL(layout.sz_hash_pos); ------------------ | | 355| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 226| 9.21k| layout.off_hash_tags = layout.total; 227| 9.21k| layout.total += ZXC_ALIGN_CL(layout.sz_hash_tags); ------------------ | | 355| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 228| 9.21k| layout.off_chain = layout.total; 229| 9.21k| layout.total += ZXC_ALIGN_CL(sz_chain); ------------------ | | 355| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 230| 9.21k| layout.off_seq_union = layout.total; 231| 9.21k| layout.total += ZXC_ALIGN_CL(sz_seq_union); ------------------ | | 355| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 232| 9.21k| layout.off_extras = layout.total; 233| 9.21k| layout.total += ZXC_ALIGN_CL(sz_extras); ------------------ | | 355| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 234| 9.21k| layout.off_lit_cctx = layout.total; 235| 9.21k| layout.total += ZXC_ALIGN_CL(sz_lit); ------------------ | | 355| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 236| | /* opt_scratch is appended last so it is absent for levels 1..5 (zero 237| | * waste on the common path) and only inflates the workspace at level 6. */ 238| 9.21k| if (layout.sz_opt) { ------------------ | Branch (238:13): [True: 0, False: 9.21k] ------------------ 239| 0| layout.off_opt = layout.total; 240| 0| layout.total += ZXC_ALIGN_CL(layout.sz_opt); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 241| 0| } 242| 9.21k| } 243| | 244| | /* [dict | data] concat scratch (dict only). Compress chunk_size already 245| | * spans [dict | block]; decompress prepends dict to a (chunk + PAD) region. */ 246| 18.4k| if (dict_size > 0) { ------------------ | Branch (246:9): [True: 0, False: 18.4k] ------------------ 247| 0| layout.sz_dict = (mode == 1) ? (chunk_size + ZXC_DECOMPRESS_TAIL_PAD) ------------------ | | 349| 0|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 0|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ | Branch (247:26): [True: 0, False: 0] ------------------ 248| 0| : (dict_size + chunk_size + ZXC_DECOMPRESS_TAIL_PAD); ------------------ | | 349| 0|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 0|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 249| 0| layout.off_dict = layout.total; 250| 0| layout.total += ZXC_ALIGN_CL(layout.sz_dict); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 251| 0| layout.off_dict_huf = layout.total; 252| 0| layout.total += ZXC_ALIGN_CL(sizeof(zxc_dict_huf_state_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 253| 0| } 254| 18.4k| return layout; 255| 18.4k|} zxc_common.c:zxc_cctx_max_seq: 144| 18.4k|static ZXC_ALWAYS_INLINE size_t zxc_cctx_max_seq(const size_t chunk_size) { 145| 18.4k| return chunk_size / ZXC_LZ_MIN_MATCH_LEN + 16; ------------------ | | 508| 18.4k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 146| 18.4k|} zxc_compress_chunk_wrapper_default: 2080| 4.90k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 2081| 4.90k| const size_t dict_sz = ctx->dict_size; 2082| 4.90k| const size_t block_sz = src_sz - dict_sz; 2083| 4.90k| const uint8_t* block_data = chunk + dict_sz; 2084| 4.90k| size_t w = 0; 2085| 4.90k| int res = ZXC_OK; 2086| | 2087| 4.90k| if (ctx->compression_level <= 2) ------------------ | Branch (2087:9): [True: 4.90k, False: 0] ------------------ 2088| 4.90k| res = zxc_encode_block_ghi(ctx, chunk, src_sz, dst, dst_cap, &w); 2089| 0| else 2090| 0| res = zxc_encode_block_glo(ctx, chunk, src_sz, dst, dst_cap, &w); 2091| | 2092| | // Check expansion against block data size (excluding dict prefix). 2093| 4.90k| if (UNLIKELY(res != ZXC_OK || w >= block_sz)) { ------------------ | | 151| 9.81k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 775, False: 4.13k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 775, False: 4.13k] | | ------------------ ------------------ 2094| 775| res = zxc_encode_block_raw(block_data, block_sz, dst, dst_cap, &w); 2095| 775| if (UNLIKELY(res != ZXC_OK)) return res; ------------------ | | 151| 775|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 775] | | ------------------ ------------------ 2096| 775| } 2097| | 2098| 4.90k| if (ctx->checksum_enabled) { ------------------ | Branch (2098:9): [True: 1.63k, False: 3.27k] ------------------ 2099| | // Calculate checksum on the compressed payload (w currently excludes checksum) 2100| | // Header is at dst, data starts at dst + ZXC_BLOCK_HEADER_SIZE 2101| 1.63k| if (UNLIKELY(w < ZXC_BLOCK_HEADER_SIZE || w + ZXC_BLOCK_CHECKSUM_SIZE > dst_cap)) ------------------ | | 151| 3.27k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.63k] | | | Branch (151:42): [True: 0, False: 1.63k] | | | Branch (151:42): [True: 0, False: 1.63k] | | ------------------ ------------------ 2102| 0| return ZXC_ERROR_OVERFLOW; 2103| | 2104| 1.63k| uint32_t payload_sz = (uint32_t)(w - ZXC_BLOCK_HEADER_SIZE); ------------------ | | 397| 1.63k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2105| 1.63k| uint32_t crc = 2106| 1.63k| zxc_checksum(dst + ZXC_BLOCK_HEADER_SIZE, payload_sz, ZXC_CHECKSUM_RAPIDHASH); ------------------ | | 397| 1.63k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ zxc_checksum(dst + ZXC_BLOCK_HEADER_SIZE, payload_sz, ZXC_CHECKSUM_RAPIDHASH); ------------------ | | 424| 1.63k|#define ZXC_CHECKSUM_RAPIDHASH 0 ------------------ 2107| 1.63k| zxc_store_le32(dst + w, crc); 2108| 1.63k| w += ZXC_BLOCK_CHECKSUM_SIZE; ------------------ | | 399| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ 2109| 1.63k| } 2110| | 2111| 4.90k| return (int)w; 2112| 4.90k|} zxc_compress.c:zxc_encode_block_ghi: 1860| 4.90k| size_t* RESTRICT const out_sz) { 1861| 4.90k| const int level = ctx->compression_level; 1862| 4.90k| const size_t dict_sz = ctx->dict_size; 1863| | 1864| 4.90k| const zxc_lz77_params_t lzp = zxc_get_lz77_params(level); 1865| | 1866| 4.90k| ctx->epoch++; 1867| 4.90k| if (UNLIKELY(ctx->epoch >= ctx->max_epoch)) { ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1868| 0| ZXC_MEMSET(ctx->hash_table, 0, ZXC_LZ_HASH_SIZE * sizeof(uint32_t)); ------------------ | | 178| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 1869| 0| ZXC_MEMSET(ctx->hash_tags, 0, ZXC_LZ_HASH_SIZE * sizeof(uint8_t)); ------------------ | | 178| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 1870| 0| ctx->epoch = 1; 1871| 0| } 1872| 4.90k| const uint32_t offset_bits = ctx->offset_bits; 1873| 4.90k| const uint32_t offset_mask = ctx->offset_mask; 1874| 4.90k| const uint32_t epoch_mark = ctx->epoch << offset_bits; 1875| | 1876| 4.90k| if (dict_sz > 0) ------------------ | Branch (1876:9): [True: 0, False: 4.90k] ------------------ 1877| 0| zxc_lz_seed_dict(src, dict_sz, ctx->hash_table, ctx->hash_tags, ctx->chain_table, 1878| 0| epoch_mark, offset_mask, level); 1879| | 1880| 4.90k| const uint8_t* ip = src + dict_sz; 1881| 4.90k| const uint8_t* iend = src + src_sz; 1882| 4.90k| const uint8_t* anchor = ip; 1883| 4.90k| const uint8_t* search_limit = iend - ZXC_LZ_SEARCH_MARGIN; ------------------ | | 532| 4.90k|#define ZXC_LZ_SEARCH_MARGIN (sizeof(uint64_t)) ------------------ 1884| | 1885| 4.90k| uint32_t* const hash_table = ctx->hash_table; 1886| 4.90k| uint8_t* const hash_tags = ctx->hash_tags; 1887| 4.90k| uint8_t* const buf_extras = ctx->buf_extras; 1888| 4.90k| uint16_t* const chain_table = ctx->chain_table; 1889| 4.90k| uint8_t* const literals = ctx->literals; 1890| 4.90k| uint32_t* const buf_sequences = ctx->buf_sequences; 1891| | 1892| 4.90k| uint32_t seq_c = 0; 1893| 4.90k| size_t extras_c = 0; 1894| 4.90k| size_t lit_c = 0; 1895| | 1896| 7.98M| while (LIKELY(ip < search_limit)) { ------------------ | | 145| 7.98M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 7.97M, False: 4.90k] | | ------------------ ------------------ 1897| 7.97M| size_t dist = (size_t)(ip - anchor); 1898| 7.97M| size_t step = lzp.step_base + (dist >> lzp.step_shift); 1899| 7.97M| if (UNLIKELY(ip + step >= search_limit)) step = 1; ------------------ | | 151| 7.97M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.10M, False: 6.87M] | | ------------------ ------------------ 1900| | 1901| 7.97M| ZXC_PREFETCH_READ(ip + step * 4 + ZXC_CACHE_LINE_SIZE); ------------------ | | 162| 7.97M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 1902| | 1903| 7.97M| if (LIKELY(ip + step + sizeof(uint64_t) <= iend)) { ------------------ | | 145| 7.97M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 7.97M, False: 0] | | ------------------ ------------------ 1904| 7.97M| const uint64_t v_next = zxc_le64(ip + step); 1905| | // cppcheck-suppress unreadVariable 1906| 7.97M| const uint32_t h_next = zxc_hash_func(v_next, 0); 1907| 7.97M| ZXC_PREFETCH_READ(&hash_tags[h_next]); ------------------ | | 162| 7.97M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 1908| 7.97M| ZXC_PREFETCH_READ(&hash_table[h_next]); ------------------ | | 162| 7.97M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 1909| 7.97M| } 1910| | 1911| 7.97M| const zxc_match_t m = 1912| 7.97M| zxc_lz77_find_best_match(src, ip, iend, search_limit, anchor, hash_table, hash_tags, 1913| 7.97M| chain_table, epoch_mark, offset_mask, level, lzp, 1914| 7.97M| /*last_off=*/0U); 1915| | 1916| 7.97M| if (m.ref) { ------------------ | Branch (1916:13): [True: 2.62M, False: 5.34M] ------------------ 1917| 2.62M| ip -= m.backtrack; 1918| 2.62M| const uint32_t ll = (uint32_t)(ip - anchor); 1919| 2.62M| const uint32_t ml = m.len - ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 2.62M|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1920| 2.62M| const uint32_t off = (uint32_t)(ip - m.ref); 1921| | 1922| 2.62M| if (ll > 0) { ------------------ | Branch (1922:17): [True: 1.09M, False: 1.53M] ------------------ 1923| 1.09M| if (LIKELY(anchor + ZXC_PAD_SIZE <= iend)) { ------------------ | | 145| 1.09M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 1.09M, False: 1.04k] | | ------------------ ------------------ 1924| 1.09M| zxc_copy32(literals + lit_c, anchor); 1925| 1.09M| if (UNLIKELY(ll > ZXC_PAD_SIZE)) { ------------------ | | 151| 1.09M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 114k, False: 976k] | | ------------------ ------------------ 1926| 114k| ZXC_MEMCPY(literals + lit_c + ZXC_PAD_SIZE, anchor + ZXC_PAD_SIZE, ------------------ | | 173| 114k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1927| 114k| ll - ZXC_PAD_SIZE); 1928| 114k| } 1929| 1.09M| } else { 1930| 1.04k| ZXC_MEMCPY(literals + lit_c, anchor, ll); ------------------ | | 173| 1.04k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1931| 1.04k| } 1932| 1.09M| lit_c += ll; 1933| 1.09M| } 1934| | 1935| 2.62M| const uint32_t ll_write = (ll >= ZXC_SEQ_LL_MASK) ? 255U : ll; ------------------ | | 469| 2.62M|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 463| 2.62M|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ | Branch (1935:39): [True: 15.6k, False: 2.61M] ------------------ 1936| 2.62M| const uint32_t ml_write = (ml >= ZXC_SEQ_ML_MASK) ? 255U : ml; ------------------ | | 471| 2.62M|#define ZXC_SEQ_ML_MASK ((1U << ZXC_SEQ_ML_BITS) - 1) | | ------------------ | | | | 465| 2.62M|#define ZXC_SEQ_ML_BITS 8 | | ------------------ ------------------ | Branch (1936:39): [True: 89.0k, False: 2.53M] ------------------ 1937| 2.62M| const uint32_t seq_val = (ll_write << (ZXC_SEQ_ML_BITS + ZXC_SEQ_OFF_BITS)) | ------------------ | | 465| 2.62M|#define ZXC_SEQ_ML_BITS 8 ------------------ const uint32_t seq_val = (ll_write << (ZXC_SEQ_ML_BITS + ZXC_SEQ_OFF_BITS)) | ------------------ | | 467| 2.62M|#define ZXC_SEQ_OFF_BITS 16 ------------------ 1938| 2.62M| (ml_write << ZXC_SEQ_OFF_BITS) | ------------------ | | 467| 2.62M|#define ZXC_SEQ_OFF_BITS 16 ------------------ 1939| 2.62M| ((off - ZXC_LZ_OFFSET_BIAS) & ZXC_SEQ_OFF_MASK); ------------------ | | 527| 2.62M|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ ((off - ZXC_LZ_OFFSET_BIAS) & ZXC_SEQ_OFF_MASK); ------------------ | | 473| 2.62M|#define ZXC_SEQ_OFF_MASK ((1U << ZXC_SEQ_OFF_BITS) - 1) | | ------------------ | | | | 467| 2.62M|#define ZXC_SEQ_OFF_BITS 16 | | ------------------ ------------------ 1940| 2.62M| buf_sequences[seq_c] = seq_val; 1941| 2.62M| seq_c++; 1942| | 1943| 2.62M| if (ll >= ZXC_SEQ_LL_MASK) { ------------------ | | 469| 2.62M|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 463| 2.62M|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ | Branch (1943:17): [True: 15.6k, False: 2.61M] ------------------ 1944| 15.6k| const size_t n = zxc_write_varint(buf_extras + extras_c, ll - ZXC_SEQ_LL_MASK); ------------------ | | 469| 15.6k|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 463| 15.6k|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ 1945| 15.6k| if (UNLIKELY(n == 0)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 15.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 15.6k] | | ------------------ ------------------ 1946| 15.6k| extras_c += n; 1947| 15.6k| } 1948| 2.62M| if (ml >= ZXC_SEQ_ML_MASK) { ------------------ | | 471| 2.62M|#define ZXC_SEQ_ML_MASK ((1U << ZXC_SEQ_ML_BITS) - 1) | | ------------------ | | | | 465| 2.62M|#define ZXC_SEQ_ML_BITS 8 | | ------------------ ------------------ | Branch (1948:17): [True: 89.0k, False: 2.53M] ------------------ 1949| 89.0k| const size_t n = zxc_write_varint(buf_extras + extras_c, ml - ZXC_SEQ_ML_MASK); ------------------ | | 471| 89.0k|#define ZXC_SEQ_ML_MASK ((1U << ZXC_SEQ_ML_BITS) - 1) | | ------------------ | | | | 465| 89.0k|#define ZXC_SEQ_ML_BITS 8 | | ------------------ ------------------ 1950| 89.0k| if (UNLIKELY(n == 0)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 89.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 89.0k] | | ------------------ ------------------ 1951| 89.0k| extras_c += n; 1952| 89.0k| } 1953| | 1954| 2.62M| ip += m.len; 1955| 2.62M| anchor = ip; 1956| 5.34M| } else { 1957| 5.34M| ip += step; 1958| 5.34M| } 1959| 7.97M| } 1960| | 1961| 4.90k| const size_t last_lits = iend - anchor; 1962| 4.90k| if (last_lits > 0) { ------------------ | Branch (1962:9): [True: 4.05k, False: 854] ------------------ 1963| 4.05k| ZXC_MEMCPY(literals + lit_c, anchor, last_lits); ------------------ | | 173| 4.05k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1964| 4.05k| lit_c += last_lits; 1965| 4.05k| } 1966| | 1967| 4.90k| zxc_block_header_t bh = {.block_type = ZXC_BLOCK_GHI}; 1968| 4.90k| uint8_t* const p = dst + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1969| 4.90k| size_t rem = dst_cap - ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1970| | 1971| 4.90k| const zxc_gnr_header_t gh = {.n_sequences = seq_c, 1972| 4.90k| .n_literals = (uint32_t)lit_c, 1973| 4.90k| .enc_lit = ZXC_SECTION_ENCODING_RAW, 1974| 4.90k| .enc_litlen = 0, 1975| 4.90k| .enc_mlen = 0, 1976| 4.90k| .enc_off = 0}; 1977| | 1978| 4.90k| zxc_section_desc_t desc[ZXC_GHI_SECTIONS] = {0}; 1979| 4.90k| desc[0].sizes = (uint64_t)lit_c | ((uint64_t)lit_c << 32); 1980| 4.90k| size_t sz_seqs = seq_c * sizeof(uint32_t); 1981| 4.90k| desc[1].sizes = (uint64_t)sz_seqs | ((uint64_t)sz_seqs << 32); 1982| 4.90k| desc[2].sizes = (uint64_t)extras_c | ((uint64_t)extras_c << 32); 1983| | 1984| 4.90k| const int ghs = zxc_write_ghi_header_and_desc(p, rem, &gh, desc); 1985| 4.90k| if (UNLIKELY(ghs < 0)) return ghs; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1986| | 1987| 4.90k| uint8_t* p_curr = p + ghs; 1988| 4.90k| rem -= ghs; 1989| | 1990| 4.90k| const size_t sz_lit = (size_t)(desc[0].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 4.90k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 1991| 4.90k| const size_t sz_seq = (size_t)(desc[1].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 4.90k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 1992| 4.90k| const size_t sz_ext = (size_t)(desc[2].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 4.90k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 1993| | 1994| 4.90k| if (UNLIKELY(rem < sz_lit + sz_seq + sz_ext)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1995| | 1996| 4.90k| ZXC_MEMCPY(p_curr, literals, lit_c); ------------------ | | 173| 4.90k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1997| 4.90k| p_curr += lit_c; 1998| 4.90k| rem -= sz_lit; 1999| | 2000| 4.90k| if (UNLIKELY(rem < sz_seq)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 2001| | // Write sequences in little-endian order 2002| |#ifdef ZXC_BIG_ENDIAN 2003| | for (uint32_t i = 0; i < seq_c; i++) { 2004| | zxc_store_le32(p_curr, buf_sequences[i]); 2005| | p_curr += sizeof(uint32_t); 2006| | } 2007| |#else 2008| 4.90k| ZXC_MEMCPY(p_curr, buf_sequences, sz_seq); ------------------ | | 173| 4.90k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 2009| 4.90k| p_curr += sz_seq; 2010| 4.90k|#endif 2011| | 2012| | // --- WRITE EXTRAS --- 2013| 4.90k| ZXC_MEMCPY(p_curr, buf_extras, sz_ext); ------------------ | | 173| 4.90k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 2014| 4.90k| p_curr += sz_ext; 2015| | 2016| 4.90k| bh.comp_size = (uint32_t)(p_curr - (dst + ZXC_BLOCK_HEADER_SIZE)); ------------------ | | 397| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2017| 4.90k| const int hw = zxc_write_block_header(dst, dst_cap, &bh); 2018| 4.90k| if (UNLIKELY(hw < 0)) return hw; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 2019| | 2020| | // Checksum will be appended by the wrapper 2021| 4.90k| *out_sz = ZXC_BLOCK_HEADER_SIZE + bh.comp_size; ------------------ | | 397| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2022| 4.90k| return ZXC_OK; 2023| 4.90k|} zxc_compress.c:zxc_hash_func: 47| 15.9M|static ZXC_ALWAYS_INLINE uint32_t zxc_hash_func(const uint64_t val, const int use_hash5) { 48| 15.9M| if (use_hash5) { ------------------ | Branch (48:9): [True: 0, False: 15.9M] ------------------ 49| 0| const uint64_t v5 = val & 0xFFFFFFFFFFULL; 50| 0| return (uint32_t)((v5 * ZXC_LZ_HASH_PRIME2) >> (64 - ZXC_LZ_HASH_BITS)); ------------------ | | 500| 0|#define ZXC_LZ_HASH_PRIME2 0x2545F4914F6CDD1DULL ------------------ return (uint32_t)((v5 * ZXC_LZ_HASH_PRIME2) >> (64 - ZXC_LZ_HASH_BITS)); ------------------ | | 496| 0|#define ZXC_LZ_HASH_BITS 15 ------------------ 51| 15.9M| } else { 52| 15.9M| const uint64_t v4 = val ^ (val >> 15); 53| 15.9M| return ((uint32_t)v4 * ZXC_LZ_HASH_PRIME1) >> (32 - ZXC_LZ_HASH_BITS); ------------------ | | 498| 15.9M|#define ZXC_LZ_HASH_PRIME1 0x2D35182DU ------------------ return ((uint32_t)v4 * ZXC_LZ_HASH_PRIME1) >> (32 - ZXC_LZ_HASH_BITS); ------------------ | | 496| 15.9M|#define ZXC_LZ_HASH_BITS 15 ------------------ 54| 15.9M| } 55| 15.9M|} zxc_compress.c:zxc_lz77_find_best_match: 193| 7.97M| const int level, const zxc_lz77_params_t p, const uint32_t last_off) { 194| 7.97M| const int use_hash5 = (level >= 3); 195| | // Sentinel length just below the minimum, so any valid match replaces it. 196| 7.97M| zxc_match_t best = (zxc_match_t){NULL, ZXC_LZ_MIN_MATCH_LEN - 1, 0}; ------------------ | | 508| 7.97M|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 197| | 198| 7.97M| uint64_t cur_val8 = zxc_le64(ip); 199| 7.97M| uint32_t cur_val = (uint32_t)cur_val8; 200| 7.97M| uint32_t h = zxc_hash_func(cur_val8, use_hash5); 201| | 202| | // 8-bit tag: XOR fold of first 4 bytes for fast rejection 203| 7.97M| const uint8_t cur_tag = (uint8_t)(cur_val ^ (cur_val >> 16)); 204| | 205| 7.97M| const uint32_t cur_pos = (uint32_t)(ip - src); 206| | 207| | // Tag-first filter on fast levels. 208| 7.97M| const uint8_t stored_tag = hash_tags[h]; 209| 7.97M| uint32_t match_idx; 210| 7.97M| if (level <= ZXC_LEVEL_FAST && stored_tag != cur_tag) { ------------------ | Branch (210:9): [True: 7.97M, False: 0] | Branch (210:36): [True: 4.20M, False: 3.77M] ------------------ 211| 4.20M| match_idx = 0; 212| 4.20M| } else { 213| 3.77M| const uint32_t raw_head = hash_table[h]; 214| 3.77M| match_idx = ((raw_head & ~offset_mask) == epoch_mark) ? (raw_head & offset_mask) : 0; ------------------ | Branch (214:21): [True: 3.21M, False: 560k] ------------------ 215| 3.77M| } 216| | 217| | // skip_head still drives the chain walk on level >= 3 (advances past the 218| | // mismatched head without comparing). On level <= 2 it is always 0 here: 219| | // either match_idx == 0 (filter-skip) or stored_tag == cur_tag. 220| 7.97M| const int skip_head = (match_idx != 0) & (stored_tag != cur_tag); 221| | 222| | // Split table writes 223| 7.97M| hash_table[h] = epoch_mark | cur_pos; 224| 7.97M| hash_tags[h] = cur_tag; 225| | 226| | // Branchless chain table update 227| 7.97M| const uint32_t dist = cur_pos - match_idx; 228| 7.97M| const uint32_t valid_mask = -((int32_t)((match_idx != 0) & (dist < ZXC_LZ_WINDOW_SIZE))); ------------------ | | 504| 7.97M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) ------------------ 229| 7.97M| chain_table[cur_pos & ZXC_LZ_WINDOW_MASK] = (uint16_t)(dist & valid_mask); ------------------ | | 506| 7.97M|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 504| 7.97M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 230| | 231| 7.97M| int attempts = p.search_depth; 232| | 233| | /* Repeat-offset seed (level-6 parser passes last_off; others pass 0). 234| | * Probing the previous offset first often finds the longest match right 235| | * away, speeding up the chain walk. It only raises best.len, never lowers 236| | * it, so the result is unchanged - this is purely a speed optimization. */ 237| 7.97M| if (last_off != 0U && last_off <= (uint32_t)ZXC_LZ_MAX_DIST && last_off <= cur_pos) { ------------------ | | 529| 7.97M|#define ZXC_LZ_MAX_DIST (ZXC_LZ_WINDOW_SIZE - 1) | | ------------------ | | | | 504| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ | Branch (237:9): [True: 0, False: 7.97M] | Branch (237:27): [True: 0, False: 0] | Branch (237:68): [True: 0, False: 0] ------------------ 238| 0| const uint8_t* const rep_ref = src + (cur_pos - last_off); 239| 0| if (zxc_le32(rep_ref) == cur_val) { ------------------ | Branch (239:13): [True: 0, False: 0] ------------------ 240| 0| uint32_t mlen = sizeof(uint32_t); 241| 0| const uint8_t* const limit_8 = iend - sizeof(uint64_t); 242| 0| while (ip + mlen < limit_8) { ------------------ | Branch (242:20): [True: 0, False: 0] ------------------ 243| 0| const uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(rep_ref + mlen); 244| 0| if (diff == 0) { ------------------ | Branch (244:21): [True: 0, False: 0] ------------------ 245| 0| mlen += sizeof(uint64_t); 246| 0| } else { 247| 0| mlen += (uint32_t)(zxc_ctz64(diff) >> 3); 248| 0| goto _rep_done; 249| 0| } 250| 0| } 251| 0| while (ip + mlen < iend && rep_ref[mlen] == ip[mlen]) mlen++; ------------------ | Branch (251:20): [True: 0, False: 0] | Branch (251:40): [True: 0, False: 0] ------------------ 252| 0| _rep_done:; 253| 0| best.len = mlen; 254| 0| best.ref = rep_ref; 255| 0| if (UNLIKELY(best.len >= (uint32_t)p.sufficient_len || ip + best.len >= iend)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 256| 0| goto _finalize_match; 257| 0| } 258| 0| } 259| | 260| 7.97M| if (match_idx == 0) goto _finalize_match; ------------------ | Branch (260:9): [True: 4.76M, False: 3.21M] ------------------ 261| | 262| | // Optimization: If head tag doesn't match, advance immediately without loading the first 263| | // mismatch. 264| 3.21M| if (skip_head) { ------------------ | Branch (264:9): [True: 0, False: 3.21M] ------------------ 265| 0| const uint16_t delta = chain_table[match_idx & ZXC_LZ_WINDOW_MASK]; ------------------ | | 506| 0|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 504| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 266| 0| const uint32_t next_idx = match_idx - delta; 267| 0| match_idx = (delta != 0) ? next_idx : 0; ------------------ | Branch (267:21): [True: 0, False: 0] ------------------ 268| 0| attempts--; 269| 0| } 270| | 271| 8.99M| while (match_idx > 0) { ------------------ | Branch (271:12): [True: 8.65M, False: 345k] ------------------ 272| 8.65M| if (UNLIKELY(attempts-- < 0 || cur_pos - match_idx > ZXC_LZ_MAX_DIST)) break; ------------------ | | 151| 16.1M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.77M, False: 6.87M] | | | Branch (151:42): [True: 1.18M, False: 7.47M] | | | Branch (151:42): [True: 599k, False: 6.87M] | | ------------------ ------------------ 273| 6.87M| const uint8_t* ref = src + match_idx; 274| | 275| | // Load the next chain link early (before the compare) so its address 276| | // resolves while we prefetch. 277| 6.87M| const uint16_t delta = chain_table[match_idx & ZXC_LZ_WINDOW_MASK]; ------------------ | | 506| 6.87M|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 504| 6.87M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 278| 6.87M| const uint32_t next_idx = match_idx - delta; 279| 6.87M| ZXC_PREFETCH_READ(src + next_idx); ------------------ | | 162| 6.87M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 280| | 281| 6.87M| const uint32_t ref_val = zxc_le32(ref); 282| 6.87M| const int tag_match = (ref_val == cur_val); 283| | // Cheap gate: 4-byte tag match, then check the byte past the current 284| | // best (the && skips that load unless the tag already matched). 285| 6.87M| const int should_compare = tag_match && (ref[best.len] == ip[best.len]); ------------------ | Branch (285:36): [True: 6.65M, False: 214k] | Branch (285:49): [True: 3.73M, False: 2.92M] ------------------ 286| | 287| 6.87M| if (should_compare) { ------------------ | Branch (287:13): [True: 3.73M, False: 3.13M] ------------------ 288| 3.73M| uint32_t mlen = sizeof(uint32_t); // We already know the first 4 bytes match 289| | 290| | // Fast path: Scalar 64-bit comparison for short matches (=< 64 bytes) 291| | // Most matches are short, so this avoids SIMD overhead for common cases 292| 3.73M| const uint8_t* limit_8 = iend - sizeof(uint64_t); 293| 3.73M| const uint8_t* scalar_limit = ip + mlen + 64; 294| 3.73M| if (scalar_limit > limit_8) scalar_limit = limit_8; ------------------ | Branch (294:17): [True: 7.85k, False: 3.72M] ------------------ 295| | 296| 8.37M| while (ip + mlen < scalar_limit) { ------------------ | Branch (296:20): [True: 8.12M, False: 253k] ------------------ 297| 8.12M| uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(ref + mlen); 298| 8.12M| if (diff == 0) ------------------ | Branch (298:21): [True: 4.64M, False: 3.47M] ------------------ 299| 4.64M| mlen += sizeof(uint64_t); 300| 3.47M| else { 301| 3.47M| mlen += (zxc_ctz64(diff) >> 3); 302| 3.47M| goto _match_len_done; 303| 3.47M| } 304| 8.12M| } 305| | 306| | // Long match path: Use SIMD for matches exceeding 64 bytes 307| |#if defined(ZXC_USE_AVX512) 308| | const uint8_t* limit_64 = iend - 64; 309| | while (ip + mlen < limit_64) { 310| | const __m512i v_src = _mm512_loadu_si512((const void*)(ip + mlen)); 311| | const __m512i v_ref = _mm512_loadu_si512((const void*)(ref + mlen)); 312| | const __mmask64 mask = _mm512_cmpeq_epi8_mask(v_src, v_ref); 313| | if (mask == 0xFFFFFFFFFFFFFFFF) 314| | mlen += 64; 315| | else { 316| | mlen += (uint32_t)zxc_ctz64(~mask); 317| | goto _match_len_done; 318| | } 319| | } 320| |#elif defined(ZXC_USE_AVX2) 321| | const uint8_t* limit_32 = iend - 32; 322| | while (ip + mlen < limit_32) { 323| | const __m256i v_src = _mm256_loadu_si256((const __m256i*)(ip + mlen)); 324| | const __m256i v_ref = _mm256_loadu_si256((const __m256i*)(ref + mlen)); 325| | const __m256i v_cmp = _mm256_cmpeq_epi8(v_src, v_ref); 326| | const uint32_t mask = (uint32_t)_mm256_movemask_epi8(v_cmp); 327| | if (mask == 0xFFFFFFFF) 328| | mlen += 32; 329| | else { 330| | mlen += zxc_ctz32(~mask); 331| | goto _match_len_done; 332| | } 333| | } 334| |#elif defined(ZXC_USE_SSE2) 335| 253k| const uint8_t* limit_16 = iend - 16; 336| 11.2M| while (ip + mlen < limit_16) { ------------------ | Branch (336:20): [True: 11.2M, False: 2.18k] ------------------ 337| 11.2M| const __m128i v_src = _mm_loadu_si128((const __m128i*)(ip + mlen)); 338| 11.2M| const __m128i v_ref = _mm_loadu_si128((const __m128i*)(ref + mlen)); 339| 11.2M| const __m128i v_cmp = _mm_cmpeq_epi8(v_src, v_ref); 340| 11.2M| const uint32_t mask = (uint32_t)_mm_movemask_epi8(v_cmp); 341| 11.2M| if (mask == 0xFFFFU) ------------------ | Branch (341:21): [True: 11.0M, False: 251k] ------------------ 342| 11.0M| mlen += 16; 343| 251k| else { 344| | // mask != 0xFFFF => a differing byte exists in bits 0..15, 345| | // so the lowest set bit of ~mask lies in that range. 346| 251k| mlen += zxc_ctz32(~mask); 347| 251k| goto _match_len_done; 348| 251k| } 349| 11.2M| } 350| |#elif defined(ZXC_USE_NEON64) 351| | { 352| | const uint8_t* limit_32 = iend - 32; 353| | while (ip + mlen < limit_32) { 354| | const uint8x16_t s0 = vld1q_u8(ip + mlen); 355| | const uint8x16_t r0 = vld1q_u8(ref + mlen); 356| | const uint8x16_t c0 = vceqq_u8(s0, r0); 357| | const uint64_t m0 = vget_lane_u64( 358| | vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(c0), 4)), 0); 359| | if (UNLIKELY(m0 != ~(uint64_t)0)) { 360| | mlen += (uint32_t)(zxc_ctz64(~m0) >> 2); 361| | goto _match_len_done; 362| | } 363| | const uint8x16_t s1 = vld1q_u8(ip + mlen + 16); 364| | const uint8x16_t r1 = vld1q_u8(ref + mlen + 16); 365| | const uint8x16_t c1 = vceqq_u8(s1, r1); 366| | const uint64_t m1 = vget_lane_u64( 367| | vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(c1), 4)), 0); 368| | if (UNLIKELY(m1 != ~(uint64_t)0)) { 369| | mlen += 16 + (uint32_t)(zxc_ctz64(~m1) >> 2); 370| | goto _match_len_done; 371| | } 372| | mlen += 32; 373| | } 374| | if (ip + mlen < iend - 16) { 375| | const uint8x16_t v_src = vld1q_u8(ip + mlen); 376| | const uint8x16_t v_ref = vld1q_u8(ref + mlen); 377| | const uint8x16_t v_cmp = vceqq_u8(v_src, v_ref); 378| | const uint64_t mask = vget_lane_u64( 379| | vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(v_cmp), 4)), 0); 380| | if (LIKELY(mask == ~(uint64_t)0)) 381| | mlen += 16; 382| | else { 383| | mlen += (uint32_t)(zxc_ctz64(~mask) >> 2); 384| | goto _match_len_done; 385| | } 386| | } 387| | } 388| |#elif defined(ZXC_USE_NEON32) 389| | { 390| | const uint8_t* limit_16 = iend - 16; 391| | while (ip + mlen < limit_16) { 392| | const uint8x16_t v_src = vld1q_u8(ip + mlen); 393| | const uint8x16_t v_ref = vld1q_u8(ref + mlen); 394| | const uint8x16_t v_cmp = vceqq_u8(v_src, v_ref); 395| | uint8x8_t p1 = vpmin_u8(vget_low_u8(v_cmp), vget_high_u8(v_cmp)); 396| | uint8x8_t p2 = vpmin_u8(p1, p1); 397| | uint8x8_t p3 = vpmin_u8(p2, p2); 398| | uint8x8_t p4 = vpmin_u8(p3, p3); 399| | uint8_t min_val = vget_lane_u8(p4, 0); 400| | if (min_val == 0xFF) 401| | mlen += 16; 402| | else { 403| | uint8x16_t v_diff = vmvnq_u8(v_cmp); 404| | uint64_t lo = 405| | (uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 0) | 406| | ((uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 1) << 32); 407| | if (lo != 0) 408| | mlen += (zxc_ctz64(lo) >> 3); 409| | else 410| | mlen += 411| | 8 + (zxc_ctz64( 412| | (uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 2) | 413| | ((uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 3) 414| | << 32)) >> 415| | 3); 416| | goto _match_len_done; 417| | } 418| | } 419| | } 420| |#endif 421| 2.40k| while (ip + mlen < limit_8) { ------------------ | Branch (421:20): [True: 393, False: 2.01k] ------------------ 422| 393| const uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(ref + mlen); 423| 393| if (diff == 0) ------------------ | Branch (423:21): [True: 227, False: 166] ------------------ 424| 227| mlen += sizeof(uint64_t); 425| 166| else { 426| 166| mlen += (zxc_ctz64(diff) >> 3); 427| 166| goto _match_len_done; 428| 166| } 429| 393| } 430| 9.30k| while (ip + mlen < iend && ref[mlen] == ip[mlen]) mlen++; ------------------ | Branch (430:20): [True: 8.44k, False: 854] | Branch (430:40): [True: 7.28k, False: 1.16k] ------------------ 431| | 432| 3.73M| _match_len_done:; 433| 3.73M| const int better = (mlen > best.len); 434| 3.73M| best.len = better ? mlen : best.len; ------------------ | Branch (434:24): [True: 3.40M, False: 324k] ------------------ 435| 3.73M| best.ref = better ? ref : best.ref; ------------------ | Branch (435:24): [True: 3.40M, False: 324k] ------------------ 436| | 437| 3.73M| if (UNLIKELY(best.len >= (uint32_t)p.sufficient_len || ip + best.len >= iend)) break; ------------------ | | 151| 6.37M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.08M, False: 2.64M] | | | Branch (151:42): [True: 1.08M, False: 2.64M] | | | Branch (151:42): [True: 263, False: 2.64M] | | ------------------ ------------------ 438| 3.73M| } 439| | 440| 5.78M| match_idx = (delta != 0) ? next_idx : 0; ------------------ | Branch (440:21): [True: 5.43M, False: 345k] ------------------ 441| 5.78M| } 442| | 443| 7.97M|_finalize_match: 444| 7.97M| if (best.ref) { ------------------ | Branch (444:9): [True: 2.62M, False: 5.34M] ------------------ 445| | // Backtrack to extend match backwards 446| 2.62M| const uint8_t* b_ip = ip; 447| 2.62M| const uint8_t* b_ref = best.ref; 448| 5.15M| while (b_ip > anchor && b_ref > src && b_ip[-1] == b_ref[-1]) { ------------------ | Branch (448:16): [True: 3.61M, False: 1.53M] | Branch (448:33): [True: 3.61M, False: 787] | Branch (448:48): [True: 2.52M, False: 1.09M] ------------------ 449| 2.52M| b_ip--; 450| 2.52M| b_ref--; 451| 2.52M| best.len++; 452| 2.52M| best.backtrack++; 453| 2.52M| } 454| 2.62M| best.ref = b_ref; 455| 2.62M| } 456| | 457| 7.97M| if (p.use_lazy && best.ref && best.len < (uint32_t)p.lazy_len_threshold && ------------------ | Branch (457:9): [True: 0, False: 7.97M] | Branch (457:23): [True: 0, False: 0] | Branch (457:35): [True: 0, False: 0] ------------------ 458| 0| ip + 1 < search_limit) { ------------------ | Branch (458:9): [True: 0, False: 0] ------------------ 459| | // --- Lazy evaluation at ip+1 --- 460| 0| const uint64_t next_val8 = zxc_le64(ip + 1); 461| 0| const uint32_t next_val = (uint32_t)next_val8; 462| 0| const uint32_t h2 = zxc_hash_func(next_val8, use_hash5); 463| 0| const uint8_t next_stored_tag = hash_tags[h2]; 464| 0| const uint32_t next_head = hash_table[h2]; 465| 0| uint32_t next_idx = 466| 0| (next_head & ~offset_mask) == epoch_mark ? (next_head & offset_mask) : 0; ------------------ | Branch (466:13): [True: 0, False: 0] ------------------ 467| 0| const uint8_t next_tag = (uint8_t)(next_val ^ (next_val >> 16)); 468| 0| const int skip_lazy_head = (next_idx > 0 && next_stored_tag != next_tag); ------------------ | Branch (468:37): [True: 0, False: 0] | Branch (468:53): [True: 0, False: 0] ------------------ 469| 0| uint32_t max_lazy2 = 0; 470| 0| int lazy_att = p.lazy_attempts; 471| 0| int is_lazy_first = 1; 472| | 473| 0| while (next_idx > 0) { ------------------ | Branch (473:16): [True: 0, False: 0] ------------------ 474| 0| if (UNLIKELY(lazy_att-- <= 0 || (uint32_t)(ip + 1 - src) - next_idx > ZXC_LZ_MAX_DIST)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 475| 0| break; 476| 0| const uint8_t* ref2 = src + next_idx; 477| | 478| 0| if ((!is_lazy_first || !skip_lazy_head) && zxc_le32(ref2) == next_val) { ------------------ | Branch (478:18): [True: 0, False: 0] | Branch (478:36): [True: 0, False: 0] | Branch (478:56): [True: 0, False: 0] ------------------ 479| 0| uint32_t l2 = sizeof(uint32_t); 480| 0| const uint8_t* limit = iend - sizeof(uint64_t); 481| | 482| 0| while (ip + 1 + l2 < limit) { ------------------ | Branch (482:24): [True: 0, False: 0] ------------------ 483| 0| const uint64_t v1 = zxc_le64(ip + 1 + l2); 484| 0| const uint64_t v2 = zxc_le64(ref2 + l2); 485| 0| if (v1 != v2) { ------------------ | Branch (485:25): [True: 0, False: 0] ------------------ 486| 0| l2 += (uint32_t)(zxc_ctz64(v1 ^ v2) >> 3); 487| 0| goto lazy2_done; 488| 0| } 489| 0| l2 += sizeof(uint64_t); 490| 0| } 491| 0| while (ip + 1 + l2 < iend && ref2[l2] == ip[1 + l2]) l2++; ------------------ | Branch (491:24): [True: 0, False: 0] | Branch (491:46): [True: 0, False: 0] ------------------ 492| 0| lazy2_done: 493| 0| max_lazy2 = l2 > max_lazy2 ? l2 : max_lazy2; ------------------ | Branch (493:29): [True: 0, False: 0] ------------------ 494| 0| } 495| | 496| 0| const uint16_t delta = chain_table[next_idx & ZXC_LZ_WINDOW_MASK]; ------------------ | | 506| 0|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 504| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 497| 0| if (UNLIKELY(delta == 0)) break; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 498| 0| next_idx -= delta; 499| 0| is_lazy_first = 0; 500| 0| } 501| | 502| | // --- Lazy evaluation at ip+2 (computed in parallel, no dependency on lazy 1) --- 503| 0| uint32_t max_lazy3 = 0; 504| 0| if (level >= ZXC_LEVEL_BALANCED && ip + 2 < search_limit) { ------------------ | Branch (504:13): [True: 0, False: 0] | Branch (504:44): [True: 0, False: 0] ------------------ 505| 0| const uint64_t val3_8 = zxc_le64(ip + 2); 506| 0| const uint32_t val3 = (uint32_t)val3_8; 507| 0| const uint32_t h3 = zxc_hash_func(val3_8, use_hash5); 508| 0| const uint8_t tag3 = hash_tags[h3]; 509| 0| const uint32_t head3 = hash_table[h3]; 510| 0| uint32_t idx3 = (head3 & ~offset_mask) == epoch_mark ? (head3 & offset_mask) : 0; ------------------ | Branch (510:29): [True: 0, False: 0] ------------------ 511| 0| const uint8_t cur_tag3 = (uint8_t)(val3 ^ (val3 >> 16)); 512| 0| const int skip_head3 = (idx3 > 0 && tag3 != cur_tag3); ------------------ | Branch (512:37): [True: 0, False: 0] | Branch (512:49): [True: 0, False: 0] ------------------ 513| | 514| 0| int is_first3 = 1; 515| 0| lazy_att = p.lazy_attempts; 516| 0| while (idx3 > 0) { ------------------ | Branch (516:20): [True: 0, False: 0] ------------------ 517| 0| if (UNLIKELY(lazy_att-- <= 0 || (uint32_t)(ip + 2 - src) - idx3 > ZXC_LZ_MAX_DIST)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 518| 0| break; 519| | 520| 0| const uint8_t* ref3 = src + idx3; 521| 0| if ((!is_first3 || !skip_head3) && zxc_le32(ref3) == val3) { ------------------ | Branch (521:22): [True: 0, False: 0] | Branch (521:36): [True: 0, False: 0] | Branch (521:52): [True: 0, False: 0] ------------------ 522| 0| uint32_t l3 = sizeof(uint32_t); 523| 0| const uint8_t* limit = iend - sizeof(uint64_t); 524| | 525| 0| while (ip + 2 + l3 < limit) { ------------------ | Branch (525:28): [True: 0, False: 0] ------------------ 526| 0| const uint64_t v1 = zxc_le64(ip + 2 + l3); 527| 0| const uint64_t v2 = zxc_le64(ref3 + l3); 528| 0| if (v1 != v2) { ------------------ | Branch (528:29): [True: 0, False: 0] ------------------ 529| 0| l3 += (uint32_t)(zxc_ctz64(v1 ^ v2) >> 3); 530| 0| goto lazy3_done; 531| 0| } 532| 0| l3 += sizeof(uint64_t); 533| 0| } 534| 0| while (ip + 2 + l3 < iend && ref3[l3] == ip[2 + l3]) l3++; ------------------ | Branch (534:28): [True: 0, False: 0] | Branch (534:50): [True: 0, False: 0] ------------------ 535| 0| lazy3_done: 536| 0| max_lazy3 = l3 > max_lazy3 ? l3 : max_lazy3; ------------------ | Branch (536:33): [True: 0, False: 0] ------------------ 537| 0| } 538| | 539| 0| const uint16_t delta = chain_table[idx3 & ZXC_LZ_WINDOW_MASK]; ------------------ | | 506| 0|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 504| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 540| 0| if (UNLIKELY(delta == 0)) break; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 541| 0| idx3 -= delta; 542| 0| is_first3 = 0; 543| 0| } 544| 0| } 545| | 546| | // Single decision: invalidate if either lazy position found a better match 547| 0| if (max_lazy2 > best.len + 1 || max_lazy3 > best.len + 2) best.ref = NULL; ------------------ | Branch (547:13): [True: 0, False: 0] | Branch (547:41): [True: 0, False: 0] ------------------ 548| 0| } 549| | 550| 7.97M| return best; 551| 7.97M|} zxc_compress.c:zxc_write_varint: 119| 104k|static ZXC_ALWAYS_INLINE size_t zxc_write_varint(uint8_t* RESTRICT dst, const uint32_t val) { 120| | // Values above ZXC_MAX_VARINT_VALUE are out-of-spec and the decoder would 121| | // reject them. Unreachable from the Block API, so this is defence in depth; 122| | // callers must treat a return of 0 as an encoding error. 123| 104k| if (UNLIKELY(val > ZXC_MAX_VARINT_VALUE)) { ------------------ | | 151| 104k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 104k] | | ------------------ ------------------ 124| 0| return 0; 125| 0| } 126| | 127| | // 1 byte: 0xxxxxxx (7 bits) = 2^7 = 128 128| 104k| if (LIKELY(val < (1U << 7))) { ------------------ | | 145| 104k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 30.2k, False: 74.5k] | | ------------------ ------------------ 129| 30.2k| dst[0] = (uint8_t)val; 130| 30.2k| return 1; 131| 30.2k| } 132| | 133| | // 2 bytes: 10xxxxxx xxxxxxxx (14 bits) = 2^14 = 16384 134| 74.5k| if (LIKELY(val < (1U << 14))) { ------------------ | | 145| 74.5k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 71.7k, False: 2.80k] | | ------------------ ------------------ 135| 71.7k| dst[0] = (uint8_t)(0x80 | (val & 0x3F)); 136| 71.7k| dst[1] = (uint8_t)(val >> 6); 137| 71.7k| return 2; 138| 71.7k| } 139| | 140| | // 3 bytes: 110xxxxx xxxxxxxx xxxxxxxx (21 bits) -> max emittable value, 141| | // matching ZXC_MAX_VARINT_VALUE = ZXC_BLOCK_SIZE_MAX - 1. 142| 2.80k| dst[0] = (uint8_t)(0xC0 | (val & 0x1F)); 143| 2.80k| dst[1] = (uint8_t)(val >> 5); 144| 2.80k| dst[2] = (uint8_t)(val >> 13); 145| 2.80k| return 3; 146| 74.5k|} zxc_compress.c:zxc_encode_block_raw: 2044| 775| size_t* RESTRICT const out_sz) { 2045| 775| if (UNLIKELY(dst_cap < ZXC_BLOCK_HEADER_SIZE + src_sz)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 775|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 775] | | ------------------ ------------------ 2046| | 2047| 775| zxc_block_header_t bh; 2048| 775| bh.block_type = ZXC_BLOCK_RAW; 2049| 775| bh.block_flags = 0; // Checksum flag moved to file header 2050| 775| bh.reserved = 0; 2051| 775| bh.comp_size = (uint32_t)src_sz; 2052| | 2053| 775| const int hw = zxc_write_block_header(dst, dst_cap, &bh); 2054| 775| if (UNLIKELY(hw < 0)) return hw; ------------------ | | 151| 775|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 775] | | ------------------ ------------------ 2055| | 2056| 775| ZXC_MEMCPY(dst + ZXC_BLOCK_HEADER_SIZE, src, src_sz); ------------------ | | 173| 775|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 2057| | 2058| | // Checksum will be appended by the wrapper 2059| 775| *out_sz = ZXC_BLOCK_HEADER_SIZE + src_sz; ------------------ | | 397| 775|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2060| 775| return ZXC_OK; 2061| 775|} zxc_decompress_chunk_wrapper_default: 1532| 4.90k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 1533| 4.90k| return zxc_decompress_chunk_wrapper_body(ctx, src, src_sz, dst, dst_cap, 0, 0); 1534| 4.90k|} zxc_decompress.c:zxc_decompress_chunk_wrapper_body: 1468| 4.90k| uint8_t* RESTRICT dst, const size_t dst_cap, const int has_dict, const int safe) { 1469| 4.90k| if (UNLIKELY(src_sz < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1470| | 1471| 4.90k| const uint8_t type = src[0]; 1472| 4.90k| const uint32_t comp_sz = zxc_le32(src + 3); 1473| 4.90k| const int has_crc = ctx->checksum_enabled; 1474| | 1475| | // Check bounds: Header + Body + Checksum(if any) 1476| 4.90k| const size_t expected_sz = 1477| 4.90k| (size_t)ZXC_BLOCK_HEADER_SIZE + comp_sz + (has_crc ? ZXC_BLOCK_CHECKSUM_SIZE : 0); ------------------ | | 397| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ (size_t)ZXC_BLOCK_HEADER_SIZE + comp_sz + (has_crc ? ZXC_BLOCK_CHECKSUM_SIZE : 0); ------------------ | | 399| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (1477:52): [True: 1.63k, False: 3.27k] ------------------ 1478| 4.90k| if (UNLIKELY(src_sz < expected_sz)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1479| | 1480| 4.90k| const uint8_t* data = src + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1481| | 1482| 4.90k| if (has_crc) { ------------------ | Branch (1482:9): [True: 1.63k, False: 3.27k] ------------------ 1483| 1.63k| const uint32_t stored = zxc_le32(data + comp_sz); 1484| 1.63k| const uint32_t calc = zxc_checksum(data, comp_sz, ZXC_CHECKSUM_RAPIDHASH); ------------------ | | 424| 1.63k|#define ZXC_CHECKSUM_RAPIDHASH 0 ------------------ 1485| 1.63k| if (UNLIKELY(stored != calc)) return ZXC_ERROR_BAD_CHECKSUM; ------------------ | | 151| 1.63k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.63k] | | ------------------ ------------------ 1486| 1.63k| } 1487| | 1488| 4.90k| int decoded_sz = ZXC_ERROR_BAD_BLOCK_TYPE; 1489| | 1490| 4.90k| switch (type) { 1491| 0| case ZXC_BLOCK_GLO: ------------------ | Branch (1491:9): [True: 0, False: 4.90k] ------------------ 1492| 0| decoded_sz = safe ? zxc_decode_block_glo_safe(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1492:26): [True: 0, False: 0] ------------------ 1493| 0| : has_dict ? zxc_decode_block_glo_dict(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1493:28): [True: 0, False: 0] ------------------ 1494| 0| : zxc_decode_block_glo(ctx, data, comp_sz, dst, dst_cap); 1495| 0| break; 1496| 4.13k| case ZXC_BLOCK_GHI: ------------------ | Branch (1496:9): [True: 4.13k, False: 775] ------------------ 1497| 4.13k| decoded_sz = safe ? zxc_decode_block_ghi_safe(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1497:26): [True: 0, False: 4.13k] ------------------ 1498| 4.13k| : has_dict ? zxc_decode_block_ghi_dict(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1498:28): [True: 0, False: 4.13k] ------------------ 1499| 4.13k| : zxc_decode_block_ghi(ctx, data, comp_sz, dst, dst_cap); 1500| 4.13k| break; 1501| 775| case ZXC_BLOCK_RAW: ------------------ | Branch (1501:9): [True: 775, False: 4.13k] ------------------ 1502| | // For RAW blocks, comp_sz == raw_sz (uncompressed data stored as-is) 1503| 775| if (UNLIKELY(comp_sz > dst_cap)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 775|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 775] | | ------------------ ------------------ 1504| 775| ZXC_MEMCPY(dst, data, comp_sz); ------------------ | | 173| 775|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1505| 775| decoded_sz = (int)comp_sz; 1506| 775| break; 1507| 0| case ZXC_BLOCK_EOF: ------------------ | Branch (1507:9): [True: 0, False: 4.90k] ------------------ 1508| | // EOF should be handled by the dispatcher, not here 1509| 0| return ZXC_ERROR_CORRUPT_DATA; 1510| 0| default: ------------------ | Branch (1510:9): [True: 0, False: 4.90k] ------------------ 1511| 0| return ZXC_ERROR_BAD_BLOCK_TYPE; 1512| 4.90k| } 1513| | 1514| 4.90k| return decoded_sz; 1515| 4.90k|} zxc_decompress.c:zxc_stage_raw_literals: 594| 900| const uint8_t** RESTRICT l_end) { 595| 900| const size_t lit_size = (size_t)(*l_end - *l_ptr); 596| 900| uint8_t* const stage = ctx->lit_buffer; 597| 900| if (UNLIKELY(!stage || ctx->lit_buffer_cap < lit_size + ZXC_PAD_SIZE)) ------------------ | | 151| 1.80k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 900] | | | Branch (151:42): [True: 0, False: 900] | | | Branch (151:42): [True: 0, False: 900] | | ------------------ ------------------ 598| 0| return ZXC_ERROR_CORRUPT_DATA; 599| | 600| 900| ZXC_MEMCPY(stage, *l_ptr, lit_size); ------------------ | | 173| 900|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 601| 900| *l_ptr = stage; 602| 900| *l_end = stage + lit_size; 603| 900| return ZXC_OK; 604| 900|} zxc_decompress.c:zxc_read_varint: 54| 104k|static ZXC_ALWAYS_INLINE uint32_t zxc_read_varint(const uint8_t** ptr, const uint8_t* end) { 55| 104k| const uint8_t* p = *ptr; 56| 104k| if (UNLIKELY(p >= end)) return 0; ------------------ | | 151| 104k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 104k] | | ------------------ ------------------ 57| | 58| 104k| const uint32_t b0 = p[0]; 59| | 60| | // 1 Byte: 0xxxxxxx (7 bits) -> val < 128 (2^7) 61| 104k| if (LIKELY(b0 < 0x80)) { ------------------ | | 145| 104k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 30.2k, False: 74.5k] | | ------------------ ------------------ 62| 30.2k| *ptr = p + 1; 63| 30.2k| return b0; 64| 30.2k| } 65| | 66| | // 2 Bytes: 10xxxxxx xxxxxxxx (14 bits) -> val < 16384 (2^14) 67| 74.5k| if (LIKELY(b0 < 0xC0)) { ------------------ | | 145| 74.5k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 71.7k, False: 2.79k] | | ------------------ ------------------ 68| 71.7k| if (UNLIKELY(p + 1 >= end)) { ------------------ | | 151| 71.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 71.7k] | | ------------------ ------------------ 69| 0| *ptr = end; 70| 0| return 0; 71| 0| } 72| 71.7k| *ptr = p + 2; 73| 71.7k| return (b0 & 0x3F) | ((uint32_t)p[1] << 6); 74| 71.7k| } 75| | 76| | // 3 Bytes: 110xxxxx xxxxxxxx xxxxxxxx (21 bits) -> val < 2^21. The longest 77| | // a legitimate varint can be: values are (ll - MASK) or (ml - MASK), always 78| | // strictly below block_size_max = 2^21. 79| 2.79k| if (LIKELY(b0 < 0xE0)) { ------------------ | | 145| 2.79k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 2.79k, False: 0] | | ------------------ ------------------ 80| 2.79k| if (UNLIKELY(p + 2 >= end)) { ------------------ | | 151| 2.79k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 2.79k] | | ------------------ ------------------ 81| 0| *ptr = end; 82| 0| return 0; 83| 0| } 84| 2.79k| *ptr = p + 3; 85| 2.79k| return (b0 & 0x1F) | ((uint32_t)p[1] << 5) | ((uint32_t)p[2] << 13); 86| 2.79k| } 87| | 88| | // extra encoding: out-of-spec for the current format, reject. 89| 0| *ptr = end; 90| 0| return 0; 91| 2.79k|} zxc_decompress.c:zxc_decode_copy_literals: 297| 2.45M| const uint64_t ll) { 298| 2.45M| zxc_copy32(dst, src); 299| 2.45M| if (UNLIKELY(ll > ZXC_PAD_SIZE)) { ------------------ | | 151| 2.45M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 112k, False: 2.34M] | | ------------------ ------------------ 300| 112k| dst += ZXC_PAD_SIZE; ------------------ | | 337| 112k|#define ZXC_PAD_SIZE 32 ------------------ 301| 112k| src += ZXC_PAD_SIZE; ------------------ | | 337| 112k|#define ZXC_PAD_SIZE 32 ------------------ 302| 112k| size_t rem = ll - ZXC_PAD_SIZE; ------------------ | | 337| 112k|#define ZXC_PAD_SIZE 32 ------------------ 303| 4.78M| while (rem > ZXC_PAD_SIZE) { ------------------ | | 337| 4.78M|#define ZXC_PAD_SIZE 32 ------------------ | Branch (303:16): [True: 4.67M, False: 112k] ------------------ 304| 4.67M| zxc_copy32(dst, src); 305| 4.67M| dst += ZXC_PAD_SIZE; ------------------ | | 337| 4.67M|#define ZXC_PAD_SIZE 32 ------------------ 306| 4.67M| src += ZXC_PAD_SIZE; ------------------ | | 337| 4.67M|#define ZXC_PAD_SIZE 32 ------------------ 307| 4.67M| rem -= ZXC_PAD_SIZE; ------------------ | | 337| 4.67M|#define ZXC_PAD_SIZE 32 ------------------ 308| 4.67M| } 309| 112k| zxc_copy32(dst, src); 310| 112k| } 311| 2.45M|} zxc_decompress.c:zxc_decode_copy_match: 333| 2.45M| const uint64_t ml) { 334| 2.45M| const uint8_t* match_src = d_ptr - off; 335| 2.45M| if (LIKELY(off >= ZXC_PAD_SIZE)) { ------------------ | | 145| 2.45M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 2.12M, False: 330k] | | ------------------ ------------------ 336| 2.12M| zxc_copy32(d_ptr, match_src); 337| 2.12M| if (UNLIKELY(ml > ZXC_PAD_SIZE)) { ------------------ | | 151| 2.12M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 495k, False: 1.62M] | | ------------------ ------------------ 338| 495k| uint8_t* out = d_ptr + ZXC_PAD_SIZE; ------------------ | | 337| 495k|#define ZXC_PAD_SIZE 32 ------------------ 339| 495k| const uint8_t* ref = match_src + ZXC_PAD_SIZE; ------------------ | | 337| 495k|#define ZXC_PAD_SIZE 32 ------------------ 340| 495k| size_t rem = ml - ZXC_PAD_SIZE; ------------------ | | 337| 495k|#define ZXC_PAD_SIZE 32 ------------------ 341| 4.27M| while (rem > ZXC_PAD_SIZE) { ------------------ | | 337| 4.27M|#define ZXC_PAD_SIZE 32 ------------------ | Branch (341:20): [True: 3.77M, False: 495k] ------------------ 342| 3.77M| zxc_copy32(out, ref); 343| 3.77M| out += ZXC_PAD_SIZE; ------------------ | | 337| 3.77M|#define ZXC_PAD_SIZE 32 ------------------ 344| 3.77M| ref += ZXC_PAD_SIZE; ------------------ | | 337| 3.77M|#define ZXC_PAD_SIZE 32 ------------------ 345| 3.77M| rem -= ZXC_PAD_SIZE; ------------------ | | 337| 3.77M|#define ZXC_PAD_SIZE 32 ------------------ 346| 3.77M| } 347| 495k| zxc_copy32(out, ref); 348| 495k| } 349| 2.12M| } else if (off >= (ZXC_PAD_SIZE / 2)) { ------------------ | | 337| 330k|#define ZXC_PAD_SIZE 32 ------------------ | Branch (349:16): [True: 220k, False: 109k] ------------------ 350| 220k| zxc_copy16(d_ptr, match_src); 351| 220k| if (UNLIKELY(ml > (ZXC_PAD_SIZE / 2))) { ------------------ | | 151| 220k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 77.2k, False: 143k] | | ------------------ ------------------ 352| 77.2k| uint8_t* out = d_ptr + (ZXC_PAD_SIZE / 2); ------------------ | | 337| 77.2k|#define ZXC_PAD_SIZE 32 ------------------ 353| 77.2k| const uint8_t* ref = match_src + (ZXC_PAD_SIZE / 2); ------------------ | | 337| 77.2k|#define ZXC_PAD_SIZE 32 ------------------ 354| 77.2k| size_t rem = ml - (ZXC_PAD_SIZE / 2); ------------------ | | 337| 77.2k|#define ZXC_PAD_SIZE 32 ------------------ 355| 585k| while (rem > (ZXC_PAD_SIZE / 2)) { ------------------ | | 337| 585k|#define ZXC_PAD_SIZE 32 ------------------ | Branch (355:20): [True: 508k, False: 77.2k] ------------------ 356| 508k| zxc_copy16(out, ref); 357| 508k| out += (ZXC_PAD_SIZE / 2); ------------------ | | 337| 508k|#define ZXC_PAD_SIZE 32 ------------------ 358| 508k| ref += (ZXC_PAD_SIZE / 2); ------------------ | | 337| 508k|#define ZXC_PAD_SIZE 32 ------------------ 359| 508k| rem -= (ZXC_PAD_SIZE / 2); ------------------ | | 337| 508k|#define ZXC_PAD_SIZE 32 ------------------ 360| 508k| } 361| 77.2k| zxc_copy16(out, ref); 362| 77.2k| } 363| 220k| } else if (off == 1) { ------------------ | Branch (363:16): [True: 714, False: 108k] ------------------ 364| 714| zxc_decode_fill_run(d_ptr, match_src[0], ml); 365| 108k| } else { 366| 108k| zxc_decode_copy_overlap_run(d_ptr, off, ml); 367| 108k| } 368| 2.45M|} zxc_decompress.c:zxc_decode_fill_run: 222| 714| const uint64_t ml) { 223| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 224| | const __m256i v = _mm256_set1_epi8((char)byte); 225| | _mm256_storeu_si256((__m256i*)dst, v); 226| | if (UNLIKELY(ml > ZXC_PAD_SIZE)) { 227| | uint8_t* out = dst + ZXC_PAD_SIZE; 228| | size_t rem = ml - ZXC_PAD_SIZE; 229| | while (rem > ZXC_PAD_SIZE) { 230| | _mm256_storeu_si256((__m256i*)out, v); 231| | out += ZXC_PAD_SIZE; 232| | rem -= ZXC_PAD_SIZE; 233| | } 234| | _mm256_storeu_si256((__m256i*)out, v); 235| | } 236| |#elif defined(ZXC_USE_SSE2) 237| | const __m128i v = _mm_set1_epi8((char)byte); 238| 714| _mm_storeu_si128((__m128i*)dst, v); 239| 714| _mm_storeu_si128((__m128i*)(dst + 16), v); 240| 714| if (UNLIKELY(ml > ZXC_PAD_SIZE)) { ------------------ | | 151| 714|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 442, False: 272] | | ------------------ ------------------ 241| 442| uint8_t* out = dst + ZXC_PAD_SIZE; ------------------ | | 337| 442|#define ZXC_PAD_SIZE 32 ------------------ 242| 442| size_t rem = ml - ZXC_PAD_SIZE; ------------------ | | 337| 442|#define ZXC_PAD_SIZE 32 ------------------ 243| 17.7k| while (rem > ZXC_PAD_SIZE) { ------------------ | | 337| 17.7k|#define ZXC_PAD_SIZE 32 ------------------ | Branch (243:16): [True: 17.3k, False: 442] ------------------ 244| 17.3k| _mm_storeu_si128((__m128i*)out, v); 245| 17.3k| _mm_storeu_si128((__m128i*)(out + 16), v); 246| 17.3k| out += ZXC_PAD_SIZE; ------------------ | | 337| 17.3k|#define ZXC_PAD_SIZE 32 ------------------ 247| 17.3k| rem -= ZXC_PAD_SIZE; ------------------ | | 337| 17.3k|#define ZXC_PAD_SIZE 32 ------------------ 248| 17.3k| } 249| 442| _mm_storeu_si128((__m128i*)out, v); 250| 442| _mm_storeu_si128((__m128i*)(out + 16), v); 251| 442| } 252| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 253| | const uint8x16_t v = vdupq_n_u8(byte); 254| | vst1q_u8(dst, v); 255| | vst1q_u8(dst + 16, v); 256| | if (UNLIKELY(ml > ZXC_PAD_SIZE)) { 257| | uint8_t* out = dst + ZXC_PAD_SIZE; 258| | size_t rem = ml - ZXC_PAD_SIZE; 259| | while (rem > ZXC_PAD_SIZE) { 260| | vst1q_u8(out, v); 261| | vst1q_u8(out + 16, v); 262| | out += ZXC_PAD_SIZE; 263| | rem -= ZXC_PAD_SIZE; 264| | } 265| | vst1q_u8(out, v); 266| | vst1q_u8(out + 16, v); 267| | } 268| |#else 269| | ZXC_MEMSET(dst, byte, ml); 270| |#endif 271| 714|} zxc_decompress.c:zxc_decode_copy_overlap_run: 157| 108k| const uint64_t ml) { 158| 108k| const size_t stride = zxc_overlap_strides[off]; 159| 108k| size_t copied = 0; 160| |#if defined(ZXC_USE_NEON64) 161| | const uint8x16_t mask = vld1q_u8(zxc_overlap_masks[off]); 162| | const uint8x16_t pat = vqtbl1q_u8(vld1q_u8(dst - off), mask); 163| | do { 164| | vst1q_u8(dst + copied, pat); 165| | copied += stride; 166| | } while (copied < ml); 167| | 168| |#elif defined(ZXC_USE_NEON32) 169| | uint8x8x2_t src_tbl; 170| | src_tbl.val[0] = vld1_u8(dst - off); 171| | src_tbl.val[1] = vld1_u8(dst - off + 8); 172| | const uint8x8_t pat_lo = vtbl2_u8(src_tbl, vld1_u8(zxc_overlap_masks[off])); 173| | const uint8x8_t pat_hi = vtbl2_u8(src_tbl, vld1_u8(zxc_overlap_masks[off] + 8)); 174| | do { 175| | vst1_u8(dst + copied, pat_lo); 176| | vst1_u8(dst + copied + 8, pat_hi); 177| | copied += stride; 178| | } while (copied < ml); 179| | 180| |#elif defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 181| | const __m128i mask = _mm_load_si128((const __m128i*)zxc_overlap_masks[off]); 182| | const __m128i src_data = _mm_loadu_si128((const __m128i*)(dst - off)); 183| | const __m128i pat = _mm_shuffle_epi8(src_data, mask); 184| | do { 185| | _mm_storeu_si128((__m128i*)(dst + copied), pat); 186| | copied += stride; 187| | } while (copied < ml); 188| | 189| |#else 190| | // SSE2-only tier and non-SIMD builds: no PSHUFB, build the pattern with a 191| | // wrap counter (no per-byte modulo), then store it via zxc_copy16. 192| 108k| const uint8_t* src = dst - off; 193| 108k| uint8_t pat[16]; 194| 108k| uint32_t k = 0; 195| 1.84M| for (size_t i = 0; i < 16; i++) { ------------------ | Branch (195:24): [True: 1.73M, False: 108k] ------------------ 196| 1.73M| pat[i] = src[k]; 197| 1.73M| if (++k == off) k = 0; ------------------ | Branch (197:13): [True: 224k, False: 1.51M] ------------------ 198| 1.73M| } 199| 1.05M| do { 200| 1.05M| zxc_copy16(dst + copied, pat); 201| 1.05M| copied += stride; 202| 1.05M| } while (copied < ml); ------------------ | Branch (202:14): [True: 941k, False: 108k] ------------------ 203| 108k|#endif 204| 108k|} zxc_decompress.c:zxc_decode_copy_match_exact: 387| 172k| const size_t off, const size_t ml) { 388| 172k| if (off >= ml) { ------------------ | Branch (388:9): [True: 163k, False: 9.23k] ------------------ 389| 163k| ZXC_MEMCPY(d_ptr, match_src, ml); ------------------ | | 173| 163k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 390| 163k| } else if (ml < 16) { ------------------ | Branch (390:16): [True: 1.30k, False: 7.92k] ------------------ 391| 15.3k| for (size_t i = 0; i < ml; i++) d_ptr[i] = match_src[i]; ------------------ | Branch (391:28): [True: 14.0k, False: 1.30k] ------------------ 392| 7.92k| } else { 393| 7.92k| size_t n = off; 394| 7.92k| ZXC_MEMCPY(d_ptr, match_src, n); ------------------ | | 173| 7.92k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 395| 21.6k| while (n <= ml - n) { ------------------ | Branch (395:16): [True: 13.7k, False: 7.92k] ------------------ 396| 13.7k| ZXC_MEMCPY(d_ptr + n, d_ptr, n); ------------------ | | 173| 13.7k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 397| 13.7k| n <<= 1; 398| 13.7k| } 399| 7.92k| ZXC_MEMCPY(d_ptr + n, d_ptr, ml - n); ------------------ | | 173| 7.92k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 400| 7.92k| } 401| 172k|} zxc_decompress.c:zxc_decode_block_ghi_impl: 1060| 4.13k| const int has_dict) { 1061| 4.13k| zxc_gnr_header_t gh; 1062| | 1063| | /* 0 when !has_dict (safe path) -> folds `d_floor` to `dst`. */ 1064| 4.13k| const size_t dict_size = has_dict ? ctx->dict_size : 0; ------------------ | Branch (1064:30): [True: 0, False: 4.13k] ------------------ 1065| 4.13k| zxc_section_desc_t desc[ZXC_GHI_SECTIONS]; 1066| | 1067| 4.13k| if (UNLIKELY(zxc_read_ghi_header_and_desc(src, src_size, &gh, desc) != ZXC_OK)) ------------------ | | 151| 4.13k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.13k] | | ------------------ ------------------ 1068| 0| return ZXC_ERROR_BAD_HEADER; 1069| | 1070| 4.13k| const uint8_t* p_curr = 1071| 4.13k| src + ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 403| 4.13k|#define ZXC_GHI_HEADER_BINARY_SIZE 16 ------------------ src + ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 421| 4.13k|#define ZXC_GHI_SECTIONS 3 ------------------ src + ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 4.13k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 1072| | 1073| | // --- Stream Pointers & Validation --- 1074| 4.13k| const size_t sz_lit = (uint32_t)desc[0].sizes; 1075| 4.13k| const size_t sz_seqs = (uint32_t)desc[1].sizes; 1076| 4.13k| const size_t sz_exts = (uint32_t)desc[2].sizes; 1077| 4.13k| const uint8_t* l_ptr = p_curr; 1078| 4.13k| const uint8_t* l_end = l_ptr + sz_lit; 1079| 4.13k| p_curr += sz_lit; 1080| | 1081| 4.13k| const uint8_t* seq_ptr = p_curr; 1082| 4.13k| const uint8_t* extras_ptr = p_curr + sz_seqs; 1083| 4.13k| const uint8_t* const extras_end = extras_ptr + sz_exts; 1084| | 1085| | // Validate streams don't overflow source buffer + 1086| | // Validate sequence stream size matches sequence count 1087| 4.13k| if (UNLIKELY((extras_end != src + src_size) || ------------------ | | 151| 8.26k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.13k] | | | Branch (151:42): [True: 0, False: 4.13k] | | | Branch (151:42): [True: 0, False: 4.13k] | | ------------------ ------------------ 1088| 4.13k| ((uint64_t)sz_seqs < (uint64_t)gh.n_sequences * 4))) 1089| 0| return ZXC_ERROR_CORRUPT_DATA; 1090| | 1091| 4.13k| if (UNLIKELY((size_t)(src + src_size - l_end) < ZXC_PAD_SIZE)) { ------------------ | | 151| 4.13k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 900, False: 3.23k] | | ------------------ ------------------ 1092| 900| const int rc = zxc_stage_raw_literals(ctx, &l_ptr, &l_end); 1093| 900| if (UNLIKELY(rc != ZXC_OK)) return rc; ------------------ | | 151| 900|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 900] | | ------------------ ------------------ 1094| 900| } 1095| | 1096| 4.13k| uint8_t* d_ptr = dst; 1097| 4.13k| const uint8_t* const d_end = dst + dst_capacity; 1098| | // Lowest address a match may reach: the dictionary prefix if any, else dst. 1099| 4.13k| const uint8_t* const d_floor = dst - dict_size; 1100| 4.13k| const uint8_t* const d_end_safe = d_end - (ZXC_PAD_SIZE * 4); // 128 ------------------ | | 337| 4.13k|#define ZXC_PAD_SIZE 32 ------------------ 1101| | // Safety margin for 4x unrolled loop: 4 * (ZXC_SEQ_LL_MASK LL + 1102| | // ZXC_SEQ_ML_MASK+ZXC_LZ_MIN_MATCH_LEN ML) + ZXC_PAD_SIZE Pad = 4 x (255 + 255 + 5) + 32 = 2092 1103| 4.13k| const uint8_t* const d_end_fast = d_end - ZXC_DECOMPRESS_TAIL_PAD; // 2112 ------------------ | | 349| 4.13k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 4.13k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 1104| | 1105| | // Literal margin for the GHI loops: without a varint, ll <= 254 per sequence, 1106| | // so 4 * 254 = 1016. Past that only the cold varint path checks l_ptr. 1107| 4.13k| const size_t ghi_margin_4x = 4 * (ZXC_SEQ_LL_MASK - 1); // 1016 ------------------ | | 469| 4.13k|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 463| 4.13k|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ 1108| 4.13k| const size_t ghi_margin_1x = ZXC_SEQ_LL_MASK - 1; // 254 ------------------ | | 469| 4.13k|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 463| 4.13k|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ 1109| 4.13k| const uint8_t* const l_end_safe_4x = (sz_lit > ghi_margin_4x) ? l_end - ghi_margin_4x : l_ptr; ------------------ | Branch (1109:42): [True: 3.01k, False: 1.12k] ------------------ 1110| 4.13k| const uint8_t* const l_end_safe_1x = (sz_lit > ghi_margin_1x) ? l_end - ghi_margin_1x : l_ptr; ------------------ | Branch (1110:42): [True: 3.43k, False: 704] ------------------ 1111| | 1112| 4.13k| uint32_t n_seq = gh.n_sequences; 1113| | 1114| | // Same handover as GLO, but the GHI offset is inline in the sequence word and 1115| | // always 16 bits -- enc_off is not a width here, so the span is fixed. 1116| 4.13k| const size_t off_span = 1U << 16; 1117| 4.13k| const size_t span = (dict_size >= off_span) ? 0 : off_span - dict_size; ------------------ | Branch (1117:25): [True: 0, False: 4.13k] ------------------ 1118| 4.13k| const uint8_t* const d_bounds = (span >= dst_capacity) ? d_end : dst + span; ------------------ | Branch (1118:37): [True: 0, False: 4.13k] ------------------ 1119| | 1120| | // --- SAFE loop: validate offsets until d_bounds (4x unroll) --- 1121| 4.13k| if (safe) { ------------------ | Branch (1121:9): [True: 0, False: 4.13k] ------------------ 1122| | /* SAFE variant: save per-batch state so an OVERFLOW can rollback and 1123| | * hand over to the 1x loop / Safe Path. Wild writes already committed 1124| | * are deterministically overwritten when the 1x loop replays. */ 1125| 0| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (1125:16): [True: 0, False: 0] | Branch (1125:30): [True: 0, False: 0] | Branch (1125:52): [True: 0, False: 0] | Branch (1125:77): [True: 0, False: 0] ------------------ 1126| 0| const uint8_t* const t_save = seq_ptr; 1127| 0| const uint8_t* const e_save = extras_ptr; 1128| 0| uint8_t* const d_save = d_ptr; 1129| 0| const uint8_t* const l_save = l_ptr; 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); ------------------ | | 543| 0| do { \ | | 544| 0| uint32_t s1 = zxc_le32(seq_ptr); \ | | 545| 0| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 546| 0| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 547| 0| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 548| 0| seq_ptr += 4 * sizeof(uint32_t); \ | | 549| 0| PREFETCH; \ | | 550| 0| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 551| 0| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 552| 0| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 553| 0| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 554| 0| n_seq -= 4; \ | | 555| 0| } while (0) | | ------------------ | | | Branch (555:14): [Folded, False: 0] | | ------------------ ------------------ 1131| 0| continue; 1132| | 1133| 0| rollback_safe_4x: 1134| 0| seq_ptr = t_save; 1135| 0| extras_ptr = e_save; 1136| 0| d_ptr = d_save; 1137| 0| l_ptr = l_save; 1138| 0| break; 1139| 0| } 1140| 4.13k| } else { 1141| 161k| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (1141:16): [True: 159k, False: 1.75k] | Branch (1141:30): [True: 159k, False: 0] | Branch (1141:52): [True: 157k, False: 1.50k] | Branch (1141:77): [True: 157k, False: 868] ------------------ 1142| 157k| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); ------------------ | | 543| 157k| do { \ | | 544| 157k| uint32_t s1 = zxc_le32(seq_ptr); \ | | 545| 157k| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 546| 157k| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 547| 157k| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 548| 157k| seq_ptr += 4 * sizeof(uint32_t); \ | | 549| 157k| PREFETCH; \ | | 550| 157k| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 157k| do { \ | | | | 517| 157k| uint64_t ll = (S) >> 24; \ | | | | 518| 157k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 157k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.85k, False: 155k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 1.85k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 1.85k| const uint64_t reserve = (RESERVE); \ | | | | 521| 1.85k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.70k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.85k] | | | | | | | Branch (151:42): [True: 0, False: 1.85k] | | | | | | | Branch (151:42): [True: 0, False: 1.85k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 1.85k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 1.85k| ON_FAIL; \ | | | | 524| 1.85k| } \ | | | | 525| 157k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 157k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 157k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 157k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 157k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 6.78k, False: 150k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 6.78k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 6.78k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 6.78k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 6.78k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 6.78k| (size_t)(d_end - d_ptr))) \ | | | | 531| 6.78k| ON_FAIL; \ | | | | 532| 6.78k| } \ | | | | 533| 157k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 157k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 157k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1142| 157k| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 405| 157k| do { \ | | | | | | | | 406| 157k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 157k| l_ptr += ll; \ | | | | | | | | 408| 157k| d_ptr += ll; \ | | | | | | | | 409| 157k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 157k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 157k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 157k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 157k| d_ptr += ml; \ | | | | | | | | 412| 157k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 157k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 157k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 157k] | | | | ------------------ | | ------------------ | | 551| 157k| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 157k| do { \ | | | | 517| 157k| uint64_t ll = (S) >> 24; \ | | | | 518| 157k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 157k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.61k, False: 155k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 1.61k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 1.61k| const uint64_t reserve = (RESERVE); \ | | | | 521| 1.61k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.23k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.61k] | | | | | | | Branch (151:42): [True: 0, False: 1.61k] | | | | | | | Branch (151:42): [True: 0, False: 1.61k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 1.61k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 1.61k| ON_FAIL; \ | | | | 524| 1.61k| } \ | | | | 525| 157k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 157k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 157k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 157k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 157k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 6.81k, False: 150k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 6.81k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 6.81k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 6.81k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 6.81k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 6.81k| (size_t)(d_end - d_ptr))) \ | | | | 531| 6.81k| ON_FAIL; \ | | | | 532| 6.81k| } \ | | | | 533| 157k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 157k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 157k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1142| 157k| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 405| 157k| do { \ | | | | | | | | 406| 157k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 157k| l_ptr += ll; \ | | | | | | | | 408| 157k| d_ptr += ll; \ | | | | | | | | 409| 157k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 157k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 157k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 157k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 157k| d_ptr += ml; \ | | | | | | | | 412| 157k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 157k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 157k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 157k] | | | | ------------------ | | ------------------ | | 552| 157k| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 157k| do { \ | | | | 517| 157k| uint64_t ll = (S) >> 24; \ | | | | 518| 157k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 157k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.59k, False: 155k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 1.59k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 1.59k| const uint64_t reserve = (RESERVE); \ | | | | 521| 1.59k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.19k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.59k] | | | | | | | Branch (151:42): [True: 0, False: 1.59k] | | | | | | | Branch (151:42): [True: 0, False: 1.59k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 1.59k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 1.59k| ON_FAIL; \ | | | | 524| 1.59k| } \ | | | | 525| 157k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 157k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 157k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 157k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 157k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 6.99k, False: 150k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 6.99k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 6.99k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 6.99k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 6.99k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 6.99k| (size_t)(d_end - d_ptr))) \ | | | | 531| 6.99k| ON_FAIL; \ | | | | 532| 6.99k| } \ | | | | 533| 157k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 157k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 157k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1142| 157k| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 405| 157k| do { \ | | | | | | | | 406| 157k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 157k| l_ptr += ll; \ | | | | | | | | 408| 157k| d_ptr += ll; \ | | | | | | | | 409| 157k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 157k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 157k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 157k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 157k| d_ptr += ml; \ | | | | | | | | 412| 157k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 157k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 157k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 157k] | | | | ------------------ | | ------------------ | | 553| 157k| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 157k| do { \ | | | | 517| 157k| uint64_t ll = (S) >> 24; \ | | | | 518| 157k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 157k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.57k, False: 155k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 1.57k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 1.57k| const uint64_t reserve = (RESERVE); \ | | | | 521| 1.57k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.14k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.57k] | | | | | | | Branch (151:42): [True: 0, False: 1.57k] | | | | | | | Branch (151:42): [True: 0, False: 1.57k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 1.57k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 1.57k| ON_FAIL; \ | | | | 524| 1.57k| } \ | | | | 525| 157k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 157k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 157k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 157k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 157k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 6.91k, False: 150k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 6.91k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 6.91k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 6.91k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 6.91k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 6.91k| (size_t)(d_end - d_ptr))) \ | | | | 531| 6.91k| ON_FAIL; \ | | | | 532| 6.91k| } \ | | | | 533| 157k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 157k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 157k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1142| 157k| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 405| 157k| do { \ | | | | | | | | 406| 157k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 157k| l_ptr += ll; \ | | | | | | | | 408| 157k| d_ptr += ll; \ | | | | | | | | 409| 157k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 157k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 157k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 157k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 157k| d_ptr += ml; \ | | | | | | | | 412| 157k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 157k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 157k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 157k] | | | | ------------------ | | ------------------ | | 554| 157k| n_seq -= 4; \ | | 555| 157k| } while (0) | | ------------------ | | | Branch (555:14): [Folded, False: 157k] | | ------------------ ------------------ 1143| 157k| } 1144| 4.13k| } 1145| | 1146| | // --- SAFE Loop tail: remaining sequences with offset validation (1x) --- 1147| 138k| while (n_seq > 0 && d_ptr < d_end_safe && d_ptr < d_bounds) { ------------------ | Branch (1147:12): [True: 136k, False: 2.94k] | Branch (1147:25): [True: 136k, False: 0] | Branch (1147:47): [True: 134k, False: 1.18k] ------------------ 1148| 134k| uint32_t seq = zxc_le32(seq_ptr); 1149| 134k| seq_ptr += sizeof(uint32_t); 1150| | 1151| 134k| uint64_t ll = seq >> 24; 1152| 134k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 134k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 661, False: 134k] | | ------------------ ------------------ 1153| | 1154| 134k| uint32_t m_bits = (seq >> 16) & 0xFF; 1155| 134k| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 134k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1156| 134k| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 134k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 4.74k, False: 130k] | | ------------------ ------------------ 1157| | 1158| 134k| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 527| 134k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1159| | 1160| | // Strict bounds check: sequence must fit, AND wild copies must not overshoot 1161| | // Check both destination (d_ptr) and source literal stream (l_ptr) 1162| 134k| if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr) || ------------------ | | 151| 269k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 16.9k, False: 117k] | | | Branch (151:42): [True: 0, False: 134k] | | | Branch (151:42): [True: 16.9k, False: 117k] | | ------------------ ------------------ 1163| 134k| ll + ZXC_PAD_SIZE > (size_t)(l_end - l_ptr))) { 1164| | // Fallback to exact copy (slow but safe) 1165| 16.9k| if (UNLIKELY(d_ptr + ll > d_end || l_ptr + ll > l_end)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 33.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 16.9k] | | | Branch (151:42): [True: 0, False: 16.9k] | | | Branch (151:42): [True: 0, False: 16.9k] | | ------------------ ------------------ 1166| 16.9k| ZXC_MEMCPY(d_ptr, l_ptr, ll); ------------------ | | 173| 16.9k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1167| 16.9k| l_ptr += ll; 1168| 16.9k| d_ptr += ll; 1169| | 1170| 16.9k| if (UNLIKELY(d_ptr + ml > d_end)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 16.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 16.9k] | | ------------------ ------------------ 1171| 16.9k| if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET; ------------------ | | 151| 16.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 16.9k] | | ------------------ ------------------ 1172| 16.9k| const uint8_t* match_src = d_ptr - offset; 1173| | 1174| 16.9k| zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml); 1175| 16.9k| d_ptr += ml; 1176| 117k| } else { 1177| 117k| DECODE_SEQ_SAFE(ll, ml, offset); ------------------ | | 405| 117k| do { \ | | 406| 117k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | 407| 117k| l_ptr += ll; \ | | 408| 117k| d_ptr += ll; \ | | 409| 117k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | ------------------ | | | | 151| 117k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | ------------------ | | | | | Branch (151:21): [True: 0, False: 117k] | | | | ------------------ | | ------------------ | | 410| 117k| zxc_decode_copy_match(d_ptr, off, ml); \ | | 411| 117k| d_ptr += ml; \ | | 412| 117k| } while (0) | | ------------------ | | | Branch (412:14): [Folded, False: 117k] | | ------------------ ------------------ 1178| 117k| } 1179| 134k| n_seq--; 1180| 134k| } 1181| | 1182| | // --- FAST Loop: After threshold, check large margin to avoid individual bounds checks --- 1183| 4.13k| if (safe) { ------------------ | Branch (1183:9): [True: 0, False: 4.13k] ------------------ 1184| 0| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) { ------------------ | Branch (1184:16): [True: 0, False: 0] | Branch (1184:30): [True: 0, False: 0] | Branch (1184:52): [True: 0, False: 0] ------------------ 1185| 0| const uint8_t* const t_save = seq_ptr; 1186| 0| const uint8_t* const e_save = extras_ptr; 1187| 0| uint8_t* const d_save = d_ptr; 1188| 0| const uint8_t* const l_save = l_ptr; 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, ------------------ | | 543| 0| do { \ | | 544| 0| uint32_t s1 = zxc_le32(seq_ptr); \ | | 545| 0| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 546| 0| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 547| 0| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 548| 0| seq_ptr += 4 * sizeof(uint32_t); \ | | 549| 0| PREFETCH; \ | | 550| 0| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 551| 0| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 552| 0| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 553| 0| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 554| 0| n_seq -= 4; \ | | 555| 0| } while (0) | | ------------------ | | | Branch (555:14): [Folded, False: 0] | | ------------------ ------------------ 1190| 0| ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE)); 1191| 0| continue; 1192| | 1193| 0| rollback_fast_4x: 1194| 0| seq_ptr = t_save; 1195| 0| extras_ptr = e_save; 1196| 0| d_ptr = d_save; 1197| 0| l_ptr = l_save; 1198| 0| break; 1199| 0| } 1200| 4.13k| } else { 1201| 391k| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) { ------------------ | Branch (1201:16): [True: 388k, False: 3.31k] | Branch (1201:30): [True: 388k, False: 0] | Branch (1201:52): [True: 387k, False: 823] ------------------ 1202| 387k| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, ------------------ | | 543| 387k| do { \ | | 544| 387k| uint32_t s1 = zxc_le32(seq_ptr); \ | | 545| 387k| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 546| 387k| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 547| 387k| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 548| 387k| seq_ptr += 4 * sizeof(uint32_t); \ | | 549| 387k| PREFETCH; \ | | 550| 387k| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 387k| do { \ | | | | 517| 387k| uint64_t ll = (S) >> 24; \ | | | | 518| 387k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.92k, False: 385k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 1.92k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 1.92k| const uint64_t reserve = (RESERVE); \ | | | | 521| 1.92k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.85k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.92k] | | | | | | | Branch (151:42): [True: 0, False: 1.92k] | | | | | | | Branch (151:42): [True: 0, False: 1.92k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 1.92k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 1.92k| ON_FAIL; \ | | | | 524| 1.92k| } \ | | | | 525| 387k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 387k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 387k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 387k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 10.9k, False: 376k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 10.9k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 10.9k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 10.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 10.9k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 10.9k| (size_t)(d_end - d_ptr))) \ | | | | 531| 10.9k| ON_FAIL; \ | | | | 532| 10.9k| } \ | | | | 533| 387k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 387k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 387k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1202| 387k| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 417| 387k| do { \ | | | | | | | | 418| 387k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 387k| l_ptr += ll; \ | | | | | | | | 420| 387k| d_ptr += ll; \ | | | | | | | | 421| 387k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 387k| d_ptr += ml; \ | | | | | | | | 423| 387k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 387k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 387k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 387k] | | | | ------------------ | | ------------------ | | 551| 387k| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 387k| do { \ | | | | 517| 387k| uint64_t ll = (S) >> 24; \ | | | | 518| 387k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.98k, False: 385k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 1.98k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 1.98k| const uint64_t reserve = (RESERVE); \ | | | | 521| 1.98k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.96k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.98k] | | | | | | | Branch (151:42): [True: 0, False: 1.98k] | | | | | | | Branch (151:42): [True: 0, False: 1.98k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 1.98k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 1.98k| ON_FAIL; \ | | | | 524| 1.98k| } \ | | | | 525| 387k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 387k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 387k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 387k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 10.8k, False: 377k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 10.8k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 10.8k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 10.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 10.8k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 10.8k| (size_t)(d_end - d_ptr))) \ | | | | 531| 10.8k| ON_FAIL; \ | | | | 532| 10.8k| } \ | | | | 533| 387k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 387k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 387k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1202| 387k| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 417| 387k| do { \ | | | | | | | | 418| 387k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 387k| l_ptr += ll; \ | | | | | | | | 420| 387k| d_ptr += ll; \ | | | | | | | | 421| 387k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 387k| d_ptr += ml; \ | | | | | | | | 423| 387k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 387k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 387k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 387k] | | | | ------------------ | | ------------------ | | 552| 387k| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 387k| do { \ | | | | 517| 387k| uint64_t ll = (S) >> 24; \ | | | | 518| 387k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.94k, False: 385k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 1.94k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 1.94k| const uint64_t reserve = (RESERVE); \ | | | | 521| 1.94k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.88k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.94k] | | | | | | | Branch (151:42): [True: 0, False: 1.94k] | | | | | | | Branch (151:42): [True: 0, False: 1.94k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 1.94k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 1.94k| ON_FAIL; \ | | | | 524| 1.94k| } \ | | | | 525| 387k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 387k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 387k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 387k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 10.8k, False: 376k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 10.8k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 10.8k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 10.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 10.8k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 10.8k| (size_t)(d_end - d_ptr))) \ | | | | 531| 10.8k| ON_FAIL; \ | | | | 532| 10.8k| } \ | | | | 533| 387k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 387k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 387k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1202| 387k| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 417| 387k| do { \ | | | | | | | | 418| 387k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 387k| l_ptr += ll; \ | | | | | | | | 420| 387k| d_ptr += ll; \ | | | | | | | | 421| 387k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 387k| d_ptr += ml; \ | | | | | | | | 423| 387k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 387k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 387k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 387k] | | | | ------------------ | | ------------------ | | 553| 387k| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 387k| do { \ | | | | 517| 387k| uint64_t ll = (S) >> 24; \ | | | | 518| 387k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 2.06k, False: 385k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 2.06k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 2.06k| const uint64_t reserve = (RESERVE); \ | | | | 521| 2.06k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 4.12k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 2.06k] | | | | | | | Branch (151:42): [True: 0, False: 2.06k] | | | | | | | Branch (151:42): [True: 0, False: 2.06k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 2.06k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 2.06k| ON_FAIL; \ | | | | 524| 2.06k| } \ | | | | 525| 387k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 387k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 387k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 387k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 10.6k, False: 377k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 10.6k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 10.6k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 10.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 10.6k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 10.6k| (size_t)(d_end - d_ptr))) \ | | | | 531| 10.6k| ON_FAIL; \ | | | | 532| 10.6k| } \ | | | | 533| 387k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 387k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 387k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1202| 387k| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 417| 387k| do { \ | | | | | | | | 418| 387k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 387k| l_ptr += ll; \ | | | | | | | | 420| 387k| d_ptr += ll; \ | | | | | | | | 421| 387k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 387k| d_ptr += ml; \ | | | | | | | | 423| 387k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 387k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 387k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 387k] | | | | ------------------ | | ------------------ | | 554| 387k| n_seq -= 4; \ | | 555| 387k| } while (0) | | ------------------ | | | Branch (555:14): [Folded, False: 387k] | | ------------------ ------------------ 1203| 387k| ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE)); 1204| 387k| } 1205| 4.13k| } 1206| | 1207| | // --- Remaining 1 sequence (Fast Path) --- 1208| 159k| while (n_seq > 0 && d_ptr < d_end_safe && l_ptr < l_end_safe_1x) { ------------------ | Branch (1208:12): [True: 156k, False: 3.14k] | Branch (1208:25): [True: 156k, False: 0] | Branch (1208:47): [True: 155k, False: 888] ------------------ 1209| | // Save state for fallback 1210| 155k| const uint8_t* seq_save = seq_ptr; 1211| 155k| const uint8_t* ext_save = extras_ptr; 1212| | 1213| 155k| const uint32_t seq = zxc_le32(seq_ptr); 1214| 155k| seq_ptr += sizeof(uint32_t); 1215| | 1216| 155k| uint64_t ll = seq >> 24; 1217| 155k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 358, False: 155k] | | ------------------ ------------------ 1218| 358| ll += zxc_read_varint(&extras_ptr, extras_end); 1219| 358| if (UNLIKELY(l_ptr + ll > l_end)) { ------------------ | | 151| 358|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 358] | | ------------------ ------------------ 1220| 0| seq_ptr = seq_save; 1221| 0| extras_ptr = ext_save; 1222| 0| break; 1223| 0| } 1224| 358| } 1225| | 1226| 155k| uint32_t m_bits = (seq >> 16) & 0xFF; 1227| 155k| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 155k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1228| 155k| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 6.86k, False: 149k] | | ------------------ ------------------ 1229| | 1230| | // Strict bounds checks (including wild copy overrun safety) 1231| 155k| if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr) || ------------------ | | 151| 311k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 106, False: 155k] | | | Branch (151:42): [True: 0, False: 155k] | | | Branch (151:42): [True: 106, False: 155k] | | ------------------ ------------------ 1232| 155k| ll + ZXC_PAD_SIZE > (size_t)(l_end - l_ptr))) { 1233| 106| seq_ptr = seq_save; 1234| 106| extras_ptr = ext_save; 1235| 106| break; 1236| 106| } 1237| 155k| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 527| 155k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1238| | 1239| 155k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); 1240| 155k| l_ptr += ll; 1241| 155k| d_ptr += ll; 1242| | 1243| 155k| if (UNLIKELY(d_ptr < d_bounds && (size_t)(d_ptr - d_floor) < offset)) ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 155k] | | | Branch (151:42): [True: 0, False: 155k] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1244| 0| return ZXC_ERROR_BAD_OFFSET; 1245| | 1246| | /* The loop entry check guarantees ll + ml + ZXC_PAD_SIZE bytes of 1247| | * headroom, so the wild-copy ladder (incl. overlap/fill runs) is safe. */ 1248| 155k| zxc_decode_copy_match(d_ptr, offset, ml); 1249| 155k| d_ptr += ml; 1250| 155k| n_seq--; 1251| 155k| } 1252| | 1253| | // --- Safe Path for Remaining Sequences --- 1254| 159k| while (n_seq > 0) { ------------------ | Branch (1254:12): [True: 155k, False: 4.13k] ------------------ 1255| 155k| uint32_t seq = zxc_le32(seq_ptr); 1256| 155k| seq_ptr += sizeof(uint32_t); 1257| | 1258| 155k| uint64_t ll = seq >> 24; 1259| 155k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 102, False: 155k] | | ------------------ ------------------ 1260| | 1261| 155k| uint32_t m_bits = (seq >> 16) & 0xFF; 1262| 155k| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 155k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1263| 155k| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 6.70k, False: 149k] | | ------------------ ------------------ 1264| 155k| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 527| 155k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1265| | 1266| 155k| if (UNLIKELY(ll + ml > (size_t)(d_end - d_ptr) || l_ptr + ll > l_end)) ------------------ | | 151| 311k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 155k] | | | Branch (151:42): [True: 0, False: 155k] | | | Branch (151:42): [True: 0, False: 155k] | | ------------------ ------------------ 1267| 0| return ZXC_ERROR_OVERFLOW; 1268| 155k| ZXC_MEMCPY(d_ptr, l_ptr, ll); ------------------ | | 173| 155k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1269| 155k| l_ptr += ll; 1270| 155k| d_ptr += ll; 1271| | 1272| 155k| if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET; ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 155k] | | ------------------ ------------------ 1273| 155k| const uint8_t* match_src = d_ptr - offset; 1274| | 1275| 155k| zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml); 1276| 155k| d_ptr += ml; 1277| 155k| n_seq--; 1278| 155k| } 1279| | 1280| | // --- Trailing Literals --- 1281| | // Copy remaining literals from source stream (literal exhaustion) 1282| 4.13k| if (UNLIKELY(l_ptr > l_end)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 4.13k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.13k] | | ------------------ ------------------ 1283| 4.13k| const size_t remaining_literals = (size_t)(l_end - l_ptr); 1284| 4.13k| if (UNLIKELY(remaining_literals > (size_t)(d_end - d_ptr))) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 4.13k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.13k] | | ------------------ ------------------ 1285| 4.13k| ZXC_MEMCPY(d_ptr, l_ptr, remaining_literals); ------------------ | | 173| 4.13k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1286| 4.13k| d_ptr += remaining_literals; 1287| | 1288| 4.13k| return (int)(d_ptr - dst); 1289| 4.13k|} zxc_decompress.c:zxc_decode_block_ghi: 1352| 4.13k| const size_t dst_capacity) { 1353| 4.13k| return zxc_decode_block_ghi_impl(ctx, src, src_size, dst, dst_capacity, 0, 0); 1354| 4.13k|} zxc_decompress_chunk_wrapper: 488| 4.90k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 489| | /* dict_size is constant for a stream; this per-block branch (outside the decode 490| | * loop) routes to the dict variant only when a dictionary is active, so the 491| | * no-dict path runs the dict-free chunk wrapper (identical codegen to main). */ 492| 4.90k|#if ZXC_USE_C11_ATOMICS 493| 4.90k| const zxc_decompress_func_t func = atomic_load_explicit( 494| 4.90k| ctx->dict_size ? &zxc_decompress_dict_ptr : &zxc_decompress_ptr, memory_order_acquire); ------------------ | Branch (494:9): [True: 0, False: 4.90k] ------------------ 495| |#else 496| | const zxc_decompress_func_t func = 497| | ctx->dict_size ? zxc_decompress_dict_ptr : zxc_decompress_ptr; 498| |#endif 499| 4.90k| if (UNLIKELY(!func)) return zxc_decompress_dispatch_init(ctx, src, src_sz, dst, dst_cap); ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1, False: 4.90k] | | ------------------ ------------------ 500| 4.90k| return func(ctx, src, src_sz, dst, dst_cap); 501| 4.90k|} zxc_compress_chunk_wrapper: 540| 4.90k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 541| 4.90k|#if ZXC_USE_C11_ATOMICS 542| 4.90k| const zxc_compress_func_t func = atomic_load_explicit(&zxc_compress_ptr, memory_order_acquire); 543| |#else 544| | const zxc_compress_func_t func = zxc_compress_ptr; 545| |#endif 546| 4.90k| if (UNLIKELY(!func)) return zxc_compress_dispatch_init(ctx, src, src_sz, dst, dst_cap); ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1, False: 4.90k] | | ------------------ ------------------ 547| 4.90k| return func(ctx, src, src_sz, dst, dst_cap); 548| 4.90k|} zxc_create_cctx: 1253| 4.60k|zxc_cctx* zxc_create_cctx(const zxc_compress_opts_t* opts) { 1254| 4.60k| zxc_cctx* const cctx = (zxc_cctx*)ZXC_CALLOC(1, sizeof(zxc_cctx)); ------------------ | | 72| 4.60k|#define ZXC_CALLOC(nmemb, size) calloc(nmemb, size) ------------------ 1255| 4.60k| if (UNLIKELY(!cctx)) return NULL; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 1256| | 1257| | /* Resolve and store sticky defaults. */ 1258| 4.60k| cctx->stored_level = 1259| 4.60k| zxc_level_clamp((opts && opts->level > 0) ? opts->level : ZXC_LEVEL_DEFAULT); ------------------ | Branch (1259:26): [True: 4.60k, False: 0] | Branch (1259:34): [True: 4.60k, False: 0] ------------------ 1260| 4.60k| cctx->stored_block_size = 1261| 4.60k| (opts && opts->block_size > 0) ? opts->block_size : ZXC_BLOCK_SIZE_DEFAULT; ------------------ | | 60| 4.60k|#define ZXC_BLOCK_SIZE_DEFAULT (512 * 1024) ------------------ | Branch (1261:10): [True: 4.60k, False: 0] | Branch (1261:18): [True: 4.60k, False: 0] ------------------ 1262| 4.60k| cctx->stored_checksum = opts ? opts->checksum_enabled : 0; ------------------ | Branch (1262:29): [True: 4.60k, False: 0] ------------------ 1263| | 1264| 4.60k| if (opts) { ------------------ | Branch (1264:9): [True: 4.60k, False: 0] ------------------ 1265| | // LCOV_EXCL_START 1266| 4.60k| if (UNLIKELY(!zxc_validate_block_size(cctx->stored_block_size) || ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 1267| 4.60k| zxc_cctx_init(&cctx->inner, cctx->stored_block_size, 1, cctx->stored_level, 1268| 4.60k| cctx->stored_checksum, 0) != ZXC_OK)) { 1269| 0| ZXC_FREE(cctx); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 1270| 0| return NULL; 1271| 0| } 1272| | // LCOV_EXCL_STOP 1273| 4.60k| cctx->last_block_size = cctx->stored_block_size; 1274| 4.60k| cctx->initialized = 1; 1275| 4.60k| } 1276| | 1277| 4.60k| return cctx; 1278| 4.60k|} zxc_free_cctx: 1289| 4.60k|void zxc_free_cctx(zxc_cctx* cctx) { 1290| 4.60k| if (UNLIKELY(!cctx)) return; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 1291| | /* Static cctx: handle + inner buffers live inside the caller's workspace, 1292| | * which we do not own. Free is a no-op; the caller owns the workspace. */ 1293| 4.60k| if (cctx->owns_workspace) return; ------------------ | Branch (1293:9): [True: 0, False: 4.60k] ------------------ 1294| 4.60k| if (cctx->initialized) zxc_cctx_free(&cctx->inner); ------------------ | Branch (1294:9): [True: 4.60k, False: 0] ------------------ 1295| 4.60k| ZXC_FREE(cctx); ------------------ | | 84| 4.60k|#define ZXC_FREE(ptr) free(ptr) ------------------ 1296| 4.60k|} zxc_compress_block: 1619| 4.90k| const zxc_compress_opts_t* opts) { 1620| 4.90k| if (UNLIKELY(!cctx || !src || !dst || src_size == 0 || dst_capacity == 0)) ------------------ | | 151| 39.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | ------------------ ------------------ 1621| 0| return ZXC_ERROR_NULL_INPUT; 1622| | 1623| | /* Block API processes a single format-conformant block: src_size must not 1624| | * exceed ZXC_BLOCK_SIZE_MAX. Callers with larger inputs should use the 1625| | * frame or streaming APIs which chunk transparently. */ 1626| 4.90k| if (UNLIKELY(src_size > ZXC_BLOCK_SIZE_MAX)) return ZXC_ERROR_BAD_BLOCK_SIZE; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1627| | 1628| 4.90k| const int checksum_enabled = opts ? opts->checksum_enabled : cctx->stored_checksum; ------------------ | Branch (1628:34): [True: 4.90k, False: 0] ------------------ 1629| 4.90k| const int level = zxc_level_clamp((opts && opts->level > 0) ? opts->level : cctx->stored_level); ------------------ | Branch (1629:40): [True: 4.90k, False: 0] | Branch (1629:48): [True: 4.90k, False: 0] ------------------ 1630| | /* For block API, block_size == src_size (the caller compresses one block at a time). */ 1631| 4.90k| const size_t block_size = 1632| 4.90k| (opts && opts->block_size > 0) ? opts->block_size : cctx->stored_block_size; ------------------ | Branch (1632:10): [True: 4.90k, False: 0] | Branch (1632:18): [True: 4.90k, False: 0] ------------------ 1633| 4.90k| const size_t min_bs = zxc_block_size_ceil(src_size); 1634| | 1635| | /* Always ensure internal buffers can hold src_size. 1636| | * When a dictionary is active, offset_bits must accommodate dict + block. */ 1637| 4.90k| const uint8_t* b_dict = opts ? (const uint8_t*)opts->dict : NULL; ------------------ | Branch (1637:29): [True: 4.90k, False: 0] ------------------ 1638| 4.90k| const size_t b_dict_size = (opts && opts->dict) ? opts->dict_size : 0; ------------------ | Branch (1638:33): [True: 4.90k, False: 0] | Branch (1638:41): [True: 0, False: 4.90k] ------------------ 1639| 4.90k| const size_t base_block_size = (block_size > min_bs) ? block_size : min_bs; ------------------ | Branch (1639:36): [True: 4.22k, False: 687] ------------------ 1640| 4.90k| const size_t effective_block_size = 1641| 4.90k| b_dict_size > 0 ? zxc_block_size_ceil(b_dict_size + base_block_size) : base_block_size; ------------------ | Branch (1641:9): [True: 0, False: 4.90k] ------------------ 1642| | 1643| | /* Static cctx: the workspace cannot grow, so reject anything forcing a 1644| | * re-partition - another block size, or a level raise into the 1645| | * optimal-parser tier it carries no opt_scratch for. Re-initing on the heap 1646| | * would break the no-allocation contract and leak: zxc_free_cctx is a no-op 1647| | * for static contexts. */ 1648| 4.90k| if (UNLIKELY(cctx->owns_workspace && effective_block_size != cctx->last_block_size)) ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1649| 0| return ZXC_ERROR_BAD_BLOCK_SIZE; 1650| 4.90k| if (UNLIKELY(cctx->owns_workspace && level >= ZXC_LEVEL_DENSITY && !cctx->inner.opt_scratch)) ------------------ | | 151| 9.81k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1651| 0| return ZXC_ERROR_BAD_LEVEL; 1652| | 1653| 4.90k| cctx->stored_level = level; 1654| 4.90k| cctx->stored_block_size = effective_block_size; 1655| 4.90k| cctx->stored_checksum = checksum_enabled; 1656| | 1657| | /* Re-init when block_size changed, or when a per-call level raise into 1658| | * the optimal-parser tier requires the opt_scratch region that inits at 1659| | * level < ZXC_LEVEL_DENSITY do not allocate (using it NULL would crash). */ 1660| 4.90k| if (UNLIKELY(!cctx->initialized || cctx->last_block_size != effective_block_size || ------------------ | | 151| 24.5k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1661| 4.90k| (level >= ZXC_LEVEL_DENSITY && !cctx->inner.opt_scratch))) { 1662| 0| if (cctx->initialized) { ------------------ | Branch (1662:13): [True: 0, False: 0] ------------------ 1663| | // LCOV_EXCL_START 1664| 0| zxc_cctx_free(&cctx->inner); 1665| 0| cctx->initialized = 0; 1666| | // LCOV_EXCL_STOP 1667| 0| } 1668| | // LCOV_EXCL_START 1669| 0| if (UNLIKELY(zxc_cctx_init(&cctx->inner, effective_block_size, 1, level, checksum_enabled, ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1670| 0| b_dict_size) != ZXC_OK)) 1671| 0| return ZXC_ERROR_MEMORY; 1672| | // LCOV_EXCL_STOP 1673| 0| cctx->last_block_size = effective_block_size; 1674| 0| cctx->initialized = 1; 1675| 4.90k| } else { 1676| 4.90k| cctx->inner.compression_level = level; 1677| 4.90k| cctx->inner.checksum_enabled = checksum_enabled; 1678| 4.90k| } 1679| | 1680| 4.90k| cctx->inner.dict_size = b_dict_size; 1681| | 1682| 4.90k| int res; 1683| 4.90k| if (b_dict && b_dict_size > 0) { ------------------ | Branch (1683:9): [True: 0, False: 4.90k] | Branch (1683:19): [True: 0, False: 0] ------------------ 1684| | /* [dict | block] assembled in the cctx-owned dict_buffer */ 1685| 0| uint8_t* const combined = cctx->inner.dict_buffer; 1686| 0| ZXC_MEMCPY(combined, b_dict, b_dict_size); ------------------ | | 173| 0|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1687| 0| ZXC_MEMCPY(combined + b_dict_size, src, src_size); ------------------ | | 173| 0|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1688| 0| res = zxc_compress_chunk_wrapper(&cctx->inner, combined, b_dict_size + src_size, 1689| 0| (uint8_t*)dst, dst_capacity); 1690| 4.90k| } else { 1691| 4.90k| res = zxc_compress_chunk_wrapper(&cctx->inner, (const uint8_t*)src, src_size, (uint8_t*)dst, 1692| 4.90k| dst_capacity); 1693| 4.90k| } 1694| 4.90k| if (UNLIKELY(res < 0)) return res; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1695| 4.90k| return (int64_t)res; 1696| 4.90k|} zxc_dispatch.c:zxc_decompress_dispatch_init: 319| 1| const size_t dst_cap) { 320| 1| const zxc_cpu_feature_t cpu = zxc_detect_cpu_features(); 321| 1| zxc_decompress_func_t zxc_decompress_ptr_local = NULL; 322| 1| zxc_decompress_func_t zxc_decompress_dict_ptr_local = NULL; 323| | 324| |#ifndef ZXC_ONLY_DEFAULT 325| |#if defined(__x86_64__) || defined(_M_X64) 326| | if (cpu == ZXC_CPU_AVX512) { 327| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_avx512; 328| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_avx512; 329| | } else if (cpu == ZXC_CPU_AVX2) { 330| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_avx2; 331| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_avx2; 332| | } else { 333| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; 334| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_default; 335| | } 336| |#elif defined(__arm__) || defined(_M_ARM) 337| | // 32-bit ARM: the only arch with a real runtime NEON probe (getauxval). 338| | // cppcheck-suppress knownConditionTrueFalse 339| | if (cpu == ZXC_CPU_NEON) { 340| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_neon32; 341| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_neon32; 342| | } else { 343| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; 344| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_default; 345| | } 346| |#else 347| | (void)cpu; 348| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; 349| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_default; 350| |#endif 351| |#else 352| 1| (void)cpu; 353| 1| zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; 354| 1| zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_default; 355| 1|#endif 356| | 357| 1|#if ZXC_USE_C11_ATOMICS 358| 1| atomic_store_explicit(&zxc_decompress_ptr, zxc_decompress_ptr_local, memory_order_release); 359| 1| atomic_store_explicit(&zxc_decompress_dict_ptr, zxc_decompress_dict_ptr_local, 360| 1| memory_order_release); 361| |#else 362| | zxc_decompress_ptr = zxc_decompress_ptr_local; 363| | zxc_decompress_dict_ptr = zxc_decompress_dict_ptr_local; 364| |#endif 365| 1| return (ctx->dict_size ? zxc_decompress_dict_ptr_local : zxc_decompress_ptr_local)( ------------------ | Branch (365:13): [True: 0, False: 1] ------------------ 366| 1| ctx, src, src_sz, dst, dst_cap); 367| 1|} zxc_dispatch.c:zxc_detect_cpu_features: 206| 2|static zxc_cpu_feature_t zxc_detect_cpu_features(void) { 207| 2|#ifdef ZXC_ONLY_DEFAULT 208| 2| return ZXC_CPU_GENERIC; 209| |#else 210| | zxc_cpu_feature_t features = ZXC_CPU_GENERIC; 211| | 212| |#if defined(__x86_64__) || defined(_M_X64) 213| | // AVX2/AVX512 need OS-enabled YMM/ZMM state: gate on OSXSAVE + XGETBV/XCR0, 214| | // not CPUID alone (else a VEX/EVEX op faults #UD when the OS hasn't enabled it). 215| | uint32_t regs[4]; 216| | int sse2 = 0; 217| | int avx2 = 0; 218| | int avx512 = 0; 219| | int bmi_lzcnt = 0; 220| | 221| | zxc_cpuid(0, 0, regs); 222| | const uint32_t max_leaf = regs[0]; 223| | 224| | zxc_cpuid(1, 0, regs); 225| | if (regs[3] & (1U << 26)) sse2 = 1; // CPUID.1:EDX[26] 226| | if ((regs[2] & (1U << 27)) && max_leaf >= 7) { // OSXSAVE, and leaf 7 is real 227| | const uint64_t xcr0 = zxc_xgetbv0(); 228| | if ((xcr0 & 0x6) == 0x6) { // SSE+YMM enabled 229| | zxc_cpuid(7, 0, regs); 230| | const int bmi1 = (regs[1] >> 3) & 1; // CPUID.7.0:EBX[3] 231| | const int bmi2 = (regs[1] >> 8) & 1; // CPUID.7.0:EBX[8] 232| | if (regs[1] & (1U << 5)) avx2 = 1; 233| | // AVX512 also needs XCR0[5..7] (opmask/ZMM) 234| | if ((regs[1] & (1U << 16)) && (regs[1] & (1U << 30)) && (regs[2] & (1U << 6)) && 235| | (xcr0 & 0xE0) == 0xE0) 236| | avx512 = 1; /* AVX512 tier = F+BW+VBMI2 (variant built with -mavx512vbmi2) */ 237| | // The AVX2/AVX512 variants are compiled with BMI1/BMI2/LZCNT enabled, 238| | // so both gates must prove those bits too. LZCNT (ABM) lives in 239| | // CPUID.80000001H:ECX[5]; that leaf is architectural on x86-64. 240| | zxc_cpuid(0x80000001U, 0, regs); 241| | bmi_lzcnt = bmi1 && bmi2 && ((regs[2] >> 5) & 1); 242| | } 243| | } 244| | 245| | if (avx512 && bmi_lzcnt) { 246| | features = ZXC_CPU_AVX512; 247| | } else if (avx2 && bmi_lzcnt) { 248| | features = ZXC_CPU_AVX2; 249| | } else if (sse2) { 250| | features = ZXC_CPU_SSE2; 251| | } 252| | 253| |#elif defined(__aarch64__) || defined(_M_ARM64) 254| | // ARM64 usually guarantees NEON 255| | features = ZXC_CPU_NEON; 256| | 257| |#elif defined(__arm__) || defined(_M_ARM) 258| | // ARM32 Runtime detection for Linux 259| |#if defined(__linux__) 260| | const unsigned long hwcaps = getauxval(AT_HWCAP); 261| | if (hwcaps & HWCAP_NEON) { 262| | features = ZXC_CPU_NEON; 263| | } 264| |#else 265| |// Fallback for non-Linux: rely on compiler flags. 266| |// If compiled with -mfpu=neon, we assume target supports it. 267| |// Otherwise, safe default is GENERIC. 268| |#if defined(__ARM_NEON) 269| | features = ZXC_CPU_NEON; 270| |#endif 271| |#endif 272| |#endif 273| | 274| | return features; 275| |#endif 276| 2|} zxc_dispatch.c:zxc_compress_dispatch_init: 441| 1| const size_t dst_cap) { 442| 1| const zxc_cpu_feature_t cpu = zxc_detect_cpu_features(); 443| 1| zxc_compress_func_t zxc_compress_ptr_local = NULL; 444| | 445| |#ifndef ZXC_ONLY_DEFAULT 446| |#if defined(__x86_64__) || defined(_M_X64) 447| | if (cpu == ZXC_CPU_AVX512) 448| | zxc_compress_ptr_local = zxc_compress_chunk_wrapper_avx512; 449| | else if (cpu == ZXC_CPU_AVX2) 450| | zxc_compress_ptr_local = zxc_compress_chunk_wrapper_avx2; 451| | else 452| | zxc_compress_ptr_local = zxc_compress_chunk_wrapper_default; 453| |#elif defined(__arm__) || defined(_M_ARM) 454| | // cppcheck-suppress knownConditionTrueFalse 455| | if (cpu == ZXC_CPU_NEON) 456| | zxc_compress_ptr_local = zxc_compress_chunk_wrapper_neon32; 457| | else 458| | zxc_compress_ptr_local = zxc_compress_chunk_wrapper_default; 459| |#else 460| | (void)cpu; 461| | zxc_compress_ptr_local = zxc_compress_chunk_wrapper_default; 462| |#endif 463| |#else 464| 1| (void)cpu; 465| 1| zxc_compress_ptr_local = zxc_compress_chunk_wrapper_default; 466| 1|#endif 467| | 468| 1|#if ZXC_USE_C11_ATOMICS 469| 1| atomic_store_explicit(&zxc_compress_ptr, zxc_compress_ptr_local, memory_order_release); 470| |#else 471| | zxc_compress_ptr = zxc_compress_ptr_local; 472| |#endif 473| 1| return zxc_compress_ptr_local(ctx, src, src_sz, dst, dst_cap); 474| 1|} zxc_common.c:zxc_log2_u32: 856| 23.0k|static ZXC_ALWAYS_INLINE uint32_t zxc_log2_u32(const uint32_t v) { 857| |#ifdef _MSC_VER 858| | unsigned long index; 859| | return (v == 0) ? 0 : (_BitScanReverse(&index, v) ? index : 0); 860| |#else 861| 23.0k| return (v == 0) ? 0 : (uint32_t)(31 - __builtin_clz(v)); ------------------ | Branch (861:12): [True: 0, False: 23.0k] ------------------ 862| 23.0k|#endif 863| 23.0k|} zxc_common.c:zxc_store_le32: 1160| 31.1k|static ZXC_ALWAYS_INLINE void zxc_store_le32(void* p, const uint32_t v) { 1161| |#ifdef ZXC_BIG_ENDIAN 1162| | const uint32_t s = ZXC_BSWAP32(v); 1163| | ZXC_MEMCPY(p, &s, sizeof(s)); 1164| |#else 1165| 31.1k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 173| 31.1k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1166| 31.1k|#endif 1167| 31.1k|} zxc_common.c:zxc_store_le16: 1139| 9.21k|static ZXC_ALWAYS_INLINE void zxc_store_le16(void* p, const uint16_t v) { 1140| |#ifdef ZXC_BIG_ENDIAN 1141| | const uint16_t s = ZXC_BSWAP16(v); 1142| | ZXC_MEMCPY(p, &s, sizeof(s)); 1143| |#else 1144| 9.21k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 173| 9.21k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1145| 9.21k|#endif 1146| 9.21k|} zxc_common.c:zxc_hash16: 1219| 9.24k|static ZXC_ALWAYS_INLINE uint16_t zxc_hash16(const uint8_t* p) { 1220| 9.24k| const uint64_t v1 = zxc_le64(p); 1221| 9.24k| const uint64_t v2 = zxc_le64(p + 8); 1222| 9.24k| uint64_t h = v1 ^ v2 ^ ZXC_HASH_PRIME2; ------------------ | | 559| 9.24k|#define ZXC_HASH_PRIME2 0xD2D84A61D2D84A61ULL ------------------ 1223| 9.24k| h ^= h << 13; 1224| 9.24k| h ^= h >> 7; 1225| 9.24k| h ^= h << 17; 1226| 9.24k| const uint32_t res = (uint32_t)((h >> 32) ^ h); 1227| 9.24k| return (uint16_t)((res >> 16) ^ res); 1228| 9.24k|} zxc_common.c:zxc_le32: 1094| 25.5k|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1095| 25.5k| uint32_t v; 1096| 25.5k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 25.5k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1097| |#ifdef ZXC_BIG_ENDIAN 1098| | return ZXC_BSWAP32(v); 1099| |#else 1100| 25.5k| return v; 1101| 25.5k|#endif 1102| 25.5k|} zxc_common.c:zxc_le16: 1074| 4.64k|static ZXC_ALWAYS_INLINE uint16_t zxc_le16(const void* p) { 1075| 4.64k| uint16_t v; 1076| 4.64k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 4.64k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1077| |#ifdef ZXC_BIG_ENDIAN 1078| | return ZXC_BSWAP16(v); 1079| |#else 1080| 4.64k| return v; 1081| 4.64k|#endif 1082| 4.64k|} zxc_common.c:zxc_hash8: 1200| 21.6k|static ZXC_ALWAYS_INLINE uint8_t zxc_hash8(const uint8_t* p) { 1201| 21.6k| const uint64_t v = zxc_le64(p); 1202| 21.6k| uint64_t h = v ^ ZXC_HASH_PRIME1; ------------------ | | 557| 21.6k|#define ZXC_HASH_PRIME1 0x9E3779B97F4A7C15ULL ------------------ 1203| 21.6k| h ^= h << 13; 1204| 21.6k| h ^= h >> 7; 1205| 21.6k| h ^= h << 17; 1206| 21.6k| return (uint8_t)((h >> 32) ^ h); 1207| 21.6k|} zxc_common.c:zxc_store_le64: 1183| 19.3k|static ZXC_ALWAYS_INLINE void zxc_store_le64(void* p, const uint64_t v) { 1184| |#ifdef ZXC_BIG_ENDIAN 1185| | const uint64_t s = ZXC_BSWAP64(v); 1186| | ZXC_MEMCPY(p, &s, sizeof(s)); 1187| |#else 1188| 19.3k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 173| 19.3k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1189| 19.3k|#endif 1190| 19.3k|} zxc_common.c:zxc_le64: 1114| 52.5k|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1115| 52.5k| uint64_t v; 1116| 52.5k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 52.5k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1117| |#ifdef ZXC_BIG_ENDIAN 1118| | return ZXC_BSWAP64(v); 1119| |#else 1120| 52.5k| return v; 1121| 52.5k|#endif 1122| 52.5k|} zxc_compress.c:zxc_get_lz77_params: 950| 4.90k|static ZXC_ALWAYS_INLINE zxc_lz77_params_t zxc_get_lz77_params(const int level) { 951| 4.90k| if (level >= ZXC_LEVEL_ULTRA) return (zxc_lz77_params_t){128, 256, 0, 0, 0, 1, 8}; ------------------ | Branch (951:9): [True: 0, False: 4.90k] ------------------ 952| | // search_depth, sufficient_len, use_lazy, lazy_attempts, lazy_len_threshold, step_base, 953| | // step_shift 954| 4.90k| static const zxc_lz77_params_t table[7] = { 955| 4.90k| {3, 16, 0, 0, 0, 4, 4}, // fallback 956| 4.90k| {3, 16, 0, 0, 0, 4, 4}, // level 1 957| 4.90k| {3, 18, 0, 0, 0, 3, 6}, // level 2 958| 4.90k| {3, 16, 1, 4, 128, 1, 4}, // level 3 959| 4.90k| {3, 18, 1, 4, 128, 1, 5}, // level 4 960| 4.90k| {64, 256, 1, 16, 128, 1, 8}, // level 5 961| 4.90k| {64, 256, 0, 0, 0, 1, 8} // level 6 962| 4.90k| }; 963| 4.90k| return table[level < ZXC_LEVEL_FASTEST ? ZXC_LEVEL_FASTEST : level]; ------------------ | Branch (963:18): [True: 0, False: 4.90k] ------------------ 964| 4.90k|} zxc_compress.c:zxc_le64: 1114| 32.2M|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1115| 32.2M| uint64_t v; 1116| 32.2M| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 32.2M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1117| |#ifdef ZXC_BIG_ENDIAN 1118| | return ZXC_BSWAP64(v); 1119| |#else 1120| 32.2M| return v; 1121| 32.2M|#endif 1122| 32.2M|} zxc_compress.c:zxc_le32: 1094| 6.87M|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1095| 6.87M| uint32_t v; 1096| 6.87M| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 6.87M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1097| |#ifdef ZXC_BIG_ENDIAN 1098| | return ZXC_BSWAP32(v); 1099| |#else 1100| 6.87M| return v; 1101| 6.87M|#endif 1102| 6.87M|} zxc_compress.c:zxc_ctz64: 1319| 3.47M|static ZXC_ALWAYS_INLINE int zxc_ctz64(const uint64_t x) { 1320| 3.47M| if (x == 0) return 64; ------------------ | Branch (1320:9): [True: 0, False: 3.47M] ------------------ 1321| 3.47M|#if defined(__GNUC__) || defined(__clang__) 1322| 3.47M| return __builtin_ctzll(x); 1323| |#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) 1324| | unsigned long r; 1325| | _BitScanForward64(&r, x); 1326| | return (int)r; 1327| |#elif defined(_MSC_VER) 1328| | // Use two 32-bit scans to avoid fragile 64-bit De Bruijn multiplication. 1329| | unsigned long r; 1330| | const uint32_t lo = (uint32_t)x; 1331| | if (_BitScanForward(&r, lo)) return (int)r; 1332| | _BitScanForward(&r, (uint32_t)(x >> 32)); 1333| | return 32 + (int)r; 1334| |#else 1335| | // Fallback De Bruijn for non-GCC/non-MSVC compilers 1336| | static const int Debruijn64[64] = { 1337| | 0, 1, 48, 2, 57, 49, 28, 3, 61, 58, 50, 42, 38, 29, 17, 4, 62, 55, 59, 36, 53, 51, 1338| | 43, 22, 45, 39, 33, 30, 24, 18, 12, 5, 63, 47, 56, 27, 60, 41, 37, 16, 54, 35, 52, 21, 1339| | 44, 32, 23, 11, 46, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6}; 1340| | return Debruijn64[((x & (0ULL - x)) * 0x03F79D71B4CA8B09ULL) >> 58]; 1341| |#endif 1342| 3.47M|} zxc_compress.c:zxc_ctz32: 1289| 251k|static ZXC_ALWAYS_INLINE int zxc_ctz32(const uint32_t x) { 1290| 251k| if (x == 0) return 32; ------------------ | Branch (1290:9): [True: 0, False: 251k] ------------------ 1291| 251k|#if defined(__GNUC__) || defined(__clang__) 1292| 251k| return __builtin_ctz(x); 1293| |#elif defined(_MSC_VER) 1294| | unsigned long r; 1295| | _BitScanForward(&r, x); 1296| | return (int)r; 1297| |#else 1298| | // Fallback De Bruijn (32 bits) 1299| | static const int DeBruijn32[32] = {0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 1300| | 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 1301| | 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; 1302| | return DeBruijn32[((uint32_t)((x & (0U - x)) * 0x077CB531U)) >> 27]; 1303| |#endif 1304| 251k|} zxc_compress.c:zxc_copy32: 1258| 1.09M|static ZXC_ALWAYS_INLINE void zxc_copy32(void* dst, const void* src) { 1259| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 1260| | // AVX2/AVX512: Single 256-bit (32 byte) unaligned load/store 1261| | _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src)); 1262| |#elif defined(ZXC_USE_SSE2) 1263| | // SSE2: Two 128-bit (16 byte) unaligned load/stores (no 256-bit regs) 1264| 1.09M| _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); 1265| 1.09M| _mm_storeu_si128((__m128i*)((uint8_t*)dst + 16), 1266| 1.09M| _mm_loadu_si128((const __m128i*)((const uint8_t*)src + 16))); 1267| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1268| | // NEON: Two 128-bit (16 byte) unaligned load/stores 1269| | vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); 1270| | vst1q_u8((uint8_t*)dst + 16, vld1q_u8((const uint8_t*)src + 16)); 1271| |#else 1272| | ZXC_MEMCPY(dst, src, 32); 1273| |#endif 1274| 1.09M|} zxc_compress.c:zxc_checksum: 1390| 1.63k| const uint8_t hash_method) { 1391| 1.63k| (void)hash_method; /* single algorithm for now; extend when adding more */ 1392| 1.63k| const uint64_t hash = rapidhash(input, len); 1393| | 1394| | return (uint32_t)(hash ^ (hash >> (sizeof(uint32_t) * CHAR_BIT))); 1395| 1.63k|} zxc_compress.c:zxc_store_le32: 1160| 1.63k|static ZXC_ALWAYS_INLINE void zxc_store_le32(void* p, const uint32_t v) { 1161| |#ifdef ZXC_BIG_ENDIAN 1162| | const uint32_t s = ZXC_BSWAP32(v); 1163| | ZXC_MEMCPY(p, &s, sizeof(s)); 1164| |#else 1165| 1.63k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 173| 1.63k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1166| 1.63k|#endif 1167| 1.63k|} zxc_decompress.c:zxc_le32: 1094| 2.63M|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1095| 2.63M| uint32_t v; 1096| 2.63M| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 2.63M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1097| |#ifdef ZXC_BIG_ENDIAN 1098| | return ZXC_BSWAP32(v); 1099| |#else 1100| 2.63M| return v; 1101| 2.63M|#endif 1102| 2.63M|} zxc_decompress.c:zxc_checksum: 1390| 1.63k| const uint8_t hash_method) { 1391| 1.63k| (void)hash_method; /* single algorithm for now; extend when adding more */ 1392| 1.63k| const uint64_t hash = rapidhash(input, len); 1393| | 1394| | return (uint32_t)(hash ^ (hash >> (sizeof(uint32_t) * CHAR_BIT))); 1395| 1.63k|} zxc_decompress.c:zxc_copy32: 1258| 13.6M|static ZXC_ALWAYS_INLINE void zxc_copy32(void* dst, const void* src) { 1259| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 1260| | // AVX2/AVX512: Single 256-bit (32 byte) unaligned load/store 1261| | _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src)); 1262| |#elif defined(ZXC_USE_SSE2) 1263| | // SSE2: Two 128-bit (16 byte) unaligned load/stores (no 256-bit regs) 1264| 13.6M| _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); 1265| 13.6M| _mm_storeu_si128((__m128i*)((uint8_t*)dst + 16), 1266| 13.6M| _mm_loadu_si128((const __m128i*)((const uint8_t*)src + 16))); 1267| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1268| | // NEON: Two 128-bit (16 byte) unaligned load/stores 1269| | vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); 1270| | vst1q_u8((uint8_t*)dst + 16, vld1q_u8((const uint8_t*)src + 16)); 1271| |#else 1272| | ZXC_MEMCPY(dst, src, 32); 1273| |#endif 1274| 13.6M|} zxc_decompress.c:zxc_copy16: 1239| 1.85M|static ZXC_ALWAYS_INLINE void zxc_copy16(void* dst, const void* src) { 1240| 1.85M|#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) || defined(ZXC_USE_SSE2) 1241| | // x86 SSE2/AVX2/AVX512: Single 128-bit unaligned load/store 1242| 1.85M| _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); 1243| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1244| | vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); 1245| |#else 1246| | ZXC_MEMCPY(dst, src, 16); 1247| |#endif 1248| 1.85M|} zxc_dispatch.c:zxc_level_clamp: 800| 9.51k|static inline int zxc_level_clamp(const int level) { 801| 9.51k| return (level > ZXC_LEVEL_ULTRA) ? ZXC_LEVEL_ULTRA : level; ------------------ | Branch (801:12): [True: 0, False: 9.51k] ------------------ 802| 9.51k|} zxc_dispatch.c:zxc_validate_block_size: 889| 4.60k|static ZXC_ALWAYS_INLINE int zxc_validate_block_size(const size_t bs) { 890| 4.60k| return bs >= ZXC_BLOCK_SIZE_MIN && bs <= ZXC_BLOCK_SIZE_MAX && (bs & (bs - 1)) == 0; ------------------ | | 62| 9.21k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 4.60k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ return bs >= ZXC_BLOCK_SIZE_MIN && bs <= ZXC_BLOCK_SIZE_MAX && (bs & (bs - 1)) == 0; ------------------ | | 64| 9.21k|#define ZXC_BLOCK_SIZE_MAX (1U << ZXC_BLOCK_SIZE_MAX_LOG2) | | ------------------ | | | | 58| 4.60k|#define ZXC_BLOCK_SIZE_MAX_LOG2 21 | | ------------------ ------------------ | Branch (890:12): [True: 4.60k, False: 0] | Branch (890:40): [True: 4.60k, False: 0] | Branch (890:68): [True: 4.60k, False: 0] ------------------ 891| 4.60k|} zxc_dispatch.c:zxc_block_size_ceil: 870| 4.90k|static ZXC_ALWAYS_INLINE size_t zxc_block_size_ceil(const size_t v) { 871| 4.90k| uint64_t x = (uint64_t)v - 1; 872| 4.90k| x |= x >> 1; 873| 4.90k| x |= x >> 2; 874| 4.90k| x |= x >> 4; 875| 4.90k| x |= x >> 8; 876| 4.90k| x |= x >> 16; 877| 4.90k| x |= x >> 32; 878| 4.90k| x++; 879| 4.90k| const size_t bs = (size_t)x; 880| 4.90k| return (bs < ZXC_BLOCK_SIZE_MIN) ? ZXC_BLOCK_SIZE_MIN : bs; ------------------ | | 62| 4.90k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 4.90k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ return (bs < ZXC_BLOCK_SIZE_MIN) ? ZXC_BLOCK_SIZE_MIN : bs; ------------------ | | 62| 2.54k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 2.54k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ | Branch (880:12): [True: 2.54k, False: 2.36k] ------------------ 881| 4.90k|} zxc_pstream.c:zxc_level_clamp: 800| 4.60k|static inline int zxc_level_clamp(const int level) { 801| 4.60k| return (level > ZXC_LEVEL_ULTRA) ? ZXC_LEVEL_ULTRA : level; ------------------ | Branch (801:12): [True: 0, False: 4.60k] ------------------ 802| 4.60k|} zxc_pstream.c:zxc_le32: 1094| 4.29k|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1095| 4.29k| uint32_t v; 1096| 4.29k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 4.29k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1097| |#ifdef ZXC_BIG_ENDIAN 1098| | return ZXC_BSWAP32(v); 1099| |#else 1100| 4.29k| return v; 1101| 4.29k|#endif 1102| 4.29k|} zxc_pstream.c:zxc_hash_combine_rotate: 1430| 3.27k| const uint32_t block_hash) { 1431| 3.27k| return ((hash << 1) | (hash >> 31)) ^ block_hash; 1432| 3.27k|} zxc_pstream.c:zxc_le64: 1114| 3.11k|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1115| 3.11k| uint64_t v; 1116| 3.11k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 3.11k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1117| |#ifdef ZXC_BIG_ENDIAN 1118| | return ZXC_BSWAP64(v); 1119| |#else 1120| 3.11k| return v; 1121| 3.11k|#endif 1122| 3.11k|} zxc_cstream_create: 269| 4.60k|zxc_cstream* zxc_cstream_create(const zxc_compress_opts_t* opts) { 270| 4.60k| zxc_cstream* cs = (zxc_cstream*)ZXC_CALLOC(1, sizeof(*cs)); ------------------ | | 72| 4.60k|#define ZXC_CALLOC(nmemb, size) calloc(nmemb, size) ------------------ 271| 4.60k| if (UNLIKELY(!cs)) return NULL; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 272| | 273| 4.60k| if (opts) cs->opts = *opts; ------------------ | Branch (273:9): [True: 4.60k, False: 0] ------------------ 274| | 275| | /* The push-stream format carries no dict_id, so a dictionary here would 276| | * produce archives that decode wrong (or not at all) elsewhere. */ 277| 4.60k| if (UNLIKELY(cs->opts.dict || cs->opts.dict_size || cs->opts.dict_huf)) { ------------------ | | 151| 18.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 278| 0| ZXC_FREE(cs); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 279| 0| return NULL; 280| 0| } 281| | 282| 4.60k| if (cs->opts.level == 0) cs->opts.level = ZXC_LEVEL_DEFAULT; ------------------ | Branch (282:9): [True: 0, False: 4.60k] ------------------ 283| 4.60k| cs->opts.level = zxc_level_clamp(cs->opts.level); 284| 4.60k| if (cs->opts.block_size == 0) cs->opts.block_size = ZXC_BLOCK_SIZE_DEFAULT; ------------------ | | 60| 4.60k|#define ZXC_BLOCK_SIZE_DEFAULT (512 * 1024) ------------------ | Branch (284:9): [True: 4.60k, False: 0] ------------------ 285| | /* n_threads is ignored on this single-threaded path. */ 286| 4.60k| cs->opts.n_threads = 0; 287| 4.60k| cs->opts.progress_cb = NULL; 288| 4.60k| cs->opts.user_data = NULL; 289| 4.60k| cs->opts.seekable = 0; 290| 4.60k| cs->block_size = cs->opts.block_size; 291| | 292| 4.60k| cs->cctx = zxc_create_cctx(&cs->opts); 293| | // LCOV_EXCL_START 294| 4.60k| if (UNLIKELY(!cs->cctx)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 295| 0| ZXC_FREE(cs); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 296| 0| return NULL; 297| 0| } 298| 4.60k| cs->in_block = (uint8_t*)ZXC_MALLOC(cs->block_size); ------------------ | | 66| 4.60k|#define ZXC_MALLOC(size) malloc(size) ------------------ 299| 4.60k| if (UNLIKELY(!cs->in_block)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 300| 0| zxc_free_cctx(cs->cctx); 301| 0| ZXC_FREE(cs); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 302| 0| return NULL; 303| 0| } 304| | // LCOV_EXCL_STOP 305| | /* Pre-size pending so the file header path never needs realloc. */ 306| 4.60k| cs->pending_cap = 307| 4.60k| ZXC_FILE_HEADER_SIZE > ZXC_FILE_FOOTER_SIZE ? ZXC_FILE_HEADER_SIZE : ZXC_FILE_FOOTER_SIZE; ------------------ | | 103| 4.60k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ ZXC_FILE_HEADER_SIZE > ZXC_FILE_FOOTER_SIZE ? ZXC_FILE_HEADER_SIZE : ZXC_FILE_FOOTER_SIZE; ------------------ | | 105| 4.60k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ ZXC_FILE_HEADER_SIZE > ZXC_FILE_FOOTER_SIZE ? ZXC_FILE_HEADER_SIZE : ZXC_FILE_FOOTER_SIZE; ------------------ | | 103| 4.60k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ ZXC_FILE_HEADER_SIZE > ZXC_FILE_FOOTER_SIZE ? ZXC_FILE_HEADER_SIZE : ZXC_FILE_FOOTER_SIZE; ------------------ | | 105| 4.60k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ | Branch (307:9): [True: 4.60k, Folded] ------------------ 308| 4.60k| cs->pending = (uint8_t*)ZXC_MALLOC(cs->pending_cap); ------------------ | | 66| 4.60k|#define ZXC_MALLOC(size) malloc(size) ------------------ 309| | // LCOV_EXCL_START 310| 4.60k| if (UNLIKELY(!cs->pending)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 311| 0| ZXC_FREE(cs->in_block); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 312| 0| zxc_free_cctx(cs->cctx); 313| 0| ZXC_FREE(cs); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 314| 0| return NULL; 315| 0| } 316| | // LCOV_EXCL_STOP 317| 4.60k| cs->state = CS_INIT; 318| 4.60k| return cs; 319| 4.60k|} zxc_cstream_free: 401| 4.60k|void zxc_cstream_free(zxc_cstream* cs) { 402| 4.60k| if (!cs) return; ------------------ | Branch (402:9): [True: 0, False: 4.60k] ------------------ 403| 4.60k| ZXC_FREE(cs->pending); ------------------ | | 84| 4.60k|#define ZXC_FREE(ptr) free(ptr) ------------------ 404| 4.60k| ZXC_FREE(cs->in_block); ------------------ | | 84| 4.60k|#define ZXC_FREE(ptr) free(ptr) ------------------ 405| 4.60k| zxc_free_cctx(cs->cctx); 406| 4.60k| ZXC_FREE(cs); ------------------ | | 84| 4.60k|#define ZXC_FREE(ptr) free(ptr) ------------------ 407| 4.60k|} zxc_cstream_compress: 453| 109M|int64_t zxc_cstream_compress(zxc_cstream* cs, zxc_outbuf_t* out, zxc_inbuf_t* in) { 454| 109M| if (UNLIKELY(!cs || !out || !in || in->pos > in->size || out->pos > out->size || ------------------ | | 151| 1.96G|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 109M, False: 83.7k] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 109M, False: 0] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | ------------------ ------------------ 455| 109M| (in->size > in->pos && !in->src) || (out->size > out->pos && !out->dst) || 456| 109M| cs->state == CS_DONE)) { 457| 0| return ZXC_ERROR_NULL_INPUT; 458| 0| } 459| 109M| if (UNLIKELY(cs->state == CS_ERRORED)) return cs->error_code; ------------------ | | 151| 109M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 109M] | | ------------------ ------------------ 460| | 461| 109M| for (;;) { 462| 109M| switch (cs->state) { ------------------ | Branch (462:17): [True: 109M, False: 0] ------------------ 463| 4.60k| case CS_INIT: { ------------------ | Branch (463:13): [True: 4.60k, False: 109M] ------------------ 464| 4.60k| const int rc = cs_stage_file_header(cs); 465| 4.60k| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 466| 4.60k| cs->state = CS_DRAIN_HEADER; 467| 4.60k| break; 468| 4.60k| } 469| | 470| 4.60k| case CS_DRAIN_HEADER: ------------------ | Branch (470:13): [True: 4.60k, False: 109M] ------------------ 471| 323k| case CS_DRAIN_BLOCK: { ------------------ | Branch (471:13): [True: 318k, False: 109M] ------------------ 472| 323k| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (472:21): [True: 318k, False: 4.91k] ------------------ 473| 4.91k| cs->state = CS_ACCUMULATE; 474| 4.91k| break; 475| 323k| } 476| | 477| 109M| case CS_ACCUMULATE: { ------------------ | Branch (477:13): [True: 109M, False: 328k] ------------------ 478| 109M| const size_t avail_in = in->size - in->pos; 479| 109M| if (cs->in_used == 0 && avail_in >= cs->block_size) { ------------------ | Branch (479:21): [True: 4.99k, False: 109M] | Branch (479:41): [True: 0, False: 4.99k] ------------------ 480| 0| const int rc = cs_compress_block_from(cs, (const uint8_t*)in->src + in->pos, 481| 0| cs->block_size); 482| 0| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 483| 0| in->pos += cs->block_size; 484| 0| cs->state = CS_DRAIN_BLOCK; 485| 0| break; 486| 0| } 487| | 488| 109M| const size_t room = cs->block_size - cs->in_used; 489| 109M| const size_t n = avail_in < room ? avail_in : room; ------------------ | Branch (489:34): [True: 109M, False: 313] ------------------ 490| 109M| if (n) { ------------------ | Branch (490:21): [True: 109M, False: 90] ------------------ 491| 109M| ZXC_MEMCPY(cs->in_block + cs->in_used, (const uint8_t*)in->src + in->pos, n); ------------------ | | 173| 109M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 492| 109M| in->pos += n; 493| 109M| cs->in_used += n; 494| 109M| } 495| | 496| 109M| if (cs->in_used == cs->block_size) { ------------------ | Branch (496:21): [True: 313, False: 109M] ------------------ 497| 313| const int rc = cs_compress_one_block(cs); 498| 313| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 313|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 313] | | ------------------ ------------------ 499| 313| cs->state = CS_DRAIN_BLOCK; 500| 313| break; 501| 313| } 502| | /* Block not yet full either in is empty or we made no progress. */ 503| 109M| return 0; 504| 109M| } 505| | 506| 0| case CS_DRAIN_LAST: ------------------ | Branch (506:13): [True: 0, False: 109M] ------------------ 507| 0| case CS_DRAIN_EOF: ------------------ | Branch (507:13): [True: 0, False: 109M] ------------------ 508| 0| case CS_DRAIN_FOOTER: ------------------ | Branch (508:13): [True: 0, False: 109M] ------------------ 509| 0| case CS_DONE: ------------------ | Branch (509:13): [True: 0, False: 109M] ------------------ 510| 0| case CS_ERRORED: ------------------ | Branch (510:13): [True: 0, False: 109M] ------------------ 511| | /* These states are owned by _end(). */ 512| 0| return ZXC_ERROR_NULL_INPUT; 513| 109M| } 514| 109M| } 515| 109M|} zxc_cstream_end: 532| 510k|int64_t zxc_cstream_end(zxc_cstream* cs, zxc_outbuf_t* out) { 533| 510k| if (UNLIKELY(!cs || !out || cs->state == CS_DONE)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 2.04M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 510k] | | | Branch (151:42): [True: 0, False: 510k] | | | Branch (151:42): [True: 0, False: 510k] | | | Branch (151:42): [True: 0, False: 510k] | | ------------------ ------------------ 534| 510k| if (UNLIKELY(cs->state == CS_ERRORED)) return cs->error_code; ------------------ | | 151| 510k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 510k] | | ------------------ ------------------ 535| | 536| 524k| for (;;) { 537| 524k| switch (cs->state) { ------------------ | Branch (537:17): [True: 524k, False: 0] ------------------ 538| 0| case CS_INIT: { ------------------ | Branch (538:13): [True: 0, False: 524k] ------------------ 539| | /* _end before any input, still need to emit file header. */ 540| 0| const int rc = cs_stage_file_header(cs); 541| 0| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 542| 0| cs->state = CS_DRAIN_HEADER; 543| 0| break; 544| 0| } 545| | 546| 0| case CS_DRAIN_HEADER: ------------------ | Branch (546:13): [True: 0, False: 524k] ------------------ 547| 0| case CS_DRAIN_BLOCK: { ------------------ | Branch (547:13): [True: 0, False: 524k] ------------------ 548| 0| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (548:21): [True: 0, False: 0] ------------------ 549| 0| cs->state = CS_ACCUMULATE; 550| 0| break; 551| 0| } 552| | 553| 4.60k| case CS_ACCUMULATE: { ------------------ | Branch (553:13): [True: 4.60k, False: 520k] ------------------ 554| | /* Compress the residual partial block (if any), then EOF + footer. */ 555| 4.60k| if (cs->in_used > 0) { ------------------ | Branch (555:21): [True: 4.59k, False: 10] ------------------ 556| 4.59k| const int rc = cs_compress_one_block(cs); 557| 4.59k| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 4.59k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.59k] | | ------------------ ------------------ 558| 4.59k| cs->state = CS_DRAIN_LAST; 559| 4.59k| break; 560| 4.59k| } 561| | /* No residual data: go straight to EOF. */ 562| 10| { 563| 10| const int rc = cs_stage_eof(cs); 564| 10| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 10|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 10] | | ------------------ ------------------ 565| 10| cs->state = CS_DRAIN_EOF; 566| 10| break; 567| 10| } 568| 10| } 569| | 570| 510k| case CS_DRAIN_LAST: { ------------------ | Branch (570:13): [True: 510k, False: 14.1k] ------------------ 571| 510k| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (571:21): [True: 505k, False: 4.59k] ------------------ 572| | /* After last data block -> EOF. */ 573| 4.59k| const int rc = cs_stage_eof(cs); 574| 4.59k| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 4.59k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.59k] | | ------------------ ------------------ 575| 4.59k| cs->state = CS_DRAIN_EOF; 576| 4.59k| break; 577| 4.59k| } 578| | 579| 4.76k| case CS_DRAIN_EOF: { ------------------ | Branch (579:13): [True: 4.76k, False: 519k] ------------------ 580| 4.76k| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (580:21): [True: 155, False: 4.60k] ------------------ 581| 4.60k| const int rc = cs_stage_footer(cs); 582| 4.60k| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 583| 4.60k| cs->state = CS_DRAIN_FOOTER; 584| 4.60k| break; 585| 4.60k| } 586| | 587| 4.78k| case CS_DRAIN_FOOTER: { ------------------ | Branch (587:13): [True: 4.78k, False: 519k] ------------------ 588| 4.78k| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (588:21): [True: 179, False: 4.60k] ------------------ 589| 4.60k| cs->state = CS_DONE; 590| 4.60k| return 0; 591| 4.78k| } 592| | 593| 0| case CS_DONE: ------------------ | Branch (593:13): [True: 0, False: 524k] ------------------ 594| 0| case CS_ERRORED: ------------------ | Branch (594:13): [True: 0, False: 524k] ------------------ 595| 0| return cs->state == CS_ERRORED ? cs->error_code : 0; // LCOV_EXCL_LINE ------------------ | Branch (595:24): [True: 0, False: 0] ------------------ 596| 524k| } 597| 524k| } 598| 510k|} zxc_dstream_create: 826| 9.21k|zxc_dstream* zxc_dstream_create(const zxc_decompress_opts_t* opts) { 827| 9.21k| zxc_dstream* ds = (zxc_dstream*)ZXC_CALLOC(1, sizeof(*ds)); ------------------ | | 72| 9.21k|#define ZXC_CALLOC(nmemb, size) calloc(nmemb, size) ------------------ 828| 9.21k| if (UNLIKELY(!ds)) return NULL; // LCOV_EXCL_LINE ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 829| 9.21k| if (opts) ds->opts = *opts; ------------------ | Branch (829:9): [True: 9.21k, False: 0] ------------------ 830| 9.21k| if (UNLIKELY(ds->opts.dict || ds->opts.dict_size || ds->opts.dict_huf)) { ------------------ | | 151| 36.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | ------------------ ------------------ 831| 0| ZXC_FREE(ds); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 832| 0| return NULL; 833| 0| } 834| 9.21k| ds->opts.n_threads = 0; 835| 9.21k| ds->opts.progress_cb = NULL; 836| 9.21k| ds->opts.user_data = NULL; 837| 9.21k| ds->state = DS_NEED_FILE_HEADER; 838| 9.21k| ds->scratch_need = ZXC_FILE_HEADER_SIZE; ------------------ | | 103| 9.21k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ 839| 9.21k| return ds; 840| 9.21k|} zxc_dstream_free: 849| 9.21k|void zxc_dstream_free(zxc_dstream* ds) { 850| 9.21k| if (UNLIKELY(!ds)) return; // LCOV_EXCL_LINE ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 851| 9.21k| ZXC_FREE(ds->payload); ------------------ | | 84| 9.21k|#define ZXC_FREE(ptr) free(ptr) ------------------ 852| 9.21k| ZXC_FREE(ds->decoded); ------------------ | | 84| 9.21k|#define ZXC_FREE(ptr) free(ptr) ------------------ 853| 9.21k| if (ds->inner_initialized) zxc_cctx_free(&ds->inner); ------------------ | Branch (853:9): [True: 4.60k, False: 4.60k] ------------------ 854| 9.21k| ZXC_FREE(ds); ------------------ | | 84| 9.21k|#define ZXC_FREE(ptr) free(ptr) ------------------ 855| 9.21k|} zxc_dstream_finished: 863| 3.12k|int zxc_dstream_finished(const zxc_dstream* ds) { return (ds && ds->state == DS_DONE) ? 1 : 0; } ------------------ | Branch (863:59): [True: 3.12k, False: 0] | Branch (863:65): [True: 3.11k, False: 4] ------------------ zxc_dstream_decompress: 1053| 42.5M|int64_t zxc_dstream_decompress(zxc_dstream* ds, zxc_outbuf_t* out, zxc_inbuf_t* in) { 1054| 42.5M| if (UNLIKELY(!ds || !out || !in || in->pos > in->size || out->pos > out->size || ------------------ | | 151| 680M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 42.5M] | | | Branch (151:42): [True: 0, False: 42.5M] | | | Branch (151:42): [True: 0, False: 42.5M] | | | Branch (151:42): [True: 0, False: 42.5M] | | | Branch (151:42): [True: 0, False: 42.5M] | | | Branch (151:42): [True: 0, False: 42.5M] | | | Branch (151:42): [True: 42.5M, False: 0] | | | Branch (151:42): [True: 0, False: 42.5M] | | | Branch (151:42): [True: 42.5M, False: 0] | | | Branch (151:42): [True: 0, False: 42.5M] | | ------------------ ------------------ 1055| 42.5M| (in->size > in->pos && !in->src) || (out->size > out->pos && !out->dst))) { 1056| 0| return ZXC_ERROR_NULL_INPUT; 1057| 0| } 1058| 42.5M| if (UNLIKELY(ds->state == DS_ERRORED)) return ds->error_code; ------------------ | | 151| 42.5M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 42.5M] | | ------------------ ------------------ 1059| 42.5M| if (UNLIKELY(ds->state == DS_DONE)) return 0; ------------------ | | 151| 42.5M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 42.5M] | | ------------------ ------------------ 1060| | 1061| 42.5M| size_t produced = 0; 1062| | 1063| 42.5M| for (;;) { 1064| 42.5M| switch (ds->state) { ------------------ | Branch (1064:17): [True: 42.5M, False: 0] ------------------ 1065| 47.9k| case DS_NEED_FILE_HEADER: { ------------------ | Branch (1065:13): [True: 47.9k, False: 42.5M] ------------------ 1066| 47.9k| const int rc = ds_handle_need_file_header(ds, in); 1067| 47.9k| if (rc == 1) return (int64_t)produced; ------------------ | Branch (1067:21): [True: 38.8k, False: 9.10k] ------------------ 1068| 9.10k| if (rc < 0) return rc; ------------------ | Branch (1068:21): [True: 4.50k, False: 4.60k] ------------------ 1069| 4.60k| break; 1070| 9.10k| } 1071| | 1072| 18.7k| case DS_NEED_BLOCK_HEADER: { ------------------ | Branch (1072:13): [True: 18.7k, False: 42.5M] ------------------ 1073| 18.7k| const int rc = ds_handle_need_block_header(ds, in); 1074| 18.7k| if (rc == 1) return (int64_t)produced; ------------------ | Branch (1074:21): [True: 10.6k, False: 8.15k] ------------------ 1075| 8.15k| if (rc < 0) return rc; ------------------ | Branch (1075:21): [True: 0, False: 8.15k] ------------------ 1076| 8.15k| break; 1077| 8.15k| } 1078| | 1079| 40.8M| case DS_NEED_BLOCK_PAYLOAD: { ------------------ | Branch (1079:13): [True: 40.8M, False: 1.78M] ------------------ 1080| 40.8M| if (!ds_pull_payload(ds, in)) return (int64_t)produced; ------------------ | Branch (1080:21): [True: 40.8M, False: 4.90k] ------------------ 1081| 4.90k| ds->state = DS_DECODE_BLOCK; 1082| 4.90k| break; 1083| 40.8M| } 1084| | 1085| 4.90k| case DS_DECODE_BLOCK: { ------------------ | Branch (1085:13): [True: 4.90k, False: 42.5M] ------------------ 1086| 4.90k| const int direct = (out->size - out->pos) >= ds->decoded_cap; 1087| 4.90k| uint8_t* const ddst = direct ? (uint8_t*)out->dst + out->pos : ds->decoded; ------------------ | Branch (1087:39): [True: 0, False: 4.90k] ------------------ 1088| 4.90k| const int dsz = zxc_decompress_chunk_wrapper( 1089| 4.90k| &ds->inner, ds->payload, ds->payload_used, ddst, ds->decoded_cap); 1090| 4.90k| if (UNLIKELY(dsz < 0)) return ds_set_error(ds, dsz); ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1091| | 1092| | /* If file-level checksum verification is enabled, fold this 1093| | * block's trailer into the rolling global hash (last 1094| | * ZXC_BLOCK_CHECKSUM_SIZE bytes of the *raw* block). */ 1095| 4.90k| if (ds->opts.checksum_enabled && ds->file_has_checksum && ------------------ | Branch (1095:21): [True: 1.63k, False: 3.27k] | Branch (1095:50): [True: 1.63k, False: 0] ------------------ 1096| 1.63k| ds->payload_used >= ZXC_BLOCK_CHECKSUM_SIZE) { ------------------ | | 399| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (1096:21): [True: 1.63k, False: 0] ------------------ 1097| 1.63k| const uint32_t bh = 1098| 1.63k| zxc_le32(ds->payload + ds->payload_used - ZXC_BLOCK_CHECKSUM_SIZE); ------------------ | | 399| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ 1099| 1.63k| ds->global_hash = zxc_hash_combine_rotate(ds->global_hash, bh); 1100| 1.63k| } 1101| | 1102| 4.90k| if (direct) { ------------------ | Branch (1102:21): [True: 0, False: 4.90k] ------------------ 1103| 0| out->pos += (size_t)dsz; 1104| 0| produced += (size_t)dsz; 1105| 0| ds->total_out += (size_t)dsz; 1106| 0| ds->decoded_size = 0; 1107| 0| ds->decoded_pos = 0; 1108| 0| ds->state = DS_NEED_BLOCK_HEADER; 1109| 0| ds->scratch_used = 0; 1110| 0| ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 0|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1111| 0| break; 1112| 0| } 1113| | 1114| 4.90k| ds->decoded_size = (size_t)dsz; 1115| 4.90k| ds->decoded_pos = 0; 1116| 4.90k| ds->state = DS_EMIT_DECODED; 1117| 4.90k| break; 1118| 4.90k| } 1119| | 1120| 1.70M| case DS_EMIT_DECODED: { ------------------ | Branch (1120:13): [True: 1.70M, False: 40.8M] ------------------ 1121| 1.70M| const int done = ds_drain_decoded(ds, out, &produced); 1122| 1.70M| if (!done) return (int64_t)produced; ------------------ | Branch (1122:21): [True: 1.70M, False: 4.90k] ------------------ 1123| 4.90k| ds->state = DS_NEED_BLOCK_HEADER; 1124| 4.90k| ds->scratch_used = 0; 1125| 4.90k| ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1126| 4.90k| break; 1127| 1.70M| } 1128| | 1129| 3.24k| case DS_PEEK_TAIL: { ------------------ | Branch (1129:13): [True: 3.24k, False: 42.5M] ------------------ 1130| 3.24k| if (!ds_pull_scratch(ds, in)) return (int64_t)produced; ------------------ | Branch (1130:21): [True: 84, False: 3.16k] ------------------ 1131| | /* Try to interpret as a block header (SEK). */ 1132| 3.16k| zxc_block_header_t peek; 1133| 3.16k| const int sek_rc = zxc_read_block_header(ds->scratch, ds->scratch_used, &peek); 1134| 3.16k| if (sek_rc == ZXC_OK && peek.block_type == (uint8_t)ZXC_BLOCK_SEK) { ------------------ | Branch (1134:21): [True: 28, False: 3.13k] | Branch (1134:41): [True: 4, False: 24] ------------------ 1135| | /* SEK block: skip its payload (peek.comp_size bytes). */ 1136| 4| ds->sek_remaining = (size_t)peek.comp_size; 1137| 4| ds->state = DS_DRAIN_SEK_PAYLOAD; 1138| 4| break; 1139| 4| } 1140| | /* Not SEK -> these 8 bytes are the first 8 of the 12-byte footer. */ 1141| 3.15k| ds->state = DS_NEED_FOOTER_REST; 1142| 3.15k| ds->scratch_need = ZXC_FILE_FOOTER_SIZE; /* keep first 8, want 4 more */ ------------------ | | 105| 3.15k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 1143| 3.15k| break; 1144| 3.16k| } 1145| | 1146| 4| case DS_DRAIN_SEK_PAYLOAD: { ------------------ | Branch (1146:13): [True: 4, False: 42.5M] ------------------ 1147| 4| const size_t avail = in->size - in->pos; 1148| 4| const size_t n = avail < ds->sek_remaining ? avail : ds->sek_remaining; ------------------ | Branch (1148:34): [True: 0, False: 4] ------------------ 1149| 4| in->pos += n; 1150| 4| ds->sek_remaining -= n; 1151| 4| if (ds->sek_remaining > 0) return (int64_t)produced; ------------------ | Branch (1151:21): [True: 0, False: 4] ------------------ 1152| 4| ds->state = DS_NEED_FOOTER_FULL; 1153| 4| ds->scratch_used = 0; 1154| 4| ds->scratch_need = ZXC_FILE_FOOTER_SIZE; ------------------ | | 105| 4|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 1155| 4| break; 1156| 4| } 1157| | 1158| 3.15k| case DS_NEED_FOOTER_REST: ------------------ | Branch (1158:13): [True: 3.15k, False: 42.5M] ------------------ 1159| 3.16k| case DS_NEED_FOOTER_FULL: { ------------------ | Branch (1159:13): [True: 4, False: 42.5M] ------------------ 1160| 3.16k| if (!ds_pull_scratch(ds, in)) return (int64_t)produced; ------------------ | Branch (1160:21): [True: 45, False: 3.11k] ------------------ 1161| 3.11k| ds->state = DS_VALIDATE_FOOTER; 1162| 3.11k| break; 1163| 3.16k| } 1164| | 1165| 3.11k| case DS_VALIDATE_FOOTER: { ------------------ | Branch (1165:13): [True: 3.11k, False: 42.5M] ------------------ 1166| 3.11k| const uint64_t declared = zxc_le64(ds->scratch); 1167| 3.11k| if (UNLIKELY(declared != ds->total_out)) ------------------ | | 151| 3.11k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 3.11k] | | ------------------ ------------------ 1168| 0| return ds_set_error(ds, ZXC_ERROR_CORRUPT_DATA); 1169| 3.11k| if (ds->opts.checksum_enabled && ds->file_has_checksum) { ------------------ | Branch (1169:21): [True: 1.02k, False: 2.09k] | Branch (1169:50): [True: 1.02k, False: 0] ------------------ 1170| 1.02k| const uint32_t fh = zxc_le32(ds->scratch + sizeof(uint64_t)); 1171| 1.02k| if (UNLIKELY(fh != ds->global_hash)) ------------------ | | 151| 1.02k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.02k] | | ------------------ ------------------ 1172| 0| return ds_set_error(ds, ZXC_ERROR_BAD_CHECKSUM); 1173| 1.02k| } 1174| 3.11k| ds->state = DS_DONE; 1175| 3.11k| return (int64_t)produced; 1176| 3.11k| } 1177| | 1178| 0| case DS_DONE: ------------------ | Branch (1178:13): [True: 0, False: 42.5M] ------------------ 1179| 0| case DS_ERRORED: ------------------ | Branch (1179:13): [True: 0, False: 42.5M] ------------------ 1180| 0| return ds->state == DS_ERRORED ? ds->error_code : (int64_t)produced; ------------------ | Branch (1180:24): [True: 0, False: 0] ------------------ 1181| 42.5M| } 1182| 42.5M| } 1183| 42.5M|} zxc_pstream.c:cs_stage_file_header: 327| 4.60k|static int cs_stage_file_header(zxc_cstream* cs) { 328| 4.60k| const int w = zxc_write_file_header(cs->pending, cs->pending_cap, cs->block_size, 329| 4.60k| cs->opts.checksum_enabled, 0); 330| 4.60k| if (UNLIKELY(w < 0)) return w; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 331| 4.60k| cs->pending_len = (size_t)w; 332| 4.60k| cs->pending_pos = 0; 333| 4.60k| return ZXC_OK; 334| 4.60k|} zxc_pstream.c:cs_drain_pending: 238| 843k|static int cs_drain_pending(zxc_cstream* cs, zxc_outbuf_t* out) { 239| 843k| const size_t avail_out = out->size - out->pos; 240| 843k| const size_t avail_pen = cs->pending_len - cs->pending_pos; 241| 843k| const size_t n = avail_out < avail_pen ? avail_out : avail_pen; ------------------ | Branch (241:22): [True: 824k, False: 18.7k] ------------------ 242| 843k| if (n) { ------------------ | Branch (242:9): [True: 843k, False: 35] ------------------ 243| 843k| ZXC_MEMCPY((uint8_t*)out->dst + out->pos, cs->pending + cs->pending_pos, n); ------------------ | | 173| 843k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 244| 843k| out->pos += n; 245| 843k| cs->pending_pos += n; 246| 843k| } 247| 843k| return cs->pending_pos == cs->pending_len; 248| 843k|} zxc_pstream.c:cs_compress_block_from: 179| 4.90k|static int cs_compress_block_from(zxc_cstream* cs, const uint8_t* RESTRICT src, const size_t len) { 180| 4.90k| const uint64_t bound = zxc_compress_block_bound(len); 181| | // LCOV_EXCL_START 182| 4.90k| if (UNLIKELY(bound == 0 || bound > SIZE_MAX)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 9.81k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | ------------------ ------------------ 183| 4.90k| if (UNLIKELY(bound > cs->pending_cap)) { ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 4.60k, False: 303] | | ------------------ ------------------ 184| 4.60k| uint8_t* nb = (uint8_t*)ZXC_REALLOC(cs->pending, (size_t)bound); ------------------ | | 78| 4.60k|#define ZXC_REALLOC(ptr, size) realloc(ptr, size) ------------------ 185| 4.60k| if (UNLIKELY(!nb)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 186| 4.60k| cs->pending = nb; 187| 4.60k| cs->pending_cap = (size_t)bound; 188| 4.60k| } 189| | // LCOV_EXCL_STOP 190| 4.90k| const int64_t csize = 191| 4.90k| zxc_compress_block(cs->cctx, src, len, cs->pending, cs->pending_cap, &cs->opts); 192| 4.90k| if (UNLIKELY(csize < 0)) return (int)csize; // LCOV_EXCL_LINE ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 193| | 194| 4.90k| cs->pending_len = (size_t)csize; 195| 4.90k| cs->pending_pos = 0; 196| 4.90k| cs->total_in += len; 197| | 198| | /* If checksums are on, the block trailer is the last ZXC_BLOCK_CHECKSUM_SIZE 199| | * bytes of pending; fold it into the rolling global hash. */ 200| 4.90k| if (cs->opts.checksum_enabled && cs->pending_len >= ZXC_BLOCK_CHECKSUM_SIZE) { ------------------ | | 399| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (200:9): [True: 1.63k, False: 3.27k] | Branch (200:38): [True: 1.63k, False: 0] ------------------ 201| 1.63k| const uint32_t bh = zxc_le32(cs->pending + cs->pending_len - ZXC_BLOCK_CHECKSUM_SIZE); ------------------ | | 399| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ 202| 1.63k| cs->global_hash = zxc_hash_combine_rotate(cs->global_hash, bh); 203| 1.63k| } 204| 4.90k| return ZXC_OK; 205| 4.90k|} zxc_pstream.c:cs_compress_one_block: 220| 4.90k|static int cs_compress_one_block(zxc_cstream* cs) { 221| 4.90k| const int rc = cs_compress_block_from(cs, cs->in_block, cs->in_used); 222| 4.90k| if (LIKELY(rc == ZXC_OK)) cs->in_used = 0; ------------------ | | 145| 4.90k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 4.90k, False: 0] | | ------------------ ------------------ 223| 4.90k| return rc; 224| 4.90k|} zxc_pstream.c:cs_stage_eof: 345| 4.60k|static int cs_stage_eof(zxc_cstream* cs) { 346| | // LCOV_EXCL_START 347| 4.60k| if (UNLIKELY(ZXC_BLOCK_HEADER_SIZE > cs->pending_cap)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 348| 0| uint8_t* nb = (uint8_t*)ZXC_REALLOC(cs->pending, ZXC_BLOCK_HEADER_SIZE); ------------------ | | 78| 0|#define ZXC_REALLOC(ptr, size) realloc(ptr, size) ------------------ 349| 0| if (UNLIKELY(!nb)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 350| 0| cs->pending = nb; 351| 0| cs->pending_cap = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 0|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 352| 0| } 353| | // LCOV_EXCL_STOP 354| 4.60k| const zxc_block_header_t eof = { 355| 4.60k| .block_type = (uint8_t)ZXC_BLOCK_EOF, 356| 4.60k| .block_flags = 0, 357| 4.60k| .reserved = 0, 358| 4.60k| .header_crc = 0, 359| 4.60k| .comp_size = 0, 360| 4.60k| }; 361| 4.60k| const int w = zxc_write_block_header(cs->pending, cs->pending_cap, &eof); 362| 4.60k| if (UNLIKELY(w < 0)) return w; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 363| 4.60k| cs->pending_len = (size_t)w; 364| 4.60k| cs->pending_pos = 0; 365| 4.60k| return ZXC_OK; 366| 4.60k|} zxc_pstream.c:cs_stage_footer: 377| 4.60k|static int cs_stage_footer(zxc_cstream* cs) { 378| | // LCOV_EXCL_START 379| 4.60k| if (UNLIKELY(ZXC_FILE_FOOTER_SIZE > cs->pending_cap)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 380| 0| uint8_t* nb = (uint8_t*)ZXC_REALLOC(cs->pending, ZXC_FILE_FOOTER_SIZE); ------------------ | | 78| 0|#define ZXC_REALLOC(ptr, size) realloc(ptr, size) ------------------ 381| 0| if (UNLIKELY(!nb)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 382| 0| cs->pending = nb; 383| 0| cs->pending_cap = ZXC_FILE_FOOTER_SIZE; ------------------ | | 105| 0|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 384| 0| } 385| | // LCOV_EXCL_STOP 386| 4.60k| const int w = zxc_write_file_footer(cs->pending, cs->pending_cap, cs->total_in, cs->global_hash, 387| 4.60k| cs->opts.checksum_enabled); 388| 4.60k| if (UNLIKELY(w < 0)) return w; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 389| 4.60k| cs->pending_len = (size_t)w; 390| 4.60k| cs->pending_pos = 0; 391| 4.60k| return ZXC_OK; 392| 4.60k|} zxc_pstream.c:ds_handle_need_file_header: 938| 47.9k|static int ds_handle_need_file_header(zxc_dstream* ds, zxc_inbuf_t* in) { 939| 47.9k| if (!ds_pull_scratch(ds, in)) return 1; ------------------ | Branch (939:9): [True: 38.8k, False: 9.10k] ------------------ 940| | 941| 9.10k| size_t bs = 0; 942| 9.10k| int has_csum = 0; 943| 9.10k| const int rc = zxc_read_file_header(ds->scratch, ds->scratch_used, &bs, &has_csum, NULL); 944| 9.10k| if (UNLIKELY(rc != ZXC_OK)) return ds_set_error(ds, rc); // LCOV_EXCL_LINE ------------------ | | 151| 9.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 4.50k, False: 4.60k] | | ------------------ ------------------ 945| 4.60k| ds->block_size = bs; 946| 4.60k| ds->file_has_checksum = has_csum; 947| | 948| | /* Allocate payload + decoded buffers now that block_size is known. */ 949| 4.60k| const uint64_t pb = zxc_compress_block_bound(ds->block_size); 950| | // LCOV_EXCL_START 951| 4.60k| if (UNLIKELY(pb == 0 || pb > SIZE_MAX)) return ds_set_error(ds, ZXC_ERROR_OVERFLOW); ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 952| | // LCOV_EXCL_STOP 953| 4.60k| ds->payload_cap = (size_t)pb; 954| 4.60k| ds->payload = (uint8_t*)ZXC_MALLOC(ds->payload_cap); ------------------ | | 66| 4.60k|#define ZXC_MALLOC(size) malloc(size) ------------------ 955| | 956| 4.60k| ds->decoded_cap = ds->block_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 349| 4.60k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 4.60k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 957| 4.60k| ds->decoded = (uint8_t*)ZXC_MALLOC(ds->decoded_cap); ------------------ | | 66| 4.60k|#define ZXC_MALLOC(size) malloc(size) ------------------ 958| | // LCOV_EXCL_START 959| 4.60k| if (UNLIKELY(!ds->payload || !ds->decoded)) return ds_set_error(ds, ZXC_ERROR_MEMORY); ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 960| | 961| 4.60k| if (UNLIKELY(zxc_cctx_init(&ds->inner, ds->block_size, 0, 0, ------------------ | | 151| 6.14k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 1.53k, False: 3.06k] | | | Branch (151:42): [True: 1.53k, False: 0] | | ------------------ ------------------ 962| 4.60k| ds->file_has_checksum && ds->opts.checksum_enabled, 0) != ZXC_OK)) { 963| 0| return ds_set_error(ds, ZXC_ERROR_MEMORY); 964| 0| } 965| | // LCOV_EXCL_STOP 966| 4.60k| ds->inner_initialized = 1; 967| | 968| 4.60k| ds->state = DS_NEED_BLOCK_HEADER; 969| 4.60k| ds->scratch_used = 0; 970| 4.60k| ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 4.60k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 971| 4.60k| return 0; 972| 4.60k|} zxc_pstream.c:ds_handle_need_block_header: 990| 18.7k|static int ds_handle_need_block_header(zxc_dstream* ds, zxc_inbuf_t* in) { 991| 18.7k| if (!ds_pull_scratch(ds, in)) return 1; ------------------ | Branch (991:9): [True: 10.6k, False: 8.15k] ------------------ 992| | 993| 8.15k| const int rc = zxc_read_block_header(ds->scratch, ds->scratch_used, &ds->cur_bh); 994| 8.15k| if (UNLIKELY(rc != ZXC_OK)) return ds_set_error(ds, rc); // LCOV_EXCL_LINE ------------------ | | 151| 8.15k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.15k] | | ------------------ ------------------ 995| | 996| 8.15k| if (ds->cur_bh.block_type == (uint8_t)ZXC_BLOCK_EOF) { ------------------ | Branch (996:9): [True: 3.24k, False: 4.90k] ------------------ 997| | /* EOF block: comp_size must be 0; no payload, no checksum. */ 998| 3.24k| if (UNLIKELY(ds->cur_bh.comp_size != 0)) return ds_set_error(ds, ZXC_ERROR_BAD_BLOCK_SIZE); ------------------ | | 151| 3.24k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 3.24k] | | ------------------ ------------------ 999| 3.24k| ds->state = DS_PEEK_TAIL; 1000| 3.24k| ds->scratch_used = 0; 1001| 3.24k| ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; /* sniff */ ------------------ | | 397| 3.24k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1002| 3.24k| return 0; 1003| 3.24k| } 1004| | 1005| | /* Normal data block: read comp_size [+ ZXC_BLOCK_CHECKSUM_SIZE if file-level checksums]. */ 1006| 4.90k| const uint64_t need = (uint64_t)ds->cur_bh.comp_size + 1007| 4.90k| (ds->file_has_checksum ? (uint64_t)ZXC_BLOCK_CHECKSUM_SIZE : 0u); ------------------ | | 399| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (1007:28): [True: 1.63k, False: 3.27k] ------------------ 1008| 4.90k| if (UNLIKELY(need > ds->payload_cap)) return ds_set_error(ds, ZXC_ERROR_BAD_BLOCK_SIZE); ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1009| | 1010| | /* Feed the full block (header + payload + opt csum) to zxc_decompress_block, 1011| | * so prefix with the 8-byte header we just parsed. */ 1012| 4.90k| ZXC_MEMCPY(ds->payload, ds->scratch, ZXC_BLOCK_HEADER_SIZE); ------------------ | | 173| 4.90k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1013| 4.90k| ds->payload_used = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1014| 4.90k| ds->payload_need = (size_t)need + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1015| | // LCOV_EXCL_START 1016| 4.90k| if (UNLIKELY(ds->payload_need > ds->payload_cap)) { ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1017| | /* grow */ 1018| 0| uint8_t* nb = (uint8_t*)ZXC_REALLOC(ds->payload, ds->payload_need); ------------------ | | 78| 0|#define ZXC_REALLOC(ptr, size) realloc(ptr, size) ------------------ 1019| 0| if (UNLIKELY(!nb)) return ds_set_error(ds, ZXC_ERROR_MEMORY); ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1020| 0| ds->payload = nb; 1021| 0| ds->payload_cap = ds->payload_need; 1022| 0| } 1023| | // LCOV_EXCL_STOP 1024| 4.90k| ds->state = DS_NEED_BLOCK_PAYLOAD; 1025| 4.90k| return 0; 1026| 4.90k|} zxc_pstream.c:ds_pull_payload: 802| 40.8M|static int ds_pull_payload(zxc_dstream* ds, zxc_inbuf_t* in) { 803| 40.8M| const size_t want = ds->payload_need - ds->payload_used; 804| 40.8M| const size_t avail = in->size - in->pos; 805| 40.8M| const size_t n = want < avail ? want : avail; ------------------ | Branch (805:22): [True: 3.72k, False: 40.8M] ------------------ 806| 40.8M| if (n) { ------------------ | Branch (806:9): [True: 40.8M, False: 1.41k] ------------------ 807| 40.8M| ZXC_MEMCPY(ds->payload + ds->payload_used, (const uint8_t*)in->src + in->pos, n); ------------------ | | 173| 40.8M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 808| 40.8M| in->pos += n; 809| 40.8M| ds->payload_used += n; 810| 40.8M| } 811| 40.8M| return ds->payload_used == ds->payload_need; 812| 40.8M|} zxc_pstream.c:ds_set_error: 762| 4.50k|static int ds_set_error(zxc_dstream* ds, const int code) { 763| 4.50k| ds->error_code = code; 764| 4.50k| ds->state = DS_ERRORED; 765| 4.50k| return code; 766| 4.50k|} zxc_pstream.c:ds_drain_decoded: 908| 1.70M|static int ds_drain_decoded(zxc_dstream* ds, zxc_outbuf_t* out, size_t* produced) { 909| 1.70M| const size_t avail_out = out->size - out->pos; 910| 1.70M| const size_t avail_dec = ds->decoded_size - ds->decoded_pos; 911| 1.70M| const size_t n = avail_out < avail_dec ? avail_out : avail_dec; ------------------ | Branch (911:22): [True: 1.70M, False: 4.90k] ------------------ 912| 1.70M| if (n) { ------------------ | Branch (912:9): [True: 1.70M, False: 39] ------------------ 913| 1.70M| ZXC_MEMCPY((uint8_t*)out->dst + out->pos, ds->decoded + ds->decoded_pos, n); ------------------ | | 173| 1.70M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 914| 1.70M| out->pos += n; 915| 1.70M| ds->decoded_pos += n; 916| 1.70M| ds->total_out += n; 917| 1.70M| if (produced) *produced += n; ------------------ | Branch (917:13): [True: 1.70M, False: 0] ------------------ 918| 1.70M| } 919| 1.70M| return ds->decoded_pos == ds->decoded_size; 920| 1.70M|} zxc_pstream.c:ds_pull_scratch: 779| 73.1k|static int ds_pull_scratch(zxc_dstream* ds, zxc_inbuf_t* in) { 780| 73.1k| const size_t want = ds->scratch_need - ds->scratch_used; 781| 73.1k| const size_t avail = in->size - in->pos; 782| 73.1k| const size_t n = want < avail ? want : avail; ------------------ | Branch (782:22): [True: 16.2k, False: 56.9k] ------------------ 783| 73.1k| if (n) { ------------------ | Branch (783:9): [True: 71.6k, False: 1.49k] ------------------ 784| 71.6k| ZXC_MEMCPY(ds->scratch + ds->scratch_used, (const uint8_t*)in->src + in->pos, n); ------------------ | | 173| 71.6k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 785| 71.6k| in->pos += n; 786| 71.6k| ds->scratch_used += n; 787| 71.6k| } 788| 73.1k| return ds->scratch_used == ds->scratch_need; 789| 73.1k|} LLVMFuzzerTestOneInput: 104| 4.62k|int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { 105| 4.62k| if (size < 3) return 0; ------------------ | Branch (105:9): [True: 2, False: 4.62k] ------------------ 106| | 107| | /* Save raw input for the parser-fuzzing phase. */ 108| 4.62k| const uint8_t* const raw_data = data; 109| 4.62k| const size_t raw_size = size; 110| | 111| | /* Header bytes drive pstream-specific axes. data[0] is reserved (the 112| | * compression level is intentionally fixed to 1, see copts below); it 113| | * is still consumed so corpus byte offsets stay stable. */ 114| 4.62k| const int checksum_enabled = data[1] & 1; 115| 4.62k| const size_t in_chunk = derive_chunk_size(data[2]); 116| 4.62k| data += 3; 117| 4.62k| size -= 3; 118| | 119| 4.62k| if (size == 0 || size > FUZZ_PSTREAM_MAX_INPUT) return 0; ------------------ | | 41| 4.60k|#define FUZZ_PSTREAM_MAX_INPUT (2 << 20) /* 2 MiB */ ------------------ | Branch (119:9): [True: 17, False: 4.60k] | Branch (119:22): [True: 0, False: 4.60k] ------------------ 120| | 121| | /* Persistent buffers - reused across iterations to reduce allocator 122| | * pressure (same pattern as fuzz_seekable.c). */ 123| 4.60k| static uint8_t* comp_buf = NULL; 124| 4.60k| static size_t comp_cap = 0; 125| 4.60k| static uint8_t* round_buf = NULL; 126| 4.60k| static size_t round_cap = 0; 127| | 128| | /* ------------------------------------------------------------------ */ 129| | /* Phase 1: Push compression in chunks */ 130| | /* ------------------------------------------------------------------ */ 131| | /* Level forced to 1: pstream's state-machine surface (framing, drain, 132| | * resumption, sticky errors) is independent of compression strength, 133| | * and buffer-API fuzzers (roundtrip, decompress) already exercise the 134| | * level dimension. Cheaper compression = more iterations/sec. */ 135| 4.60k| zxc_compress_opts_t copts = { 136| 4.60k| .level = 1, 137| 4.60k| .checksum_enabled = checksum_enabled, 138| 4.60k| }; 139| 4.60k| zxc_cstream* cs = zxc_cstream_create(&copts); 140| 4.60k| if (!cs) return 0; ------------------ | Branch (140:9): [True: 0, False: 4.60k] ------------------ 141| | 142| | /* Sized so the compressor never runs out - bound is a safe upper limit. */ 143| | /* Use the public buffer-API bound function via a local extern declaration 144| | * to avoid pulling zxc_buffer.h (kept narrow on purpose). */ 145| 4.60k| extern uint64_t zxc_compress_bound(size_t input_size); 146| 4.60k| const uint64_t bound64 = zxc_compress_bound(size); 147| 4.60k| if (bound64 == 0 || bound64 > SIZE_MAX) { ------------------ | Branch (147:9): [True: 0, False: 4.60k] | Branch (147:25): [True: 0, False: 4.60k] ------------------ 148| 0| zxc_cstream_free(cs); 149| 0| return 0; 150| 0| } 151| 4.60k| const size_t bound = (size_t)bound64; 152| 4.60k| if (bound > comp_cap) { ------------------ | Branch (152:9): [True: 2.98k, False: 1.62k] ------------------ 153| 2.98k| void* nb = realloc(comp_buf, bound); 154| 2.98k| if (!nb) { ------------------ | Branch (154:13): [True: 0, False: 2.98k] ------------------ 155| 0| zxc_cstream_free(cs); 156| 0| return 0; 157| 0| } 158| 2.98k| comp_buf = (uint8_t*)nb; 159| 2.98k| comp_cap = bound; 160| 2.98k| } 161| | 162| 4.60k| size_t comp_len = 0; 163| 4.60k| size_t in_pos = 0; 164| | 165| | /* Feed input in `in_chunk`-sized slices. */ 166| 109M| while (in_pos < size) { ------------------ | Branch (166:12): [True: 109M, False: 4.60k] ------------------ 167| 109M| const size_t feed = (size - in_pos < in_chunk) ? (size - in_pos) : in_chunk; ------------------ | Branch (167:29): [True: 3.47k, False: 109M] ------------------ 168| 109M| zxc_inbuf_t in = {.src = data + in_pos, .size = feed, .pos = 0}; 169| | 170| | /* Drain repeatedly until this slice is fully consumed. */ 171| 109M| for (;;) { 172| 109M| size_t window = bound - comp_len; 173| 109M| if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 109M|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 109M|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ | Branch (173:17): [True: 109M, False: 15.4k] ------------------ 174| 109M| if (window == 0) { ------------------ | Branch (174:17): [True: 0, False: 109M] ------------------ 175| | /* Bound exhausted - shouldn't happen with zxc_compress_bound. */ 176| 0| zxc_cstream_free(cs); 177| 0| return 0; 178| 0| } 179| 109M| zxc_outbuf_t ob = {.dst = comp_buf + comp_len, .size = window, .pos = 0}; 180| 109M| const int64_t r = zxc_cstream_compress(cs, &ob, &in); 181| 109M| if (r < 0) { ------------------ | Branch (181:17): [True: 0, False: 109M] ------------------ 182| 0| zxc_cstream_free(cs); 183| 0| return 0; 184| 0| } 185| 109M| comp_len += ob.pos; 186| 109M| if (in.pos == in.size && r == 0) break; ------------------ | Branch (186:17): [True: 109M, False: 234k] | Branch (186:38): [True: 109M, False: 83.7k] ------------------ 187| 109M| } 188| 109M| in_pos += in.pos; 189| 109M| } 190| | 191| | /* Finalize: zxc_cstream_end may report >0 pending bytes; drain until 0. */ 192| 510k| for (;;) { 193| 510k| size_t window = bound - comp_len; 194| 510k| if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 510k|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 509k|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ | Branch (194:13): [True: 509k, False: 1.18k] ------------------ 195| 510k| if (window == 0) { ------------------ | Branch (195:13): [True: 0, False: 510k] ------------------ 196| 0| zxc_cstream_free(cs); 197| 0| return 0; 198| 0| } 199| 510k| zxc_outbuf_t ob = {.dst = comp_buf + comp_len, .size = window, .pos = 0}; 200| 510k| const int64_t pending = zxc_cstream_end(cs, &ob); 201| 510k| if (pending < 0) { ------------------ | Branch (201:13): [True: 0, False: 510k] ------------------ 202| 0| zxc_cstream_free(cs); 203| 0| return 0; 204| 0| } 205| 510k| comp_len += ob.pos; 206| 510k| if (pending == 0) break; ------------------ | Branch (206:13): [True: 4.60k, False: 506k] ------------------ 207| 510k| } 208| | 209| 4.60k| zxc_cstream_free(cs); 210| | 211| | /* ------------------------------------------------------------------ */ 212| | /* Phase 2: Push decompression with chunked feeding + roundtrip */ 213| | /* ------------------------------------------------------------------ */ 214| 4.60k| if (size > round_cap) { ------------------ | Branch (214:9): [True: 2.98k, False: 1.62k] ------------------ 215| 2.98k| void* nb = realloc(round_buf, size); 216| 2.98k| if (!nb) return 0; ------------------ | Branch (216:13): [True: 0, False: 2.98k] ------------------ 217| 2.98k| round_buf = (uint8_t*)nb; 218| 2.98k| round_cap = size; 219| 2.98k| } 220| | 221| 4.60k| const int64_t dsize = 222| 4.60k| drip_decompress(comp_buf, comp_len, round_buf, size, in_chunk, checksum_enabled); 223| 4.60k| if (dsize >= 0) { ------------------ | Branch (223:9): [True: 4.60k, False: 0] ------------------ 224| 4.60k| assert((size_t)dsize == size); ------------------ | Branch (224:9): [True: 0, False: 4.60k] | Branch (224:9): [True: 4.60k, False: 0] ------------------ 225| 4.60k| assert(memcmp(data, round_buf, size) == 0); ------------------ | Branch (225:9): [True: 0, False: 4.60k] | Branch (225:9): [True: 4.60k, False: 0] ------------------ 226| 4.60k| } 227| | 228| | /* ------------------------------------------------------------------ */ 229| | /* Phase 3: Parser fuzzing - feed raw input directly */ 230| | /* */ 231| | /* This is where malformed-frame and truncation bugs live. We don't */ 232| | /* care about the result, only that no UB / crash occurs. */ 233| | /* ------------------------------------------------------------------ */ 234| 4.60k| if (raw_size > 0) { ------------------ | Branch (234:9): [True: 4.60k, False: 0] ------------------ 235| | /* Output is discarded - a single small stack buffer suffices and 236| | * keeps the parser's resumption code (small ob.size) under fuzz. */ 237| 4.60k| uint8_t scratch[FUZZ_PSTREAM_OUT_CAP]; 238| 4.60k| const size_t parser_chunk = derive_chunk_size(raw_data[raw_size - 1]); 239| | 240| 4.60k| zxc_decompress_opts_t opts = {.checksum_enabled = checksum_enabled}; 241| 4.60k| zxc_dstream* ds = zxc_dstream_create(&opts); 242| 4.60k| if (ds) { ------------------ | Branch (242:13): [True: 4.60k, False: 0] ------------------ 243| 4.60k| size_t pos = 0; 244| 4.60k| int rounds = 0; 245| 26.9k| while (pos < raw_size && rounds < 1024) { ------------------ | Branch (245:20): [True: 26.8k, False: 105] | Branch (245:38): [True: 26.8k, False: 0] ------------------ 246| 26.8k| const size_t feed = 247| 26.8k| (raw_size - pos < parser_chunk) ? (raw_size - pos) : parser_chunk; ------------------ | Branch (247:21): [True: 515, False: 26.2k] ------------------ 248| 26.8k| zxc_inbuf_t in = {.src = raw_data + pos, .size = feed, .pos = 0}; 249| 26.8k| zxc_outbuf_t ob = {.dst = scratch, .size = sizeof scratch, .pos = 0}; 250| 26.8k| const size_t before_in = in.pos; 251| 26.8k| const size_t before_out = ob.pos; 252| 26.8k| const int64_t r = zxc_dstream_decompress(ds, &ob, &in); 253| 26.8k| pos += in.pos; 254| 26.8k| if (r < 0) break; ------------------ | Branch (254:21): [True: 4.50k, False: 22.3k] ------------------ 255| 22.3k| if (in.pos == before_in && ob.pos == before_out) break; ------------------ | Branch (255:21): [True: 0, False: 22.3k] | Branch (255:44): [True: 0, False: 0] ------------------ 256| 22.3k| rounds++; 257| 22.3k| } 258| 4.60k| zxc_dstream_free(ds); 259| 4.60k| } 260| 4.60k| } 261| | 262| 4.60k| return 0; 263| 4.60k|} fuzz_pstream.c:derive_chunk_size: 47| 9.22k|static size_t derive_chunk_size(uint8_t b) { 48| | /* Map [0..255] -> [1..512] with a non-linear distribution biased toward 49| | * small values (small chunks stress the state machine the most). */ 50| 9.22k| if (b == 0) return 1; ------------------ | Branch (50:9): [True: 2.18k, False: 7.04k] ------------------ 51| 7.04k| if (b < 64) return (size_t)b; /* 1..63 */ ------------------ | Branch (51:9): [True: 2.41k, False: 4.62k] ------------------ 52| 4.62k| if (b < 192) return (size_t)b * 2; /* 128..382 */ ------------------ | Branch (52:9): [True: 1.61k, False: 3.01k] ------------------ 53| 3.01k| return (size_t)b + 256; /* 448..511 */ 54| 4.62k|} fuzz_pstream.c:drip_decompress: 65| 4.60k| size_t in_chunk, int checksum_enabled) { 66| 4.60k| zxc_decompress_opts_t opts = {.checksum_enabled = checksum_enabled}; 67| 4.60k| zxc_dstream* ds = zxc_dstream_create(&opts); 68| 4.60k| if (!ds) return -1; ------------------ | Branch (68:9): [True: 0, False: 4.60k] ------------------ 69| | 70| 4.60k| size_t in_pos = 0; 71| 4.60k| size_t out_total = 0; 72| | 73| 42.5M| while (in_pos < csize || !zxc_dstream_finished(ds)) { ------------------ | Branch (73:12): [True: 42.5M, False: 3.12k] | Branch (73:30): [True: 4, False: 3.11k] ------------------ 74| 42.5M| if (out_total >= out_cap) { ------------------ | Branch (74:13): [True: 1.48k, False: 42.5M] ------------------ 75| | /* Caller's buffer full - stop without asserting. */ 76| 1.48k| zxc_dstream_free(ds); 77| 1.48k| return (int64_t)out_total; 78| 1.48k| } 79| 42.5M| const size_t feed = (csize - in_pos < in_chunk) ? (csize - in_pos) : in_chunk; ------------------ | Branch (79:29): [True: 858k, False: 41.6M] ------------------ 80| 42.5M| size_t window = out_cap - out_total; 81| 42.5M| if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 42.5M|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 42.5M|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ | Branch (81:13): [True: 42.5M, False: 32.2k] ------------------ 82| 42.5M| zxc_inbuf_t in = {.src = comp + in_pos, .size = feed, .pos = 0}; 83| 42.5M| zxc_outbuf_t ob = {.dst = out + out_total, .size = window, .pos = 0}; 84| | 85| 42.5M| const size_t before_in = in.pos; 86| 42.5M| const size_t before_out = ob.pos; 87| 42.5M| const int64_t r = zxc_dstream_decompress(ds, &ob, &in); 88| 42.5M| if (r < 0) { ------------------ | Branch (88:13): [True: 0, False: 42.5M] ------------------ 89| 0| zxc_dstream_free(ds); 90| 0| return r; 91| 0| } 92| | 93| 42.5M| out_total += ob.pos; 94| 42.5M| in_pos += in.pos; 95| | 96| | /* No progress and no more input: prevent infinite loop on truncation. */ 97| 42.5M| if (in.pos == before_in && ob.pos == before_out && in_pos == csize) break; ------------------ | Branch (97:13): [True: 1.69M, False: 40.8M] | Branch (97:36): [True: 0, False: 1.69M] | Branch (97:60): [True: 0, False: 0] ------------------ 98| 42.5M| } 99| | 100| 3.11k| zxc_dstream_free(ds); 101| 3.11k| return (int64_t)out_total; 102| 4.60k|}