zxc_compress.c:rapidhash_withSeed: 486| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_withSeed(const void *key, size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT { 487| 1.63k| return rapidhash_internal(key, len, seed, rapid_secret); 488| 1.63k|} zxc_compress.c:rapidhash_internal: 243| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_internal(const void *key, size_t len, uint64_t seed, const uint64_t* secret) RAPIDHASH_NOEXCEPT { 244| 1.63k| const uint8_t *p=(const uint8_t *)key; 245| 1.63k| seed ^= rapid_mix(seed ^ secret[2], secret[1]); 246| 1.63k| uint64_t a=0, b=0; 247| 1.63k| size_t i = len; 248| 1.63k| if (_likely_(len <= 16)) { ------------------ | | 106| 1.63k| # define _likely_(x) __builtin_expect(x,1) | | ------------------ | | | Branch (106:24): [True: 73, False: 1.56k] | | ------------------ ------------------ 249| 73| if (len >= 4) { ------------------ | Branch (249:9): [True: 52, False: 21] ------------------ 250| 52| seed ^= len; 251| 52| if (len >= 8) { ------------------ | Branch (251:11): [True: 46, False: 6] ------------------ 252| 46| const uint8_t* plast = p + len - 8; 253| 46| a = rapid_read64(p); 254| 46| b = rapid_read64(plast); 255| 46| } else { 256| 6| const uint8_t* plast = p + len - 4; 257| 6| a = rapid_read32(p); 258| 6| b = rapid_read32(plast); 259| 6| } 260| 52| } else if (len > 0) { ------------------ | Branch (260:16): [True: 21, False: 0] ------------------ 261| 21| a = (((uint64_t)p[0])<<45)|p[len-1]; 262| 21| b = p[len>>1]; 263| 21| } else 264| 0| a = b = 0; 265| 1.56k| } else { 266| 1.56k| if (len > 112) { ------------------ | Branch (266:9): [True: 1.35k, False: 209] ------------------ 267| 1.35k| uint64_t see1 = seed, see2 = seed; 268| 1.35k| uint64_t see3 = seed, see4 = seed; 269| 1.35k| uint64_t see5 = seed, see6 = seed; 270| 1.35k|#ifdef RAPIDHASH_COMPACT 271| 1.12M| do { 272| 1.12M| seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 273| 1.12M| see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 274| 1.12M| see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 275| 1.12M| see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 276| 1.12M| see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 277| 1.12M| see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 278| 1.12M| see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 279| 1.12M| p += 112; 280| 1.12M| i -= 112; 281| 1.12M| } while(i > 112); ------------------ | Branch (281:15): [True: 1.12M, False: 1.35k] ------------------ 282| |#else 283| | while (i > 224) { 284| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 285| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 286| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 287| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 288| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 289| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 290| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 291| | seed = rapid_mix(rapid_read64(p + 112) ^ secret[0], rapid_read64(p + 120) ^ seed); 292| | see1 = rapid_mix(rapid_read64(p + 128) ^ secret[1], rapid_read64(p + 136) ^ see1); 293| | see2 = rapid_mix(rapid_read64(p + 144) ^ secret[2], rapid_read64(p + 152) ^ see2); 294| | see3 = rapid_mix(rapid_read64(p + 160) ^ secret[3], rapid_read64(p + 168) ^ see3); 295| | see4 = rapid_mix(rapid_read64(p + 176) ^ secret[4], rapid_read64(p + 184) ^ see4); 296| | see5 = rapid_mix(rapid_read64(p + 192) ^ secret[5], rapid_read64(p + 200) ^ see5); 297| | see6 = rapid_mix(rapid_read64(p + 208) ^ secret[6], rapid_read64(p + 216) ^ see6); 298| | p += 224; 299| | i -= 224; 300| | } 301| | if (i > 112) { 302| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 303| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 304| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 305| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 306| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 307| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 308| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 309| | p += 112; 310| | i -= 112; 311| | } 312| |#endif 313| 1.35k| seed ^= see1; 314| 1.35k| see2 ^= see3; 315| 1.35k| see4 ^= see5; 316| 1.35k| seed ^= see6; 317| 1.35k| see2 ^= see4; 318| 1.35k| seed ^= see2; 319| 1.35k| } 320| 1.56k| if (i > 16) { ------------------ | Branch (320:9): [True: 1.20k, False: 360] ------------------ 321| 1.20k| seed = rapid_mix(rapid_read64(p) ^ secret[2], rapid_read64(p + 8) ^ seed); 322| 1.20k| if (i > 32) { ------------------ | Branch (322:11): [True: 975, False: 228] ------------------ 323| 975| seed = rapid_mix(rapid_read64(p + 16) ^ secret[2], rapid_read64(p + 24) ^ seed); 324| 975| if (i > 48) { ------------------ | Branch (324:15): [True: 786, False: 189] ------------------ 325| 786| seed = rapid_mix(rapid_read64(p + 32) ^ secret[1], rapid_read64(p + 40) ^ seed); 326| 786| if (i > 64) { ------------------ | Branch (326:19): [True: 573, False: 213] ------------------ 327| 573| seed = rapid_mix(rapid_read64(p + 48) ^ secret[1], rapid_read64(p + 56) ^ seed); 328| 573| if (i > 80) { ------------------ | Branch (328:23): [True: 386, False: 187] ------------------ 329| 386| seed = rapid_mix(rapid_read64(p + 64) ^ secret[2], rapid_read64(p + 72) ^ seed); 330| 386| if (i > 96) { ------------------ | Branch (330:27): [True: 164, False: 222] ------------------ 331| 164| seed = rapid_mix(rapid_read64(p + 80) ^ secret[1], rapid_read64(p + 88) ^ seed); 332| 164| } 333| 386| } 334| 573| } 335| 786| } 336| 975| } 337| 1.20k| } 338| 1.56k| a=rapid_read64(p+i-16) ^ i; b=rapid_read64(p+i-8); 339| 1.56k| } 340| 1.63k| a ^= secret[1]; 341| 1.63k| b ^= seed; 342| 1.63k| rapid_mum(&a, &b); 343| 1.63k| return rapid_mix(a ^ secret[7], b ^ secret[1] ^ i); 344| 1.63k|} zxc_compress.c:rapid_mix: 208| 7.91M| RAPIDHASH_INLINE_CONSTEXPR uint64_t rapid_mix(uint64_t A, uint64_t B) RAPIDHASH_NOEXCEPT { rapid_mum(&A,&B); return A^B; } zxc_compress.c:rapid_read64: 214| 15.8M| RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return v;} zxc_compress.c:rapid_read32: 215| 12| RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return v;} zxc_compress.c:rapid_mum: 156| 7.91M| RAPIDHASH_INLINE_CONSTEXPR void rapid_mum(uint64_t *A, uint64_t *B) RAPIDHASH_NOEXCEPT { 157| 7.91M| #if defined(__SIZEOF_INT128__) 158| 7.91M| __uint128_t r=*A; r*=*B; 159| | #ifdef RAPIDHASH_PROTECTED 160| | *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); 161| | #else 162| 7.91M| *A=(uint64_t)r; *B=(uint64_t)(r>>64); 163| 7.91M| #endif 164| | #elif defined(_MSC_VER) && (defined(_WIN64) || defined(_M_HYBRID_CHPE_ARM64)) 165| | #if defined(_M_X64) 166| | #ifdef RAPIDHASH_PROTECTED 167| | uint64_t a, b; 168| | a=_umul128(*A,*B,&b); 169| | *A^=a; *B^=b; 170| | #else 171| | *A=_umul128(*A,*B,B); 172| | #endif 173| | #else 174| | #ifdef RAPIDHASH_PROTECTED 175| | uint64_t a, b; 176| | b = __umulh(*A, *B); 177| | a = *A * *B; 178| | *A^=a; *B^=b; 179| | #else 180| | uint64_t c = __umulh(*A, *B); 181| | *A = *A * *B; 182| | *B = c; 183| | #endif 184| | #endif 185| | #else 186| | uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B; 187| | uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t>32)+(rm1>>32)+c; 191| | #ifdef RAPIDHASH_PROTECTED 192| | *A^=lo; *B^=hi; 193| | #else 194| | *A=lo; *B=hi; 195| | #endif 196| | #endif 197| 7.91M| } zxc_decompress.c:rapidhash_withSeed: 486| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_withSeed(const void *key, size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT { 487| 1.63k| return rapidhash_internal(key, len, seed, rapid_secret); 488| 1.63k|} zxc_decompress.c:rapidhash_internal: 243| 1.63k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_internal(const void *key, size_t len, uint64_t seed, const uint64_t* secret) RAPIDHASH_NOEXCEPT { 244| 1.63k| const uint8_t *p=(const uint8_t *)key; 245| 1.63k| seed ^= rapid_mix(seed ^ secret[2], secret[1]); 246| 1.63k| uint64_t a=0, b=0; 247| 1.63k| size_t i = len; 248| 1.63k| if (_likely_(len <= 16)) { ------------------ | | 106| 1.63k| # define _likely_(x) __builtin_expect(x,1) | | ------------------ | | | Branch (106:24): [True: 73, False: 1.56k] | | ------------------ ------------------ 249| 73| if (len >= 4) { ------------------ | Branch (249:9): [True: 52, False: 21] ------------------ 250| 52| seed ^= len; 251| 52| if (len >= 8) { ------------------ | Branch (251:11): [True: 46, False: 6] ------------------ 252| 46| const uint8_t* plast = p + len - 8; 253| 46| a = rapid_read64(p); 254| 46| b = rapid_read64(plast); 255| 46| } else { 256| 6| const uint8_t* plast = p + len - 4; 257| 6| a = rapid_read32(p); 258| 6| b = rapid_read32(plast); 259| 6| } 260| 52| } else if (len > 0) { ------------------ | Branch (260:16): [True: 21, False: 0] ------------------ 261| 21| a = (((uint64_t)p[0])<<45)|p[len-1]; 262| 21| b = p[len>>1]; 263| 21| } else 264| 0| a = b = 0; 265| 1.56k| } else { 266| 1.56k| if (len > 112) { ------------------ | Branch (266:9): [True: 1.35k, False: 209] ------------------ 267| 1.35k| uint64_t see1 = seed, see2 = seed; 268| 1.35k| uint64_t see3 = seed, see4 = seed; 269| 1.35k| uint64_t see5 = seed, see6 = seed; 270| 1.35k|#ifdef RAPIDHASH_COMPACT 271| 1.12M| do { 272| 1.12M| seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 273| 1.12M| see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 274| 1.12M| see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 275| 1.12M| see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 276| 1.12M| see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 277| 1.12M| see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 278| 1.12M| see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 279| 1.12M| p += 112; 280| 1.12M| i -= 112; 281| 1.12M| } while(i > 112); ------------------ | Branch (281:15): [True: 1.12M, False: 1.35k] ------------------ 282| |#else 283| | while (i > 224) { 284| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 285| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 286| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 287| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 288| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 289| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 290| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 291| | seed = rapid_mix(rapid_read64(p + 112) ^ secret[0], rapid_read64(p + 120) ^ seed); 292| | see1 = rapid_mix(rapid_read64(p + 128) ^ secret[1], rapid_read64(p + 136) ^ see1); 293| | see2 = rapid_mix(rapid_read64(p + 144) ^ secret[2], rapid_read64(p + 152) ^ see2); 294| | see3 = rapid_mix(rapid_read64(p + 160) ^ secret[3], rapid_read64(p + 168) ^ see3); 295| | see4 = rapid_mix(rapid_read64(p + 176) ^ secret[4], rapid_read64(p + 184) ^ see4); 296| | see5 = rapid_mix(rapid_read64(p + 192) ^ secret[5], rapid_read64(p + 200) ^ see5); 297| | see6 = rapid_mix(rapid_read64(p + 208) ^ secret[6], rapid_read64(p + 216) ^ see6); 298| | p += 224; 299| | i -= 224; 300| | } 301| | if (i > 112) { 302| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 303| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 304| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 305| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 306| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 307| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 308| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 309| | p += 112; 310| | i -= 112; 311| | } 312| |#endif 313| 1.35k| seed ^= see1; 314| 1.35k| see2 ^= see3; 315| 1.35k| see4 ^= see5; 316| 1.35k| seed ^= see6; 317| 1.35k| see2 ^= see4; 318| 1.35k| seed ^= see2; 319| 1.35k| } 320| 1.56k| if (i > 16) { ------------------ | Branch (320:9): [True: 1.20k, False: 360] ------------------ 321| 1.20k| seed = rapid_mix(rapid_read64(p) ^ secret[2], rapid_read64(p + 8) ^ seed); 322| 1.20k| if (i > 32) { ------------------ | Branch (322:11): [True: 975, False: 228] ------------------ 323| 975| seed = rapid_mix(rapid_read64(p + 16) ^ secret[2], rapid_read64(p + 24) ^ seed); 324| 975| if (i > 48) { ------------------ | Branch (324:15): [True: 786, False: 189] ------------------ 325| 786| seed = rapid_mix(rapid_read64(p + 32) ^ secret[1], rapid_read64(p + 40) ^ seed); 326| 786| if (i > 64) { ------------------ | Branch (326:19): [True: 573, False: 213] ------------------ 327| 573| seed = rapid_mix(rapid_read64(p + 48) ^ secret[1], rapid_read64(p + 56) ^ seed); 328| 573| if (i > 80) { ------------------ | Branch (328:23): [True: 386, False: 187] ------------------ 329| 386| seed = rapid_mix(rapid_read64(p + 64) ^ secret[2], rapid_read64(p + 72) ^ seed); 330| 386| if (i > 96) { ------------------ | Branch (330:27): [True: 164, False: 222] ------------------ 331| 164| seed = rapid_mix(rapid_read64(p + 80) ^ secret[1], rapid_read64(p + 88) ^ seed); 332| 164| } 333| 386| } 334| 573| } 335| 786| } 336| 975| } 337| 1.20k| } 338| 1.56k| a=rapid_read64(p+i-16) ^ i; b=rapid_read64(p+i-8); 339| 1.56k| } 340| 1.63k| a ^= secret[1]; 341| 1.63k| b ^= seed; 342| 1.63k| rapid_mum(&a, &b); 343| 1.63k| return rapid_mix(a ^ secret[7], b ^ secret[1] ^ i); 344| 1.63k|} zxc_decompress.c:rapid_mix: 208| 7.91M| RAPIDHASH_INLINE_CONSTEXPR uint64_t rapid_mix(uint64_t A, uint64_t B) RAPIDHASH_NOEXCEPT { rapid_mum(&A,&B); return A^B; } zxc_decompress.c:rapid_read64: 214| 15.8M| RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return v;} zxc_decompress.c:rapid_read32: 215| 12| RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return v;} zxc_decompress.c:rapid_mum: 156| 7.91M| RAPIDHASH_INLINE_CONSTEXPR void rapid_mum(uint64_t *A, uint64_t *B) RAPIDHASH_NOEXCEPT { 157| 7.91M| #if defined(__SIZEOF_INT128__) 158| 7.91M| __uint128_t r=*A; r*=*B; 159| | #ifdef RAPIDHASH_PROTECTED 160| | *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); 161| | #else 162| 7.91M| *A=(uint64_t)r; *B=(uint64_t)(r>>64); 163| 7.91M| #endif 164| | #elif defined(_MSC_VER) && (defined(_WIN64) || defined(_M_HYBRID_CHPE_ARM64)) 165| | #if defined(_M_X64) 166| | #ifdef RAPIDHASH_PROTECTED 167| | uint64_t a, b; 168| | a=_umul128(*A,*B,&b); 169| | *A^=a; *B^=b; 170| | #else 171| | *A=_umul128(*A,*B,B); 172| | #endif 173| | #else 174| | #ifdef RAPIDHASH_PROTECTED 175| | uint64_t a, b; 176| | b = __umulh(*A, *B); 177| | a = *A * *B; 178| | *A^=a; *B^=b; 179| | #else 180| | uint64_t c = __umulh(*A, *B); 181| | *A = *A * *B; 182| | *B = c; 183| | #endif 184| | #endif 185| | #else 186| | uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B; 187| | uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t>32)+(rm1>>32)+c; 191| | #ifdef RAPIDHASH_PROTECTED 192| | *A^=lo; *B^=hi; 193| | #else 194| | *A=lo; *B=hi; 195| | #endif 196| | #endif 197| 7.91M| } zxc_cctx_init_in_workspace: 255| 9.21k| const int defer_entropy_scratch) { 256| 9.21k| if (UNLIKELY(!ctx || !workspace || chunk_size == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 36.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | ------------------ ------------------ 257| | 258| 9.21k| const zxc_cctx_layout_t layout = 259| 9.21k| compute_cctx_layout(chunk_size, mode, level, dict_size, defer_entropy_scratch); 260| 9.21k| if (UNLIKELY(workspace_size < layout.total)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 261| | 262| 9.21k| ZXC_MEMSET(ctx, 0, sizeof(zxc_cctx_t)); ------------------ | | 172| 9.21k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 263| 9.21k| ctx->checksum_enabled = checksum_enabled; 264| 9.21k| ctx->chunk_size = chunk_size; 265| 9.21k| const uint32_t offset_bits = zxc_log2_u32((uint32_t)chunk_size); 266| 9.21k| ctx->offset_bits = offset_bits; 267| 9.21k| ctx->offset_mask = (uint32_t)((1ULL << offset_bits) - 1); 268| 9.21k| ctx->max_epoch = (uint32_t)(1ULL << (32 - offset_bits)); 269| | 270| | // memory_block stays NULL on the static-init path so zxc_cctx_free does 271| | // not try to free the caller's workspace. Sub-buffer pointers carry the 272| | // partition; ownership is implicit (the caller owns @p workspace). 273| 9.21k| uint8_t* const mem = (uint8_t*)workspace; 274| | 275| | // Dictionary concat scratch (both modes); init owns dict_size now so callers 276| | // no longer assign ctx->dict_size after init. 277| 9.21k| ctx->dict_size = dict_size; 278| 9.21k| if (dict_size > 0) { ------------------ | Branch (278:9): [True: 0, False: 9.21k] ------------------ 279| 0| ctx->dict_buffer = mem + layout.off_dict; 280| 0| ctx->dict_buffer_cap = layout.sz_dict; 281| 0| ctx->dict_huf = (zxc_dict_huf_state_t*)(void*)(mem + layout.off_dict_huf); 282| 0| } 283| | 284| 9.21k| if (mode == 0) { ------------------ | Branch (284:9): [True: 4.60k, False: 4.60k] ------------------ 285| 4.60k| ctx->work_buf = mem + layout.off_work; 286| 4.60k| ctx->work_buf_cap = chunk_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 354| 4.60k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 328| 4.60k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 287| 4.60k| ctx->lit_buffer = mem + layout.off_lit_dctx; 288| 4.60k| ctx->lit_buffer_cap = chunk_size + ZXC_PAD_SIZE; ------------------ | | 328| 4.60k|#define ZXC_PAD_SIZE 32 ------------------ 289| 4.60k| if (layout.sz_pivco_dctx) { ------------------ | Branch (289:13): [True: 0, False: 4.60k] ------------------ 290| 0| ctx->tok_buffer = mem + layout.off_tok_dctx; 291| 0| ctx->tok_buffer_cap = layout.sz_tok_dctx; 292| 0| ctx->pivco_scratch = mem + layout.off_pivco_dctx; 293| 0| ctx->pivco_scratch_cap = layout.sz_pivco_dctx; 294| 0| } 295| 4.60k| return ZXC_OK; 296| 4.60k| } 297| | 298| 4.60k| ctx->hash_table = (uint32_t*)(mem + layout.off_hash_pos); 299| 4.60k| ctx->hash_tags = mem + layout.off_hash_tags; 300| 4.60k| ctx->chain_table = (uint16_t*)(mem + layout.off_chain); 301| 4.60k| ctx->buf_sequences = (uint32_t*)(mem + layout.off_seq_union); 302| 4.60k| ctx->buf_offsets = (uint16_t*)(mem + layout.off_seq_union); 303| 4.60k| ctx->buf_tokens = mem + layout.off_seq_union + layout.max_seq * sizeof(uint16_t); 304| 4.60k| ctx->buf_split = ctx->buf_tokens + layout.max_seq; 305| 4.60k| ctx->buf_split_hist = (zxc_glo_split_hist_t*)(void*)(mem + layout.off_split_hist); 306| 4.60k| ctx->buf_extras = mem + layout.off_extras; 307| 4.60k| ctx->literals = mem + layout.off_lit_cctx; 308| 4.60k| if (layout.sz_opt) { ------------------ | Branch (308:9): [True: 0, False: 4.60k] ------------------ 309| 0| ctx->opt_scratch = mem + layout.off_opt; 310| 0| ctx->opt_scratch_cap = layout.sz_opt; 311| 0| } 312| | 313| 4.60k| ctx->compression_level = level; 314| 4.60k| ctx->epoch = 1; 315| | 316| 4.60k| ZXC_MEMSET(ctx->hash_table, 0, layout.sz_hash_pos); ------------------ | | 172| 4.60k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 317| 4.60k| ZXC_MEMSET(ctx->hash_tags, 0, layout.sz_hash_tags); ------------------ | | 172| 4.60k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 318| 4.60k| return ZXC_OK; 319| 9.21k|} zxc_cctx_init: 332| 9.21k| const int level, const int checksum_enabled, const size_t dict_size) { 333| 9.21k| if (UNLIKELY(chunk_size == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 334| | // Heap contexts defer the decode-side entropy scratch to the first entropy 335| | // section; static workspaces must have it already, they may never allocate. 336| 9.21k| const int defer_entropy = (mode == 0); 337| 9.21k| const size_t total = 338| 9.21k| compute_cctx_layout(chunk_size, mode, level, dict_size, defer_entropy).total; 339| 9.21k| if (UNLIKELY(total == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 340| | 341| 9.21k| uint8_t* const mem = (uint8_t*)ZXC_ALIGNED_MALLOC(total, ZXC_CACHE_LINE_SIZE); ------------------ | | 133| 9.21k|#define ZXC_ALIGNED_MALLOC(size, alignment) zxc_aligned_malloc(size, alignment) ------------------ 342| 9.21k| if (UNLIKELY(!mem)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 343| | 344| 9.21k| const int rc = zxc_cctx_init_in_workspace(ctx, mem, total, chunk_size, mode, level, 345| 9.21k| checksum_enabled, dict_size, defer_entropy); 346| 9.21k| if (UNLIKELY(rc != ZXC_OK)) { ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 347| | // LCOV_EXCL_START 348| 0| ZXC_ALIGNED_FREE(mem); ------------------ | | 136| 0|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 349| 0| return rc; 350| | // LCOV_EXCL_STOP 351| 0| } 352| | // Library-owned buffer: record the allocation so zxc_cctx_free frees it. 353| 9.21k| ctx->memory_block = mem; 354| 9.21k| return ZXC_OK; 355| 9.21k|} zxc_cctx_free: 390| 9.21k|void zxc_cctx_free(zxc_cctx_t* ctx) { 391| 9.21k| if (ctx->memory_block) { ------------------ | Branch (391:9): [True: 9.21k, False: 0] ------------------ 392| 9.21k| ZXC_ALIGNED_FREE(ctx->memory_block); ------------------ | | 136| 9.21k|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 393| 9.21k| ctx->memory_block = NULL; 394| 9.21k| } 395| 9.21k| if (ctx->entropy_block) { ------------------ | Branch (395:9): [True: 0, False: 9.21k] ------------------ 396| 0| ZXC_ALIGNED_FREE(ctx->entropy_block); ------------------ | | 136| 0|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 397| 0| ctx->entropy_block = NULL; 398| 0| } 399| | 400| 9.21k| ctx->lit_buffer = NULL; 401| 9.21k| ctx->hash_table = NULL; 402| 9.21k| ctx->hash_tags = NULL; 403| 9.21k| ctx->chain_table = NULL; 404| 9.21k| ctx->buf_sequences = NULL; 405| 9.21k| ctx->buf_tokens = NULL; 406| 9.21k| ctx->buf_offsets = NULL; 407| 9.21k| ctx->buf_extras = NULL; 408| 9.21k| ctx->buf_split = NULL; 409| 9.21k| ctx->buf_split_hist = NULL; 410| 9.21k| ctx->literals = NULL; 411| 9.21k| ctx->work_buf = NULL; 412| 9.21k| ctx->tok_buffer = NULL; 413| 9.21k| ctx->pivco_scratch = NULL; 414| 9.21k| ctx->opt_scratch = NULL; 415| 9.21k| ctx->dict_buffer = NULL; 416| 9.21k| ctx->dict_huf = NULL; 417| | 418| 9.21k| ctx->epoch = 0; 419| 9.21k| ctx->lit_buffer_cap = 0; 420| 9.21k| ctx->work_buf_cap = 0; 421| 9.21k| ctx->tok_buffer_cap = 0; 422| 9.21k| ctx->pivco_scratch_cap = 0; 423| 9.21k| ctx->opt_scratch_cap = 0; 424| 9.21k| ctx->dict_buffer_cap = 0; 425| 9.21k| ctx->dict_size = 0; 426| 9.21k| ctx->dict_huf_tree_ok = 0; 427| | ctx->lit_freq_acc = NULL; 428| 9.21k|} zxc_write_file_header: 477| 4.60k| const int has_checksum, const uint32_t dict_id, const int has_seek) { 478| 4.60k| if (UNLIKELY(dst_capacity < ZXC_FILE_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 479| | 480| 4.60k| zxc_store_le32(dst, ZXC_MAGIC_WORD); ------------------ | | 322| 4.60k|#define ZXC_MAGIC_WORD 0x9CB02EF5U ------------------ 481| 4.60k| dst[4] = ZXC_FILE_FORMAT_VERSION; ------------------ | | 325| 4.60k|#define ZXC_FILE_FORMAT_VERSION 9 ------------------ 482| | 483| | // Block size stored as log2 exponent (e.g. 18 = 256 KB) 484| 4.60k| dst[5] = (uint8_t)zxc_log2_u32((uint32_t)chunk_size); 485| | 486| 4.60k| uint8_t flags = has_checksum ? (ZXC_FILE_FLAG_HAS_CHECKSUM | ZXC_CHECKSUM_RAPIDHASH) : 0; ------------------ | | 370| 1.53k|#define ZXC_FILE_FLAG_HAS_CHECKSUM 0x80U ------------------ uint8_t flags = has_checksum ? (ZXC_FILE_FLAG_HAS_CHECKSUM | ZXC_CHECKSUM_RAPIDHASH) : 0; ------------------ | | 426| 1.53k|#define ZXC_CHECKSUM_RAPIDHASH 0 ------------------ | Branch (486:21): [True: 1.53k, False: 3.06k] ------------------ 487| 4.60k| if (dict_id != 0) flags |= ZXC_FILE_FLAG_HAS_DICTIONARY; ------------------ | | 372| 0|#define ZXC_FILE_FLAG_HAS_DICTIONARY 0x40U ------------------ | Branch (487:9): [True: 0, False: 4.60k] ------------------ 488| 4.60k| if (has_seek) flags |= ZXC_FILE_FLAG_HAS_SEEK_TABLE; ------------------ | | 375| 0|#define ZXC_FILE_FLAG_HAS_SEEK_TABLE 0x20U ------------------ | Branch (488:9): [True: 0, False: 4.60k] ------------------ 489| 4.60k| dst[6] = flags; 490| | 491| | // Bytes 7-13: Reserved / Dictionary ID 492| 4.60k| ZXC_MEMSET(dst + 7, 0, 7); ------------------ | | 172| 4.60k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 493| 4.60k| if (dict_id != 0) zxc_store_le32(dst + 7, dict_id); ------------------ | Branch (493:9): [True: 0, False: 4.60k] ------------------ 494| | 495| | // Bytes 14-15: Header Checksum (16-bit) 496| 4.60k| zxc_file_header_sign(dst); 497| | 498| 4.60k| return ZXC_FILE_HEADER_SIZE; ------------------ | | 104| 4.60k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ 499| 4.60k|} zxc_read_file_header: 508| 9.10k| uint32_t* RESTRICT out_dict_id, int* RESTRICT out_has_seek) { 509| 9.10k| if (UNLIKELY(src_size < ZXC_FILE_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 9.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.10k] | | ------------------ ------------------ 510| 9.10k| if (UNLIKELY(zxc_le32(src) != ZXC_MAGIC_WORD)) return ZXC_ERROR_BAD_MAGIC; ------------------ | | 151| 9.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 4.37k, False: 4.72k] | | ------------------ ------------------ 511| 4.72k| if (UNLIKELY(src[4] != ZXC_FILE_FORMAT_VERSION)) return ZXC_ERROR_BAD_VERSION; ------------------ | | 151| 4.72k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 123, False: 4.60k] | | ------------------ ------------------ 512| | 513| 4.60k| uint8_t temp[ZXC_FILE_HEADER_SIZE]; 514| 4.60k| ZXC_MEMCPY(temp, src, ZXC_FILE_HEADER_SIZE); ------------------ | | 167| 4.60k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 515| | // Zero out checksum bytes (14-15) before hash check 516| 4.60k| temp[14] = 0; 517| 4.60k| temp[15] = 0; 518| | // Header checksum (integrity), then the checksum-algorithm id in flags bits 0-3 519| | // (only 0 = RapidHash is defined). It is checked first via short-circuit. 520| 4.60k| if (UNLIKELY(zxc_le16(src + 14) != zxc_hash16(temp) || ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 521| 4.60k| (src[6] & ZXC_FILE_CHECKSUM_ALGO_MASK) != ZXC_CHECKSUM_RAPIDHASH)) 522| 0| return ZXC_ERROR_BAD_HEADER; 523| | 524| 4.60k| if (out_block_size) { ------------------ | Branch (524:9): [True: 4.60k, False: 0] ------------------ 525| 4.60k| const uint8_t code = src[5]; 526| 4.60k| if (UNLIKELY(code < ZXC_BLOCK_SIZE_MIN_LOG2 || code > ZXC_BLOCK_SIZE_MAX_LOG2)) ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 527| 0| return ZXC_ERROR_BAD_BLOCK_SIZE; 528| | // Exponent encoding: block_size = 2^code (4 KB - 2 MB) 529| 4.60k| *out_block_size = (size_t)1U << code; 530| 4.60k| } 531| 4.60k| if (out_has_checksum) *out_has_checksum = (src[6] & ZXC_FILE_FLAG_HAS_CHECKSUM) ? 1 : 0; ------------------ | | 370| 4.60k|#define ZXC_FILE_FLAG_HAS_CHECKSUM 0x80U ------------------ | Branch (531:9): [True: 4.60k, False: 0] | Branch (531:47): [True: 1.53k, False: 3.06k] ------------------ 532| 4.60k| if (out_dict_id) *out_dict_id = (src[6] & ZXC_FILE_FLAG_HAS_DICTIONARY) ? zxc_le32(src + 7) : 0; ------------------ | | 372| 4.60k|#define ZXC_FILE_FLAG_HAS_DICTIONARY 0x40U ------------------ | Branch (532:9): [True: 4.60k, False: 0] | Branch (532:37): [True: 0, False: 4.60k] ------------------ 533| 4.60k| if (out_has_seek) *out_has_seek = (src[6] & ZXC_FILE_FLAG_HAS_SEEK_TABLE) ? 1 : 0; ------------------ | | 375| 4.60k|#define ZXC_FILE_FLAG_HAS_SEEK_TABLE 0x20U ------------------ | Branch (533:9): [True: 4.60k, False: 0] | Branch (533:39): [True: 0, False: 4.60k] ------------------ 534| | 535| 4.60k| return ZXC_OK; 536| 4.60k|} zxc_write_block_header: 542| 10.3k| const zxc_block_header_t* RESTRICT bh) { 543| 10.3k| if (UNLIKELY(dst_capacity < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 10.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 10.3k] | | ------------------ ------------------ 544| | 545| 10.3k| dst[0] = bh->block_type; 546| 10.3k| dst[1] = 0; // Flags not used currently 547| 10.3k| dst[2] = 0; // Reserved 548| 10.3k| zxc_store_le32(dst + 3, bh->comp_size); 549| 10.3k| dst[7] = 0; // Zero before hashing 550| 10.3k| dst[7] = zxc_hash8(dst); // Checksum at the end 551| | 552| 10.3k| return ZXC_BLOCK_HEADER_SIZE; ------------------ | | 407| 10.3k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 553| 10.3k|} zxc_read_block_header: 561| 8.15k| zxc_block_header_t* RESTRICT bh) { 562| 8.15k| if (UNLIKELY(src_size < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 8.15k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.15k] | | ------------------ ------------------ 563| | 564| 8.15k| uint8_t temp[ZXC_BLOCK_HEADER_SIZE]; 565| 8.15k| ZXC_MEMCPY(temp, src, ZXC_BLOCK_HEADER_SIZE); ------------------ | | 167| 8.15k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 566| 8.15k| temp[7] = 0; // Zero out checksum byte before hashing 567| 8.15k| if (UNLIKELY(src[7] != zxc_hash8(temp))) return ZXC_ERROR_BAD_HEADER; ------------------ | | 151| 8.15k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.15k] | | ------------------ ------------------ 568| | 569| 8.15k| bh->block_type = src[0]; 570| 8.15k| bh->block_flags = 0; // Flags not used currently 571| 8.15k| bh->reserved = src[2]; 572| 8.15k| bh->comp_size = zxc_le32(src + 3); 573| 8.15k| bh->header_checksum = src[7]; 574| | 575| 8.15k| return ZXC_OK; 576| 8.15k|} zxc_write_file_footer: 645| 4.60k| const uint64_t digest, const int checksum_enabled) { 646| 4.60k| const size_t need = zxc_footer_bytes(checksum_enabled); 647| 4.60k| if (UNLIKELY(dst_capacity < need)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 648| 4.60k| zxc_store_le64(dst, src_size); 649| 4.60k| if (checksum_enabled) zxc_store_le64(dst + ZXC_FILE_FOOTER_SIZE, digest); ------------------ | | 106| 1.53k|#define ZXC_FILE_FOOTER_SIZE 8 ------------------ | Branch (649:9): [True: 1.53k, False: 3.06k] ------------------ 650| 4.60k| return (int)need; 651| 4.60k|} zxc_write_ghi_header: 763| 4.90k| const zxc_gnr_header_t* RESTRICT gh) { 764| 4.90k| if (UNLIKELY(rem < ZXC_GHI_HEADER_BINARY_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 765| | 766| 4.90k| zxc_write_gnr_header(dst, gh); 767| 4.90k| return ZXC_GHI_HEADER_BINARY_SIZE; ------------------ | | 413| 4.90k|#define ZXC_GHI_HEADER_BINARY_SIZE 12 ------------------ 768| 4.90k|} zxc_read_ghi_header: 774| 4.10k| zxc_gnr_header_t* RESTRICT gh) { 775| 4.10k| if (UNLIKELY(len < ZXC_GHI_HEADER_BINARY_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 776| | 777| 4.10k| zxc_read_gnr_header(src, gh); 778| 4.10k| return ZXC_OK; 779| 4.10k|} zxc_compress_bound: 794| 4.60k|uint64_t zxc_compress_bound(const size_t input_size) { 795| | // Guard against uint64 overflow when summing per-block overhead 796| | // across very large inputs (input_size approaching SIZE_MAX). 797| 4.60k| if (UNLIKELY(input_size > (SIZE_MAX - (SIZE_MAX >> 8)))) return 0; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 798| 4.60k| uint64_t n = ((uint64_t)input_size + ZXC_BLOCK_SIZE_MIN - 1) / ZXC_BLOCK_SIZE_MIN; ------------------ | | 62| 4.60k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 4.60k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ uint64_t n = ((uint64_t)input_size + ZXC_BLOCK_SIZE_MIN - 1) / ZXC_BLOCK_SIZE_MIN; ------------------ | | 62| 4.60k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 4.60k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ 799| 4.60k| if (n == 0) n = 1; ------------------ | Branch (799:9): [True: 0, False: 4.60k] ------------------ 800| 4.60k| return ZXC_FILE_HEADER_SIZE + ------------------ | | 104| 4.60k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ 801| 4.60k| (n * (ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_CHECKSUM_SIZE + ZXC_BLOCK_FORMAT_OVERHEAD)) + ------------------ | | 407| 4.60k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ (n * (ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_CHECKSUM_SIZE + ZXC_BLOCK_FORMAT_OVERHEAD)) + ------------------ | | 409| 4.60k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ (n * (ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_CHECKSUM_SIZE + ZXC_BLOCK_FORMAT_OVERHEAD)) + ------------------ | | 423| 4.60k|#define ZXC_BLOCK_FORMAT_OVERHEAD 68 ------------------ 802| 4.60k| (uint64_t)input_size + ZXC_BLOCK_HEADER_SIZE + /* EOF block */ ------------------ | | 407| 4.60k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 803| 4.60k| ZXC_BLOCK_HEADER_SIZE + /* SEK block header (seekable) */ ------------------ | | 407| 4.60k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 804| 4.60k| zxc_seek_table_bytes(n) + /* SEK groups (seekable) */ 805| 4.60k| ZXC_FILE_FOOTER_SIZE + ZXC_FILE_DIGEST_SIZE; /* footer + optional digest */ ------------------ | | 106| 4.60k|#define ZXC_FILE_FOOTER_SIZE 8 ------------------ ZXC_FILE_FOOTER_SIZE + ZXC_FILE_DIGEST_SIZE; /* footer + optional digest */ ------------------ | | 108| 4.60k|#define ZXC_FILE_DIGEST_SIZE 8 ------------------ 806| 4.60k|} zxc_compress_block_bound: 811| 9.51k|uint64_t zxc_compress_block_bound(const size_t input_size) { 812| | // Mirrors the Block API contract: outside [1, ZXC_BLOCK_SIZE_MAX] the call 813| | // would fail anyway, so the bound is undefined and 0 says "unusable". The cap 814| | // also makes the addition below trivially overflow-free. 815| 9.51k| if (UNLIKELY(input_size == 0 || input_size > ZXC_BLOCK_SIZE_MAX)) return 0; ------------------ | | 151| 19.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.51k] | | | Branch (151:42): [True: 0, False: 9.51k] | | | Branch (151:42): [True: 0, False: 9.51k] | | ------------------ ------------------ 816| | // Outer block header + payload (worst case: incompressible, raw bytes) 817| | // + inner format overhead + optional checksum. 818| 9.51k| return (uint64_t)ZXC_BLOCK_HEADER_SIZE + (uint64_t)input_size + ZXC_BLOCK_FORMAT_OVERHEAD + ------------------ | | 407| 9.51k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ return (uint64_t)ZXC_BLOCK_HEADER_SIZE + (uint64_t)input_size + ZXC_BLOCK_FORMAT_OVERHEAD + ------------------ | | 423| 9.51k|#define ZXC_BLOCK_FORMAT_OVERHEAD 68 ------------------ 819| 9.51k| ZXC_BLOCK_CHECKSUM_SIZE; ------------------ | | 409| 9.51k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ 820| 9.51k|} zxc_common.c:compute_cctx_layout: 134| 18.4k| const int defer_entropy_scratch) { 135| 18.4k| zxc_cctx_layout_t layout = {0}; 136| | 137| 18.4k| const size_t max_seq = zxc_cctx_max_seq(chunk_size); 138| | 139| 18.4k| if (mode == 0) { ------------------ | Branch (139:9): [True: 9.21k, False: 9.21k] ------------------ 140| | // Decompress: work_buf + lit_buffer, padded for wild-copy overshoot and 141| | // sized worst-case. lit_buffer is provisioned at every level - the decoder 142| | // cannot predict a block's literal encoding (RAW / RLE / HUFFMAN). 143| 9.21k| const size_t sz_work = chunk_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 354| 9.21k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 328| 9.21k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 144| 9.21k| const size_t sz_lit = chunk_size + ZXC_PAD_SIZE; ------------------ | | 328| 9.21k|#define ZXC_PAD_SIZE 32 ------------------ 145| | 146| 9.21k| layout.off_work = layout.total; 147| 9.21k| layout.total += ZXC_ALIGN_CL(sz_work); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 148| 9.21k| layout.off_lit_dctx = layout.total; 149| 9.21k| layout.total += ZXC_ALIGN_CL(sz_lit); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 150| | // Token-section decode scratch (level-7 GLO Huffman-codes the tokens) + 151| | // PivCo ping-pong scratch. enc_lit/enc_tok are unpredictable, so static 152| | // workspaces provision both up front (no-alloc contract); heap contexts 153| | // defer them to the first entropy section, sparing L1-5 archives ~1.2x 154| | // chunk_size. See zxc_cctx_alloc_entropy_scratch. 155| 9.21k| if (!defer_entropy_scratch) { ------------------ | Branch (155:13): [True: 0, False: 9.21k] ------------------ 156| 0| size_t sz_tok = 0; 157| 0| size_t sz_pivco = 0; 158| 0| zxc_dctx_entropy_sizes(chunk_size, &sz_tok, &sz_pivco); 159| 0| layout.sz_tok_dctx = sz_tok; 160| 0| layout.off_tok_dctx = layout.total; 161| 0| layout.total += ZXC_ALIGN_CL(layout.sz_tok_dctx); ------------------ | | 360| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 162| 0| layout.sz_pivco_dctx = sz_pivco; 163| 0| layout.off_pivco_dctx = layout.total; 164| 0| layout.total += ZXC_ALIGN_CL(layout.sz_pivco_dctx); ------------------ | | 360| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 165| 0| } 166| 9.21k| } else { 167| | // Compress: 7 partitions + optional opt_scratch at level >= ZXC_LEVEL_DENSITY. 168| 9.21k| const uint32_t offset_bits = zxc_log2_u32((uint32_t)chunk_size); 169| 9.21k| layout.max_seq = max_seq; 170| 9.21k| layout.sz_hash_pos = ZXC_LZ_HASH_SIZE * sizeof(uint32_t); ------------------ | | 529| 9.21k|#define ZXC_LZ_HASH_SIZE (1U << ZXC_LZ_HASH_BITS) | | ------------------ | | | | 527| 9.21k|#define ZXC_LZ_HASH_BITS 15 | | ------------------ ------------------ 171| 9.21k| layout.sz_hash_tags = ZXC_LZ_HASH_SIZE * sizeof(uint8_t); ------------------ | | 529| 9.21k|#define ZXC_LZ_HASH_SIZE (1U << ZXC_LZ_HASH_BITS) | | ------------------ | | | | 527| 9.21k|#define ZXC_LZ_HASH_BITS 15 | | ------------------ ------------------ 172| 9.21k| const size_t sz_chain = ZXC_LZ_WINDOW_SIZE * sizeof(uint16_t); ------------------ | | 531| 9.21k|#define ZXC_LZ_WINDOW_SIZE (1U << 16) ------------------ 173| | // buf_sequences (GHI, level <= ZXC_LEVEL_FAST) aliases buf_offsets, buf_tokens and 174| | // buf_split (GLO, level >= ZXC_LEVEL_DEFAULT): 4 bytes per sequence either way. 175| 9.21k| const size_t sz_seq_union = layout.max_seq * sizeof(uint32_t); 176| 9.21k| const size_t vbyte_len = (offset_bits + 6) / 7; 177| 9.21k| const size_t sz_extras = layout.max_seq * 2 * vbyte_len; 178| 9.21k| const size_t sz_lit = chunk_size + ZXC_PAD_SIZE; ------------------ | | 328| 9.21k|#define ZXC_PAD_SIZE 32 ------------------ 179| | 180| | // opt_scratch (level >= ZXC_LEVEL_DENSITY): the optimal parser's DP arrays, 181| | // reused transiently by the code-length builder and its nudge, so sized to 182| | // the larger demand. Keep in sync with zxc_estimate_cctx_size() and its 183| | // consumer in zxc_compress.c. 184| 9.21k| if (level >= ZXC_LEVEL_DENSITY) { ------------------ | Branch (184:13): [True: 0, False: 9.21k] ------------------ 185| 0| size_t sz_dp; 186| 0| size_t sz_pl; 187| 0| size_t sz_po; 188| 0| size_t sz_bm; 189| 0| zxc_opt_dp_sizes(chunk_size, &sz_dp, &sz_pl, &sz_po, &sz_bm); 190| 0| const size_t dp_needed = sz_dp + sz_pl + sz_po + sz_bm; 191| 0| layout.sz_opt = 192| 0| (dp_needed > ZXC_HUF_NUDGE_SCRATCH_SIZE) ? dp_needed : ZXC_HUF_NUDGE_SCRATCH_SIZE; ------------------ | | 998| 0|#define ZXC_HUF_NUDGE_SCRATCH_SIZE (ZXC_HUF_NUDGE_SCRATCH_OFF + sizeof(zxc_huf_nudge_ws_t)) | | ------------------ | | | | 996| 0|#define ZXC_HUF_NUDGE_SCRATCH_OFF ZXC_ALIGN_CL(ZXC_HUF_BUILD_SCRATCH_SIZE) | | | | ------------------ | | | | | | 360| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | | | | | ------------------ | | | | | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | | | | | ------------------ | | | | | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | | | | | ------------------ | | | | | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | | | | | ------------------ | | | | | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | ------------------ ------------------ (dp_needed > ZXC_HUF_NUDGE_SCRATCH_SIZE) ? dp_needed : ZXC_HUF_NUDGE_SCRATCH_SIZE; ------------------ | | 998| 0|#define ZXC_HUF_NUDGE_SCRATCH_SIZE (ZXC_HUF_NUDGE_SCRATCH_OFF + sizeof(zxc_huf_nudge_ws_t)) | | ------------------ | | | | 996| 0|#define ZXC_HUF_NUDGE_SCRATCH_OFF ZXC_ALIGN_CL(ZXC_HUF_BUILD_SCRATCH_SIZE) | | | | ------------------ | | | | | | 360| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | | | | | ------------------ | | | | | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | | | | | ------------------ | | | | | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | | | | | ------------------ | | | | | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | | | | | ------------------ | | | | | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | ------------------ ------------------ | Branch (192:17): [True: 0, False: 0] ------------------ 193| 0| } 194| | 195| 9.21k| layout.off_hash_pos = layout.total; 196| 9.21k| layout.total += ZXC_ALIGN_CL(layout.sz_hash_pos); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 197| 9.21k| layout.off_hash_tags = layout.total; 198| 9.21k| layout.total += ZXC_ALIGN_CL(layout.sz_hash_tags); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 199| 9.21k| layout.off_chain = layout.total; 200| 9.21k| layout.total += ZXC_ALIGN_CL(sz_chain); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 201| 9.21k| layout.off_seq_union = layout.total; 202| 9.21k| layout.total += ZXC_ALIGN_CL(sz_seq_union); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 203| 9.21k| layout.off_split_hist = layout.total; 204| 9.21k| layout.total += ZXC_ALIGN_CL(sizeof(zxc_glo_split_hist_t)); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 205| 9.21k| layout.off_extras = layout.total; 206| 9.21k| layout.total += ZXC_ALIGN_CL(sz_extras); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 207| 9.21k| layout.off_lit_cctx = layout.total; 208| 9.21k| layout.total += ZXC_ALIGN_CL(sz_lit); ------------------ | | 360| 9.21k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 9.21k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 9.21k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 209| | // opt_scratch is appended last so it is absent for levels 1..5 (zero 210| | // waste on the common path) and only inflates the workspace at level 6. 211| 9.21k| if (layout.sz_opt) { ------------------ | Branch (211:13): [True: 0, False: 9.21k] ------------------ 212| 0| layout.off_opt = layout.total; 213| 0| layout.total += ZXC_ALIGN_CL(layout.sz_opt); ------------------ | | 360| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 214| 0| } 215| 9.21k| } 216| | 217| | // [dict | data] concat scratch (dict only). Compress chunk_size already 218| | // spans [dict | block]; decompress prepends dict to a (chunk + PAD) region. 219| 18.4k| if (dict_size > 0) { ------------------ | Branch (219:9): [True: 0, False: 18.4k] ------------------ 220| 0| layout.sz_dict = (mode == 1) ? (chunk_size + ZXC_DECOMPRESS_TAIL_PAD) ------------------ | | 354| 0|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 328| 0|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ | Branch (220:26): [True: 0, False: 0] ------------------ 221| 0| : (dict_size + chunk_size + ZXC_DECOMPRESS_TAIL_PAD); ------------------ | | 354| 0|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 328| 0|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 222| 0| layout.off_dict = layout.total; 223| 0| layout.total += ZXC_ALIGN_CL(layout.sz_dict); ------------------ | | 360| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 224| 0| layout.off_dict_huf = layout.total; 225| 0| layout.total += ZXC_ALIGN_CL(sizeof(zxc_dict_huf_state_t)); ------------------ | | 360| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 358| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 356| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 226| 0| } 227| 18.4k| return layout; 228| 18.4k|} zxc_common.c:zxc_cctx_max_seq: 86| 18.4k|static ZXC_ALWAYS_INLINE size_t zxc_cctx_max_seq(const size_t chunk_size) { 87| 18.4k| return chunk_size / ZXC_LZ_MIN_MATCH_LEN + 16; ------------------ | | 535| 18.4k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 88| 18.4k|} zxc_common.c:zxc_write_gnr_header: 660| 4.90k| const zxc_gnr_header_t* RESTRICT gh) { 661| 4.90k| zxc_store_le32(dst, gh->n_sequences); 662| 4.90k| zxc_store_le32(dst + 4, gh->n_literals); 663| | 664| 4.90k| dst[8] = gh->enc_lit; 665| 4.90k| dst[9] = gh->enc_tok; 666| 4.90k| dst[10] = gh->enc_mlen; 667| 4.90k| dst[11] = gh->enc_off; 668| 4.90k|} zxc_common.c:zxc_read_gnr_header: 677| 4.10k| zxc_gnr_header_t* RESTRICT gh) { 678| 4.10k| gh->n_sequences = zxc_le32(src); 679| 4.10k| gh->n_literals = zxc_le32(src + 4); 680| 4.10k| gh->enc_lit = src[8]; 681| 4.10k| gh->enc_tok = src[9]; 682| 4.10k| gh->enc_mlen = src[10]; 683| 4.10k| gh->enc_off = src[11]; 684| 4.10k|} zxc_compress_chunk_wrapper_default: 2197| 4.90k| const uint64_t block_index) { 2198| 4.90k| if (UNLIKELY(dst_cap < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 2199| | 2200| 4.90k| const size_t dict_sz = ctx->dict_size; 2201| 4.90k| const size_t block_sz = src_sz - dict_sz; 2202| 4.90k| const uint8_t* block_data = chunk + dict_sz; 2203| 4.90k| size_t w = 0; 2204| 4.90k| int res = ZXC_OK; 2205| | 2206| 4.90k| if (ctx->compression_level <= 2) ------------------ | Branch (2206:9): [True: 4.90k, False: 0] ------------------ 2207| 4.90k| res = zxc_encode_block_ghi(ctx, chunk, src_sz, dst, dst_cap, &w); 2208| 0| else 2209| 0| res = zxc_encode_block_glo(ctx, chunk, src_sz, dst, dst_cap, &w); 2210| | 2211| | // Check expansion against block data size (excluding dict prefix). 2212| 4.90k| if (UNLIKELY(res != ZXC_OK || w >= block_sz)) { ------------------ | | 151| 9.81k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 802, False: 4.10k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 802, False: 4.10k] | | ------------------ ------------------ 2213| 802| res = zxc_encode_block_raw(block_data, block_sz, dst, dst_cap, &w); 2214| 802| if (UNLIKELY(res != ZXC_OK)) return res; ------------------ | | 151| 802|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 802] | | ------------------ ------------------ 2215| 802| } 2216| | 2217| 4.90k| if (ctx->checksum_enabled) { ------------------ | Branch (2217:9): [True: 1.63k, False: 3.27k] ------------------ 2218| 1.63k| if (UNLIKELY(w + ZXC_BLOCK_CHECKSUM_SIZE > dst_cap)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 1.63k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.63k] | | ------------------ ------------------ 2219| | 2220| 1.63k| zxc_store_le32(dst + w, 2221| 1.63k| zxc_checksum(block_data, block_sz, block_index, ZXC_CHECKSUM_RAPIDHASH)); ------------------ | | 426| 1.63k|#define ZXC_CHECKSUM_RAPIDHASH 0 ------------------ 2222| 1.63k| w += ZXC_BLOCK_CHECKSUM_SIZE; ------------------ | | 409| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ 2223| 1.63k| } 2224| | 2225| 4.90k| return (int)w; 2226| 4.90k|} zxc_compress.c:zxc_encode_block_ghi: 1997| 4.90k| size_t* RESTRICT const out_sz) { 1998| 4.90k| if (UNLIKELY(dst_cap < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1999| | 2000| 4.90k| const int level = ctx->compression_level; 2001| 4.90k| const size_t dict_sz = ctx->dict_size; 2002| | 2003| 4.90k| zxc_lz77_params_t lzp = zxc_get_lz77_params(level); 2004| | // The floor is a candidate until the block itself clears it. 2005| 4.90k| if (lzp.min_offset > 1 && zxc_block_is_short_dist_bound(src + dict_sz, src_sz - dict_sz)) ------------------ | Branch (2005:9): [True: 4.90k, False: 0] | Branch (2005:31): [True: 4.03k, False: 870] ------------------ 2006| 4.03k| lzp.min_offset = 1; 2007| | 2008| 4.90k| ctx->epoch++; 2009| 4.90k| if (UNLIKELY(ctx->epoch >= ctx->max_epoch)) { ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 2010| 0| ZXC_MEMSET(ctx->hash_table, 0, ZXC_LZ_HASH_SIZE * sizeof(uint32_t)); ------------------ | | 172| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 2011| 0| ZXC_MEMSET(ctx->hash_tags, 0, ZXC_LZ_HASH_SIZE * sizeof(uint8_t)); ------------------ | | 172| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 2012| 0| ctx->epoch = 1; 2013| 0| } 2014| 4.90k| const uint32_t offset_bits = ctx->offset_bits; 2015| 4.90k| const uint32_t offset_mask = ctx->offset_mask; 2016| 4.90k| const uint32_t epoch_mark = ctx->epoch << offset_bits; 2017| | 2018| 4.90k| if (dict_sz > 0) ------------------ | Branch (2018:9): [True: 0, False: 4.90k] ------------------ 2019| 0| zxc_lz_seed_dict(src, dict_sz, ctx->hash_table, ctx->hash_tags, ctx->chain_table, 2020| 0| epoch_mark, offset_mask, level); 2021| | 2022| 4.90k| const uint8_t* ip = src + dict_sz; 2023| 4.90k| const uint8_t* iend = src + src_sz; 2024| 4.90k| const uint8_t* anchor = ip; 2025| 4.90k| const uint8_t* search_limit = iend - ZXC_LZ_SEARCH_MARGIN; ------------------ | | 649| 4.90k|#define ZXC_LZ_SEARCH_MARGIN (sizeof(uint64_t)) ------------------ 2026| | 2027| 4.90k| uint32_t* const hash_table = ctx->hash_table; 2028| 4.90k| uint8_t* const hash_tags = ctx->hash_tags; 2029| 4.90k| uint8_t* const buf_extras = ctx->buf_extras; 2030| 4.90k| uint16_t* const chain_table = ctx->chain_table; 2031| 4.90k| uint8_t* const literals = ctx->literals; 2032| 4.90k| uint32_t* const buf_sequences = ctx->buf_sequences; 2033| | 2034| 4.90k| uint32_t seq_c = 0; 2035| 4.90k| size_t extras_c = 0; 2036| 4.90k| size_t lit_c = 0; 2037| | 2038| 8.29M| while (LIKELY(ip < search_limit)) { ------------------ | | 145| 8.29M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 8.28M, False: 4.90k] | | ------------------ ------------------ 2039| 8.28M| size_t dist = (size_t)(ip - anchor); 2040| 8.28M| size_t step = lzp.step_base + (dist >> lzp.step_shift); 2041| 8.28M| if (UNLIKELY(ip + step >= search_limit)) step = 1; ------------------ | | 151| 8.28M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.43M, False: 6.85M] | | ------------------ ------------------ 2042| | 2043| 8.28M| ZXC_PREFETCH_READ(ip + step * 4 + ZXC_CACHE_LINE_SIZE); ------------------ | | 162| 8.28M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 2044| | 2045| 8.28M| if (LIKELY(ip + step + sizeof(uint64_t) <= iend)) { ------------------ | | 145| 8.28M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 8.28M, False: 0] | | ------------------ ------------------ 2046| 8.28M| const uint64_t v_next = zxc_le64(ip + step); 2047| | // cppcheck-suppress unreadVariable 2048| 8.28M| const uint32_t h_next = zxc_hash_func(v_next, 0); 2049| 8.28M| ZXC_PREFETCH_READ(&hash_tags[h_next]); ------------------ | | 162| 8.28M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 2050| 8.28M| ZXC_PREFETCH_READ(&hash_table[h_next]); ------------------ | | 162| 8.28M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 2051| 8.28M| } 2052| | 2053| 8.28M| const zxc_match_t m = 2054| 8.28M| zxc_lz77_find_best_match(src, ip, iend, search_limit, anchor, hash_table, hash_tags, 2055| 8.28M| chain_table, epoch_mark, offset_mask, level, lzp, 2056| 8.28M| /*last_off=*/0U); 2057| | 2058| 8.28M| if (m.ref) { ------------------ | Branch (2058:13): [True: 2.62M, False: 5.66M] ------------------ 2059| 2.62M| ip -= m.backtrack; 2060| 2.62M| const uint32_t ll = (uint32_t)(ip - anchor); 2061| 2.62M| const uint32_t ml = m.len - ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 535| 2.62M|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 2062| 2.62M| const uint32_t off = (uint32_t)(ip - m.ref); 2063| | 2064| 2.62M| if (ll > 0) zxc_flush_literals(literals, &lit_c, anchor, ll, iend); ------------------ | Branch (2064:17): [True: 1.08M, False: 1.53M] ------------------ 2065| | 2066| 2.62M| const uint32_t ll_write = (ll >= ZXC_SEQ_LL_MASK) ? 255U : ll; ------------------ | | 500| 2.62M|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 494| 2.62M|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ | Branch (2066:39): [True: 15.4k, False: 2.60M] ------------------ 2067| 2.62M| const uint32_t ml_write = (ml >= ZXC_SEQ_ML_MASK) ? 255U : ml; ------------------ | | 502| 2.62M|#define ZXC_SEQ_ML_MASK ((1U << ZXC_SEQ_ML_BITS) - 1) | | ------------------ | | | | 496| 2.62M|#define ZXC_SEQ_ML_BITS 8 | | ------------------ ------------------ | Branch (2067:39): [True: 88.8k, False: 2.53M] ------------------ 2068| 2.62M| const uint32_t seq_val = (ll_write << (ZXC_SEQ_ML_BITS + ZXC_SEQ_OFF_BITS)) | ------------------ | | 496| 2.62M|#define ZXC_SEQ_ML_BITS 8 ------------------ const uint32_t seq_val = (ll_write << (ZXC_SEQ_ML_BITS + ZXC_SEQ_OFF_BITS)) | ------------------ | | 498| 2.62M|#define ZXC_SEQ_OFF_BITS 16 ------------------ 2069| 2.62M| (ml_write << ZXC_SEQ_OFF_BITS) | ------------------ | | 498| 2.62M|#define ZXC_SEQ_OFF_BITS 16 ------------------ 2070| 2.62M| ((off - ZXC_LZ_OFFSET_BIAS) & ZXC_SEQ_OFF_MASK); ------------------ | | 618| 2.62M|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ ((off - ZXC_LZ_OFFSET_BIAS) & ZXC_SEQ_OFF_MASK); ------------------ | | 504| 2.62M|#define ZXC_SEQ_OFF_MASK ((1U << ZXC_SEQ_OFF_BITS) - 1) | | ------------------ | | | | 498| 2.62M|#define ZXC_SEQ_OFF_BITS 16 | | ------------------ ------------------ 2071| 2.62M| buf_sequences[seq_c] = seq_val; 2072| 2.62M| seq_c++; 2073| | 2074| 2.62M| if (UNLIKELY(!zxc_emit_extra(buf_extras, &extras_c, ll, ZXC_SEQ_LL_MASK) || ------------------ | | 151| 5.24M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 2.62M] | | | Branch (151:42): [True: 0, False: 2.62M] | | | Branch (151:42): [True: 0, False: 2.62M] | | ------------------ ------------------ 2075| 2.62M| !zxc_emit_extra(buf_extras, &extras_c, ml, ZXC_SEQ_ML_MASK))) 2076| 0| return ZXC_ERROR_OVERFLOW; 2077| | 2078| 2.62M| ip += m.len; 2079| 2.62M| anchor = ip; 2080| 5.66M| } else { 2081| 5.66M| ip += step; 2082| 5.66M| } 2083| 8.28M| } 2084| | 2085| 4.90k| const size_t last_lits = iend - anchor; 2086| 4.90k| if (last_lits > 0) { ------------------ | Branch (2086:9): [True: 4.07k, False: 833] ------------------ 2087| 4.07k| ZXC_MEMCPY(literals + lit_c, anchor, last_lits); ------------------ | | 167| 4.07k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 2088| 4.07k| lit_c += last_lits; 2089| 4.07k| } 2090| | 2091| 4.90k| zxc_block_header_t bh = {.block_type = ZXC_BLOCK_GHI}; 2092| 4.90k| uint8_t* const p = dst + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 407| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2093| 4.90k| size_t rem = dst_cap - ZXC_BLOCK_HEADER_SIZE; ------------------ | | 407| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2094| | 2095| 4.90k| const zxc_gnr_header_t gh = {.n_sequences = seq_c, 2096| 4.90k| .n_literals = (uint32_t)lit_c, 2097| 4.90k| .enc_lit = ZXC_SECTION_ENCODING_RAW, 2098| 4.90k| .enc_tok = 0, 2099| 4.90k| .enc_mlen = 0, 2100| 4.90k| .enc_off = 0}; 2101| | 2102| 4.90k| const int ghs = zxc_write_ghi_header(p, rem, &gh); 2103| 4.90k| if (UNLIKELY(ghs < 0)) return ghs; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 2104| | 2105| 4.90k| uint8_t* p_curr = p + ghs; 2106| 4.90k| rem -= ghs; 2107| | 2108| 4.90k| const size_t sz_lit = lit_c; 2109| 4.90k| const size_t sz_seq = seq_c * sizeof(uint32_t); 2110| 4.90k| const size_t sz_ext = extras_c; 2111| | 2112| | // See the GLO twin. GHI literals are always RAW, so the slack always 2113| | // matters here. 2114| 4.90k| const size_t behind_lit = sz_seq + sz_ext; 2115| 4.90k| const size_t pad = (behind_lit < ZXC_BLOCK_LIT_SLACK) ? ZXC_BLOCK_LIT_SLACK - behind_lit : 0; ------------------ | | 342| 4.90k|#define ZXC_BLOCK_LIT_SLACK 32 ------------------ const size_t pad = (behind_lit < ZXC_BLOCK_LIT_SLACK) ? ZXC_BLOCK_LIT_SLACK - behind_lit : 0; ------------------ | | 342| 1.69k|#define ZXC_BLOCK_LIT_SLACK 32 ------------------ | Branch (2115:24): [True: 1.69k, False: 3.21k] ------------------ 2116| | 2117| 4.90k| if (UNLIKELY(rem < sz_lit + sz_seq + sz_ext + pad)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 2118| | 2119| 4.90k| ZXC_MEMCPY(p_curr, literals, lit_c); ------------------ | | 167| 4.90k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 2120| 4.90k| p_curr += lit_c; 2121| | 2122| | // Write sequences in little-endian order 2123| |#ifdef ZXC_BIG_ENDIAN 2124| | for (uint32_t i = 0; i < seq_c; i++) { 2125| | zxc_store_le32(p_curr, buf_sequences[i]); 2126| | p_curr += sizeof(uint32_t); 2127| | } 2128| |#else 2129| 4.90k| ZXC_MEMCPY(p_curr, buf_sequences, sz_seq); ------------------ | | 167| 4.90k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 2130| 4.90k| p_curr += sz_seq; 2131| 4.90k|#endif 2132| | 2133| | // --- WRITE EXTRAS --- 2134| 4.90k| ZXC_MEMCPY(p_curr, buf_extras, sz_ext); ------------------ | | 167| 4.90k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 2135| 4.90k| p_curr += sz_ext; 2136| | 2137| 4.90k| ZXC_MEMSET(p_curr, 0, pad); ------------------ | | 172| 4.90k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 2138| 4.90k| p_curr += pad; 2139| | 2140| 4.90k| bh.comp_size = (uint32_t)(p_curr - (dst + ZXC_BLOCK_HEADER_SIZE)); ------------------ | | 407| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2141| 4.90k| const int hw = zxc_write_block_header(dst, dst_cap, &bh); 2142| 4.90k| if (UNLIKELY(hw < 0)) return hw; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 2143| | 2144| | // Checksum will be appended by the wrapper 2145| 4.90k| *out_sz = ZXC_BLOCK_HEADER_SIZE + bh.comp_size; ------------------ | | 407| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2146| 4.90k| return ZXC_OK; 2147| 4.90k|} zxc_compress.c:zxc_hash_tag: 61| 8.28M|static ZXC_ALWAYS_INLINE uint8_t zxc_hash_tag(const uint32_t val4) { 62| 8.28M| return (uint8_t)(val4 ^ (val4 >> 16)); 63| 8.28M|} zxc_compress.c:zxc_epoch_pos: 72| 3.91M| const uint32_t epoch_mark) { 73| 3.91M| return ((head & ~offset_mask) == epoch_mark) ? (head & offset_mask) : 0; ------------------ | Branch (73:12): [True: 3.34M, False: 571k] ------------------ 74| 3.91M|} zxc_compress.c:zxc_chain_delta: 82| 8.28M|static ZXC_ALWAYS_INLINE uint16_t zxc_chain_delta(const uint32_t cur_pos, const uint32_t prev_idx) { 83| 8.28M| const uint32_t dist = cur_pos - prev_idx; 84| 8.28M| const uint32_t valid = -((int32_t)((prev_idx != 0) & (dist < ZXC_LZ_WINDOW_SIZE))); ------------------ | | 531| 8.28M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) ------------------ 85| 8.28M| return (uint16_t)(dist & valid); 86| 8.28M|} zxc_compress.c:zxc_hash_func: 45| 16.5M|static ZXC_ALWAYS_INLINE uint32_t zxc_hash_func(const uint64_t val, const int use_hash5) { 46| 16.5M| if (use_hash5) { ------------------ | Branch (46:9): [True: 0, False: 16.5M] ------------------ 47| 0| const uint64_t v5 = val & 0xFFFFFFFFFFULL; 48| 0| return (uint32_t)((v5 * ZXC_HASH_XORSHIFT64) >> (64 - ZXC_LZ_HASH_BITS)); ------------------ | | 678| 0|#define ZXC_HASH_XORSHIFT64 0x2545F4914F6CDD1DULL ------------------ return (uint32_t)((v5 * ZXC_HASH_XORSHIFT64) >> (64 - ZXC_LZ_HASH_BITS)); ------------------ | | 527| 0|#define ZXC_LZ_HASH_BITS 15 ------------------ 49| 16.5M| } else { 50| 16.5M| const uint64_t v4 = val ^ (val >> 15); 51| 16.5M| return ((uint32_t)v4 * ZXC_HASH_MULT32) >> (32 - ZXC_LZ_HASH_BITS); ------------------ | | 680| 16.5M|#define ZXC_HASH_MULT32 0x2D35182DU ------------------ return ((uint32_t)v4 * ZXC_HASH_MULT32) >> (32 - ZXC_LZ_HASH_BITS); ------------------ | | 527| 16.5M|#define ZXC_LZ_HASH_BITS 15 ------------------ 52| 16.5M| } 53| 16.5M|} zxc_compress.c:zxc_lz77_find_best_match: 489| 8.28M| const int level, const zxc_lz77_params_t p, const uint32_t last_off) { 490| 8.28M| const int use_hash5 = (level >= 3); 491| | // Sentinel length just below the minimum, so any valid match replaces it. 492| 8.28M| zxc_match_t best = (zxc_match_t){NULL, ZXC_LZ_MIN_MATCH_LEN - 1, 0}; ------------------ | | 535| 8.28M|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 493| | 494| 8.28M| uint64_t cur_val8 = zxc_le64(ip); 495| 8.28M| uint32_t cur_val = (uint32_t)cur_val8; 496| 8.28M| uint32_t h = zxc_hash_func(cur_val8, use_hash5); 497| | 498| | // 8-bit tag: XOR fold of first 4 bytes for fast rejection 499| 8.28M| const uint8_t cur_tag = zxc_hash_tag(cur_val); 500| | 501| 8.28M| const uint32_t cur_pos = (uint32_t)(ip - src); 502| | 503| | // Tag-first filter on fast levels. 504| 8.28M| const uint8_t stored_tag = hash_tags[h]; 505| 8.28M| uint32_t match_idx; 506| 8.28M| if (level <= ZXC_LEVEL_FAST && stored_tag != cur_tag) { ------------------ | Branch (506:9): [True: 8.28M, False: 0] | Branch (506:36): [True: 4.37M, False: 3.91M] ------------------ 507| 4.37M| match_idx = 0; 508| 4.37M| } else { 509| 3.91M| match_idx = zxc_epoch_pos(hash_table[h], offset_mask, epoch_mark); 510| 3.91M| } 511| | 512| | // skip_head still drives the chain walk on level >= 3 (advances past the 513| | // mismatched head without comparing). On level <= 2 it is always 0 here: 514| | // either match_idx == 0 (filter-skip) or stored_tag == cur_tag. 515| 8.28M| const int skip_head = (match_idx != 0) & (stored_tag != cur_tag); 516| | 517| | // Split table writes 518| 8.28M| hash_table[h] = epoch_mark | cur_pos; 519| 8.28M| hash_tags[h] = cur_tag; 520| | 521| 8.28M| chain_table[cur_pos & ZXC_LZ_WINDOW_MASK] = zxc_chain_delta(cur_pos, match_idx); ------------------ | | 533| 8.28M|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 531| 8.28M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 522| | 523| 8.28M| int attempts = p.search_depth; 524| | 525| | // Repeat-offset seed (level-6 parser passes last_off; others pass 0). 526| | // Probing the previous offset first often finds the longest match right 527| | // away, speeding up the chain walk. It only raises best.len, never lowers 528| | // it, so the result is unchanged - this is purely a speed optimization. 529| 8.28M| if (last_off != 0U && last_off >= p.min_offset && last_off <= (uint32_t)ZXC_LZ_MAX_DIST && ------------------ | | 620| 8.28M|#define ZXC_LZ_MAX_DIST (ZXC_LZ_WINDOW_SIZE - 1) | | ------------------ | | | | 531| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ | Branch (529:9): [True: 0, False: 8.28M] | Branch (529:27): [True: 0, False: 0] | Branch (529:55): [True: 0, False: 0] ------------------ 530| 0| last_off <= cur_pos) { ------------------ | Branch (530:9): [True: 0, False: 0] ------------------ 531| 0| const uint8_t* const rep_ref = src + (cur_pos - last_off); 532| 0| if (zxc_le32(rep_ref) == cur_val) { ------------------ | Branch (532:13): [True: 0, False: 0] ------------------ 533| 0| uint32_t mlen = sizeof(uint32_t); 534| 0| const uint8_t* const limit_8 = iend - sizeof(uint64_t); 535| 0| while (ip + mlen < limit_8) { ------------------ | Branch (535:20): [True: 0, False: 0] ------------------ 536| 0| const uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(rep_ref + mlen); 537| 0| if (diff == 0) { ------------------ | Branch (537:21): [True: 0, False: 0] ------------------ 538| 0| mlen += sizeof(uint64_t); 539| 0| } else { 540| 0| mlen += (uint32_t)(zxc_ctz64(diff) >> 3); 541| 0| goto _rep_done; 542| 0| } 543| 0| } 544| 0| while (ip + mlen < iend && rep_ref[mlen] == ip[mlen]) mlen++; ------------------ | Branch (544:20): [True: 0, False: 0] | Branch (544:40): [True: 0, False: 0] ------------------ 545| 0| _rep_done:; 546| 0| best.len = mlen; 547| 0| best.ref = rep_ref; 548| 0| if (UNLIKELY(best.len >= (uint32_t)p.sufficient_len || ip + best.len >= iend)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 549| 0| goto _finalize_match; 550| 0| } 551| 0| } 552| | 553| 8.28M| if (match_idx == 0) goto _finalize_match; ------------------ | Branch (553:9): [True: 4.94M, False: 3.34M] ------------------ 554| | 555| | // Optimization: If head tag doesn't match, advance immediately without loading the first 556| | // mismatch. 557| 3.34M| if (skip_head) { ------------------ | Branch (557:9): [True: 0, False: 3.34M] ------------------ 558| 0| const uint16_t delta = chain_table[match_idx & ZXC_LZ_WINDOW_MASK]; ------------------ | | 533| 0|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 531| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 559| 0| const uint32_t next_idx = match_idx - delta; 560| 0| match_idx = (delta != 0) ? next_idx : 0; ------------------ | Branch (560:21): [True: 0, False: 0] ------------------ 561| 0| attempts--; 562| 0| } 563| | 564| 9.63M| while (match_idx > 0) { ------------------ | Branch (564:12): [True: 9.26M, False: 362k] ------------------ 565| 9.26M| if (UNLIKELY(attempts-- < 0 || cur_pos - match_idx > ZXC_LZ_MAX_DIST)) break; ------------------ | | 151| 17.2M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.89M, False: 7.37M] | | | Branch (151:42): [True: 1.29M, False: 7.97M] | | | Branch (151:42): [True: 598k, False: 7.37M] | | ------------------ ------------------ 566| 7.37M| const uint8_t* ref = src + match_idx; 567| | 568| | // Load the next chain link early (before the compare) so its address 569| | // resolves while we prefetch. 570| 7.37M| const uint16_t delta = chain_table[match_idx & ZXC_LZ_WINDOW_MASK]; ------------------ | | 533| 7.37M|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 531| 7.37M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 571| 7.37M| const uint32_t next_idx = match_idx - delta; 572| 7.37M| ZXC_PREFETCH_READ(src + next_idx); ------------------ | | 162| 7.37M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 573| | 574| 7.37M| const uint32_t ref_val = zxc_le32(ref); 575| 7.37M| const int tag_match = (ref_val == cur_val); 576| | // Cheap gate: 4-byte tag match, then check the byte past the current 577| | // best (the && skips that load unless the tag already matched). 578| 7.37M| const int should_compare = 579| 7.37M| (cur_pos - match_idx) >= p.min_offset && tag_match && (ref[best.len] == ip[best.len]); ------------------ | Branch (579:13): [True: 6.84M, False: 529k] | Branch (579:54): [True: 6.62M, False: 216k] | Branch (579:67): [True: 3.72M, False: 2.90M] ------------------ 580| | 581| 7.37M| if (should_compare) { ------------------ | Branch (581:13): [True: 3.72M, False: 3.64M] ------------------ 582| 3.72M| uint32_t mlen = sizeof(uint32_t); // We already know the first 4 bytes match 583| | 584| | // Fast path: Scalar 64-bit comparison for short matches (=< 64 bytes) 585| | // Most matches are short, so this avoids SIMD overhead for common cases 586| 3.72M| const uint8_t* limit_8 = iend - sizeof(uint64_t); 587| 3.72M| const uint8_t* scalar_limit = ip + mlen + 64; 588| 3.72M| if (scalar_limit > limit_8) scalar_limit = limit_8; ------------------ | Branch (588:17): [True: 7.56k, False: 3.72M] ------------------ 589| | 590| 8.35M| while (ip + mlen < scalar_limit) { ------------------ | Branch (590:20): [True: 8.10M, False: 252k] ------------------ 591| 8.10M| uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(ref + mlen); 592| 8.10M| if (diff == 0) ------------------ | Branch (592:21): [True: 4.62M, False: 3.47M] ------------------ 593| 4.62M| mlen += sizeof(uint64_t); 594| 3.47M| else { 595| 3.47M| mlen += (zxc_ctz64(diff) >> 3); 596| 3.47M| goto _match_len_done; 597| 3.47M| } 598| 8.10M| } 599| | 600| | // Long match path: Use SIMD for matches exceeding 64 bytes 601| |#if defined(ZXC_USE_AVX512) 602| | const uint8_t* limit_64 = iend - 64; 603| | while (ip + mlen < limit_64) { 604| | const __m512i v_src = _mm512_loadu_si512((const void*)(ip + mlen)); 605| | const __m512i v_ref = _mm512_loadu_si512((const void*)(ref + mlen)); 606| | const __mmask64 mask = _mm512_cmpeq_epi8_mask(v_src, v_ref); 607| | if (mask == 0xFFFFFFFFFFFFFFFF) 608| | mlen += 64; 609| | else { 610| | mlen += (uint32_t)zxc_ctz64(~mask); 611| | goto _match_len_done; 612| | } 613| | } 614| |#elif defined(ZXC_USE_AVX2) 615| | const uint8_t* limit_32 = iend - 32; 616| | while (ip + mlen < limit_32) { 617| | const __m256i v_src = _mm256_loadu_si256((const __m256i*)(ip + mlen)); 618| | const __m256i v_ref = _mm256_loadu_si256((const __m256i*)(ref + mlen)); 619| | const __m256i v_cmp = _mm256_cmpeq_epi8(v_src, v_ref); 620| | const uint32_t mask = (uint32_t)_mm256_movemask_epi8(v_cmp); 621| | if (mask == 0xFFFFFFFF) 622| | mlen += 32; 623| | else { 624| | mlen += zxc_ctz32(~mask); 625| | goto _match_len_done; 626| | } 627| | } 628| |#elif defined(ZXC_USE_SSE2) 629| 252k| const uint8_t* limit_16 = iend - 16; 630| 11.2M| while (ip + mlen < limit_16) { ------------------ | Branch (630:20): [True: 11.2M, False: 2.09k] ------------------ 631| 11.2M| const __m128i v_src = _mm_loadu_si128((const __m128i*)(ip + mlen)); 632| 11.2M| const __m128i v_ref = _mm_loadu_si128((const __m128i*)(ref + mlen)); 633| 11.2M| const __m128i v_cmp = _mm_cmpeq_epi8(v_src, v_ref); 634| 11.2M| const uint32_t mask = (uint32_t)_mm_movemask_epi8(v_cmp); 635| 11.2M| if (mask == 0xFFFFU) ------------------ | Branch (635:21): [True: 11.0M, False: 250k] ------------------ 636| 11.0M| mlen += 16; 637| 250k| else { 638| | // mask != 0xFFFF => a differing byte exists in bits 0..15, 639| | // so the lowest set bit of ~mask lies in that range. 640| 250k| mlen += zxc_ctz32(~mask); 641| 250k| goto _match_len_done; 642| 250k| } 643| 11.2M| } 644| |#elif defined(ZXC_USE_NEON64) 645| | { 646| | const uint8_t* limit_32 = iend - 32; 647| | while (ip + mlen < limit_32) { 648| | const uint8x16_t s0 = vld1q_u8(ip + mlen); 649| | const uint8x16_t r0 = vld1q_u8(ref + mlen); 650| | const uint8x16_t c0 = vceqq_u8(s0, r0); 651| | const uint64_t m0 = vget_lane_u64( 652| | vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(c0), 4)), 0); 653| | if (UNLIKELY(m0 != ~(uint64_t)0)) { 654| | mlen += (uint32_t)(zxc_ctz64(~m0) >> 2); 655| | goto _match_len_done; 656| | } 657| | const uint8x16_t s1 = vld1q_u8(ip + mlen + 16); 658| | const uint8x16_t r1 = vld1q_u8(ref + mlen + 16); 659| | const uint8x16_t c1 = vceqq_u8(s1, r1); 660| | const uint64_t m1 = vget_lane_u64( 661| | vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(c1), 4)), 0); 662| | if (UNLIKELY(m1 != ~(uint64_t)0)) { 663| | mlen += 16 + (uint32_t)(zxc_ctz64(~m1) >> 2); 664| | goto _match_len_done; 665| | } 666| | mlen += 32; 667| | } 668| | if (ip + mlen < iend - 16) { 669| | const uint8x16_t v_src = vld1q_u8(ip + mlen); 670| | const uint8x16_t v_ref = vld1q_u8(ref + mlen); 671| | const uint8x16_t v_cmp = vceqq_u8(v_src, v_ref); 672| | const uint64_t mask = vget_lane_u64( 673| | vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(v_cmp), 4)), 0); 674| | if (LIKELY(mask == ~(uint64_t)0)) 675| | mlen += 16; 676| | else { 677| | mlen += (uint32_t)(zxc_ctz64(~mask) >> 2); 678| | goto _match_len_done; 679| | } 680| | } 681| | } 682| |#elif defined(ZXC_USE_NEON32) 683| | { 684| | const uint8_t* limit_16 = iend - 16; 685| | while (ip + mlen < limit_16) { 686| | const uint8x16_t v_src = vld1q_u8(ip + mlen); 687| | const uint8x16_t v_ref = vld1q_u8(ref + mlen); 688| | const uint8x16_t v_cmp = vceqq_u8(v_src, v_ref); 689| | uint8x8_t p1 = vpmin_u8(vget_low_u8(v_cmp), vget_high_u8(v_cmp)); 690| | uint8x8_t p2 = vpmin_u8(p1, p1); 691| | uint8x8_t p3 = vpmin_u8(p2, p2); 692| | uint8x8_t p4 = vpmin_u8(p3, p3); 693| | uint8_t min_val = vget_lane_u8(p4, 0); 694| | if (min_val == 0xFF) 695| | mlen += 16; 696| | else { 697| | uint8x16_t v_diff = vmvnq_u8(v_cmp); 698| | uint64_t lo = 699| | (uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 0) | 700| | ((uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 1) << 32); 701| | if (lo != 0) 702| | mlen += (zxc_ctz64(lo) >> 3); 703| | else 704| | mlen += 705| | 8 + (zxc_ctz64( 706| | (uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 2) | 707| | ((uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 3) 708| | << 32)) >> 709| | 3); 710| | goto _match_len_done; 711| | } 712| | } 713| | } 714| |#endif 715| 2.31k| while (ip + mlen < limit_8) { ------------------ | Branch (715:20): [True: 382, False: 1.93k] ------------------ 716| 382| const uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(ref + mlen); 717| 382| if (diff == 0) ------------------ | Branch (717:21): [True: 220, False: 162] ------------------ 718| 220| mlen += sizeof(uint64_t); 719| 162| else { 720| 162| mlen += (zxc_ctz64(diff) >> 3); 721| 162| goto _match_len_done; 722| 162| } 723| 382| } 724| 8.94k| while (ip + mlen < iend && ref[mlen] == ip[mlen]) mlen++; ------------------ | Branch (724:20): [True: 8.11k, False: 833] | Branch (724:40): [True: 7.01k, False: 1.09k] ------------------ 725| | 726| 3.72M| _match_len_done:; 727| 3.72M| const int better = (mlen > best.len); 728| 3.72M| best.len = better ? mlen : best.len; ------------------ | Branch (728:24): [True: 3.40M, False: 324k] ------------------ 729| 3.72M| best.ref = better ? ref : best.ref; ------------------ | Branch (729:24): [True: 3.40M, False: 324k] ------------------ 730| | 731| 3.72M| if (UNLIKELY(best.len >= (uint32_t)p.sufficient_len || ip + best.len >= iend)) break; ------------------ | | 151| 6.37M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.08M, False: 2.64M] | | | Branch (151:42): [True: 1.08M, False: 2.64M] | | | Branch (151:42): [True: 247, False: 2.64M] | | ------------------ ------------------ 732| 3.72M| } 733| | 734| 6.29M| match_idx = (delta != 0) ? next_idx : 0; ------------------ | Branch (734:21): [True: 5.92M, False: 362k] ------------------ 735| 6.29M| } 736| | 737| 8.28M|_finalize_match: 738| 8.28M| if (best.ref) { ------------------ | Branch (738:9): [True: 2.62M, False: 5.66M] ------------------ 739| | // Backtrack to extend match backwards 740| 2.62M| const uint8_t* b_ip = ip; 741| 2.62M| const uint8_t* b_ref = best.ref; 742| 5.03M| while (b_ip > anchor && b_ref > src && b_ip[-1] == b_ref[-1]) { ------------------ | Branch (742:16): [True: 3.49M, False: 1.53M] | Branch (742:33): [True: 3.49M, False: 751] | Branch (742:48): [True: 2.40M, False: 1.08M] ------------------ 743| 2.40M| b_ip--; 744| 2.40M| b_ref--; 745| 2.40M| best.len++; 746| 2.40M| best.backtrack++; 747| 2.40M| } 748| 2.62M| best.ref = b_ref; 749| 2.62M| } 750| | 751| 8.28M| if (p.use_lazy && best.ref && best.len < (uint32_t)p.lazy_len_threshold && ------------------ | Branch (751:9): [True: 0, False: 8.28M] | Branch (751:23): [True: 0, False: 0] | Branch (751:35): [True: 0, False: 0] ------------------ 752| 0| ip + 1 < search_limit) { ------------------ | Branch (752:9): [True: 0, False: 0] ------------------ 753| | // --- Lazy evaluation at ip+1 --- 754| 0| const uint64_t next_val8 = zxc_le64(ip + 1); 755| 0| const uint32_t next_val = (uint32_t)next_val8; 756| 0| const uint32_t h2 = zxc_hash_func(next_val8, use_hash5); 757| 0| const uint8_t next_stored_tag = hash_tags[h2]; 758| 0| const uint32_t next_head = hash_table[h2]; 759| 0| uint32_t next_idx = zxc_epoch_pos(next_head, offset_mask, epoch_mark); 760| 0| const uint8_t next_tag = zxc_hash_tag(next_val); 761| 0| const int skip_lazy_head = (next_idx > 0 && next_stored_tag != next_tag); ------------------ | Branch (761:37): [True: 0, False: 0] | Branch (761:53): [True: 0, False: 0] ------------------ 762| 0| uint32_t max_lazy2 = 0; 763| 0| int lazy_att = p.lazy_attempts; 764| 0| int is_lazy_first = 1; 765| | 766| 0| while (next_idx > 0) { ------------------ | Branch (766:16): [True: 0, False: 0] ------------------ 767| 0| if (UNLIKELY(lazy_att-- <= 0 || (uint32_t)(ip + 1 - src) - next_idx > ZXC_LZ_MAX_DIST)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 768| 0| break; 769| 0| const uint8_t* ref2 = src + next_idx; 770| | 771| | // Filtered here too: a lazy hit only cancels best.ref, so a match 772| | // we could never emit would drop a legal one for nothing. 773| 0| if (((uint32_t)(ip + 1 - src) - next_idx) >= p.min_offset && ------------------ | Branch (773:17): [True: 0, False: 0] ------------------ 774| 0| (!is_lazy_first || !skip_lazy_head) && zxc_le32(ref2) == next_val) { ------------------ | Branch (774:18): [True: 0, False: 0] | Branch (774:36): [True: 0, False: 0] | Branch (774:56): [True: 0, False: 0] ------------------ 775| 0| uint32_t l2 = sizeof(uint32_t); 776| 0| const uint8_t* limit = iend - sizeof(uint64_t); 777| | 778| 0| while (ip + 1 + l2 < limit) { ------------------ | Branch (778:24): [True: 0, False: 0] ------------------ 779| 0| const uint64_t v1 = zxc_le64(ip + 1 + l2); 780| 0| const uint64_t v2 = zxc_le64(ref2 + l2); 781| 0| if (v1 != v2) { ------------------ | Branch (781:25): [True: 0, False: 0] ------------------ 782| 0| l2 += (uint32_t)(zxc_ctz64(v1 ^ v2) >> 3); 783| 0| goto lazy2_done; 784| 0| } 785| 0| l2 += sizeof(uint64_t); 786| 0| } 787| 0| while (ip + 1 + l2 < iend && ref2[l2] == ip[1 + l2]) l2++; ------------------ | Branch (787:24): [True: 0, False: 0] | Branch (787:46): [True: 0, False: 0] ------------------ 788| 0| lazy2_done: 789| 0| max_lazy2 = l2 > max_lazy2 ? l2 : max_lazy2; ------------------ | Branch (789:29): [True: 0, False: 0] ------------------ 790| 0| } 791| | 792| 0| const uint16_t delta = chain_table[next_idx & ZXC_LZ_WINDOW_MASK]; ------------------ | | 533| 0|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 531| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 793| 0| if (UNLIKELY(delta == 0)) break; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 794| 0| next_idx -= delta; 795| 0| is_lazy_first = 0; 796| 0| } 797| | 798| | // --- Lazy evaluation at ip+2 (computed in parallel, no dependency on lazy 1) --- 799| 0| uint32_t max_lazy3 = 0; 800| 0| if (level >= ZXC_LEVEL_BALANCED && ip + 2 < search_limit) { ------------------ | Branch (800:13): [True: 0, False: 0] | Branch (800:44): [True: 0, False: 0] ------------------ 801| 0| const uint64_t val3_8 = zxc_le64(ip + 2); 802| 0| const uint32_t val3 = (uint32_t)val3_8; 803| 0| const uint32_t h3 = zxc_hash_func(val3_8, use_hash5); 804| 0| const uint8_t tag3 = hash_tags[h3]; 805| 0| const uint32_t head3 = hash_table[h3]; 806| 0| uint32_t idx3 = zxc_epoch_pos(head3, offset_mask, epoch_mark); 807| 0| const uint8_t cur_tag3 = zxc_hash_tag(val3); 808| 0| const int skip_head3 = (idx3 > 0 && tag3 != cur_tag3); ------------------ | Branch (808:37): [True: 0, False: 0] | Branch (808:49): [True: 0, False: 0] ------------------ 809| | 810| 0| int is_first3 = 1; 811| 0| lazy_att = p.lazy_attempts; 812| 0| while (idx3 > 0) { ------------------ | Branch (812:20): [True: 0, False: 0] ------------------ 813| 0| if (UNLIKELY(lazy_att-- <= 0 || (uint32_t)(ip + 2 - src) - idx3 > ZXC_LZ_MAX_DIST)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 814| 0| break; 815| | 816| 0| const uint8_t* ref3 = src + idx3; 817| 0| if (((uint32_t)(ip + 2 - src) - idx3) >= p.min_offset && ------------------ | Branch (817:21): [True: 0, False: 0] ------------------ 818| 0| (!is_first3 || !skip_head3) && zxc_le32(ref3) == val3) { ------------------ | Branch (818:22): [True: 0, False: 0] | Branch (818:36): [True: 0, False: 0] | Branch (818:52): [True: 0, False: 0] ------------------ 819| 0| uint32_t l3 = sizeof(uint32_t); 820| 0| const uint8_t* limit = iend - sizeof(uint64_t); 821| | 822| 0| while (ip + 2 + l3 < limit) { ------------------ | Branch (822:28): [True: 0, False: 0] ------------------ 823| 0| const uint64_t v1 = zxc_le64(ip + 2 + l3); 824| 0| const uint64_t v2 = zxc_le64(ref3 + l3); 825| 0| if (v1 != v2) { ------------------ | Branch (825:29): [True: 0, False: 0] ------------------ 826| 0| l3 += (uint32_t)(zxc_ctz64(v1 ^ v2) >> 3); 827| 0| goto lazy3_done; 828| 0| } 829| 0| l3 += sizeof(uint64_t); 830| 0| } 831| 0| while (ip + 2 + l3 < iend && ref3[l3] == ip[2 + l3]) l3++; ------------------ | Branch (831:28): [True: 0, False: 0] | Branch (831:50): [True: 0, False: 0] ------------------ 832| 0| lazy3_done: 833| 0| max_lazy3 = l3 > max_lazy3 ? l3 : max_lazy3; ------------------ | Branch (833:33): [True: 0, False: 0] ------------------ 834| 0| } 835| | 836| 0| const uint16_t delta = chain_table[idx3 & ZXC_LZ_WINDOW_MASK]; ------------------ | | 533| 0|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 531| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 837| 0| if (UNLIKELY(delta == 0)) break; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 838| 0| idx3 -= delta; 839| 0| is_first3 = 0; 840| 0| } 841| 0| } 842| | 843| | // Single decision: invalidate if either lazy position found a better match 844| 0| if (max_lazy2 > best.len + 1 || max_lazy3 > best.len + 2) best.ref = NULL; ------------------ | Branch (844:13): [True: 0, False: 0] | Branch (844:41): [True: 0, False: 0] ------------------ 845| 0| } 846| | 847| 8.28M| return best; 848| 8.28M|} zxc_compress.c:zxc_flush_literals: 433| 1.08M| const uint8_t* iend) { 434| 1.08M| if (LIKELY(anchor + ZXC_PAD_SIZE <= iend)) { ------------------ | | 145| 1.08M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 1.08M, False: 970] | | ------------------ ------------------ 435| 1.08M| zxc_copy32(literals + *lit_c, anchor); 436| 1.08M| if (UNLIKELY(ll > ZXC_PAD_SIZE)) ------------------ | | 151| 1.08M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 113k, False: 971k] | | ------------------ ------------------ 437| 113k| ZXC_MEMCPY(literals + *lit_c + ZXC_PAD_SIZE, anchor + ZXC_PAD_SIZE, ll - ZXC_PAD_SIZE); ------------------ | | 167| 113k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 438| 1.08M| } else { 439| 970| ZXC_MEMCPY(literals + *lit_c, anchor, ll); ------------------ | | 167| 970|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 440| 970| } 441| 1.08M| *lit_c += ll; 442| 1.08M|} zxc_compress.c:zxc_emit_extra: 233| 5.24M| const uint32_t val, const uint32_t mask) { 234| 5.24M| if (LIKELY(val < mask)) return 1; ------------------ | | 145| 5.24M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 5.14M, False: 104k] | | ------------------ ------------------ 235| 104k| const size_t n = zxc_write_varint(extras + *used, val - mask); 236| 104k| if (UNLIKELY(n == 0)) return 0; ------------------ | | 151| 104k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 104k] | | ------------------ ------------------ 237| 104k| *used += n; 238| 104k| return 1; 239| 104k|} zxc_compress.c:zxc_write_varint: 197| 104k|static ZXC_ALWAYS_INLINE size_t zxc_write_varint(uint8_t* RESTRICT dst, const uint32_t val) { 198| | // Values above ZXC_MAX_VARINT_VALUE are out-of-spec and the decoder would 199| | // reject them. Unreachable from the Block API, so this is defence in depth; 200| | // callers must treat a return of 0 as an encoding error. 201| 104k| if (UNLIKELY(val > ZXC_MAX_VARINT_VALUE)) { ------------------ | | 151| 104k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 104k] | | ------------------ ------------------ 202| 0| return 0; 203| 0| } 204| | 205| | // 1 byte: 0xxxxxxx (7 bits) = 2^7 = 128 206| 104k| if (LIKELY(val < (1U << 7))) { ------------------ | | 145| 104k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 30.1k, False: 74.1k] | | ------------------ ------------------ 207| 30.1k| dst[0] = (uint8_t)val; 208| 30.1k| return 1; 209| 30.1k| } 210| | 211| | // 2 bytes: 10xxxxxx xxxxxxxx (14 bits) = 2^14 = 16384 212| 74.1k| if (LIKELY(val < (1U << 14))) { ------------------ | | 145| 74.1k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 71.4k, False: 2.71k] | | ------------------ ------------------ 213| 71.4k| dst[0] = (uint8_t)(0x80 | (val & 0x3F)); 214| 71.4k| dst[1] = (uint8_t)(val >> 6); 215| 71.4k| return 2; 216| 71.4k| } 217| | 218| | // 3 bytes: 110xxxxx xxxxxxxx xxxxxxxx (21 bits) -> max emittable value, 219| | // matching ZXC_MAX_VARINT_VALUE = ZXC_BLOCK_SIZE_MAX - 1. 220| 2.71k| dst[0] = (uint8_t)(0xC0 | (val & 0x1F)); 221| 2.71k| dst[1] = (uint8_t)(val >> 5); 222| 2.71k| dst[2] = (uint8_t)(val >> 13); 223| 2.71k| return 3; 224| 74.1k|} zxc_compress.c:zxc_encode_block_raw: 2166| 802| size_t* RESTRICT const out_sz) { 2167| 802| if (UNLIKELY(dst_cap < ZXC_BLOCK_HEADER_SIZE + src_sz)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 802|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 802] | | ------------------ ------------------ 2168| | 2169| 802| zxc_block_header_t bh; 2170| 802| bh.block_type = ZXC_BLOCK_RAW; 2171| 802| bh.block_flags = 0; // Checksum flag moved to file header 2172| 802| bh.reserved = 0; 2173| 802| bh.comp_size = (uint32_t)src_sz; 2174| | 2175| 802| const int hw = zxc_write_block_header(dst, dst_cap, &bh); 2176| 802| if (UNLIKELY(hw < 0)) return hw; ------------------ | | 151| 802|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 802] | | ------------------ ------------------ 2177| | 2178| 802| ZXC_MEMCPY(dst + ZXC_BLOCK_HEADER_SIZE, src, src_sz); ------------------ | | 167| 802|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 2179| | 2180| | // Checksum will be appended by the wrapper 2181| 802| *out_sz = ZXC_BLOCK_HEADER_SIZE + src_sz; ------------------ | | 407| 802|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2182| 802| return ZXC_OK; 2183| 802|} zxc_decompress_chunk_wrapper_default: 1657| 4.90k| const uint64_t block_index) { 1658| 4.90k| return zxc_decompress_chunk_wrapper_body(ctx, src, src_sz, dst, dst_cap, block_index, 0, 0); 1659| 4.90k|} zxc_decompress.c:zxc_decompress_chunk_wrapper_body: 1595| 4.90k| const int safe) { 1596| 4.90k| if (UNLIKELY(src_sz < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1597| | 1598| 4.90k| const uint8_t type = src[0]; 1599| 4.90k| const uint32_t comp_sz = zxc_le32(src + 3); 1600| 4.90k| const int has_checksum = ctx->checksum_enabled; 1601| | 1602| | // A block never compresses past its own content size 1603| 4.90k| if (UNLIKELY((uint64_t)comp_sz > (uint64_t)ctx->chunk_size)) return ZXC_ERROR_BAD_BLOCK_SIZE; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1604| | 1605| | // Check bounds: Header + Body + Checksum(if any) 1606| 4.90k| const size_t expected_sz = 1607| 4.90k| (size_t)ZXC_BLOCK_HEADER_SIZE + comp_sz + (has_checksum ? ZXC_BLOCK_CHECKSUM_SIZE : 0); ------------------ | | 407| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ (size_t)ZXC_BLOCK_HEADER_SIZE + comp_sz + (has_checksum ? ZXC_BLOCK_CHECKSUM_SIZE : 0); ------------------ | | 409| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (1607:52): [True: 1.63k, False: 3.27k] ------------------ 1608| 4.90k| if (UNLIKELY(src_sz < expected_sz)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1609| | 1610| 4.90k| const uint8_t* data = src + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 407| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1611| | 1612| 4.90k| int decoded_sz = ZXC_ERROR_BAD_BLOCK_TYPE; 1613| | 1614| 4.90k| switch (type) { 1615| 0| case ZXC_BLOCK_GLO: ------------------ | Branch (1615:9): [True: 0, False: 4.90k] ------------------ 1616| 0| decoded_sz = safe ? zxc_decode_block_glo_safe(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1616:26): [True: 0, False: 0] ------------------ 1617| 0| : has_dict ? zxc_decode_block_glo_dict(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1617:28): [True: 0, False: 0] ------------------ 1618| 0| : zxc_decode_block_glo(ctx, data, comp_sz, dst, dst_cap); 1619| 0| break; 1620| 4.10k| case ZXC_BLOCK_GHI: ------------------ | Branch (1620:9): [True: 4.10k, False: 802] ------------------ 1621| 4.10k| decoded_sz = safe ? zxc_decode_block_ghi_safe(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1621:26): [True: 0, False: 4.10k] ------------------ 1622| 4.10k| : has_dict ? zxc_decode_block_ghi_dict(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1622:28): [True: 0, False: 4.10k] ------------------ 1623| 4.10k| : zxc_decode_block_ghi(ctx, data, comp_sz, dst, dst_cap); 1624| 4.10k| break; 1625| 802| case ZXC_BLOCK_RAW: ------------------ | Branch (1625:9): [True: 802, False: 4.10k] ------------------ 1626| | // For RAW blocks, comp_sz == raw_sz (uncompressed data stored as-is) 1627| 802| if (UNLIKELY(comp_sz > dst_cap)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 802|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 802] | | ------------------ ------------------ 1628| 802| ZXC_MEMCPY(dst, data, comp_sz); ------------------ | | 167| 802|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1629| 802| decoded_sz = (int)comp_sz; 1630| 802| break; 1631| 0| case ZXC_BLOCK_EOF: ------------------ | Branch (1631:9): [True: 0, False: 4.90k] ------------------ 1632| | // EOF should be handled by the dispatcher, not here 1633| 0| return ZXC_ERROR_CORRUPT_DATA; 1634| 0| default: ------------------ | Branch (1634:9): [True: 0, False: 4.90k] ------------------ 1635| 0| return ZXC_ERROR_BAD_BLOCK_TYPE; 1636| 4.90k| } 1637| | 1638| 4.90k| if (has_checksum && LIKELY(decoded_sz >= 0)) { ------------------ | | 145| 1.63k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 1.63k, False: 0] | | ------------------ ------------------ | Branch (1638:9): [True: 1.63k, False: 3.27k] ------------------ 1639| 1.63k| const uint32_t stored = zxc_le32(data + comp_sz); 1640| 1.63k| if (UNLIKELY(stored != ------------------ | | 151| 1.63k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.63k] | | ------------------ ------------------ 1641| 1.63k| zxc_checksum(dst, (size_t)decoded_sz, block_index, ZXC_CHECKSUM_RAPIDHASH))) 1642| 0| return ZXC_ERROR_BAD_CHECKSUM; 1643| 1.63k| } 1644| | 1645| 4.90k| return decoded_sz; 1646| 4.90k|} zxc_decompress.c:zxc_decode_copy_literals: 405| 2.45M| const uint64_t ll) { 406| 2.45M| zxc_wild_copy32(dst, src, ll); 407| 2.45M|} zxc_decompress.c:zxc_wild_copy32: 372| 4.60M|static ZXC_ALWAYS_INLINE void zxc_wild_copy32(uint8_t* dst, const uint8_t* src, const size_t len) { 373| 4.60M| zxc_copy32(dst, src); 374| 4.60M| if (UNLIKELY(len > 32)) { ------------------ | | 151| 4.60M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 607k, False: 3.99M] | | ------------------ ------------------ 375| 607k| size_t rem = len - 32; 376| 607k| dst += 32; 377| 607k| src += 32; 378| 8.76M| while (rem > 32) { ------------------ | Branch (378:16): [True: 8.15M, False: 607k] ------------------ 379| 8.15M| zxc_copy32(dst, src); 380| 8.15M| dst += 32; 381| 8.15M| src += 32; 382| 8.15M| rem -= 32; 383| 8.15M| } 384| 607k| zxc_copy32(dst, src); 385| 607k| } 386| 4.60M|} zxc_decompress.c:zxc_decode_fill_run: 310| 466| const uint64_t ml) { 311| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 312| | const __m256i v = _mm256_set1_epi8((char)byte); 313| | _mm256_storeu_si256((__m256i*)dst, v); 314| | if (UNLIKELY(ml > 32)) { 315| | uint8_t* p = dst; 316| | const uint8_t* const last = dst + ml - 32; 317| | do { 318| | p += 32; 319| | _mm256_storeu_si256((__m256i*)p, v); 320| | } while (p < last); 321| | } 322| |#elif defined(ZXC_USE_SSE2) 323| | const __m128i v = _mm_set1_epi8((char)byte); 324| 466| _mm_storeu_si128((__m128i*)dst, v); 325| 466| _mm_storeu_si128((__m128i*)(dst + 16), v); 326| 466| if (UNLIKELY(ml > 32)) { ------------------ | | 151| 466|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 278, False: 188] | | ------------------ ------------------ 327| 278| uint8_t* p = dst; 328| 278| const uint8_t* const last = dst + ml - 32; 329| 13.1k| do { 330| 13.1k| p += 32; 331| 13.1k| _mm_storeu_si128((__m128i*)p, v); 332| 13.1k| _mm_storeu_si128((__m128i*)(p + 16), v); 333| 13.1k| } while (p < last); ------------------ | Branch (333:18): [True: 12.8k, False: 278] ------------------ 334| 278| } 335| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 336| | const uint8x16_t v = vdupq_n_u8(byte); 337| | vst1q_u8(dst, v); 338| | vst1q_u8(dst + 16, v); 339| | if (UNLIKELY(ml > 32)) { 340| | uint8_t* p = dst; 341| | const uint8_t* const last = dst + ml - 32; 342| | do { 343| | p += 32; 344| | vst1q_u8(p, v); 345| | vst1q_u8(p + 16, v); 346| | } while (p < last); 347| | } 348| |#else 349| | ZXC_MEMSET(dst, byte, ml); 350| |#endif 351| 466|} zxc_decompress.c:zxc_decode_copy_match: 433| 2.45M| const uint64_t ml) { 434| 2.45M| const uint8_t* match_src = d_ptr - off; 435| 2.45M| if (LIKELY(off >= 32)) { ------------------ | | 145| 2.45M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 2.15M, False: 301k] | | ------------------ ------------------ 436| 2.15M| zxc_wild_copy32(d_ptr, match_src, ml); 437| 2.15M| } else if (off == 1) { ------------------ | Branch (437:16): [True: 466, False: 300k] ------------------ 438| 466| zxc_decode_fill_run(d_ptr, match_src[0], ml); 439| 300k| } else { 440| 300k| zxc_decode_copy_overlap_run32(d_ptr, off, ml); 441| 300k| } 442| 2.45M|} zxc_decompress.c:zxc_decode_copy_overlap_run32: 144| 300k| const uint64_t ml) { 145| 300k| const size_t stride = zxc_overlap_strides32[off]; 146| 300k| size_t copied = 0; 147| |#if defined(ZXC_USE_NEON64) 148| | uint8x16x2_t tbl; 149| | tbl.val[0] = vld1q_u8(dst - off); 150| | tbl.val[1] = vld1q_u8(dst - off + 16); 151| | const uint8x16_t pat_lo = vqtbl2q_u8(tbl, vld1q_u8(zxc_overlap_masks32[off])); 152| | const uint8x16_t pat_hi = vqtbl2q_u8(tbl, vld1q_u8(zxc_overlap_masks32[off] + 16)); 153| | do { 154| | vst1q_u8(dst + copied, pat_lo); 155| | vst1q_u8(dst + copied + 16, pat_hi); 156| | copied += stride; 157| | } while (copied < ml); 158| | 159| |#elif defined(ZXC_USE_NEON32) 160| | // VTBL reaches 8 bytes per lookup from a 4-register (32-byte) table, so the 161| | // pattern takes four lookups instead of NEON64's two. 162| | uint8x8x4_t tbl; 163| | tbl.val[0] = vld1_u8(dst - off); 164| | tbl.val[1] = vld1_u8(dst - off + 8); 165| | tbl.val[2] = vld1_u8(dst - off + 16); 166| | tbl.val[3] = vld1_u8(dst - off + 24); 167| | const uint8_t* const m = zxc_overlap_masks32[off]; 168| | const uint8x8_t p0 = vtbl4_u8(tbl, vld1_u8(m)); 169| | const uint8x8_t p1 = vtbl4_u8(tbl, vld1_u8(m + 8)); 170| | const uint8x8_t p2 = vtbl4_u8(tbl, vld1_u8(m + 16)); 171| | const uint8x8_t p3 = vtbl4_u8(tbl, vld1_u8(m + 24)); 172| | do { 173| | vst1_u8(dst + copied, p0); 174| | vst1_u8(dst + copied + 8, p1); 175| | vst1_u8(dst + copied + 16, p2); 176| | vst1_u8(dst + copied + 24, p3); 177| | copied += stride; 178| | } while (copied < ml); 179| | 180| |#elif defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 181| | // pshufb only reaches 16 bytes per lane, so the 32-byte table is emulated: 182| | // shuffle both halves, then pick per byte on bit 4 of the index. 183| | const __m128i t0 = _mm_loadu_si128((const __m128i*)(dst - off)); 184| | const __m128i t1 = _mm_loadu_si128((const __m128i*)(dst - off + 16)); 185| | const __m128i sixteen = _mm_set1_epi8(16); 186| | const __m128i m_lo = _mm_load_si128((const __m128i*)zxc_overlap_masks32[off]); 187| | const __m128i m_hi = _mm_load_si128((const __m128i*)(zxc_overlap_masks32[off] + 16)); 188| | const __m128i pat_lo = 189| | _mm_blendv_epi8(_mm_shuffle_epi8(t0, m_lo), 190| | _mm_shuffle_epi8(t1, _mm_sub_epi8(m_lo, sixteen)), _mm_slli_epi16(m_lo, 3)); 191| | const __m128i pat_hi = 192| | _mm_blendv_epi8(_mm_shuffle_epi8(t0, m_hi), 193| | _mm_shuffle_epi8(t1, _mm_sub_epi8(m_hi, sixteen)), _mm_slli_epi16(m_hi, 3)); 194| | do { 195| | _mm_storeu_si128((__m128i*)(dst + copied), pat_lo); 196| | _mm_storeu_si128((__m128i*)(dst + copied + 16), pat_hi); 197| | copied += stride; 198| | } while (copied < ml); 199| | 200| |#else 201| | // SSE2 (no PSHUFB) and non-SIMD tiers: build the 32-byte pattern with a 202| | // wrap counter (no per-byte modulo), then store it as two 16-byte halves. 203| 300k| const uint8_t* src = dst - off; 204| 300k| uint8_t pat[32]; 205| 300k| uint32_t k = 0; 206| 9.92M| for (size_t i = 0; i < 32; i++) { ------------------ | Branch (206:24): [True: 9.62M, False: 300k] ------------------ 207| 9.62M| pat[i] = src[k]; 208| 9.62M| if (++k == off) k = 0; ------------------ | Branch (208:13): [True: 657k, False: 8.96M] ------------------ 209| 9.62M| } 210| 1.12M| do { 211| 1.12M| zxc_copy16(dst + copied, pat); 212| 1.12M| zxc_copy16(dst + copied + 16, pat + 16); 213| 1.12M| copied += stride; 214| 1.12M| } while (copied < ml); ------------------ | Branch (214:14): [True: 824k, False: 300k] ------------------ 215| 300k|#endif 216| 300k|} zxc_decompress.c:zxc_decode_copy_match_exact: 504| 172k| const size_t off, const size_t ml) { 505| 172k| if (off >= ml) { ------------------ | Branch (505:9): [True: 163k, False: 9.05k] ------------------ 506| 163k| ZXC_MEMCPY(d_ptr, match_src, ml); ------------------ | | 167| 163k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 507| 163k| } else if (ml < 16) { ------------------ | Branch (507:16): [True: 1.28k, False: 7.76k] ------------------ 508| 15.0k| for (size_t i = 0; i < ml; i++) d_ptr[i] = match_src[i]; ------------------ | Branch (508:28): [True: 13.8k, False: 1.28k] ------------------ 509| 7.76k| } else { 510| 7.76k| size_t n = off; 511| 7.76k| ZXC_MEMCPY(d_ptr, match_src, n); ------------------ | | 167| 7.76k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 512| 21.2k| while (n <= ml - n) { ------------------ | Branch (512:16): [True: 13.4k, False: 7.76k] ------------------ 513| 13.4k| ZXC_MEMCPY(d_ptr + n, d_ptr, n); ------------------ | | 167| 13.4k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 514| 13.4k| n <<= 1; 515| 13.4k| } 516| 7.76k| ZXC_MEMCPY(d_ptr + n, d_ptr, ml - n); ------------------ | | 167| 7.76k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 517| 7.76k| } 518| 172k|} zxc_decompress.c:zxc_decode_block_ghi_impl: 1178| 4.10k| const int has_dict) { 1179| 4.10k| zxc_gnr_header_t gh; 1180| | 1181| | // 0 when !has_dict (safe path) -> folds `d_floor` to `dst`. 1182| 4.10k| const size_t dict_size = has_dict ? ctx->dict_size : 0; ------------------ | Branch (1182:30): [True: 0, False: 4.10k] ------------------ 1183| | 1184| 4.10k| if (UNLIKELY(zxc_read_ghi_header(src, src_size, &gh) != ZXC_OK)) return ZXC_ERROR_BAD_HEADER; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 1185| 4.10k| if (UNLIKELY(gh.enc_lit != ZXC_SECTION_ENCODING_RAW || gh.enc_tok != 0)) ------------------ | | 151| 8.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | | Branch (151:42): [True: 0, False: 4.10k] | | | Branch (151:42): [True: 0, False: 4.10k] | | ------------------ ------------------ 1186| 0| return ZXC_ERROR_CORRUPT_DATA; 1187| | 1188| 4.10k| const uint8_t* const p_data = src + ZXC_GHI_HEADER_BINARY_SIZE; ------------------ | | 413| 4.10k|#define ZXC_GHI_HEADER_BINARY_SIZE 12 ------------------ 1189| 4.10k| const uint8_t* p_curr = p_data; 1190| | 1191| | // --- Stream Pointers & Validation --- 1192| | // GHI carries no section descriptors: literals are always RAW, the sequence 1193| | // stream is four bytes per sequence, and extras run to the payload end. 1194| 4.10k| const size_t sz_lit = gh.n_literals; 1195| 4.10k| const uint64_t sz_seqs = (uint64_t)gh.n_sequences * sizeof(uint32_t); 1196| | 1197| 4.10k| const size_t payload_avail = (size_t)(src + src_size - p_data); 1198| 4.10k| const uint64_t consumed = (uint64_t)sz_lit + sz_seqs; 1199| 4.10k| if (UNLIKELY(consumed > (uint64_t)payload_avail)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 1200| 4.10k| const size_t sz_exts = payload_avail - (size_t)consumed; /* slack padding included */ 1201| | 1202| | // GHI literals are always RAW, so the slack is always load-bearing here. 1203| | // No underflow: sz_lit <= consumed <= payload_avail. 1204| 4.10k| if (UNLIKELY(payload_avail - sz_lit < ZXC_BLOCK_LIT_SLACK)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 1205| | 1206| 4.10k| const uint8_t* l_ptr = p_curr; 1207| 4.10k| const uint8_t* l_end = l_ptr + sz_lit; 1208| 4.10k| p_curr += sz_lit; 1209| | 1210| 4.10k| const uint8_t* seq_ptr = p_curr; 1211| 4.10k| const uint8_t* extras_ptr = p_curr + (size_t)sz_seqs; 1212| 4.10k| const uint8_t* const extras_end = extras_ptr + sz_exts; 1213| | 1214| 4.10k| uint8_t* d_ptr = dst; 1215| 4.10k| const uint8_t* const d_end = dst + dst_capacity; 1216| | // Lowest address a match may reach: the dictionary prefix if any, else dst. 1217| 4.10k| const uint8_t* const d_floor = dst - dict_size; 1218| 4.10k| const uint8_t* const d_end_safe = 1219| 4.10k| (dst_capacity > ZXC_PAD_SIZE * 4) ? d_end - (ZXC_PAD_SIZE * 4) : dst; // 128 ------------------ | | 328| 4.10k|#define ZXC_PAD_SIZE 32 ------------------ (dst_capacity > ZXC_PAD_SIZE * 4) ? d_end - (ZXC_PAD_SIZE * 4) : dst; // 128 ------------------ | | 328| 4.10k|#define ZXC_PAD_SIZE 32 ------------------ | Branch (1219:9): [True: 4.10k, False: 0] ------------------ 1220| | // Safety margin for 4x unrolled loop: 4 * (ZXC_SEQ_LL_MASK LL + 1221| | // ZXC_SEQ_ML_MASK+ZXC_LZ_MIN_MATCH_LEN ML) + ZXC_PAD_SIZE Pad = 4 x (255 + 255 + 5) + 32 = 2092 1222| 4.10k| const uint8_t* const d_end_fast = 1223| 4.10k| (dst_capacity > ZXC_DECOMPRESS_TAIL_PAD) ? d_end - ZXC_DECOMPRESS_TAIL_PAD : dst; // 2112 ------------------ | | 354| 4.10k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 328| 4.10k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ (dst_capacity > ZXC_DECOMPRESS_TAIL_PAD) ? d_end - ZXC_DECOMPRESS_TAIL_PAD : dst; // 2112 ------------------ | | 354| 4.10k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 328| 4.10k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ | Branch (1223:9): [True: 4.10k, False: 0] ------------------ 1224| | 1225| | // Literal margin for the GHI loops: without a varint, ll <= 254 per sequence, 1226| | // so 4 * 254 = 1016. Past that only the cold varint path checks l_ptr. 1227| 4.10k| const size_t ghi_margin_4x = 4 * (ZXC_SEQ_LL_MASK - 1); // 1016 ------------------ | | 500| 4.10k|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 494| 4.10k|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ 1228| 4.10k| const size_t ghi_margin_1x = ZXC_SEQ_LL_MASK - 1; // 254 ------------------ | | 500| 4.10k|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 494| 4.10k|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ 1229| 4.10k| const uint8_t* const l_end_safe_4x = (sz_lit > ghi_margin_4x) ? l_end - ghi_margin_4x : l_ptr; ------------------ | Branch (1229:42): [True: 2.93k, False: 1.16k] ------------------ 1230| 4.10k| const uint8_t* const l_end_safe_1x = (sz_lit > ghi_margin_1x) ? l_end - ghi_margin_1x : l_ptr; ------------------ | Branch (1230:42): [True: 3.36k, False: 740] ------------------ 1231| | 1232| 4.10k| uint32_t n_seq = gh.n_sequences; 1233| | 1234| | // Same handover as GLO, but the GHI offset is inline in the sequence word and 1235| | // always 16 bits -- enc_off is not a width here, so the span is fixed. 1236| 4.10k| const size_t off_span = 1U << 16; 1237| 4.10k| const size_t span = (dict_size >= off_span) ? 0 : off_span - dict_size; ------------------ | Branch (1237:25): [True: 0, False: 4.10k] ------------------ 1238| 4.10k| const uint8_t* const d_bounds = (span >= dst_capacity) ? d_end : dst + span; ------------------ | Branch (1238:37): [True: 0, False: 4.10k] ------------------ 1239| | 1240| | // --- SAFE loop: validate offsets until d_bounds (4x unroll) --- 1241| 4.10k| if (safe) { ------------------ | Branch (1241:9): [True: 0, False: 4.10k] ------------------ 1242| | // SAFE variant: save per-batch state so an OVERFLOW can rollback and 1243| | // hand over to the 1x loop / Safe Path. Wild writes already committed 1244| | // are deterministically overwritten when the 1x loop replays. 1245| 0| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (1245:16): [True: 0, False: 0] | Branch (1245:30): [True: 0, False: 0] | Branch (1245:52): [True: 0, False: 0] | Branch (1245:77): [True: 0, False: 0] ------------------ 1246| 0| const uint8_t* const t_save = seq_ptr; 1247| 0| const uint8_t* const e_save = extras_ptr; 1248| 0| uint8_t* const d_save = d_ptr; 1249| 0| const uint8_t* const l_save = l_ptr; 1250| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, goto rollback_safe_4x, (void)0); ------------------ | | 678| 0| do { \ | | 679| 0| uint32_t s1 = zxc_le32(seq_ptr); \ | | 680| 0| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 681| 0| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 682| 0| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 683| 0| seq_ptr += 4 * sizeof(uint32_t); \ | | 684| 0| PREFETCH; \ | | 685| 0| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 0| do { \ | | | | 646| 0| uint64_t ll = (S) >> 24; \ | | | | 647| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 650| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 0| const uint64_t reserve = (RESERVE); \ | | | | 652| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 654| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 0| (size_t)(d_end - d_ptr))) \ | | | | 656| 0| ON_FAIL; \ | | | | 657| 0| } \ | | | | 658| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 659| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 661| 0| (size_t)(d_end - d_ptr))) \ | | | | 662| 0| ON_FAIL; \ | | | | 663| 0| } \ | | | | 664| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 0| l_ptr += ll; \ | | | | 668| 0| d_ptr += ll; \ | | | | 669| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1250| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 528| 0| do { \ | | | | | | | | 529| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 530| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 531| 0| d_ptr += ml; \ | | | | | | | | 532| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (532:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 0| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 686| 0| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 0| do { \ | | | | 646| 0| uint64_t ll = (S) >> 24; \ | | | | 647| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 650| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 0| const uint64_t reserve = (RESERVE); \ | | | | 652| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 654| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 0| (size_t)(d_end - d_ptr))) \ | | | | 656| 0| ON_FAIL; \ | | | | 657| 0| } \ | | | | 658| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 659| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 661| 0| (size_t)(d_end - d_ptr))) \ | | | | 662| 0| ON_FAIL; \ | | | | 663| 0| } \ | | | | 664| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 0| l_ptr += ll; \ | | | | 668| 0| d_ptr += ll; \ | | | | 669| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1250| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 528| 0| do { \ | | | | | | | | 529| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 530| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 531| 0| d_ptr += ml; \ | | | | | | | | 532| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (532:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 0| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 687| 0| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 0| do { \ | | | | 646| 0| uint64_t ll = (S) >> 24; \ | | | | 647| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 650| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 0| const uint64_t reserve = (RESERVE); \ | | | | 652| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 654| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 0| (size_t)(d_end - d_ptr))) \ | | | | 656| 0| ON_FAIL; \ | | | | 657| 0| } \ | | | | 658| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 659| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 661| 0| (size_t)(d_end - d_ptr))) \ | | | | 662| 0| ON_FAIL; \ | | | | 663| 0| } \ | | | | 664| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 0| l_ptr += ll; \ | | | | 668| 0| d_ptr += ll; \ | | | | 669| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1250| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 528| 0| do { \ | | | | | | | | 529| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 530| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 531| 0| d_ptr += ml; \ | | | | | | | | 532| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (532:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 0| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 688| 0| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 0| do { \ | | | | 646| 0| uint64_t ll = (S) >> 24; \ | | | | 647| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 650| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 0| const uint64_t reserve = (RESERVE); \ | | | | 652| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 654| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 0| (size_t)(d_end - d_ptr))) \ | | | | 656| 0| ON_FAIL; \ | | | | 657| 0| } \ | | | | 658| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 659| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 661| 0| (size_t)(d_end - d_ptr))) \ | | | | 662| 0| ON_FAIL; \ | | | | 663| 0| } \ | | | | 664| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 0| l_ptr += ll; \ | | | | 668| 0| d_ptr += ll; \ | | | | 669| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1250| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 528| 0| do { \ | | | | | | | | 529| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 530| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 531| 0| d_ptr += ml; \ | | | | | | | | 532| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (532:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 0| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 689| 0| n_seq -= 4; \ | | 690| 0| } while (0) | | ------------------ | | | Branch (690:14): [Folded, False: 0] | | ------------------ ------------------ 1251| 0| continue; 1252| | 1253| 0| rollback_safe_4x: 1254| 0| seq_ptr = t_save; 1255| 0| extras_ptr = e_save; 1256| 0| d_ptr = d_save; 1257| 0| l_ptr = l_save; 1258| 0| break; 1259| 0| } 1260| 4.10k| } else { 1261| 160k| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (1261:16): [True: 158k, False: 1.66k] | Branch (1261:30): [True: 158k, False: 0] | Branch (1261:52): [True: 157k, False: 1.58k] | Branch (1261:77): [True: 156k, False: 862] ------------------ 1262| 156k| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); ------------------ | | 678| 156k| do { \ | | 679| 156k| uint32_t s1 = zxc_le32(seq_ptr); \ | | 680| 156k| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 681| 156k| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 682| 156k| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 683| 156k| seq_ptr += 4 * sizeof(uint32_t); \ | | 684| 156k| PREFETCH; \ | | 685| 156k| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 156k| do { \ | | | | 646| 156k| uint64_t ll = (S) >> 24; \ | | | | 647| 156k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 156k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 156k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 156k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.93k, False: 154k] | | | | | | ------------------ | | | | ------------------ | | | | 650| 1.93k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 1.93k| const uint64_t reserve = (RESERVE); \ | | | | 652| 1.93k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 1.93k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.87k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.93k] | | | | | | | Branch (151:42): [True: 0, False: 1.93k] | | | | | | | Branch (151:42): [True: 0, False: 1.93k] | | | | | | ------------------ | | | | ------------------ | | | | 654| 1.93k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 1.93k| (size_t)(d_end - d_ptr))) \ | | | | 656| 1.93k| ON_FAIL; \ | | | | 657| 1.93k| } \ | | | | 658| 156k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 6.76k, False: 149k] | | | | | | ------------------ | | | | ------------------ | | | | 659| 6.76k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 6.76k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 6.76k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 6.76k] | | | | | | ------------------ | | | | ------------------ | | | | 661| 6.76k| (size_t)(d_end - d_ptr))) \ | | | | 662| 6.76k| ON_FAIL; \ | | | | 663| 6.76k| } \ | | | | 664| 156k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 156k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 156k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 156k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 156k| l_ptr += ll; \ | | | | 668| 156k| d_ptr += ll; \ | | | | 669| 156k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1262| 156k| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 528| 156k| do { \ | | | | | | | | 529| 156k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 156k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 530| 156k| COPY(d_ptr, off, ml); \ | | | | | | | | 531| 156k| d_ptr += ml; \ | | | | | | | | 532| 156k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (532:14): [Folded, False: 156k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 156k| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 156k] | | | | ------------------ | | ------------------ | | 686| 156k| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 156k| do { \ | | | | 646| 156k| uint64_t ll = (S) >> 24; \ | | | | 647| 156k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 156k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 156k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 156k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.56k, False: 154k] | | | | | | ------------------ | | | | ------------------ | | | | 650| 1.56k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 1.56k| const uint64_t reserve = (RESERVE); \ | | | | 652| 1.56k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 1.56k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.12k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.56k] | | | | | | | Branch (151:42): [True: 0, False: 1.56k] | | | | | | | Branch (151:42): [True: 0, False: 1.56k] | | | | | | ------------------ | | | | ------------------ | | | | 654| 1.56k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 1.56k| (size_t)(d_end - d_ptr))) \ | | | | 656| 1.56k| ON_FAIL; \ | | | | 657| 1.56k| } \ | | | | 658| 156k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 6.82k, False: 149k] | | | | | | ------------------ | | | | ------------------ | | | | 659| 6.82k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 6.82k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 6.82k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 6.82k] | | | | | | ------------------ | | | | ------------------ | | | | 661| 6.82k| (size_t)(d_end - d_ptr))) \ | | | | 662| 6.82k| ON_FAIL; \ | | | | 663| 6.82k| } \ | | | | 664| 156k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 156k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 156k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 156k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 156k| l_ptr += ll; \ | | | | 668| 156k| d_ptr += ll; \ | | | | 669| 156k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1262| 156k| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 528| 156k| do { \ | | | | | | | | 529| 156k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 156k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 530| 156k| COPY(d_ptr, off, ml); \ | | | | | | | | 531| 156k| d_ptr += ml; \ | | | | | | | | 532| 156k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (532:14): [Folded, False: 156k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 156k| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 156k] | | | | ------------------ | | ------------------ | | 687| 156k| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 156k| do { \ | | | | 646| 156k| uint64_t ll = (S) >> 24; \ | | | | 647| 156k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 156k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 156k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 156k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.48k, False: 154k] | | | | | | ------------------ | | | | ------------------ | | | | 650| 1.48k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 1.48k| const uint64_t reserve = (RESERVE); \ | | | | 652| 1.48k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 1.48k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 2.96k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.48k] | | | | | | | Branch (151:42): [True: 0, False: 1.48k] | | | | | | | Branch (151:42): [True: 0, False: 1.48k] | | | | | | ------------------ | | | | ------------------ | | | | 654| 1.48k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 1.48k| (size_t)(d_end - d_ptr))) \ | | | | 656| 1.48k| ON_FAIL; \ | | | | 657| 1.48k| } \ | | | | 658| 156k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 6.96k, False: 149k] | | | | | | ------------------ | | | | ------------------ | | | | 659| 6.96k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 6.96k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 6.96k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 6.96k] | | | | | | ------------------ | | | | ------------------ | | | | 661| 6.96k| (size_t)(d_end - d_ptr))) \ | | | | 662| 6.96k| ON_FAIL; \ | | | | 663| 6.96k| } \ | | | | 664| 156k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 156k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 156k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 156k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 156k| l_ptr += ll; \ | | | | 668| 156k| d_ptr += ll; \ | | | | 669| 156k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1262| 156k| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 528| 156k| do { \ | | | | | | | | 529| 156k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 156k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 530| 156k| COPY(d_ptr, off, ml); \ | | | | | | | | 531| 156k| d_ptr += ml; \ | | | | | | | | 532| 156k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (532:14): [Folded, False: 156k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 156k| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 156k] | | | | ------------------ | | ------------------ | | 688| 156k| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 156k| do { \ | | | | 646| 156k| uint64_t ll = (S) >> 24; \ | | | | 647| 156k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 156k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 156k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 156k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.52k, False: 154k] | | | | | | ------------------ | | | | ------------------ | | | | 650| 1.52k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 1.52k| const uint64_t reserve = (RESERVE); \ | | | | 652| 1.52k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 1.52k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.05k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.52k] | | | | | | | Branch (151:42): [True: 0, False: 1.52k] | | | | | | | Branch (151:42): [True: 0, False: 1.52k] | | | | | | ------------------ | | | | ------------------ | | | | 654| 1.52k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 1.52k| (size_t)(d_end - d_ptr))) \ | | | | 656| 1.52k| ON_FAIL; \ | | | | 657| 1.52k| } \ | | | | 658| 156k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 6.88k, False: 149k] | | | | | | ------------------ | | | | ------------------ | | | | 659| 6.88k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 6.88k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 6.88k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 6.88k] | | | | | | ------------------ | | | | ------------------ | | | | 661| 6.88k| (size_t)(d_end - d_ptr))) \ | | | | 662| 6.88k| ON_FAIL; \ | | | | 663| 6.88k| } \ | | | | 664| 156k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 156k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 156k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 156k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 156k| l_ptr += ll; \ | | | | 668| 156k| d_ptr += ll; \ | | | | 669| 156k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1262| 156k| DECODE_GHI_BATCH_4X(DECODE_MATCH_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 528| 156k| do { \ | | | | | | | | 529| 156k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 156k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 530| 156k| COPY(d_ptr, off, ml); \ | | | | | | | | 531| 156k| d_ptr += ml; \ | | | | | | | | 532| 156k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (532:14): [Folded, False: 156k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 156k| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 156k] | | | | ------------------ | | ------------------ | | 689| 156k| n_seq -= 4; \ | | 690| 156k| } while (0) | | ------------------ | | | Branch (690:14): [Folded, False: 156k] | | ------------------ ------------------ 1263| 156k| } 1264| 4.10k| } 1265| | 1266| | // --- SAFE Loop tail: remaining sequences with offset validation (1x) --- 1267| 138k| while (n_seq > 0 && d_ptr < d_end_safe && d_ptr < d_bounds) { ------------------ | Branch (1267:12): [True: 136k, False: 2.93k] | Branch (1267:25): [True: 136k, False: 0] | Branch (1267:47): [True: 134k, False: 1.17k] ------------------ 1268| 134k| uint32_t seq = zxc_le32(seq_ptr); 1269| 134k| seq_ptr += sizeof(uint32_t); 1270| | 1271| 134k| uint64_t ll = seq >> 24; 1272| 134k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 134k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 669, False: 134k] | | ------------------ ------------------ 1273| | 1274| 134k| uint32_t m_bits = (seq >> 16) & 0xFF; 1275| 134k| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 535| 134k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1276| 134k| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 134k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 4.76k, False: 130k] | | ------------------ ------------------ 1277| | 1278| 134k| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 618| 134k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1279| | 1280| | // Strict bounds check: sequence must fit, AND wild copies must not overshoot 1281| | // Check both destination (d_ptr) and source literal stream (l_ptr) 1282| 134k| if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr) || ------------------ | | 151| 269k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 17.0k, False: 117k] | | | Branch (151:42): [True: 0, False: 134k] | | | Branch (151:42): [True: 17.0k, False: 117k] | | ------------------ ------------------ 1283| 134k| ll + ZXC_PAD_SIZE > (size_t)(l_end - l_ptr))) { 1284| | // Fallback to exact copy (slow but safe) 1285| 17.0k| if (UNLIKELY(d_ptr + ll > d_end || l_ptr + ll > l_end)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 34.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 17.0k] | | | Branch (151:42): [True: 0, False: 17.0k] | | | Branch (151:42): [True: 0, False: 17.0k] | | ------------------ ------------------ 1286| 17.0k| ZXC_MEMCPY(d_ptr, l_ptr, ll); ------------------ | | 167| 17.0k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1287| 17.0k| l_ptr += ll; 1288| 17.0k| d_ptr += ll; 1289| | 1290| 17.0k| if (UNLIKELY(d_ptr + ml > d_end)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 17.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 17.0k] | | ------------------ ------------------ 1291| 17.0k| if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET; ------------------ | | 151| 17.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 17.0k] | | ------------------ ------------------ 1292| 17.0k| const uint8_t* match_src = d_ptr - offset; 1293| | 1294| 17.0k| zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml); 1295| 17.0k| d_ptr += ml; 1296| 117k| } else { 1297| 117k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); 1298| 117k| l_ptr += ll; 1299| 117k| d_ptr += ll; 1300| 117k| DECODE_MATCH_SAFE(ml, offset, zxc_decode_copy_match); ------------------ | | 528| 117k| do { \ | | 529| 117k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | ------------------ | | | | 151| 117k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | ------------------ | | | | | Branch (151:21): [True: 0, False: 117k] | | | | ------------------ | | ------------------ | | 530| 117k| COPY(d_ptr, off, ml); \ | | 531| 117k| d_ptr += ml; \ | | 532| 117k| } while (0) | | ------------------ | | | Branch (532:14): [Folded, False: 117k] | | ------------------ ------------------ 1301| 117k| } 1302| 134k| n_seq--; 1303| 134k| } 1304| | 1305| | // --- FAST Loop: After threshold, check large margin to avoid individual bounds checks --- 1306| 4.10k| if (safe) { ------------------ | Branch (1306:9): [True: 0, False: 4.10k] ------------------ 1307| 0| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) { ------------------ | Branch (1307:16): [True: 0, False: 0] | Branch (1307:30): [True: 0, False: 0] | Branch (1307:52): [True: 0, False: 0] ------------------ 1308| 0| const uint8_t* const t_save = seq_ptr; 1309| 0| const uint8_t* const e_save = extras_ptr; 1310| 0| uint8_t* const d_save = d_ptr; 1311| 0| const uint8_t* const l_save = l_ptr; 1312| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, goto rollback_fast_4x, ------------------ | | 678| 0| do { \ | | 679| 0| uint32_t s1 = zxc_le32(seq_ptr); \ | | 680| 0| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 681| 0| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 682| 0| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 683| 0| seq_ptr += 4 * sizeof(uint32_t); \ | | 684| 0| PREFETCH; \ | | 685| 0| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 0| do { \ | | | | 646| 0| uint64_t ll = (S) >> 24; \ | | | | 647| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 650| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 0| const uint64_t reserve = (RESERVE); \ | | | | 652| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 654| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 0| (size_t)(d_end - d_ptr))) \ | | | | 656| 0| ON_FAIL; \ | | | | 657| 0| } \ | | | | 658| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 659| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 661| 0| (size_t)(d_end - d_ptr))) \ | | | | 662| 0| ON_FAIL; \ | | | | 663| 0| } \ | | | | 664| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 0| l_ptr += ll; \ | | | | 668| 0| d_ptr += ll; \ | | | | 669| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1312| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 537| 0| do { \ | | | | | | | | 538| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 539| 0| d_ptr += ml; \ | | | | | | | | 540| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (540:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 0| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 686| 0| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 0| do { \ | | | | 646| 0| uint64_t ll = (S) >> 24; \ | | | | 647| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 650| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 0| const uint64_t reserve = (RESERVE); \ | | | | 652| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 654| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 0| (size_t)(d_end - d_ptr))) \ | | | | 656| 0| ON_FAIL; \ | | | | 657| 0| } \ | | | | 658| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 659| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 661| 0| (size_t)(d_end - d_ptr))) \ | | | | 662| 0| ON_FAIL; \ | | | | 663| 0| } \ | | | | 664| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 0| l_ptr += ll; \ | | | | 668| 0| d_ptr += ll; \ | | | | 669| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1312| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 537| 0| do { \ | | | | | | | | 538| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 539| 0| d_ptr += ml; \ | | | | | | | | 540| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (540:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 0| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 687| 0| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 0| do { \ | | | | 646| 0| uint64_t ll = (S) >> 24; \ | | | | 647| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 650| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 0| const uint64_t reserve = (RESERVE); \ | | | | 652| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 654| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 0| (size_t)(d_end - d_ptr))) \ | | | | 656| 0| ON_FAIL; \ | | | | 657| 0| } \ | | | | 658| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 659| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 661| 0| (size_t)(d_end - d_ptr))) \ | | | | 662| 0| ON_FAIL; \ | | | | 663| 0| } \ | | | | 664| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 0| l_ptr += ll; \ | | | | 668| 0| d_ptr += ll; \ | | | | 669| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1312| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 537| 0| do { \ | | | | | | | | 538| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 539| 0| d_ptr += ml; \ | | | | | | | | 540| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (540:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 0| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 688| 0| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 0| do { \ | | | | 646| 0| uint64_t ll = (S) >> 24; \ | | | | 647| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 650| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 0| const uint64_t reserve = (RESERVE); \ | | | | 652| 0| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 654| 0| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 0| (size_t)(d_end - d_ptr))) \ | | | | 656| 0| ON_FAIL; \ | | | | 657| 0| } \ | | | | 658| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 659| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 661| 0| (size_t)(d_end - d_ptr))) \ | | | | 662| 0| ON_FAIL; \ | | | | 663| 0| } \ | | | | 664| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 0| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 0| l_ptr += ll; \ | | | | 668| 0| d_ptr += ll; \ | | | | 669| 0| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1312| 0| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 537| 0| do { \ | | | | | | | | 538| 0| COPY(d_ptr, off, ml); \ | | | | | | | | 539| 0| d_ptr += ml; \ | | | | | | | | 540| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (540:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 0| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 689| 0| n_seq -= 4; \ | | 690| 0| } while (0) | | ------------------ | | | Branch (690:14): [Folded, False: 0] | | ------------------ ------------------ 1313| 0| ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE)); 1314| 0| continue; 1315| | 1316| 0| rollback_fast_4x: 1317| 0| seq_ptr = t_save; 1318| 0| extras_ptr = e_save; 1319| 0| d_ptr = d_save; 1320| 0| l_ptr = l_save; 1321| 0| break; 1322| 0| } 1323| 4.10k| } else { 1324| 392k| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) { ------------------ | Branch (1324:16): [True: 388k, False: 3.28k] | Branch (1324:30): [True: 388k, False: 0] | Branch (1324:52): [True: 387k, False: 827] ------------------ 1325| 387k| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, return ZXC_ERROR_OVERFLOW, ------------------ | | 678| 387k| do { \ | | 679| 387k| uint32_t s1 = zxc_le32(seq_ptr); \ | | 680| 387k| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 681| 387k| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 682| 387k| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 683| 387k| seq_ptr += 4 * sizeof(uint32_t); \ | | 684| 387k| PREFETCH; \ | | 685| 387k| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 387k| do { \ | | | | 646| 387k| uint64_t ll = (S) >> 24; \ | | | | 647| 387k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 387k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 387k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 387k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.90k, False: 386k] | | | | | | ------------------ | | | | ------------------ | | | | 650| 1.90k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 1.90k| const uint64_t reserve = (RESERVE); \ | | | | 652| 1.90k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 1.90k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.81k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.90k] | | | | | | | Branch (151:42): [True: 0, False: 1.90k] | | | | | | | Branch (151:42): [True: 0, False: 1.90k] | | | | | | ------------------ | | | | ------------------ | | | | 654| 1.90k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 1.90k| (size_t)(d_end - d_ptr))) \ | | | | 656| 1.90k| ON_FAIL; \ | | | | 657| 1.90k| } \ | | | | 658| 387k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 10.8k, False: 377k] | | | | | | ------------------ | | | | ------------------ | | | | 659| 10.8k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 10.8k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 10.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 10.8k] | | | | | | ------------------ | | | | ------------------ | | | | 661| 10.8k| (size_t)(d_end - d_ptr))) \ | | | | 662| 10.8k| ON_FAIL; \ | | | | 663| 10.8k| } \ | | | | 664| 387k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 387k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 387k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 387k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 387k| l_ptr += ll; \ | | | | 668| 387k| d_ptr += ll; \ | | | | 669| 387k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1325| 387k| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 537| 387k| do { \ | | | | | | | | 538| 387k| COPY(d_ptr, off, ml); \ | | | | | | | | 539| 387k| d_ptr += ml; \ | | | | | | | | 540| 387k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (540:14): [Folded, False: 387k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 387k| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 387k] | | | | ------------------ | | ------------------ | | 686| 387k| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 387k| do { \ | | | | 646| 387k| uint64_t ll = (S) >> 24; \ | | | | 647| 387k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 387k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 387k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 387k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.96k, False: 386k] | | | | | | ------------------ | | | | ------------------ | | | | 650| 1.96k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 1.96k| const uint64_t reserve = (RESERVE); \ | | | | 652| 1.96k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 1.96k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.93k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.96k] | | | | | | | Branch (151:42): [True: 0, False: 1.96k] | | | | | | | Branch (151:42): [True: 0, False: 1.96k] | | | | | | ------------------ | | | | ------------------ | | | | 654| 1.96k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 1.96k| (size_t)(d_end - d_ptr))) \ | | | | 656| 1.96k| ON_FAIL; \ | | | | 657| 1.96k| } \ | | | | 658| 387k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 10.8k, False: 377k] | | | | | | ------------------ | | | | ------------------ | | | | 659| 10.8k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 10.8k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 10.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 10.8k] | | | | | | ------------------ | | | | ------------------ | | | | 661| 10.8k| (size_t)(d_end - d_ptr))) \ | | | | 662| 10.8k| ON_FAIL; \ | | | | 663| 10.8k| } \ | | | | 664| 387k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 387k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 387k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 387k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 387k| l_ptr += ll; \ | | | | 668| 387k| d_ptr += ll; \ | | | | 669| 387k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1325| 387k| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 537| 387k| do { \ | | | | | | | | 538| 387k| COPY(d_ptr, off, ml); \ | | | | | | | | 539| 387k| d_ptr += ml; \ | | | | | | | | 540| 387k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (540:14): [Folded, False: 387k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 387k| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 387k] | | | | ------------------ | | ------------------ | | 687| 387k| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 387k| do { \ | | | | 646| 387k| uint64_t ll = (S) >> 24; \ | | | | 647| 387k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 387k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 387k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 387k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 1.91k, False: 386k] | | | | | | ------------------ | | | | ------------------ | | | | 650| 1.91k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 1.91k| const uint64_t reserve = (RESERVE); \ | | | | 652| 1.91k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 1.91k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 3.82k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 1.91k] | | | | | | | Branch (151:42): [True: 0, False: 1.91k] | | | | | | | Branch (151:42): [True: 0, False: 1.91k] | | | | | | ------------------ | | | | ------------------ | | | | 654| 1.91k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 1.91k| (size_t)(d_end - d_ptr))) \ | | | | 656| 1.91k| ON_FAIL; \ | | | | 657| 1.91k| } \ | | | | 658| 387k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 10.8k, False: 377k] | | | | | | ------------------ | | | | ------------------ | | | | 659| 10.8k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 10.8k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 10.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 10.8k] | | | | | | ------------------ | | | | ------------------ | | | | 661| 10.8k| (size_t)(d_end - d_ptr))) \ | | | | 662| 10.8k| ON_FAIL; \ | | | | 663| 10.8k| } \ | | | | 664| 387k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 387k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 387k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 387k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 387k| l_ptr += ll; \ | | | | 668| 387k| d_ptr += ll; \ | | | | 669| 387k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1325| 387k| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 537| 387k| do { \ | | | | | | | | 538| 387k| COPY(d_ptr, off, ml); \ | | | | | | | | 539| 387k| d_ptr += ml; \ | | | | | | | | 540| 387k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (540:14): [Folded, False: 387k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 387k| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 387k] | | | | ------------------ | | ------------------ | | 688| 387k| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 645| 387k| do { \ | | | | 646| 387k| uint64_t ll = (S) >> 24; \ | | | | 647| 387k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 648| 387k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 535| 387k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 649| 387k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 2.03k, False: 385k] | | | | | | ------------------ | | | | ------------------ | | | | 650| 2.03k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 651| 2.03k| const uint64_t reserve = (RESERVE); \ | | | | 652| 2.03k| /* Same reservation as the GLO twin; GHI's inline ml reaches 259. */ \ | | | | 653| 2.03k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 4.07k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 2.03k] | | | | | | | Branch (151:42): [True: 0, False: 2.03k] | | | | | | | Branch (151:42): [True: 0, False: 2.03k] | | | | | | ------------------ | | | | ------------------ | | | | 654| 2.03k| ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 655| 2.03k| (size_t)(d_end - d_ptr))) \ | | | | 656| 2.03k| ON_FAIL; \ | | | | 657| 2.03k| } \ | | | | 658| 387k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 387k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 10.6k, False: 377k] | | | | | | ------------------ | | | | ------------------ | | | | 659| 10.6k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 660| 10.6k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 10.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 10.6k] | | | | | | ------------------ | | | | ------------------ | | | | 661| 10.6k| (size_t)(d_end - d_ptr))) \ | | | | 662| 10.6k| ON_FAIL; \ | | | | 663| 10.6k| } \ | | | | 664| 387k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 618| 387k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 665| 387k| /* GHI keeps the ladder: its inline ll reaches 254. */ \ | | | | 666| 387k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | 667| 387k| l_ptr += ll; \ | | | | 668| 387k| d_ptr += ll; \ | | | | 669| 387k| DECODE(ml, off, zxc_decode_copy_match); \ | | | | ------------------ | | | | | | 1325| 387k| DECODE_GHI_BATCH_4X(DECODE_MATCH_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 537| 387k| do { \ | | | | | | | | 538| 387k| COPY(d_ptr, off, ml); \ | | | | | | | | 539| 387k| d_ptr += ml; \ | | | | | | | | 540| 387k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (540:14): [Folded, False: 387k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 670| 387k| } while (0) | | | | ------------------ | | | | | Branch (670:14): [Folded, False: 387k] | | | | ------------------ | | ------------------ | | 689| 387k| n_seq -= 4; \ | | 690| 387k| } while (0) | | ------------------ | | | Branch (690:14): [Folded, False: 387k] | | ------------------ ------------------ 1326| 387k| ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE)); 1327| 387k| } 1328| 4.10k| } 1329| | 1330| | // --- Remaining 1 sequence (Fast Path) --- 1331| 160k| while (n_seq > 0 && d_ptr < d_end_safe && l_ptr < l_end_safe_1x) { ------------------ | Branch (1331:12): [True: 156k, False: 3.12k] | Branch (1331:25): [True: 156k, False: 0] | Branch (1331:47): [True: 156k, False: 878] ------------------ 1332| | // Save state for fallback 1333| 156k| const uint8_t* seq_save = seq_ptr; 1334| 156k| const uint8_t* ext_save = extras_ptr; 1335| | 1336| 156k| const uint32_t seq = zxc_le32(seq_ptr); 1337| 156k| seq_ptr += sizeof(uint32_t); 1338| | 1339| 156k| uint64_t ll = seq >> 24; 1340| 156k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { ------------------ | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 318, False: 155k] | | ------------------ ------------------ 1341| 318| ll += zxc_read_varint(&extras_ptr, extras_end); 1342| 318| if (UNLIKELY(l_ptr + ll > l_end)) { ------------------ | | 151| 318|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 318] | | ------------------ ------------------ 1343| 0| seq_ptr = seq_save; 1344| 0| extras_ptr = ext_save; 1345| 0| break; 1346| 0| } 1347| 318| } 1348| | 1349| 156k| uint32_t m_bits = (seq >> 16) & 0xFF; 1350| 156k| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 535| 156k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1351| 156k| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 156k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 6.86k, False: 149k] | | ------------------ ------------------ 1352| | 1353| | // Strict bounds checks (including wild copy overrun safety) 1354| 156k| if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr) || ------------------ | | 151| 312k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 100, False: 155k] | | | Branch (151:42): [True: 0, False: 156k] | | | Branch (151:42): [True: 100, False: 155k] | | ------------------ ------------------ 1355| 156k| ll + ZXC_PAD_SIZE > (size_t)(l_end - l_ptr))) { 1356| 100| seq_ptr = seq_save; 1357| 100| extras_ptr = ext_save; 1358| 100| break; 1359| 100| } 1360| 155k| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 618| 155k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1361| | 1362| 155k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); 1363| 155k| l_ptr += ll; 1364| 155k| d_ptr += ll; 1365| | 1366| 155k| if (UNLIKELY(d_ptr < d_bounds && (size_t)(d_ptr - d_floor) < offset)) ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 155k] | | | Branch (151:42): [True: 0, False: 155k] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1367| 0| return ZXC_ERROR_BAD_OFFSET; 1368| | 1369| | // The loop entry check guarantees ll + ml + ZXC_PAD_SIZE bytes of 1370| | // headroom, so the wild-copy ladder (incl. overlap/fill runs) is safe. 1371| 155k| zxc_decode_copy_match(d_ptr, offset, ml); 1372| 155k| d_ptr += ml; 1373| 155k| n_seq--; 1374| 155k| } 1375| | 1376| | // --- Safe Path for Remaining Sequences --- 1377| 159k| while (n_seq > 0) { ------------------ | Branch (1377:12): [True: 155k, False: 4.10k] ------------------ 1378| 155k| uint32_t seq = zxc_le32(seq_ptr); 1379| 155k| seq_ptr += sizeof(uint32_t); 1380| | 1381| 155k| uint64_t ll = seq >> 24; 1382| 155k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 97, False: 155k] | | ------------------ ------------------ 1383| | 1384| 155k| uint32_t m_bits = (seq >> 16) & 0xFF; 1385| 155k| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 535| 155k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1386| 155k| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 6.71k, False: 148k] | | ------------------ ------------------ 1387| 155k| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 618| 155k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1388| | 1389| 155k| if (UNLIKELY(ll + ml > (size_t)(d_end - d_ptr) || l_ptr + ll > l_end)) ------------------ | | 151| 310k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 155k] | | | Branch (151:42): [True: 0, False: 155k] | | | Branch (151:42): [True: 0, False: 155k] | | ------------------ ------------------ 1390| 0| return ZXC_ERROR_OVERFLOW; 1391| 155k| ZXC_MEMCPY(d_ptr, l_ptr, ll); ------------------ | | 167| 155k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1392| 155k| l_ptr += ll; 1393| 155k| d_ptr += ll; 1394| | 1395| 155k| if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET; ------------------ | | 151| 155k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 155k] | | ------------------ ------------------ 1396| 155k| const uint8_t* match_src = d_ptr - offset; 1397| | 1398| 155k| zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml); 1399| 155k| d_ptr += ml; 1400| 155k| n_seq--; 1401| 155k| } 1402| | 1403| | // --- Trailing Literals --- 1404| | // Copy remaining literals from source stream (literal exhaustion) 1405| 4.10k| if (UNLIKELY(l_ptr > l_end)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 1406| 4.10k| if (UNLIKELY(d_ptr > d_end)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 1407| | 1408| 4.10k| const size_t remaining_literals = (size_t)(l_end - l_ptr); 1409| 4.10k| if (UNLIKELY(remaining_literals > (size_t)(d_end - d_ptr))) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 4.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.10k] | | ------------------ ------------------ 1410| 4.10k| ZXC_MEMCPY(d_ptr, l_ptr, remaining_literals); ------------------ | | 167| 4.10k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1411| 4.10k| d_ptr += remaining_literals; 1412| | 1413| 4.10k| return (int)(d_ptr - dst); 1414| 4.10k|} zxc_decompress.c:zxc_decode_block_ghi: 1477| 4.10k| const size_t dst_capacity) { 1478| 4.10k| return zxc_decode_block_ghi_impl(ctx, src, src_size, dst, dst_capacity, 0, 0); 1479| 4.10k|} zxc_common.c:zxc_aligned_malloc: 112| 9.21k|static inline void* zxc_aligned_malloc(const size_t size, const size_t alignment) { 113| |#if defined(_WIN32) 114| | return _aligned_malloc(size, alignment); 115| |#else 116| 9.21k| void* ptr = NULL; 117| 9.21k| if (posix_memalign(&ptr, alignment, size) != 0) return NULL; ------------------ | Branch (117:9): [True: 0, False: 9.21k] ------------------ 118| 9.21k| return ptr; 119| 9.21k|#endif 120| 9.21k|} zxc_common.c:zxc_aligned_free: 123| 9.21k|static inline void zxc_aligned_free(void* ptr) { 124| |#if defined(_WIN32) 125| | _aligned_free(ptr); 126| |#else 127| 9.21k| free(ptr); 128| 9.21k|#endif 129| 9.21k|} zxc_decompress_chunk_wrapper: 436| 4.90k| const uint64_t block_index) { 437| | // dict_size is constant for a stream; this per-block branch (outside the decode 438| | // loop) routes to the dict variant only when a dictionary is active, so the 439| | // no-dict path runs the dict-free chunk wrapper (identical codegen to main). 440| 4.90k| const zxc_variant_set_t* v = zxc_variants(); 441| 4.90k| return (ctx->dict_size ? v->decompress_dict : v->decompress)(ctx, src, src_sz, dst, dst_cap, ------------------ | Branch (441:13): [True: 0, False: 4.90k] ------------------ 442| 4.90k| block_index); 443| 4.90k|} zxc_compress_chunk_wrapper: 467| 4.90k| const uint64_t block_index) { 468| 4.90k| return zxc_variants()->compress(ctx, src, src_sz, dst, dst_cap, block_index); 469| 4.90k|} zxc_create_cctx: 1253| 4.60k|zxc_cctx* zxc_create_cctx(const zxc_compress_opts_t* opts) { 1254| 4.60k| zxc_cctx* const cctx = (zxc_cctx*)ZXC_CALLOC(1, sizeof(zxc_cctx)); ------------------ | | 73| 4.60k|#define ZXC_CALLOC(nmemb, size) calloc(nmemb, size) ------------------ 1255| 4.60k| if (UNLIKELY(!cctx)) return NULL; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 1256| | 1257| | // Resolve and store sticky defaults. 1258| 4.60k| cctx->stored_level = ZXC_OPTS_LEVEL(opts, ZXC_LEVEL_DEFAULT); ------------------ | | 899| 4.60k|#define ZXC_OPTS_LEVEL(o, dflt) zxc_level_clamp(((o) && (o)->level > 0) ? (o)->level : (dflt)) | | ------------------ | | | Branch (899:50): [True: 4.60k, False: 0] | | | Branch (899:57): [True: 4.60k, False: 0] | | ------------------ ------------------ 1259| 4.60k| cctx->stored_block_size = ZXC_OPTS_BLOCK_SIZE(opts, ZXC_BLOCK_SIZE_DEFAULT); ------------------ | | 901| 4.60k|#define ZXC_OPTS_BLOCK_SIZE(o, dflt) (((o) && (o)->block_size > 0) ? (o)->block_size : (dflt)) | | ------------------ | | | Branch (901:40): [True: 4.60k, False: 0] | | | Branch (901:47): [True: 4.60k, False: 0] | | ------------------ ------------------ 1260| 4.60k| cctx->stored_checksum = opts ? opts->checksum_enabled : 0; ------------------ | Branch (1260:29): [True: 4.60k, False: 0] ------------------ 1261| | 1262| 4.60k| if (opts) { ------------------ | Branch (1262:9): [True: 4.60k, False: 0] ------------------ 1263| | // LCOV_EXCL_START 1264| 4.60k| if (UNLIKELY(!zxc_validate_block_size(cctx->stored_block_size) || ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 1265| 4.60k| zxc_cctx_init(&cctx->inner, cctx->stored_block_size, 1, cctx->stored_level, 1266| 4.60k| cctx->stored_checksum, 0) != ZXC_OK)) { 1267| 0| ZXC_FREE(cctx); ------------------ | | 85| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 1268| 0| return NULL; 1269| 0| } 1270| | // LCOV_EXCL_STOP 1271| 4.60k| cctx->last_block_size = cctx->stored_block_size; 1272| 4.60k| cctx->initialized = 1; 1273| 4.60k| } 1274| | 1275| 4.60k| return cctx; 1276| 4.60k|} zxc_free_cctx: 1285| 4.60k|void zxc_free_cctx(zxc_cctx* cctx) { 1286| 4.60k| if (UNLIKELY(!cctx)) return; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 1287| | // Static cctx: handle + inner buffers live inside the caller's workspace, 1288| | // which we do not own. Free is a no-op; the caller owns the workspace. 1289| 4.60k| if (cctx->owns_workspace) return; ------------------ | Branch (1289:9): [True: 0, False: 4.60k] ------------------ 1290| 4.60k| if (cctx->initialized) zxc_cctx_free(&cctx->inner); ------------------ | Branch (1290:9): [True: 4.60k, False: 0] ------------------ 1291| 4.60k| ZXC_FREE(cctx); ------------------ | | 85| 4.60k|#define ZXC_FREE(ptr) free(ptr) ------------------ 1292| 4.60k|} zxc_compress_block_at: 1674| 4.90k| const zxc_compress_opts_t* opts, const uint64_t block_index) { 1675| 4.90k| if (UNLIKELY(!cctx || !src || !dst || src_size == 0 || dst_capacity == 0)) ------------------ | | 151| 39.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | ------------------ ------------------ 1676| 0| return ZXC_ERROR_NULL_INPUT; 1677| | 1678| | // Block API processes a single format-conformant block: src_size must not 1679| | // exceed ZXC_BLOCK_SIZE_MAX. Callers with larger inputs should use the 1680| | // frame or streaming APIs which chunk transparently. 1681| 4.90k| if (UNLIKELY(src_size > ZXC_BLOCK_SIZE_MAX)) return ZXC_ERROR_BAD_BLOCK_SIZE; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1682| | 1683| 4.90k| const int checksum_enabled = opts ? opts->checksum_enabled : cctx->stored_checksum; ------------------ | Branch (1683:34): [True: 4.90k, False: 0] ------------------ 1684| 4.90k| const int level = ZXC_OPTS_LEVEL(opts, cctx->stored_level); ------------------ | | 899| 4.90k|#define ZXC_OPTS_LEVEL(o, dflt) zxc_level_clamp(((o) && (o)->level > 0) ? (o)->level : (dflt)) | | ------------------ | | | Branch (899:50): [True: 4.90k, False: 0] | | | Branch (899:57): [True: 4.90k, False: 0] | | ------------------ ------------------ 1685| | // For block API, block_size == src_size (the caller compresses one block at a time). 1686| 4.90k| const size_t block_size = ZXC_OPTS_BLOCK_SIZE(opts, cctx->stored_block_size); ------------------ | | 901| 4.90k|#define ZXC_OPTS_BLOCK_SIZE(o, dflt) (((o) && (o)->block_size > 0) ? (o)->block_size : (dflt)) | | ------------------ | | | Branch (901:40): [True: 4.90k, False: 0] | | | Branch (901:47): [True: 4.90k, False: 0] | | ------------------ ------------------ 1687| 4.90k| const size_t min_bs = zxc_block_size_ceil(src_size); 1688| | 1689| | // Always ensure internal buffers can hold src_size. 1690| | // When a dictionary is active, offset_bits must accommodate dict + block. 1691| 4.90k| const uint8_t* b_dict = opts ? (const uint8_t*)opts->dict : NULL; ------------------ | Branch (1691:29): [True: 4.90k, False: 0] ------------------ 1692| 4.90k| const size_t b_dict_size = ZXC_OPTS_DICT_SIZE(opts); ------------------ | | 893| 4.90k|#define ZXC_OPTS_DICT_SIZE(o) (((o) && (o)->dict) ? (o)->dict_size : (size_t)0) | | ------------------ | | | Branch (893:33): [True: 4.90k, False: 0] | | | Branch (893:40): [True: 0, False: 4.90k] | | ------------------ ------------------ 1693| 4.90k| if (UNLIKELY(b_dict_size > ZXC_DICT_SIZE_MAX)) return ZXC_ERROR_DICT_TOO_LARGE; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1694| 4.90k| const size_t base_block_size = (block_size > min_bs) ? block_size : min_bs; ------------------ | Branch (1694:36): [True: 4.22k, False: 687] ------------------ 1695| 4.90k| const size_t effective_block_size = 1696| 4.90k| b_dict_size > 0 ? zxc_block_size_ceil(b_dict_size + base_block_size) : base_block_size; ------------------ | Branch (1696:9): [True: 0, False: 4.90k] ------------------ 1697| | 1698| | // Static cctx: the workspace cannot grow, so reject anything forcing a 1699| | // re-partition - another block size, or a level raise into the 1700| | // optimal-parser tier it carries no opt_scratch for. Re-initing on the heap 1701| | // would break the no-allocation contract and leak: zxc_free_cctx is a no-op 1702| | // for static contexts. 1703| 4.90k| if (UNLIKELY(cctx->owns_workspace && b_dict_size > 0)) return ZXC_ERROR_DICT_UNSUPPORTED; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1704| 4.90k| if (UNLIKELY(cctx->owns_workspace && effective_block_size != cctx->last_block_size)) ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1705| 0| return ZXC_ERROR_BAD_BLOCK_SIZE; // LCOV_EXCL_LINE 1706| 4.90k| if (UNLIKELY(cctx->owns_workspace && level >= ZXC_LEVEL_DENSITY && !cctx->inner.opt_scratch)) ------------------ | | 151| 9.81k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1707| 0| return ZXC_ERROR_BAD_LEVEL; 1708| | 1709| 4.90k| cctx->stored_level = level; 1710| 4.90k| cctx->stored_block_size = base_block_size; 1711| 4.90k| cctx->stored_checksum = checksum_enabled; 1712| | 1713| | // Re-init when block_size changed, a level raise needs the optimal-parser 1714| | // scratch, or a dictionary arrives on a context carved without its prefix 1715| | // (a block_size switch can round [dict | block] back to the same size). 1716| 4.90k| if (UNLIKELY(!cctx->initialized || cctx->last_block_size != effective_block_size || ------------------ | | 151| 39.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1717| 4.90k| (level >= ZXC_LEVEL_DENSITY && !cctx->inner.opt_scratch) || 1718| 4.90k| (b_dict_size > 0 && !cctx->inner.dict_buffer))) { 1719| 0| if (cctx->initialized) { ------------------ | Branch (1719:13): [True: 0, False: 0] ------------------ 1720| | // LCOV_EXCL_START 1721| 0| zxc_cctx_free(&cctx->inner); 1722| 0| cctx->initialized = 0; 1723| | // LCOV_EXCL_STOP 1724| 0| } 1725| | // LCOV_EXCL_START 1726| 0| if (UNLIKELY(zxc_cctx_init(&cctx->inner, effective_block_size, 1, level, checksum_enabled, ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1727| 0| b_dict_size) != ZXC_OK)) 1728| 0| return ZXC_ERROR_MEMORY; 1729| | // LCOV_EXCL_STOP 1730| 0| cctx->last_block_size = effective_block_size; 1731| 0| cctx->initialized = 1; 1732| 0| cctx->huf_cached = 0; 1733| 4.90k| } else { 1734| 4.90k| cctx->inner.compression_level = level; 1735| 4.90k| cctx->inner.checksum_enabled = checksum_enabled; 1736| 4.90k| } 1737| | 1738| 4.90k| cctx->inner.dict_size = b_dict_size; 1739| 4.90k| if (UNLIKELY(zxc_ctx_sync_dict_huf(&cctx->inner, cctx->huf_cache, &cctx->huf_cached, ------------------ | | 151| 24.5k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 4.90k, False: 0] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1740| 4.90k| ZXC_OPTS_DICT_HUF(opts)) != ZXC_OK)) 1741| 0| return ZXC_ERROR_CORRUPT_DATA; // LCOV_EXCL_LINE 1742| | 1743| 4.90k| int res; 1744| 4.90k| if (b_dict && b_dict_size > 0) { ------------------ | Branch (1744:9): [True: 0, False: 4.90k] | Branch (1744:19): [True: 0, False: 0] ------------------ 1745| | // [dict | block] assembled in the cctx-owned dict_buffer 1746| 0| uint8_t* const combined = cctx->inner.dict_buffer; 1747| 0| ZXC_MEMCPY(combined, b_dict, b_dict_size); ------------------ | | 167| 0|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1748| 0| ZXC_MEMCPY(combined + b_dict_size, src, src_size); ------------------ | | 167| 0|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1749| 0| res = zxc_compress_chunk_wrapper(&cctx->inner, combined, b_dict_size + src_size, 1750| 0| (uint8_t*)dst, dst_capacity, block_index); 1751| 4.90k| } else { 1752| 4.90k| res = zxc_compress_chunk_wrapper(&cctx->inner, (const uint8_t*)src, src_size, (uint8_t*)dst, 1753| 4.90k| dst_capacity, block_index); 1754| 4.90k| } 1755| 4.90k| if (UNLIKELY(res < 0)) return res; ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1756| 4.90k| return (int64_t)res; 1757| 4.90k|} zxc_dispatch.c:zxc_variants: 335| 9.81k|static ZXC_ALWAYS_INLINE const zxc_variant_set_t* zxc_variants(void) { 336| 9.81k| return &zxc_variants_default; 337| 9.81k|} zxc_dispatch.c:zxc_ctx_sync_dict_huf: 1237| 4.90k| int* RESTRICT cached, const uint8_t* RESTRICT dict_huf) { 1238| 4.90k| if (dict_huf) { ------------------ | Branch (1238:9): [True: 0, False: 4.90k] ------------------ 1239| 0| if (!*cached || memcmp(cache, dict_huf, ZXC_HUF_TABLE_SIZE) != 0) { ------------------ | | 83| 0|#define ZXC_HUF_TABLE_SIZE 128 ------------------ | Branch (1239:13): [True: 0, False: 0] | Branch (1239:25): [True: 0, False: 0] ------------------ 1240| 0| *cached = 0; /* a failed attach leaves inner without a tree */ 1241| 0| if (UNLIKELY(zxc_cctx_attach_dict_huf(inner, dict_huf) != ZXC_OK)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1242| 0| return ZXC_ERROR_CORRUPT_DATA; 1243| 0| ZXC_MEMCPY(cache, dict_huf, ZXC_HUF_TABLE_SIZE); ------------------ | | 167| 0|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1244| 0| *cached = 1; 1245| 0| } 1246| 4.90k| } else if (*cached) { ------------------ | Branch (1246:16): [True: 0, False: 4.90k] ------------------ 1247| 0| (void)zxc_cctx_attach_dict_huf(inner, NULL); 1248| 0| *cached = 0; 1249| 0| } 1250| 4.90k| return ZXC_OK; 1251| 4.90k|} zxc_common.c:zxc_log2_u32: 1024| 23.0k|static ZXC_ALWAYS_INLINE uint32_t zxc_log2_u32(const uint32_t v) { 1025| |#ifdef _MSC_VER 1026| | unsigned long index; 1027| | return (v == 0) ? 0 : (_BitScanReverse(&index, v) ? index : 0); 1028| |#else 1029| 23.0k| return (v == 0) ? 0 : (uint32_t)(31 - __builtin_clz(v)); ------------------ | Branch (1029:12): [True: 0, False: 23.0k] ------------------ 1030| 23.0k|#endif 1031| 23.0k|} zxc_common.c:zxc_store_le32: 1363| 24.7k|static ZXC_ALWAYS_INLINE void zxc_store_le32(void* p, const uint32_t v) { 1364| |#ifdef ZXC_BIG_ENDIAN 1365| | const uint32_t s = ZXC_BSWAP32(v); 1366| | ZXC_MEMCPY(p, &s, sizeof(s)); 1367| |#else 1368| 24.7k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 167| 24.7k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1369| 24.7k|#endif 1370| 24.7k|} zxc_common.c:zxc_file_header_sign: 1432| 4.60k|static ZXC_ALWAYS_INLINE void zxc_file_header_sign(uint8_t* h) { 1433| 4.60k| zxc_store_le16(h + 14, 0); 1434| 4.60k| zxc_store_le16(h + 14, zxc_hash16(h)); 1435| 4.60k|} zxc_common.c:zxc_store_le16: 1345| 9.21k|static ZXC_ALWAYS_INLINE void zxc_store_le16(void* p, const uint16_t v) { 1346| |#ifdef ZXC_BIG_ENDIAN 1347| | const uint16_t s = ZXC_BSWAP16(v); 1348| | ZXC_MEMCPY(p, &s, sizeof(s)); 1349| |#else 1350| 9.21k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 167| 9.21k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1351| 9.21k|#endif 1352| 9.21k|} zxc_common.c:zxc_le32: 1263| 25.4k|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1264| 25.4k| uint32_t v; 1265| 25.4k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 25.4k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1266| |#ifdef ZXC_BIG_ENDIAN 1267| | return ZXC_BSWAP32(v); 1268| |#else 1269| 25.4k| return v; 1270| 25.4k|#endif 1271| 25.4k|} zxc_common.c:zxc_le16: 1244| 4.60k|static ZXC_ALWAYS_INLINE uint16_t zxc_le16(const void* p) { 1245| 4.60k| uint16_t v; 1246| 4.60k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 4.60k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1247| |#ifdef ZXC_BIG_ENDIAN 1248| | return ZXC_BSWAP16(v); 1249| |#else 1250| 4.60k| return v; 1251| 4.60k|#endif 1252| 4.60k|} zxc_common.c:zxc_hash16: 1421| 9.21k|static ZXC_ALWAYS_INLINE uint16_t zxc_hash16(const uint8_t* p) { 1422| 9.21k| const uint64_t h1 = (zxc_le64(p) ^ ZXC_HASH_MULT64) * ZXC_HASH_MULT64; ------------------ | | 682| 9.21k|#define ZXC_HASH_MULT64 0xD2D84A61D2D84A61ULL ------------------ const uint64_t h1 = (zxc_le64(p) ^ ZXC_HASH_MULT64) * ZXC_HASH_MULT64; ------------------ | | 682| 9.21k|#define ZXC_HASH_MULT64 0xD2D84A61D2D84A61ULL ------------------ 1423| | 1424| 9.21k| return (uint16_t)(((h1 + zxc_le64(p + 8) + ZXC_HASH_GOLDEN64) * ZXC_HASH_GOLDEN64) >> 48); ------------------ | | 676| 9.21k|#define ZXC_HASH_GOLDEN64 0x9E3779B97F4A7C15ULL ------------------ return (uint16_t)(((h1 + zxc_le64(p + 8) + ZXC_HASH_GOLDEN64) * ZXC_HASH_GOLDEN64) >> 48); ------------------ | | 676| 9.21k|#define ZXC_HASH_GOLDEN64 0x9E3779B97F4A7C15ULL ------------------ 1425| 9.21k|} zxc_common.c:zxc_le64: 1326| 36.9k|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1327| 36.9k| uint64_t v; 1328| 36.9k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 36.9k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1329| |#ifdef ZXC_BIG_ENDIAN 1330| | return ZXC_BSWAP64(v); 1331| |#else 1332| 36.9k| return v; 1333| 36.9k|#endif 1334| 36.9k|} zxc_common.c:zxc_hash8: 1402| 18.4k|static ZXC_ALWAYS_INLINE uint8_t zxc_hash8(const uint8_t* p) { 1403| 18.4k| const uint64_t h = (zxc_le64(p) ^ ZXC_HASH_GOLDEN64) * ZXC_HASH_GOLDEN64; ------------------ | | 676| 18.4k|#define ZXC_HASH_GOLDEN64 0x9E3779B97F4A7C15ULL ------------------ const uint64_t h = (zxc_le64(p) ^ ZXC_HASH_GOLDEN64) * ZXC_HASH_GOLDEN64; ------------------ | | 676| 18.4k|#define ZXC_HASH_GOLDEN64 0x9E3779B97F4A7C15ULL ------------------ 1404| | 1405| 18.4k| return (uint8_t)(h >> 56); 1406| 18.4k|} zxc_common.c:zxc_seek_table_bytes: 457| 4.60k|static ZXC_ALWAYS_INLINE uint64_t zxc_seek_table_bytes(const uint64_t nblocks) { 458| 4.60k| return zxc_seek_group_count(nblocks) * ZXC_SEEK_ANCHOR_SIZE + nblocks * ZXC_SEEK_SIZE_ENTRY; ------------------ | | 439| 4.60k|#define ZXC_SEEK_ANCHOR_SIZE 8U ------------------ return zxc_seek_group_count(nblocks) * ZXC_SEEK_ANCHOR_SIZE + nblocks * ZXC_SEEK_SIZE_ENTRY; ------------------ | | 441| 4.60k|#define ZXC_SEEK_SIZE_ENTRY 4U ------------------ 459| 4.60k|} zxc_common.c:zxc_seek_group_count: 446| 4.60k|static ZXC_ALWAYS_INLINE uint64_t zxc_seek_group_count(const uint64_t nblocks) { 447| 4.60k| return (nblocks + ZXC_SEEK_GROUP - 1) / ZXC_SEEK_GROUP; ------------------ | | 437| 4.60k|#define ZXC_SEEK_GROUP 64U ------------------ return (nblocks + ZXC_SEEK_GROUP - 1) / ZXC_SEEK_GROUP; ------------------ | | 437| 4.60k|#define ZXC_SEEK_GROUP 64U ------------------ 448| 4.60k|} zxc_common.c:zxc_store_le64: 1381| 6.14k|static ZXC_ALWAYS_INLINE void zxc_store_le64(void* p, const uint64_t v) { 1382| |#ifdef ZXC_BIG_ENDIAN 1383| | const uint64_t s = ZXC_BSWAP64(v); 1384| | ZXC_MEMCPY(p, &s, sizeof(s)); 1385| |#else 1386| 6.14k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 167| 6.14k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1387| 6.14k|#endif 1388| 6.14k|} zxc_common.c:zxc_footer_bytes: 2190| 4.60k|static ZXC_ALWAYS_INLINE size_t zxc_footer_bytes(const int checksum_enabled) { 2191| 4.60k| return ZXC_FILE_FOOTER_SIZE + (checksum_enabled ? (size_t)ZXC_FILE_DIGEST_SIZE : 0U); ------------------ | | 106| 4.60k|#define ZXC_FILE_FOOTER_SIZE 8 ------------------ return ZXC_FILE_FOOTER_SIZE + (checksum_enabled ? (size_t)ZXC_FILE_DIGEST_SIZE : 0U); ------------------ | | 108| 1.53k|#define ZXC_FILE_DIGEST_SIZE 8 ------------------ | Branch (2191:36): [True: 1.53k, False: 3.06k] ------------------ 2192| 4.60k|} zxc_compress.c:zxc_get_lz77_params: 1128| 4.90k|static ZXC_ALWAYS_INLINE zxc_lz77_params_t zxc_get_lz77_params(const int level) { 1129| | // The distance floor stops at level 5: the slow levels keep every distance. 1130| | // Match splitting at levels 3-5 only: 1-2 are GHI, 6-7 keep every escape. 1131| | // search_depth, sufficient_len, use_lazy, lazy_attempts, lazy_len_threshold, step_base, 1132| | // step_shift, min_offset, split_max 1133| 4.90k| static const zxc_lz77_params_t table[7] = { 1134| 4.90k| {3, 16, 0, 0, 0, 4, 4, ZXC_LZ_MINDIST, 0}, // fallback ------------------ | | 629| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1135| 4.90k| {3, 16, 0, 0, 0, 4, 4, ZXC_LZ_MINDIST, 0}, // level 1 ------------------ | | 629| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1136| 4.90k| {3, 18, 0, 0, 0, 3, 6, ZXC_LZ_MINDIST, 0}, // level 2 ------------------ | | 629| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1137| 4.90k| {3, 16, 1, 4, 128, 1, 4, ZXC_LZ_MINDIST, 33}, // level 3 ------------------ | | 629| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1138| 4.90k| {3, 18, 1, 4, 128, 1, 5, ZXC_LZ_MINDIST, 24}, // level 4 ------------------ | | 629| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1139| 4.90k| {64, 256, 1, 16, 128, 1, 8, ZXC_LZ_MINDIST, 23}, // level 5 ------------------ | | 629| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1140| 4.90k| {64, 256, 0, 0, 0, 1, 8, 1, 0} // level 6 1141| 4.90k| }; 1142| 4.90k| return (level >= ZXC_LEVEL_ULTRA) ------------------ | Branch (1142:12): [True: 0, False: 4.90k] ------------------ 1143| 4.90k| ? (zxc_lz77_params_t){128, 256, 0, 0, 0, 1, 8, 1, 0} 1144| 4.90k| : table[level < ZXC_LEVEL_FASTEST ? ZXC_LEVEL_FASTEST : level]; ------------------ | Branch (1144:24): [True: 0, False: 4.90k] ------------------ 1145| 4.90k|} zxc_compress.c:zxc_block_is_short_dist_bound: 1286| 4.90k| const size_t size) { 1287| 4.90k| const size_t d_max = ZXC_LZ_MINDIST; ------------------ | | 629| 4.90k|#define ZXC_LZ_MINDIST 32 ------------------ 1288| | // Too small to sample, and too small to gain: keep every distance. 1289| 4.90k| if (size < d_max + sizeof(uint32_t)) return 1; ------------------ | Branch (1289:9): [True: 377, False: 4.53k] ------------------ 1290| | 1291| 4.53k| size_t want = size / ZXC_LZ_MINDIST_PROBE_PER_KB; ------------------ | | 636| 4.53k|#define ZXC_LZ_MINDIST_PROBE_PER_KB 1024 ------------------ 1292| 4.53k| if (want < ZXC_LZ_MINDIST_PROBE_MIN) want = ZXC_LZ_MINDIST_PROBE_MIN; ------------------ | | 637| 4.53k|#define ZXC_LZ_MINDIST_PROBE_MIN 16 ------------------ if (want < ZXC_LZ_MINDIST_PROBE_MIN) want = ZXC_LZ_MINDIST_PROBE_MIN; ------------------ | | 637| 2.82k|#define ZXC_LZ_MINDIST_PROBE_MIN 16 ------------------ | Branch (1292:9): [True: 2.82k, False: 1.70k] ------------------ 1293| 4.53k| if (want > ZXC_LZ_MINDIST_PROBE_MAX) want = ZXC_LZ_MINDIST_PROBE_MAX; ------------------ | | 638| 4.53k|#define ZXC_LZ_MINDIST_PROBE_MAX 64 ------------------ if (want > ZXC_LZ_MINDIST_PROBE_MAX) want = ZXC_LZ_MINDIST_PROBE_MAX; ------------------ | | 638| 1.34k|#define ZXC_LZ_MINDIST_PROBE_MAX 64 ------------------ | Branch (1293:9): [True: 1.34k, False: 3.18k] ------------------ 1294| 4.53k| size_t stride = size / want; 1295| 4.53k| if (stride < d_max) stride = d_max; ------------------ | Branch (1295:9): [True: 840, False: 3.69k] ------------------ 1296| | 1297| 4.53k| const size_t planned = (size - sizeof(uint32_t) - d_max) / stride + 1; 1298| 4.53k| const size_t bar = (size_t)ZXC_LZ_MINDIST_MAX_SHORT_PCT * planned; ------------------ | | 646| 4.53k|#define ZXC_LZ_MINDIST_MAX_SHORT_PCT 20 ------------------ 1299| | 1300| 4.53k| size_t samples = 0; 1301| 4.53k| size_t hits = 0; 1302| 70.6k| for (size_t i = d_max; i + sizeof(uint32_t) <= size; i += stride) { ------------------ | Branch (1302:28): [True: 70.6k, False: 0] ------------------ 1303| 70.6k| const uint32_t cur = zxc_le32(blk + i); 1304| 70.6k| samples++; 1305| 1.50M| for (size_t d = 1; d < d_max; d++) { ------------------ | Branch (1305:28): [True: 1.45M, False: 44.7k] ------------------ 1306| 1.45M| if (zxc_le32(blk + i - d) == cur) { ------------------ | Branch (1306:17): [True: 25.9k, False: 1.43M] ------------------ 1307| 25.9k| hits++; 1308| 25.9k| break; 1309| 25.9k| } 1310| 1.45M| } 1311| 70.6k| if (hits * 100U > bar) return 1; // can only rise ------------------ | Branch (1311:13): [True: 3.66k, False: 67.0k] ------------------ 1312| 67.0k| if ((hits + (planned - samples)) * 100U <= bar) return 0; // out of reach ------------------ | Branch (1312:13): [True: 870, False: 66.1k] ------------------ 1313| 67.0k| } 1314| 0| return hits * 100U > bar; 1315| 4.53k|} zxc_compress.c:zxc_le32: 1263| 8.90M|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1264| 8.90M| uint32_t v; 1265| 8.90M| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 8.90M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1266| |#ifdef ZXC_BIG_ENDIAN 1267| | return ZXC_BSWAP32(v); 1268| |#else 1269| 8.90M| return v; 1270| 8.90M|#endif 1271| 8.90M|} zxc_compress.c:zxc_le64: 1326| 32.7M|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1327| 32.7M| uint64_t v; 1328| 32.7M| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 32.7M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1329| |#ifdef ZXC_BIG_ENDIAN 1330| | return ZXC_BSWAP64(v); 1331| |#else 1332| 32.7M| return v; 1333| 32.7M|#endif 1334| 32.7M|} zxc_compress.c:zxc_ctz64: 1519| 3.47M|static ZXC_ALWAYS_INLINE int zxc_ctz64(const uint64_t x) { 1520| 3.47M| if (x == 0) return 64; ------------------ | Branch (1520:9): [True: 0, False: 3.47M] ------------------ 1521| 3.47M|#if defined(__GNUC__) || defined(__clang__) 1522| 3.47M| return __builtin_ctzll(x); 1523| |#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) 1524| | unsigned long r; 1525| | _BitScanForward64(&r, x); 1526| | return (int)r; 1527| |#elif defined(_MSC_VER) 1528| | // Use two 32-bit scans to avoid fragile 64-bit De Bruijn multiplication. 1529| | unsigned long r; 1530| | const uint32_t lo = (uint32_t)x; 1531| | if (_BitScanForward(&r, lo)) return (int)r; 1532| | _BitScanForward(&r, (uint32_t)(x >> 32)); 1533| | return 32 + (int)r; 1534| |#else 1535| | // Fallback De Bruijn for non-GCC/non-MSVC compilers 1536| | static const int Debruijn64[64] = { 1537| | 0, 1, 48, 2, 57, 49, 28, 3, 61, 58, 50, 42, 38, 29, 17, 4, 62, 55, 59, 36, 53, 51, 1538| | 43, 22, 45, 39, 33, 30, 24, 18, 12, 5, 63, 47, 56, 27, 60, 41, 37, 16, 54, 35, 52, 21, 1539| | 44, 32, 23, 11, 46, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6}; 1540| | return Debruijn64[((x & (0ULL - x)) * 0x03F79D71B4CA8B09ULL) >> 58]; 1541| |#endif 1542| 3.47M|} zxc_compress.c:zxc_ctz32: 1493| 250k|static ZXC_ALWAYS_INLINE int zxc_ctz32(const uint32_t x) { 1494| 250k| if (x == 0) return 32; ------------------ | Branch (1494:9): [True: 0, False: 250k] ------------------ 1495| 250k|#if defined(__GNUC__) || defined(__clang__) 1496| 250k| return __builtin_ctz(x); 1497| |#elif defined(_MSC_VER) 1498| | unsigned long r; 1499| | _BitScanForward(&r, x); 1500| | return (int)r; 1501| |#else 1502| | // Fallback De Bruijn (32 bits) 1503| | static const int DeBruijn32[32] = {0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 1504| | 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 1505| | 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; 1506| | return DeBruijn32[((uint32_t)((x & (0U - x)) * 0x077CB531U)) >> 27]; 1507| |#endif 1508| 250k|} zxc_compress.c:zxc_copy32: 1465| 1.08M|static ZXC_ALWAYS_INLINE void zxc_copy32(void* dst, const void* src) { 1466| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 1467| | // AVX2/AVX512: Single 256-bit (32 byte) unaligned load/store 1468| | _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src)); 1469| |#elif defined(ZXC_USE_SSE2) 1470| | const __m128i a = _mm_loadu_si128((const __m128i*)src); 1471| 1.08M| const __m128i b = _mm_loadu_si128((const __m128i*)((const uint8_t*)src + 16)); 1472| 1.08M| _mm_storeu_si128((__m128i*)dst, a); 1473| 1.08M| _mm_storeu_si128((__m128i*)((uint8_t*)dst + 16), b); 1474| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1475| | const uint8x16_t a = vld1q_u8((const uint8_t*)src); 1476| | const uint8x16_t b = vld1q_u8((const uint8_t*)src + 16); 1477| | vst1q_u8((uint8_t*)dst, a); 1478| | vst1q_u8((uint8_t*)dst + 16, b); 1479| |#else 1480| | ZXC_MEMCPY(dst, src, 32); 1481| |#endif 1482| 1.08M|} zxc_compress.c:zxc_store_le32: 1363| 1.63k|static ZXC_ALWAYS_INLINE void zxc_store_le32(void* p, const uint32_t v) { 1364| |#ifdef ZXC_BIG_ENDIAN 1365| | const uint32_t s = ZXC_BSWAP32(v); 1366| | ZXC_MEMCPY(p, &s, sizeof(s)); 1367| |#else 1368| 1.63k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 167| 1.63k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1369| 1.63k|#endif 1370| 1.63k|} zxc_compress.c:zxc_checksum: 1579| 1.63k| const uint64_t seed, const uint8_t hash_method) { 1580| 1.63k| (void)hash_method; /* single algorithm for now; extend when adding more */ 1581| 1.63k| const uint64_t hash = rapidhash_withSeed(input, len, seed); 1582| | 1583| | return (uint32_t)(hash ^ (hash >> (sizeof(uint32_t) * CHAR_BIT))); 1584| 1.63k|} zxc_decompress.c:zxc_le32: 1263| 2.63M|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1264| 2.63M| uint32_t v; 1265| 2.63M| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 2.63M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1266| |#ifdef ZXC_BIG_ENDIAN 1267| | return ZXC_BSWAP32(v); 1268| |#else 1269| 2.63M| return v; 1270| 2.63M|#endif 1271| 2.63M|} zxc_decompress.c:zxc_copy32: 1465| 13.3M|static ZXC_ALWAYS_INLINE void zxc_copy32(void* dst, const void* src) { 1466| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 1467| | // AVX2/AVX512: Single 256-bit (32 byte) unaligned load/store 1468| | _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src)); 1469| |#elif defined(ZXC_USE_SSE2) 1470| | const __m128i a = _mm_loadu_si128((const __m128i*)src); 1471| 13.3M| const __m128i b = _mm_loadu_si128((const __m128i*)((const uint8_t*)src + 16)); 1472| 13.3M| _mm_storeu_si128((__m128i*)dst, a); 1473| 13.3M| _mm_storeu_si128((__m128i*)((uint8_t*)dst + 16), b); 1474| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1475| | const uint8x16_t a = vld1q_u8((const uint8_t*)src); 1476| | const uint8x16_t b = vld1q_u8((const uint8_t*)src + 16); 1477| | vst1q_u8((uint8_t*)dst, a); 1478| | vst1q_u8((uint8_t*)dst + 16, b); 1479| |#else 1480| | ZXC_MEMCPY(dst, src, 32); 1481| |#endif 1482| 13.3M|} zxc_decompress.c:zxc_read_varint: 561| 104k|static ZXC_ALWAYS_INLINE uint32_t zxc_read_varint(const uint8_t** ptr, const uint8_t* end) { 562| 104k| const uint8_t* p = *ptr; 563| 104k| if (UNLIKELY(p >= end)) return 0; ------------------ | | 151| 104k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 104k] | | ------------------ ------------------ 564| | 565| 104k| const uint32_t b0 = p[0]; 566| | 567| | // 1 Byte: 0xxxxxxx (7 bits) -> val < 128 (2^7) 568| 104k| if (LIKELY(b0 < 0x80)) { ------------------ | | 145| 104k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 30.1k, False: 74.1k] | | ------------------ ------------------ 569| 30.1k| *ptr = p + 1; 570| 30.1k| return b0; 571| 30.1k| } 572| | 573| | // 2 Bytes: 10xxxxxx xxxxxxxx (14 bits) -> val < 16384 (2^14) 574| 74.1k| if (LIKELY(b0 < 0xC0)) { ------------------ | | 145| 74.1k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 71.4k, False: 2.70k] | | ------------------ ------------------ 575| 71.4k| if (UNLIKELY(p + 1 >= end)) { ------------------ | | 151| 71.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 71.4k] | | ------------------ ------------------ 576| 0| *ptr = end; 577| 0| return 0; 578| 0| } 579| 71.4k| *ptr = p + 2; 580| 71.4k| return (b0 & 0x3F) | ((uint32_t)p[1] << 6); 581| 71.4k| } 582| | 583| | // 3 Bytes: 110xxxxx xxxxxxxx xxxxxxxx (21 bits) -> val < 2^21. The longest 584| | // a legitimate varint can be: values are (ll - MASK) or (ml - MASK), always 585| | // strictly below block_size_max = 2^21. 586| 2.70k| if (LIKELY(b0 < 0xE0)) { ------------------ | | 145| 2.70k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 2.70k, False: 0] | | ------------------ ------------------ 587| 2.70k| if (UNLIKELY(p + 2 >= end)) { ------------------ | | 151| 2.70k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 2.70k] | | ------------------ ------------------ 588| 0| *ptr = end; 589| 0| return 0; 590| 0| } 591| 2.70k| *ptr = p + 3; 592| 2.70k| return (b0 & 0x1F) | ((uint32_t)p[1] << 5) | ((uint32_t)p[2] << 13); 593| 2.70k| } 594| | 595| | // extra encoding: out-of-spec for the current format, reject. 596| 0| *ptr = end; 597| 0| return 0; 598| 2.70k|} zxc_decompress.c:zxc_copy16: 1446| 2.25M|static ZXC_ALWAYS_INLINE void zxc_copy16(void* dst, const void* src) { 1447| 2.25M|#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) || defined(ZXC_USE_SSE2) 1448| | // x86 SSE2/AVX2/AVX512: Single 128-bit unaligned load/store 1449| 2.25M| _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); 1450| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1451| | vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); 1452| |#else 1453| | ZXC_MEMCPY(dst, src, 16); 1454| |#endif 1455| 2.25M|} zxc_decompress.c:zxc_checksum: 1579| 1.63k| const uint64_t seed, const uint8_t hash_method) { 1580| 1.63k| (void)hash_method; /* single algorithm for now; extend when adding more */ 1581| 1.63k| const uint64_t hash = rapidhash_withSeed(input, len, seed); 1582| | 1583| | return (uint32_t)(hash ^ (hash >> (sizeof(uint32_t) * CHAR_BIT))); 1584| 1.63k|} zxc_dispatch.c:zxc_level_clamp: 884| 9.51k|static inline int zxc_level_clamp(const int level) { 885| 9.51k| return (level > ZXC_LEVEL_ULTRA) ? ZXC_LEVEL_ULTRA : level; ------------------ | Branch (885:12): [True: 0, False: 9.51k] ------------------ 886| 9.51k|} zxc_dispatch.c:zxc_validate_block_size: 1057| 4.60k|static ZXC_ALWAYS_INLINE int zxc_validate_block_size(const size_t bs) { 1058| 4.60k| return bs >= ZXC_BLOCK_SIZE_MIN && bs <= ZXC_BLOCK_SIZE_MAX && (bs & (bs - 1)) == 0; ------------------ | | 62| 9.21k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 4.60k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ return bs >= ZXC_BLOCK_SIZE_MIN && bs <= ZXC_BLOCK_SIZE_MAX && (bs & (bs - 1)) == 0; ------------------ | | 64| 9.21k|#define ZXC_BLOCK_SIZE_MAX (1U << ZXC_BLOCK_SIZE_MAX_LOG2) | | ------------------ | | | | 58| 4.60k|#define ZXC_BLOCK_SIZE_MAX_LOG2 21 | | ------------------ ------------------ | Branch (1058:12): [True: 4.60k, False: 0] | Branch (1058:40): [True: 4.60k, False: 0] | Branch (1058:68): [True: 4.60k, False: 0] ------------------ 1059| 4.60k|} zxc_dispatch.c:zxc_block_size_ceil: 1038| 4.90k|static ZXC_ALWAYS_INLINE size_t zxc_block_size_ceil(const size_t v) { 1039| 4.90k| uint64_t x = (uint64_t)v - 1; 1040| 4.90k| x |= x >> 1; 1041| 4.90k| x |= x >> 2; 1042| 4.90k| x |= x >> 4; 1043| 4.90k| x |= x >> 8; 1044| 4.90k| x |= x >> 16; 1045| 4.90k| x |= x >> 32; 1046| 4.90k| x++; 1047| 4.90k| const size_t bs = (size_t)x; 1048| 4.90k| return (bs < ZXC_BLOCK_SIZE_MIN) ? ZXC_BLOCK_SIZE_MIN : bs; ------------------ | | 62| 4.90k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 4.90k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ return (bs < ZXC_BLOCK_SIZE_MIN) ? ZXC_BLOCK_SIZE_MIN : bs; ------------------ | | 62| 2.54k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 2.54k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ | Branch (1048:12): [True: 2.54k, False: 2.36k] ------------------ 1049| 4.90k|} zxc_pstream.c:zxc_level_clamp: 884| 4.60k|static inline int zxc_level_clamp(const int level) { 885| 4.60k| return (level > ZXC_LEVEL_ULTRA) ? ZXC_LEVEL_ULTRA : level; ------------------ | Branch (885:12): [True: 0, False: 4.60k] ------------------ 886| 4.60k|} zxc_pstream.c:zxc_digest_combine: 1599| 3.27k| const uint32_t block_checksum) { 1600| 3.27k| const uint64_t a = acc ^ ((uint64_t)block_checksum + 1U) * ZXC_HASH_GOLDEN64; ------------------ | | 676| 3.27k|#define ZXC_HASH_GOLDEN64 0x9E3779B97F4A7C15ULL ------------------ 1601| 3.27k| const uint64_t b = ZXC_HASH_XORSHIFT64; ------------------ | | 678| 3.27k|#define ZXC_HASH_XORSHIFT64 0x2545F4914F6CDD1DULL ------------------ 1602| 3.27k|#if defined(__SIZEOF_INT128__) 1603| 3.27k| const __uint128_t r = (__uint128_t)a * b; 1604| 3.27k| return (uint64_t)r ^ (uint64_t)(r >> 64); 1605| |#else 1606| | const uint64_t alo = (uint32_t)a, ahi = a >> 32, blo = (uint32_t)b, bhi = b >> 32; 1607| | const uint64_t lolo = alo * blo, lohi = alo * bhi, hilo = ahi * blo, hihi = ahi * bhi; 1608| | const uint64_t cross = (lolo >> 32) + (uint32_t)lohi + (uint32_t)hilo; 1609| | const uint64_t hi = hihi + (lohi >> 32) + (hilo >> 32) + (cross >> 32); 1610| | const uint64_t lo = (cross << 32) | (uint32_t)lolo; 1611| | return lo ^ hi; 1612| |#endif 1613| 3.27k|} zxc_pstream.c:zxc_le32: 1263| 3.27k|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1264| 3.27k| uint32_t v; 1265| 3.27k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 3.27k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1266| |#ifdef ZXC_BIG_ENDIAN 1267| | return ZXC_BSWAP32(v); 1268| |#else 1269| 3.27k| return v; 1270| 3.27k|#endif 1271| 3.27k|} zxc_pstream.c:zxc_footer_bytes: 2190| 7.85k|static ZXC_ALWAYS_INLINE size_t zxc_footer_bytes(const int checksum_enabled) { 2191| 7.85k| return ZXC_FILE_FOOTER_SIZE + (checksum_enabled ? (size_t)ZXC_FILE_DIGEST_SIZE : 0U); ------------------ | | 106| 7.85k|#define ZXC_FILE_FOOTER_SIZE 8 ------------------ return ZXC_FILE_FOOTER_SIZE + (checksum_enabled ? (size_t)ZXC_FILE_DIGEST_SIZE : 0U); ------------------ | | 108| 2.60k|#define ZXC_FILE_DIGEST_SIZE 8 ------------------ | Branch (2191:36): [True: 2.60k, False: 5.25k] ------------------ 2192| 7.85k|} zxc_pstream.c:zxc_le64: 1326| 4.16k|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1327| 4.16k| uint64_t v; 1328| 4.16k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 167| 4.16k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1329| |#ifdef ZXC_BIG_ENDIAN 1330| | return ZXC_BSWAP64(v); 1331| |#else 1332| 4.16k| return v; 1333| 4.16k|#endif 1334| 4.16k|} zxc_cstream_create: 262| 4.60k|zxc_cstream* zxc_cstream_create(const zxc_compress_opts_t* opts) { 263| 4.60k| zxc_cstream* cs = (zxc_cstream*)ZXC_CALLOC(1, sizeof(*cs)); ------------------ | | 73| 4.60k|#define ZXC_CALLOC(nmemb, size) calloc(nmemb, size) ------------------ 264| 4.60k| if (UNLIKELY(!cs)) return NULL; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 265| | 266| 4.60k| if (opts) cs->opts = *opts; ------------------ | Branch (266:9): [True: 4.60k, False: 0] ------------------ 267| | 268| | // Push streams take no dictionary yet (deferred until a need arises): the 269| | // header id stays 0, so a dictionary here would produce archives that 270| | // decode wrong elsewhere. 271| 4.60k| if (UNLIKELY(cs->opts.dict || cs->opts.dict_size || cs->opts.dict_huf)) { ------------------ | | 151| 18.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 272| 0| ZXC_FREE(cs); ------------------ | | 85| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 273| 0| return NULL; 274| 0| } 275| | 276| 4.60k| cs->opts.level = ZXC_OPTS_LEVEL(&cs->opts, ZXC_LEVEL_DEFAULT); ------------------ | | 899| 4.60k|#define ZXC_OPTS_LEVEL(o, dflt) zxc_level_clamp(((o) && (o)->level > 0) ? (o)->level : (dflt)) | | ------------------ | | | Branch (899:50): [True: 4.60k, False: 0] | | | Branch (899:57): [True: 4.60k, False: 0] | | ------------------ ------------------ 277| 4.60k| cs->opts.block_size = ZXC_OPTS_BLOCK_SIZE(&cs->opts, ZXC_BLOCK_SIZE_DEFAULT); ------------------ | | 901| 4.60k|#define ZXC_OPTS_BLOCK_SIZE(o, dflt) (((o) && (o)->block_size > 0) ? (o)->block_size : (dflt)) | | ------------------ | | | Branch (901:40): [True: 4.60k, False: 0] | | | Branch (901:47): [True: 0, False: 4.60k] | | ------------------ ------------------ 278| | // n_threads is ignored on this single-threaded path. 279| 4.60k| cs->opts.n_threads = 0; 280| 4.60k| cs->opts.progress_cb = NULL; 281| 4.60k| cs->opts.user_data = NULL; 282| 4.60k| cs->opts.seekable = 0; 283| 4.60k| cs->block_size = cs->opts.block_size; 284| | 285| 4.60k| cs->cctx = zxc_create_cctx(&cs->opts); 286| | // LCOV_EXCL_START 287| 4.60k| if (UNLIKELY(!cs->cctx)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 288| 0| ZXC_FREE(cs); ------------------ | | 85| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 289| 0| return NULL; 290| 0| } 291| 4.60k| cs->in_block = (uint8_t*)ZXC_MALLOC(cs->block_size); ------------------ | | 67| 4.60k|#define ZXC_MALLOC(size) malloc(size) ------------------ 292| 4.60k| if (UNLIKELY(!cs->in_block)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 293| 0| zxc_free_cctx(cs->cctx); 294| 0| ZXC_FREE(cs); ------------------ | | 85| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 295| 0| return NULL; 296| 0| } 297| | // LCOV_EXCL_STOP 298| | // Pre-size pending so the file header path never needs realloc. 299| 4.60k| cs->pending_cap = 300| 4.60k| ZXC_FILE_HEADER_SIZE > ZXC_FILE_FOOTER_SIZE ? ZXC_FILE_HEADER_SIZE : ZXC_FILE_FOOTER_SIZE; ------------------ | | 104| 4.60k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ ZXC_FILE_HEADER_SIZE > ZXC_FILE_FOOTER_SIZE ? ZXC_FILE_HEADER_SIZE : ZXC_FILE_FOOTER_SIZE; ------------------ | | 106| 4.60k|#define ZXC_FILE_FOOTER_SIZE 8 ------------------ ZXC_FILE_HEADER_SIZE > ZXC_FILE_FOOTER_SIZE ? ZXC_FILE_HEADER_SIZE : ZXC_FILE_FOOTER_SIZE; ------------------ | | 104| 4.60k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ ZXC_FILE_HEADER_SIZE > ZXC_FILE_FOOTER_SIZE ? ZXC_FILE_HEADER_SIZE : ZXC_FILE_FOOTER_SIZE; ------------------ | | 106| 4.60k|#define ZXC_FILE_FOOTER_SIZE 8 ------------------ | Branch (300:9): [True: 4.60k, Folded] ------------------ 301| 4.60k| cs->pending = (uint8_t*)ZXC_MALLOC(cs->pending_cap); ------------------ | | 67| 4.60k|#define ZXC_MALLOC(size) malloc(size) ------------------ 302| | // LCOV_EXCL_START 303| 4.60k| if (UNLIKELY(!cs->pending)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 304| 0| ZXC_FREE(cs->in_block); ------------------ | | 85| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 305| 0| zxc_free_cctx(cs->cctx); 306| 0| ZXC_FREE(cs); ------------------ | | 85| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 307| 0| return NULL; 308| 0| } 309| | // LCOV_EXCL_STOP 310| 4.60k| cs->state = CS_INIT; 311| 4.60k| return cs; 312| 4.60k|} zxc_cstream_free: 387| 4.60k|void zxc_cstream_free(zxc_cstream* cs) { 388| 4.60k| if (!cs) return; ------------------ | Branch (388:9): [True: 0, False: 4.60k] ------------------ 389| 4.60k| ZXC_FREE(cs->pending); ------------------ | | 85| 4.60k|#define ZXC_FREE(ptr) free(ptr) ------------------ 390| 4.60k| ZXC_FREE(cs->in_block); ------------------ | | 85| 4.60k|#define ZXC_FREE(ptr) free(ptr) ------------------ 391| 4.60k| zxc_free_cctx(cs->cctx); 392| 4.60k| ZXC_FREE(cs); ------------------ | | 85| 4.60k|#define ZXC_FREE(ptr) free(ptr) ------------------ 393| 4.60k|} zxc_cstream_compress: 426| 109M|int64_t zxc_cstream_compress(zxc_cstream* cs, zxc_outbuf_t* out, zxc_inbuf_t* in) { 427| 109M| if (UNLIKELY(!cs || !out || !in || in->pos > in->size || out->pos > out->size || ------------------ | | 151| 1.96G|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 109M, False: 84.1k] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 109M, False: 0] | | | Branch (151:42): [True: 0, False: 109M] | | | Branch (151:42): [True: 0, False: 109M] | | ------------------ ------------------ 428| 109M| (in->size > in->pos && !in->src) || (out->size > out->pos && !out->dst) || 429| 109M| cs->state == CS_DONE)) { 430| 0| return ZXC_ERROR_NULL_INPUT; 431| 0| } 432| 109M| if (UNLIKELY(cs->state == CS_ERRORED)) return cs->error_code; ------------------ | | 151| 109M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 109M] | | ------------------ ------------------ 433| | 434| 109M| for (;;) { 435| 109M| switch (cs->state) { ------------------ | Branch (435:17): [True: 109M, False: 0] ------------------ 436| 4.60k| case CS_INIT: { ------------------ | Branch (436:13): [True: 4.60k, False: 109M] ------------------ 437| 4.60k| const int rc = cs_stage_file_header(cs); 438| 4.60k| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 439| 4.60k| cs->state = CS_DRAIN_HEADER; 440| 4.60k| break; 441| 4.60k| } 442| | 443| 4.60k| case CS_DRAIN_HEADER: ------------------ | Branch (443:13): [True: 4.60k, False: 109M] ------------------ 444| 323k| case CS_DRAIN_BLOCK: { ------------------ | Branch (444:13): [True: 319k, False: 109M] ------------------ 445| 323k| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (445:21): [True: 318k, False: 4.91k] ------------------ 446| 4.91k| cs->state = CS_ACCUMULATE; 447| 4.91k| break; 448| 323k| } 449| | 450| 109M| case CS_ACCUMULATE: { ------------------ | Branch (450:13): [True: 109M, False: 328k] ------------------ 451| 109M| const size_t avail_in = in->size - in->pos; 452| 109M| if (cs->in_used == 0 && avail_in >= cs->block_size) { ------------------ | Branch (452:21): [True: 4.99k, False: 109M] | Branch (452:41): [True: 0, False: 4.99k] ------------------ 453| 0| const int rc = cs_compress_block_from(cs, (const uint8_t*)in->src + in->pos, 454| 0| cs->block_size); 455| 0| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 456| 0| in->pos += cs->block_size; 457| 0| cs->state = CS_DRAIN_BLOCK; 458| 0| break; 459| 0| } 460| | 461| 109M| const size_t room = cs->block_size - cs->in_used; 462| 109M| const size_t n = avail_in < room ? avail_in : room; ------------------ | Branch (462:34): [True: 109M, False: 313] ------------------ 463| 109M| if (n) { ------------------ | Branch (463:21): [True: 109M, False: 90] ------------------ 464| 109M| ZXC_MEMCPY(cs->in_block + cs->in_used, (const uint8_t*)in->src + in->pos, n); ------------------ | | 167| 109M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 465| 109M| in->pos += n; 466| 109M| cs->in_used += n; 467| 109M| } 468| | 469| 109M| if (cs->in_used == cs->block_size) { ------------------ | Branch (469:21): [True: 313, False: 109M] ------------------ 470| 313| const int rc = cs_compress_one_block(cs); 471| 313| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 313|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 313] | | ------------------ ------------------ 472| 313| cs->state = CS_DRAIN_BLOCK; 473| 313| break; 474| 313| } 475| | // Block not yet full either in is empty or we made no progress. 476| 109M| return 0; 477| 109M| } 478| | 479| 0| case CS_DRAIN_LAST: ------------------ | Branch (479:13): [True: 0, False: 109M] ------------------ 480| 0| case CS_DRAIN_EOF: ------------------ | Branch (480:13): [True: 0, False: 109M] ------------------ 481| 0| case CS_DRAIN_FOOTER: ------------------ | Branch (481:13): [True: 0, False: 109M] ------------------ 482| 0| case CS_DONE: ------------------ | Branch (482:13): [True: 0, False: 109M] ------------------ 483| 0| case CS_ERRORED: ------------------ | Branch (483:13): [True: 0, False: 109M] ------------------ 484| | // These states are owned by _end(). 485| 0| return ZXC_ERROR_NULL_INPUT; 486| 109M| } 487| 109M| } 488| 109M|} zxc_cstream_end: 499| 512k|int64_t zxc_cstream_end(zxc_cstream* cs, zxc_outbuf_t* out) { 500| 512k| if (UNLIKELY(!cs || !out || cs->state == CS_DONE)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 2.05M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 512k] | | | Branch (151:42): [True: 0, False: 512k] | | | Branch (151:42): [True: 0, False: 512k] | | | Branch (151:42): [True: 0, False: 512k] | | ------------------ ------------------ 501| 512k| if (UNLIKELY(cs->state == CS_ERRORED)) return cs->error_code; ------------------ | | 151| 512k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 512k] | | ------------------ ------------------ 502| | 503| 526k| for (;;) { 504| 526k| switch (cs->state) { ------------------ | Branch (504:17): [True: 526k, False: 0] ------------------ 505| 0| case CS_INIT: { ------------------ | Branch (505:13): [True: 0, False: 526k] ------------------ 506| | // _end before any input, still need to emit file header. 507| 0| const int rc = cs_stage_file_header(cs); 508| 0| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 509| 0| cs->state = CS_DRAIN_HEADER; 510| 0| break; 511| 0| } 512| | 513| 0| case CS_DRAIN_HEADER: ------------------ | Branch (513:13): [True: 0, False: 526k] ------------------ 514| 0| case CS_DRAIN_BLOCK: { ------------------ | Branch (514:13): [True: 0, False: 526k] ------------------ 515| 0| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (515:21): [True: 0, False: 0] ------------------ 516| 0| cs->state = CS_ACCUMULATE; 517| 0| break; 518| 0| } 519| | 520| 4.60k| case CS_ACCUMULATE: { ------------------ | Branch (520:13): [True: 4.60k, False: 521k] ------------------ 521| | // Compress the residual partial block (if any), then EOF + footer. 522| 4.60k| if (cs->in_used > 0) { ------------------ | Branch (522:21): [True: 4.59k, False: 10] ------------------ 523| 4.59k| const int rc = cs_compress_one_block(cs); 524| 4.59k| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 4.59k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.59k] | | ------------------ ------------------ 525| 4.59k| cs->state = CS_DRAIN_LAST; 526| 4.59k| break; 527| 4.59k| } 528| | // No residual data: go straight to EOF. 529| 10| { 530| 10| const int rc = cs_stage_eof(cs); 531| 10| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 10|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 10] | | ------------------ ------------------ 532| 10| cs->state = CS_DRAIN_EOF; 533| 10| break; 534| 10| } 535| 10| } 536| | 537| 512k| case CS_DRAIN_LAST: { ------------------ | Branch (537:13): [True: 512k, False: 14.0k] ------------------ 538| 512k| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (538:21): [True: 507k, False: 4.59k] ------------------ 539| | // After last data block -> EOF. 540| 4.59k| const int rc = cs_stage_eof(cs); 541| 4.59k| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 4.59k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.59k] | | ------------------ ------------------ 542| 4.59k| cs->state = CS_DRAIN_EOF; 543| 4.59k| break; 544| 4.59k| } 545| | 546| 4.71k| case CS_DRAIN_EOF: { ------------------ | Branch (546:13): [True: 4.71k, False: 521k] ------------------ 547| 4.71k| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (547:21): [True: 106, False: 4.60k] ------------------ 548| 4.60k| const int rc = cs_stage_footer(cs); 549| 4.60k| if (UNLIKELY(rc < 0)) return cs_set_error(cs, rc); // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 550| 4.60k| cs->state = CS_DRAIN_FOOTER; 551| 4.60k| break; 552| 4.60k| } 553| | 554| 4.78k| case CS_DRAIN_FOOTER: { ------------------ | Branch (554:13): [True: 4.78k, False: 521k] ------------------ 555| 4.78k| if (!cs_drain_pending(cs, out)) return (int64_t)(cs->pending_len - cs->pending_pos); ------------------ | Branch (555:21): [True: 174, False: 4.60k] ------------------ 556| 4.60k| cs->state = CS_DONE; 557| 4.60k| return 0; 558| 4.78k| } 559| | 560| 0| case CS_DONE: ------------------ | Branch (560:13): [True: 0, False: 526k] ------------------ 561| 0| case CS_ERRORED: ------------------ | Branch (561:13): [True: 0, False: 526k] ------------------ 562| 0| return cs->state == CS_ERRORED ? cs->error_code : 0; // LCOV_EXCL_LINE ------------------ | Branch (562:24): [True: 0, False: 0] ------------------ 563| 526k| } 564| 526k| } 565| 512k|} zxc_dstream_create: 763| 9.21k|zxc_dstream* zxc_dstream_create(const zxc_decompress_opts_t* opts) { 764| 9.21k| zxc_dstream* ds = (zxc_dstream*)ZXC_CALLOC(1, sizeof(*ds)); ------------------ | | 73| 9.21k|#define ZXC_CALLOC(nmemb, size) calloc(nmemb, size) ------------------ 765| 9.21k| if (UNLIKELY(!ds)) return NULL; // LCOV_EXCL_LINE ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 766| 9.21k| if (opts) ds->opts = *opts; ------------------ | Branch (766:9): [True: 9.21k, False: 0] ------------------ 767| 9.21k| if (UNLIKELY(ds->opts.dict || ds->opts.dict_size || ds->opts.dict_huf)) { ------------------ | | 151| 36.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | | Branch (151:42): [True: 0, False: 9.21k] | | ------------------ ------------------ 768| 0| ZXC_FREE(ds); ------------------ | | 85| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 769| 0| return NULL; 770| 0| } 771| 9.21k| ds->opts.n_threads = 0; 772| 9.21k| ds->opts.progress_cb = NULL; 773| 9.21k| ds->opts.user_data = NULL; 774| 9.21k| ds->state = DS_NEED_FILE_HEADER; 775| 9.21k| ds->scratch_need = ZXC_FILE_HEADER_SIZE; ------------------ | | 104| 9.21k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ 776| 9.21k| return ds; 777| 9.21k|} zxc_dstream_free: 784| 9.21k|void zxc_dstream_free(zxc_dstream* ds) { 785| 9.21k| if (UNLIKELY(!ds)) return; // LCOV_EXCL_LINE ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.21k] | | ------------------ ------------------ 786| 9.21k| ZXC_FREE(ds->payload); ------------------ | | 85| 9.21k|#define ZXC_FREE(ptr) free(ptr) ------------------ 787| 9.21k| ZXC_FREE(ds->decoded); ------------------ | | 85| 9.21k|#define ZXC_FREE(ptr) free(ptr) ------------------ 788| 9.21k| if (ds->inner_initialized) zxc_cctx_free(&ds->inner); ------------------ | Branch (788:9): [True: 4.60k, False: 4.60k] ------------------ 789| 9.21k| ZXC_FREE(ds); ------------------ | | 85| 9.21k|#define ZXC_FREE(ptr) free(ptr) ------------------ 790| 9.21k|} zxc_dstream_finished: 795| 3.14k|int zxc_dstream_finished(const zxc_dstream* ds) { return (ds && ds->state == DS_DONE) ? 1 : 0; } ------------------ | Branch (795:59): [True: 3.14k, False: 0] | Branch (795:65): [True: 3.14k, False: 0] ------------------ zxc_dstream_decompress: 965| 42.7M|int64_t zxc_dstream_decompress(zxc_dstream* ds, zxc_outbuf_t* out, zxc_inbuf_t* in) { 966| 42.7M| if (UNLIKELY(!ds || !out || !in || in->pos > in->size || out->pos > out->size || ------------------ | | 151| 683M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 42.7M, False: 0] | | | Branch (151:42): [True: 0, False: 42.7M] | | | Branch (151:42): [True: 42.7M, False: 0] | | | Branch (151:42): [True: 0, False: 42.7M] | | ------------------ ------------------ 967| 42.7M| (in->size > in->pos && !in->src) || (out->size > out->pos && !out->dst))) { 968| 0| return ZXC_ERROR_NULL_INPUT; 969| 0| } 970| 42.7M| if (UNLIKELY(ds->state == DS_ERRORED)) return ds->error_code; ------------------ | | 151| 42.7M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 42.7M] | | ------------------ ------------------ 971| 42.7M| if (UNLIKELY(ds->state == DS_DONE)) return 0; ------------------ | | 151| 42.7M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 42.7M] | | ------------------ ------------------ 972| | 973| 42.7M| size_t produced = 0; 974| | 975| 42.7M| for (;;) { 976| 42.7M| switch (ds->state) { ------------------ | Branch (976:17): [True: 42.7M, False: 0] ------------------ 977| 47.9k| case DS_NEED_FILE_HEADER: { ------------------ | Branch (977:13): [True: 47.9k, False: 42.7M] ------------------ 978| 47.9k| const int rc = ds_handle_need_file_header(ds, in); 979| 47.9k| if (rc == 1) return (int64_t)produced; ------------------ | Branch (979:21): [True: 38.8k, False: 9.10k] ------------------ 980| 9.10k| if (rc < 0) return rc; ------------------ | Branch (980:21): [True: 4.50k, False: 4.60k] ------------------ 981| 4.60k| break; 982| 9.10k| } 983| | 984| 18.7k| case DS_NEED_BLOCK_HEADER: { ------------------ | Branch (984:13): [True: 18.7k, False: 42.7M] ------------------ 985| 18.7k| const int rc = ds_handle_need_block_header(ds, in); 986| 18.7k| if (rc == 1) return (int64_t)produced; ------------------ | Branch (986:21): [True: 10.6k, False: 8.15k] ------------------ 987| 8.15k| if (rc < 0) return rc; ------------------ | Branch (987:21): [True: 0, False: 8.15k] ------------------ 988| 8.15k| break; 989| 8.15k| } 990| | 991| 40.9M| case DS_NEED_BLOCK_PAYLOAD: { ------------------ | Branch (991:13): [True: 40.9M, False: 1.78M] ------------------ 992| 40.9M| if (!ds_pull(ds->payload, &ds->payload_used, ds->payload_need, in)) ------------------ | Branch (992:21): [True: 40.9M, False: 4.90k] ------------------ 993| 40.9M| return (int64_t)produced; 994| 4.90k| ds->state = DS_DECODE_BLOCK; 995| 4.90k| break; 996| 40.9M| } 997| | 998| 4.90k| case DS_DECODE_BLOCK: { ------------------ | Branch (998:13): [True: 4.90k, False: 42.7M] ------------------ 999| 4.90k| const int direct = (out->size - out->pos) >= ds->decoded_cap; 1000| 4.90k| uint8_t* const ddst = direct ? (uint8_t*)out->dst + out->pos : ds->decoded; ------------------ | Branch (1000:39): [True: 0, False: 4.90k] ------------------ 1001| 4.90k| const int dsz = 1002| 4.90k| zxc_decompress_chunk_wrapper(&ds->inner, ds->payload, ds->payload_used, ddst, 1003| 4.90k| ds->decoded_cap, ds->block_index); 1004| 4.90k| if (UNLIKELY(dsz < 0)) return ds_set_error(ds, dsz); ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 1005| 4.90k| ds->block_index++; 1006| | // Folded only when DS_VALIDATE_FOOTER will compare it. 1007| 4.90k| if (ds->file_has_checksum && ds->opts.checksum_enabled) ------------------ | Branch (1007:21): [True: 1.63k, False: 3.27k] | Branch (1007:46): [True: 1.63k, False: 0] ------------------ 1008| 1.63k| ds->digest = zxc_digest_combine( 1009| 1.63k| ds->digest, 1010| 1.63k| zxc_le32(ds->payload + ds->payload_used - ZXC_BLOCK_CHECKSUM_SIZE)); ------------------ | | 409| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ 1011| | 1012| 4.90k| if (direct) { ------------------ | Branch (1012:21): [True: 0, False: 4.90k] ------------------ 1013| 0| out->pos += (size_t)dsz; 1014| 0| produced += (size_t)dsz; 1015| 0| ds->total_out += (size_t)dsz; 1016| 0| ds->decoded_size = 0; 1017| 0| ds->decoded_pos = 0; 1018| 0| ds->state = DS_NEED_BLOCK_HEADER; 1019| 0| ds->scratch_used = 0; 1020| 0| ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 407| 0|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1021| 0| break; 1022| 0| } 1023| | 1024| 4.90k| ds->decoded_size = (size_t)dsz; 1025| 4.90k| ds->decoded_pos = 0; 1026| 4.90k| ds->state = DS_EMIT_DECODED; 1027| 4.90k| break; 1028| 4.90k| } 1029| | 1030| 1.70M| case DS_EMIT_DECODED: { ------------------ | Branch (1030:13): [True: 1.70M, False: 41.0M] ------------------ 1031| 1.70M| const int done = ds_drain_decoded(ds, out, &produced); 1032| 1.70M| if (!done) return (int64_t)produced; ------------------ | Branch (1032:21): [True: 1.70M, False: 4.90k] ------------------ 1033| 4.90k| ds->state = DS_NEED_BLOCK_HEADER; 1034| 4.90k| ds->scratch_used = 0; 1035| 4.90k| ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 407| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1036| 4.90k| break; 1037| 1.70M| } 1038| | 1039| 0| case DS_NEED_SEK_HEADER: { ------------------ | Branch (1039:13): [True: 0, False: 42.7M] ------------------ 1040| 0| if (!ds_pull(ds->scratch, &ds->scratch_used, ds->scratch_need, in)) ------------------ | Branch (1040:21): [True: 0, False: 0] ------------------ 1041| 0| return (int64_t)produced; 1042| 0| if (UNLIKELY(!zxc_seek_header_ok(ds->scratch, ds->total_out, ds->block_size, ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1043| 0| &ds->sek_remaining))) 1044| 0| return ds_set_error(ds, ZXC_ERROR_CORRUPT_DATA); 1045| 0| ds->state = DS_DRAIN_SEK_PAYLOAD; 1046| 0| break; 1047| 0| } 1048| | 1049| 0| case DS_DRAIN_SEK_PAYLOAD: { ------------------ | Branch (1049:13): [True: 0, False: 42.7M] ------------------ 1050| 0| const size_t avail = in->size - in->pos; 1051| 0| const size_t n = 1052| 0| (uint64_t)avail < ds->sek_remaining ? avail : (size_t)ds->sek_remaining; ------------------ | Branch (1052:21): [True: 0, False: 0] ------------------ 1053| 0| in->pos += n; 1054| 0| ds->sek_remaining -= n; 1055| 0| if (ds->sek_remaining > 0) return (int64_t)produced; ------------------ | Branch (1055:21): [True: 0, False: 0] ------------------ 1056| 0| ds->state = DS_NEED_FOOTER; 1057| 0| ds->scratch_used = 0; 1058| 0| ds->scratch_need = zxc_footer_bytes(ds->file_has_checksum); 1059| 0| break; 1060| 0| } 1061| | 1062| 3.25k| case DS_NEED_FOOTER: { ------------------ | Branch (1062:13): [True: 3.25k, False: 42.7M] ------------------ 1063| 3.25k| if (!ds_pull(ds->scratch, &ds->scratch_used, ds->scratch_need, in)) ------------------ | Branch (1063:21): [True: 102, False: 3.14k] ------------------ 1064| 102| return (int64_t)produced; 1065| 3.14k| ds->state = DS_VALIDATE_FOOTER; 1066| 3.14k| break; 1067| 3.25k| } 1068| | 1069| 3.14k| case DS_VALIDATE_FOOTER: { ------------------ | Branch (1069:13): [True: 3.14k, False: 42.7M] ------------------ 1070| 3.14k| if (UNLIKELY(zxc_le64(ds->scratch) != ds->total_out)) ------------------ | | 151| 3.14k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 3.14k] | | ------------------ ------------------ 1071| 0| return ds_set_error(ds, ZXC_ERROR_CORRUPT_DATA); 1072| 3.14k| if (ds->file_has_checksum && ds->opts.checksum_enabled && ------------------ | Branch (1072:21): [True: 1.01k, False: 2.13k] | Branch (1072:46): [True: 1.01k, False: 0] ------------------ 1073| 1.01k| UNLIKELY(zxc_le64(ds->scratch + ZXC_FILE_FOOTER_SIZE) != ds->digest)) ------------------ | | 151| 1.01k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.01k] | | ------------------ ------------------ 1074| 0| return ds_set_error(ds, ZXC_ERROR_BAD_CHECKSUM); 1075| 3.14k| ds->state = DS_DONE; 1076| 3.14k| return (int64_t)produced; 1077| 3.14k| } 1078| | 1079| 0| case DS_DONE: ------------------ | Branch (1079:13): [True: 0, False: 42.7M] ------------------ 1080| 0| case DS_ERRORED: ------------------ | Branch (1080:13): [True: 0, False: 42.7M] ------------------ 1081| 0| return ds->state == DS_ERRORED ? ds->error_code : (int64_t)produced; ------------------ | Branch (1081:24): [True: 0, False: 0] ------------------ 1082| 42.7M| } 1083| 42.7M| } 1084| 42.7M|} zxc_pstream.c:cs_stage_file_header: 320| 4.60k|static int cs_stage_file_header(zxc_cstream* cs) { 321| 4.60k| const int w = zxc_write_file_header(cs->pending, cs->pending_cap, cs->block_size, 322| 4.60k| cs->opts.checksum_enabled, 0, 0); 323| 4.60k| if (UNLIKELY(w < 0)) return w; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 324| 4.60k| cs->pending_len = (size_t)w; 325| 4.60k| cs->pending_pos = 0; 326| 4.60k| return ZXC_OK; 327| 4.60k|} zxc_pstream.c:cs_drain_pending: 242| 845k|static int cs_drain_pending(zxc_cstream* cs, zxc_outbuf_t* out) { 243| 845k| zxc_outbuf_push(out, cs->pending, &cs->pending_pos, cs->pending_len); 244| 845k| return cs->pending_pos == cs->pending_len; 245| 845k|} zxc_pstream.c:zxc_outbuf_push: 229| 2.55M| const size_t len) { 230| 2.55M| const size_t avail_out = out->size - out->pos; 231| 2.55M| const size_t avail_src = len - *pos; 232| 2.55M| const size_t n = avail_out < avail_src ? avail_out : avail_src; ------------------ | Branch (232:22): [True: 2.52M, False: 23.6k] ------------------ 233| 2.55M| if (n) { ------------------ | Branch (233:9): [True: 2.55M, False: 71] ------------------ 234| 2.55M| ZXC_MEMCPY((uint8_t*)out->dst + out->pos, src + *pos, n); ------------------ | | 167| 2.55M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 235| 2.55M| out->pos += n; 236| 2.55M| *pos += n; 237| 2.55M| } 238| 2.55M| return n; 239| 2.55M|} zxc_pstream.c:cs_compress_block_from: 178| 4.90k|static int cs_compress_block_from(zxc_cstream* cs, const uint8_t* RESTRICT src, const size_t len) { 179| 4.90k| const uint64_t bound = zxc_compress_block_bound(len); 180| | // LCOV_EXCL_START 181| 4.90k| if (UNLIKELY(bound == 0 || bound > SIZE_MAX)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 9.81k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | | Branch (151:42): [True: 0, False: 4.90k] | | ------------------ ------------------ 182| 4.90k| if (UNLIKELY(bound > cs->pending_cap)) { ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 4.60k, False: 303] | | ------------------ ------------------ 183| 4.60k| uint8_t* nb = (uint8_t*)ZXC_REALLOC(cs->pending, (size_t)bound); ------------------ | | 79| 4.60k|#define ZXC_REALLOC(ptr, size) realloc(ptr, size) ------------------ 184| 4.60k| if (UNLIKELY(!nb)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 185| 4.60k| cs->pending = nb; 186| 4.60k| cs->pending_cap = (size_t)bound; 187| 4.60k| } 188| | // LCOV_EXCL_STOP 189| 4.90k| const int64_t csize = zxc_compress_block_at(cs->cctx, src, len, cs->pending, cs->pending_cap, 190| 4.90k| &cs->opts, cs->block_index); 191| 4.90k| if (UNLIKELY(csize < 0)) return (int)csize; // LCOV_EXCL_LINE ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 192| 4.90k| cs->block_index++; 193| | 194| 4.90k| cs->pending_len = (size_t)csize; 195| 4.90k| cs->pending_pos = 0; 196| 4.90k| cs->total_in += len; 197| 4.90k| if (cs->opts.checksum_enabled && cs->pending_len >= ZXC_BLOCK_CHECKSUM_SIZE) ------------------ | | 409| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (197:9): [True: 1.63k, False: 3.27k] | Branch (197:38): [True: 1.63k, False: 0] ------------------ 198| 1.63k| cs->digest = zxc_digest_combine( 199| 1.63k| cs->digest, zxc_le32(cs->pending + cs->pending_len - ZXC_BLOCK_CHECKSUM_SIZE)); ------------------ | | 409| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ 200| 4.90k| return ZXC_OK; 201| 4.90k|} zxc_pstream.c:cs_compress_one_block: 216| 4.90k|static int cs_compress_one_block(zxc_cstream* cs) { 217| 4.90k| const int rc = cs_compress_block_from(cs, cs->in_block, cs->in_used); 218| 4.90k| if (LIKELY(rc == ZXC_OK)) cs->in_used = 0; ------------------ | | 145| 4.90k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 4.90k, False: 0] | | ------------------ ------------------ 219| 4.90k| return rc; 220| 4.90k|} zxc_pstream.c:cs_stage_eof: 338| 4.60k|static int cs_stage_eof(zxc_cstream* cs) { 339| | // LCOV_EXCL_START 340| 4.60k| if (UNLIKELY(ZXC_BLOCK_HEADER_SIZE > cs->pending_cap)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 341| 0| uint8_t* nb = (uint8_t*)ZXC_REALLOC(cs->pending, ZXC_BLOCK_HEADER_SIZE); ------------------ | | 79| 0|#define ZXC_REALLOC(ptr, size) realloc(ptr, size) ------------------ 342| 0| if (UNLIKELY(!nb)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 343| 0| cs->pending = nb; 344| 0| cs->pending_cap = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 407| 0|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 345| 0| } 346| | // LCOV_EXCL_STOP 347| 4.60k| const zxc_block_header_t eof = { 348| 4.60k| .block_type = (uint8_t)ZXC_BLOCK_EOF, 349| 4.60k| .block_flags = 0, 350| 4.60k| .reserved = 0, 351| 4.60k| .header_checksum = 0, 352| 4.60k| .comp_size = 0, 353| 4.60k| }; 354| 4.60k| const int w = zxc_write_block_header(cs->pending, cs->pending_cap, &eof); 355| 4.60k| if (UNLIKELY(w < 0)) return w; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 356| 4.60k| cs->pending_len = (size_t)w; 357| 4.60k| cs->pending_pos = 0; 358| 4.60k| return ZXC_OK; 359| 4.60k|} zxc_pstream.c:cs_stage_footer: 369| 4.60k|static int cs_stage_footer(zxc_cstream* cs) { 370| 4.60k| const size_t footer_len = zxc_footer_bytes(cs->opts.checksum_enabled); 371| | // LCOV_EXCL_START 372| 4.60k| if (UNLIKELY(footer_len > cs->pending_cap)) { ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 373| 0| uint8_t* nb = (uint8_t*)ZXC_REALLOC(cs->pending, footer_len); ------------------ | | 79| 0|#define ZXC_REALLOC(ptr, size) realloc(ptr, size) ------------------ 374| 0| if (UNLIKELY(!nb)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 375| 0| cs->pending = nb; 376| 0| cs->pending_cap = footer_len; 377| 0| } 378| | // LCOV_EXCL_STOP 379| 4.60k| const int w = zxc_write_file_footer(cs->pending, cs->pending_cap, cs->total_in, cs->digest, 380| 4.60k| cs->opts.checksum_enabled); 381| 4.60k| if (UNLIKELY(w < 0)) return w; // LCOV_EXCL_LINE ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 382| 4.60k| cs->pending_len = (size_t)w; 383| 4.60k| cs->pending_pos = 0; 384| 4.60k| return ZXC_OK; 385| 4.60k|} zxc_pstream.c:ds_handle_need_file_header: 846| 47.9k|static int ds_handle_need_file_header(zxc_dstream* ds, zxc_inbuf_t* in) { 847| 47.9k| if (!ds_pull(ds->scratch, &ds->scratch_used, ds->scratch_need, in)) return 1; ------------------ | Branch (847:9): [True: 38.8k, False: 9.10k] ------------------ 848| | 849| 9.10k| size_t bs = 0; 850| 9.10k| int has_csum = 0; 851| 9.10k| int has_seek = 0; 852| 9.10k| uint32_t dict_id = 0; 853| 9.10k| const int rc = 854| 9.10k| zxc_read_file_header(ds->scratch, ds->scratch_used, &bs, &has_csum, &dict_id, &has_seek); 855| 9.10k| if (UNLIKELY(rc != ZXC_OK)) return ds_set_error(ds, rc); // LCOV_EXCL_LINE ------------------ | | 151| 9.10k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 4.50k, False: 4.60k] | | ------------------ ------------------ 856| | // Push streams take no dictionary yet, so an archive requiring one is refused. 857| 4.60k| if (UNLIKELY(dict_id != 0)) return ds_set_error(ds, ZXC_ERROR_DICT_REQUIRED); ------------------ | | 151| 4.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | ------------------ ------------------ 858| 4.60k| ds->block_size = bs; 859| 4.60k| ds->file_has_checksum = has_csum; 860| 4.60k| ds->file_has_seek = has_seek; 861| | 862| | // Allocate payload + decoded buffers now that block_size is known. 863| 4.60k| const uint64_t pb = zxc_compress_block_bound(ds->block_size); 864| | // LCOV_EXCL_START 865| 4.60k| if (UNLIKELY(pb == 0 || pb > SIZE_MAX)) return ds_set_error(ds, ZXC_ERROR_OVERFLOW); ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 866| | // LCOV_EXCL_STOP 867| 4.60k| ds->payload_cap = (size_t)pb; 868| 4.60k| ds->payload = (uint8_t*)ZXC_MALLOC(ds->payload_cap); ------------------ | | 67| 4.60k|#define ZXC_MALLOC(size) malloc(size) ------------------ 869| | 870| 4.60k| ds->decoded_cap = ds->block_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 354| 4.60k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 328| 4.60k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 871| 4.60k| ds->decoded = (uint8_t*)ZXC_MALLOC(ds->decoded_cap); ------------------ | | 67| 4.60k|#define ZXC_MALLOC(size) malloc(size) ------------------ 872| | // LCOV_EXCL_START 873| 4.60k| if (UNLIKELY(!ds->payload || !ds->decoded)) return ds_set_error(ds, ZXC_ERROR_MEMORY); ------------------ | | 151| 9.21k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 0, False: 4.60k] | | ------------------ ------------------ 874| | 875| 4.60k| if (UNLIKELY(zxc_cctx_init(&ds->inner, ds->block_size, 0, 0, ------------------ | | 151| 6.14k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.60k] | | | Branch (151:42): [True: 1.53k, False: 3.06k] | | | Branch (151:42): [True: 1.53k, False: 0] | | ------------------ ------------------ 876| 4.60k| ds->file_has_checksum && ds->opts.checksum_enabled, 0) != ZXC_OK)) { 877| 0| return ds_set_error(ds, ZXC_ERROR_MEMORY); 878| 0| } 879| | // LCOV_EXCL_STOP 880| 4.60k| ds->inner_initialized = 1; 881| | 882| 4.60k| ds->state = DS_NEED_BLOCK_HEADER; 883| 4.60k| ds->scratch_used = 0; 884| 4.60k| ds->scratch_need = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 407| 4.60k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 885| 4.60k| return 0; 886| 4.60k|} zxc_pstream.c:ds_handle_need_block_header: 904| 18.7k|static int ds_handle_need_block_header(zxc_dstream* ds, zxc_inbuf_t* in) { 905| 18.7k| if (!ds_pull(ds->scratch, &ds->scratch_used, ds->scratch_need, in)) return 1; ------------------ | Branch (905:9): [True: 10.6k, False: 8.15k] ------------------ 906| | 907| 8.15k| const int rc = zxc_read_block_header(ds->scratch, ds->scratch_used, &ds->cur_bh); 908| 8.15k| if (UNLIKELY(rc != ZXC_OK)) return ds_set_error(ds, rc); // LCOV_EXCL_LINE ------------------ | | 151| 8.15k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.15k] | | ------------------ ------------------ 909| | 910| 8.15k| if (ds->cur_bh.block_type == (uint8_t)ZXC_BLOCK_EOF) { ------------------ | Branch (910:9): [True: 3.25k, False: 4.90k] ------------------ 911| | // EOF block: comp_size must be 0; no payload, no checksum. 912| 3.25k| if (UNLIKELY(ds->cur_bh.comp_size != 0)) return ds_set_error(ds, ZXC_ERROR_BAD_BLOCK_SIZE); ------------------ | | 151| 3.25k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 3.25k] | | ------------------ ------------------ 913| 3.25k| ds->state = ds->file_has_seek ? DS_NEED_SEK_HEADER : DS_NEED_FOOTER; ------------------ | Branch (913:21): [True: 0, False: 3.25k] ------------------ 914| 3.25k| ds->scratch_used = 0; 915| 3.25k| ds->scratch_need = 916| 3.25k| ds->file_has_seek ? ZXC_BLOCK_HEADER_SIZE : zxc_footer_bytes(ds->file_has_checksum); ------------------ | | 407| 0|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ | Branch (916:13): [True: 0, False: 3.25k] ------------------ 917| 3.25k| return 0; 918| 3.25k| } 919| | 920| | // Data block: validate comp_size against the block_size negotiated in the file header. 921| 4.90k| if (UNLIKELY((uint64_t)ds->cur_bh.comp_size > (uint64_t)ds->block_size)) ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 922| 0| return ds_set_error(ds, ZXC_ERROR_BAD_BLOCK_SIZE); 923| | 924| | // Read comp_size [+ ZXC_BLOCK_CHECKSUM_SIZE if file-level checksums]. 925| 4.90k| const uint64_t need = (uint64_t)ds->cur_bh.comp_size + 926| 4.90k| (ds->file_has_checksum ? (uint64_t)ZXC_BLOCK_CHECKSUM_SIZE : 0U); ------------------ | | 409| 1.63k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (926:28): [True: 1.63k, False: 3.27k] ------------------ 927| 4.90k| if (UNLIKELY(need > ds->payload_cap)) return ds_set_error(ds, ZXC_ERROR_BAD_BLOCK_SIZE); ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 928| | 929| | // Feed the full block (header + payload + opt csum) to zxc_decompress_block, 930| | // so prefix with the 8-byte header we just parsed. 931| 4.90k| ZXC_MEMCPY(ds->payload, ds->scratch, ZXC_BLOCK_HEADER_SIZE); ------------------ | | 167| 4.90k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 932| 4.90k| ds->payload_used = ZXC_BLOCK_HEADER_SIZE; ------------------ | | 407| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 933| 4.90k| ds->payload_need = (size_t)need + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 407| 4.90k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 934| | // LCOV_EXCL_START 935| 4.90k| if (UNLIKELY(ds->payload_need > ds->payload_cap)) { ------------------ | | 151| 4.90k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.90k] | | ------------------ ------------------ 936| | // grow 937| 0| uint8_t* nb = (uint8_t*)ZXC_REALLOC(ds->payload, ds->payload_need); ------------------ | | 79| 0|#define ZXC_REALLOC(ptr, size) realloc(ptr, size) ------------------ 938| 0| if (UNLIKELY(!nb)) return ds_set_error(ds, ZXC_ERROR_MEMORY); ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 939| 0| ds->payload = nb; 940| 0| ds->payload_cap = ds->payload_need; 941| 0| } 942| | // LCOV_EXCL_STOP 943| 4.90k| ds->state = DS_NEED_BLOCK_PAYLOAD; 944| 4.90k| return 0; 945| 4.90k|} zxc_pstream.c:ds_pull: 742| 41.0M| zxc_inbuf_t* in) { 743| 41.0M| const size_t want = need - *used; 744| 41.0M| const size_t avail = in->size - in->pos; 745| 41.0M| const size_t n = want < avail ? want : avail; ------------------ | Branch (745:22): [True: 16.7k, False: 41.0M] ------------------ 746| 41.0M| if (n) { ------------------ | Branch (746:9): [True: 41.0M, False: 2.88k] ------------------ 747| 41.0M| ZXC_MEMCPY(dst + *used, (const uint8_t*)in->src + in->pos, n); ------------------ | | 167| 41.0M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 748| 41.0M| in->pos += n; 749| 41.0M| *used += n; 750| 41.0M| } 751| 41.0M| return *used == need; 752| 41.0M|} zxc_pstream.c:ds_set_error: 728| 4.50k|static int ds_set_error(zxc_dstream* ds, const int code) { 729| 4.50k| ds->error_code = code; 730| 4.50k| ds->state = DS_ERRORED; 731| 4.50k| return code; 732| 4.50k|} zxc_pstream.c:ds_drain_decoded: 823| 1.70M|static int ds_drain_decoded(zxc_dstream* ds, zxc_outbuf_t* out, size_t* produced) { 824| 1.70M| const size_t n = zxc_outbuf_push(out, ds->decoded, &ds->decoded_pos, ds->decoded_size); 825| 1.70M| ds->total_out += n; 826| 1.70M| if (produced) *produced += n; ------------------ | Branch (826:9): [True: 1.70M, False: 0] ------------------ 827| 1.70M| return ds->decoded_pos == ds->decoded_size; 828| 1.70M|} LLVMFuzzerTestOneInput: 104| 4.62k|int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { 105| 4.62k| if (size < 3) return 0; ------------------ | Branch (105:9): [True: 2, False: 4.62k] ------------------ 106| | 107| | /* Save raw input for the parser-fuzzing phase. */ 108| 4.62k| const uint8_t* const raw_data = data; 109| 4.62k| const size_t raw_size = size; 110| | 111| | /* Header bytes drive pstream-specific axes. data[0] is reserved (the 112| | * compression level is intentionally fixed to 1, see copts below); it 113| | * is still consumed so corpus byte offsets stay stable. */ 114| 4.62k| const int checksum_enabled = data[1] & 1; 115| 4.62k| const size_t in_chunk = derive_chunk_size(data[2]); 116| 4.62k| data += 3; 117| 4.62k| size -= 3; 118| | 119| 4.62k| if (size == 0 || size > FUZZ_PSTREAM_MAX_INPUT) return 0; ------------------ | | 41| 4.60k|#define FUZZ_PSTREAM_MAX_INPUT (2 << 20) /* 2 MiB */ ------------------ | Branch (119:9): [True: 17, False: 4.60k] | Branch (119:22): [True: 0, False: 4.60k] ------------------ 120| | 121| | /* Persistent buffers - reused across iterations to reduce allocator 122| | * pressure (same pattern as fuzz_seekable.c). */ 123| 4.60k| static uint8_t* comp_buf = NULL; 124| 4.60k| static size_t comp_cap = 0; 125| 4.60k| static uint8_t* round_buf = NULL; 126| 4.60k| static size_t round_cap = 0; 127| | 128| | /* ------------------------------------------------------------------ */ 129| | /* Phase 1: Push compression in chunks */ 130| | /* ------------------------------------------------------------------ */ 131| | /* Level forced to 1: pstream's state-machine surface (framing, drain, 132| | * resumption, sticky errors) is independent of compression strength, 133| | * and buffer-API fuzzers (roundtrip, decompress) already exercise the 134| | * level dimension. Cheaper compression = more iterations/sec. */ 135| 4.60k| zxc_compress_opts_t copts = { 136| 4.60k| .level = 1, 137| 4.60k| .checksum_enabled = checksum_enabled, 138| 4.60k| }; 139| 4.60k| zxc_cstream* cs = zxc_cstream_create(&copts); 140| 4.60k| if (!cs) return 0; ------------------ | Branch (140:9): [True: 0, False: 4.60k] ------------------ 141| | 142| | /* Sized so the compressor never runs out - bound is a safe upper limit. */ 143| | /* Use the public buffer-API bound function via a local extern declaration 144| | * to avoid pulling zxc_buffer.h (kept narrow on purpose). */ 145| 4.60k| extern uint64_t zxc_compress_bound(size_t input_size); 146| 4.60k| const uint64_t bound64 = zxc_compress_bound(size); 147| 4.60k| if (bound64 == 0 || bound64 > SIZE_MAX) { ------------------ | Branch (147:9): [True: 0, False: 4.60k] | Branch (147:25): [True: 0, False: 4.60k] ------------------ 148| 0| zxc_cstream_free(cs); 149| 0| return 0; 150| 0| } 151| 4.60k| const size_t bound = (size_t)bound64; 152| 4.60k| if (bound > comp_cap) { ------------------ | Branch (152:9): [True: 2.98k, False: 1.62k] ------------------ 153| 2.98k| void* nb = realloc(comp_buf, bound); 154| 2.98k| if (!nb) { ------------------ | Branch (154:13): [True: 0, False: 2.98k] ------------------ 155| 0| zxc_cstream_free(cs); 156| 0| return 0; 157| 0| } 158| 2.98k| comp_buf = (uint8_t*)nb; 159| 2.98k| comp_cap = bound; 160| 2.98k| } 161| | 162| 4.60k| size_t comp_len = 0; 163| 4.60k| size_t in_pos = 0; 164| | 165| | /* Feed input in `in_chunk`-sized slices. */ 166| 109M| while (in_pos < size) { ------------------ | Branch (166:12): [True: 109M, False: 4.60k] ------------------ 167| 109M| const size_t feed = (size - in_pos < in_chunk) ? (size - in_pos) : in_chunk; ------------------ | Branch (167:29): [True: 3.47k, False: 109M] ------------------ 168| 109M| zxc_inbuf_t in = {.src = data + in_pos, .size = feed, .pos = 0}; 169| | 170| | /* Drain repeatedly until this slice is fully consumed. */ 171| 109M| for (;;) { 172| 109M| size_t window = bound - comp_len; 173| 109M| if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 109M|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 109M|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ | Branch (173:17): [True: 109M, False: 14.3k] ------------------ 174| 109M| if (window == 0) { ------------------ | Branch (174:17): [True: 0, False: 109M] ------------------ 175| | /* Bound exhausted - shouldn't happen with zxc_compress_bound. */ 176| 0| zxc_cstream_free(cs); 177| 0| return 0; 178| 0| } 179| 109M| zxc_outbuf_t ob = {.dst = comp_buf + comp_len, .size = window, .pos = 0}; 180| 109M| const int64_t r = zxc_cstream_compress(cs, &ob, &in); 181| 109M| if (r < 0) { ------------------ | Branch (181:17): [True: 0, False: 109M] ------------------ 182| 0| zxc_cstream_free(cs); 183| 0| return 0; 184| 0| } 185| 109M| comp_len += ob.pos; 186| 109M| if (in.pos == in.size && r == 0) break; ------------------ | Branch (186:17): [True: 109M, False: 234k] | Branch (186:38): [True: 109M, False: 84.1k] ------------------ 187| 109M| } 188| 109M| in_pos += in.pos; 189| 109M| } 190| | 191| | /* Finalize: zxc_cstream_end may report >0 pending bytes; drain until 0. */ 192| 512k| for (;;) { 193| 512k| size_t window = bound - comp_len; 194| 512k| if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 512k|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 511k|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ | Branch (194:13): [True: 511k, False: 1.06k] ------------------ 195| 512k| if (window == 0) { ------------------ | Branch (195:13): [True: 0, False: 512k] ------------------ 196| 0| zxc_cstream_free(cs); 197| 0| return 0; 198| 0| } 199| 512k| zxc_outbuf_t ob = {.dst = comp_buf + comp_len, .size = window, .pos = 0}; 200| 512k| const int64_t pending = zxc_cstream_end(cs, &ob); 201| 512k| if (pending < 0) { ------------------ | Branch (201:13): [True: 0, False: 512k] ------------------ 202| 0| zxc_cstream_free(cs); 203| 0| return 0; 204| 0| } 205| 512k| comp_len += ob.pos; 206| 512k| if (pending == 0) break; ------------------ | Branch (206:13): [True: 4.60k, False: 508k] ------------------ 207| 512k| } 208| | 209| 4.60k| zxc_cstream_free(cs); 210| | 211| | /* ------------------------------------------------------------------ */ 212| | /* Phase 2: Push decompression with chunked feeding + roundtrip */ 213| | /* ------------------------------------------------------------------ */ 214| 4.60k| if (size > round_cap) { ------------------ | Branch (214:9): [True: 2.98k, False: 1.62k] ------------------ 215| 2.98k| void* nb = realloc(round_buf, size); 216| 2.98k| if (!nb) return 0; ------------------ | Branch (216:13): [True: 0, False: 2.98k] ------------------ 217| 2.98k| round_buf = (uint8_t*)nb; 218| 2.98k| round_cap = size; 219| 2.98k| } 220| | 221| 4.60k| const int64_t dsize = 222| 4.60k| drip_decompress(comp_buf, comp_len, round_buf, size, in_chunk, checksum_enabled); 223| 4.60k| if (dsize >= 0) { ------------------ | Branch (223:9): [True: 4.60k, False: 0] ------------------ 224| 4.60k| assert((size_t)dsize == size); ------------------ | Branch (224:9): [True: 0, False: 4.60k] | Branch (224:9): [True: 4.60k, False: 0] ------------------ 225| 4.60k| assert(memcmp(data, round_buf, size) == 0); ------------------ | Branch (225:9): [True: 0, False: 4.60k] | Branch (225:9): [True: 4.60k, False: 0] ------------------ 226| 4.60k| } 227| | 228| | /* ------------------------------------------------------------------ */ 229| | /* Phase 3: Parser fuzzing - feed raw input directly */ 230| | /* */ 231| | /* This is where malformed-frame and truncation bugs live. We don't */ 232| | /* care about the result, only that no UB / crash occurs. */ 233| | /* ------------------------------------------------------------------ */ 234| 4.60k| if (raw_size > 0) { ------------------ | Branch (234:9): [True: 4.60k, False: 0] ------------------ 235| | /* Output is discarded - a single small stack buffer suffices and 236| | * keeps the parser's resumption code (small ob.size) under fuzz. */ 237| 4.60k| uint8_t scratch[FUZZ_PSTREAM_OUT_CAP]; 238| 4.60k| const size_t parser_chunk = derive_chunk_size(raw_data[raw_size - 1]); 239| | 240| 4.60k| zxc_decompress_opts_t opts = {.checksum_enabled = checksum_enabled}; 241| 4.60k| zxc_dstream* ds = zxc_dstream_create(&opts); 242| 4.60k| if (ds) { ------------------ | Branch (242:13): [True: 4.60k, False: 0] ------------------ 243| 4.60k| size_t pos = 0; 244| 4.60k| int rounds = 0; 245| 26.9k| while (pos < raw_size && rounds < 1024) { ------------------ | Branch (245:20): [True: 26.8k, False: 105] | Branch (245:38): [True: 26.8k, False: 0] ------------------ 246| 26.8k| const size_t feed = 247| 26.8k| (raw_size - pos < parser_chunk) ? (raw_size - pos) : parser_chunk; ------------------ | Branch (247:21): [True: 515, False: 26.2k] ------------------ 248| 26.8k| zxc_inbuf_t in = {.src = raw_data + pos, .size = feed, .pos = 0}; 249| 26.8k| zxc_outbuf_t ob = {.dst = scratch, .size = sizeof scratch, .pos = 0}; 250| 26.8k| const size_t before_in = in.pos; 251| 26.8k| const size_t before_out = ob.pos; 252| 26.8k| const int64_t r = zxc_dstream_decompress(ds, &ob, &in); 253| 26.8k| pos += in.pos; 254| 26.8k| if (r < 0) break; ------------------ | Branch (254:21): [True: 4.50k, False: 22.3k] ------------------ 255| 22.3k| if (in.pos == before_in && ob.pos == before_out) break; ------------------ | Branch (255:21): [True: 0, False: 22.3k] | Branch (255:44): [True: 0, False: 0] ------------------ 256| 22.3k| rounds++; 257| 22.3k| } 258| 4.60k| zxc_dstream_free(ds); 259| 4.60k| } 260| 4.60k| } 261| | 262| 4.60k| return 0; 263| 4.60k|} fuzz_pstream.c:derive_chunk_size: 47| 9.22k|static size_t derive_chunk_size(uint8_t b) { 48| | /* Map [0..255] -> [1..512] with a non-linear distribution biased toward 49| | * small values (small chunks stress the state machine the most). */ 50| 9.22k| if (b == 0) return 1; ------------------ | Branch (50:9): [True: 2.18k, False: 7.04k] ------------------ 51| 7.04k| if (b < 64) return (size_t)b; /* 1..63 */ ------------------ | Branch (51:9): [True: 2.41k, False: 4.62k] ------------------ 52| 4.62k| if (b < 192) return (size_t)b * 2; /* 128..382 */ ------------------ | Branch (52:9): [True: 1.61k, False: 3.01k] ------------------ 53| 3.01k| return (size_t)b + 256; /* 448..511 */ 54| 4.62k|} fuzz_pstream.c:drip_decompress: 65| 4.60k| size_t in_chunk, int checksum_enabled) { 66| 4.60k| zxc_decompress_opts_t opts = {.checksum_enabled = checksum_enabled}; 67| 4.60k| zxc_dstream* ds = zxc_dstream_create(&opts); 68| 4.60k| if (!ds) return -1; ------------------ | Branch (68:9): [True: 0, False: 4.60k] ------------------ 69| | 70| 4.60k| size_t in_pos = 0; 71| 4.60k| size_t out_total = 0; 72| | 73| 42.7M| while (in_pos < csize || !zxc_dstream_finished(ds)) { ------------------ | Branch (73:12): [True: 42.7M, False: 3.14k] | Branch (73:30): [True: 0, False: 3.14k] ------------------ 74| 42.7M| if (out_total >= out_cap) { ------------------ | Branch (74:13): [True: 1.45k, False: 42.7M] ------------------ 75| | /* Caller's buffer full - stop without asserting. */ 76| 1.45k| zxc_dstream_free(ds); 77| 1.45k| return (int64_t)out_total; 78| 1.45k| } 79| 42.7M| const size_t feed = (csize - in_pos < in_chunk) ? (csize - in_pos) : in_chunk; ------------------ | Branch (79:29): [True: 858k, False: 41.8M] ------------------ 80| 42.7M| size_t window = out_cap - out_total; 81| 42.7M| if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 42.7M|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ if (window > FUZZ_PSTREAM_OUT_CAP) window = FUZZ_PSTREAM_OUT_CAP; ------------------ | | 45| 42.6M|#define FUZZ_PSTREAM_OUT_CAP 256 ------------------ | Branch (81:13): [True: 42.6M, False: 30.8k] ------------------ 82| 42.7M| zxc_inbuf_t in = {.src = comp + in_pos, .size = feed, .pos = 0}; 83| 42.7M| zxc_outbuf_t ob = {.dst = out + out_total, .size = window, .pos = 0}; 84| | 85| 42.7M| const size_t before_in = in.pos; 86| 42.7M| const size_t before_out = ob.pos; 87| 42.7M| const int64_t r = zxc_dstream_decompress(ds, &ob, &in); 88| 42.7M| if (r < 0) { ------------------ | Branch (88:13): [True: 0, False: 42.7M] ------------------ 89| 0| zxc_dstream_free(ds); 90| 0| return r; 91| 0| } 92| | 93| 42.7M| out_total += ob.pos; 94| 42.7M| in_pos += in.pos; 95| | 96| | /* No progress and no more input: prevent infinite loop on truncation. */ 97| 42.7M| if (in.pos == before_in && ob.pos == before_out && in_pos == csize) break; ------------------ | Branch (97:13): [True: 1.69M, False: 41.0M] | Branch (97:36): [True: 0, False: 1.69M] | Branch (97:60): [True: 0, False: 0] ------------------ 98| 42.7M| } 99| | 100| 3.14k| zxc_dstream_free(ds); 101| 3.14k| return (int64_t)out_total; 102| 4.60k|}