zxc_compress.c:rapidhash: 500| 5.38k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash(const void *key, size_t len) RAPIDHASH_NOEXCEPT { 501| 5.38k| return rapidhash_withSeed(key, len, 0); 502| 5.38k|} zxc_compress.c:rapidhash_withSeed: 486| 5.38k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_withSeed(const void *key, size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT { 487| 5.38k| return rapidhash_internal(key, len, seed, rapid_secret); 488| 5.38k|} zxc_compress.c:rapidhash_internal: 243| 5.38k|RAPIDHASH_INLINE_CONSTEXPR uint64_t rapidhash_internal(const void *key, size_t len, uint64_t seed, const uint64_t* secret) RAPIDHASH_NOEXCEPT { 244| 5.38k| const uint8_t *p=(const uint8_t *)key; 245| 5.38k| seed ^= rapid_mix(seed ^ secret[2], secret[1]); 246| 5.38k| uint64_t a=0, b=0; 247| 5.38k| size_t i = len; 248| 5.38k| if (_likely_(len <= 16)) { ------------------ | | 106| 5.38k| # define _likely_(x) __builtin_expect(x,1) | | ------------------ | | | Branch (106:24): [True: 180, False: 5.20k] | | ------------------ ------------------ 249| 180| if (len >= 4) { ------------------ | Branch (249:9): [True: 151, False: 29] ------------------ 250| 151| seed ^= len; 251| 151| if (len >= 8) { ------------------ | Branch (251:11): [True: 111, False: 40] ------------------ 252| 111| const uint8_t* plast = p + len - 8; 253| 111| a = rapid_read64(p); 254| 111| b = rapid_read64(plast); 255| 111| } else { 256| 40| const uint8_t* plast = p + len - 4; 257| 40| a = rapid_read32(p); 258| 40| b = rapid_read32(plast); 259| 40| } 260| 151| } else if (len > 0) { ------------------ | Branch (260:16): [True: 29, False: 0] ------------------ 261| 29| a = (((uint64_t)p[0])<<45)|p[len-1]; 262| 29| b = p[len>>1]; 263| 29| } else 264| 0| a = b = 0; 265| 5.20k| } else { 266| 5.20k| if (len > 112) { ------------------ | Branch (266:9): [True: 4.27k, False: 929] ------------------ 267| 4.27k| uint64_t see1 = seed, see2 = seed; 268| 4.27k| uint64_t see3 = seed, see4 = seed; 269| 4.27k| uint64_t see5 = seed, see6 = seed; 270| 4.27k|#ifdef RAPIDHASH_COMPACT 271| 1.59M| do { 272| 1.59M| seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 273| 1.59M| see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 274| 1.59M| see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 275| 1.59M| see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 276| 1.59M| see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 277| 1.59M| see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 278| 1.59M| see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 279| 1.59M| p += 112; 280| 1.59M| i -= 112; 281| 1.59M| } while(i > 112); ------------------ | Branch (281:15): [True: 1.58M, False: 4.27k] ------------------ 282| |#else 283| | while (i > 224) { 284| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 285| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 286| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 287| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 288| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 289| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 290| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 291| | seed = rapid_mix(rapid_read64(p + 112) ^ secret[0], rapid_read64(p + 120) ^ seed); 292| | see1 = rapid_mix(rapid_read64(p + 128) ^ secret[1], rapid_read64(p + 136) ^ see1); 293| | see2 = rapid_mix(rapid_read64(p + 144) ^ secret[2], rapid_read64(p + 152) ^ see2); 294| | see3 = rapid_mix(rapid_read64(p + 160) ^ secret[3], rapid_read64(p + 168) ^ see3); 295| | see4 = rapid_mix(rapid_read64(p + 176) ^ secret[4], rapid_read64(p + 184) ^ see4); 296| | see5 = rapid_mix(rapid_read64(p + 192) ^ secret[5], rapid_read64(p + 200) ^ see5); 297| | see6 = rapid_mix(rapid_read64(p + 208) ^ secret[6], rapid_read64(p + 216) ^ see6); 298| | p += 224; 299| | i -= 224; 300| | } 301| | if (i > 112) { 302| | seed = rapid_mix(rapid_read64(p) ^ secret[0], rapid_read64(p + 8) ^ seed); 303| | see1 = rapid_mix(rapid_read64(p + 16) ^ secret[1], rapid_read64(p + 24) ^ see1); 304| | see2 = rapid_mix(rapid_read64(p + 32) ^ secret[2], rapid_read64(p + 40) ^ see2); 305| | see3 = rapid_mix(rapid_read64(p + 48) ^ secret[3], rapid_read64(p + 56) ^ see3); 306| | see4 = rapid_mix(rapid_read64(p + 64) ^ secret[4], rapid_read64(p + 72) ^ see4); 307| | see5 = rapid_mix(rapid_read64(p + 80) ^ secret[5], rapid_read64(p + 88) ^ see5); 308| | see6 = rapid_mix(rapid_read64(p + 96) ^ secret[6], rapid_read64(p + 104) ^ see6); 309| | p += 112; 310| | i -= 112; 311| | } 312| |#endif 313| 4.27k| seed ^= see1; 314| 4.27k| see2 ^= see3; 315| 4.27k| see4 ^= see5; 316| 4.27k| seed ^= see6; 317| 4.27k| see2 ^= see4; 318| 4.27k| seed ^= see2; 319| 4.27k| } 320| 5.20k| if (i > 16) { ------------------ | Branch (320:9): [True: 4.51k, False: 692] ------------------ 321| 4.51k| seed = rapid_mix(rapid_read64(p) ^ secret[2], rapid_read64(p + 8) ^ seed); 322| 4.51k| if (i > 32) { ------------------ | Branch (322:11): [True: 3.68k, False: 828] ------------------ 323| 3.68k| seed = rapid_mix(rapid_read64(p + 16) ^ secret[2], rapid_read64(p + 24) ^ seed); 324| 3.68k| if (i > 48) { ------------------ | Branch (324:15): [True: 2.93k, False: 749] ------------------ 325| 2.93k| seed = rapid_mix(rapid_read64(p + 32) ^ secret[1], rapid_read64(p + 40) ^ seed); 326| 2.93k| if (i > 64) { ------------------ | Branch (326:19): [True: 2.21k, False: 726] ------------------ 327| 2.21k| seed = rapid_mix(rapid_read64(p + 48) ^ secret[1], rapid_read64(p + 56) ^ seed); 328| 2.21k| if (i > 80) { ------------------ | Branch (328:23): [True: 1.36k, False: 845] ------------------ 329| 1.36k| seed = rapid_mix(rapid_read64(p + 64) ^ secret[2], rapid_read64(p + 72) ^ seed); 330| 1.36k| if (i > 96) { ------------------ | Branch (330:27): [True: 705, False: 663] ------------------ 331| 705| seed = rapid_mix(rapid_read64(p + 80) ^ secret[1], rapid_read64(p + 88) ^ seed); 332| 705| } 333| 1.36k| } 334| 2.21k| } 335| 2.93k| } 336| 3.68k| } 337| 4.51k| } 338| 5.20k| a=rapid_read64(p+i-16) ^ i; b=rapid_read64(p+i-8); 339| 5.20k| } 340| 5.38k| a ^= secret[1]; 341| 5.38k| b ^= seed; 342| 5.38k| rapid_mum(&a, &b); 343| 5.38k| return rapid_mix(a ^ secret[7], b ^ secret[1] ^ i); 344| 5.38k|} zxc_compress.c:rapid_mix: 208| 11.1M| RAPIDHASH_INLINE_CONSTEXPR uint64_t rapid_mix(uint64_t A, uint64_t B) RAPIDHASH_NOEXCEPT { rapid_mum(&A,&B); return A^B; } zxc_compress.c:rapid_read64: 214| 22.3M| RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return v;} zxc_compress.c:rapid_read32: 215| 80| RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return v;} zxc_compress.c:rapid_mum: 156| 11.1M| RAPIDHASH_INLINE_CONSTEXPR void rapid_mum(uint64_t *A, uint64_t *B) RAPIDHASH_NOEXCEPT { 157| 11.1M| #if defined(__SIZEOF_INT128__) 158| 11.1M| __uint128_t r=*A; r*=*B; 159| | #ifdef RAPIDHASH_PROTECTED 160| | *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); 161| | #else 162| 11.1M| *A=(uint64_t)r; *B=(uint64_t)(r>>64); 163| 11.1M| #endif 164| | #elif defined(_MSC_VER) && (defined(_WIN64) || defined(_M_HYBRID_CHPE_ARM64)) 165| | #if defined(_M_X64) 166| | #ifdef RAPIDHASH_PROTECTED 167| | uint64_t a, b; 168| | a=_umul128(*A,*B,&b); 169| | *A^=a; *B^=b; 170| | #else 171| | *A=_umul128(*A,*B,B); 172| | #endif 173| | #else 174| | #ifdef RAPIDHASH_PROTECTED 175| | uint64_t a, b; 176| | b = __umulh(*A, *B); 177| | a = *A * *B; 178| | *A^=a; *B^=b; 179| | #else 180| | uint64_t c = __umulh(*A, *B); 181| | *A = *A * *B; 182| | *B = c; 183| | #endif 184| | #endif 185| | #else 186| | uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B; 187| | uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t>32)+(rm1>>32)+c; 191| | #ifdef RAPIDHASH_PROTECTED 192| | *A^=lo; *B^=hi; 193| | #else 194| | *A=lo; *B=hi; 195| | #endif 196| | #endif 197| 11.1M| } zxc_aligned_malloc: 36| 20.1k|void* zxc_aligned_malloc(const size_t size, const size_t alignment) { 37| |#if defined(_WIN32) 38| | return _aligned_malloc(size, alignment); 39| |#else 40| 20.1k| void* ptr = NULL; 41| 20.1k| if (posix_memalign(&ptr, alignment, size) != 0) return NULL; ------------------ | Branch (41:9): [True: 0, False: 20.1k] ------------------ 42| 20.1k| return ptr; 43| 20.1k|#endif 44| 20.1k|} zxc_aligned_free: 51| 20.1k|void zxc_aligned_free(void* ptr) { 52| |#if defined(_WIN32) 53| | _aligned_free(ptr); 54| |#else 55| 20.1k| free(ptr); 56| 20.1k|#endif 57| 20.1k|} zxc_cctx_init_in_workspace: 298| 20.1k| const int defer_entropy_scratch) { 299| 20.1k| if (UNLIKELY(!ctx || !workspace || chunk_size == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 80.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 20.1k] | | | Branch (151:42): [True: 0, False: 20.1k] | | | Branch (151:42): [True: 0, False: 20.1k] | | | Branch (151:42): [True: 1, False: 20.1k] | | ------------------ ------------------ 300| | 301| 20.1k| const zxc_cctx_layout_t layout = 302| 20.1k| compute_cctx_layout(chunk_size, mode, level, dict_size, defer_entropy_scratch); 303| 20.1k| if (UNLIKELY(workspace_size < layout.total)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 20.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 20.1k] | | ------------------ ------------------ 304| | 305| 20.1k| ZXC_MEMSET(ctx, 0, sizeof(zxc_cctx_t)); ------------------ | | 178| 20.1k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 306| 20.1k| ctx->checksum_enabled = checksum_enabled; 307| 20.1k| ctx->chunk_size = chunk_size; 308| 20.1k| const uint32_t offset_bits = zxc_log2_u32((uint32_t)chunk_size); 309| 20.1k| ctx->offset_bits = offset_bits; 310| 20.1k| ctx->offset_mask = (uint32_t)((1ULL << offset_bits) - 1); 311| 20.1k| ctx->max_epoch = (uint32_t)(1ULL << (32 - offset_bits)); 312| | 313| | /* memory_block stays NULL on the static-init path so zxc_cctx_free does 314| | * not try to free the caller's workspace. Sub-buffer pointers carry the 315| | * partition; ownership is implicit (the caller owns @p workspace). */ 316| 20.1k| uint8_t* const mem = (uint8_t*)workspace; 317| | 318| | /* Dictionary concat scratch (both modes); init owns dict_size now so callers 319| | * no longer assign ctx->dict_size after init. */ 320| 20.1k| ctx->dict_size = dict_size; 321| 20.1k| if (dict_size > 0) { ------------------ | Branch (321:9): [True: 0, False: 20.1k] ------------------ 322| 0| ctx->dict_buffer = mem + layout.off_dict; 323| 0| ctx->dict_buffer_cap = layout.sz_dict; 324| 0| ctx->dict_huf = (zxc_dict_huf_state_t*)(void*)(mem + layout.off_dict_huf); 325| 0| } 326| | 327| 20.1k| if (mode == 0) { ------------------ | Branch (327:9): [True: 10.6k, False: 9.45k] ------------------ 328| 10.6k| ctx->work_buf = mem + layout.off_work; 329| 10.6k| ctx->work_buf_cap = chunk_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 349| 10.6k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 10.6k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 330| 10.6k| ctx->lit_buffer = mem + layout.off_lit_dctx; 331| 10.6k| ctx->lit_buffer_cap = chunk_size + ZXC_PAD_SIZE; ------------------ | | 337| 10.6k|#define ZXC_PAD_SIZE 32 ------------------ 332| 10.6k| if (layout.sz_pivco_dctx) { ------------------ | Branch (332:13): [True: 0, False: 10.6k] ------------------ 333| 0| ctx->tok_buffer = mem + layout.off_tok_dctx; 334| 0| ctx->tok_buffer_cap = layout.sz_tok_dctx; 335| 0| ctx->pivco_scratch = mem + layout.off_pivco_dctx; 336| 0| ctx->pivco_scratch_cap = layout.sz_pivco_dctx; 337| 0| } 338| 10.6k| return ZXC_OK; 339| 10.6k| } 340| | 341| 9.45k| ctx->hash_table = (uint32_t*)(mem + layout.off_hash_pos); 342| 9.45k| ctx->hash_tags = mem + layout.off_hash_tags; 343| 9.45k| ctx->chain_table = (uint16_t*)(mem + layout.off_chain); 344| 9.45k| ctx->buf_sequences = (uint32_t*)(mem + layout.off_seq_union); 345| 9.45k| ctx->buf_offsets = (uint16_t*)(mem + layout.off_seq_union); 346| 9.45k| ctx->buf_tokens = mem + layout.off_seq_union + layout.max_seq * sizeof(uint16_t); 347| 9.45k| ctx->buf_extras = mem + layout.off_extras; 348| 9.45k| ctx->literals = mem + layout.off_lit_cctx; 349| 9.45k| if (layout.sz_opt) { ------------------ | Branch (349:9): [True: 0, False: 9.45k] ------------------ 350| 0| ctx->opt_scratch = mem + layout.off_opt; 351| 0| ctx->opt_scratch_cap = layout.sz_opt; 352| 0| } 353| | 354| 9.45k| ctx->compression_level = level; 355| 9.45k| ctx->epoch = 1; 356| | 357| 9.45k| ZXC_MEMSET(ctx->hash_table, 0, layout.sz_hash_pos); ------------------ | | 178| 9.45k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 358| 9.45k| ZXC_MEMSET(ctx->hash_tags, 0, layout.sz_hash_tags); ------------------ | | 178| 9.45k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 359| 9.45k| return ZXC_OK; 360| 20.1k|} zxc_cctx_init: 381| 20.1k| const int level, const int checksum_enabled, const size_t dict_size) { 382| 20.1k| if (UNLIKELY(chunk_size == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 20.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 20.1k] | | ------------------ ------------------ 383| | /* Heap contexts defer the decode-side entropy scratch to the first entropy 384| | * section; static workspaces must have it already, they may never allocate. */ 385| 20.1k| const int defer_entropy = (mode == 0); 386| 20.1k| const size_t total = 387| 20.1k| compute_cctx_layout(chunk_size, mode, level, dict_size, defer_entropy).total; 388| 20.1k| if (UNLIKELY(total == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 20.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 20.1k] | | ------------------ ------------------ 389| | 390| 20.1k| uint8_t* const mem = (uint8_t*)ZXC_ALIGNED_MALLOC(total, ZXC_CACHE_LINE_SIZE); ------------------ | | 110| 20.1k|#define ZXC_ALIGNED_MALLOC(size, alignment) zxc_aligned_malloc(size, alignment) ------------------ 391| 20.1k| if (UNLIKELY(!mem)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 20.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 20.1k] | | ------------------ ------------------ 392| | 393| 20.1k| const int rc = zxc_cctx_init_in_workspace(ctx, mem, total, chunk_size, mode, level, 394| 20.1k| checksum_enabled, dict_size, defer_entropy); 395| 20.1k| if (UNLIKELY(rc != ZXC_OK)) { ------------------ | | 151| 20.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 20.1k] | | ------------------ ------------------ 396| | // LCOV_EXCL_START 397| 0| ZXC_ALIGNED_FREE(mem); ------------------ | | 116| 0|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 398| 0| return rc; 399| | // LCOV_EXCL_STOP 400| 0| } 401| | /* Library-owned buffer: record the allocation so zxc_cctx_free frees it. */ 402| 20.1k| ctx->memory_block = mem; 403| 20.1k| return ZXC_OK; 404| 20.1k|} zxc_cctx_free: 441| 20.1k|void zxc_cctx_free(zxc_cctx_t* ctx) { 442| 20.1k| if (ctx->memory_block) { ------------------ | Branch (442:9): [True: 20.1k, False: 0] ------------------ 443| 20.1k| ZXC_ALIGNED_FREE(ctx->memory_block); ------------------ | | 116| 20.1k|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 444| 20.1k| ctx->memory_block = NULL; 445| 20.1k| } 446| 20.1k| if (ctx->entropy_block) { ------------------ | Branch (446:9): [True: 0, False: 20.1k] ------------------ 447| 0| ZXC_ALIGNED_FREE(ctx->entropy_block); ------------------ | | 116| 0|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 448| 0| ctx->entropy_block = NULL; 449| 0| } 450| | 451| 20.1k| ctx->lit_buffer = NULL; 452| 20.1k| ctx->hash_table = NULL; 453| 20.1k| ctx->hash_tags = NULL; 454| 20.1k| ctx->chain_table = NULL; 455| 20.1k| ctx->buf_sequences = NULL; 456| 20.1k| ctx->buf_tokens = NULL; 457| 20.1k| ctx->buf_offsets = NULL; 458| 20.1k| ctx->buf_extras = NULL; 459| 20.1k| ctx->literals = NULL; 460| 20.1k| ctx->work_buf = NULL; 461| 20.1k| ctx->tok_buffer = NULL; 462| 20.1k| ctx->pivco_scratch = NULL; 463| 20.1k| ctx->opt_scratch = NULL; 464| 20.1k| ctx->dict_buffer = NULL; 465| 20.1k| ctx->dict_huf = NULL; 466| | 467| 20.1k| ctx->epoch = 0; 468| 20.1k| ctx->lit_buffer_cap = 0; 469| 20.1k| ctx->work_buf_cap = 0; 470| 20.1k| ctx->tok_buffer_cap = 0; 471| 20.1k| ctx->pivco_scratch_cap = 0; 472| 20.1k| ctx->opt_scratch_cap = 0; 473| 20.1k| ctx->dict_buffer_cap = 0; 474| 20.1k| ctx->dict_size = 0; 475| 20.1k| ctx->dict_huf_tree_ok = 0; 476| | ctx->lit_freq_acc = NULL; 477| 20.1k|} zxc_cctx_attach_dict_huf: 492| 20.1k|int zxc_cctx_attach_dict_huf(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT lengths) { 493| 20.1k| if (UNLIKELY(!ctx)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 20.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 20.1k] | | ------------------ ------------------ 494| 20.1k| ctx->dict_huf_tree_ok = 0; 495| 20.1k| if (lengths == NULL) return ZXC_OK; ------------------ | Branch (495:9): [True: 20.1k, False: 0] ------------------ 496| | 497| | /* Empty (all-zero) table from a low-entropy corpus: treat it as "no shared table". */ 498| 0| int empty = 1; 499| 0| for (size_t i = 0; i < ZXC_HUF_TABLE_SIZE; i++) { ------------------ | | 83| 0|#define ZXC_HUF_TABLE_SIZE 128 ------------------ | Branch (499:24): [True: 0, False: 0] ------------------ 500| 0| if (lengths[i]) { ------------------ | Branch (500:13): [True: 0, False: 0] ------------------ 501| 0| empty = 0; 502| 0| break; 503| 0| } 504| 0| } 505| 0| if (UNLIKELY(empty || ctx->dict_huf == NULL)) return ZXC_OK; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 506| | 507| | /* Tree-at-attach: unpack + build the PivCo tree, codes and decoder tables 508| | * once here; the per-block encode/estimate/decode paths reuse them via 509| | * the context. */ 510| 0| const int rc = zxc_huf_dict_tree_build(lengths, &ctx->dict_huf->tree, ctx->dict_huf->codes, 511| 0| ctx->dict_huf->code_len, &ctx->dict_huf->dec); 512| 0| if (UNLIKELY(rc != ZXC_OK)) return rc; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 513| 0| ctx->dict_huf_tree_ok = 1; 514| 0| return ZXC_OK; 515| 0|} zxc_write_file_header: 539| 9.45k| const int has_checksum, const uint32_t dict_id) { 540| 9.45k| if (UNLIKELY(dst_capacity < ZXC_FILE_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 541| | 542| 9.45k| zxc_store_le32(dst, ZXC_MAGIC_WORD); ------------------ | | 331| 9.45k|#define ZXC_MAGIC_WORD 0x9CB02EF5U ------------------ 543| 9.45k| dst[4] = ZXC_FILE_FORMAT_VERSION; ------------------ | | 334| 9.45k|#define ZXC_FILE_FORMAT_VERSION 7 ------------------ 544| | 545| | // Block size stored as log2 exponent (e.g. 18 = 256 KB) 546| 9.45k| dst[5] = (uint8_t)zxc_log2_u32((uint32_t)chunk_size); 547| | 548| 9.45k| uint8_t flags = has_checksum ? (ZXC_FILE_FLAG_HAS_CHECKSUM | ZXC_CHECKSUM_RAPIDHASH) : 0; ------------------ | | 365| 4.72k|#define ZXC_FILE_FLAG_HAS_CHECKSUM 0x80U ------------------ uint8_t flags = has_checksum ? (ZXC_FILE_FLAG_HAS_CHECKSUM | ZXC_CHECKSUM_RAPIDHASH) : 0; ------------------ | | 424| 4.72k|#define ZXC_CHECKSUM_RAPIDHASH 0 ------------------ | Branch (548:21): [True: 4.72k, False: 4.72k] ------------------ 549| 9.45k| if (dict_id != 0) flags |= ZXC_FILE_FLAG_HAS_DICTIONARY; ------------------ | | 367| 0|#define ZXC_FILE_FLAG_HAS_DICTIONARY 0x40U ------------------ | Branch (549:9): [True: 0, False: 9.45k] ------------------ 550| 9.45k| dst[6] = flags; 551| | 552| | // Bytes 7-13: Reserved / dict_id 553| 9.45k| ZXC_MEMSET(dst + 7, 0, 7); ------------------ | | 178| 9.45k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 554| 9.45k| if (dict_id != 0) zxc_store_le32(dst + 7, dict_id); ------------------ | Branch (554:9): [True: 0, False: 9.45k] ------------------ 555| | 556| | // Bytes 14-15: CRC (16-bit) 557| 9.45k| zxc_store_le16(dst + 14, 0); // Zero out before hashing 558| 9.45k| const uint16_t crc = zxc_hash16(dst); 559| 9.45k| zxc_store_le16(dst + 14, crc); 560| | 561| 9.45k| return ZXC_FILE_HEADER_SIZE; ------------------ | | 103| 9.45k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ 562| 9.45k|} zxc_read_file_header: 580| 18.0k| uint32_t* RESTRICT out_dict_id) { 581| 18.0k| if (UNLIKELY(src_size < ZXC_FILE_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 18.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 18.0k] | | ------------------ ------------------ 582| 18.0k| if (UNLIKELY(zxc_le32(src) != ZXC_MAGIC_WORD)) return ZXC_ERROR_BAD_MAGIC; ------------------ | | 151| 18.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 8.45k, False: 9.56k] | | ------------------ ------------------ 583| 9.56k| if (UNLIKELY(src[4] != ZXC_FILE_FORMAT_VERSION)) return ZXC_ERROR_BAD_VERSION; ------------------ | | 151| 9.56k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 78, False: 9.48k] | | ------------------ ------------------ 584| | 585| 9.48k| uint8_t temp[ZXC_FILE_HEADER_SIZE]; 586| 9.48k| ZXC_MEMCPY(temp, src, ZXC_FILE_HEADER_SIZE); ------------------ | | 173| 9.48k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 587| | // Zero out CRC bytes (14-15) before hash check 588| 9.48k| temp[14] = 0; 589| 9.48k| temp[15] = 0; 590| | // Header CRC16 (integrity), then the checksum-algorithm id in flags bits 0-3 591| | // (only 0 = RapidHash is defined). CRC is checked first via short-circuit. 592| 9.48k| if (UNLIKELY(zxc_le16(src + 14) != zxc_hash16(temp) || ------------------ | | 151| 18.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 27, False: 9.45k] | | | Branch (151:42): [True: 25, False: 9.45k] | | | Branch (151:42): [True: 2, False: 9.45k] | | ------------------ ------------------ 593| 9.48k| (src[6] & 0x0FU) != ZXC_CHECKSUM_RAPIDHASH)) 594| 27| return ZXC_ERROR_BAD_HEADER; 595| | 596| 9.45k| if (out_block_size) { ------------------ | Branch (596:9): [True: 9.45k, False: 0] ------------------ 597| 9.45k| const uint8_t code = src[5]; 598| 9.45k| if (UNLIKELY(code < ZXC_BLOCK_SIZE_MIN_LOG2 || code > ZXC_BLOCK_SIZE_MAX_LOG2)) ------------------ | | 151| 18.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 2, False: 9.45k] | | | Branch (151:42): [True: 1, False: 9.45k] | | | Branch (151:42): [True: 1, False: 9.45k] | | ------------------ ------------------ 599| 2| return ZXC_ERROR_BAD_BLOCK_SIZE; 600| | // Exponent encoding: block_size = 2^code (4 KB - 2 MB) 601| 9.45k| *out_block_size = (size_t)1U << code; 602| 9.45k| } 603| 9.45k| if (out_has_checksum) *out_has_checksum = (src[6] & ZXC_FILE_FLAG_HAS_CHECKSUM) ? 1 : 0; ------------------ | | 365| 9.45k|#define ZXC_FILE_FLAG_HAS_CHECKSUM 0x80U ------------------ | Branch (603:9): [True: 9.45k, False: 0] | Branch (603:47): [True: 4.72k, False: 4.72k] ------------------ 604| 9.45k| if (out_dict_id) *out_dict_id = (src[6] & ZXC_FILE_FLAG_HAS_DICTIONARY) ? zxc_le32(src + 7) : 0; ------------------ | | 367| 9.45k|#define ZXC_FILE_FLAG_HAS_DICTIONARY 0x40U ------------------ | Branch (604:9): [True: 9.45k, False: 0] | Branch (604:37): [True: 0, False: 9.45k] ------------------ 605| | 606| 9.45k| return ZXC_OK; 607| 9.45k|} zxc_write_block_header: 619| 31.1k| const zxc_block_header_t* RESTRICT bh) { 620| 31.1k| if (UNLIKELY(dst_capacity < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 31.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 31.1k] | | ------------------ ------------------ 621| | 622| 31.1k| dst[0] = bh->block_type; 623| 31.1k| dst[1] = 0; // Flags not used currently 624| 31.1k| dst[2] = 0; // Reserved 625| 31.1k| zxc_store_le32(dst + 3, bh->comp_size); 626| 31.1k| dst[7] = 0; // Zero before hashing 627| 31.1k| dst[7] = zxc_hash8(dst); // Checksum at the end 628| | 629| 31.1k| return ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 31.1k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 630| 31.1k|} zxc_read_block_header: 643| 18.9k| zxc_block_header_t* RESTRICT bh) { 644| 18.9k| if (UNLIKELY(src_size < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 18.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 18.9k] | | ------------------ ------------------ 645| | 646| 18.9k| uint8_t temp[ZXC_BLOCK_HEADER_SIZE]; 647| 18.9k| ZXC_MEMCPY(temp, src, ZXC_BLOCK_HEADER_SIZE); ------------------ | | 173| 18.9k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 648| 18.9k| temp[7] = 0; // Zero out checksum byte before hashing 649| 18.9k| if (UNLIKELY(src[7] != zxc_hash8(temp))) return ZXC_ERROR_BAD_HEADER; ------------------ | | 151| 18.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 18.9k] | | ------------------ ------------------ 650| | 651| 18.9k| bh->block_type = src[0]; 652| 18.9k| bh->block_flags = 0; // Flags not used currently 653| 18.9k| bh->reserved = src[2]; 654| 18.9k| bh->comp_size = zxc_le32(src + 3); 655| 18.9k| bh->header_crc = src[7]; 656| | 657| 18.9k| return ZXC_OK; 658| 18.9k|} zxc_write_file_footer: 672| 9.45k| const uint32_t global_hash, const int checksum_enabled) { 673| 9.45k| if (UNLIKELY(dst_capacity < ZXC_FILE_FOOTER_SIZE)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 674| | 675| 9.45k| zxc_store_le64(dst, src_size); 676| | 677| 9.45k| if (checksum_enabled) { ------------------ | Branch (677:9): [True: 4.72k, False: 4.72k] ------------------ 678| 4.72k| zxc_store_le32(dst + sizeof(uint64_t), global_hash); 679| 4.72k| } else { 680| 4.72k| ZXC_MEMSET(dst + sizeof(uint64_t), 0, sizeof(uint32_t)); ------------------ | | 178| 4.72k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 681| 4.72k| } 682| | 683| 9.45k| return ZXC_FILE_FOOTER_SIZE; ------------------ | | 105| 9.45k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 684| 9.45k|} zxc_write_glo_header_and_desc: 697| 5.80k| const zxc_section_desc_t desc[ZXC_GLO_SECTIONS]) { 698| 5.80k| const size_t needed = 699| 5.80k| ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 401| 5.80k|#define ZXC_GLO_HEADER_BINARY_SIZE 16 ------------------ ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 419| 5.80k|#define ZXC_GLO_SECTIONS 4 ------------------ ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 5.80k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 700| | 701| 5.80k| if (UNLIKELY(rem < needed)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 5.80k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 5.80k] | | ------------------ ------------------ 702| | 703| 5.80k| zxc_store_le32(dst, gh->n_sequences); 704| 5.80k| zxc_store_le32(dst + 4, gh->n_literals); 705| | 706| 5.80k| dst[8] = gh->enc_lit; 707| 5.80k| dst[9] = gh->enc_litlen; 708| 5.80k| dst[10] = gh->enc_mlen; 709| 5.80k| dst[11] = gh->enc_off; 710| | 711| 5.80k| zxc_store_le32(dst + 12, 0); 712| 5.80k| uint8_t* p = dst + ZXC_GLO_HEADER_BINARY_SIZE; ------------------ | | 401| 5.80k|#define ZXC_GLO_HEADER_BINARY_SIZE 16 ------------------ 713| | 714| 29.0k| for (int i = 0; i < ZXC_GLO_SECTIONS; i++) { ------------------ | | 419| 29.0k|#define ZXC_GLO_SECTIONS 4 ------------------ | Branch (714:21): [True: 23.2k, False: 5.80k] ------------------ 715| 23.2k| zxc_store_le64(p, desc[i].sizes); 716| 23.2k| p += ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 23.2k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 717| 23.2k| } 718| | 719| 5.80k| return (int)needed; 720| 5.80k|} zxc_read_glo_header_and_desc: 733| 8.99k| zxc_section_desc_t desc[ZXC_GLO_SECTIONS]) { 734| 8.99k| const size_t needed = 735| 8.99k| ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 401| 8.99k|#define ZXC_GLO_HEADER_BINARY_SIZE 16 ------------------ ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 419| 8.99k|#define ZXC_GLO_SECTIONS 4 ------------------ ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 8.99k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 736| | 737| 8.99k| if (UNLIKELY(len < needed)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 8.99k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.99k] | | ------------------ ------------------ 738| | 739| 8.99k| gh->n_sequences = zxc_le32(src); 740| 8.99k| gh->n_literals = zxc_le32(src + 4); 741| 8.99k| gh->enc_lit = src[8]; 742| 8.99k| gh->enc_litlen = src[9]; 743| 8.99k| gh->enc_mlen = src[10]; 744| 8.99k| gh->enc_off = src[11]; 745| | 746| 8.99k| const uint8_t* p = src + ZXC_GLO_HEADER_BINARY_SIZE; ------------------ | | 401| 8.99k|#define ZXC_GLO_HEADER_BINARY_SIZE 16 ------------------ 747| | 748| 44.9k| for (int i = 0; i < ZXC_GLO_SECTIONS; i++) { ------------------ | | 419| 44.9k|#define ZXC_GLO_SECTIONS 4 ------------------ | Branch (748:21): [True: 35.9k, False: 8.99k] ------------------ 749| 35.9k| desc[i].sizes = zxc_le64(p); 750| 35.9k| p += ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 35.9k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 751| 35.9k| } 752| 8.99k| return ZXC_OK; 753| 8.99k|} zxc_write_ghi_header_and_desc: 766| 4.86k| const zxc_section_desc_t desc[ZXC_GHI_SECTIONS]) { 767| 4.86k| const size_t needed = 768| 4.86k| ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 403| 4.86k|#define ZXC_GHI_HEADER_BINARY_SIZE 16 ------------------ ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 421| 4.86k|#define ZXC_GHI_SECTIONS 3 ------------------ ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 4.86k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 769| | 770| 4.86k| if (UNLIKELY(rem < needed)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.86k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.86k] | | ------------------ ------------------ 771| | 772| 4.86k| zxc_store_le32(dst, gh->n_sequences); 773| 4.86k| zxc_store_le32(dst + 4, gh->n_literals); 774| | 775| 4.86k| dst[8] = gh->enc_lit; 776| 4.86k| dst[9] = gh->enc_litlen; 777| 4.86k| dst[10] = gh->enc_mlen; 778| 4.86k| dst[11] = gh->enc_off; 779| | 780| 4.86k| zxc_store_le32(dst + 12, 0); 781| 4.86k| uint8_t* p = dst + ZXC_GHI_HEADER_BINARY_SIZE; ------------------ | | 403| 4.86k|#define ZXC_GHI_HEADER_BINARY_SIZE 16 ------------------ 782| | 783| 19.4k| for (int i = 0; i < ZXC_GHI_SECTIONS; i++) { ------------------ | | 421| 19.4k|#define ZXC_GHI_SECTIONS 3 ------------------ | Branch (783:21): [True: 14.5k, False: 4.86k] ------------------ 784| 14.5k| zxc_store_le64(p, desc[i].sizes); 785| 14.5k| p += ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 14.5k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 786| 14.5k| } 787| | 788| 4.86k| return (int)needed; 789| 4.86k|} zxc_read_ghi_header_and_desc: 802| 8.18k| zxc_section_desc_t desc[ZXC_GHI_SECTIONS]) { 803| 8.18k| const size_t needed = 804| 8.18k| ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 403| 8.18k|#define ZXC_GHI_HEADER_BINARY_SIZE 16 ------------------ ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 421| 8.18k|#define ZXC_GHI_SECTIONS 3 ------------------ ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 8.18k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 805| | 806| 8.18k| if (UNLIKELY(len < needed)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 8.18k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.18k] | | ------------------ ------------------ 807| | 808| 8.18k| gh->n_sequences = zxc_le32(src); 809| 8.18k| gh->n_literals = zxc_le32(src + 4); 810| 8.18k| gh->enc_lit = src[8]; 811| 8.18k| gh->enc_litlen = src[9]; 812| 8.18k| gh->enc_mlen = src[10]; 813| 8.18k| gh->enc_off = src[11]; 814| | 815| 8.18k| const uint8_t* p = src + ZXC_GHI_HEADER_BINARY_SIZE; ------------------ | | 403| 8.18k|#define ZXC_GHI_HEADER_BINARY_SIZE 16 ------------------ 816| | 817| 32.7k| for (int i = 0; i < ZXC_GHI_SECTIONS; i++) { ------------------ | | 421| 32.7k|#define ZXC_GHI_SECTIONS 3 ------------------ | Branch (817:21): [True: 24.5k, False: 8.18k] ------------------ 818| 24.5k| desc[i].sizes = zxc_le64(p); 819| 24.5k| p += ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 24.5k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 820| 24.5k| } 821| 8.18k| return ZXC_OK; 822| 8.18k|} zxc_compress_bound: 842| 9.45k|uint64_t zxc_compress_bound(const size_t input_size) { 843| | // Guard against uint64 overflow when summing per-block overhead 844| | // across very large inputs (input_size approaching SIZE_MAX). 845| 9.45k| if (UNLIKELY(input_size > (SIZE_MAX - (SIZE_MAX >> 8)))) return 0; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 846| 9.45k| uint64_t n = ((uint64_t)input_size + ZXC_BLOCK_SIZE_MIN - 1) / ZXC_BLOCK_SIZE_MIN; ------------------ | | 62| 9.45k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 9.45k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ uint64_t n = ((uint64_t)input_size + ZXC_BLOCK_SIZE_MIN - 1) / ZXC_BLOCK_SIZE_MIN; ------------------ | | 62| 9.45k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 9.45k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ 847| 9.45k| if (n == 0) n = 1; ------------------ | Branch (847:9): [True: 0, False: 9.45k] ------------------ 848| 9.45k| return ZXC_FILE_HEADER_SIZE + ------------------ | | 103| 9.45k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ 849| 9.45k| (n * (ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_CHECKSUM_SIZE + ZXC_BLOCK_FORMAT_OVERHEAD)) + ------------------ | | 397| 9.45k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ (n * (ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_CHECKSUM_SIZE + ZXC_BLOCK_FORMAT_OVERHEAD)) + ------------------ | | 399| 9.45k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ (n * (ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_CHECKSUM_SIZE + ZXC_BLOCK_FORMAT_OVERHEAD)) + ------------------ | | 412| 9.45k|#define ZXC_BLOCK_FORMAT_OVERHEAD 64 ------------------ 850| 9.45k| (uint64_t)input_size + ZXC_BLOCK_HEADER_SIZE + /* EOF block */ ------------------ | | 397| 9.45k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 851| 9.45k| ZXC_BLOCK_HEADER_SIZE + /* SEK block header (seekable) */ ------------------ | | 397| 9.45k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 852| 9.45k| (n * ZXC_SEEK_ENTRY_SIZE) + /* SEK entries: 4 bytes per block */ ------------------ | | 443| 9.45k|#define ZXC_SEEK_ENTRY_SIZE 4 ------------------ 853| 9.45k| ZXC_FILE_FOOTER_SIZE; ------------------ | | 105| 9.45k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 854| 9.45k|} zxc_common.c:compute_cctx_layout: 163| 40.2k| const int defer_entropy_scratch) { 164| 40.2k| zxc_cctx_layout_t layout = {0}; 165| | 166| 40.2k| const size_t max_seq = zxc_cctx_max_seq(chunk_size); 167| | 168| 40.2k| if (mode == 0) { ------------------ | Branch (168:9): [True: 21.3k, False: 18.9k] ------------------ 169| | /* Decompress: work_buf + lit_buffer, padded for wild-copy overshoot and 170| | * sized worst-case. lit_buffer is provisioned at every level - the decoder 171| | * cannot predict a block's literal encoding (RAW / RLE / HUFFMAN). */ 172| 21.3k| const size_t sz_work = chunk_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 349| 21.3k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 21.3k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 173| 21.3k| const size_t sz_lit = chunk_size + ZXC_PAD_SIZE; ------------------ | | 337| 21.3k|#define ZXC_PAD_SIZE 32 ------------------ 174| | 175| 21.3k| layout.off_work = layout.total; 176| 21.3k| layout.total += ZXC_ALIGN_CL(sz_work); ------------------ | | 355| 21.3k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 21.3k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 21.3k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 21.3k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 21.3k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 177| 21.3k| layout.off_lit_dctx = layout.total; 178| 21.3k| layout.total += ZXC_ALIGN_CL(sz_lit); ------------------ | | 355| 21.3k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 21.3k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 21.3k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 21.3k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 21.3k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 179| | /* Token-section decode scratch (level-7 GLO Huffman-codes the tokens) + 180| | * PivCo ping-pong scratch. enc_lit/enc_litlen are unpredictable, so static 181| | * workspaces provision both up front (no-alloc contract); heap contexts 182| | * defer them to the first entropy section, sparing L1-5 archives ~1.2x 183| | * chunk_size. See zxc_cctx_alloc_entropy_scratch. */ 184| 21.3k| if (!defer_entropy_scratch) { ------------------ | Branch (184:13): [True: 0, False: 21.3k] ------------------ 185| 0| size_t sz_tok = 0; 186| 0| size_t sz_pivco = 0; 187| 0| zxc_dctx_entropy_sizes(chunk_size, &sz_tok, &sz_pivco); 188| 0| layout.sz_tok_dctx = sz_tok; 189| 0| layout.off_tok_dctx = layout.total; 190| 0| layout.total += ZXC_ALIGN_CL(layout.sz_tok_dctx); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 191| 0| layout.sz_pivco_dctx = sz_pivco; 192| 0| layout.off_pivco_dctx = layout.total; 193| 0| layout.total += ZXC_ALIGN_CL(layout.sz_pivco_dctx); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 194| 0| } 195| 21.3k| } else { 196| | /* Compress: 6 partitions + optional opt_scratch at level >= ZXC_LEVEL_DENSITY. */ 197| 18.9k| const uint32_t offset_bits = zxc_log2_u32((uint32_t)chunk_size); 198| 18.9k| layout.max_seq = max_seq; 199| 18.9k| layout.sz_hash_pos = ZXC_LZ_HASH_SIZE * sizeof(uint32_t); ------------------ | | 502| 18.9k|#define ZXC_LZ_HASH_SIZE (1U << ZXC_LZ_HASH_BITS) | | ------------------ | | | | 496| 18.9k|#define ZXC_LZ_HASH_BITS 15 | | ------------------ ------------------ 200| 18.9k| layout.sz_hash_tags = ZXC_LZ_HASH_SIZE * sizeof(uint8_t); ------------------ | | 502| 18.9k|#define ZXC_LZ_HASH_SIZE (1U << ZXC_LZ_HASH_BITS) | | ------------------ | | | | 496| 18.9k|#define ZXC_LZ_HASH_BITS 15 | | ------------------ ------------------ 201| 18.9k| const size_t sz_chain = ZXC_LZ_WINDOW_SIZE * sizeof(uint16_t); ------------------ | | 504| 18.9k|#define ZXC_LZ_WINDOW_SIZE (1U << 16) ------------------ 202| | /* buf_sequences (GHI, level <= ZXC_LEVEL_FAST) aliases buf_offsets + buf_tokens (GLO, 203| | * level >= ZXC_LEVEL_DEFAULT). Mutually exclusive per block; sized for the larger. */ 204| 18.9k| const size_t sz_seq_union = layout.max_seq * sizeof(uint32_t); 205| 18.9k| const size_t vbyte_len = (offset_bits + 6) / 7; 206| 18.9k| const size_t sz_extras = layout.max_seq * 2 * vbyte_len; 207| 18.9k| const size_t sz_lit = chunk_size + ZXC_PAD_SIZE; ------------------ | | 337| 18.9k|#define ZXC_PAD_SIZE 32 ------------------ 208| | 209| | /* opt_scratch (level >= ZXC_LEVEL_DENSITY): the optimal parser's DP arrays, 210| | * reused transiently as package-merge scratch by the code-length builder, 211| | * so sized to the larger demand. Keep in sync with zxc_estimate_cctx_size() 212| | * and its consumer in zxc_compress.c. */ 213| 18.9k| if (level >= ZXC_LEVEL_DENSITY) { ------------------ | Branch (213:13): [True: 0, False: 18.9k] ------------------ 214| 0| const size_t sz_dp = ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint32_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 215| 0| const size_t sz_pl = ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint16_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 216| 0| const size_t sz_po = ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint16_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 217| 0| const size_t n_bm_words = ZXC_BITMAP_WORDS(chunk_size + 1); ------------------ | | 362| 0|#define ZXC_BITMAP_WORDS(n_bits) (((n_bits) + 63) / 64) ------------------ 218| 0| const size_t sz_bm = ZXC_ALIGN_CL(n_bm_words * sizeof(uint64_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 219| 0| const size_t dp_needed = sz_dp + sz_pl + sz_po + sz_bm; 220| 0| layout.sz_opt = 221| 0| (dp_needed > ZXC_HUF_BUILD_SCRATCH_SIZE) ? dp_needed : ZXC_HUF_BUILD_SCRATCH_SIZE; ------------------ | | 841| 0| ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 834| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 589| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 842| 0| sizeof(zxc_huf_pm_item_t) + \ | | 843| 0| 8U + (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * sizeof(int) + 8U + \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | 844| 0| (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 834| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 589| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 845| 0| sizeof(zxc_huf_pm_frame_t)) ------------------ (dp_needed > ZXC_HUF_BUILD_SCRATCH_SIZE) ? dp_needed : ZXC_HUF_BUILD_SCRATCH_SIZE; ------------------ | | 841| 0| ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 834| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 589| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 842| 0| sizeof(zxc_huf_pm_item_t) + \ | | 843| 0| 8U + (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * sizeof(int) + 8U + \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | 844| 0| (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 834| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 589| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 845| 0| sizeof(zxc_huf_pm_frame_t)) ------------------ | Branch (221:17): [True: 0, False: 0] ------------------ 222| 0| } 223| | 224| 18.9k| layout.off_hash_pos = layout.total; 225| 18.9k| layout.total += ZXC_ALIGN_CL(layout.sz_hash_pos); ------------------ | | 355| 18.9k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 18.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 18.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 18.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 18.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 226| 18.9k| layout.off_hash_tags = layout.total; 227| 18.9k| layout.total += ZXC_ALIGN_CL(layout.sz_hash_tags); ------------------ | | 355| 18.9k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 18.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 18.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 18.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 18.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 228| 18.9k| layout.off_chain = layout.total; 229| 18.9k| layout.total += ZXC_ALIGN_CL(sz_chain); ------------------ | | 355| 18.9k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 18.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 18.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 18.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 18.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 230| 18.9k| layout.off_seq_union = layout.total; 231| 18.9k| layout.total += ZXC_ALIGN_CL(sz_seq_union); ------------------ | | 355| 18.9k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 18.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 18.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 18.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 18.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 232| 18.9k| layout.off_extras = layout.total; 233| 18.9k| layout.total += ZXC_ALIGN_CL(sz_extras); ------------------ | | 355| 18.9k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 18.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 18.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 18.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 18.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 234| 18.9k| layout.off_lit_cctx = layout.total; 235| 18.9k| layout.total += ZXC_ALIGN_CL(sz_lit); ------------------ | | 355| 18.9k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 18.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 18.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 18.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 18.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 236| | /* opt_scratch is appended last so it is absent for levels 1..5 (zero 237| | * waste on the common path) and only inflates the workspace at level 6. */ 238| 18.9k| if (layout.sz_opt) { ------------------ | Branch (238:13): [True: 0, False: 18.9k] ------------------ 239| 0| layout.off_opt = layout.total; 240| 0| layout.total += ZXC_ALIGN_CL(layout.sz_opt); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 241| 0| } 242| 18.9k| } 243| | 244| | /* [dict | data] concat scratch (dict only). Compress chunk_size already 245| | * spans [dict | block]; decompress prepends dict to a (chunk + PAD) region. */ 246| 40.2k| if (dict_size > 0) { ------------------ | Branch (246:9): [True: 0, False: 40.2k] ------------------ 247| 0| layout.sz_dict = (mode == 1) ? (chunk_size + ZXC_DECOMPRESS_TAIL_PAD) ------------------ | | 349| 0|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 0|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ | Branch (247:26): [True: 0, False: 0] ------------------ 248| 0| : (dict_size + chunk_size + ZXC_DECOMPRESS_TAIL_PAD); ------------------ | | 349| 0|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 0|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 249| 0| layout.off_dict = layout.total; 250| 0| layout.total += ZXC_ALIGN_CL(layout.sz_dict); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 251| 0| layout.off_dict_huf = layout.total; 252| 0| layout.total += ZXC_ALIGN_CL(sizeof(zxc_dict_huf_state_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 253| 0| } 254| 40.2k| return layout; 255| 40.2k|} zxc_common.c:zxc_cctx_max_seq: 144| 40.2k|static ZXC_ALWAYS_INLINE size_t zxc_cctx_max_seq(const size_t chunk_size) { 145| 40.2k| return chunk_size / ZXC_LZ_MIN_MATCH_LEN + 16; ------------------ | | 508| 40.2k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 146| 40.2k|} zxc_compress_chunk_wrapper_default: 2080| 10.6k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 2081| 10.6k| const size_t dict_sz = ctx->dict_size; 2082| 10.6k| const size_t block_sz = src_sz - dict_sz; 2083| 10.6k| const uint8_t* block_data = chunk + dict_sz; 2084| 10.6k| size_t w = 0; 2085| 10.6k| int res = ZXC_OK; 2086| | 2087| 10.6k| if (ctx->compression_level <= 2) ------------------ | Branch (2087:9): [True: 4.86k, False: 5.80k] ------------------ 2088| 4.86k| res = zxc_encode_block_ghi(ctx, chunk, src_sz, dst, dst_cap, &w); 2089| 5.80k| else 2090| 5.80k| res = zxc_encode_block_glo(ctx, chunk, src_sz, dst, dst_cap, &w); 2091| | 2092| | // Check expansion against block data size (excluding dict prefix). 2093| 10.6k| if (UNLIKELY(res != ZXC_OK || w >= block_sz)) { ------------------ | | 151| 21.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.58k, False: 9.08k] | | | Branch (151:42): [True: 0, False: 10.6k] | | | Branch (151:42): [True: 1.58k, False: 9.08k] | | ------------------ ------------------ 2094| 1.58k| res = zxc_encode_block_raw(block_data, block_sz, dst, dst_cap, &w); 2095| 1.58k| if (UNLIKELY(res != ZXC_OK)) return res; ------------------ | | 151| 1.58k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.58k] | | ------------------ ------------------ 2096| 1.58k| } 2097| | 2098| 10.6k| if (ctx->checksum_enabled) { ------------------ | Branch (2098:9): [True: 5.38k, False: 5.28k] ------------------ 2099| | // Calculate checksum on the compressed payload (w currently excludes checksum) 2100| | // Header is at dst, data starts at dst + ZXC_BLOCK_HEADER_SIZE 2101| 5.38k| if (UNLIKELY(w < ZXC_BLOCK_HEADER_SIZE || w + ZXC_BLOCK_CHECKSUM_SIZE > dst_cap)) ------------------ | | 151| 10.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 5.38k] | | | Branch (151:42): [True: 0, False: 5.38k] | | | Branch (151:42): [True: 0, False: 5.38k] | | ------------------ ------------------ 2102| 0| return ZXC_ERROR_OVERFLOW; 2103| | 2104| 5.38k| uint32_t payload_sz = (uint32_t)(w - ZXC_BLOCK_HEADER_SIZE); ------------------ | | 397| 5.38k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2105| 5.38k| uint32_t crc = 2106| 5.38k| zxc_checksum(dst + ZXC_BLOCK_HEADER_SIZE, payload_sz, ZXC_CHECKSUM_RAPIDHASH); ------------------ | | 397| 5.38k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ zxc_checksum(dst + ZXC_BLOCK_HEADER_SIZE, payload_sz, ZXC_CHECKSUM_RAPIDHASH); ------------------ | | 424| 5.38k|#define ZXC_CHECKSUM_RAPIDHASH 0 ------------------ 2107| 5.38k| zxc_store_le32(dst + w, crc); 2108| 5.38k| w += ZXC_BLOCK_CHECKSUM_SIZE; ------------------ | | 399| 5.38k|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ 2109| 5.38k| } 2110| | 2111| 10.6k| return (int)w; 2112| 10.6k|} zxc_compress.c:zxc_encode_block_ghi: 1860| 4.86k| size_t* RESTRICT const out_sz) { 1861| 4.86k| const int level = ctx->compression_level; 1862| 4.86k| const size_t dict_sz = ctx->dict_size; 1863| | 1864| 4.86k| const zxc_lz77_params_t lzp = zxc_get_lz77_params(level); 1865| | 1866| 4.86k| ctx->epoch++; 1867| 4.86k| if (UNLIKELY(ctx->epoch >= ctx->max_epoch)) { ------------------ | | 151| 4.86k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.86k] | | ------------------ ------------------ 1868| 0| ZXC_MEMSET(ctx->hash_table, 0, ZXC_LZ_HASH_SIZE * sizeof(uint32_t)); ------------------ | | 178| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 1869| 0| ZXC_MEMSET(ctx->hash_tags, 0, ZXC_LZ_HASH_SIZE * sizeof(uint8_t)); ------------------ | | 178| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 1870| 0| ctx->epoch = 1; 1871| 0| } 1872| 4.86k| const uint32_t offset_bits = ctx->offset_bits; 1873| 4.86k| const uint32_t offset_mask = ctx->offset_mask; 1874| 4.86k| const uint32_t epoch_mark = ctx->epoch << offset_bits; 1875| | 1876| 4.86k| if (dict_sz > 0) ------------------ | Branch (1876:9): [True: 0, False: 4.86k] ------------------ 1877| 0| zxc_lz_seed_dict(src, dict_sz, ctx->hash_table, ctx->hash_tags, ctx->chain_table, 1878| 0| epoch_mark, offset_mask, level); 1879| | 1880| 4.86k| const uint8_t* ip = src + dict_sz; 1881| 4.86k| const uint8_t* iend = src + src_sz; 1882| 4.86k| const uint8_t* anchor = ip; 1883| 4.86k| const uint8_t* search_limit = iend - ZXC_LZ_SEARCH_MARGIN; ------------------ | | 532| 4.86k|#define ZXC_LZ_SEARCH_MARGIN (sizeof(uint64_t)) ------------------ 1884| | 1885| 4.86k| uint32_t* const hash_table = ctx->hash_table; 1886| 4.86k| uint8_t* const hash_tags = ctx->hash_tags; 1887| 4.86k| uint8_t* const buf_extras = ctx->buf_extras; 1888| 4.86k| uint16_t* const chain_table = ctx->chain_table; 1889| 4.86k| uint8_t* const literals = ctx->literals; 1890| 4.86k| uint32_t* const buf_sequences = ctx->buf_sequences; 1891| | 1892| 4.86k| uint32_t seq_c = 0; 1893| 4.86k| size_t extras_c = 0; 1894| 4.86k| size_t lit_c = 0; 1895| | 1896| 8.70M| while (LIKELY(ip < search_limit)) { ------------------ | | 145| 8.70M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 8.70M, False: 4.86k] | | ------------------ ------------------ 1897| 8.70M| size_t dist = (size_t)(ip - anchor); 1898| 8.70M| size_t step = lzp.step_base + (dist >> lzp.step_shift); 1899| 8.70M| if (UNLIKELY(ip + step >= search_limit)) step = 1; ------------------ | | 151| 8.70M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 613k, False: 8.08M] | | ------------------ ------------------ 1900| | 1901| 8.70M| ZXC_PREFETCH_READ(ip + step * 4 + ZXC_CACHE_LINE_SIZE); ------------------ | | 162| 8.70M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 1902| | 1903| 8.70M| if (LIKELY(ip + step + sizeof(uint64_t) <= iend)) { ------------------ | | 145| 8.70M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 8.70M, False: 0] | | ------------------ ------------------ 1904| 8.70M| const uint64_t v_next = zxc_le64(ip + step); 1905| | // cppcheck-suppress unreadVariable 1906| 8.70M| const uint32_t h_next = zxc_hash_func(v_next, 0); 1907| 8.70M| ZXC_PREFETCH_READ(&hash_tags[h_next]); ------------------ | | 162| 8.70M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 1908| 8.70M| ZXC_PREFETCH_READ(&hash_table[h_next]); ------------------ | | 162| 8.70M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 1909| 8.70M| } 1910| | 1911| 8.70M| const zxc_match_t m = 1912| 8.70M| zxc_lz77_find_best_match(src, ip, iend, search_limit, anchor, hash_table, hash_tags, 1913| 8.70M| chain_table, epoch_mark, offset_mask, level, lzp, 1914| 8.70M| /*last_off=*/0U); 1915| | 1916| 8.70M| if (m.ref) { ------------------ | Branch (1916:13): [True: 1.90M, False: 6.79M] ------------------ 1917| 1.90M| ip -= m.backtrack; 1918| 1.90M| const uint32_t ll = (uint32_t)(ip - anchor); 1919| 1.90M| const uint32_t ml = m.len - ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 1.90M|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1920| 1.90M| const uint32_t off = (uint32_t)(ip - m.ref); 1921| | 1922| 1.90M| if (ll > 0) { ------------------ | Branch (1922:17): [True: 732k, False: 1.17M] ------------------ 1923| 732k| if (LIKELY(anchor + ZXC_PAD_SIZE <= iend)) { ------------------ | | 145| 732k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 731k, False: 948] | | ------------------ ------------------ 1924| 731k| zxc_copy32(literals + lit_c, anchor); 1925| 731k| if (UNLIKELY(ll > ZXC_PAD_SIZE)) { ------------------ | | 151| 731k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 149k, False: 582k] | | ------------------ ------------------ 1926| 149k| ZXC_MEMCPY(literals + lit_c + ZXC_PAD_SIZE, anchor + ZXC_PAD_SIZE, ------------------ | | 173| 149k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1927| 149k| ll - ZXC_PAD_SIZE); 1928| 149k| } 1929| 731k| } else { 1930| 948| ZXC_MEMCPY(literals + lit_c, anchor, ll); ------------------ | | 173| 948|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1931| 948| } 1932| 732k| lit_c += ll; 1933| 732k| } 1934| | 1935| 1.90M| const uint32_t ll_write = (ll >= ZXC_SEQ_LL_MASK) ? 255U : ll; ------------------ | | 469| 1.90M|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 463| 1.90M|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ | Branch (1935:39): [True: 29.6k, False: 1.87M] ------------------ 1936| 1.90M| const uint32_t ml_write = (ml >= ZXC_SEQ_ML_MASK) ? 255U : ml; ------------------ | | 471| 1.90M|#define ZXC_SEQ_ML_MASK ((1U << ZXC_SEQ_ML_BITS) - 1) | | ------------------ | | | | 465| 1.90M|#define ZXC_SEQ_ML_BITS 8 | | ------------------ ------------------ | Branch (1936:39): [True: 116k, False: 1.78M] ------------------ 1937| 1.90M| const uint32_t seq_val = (ll_write << (ZXC_SEQ_ML_BITS + ZXC_SEQ_OFF_BITS)) | ------------------ | | 465| 1.90M|#define ZXC_SEQ_ML_BITS 8 ------------------ const uint32_t seq_val = (ll_write << (ZXC_SEQ_ML_BITS + ZXC_SEQ_OFF_BITS)) | ------------------ | | 467| 1.90M|#define ZXC_SEQ_OFF_BITS 16 ------------------ 1938| 1.90M| (ml_write << ZXC_SEQ_OFF_BITS) | ------------------ | | 467| 1.90M|#define ZXC_SEQ_OFF_BITS 16 ------------------ 1939| 1.90M| ((off - ZXC_LZ_OFFSET_BIAS) & ZXC_SEQ_OFF_MASK); ------------------ | | 527| 1.90M|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ ((off - ZXC_LZ_OFFSET_BIAS) & ZXC_SEQ_OFF_MASK); ------------------ | | 473| 1.90M|#define ZXC_SEQ_OFF_MASK ((1U << ZXC_SEQ_OFF_BITS) - 1) | | ------------------ | | | | 467| 1.90M|#define ZXC_SEQ_OFF_BITS 16 | | ------------------ ------------------ 1940| 1.90M| buf_sequences[seq_c] = seq_val; 1941| 1.90M| seq_c++; 1942| | 1943| 1.90M| if (ll >= ZXC_SEQ_LL_MASK) { ------------------ | | 469| 1.90M|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 463| 1.90M|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ | Branch (1943:17): [True: 29.6k, False: 1.87M] ------------------ 1944| 29.6k| const size_t n = zxc_write_varint(buf_extras + extras_c, ll - ZXC_SEQ_LL_MASK); ------------------ | | 469| 29.6k|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 463| 29.6k|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ 1945| 29.6k| if (UNLIKELY(n == 0)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 29.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 29.6k] | | ------------------ ------------------ 1946| 29.6k| extras_c += n; 1947| 29.6k| } 1948| 1.90M| if (ml >= ZXC_SEQ_ML_MASK) { ------------------ | | 471| 1.90M|#define ZXC_SEQ_ML_MASK ((1U << ZXC_SEQ_ML_BITS) - 1) | | ------------------ | | | | 465| 1.90M|#define ZXC_SEQ_ML_BITS 8 | | ------------------ ------------------ | Branch (1948:17): [True: 116k, False: 1.78M] ------------------ 1949| 116k| const size_t n = zxc_write_varint(buf_extras + extras_c, ml - ZXC_SEQ_ML_MASK); ------------------ | | 471| 116k|#define ZXC_SEQ_ML_MASK ((1U << ZXC_SEQ_ML_BITS) - 1) | | ------------------ | | | | 465| 116k|#define ZXC_SEQ_ML_BITS 8 | | ------------------ ------------------ 1950| 116k| if (UNLIKELY(n == 0)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 116k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 116k] | | ------------------ ------------------ 1951| 116k| extras_c += n; 1952| 116k| } 1953| | 1954| 1.90M| ip += m.len; 1955| 1.90M| anchor = ip; 1956| 6.79M| } else { 1957| 6.79M| ip += step; 1958| 6.79M| } 1959| 8.70M| } 1960| | 1961| 4.86k| const size_t last_lits = iend - anchor; 1962| 4.86k| if (last_lits > 0) { ------------------ | Branch (1962:9): [True: 3.82k, False: 1.04k] ------------------ 1963| 3.82k| ZXC_MEMCPY(literals + lit_c, anchor, last_lits); ------------------ | | 173| 3.82k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1964| 3.82k| lit_c += last_lits; 1965| 3.82k| } 1966| | 1967| 4.86k| zxc_block_header_t bh = {.block_type = ZXC_BLOCK_GHI}; 1968| 4.86k| uint8_t* const p = dst + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 4.86k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1969| 4.86k| size_t rem = dst_cap - ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 4.86k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1970| | 1971| 4.86k| const zxc_gnr_header_t gh = {.n_sequences = seq_c, 1972| 4.86k| .n_literals = (uint32_t)lit_c, 1973| 4.86k| .enc_lit = ZXC_SECTION_ENCODING_RAW, 1974| 4.86k| .enc_litlen = 0, 1975| 4.86k| .enc_mlen = 0, 1976| 4.86k| .enc_off = 0}; 1977| | 1978| 4.86k| zxc_section_desc_t desc[ZXC_GHI_SECTIONS] = {0}; 1979| 4.86k| desc[0].sizes = (uint64_t)lit_c | ((uint64_t)lit_c << 32); 1980| 4.86k| size_t sz_seqs = seq_c * sizeof(uint32_t); 1981| 4.86k| desc[1].sizes = (uint64_t)sz_seqs | ((uint64_t)sz_seqs << 32); 1982| 4.86k| desc[2].sizes = (uint64_t)extras_c | ((uint64_t)extras_c << 32); 1983| | 1984| 4.86k| const int ghs = zxc_write_ghi_header_and_desc(p, rem, &gh, desc); 1985| 4.86k| if (UNLIKELY(ghs < 0)) return ghs; ------------------ | | 151| 4.86k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.86k] | | ------------------ ------------------ 1986| | 1987| 4.86k| uint8_t* p_curr = p + ghs; 1988| 4.86k| rem -= ghs; 1989| | 1990| 4.86k| const size_t sz_lit = (size_t)(desc[0].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 4.86k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 1991| 4.86k| const size_t sz_seq = (size_t)(desc[1].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 4.86k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 1992| 4.86k| const size_t sz_ext = (size_t)(desc[2].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 4.86k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 1993| | 1994| 4.86k| if (UNLIKELY(rem < sz_lit + sz_seq + sz_ext)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.86k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.86k] | | ------------------ ------------------ 1995| | 1996| 4.86k| ZXC_MEMCPY(p_curr, literals, lit_c); ------------------ | | 173| 4.86k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1997| 4.86k| p_curr += lit_c; 1998| 4.86k| rem -= sz_lit; 1999| | 2000| 4.86k| if (UNLIKELY(rem < sz_seq)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 4.86k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.86k] | | ------------------ ------------------ 2001| | // Write sequences in little-endian order 2002| |#ifdef ZXC_BIG_ENDIAN 2003| | for (uint32_t i = 0; i < seq_c; i++) { 2004| | zxc_store_le32(p_curr, buf_sequences[i]); 2005| | p_curr += sizeof(uint32_t); 2006| | } 2007| |#else 2008| 4.86k| ZXC_MEMCPY(p_curr, buf_sequences, sz_seq); ------------------ | | 173| 4.86k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 2009| 4.86k| p_curr += sz_seq; 2010| 4.86k|#endif 2011| | 2012| | // --- WRITE EXTRAS --- 2013| 4.86k| ZXC_MEMCPY(p_curr, buf_extras, sz_ext); ------------------ | | 173| 4.86k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 2014| 4.86k| p_curr += sz_ext; 2015| | 2016| 4.86k| bh.comp_size = (uint32_t)(p_curr - (dst + ZXC_BLOCK_HEADER_SIZE)); ------------------ | | 397| 4.86k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2017| 4.86k| const int hw = zxc_write_block_header(dst, dst_cap, &bh); 2018| 4.86k| if (UNLIKELY(hw < 0)) return hw; ------------------ | | 151| 4.86k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.86k] | | ------------------ ------------------ 2019| | 2020| | // Checksum will be appended by the wrapper 2021| 4.86k| *out_sz = ZXC_BLOCK_HEADER_SIZE + bh.comp_size; ------------------ | | 397| 4.86k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2022| 4.86k| return ZXC_OK; 2023| 4.86k|} zxc_compress.c:zxc_hash_func: 47| 62.8M|static ZXC_ALWAYS_INLINE uint32_t zxc_hash_func(const uint64_t val, const int use_hash5) { 48| 62.8M| if (use_hash5) { ------------------ | Branch (48:9): [True: 45.4M, False: 17.4M] ------------------ 49| 45.4M| const uint64_t v5 = val & 0xFFFFFFFFFFULL; 50| 45.4M| return (uint32_t)((v5 * ZXC_LZ_HASH_PRIME2) >> (64 - ZXC_LZ_HASH_BITS)); ------------------ | | 500| 45.4M|#define ZXC_LZ_HASH_PRIME2 0x2545F4914F6CDD1DULL ------------------ return (uint32_t)((v5 * ZXC_LZ_HASH_PRIME2) >> (64 - ZXC_LZ_HASH_BITS)); ------------------ | | 496| 45.4M|#define ZXC_LZ_HASH_BITS 15 ------------------ 51| 45.4M| } else { 52| 17.4M| const uint64_t v4 = val ^ (val >> 15); 53| 17.4M| return ((uint32_t)v4 * ZXC_LZ_HASH_PRIME1) >> (32 - ZXC_LZ_HASH_BITS); ------------------ | | 498| 17.4M|#define ZXC_LZ_HASH_PRIME1 0x2D35182DU ------------------ return ((uint32_t)v4 * ZXC_LZ_HASH_PRIME1) >> (32 - ZXC_LZ_HASH_BITS); ------------------ | | 496| 17.4M|#define ZXC_LZ_HASH_BITS 15 ------------------ 54| 17.4M| } 55| 62.8M|} zxc_compress.c:zxc_lz77_find_best_match: 193| 28.5M| const int level, const zxc_lz77_params_t p, const uint32_t last_off) { 194| 28.5M| const int use_hash5 = (level >= 3); 195| | // Sentinel length just below the minimum, so any valid match replaces it. 196| 28.5M| zxc_match_t best = (zxc_match_t){NULL, ZXC_LZ_MIN_MATCH_LEN - 1, 0}; ------------------ | | 508| 28.5M|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 197| | 198| 28.5M| uint64_t cur_val8 = zxc_le64(ip); 199| 28.5M| uint32_t cur_val = (uint32_t)cur_val8; 200| 28.5M| uint32_t h = zxc_hash_func(cur_val8, use_hash5); 201| | 202| | // 8-bit tag: XOR fold of first 4 bytes for fast rejection 203| 28.5M| const uint8_t cur_tag = (uint8_t)(cur_val ^ (cur_val >> 16)); 204| | 205| 28.5M| const uint32_t cur_pos = (uint32_t)(ip - src); 206| | 207| | // Tag-first filter on fast levels. 208| 28.5M| const uint8_t stored_tag = hash_tags[h]; 209| 28.5M| uint32_t match_idx; 210| 28.5M| if (level <= ZXC_LEVEL_FAST && stored_tag != cur_tag) { ------------------ | Branch (210:9): [True: 8.70M, False: 19.8M] | Branch (210:36): [True: 5.23M, False: 3.46M] ------------------ 211| 5.23M| match_idx = 0; 212| 23.3M| } else { 213| 23.3M| const uint32_t raw_head = hash_table[h]; 214| 23.3M| match_idx = ((raw_head & ~offset_mask) == epoch_mark) ? (raw_head & offset_mask) : 0; ------------------ | Branch (214:21): [True: 14.7M, False: 8.57M] ------------------ 215| 23.3M| } 216| | 217| | // skip_head still drives the chain walk on level >= 3 (advances past the 218| | // mismatched head without comparing). On level <= 2 it is always 0 here: 219| | // either match_idx == 0 (filter-skip) or stored_tag == cur_tag. 220| 28.5M| const int skip_head = (match_idx != 0) & (stored_tag != cur_tag); 221| | 222| | // Split table writes 223| 28.5M| hash_table[h] = epoch_mark | cur_pos; 224| 28.5M| hash_tags[h] = cur_tag; 225| | 226| | // Branchless chain table update 227| 28.5M| const uint32_t dist = cur_pos - match_idx; 228| 28.5M| const uint32_t valid_mask = -((int32_t)((match_idx != 0) & (dist < ZXC_LZ_WINDOW_SIZE))); ------------------ | | 504| 28.5M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) ------------------ 229| 28.5M| chain_table[cur_pos & ZXC_LZ_WINDOW_MASK] = (uint16_t)(dist & valid_mask); ------------------ | | 506| 28.5M|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 504| 28.5M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 230| | 231| 28.5M| int attempts = p.search_depth; 232| | 233| | /* Repeat-offset seed (level-6 parser passes last_off; others pass 0). 234| | * Probing the previous offset first often finds the longest match right 235| | * away, speeding up the chain walk. It only raises best.len, never lowers 236| | * it, so the result is unchanged - this is purely a speed optimization. */ 237| 28.5M| if (last_off != 0U && last_off <= (uint32_t)ZXC_LZ_MAX_DIST && last_off <= cur_pos) { ------------------ | | 529| 28.5M|#define ZXC_LZ_MAX_DIST (ZXC_LZ_WINDOW_SIZE - 1) | | ------------------ | | | | 504| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ | Branch (237:9): [True: 0, False: 28.5M] | Branch (237:27): [True: 0, False: 0] | Branch (237:68): [True: 0, False: 0] ------------------ 238| 0| const uint8_t* const rep_ref = src + (cur_pos - last_off); 239| 0| if (zxc_le32(rep_ref) == cur_val) { ------------------ | Branch (239:13): [True: 0, False: 0] ------------------ 240| 0| uint32_t mlen = sizeof(uint32_t); 241| 0| const uint8_t* const limit_8 = iend - sizeof(uint64_t); 242| 0| while (ip + mlen < limit_8) { ------------------ | Branch (242:20): [True: 0, False: 0] ------------------ 243| 0| const uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(rep_ref + mlen); 244| 0| if (diff == 0) { ------------------ | Branch (244:21): [True: 0, False: 0] ------------------ 245| 0| mlen += sizeof(uint64_t); 246| 0| } else { 247| 0| mlen += (uint32_t)(zxc_ctz64(diff) >> 3); 248| 0| goto _rep_done; 249| 0| } 250| 0| } 251| 0| while (ip + mlen < iend && rep_ref[mlen] == ip[mlen]) mlen++; ------------------ | Branch (251:20): [True: 0, False: 0] | Branch (251:40): [True: 0, False: 0] ------------------ 252| 0| _rep_done:; 253| 0| best.len = mlen; 254| 0| best.ref = rep_ref; 255| 0| if (UNLIKELY(best.len >= (uint32_t)p.sufficient_len || ip + best.len >= iend)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 256| 0| goto _finalize_match; 257| 0| } 258| 0| } 259| | 260| 28.5M| if (match_idx == 0) goto _finalize_match; ------------------ | Branch (260:9): [True: 13.8M, False: 14.7M] ------------------ 261| | 262| | // Optimization: If head tag doesn't match, advance immediately without loading the first 263| | // mismatch. 264| 14.7M| if (skip_head) { ------------------ | Branch (264:9): [True: 5.19M, False: 9.56M] ------------------ 265| 5.19M| const uint16_t delta = chain_table[match_idx & ZXC_LZ_WINDOW_MASK]; ------------------ | | 506| 5.19M|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 504| 5.19M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 266| 5.19M| const uint32_t next_idx = match_idx - delta; 267| 5.19M| match_idx = (delta != 0) ? next_idx : 0; ------------------ | Branch (267:21): [True: 2.34M, False: 2.84M] ------------------ 268| 5.19M| attempts--; 269| 5.19M| } 270| | 271| 47.8M| while (match_idx > 0) { ------------------ | Branch (271:12): [True: 42.4M, False: 5.33M] ------------------ 272| 42.4M| if (UNLIKELY(attempts-- < 0 || cur_pos - match_idx > ZXC_LZ_MAX_DIST)) break; ------------------ | | 151| 82.6M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 7.87M, False: 34.5M] | | | Branch (151:42): [True: 2.25M, False: 40.2M] | | | Branch (151:42): [True: 5.62M, False: 34.5M] | | ------------------ ------------------ 273| 34.5M| const uint8_t* ref = src + match_idx; 274| | 275| | // Load the next chain link early (before the compare) so its address 276| | // resolves while we prefetch. 277| 34.5M| const uint16_t delta = chain_table[match_idx & ZXC_LZ_WINDOW_MASK]; ------------------ | | 506| 34.5M|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 504| 34.5M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 278| 34.5M| const uint32_t next_idx = match_idx - delta; 279| 34.5M| ZXC_PREFETCH_READ(src + next_idx); ------------------ | | 162| 34.5M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 280| | 281| 34.5M| const uint32_t ref_val = zxc_le32(ref); 282| 34.5M| const int tag_match = (ref_val == cur_val); 283| | // Cheap gate: 4-byte tag match, then check the byte past the current 284| | // best (the && skips that load unless the tag already matched). 285| 34.5M| const int should_compare = tag_match && (ref[best.len] == ip[best.len]); ------------------ | Branch (285:36): [True: 31.8M, False: 2.76M] | Branch (285:49): [True: 11.6M, False: 20.1M] ------------------ 286| | 287| 34.5M| if (should_compare) { ------------------ | Branch (287:13): [True: 11.6M, False: 22.9M] ------------------ 288| 11.6M| uint32_t mlen = sizeof(uint32_t); // We already know the first 4 bytes match 289| | 290| | // Fast path: Scalar 64-bit comparison for short matches (=< 64 bytes) 291| | // Most matches are short, so this avoids SIMD overhead for common cases 292| 11.6M| const uint8_t* limit_8 = iend - sizeof(uint64_t); 293| 11.6M| const uint8_t* scalar_limit = ip + mlen + 64; 294| 11.6M| if (scalar_limit > limit_8) scalar_limit = limit_8; ------------------ | Branch (294:17): [True: 23.5k, False: 11.6M] ------------------ 295| | 296| 21.9M| while (ip + mlen < scalar_limit) { ------------------ | Branch (296:20): [True: 21.3M, False: 526k] ------------------ 297| 21.3M| uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(ref + mlen); 298| 21.3M| if (diff == 0) ------------------ | Branch (298:21): [True: 10.2M, False: 11.1M] ------------------ 299| 10.2M| mlen += sizeof(uint64_t); 300| 11.1M| else { 301| 11.1M| mlen += (zxc_ctz64(diff) >> 3); 302| 11.1M| goto _match_len_done; 303| 11.1M| } 304| 21.3M| } 305| | 306| | // Long match path: Use SIMD for matches exceeding 64 bytes 307| |#if defined(ZXC_USE_AVX512) 308| | const uint8_t* limit_64 = iend - 64; 309| | while (ip + mlen < limit_64) { 310| | const __m512i v_src = _mm512_loadu_si512((const void*)(ip + mlen)); 311| | const __m512i v_ref = _mm512_loadu_si512((const void*)(ref + mlen)); 312| | const __mmask64 mask = _mm512_cmpeq_epi8_mask(v_src, v_ref); 313| | if (mask == 0xFFFFFFFFFFFFFFFF) 314| | mlen += 64; 315| | else { 316| | mlen += (uint32_t)zxc_ctz64(~mask); 317| | goto _match_len_done; 318| | } 319| | } 320| |#elif defined(ZXC_USE_AVX2) 321| | const uint8_t* limit_32 = iend - 32; 322| | while (ip + mlen < limit_32) { 323| | const __m256i v_src = _mm256_loadu_si256((const __m256i*)(ip + mlen)); 324| | const __m256i v_ref = _mm256_loadu_si256((const __m256i*)(ref + mlen)); 325| | const __m256i v_cmp = _mm256_cmpeq_epi8(v_src, v_ref); 326| | const uint32_t mask = (uint32_t)_mm256_movemask_epi8(v_cmp); 327| | if (mask == 0xFFFFFFFF) 328| | mlen += 32; 329| | else { 330| | mlen += zxc_ctz32(~mask); 331| | goto _match_len_done; 332| | } 333| | } 334| |#elif defined(ZXC_USE_SSE2) 335| 526k| const uint8_t* limit_16 = iend - 16; 336| 31.6M| while (ip + mlen < limit_16) { ------------------ | Branch (336:20): [True: 31.6M, False: 5.45k] ------------------ 337| 31.6M| const __m128i v_src = _mm_loadu_si128((const __m128i*)(ip + mlen)); 338| 31.6M| const __m128i v_ref = _mm_loadu_si128((const __m128i*)(ref + mlen)); 339| 31.6M| const __m128i v_cmp = _mm_cmpeq_epi8(v_src, v_ref); 340| 31.6M| const uint32_t mask = (uint32_t)_mm_movemask_epi8(v_cmp); 341| 31.6M| if (mask == 0xFFFFU) ------------------ | Branch (341:21): [True: 31.1M, False: 520k] ------------------ 342| 31.1M| mlen += 16; 343| 520k| else { 344| | // mask != 0xFFFF => a differing byte exists in bits 0..15, 345| | // so the lowest set bit of ~mask lies in that range. 346| 520k| mlen += zxc_ctz32(~mask); 347| 520k| goto _match_len_done; 348| 520k| } 349| 31.6M| } 350| |#elif defined(ZXC_USE_NEON64) 351| | { 352| | const uint8_t* limit_32 = iend - 32; 353| | while (ip + mlen < limit_32) { 354| | const uint8x16_t s0 = vld1q_u8(ip + mlen); 355| | const uint8x16_t r0 = vld1q_u8(ref + mlen); 356| | const uint8x16_t c0 = vceqq_u8(s0, r0); 357| | const uint64_t m0 = vget_lane_u64( 358| | vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(c0), 4)), 0); 359| | if (UNLIKELY(m0 != ~(uint64_t)0)) { 360| | mlen += (uint32_t)(zxc_ctz64(~m0) >> 2); 361| | goto _match_len_done; 362| | } 363| | const uint8x16_t s1 = vld1q_u8(ip + mlen + 16); 364| | const uint8x16_t r1 = vld1q_u8(ref + mlen + 16); 365| | const uint8x16_t c1 = vceqq_u8(s1, r1); 366| | const uint64_t m1 = vget_lane_u64( 367| | vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(c1), 4)), 0); 368| | if (UNLIKELY(m1 != ~(uint64_t)0)) { 369| | mlen += 16 + (uint32_t)(zxc_ctz64(~m1) >> 2); 370| | goto _match_len_done; 371| | } 372| | mlen += 32; 373| | } 374| | if (ip + mlen < iend - 16) { 375| | const uint8x16_t v_src = vld1q_u8(ip + mlen); 376| | const uint8x16_t v_ref = vld1q_u8(ref + mlen); 377| | const uint8x16_t v_cmp = vceqq_u8(v_src, v_ref); 378| | const uint64_t mask = vget_lane_u64( 379| | vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(v_cmp), 4)), 0); 380| | if (LIKELY(mask == ~(uint64_t)0)) 381| | mlen += 16; 382| | else { 383| | mlen += (uint32_t)(zxc_ctz64(~mask) >> 2); 384| | goto _match_len_done; 385| | } 386| | } 387| | } 388| |#elif defined(ZXC_USE_NEON32) 389| | { 390| | const uint8_t* limit_16 = iend - 16; 391| | while (ip + mlen < limit_16) { 392| | const uint8x16_t v_src = vld1q_u8(ip + mlen); 393| | const uint8x16_t v_ref = vld1q_u8(ref + mlen); 394| | const uint8x16_t v_cmp = vceqq_u8(v_src, v_ref); 395| | uint8x8_t p1 = vpmin_u8(vget_low_u8(v_cmp), vget_high_u8(v_cmp)); 396| | uint8x8_t p2 = vpmin_u8(p1, p1); 397| | uint8x8_t p3 = vpmin_u8(p2, p2); 398| | uint8x8_t p4 = vpmin_u8(p3, p3); 399| | uint8_t min_val = vget_lane_u8(p4, 0); 400| | if (min_val == 0xFF) 401| | mlen += 16; 402| | else { 403| | uint8x16_t v_diff = vmvnq_u8(v_cmp); 404| | uint64_t lo = 405| | (uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 0) | 406| | ((uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 1) << 32); 407| | if (lo != 0) 408| | mlen += (zxc_ctz64(lo) >> 3); 409| | else 410| | mlen += 411| | 8 + (zxc_ctz64( 412| | (uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 2) | 413| | ((uint64_t)vgetq_lane_u32(vreinterpretq_u32_u8(v_diff), 3) 414| | << 32)) >> 415| | 3); 416| | goto _match_len_done; 417| | } 418| | } 419| | } 420| |#endif 421| 6.05k| while (ip + mlen < limit_8) { ------------------ | Branch (421:20): [True: 1.00k, False: 5.04k] ------------------ 422| 1.00k| const uint64_t diff = zxc_le64(ip + mlen) ^ zxc_le64(ref + mlen); 423| 1.00k| if (diff == 0) ------------------ | Branch (423:21): [True: 602, False: 407] ------------------ 424| 602| mlen += sizeof(uint64_t); 425| 407| else { 426| 407| mlen += (zxc_ctz64(diff) >> 3); 427| 407| goto _match_len_done; 428| 407| } 429| 1.00k| } 430| 22.5k| while (ip + mlen < iend && ref[mlen] == ip[mlen]) mlen++; ------------------ | Branch (430:20): [True: 20.2k, False: 2.24k] | Branch (430:40): [True: 17.4k, False: 2.79k] ------------------ 431| | 432| 11.6M| _match_len_done:; 433| 11.6M| const int better = (mlen > best.len); 434| 11.6M| best.len = better ? mlen : best.len; ------------------ | Branch (434:24): [True: 7.63M, False: 4.01M] ------------------ 435| 11.6M| best.ref = better ? ref : best.ref; ------------------ | Branch (435:24): [True: 7.63M, False: 4.01M] ------------------ 436| | 437| 11.6M| if (UNLIKELY(best.len >= (uint32_t)p.sufficient_len || ip + best.len >= iend)) break; ------------------ | | 151| 21.7M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.55M, False: 10.0M] | | | Branch (151:42): [True: 1.55M, False: 10.0M] | | | Branch (151:42): [True: 939, False: 10.0M] | | ------------------ ------------------ 438| 11.6M| } 439| | 440| 33.0M| match_idx = (delta != 0) ? next_idx : 0; ------------------ | Branch (440:21): [True: 30.5M, False: 2.48M] ------------------ 441| 33.0M| } 442| | 443| 28.5M|_finalize_match: 444| 28.5M| if (best.ref) { ------------------ | Branch (444:9): [True: 5.50M, False: 23.0M] ------------------ 445| | // Backtrack to extend match backwards 446| 5.50M| const uint8_t* b_ip = ip; 447| 5.50M| const uint8_t* b_ref = best.ref; 448| 11.2M| while (b_ip > anchor && b_ref > src && b_ip[-1] == b_ref[-1]) { ------------------ | Branch (448:16): [True: 8.24M, False: 3.04M] | Branch (448:33): [True: 8.24M, False: 1.95k] | Branch (448:48): [True: 5.77M, False: 2.46M] ------------------ 449| 5.77M| b_ip--; 450| 5.77M| b_ref--; 451| 5.77M| best.len++; 452| 5.77M| best.backtrack++; 453| 5.77M| } 454| 5.50M| best.ref = b_ref; 455| 5.50M| } 456| | 457| 28.5M| if (p.use_lazy && best.ref && best.len < (uint32_t)p.lazy_len_threshold && ------------------ | Branch (457:9): [True: 19.8M, False: 8.70M] | Branch (457:23): [True: 3.60M, False: 16.2M] | Branch (457:35): [True: 3.42M, False: 176k] ------------------ 458| 3.42M| ip + 1 < search_limit) { ------------------ | Branch (458:9): [True: 3.42M, False: 328] ------------------ 459| | // --- Lazy evaluation at ip+1 --- 460| 3.42M| const uint64_t next_val8 = zxc_le64(ip + 1); 461| 3.42M| const uint32_t next_val = (uint32_t)next_val8; 462| 3.42M| const uint32_t h2 = zxc_hash_func(next_val8, use_hash5); 463| 3.42M| const uint8_t next_stored_tag = hash_tags[h2]; 464| 3.42M| const uint32_t next_head = hash_table[h2]; 465| 3.42M| uint32_t next_idx = 466| 3.42M| (next_head & ~offset_mask) == epoch_mark ? (next_head & offset_mask) : 0; ------------------ | Branch (466:13): [True: 3.19M, False: 227k] ------------------ 467| 3.42M| const uint8_t next_tag = (uint8_t)(next_val ^ (next_val >> 16)); 468| 3.42M| const int skip_lazy_head = (next_idx > 0 && next_stored_tag != next_tag); ------------------ | Branch (468:37): [True: 3.19M, False: 227k] | Branch (468:53): [True: 164k, False: 3.03M] ------------------ 469| 3.42M| uint32_t max_lazy2 = 0; 470| 3.42M| int lazy_att = p.lazy_attempts; 471| 3.42M| int is_lazy_first = 1; 472| | 473| 18.1M| while (next_idx > 0) { ------------------ | Branch (473:16): [True: 17.9M, False: 227k] ------------------ 474| 17.9M| if (UNLIKELY(lazy_att-- <= 0 || (uint32_t)(ip + 1 - src) - next_idx > ZXC_LZ_MAX_DIST)) ------------------ | | 151| 33.8M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 2.32M, False: 15.6M] | | | Branch (151:42): [True: 2.05M, False: 15.8M] | | | Branch (151:42): [True: 267k, False: 15.6M] | | ------------------ ------------------ 475| 2.32M| break; 476| 15.6M| const uint8_t* ref2 = src + next_idx; 477| | 478| 15.6M| if ((!is_lazy_first || !skip_lazy_head) && zxc_le32(ref2) == next_val) { ------------------ | Branch (478:18): [True: 12.5M, False: 3.06M] | Branch (478:36): [True: 2.93M, False: 127k] | Branch (478:56): [True: 15.2M, False: 209k] ------------------ 479| 15.2M| uint32_t l2 = sizeof(uint32_t); 480| 15.2M| const uint8_t* limit = iend - sizeof(uint64_t); 481| | 482| 38.2M| while (ip + 1 + l2 < limit) { ------------------ | Branch (482:24): [True: 38.2M, False: 6.70k] ------------------ 483| 38.2M| const uint64_t v1 = zxc_le64(ip + 1 + l2); 484| 38.2M| const uint64_t v2 = zxc_le64(ref2 + l2); 485| 38.2M| if (v1 != v2) { ------------------ | Branch (485:25): [True: 15.2M, False: 23.0M] ------------------ 486| 15.2M| l2 += (uint32_t)(zxc_ctz64(v1 ^ v2) >> 3); 487| 15.2M| goto lazy2_done; 488| 15.2M| } 489| 23.0M| l2 += sizeof(uint64_t); 490| 23.0M| } 491| 28.9k| while (ip + 1 + l2 < iend && ref2[l2] == ip[1 + l2]) l2++; ------------------ | Branch (491:24): [True: 26.2k, False: 2.69k] | Branch (491:46): [True: 22.2k, False: 4.01k] ------------------ 492| 15.2M| lazy2_done: 493| 15.2M| max_lazy2 = l2 > max_lazy2 ? l2 : max_lazy2; ------------------ | Branch (493:29): [True: 3.93M, False: 11.3M] ------------------ 494| 15.2M| } 495| | 496| 15.6M| const uint16_t delta = chain_table[next_idx & ZXC_LZ_WINDOW_MASK]; ------------------ | | 506| 15.6M|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 504| 15.6M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 497| 15.6M| if (UNLIKELY(delta == 0)) break; ------------------ | | 151| 15.6M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 873k, False: 14.7M] | | ------------------ ------------------ 498| 14.7M| next_idx -= delta; 499| 14.7M| is_lazy_first = 0; 500| 14.7M| } 501| | 502| | // --- Lazy evaluation at ip+2 (computed in parallel, no dependency on lazy 1) --- 503| 3.42M| uint32_t max_lazy3 = 0; 504| 3.42M| if (level >= ZXC_LEVEL_BALANCED && ip + 2 < search_limit) { ------------------ | Branch (504:13): [True: 1.41M, False: 2.01M] | Branch (504:44): [True: 1.41M, False: 153] ------------------ 505| 1.41M| const uint64_t val3_8 = zxc_le64(ip + 2); 506| 1.41M| const uint32_t val3 = (uint32_t)val3_8; 507| 1.41M| const uint32_t h3 = zxc_hash_func(val3_8, use_hash5); 508| 1.41M| const uint8_t tag3 = hash_tags[h3]; 509| 1.41M| const uint32_t head3 = hash_table[h3]; 510| 1.41M| uint32_t idx3 = (head3 & ~offset_mask) == epoch_mark ? (head3 & offset_mask) : 0; ------------------ | Branch (510:29): [True: 1.20M, False: 205k] ------------------ 511| 1.41M| const uint8_t cur_tag3 = (uint8_t)(val3 ^ (val3 >> 16)); 512| 1.41M| const int skip_head3 = (idx3 > 0 && tag3 != cur_tag3); ------------------ | Branch (512:37): [True: 1.20M, False: 206k] | Branch (512:49): [True: 144k, False: 1.06M] ------------------ 513| | 514| 1.41M| int is_first3 = 1; 515| 1.41M| lazy_att = p.lazy_attempts; 516| 8.69M| while (idx3 > 0) { ------------------ | Branch (516:20): [True: 8.49M, False: 206k] ------------------ 517| 8.49M| if (UNLIKELY(lazy_att-- <= 0 || (uint32_t)(ip + 2 - src) - idx3 > ZXC_LZ_MAX_DIST)) ------------------ | | 151| 16.4M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 733k, False: 7.75M] | | | Branch (151:42): [True: 570k, False: 7.92M] | | | Branch (151:42): [True: 162k, False: 7.75M] | | ------------------ ------------------ 518| 733k| break; 519| | 520| 7.75M| const uint8_t* ref3 = src + idx3; 521| 7.75M| if ((!is_first3 || !skip_head3) && zxc_le32(ref3) == val3) { ------------------ | Branch (521:22): [True: 6.63M, False: 1.12M] | Branch (521:36): [True: 1.01M, False: 111k] | Branch (521:52): [True: 7.50M, False: 143k] ------------------ 522| 7.50M| uint32_t l3 = sizeof(uint32_t); 523| 7.50M| const uint8_t* limit = iend - sizeof(uint64_t); 524| | 525| 18.2M| while (ip + 2 + l3 < limit) { ------------------ | Branch (525:28): [True: 18.2M, False: 3.51k] ------------------ 526| 18.2M| const uint64_t v1 = zxc_le64(ip + 2 + l3); 527| 18.2M| const uint64_t v2 = zxc_le64(ref3 + l3); 528| 18.2M| if (v1 != v2) { ------------------ | Branch (528:29): [True: 7.50M, False: 10.7M] ------------------ 529| 7.50M| l3 += (uint32_t)(zxc_ctz64(v1 ^ v2) >> 3); 530| 7.50M| goto lazy3_done; 531| 7.50M| } 532| 10.7M| l3 += sizeof(uint64_t); 533| 10.7M| } 534| 14.8k| while (ip + 2 + l3 < iend && ref3[l3] == ip[2 + l3]) l3++; ------------------ | Branch (534:28): [True: 13.5k, False: 1.28k] | Branch (534:50): [True: 11.3k, False: 2.22k] ------------------ 535| 7.50M| lazy3_done: 536| 7.50M| max_lazy3 = l3 > max_lazy3 ? l3 : max_lazy3; ------------------ | Branch (536:33): [True: 1.41M, False: 6.08M] ------------------ 537| 7.50M| } 538| | 539| 7.75M| const uint16_t delta = chain_table[idx3 & ZXC_LZ_WINDOW_MASK]; ------------------ | | 506| 7.75M|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 504| 7.75M|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 540| 7.75M| if (UNLIKELY(delta == 0)) break; ------------------ | | 151| 7.75M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 474k, False: 7.28M] | | ------------------ ------------------ 541| 7.28M| idx3 -= delta; 542| 7.28M| is_first3 = 0; 543| 7.28M| } 544| 1.41M| } 545| | 546| | // Single decision: invalidate if either lazy position found a better match 547| 3.42M| if (max_lazy2 > best.len + 1 || max_lazy3 > best.len + 2) best.ref = NULL; ------------------ | Branch (547:13): [True: 551k, False: 2.87M] | Branch (547:41): [True: 40.8k, False: 2.83M] ------------------ 548| 3.42M| } 549| | 550| 28.5M| return best; 551| 28.5M|} zxc_compress.c:zxc_write_varint: 119| 1.15M|static ZXC_ALWAYS_INLINE size_t zxc_write_varint(uint8_t* RESTRICT dst, const uint32_t val) { 120| | // Values above ZXC_MAX_VARINT_VALUE are out-of-spec and the decoder would 121| | // reject them. Unreachable from the Block API, so this is defence in depth; 122| | // callers must treat a return of 0 as an encoding error. 123| 1.15M| if (UNLIKELY(val > ZXC_MAX_VARINT_VALUE)) { ------------------ | | 151| 1.15M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.15M] | | ------------------ ------------------ 124| 0| return 0; 125| 0| } 126| | 127| | // 1 byte: 0xxxxxxx (7 bits) = 2^7 = 128 128| 1.15M| if (LIKELY(val < (1U << 7))) { ------------------ | | 145| 1.15M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 852k, False: 304k] | | ------------------ ------------------ 129| 852k| dst[0] = (uint8_t)val; 130| 852k| return 1; 131| 852k| } 132| | 133| | // 2 bytes: 10xxxxxx xxxxxxxx (14 bits) = 2^14 = 16384 134| 304k| if (LIKELY(val < (1U << 14))) { ------------------ | | 145| 304k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 297k, False: 7.17k] | | ------------------ ------------------ 135| 297k| dst[0] = (uint8_t)(0x80 | (val & 0x3F)); 136| 297k| dst[1] = (uint8_t)(val >> 6); 137| 297k| return 2; 138| 297k| } 139| | 140| | // 3 bytes: 110xxxxx xxxxxxxx xxxxxxxx (21 bits) -> max emittable value, 141| | // matching ZXC_MAX_VARINT_VALUE = ZXC_BLOCK_SIZE_MAX - 1. 142| 7.17k| dst[0] = (uint8_t)(0xC0 | (val & 0x1F)); 143| 7.17k| dst[1] = (uint8_t)(val >> 5); 144| 7.17k| dst[2] = (uint8_t)(val >> 13); 145| 7.17k| return 3; 146| 304k|} zxc_compress.c:zxc_encode_block_glo: 1155| 5.80k| size_t* RESTRICT out_sz) { 1156| 5.80k| const int level = ctx->compression_level; 1157| 5.80k| const size_t dict_sz = ctx->dict_size; 1158| | 1159| 5.80k| const zxc_lz77_params_t lzp = zxc_get_lz77_params(level); 1160| | 1161| 5.80k| ctx->epoch++; 1162| 5.80k| if (UNLIKELY(ctx->epoch >= ctx->max_epoch)) { ------------------ | | 151| 5.80k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 5.80k] | | ------------------ ------------------ 1163| 0| ZXC_MEMSET(ctx->hash_table, 0, ZXC_LZ_HASH_SIZE * sizeof(uint32_t)); ------------------ | | 178| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 1164| 0| ZXC_MEMSET(ctx->hash_tags, 0, ZXC_LZ_HASH_SIZE * sizeof(uint8_t)); ------------------ | | 178| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 1165| 0| ctx->epoch = 1; 1166| 0| } 1167| 5.80k| const uint32_t offset_bits = ctx->offset_bits; 1168| 5.80k| const uint32_t offset_mask = ctx->offset_mask; 1169| 5.80k| const uint32_t epoch_mark = ctx->epoch << offset_bits; 1170| | 1171| 5.80k| if (dict_sz > 0) ------------------ | Branch (1171:9): [True: 0, False: 5.80k] ------------------ 1172| 0| zxc_lz_seed_dict(src, dict_sz, ctx->hash_table, ctx->hash_tags, ctx->chain_table, 1173| 0| epoch_mark, offset_mask, level); 1174| | 1175| 5.80k| const uint8_t* ip = src + dict_sz; 1176| 5.80k| const uint8_t* iend = src + src_sz; 1177| 5.80k| const uint8_t* anchor = ip; 1178| 5.80k| const uint8_t* search_limit = iend - ZXC_LZ_SEARCH_MARGIN; ------------------ | | 532| 5.80k|#define ZXC_LZ_SEARCH_MARGIN (sizeof(uint64_t)) ------------------ 1179| | 1180| 5.80k| uint32_t* const hash_table = ctx->hash_table; 1181| 5.80k| uint8_t* const hash_tags = ctx->hash_tags; 1182| 5.80k| uint16_t* const chain_table = ctx->chain_table; 1183| 5.80k| uint8_t* const literals = ctx->literals; 1184| 5.80k| uint8_t* const buf_tokens = ctx->buf_tokens; 1185| 5.80k| uint16_t* const buf_offsets = ctx->buf_offsets; 1186| 5.80k| uint8_t* const buf_extras = ctx->buf_extras; 1187| | 1188| 5.80k| uint32_t seq_c = 0; 1189| 5.80k| size_t lit_c = 0; 1190| 5.80k| size_t extras_sz = 0; 1191| 5.80k| uint16_t max_offset = 0; // Track max offset for 1-byte/2-byte mode decision 1192| | 1193| | /* Level 6+: price-based optimal parser (fills outputs and skips the 1194| | * lazy loop + last_lits handling below via `goto parse_done`). */ 1195| 5.80k| if (level >= ZXC_LEVEL_DENSITY) { ------------------ | Branch (1195:9): [True: 0, False: 5.80k] ------------------ 1196| 0| const int rc = zxc_lz77_optimal_parse_glo( 1197| 0| ctx, src, src_sz, hash_table, hash_tags, chain_table, epoch_mark, offset_mask, level, 1198| 0| literals, buf_tokens, buf_offsets, buf_extras, &seq_c, &lit_c, &extras_sz, &max_offset); 1199| 0| if (UNLIKELY(rc != 0)) return rc; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1200| 0| goto parse_done; 1201| 0| } 1202| | 1203| 19.8M| while (LIKELY(ip < search_limit)) { ------------------ | | 145| 19.8M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 19.8M, False: 5.80k] | | ------------------ ------------------ 1204| 19.8M| const size_t dist = (size_t)(ip - anchor); 1205| 19.8M| size_t step = lzp.step_base + (dist >> lzp.step_shift); 1206| 19.8M| if (UNLIKELY(ip + step >= search_limit)) step = 1; ------------------ | | 151| 19.8M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 465k, False: 19.4M] | | ------------------ ------------------ 1207| | 1208| 19.8M| if (LIKELY(ip + step + sizeof(uint64_t) <= iend)) { ------------------ | | 145| 19.8M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 19.8M, False: 0] | | ------------------ ------------------ 1209| 19.8M| const uint64_t v_next = zxc_le64(ip + step); 1210| | // cppcheck-suppress unreadVariable 1211| 19.8M| const uint32_t h_next = zxc_hash_func(v_next, 1); 1212| 19.8M| ZXC_PREFETCH_READ(&hash_tags[h_next]); ------------------ | | 162| 19.8M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 1213| 19.8M| ZXC_PREFETCH_READ(&hash_table[h_next]); ------------------ | | 162| 19.8M|#define ZXC_PREFETCH_READ(ptr) __builtin_prefetch((const void*)(ptr), 0, 3) ------------------ 1214| 19.8M| } 1215| | 1216| 19.8M| const zxc_match_t m = 1217| 19.8M| zxc_lz77_find_best_match(src, ip, iend, search_limit, anchor, hash_table, hash_tags, 1218| 19.8M| chain_table, epoch_mark, offset_mask, level, lzp, 1219| 19.8M| /*last_off=*/0U); 1220| | 1221| 19.8M| if (m.ref) { ------------------ | Branch (1221:13): [True: 3.00M, False: 16.8M] ------------------ 1222| 3.00M| ip -= m.backtrack; 1223| 3.00M| const uint32_t ll = (uint32_t)(ip - anchor); 1224| 3.00M| const uint32_t ml = m.len - ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 3.00M|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1225| 3.00M| const uint32_t off = (uint32_t)(ip - m.ref); 1226| | 1227| 3.00M| if (ll > 0) { ------------------ | Branch (1227:17): [True: 1.49M, False: 1.51M] ------------------ 1228| 1.49M| if (LIKELY(anchor + ZXC_PAD_SIZE <= iend)) { ------------------ | | 145| 1.49M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 1.49M, False: 2.40k] | | ------------------ ------------------ 1229| 1.49M| zxc_copy32(literals + lit_c, anchor); 1230| 1.49M| if (UNLIKELY(ll > ZXC_PAD_SIZE)) { ------------------ | | 151| 1.49M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 105k, False: 1.38M] | | ------------------ ------------------ 1231| 105k| ZXC_MEMCPY(literals + lit_c + ZXC_PAD_SIZE, anchor + ZXC_PAD_SIZE, ------------------ | | 173| 105k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1232| 105k| ll - ZXC_PAD_SIZE); 1233| 105k| } 1234| 1.49M| } else { 1235| 2.40k| ZXC_MEMCPY(literals + lit_c, anchor, ll); ------------------ | | 173| 2.40k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1236| 2.40k| } 1237| 1.49M| lit_c += ll; 1238| 1.49M| } 1239| | 1240| 3.00M| const uint8_t ll_code = (ll >= ZXC_TOKEN_LL_MASK) ? ZXC_TOKEN_LL_MASK : (uint8_t)ll; ------------------ | | 454| 3.00M|#define ZXC_TOKEN_LL_MASK ((1U << ZXC_TOKEN_LIT_BITS) - 1) | | ------------------ | | | | 450| 3.00M|#define ZXC_TOKEN_LIT_BITS 4 | | ------------------ ------------------ const uint8_t ll_code = (ll >= ZXC_TOKEN_LL_MASK) ? ZXC_TOKEN_LL_MASK : (uint8_t)ll; ------------------ | | 454| 224k|#define ZXC_TOKEN_LL_MASK ((1U << ZXC_TOKEN_LIT_BITS) - 1) | | ------------------ | | | | 450| 224k|#define ZXC_TOKEN_LIT_BITS 4 | | ------------------ ------------------ | Branch (1240:37): [True: 224k, False: 2.78M] ------------------ 1241| 3.00M| const uint8_t ml_code = (ml >= ZXC_TOKEN_ML_MASK) ? ZXC_TOKEN_ML_MASK : (uint8_t)ml; ------------------ | | 456| 3.00M|#define ZXC_TOKEN_ML_MASK ((1U << ZXC_TOKEN_ML_BITS) - 1) | | ------------------ | | | | 452| 3.00M|#define ZXC_TOKEN_ML_BITS 4 | | ------------------ ------------------ const uint8_t ml_code = (ml >= ZXC_TOKEN_ML_MASK) ? ZXC_TOKEN_ML_MASK : (uint8_t)ml; ------------------ | | 456| 787k|#define ZXC_TOKEN_ML_MASK ((1U << ZXC_TOKEN_ML_BITS) - 1) | | ------------------ | | | | 452| 787k|#define ZXC_TOKEN_ML_BITS 4 | | ------------------ ------------------ | Branch (1241:37): [True: 787k, False: 2.22M] ------------------ 1242| 3.00M| buf_tokens[seq_c] = (ll_code << ZXC_TOKEN_LIT_BITS) | ml_code; ------------------ | | 450| 3.00M|#define ZXC_TOKEN_LIT_BITS 4 ------------------ 1243| 3.00M| buf_offsets[seq_c] = (uint16_t)(off - ZXC_LZ_OFFSET_BIAS); ------------------ | | 527| 3.00M|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1244| 3.00M| if ((off - ZXC_LZ_OFFSET_BIAS) > max_offset) ------------------ | | 527| 3.00M|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ | Branch (1244:17): [True: 48.0k, False: 2.96M] ------------------ 1245| 48.0k| max_offset = (uint16_t)(off - ZXC_LZ_OFFSET_BIAS); ------------------ | | 527| 48.0k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1246| | 1247| 3.00M| if (ll >= ZXC_TOKEN_LL_MASK) { ------------------ | | 454| 3.00M|#define ZXC_TOKEN_LL_MASK ((1U << ZXC_TOKEN_LIT_BITS) - 1) | | ------------------ | | | | 450| 3.00M|#define ZXC_TOKEN_LIT_BITS 4 | | ------------------ ------------------ | Branch (1247:17): [True: 224k, False: 2.78M] ------------------ 1248| 224k| const size_t n = zxc_write_varint(buf_extras + extras_sz, ll - ZXC_TOKEN_LL_MASK); ------------------ | | 454| 224k|#define ZXC_TOKEN_LL_MASK ((1U << ZXC_TOKEN_LIT_BITS) - 1) | | ------------------ | | | | 450| 224k|#define ZXC_TOKEN_LIT_BITS 4 | | ------------------ ------------------ 1249| 224k| if (UNLIKELY(n == 0)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 224k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 224k] | | ------------------ ------------------ 1250| 224k| extras_sz += n; 1251| 224k| } 1252| 3.00M| if (ml >= ZXC_TOKEN_ML_MASK) { ------------------ | | 456| 3.00M|#define ZXC_TOKEN_ML_MASK ((1U << ZXC_TOKEN_ML_BITS) - 1) | | ------------------ | | | | 452| 3.00M|#define ZXC_TOKEN_ML_BITS 4 | | ------------------ ------------------ | Branch (1252:17): [True: 787k, False: 2.22M] ------------------ 1253| 787k| const size_t n = zxc_write_varint(buf_extras + extras_sz, ml - ZXC_TOKEN_ML_MASK); ------------------ | | 456| 787k|#define ZXC_TOKEN_ML_MASK ((1U << ZXC_TOKEN_ML_BITS) - 1) | | ------------------ | | | | 452| 787k|#define ZXC_TOKEN_ML_BITS 4 | | ------------------ ------------------ 1254| 787k| if (UNLIKELY(n == 0)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 787k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 787k] | | ------------------ ------------------ 1255| 787k| extras_sz += n; 1256| 787k| } 1257| | 1258| 3.00M| seq_c++; 1259| | 1260| 3.00M| if (m.len > 2 && level > ZXC_LEVEL_BALANCED) { ------------------ | Branch (1260:17): [True: 3.00M, False: 0] | Branch (1260:30): [True: 889k, False: 2.12M] ------------------ 1261| 889k| const uint8_t* match_end = ip + m.len; 1262| 889k| if (match_end < iend - 7) { ------------------ | Branch (1262:21): [True: 888k, False: 818] ------------------ 1263| 888k| const uint32_t pos_u = (uint32_t)((match_end - 2) - src); 1264| 888k| const uint64_t val_u8 = zxc_le64(match_end - 2); 1265| 888k| const uint32_t val_u = (uint32_t)val_u8; 1266| 888k| const uint32_t h_u = zxc_hash_func(val_u8, 1); 1267| 888k| const uint32_t prev_head = hash_table[h_u]; 1268| 888k| const uint32_t prev_idx = 1269| 888k| (prev_head & ~offset_mask) == epoch_mark ? (prev_head & offset_mask) : 0; ------------------ | Branch (1269:25): [True: 672k, False: 215k] ------------------ 1270| 888k| hash_table[h_u] = epoch_mark | pos_u; 1271| 888k| hash_tags[h_u] = (uint8_t)(val_u ^ (val_u >> 16)); 1272| 888k| chain_table[pos_u & ZXC_LZ_WINDOW_MASK] = ------------------ | | 506| 888k|#define ZXC_LZ_WINDOW_MASK (ZXC_LZ_WINDOW_SIZE - 1U) | | ------------------ | | | | 504| 888k|#define ZXC_LZ_WINDOW_SIZE (1U << 16) | | ------------------ ------------------ 1273| 888k| (prev_idx > 0 && (pos_u - prev_idx) < ZXC_LZ_WINDOW_SIZE) ------------------ | | 504| 672k|#define ZXC_LZ_WINDOW_SIZE (1U << 16) ------------------ | Branch (1273:26): [True: 672k, False: 215k] | Branch (1273:42): [True: 528k, False: 144k] ------------------ 1274| 888k| ? (uint16_t)(pos_u - prev_idx) 1275| 888k| : 0; 1276| 888k| } 1277| 889k| } 1278| | 1279| 3.00M| ip += m.len; 1280| 3.00M| anchor = ip; 1281| 16.8M| } else { 1282| 16.8M| ip += step; 1283| 16.8M| } 1284| 19.8M| } 1285| | 1286| 5.80k| const size_t last_lits = iend - anchor; 1287| 5.80k| if (last_lits > 0) { ------------------ | Branch (1287:9): [True: 4.60k, False: 1.19k] ------------------ 1288| 4.60k| ZXC_MEMCPY(literals + lit_c, anchor, last_lits); ------------------ | | 173| 4.60k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1289| 4.60k| lit_c += last_lits; 1290| 4.60k| } 1291| | 1292| 5.80k|parse_done:; 1293| | /* Dictionary-table trainer hook: accumulate the REAL post-LZ literal 1294| | * frequencies (see zxc_train_dict_huf). Cold path, NULL outside training. */ 1295| 5.80k| if (UNLIKELY(ctx->lit_freq_acc != NULL)) { ------------------ | | 151| 5.80k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 5.80k] | | ------------------ ------------------ 1296| 0| for (size_t i = 0; i < lit_c; i++) ctx->lit_freq_acc[literals[i]]++; ------------------ | Branch (1296:28): [True: 0, False: 0] ------------------ 1297| 0| } 1298| | 1299| | // --- RLE ANALYSIS --- 1300| 5.80k| size_t rle_size = 0; 1301| 5.80k| int enc_lit = ZXC_SECTION_ENCODING_RAW; 1302| 5.80k| size_t best_j = lit_c; 1303| | 1304| 5.80k| if (lit_c > 0) { ------------------ | Branch (1304:9): [True: 5.80k, False: 0] ------------------ 1305| 5.80k| const uint8_t* p = literals; 1306| 5.80k| const uint8_t* const p_end = literals + lit_c; 1307| 5.80k| const uint8_t* const p_end_4 = p_end - 3; // Safe limit for 4-byte lookahead 1308| | 1309| 305k| while (LIKELY(p < p_end)) { ------------------ | | 145| 305k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 300k, False: 5.80k] | | ------------------ ------------------ 1310| 300k| const uint8_t b = *p; 1311| 300k| const uint8_t* run_start = p++; 1312| | 1313| | // Fast run counting with early SIMD exit 1314| |#if defined(ZXC_USE_AVX512) 1315| | const __m512i vb = _mm512_set1_epi8((char)b); 1316| | while (p <= p_end - 64) { 1317| | const __m512i v = _mm512_loadu_si512((const void*)p); 1318| | const __mmask64 mask = _mm512_cmpeq_epi8_mask(v, vb); 1319| | if (mask != 0xFFFFFFFFFFFFFFFFULL) { 1320| | p += (size_t)zxc_ctz64(~mask); 1321| | goto _run_done; 1322| | } 1323| | p += 64; 1324| | } 1325| |#elif defined(ZXC_USE_AVX2) 1326| | const __m256i vb = _mm256_set1_epi8((char)b); 1327| | while (p <= p_end - 32) { 1328| | const __m256i v = _mm256_loadu_si256((const __m256i*)p); 1329| | const uint32_t mask = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, vb)); 1330| | if (mask != 0xFFFFFFFF) { 1331| | p += zxc_ctz32(~mask); 1332| | goto _run_done; 1333| | } 1334| | p += 32; 1335| | } 1336| |#elif defined(ZXC_USE_SSE2) 1337| | const __m128i vb = _mm_set1_epi8((char)b); 1338| 622k| while (p <= p_end - 16) { ------------------ | Branch (1338:20): [True: 617k, False: 4.66k] ------------------ 1339| 617k| const __m128i v = _mm_loadu_si128((const __m128i*)p); 1340| 617k| const uint32_t mask = (uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v, vb)); 1341| 617k| if (mask != 0xFFFFU) { ------------------ | Branch (1341:21): [True: 295k, False: 322k] ------------------ 1342| 295k| p += zxc_ctz32(~mask); 1343| 295k| goto _run_done; 1344| 295k| } 1345| 322k| p += 16; 1346| 322k| } 1347| |#elif defined(ZXC_USE_NEON64) 1348| | const uint8x16_t vb = vdupq_n_u8(b); 1349| | while (p <= p_end - 16) { 1350| | const uint8x16_t v = vld1q_u8(p); 1351| | const uint8x16_t eq = vceqq_u8(v, vb); 1352| | /* SHRN nibble-mask: see find_best_match above for rationale. */ 1353| | const uint64_t mask = 1354| | vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(eq), 4)), 0); 1355| | if (LIKELY(mask == ~(uint64_t)0)) { 1356| | p += 16; 1357| | } else { 1358| | p += (size_t)(zxc_ctz64(~mask) >> 2); 1359| | goto _run_done; 1360| | } 1361| | } 1362| |#elif defined(ZXC_USE_NEON32) 1363| | uint8x16_t vb = vdupq_n_u8(b); 1364| | while (p <= p_end - 16) { 1365| | uint8x16_t v = vld1q_u8(p); 1366| | uint8x16_t eq = vceqq_u8(v, vb); 1367| | uint8x16_t not_eq = vmvnq_u8(eq); 1368| | 1369| | // 32-bit ARM NEON doesn't always support vgetq_lane_u64 / vreinterpretq_u64_u8 so 1370| | // we treat the 128-bit vector as 4 x 32-bit lanes */ 1371| | const uint32x4_t neq32 = vreinterpretq_u32_u8(not_eq); 1372| | const uint32_t l0 = vgetq_lane_u32(neq32, 0); 1373| | const uint32_t l1 = vgetq_lane_u32(neq32, 1); 1374| | 1375| | const uint64_t lo = ((uint64_t)l1 << 32) | l0; 1376| | if (lo != 0) { 1377| | p += (size_t)(zxc_ctz64(lo) >> 3); 1378| | goto _run_done; 1379| | } 1380| | 1381| | const uint32_t h0 = vgetq_lane_u32(neq32, 2); 1382| | const uint32_t h1 = vgetq_lane_u32(neq32, 3); 1383| | const uint64_t hi = ((uint64_t)h1 << 32) | h0; 1384| | 1385| | if (hi != 0) { 1386| | p += 8 + (zxc_ctz64(hi) >> 3); 1387| | goto _run_done; 1388| | } 1389| | p += 16; 1390| | } 1391| |#endif 1392| 13.8k| while (p < p_end && *p == b) p++; ------------------ | Branch (1392:20): [True: 12.9k, False: 936] | Branch (1392:33): [True: 9.21k, False: 3.73k] ------------------ 1393| | 1394| 4.66k|#if defined(ZXC_USE_AVX512) || defined(ZXC_USE_AVX2) || defined(ZXC_USE_NEON64) || \ 1395| 4.66k| defined(ZXC_USE_NEON32) || defined(ZXC_USE_SSE2) 1396| 300k| _run_done:; 1397| 300k|#endif 1398| 300k| const size_t run = (size_t)(p - run_start); 1399| | 1400| 300k| if (run >= 4) { ------------------ | Branch (1400:17): [True: 158k, False: 141k] ------------------ 1401| | // RLE run: 2 bytes per 131 values, then remainder 1402| | // Branchless: full_chunks * 2 + remainder handling 1403| 158k| const size_t full_chunks = run / 131; 1404| 158k| const size_t rem = run - full_chunks * 131; // Avoid modulo 1405| 158k| rle_size += full_chunks * 2; 1406| | // Remainder: if >= 4 -> 2 bytes (RLE), else 1 + rem (literal) 1407| 158k| if (rem >= 4) ------------------ | Branch (1407:21): [True: 157k, False: 948] ------------------ 1408| 157k| rle_size += 2; 1409| 948| else if (rem > 0) ------------------ | Branch (1409:26): [True: 663, False: 285] ------------------ 1410| 663| rle_size += 1 + rem; 1411| 158k| } else { 1412| | // Literal run: scan ahead with fast SIMD lookahead 1413| 141k| const uint8_t* lit_start = run_start; 1414| | 1415| |#if defined(ZXC_USE_AVX512) 1416| | while (p <= p_end_4 - 64) { 1417| | const __m512i v0 = _mm512_loadu_si512((const void*)p); 1418| | const __m512i v1 = _mm512_loadu_si512((const void*)(p + 1)); 1419| | const uint64_t m = (uint64_t)_mm512_cmpeq_epi8_mask(v0, v1); 1420| | const uint64_t mask = m & (m >> 1) & (m >> 2); 1421| | if (LIKELY(mask == 0)) { 1422| | p += 62; 1423| | continue; 1424| | } 1425| | const unsigned rpos = (unsigned)zxc_ctz64(mask); 1426| | const unsigned rpairs = (unsigned)zxc_ctz64(~(m >> rpos)); 1427| | if (UNLIKELY(rpos + rpairs >= 64)) { 1428| | p += rpos; 1429| | goto _lit_done; /* run may extend past the window */ 1430| | } 1431| | const size_t seg = (size_t)(p + rpos - lit_start); 1432| | rle_size += seg + ((seg + 127) >> 7); 1433| | rle_size += 2; /* run in [4,63]: full_chunks=0, rem>=4 */ 1434| | p += rpos + rpairs + 1; 1435| | lit_start = p; 1436| | } 1437| |#elif defined(ZXC_USE_AVX2) 1438| | while (p <= p_end_4 - 32) { 1439| | __m256i v0 = _mm256_loadu_si256((const __m256i*)p); 1440| | __m256i v1 = _mm256_loadu_si256((const __m256i*)(p + 1)); 1441| | const uint32_t m = (uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v0, v1)); 1442| | const uint32_t mask = m & (m >> 1) & (m >> 2); 1443| | if (LIKELY(mask == 0)) { 1444| | p += 30; 1445| | continue; 1446| | } 1447| | const unsigned rpos = zxc_ctz32(mask); 1448| | const unsigned rpairs = zxc_ctz32(~(m >> rpos)); 1449| | if (UNLIKELY(rpos + rpairs >= 32)) { 1450| | p += rpos; 1451| | goto _lit_done; /* run may extend past the window */ 1452| | } 1453| | const size_t seg = (size_t)(p + rpos - lit_start); 1454| | rle_size += seg + ((seg + 127) >> 7); 1455| | rle_size += 2; /* run in [4,31]: full_chunks=0, rem>=4 */ 1456| | p += rpos + rpairs + 1; 1457| | lit_start = p; 1458| | } 1459| |#elif defined(ZXC_USE_SSE2) 1460| 10.4M| while (p <= p_end_4 - 16) { ------------------ | Branch (1460:24): [True: 10.4M, False: 6.95k] ------------------ 1461| 10.4M| __m128i v0 = _mm_loadu_si128((const __m128i*)p); 1462| 10.4M| __m128i v1 = _mm_loadu_si128((const __m128i*)(p + 1)); 1463| 10.4M| const uint32_t m = (uint32_t)_mm_movemask_epi8(_mm_cmpeq_epi8(v0, v1)); 1464| 10.4M| const uint32_t mask = m & (m >> 1) & (m >> 2); 1465| 10.4M| if (LIKELY(mask == 0)) { ------------------ | | 145| 10.4M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 9.80M, False: 670k] | | ------------------ ------------------ 1466| 9.80M| p += 14; 1467| 9.80M| continue; 1468| 9.80M| } 1469| 670k| const unsigned rpos = zxc_ctz32(mask); 1470| 670k| const unsigned rpairs = zxc_ctz32(~(m >> rpos)); 1471| 670k| if (UNLIKELY(rpos + rpairs >= 16)) { ------------------ | | 151| 670k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 134k, False: 536k] | | ------------------ ------------------ 1472| 134k| p += rpos; 1473| 134k| goto _lit_done; /* run may extend past the window */ 1474| 134k| } 1475| 536k| const size_t seg = (size_t)(p + rpos - lit_start); 1476| 536k| rle_size += seg + ((seg + 127) >> 7); 1477| 536k| rle_size += 2; /* run in [4,16]: full_chunks=0, rem>=4 */ 1478| 536k| p += rpos + rpairs + 1; 1479| 536k| lit_start = p; 1480| 536k| } 1481| |#elif defined(ZXC_USE_NEON64) 1482| | while (p <= p_end_4 - 16) { 1483| | uint8x16_t v0 = vld1q_u8(p); 1484| | uint8x16_t v1 = vld1q_u8(p + 1); 1485| | const uint8x16_t eq = vceqq_u8(v0, v1); 1486| | const uint64_t m = vget_lane_u64( 1487| | vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(eq), 4)), 0); 1488| | const uint64_t mask = m & (m >> 4) & (m >> 8); 1489| | if (LIKELY(mask == 0)) { 1490| | p += 14; 1491| | continue; 1492| | } 1493| | const unsigned rpos = (unsigned)(zxc_ctz64(mask) >> 2); 1494| | const unsigned rpairs = (unsigned)(zxc_ctz64(~(m >> (rpos * 4))) >> 2); 1495| | if (UNLIKELY(rpos + rpairs >= 16)) { 1496| | p += rpos; 1497| | goto _lit_done; /* run may extend past the window */ 1498| | } 1499| | const size_t seg = (size_t)(p + rpos - lit_start); 1500| | rle_size += seg + ((seg + 127) >> 7); 1501| | rle_size += 2; /* run in [4,16]: full_chunks=0, rem>=4 */ 1502| | p += rpos + rpairs + 1; 1503| | lit_start = p; 1504| | } 1505| |#elif defined(ZXC_USE_NEON32) 1506| | while (p <= p_end_4 - 16) { 1507| | uint8x16_t v0 = vld1q_u8(p); 1508| | uint8x16_t v1 = vld1q_u8(p + 1); 1509| | const uint8x16_t pe = vceqq_u8(v0, v1); 1510| | const uint8x16_t zv = vdupq_n_u8(0); 1511| | const uint8x16_t eq = 1512| | vandq_u8(pe, vandq_u8(vextq_u8(pe, zv, 1), vextq_u8(pe, zv, 2))); 1513| | 1514| | uint32x4_t eq32 = vreinterpretq_u32_u8(eq); 1515| | uint32_t l0 = vgetq_lane_u32(eq32, 0); 1516| | uint32_t l1 = vgetq_lane_u32(eq32, 1); 1517| | uint64_t lo = ((uint64_t)l1 << 32) | l0; 1518| | 1519| | if (lo != 0) { 1520| | p += (zxc_ctz64(lo) >> 3); 1521| | goto _lit_done; 1522| | } 1523| | 1524| | uint32_t h0 = vgetq_lane_u32(eq32, 2); 1525| | uint32_t h1 = vgetq_lane_u32(eq32, 3); 1526| | uint64_t hi = ((uint64_t)h1 << 32) | h0; 1527| | 1528| | if (hi != 0) { 1529| | p += 8 + (zxc_ctz64(hi) >> 3); 1530| | goto _lit_done; 1531| | } 1532| | p += 14; 1533| | } 1534| |#endif 1535| 44.7k| while (p < p_end_4) { ------------------ | Branch (1535:24): [True: 39.5k, False: 5.20k] ------------------ 1536| 39.5k| if (UNLIKELY(p[0] == p[1] && p[1] == p[2] && p[2] == p[3])) break; ------------------ | | 151| 88.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.74k, False: 37.8k] | | | Branch (151:42): [True: 6.82k, False: 32.7k] | | | Branch (151:42): [True: 2.96k, False: 3.85k] | | | Branch (151:42): [True: 1.74k, False: 1.21k] | | ------------------ ------------------ 1537| 37.8k| p++; 1538| 37.8k| } 1539| 20.8k| while (p < p_end) { ------------------ | Branch (1539:24): [True: 15.6k, False: 5.20k] ------------------ 1540| 15.6k| if (UNLIKELY(p + 3 < p_end && p[0] == p[1] && p[1] == p[2] && p[2] == p[3])) ------------------ | | 151| 52.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.74k, False: 13.9k] | | | Branch (151:42): [True: 1.74k, False: 13.9k] | | | Branch (151:42): [True: 1.74k, False: 0] | | | Branch (151:42): [True: 1.74k, False: 0] | | | Branch (151:42): [True: 1.74k, False: 0] | | ------------------ ------------------ 1541| 1.74k| break; 1542| 13.9k| p++; 1543| 13.9k| } 1544| | 1545| 6.95k|#if defined(ZXC_USE_AVX512) || defined(ZXC_USE_AVX2) || defined(ZXC_USE_NEON64) || \ 1546| 6.95k| defined(ZXC_USE_NEON32) || defined(ZXC_USE_SSE2) 1547| 141k| _lit_done:; 1548| 141k|#endif 1549| 141k| const size_t lit_run = (size_t)(p - lit_start); 1550| | // 1 header per 128 bytes + all data bytes 1551| | // lit_run + ceil(lit_run / 128) 1552| 141k| rle_size += lit_run + ((lit_run + 127) >> 7); 1553| 141k| } 1554| 300k| } 1555| | 1556| | /* RLE over RAW: space-speed J comparison. The per-level premium 1557| | * reproduces the historical margin exactly below ULTRA. */ 1558| 5.80k| const size_t rle_j = rle_size + ZXC_SS_TAX(lit_c, zxc_ss_prem_rle_q8(level)); ------------------ | | 741| 5.80k|#define ZXC_SS_TAX(n, prem_q8) (((size_t)(n) * (size_t)(prem_q8)) >> 8) ------------------ 1559| 5.80k| if (rle_j < best_j) { ------------------ | Branch (1559:13): [True: 2.65k, False: 3.14k] ------------------ 1560| 2.65k| enc_lit = ZXC_SECTION_ENCODING_RLE; 1561| 2.65k| best_j = rle_j; 1562| 2.65k| } 1563| 5.80k| } 1564| | 1565| | /* Level >= 6: Huffman as a third literal-encoding candidate. Histogram, 1566| | * length-limited code lengths, exact PivCo section size, and switch if its 1567| | * J (size + decode tax) beats the current winner's. */ 1568| 5.80k| uint8_t huf_code_len[ZXC_HUF_NUM_SYMBOLS]; 1569| 5.80k| size_t huf_total_size = SIZE_MAX; 1570| 5.80k| uint32_t lit_freq[ZXC_HUF_NUM_SYMBOLS]; /* valid iff huf_total_size != SIZE_MAX */ 1571| 5.80k| if (level >= ZXC_LEVEL_DENSITY && lit_c >= ZXC_HUF_MIN_LITERALS) { ------------------ | | 790| 0|#define ZXC_HUF_MIN_LITERALS 139 ------------------ | Branch (1571:9): [True: 0, False: 5.80k] | Branch (1571:39): [True: 0, False: 0] ------------------ 1572| 0| uint32_t freq0[ZXC_HUF_NUM_SYMBOLS] = {0}; 1573| 0| uint32_t freq1[ZXC_HUF_NUM_SYMBOLS] = {0}; 1574| 0| uint32_t freq2[ZXC_HUF_NUM_SYMBOLS] = {0}; 1575| 0| uint32_t freq3[ZXC_HUF_NUM_SYMBOLS] = {0}; 1576| 0| { 1577| 0| size_t i = 0; 1578| 0| for (; i + 4 <= lit_c; i += 4) { ------------------ | Branch (1578:20): [True: 0, False: 0] ------------------ 1579| 0| freq0[literals[i + 0]]++; 1580| 0| freq1[literals[i + 1]]++; 1581| 0| freq2[literals[i + 2]]++; 1582| 0| freq3[literals[i + 3]]++; 1583| 0| } 1584| 0| for (; i < lit_c; i++) freq0[literals[i]]++; ------------------ | Branch (1584:20): [True: 0, False: 0] ------------------ 1585| 0| } 1586| 0| uint32_t* const freq = lit_freq; 1587| 0| for (int k = 0; k < ZXC_HUF_NUM_SYMBOLS; k++) { ------------------ | | 589| 0|#define ZXC_HUF_NUM_SYMBOLS 256 ------------------ | Branch (1587:25): [True: 0, False: 0] ------------------ 1588| 0| freq[k] = freq0[k] + freq1[k] + freq2[k] + freq3[k]; 1589| 0| } 1590| | 1591| 0| if (zxc_huf_build_code_lengths(freq, huf_code_len, ctx->opt_scratch, ------------------ | | 28| 0|#define zxc_huf_build_code_lengths ZXC_CAT(zxc_huf_build_code_lengths, ZXC_FUNCTION_SUFFIX) | | ------------------ | | | | 26| 0|#define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) | | | | ------------------ | | | | | | 25| 0|#define ZXC_CAT_IMPL(x, y) x##y | | | | ------------------ | | ------------------ ------------------ | Branch (1591:13): [True: 0, False: 0] ------------------ 1592| 0| zxc_huf_enc_max_code_len(level)) == ZXC_OK) { 1593| 0| (void)zxc_huf_nudge_code_lengths(freq, huf_code_len, ctx->opt_scratch, 1594| 0| zxc_huf_enc_max_code_len(level)); 1595| 0| huf_total_size = zxc_huf_calc_size(freq, huf_code_len, 1); ------------------ | | 30| 0|#define zxc_huf_calc_size ZXC_CAT(zxc_huf_calc_size, ZXC_FUNCTION_SUFFIX) | | ------------------ | | | | 26| 0|#define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) | | | | ------------------ | | | | | | 25| 0|#define ZXC_CAT_IMPL(x, y) x##y | | | | ------------------ | | ------------------ ------------------ 1596| | /* Space-speed: the entropy candidate must beat the current winner's 1597| | * J, paying its own decode tax over the copy path. */ 1598| 0| if (huf_total_size != SIZE_MAX) { ------------------ | Branch (1598:17): [True: 0, False: 0] ------------------ 1599| 0| const size_t huf_j = huf_total_size + ZXC_SS_TAX(lit_c, zxc_ss_prem_huf_q8(level)); ------------------ | | 741| 0|#define ZXC_SS_TAX(n, prem_q8) (((size_t)(n) * (size_t)(prem_q8)) >> 8) ------------------ 1600| 0| if (huf_j < best_j) { ------------------ | Branch (1600:21): [True: 0, False: 0] ------------------ 1601| 0| enc_lit = ZXC_SECTION_ENCODING_HUFFMAN; 1602| 0| best_j = huf_j; 1603| 0| } 1604| 0| } 1605| 0| } 1606| 0| } 1607| | 1608| | /* Shared dictionary table (level >= 6, dict with a table): the HUFFMAN 1609| | * bitstream minus the 128-byte header, so it stays viable far below 1610| | * ZXC_HUF_MIN_LITERALS. A literal without a code drops the candidate. */ 1611| 5.80k| size_t huf_dict_total_size = SIZE_MAX; 1612| 5.80k| if (level >= ZXC_LEVEL_DENSITY && ctx->dict_huf_tree_ok && lit_c > 0) { ------------------ | Branch (1612:9): [True: 0, False: 5.80k] | Branch (1612:39): [True: 0, False: 0] | Branch (1612:64): [True: 0, False: 0] ------------------ 1613| | /* The dict candidate runs on sections too small for the per-block 1614| | * table, so the histogram may not have been built above. */ 1615| 0| if (huf_total_size == SIZE_MAX) { ------------------ | Branch (1615:13): [True: 0, False: 0] ------------------ 1616| 0| ZXC_MEMSET(lit_freq, 0, sizeof(uint32_t) * ZXC_HUF_NUM_SYMBOLS); ------------------ | | 178| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 1617| 0| for (size_t i = 0; i < lit_c; i++) lit_freq[literals[i]]++; ------------------ | Branch (1617:32): [True: 0, False: 0] ------------------ 1618| 0| } 1619| | /* zxc_huf_calc_size returns SIZE_MAX for unencodable candidates 1620| | * (a literal byte without a code in the shared table). */ 1621| 0| huf_dict_total_size = 1622| 0| zxc_huf_calc_size_dict(lit_freq, ctx->dict_huf->code_len, &ctx->dict_huf->tree); ------------------ | | 31| 0|#define zxc_huf_calc_size_dict ZXC_CAT(zxc_huf_calc_size_dict, ZXC_FUNCTION_SUFFIX) | | ------------------ | | | | 26| 0|#define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) | | | | ------------------ | | | | | | 25| 0|#define ZXC_CAT_IMPL(x, y) x##y | | | | ------------------ | | ------------------ ------------------ 1623| 0| if (huf_dict_total_size != SIZE_MAX) { ------------------ | Branch (1623:13): [True: 0, False: 0] ------------------ 1624| | /* Same J rule against the running winner: vs the per-block Huffman 1625| | * candidate the equal entropy taxes cancel (pure size comparison), 1626| | * while RAW/RLE baselines keep their decode-tax edge. */ 1627| 0| const size_t huf_dict_j = 1628| 0| huf_dict_total_size + ZXC_SS_TAX(lit_c, zxc_ss_prem_huf_q8(level)); ------------------ | | 741| 0|#define ZXC_SS_TAX(n, prem_q8) (((size_t)(n) * (size_t)(prem_q8)) >> 8) ------------------ 1629| | /* Last candidate: no need to keep best_j updated past this point. */ 1630| 0| if (huf_dict_j < best_j) enc_lit = ZXC_SECTION_ENCODING_HUFFMAN_DICT; ------------------ | Branch (1630:17): [True: 0, False: 0] ------------------ 1631| 0| } 1632| 0| } 1633| | 1634| | /* Level-7 (ULTRA): Huffman-code the token stream with the literal codec - 1635| | * tokens are a <= 256-symbol alphabet. Same J rule; the decoder picks the 1636| | * path from gh.enc_litlen. opt_scratch is free by now. */ 1637| 5.80k| uint8_t tok_code_len[ZXC_HUF_NUM_SYMBOLS]; 1638| 5.80k| uint8_t enc_tok = ZXC_SECTION_ENCODING_RAW; 1639| 5.80k| size_t tok_huf_size = 0; 1640| 5.80k| uint32_t tfreq[ZXC_HUF_NUM_SYMBOLS]; 1641| 5.80k| if (level >= ZXC_LEVEL_ULTRA && seq_c >= ZXC_HUF_MIN_LITERALS) { ------------------ | | 790| 0|#define ZXC_HUF_MIN_LITERALS 139 ------------------ | Branch (1641:9): [True: 0, False: 5.80k] | Branch (1641:37): [True: 0, False: 0] ------------------ 1642| 0| ZXC_MEMSET(tfreq, 0, sizeof(tfreq)); /* only the ULTRA path reads tfreq */ ------------------ | | 178| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 1643| 0| for (uint32_t i = 0; i < seq_c; i++) tfreq[buf_tokens[i]]++; ------------------ | Branch (1643:30): [True: 0, False: 0] ------------------ 1644| 0| if (zxc_huf_build_code_lengths(tfreq, tok_code_len, ctx->opt_scratch, ------------------ | | 28| 0|#define zxc_huf_build_code_lengths ZXC_CAT(zxc_huf_build_code_lengths, ZXC_FUNCTION_SUFFIX) | | ------------------ | | | | 26| 0|#define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) | | | | ------------------ | | | | | | 25| 0|#define ZXC_CAT_IMPL(x, y) x##y | | | | ------------------ | | ------------------ ------------------ | Branch (1644:13): [True: 0, False: 0] ------------------ 1645| 0| zxc_huf_enc_max_code_len(level)) == ZXC_OK) { 1646| 0| (void)zxc_huf_nudge_code_lengths(tfreq, tok_code_len, ctx->opt_scratch, 1647| 0| zxc_huf_enc_max_code_len(level)); 1648| 0| tok_huf_size = zxc_huf_calc_size(tfreq, tok_code_len, 1); ------------------ | | 30| 0|#define zxc_huf_calc_size ZXC_CAT(zxc_huf_calc_size, ZXC_FUNCTION_SUFFIX) | | ------------------ | | | | 26| 0|#define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) | | | | ------------------ | | | | | | 25| 0|#define ZXC_CAT_IMPL(x, y) x##y | | | | ------------------ | | ------------------ ------------------ 1649| | /* Space-speed J comparison (this path is ULTRA-only): the PivCo 1650| | * token section pays the same decode tax as PivCo literals. */ 1651| 0| if (tok_huf_size != SIZE_MAX && ------------------ | Branch (1651:17): [True: 0, False: 0] ------------------ 1652| 0| tok_huf_size + ZXC_SS_TAX((size_t)seq_c, zxc_ss_prem_huf_q8(level)) < (size_t)seq_c) ------------------ | | 741| 0|#define ZXC_SS_TAX(n, prem_q8) (((size_t)(n) * (size_t)(prem_q8)) >> 8) ------------------ | Branch (1652:17): [True: 0, False: 0] ------------------ 1653| 0| enc_tok = ZXC_SECTION_ENCODING_HUFFMAN; 1654| 0| } 1655| 0| } 1656| | 1657| 5.80k| zxc_block_header_t bh = {.block_type = ZXC_BLOCK_GLO}; 1658| 5.80k| uint8_t* const p = dst + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 5.80k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1659| 5.80k| size_t rem = dst_cap - ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 5.80k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1660| | 1661| 5.80k| const int use_8bit_off = (max_offset <= 255) ? 1 : 0; ------------------ | Branch (1661:30): [True: 3.59k, False: 2.20k] ------------------ 1662| 5.80k| const size_t off_stream_size = use_8bit_off ? seq_c : (seq_c * 2); ------------------ | Branch (1662:36): [True: 3.59k, False: 2.20k] ------------------ 1663| | 1664| 5.80k| const zxc_gnr_header_t gh = {.n_sequences = seq_c, 1665| 5.80k| .n_literals = (uint32_t)lit_c, 1666| 5.80k| .enc_lit = enc_lit, 1667| 5.80k| .enc_litlen = enc_tok, 1668| 5.80k| .enc_mlen = 0, 1669| 5.80k| .enc_off = (uint8_t)use_8bit_off}; 1670| | 1671| 5.80k| zxc_section_desc_t desc[ZXC_GLO_SECTIONS] = {0}; 1672| 5.80k| const size_t lit_section_size = (enc_lit == ZXC_SECTION_ENCODING_RLE) ? rle_size ------------------ | Branch (1672:37): [True: 2.65k, False: 3.14k] ------------------ 1673| 5.80k| : (enc_lit == ZXC_SECTION_ENCODING_HUFFMAN) ? huf_total_size ------------------ | Branch (1673:39): [True: 0, False: 3.14k] ------------------ 1674| 3.14k| : (enc_lit == ZXC_SECTION_ENCODING_HUFFMAN_DICT) ------------------ | Branch (1674:39): [True: 0, False: 3.14k] ------------------ 1675| 3.14k| ? huf_dict_total_size 1676| 3.14k| : lit_c; 1677| 5.80k| desc[0].sizes = (uint64_t)lit_section_size | ((uint64_t)lit_c << 32); 1678| 5.80k| const size_t tok_section_size = 1679| 5.80k| (enc_tok == ZXC_SECTION_ENCODING_HUFFMAN) ? tok_huf_size : seq_c; ------------------ | Branch (1679:9): [True: 0, False: 5.80k] ------------------ 1680| 5.80k| desc[1].sizes = (uint64_t)tok_section_size | ((uint64_t)seq_c << 32); 1681| 5.80k| desc[2].sizes = (uint64_t)off_stream_size | ((uint64_t)off_stream_size << 32); 1682| 5.80k| desc[3].sizes = (uint64_t)extras_sz | ((uint64_t)extras_sz << 32); 1683| | 1684| 5.80k| const int ghs = zxc_write_glo_header_and_desc(p, rem, &gh, desc); 1685| 5.80k| if (UNLIKELY(ghs < 0)) return ghs; ------------------ | | 151| 5.80k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 5.80k] | | ------------------ ------------------ 1686| | 1687| 5.80k| uint8_t* p_curr = p + ghs; 1688| 5.80k| rem -= ghs; 1689| | 1690| 5.80k| const size_t sz_lit = (size_t)(desc[0].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 5.80k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 1691| 5.80k| const size_t sz_tok = (size_t)(desc[1].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 5.80k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 1692| 5.80k| const size_t sz_off = (size_t)(desc[2].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 5.80k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 1693| 5.80k| const size_t sz_ext = (size_t)(desc[3].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 5.80k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 1694| | 1695| 5.80k| if (UNLIKELY(rem < sz_lit)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 5.80k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 5.80k] | | ------------------ ------------------ 1696| | 1697| 5.80k| if (enc_lit == ZXC_SECTION_ENCODING_HUFFMAN) { ------------------ | Branch (1697:9): [True: 0, False: 5.80k] ------------------ 1698| 0| const int written = 1699| 0| zxc_huf_encode_section(literals, lit_c, lit_freq, huf_code_len, p_curr, rem); ------------------ | | 32| 0|#define zxc_huf_encode_section ZXC_CAT(zxc_huf_encode_section, ZXC_FUNCTION_SUFFIX) | | ------------------ | | | | 26| 0|#define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) | | | | ------------------ | | | | | | 25| 0|#define ZXC_CAT_IMPL(x, y) x##y | | | | ------------------ | | ------------------ ------------------ 1700| 0| if (UNLIKELY(written < 0)) return written; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1701| 0| if (UNLIKELY((size_t)written != huf_total_size)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1702| 0| p_curr += written; 1703| 5.80k| } else if (enc_lit == ZXC_SECTION_ENCODING_HUFFMAN_DICT) { ------------------ | Branch (1703:16): [True: 0, False: 5.80k] ------------------ 1704| 0| const int written = 1705| 0| zxc_huf_encode_section_dict(literals, lit_c, lit_freq, ctx->dict_huf->code_len, ------------------ | | 33| 0|#define zxc_huf_encode_section_dict ZXC_CAT(zxc_huf_encode_section_dict, ZXC_FUNCTION_SUFFIX) | | ------------------ | | | | 26| 0|#define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) | | | | ------------------ | | | | | | 25| 0|#define ZXC_CAT_IMPL(x, y) x##y | | | | ------------------ | | ------------------ ------------------ 1706| 0| &ctx->dict_huf->tree, ctx->dict_huf->codes, p_curr, rem); 1707| 0| if (UNLIKELY(written < 0)) return written; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1708| 0| if (UNLIKELY((size_t)written != huf_dict_total_size)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1709| 0| p_curr += written; 1710| 5.80k| } else if (enc_lit == ZXC_SECTION_ENCODING_RLE) { ------------------ | Branch (1710:16): [True: 2.65k, False: 3.14k] ------------------ 1711| | // Write RLE - optimized single-pass encoding 1712| 2.65k| const uint8_t* lit_ptr = literals; 1713| 2.65k| const uint8_t* const lit_end = literals + lit_c; 1714| | 1715| 901k| while (lit_ptr < lit_end) { ------------------ | Branch (1715:16): [True: 898k, False: 2.65k] ------------------ 1716| 898k| uint8_t b = *lit_ptr; 1717| 898k| const uint8_t* run_start = lit_ptr++; 1718| | 1719| 7.91M| while (lit_ptr < lit_end && *lit_ptr == b) lit_ptr++; ------------------ | Branch (1719:20): [True: 7.91M, False: 706] | Branch (1719:41): [True: 7.01M, False: 897k] ------------------ 1720| 898k| size_t run = (size_t)(lit_ptr - run_start); 1721| | 1722| 898k| if (run >= 4) { ------------------ | Branch (1722:17): [True: 469k, False: 428k] ------------------ 1723| | // RLE runs: emit 2-byte tokens (header + value) 1724| 954k| while (run >= 4) { ------------------ | Branch (1724:24): [True: 484k, False: 469k] ------------------ 1725| 484k| size_t chunk = (run > 131) ? 131 : run; ------------------ | Branch (1725:36): [True: 15.4k, False: 469k] ------------------ 1726| 484k| *p_curr++ = (uint8_t)(ZXC_LIT_RLE_FLAG | (chunk - 4)); ------------------ | | 479| 484k|#define ZXC_LIT_RLE_FLAG 0x80U ------------------ 1727| 484k| *p_curr++ = b; 1728| 484k| run -= chunk; 1729| 484k| } 1730| | // Leftover < 4 bytes: emit as literal 1731| 469k| if (run > 0) { ------------------ | Branch (1731:21): [True: 605, False: 469k] ------------------ 1732| 605| *p_curr++ = (uint8_t)(run - 1); 1733| 605| ZXC_MEMCPY(p_curr, lit_ptr - run, run); ------------------ | | 173| 605|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1734| 605| p_curr += run; 1735| 605| } 1736| 469k| } else { 1737| | // Literal run: scan ahead to find next RLE opportunity 1738| 428k| const uint8_t* lit_run_start = run_start; 1739| | 1740| 67.6M| while (lit_ptr < lit_end) { ------------------ | Branch (1740:24): [True: 67.6M, False: 2.15k] ------------------ 1741| 67.6M| if (UNLIKELY(lit_ptr + 3 < lit_end && lit_ptr[0] == lit_ptr[1] && ------------------ | | 151| 273M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 426k, False: 67.2M] | | | Branch (151:42): [True: 67.6M, False: 5.49k] | | | Branch (151:42): [True: 2.63M, False: 64.9M] | | | Branch (151:42): [True: 649k, False: 1.98M] | | | Branch (151:42): [True: 426k, False: 222k] | | ------------------ ------------------ 1742| 67.6M| lit_ptr[1] == lit_ptr[2] && lit_ptr[2] == lit_ptr[3])) { 1743| 426k| break; 1744| 426k| } 1745| 67.2M| lit_ptr++; 1746| 67.2M| } 1747| | 1748| 428k| size_t lit_run = (size_t)(lit_ptr - lit_run_start); 1749| 428k| const uint8_t* src_ptr = lit_run_start; 1750| | 1751| | // Emit literal chunks (max 128 bytes each) 1752| 1.32M| while (lit_run > 0) { ------------------ | Branch (1752:24): [True: 892k, False: 428k] ------------------ 1753| 892k| size_t chunk = (lit_run > 128) ? 128 : lit_run; ------------------ | Branch (1753:36): [True: 463k, False: 428k] ------------------ 1754| 892k| *p_curr++ = (uint8_t)(chunk - 1); 1755| 892k| ZXC_MEMCPY(p_curr, src_ptr, chunk); ------------------ | | 173| 892k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1756| 892k| p_curr += chunk; 1757| 892k| src_ptr += chunk; 1758| 892k| lit_run -= chunk; 1759| 892k| } 1760| 428k| } 1761| 898k| } 1762| 3.14k| } else { 1763| 3.14k| ZXC_MEMCPY(p_curr, literals, lit_c); ------------------ | | 173| 3.14k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1764| 3.14k| p_curr += lit_c; 1765| 3.14k| } 1766| 5.80k| rem -= sz_lit; 1767| | 1768| 5.80k| if (UNLIKELY(rem < sz_tok)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 5.80k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 5.80k] | | ------------------ ------------------ 1769| | 1770| 5.80k| if (enc_tok == ZXC_SECTION_ENCODING_HUFFMAN) { ------------------ | Branch (1770:9): [True: 0, False: 5.80k] ------------------ 1771| 0| const int written = 1772| 0| zxc_huf_encode_section(buf_tokens, seq_c, tfreq, tok_code_len, p_curr, rem); ------------------ | | 32| 0|#define zxc_huf_encode_section ZXC_CAT(zxc_huf_encode_section, ZXC_FUNCTION_SUFFIX) | | ------------------ | | | | 26| 0|#define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) | | | | ------------------ | | | | | | 25| 0|#define ZXC_CAT_IMPL(x, y) x##y | | | | ------------------ | | ------------------ ------------------ 1773| 0| if (UNLIKELY(written < 0)) return written; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1774| 0| if (UNLIKELY((size_t)written != tok_huf_size)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1775| 0| p_curr += written; 1776| 5.80k| } else { 1777| 5.80k| ZXC_MEMCPY(p_curr, buf_tokens, seq_c); ------------------ | | 173| 5.80k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1778| 5.80k| p_curr += seq_c; 1779| 5.80k| } 1780| 5.80k| rem -= sz_tok; 1781| | 1782| 5.80k| if (UNLIKELY(rem < sz_off)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 5.80k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 5.80k] | | ------------------ ------------------ 1783| | 1784| 5.80k| if (use_8bit_off) { ------------------ | Branch (1784:9): [True: 3.59k, False: 2.20k] ------------------ 1785| | // Write 1-byte offsets - unroll for better throughput 1786| 3.59k| uint32_t i = 0; 1787| 9.43k| for (; i + 8 <= seq_c; i += 8) { ------------------ | Branch (1787:16): [True: 5.83k, False: 3.59k] ------------------ 1788| 5.83k| p_curr[0] = (uint8_t)buf_offsets[i + 0]; 1789| 5.83k| p_curr[1] = (uint8_t)buf_offsets[i + 1]; 1790| 5.83k| p_curr[2] = (uint8_t)buf_offsets[i + 2]; 1791| 5.83k| p_curr[3] = (uint8_t)buf_offsets[i + 3]; 1792| 5.83k| p_curr[4] = (uint8_t)buf_offsets[i + 4]; 1793| 5.83k| p_curr[5] = (uint8_t)buf_offsets[i + 5]; 1794| 5.83k| p_curr[6] = (uint8_t)buf_offsets[i + 6]; 1795| 5.83k| p_curr[7] = (uint8_t)buf_offsets[i + 7]; 1796| 5.83k| p_curr += 8; 1797| 5.83k| } 1798| 12.3k| for (; i < seq_c; i++) { ------------------ | Branch (1798:16): [True: 8.76k, False: 3.59k] ------------------ 1799| 8.76k| *p_curr++ = (uint8_t)buf_offsets[i]; 1800| 8.76k| } 1801| 3.59k| } else { 1802| | // Write 2-byte offsets in little-endian order 1803| |#ifdef ZXC_BIG_ENDIAN 1804| | for (uint32_t i = 0; i < seq_c; i++) { 1805| | zxc_store_le16(p_curr, buf_offsets[i]); 1806| | p_curr += sizeof(uint16_t); 1807| | } 1808| |#else 1809| 2.20k| ZXC_MEMCPY(p_curr, buf_offsets, seq_c * sizeof(uint16_t)); ------------------ | | 173| 2.20k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1810| 2.20k| p_curr += seq_c * sizeof(uint16_t); 1811| 2.20k|#endif 1812| 2.20k| } 1813| 5.80k| rem -= sz_off; 1814| | 1815| 5.80k| if (UNLIKELY(rem < sz_ext)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 5.80k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 5.80k] | | ------------------ ------------------ 1816| | 1817| 5.80k| ZXC_MEMCPY(p_curr, buf_extras, extras_sz); ------------------ | | 173| 5.80k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1818| 5.80k| p_curr += extras_sz; 1819| | 1820| 5.80k| bh.comp_size = (uint32_t)(p_curr - (dst + ZXC_BLOCK_HEADER_SIZE)); ------------------ | | 397| 5.80k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1821| 5.80k| const int hw = zxc_write_block_header(dst, dst_cap, &bh); 1822| 5.80k| if (UNLIKELY(hw < 0)) return hw; ------------------ | | 151| 5.80k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 5.80k] | | ------------------ ------------------ 1823| | 1824| | // Checksum will be appended by the wrapper 1825| 5.80k| *out_sz = ZXC_BLOCK_HEADER_SIZE + bh.comp_size; ------------------ | | 397| 5.80k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1826| 5.80k| return ZXC_OK; 1827| 5.80k|} zxc_compress.c:zxc_encode_block_raw: 2044| 1.58k| size_t* RESTRICT const out_sz) { 2045| 1.58k| if (UNLIKELY(dst_cap < ZXC_BLOCK_HEADER_SIZE + src_sz)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 1.58k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.58k] | | ------------------ ------------------ 2046| | 2047| 1.58k| zxc_block_header_t bh; 2048| 1.58k| bh.block_type = ZXC_BLOCK_RAW; 2049| 1.58k| bh.block_flags = 0; // Checksum flag moved to file header 2050| 1.58k| bh.reserved = 0; 2051| 1.58k| bh.comp_size = (uint32_t)src_sz; 2052| | 2053| 1.58k| const int hw = zxc_write_block_header(dst, dst_cap, &bh); 2054| 1.58k| if (UNLIKELY(hw < 0)) return hw; ------------------ | | 151| 1.58k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.58k] | | ------------------ ------------------ 2055| | 2056| 1.58k| ZXC_MEMCPY(dst + ZXC_BLOCK_HEADER_SIZE, src, src_sz); ------------------ | | 173| 1.58k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 2057| | 2058| | // Checksum will be appended by the wrapper 2059| 1.58k| *out_sz = ZXC_BLOCK_HEADER_SIZE + src_sz; ------------------ | | 397| 1.58k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 2060| 1.58k| return ZXC_OK; 2061| 1.58k|} zxc_decompress_chunk_wrapper_default: 1532| 20.2k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 1533| 20.2k| return zxc_decompress_chunk_wrapper_body(ctx, src, src_sz, dst, dst_cap, 0, 0); 1534| 20.2k|} zxc_decompress.c:zxc_decompress_chunk_wrapper_body: 1468| 20.2k| uint8_t* RESTRICT dst, const size_t dst_cap, const int has_dict, const int safe) { 1469| 20.2k| if (UNLIKELY(src_sz < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 20.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 20.2k] | | ------------------ ------------------ 1470| | 1471| 20.2k| const uint8_t type = src[0]; 1472| 20.2k| const uint32_t comp_sz = zxc_le32(src + 3); 1473| 20.2k| const int has_crc = ctx->checksum_enabled; 1474| | 1475| | // Check bounds: Header + Body + Checksum(if any) 1476| 20.2k| const size_t expected_sz = 1477| 20.2k| (size_t)ZXC_BLOCK_HEADER_SIZE + comp_sz + (has_crc ? ZXC_BLOCK_CHECKSUM_SIZE : 0); ------------------ | | 397| 20.2k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ (size_t)ZXC_BLOCK_HEADER_SIZE + comp_sz + (has_crc ? ZXC_BLOCK_CHECKSUM_SIZE : 0); ------------------ | | 399| 0|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (1477:52): [True: 0, False: 20.2k] ------------------ 1478| 20.2k| if (UNLIKELY(src_sz < expected_sz)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 20.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 20.2k] | | ------------------ ------------------ 1479| | 1480| 20.2k| const uint8_t* data = src + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 20.2k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1481| | 1482| 20.2k| if (has_crc) { ------------------ | Branch (1482:9): [True: 0, False: 20.2k] ------------------ 1483| 0| const uint32_t stored = zxc_le32(data + comp_sz); 1484| 0| const uint32_t calc = zxc_checksum(data, comp_sz, ZXC_CHECKSUM_RAPIDHASH); ------------------ | | 424| 0|#define ZXC_CHECKSUM_RAPIDHASH 0 ------------------ 1485| 0| if (UNLIKELY(stored != calc)) return ZXC_ERROR_BAD_CHECKSUM; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1486| 0| } 1487| | 1488| 20.2k| int decoded_sz = ZXC_ERROR_BAD_BLOCK_TYPE; 1489| | 1490| 20.2k| switch (type) { 1491| 8.99k| case ZXC_BLOCK_GLO: ------------------ | Branch (1491:9): [True: 8.99k, False: 11.2k] ------------------ 1492| 8.99k| decoded_sz = safe ? zxc_decode_block_glo_safe(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1492:26): [True: 0, False: 8.99k] ------------------ 1493| 8.99k| : has_dict ? zxc_decode_block_glo_dict(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1493:28): [True: 0, False: 8.99k] ------------------ 1494| 8.99k| : zxc_decode_block_glo(ctx, data, comp_sz, dst, dst_cap); 1495| 8.99k| break; 1496| 8.18k| case ZXC_BLOCK_GHI: ------------------ | Branch (1496:9): [True: 8.18k, False: 12.0k] ------------------ 1497| 8.18k| decoded_sz = safe ? zxc_decode_block_ghi_safe(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1497:26): [True: 0, False: 8.18k] ------------------ 1498| 8.18k| : has_dict ? zxc_decode_block_ghi_dict(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1498:28): [True: 0, False: 8.18k] ------------------ 1499| 8.18k| : zxc_decode_block_ghi(ctx, data, comp_sz, dst, dst_cap); 1500| 8.18k| break; 1501| 3.04k| case ZXC_BLOCK_RAW: ------------------ | Branch (1501:9): [True: 3.04k, False: 17.1k] ------------------ 1502| | // For RAW blocks, comp_sz == raw_sz (uncompressed data stored as-is) 1503| 3.04k| if (UNLIKELY(comp_sz > dst_cap)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 3.04k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 3.04k] | | ------------------ ------------------ 1504| 3.04k| ZXC_MEMCPY(dst, data, comp_sz); ------------------ | | 173| 3.04k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1505| 3.04k| decoded_sz = (int)comp_sz; 1506| 3.04k| break; 1507| 0| case ZXC_BLOCK_EOF: ------------------ | Branch (1507:9): [True: 0, False: 20.2k] ------------------ 1508| | // EOF should be handled by the dispatcher, not here 1509| 0| return ZXC_ERROR_CORRUPT_DATA; 1510| 0| default: ------------------ | Branch (1510:9): [True: 0, False: 20.2k] ------------------ 1511| 0| return ZXC_ERROR_BAD_BLOCK_TYPE; 1512| 20.2k| } 1513| | 1514| 20.2k| return decoded_sz; 1515| 20.2k|} zxc_decompress.c:zxc_decode_block_glo_impl: 689| 8.99k| const int has_dict, const int tok_entropy) { 690| 8.99k| zxc_gnr_header_t gh; 691| | 692| | /* Constant 0 when !has_dict, so `d_floor` folds to `dst`. */ 693| 8.99k| const size_t dict_size = has_dict ? ctx->dict_size : 0; ------------------ | Branch (693:30): [True: 0, False: 8.99k] ------------------ 694| 8.99k| zxc_section_desc_t desc[ZXC_GLO_SECTIONS]; 695| | 696| 8.99k| if (UNLIKELY(zxc_read_glo_header_and_desc(src, src_size, &gh, desc) != ZXC_OK)) ------------------ | | 151| 8.99k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.99k] | | ------------------ ------------------ 697| 0| return ZXC_ERROR_BAD_HEADER; 698| | 699| | /* Entropy-coded tokens (level 7): tail-call the dedicated instantiation 700| | * before any section work - it restarts from the header, so only the parse 701| | * above is duplicated. Flags are constant per instantiation, so exactly one 702| | * call survives and t_ptr keeps a single provenance below. */ 703| 8.99k| if (!tok_entropy && UNLIKELY(gh.enc_litlen == ZXC_SECTION_ENCODING_HUFFMAN)) { ------------------ | | 151| 8.99k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.99k] | | ------------------ ------------------ | Branch (703:9): [True: 8.99k, False: 0] ------------------ 704| 0| if (safe) return zxc_decode_block_glo_entropy_safe(ctx, src, src_size, dst, dst_capacity); ------------------ | Branch (704:13): [True: 0, False: 0] ------------------ 705| 0| if (has_dict) ------------------ | Branch (705:13): [True: 0, False: 0] ------------------ 706| 0| return zxc_decode_block_glo_entropy_dict(ctx, src, src_size, dst, dst_capacity); 707| 0| return zxc_decode_block_glo_entropy(ctx, src, src_size, dst, dst_capacity); 708| 0| } 709| | 710| 8.99k| const uint8_t* p_data = 711| 8.99k| src + ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 401| 8.99k|#define ZXC_GLO_HEADER_BINARY_SIZE 16 ------------------ src + ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 419| 8.99k|#define ZXC_GLO_SECTIONS 4 ------------------ src + ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 8.99k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 712| 8.99k| const uint8_t* p_curr = p_data; 713| | 714| | // --- Literal Stream Setup --- 715| 8.99k| const uint8_t* l_ptr; 716| 8.99k| const uint8_t* l_end; 717| 8.99k| uint8_t* rle_buf = NULL; 718| | 719| 8.99k| size_t lit_stream_size = (size_t)(desc[0].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 8.99k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 720| | /* Decoded size of an encoded literal section (RAW ignores it). */ 721| 8.99k| const size_t required_size = (size_t)(desc[0].sizes >> 32); 722| | 723| 8.99k| if (gh.enc_lit == ZXC_SECTION_ENCODING_HUFFMAN || ------------------ | Branch (723:9): [True: 0, False: 8.99k] ------------------ 724| 8.99k| gh.enc_lit == ZXC_SECTION_ENCODING_HUFFMAN_DICT) { ------------------ | Branch (724:9): [True: 18.4E, False: 8.99k] ------------------ 725| 0| if (UNLIKELY(lit_stream_size > (size_t)(src + src_size - p_curr))) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 726| 0| return ZXC_ERROR_CORRUPT_DATA; 727| 0| if (required_size == 0) { ------------------ | Branch (727:13): [True: 0, False: 0] ------------------ 728| 0| l_ptr = p_curr; 729| 0| l_end = p_curr; 730| 0| } else { 731| 0| if (UNLIKELY(required_size > dst_capacity || required_size > SIZE_MAX - ZXC_PAD_SIZE)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 732| 0| return ZXC_ERROR_DST_TOO_SMALL; 733| 0| const int rc = 734| 0| (gh.enc_lit == ZXC_SECTION_ENCODING_HUFFMAN) ------------------ | Branch (734:17): [True: 0, False: 0] ------------------ 735| 0| ? zxc_decode_lit_pivco(ctx, p_curr, lit_stream_size, required_size) 736| 0| : zxc_decode_lit_pivco_dict(ctx, p_curr, lit_stream_size, required_size); 737| 0| if (UNLIKELY(rc != ZXC_OK)) return rc; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 738| 0| l_ptr = ctx->lit_buffer; 739| 0| l_end = ctx->lit_buffer + required_size; 740| 0| } 741| 8.99k| } else if (gh.enc_lit == ZXC_SECTION_ENCODING_RLE) { ------------------ | Branch (741:16): [True: 4.32k, False: 4.66k] ------------------ 742| 4.32k| if (required_size > 0) { ------------------ | Branch (742:13): [True: 4.32k, False: 0] ------------------ 743| 4.32k| if (UNLIKELY(required_size > dst_capacity || required_size > SIZE_MAX - ZXC_PAD_SIZE)) ------------------ | | 151| 8.65k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.32k] | | | Branch (151:42): [True: 0, False: 4.32k] | | | Branch (151:42): [True: 0, False: 4.32k] | | ------------------ ------------------ 744| 0| return ZXC_ERROR_DST_TOO_SMALL; 745| 4.32k| const size_t alloc_size = required_size + ZXC_PAD_SIZE; ------------------ | | 337| 4.32k|#define ZXC_PAD_SIZE 32 ------------------ 746| | 747| | /* lit_buffer is pre-allocated to chunk_size + ZXC_PAD_SIZE by 748| | * zxc_cctx_init (mode == 0).*/ 749| 4.32k| if (UNLIKELY(ctx->lit_buffer_cap < alloc_size)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 4.32k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.32k] | | ------------------ ------------------ 750| | 751| 4.32k| rle_buf = ctx->lit_buffer; 752| 4.32k| if (UNLIKELY(!rle_buf || lit_stream_size > (size_t)(src + src_size - p_curr))) ------------------ | | 151| 8.65k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.32k] | | | Branch (151:42): [True: 0, False: 4.32k] | | | Branch (151:42): [True: 0, False: 4.32k] | | ------------------ ------------------ 753| 0| return ZXC_ERROR_CORRUPT_DATA; 754| | 755| 4.32k| const uint8_t* r_ptr = p_curr; 756| 4.32k| const uint8_t* r_end = r_ptr + lit_stream_size; 757| 4.32k| uint8_t* w_ptr = rle_buf; 758| 4.32k| const uint8_t* const w_end = rle_buf + required_size; 759| | 760| 2.74M| while (r_ptr < r_end && w_ptr < w_end) { ------------------ | Branch (760:20): [True: 2.73M, False: 4.36k] | Branch (760:37): [True: 2.73M, False: 18.4E] ------------------ 761| 2.73M| uint8_t token = *r_ptr++; 762| 2.73M| if (LIKELY(!(token & ZXC_LIT_RLE_FLAG))) { ------------------ | | 145| 2.73M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 1.77M, False: 962k] | | ------------------ ------------------ 763| | // Raw copy (most common path): use ZXC_PAD_SIZE-byte wild copies 764| | // token is 7-bit (0-127), so len is 1-128 bytes 765| 1.77M| const uint32_t len = (uint32_t)token + 1; 766| 1.77M| if (UNLIKELY(w_ptr + len > w_end || r_ptr + len > r_end)) ------------------ | | 151| 3.55M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.77M] | | | Branch (151:42): [True: 18.4E, False: 1.77M] | | | Branch (151:42): [True: 18.4E, False: 1.77M] | | ------------------ ------------------ 767| 0| return ZXC_ERROR_CORRUPT_DATA; 768| | 769| | // Destination has ZXC_PAD_SIZE bytes of safe overrun space. 770| | // Source may not - check before wild copy. 771| | // Fast path: source has ZXC_PAD_SIZE-byte read headroom (most common) 772| 1.77M| if (LIKELY(r_ptr + ZXC_PAD_SIZE <= r_end)) { ------------------ | | 145| 1.77M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 1.77M, False: 6.15k] | | ------------------ ------------------ 773| | // Single 32-byte copy covers len <= ZXC_PAD_SIZE (most tokens) 774| 1.77M| zxc_copy32(w_ptr, r_ptr); 775| | 776| 1.77M| if (UNLIKELY(len > ZXC_PAD_SIZE)) { ------------------ | | 151| 1.77M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.07M, False: 691k] | | ------------------ ------------------ 777| | // Unroll: max len=128, so max 4 copies total 778| | // Use unconditional stores with overlap - faster than branches 779| 1.07M| if (len <= 2 * ZXC_PAD_SIZE) { ------------------ | | 337| 1.07M|#define ZXC_PAD_SIZE 32 ------------------ | Branch (779:33): [True: 81.8k, False: 997k] ------------------ 780| 81.8k| zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); ------------------ | | 337| 81.8k|#define ZXC_PAD_SIZE 32 ------------------ zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); ------------------ | | 337| 81.8k|#define ZXC_PAD_SIZE 32 ------------------ 781| 997k| } else if (len <= 3 * ZXC_PAD_SIZE) { ------------------ | | 337| 997k|#define ZXC_PAD_SIZE 32 ------------------ | Branch (781:40): [True: 38.8k, False: 958k] ------------------ 782| 38.8k| zxc_copy32(w_ptr + ZXC_PAD_SIZE, r_ptr + ZXC_PAD_SIZE); ------------------ | | 337| 38.8k|#define ZXC_PAD_SIZE 32 ------------------ zxc_copy32(w_ptr + ZXC_PAD_SIZE, r_ptr + ZXC_PAD_SIZE); ------------------ | | 337| 38.8k|#define ZXC_PAD_SIZE 32 ------------------ 783| 38.8k| zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); ------------------ | | 337| 38.8k|#define ZXC_PAD_SIZE 32 ------------------ zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); ------------------ | | 337| 38.8k|#define ZXC_PAD_SIZE 32 ------------------ 784| 958k| } else { 785| 958k| zxc_copy32(w_ptr + ZXC_PAD_SIZE, r_ptr + ZXC_PAD_SIZE); ------------------ | | 337| 958k|#define ZXC_PAD_SIZE 32 ------------------ zxc_copy32(w_ptr + ZXC_PAD_SIZE, r_ptr + ZXC_PAD_SIZE); ------------------ | | 337| 958k|#define ZXC_PAD_SIZE 32 ------------------ 786| 958k| zxc_copy32(w_ptr + 2 * ZXC_PAD_SIZE, r_ptr + 2 * ZXC_PAD_SIZE); ------------------ | | 337| 958k|#define ZXC_PAD_SIZE 32 ------------------ zxc_copy32(w_ptr + 2 * ZXC_PAD_SIZE, r_ptr + 2 * ZXC_PAD_SIZE); ------------------ | | 337| 958k|#define ZXC_PAD_SIZE 32 ------------------ 787| 958k| zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); ------------------ | | 337| 958k|#define ZXC_PAD_SIZE 32 ------------------ zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); ------------------ | | 337| 958k|#define ZXC_PAD_SIZE 32 ------------------ 788| 958k| } 789| 1.07M| } 790| 1.77M| } else { 791| | // Near end of source: safe copy (rare cold path) 792| 6.15k| ZXC_MEMCPY(w_ptr, r_ptr, len); ------------------ | | 173| 6.15k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 793| 6.15k| } 794| | 795| 1.77M| w_ptr += len; 796| 1.77M| r_ptr += len; 797| 1.77M| } else { 798| | // RLE run: fill with single byte 799| 962k| const uint32_t len = (token & ZXC_LIT_LEN_MASK) + 4; ------------------ | | 481| 962k|#define ZXC_LIT_LEN_MASK (ZXC_LIT_RLE_FLAG - 1) | | ------------------ | | | | 479| 962k|#define ZXC_LIT_RLE_FLAG 0x80U | | ------------------ ------------------ 800| 962k| if (UNLIKELY(w_ptr + len > w_end || r_ptr >= r_end)) ------------------ | | 151| 1.92M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 962k] | | | Branch (151:42): [True: 18.4E, False: 963k] | | | Branch (151:42): [True: 18.4E, False: 963k] | | ------------------ ------------------ 801| 0| return ZXC_ERROR_CORRUPT_DATA; 802| 962k| ZXC_MEMSET(w_ptr, *r_ptr++, len); ------------------ | | 178| 962k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 803| 962k| w_ptr += len; 804| 962k| } 805| 2.73M| } 806| 4.32k| if (UNLIKELY(w_ptr != w_end)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 4.32k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 4.32k] | | ------------------ ------------------ 807| 4.32k| l_ptr = rle_buf; 808| 4.32k| l_end = rle_buf + required_size; 809| 4.32k| } else { 810| 0| l_ptr = p_curr; 811| 0| l_end = p_curr; 812| 0| } 813| 4.66k| } else if (gh.enc_lit == ZXC_SECTION_ENCODING_RAW) { ------------------ | Branch (813:16): [True: 4.66k, False: 18.4E] ------------------ 814| 4.66k| l_ptr = p_curr; 815| 4.66k| l_end = p_curr + lit_stream_size; 816| 18.4E| } else { 817| 18.4E| return ZXC_ERROR_CORRUPT_DATA; 818| 18.4E| } 819| | 820| 8.99k| p_curr += lit_stream_size; 821| | 822| | // --- Stream Pointers & Validation --- 823| 8.99k| const size_t sz_tokens = (size_t)(desc[1].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 8.99k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 824| 8.99k| const size_t sz_offsets = (size_t)(desc[2].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 8.99k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 825| 8.99k| const size_t sz_extras = (size_t)(desc[3].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 8.99k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 826| | 827| | // Validate stream sizes match sequence count (early rejection of malformed data) 828| 8.99k| const uint64_t expected_off_size = 829| 8.99k| GLO_OFF8 ? (uint64_t)gh.n_sequences : (uint64_t)gh.n_sequences * 2; ------------------ | | 436| 8.99k|#define GLO_OFF8 (gh.enc_off == 1) | | ------------------ | | | Branch (436:18): [True: 4.70k, False: 4.29k] | | ------------------ ------------------ 830| | 831| | /* Offsets/extras follow the on-disk token SECTION; sz_tokens is its size 832| | * (== n_sequences when RAW, the Huffman payload size when enc_litlen set). */ 833| 8.99k| const uint8_t* o_ptr = p_curr + sz_tokens; 834| 8.99k| const uint8_t* e_ptr = o_ptr + sz_offsets; 835| 8.99k| const uint8_t* const e_end = e_ptr + sz_extras; // For vbyte overflow detection 836| | 837| | // Validate streams don't overflow source buffer + match sequence count. 838| 8.99k| if (UNLIKELY((e_end != src + src_size) || (uint64_t)sz_offsets < expected_off_size)) ------------------ | | 151| 17.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.99k] | | | Branch (151:42): [True: 0, False: 8.99k] | | | Branch (151:42): [True: 0, False: 8.99k] | | ------------------ ------------------ 839| 0| return ZXC_ERROR_CORRUPT_DATA; 840| | 841| 8.99k| if (UNLIKELY(gh.enc_lit == ZXC_SECTION_ENCODING_RAW && ------------------ | | 151| 13.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.28k, False: 7.70k] | | | Branch (151:42): [True: 4.66k, False: 4.32k] | | | Branch (151:42): [True: 1.28k, False: 3.37k] | | ------------------ ------------------ 842| 8.99k| (size_t)(src + src_size - l_end) < ZXC_PAD_SIZE)) { 843| 1.28k| const int rc = zxc_stage_raw_literals(ctx, &l_ptr, &l_end); 844| 1.28k| if (UNLIKELY(rc != ZXC_OK)) return rc; ------------------ | | 151| 1.28k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.28k] | | ------------------ ------------------ 845| 1.28k| } 846| | 847| 8.99k| const uint8_t* RESTRICT t_ptr; 848| 8.99k| if (!tok_entropy) { ------------------ | Branch (848:9): [True: 8.99k, False: 0] ------------------ 849| | /* enc_litlen == 2 was re-routed right after the header parse. */ 850| 8.99k| if (UNLIKELY(sz_tokens < gh.n_sequences)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 8.99k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.99k] | | ------------------ ------------------ 851| 8.99k| t_ptr = p_curr; 852| 8.99k| } else { 853| 0| if (UNLIKELY(gh.enc_litlen != ZXC_SECTION_ENCODING_HUFFMAN)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 854| 0| const int rc = zxc_decode_tok_pivco(ctx, p_curr, sz_tokens, gh.n_sequences); 855| 0| if (UNLIKELY(rc != ZXC_OK)) return rc; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 856| 0| t_ptr = ctx->tok_buffer; 857| 0| } 858| | 859| 8.99k| uint8_t* d_ptr = dst; 860| 8.99k| const uint8_t* const d_end = dst + dst_capacity; 861| | // Lowest address a match may reach: the dictionary prefix if any, else dst. 862| 8.99k| const uint8_t* const d_floor = dst - dict_size; 863| | // Destination safe margin for 4x loop: max output without varint extension. 864| | // ll_max = 14, ml_max = 14 + 5 = 19, per-seq = 33, 4x = 132. 865| | // Add ZXC_PAD_SIZE (32) for the wild zxc_copy32 overshoot + 4 safety = 168. 866| 8.99k| const uint8_t* const d_end_safe = d_end - (132 + ZXC_PAD_SIZE + 4); ------------------ | | 337| 8.99k|#define ZXC_PAD_SIZE 32 ------------------ 867| | 868| | // Literal margin for the 4x loops: without a varint, ll <= 14 per sequence, 869| | // so 4 * 14 = 56. Past that margin only the cold varint path checks l_ptr. 870| 8.99k| const size_t glo_sz_lit = (size_t)(l_end - l_ptr); 871| 8.99k| const size_t glo_margin_4x = 4 * (ZXC_TOKEN_LL_MASK - 1); // 56 ------------------ | | 454| 8.99k|#define ZXC_TOKEN_LL_MASK ((1U << ZXC_TOKEN_LIT_BITS) - 1) | | ------------------ | | | | 450| 8.99k|#define ZXC_TOKEN_LIT_BITS 4 | | ------------------ ------------------ 872| 8.99k| const size_t glo_margin_1x = ZXC_TOKEN_LL_MASK - 1; // 14 ------------------ | | 454| 8.99k|#define ZXC_TOKEN_LL_MASK ((1U << ZXC_TOKEN_LIT_BITS) - 1) | | ------------------ | | | | 450| 8.99k|#define ZXC_TOKEN_LIT_BITS 4 | | ------------------ ------------------ 873| 8.99k| const uint8_t* const l_end_safe_4x = 874| 8.99k| (glo_sz_lit > glo_margin_4x) ? l_end - glo_margin_4x : l_ptr; ------------------ | Branch (874:9): [True: 7.79k, False: 1.19k] ------------------ 875| 8.99k| const uint8_t* const l_end_safe_1x = 876| 8.99k| (glo_sz_lit > glo_margin_1x) ? l_end - glo_margin_1x : l_ptr; ------------------ | Branch (876:9): [True: 8.57k, False: 419] ------------------ 877| | 878| 8.99k| uint32_t n_seq = gh.n_sequences; 879| | 880| | // Offsets only need checking until the output is wider than the widest offset 881| | // the format can encode (256 for 1-byte offsets, 65536 for 2-byte); past that 882| | // none can reach below d_floor. A dictionary counts as already written, so a 883| | // large one puts d_bounds at dst and skips the SAFE loop. 884| 8.99k| const size_t off_span = GLO_OFF8 ? (1U << 8) : (1U << 16); ------------------ | | 436| 8.99k|#define GLO_OFF8 (gh.enc_off == 1) | | ------------------ | | | Branch (436:18): [True: 4.70k, False: 4.29k] | | ------------------ ------------------ 885| 8.99k| const size_t span = (dict_size >= off_span) ? 0 : off_span - dict_size; ------------------ | Branch (885:25): [True: 0, False: 8.99k] ------------------ 886| 8.99k| const uint8_t* const d_bounds = (span >= dst_capacity) ? d_end : dst + span; ------------------ | Branch (886:37): [True: 0, False: 8.99k] ------------------ 887| | 888| | // --- SAFE Loop: offset validation until d_bounds (4x unroll) --- 889| 8.99k| if (safe) { ------------------ | Branch (889:9): [True: 0, False: 8.99k] ------------------ 890| | /* SAFE variant: save per-batch state so overflow can rollback. */ 891| 0| while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (891:16): [True: 0, False: 0] | Branch (891:30): [True: 0, False: 0] | Branch (891:52): [True: 0, False: 0] | Branch (891:77): [True: 0, False: 0] ------------------ 892| 0| const uint8_t* const t_save = t_ptr; 893| 0| const uint8_t* const o_save = o_ptr; 894| 0| const uint8_t* const e_save = e_ptr; 895| 0| uint8_t* const d_save = d_ptr; 896| 0| const uint8_t* const l_save = l_ptr; 897| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x); ------------------ | | 479| 0| do { \ | | 480| 0| uint32_t tokens = zxc_le32(t_ptr); \ | | 481| 0| t_ptr += sizeof(uint32_t); \ | | 482| 0| uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 483| 0| off4 = ZXC_LZ_OFFSET_BIAS; \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 484| 0| if (GLO_OFF8) { \ | | ------------------ | | | | 436| 0|#define GLO_OFF8 (gh.enc_off == 1) | | | | ------------------ | | | | | Branch (436:18): [True: 0, False: 0] | | | | ------------------ | | ------------------ | | 485| 0| uint32_t offsets = zxc_le32(o_ptr); \ | | 486| 0| o_ptr += sizeof(uint32_t); \ | | 487| 0| off1 += offsets & 0xFF; \ | | 488| 0| off2 += (offsets >> 8) & 0xFF; \ | | 489| 0| off3 += (offsets >> 16) & 0xFF; \ | | 490| 0| off4 += (offsets >> 24) & 0xFF; \ | | 491| 0| } else { \ | | 492| 0| uint64_t offsets = zxc_le64(o_ptr); \ | | 493| 0| o_ptr += sizeof(uint64_t); \ | | 494| 0| off1 += (uint32_t)(offsets & 0xFFFF); \ | | 495| 0| off2 += (uint32_t)((offsets >> 16) & 0xFFFF); \ | | 496| 0| off3 += (uint32_t)((offsets >> 32) & 0xFFFF); \ | | 497| 0| off4 += (uint32_t)((offsets >> 48) & 0xFFFF); \ | | 498| 0| } \ | | 499| 0| DECODE_GLO_SEQ((tokens & 0x0F0) >> 4, (tokens & 0x00F), off1, \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 897| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 500| 0| ((tokens >> 12) & 0xF) + ((tokens >> 20) & 0xF) + (tokens >> 28), 3U, \ | | 501| 0| DECODE, ON_FAIL); \ | | 502| 0| DECODE_GLO_SEQ((tokens & 0x0F000) >> 12, (tokens & 0x00F00) >> 8, off2, \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 897| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 503| 0| ((tokens >> 20) & 0xF) + (tokens >> 28), 2U, DECODE, ON_FAIL); \ | | 504| 0| DECODE_GLO_SEQ((tokens & 0x0F00000) >> 20, (tokens & 0x00F0000) >> 16, off3, \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 897| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 505| 0| (tokens >> 28), 1U, DECODE, ON_FAIL); \ | | 506| 0| DECODE_GLO_SEQ((tokens >> 28), (tokens >> 24) & 0x0F, off4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 897| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 507| 0| n_seq -= 4; \ | | 508| 0| } while (0) | | ------------------ | | | Branch (508:14): [Folded, False: 0] | | ------------------ ------------------ 898| 0| continue; 899| | 900| 0| rollback_safe_4x: 901| 0| t_ptr = t_save; 902| 0| o_ptr = o_save; 903| 0| e_ptr = e_save; 904| 0| d_ptr = d_save; 905| 0| l_ptr = l_save; 906| 0| break; 907| 0| } 908| 8.99k| } else { 909| 416k| while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (909:16): [True: 412k, False: 3.41k] | Branch (909:30): [True: 412k, False: 0] | Branch (909:52): [True: 411k, False: 1.39k] | Branch (909:77): [True: 407k, False: 4.18k] ------------------ 910| 407k| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW); ------------------ | | 479| 407k| do { \ | | 480| 407k| uint32_t tokens = zxc_le32(t_ptr); \ | | 481| 407k| t_ptr += sizeof(uint32_t); \ | | 482| 407k| uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 407k|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 407k|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 407k|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 483| 407k| off4 = ZXC_LZ_OFFSET_BIAS; \ | | ------------------ | | | | 527| 407k|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 484| 407k| if (GLO_OFF8) { \ | | ------------------ | | | | 436| 407k|#define GLO_OFF8 (gh.enc_off == 1) | | | | ------------------ | | | | | Branch (436:18): [True: 7.21k, False: 399k] | | | | ------------------ | | ------------------ | | 485| 7.21k| uint32_t offsets = zxc_le32(o_ptr); \ | | 486| 7.21k| o_ptr += sizeof(uint32_t); \ | | 487| 7.21k| off1 += offsets & 0xFF; \ | | 488| 7.21k| off2 += (offsets >> 8) & 0xFF; \ | | 489| 7.21k| off3 += (offsets >> 16) & 0xFF; \ | | 490| 7.21k| off4 += (offsets >> 24) & 0xFF; \ | | 491| 399k| } else { \ | | 492| 399k| uint64_t offsets = zxc_le64(o_ptr); \ | | 493| 399k| o_ptr += sizeof(uint64_t); \ | | 494| 399k| off1 += (uint32_t)(offsets & 0xFFFF); \ | | 495| 399k| off2 += (uint32_t)((offsets >> 16) & 0xFFFF); \ | | 496| 399k| off3 += (uint32_t)((offsets >> 32) & 0xFFFF); \ | | 497| 399k| off4 += (uint32_t)((offsets >> 48) & 0xFFFF); \ | | 498| 399k| } \ | | 499| 407k| DECODE_GLO_SEQ((tokens & 0x0F0) >> 4, (tokens & 0x00F), off1, \ | | ------------------ | | | | 453| 407k| do { \ | | | | 454| 407k| uint64_t ll = (LL); \ | | | | 455| 407k| uint64_t ml = (ML); \ | | | | 456| 407k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 407k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 45.0k, False: 362k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 45.0k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 45.0k| const uint64_t reserve = (RESERVE); \ | | | | 459| 45.0k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 90.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 45.0k] | | | | | | | Branch (151:42): [True: 0, False: 45.0k] | | | | | | | Branch (151:42): [True: 0, False: 45.0k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 45.0k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 45.0k| ON_FAIL; \ | | | | 462| 45.0k| } \ | | | | 463| 407k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 407k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 99.9k, False: 307k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 99.9k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 99.9k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 99.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 99.9k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 99.9k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 99.9k| (size_t)(d_end - d_ptr))) \ | | | | 468| 99.9k| ON_FAIL; \ | | | | 469| 99.9k| } \ | | | | 470| 407k| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 407k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 407k| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 910| 407k| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 405| 407k| do { \ | | | | | | | | 406| 407k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 407k| l_ptr += ll; \ | | | | | | | | 408| 407k| d_ptr += ll; \ | | | | | | | | 409| 407k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 407k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 407k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 407k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 407k| d_ptr += ml; \ | | | | | | | | 412| 407k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 407k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 407k| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 407k] | | | | ------------------ | | ------------------ | | 500| 407k| ((tokens >> 12) & 0xF) + ((tokens >> 20) & 0xF) + (tokens >> 28), 3U, \ | | 501| 407k| DECODE, ON_FAIL); \ | | 502| 407k| DECODE_GLO_SEQ((tokens & 0x0F000) >> 12, (tokens & 0x00F00) >> 8, off2, \ | | ------------------ | | | | 453| 407k| do { \ | | | | 454| 407k| uint64_t ll = (LL); \ | | | | 455| 407k| uint64_t ml = (ML); \ | | | | 456| 407k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 407k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 43.5k, False: 363k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 43.5k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 43.5k| const uint64_t reserve = (RESERVE); \ | | | | 459| 43.5k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 87.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 43.5k] | | | | | | | Branch (151:42): [True: 0, False: 43.5k] | | | | | | | Branch (151:42): [True: 0, False: 43.5k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 43.5k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 43.5k| ON_FAIL; \ | | | | 462| 43.5k| } \ | | | | 463| 407k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 407k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 100k, False: 306k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 100k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 100k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 100k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 100k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 100k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 100k| (size_t)(d_end - d_ptr))) \ | | | | 468| 100k| ON_FAIL; \ | | | | 469| 100k| } \ | | | | 470| 407k| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 407k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 407k| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 910| 407k| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 405| 407k| do { \ | | | | | | | | 406| 407k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 407k| l_ptr += ll; \ | | | | | | | | 408| 407k| d_ptr += ll; \ | | | | | | | | 409| 407k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 407k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 407k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 407k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 407k| d_ptr += ml; \ | | | | | | | | 412| 407k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 407k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 407k| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 407k] | | | | ------------------ | | ------------------ | | 503| 407k| ((tokens >> 20) & 0xF) + (tokens >> 28), 2U, DECODE, ON_FAIL); \ | | 504| 407k| DECODE_GLO_SEQ((tokens & 0x0F00000) >> 20, (tokens & 0x00F0000) >> 16, off3, \ | | ------------------ | | | | 453| 407k| do { \ | | | | 454| 407k| uint64_t ll = (LL); \ | | | | 455| 407k| uint64_t ml = (ML); \ | | | | 456| 407k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 407k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 43.1k, False: 363k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 43.1k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 43.1k| const uint64_t reserve = (RESERVE); \ | | | | 459| 43.1k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 86.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 43.1k] | | | | | | | Branch (151:42): [True: 0, False: 43.1k] | | | | | | | Branch (151:42): [True: 0, False: 43.1k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 43.1k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 43.1k| ON_FAIL; \ | | | | 462| 43.1k| } \ | | | | 463| 407k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 407k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 100k, False: 306k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 100k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 100k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 100k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 100k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 100k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 100k| (size_t)(d_end - d_ptr))) \ | | | | 468| 100k| ON_FAIL; \ | | | | 469| 100k| } \ | | | | 470| 407k| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 407k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 407k| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 910| 407k| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 405| 407k| do { \ | | | | | | | | 406| 407k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 407k| l_ptr += ll; \ | | | | | | | | 408| 407k| d_ptr += ll; \ | | | | | | | | 409| 407k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 407k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 407k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 407k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 407k| d_ptr += ml; \ | | | | | | | | 412| 407k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 407k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 407k| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 407k] | | | | ------------------ | | ------------------ | | 505| 407k| (tokens >> 28), 1U, DECODE, ON_FAIL); \ | | 506| 407k| DECODE_GLO_SEQ((tokens >> 28), (tokens >> 24) & 0x0F, off4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 453| 407k| do { \ | | | | 454| 407k| uint64_t ll = (LL); \ | | | | 455| 407k| uint64_t ml = (ML); \ | | | | 456| 407k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 407k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 43.3k, False: 363k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 43.3k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 43.3k| const uint64_t reserve = (RESERVE); \ | | | | 459| 43.3k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 86.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 43.3k] | | | | | | | Branch (151:42): [True: 0, False: 43.3k] | | | | | | | Branch (151:42): [True: 0, False: 43.3k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 43.3k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 43.3k| ON_FAIL; \ | | | | 462| 43.3k| } \ | | | | 463| 407k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 407k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 100k, False: 306k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 100k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 100k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 100k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 100k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 100k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 100k| (size_t)(d_end - d_ptr))) \ | | | | 468| 100k| ON_FAIL; \ | | | | 469| 100k| } \ | | | | 470| 407k| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 407k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 407k| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 910| 407k| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 405| 407k| do { \ | | | | | | | | 406| 407k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 407k| l_ptr += ll; \ | | | | | | | | 408| 407k| d_ptr += ll; \ | | | | | | | | 409| 407k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 407k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 407k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 407k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 407k| d_ptr += ml; \ | | | | | | | | 412| 407k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 407k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 407k| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 407k] | | | | ------------------ | | ------------------ | | 507| 407k| n_seq -= 4; \ | | 508| 407k| } while (0) | | ------------------ | | | Branch (508:14): [Folded, False: 407k] | | ------------------ ------------------ 911| 407k| } 912| 8.99k| } 913| | 914| | // --- FAST Loop: After threshold, no offset validation needed (4x unroll) --- 915| 8.99k| if (safe) { ------------------ | Branch (915:9): [True: 0, False: 8.99k] ------------------ 916| 0| while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x) { ------------------ | Branch (916:16): [True: 0, False: 0] | Branch (916:30): [True: 0, False: 0] | Branch (916:52): [True: 0, False: 0] ------------------ 917| 0| const uint8_t* const t_save = t_ptr; 918| 0| const uint8_t* const o_save = o_ptr; 919| 0| const uint8_t* const e_save = e_ptr; 920| 0| uint8_t* const d_save = d_ptr; 921| 0| const uint8_t* const l_save = l_ptr; 922| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x); ------------------ | | 479| 0| do { \ | | 480| 0| uint32_t tokens = zxc_le32(t_ptr); \ | | 481| 0| t_ptr += sizeof(uint32_t); \ | | 482| 0| uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 483| 0| off4 = ZXC_LZ_OFFSET_BIAS; \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 484| 0| if (GLO_OFF8) { \ | | ------------------ | | | | 436| 0|#define GLO_OFF8 (gh.enc_off == 1) | | | | ------------------ | | | | | Branch (436:18): [True: 0, False: 0] | | | | ------------------ | | ------------------ | | 485| 0| uint32_t offsets = zxc_le32(o_ptr); \ | | 486| 0| o_ptr += sizeof(uint32_t); \ | | 487| 0| off1 += offsets & 0xFF; \ | | 488| 0| off2 += (offsets >> 8) & 0xFF; \ | | 489| 0| off3 += (offsets >> 16) & 0xFF; \ | | 490| 0| off4 += (offsets >> 24) & 0xFF; \ | | 491| 0| } else { \ | | 492| 0| uint64_t offsets = zxc_le64(o_ptr); \ | | 493| 0| o_ptr += sizeof(uint64_t); \ | | 494| 0| off1 += (uint32_t)(offsets & 0xFFFF); \ | | 495| 0| off2 += (uint32_t)((offsets >> 16) & 0xFFFF); \ | | 496| 0| off3 += (uint32_t)((offsets >> 32) & 0xFFFF); \ | | 497| 0| off4 += (uint32_t)((offsets >> 48) & 0xFFFF); \ | | 498| 0| } \ | | 499| 0| DECODE_GLO_SEQ((tokens & 0x0F0) >> 4, (tokens & 0x00F), off1, \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 922| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x); | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 500| 0| ((tokens >> 12) & 0xF) + ((tokens >> 20) & 0xF) + (tokens >> 28), 3U, \ | | 501| 0| DECODE, ON_FAIL); \ | | 502| 0| DECODE_GLO_SEQ((tokens & 0x0F000) >> 12, (tokens & 0x00F00) >> 8, off2, \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 922| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x); | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 503| 0| ((tokens >> 20) & 0xF) + (tokens >> 28), 2U, DECODE, ON_FAIL); \ | | 504| 0| DECODE_GLO_SEQ((tokens & 0x0F00000) >> 20, (tokens & 0x00F0000) >> 16, off3, \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 922| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x); | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 505| 0| (tokens >> 28), 1U, DECODE, ON_FAIL); \ | | 506| 0| DECODE_GLO_SEQ((tokens >> 28), (tokens >> 24) & 0x0F, off4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 922| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x); | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 507| 0| n_seq -= 4; \ | | 508| 0| } while (0) | | ------------------ | | | Branch (508:14): [Folded, False: 0] | | ------------------ ------------------ 923| 0| continue; 924| | 925| 0| rollback_fast_4x: 926| 0| t_ptr = t_save; 927| 0| o_ptr = o_save; 928| 0| e_ptr = e_save; 929| 0| d_ptr = d_save; 930| 0| l_ptr = l_save; 931| 0| break; 932| 0| } 933| 8.99k| } else { 934| 1.06M| while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x) { ------------------ | Branch (934:16): [True: 1.06M, False: 6.31k] | Branch (934:30): [True: 1.06M, False: 18.4E] | Branch (934:52): [True: 1.05M, False: 2.67k] ------------------ 935| 1.05M| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW); ------------------ | | 479| 1.05M| do { \ | | 480| 1.05M| uint32_t tokens = zxc_le32(t_ptr); \ | | 481| 1.05M| t_ptr += sizeof(uint32_t); \ | | 482| 1.05M| uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 1.05M|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 1.05M|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 1.05M|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 483| 1.05M| off4 = ZXC_LZ_OFFSET_BIAS; \ | | ------------------ | | | | 527| 1.05M|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 484| 1.05M| if (GLO_OFF8) { \ | | ------------------ | | | | 436| 1.05M|#define GLO_OFF8 (gh.enc_off == 1) | | | | ------------------ | | | | | Branch (436:18): [True: 15.3k, False: 1.04M] | | | | ------------------ | | ------------------ | | 485| 15.3k| uint32_t offsets = zxc_le32(o_ptr); \ | | 486| 15.3k| o_ptr += sizeof(uint32_t); \ | | 487| 15.3k| off1 += offsets & 0xFF; \ | | 488| 15.3k| off2 += (offsets >> 8) & 0xFF; \ | | 489| 15.3k| off3 += (offsets >> 16) & 0xFF; \ | | 490| 15.3k| off4 += (offsets >> 24) & 0xFF; \ | | 491| 1.04M| } else { \ | | 492| 1.04M| uint64_t offsets = zxc_le64(o_ptr); \ | | 493| 1.04M| o_ptr += sizeof(uint64_t); \ | | 494| 1.04M| off1 += (uint32_t)(offsets & 0xFFFF); \ | | 495| 1.04M| off2 += (uint32_t)((offsets >> 16) & 0xFFFF); \ | | 496| 1.04M| off3 += (uint32_t)((offsets >> 32) & 0xFFFF); \ | | 497| 1.04M| off4 += (uint32_t)((offsets >> 48) & 0xFFFF); \ | | 498| 1.04M| } \ | | 499| 1.05M| DECODE_GLO_SEQ((tokens & 0x0F0) >> 4, (tokens & 0x00F), off1, \ | | ------------------ | | | | 453| 1.05M| do { \ | | | | 454| 1.05M| uint64_t ll = (LL); \ | | | | 455| 1.05M| uint64_t ml = (ML); \ | | | | 456| 1.05M| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 1.05M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 65.8k, False: 992k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 65.8k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 65.8k| const uint64_t reserve = (RESERVE); \ | | | | 459| 65.8k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 131k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 65.8k] | | | | | | | Branch (151:42): [True: 18.4E, False: 65.8k] | | | | | | | Branch (151:42): [True: 18.4E, False: 65.8k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 65.8k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 65.8k| ON_FAIL; \ | | | | 462| 65.8k| } \ | | | | 463| 1.05M| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 1.05M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 277k, False: 780k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 277k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 277k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 277k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 277k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 277k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 277k| (size_t)(d_end - d_ptr))) \ | | | | 468| 277k| ON_FAIL; \ | | | | 469| 277k| } \ | | | | 470| 1.05M| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 1.05M|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 1.05M| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 935| 1.05M| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 417| 1.05M| do { \ | | | | | | | | 418| 1.05M| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 1.05M| l_ptr += ll; \ | | | | | | | | 420| 1.05M| d_ptr += ll; \ | | | | | | | | 421| 1.05M| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 1.05M| d_ptr += ml; \ | | | | | | | | 423| 1.05M| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 1.05M] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 1.05M| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 1.05M] | | | | ------------------ | | ------------------ | | 500| 1.05M| ((tokens >> 12) & 0xF) + ((tokens >> 20) & 0xF) + (tokens >> 28), 3U, \ | | 501| 1.05M| DECODE, ON_FAIL); \ | | 502| 1.05M| DECODE_GLO_SEQ((tokens & 0x0F000) >> 12, (tokens & 0x00F00) >> 8, off2, \ | | ------------------ | | | | 453| 1.05M| do { \ | | | | 454| 1.05M| uint64_t ll = (LL); \ | | | | 455| 1.05M| uint64_t ml = (ML); \ | | | | 456| 1.05M| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 1.05M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 67.0k, False: 991k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 67.0k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 67.0k| const uint64_t reserve = (RESERVE); \ | | | | 459| 67.0k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 134k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 67.0k] | | | | | | | Branch (151:42): [True: 18.4E, False: 67.0k] | | | | | | | Branch (151:42): [True: 18.4E, False: 67.0k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 67.0k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 67.0k| ON_FAIL; \ | | | | 462| 67.0k| } \ | | | | 463| 1.05M| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 1.05M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 277k, False: 780k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 277k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 277k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 277k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 277k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 277k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 277k| (size_t)(d_end - d_ptr))) \ | | | | 468| 277k| ON_FAIL; \ | | | | 469| 277k| } \ | | | | 470| 1.05M| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 1.05M|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 1.05M| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 935| 1.05M| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 417| 1.05M| do { \ | | | | | | | | 418| 1.05M| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 1.05M| l_ptr += ll; \ | | | | | | | | 420| 1.05M| d_ptr += ll; \ | | | | | | | | 421| 1.05M| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 1.05M| d_ptr += ml; \ | | | | | | | | 423| 1.05M| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 1.05M] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 1.05M| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 1.05M] | | | | ------------------ | | ------------------ | | 503| 1.05M| ((tokens >> 20) & 0xF) + (tokens >> 28), 2U, DECODE, ON_FAIL); \ | | 504| 1.05M| DECODE_GLO_SEQ((tokens & 0x0F00000) >> 20, (tokens & 0x00F0000) >> 16, off3, \ | | ------------------ | | | | 453| 1.05M| do { \ | | | | 454| 1.05M| uint64_t ll = (LL); \ | | | | 455| 1.05M| uint64_t ml = (ML); \ | | | | 456| 1.05M| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 1.05M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 67.0k, False: 991k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 67.0k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 67.0k| const uint64_t reserve = (RESERVE); \ | | | | 459| 67.0k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 134k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 67.0k] | | | | | | | Branch (151:42): [True: 18.4E, False: 67.0k] | | | | | | | Branch (151:42): [True: 18.4E, False: 67.0k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 67.0k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 67.0k| ON_FAIL; \ | | | | 462| 67.0k| } \ | | | | 463| 1.05M| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 1.05M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 276k, False: 781k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 276k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 276k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 276k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 276k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 276k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 276k| (size_t)(d_end - d_ptr))) \ | | | | 468| 276k| ON_FAIL; \ | | | | 469| 276k| } \ | | | | 470| 1.05M| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 1.05M|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 1.05M| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 935| 1.05M| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 417| 1.05M| do { \ | | | | | | | | 418| 1.05M| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 1.05M| l_ptr += ll; \ | | | | | | | | 420| 1.05M| d_ptr += ll; \ | | | | | | | | 421| 1.05M| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 1.05M| d_ptr += ml; \ | | | | | | | | 423| 1.05M| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 1.05M] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 1.05M| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 1.05M] | | | | ------------------ | | ------------------ | | 505| 1.05M| (tokens >> 28), 1U, DECODE, ON_FAIL); \ | | 506| 1.05M| DECODE_GLO_SEQ((tokens >> 28), (tokens >> 24) & 0x0F, off4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 453| 1.05M| do { \ | | | | 454| 1.05M| uint64_t ll = (LL); \ | | | | 455| 1.05M| uint64_t ml = (ML); \ | | | | 456| 1.05M| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 1.05M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 67.2k, False: 990k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 67.2k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 67.2k| const uint64_t reserve = (RESERVE); \ | | | | 459| 67.2k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 134k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 67.2k] | | | | | | | Branch (151:42): [True: 18.4E, False: 67.2k] | | | | | | | Branch (151:42): [True: 1, False: 67.2k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 67.2k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 67.2k| ON_FAIL; \ | | | | 462| 67.2k| } \ | | | | 463| 1.05M| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 1.05M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 277k, False: 780k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 277k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 277k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 277k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 277k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 277k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 277k| (size_t)(d_end - d_ptr))) \ | | | | 468| 277k| ON_FAIL; \ | | | | 469| 277k| } \ | | | | 470| 1.05M| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 1.05M|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 1.05M| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 935| 1.05M| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 417| 1.05M| do { \ | | | | | | | | 418| 1.05M| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 1.05M| l_ptr += ll; \ | | | | | | | | 420| 1.05M| d_ptr += ll; \ | | | | | | | | 421| 1.05M| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 1.05M| d_ptr += ml; \ | | | | | | | | 423| 1.05M| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 1.05M] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 1.05M| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 1.05M] | | | | ------------------ | | ------------------ | | 507| 1.05M| n_seq -= 4; \ | | 508| 1.05M| } while (0) | | ------------------ | | | Branch (508:14): [Folded, False: 1.05M] | | ------------------ ------------------ 936| 1.05M| } 937| 8.99k| } 938| | 939| | // Validate vbyte reads didn't overflow 940| 8.99k| if (UNLIKELY(e_ptr > e_end)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 8.99k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.99k] | | ------------------ ------------------ 941| | 942| | // --- Remaining 1 sequence (Fast Path) --- 943| 99.8k| while (n_seq > 0 && d_ptr < d_end_safe && l_ptr < l_end_safe_1x) { ------------------ | Branch (943:12): [True: 94.4k, False: 5.41k] | Branch (943:25): [True: 94.4k, False: 0] | Branch (943:47): [True: 90.8k, False: 3.57k] ------------------ 944| | // Save pointers before reading (in case we need to fall back to Safe Path) 945| 90.8k| const uint8_t* t_save = t_ptr; 946| 90.8k| const uint8_t* o_save = o_ptr; 947| 90.8k| const uint8_t* e_save = e_ptr; 948| | 949| 90.8k| uint8_t token = *t_ptr++; 950| 90.8k| uint64_t ll = token >> ZXC_TOKEN_LIT_BITS; ------------------ | | 450| 90.8k|#define ZXC_TOKEN_LIT_BITS 4 ------------------ 951| 90.8k| uint64_t ml = token & ZXC_TOKEN_ML_MASK; ------------------ | | 456| 90.8k|#define ZXC_TOKEN_ML_MASK ((1U << ZXC_TOKEN_ML_BITS) - 1) | | ------------------ | | | | 452| 90.8k|#define ZXC_TOKEN_ML_BITS 4 | | ------------------ ------------------ 952| 90.8k| uint32_t offset = ZXC_LZ_OFFSET_BIAS; ------------------ | | 527| 90.8k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 953| 90.8k| if (GLO_OFF8) { ------------------ | | 436| 90.8k|#define GLO_OFF8 (gh.enc_off == 1) | | ------------------ | | | Branch (436:18): [True: 8.21k, False: 82.6k] | | ------------------ ------------------ 954| 8.21k| offset += *o_ptr++; // 1-byte offset (biased) 955| 82.6k| } else { 956| 82.6k| offset += zxc_le16(o_ptr); // 2-byte offset (biased) 957| 82.6k| o_ptr += sizeof(uint16_t); 958| 82.6k| } 959| | 960| 90.8k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { ------------------ | | 151| 90.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 2.87k, False: 87.9k] | | ------------------ ------------------ 961| 2.87k| ll += zxc_read_varint(&e_ptr, e_end); 962| 2.87k| if (UNLIKELY(l_ptr + ll > l_end)) { ------------------ | | 151| 2.87k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 2.87k] | | ------------------ ------------------ 963| 0| t_ptr = t_save; 964| 0| o_ptr = o_save; 965| 0| e_ptr = e_save; 966| 0| break; 967| 0| } 968| 2.87k| } 969| 90.8k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) ml += zxc_read_varint(&e_ptr, e_end); ------------------ | | 151| 90.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 39.1k, False: 51.6k] | | ------------------ ------------------ 970| 90.8k| ml += ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 90.8k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 971| | 972| | // Check bounds before wild copies - if too close to end, fall back to Safe Path 973| 90.8k| if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) { ------------------ | | 151| 90.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 90.8k] | | ------------------ ------------------ 974| 0| t_ptr = t_save; 975| 0| o_ptr = o_save; 976| 0| e_ptr = e_save; 977| 0| break; 978| 0| } 979| | 980| 90.8k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); 981| 90.8k| l_ptr += ll; 982| 90.8k| d_ptr += ll; 983| | 984| 90.8k| if (UNLIKELY(d_ptr < d_bounds && (size_t)(d_ptr - d_floor) < offset)) ------------------ | | 151| 137k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 90.8k] | | | Branch (151:42): [True: 47.0k, False: 43.8k] | | | Branch (151:42): [True: 0, False: 47.0k] | | ------------------ ------------------ 985| 0| return ZXC_ERROR_BAD_OFFSET; 986| | 987| | /* The loop entry check guarantees ll + ml + ZXC_PAD_SIZE bytes of 988| | * headroom, so the wild-copy ladder (incl. overlap/fill runs) is safe. */ 989| 90.8k| zxc_decode_copy_match(d_ptr, offset, ml); 990| 90.8k| d_ptr += ml; 991| 90.8k| n_seq--; 992| 90.8k| } 993| | 994| | // --- Safe Path for Remaining Sequences --- 995| 42.9k| while (n_seq > 0) { ------------------ | Branch (995:12): [True: 33.9k, False: 8.99k] ------------------ 996| 33.9k| uint8_t token = *t_ptr++; 997| 33.9k| uint64_t ll = token >> ZXC_TOKEN_LIT_BITS; ------------------ | | 450| 33.9k|#define ZXC_TOKEN_LIT_BITS 4 ------------------ 998| 33.9k| uint64_t ml = token & ZXC_TOKEN_ML_MASK; ------------------ | | 456| 33.9k|#define ZXC_TOKEN_ML_MASK ((1U << ZXC_TOKEN_ML_BITS) - 1) | | ------------------ | | | | 452| 33.9k|#define ZXC_TOKEN_ML_BITS 4 | | ------------------ ------------------ 999| 33.9k| uint32_t offset = ZXC_LZ_OFFSET_BIAS; ------------------ | | 527| 33.9k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1000| 33.9k| if (GLO_OFF8) { ------------------ | | 436| 33.9k|#define GLO_OFF8 (gh.enc_off == 1) | | ------------------ | | | Branch (436:18): [True: 4.25k, False: 29.7k] | | ------------------ ------------------ 1001| 4.25k| offset += *o_ptr++; // 1-byte offset (biased) 1002| 29.7k| } else { 1003| 29.7k| offset += zxc_le16(o_ptr); // 2-byte offset (biased) 1004| 29.7k| o_ptr += sizeof(uint16_t); 1005| 29.7k| } 1006| | 1007| 33.9k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) ll += zxc_read_varint(&e_ptr, e_end); ------------------ | | 151| 33.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 33.9k] | | ------------------ ------------------ 1008| 33.9k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) ml += zxc_read_varint(&e_ptr, e_end); ------------------ | | 151| 33.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 17.0k, False: 16.9k] | | ------------------ ------------------ 1009| 33.9k| ml += ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 33.9k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1010| | 1011| 33.9k| if (UNLIKELY(ll + ml > (size_t)(d_end - d_ptr) || l_ptr + ll > l_end)) ------------------ | | 151| 67.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 33.9k] | | | Branch (151:42): [True: 0, False: 33.9k] | | | Branch (151:42): [True: 0, False: 33.9k] | | ------------------ ------------------ 1012| 0| return ZXC_ERROR_OVERFLOW; 1013| 33.9k| ZXC_MEMCPY(d_ptr, l_ptr, ll); ------------------ | | 173| 33.9k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1014| 33.9k| l_ptr += ll; 1015| 33.9k| d_ptr += ll; 1016| | 1017| 33.9k| if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET; ------------------ | | 151| 33.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 33.9k] | | ------------------ ------------------ 1018| 33.9k| const uint8_t* match_src = d_ptr - offset; 1019| | 1020| 33.9k| zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml); 1021| 33.9k| d_ptr += ml; 1022| 33.9k| n_seq--; 1023| 33.9k| } 1024| | 1025| | // --- Trailing Literals --- 1026| | // Copy remaining literals from source stream (literal exhaustion) 1027| 8.99k| if (UNLIKELY(l_ptr > l_end)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 8.99k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.99k] | | ------------------ ------------------ 1028| 8.99k| const size_t remaining_literals = (size_t)(l_end - l_ptr); 1029| 8.99k| if (UNLIKELY(remaining_literals > (size_t)(d_end - d_ptr))) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 8.99k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.99k] | | ------------------ ------------------ 1030| 8.99k| ZXC_MEMCPY(d_ptr, l_ptr, remaining_literals); ------------------ | | 173| 8.99k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1031| 8.99k| d_ptr += remaining_literals; 1032| | 1033| 8.99k| return (int)(d_ptr - dst); 1034| 8.99k|} zxc_decompress.c:zxc_stage_raw_literals: 594| 3.24k| const uint8_t** RESTRICT l_end) { 595| 3.24k| const size_t lit_size = (size_t)(*l_end - *l_ptr); 596| 3.24k| uint8_t* const stage = ctx->lit_buffer; 597| 3.24k| if (UNLIKELY(!stage || ctx->lit_buffer_cap < lit_size + ZXC_PAD_SIZE)) ------------------ | | 151| 6.48k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 3.24k] | | | Branch (151:42): [True: 0, False: 3.24k] | | | Branch (151:42): [True: 0, False: 3.24k] | | ------------------ ------------------ 598| 0| return ZXC_ERROR_CORRUPT_DATA; 599| | 600| 3.24k| ZXC_MEMCPY(stage, *l_ptr, lit_size); ------------------ | | 173| 3.24k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 601| 3.24k| *l_ptr = stage; 602| 3.24k| *l_end = stage + lit_size; 603| 3.24k| return ZXC_OK; 604| 3.24k|} zxc_decompress.c:zxc_read_varint: 54| 2.30M|static ZXC_ALWAYS_INLINE uint32_t zxc_read_varint(const uint8_t** ptr, const uint8_t* end) { 55| 2.30M| const uint8_t* p = *ptr; 56| 2.30M| if (UNLIKELY(p >= end)) return 0; ------------------ | | 151| 2.30M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 2.30M] | | ------------------ ------------------ 57| | 58| 2.30M| const uint32_t b0 = p[0]; 59| | 60| | // 1 Byte: 0xxxxxxx (7 bits) -> val < 128 (2^7) 61| 2.30M| if (LIKELY(b0 < 0x80)) { ------------------ | | 145| 2.30M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 1.69M, False: 606k] | | ------------------ ------------------ 62| 1.69M| *ptr = p + 1; 63| 1.69M| return b0; 64| 1.69M| } 65| | 66| | // 2 Bytes: 10xxxxxx xxxxxxxx (14 bits) -> val < 16384 (2^14) 67| 606k| if (LIKELY(b0 < 0xC0)) { ------------------ | | 145| 606k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 593k, False: 13.3k] | | ------------------ ------------------ 68| 593k| if (UNLIKELY(p + 1 >= end)) { ------------------ | | 151| 593k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 593k] | | ------------------ ------------------ 69| 0| *ptr = end; 70| 0| return 0; 71| 0| } 72| 593k| *ptr = p + 2; 73| 593k| return (b0 & 0x3F) | ((uint32_t)p[1] << 6); 74| 593k| } 75| | 76| | // 3 Bytes: 110xxxxx xxxxxxxx xxxxxxxx (21 bits) -> val < 2^21. The longest 77| | // a legitimate varint can be: values are (ll - MASK) or (ml - MASK), always 78| | // strictly below block_size_max = 2^21. 79| 14.1k| if (LIKELY(b0 < 0xE0)) { ------------------ | | 145| 13.3k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 14.1k, False: 18.4E] | | ------------------ ------------------ 80| 14.1k| if (UNLIKELY(p + 2 >= end)) { ------------------ | | 151| 14.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 14.1k] | | ------------------ ------------------ 81| 0| *ptr = end; 82| 0| return 0; 83| 0| } 84| 14.1k| *ptr = p + 3; 85| 14.1k| return (b0 & 0x1F) | ((uint32_t)p[1] << 5) | ((uint32_t)p[2] << 13); 86| 14.1k| } 87| | 88| | // extra encoding: out-of-spec for the current format, reject. 89| 18.4E| *ptr = end; 90| 18.4E| return 0; 91| 13.3k|} zxc_decompress.c:zxc_decode_copy_literals: 297| 9.58M| const uint64_t ll) { 298| 9.58M| zxc_copy32(dst, src); 299| 9.58M| if (UNLIKELY(ll > ZXC_PAD_SIZE)) { ------------------ | | 151| 9.58M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 502k, False: 9.07M] | | ------------------ ------------------ 300| 502k| dst += ZXC_PAD_SIZE; ------------------ | | 337| 502k|#define ZXC_PAD_SIZE 32 ------------------ 301| 502k| src += ZXC_PAD_SIZE; ------------------ | | 337| 502k|#define ZXC_PAD_SIZE 32 ------------------ 302| 502k| size_t rem = ll - ZXC_PAD_SIZE; ------------------ | | 337| 502k|#define ZXC_PAD_SIZE 32 ------------------ 303| 19.1M| while (rem > ZXC_PAD_SIZE) { ------------------ | | 337| 19.1M|#define ZXC_PAD_SIZE 32 ------------------ | Branch (303:16): [True: 18.6M, False: 502k] ------------------ 304| 18.6M| zxc_copy32(dst, src); 305| 18.6M| dst += ZXC_PAD_SIZE; ------------------ | | 337| 18.6M|#define ZXC_PAD_SIZE 32 ------------------ 306| 18.6M| src += ZXC_PAD_SIZE; ------------------ | | 337| 18.6M|#define ZXC_PAD_SIZE 32 ------------------ 307| 18.6M| rem -= ZXC_PAD_SIZE; ------------------ | | 337| 18.6M|#define ZXC_PAD_SIZE 32 ------------------ 308| 18.6M| } 309| 502k| zxc_copy32(dst, src); 310| 502k| } 311| 9.58M|} zxc_decompress.c:zxc_decode_copy_match: 333| 9.58M| const uint64_t ml) { 334| 9.58M| const uint8_t* match_src = d_ptr - off; 335| 9.58M| if (LIKELY(off >= ZXC_PAD_SIZE)) { ------------------ | | 145| 9.58M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 7.96M, False: 1.61M] | | ------------------ ------------------ 336| 7.96M| zxc_copy32(d_ptr, match_src); 337| 7.96M| if (UNLIKELY(ml > ZXC_PAD_SIZE)) { ------------------ | | 151| 7.96M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.25M, False: 6.70M] | | ------------------ ------------------ 338| 1.25M| uint8_t* out = d_ptr + ZXC_PAD_SIZE; ------------------ | | 337| 1.25M|#define ZXC_PAD_SIZE 32 ------------------ 339| 1.25M| const uint8_t* ref = match_src + ZXC_PAD_SIZE; ------------------ | | 337| 1.25M|#define ZXC_PAD_SIZE 32 ------------------ 340| 1.25M| size_t rem = ml - ZXC_PAD_SIZE; ------------------ | | 337| 1.25M|#define ZXC_PAD_SIZE 32 ------------------ 341| 25.4M| while (rem > ZXC_PAD_SIZE) { ------------------ | | 337| 25.4M|#define ZXC_PAD_SIZE 32 ------------------ | Branch (341:20): [True: 24.1M, False: 1.25M] ------------------ 342| 24.1M| zxc_copy32(out, ref); 343| 24.1M| out += ZXC_PAD_SIZE; ------------------ | | 337| 24.1M|#define ZXC_PAD_SIZE 32 ------------------ 344| 24.1M| ref += ZXC_PAD_SIZE; ------------------ | | 337| 24.1M|#define ZXC_PAD_SIZE 32 ------------------ 345| 24.1M| rem -= ZXC_PAD_SIZE; ------------------ | | 337| 24.1M|#define ZXC_PAD_SIZE 32 ------------------ 346| 24.1M| } 347| 1.25M| zxc_copy32(out, ref); 348| 1.25M| } 349| 7.96M| } else if (off >= (ZXC_PAD_SIZE / 2)) { ------------------ | | 337| 1.61M|#define ZXC_PAD_SIZE 32 ------------------ | Branch (349:16): [True: 704k, False: 911k] ------------------ 350| 704k| zxc_copy16(d_ptr, match_src); 351| 704k| if (UNLIKELY(ml > (ZXC_PAD_SIZE / 2))) { ------------------ | | 151| 704k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 220k, False: 483k] | | ------------------ ------------------ 352| 220k| uint8_t* out = d_ptr + (ZXC_PAD_SIZE / 2); ------------------ | | 337| 220k|#define ZXC_PAD_SIZE 32 ------------------ 353| 220k| const uint8_t* ref = match_src + (ZXC_PAD_SIZE / 2); ------------------ | | 337| 220k|#define ZXC_PAD_SIZE 32 ------------------ 354| 220k| size_t rem = ml - (ZXC_PAD_SIZE / 2); ------------------ | | 337| 220k|#define ZXC_PAD_SIZE 32 ------------------ 355| 2.18M| while (rem > (ZXC_PAD_SIZE / 2)) { ------------------ | | 337| 2.18M|#define ZXC_PAD_SIZE 32 ------------------ | Branch (355:20): [True: 1.96M, False: 220k] ------------------ 356| 1.96M| zxc_copy16(out, ref); 357| 1.96M| out += (ZXC_PAD_SIZE / 2); ------------------ | | 337| 1.96M|#define ZXC_PAD_SIZE 32 ------------------ 358| 1.96M| ref += (ZXC_PAD_SIZE / 2); ------------------ | | 337| 1.96M|#define ZXC_PAD_SIZE 32 ------------------ 359| 1.96M| rem -= (ZXC_PAD_SIZE / 2); ------------------ | | 337| 1.96M|#define ZXC_PAD_SIZE 32 ------------------ 360| 1.96M| } 361| 220k| zxc_copy16(out, ref); 362| 220k| } 363| 911k| } else if (off == 1) { ------------------ | Branch (363:16): [True: 397k, False: 513k] ------------------ 364| 397k| zxc_decode_fill_run(d_ptr, match_src[0], ml); 365| 513k| } else { 366| 513k| zxc_decode_copy_overlap_run(d_ptr, off, ml); 367| 513k| } 368| 9.58M|} zxc_decompress.c:zxc_decode_fill_run: 222| 397k| const uint64_t ml) { 223| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 224| | const __m256i v = _mm256_set1_epi8((char)byte); 225| | _mm256_storeu_si256((__m256i*)dst, v); 226| | if (UNLIKELY(ml > ZXC_PAD_SIZE)) { 227| | uint8_t* out = dst + ZXC_PAD_SIZE; 228| | size_t rem = ml - ZXC_PAD_SIZE; 229| | while (rem > ZXC_PAD_SIZE) { 230| | _mm256_storeu_si256((__m256i*)out, v); 231| | out += ZXC_PAD_SIZE; 232| | rem -= ZXC_PAD_SIZE; 233| | } 234| | _mm256_storeu_si256((__m256i*)out, v); 235| | } 236| |#elif defined(ZXC_USE_SSE2) 237| | const __m128i v = _mm_set1_epi8((char)byte); 238| 397k| _mm_storeu_si128((__m128i*)dst, v); 239| 397k| _mm_storeu_si128((__m128i*)(dst + 16), v); 240| 397k| if (UNLIKELY(ml > ZXC_PAD_SIZE)) { ------------------ | | 151| 397k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 160k, False: 237k] | | ------------------ ------------------ 241| 160k| uint8_t* out = dst + ZXC_PAD_SIZE; ------------------ | | 337| 160k|#define ZXC_PAD_SIZE 32 ------------------ 242| 160k| size_t rem = ml - ZXC_PAD_SIZE; ------------------ | | 337| 160k|#define ZXC_PAD_SIZE 32 ------------------ 243| 1.85M| while (rem > ZXC_PAD_SIZE) { ------------------ | | 337| 1.85M|#define ZXC_PAD_SIZE 32 ------------------ | Branch (243:16): [True: 1.69M, False: 160k] ------------------ 244| 1.69M| _mm_storeu_si128((__m128i*)out, v); 245| 1.69M| _mm_storeu_si128((__m128i*)(out + 16), v); 246| 1.69M| out += ZXC_PAD_SIZE; ------------------ | | 337| 1.69M|#define ZXC_PAD_SIZE 32 ------------------ 247| 1.69M| rem -= ZXC_PAD_SIZE; ------------------ | | 337| 1.69M|#define ZXC_PAD_SIZE 32 ------------------ 248| 1.69M| } 249| 160k| _mm_storeu_si128((__m128i*)out, v); 250| 160k| _mm_storeu_si128((__m128i*)(out + 16), v); 251| 160k| } 252| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 253| | const uint8x16_t v = vdupq_n_u8(byte); 254| | vst1q_u8(dst, v); 255| | vst1q_u8(dst + 16, v); 256| | if (UNLIKELY(ml > ZXC_PAD_SIZE)) { 257| | uint8_t* out = dst + ZXC_PAD_SIZE; 258| | size_t rem = ml - ZXC_PAD_SIZE; 259| | while (rem > ZXC_PAD_SIZE) { 260| | vst1q_u8(out, v); 261| | vst1q_u8(out + 16, v); 262| | out += ZXC_PAD_SIZE; 263| | rem -= ZXC_PAD_SIZE; 264| | } 265| | vst1q_u8(out, v); 266| | vst1q_u8(out + 16, v); 267| | } 268| |#else 269| | ZXC_MEMSET(dst, byte, ml); 270| |#endif 271| 397k|} zxc_decompress.c:zxc_decode_copy_overlap_run: 157| 517k| const uint64_t ml) { 158| 517k| const size_t stride = zxc_overlap_strides[off]; 159| 517k| size_t copied = 0; 160| |#if defined(ZXC_USE_NEON64) 161| | const uint8x16_t mask = vld1q_u8(zxc_overlap_masks[off]); 162| | const uint8x16_t pat = vqtbl1q_u8(vld1q_u8(dst - off), mask); 163| | do { 164| | vst1q_u8(dst + copied, pat); 165| | copied += stride; 166| | } while (copied < ml); 167| | 168| |#elif defined(ZXC_USE_NEON32) 169| | uint8x8x2_t src_tbl; 170| | src_tbl.val[0] = vld1_u8(dst - off); 171| | src_tbl.val[1] = vld1_u8(dst - off + 8); 172| | const uint8x8_t pat_lo = vtbl2_u8(src_tbl, vld1_u8(zxc_overlap_masks[off])); 173| | const uint8x8_t pat_hi = vtbl2_u8(src_tbl, vld1_u8(zxc_overlap_masks[off] + 8)); 174| | do { 175| | vst1_u8(dst + copied, pat_lo); 176| | vst1_u8(dst + copied + 8, pat_hi); 177| | copied += stride; 178| | } while (copied < ml); 179| | 180| |#elif defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 181| | const __m128i mask = _mm_load_si128((const __m128i*)zxc_overlap_masks[off]); 182| | const __m128i src_data = _mm_loadu_si128((const __m128i*)(dst - off)); 183| | const __m128i pat = _mm_shuffle_epi8(src_data, mask); 184| | do { 185| | _mm_storeu_si128((__m128i*)(dst + copied), pat); 186| | copied += stride; 187| | } while (copied < ml); 188| | 189| |#else 190| | // SSE2-only tier and non-SIMD builds: no PSHUFB, build the pattern with a 191| | // wrap counter (no per-byte modulo), then store it via zxc_copy16. 192| 517k| const uint8_t* src = dst - off; 193| 517k| uint8_t pat[16]; 194| 517k| uint32_t k = 0; 195| 8.78M| for (size_t i = 0; i < 16; i++) { ------------------ | Branch (195:24): [True: 8.27M, False: 517k] ------------------ 196| 8.27M| pat[i] = src[k]; 197| 8.27M| if (++k == off) k = 0; ------------------ | Branch (197:13): [True: 1.30M, False: 6.96M] ------------------ 198| 8.27M| } 199| 3.19M| do { 200| 3.19M| zxc_copy16(dst + copied, pat); 201| 3.19M| copied += stride; 202| 3.19M| } while (copied < ml); ------------------ | Branch (202:14): [True: 2.67M, False: 517k] ------------------ 203| 517k|#endif 204| 517k|} zxc_decompress.c:zxc_decode_copy_match_exact: 387| 177k| const size_t off, const size_t ml) { 388| 177k| if (off >= ml) { ------------------ | Branch (388:9): [True: 158k, False: 18.8k] ------------------ 389| 158k| ZXC_MEMCPY(d_ptr, match_src, ml); ------------------ | | 173| 158k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 390| 158k| } else if (ml < 16) { ------------------ | Branch (390:16): [True: 3.72k, False: 15.1k] ------------------ 391| 43.1k| for (size_t i = 0; i < ml; i++) d_ptr[i] = match_src[i]; ------------------ | Branch (391:28): [True: 39.4k, False: 3.72k] ------------------ 392| 15.1k| } else { 393| 15.1k| size_t n = off; 394| 15.1k| ZXC_MEMCPY(d_ptr, match_src, n); ------------------ | | 173| 15.1k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 395| 54.4k| while (n <= ml - n) { ------------------ | Branch (395:16): [True: 39.3k, False: 15.1k] ------------------ 396| 39.3k| ZXC_MEMCPY(d_ptr + n, d_ptr, n); ------------------ | | 173| 39.3k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 397| 39.3k| n <<= 1; 398| 39.3k| } 399| 15.1k| ZXC_MEMCPY(d_ptr + n, d_ptr, ml - n); ------------------ | | 173| 15.1k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 400| 15.1k| } 401| 177k|} zxc_decompress.c:zxc_decode_block_glo: 1334| 8.99k| const size_t dst_capacity) { 1335| 8.99k| return zxc_decode_block_glo_impl(ctx, src, src_size, dst, dst_capacity, 0, 0, 0); 1336| 8.99k|} zxc_decompress.c:zxc_decode_block_ghi_impl: 1060| 8.18k| const int has_dict) { 1061| 8.18k| zxc_gnr_header_t gh; 1062| | 1063| | /* 0 when !has_dict (safe path) -> folds `d_floor` to `dst`. */ 1064| 8.18k| const size_t dict_size = has_dict ? ctx->dict_size : 0; ------------------ | Branch (1064:30): [True: 0, False: 8.18k] ------------------ 1065| 8.18k| zxc_section_desc_t desc[ZXC_GHI_SECTIONS]; 1066| | 1067| 8.18k| if (UNLIKELY(zxc_read_ghi_header_and_desc(src, src_size, &gh, desc) != ZXC_OK)) ------------------ | | 151| 8.18k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.18k] | | ------------------ ------------------ 1068| 0| return ZXC_ERROR_BAD_HEADER; 1069| | 1070| 8.18k| const uint8_t* p_curr = 1071| 8.18k| src + ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 403| 8.18k|#define ZXC_GHI_HEADER_BINARY_SIZE 16 ------------------ src + ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 421| 8.18k|#define ZXC_GHI_SECTIONS 3 ------------------ src + ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 8.18k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 1072| | 1073| | // --- Stream Pointers & Validation --- 1074| 8.18k| const size_t sz_lit = (uint32_t)desc[0].sizes; 1075| 8.18k| const size_t sz_seqs = (uint32_t)desc[1].sizes; 1076| 8.18k| const size_t sz_exts = (uint32_t)desc[2].sizes; 1077| 8.18k| const uint8_t* l_ptr = p_curr; 1078| 8.18k| const uint8_t* l_end = l_ptr + sz_lit; 1079| 8.18k| p_curr += sz_lit; 1080| | 1081| 8.18k| const uint8_t* seq_ptr = p_curr; 1082| 8.18k| const uint8_t* extras_ptr = p_curr + sz_seqs; 1083| 8.18k| const uint8_t* const extras_end = extras_ptr + sz_exts; 1084| | 1085| | // Validate streams don't overflow source buffer + 1086| | // Validate sequence stream size matches sequence count 1087| 8.18k| if (UNLIKELY((extras_end != src + src_size) || ------------------ | | 151| 16.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.18k] | | | Branch (151:42): [True: 1, False: 8.18k] | | | Branch (151:42): [True: 1, False: 8.18k] | | ------------------ ------------------ 1088| 8.18k| ((uint64_t)sz_seqs < (uint64_t)gh.n_sequences * 4))) 1089| 0| return ZXC_ERROR_CORRUPT_DATA; 1090| | 1091| 8.18k| if (UNLIKELY((size_t)(src + src_size - l_end) < ZXC_PAD_SIZE)) { ------------------ | | 151| 8.18k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.95k, False: 6.23k] | | ------------------ ------------------ 1092| 1.95k| const int rc = zxc_stage_raw_literals(ctx, &l_ptr, &l_end); 1093| 1.95k| if (UNLIKELY(rc != ZXC_OK)) return rc; ------------------ | | 151| 1.95k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.95k] | | ------------------ ------------------ 1094| 1.95k| } 1095| | 1096| 8.18k| uint8_t* d_ptr = dst; 1097| 8.18k| const uint8_t* const d_end = dst + dst_capacity; 1098| | // Lowest address a match may reach: the dictionary prefix if any, else dst. 1099| 8.18k| const uint8_t* const d_floor = dst - dict_size; 1100| 8.18k| const uint8_t* const d_end_safe = d_end - (ZXC_PAD_SIZE * 4); // 128 ------------------ | | 337| 8.18k|#define ZXC_PAD_SIZE 32 ------------------ 1101| | // Safety margin for 4x unrolled loop: 4 * (ZXC_SEQ_LL_MASK LL + 1102| | // ZXC_SEQ_ML_MASK+ZXC_LZ_MIN_MATCH_LEN ML) + ZXC_PAD_SIZE Pad = 4 x (255 + 255 + 5) + 32 = 2092 1103| 8.18k| const uint8_t* const d_end_fast = d_end - ZXC_DECOMPRESS_TAIL_PAD; // 2112 ------------------ | | 349| 8.18k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 8.18k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 1104| | 1105| | // Literal margin for the GHI loops: without a varint, ll <= 254 per sequence, 1106| | // so 4 * 254 = 1016. Past that only the cold varint path checks l_ptr. 1107| 8.18k| const size_t ghi_margin_4x = 4 * (ZXC_SEQ_LL_MASK - 1); // 1016 ------------------ | | 469| 8.18k|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 463| 8.18k|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ 1108| 8.18k| const size_t ghi_margin_1x = ZXC_SEQ_LL_MASK - 1; // 254 ------------------ | | 469| 8.18k|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 463| 8.18k|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ 1109| 8.18k| const uint8_t* const l_end_safe_4x = (sz_lit > ghi_margin_4x) ? l_end - ghi_margin_4x : l_ptr; ------------------ | Branch (1109:42): [True: 6.39k, False: 1.79k] ------------------ 1110| 8.18k| const uint8_t* const l_end_safe_1x = (sz_lit > ghi_margin_1x) ? l_end - ghi_margin_1x : l_ptr; ------------------ | Branch (1110:42): [True: 7.10k, False: 1.08k] ------------------ 1111| | 1112| 8.18k| uint32_t n_seq = gh.n_sequences; 1113| | 1114| | // Same handover as GLO, but the GHI offset is inline in the sequence word and 1115| | // always 16 bits -- enc_off is not a width here, so the span is fixed. 1116| 8.18k| const size_t off_span = 1U << 16; 1117| 8.18k| const size_t span = (dict_size >= off_span) ? 0 : off_span - dict_size; ------------------ | Branch (1117:25): [True: 0, False: 8.18k] ------------------ 1118| 8.18k| const uint8_t* const d_bounds = (span >= dst_capacity) ? d_end : dst + span; ------------------ | Branch (1118:37): [True: 0, False: 8.18k] ------------------ 1119| | 1120| | // --- SAFE loop: validate offsets until d_bounds (4x unroll) --- 1121| 8.18k| if (safe) { ------------------ | Branch (1121:9): [True: 0, False: 8.18k] ------------------ 1122| | /* SAFE variant: save per-batch state so an OVERFLOW can rollback and 1123| | * hand over to the 1x loop / Safe Path. Wild writes already committed 1124| | * are deterministically overwritten when the 1x loop replays. */ 1125| 0| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (1125:16): [True: 0, False: 0] | Branch (1125:30): [True: 0, False: 0] | Branch (1125:52): [True: 0, False: 0] | Branch (1125:77): [True: 0, False: 0] ------------------ 1126| 0| const uint8_t* const t_save = seq_ptr; 1127| 0| const uint8_t* const e_save = extras_ptr; 1128| 0| uint8_t* const d_save = d_ptr; 1129| 0| const uint8_t* const l_save = l_ptr; 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); ------------------ | | 543| 0| do { \ | | 544| 0| uint32_t s1 = zxc_le32(seq_ptr); \ | | 545| 0| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 546| 0| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 547| 0| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 548| 0| seq_ptr += 4 * sizeof(uint32_t); \ | | 549| 0| PREFETCH; \ | | 550| 0| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 551| 0| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 552| 0| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 553| 0| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 554| 0| n_seq -= 4; \ | | 555| 0| } while (0) | | ------------------ | | | Branch (555:14): [Folded, False: 0] | | ------------------ ------------------ 1131| 0| continue; 1132| | 1133| 0| rollback_safe_4x: 1134| 0| seq_ptr = t_save; 1135| 0| extras_ptr = e_save; 1136| 0| d_ptr = d_save; 1137| 0| l_ptr = l_save; 1138| 0| break; 1139| 0| } 1140| 8.18k| } else { 1141| 263k| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (1141:16): [True: 259k, False: 3.94k] | Branch (1141:30): [True: 259k, False: 18.4E] | Branch (1141:52): [True: 257k, False: 2.33k] | Branch (1141:77): [True: 255k, False: 1.91k] ------------------ 1142| 255k| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); ------------------ | | 543| 255k| do { \ | | 544| 255k| uint32_t s1 = zxc_le32(seq_ptr); \ | | 545| 255k| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 546| 255k| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 547| 255k| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 548| 255k| seq_ptr += 4 * sizeof(uint32_t); \ | | 549| 255k| PREFETCH; \ | | 550| 255k| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 255k| do { \ | | | | 517| 255k| uint64_t ll = (S) >> 24; \ | | | | 518| 255k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 255k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 5.46k, False: 249k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 5.46k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 5.46k| const uint64_t reserve = (RESERVE); \ | | | | 521| 5.46k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 10.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 5.46k] | | | | | | | Branch (151:42): [True: 0, False: 5.46k] | | | | | | | Branch (151:42): [True: 0, False: 5.46k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 5.46k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 5.46k| ON_FAIL; \ | | | | 524| 5.46k| } \ | | | | 525| 255k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 255k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 255k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 255k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 255k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 12.5k, False: 242k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 12.5k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 12.5k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 12.5k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 12.5k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 12.5k| (size_t)(d_end - d_ptr))) \ | | | | 531| 12.5k| ON_FAIL; \ | | | | 532| 12.5k| } \ | | | | 533| 255k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 255k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 255k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1142| 255k| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 405| 255k| do { \ | | | | | | | | 406| 255k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 255k| l_ptr += ll; \ | | | | | | | | 408| 255k| d_ptr += ll; \ | | | | | | | | 409| 255k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 255k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 255k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 255k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 255k| d_ptr += ml; \ | | | | | | | | 412| 255k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 255k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 255k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 255k] | | | | ------------------ | | ------------------ | | 551| 255k| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 255k| do { \ | | | | 517| 255k| uint64_t ll = (S) >> 24; \ | | | | 518| 255k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 255k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 5.06k, False: 250k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 5.06k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 5.06k| const uint64_t reserve = (RESERVE); \ | | | | 521| 5.06k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 10.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 5.06k] | | | | | | | Branch (151:42): [True: 0, False: 5.06k] | | | | | | | Branch (151:42): [True: 0, False: 5.06k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 5.06k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 5.06k| ON_FAIL; \ | | | | 524| 5.06k| } \ | | | | 525| 255k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 255k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 255k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 255k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 255k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 13.1k, False: 242k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 13.1k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 13.1k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 13.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 13.1k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 13.1k| (size_t)(d_end - d_ptr))) \ | | | | 531| 13.1k| ON_FAIL; \ | | | | 532| 13.1k| } \ | | | | 533| 255k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 255k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 255k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1142| 255k| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 405| 255k| do { \ | | | | | | | | 406| 255k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 255k| l_ptr += ll; \ | | | | | | | | 408| 255k| d_ptr += ll; \ | | | | | | | | 409| 255k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 255k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 255k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 255k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 255k| d_ptr += ml; \ | | | | | | | | 412| 255k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 255k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 255k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 255k] | | | | ------------------ | | ------------------ | | 552| 255k| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 255k| do { \ | | | | 517| 255k| uint64_t ll = (S) >> 24; \ | | | | 518| 255k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 255k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 4.89k, False: 250k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 4.89k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 4.89k| const uint64_t reserve = (RESERVE); \ | | | | 521| 4.89k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 9.78k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 4.89k] | | | | | | | Branch (151:42): [True: 0, False: 4.89k] | | | | | | | Branch (151:42): [True: 0, False: 4.89k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 4.89k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 4.89k| ON_FAIL; \ | | | | 524| 4.89k| } \ | | | | 525| 255k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 255k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 255k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 255k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 255k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 12.4k, False: 242k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 12.4k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 12.4k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 12.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 12.4k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 12.4k| (size_t)(d_end - d_ptr))) \ | | | | 531| 12.4k| ON_FAIL; \ | | | | 532| 12.4k| } \ | | | | 533| 255k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 255k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 255k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1142| 255k| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 405| 255k| do { \ | | | | | | | | 406| 255k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 255k| l_ptr += ll; \ | | | | | | | | 408| 255k| d_ptr += ll; \ | | | | | | | | 409| 255k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 255k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 255k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 255k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 255k| d_ptr += ml; \ | | | | | | | | 412| 255k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 255k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 255k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 255k] | | | | ------------------ | | ------------------ | | 553| 255k| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 255k| do { \ | | | | 517| 255k| uint64_t ll = (S) >> 24; \ | | | | 518| 255k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 255k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 5.21k, False: 249k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 5.21k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 5.21k| const uint64_t reserve = (RESERVE); \ | | | | 521| 5.21k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 10.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 5.21k] | | | | | | | Branch (151:42): [True: 0, False: 5.21k] | | | | | | | Branch (151:42): [True: 0, False: 5.21k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 5.21k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 5.21k| ON_FAIL; \ | | | | 524| 5.21k| } \ | | | | 525| 255k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 255k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 255k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 255k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 255k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 13.1k, False: 242k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 13.1k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 13.1k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 13.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 13.1k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 13.1k| (size_t)(d_end - d_ptr))) \ | | | | 531| 13.1k| ON_FAIL; \ | | | | 532| 13.1k| } \ | | | | 533| 255k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 255k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 255k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1142| 255k| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 405| 255k| do { \ | | | | | | | | 406| 255k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 255k| l_ptr += ll; \ | | | | | | | | 408| 255k| d_ptr += ll; \ | | | | | | | | 409| 255k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 255k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 255k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 255k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 255k| d_ptr += ml; \ | | | | | | | | 412| 255k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 255k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 255k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 255k] | | | | ------------------ | | ------------------ | | 554| 255k| n_seq -= 4; \ | | 555| 255k| } while (0) | | ------------------ | | | Branch (555:14): [Folded, False: 255k] | | ------------------ ------------------ 1143| 255k| } 1144| 8.18k| } 1145| | 1146| | // --- SAFE Loop tail: remaining sequences with offset validation (1x) --- 1147| 148k| while (n_seq > 0 && d_ptr < d_end_safe && d_ptr < d_bounds) { ------------------ | Branch (1147:12): [True: 143k, False: 5.76k] | Branch (1147:25): [True: 143k, False: 0] | Branch (1147:47): [True: 140k, False: 2.42k] ------------------ 1148| 140k| uint32_t seq = zxc_le32(seq_ptr); 1149| 140k| seq_ptr += sizeof(uint32_t); 1150| | 1151| 140k| uint64_t ll = seq >> 24; 1152| 140k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 140k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.40k, False: 139k] | | ------------------ ------------------ 1153| | 1154| 140k| uint32_t m_bits = (seq >> 16) & 0xFF; 1155| 140k| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 140k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1156| 140k| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 140k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 6.12k, False: 134k] | | ------------------ ------------------ 1157| | 1158| 140k| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 527| 140k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1159| | 1160| | // Strict bounds check: sequence must fit, AND wild copies must not overshoot 1161| | // Check both destination (d_ptr) and source literal stream (l_ptr) 1162| 140k| if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr) || ------------------ | | 151| 281k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 12.7k, False: 128k] | | | Branch (151:42): [True: 0, False: 140k] | | | Branch (151:42): [True: 12.7k, False: 128k] | | ------------------ ------------------ 1163| 140k| ll + ZXC_PAD_SIZE > (size_t)(l_end - l_ptr))) { 1164| | // Fallback to exact copy (slow but safe) 1165| 12.7k| if (UNLIKELY(d_ptr + ll > d_end || l_ptr + ll > l_end)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 25.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 12.7k] | | | Branch (151:42): [True: 0, False: 12.7k] | | | Branch (151:42): [True: 0, False: 12.7k] | | ------------------ ------------------ 1166| 12.7k| ZXC_MEMCPY(d_ptr, l_ptr, ll); ------------------ | | 173| 12.7k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1167| 12.7k| l_ptr += ll; 1168| 12.7k| d_ptr += ll; 1169| | 1170| 12.7k| if (UNLIKELY(d_ptr + ml > d_end)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 12.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 12.7k] | | ------------------ ------------------ 1171| 12.7k| if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET; ------------------ | | 151| 12.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 12.7k] | | ------------------ ------------------ 1172| 12.7k| const uint8_t* match_src = d_ptr - offset; 1173| | 1174| 12.7k| zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml); 1175| 12.7k| d_ptr += ml; 1176| 128k| } else { 1177| 128k| DECODE_SEQ_SAFE(ll, ml, offset); ------------------ | | 405| 128k| do { \ | | 406| 128k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | 407| 128k| l_ptr += ll; \ | | 408| 128k| d_ptr += ll; \ | | 409| 128k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | ------------------ | | | | 151| 128k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | ------------------ | | | | | Branch (151:21): [True: 0, False: 128k] | | | | ------------------ | | ------------------ | | 410| 128k| zxc_decode_copy_match(d_ptr, off, ml); \ | | 411| 128k| d_ptr += ml; \ | | 412| 128k| } while (0) | | ------------------ | | | Branch (412:14): [Folded, False: 128k] | | ------------------ ------------------ 1178| 128k| } 1179| 140k| n_seq--; 1180| 140k| } 1181| | 1182| | // --- FAST Loop: After threshold, check large margin to avoid individual bounds checks --- 1183| 8.18k| if (safe) { ------------------ | Branch (1183:9): [True: 0, False: 8.18k] ------------------ 1184| 0| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) { ------------------ | Branch (1184:16): [True: 0, False: 0] | Branch (1184:30): [True: 0, False: 0] | Branch (1184:52): [True: 0, False: 0] ------------------ 1185| 0| const uint8_t* const t_save = seq_ptr; 1186| 0| const uint8_t* const e_save = extras_ptr; 1187| 0| uint8_t* const d_save = d_ptr; 1188| 0| const uint8_t* const l_save = l_ptr; 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, ------------------ | | 543| 0| do { \ | | 544| 0| uint32_t s1 = zxc_le32(seq_ptr); \ | | 545| 0| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 546| 0| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 547| 0| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 548| 0| seq_ptr += 4 * sizeof(uint32_t); \ | | 549| 0| PREFETCH; \ | | 550| 0| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 551| 0| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 552| 0| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 553| 0| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 554| 0| n_seq -= 4; \ | | 555| 0| } while (0) | | ------------------ | | | Branch (555:14): [Folded, False: 0] | | ------------------ ------------------ 1190| 0| ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE)); 1191| 0| continue; 1192| | 1193| 0| rollback_fast_4x: 1194| 0| seq_ptr = t_save; 1195| 0| extras_ptr = e_save; 1196| 0| d_ptr = d_save; 1197| 0| l_ptr = l_save; 1198| 0| break; 1199| 0| } 1200| 8.18k| } else { 1201| 594k| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) { ------------------ | Branch (1201:16): [True: 588k, False: 6.48k] | Branch (1201:30): [True: 588k, False: 0] | Branch (1201:52): [True: 586k, False: 1.70k] ------------------ 1202| 586k| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, ------------------ | | 543| 586k| do { \ | | 544| 586k| uint32_t s1 = zxc_le32(seq_ptr); \ | | 545| 586k| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 546| 586k| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 547| 586k| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 548| 586k| seq_ptr += 4 * sizeof(uint32_t); \ | | 549| 586k| PREFETCH; \ | | 550| 586k| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 586k| do { \ | | | | 517| 586k| uint64_t ll = (S) >> 24; \ | | | | 518| 586k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 586k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 8.99k, False: 577k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 8.99k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 8.99k| const uint64_t reserve = (RESERVE); \ | | | | 521| 8.99k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 17.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 8.99k] | | | | | | | Branch (151:42): [True: 0, False: 8.99k] | | | | | | | Branch (151:42): [True: 0, False: 8.99k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 8.99k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 8.99k| ON_FAIL; \ | | | | 524| 8.99k| } \ | | | | 525| 586k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 586k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 586k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 586k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 586k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 37.2k, False: 549k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 37.2k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 37.2k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 37.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 37.2k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 37.2k| (size_t)(d_end - d_ptr))) \ | | | | 531| 37.2k| ON_FAIL; \ | | | | 532| 37.2k| } \ | | | | 533| 586k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 586k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 586k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1202| 586k| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 417| 586k| do { \ | | | | | | | | 418| 586k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 586k| l_ptr += ll; \ | | | | | | | | 420| 586k| d_ptr += ll; \ | | | | | | | | 421| 586k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 586k| d_ptr += ml; \ | | | | | | | | 423| 586k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 586k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 586k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 586k] | | | | ------------------ | | ------------------ | | 551| 586k| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 586k| do { \ | | | | 517| 586k| uint64_t ll = (S) >> 24; \ | | | | 518| 586k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 586k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 8.81k, False: 577k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 8.81k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 8.81k| const uint64_t reserve = (RESERVE); \ | | | | 521| 8.81k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 17.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 8.81k] | | | | | | | Branch (151:42): [True: 0, False: 8.81k] | | | | | | | Branch (151:42): [True: 0, False: 8.81k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 8.81k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 8.81k| ON_FAIL; \ | | | | 524| 8.81k| } \ | | | | 525| 586k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 586k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 586k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 586k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 586k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 37.2k, False: 549k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 37.2k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 37.2k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 37.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 37.2k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 37.2k| (size_t)(d_end - d_ptr))) \ | | | | 531| 37.2k| ON_FAIL; \ | | | | 532| 37.2k| } \ | | | | 533| 586k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 586k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 586k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1202| 586k| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 417| 586k| do { \ | | | | | | | | 418| 586k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 586k| l_ptr += ll; \ | | | | | | | | 420| 586k| d_ptr += ll; \ | | | | | | | | 421| 586k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 586k| d_ptr += ml; \ | | | | | | | | 423| 586k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 586k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 586k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 586k] | | | | ------------------ | | ------------------ | | 552| 586k| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 586k| do { \ | | | | 517| 586k| uint64_t ll = (S) >> 24; \ | | | | 518| 586k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 586k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 8.89k, False: 577k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 8.89k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 8.89k| const uint64_t reserve = (RESERVE); \ | | | | 521| 8.89k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 17.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 8.89k] | | | | | | | Branch (151:42): [True: 0, False: 8.89k] | | | | | | | Branch (151:42): [True: 0, False: 8.89k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 8.89k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 8.89k| ON_FAIL; \ | | | | 524| 8.89k| } \ | | | | 525| 586k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 586k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 586k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 586k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 586k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 37.1k, False: 549k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 37.1k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 37.1k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 37.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 37.1k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 37.1k| (size_t)(d_end - d_ptr))) \ | | | | 531| 37.1k| ON_FAIL; \ | | | | 532| 37.1k| } \ | | | | 533| 586k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 586k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 586k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1202| 586k| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 417| 586k| do { \ | | | | | | | | 418| 586k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 586k| l_ptr += ll; \ | | | | | | | | 420| 586k| d_ptr += ll; \ | | | | | | | | 421| 586k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 586k| d_ptr += ml; \ | | | | | | | | 423| 586k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 586k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 586k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 586k] | | | | ------------------ | | ------------------ | | 553| 586k| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 586k| do { \ | | | | 517| 586k| uint64_t ll = (S) >> 24; \ | | | | 518| 586k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 586k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 9.00k, False: 577k] | | | | | | ------------------ | | | | ------------------ | | | | 519| 9.00k| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 9.00k| const uint64_t reserve = (RESERVE); \ | | | | 521| 9.00k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 18.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 9.00k] | | | | | | | Branch (151:42): [True: 0, False: 9.00k] | | | | | | | Branch (151:42): [True: 0, False: 9.00k] | | | | | | ------------------ | | | | ------------------ | | | | 522| 9.00k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 9.00k| ON_FAIL; \ | | | | 524| 9.00k| } \ | | | | 525| 586k| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 586k| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 586k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 586k| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 586k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 37.1k, False: 549k] | | | | | | ------------------ | | | | ------------------ | | | | 528| 37.1k| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 37.1k| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 37.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 37.1k] | | | | | | ------------------ | | | | ------------------ | | | | 530| 37.1k| (size_t)(d_end - d_ptr))) \ | | | | 531| 37.1k| ON_FAIL; \ | | | | 532| 37.1k| } \ | | | | 533| 586k| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 586k|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 586k| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1202| 586k| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 417| 586k| do { \ | | | | | | | | 418| 586k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 586k| l_ptr += ll; \ | | | | | | | | 420| 586k| d_ptr += ll; \ | | | | | | | | 421| 586k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 586k| d_ptr += ml; \ | | | | | | | | 423| 586k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 586k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 586k| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 586k] | | | | ------------------ | | ------------------ | | 554| 586k| n_seq -= 4; \ | | 555| 586k| } while (0) | | ------------------ | | | Branch (555:14): [Folded, False: 586k] | | ------------------ ------------------ 1203| 586k| ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE)); 1204| 586k| } 1205| 8.18k| } 1206| | 1207| | // --- Remaining 1 sequence (Fast Path) --- 1208| 155k| while (n_seq > 0 && d_ptr < d_end_safe && l_ptr < l_end_safe_1x) { ------------------ | Branch (1208:12): [True: 149k, False: 6.22k] | Branch (1208:25): [True: 149k, False: 0] | Branch (1208:47): [True: 147k, False: 1.68k] ------------------ 1209| | // Save state for fallback 1210| 147k| const uint8_t* seq_save = seq_ptr; 1211| 147k| const uint8_t* ext_save = extras_ptr; 1212| | 1213| 147k| const uint32_t seq = zxc_le32(seq_ptr); 1214| 147k| seq_ptr += sizeof(uint32_t); 1215| | 1216| 147k| uint64_t ll = seq >> 24; 1217| 147k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { ------------------ | | 151| 147k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 780, False: 146k] | | ------------------ ------------------ 1218| 780| ll += zxc_read_varint(&extras_ptr, extras_end); 1219| 780| if (UNLIKELY(l_ptr + ll > l_end)) { ------------------ | | 151| 780|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 780] | | ------------------ ------------------ 1220| 0| seq_ptr = seq_save; 1221| 0| extras_ptr = ext_save; 1222| 0| break; 1223| 0| } 1224| 780| } 1225| | 1226| 147k| uint32_t m_bits = (seq >> 16) & 0xFF; 1227| 147k| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 147k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1228| 147k| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 147k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 11.1k, False: 136k] | | ------------------ ------------------ 1229| | 1230| | // Strict bounds checks (including wild copy overrun safety) 1231| 147k| if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr) || ------------------ | | 151| 295k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 273, False: 147k] | | | Branch (151:42): [True: 0, False: 147k] | | | Branch (151:42): [True: 273, False: 147k] | | ------------------ ------------------ 1232| 147k| ll + ZXC_PAD_SIZE > (size_t)(l_end - l_ptr))) { 1233| 273| seq_ptr = seq_save; 1234| 273| extras_ptr = ext_save; 1235| 273| break; 1236| 273| } 1237| 147k| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 527| 147k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1238| | 1239| 147k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); 1240| 147k| l_ptr += ll; 1241| 147k| d_ptr += ll; 1242| | 1243| 147k| if (UNLIKELY(d_ptr < d_bounds && (size_t)(d_ptr - d_floor) < offset)) ------------------ | | 151| 147k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 147k] | | | Branch (151:42): [True: 0, False: 147k] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1244| 0| return ZXC_ERROR_BAD_OFFSET; 1245| | 1246| | /* The loop entry check guarantees ll + ml + ZXC_PAD_SIZE bytes of 1247| | * headroom, so the wild-copy ladder (incl. overlap/fill runs) is safe. */ 1248| 147k| zxc_decode_copy_match(d_ptr, offset, ml); 1249| 147k| d_ptr += ml; 1250| 147k| n_seq--; 1251| 147k| } 1252| | 1253| | // --- Safe Path for Remaining Sequences --- 1254| 139k| while (n_seq > 0) { ------------------ | Branch (1254:12): [True: 130k, False: 8.18k] ------------------ 1255| 130k| uint32_t seq = zxc_le32(seq_ptr); 1256| 130k| seq_ptr += sizeof(uint32_t); 1257| | 1258| 130k| uint64_t ll = seq >> 24; 1259| 130k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 130k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 261, False: 130k] | | ------------------ ------------------ 1260| | 1261| 130k| uint32_t m_bits = (seq >> 16) & 0xFF; 1262| 130k| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 130k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1263| 130k| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 130k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 14.7k, False: 116k] | | ------------------ ------------------ 1264| 130k| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 527| 130k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1265| | 1266| 130k| if (UNLIKELY(ll + ml > (size_t)(d_end - d_ptr) || l_ptr + ll > l_end)) ------------------ | | 151| 261k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 130k] | | | Branch (151:42): [True: 0, False: 130k] | | | Branch (151:42): [True: 1, False: 130k] | | ------------------ ------------------ 1267| 0| return ZXC_ERROR_OVERFLOW; 1268| 130k| ZXC_MEMCPY(d_ptr, l_ptr, ll); ------------------ | | 173| 130k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1269| 130k| l_ptr += ll; 1270| 130k| d_ptr += ll; 1271| | 1272| 130k| if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET; ------------------ | | 151| 130k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 130k] | | ------------------ ------------------ 1273| 130k| const uint8_t* match_src = d_ptr - offset; 1274| | 1275| 130k| zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml); 1276| 130k| d_ptr += ml; 1277| 130k| n_seq--; 1278| 130k| } 1279| | 1280| | // --- Trailing Literals --- 1281| | // Copy remaining literals from source stream (literal exhaustion) 1282| 8.18k| if (UNLIKELY(l_ptr > l_end)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 8.18k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.18k] | | ------------------ ------------------ 1283| 8.18k| const size_t remaining_literals = (size_t)(l_end - l_ptr); 1284| 8.18k| if (UNLIKELY(remaining_literals > (size_t)(d_end - d_ptr))) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 8.18k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 8.18k] | | ------------------ ------------------ 1285| 8.18k| ZXC_MEMCPY(d_ptr, l_ptr, remaining_literals); ------------------ | | 173| 8.18k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1286| 8.18k| d_ptr += remaining_literals; 1287| | 1288| 8.18k| return (int)(d_ptr - dst); 1289| 8.18k|} zxc_decompress.c:zxc_decode_block_ghi: 1352| 8.18k| const size_t dst_capacity) { 1353| 8.18k| return zxc_decode_block_ghi_impl(ctx, src, src_size, dst, dst_capacity, 0, 0); 1354| 8.18k|} zxc_decompress_chunk_wrapper: 488| 20.2k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 489| | /* dict_size is constant for a stream; this per-block branch (outside the decode 490| | * loop) routes to the dict variant only when a dictionary is active, so the 491| | * no-dict path runs the dict-free chunk wrapper (identical codegen to main). */ 492| 20.2k|#if ZXC_USE_C11_ATOMICS 493| 20.2k| const zxc_decompress_func_t func = atomic_load_explicit( 494| 20.2k| ctx->dict_size ? &zxc_decompress_dict_ptr : &zxc_decompress_ptr, memory_order_acquire); ------------------ | Branch (494:9): [True: 0, False: 20.2k] ------------------ 495| |#else 496| | const zxc_decompress_func_t func = 497| | ctx->dict_size ? zxc_decompress_dict_ptr : zxc_decompress_ptr; 498| |#endif 499| 20.2k| if (UNLIKELY(!func)) return zxc_decompress_dispatch_init(ctx, src, src_sz, dst, dst_cap); ------------------ | | 151| 20.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1, False: 20.2k] | | ------------------ ------------------ 500| 20.2k| return func(ctx, src, src_sz, dst, dst_cap); 501| 20.2k|} zxc_compress_chunk_wrapper: 540| 10.6k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 541| 10.6k|#if ZXC_USE_C11_ATOMICS 542| 10.6k| const zxc_compress_func_t func = atomic_load_explicit(&zxc_compress_ptr, memory_order_acquire); 543| |#else 544| | const zxc_compress_func_t func = zxc_compress_ptr; 545| |#endif 546| 10.6k| if (UNLIKELY(!func)) return zxc_compress_dispatch_init(ctx, src, src_sz, dst, dst_cap); ------------------ | | 151| 10.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1, False: 10.6k] | | ------------------ ------------------ 547| 10.6k| return func(ctx, src, src_sz, dst, dst_cap); 548| 10.6k|} zxc_compress: 671| 9.45k| const size_t dst_capacity, const zxc_compress_opts_t* opts) { 672| 9.45k| if (UNLIKELY(!dst || dst_capacity == 0 || (src_size > 0 && !src))) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 56.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | | Branch (151:42): [True: 0, False: 9.45k] | | | Branch (151:42): [True: 0, False: 9.45k] | | | Branch (151:42): [True: 9.45k, False: 0] | | | Branch (151:42): [True: 0, False: 9.45k] | | ------------------ ------------------ 673| | 674| 9.45k| const int checksum_enabled = opts ? opts->checksum_enabled : 0; ------------------ | Branch (674:34): [True: 9.45k, False: 0] ------------------ 675| 9.45k| const int seekable = opts ? opts->seekable : 0; ------------------ | Branch (675:26): [True: 9.45k, False: 0] ------------------ 676| 9.45k| const int level = zxc_level_clamp((opts && opts->level > 0) ? opts->level : ZXC_LEVEL_DEFAULT); ------------------ | Branch (676:40): [True: 9.45k, False: 0] | Branch (676:48): [True: 9.45k, False: 0] ------------------ 677| 9.45k| const size_t block_size = 678| 9.45k| (opts && opts->block_size > 0) ? opts->block_size : ZXC_BLOCK_SIZE_DEFAULT; ------------------ | | 60| 9.45k|#define ZXC_BLOCK_SIZE_DEFAULT (512 * 1024) ------------------ | Branch (678:10): [True: 9.45k, False: 0] | Branch (678:18): [True: 0, False: 9.45k] ------------------ 679| 9.45k| const uint8_t* dict = opts ? (const uint8_t*)opts->dict : NULL; ------------------ | Branch (679:27): [True: 9.45k, False: 0] ------------------ 680| 9.45k| const size_t dict_size = (opts && opts->dict) ? opts->dict_size : 0; ------------------ | Branch (680:31): [True: 9.45k, False: 0] | Branch (680:39): [True: 0, False: 9.45k] ------------------ 681| 9.45k| const uint8_t* dict_huf = (opts && opts->dict) ? (const uint8_t*)opts->dict_huf : NULL; ------------------ | Branch (681:32): [True: 9.45k, False: 0] | Branch (681:40): [True: 0, False: 9.45k] ------------------ 682| | 683| 9.45k| if (UNLIKELY(dict_size > ZXC_DICT_SIZE_MAX)) return ZXC_ERROR_DICT_TOO_LARGE; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 684| 9.45k| if (UNLIKELY(!zxc_validate_block_size(block_size))) return ZXC_ERROR_BAD_BLOCK_SIZE; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 685| | 686| 9.45k| const uint32_t did = (dict && dict_size > 0) ? zxc_dict_id(dict, dict_size, dict_huf) : 0; ------------------ | Branch (686:27): [True: 0, False: 9.45k] | Branch (686:35): [True: 0, False: 0] ------------------ 687| | 688| 9.45k| const uint8_t* ip = (const uint8_t*)src; 689| 9.45k| uint8_t* op = (uint8_t*)dst; 690| 9.45k| const uint8_t* op_start = op; 691| 9.45k| const uint8_t* op_end = op + dst_capacity; 692| 9.45k| uint32_t global_hash = 0; 693| 9.45k| zxc_cctx_t ctx; 694| | 695| 9.45k| const size_t eff_chunk = 696| 9.45k| dict_size > 0 ? zxc_block_size_ceil(dict_size + block_size) : block_size; ------------------ | Branch (696:9): [True: 0, False: 9.45k] ------------------ 697| | // LCOV_EXCL_START 698| 9.45k| if (UNLIKELY(zxc_cctx_init(&ctx, eff_chunk, 1, level, checksum_enabled, dict_size) != ZXC_OK)) ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 699| 0| return ZXC_ERROR_MEMORY; 700| | // LCOV_EXCL_STOP 701| 9.45k| if (UNLIKELY(zxc_cctx_attach_dict_huf(&ctx, dict_huf) != ZXC_OK)) { ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 702| | // LCOV_EXCL_START 703| 0| zxc_cctx_free(&ctx); 704| 0| return ZXC_ERROR_CORRUPT_DATA; 705| | // LCOV_EXCL_STOP 706| 0| } 707| | 708| | /* Dict input buffer: [dict_content | block_data] for the encoder, carved 709| | * into the cctx workspace (NULL when no dictionary is active). */ 710| 9.45k| uint8_t* const dict_input = ctx.dict_buffer; 711| 9.45k| if (dict_input) ZXC_MEMCPY(dict_input, dict, dict_size); ------------------ | | 173| 0|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ | Branch (711:9): [True: 0, False: 9.45k] ------------------ 712| | 713| 9.45k| const int h_val = 714| 9.45k| zxc_write_file_header(op, (size_t)(op_end - op), block_size, checksum_enabled, did); 715| | // LCOV_EXCL_START 716| 9.45k| if (UNLIKELY(h_val < 0)) { ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 717| 0| zxc_cctx_free(&ctx); 718| 0| return h_val; 719| 0| } 720| | // LCOV_EXCL_STOP 721| 9.45k| op += h_val; 722| | 723| | /* Seekable: dynamic array for per-block compressed sizes */ 724| 9.45k| uint32_t* seek_comp = NULL; 725| 9.45k| uint32_t seek_count = 0; 726| 9.45k| uint32_t seek_cap = 0; 727| 9.45k| if (seekable) { ------------------ | Branch (727:9): [True: 9.45k, False: 0] ------------------ 728| 9.45k| const size_t block_count = src_size / block_size; 729| 9.45k| if (UNLIKELY(block_count > (size_t)UINT32_MAX - 2)) { ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 730| | // LCOV_EXCL_START 731| 0| zxc_cctx_free(&ctx); 732| 0| return ZXC_ERROR_BAD_BLOCK_SIZE; 733| | // LCOV_EXCL_STOP 734| 0| } 735| 9.45k| seek_cap = (uint32_t)(block_count + 2); 736| 9.45k| seek_comp = (uint32_t*)ZXC_MALLOC(seek_cap * sizeof(uint32_t)); ------------------ | | 66| 9.45k|#define ZXC_MALLOC(size) malloc(size) ------------------ 737| | // LCOV_EXCL_START 738| 9.45k| if (UNLIKELY(!seek_comp)) { ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 739| 0| zxc_cctx_free(&ctx); 740| 0| return ZXC_ERROR_MEMORY; 741| 0| } 742| | // LCOV_EXCL_STOP 743| 9.45k| } 744| | 745| 9.45k| size_t pos = 0; 746| 20.1k| while (pos < src_size) { ------------------ | Branch (746:12): [True: 10.6k, False: 9.45k] ------------------ 747| 10.6k| const size_t chunk_len = (src_size - pos > block_size) ? block_size : (src_size - pos); ------------------ | Branch (747:34): [True: 1.21k, False: 9.45k] ------------------ 748| 10.6k| const size_t rem_cap = (size_t)(op_end - op); 749| | 750| 10.6k| int res; 751| 10.6k| if (dict_input) { ------------------ | Branch (751:13): [True: 0, False: 10.6k] ------------------ 752| 0| ZXC_MEMCPY(dict_input + dict_size, ip + pos, chunk_len); ------------------ | | 173| 0|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 753| 0| res = zxc_compress_chunk_wrapper(&ctx, dict_input, dict_size + chunk_len, op, rem_cap); 754| 10.6k| } else { 755| 10.6k| res = zxc_compress_chunk_wrapper(&ctx, ip + pos, chunk_len, op, rem_cap); 756| 10.6k| } 757| 10.6k| if (UNLIKELY(res < 0)) { ------------------ | | 151| 10.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 10.6k] | | ------------------ ------------------ 758| 0| ZXC_FREE(seek_comp); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 759| 0| zxc_cctx_free(&ctx); 760| 0| return res; 761| 0| } 762| | 763| 10.6k| if (checksum_enabled) { ------------------ | Branch (763:13): [True: 5.38k, False: 5.28k] ------------------ 764| | // Update Global Hash (Rotation + XOR) 765| | // Block checksum is at the end of the written block data 766| 5.38k| if (LIKELY(res >= ZXC_GLOBAL_CHECKSUM_SIZE)) { ------------------ | | 145| 5.38k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 5.38k, False: 0] | | ------------------ ------------------ 767| 5.38k| const uint32_t block_hash = zxc_le32(op + res - ZXC_GLOBAL_CHECKSUM_SIZE); ------------------ | | 427| 5.38k|#define ZXC_GLOBAL_CHECKSUM_SIZE 4 ------------------ 768| 5.38k| global_hash = zxc_hash_combine_rotate(global_hash, block_hash); 769| 5.38k| } 770| 5.38k| } 771| | 772| | /* Seekable: record compressed block size */ 773| 10.6k| if (seekable) { ------------------ | Branch (773:13): [True: 10.6k, False: 0] ------------------ 774| | // LCOV_EXCL_START 775| 10.6k| if (UNLIKELY(seek_count >= seek_cap)) { ------------------ | | 151| 10.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 10.6k] | | ------------------ ------------------ 776| 0| seek_cap = seek_cap * 2; 777| 0| uint32_t* nc = (uint32_t*)ZXC_REALLOC(seek_comp, seek_cap * sizeof(uint32_t)); ------------------ | | 78| 0|#define ZXC_REALLOC(ptr, size) realloc(ptr, size) ------------------ 778| 0| if (UNLIKELY(!nc)) { ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 779| 0| ZXC_FREE(seek_comp); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 780| 0| zxc_cctx_free(&ctx); 781| 0| return ZXC_ERROR_MEMORY; 782| 0| } 783| 0| seek_comp = nc; 784| 0| } 785| | // LCOV_EXCL_STOP 786| 10.6k| seek_comp[seek_count] = (uint32_t)res; 787| 10.6k| seek_count++; 788| 10.6k| } 789| | 790| 10.6k| op += res; 791| 10.6k| pos += chunk_len; 792| 10.6k| } 793| | 794| 9.45k| zxc_cctx_free(&ctx); 795| | 796| | // Write EOF Block 797| 9.45k| const size_t rem_cap = (size_t)(op_end - op); 798| 9.45k| const zxc_block_header_t eof_bh = { 799| 9.45k| .block_type = ZXC_BLOCK_EOF, .block_flags = 0, .reserved = 0, .comp_size = 0}; 800| 9.45k| const int eof_val = zxc_write_block_header(op, rem_cap, &eof_bh); 801| | // LCOV_EXCL_START 802| 9.45k| if (UNLIKELY(eof_val < 0)) { ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 803| 0| ZXC_FREE(seek_comp); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 804| 0| return eof_val; 805| 0| } 806| | // LCOV_EXCL_STOP 807| 9.45k| op += eof_val; 808| | 809| | /* Seekable: write seek table between EOF block and footer */ 810| 9.45k| if (seekable && seek_count > 0) { ------------------ | Branch (810:9): [True: 9.45k, False: 0] | Branch (810:21): [True: 9.45k, False: 0] ------------------ 811| 9.45k| const size_t st_cap = (size_t)(op_end - op); 812| 9.45k| const int64_t st_val = zxc_write_seek_table(op, st_cap, seek_comp, seek_count); 813| 9.45k| ZXC_FREE(seek_comp); ------------------ | | 84| 9.45k|#define ZXC_FREE(ptr) free(ptr) ------------------ 814| 9.45k| if (UNLIKELY(st_val < 0)) return st_val; // LCOV_EXCL_LINE ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 815| 9.45k| op += st_val; 816| 9.45k| } else { 817| 0| ZXC_FREE(seek_comp); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 818| 0| } 819| | 820| 9.45k| if (UNLIKELY((size_t)(op_end - op) < ZXC_FILE_FOOTER_SIZE)) ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 821| 0| return ZXC_ERROR_DST_TOO_SMALL; // LCOV_EXCL_LINE 822| | 823| | // Write 12-byte Footer: [Source Size (8)] + [Global Hash (4)] 824| 9.45k| const int footer_val = 825| 9.45k| zxc_write_file_footer(op, (size_t)(op_end - op), src_size, global_hash, checksum_enabled); 826| 9.45k| if (UNLIKELY(footer_val < 0)) return footer_val; // LCOV_EXCL_LINE ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 827| 9.45k| op += footer_val; 828| | 829| 9.45k| return (int64_t)(op - op_start); 830| 9.45k|} zxc_dispatch.c:zxc_decompress_dispatch_init: 319| 1| const size_t dst_cap) { 320| 1| const zxc_cpu_feature_t cpu = zxc_detect_cpu_features(); 321| 1| zxc_decompress_func_t zxc_decompress_ptr_local = NULL; 322| 1| zxc_decompress_func_t zxc_decompress_dict_ptr_local = NULL; 323| | 324| |#ifndef ZXC_ONLY_DEFAULT 325| |#if defined(__x86_64__) || defined(_M_X64) 326| | if (cpu == ZXC_CPU_AVX512) { 327| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_avx512; 328| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_avx512; 329| | } else if (cpu == ZXC_CPU_AVX2) { 330| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_avx2; 331| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_avx2; 332| | } else { 333| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; 334| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_default; 335| | } 336| |#elif defined(__arm__) || defined(_M_ARM) 337| | // 32-bit ARM: the only arch with a real runtime NEON probe (getauxval). 338| | // cppcheck-suppress knownConditionTrueFalse 339| | if (cpu == ZXC_CPU_NEON) { 340| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_neon32; 341| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_neon32; 342| | } else { 343| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; 344| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_default; 345| | } 346| |#else 347| | (void)cpu; 348| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; 349| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_default; 350| |#endif 351| |#else 352| 1| (void)cpu; 353| 1| zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; 354| 1| zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_default; 355| 1|#endif 356| | 357| 1|#if ZXC_USE_C11_ATOMICS 358| 1| atomic_store_explicit(&zxc_decompress_ptr, zxc_decompress_ptr_local, memory_order_release); 359| 1| atomic_store_explicit(&zxc_decompress_dict_ptr, zxc_decompress_dict_ptr_local, 360| 1| memory_order_release); 361| |#else 362| | zxc_decompress_ptr = zxc_decompress_ptr_local; 363| | zxc_decompress_dict_ptr = zxc_decompress_dict_ptr_local; 364| |#endif 365| 1| return (ctx->dict_size ? zxc_decompress_dict_ptr_local : zxc_decompress_ptr_local)( ------------------ | Branch (365:13): [True: 0, False: 1] ------------------ 366| 1| ctx, src, src_sz, dst, dst_cap); 367| 1|} zxc_dispatch.c:zxc_detect_cpu_features: 206| 2|static zxc_cpu_feature_t zxc_detect_cpu_features(void) { 207| 2|#ifdef ZXC_ONLY_DEFAULT 208| 2| return ZXC_CPU_GENERIC; 209| |#else 210| | zxc_cpu_feature_t features = ZXC_CPU_GENERIC; 211| | 212| |#if defined(__x86_64__) || defined(_M_X64) 213| | // AVX2/AVX512 need OS-enabled YMM/ZMM state: gate on OSXSAVE + XGETBV/XCR0, 214| | // not CPUID alone (else a VEX/EVEX op faults #UD when the OS hasn't enabled it). 215| | uint32_t regs[4]; 216| | int sse2 = 0; 217| | int avx2 = 0; 218| | int avx512 = 0; 219| | int bmi_lzcnt = 0; 220| | 221| | zxc_cpuid(0, 0, regs); 222| | const uint32_t max_leaf = regs[0]; 223| | 224| | zxc_cpuid(1, 0, regs); 225| | if (regs[3] & (1U << 26)) sse2 = 1; // CPUID.1:EDX[26] 226| | if ((regs[2] & (1U << 27)) && max_leaf >= 7) { // OSXSAVE, and leaf 7 is real 227| | const uint64_t xcr0 = zxc_xgetbv0(); 228| | if ((xcr0 & 0x6) == 0x6) { // SSE+YMM enabled 229| | zxc_cpuid(7, 0, regs); 230| | const int bmi1 = (regs[1] >> 3) & 1; // CPUID.7.0:EBX[3] 231| | const int bmi2 = (regs[1] >> 8) & 1; // CPUID.7.0:EBX[8] 232| | if (regs[1] & (1U << 5)) avx2 = 1; 233| | // AVX512 also needs XCR0[5..7] (opmask/ZMM) 234| | if ((regs[1] & (1U << 16)) && (regs[1] & (1U << 30)) && (regs[2] & (1U << 6)) && 235| | (xcr0 & 0xE0) == 0xE0) 236| | avx512 = 1; /* AVX512 tier = F+BW+VBMI2 (variant built with -mavx512vbmi2) */ 237| | // The AVX2/AVX512 variants are compiled with BMI1/BMI2/LZCNT enabled, 238| | // so both gates must prove those bits too. LZCNT (ABM) lives in 239| | // CPUID.80000001H:ECX[5]; that leaf is architectural on x86-64. 240| | zxc_cpuid(0x80000001U, 0, regs); 241| | bmi_lzcnt = bmi1 && bmi2 && ((regs[2] >> 5) & 1); 242| | } 243| | } 244| | 245| | if (avx512 && bmi_lzcnt) { 246| | features = ZXC_CPU_AVX512; 247| | } else if (avx2 && bmi_lzcnt) { 248| | features = ZXC_CPU_AVX2; 249| | } else if (sse2) { 250| | features = ZXC_CPU_SSE2; 251| | } 252| | 253| |#elif defined(__aarch64__) || defined(_M_ARM64) 254| | // ARM64 usually guarantees NEON 255| | features = ZXC_CPU_NEON; 256| | 257| |#elif defined(__arm__) || defined(_M_ARM) 258| | // ARM32 Runtime detection for Linux 259| |#if defined(__linux__) 260| | const unsigned long hwcaps = getauxval(AT_HWCAP); 261| | if (hwcaps & HWCAP_NEON) { 262| | features = ZXC_CPU_NEON; 263| | } 264| |#else 265| |// Fallback for non-Linux: rely on compiler flags. 266| |// If compiled with -mfpu=neon, we assume target supports it. 267| |// Otherwise, safe default is GENERIC. 268| |#if defined(__ARM_NEON) 269| | features = ZXC_CPU_NEON; 270| |#endif 271| |#endif 272| |#endif 273| | 274| | return features; 275| |#endif 276| 2|} zxc_dispatch.c:zxc_compress_dispatch_init: 441| 1| const size_t dst_cap) { 442| 1| const zxc_cpu_feature_t cpu = zxc_detect_cpu_features(); 443| 1| zxc_compress_func_t zxc_compress_ptr_local = NULL; 444| | 445| |#ifndef ZXC_ONLY_DEFAULT 446| |#if defined(__x86_64__) || defined(_M_X64) 447| | if (cpu == ZXC_CPU_AVX512) 448| | zxc_compress_ptr_local = zxc_compress_chunk_wrapper_avx512; 449| | else if (cpu == ZXC_CPU_AVX2) 450| | zxc_compress_ptr_local = zxc_compress_chunk_wrapper_avx2; 451| | else 452| | zxc_compress_ptr_local = zxc_compress_chunk_wrapper_default; 453| |#elif defined(__arm__) || defined(_M_ARM) 454| | // cppcheck-suppress knownConditionTrueFalse 455| | if (cpu == ZXC_CPU_NEON) 456| | zxc_compress_ptr_local = zxc_compress_chunk_wrapper_neon32; 457| | else 458| | zxc_compress_ptr_local = zxc_compress_chunk_wrapper_default; 459| |#else 460| | (void)cpu; 461| | zxc_compress_ptr_local = zxc_compress_chunk_wrapper_default; 462| |#endif 463| |#else 464| 1| (void)cpu; 465| 1| zxc_compress_ptr_local = zxc_compress_chunk_wrapper_default; 466| 1|#endif 467| | 468| 1|#if ZXC_USE_C11_ATOMICS 469| 1| atomic_store_explicit(&zxc_compress_ptr, zxc_compress_ptr_local, memory_order_release); 470| |#else 471| | zxc_compress_ptr = zxc_compress_ptr_local; 472| |#endif 473| 1| return zxc_compress_ptr_local(ctx, src, src_sz, dst, dst_cap); 474| 1|} zxc_common.c:zxc_log2_u32: 856| 48.4k|static ZXC_ALWAYS_INLINE uint32_t zxc_log2_u32(const uint32_t v) { 857| |#ifdef _MSC_VER 858| | unsigned long index; 859| | return (v == 0) ? 0 : (_BitScanReverse(&index, v) ? index : 0); 860| |#else 861| 48.4k| return (v == 0) ? 0 : (uint32_t)(31 - __builtin_clz(v)); ------------------ | Branch (861:12): [True: 0, False: 48.4k] ------------------ 862| 48.4k|#endif 863| 48.4k|} zxc_common.c:zxc_store_le32: 1160| 77.3k|static ZXC_ALWAYS_INLINE void zxc_store_le32(void* p, const uint32_t v) { 1161| |#ifdef ZXC_BIG_ENDIAN 1162| | const uint32_t s = ZXC_BSWAP32(v); 1163| | ZXC_MEMCPY(p, &s, sizeof(s)); 1164| |#else 1165| 77.3k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 173| 77.3k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1166| 77.3k|#endif 1167| 77.3k|} zxc_common.c:zxc_store_le16: 1139| 18.9k|static ZXC_ALWAYS_INLINE void zxc_store_le16(void* p, const uint16_t v) { 1140| |#ifdef ZXC_BIG_ENDIAN 1141| | const uint16_t s = ZXC_BSWAP16(v); 1142| | ZXC_MEMCPY(p, &s, sizeof(s)); 1143| |#else 1144| 18.9k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 173| 18.9k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1145| 18.9k|#endif 1146| 18.9k|} zxc_common.c:zxc_hash16: 1219| 18.9k|static ZXC_ALWAYS_INLINE uint16_t zxc_hash16(const uint8_t* p) { 1220| 18.9k| const uint64_t v1 = zxc_le64(p); 1221| 18.9k| const uint64_t v2 = zxc_le64(p + 8); 1222| 18.9k| uint64_t h = v1 ^ v2 ^ ZXC_HASH_PRIME2; ------------------ | | 559| 18.9k|#define ZXC_HASH_PRIME2 0xD2D84A61D2D84A61ULL ------------------ 1223| 18.9k| h ^= h << 13; 1224| 18.9k| h ^= h >> 7; 1225| 18.9k| h ^= h << 17; 1226| 18.9k| const uint32_t res = (uint32_t)((h >> 32) ^ h); 1227| 18.9k| return (uint16_t)((res >> 16) ^ res); 1228| 18.9k|} zxc_common.c:zxc_le32: 1094| 71.2k|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1095| 71.2k| uint32_t v; 1096| 71.2k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 71.2k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1097| |#ifdef ZXC_BIG_ENDIAN 1098| | return ZXC_BSWAP32(v); 1099| |#else 1100| 71.2k| return v; 1101| 71.2k|#endif 1102| 71.2k|} zxc_common.c:zxc_le16: 1074| 9.48k|static ZXC_ALWAYS_INLINE uint16_t zxc_le16(const void* p) { 1075| 9.48k| uint16_t v; 1076| 9.48k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 9.48k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1077| |#ifdef ZXC_BIG_ENDIAN 1078| | return ZXC_BSWAP16(v); 1079| |#else 1080| 9.48k| return v; 1081| 9.48k|#endif 1082| 9.48k|} zxc_common.c:zxc_hash8: 1200| 50.0k|static ZXC_ALWAYS_INLINE uint8_t zxc_hash8(const uint8_t* p) { 1201| 50.0k| const uint64_t v = zxc_le64(p); 1202| 50.0k| uint64_t h = v ^ ZXC_HASH_PRIME1; ------------------ | | 557| 50.0k|#define ZXC_HASH_PRIME1 0x9E3779B97F4A7C15ULL ------------------ 1203| 50.0k| h ^= h << 13; 1204| 50.0k| h ^= h >> 7; 1205| 50.0k| h ^= h << 17; 1206| 50.0k| return (uint8_t)((h >> 32) ^ h); 1207| 50.0k|} zxc_common.c:zxc_store_le64: 1183| 47.2k|static ZXC_ALWAYS_INLINE void zxc_store_le64(void* p, const uint64_t v) { 1184| |#ifdef ZXC_BIG_ENDIAN 1185| | const uint64_t s = ZXC_BSWAP64(v); 1186| | ZXC_MEMCPY(p, &s, sizeof(s)); 1187| |#else 1188| 47.2k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 173| 47.2k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1189| 47.2k|#endif 1190| 47.2k|} zxc_common.c:zxc_le64: 1114| 148k|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1115| 148k| uint64_t v; 1116| 148k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 148k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1117| |#ifdef ZXC_BIG_ENDIAN 1118| | return ZXC_BSWAP64(v); 1119| |#else 1120| 148k| return v; 1121| 148k|#endif 1122| 148k|} zxc_compress.c:zxc_get_lz77_params: 950| 10.6k|static ZXC_ALWAYS_INLINE zxc_lz77_params_t zxc_get_lz77_params(const int level) { 951| 10.6k| if (level >= ZXC_LEVEL_ULTRA) return (zxc_lz77_params_t){128, 256, 0, 0, 0, 1, 8}; ------------------ | Branch (951:9): [True: 0, False: 10.6k] ------------------ 952| | // search_depth, sufficient_len, use_lazy, lazy_attempts, lazy_len_threshold, step_base, 953| | // step_shift 954| 10.6k| static const zxc_lz77_params_t table[7] = { 955| 10.6k| {3, 16, 0, 0, 0, 4, 4}, // fallback 956| 10.6k| {3, 16, 0, 0, 0, 4, 4}, // level 1 957| 10.6k| {3, 18, 0, 0, 0, 3, 6}, // level 2 958| 10.6k| {3, 16, 1, 4, 128, 1, 4}, // level 3 959| 10.6k| {3, 18, 1, 4, 128, 1, 5}, // level 4 960| 10.6k| {64, 256, 1, 16, 128, 1, 8}, // level 5 961| 10.6k| {64, 256, 0, 0, 0, 1, 8} // level 6 962| 10.6k| }; 963| 10.6k| return table[level < ZXC_LEVEL_FASTEST ? ZXC_LEVEL_FASTEST : level]; ------------------ | Branch (963:18): [True: 0, False: 10.6k] ------------------ 964| 10.6k|} zxc_compress.c:zxc_le64: 1114| 218M|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1115| 218M| uint64_t v; 1116| 218M| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 218M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1117| |#ifdef ZXC_BIG_ENDIAN 1118| | return ZXC_BSWAP64(v); 1119| |#else 1120| 218M| return v; 1121| 218M|#endif 1122| 218M|} zxc_compress.c:zxc_le32: 1094| 57.7M|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1095| 57.7M| uint32_t v; 1096| 57.7M| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 57.7M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1097| |#ifdef ZXC_BIG_ENDIAN 1098| | return ZXC_BSWAP32(v); 1099| |#else 1100| 57.7M| return v; 1101| 57.7M|#endif 1102| 57.7M|} zxc_compress.c:zxc_ctz64: 1319| 33.8M|static ZXC_ALWAYS_INLINE int zxc_ctz64(const uint64_t x) { 1320| 33.8M| if (x == 0) return 64; ------------------ | Branch (1320:9): [True: 0, False: 33.8M] ------------------ 1321| 33.8M|#if defined(__GNUC__) || defined(__clang__) 1322| 33.8M| return __builtin_ctzll(x); 1323| |#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) 1324| | unsigned long r; 1325| | _BitScanForward64(&r, x); 1326| | return (int)r; 1327| |#elif defined(_MSC_VER) 1328| | // Use two 32-bit scans to avoid fragile 64-bit De Bruijn multiplication. 1329| | unsigned long r; 1330| | const uint32_t lo = (uint32_t)x; 1331| | if (_BitScanForward(&r, lo)) return (int)r; 1332| | _BitScanForward(&r, (uint32_t)(x >> 32)); 1333| | return 32 + (int)r; 1334| |#else 1335| | // Fallback De Bruijn for non-GCC/non-MSVC compilers 1336| | static const int Debruijn64[64] = { 1337| | 0, 1, 48, 2, 57, 49, 28, 3, 61, 58, 50, 42, 38, 29, 17, 4, 62, 55, 59, 36, 53, 51, 1338| | 43, 22, 45, 39, 33, 30, 24, 18, 12, 5, 63, 47, 56, 27, 60, 41, 37, 16, 54, 35, 52, 21, 1339| | 44, 32, 23, 11, 46, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6}; 1340| | return Debruijn64[((x & (0ULL - x)) * 0x03F79D71B4CA8B09ULL) >> 58]; 1341| |#endif 1342| 33.8M|} zxc_compress.c:zxc_ctz32: 1289| 2.15M|static ZXC_ALWAYS_INLINE int zxc_ctz32(const uint32_t x) { 1290| 2.15M| if (x == 0) return 32; ------------------ | Branch (1290:9): [True: 0, False: 2.15M] ------------------ 1291| 2.15M|#if defined(__GNUC__) || defined(__clang__) 1292| 2.15M| return __builtin_ctz(x); 1293| |#elif defined(_MSC_VER) 1294| | unsigned long r; 1295| | _BitScanForward(&r, x); 1296| | return (int)r; 1297| |#else 1298| | // Fallback De Bruijn (32 bits) 1299| | static const int DeBruijn32[32] = {0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 1300| | 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 1301| | 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; 1302| | return DeBruijn32[((uint32_t)((x & (0U - x)) * 0x077CB531U)) >> 27]; 1303| |#endif 1304| 2.15M|} zxc_compress.c:zxc_copy32: 1258| 2.22M|static ZXC_ALWAYS_INLINE void zxc_copy32(void* dst, const void* src) { 1259| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 1260| | // AVX2/AVX512: Single 256-bit (32 byte) unaligned load/store 1261| | _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src)); 1262| |#elif defined(ZXC_USE_SSE2) 1263| | // SSE2: Two 128-bit (16 byte) unaligned load/stores (no 256-bit regs) 1264| 2.22M| _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); 1265| 2.22M| _mm_storeu_si128((__m128i*)((uint8_t*)dst + 16), 1266| 2.22M| _mm_loadu_si128((const __m128i*)((const uint8_t*)src + 16))); 1267| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1268| | // NEON: Two 128-bit (16 byte) unaligned load/stores 1269| | vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); 1270| | vst1q_u8((uint8_t*)dst + 16, vld1q_u8((const uint8_t*)src + 16)); 1271| |#else 1272| | ZXC_MEMCPY(dst, src, 32); 1273| |#endif 1274| 2.22M|} zxc_compress.c:zxc_ss_prem_rle_q8: 756| 5.80k|static inline int zxc_ss_prem_rle_q8(const int level) { 757| 5.80k| return (level >= ZXC_LEVEL_DENSITY) ? 1 : (256 >> ZXC_RLE_MARGIN_SHIFT); ------------------ | | 661| 5.80k|#define ZXC_RLE_MARGIN_SHIFT 5 ------------------ | Branch (757:12): [True: 0, False: 5.80k] ------------------ 758| 5.80k|} zxc_compress.c:zxc_checksum: 1390| 5.38k| const uint8_t hash_method) { 1391| 5.38k| (void)hash_method; /* single algorithm for now; extend when adding more */ 1392| 5.38k| const uint64_t hash = rapidhash(input, len); 1393| | 1394| | return (uint32_t)(hash ^ (hash >> (sizeof(uint32_t) * CHAR_BIT))); 1395| 5.38k|} zxc_compress.c:zxc_store_le32: 1160| 5.38k|static ZXC_ALWAYS_INLINE void zxc_store_le32(void* p, const uint32_t v) { 1161| |#ifdef ZXC_BIG_ENDIAN 1162| | const uint32_t s = ZXC_BSWAP32(v); 1163| | ZXC_MEMCPY(p, &s, sizeof(s)); 1164| |#else 1165| 5.38k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 173| 5.38k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1166| 5.38k|#endif 1167| 5.38k|} zxc_decompress.c:zxc_le32: 1094| 5.29M|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1095| 5.29M| uint32_t v; 1096| 5.29M| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 5.29M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1097| |#ifdef ZXC_BIG_ENDIAN 1098| | return ZXC_BSWAP32(v); 1099| |#else 1100| 5.29M| return v; 1101| 5.29M|#endif 1102| 5.29M|} zxc_decompress.c:zxc_copy32: 1258| 66.7M|static ZXC_ALWAYS_INLINE void zxc_copy32(void* dst, const void* src) { 1259| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 1260| | // AVX2/AVX512: Single 256-bit (32 byte) unaligned load/store 1261| | _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src)); 1262| |#elif defined(ZXC_USE_SSE2) 1263| | // SSE2: Two 128-bit (16 byte) unaligned load/stores (no 256-bit regs) 1264| 66.7M| _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); 1265| 66.7M| _mm_storeu_si128((__m128i*)((uint8_t*)dst + 16), 1266| 66.7M| _mm_loadu_si128((const __m128i*)((const uint8_t*)src + 16))); 1267| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1268| | // NEON: Two 128-bit (16 byte) unaligned load/stores 1269| | vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); 1270| | vst1q_u8((uint8_t*)dst + 16, vld1q_u8((const uint8_t*)src + 16)); 1271| |#else 1272| | ZXC_MEMCPY(dst, src, 32); 1273| |#endif 1274| 66.7M|} zxc_decompress.c:zxc_le64: 1114| 1.44M|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1115| 1.44M| uint64_t v; 1116| 1.44M| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 1.44M|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1117| |#ifdef ZXC_BIG_ENDIAN 1118| | return ZXC_BSWAP64(v); 1119| |#else 1120| 1.44M| return v; 1121| 1.44M|#endif 1122| 1.44M|} zxc_decompress.c:zxc_copy16: 1239| 6.08M|static ZXC_ALWAYS_INLINE void zxc_copy16(void* dst, const void* src) { 1240| 6.08M|#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) || defined(ZXC_USE_SSE2) 1241| | // x86 SSE2/AVX2/AVX512: Single 128-bit unaligned load/store 1242| 6.08M| _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); 1243| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1244| | vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); 1245| |#else 1246| | ZXC_MEMCPY(dst, src, 16); 1247| |#endif 1248| 6.08M|} zxc_decompress.c:zxc_le16: 1074| 112k|static ZXC_ALWAYS_INLINE uint16_t zxc_le16(const void* p) { 1075| 112k| uint16_t v; 1076| 112k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 112k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1077| |#ifdef ZXC_BIG_ENDIAN 1078| | return ZXC_BSWAP16(v); 1079| |#else 1080| 112k| return v; 1081| 112k|#endif 1082| 112k|} zxc_dispatch.c:zxc_level_clamp: 800| 9.45k|static inline int zxc_level_clamp(const int level) { 801| 9.45k| return (level > ZXC_LEVEL_ULTRA) ? ZXC_LEVEL_ULTRA : level; ------------------ | Branch (801:12): [True: 0, False: 9.45k] ------------------ 802| 9.45k|} zxc_dispatch.c:zxc_validate_block_size: 889| 9.45k|static ZXC_ALWAYS_INLINE int zxc_validate_block_size(const size_t bs) { 890| 9.45k| return bs >= ZXC_BLOCK_SIZE_MIN && bs <= ZXC_BLOCK_SIZE_MAX && (bs & (bs - 1)) == 0; ------------------ | | 62| 18.9k|#define ZXC_BLOCK_SIZE_MIN (1U << ZXC_BLOCK_SIZE_MIN_LOG2) | | ------------------ | | | | 56| 9.45k|#define ZXC_BLOCK_SIZE_MIN_LOG2 12 | | ------------------ ------------------ return bs >= ZXC_BLOCK_SIZE_MIN && bs <= ZXC_BLOCK_SIZE_MAX && (bs & (bs - 1)) == 0; ------------------ | | 64| 18.9k|#define ZXC_BLOCK_SIZE_MAX (1U << ZXC_BLOCK_SIZE_MAX_LOG2) | | ------------------ | | | | 58| 9.45k|#define ZXC_BLOCK_SIZE_MAX_LOG2 21 | | ------------------ ------------------ | Branch (890:12): [True: 9.45k, False: 0] | Branch (890:40): [True: 9.45k, False: 0] | Branch (890:68): [True: 9.45k, False: 0] ------------------ 891| 9.45k|} zxc_dispatch.c:zxc_le32: 1094| 5.38k|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1095| 5.38k| uint32_t v; 1096| 5.38k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 5.38k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1097| |#ifdef ZXC_BIG_ENDIAN 1098| | return ZXC_BSWAP32(v); 1099| |#else 1100| 5.38k| return v; 1101| 5.38k|#endif 1102| 5.38k|} zxc_dispatch.c:zxc_hash_combine_rotate: 1430| 5.38k| const uint32_t block_hash) { 1431| 5.38k| return ((hash << 1) | (hash >> 31)) ^ block_hash; 1432| 5.38k|} zxc_seekable.c:zxc_store_le32: 1160| 10.6k|static ZXC_ALWAYS_INLINE void zxc_store_le32(void* p, const uint32_t v) { 1161| |#ifdef ZXC_BIG_ENDIAN 1162| | const uint32_t s = ZXC_BSWAP32(v); 1163| | ZXC_MEMCPY(p, &s, sizeof(s)); 1164| |#else 1165| 10.6k| ZXC_MEMCPY(p, &v, sizeof(v)); ------------------ | | 173| 10.6k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1166| 10.6k|#endif 1167| 10.6k|} zxc_seekable.c:zxc_le64: 1114| 9.45k|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1115| 9.45k| uint64_t v; 1116| 9.45k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 9.45k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1117| |#ifdef ZXC_BIG_ENDIAN 1118| | return ZXC_BSWAP64(v); 1119| |#else 1120| 9.45k| return v; 1121| 9.45k|#endif 1122| 9.45k|} zxc_seekable.c:zxc_le32: 1094| 10.6k|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1095| 10.6k| uint32_t v; 1096| 10.6k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 10.6k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1097| |#ifdef ZXC_BIG_ENDIAN 1098| | return ZXC_BSWAP32(v); 1099| |#else 1100| 10.6k| return v; 1101| 10.6k|#endif 1102| 10.6k|} zxc_seek_table_size: 172| 9.45k|size_t zxc_seek_table_size(const uint32_t num_blocks) { 173| 9.45k| return ZXC_BLOCK_HEADER_SIZE + (size_t)num_blocks * ZXC_SEEK_ENTRY_SIZE; ------------------ | | 397| 9.45k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ return ZXC_BLOCK_HEADER_SIZE + (size_t)num_blocks * ZXC_SEEK_ENTRY_SIZE; ------------------ | | 443| 9.45k|#define ZXC_SEEK_ENTRY_SIZE 4 ------------------ 174| 9.45k|} zxc_write_seek_table: 191| 9.45k| const uint32_t num_blocks) { 192| 9.45k| if (UNLIKELY(num_blocks > UINT32_MAX / ZXC_SEEK_ENTRY_SIZE)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 193| | 194| 9.45k| const size_t total = zxc_seek_table_size(num_blocks); 195| 9.45k| if (UNLIKELY(dst_capacity < total)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 196| 9.45k| if (UNLIKELY(!dst || !comp_sizes)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 18.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | | Branch (151:42): [True: 0, False: 9.45k] | | | Branch (151:42): [True: 0, False: 9.45k] | | ------------------ ------------------ 197| | 198| 9.45k| const uint32_t payload_size = num_blocks * ZXC_SEEK_ENTRY_SIZE; ------------------ | | 443| 9.45k|#define ZXC_SEEK_ENTRY_SIZE 4 ------------------ 199| | 200| | /* Write standard ZXC block header */ 201| 9.45k| const zxc_block_header_t bh = { 202| 9.45k| .block_type = ZXC_BLOCK_SEK, .block_flags = 0, .reserved = 0, .comp_size = payload_size}; 203| 9.45k| const int hdr_res = zxc_write_block_header(dst, dst_capacity, &bh); 204| 9.45k| if (UNLIKELY(hdr_res < 0)) return hdr_res; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 205| 9.45k| uint8_t* p = dst + hdr_res; 206| | 207| | /* Write entries: comp_size(4) only */ 208| 20.1k| for (uint32_t i = 0; i < num_blocks; i++) { ------------------ | Branch (208:26): [True: 10.6k, False: 9.45k] ------------------ 209| 10.6k| zxc_store_le32(p, comp_sizes[i]); 210| 10.6k| p += sizeof(uint32_t); 211| 10.6k| } 212| | 213| 9.45k| return (int64_t)(p - dst); 214| 9.45k|} zxc_seekable_open: 409| 18.9k|zxc_seekable* zxc_seekable_open(const void* src, const size_t src_size) { 410| 18.9k| if (UNLIKELY(!src || src_size == 0)) return NULL; ------------------ | | 151| 37.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 18.9k] | | | Branch (151:42): [True: 0, False: 18.9k] | | | Branch (151:42): [True: 0, False: 18.9k] | | ------------------ ------------------ 411| 18.9k| return zxc_seekable_parse((const uint8_t*)src, src_size); 412| 18.9k|} zxc_seekable_get_num_blocks: 561| 18.9k|uint32_t zxc_seekable_get_num_blocks(const zxc_seekable* s) { return s ? s->num_blocks : 0; } ------------------ | Branch (561:70): [True: 9.45k, False: 9.45k] ------------------ zxc_seekable_get_decompressed_size: 568| 18.9k|uint64_t zxc_seekable_get_decompressed_size(const zxc_seekable* s) { 569| 18.9k| return s ? s->total_decomp : 0; ------------------ | Branch (569:12): [True: 9.45k, False: 9.45k] ------------------ 570| 18.9k|} zxc_seekable_get_block_comp_size: 579| 20.1k|uint32_t zxc_seekable_get_block_comp_size(const zxc_seekable* s, const uint32_t block_idx) { 580| 20.1k| if (UNLIKELY(!s || block_idx >= s->num_blocks)) return 0; ------------------ | | 151| 40.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 9.45k, False: 10.6k] | | | Branch (151:42): [True: 0, False: 20.1k] | | | Branch (151:42): [True: 9.45k, False: 10.6k] | | ------------------ ------------------ 581| 10.6k| return s->comp_sizes[block_idx]; 582| 20.1k|} zxc_seekable_get_block_decomp_size: 595| 20.1k|uint32_t zxc_seekable_get_block_decomp_size(const zxc_seekable* s, const uint32_t block_idx) { 596| 20.1k| if (UNLIKELY(!s || block_idx >= s->num_blocks)) return 0; ------------------ | | 151| 40.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 9.45k, False: 10.6k] | | | Branch (151:42): [True: 0, False: 20.1k] | | | Branch (151:42): [True: 9.45k, False: 10.6k] | | ------------------ ------------------ 597| 10.6k| const uint64_t start = (uint64_t)block_idx * (uint64_t)s->block_size; 598| 10.6k| const uint64_t remaining = s->total_decomp - start; 599| 10.6k| return (remaining >= (uint64_t)s->block_size) ? s->block_size : (uint32_t)remaining; ------------------ | Branch (599:12): [True: 1.21k, False: 9.45k] ------------------ 600| 20.1k|} zxc_seekable_decompress_range: 696| 37.1k| const uint64_t offset, const size_t len) { 697| 37.1k| if (UNLIKELY(len == 0)) return 0; ------------------ | | 151| 37.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 9.45k, False: 27.7k] | | ------------------ ------------------ 698| 27.7k| if (UNLIKELY(!s || !dst)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 55.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 27.7k] | | | Branch (151:42): [True: 0, False: 27.7k] | | | Branch (151:42): [True: 0, False: 27.7k] | | ------------------ ------------------ 699| 27.7k| if (UNLIKELY(dst_capacity < len)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 27.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 27.7k] | | ------------------ ------------------ 700| 27.7k| if (UNLIKELY(offset + len > s->total_decomp)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 27.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 9.45k, False: 18.2k] | | ------------------ ------------------ 701| 18.2k| if (UNLIKELY(s->expected_dict_id != 0 && (!s->dict || s->dict_size == 0))) ------------------ | | 151| 18.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 18.2k] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 18.2k] | | ------------------ ------------------ 702| 0| return ZXC_ERROR_DICT_REQUIRED; 703| | 704| | /* Initialize decompression context on first use */ 705| 18.2k| if (!s->dctx_initialized) { ------------------ | Branch (705:9): [True: 9.45k, False: 8.82k] ------------------ 706| | // LCOV_EXCL_START 707| 9.45k| if (UNLIKELY(zxc_cctx_init(&s->dctx, (size_t)s->block_size, 0, 0, 0, s->dict_size) != ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 708| 9.45k| ZXC_OK)) 709| 0| return ZXC_ERROR_MEMORY; 710| | // LCOV_EXCL_STOP 711| 9.45k| if (UNLIKELY(zxc_cctx_attach_dict_huf(&s->dctx, s->has_dict_huf ? s->dict_huf : NULL) != ------------------ | | 151| 18.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | | Branch (151:42): [True: 0, False: 9.45k] | | ------------------ ------------------ 712| 9.45k| ZXC_OK)) { 713| | // LCOV_EXCL_START 714| 0| zxc_cctx_free(&s->dctx); 715| 0| return ZXC_ERROR_CORRUPT_DATA; 716| | // LCOV_EXCL_STOP 717| 0| } 718| 9.45k| s->dctx_initialized = 1; 719| 9.45k| if (s->dict_size > 0) ZXC_MEMCPY(s->dctx.dict_buffer, s->dict, s->dict_size); ------------------ | | 173| 0|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ | Branch (719:13): [True: 0, False: 9.45k] ------------------ 720| 9.45k| } 721| 18.2k| s->dctx.dict_size = s->dict_size; 722| | 723| | /* work_buf is pre-sized to block_size + ZXC_DECOMPRESS_TAIL_PAD by the 724| | * matching zxc_cctx_init above. */ 725| 18.2k| const size_t work_sz = (size_t)s->block_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 349| 18.2k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 18.2k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 726| | 727| | /* Find block range - O(1) division */ 728| 18.2k| const uint32_t blk_start = zxc_seek_find_block(s->block_size, offset); 729| 18.2k| const uint32_t blk_end = zxc_seek_find_block(s->block_size, offset + len - 1); 730| | 731| 18.2k| uint8_t* out = (uint8_t*)dst; 732| 18.2k| size_t remaining = len; 733| | 734| | /* Allocate read buffer for compressed blocks */ 735| 18.2k| size_t max_comp = 0; 736| 37.3k| for (uint32_t bi = blk_start; bi <= blk_end; bi++) { ------------------ | Branch (736:35): [True: 19.0k, False: 18.2k] ------------------ 737| 19.0k| if (s->comp_sizes[bi] > max_comp) max_comp = s->comp_sizes[bi]; ------------------ | Branch (737:13): [True: 18.3k, False: 687] ------------------ 738| 19.0k| } 739| 18.2k| uint8_t* const read_buf = (uint8_t*)ZXC_MALLOC(max_comp + ZXC_PAD_SIZE); ------------------ | | 66| 18.2k|#define ZXC_MALLOC(size) malloc(size) ------------------ 740| 18.2k| if (UNLIKELY(!read_buf)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_LINE ------------------ | | 151| 18.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 18.2k] | | ------------------ ------------------ 741| | 742| 37.3k| for (uint32_t bi = blk_start; bi <= blk_end; bi++) { ------------------ | Branch (742:35): [True: 19.0k, False: 18.2k] ------------------ 743| | /* Read compressed block data */ 744| 19.0k| const int read_res = zxc_seek_read_block(s, bi, read_buf, max_comp + ZXC_PAD_SIZE); ------------------ | | 337| 19.0k|#define ZXC_PAD_SIZE 32 ------------------ 745| 19.0k| if (UNLIKELY(read_res < 0)) { ------------------ | | 151| 19.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 19.0k] | | ------------------ ------------------ 746| | // LCOV_EXCL_START 747| 0| ZXC_FREE(read_buf); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 748| 0| return read_res; 749| | // LCOV_EXCL_STOP 750| 0| } 751| | 752| | /* Decompress the block: when a dictionary is active, decode into the 753| | * cctx-owned dict_buffer (which has dict content prepended) so that 754| | * match copies referencing dictionary bytes resolve naturally. */ 755| 19.0k| uint8_t* dec_dst = 756| 19.0k| s->dctx.dict_buffer ? s->dctx.dict_buffer + s->dict_size : s->dctx.work_buf; ------------------ | Branch (756:13): [True: 0, False: 19.0k] ------------------ 757| 19.0k| const int dec_res = 758| 19.0k| zxc_decompress_chunk_wrapper(&s->dctx, read_buf, (size_t)read_res, dec_dst, work_sz); 759| 19.0k| if (UNLIKELY(dec_res < 0)) { ------------------ | | 151| 19.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 19.0k] | | ------------------ ------------------ 760| | // LCOV_EXCL_START 761| 0| ZXC_FREE(read_buf); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 762| 0| return dec_res; 763| | // LCOV_EXCL_STOP 764| 0| } 765| | 766| | /* Calculate which portion of this block's decompressed data we need */ 767| 19.0k| const uint64_t blk_decomp_start = zxc_seek_decomp_offset(s->block_size, bi); 768| 19.0k| const size_t skip = (offset > blk_decomp_start) ? (size_t)(offset - blk_decomp_start) : 0; ------------------ | Branch (768:29): [True: 9.43k, False: 9.59k] ------------------ 769| 19.0k| if (UNLIKELY((size_t)dec_res < skip)) { ------------------ | | 151| 19.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 19.0k] | | ------------------ ------------------ 770| | // LCOV_EXCL_START 771| 0| ZXC_FREE(read_buf); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 772| 0| return ZXC_ERROR_CORRUPT_DATA; 773| | // LCOV_EXCL_STOP 774| 0| } 775| 19.0k| const size_t avail = (size_t)dec_res - skip; 776| 19.0k| const size_t copy = (avail < remaining) ? avail : remaining; ------------------ | Branch (776:29): [True: 745, False: 18.2k] ------------------ 777| | 778| 19.0k| ZXC_MEMCPY(out, dec_dst + skip, copy); ------------------ | | 173| 19.0k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 779| 19.0k| out += copy; 780| 19.0k| remaining -= copy; 781| 19.0k| } 782| | 783| 18.2k| ZXC_FREE(read_buf); ------------------ | | 84| 18.2k|#define ZXC_FREE(ptr) free(ptr) ------------------ 784| 18.2k| return (int64_t)len; 785| 18.2k|} zxc_seekable_decompress_range_mt: 1000| 1.72k| const uint64_t offset, const size_t len, int n_threads) { 1001| 1.72k| if (UNLIKELY(len == 0)) return 0; ------------------ | | 151| 1.72k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.72k] | | ------------------ ------------------ 1002| 1.72k| if (UNLIKELY(!s || !dst)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 3.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.72k] | | | Branch (151:42): [True: 0, False: 1.72k] | | | Branch (151:42): [True: 0, False: 1.72k] | | ------------------ ------------------ 1003| 1.72k| if (UNLIKELY(dst_capacity < len)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 1.72k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.72k] | | ------------------ ------------------ 1004| 1.72k| if (UNLIKELY(offset + len > s->total_decomp)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 1.72k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.72k] | | ------------------ ------------------ 1005| 1.72k| if (UNLIKELY(s->expected_dict_id != 0 && (!s->dict || s->dict_size == 0))) ------------------ | | 151| 1.72k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.72k] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 1.72k] | | ------------------ ------------------ 1006| 0| return ZXC_ERROR_DICT_REQUIRED; 1007| | 1008| | /* Find block range - O(1) division */ 1009| 1.72k| const uint32_t blk_start = zxc_seek_find_block(s->block_size, offset); 1010| 1.72k| const uint32_t blk_end = zxc_seek_find_block(s->block_size, offset + len - 1); 1011| 1.72k| const uint32_t num_jobs = blk_end - blk_start + 1; 1012| | 1013| | /* Auto-detect thread count (0 = use all available cores) */ 1014| 1.72k| if (n_threads == 0) n_threads = zxc_seek_get_num_procs(); ------------------ | Branch (1014:9): [True: 0, False: 1.72k] ------------------ 1015| | 1016| | /* Fallback to single-threaded path for trivial cases */ 1017| 1.72k| if (n_threads <= 1 || num_jobs <= 1) { ------------------ | Branch (1017:9): [True: 0, False: 1.72k] | Branch (1017:27): [True: 1.12k, False: 601] ------------------ 1018| 1.12k| return zxc_seekable_decompress_range(s, dst, dst_capacity, offset, len); 1019| 1.12k| } 1020| | 1021| | /* Cap threads to number of blocks and max limit */ 1022| 601| if ((uint32_t)n_threads > num_jobs) n_threads = (int)num_jobs; ------------------ | Branch (1022:9): [True: 0, False: 601] ------------------ 1023| 601| if (n_threads > ZXC_MAX_THREADS) n_threads = ZXC_MAX_THREADS; ------------------ | | 93| 601|#define ZXC_MAX_THREADS 512 ------------------ if (n_threads > ZXC_MAX_THREADS) n_threads = ZXC_MAX_THREADS; ------------------ | | 93| 0|#define ZXC_MAX_THREADS 512 ------------------ | Branch (1023:9): [True: 0, False: 601] ------------------ 1024| | 1025| | /* Allocate job descriptors */ 1026| 601| zxc_seek_mt_job_t* const jobs = 1027| 601| (zxc_seek_mt_job_t*)ZXC_CALLOC(num_jobs, sizeof(zxc_seek_mt_job_t)); ------------------ | | 72| 601|#define ZXC_CALLOC(nmemb, size) calloc(nmemb, size) ------------------ 1028| 601| if (UNLIKELY(!jobs)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_LINE ------------------ | | 151| 601|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 601] | | ------------------ ------------------ 1029| | 1030| | /* Plan jobs: compute skip, copy_len, and dst pointer for each block */ 1031| 601| uint8_t* out = (uint8_t*)dst; 1032| 601| size_t remaining = len; 1033| 1.80k| for (uint32_t i = 0; i < num_jobs; i++) { ------------------ | Branch (1033:26): [True: 1.20k, False: 601] ------------------ 1034| 1.20k| const uint32_t bi = blk_start + i; 1035| 1.20k| const uint64_t blk_decomp_start = zxc_seek_decomp_offset(s->block_size, bi); 1036| 1.20k| const size_t skip = (offset > blk_decomp_start) ? (size_t)(offset - blk_decomp_start) : 0; ------------------ | Branch (1036:29): [True: 0, False: 1.20k] ------------------ 1037| 1.20k| const size_t blk_decomp_sz = zxc_seek_decomp_size(s->block_size, s->total_decomp, bi); 1038| 1.20k| if (UNLIKELY(blk_decomp_sz < skip)) { ------------------ | | 151| 1.20k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.20k] | | ------------------ ------------------ 1039| | // LCOV_EXCL_START 1040| 0| ZXC_FREE(jobs); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 1041| 0| return ZXC_ERROR_CORRUPT_DATA; 1042| | // LCOV_EXCL_STOP 1043| 0| } 1044| 1.20k| const size_t avail = blk_decomp_sz - skip; 1045| 1.20k| const size_t copy = (avail < remaining) ? avail : remaining; ------------------ | Branch (1045:29): [True: 601, False: 601] ------------------ 1046| | 1047| 1.20k| jobs[i].s = s; 1048| 1.20k| jobs[i].block_idx = bi; 1049| 1.20k| jobs[i].dst = out; 1050| 1.20k| jobs[i].skip = skip; 1051| 1.20k| jobs[i].copy_len = copy; 1052| 1.20k| jobs[i].result = 0; 1053| | 1054| 1.20k| out += copy; 1055| 1.20k| remaining -= copy; 1056| 1.20k| } 1057| | 1058| | /* Launch one persistent worker per thread */ 1059| 601| zxc_thread_t* const threads = 1060| 601| (zxc_thread_t*)ZXC_MALLOC((size_t)n_threads * sizeof(zxc_thread_t)); ------------------ | | 66| 601|#define ZXC_MALLOC(size) malloc(size) ------------------ 1061| 601| zxc_seek_mt_stripe_t* const stripes = 1062| 601| (zxc_seek_mt_stripe_t*)ZXC_MALLOC((size_t)n_threads * sizeof(zxc_seek_mt_stripe_t)); ------------------ | | 66| 601|#define ZXC_MALLOC(size) malloc(size) ------------------ 1063| 601| if (UNLIKELY(!threads || !stripes)) { ------------------ | | 151| 1.20k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 601] | | | Branch (151:42): [True: 0, False: 601] | | | Branch (151:42): [True: 0, False: 601] | | ------------------ ------------------ 1064| | // LCOV_EXCL_START 1065| 0| ZXC_FREE(threads); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 1066| 0| ZXC_FREE(stripes); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 1067| 0| ZXC_FREE(jobs); ------------------ | | 84| 0|#define ZXC_FREE(ptr) free(ptr) ------------------ 1068| 0| return ZXC_ERROR_MEMORY; 1069| | // LCOV_EXCL_STOP 1070| 0| } 1071| | 1072| 601| int launched = 0; 1073| 1.80k| for (int t = 0; t < n_threads; t++) { ------------------ | Branch (1073:21): [True: 1.20k, False: 601] ------------------ 1074| 1.20k| stripes[t].jobs = jobs; 1075| 1.20k| stripes[t].num_jobs = num_jobs; 1076| 1.20k| stripes[t].first = (uint32_t)t; 1077| 1.20k| stripes[t].stride = (uint32_t)n_threads; 1078| 1.20k| if (UNLIKELY(zxc_seek_thread_create(&threads[t], zxc_seek_mt_worker, &stripes[t]) != 0)) { ------------------ | | 151| 1.20k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.20k] | | ------------------ ------------------ 1079| | // LCOV_EXCL_START 1080| | /* Failed to create thread - mark its stripe as errored; already 1081| | * launched workers keep running and are joined below. */ 1082| 0| zxc_seek_mt_fail_stripe(&stripes[t], ZXC_ERROR_MEMORY); 1083| 0| continue; 1084| | // LCOV_EXCL_STOP 1085| 0| } 1086| | 1087| 1.20k| launched++; 1088| 1.20k| threads[launched - 1] = threads[t]; 1089| 1.20k| } 1090| | 1091| | /* Join phase */ 1092| 1.80k| for (int t = 0; t < launched; t++) zxc_seek_thread_join(threads[t]); ------------------ | Branch (1092:21): [True: 1.20k, False: 601] ------------------ 1093| | 1094| 601| ZXC_FREE(threads); ------------------ | | 84| 601|#define ZXC_FREE(ptr) free(ptr) ------------------ 1095| 601| ZXC_FREE(stripes); ------------------ | | 84| 601|#define ZXC_FREE(ptr) free(ptr) ------------------ 1096| | 1097| | /* Report the first error in job order, if any. */ 1098| 601| int64_t result = (int64_t)len; 1099| 1.80k| for (uint32_t i = 0; i < num_jobs; i++) { ------------------ | Branch (1099:26): [True: 1.20k, False: 601] ------------------ 1100| 1.20k| if (jobs[i].result < 0) { ------------------ | Branch (1100:13): [True: 0, False: 1.20k] ------------------ 1101| 0| result = (int64_t)jobs[i].result; 1102| 0| break; 1103| 0| } 1104| 1.20k| } 1105| | 1106| 601| ZXC_FREE(jobs); ------------------ | | 84| 601|#define ZXC_FREE(ptr) free(ptr) ------------------ 1107| 601| return result; 1108| 601|} zxc_seekable_free: 1119| 9.45k|void zxc_seekable_free(zxc_seekable* s) { 1120| 9.45k| if (UNLIKELY(!s)) return; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 1121| 9.45k| if (s->dctx_initialized) zxc_cctx_free(&s->dctx); ------------------ | Branch (1121:9): [True: 9.45k, False: 0] ------------------ 1122| 9.45k| ZXC_FREE(s->dict); ------------------ | | 84| 9.45k|#define ZXC_FREE(ptr) free(ptr) ------------------ 1123| 9.45k| ZXC_FREE(s->comp_sizes); ------------------ | | 84| 9.45k|#define ZXC_FREE(ptr) free(ptr) ------------------ 1124| 9.45k| ZXC_FREE(s->comp_offsets); ------------------ | | 84| 9.45k|#define ZXC_FREE(ptr) free(ptr) ------------------ 1125| 9.45k| ZXC_FREE(s->owned_reader_ctx); ------------------ | | 84| 9.45k|#define ZXC_FREE(ptr) free(ptr) ------------------ 1126| 9.45k| ZXC_FREE(s); ------------------ | | 84| 9.45k|#define ZXC_FREE(ptr) free(ptr) ------------------ 1127| 9.45k|} zxc_seekable.c:zxc_seekable_parse: 270| 18.9k|static zxc_seekable* zxc_seekable_parse(const uint8_t* data, const size_t data_size) { 271| | /* Minimum: file_header(16) + eof_block(8) + seek_block_header(8) 272| | * + file_footer(12) = 44 */ 273| 18.9k| const size_t MIN_SEEKABLE_SIZE = 274| 18.9k| ZXC_FILE_HEADER_SIZE + ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_HEADER_SIZE + ZXC_FILE_FOOTER_SIZE; ------------------ | | 103| 18.9k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ ZXC_FILE_HEADER_SIZE + ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_HEADER_SIZE + ZXC_FILE_FOOTER_SIZE; ------------------ | | 397| 18.9k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ ZXC_FILE_HEADER_SIZE + ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_HEADER_SIZE + ZXC_FILE_FOOTER_SIZE; ------------------ | | 397| 18.9k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ ZXC_FILE_HEADER_SIZE + ZXC_BLOCK_HEADER_SIZE + ZXC_BLOCK_HEADER_SIZE + ZXC_FILE_FOOTER_SIZE; ------------------ | | 105| 18.9k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 275| 18.9k| if (UNLIKELY(data_size < MIN_SEEKABLE_SIZE)) return NULL; ------------------ | | 151| 18.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 889, False: 18.0k] | | ------------------ ------------------ 276| | 277| | /* Step 1: validate file header => block_size */ 278| 18.0k| size_t block_size_sz = 0; 279| 18.0k| int file_has_chk = 0; 280| 18.0k| uint32_t header_dict_id = 0; 281| 18.0k| if (UNLIKELY(zxc_read_file_header(data, data_size, &block_size_sz, &file_has_chk, ------------------ | | 151| 18.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 8.56k, False: 9.45k] | | ------------------ ------------------ 282| 18.0k| &header_dict_id) != ZXC_OK)) 283| 8.56k| return NULL; // LCOV_EXCL_LINE 284| 9.45k| const uint32_t block_size = (uint32_t)block_size_sz; 285| 9.45k| if (UNLIKELY(block_size == 0)) return NULL; // LCOV_EXCL_LINE ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 286| | 287| | /* Step 2: read total decompressed size from the file footer */ 288| 9.45k| const uint8_t* const footer_ptr = data + data_size - ZXC_FILE_FOOTER_SIZE; ------------------ | | 105| 9.45k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 289| 9.45k| const uint64_t total_decomp = zxc_le64(footer_ptr); 290| | 291| | /* A value of 0 means empty file - no seek table */ 292| 9.45k| if (UNLIKELY(total_decomp == 0)) return NULL; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 293| | 294| | /* Step 3: derive num_blocks = ceil(total_decomp / block_size) */ 295| 9.45k| const uint64_t num_blocks_64 = (total_decomp + block_size - 1) / block_size; 296| 9.45k| if (UNLIKELY(num_blocks_64 > UINT32_MAX)) return NULL; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 297| 9.45k| const uint32_t num_blocks = (uint32_t)num_blocks_64; 298| | 299| | /* Step 4: compute seek block position and validate. */ 300| 9.45k| const uint64_t entries_total_64 = num_blocks_64 * ZXC_SEEK_ENTRY_SIZE; ------------------ | | 443| 9.45k|#define ZXC_SEEK_ENTRY_SIZE 4 ------------------ 301| 9.45k| if (UNLIKELY(entries_total_64 > SIZE_MAX - ZXC_BLOCK_HEADER_SIZE)) return NULL; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 302| 9.45k| const size_t entries_total = (size_t)entries_total_64; 303| 9.45k| const size_t seek_block_total = ZXC_BLOCK_HEADER_SIZE + entries_total; ------------------ | | 397| 9.45k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 304| 9.45k| if (UNLIKELY(seek_block_total + ZXC_FILE_FOOTER_SIZE > data_size)) return NULL; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 305| 9.45k| const uint8_t* const seek_block_start = 306| 9.45k| data + data_size - ZXC_FILE_FOOTER_SIZE - seek_block_total; ------------------ | | 105| 9.45k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 307| 9.45k| if (UNLIKELY(seek_block_start < data)) return NULL; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 308| | 309| | /* Read and validate SEK block header */ 310| 9.45k| zxc_block_header_t bh; 311| 9.45k| if (UNLIKELY(zxc_read_block_header(seek_block_start, seek_block_total, &bh) != ZXC_OK)) ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 312| 0| return NULL; 313| 9.45k| if (UNLIKELY(bh.block_type != ZXC_BLOCK_SEK)) return NULL; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 314| 9.45k| if (UNLIKELY(bh.comp_size != (uint32_t)entries_total)) return NULL; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 315| | 316| | /* Step 5: allocate handle and parse entries */ 317| 9.45k| zxc_seekable* const s = (zxc_seekable*)ZXC_CALLOC(1, sizeof(zxc_seekable)); ------------------ | | 72| 9.45k|#define ZXC_CALLOC(nmemb, size) calloc(nmemb, size) ------------------ 318| | // LCOV_EXCL_START 319| 9.45k| if (UNLIKELY(!s)) return NULL; ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 320| | // LCOV_EXCL_STOP 321| | 322| 9.45k| s->num_blocks = num_blocks; 323| 9.45k| s->block_size = block_size; 324| 9.45k| s->file_has_checksums = file_has_chk; 325| 9.45k| s->expected_dict_id = header_dict_id; 326| 9.45k| s->src = data; 327| 9.45k| s->src_size = (uint64_t)data_size; 328| | 329| | /* Allocate arrays */ 330| 9.45k| s->comp_sizes = (uint32_t*)ZXC_CALLOC(num_blocks, sizeof(uint32_t)); ------------------ | | 72| 9.45k|#define ZXC_CALLOC(nmemb, size) calloc(nmemb, size) ------------------ 331| 9.45k| s->comp_offsets = (uint64_t*)ZXC_CALLOC((size_t)num_blocks + 1, sizeof(uint64_t)); ------------------ | | 72| 9.45k|#define ZXC_CALLOC(nmemb, size) calloc(nmemb, size) ------------------ 332| | // LCOV_EXCL_START 333| 9.45k| if (UNLIKELY(!s->comp_sizes || !s->comp_offsets)) { ------------------ | | 151| 18.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | | Branch (151:42): [True: 0, False: 9.45k] | | | Branch (151:42): [True: 0, False: 9.45k] | | ------------------ ------------------ 334| 0| zxc_seekable_free(s); 335| 0| return NULL; 336| 0| } 337| | // LCOV_EXCL_STOP 338| 9.45k| s->total_decomp = total_decomp; 339| | 340| | /* Parse comp_sizes and build compressed prefix sums. 341| | * Validate each comp_size against data_size to prevent prefix-sum overflow 342| | * and out-of-bounds reads during decompression. */ 343| 9.45k| const uint8_t* ep = seek_block_start + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 9.45k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 344| 9.45k| uint64_t comp_acc = ZXC_FILE_HEADER_SIZE; /* blocks start after file header */ ------------------ | | 103| 9.45k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ 345| 20.1k| for (uint32_t i = 0; i < num_blocks; i++) { ------------------ | Branch (345:26): [True: 10.6k, False: 9.45k] ------------------ 346| 10.6k| s->comp_sizes[i] = zxc_le32(ep); 347| 10.6k| ep += sizeof(uint32_t); 348| | 349| | /* Reject entries below minimum (block header) or larger than the file */ 350| 10.6k| if (UNLIKELY(s->comp_sizes[i] < ZXC_BLOCK_HEADER_SIZE || ------------------ | | 151| 21.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 10.6k] | | | Branch (151:42): [True: 0, False: 10.6k] | | | Branch (151:42): [True: 0, False: 10.6k] | | ------------------ ------------------ 351| 10.6k| s->comp_sizes[i] > (uint64_t)data_size)) { 352| 0| zxc_seekable_free(s); 353| 0| return NULL; 354| 0| } 355| 10.6k| s->comp_offsets[i] = comp_acc; 356| 10.6k| comp_acc += s->comp_sizes[i]; 357| | /* Reject if cumulative offset exceeds file size (inconsistent table) */ 358| 10.6k| if (UNLIKELY(comp_acc > (uint64_t)data_size)) { ------------------ | | 151| 10.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 10.6k] | | ------------------ ------------------ 359| | // LCOV_EXCL_START 360| 0| zxc_seekable_free(s); 361| 0| return NULL; 362| | // LCOV_EXCL_STOP 363| 0| } 364| 10.6k| } 365| 9.45k| s->comp_offsets[num_blocks] = comp_acc; 366| | 367| | /* Verify prefix-sum lands exactly at the EOF block position. 368| | * Expected layout: [header 16][data blocks][EOF 8][SEK block][footer 12] 369| | * So comp_acc (end of data blocks) + EOF(8) == seek_block_start. */ 370| 9.45k| const uint64_t expected_eof_offset = 371| 9.45k| (uint64_t)(seek_block_start - data) - ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 9.45k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 372| 9.45k| if (UNLIKELY(comp_acc != expected_eof_offset)) { ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 373| | // LCOV_EXCL_START 374| 0| zxc_seekable_free(s); 375| 0| return NULL; 376| | // LCOV_EXCL_STOP 377| 0| } 378| | 379| | /* Validate that an actual EOF block header exists at the computed offset */ 380| 9.45k| if (UNLIKELY(comp_acc + ZXC_BLOCK_HEADER_SIZE > data_size)) { ------------------ | | 151| 9.45k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | ------------------ ------------------ 381| | // LCOV_EXCL_START 382| 0| zxc_seekable_free(s); 383| 0| return NULL; 384| | // LCOV_EXCL_STOP 385| 0| } 386| 9.45k| zxc_block_header_t eof_bh; 387| 9.45k| if (UNLIKELY(zxc_read_block_header(data + comp_acc, ZXC_BLOCK_HEADER_SIZE, &eof_bh) != ZXC_OK || ------------------ | | 151| 18.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 9.45k] | | | Branch (151:42): [True: 0, False: 9.45k] | | | Branch (151:42): [True: 0, False: 9.45k] | | ------------------ ------------------ 388| 9.45k| eof_bh.block_type != ZXC_BLOCK_EOF)) { 389| | // LCOV_EXCL_START 390| 0| zxc_seekable_free(s); 391| 0| return NULL; 392| | // LCOV_EXCL_STOP 393| 0| } 394| | 395| 9.45k| return s; 396| 9.45k|} zxc_seekable.c:zxc_seek_find_block: 612| 40.0k|static uint32_t zxc_seek_find_block(const uint32_t block_size, const uint64_t offset) { 613| 40.0k| return (uint32_t)(offset / (uint64_t)block_size); 614| 40.0k|} zxc_seekable.c:zxc_seek_read_block: 659| 19.0k| const size_t buf_cap) { 660| 19.0k| const uint64_t off = s->comp_offsets[block_idx]; 661| 19.0k| const uint32_t csz = s->comp_sizes[block_idx]; 662| 19.0k| if (UNLIKELY(csz > buf_cap)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 19.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 19.0k] | | ------------------ ------------------ 663| | 664| 19.0k| if (s->src) { ------------------ | Branch (664:9): [True: 19.0k, False: 0] ------------------ 665| | /* Buffer mode */ 666| 19.0k| if (UNLIKELY(off + csz > s->src_size)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 19.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 19.0k] | | ------------------ ------------------ 667| 19.0k| ZXC_MEMCPY(buf, s->src + off, csz); ------------------ | | 173| 19.0k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 668| 19.0k| } else if (s->reader.read_at) { ------------------ | Branch (668:16): [True: 0, False: 0] ------------------ 669| | /* Caller-supplied reader (also covers the FILE* variant, which 670| | * provides a pread-backed callback from zxc_seekable_file.c). */ 671| 0| const int64_t r = s->reader.read_at(s->reader.ctx, buf, csz, off); 672| 0| if (UNLIKELY(r != (int64_t)csz)) return (r < 0) ? (int)r : ZXC_ERROR_IO; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ | Branch (672:49): [True: 0, False: 0] ------------------ 673| 0| } else { 674| 0| return ZXC_ERROR_NULL_INPUT; // LCOV_EXCL_LINE 675| 0| } 676| 19.0k| return (int)csz; 677| 19.0k|} zxc_seekable.c:zxc_seek_decomp_offset: 622| 20.2k|static uint64_t zxc_seek_decomp_offset(const uint32_t block_size, const uint32_t idx) { 623| 20.2k| return (uint64_t)idx * (uint64_t)block_size; 624| 20.2k|} zxc_seekable.c:zxc_seek_decomp_size: 638| 1.20k| const uint32_t idx) { 639| 1.20k| const uint64_t start = (uint64_t)idx * (uint64_t)block_size; 640| 1.20k| const uint64_t remaining = total_decomp - start; 641| 1.20k| return (remaining >= (uint64_t)block_size) ? block_size : (uint32_t)remaining; ------------------ | Branch (641:12): [True: 601, False: 601] ------------------ 642| 1.20k|} zxc_seekable.c:zxc_seek_thread_create: 137| 1.20k|static int zxc_seek_thread_create(zxc_thread_t* t, void* (*fn)(void*), void* arg) { 138| 1.20k| return pthread_create(t, NULL, fn, arg) == 0 ? 0 : ZXC_ERROR_MEMORY; ------------------ | Branch (138:12): [True: 1.20k, False: 0] ------------------ 139| 1.20k|} zxc_seekable.c:zxc_seek_mt_worker: 902| 1.20k|static void* zxc_seek_mt_worker(void* arg) { 903| 1.20k| zxc_seek_mt_stripe_t* const st = (zxc_seek_mt_stripe_t*)arg; 904| 1.20k| zxc_seek_mt_job_t* const jobs = st->jobs; 905| 1.20k| const zxc_seekable* const s = jobs[st->first].s; 906| | 907| | /* Thread-local decompression context (mode=0 for decompress-only) */ 908| 1.20k| zxc_cctx_t dctx; 909| 1.20k| if (UNLIKELY(zxc_cctx_init(&dctx, (size_t)s->block_size, 0, 0, 0, s->dict_size) != ZXC_OK)) { ------------------ | | 151| 1.20k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.20k] | | ------------------ ------------------ 910| | // LCOV_EXCL_START 911| 0| zxc_seek_mt_fail_stripe(st, ZXC_ERROR_MEMORY); 912| 0| return NULL; 913| | // LCOV_EXCL_STOP 914| 0| } 915| | 916| 1.20k| if (UNLIKELY(zxc_cctx_attach_dict_huf(&dctx, s->has_dict_huf ? s->dict_huf : NULL) != ZXC_OK)) { ------------------ | | 151| 2.40k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.20k] | | | Branch (151:42): [True: 0, False: 1.20k] | | ------------------ ------------------ 917| | // LCOV_EXCL_START 918| 0| zxc_cctx_free(&dctx); 919| 0| zxc_seek_mt_fail_stripe(st, ZXC_ERROR_CORRUPT_DATA); 920| 0| return NULL; 921| | // LCOV_EXCL_STOP 922| 0| } 923| 1.20k| const size_t work_sz = (size_t)s->block_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 349| 1.20k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 1.20k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 924| | 925| 1.20k| uint8_t* const dict_work = dctx.dict_buffer; 926| 1.20k| if (dict_work) ZXC_MEMCPY(dict_work, s->dict, s->dict_size); ------------------ | | 173| 0|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ | Branch (926:9): [True: 0, False: 1.20k] ------------------ 927| | 928| | /* Read buffer sized for the largest compressed block of the stripe. */ 929| 1.20k| size_t max_csz = 0; 930| 2.40k| for (uint32_t i = st->first; i < st->num_jobs; i += st->stride) { ------------------ | Branch (930:34): [True: 1.20k, False: 1.20k] ------------------ 931| 1.20k| const uint32_t csz = s->comp_sizes[jobs[i].block_idx]; 932| 1.20k| if (csz > max_csz) max_csz = csz; ------------------ | Branch (932:13): [True: 1.20k, False: 1] ------------------ 933| 1.20k| } 934| 1.20k| uint8_t* const read_buf = (uint8_t*)ZXC_MALLOC(max_csz + ZXC_PAD_SIZE); ------------------ | | 66| 1.20k|#define ZXC_MALLOC(size) malloc(size) ------------------ 935| 1.20k| if (UNLIKELY(!read_buf)) { ------------------ | | 151| 1.20k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.20k] | | ------------------ ------------------ 936| | // LCOV_EXCL_START 937| 0| zxc_cctx_free(&dctx); 938| 0| zxc_seek_mt_fail_stripe(st, ZXC_ERROR_MEMORY); 939| 0| return NULL; 940| | // LCOV_EXCL_STOP 941| 0| } 942| | 943| 2.40k| for (uint32_t i = st->first; i < st->num_jobs; i += st->stride) { ------------------ | Branch (943:34): [True: 1.20k, False: 1.20k] ------------------ 944| 1.20k| zxc_seek_mt_job_t* const job = &jobs[i]; 945| | 946| 1.20k| const int read_res = 947| 1.20k| zxc_seek_read_block_mt(s, job->block_idx, read_buf, max_csz + ZXC_PAD_SIZE); ------------------ | | 337| 1.20k|#define ZXC_PAD_SIZE 32 ------------------ 948| 1.20k| if (UNLIKELY(read_res < 0)) { ------------------ | | 151| 1.20k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.20k] | | ------------------ ------------------ 949| | // LCOV_EXCL_START 950| 0| job->result = read_res; 951| 0| break; 952| | // LCOV_EXCL_STOP 953| 0| } 954| | 955| | /* Decompress: use dict bounce buffer when dictionary is active */ 956| 1.20k| uint8_t* dec_dst = dict_work ? dict_work + s->dict_size : dctx.work_buf; ------------------ | Branch (956:28): [True: 0, False: 1.20k] ------------------ 957| 1.20k| const int dec_res = 958| 1.20k| zxc_decompress_chunk_wrapper(&dctx, read_buf, (size_t)read_res, dec_dst, work_sz); 959| | 960| 1.20k| if (UNLIKELY(dec_res < 0)) { ------------------ | | 151| 1.20k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.20k] | | ------------------ ------------------ 961| | // LCOV_EXCL_START 962| 0| job->result = dec_res; 963| 0| break; 964| | // LCOV_EXCL_STOP 965| 0| } 966| 1.20k| if (UNLIKELY((size_t)dec_res < job->skip + job->copy_len)) { ------------------ | | 151| 1.20k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.20k] | | ------------------ ------------------ 967| | // LCOV_EXCL_START 968| 0| job->result = ZXC_ERROR_CORRUPT_DATA; 969| 0| break; 970| | // LCOV_EXCL_STOP 971| 0| } 972| | 973| | /* Copy the requested portion directly into the caller's output buffer */ 974| 1.20k| ZXC_MEMCPY(job->dst, dec_dst + job->skip, job->copy_len); ------------------ | | 173| 1.20k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 975| 1.20k| job->result = 0; 976| 1.20k| } 977| | 978| 1.20k| ZXC_FREE(read_buf); ------------------ | | 84| 1.20k|#define ZXC_FREE(ptr) free(ptr) ------------------ 979| 1.20k| zxc_cctx_free(&dctx); 980| | return NULL; 981| 1.20k|} zxc_seekable.c:zxc_seek_read_block_mt: 822| 1.20k| const size_t buf_cap) { 823| 1.20k| const uint64_t off = s->comp_offsets[block_idx]; 824| 1.20k| const uint32_t csz = s->comp_sizes[block_idx]; 825| 1.20k| if (UNLIKELY(csz > buf_cap)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 1.20k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.20k] | | ------------------ ------------------ 826| | 827| 1.20k| if (s->src) { ------------------ | Branch (827:9): [True: 1.20k, False: 0] ------------------ 828| | /* Buffer mode - memcpy is inherently thread-safe on const data */ 829| 1.20k| if (UNLIKELY(off + csz > s->src_size)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 1.20k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 1.20k] | | ------------------ ------------------ 830| 1.20k| ZXC_MEMCPY(buf, s->src + off, csz); ------------------ | | 173| 1.20k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 831| 1.20k| } else if (s->reader.read_at) { ------------------ | Branch (831:16): [True: 0, False: 0] ------------------ 832| | /* Reader callback - caller-supplied read_at must be thread-safe. 833| | * The FILE* variant (zxc_seekable_file.c) installs a pread-backed 834| | * callback that is naturally thread-safe. */ 835| 0| const int64_t r = s->reader.read_at(s->reader.ctx, buf, csz, off); 836| 0| if (UNLIKELY(r != (int64_t)csz)) return (r < 0) ? (int)r : ZXC_ERROR_IO; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ | Branch (836:49): [True: 0, False: 0] ------------------ 837| 0| } else { 838| 0| return ZXC_ERROR_NULL_INPUT; // LCOV_EXCL_LINE 839| 0| } 840| 1.20k| return (int)csz; 841| 1.20k|} zxc_seekable.c:zxc_seek_thread_join: 145| 1.20k|static void zxc_seek_thread_join(zxc_thread_t t) { pthread_join(t, NULL); } LLVMFuzzerTestOneInput: 28| 9.45k|int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { 29| 9.45k| if (size < 2) return 0; ------------------ | Branch (29:9): [True: 1, False: 9.45k] ------------------ 30| | 31| | /* Save original input for phase 7 (raw parser fuzzing) */ 32| 9.45k| const uint8_t* const raw_data = data; 33| 9.45k| const size_t raw_size = size; 34| | 35| | /* Use first byte as level, second as flags */ 36| 9.45k| const int level = (data[0] % 5) + 1; 37| 9.45k| const int use_checksum = data[1] & 1; 38| 9.45k| const int use_mt = data[1] & 2; 39| 9.45k| data += 2; 40| 9.45k| size -= 2; 41| | 42| 9.45k| if (size == 0 || size > FUZZ_SEEKABLE_MAX_INPUT) return 0; ------------------ | | 26| 9.45k|#define FUZZ_SEEKABLE_MAX_INPUT (4 << 20) /* 4 MiB */ ------------------ | Branch (42:9): [True: 1, False: 9.45k] | Branch (42:22): [True: 0, False: 9.45k] ------------------ 43| | 44| | /* Persistent buffers - reused across iterations to reduce allocator pressure */ 45| 9.45k| static uint8_t* comp_buf = NULL; 46| 9.45k| static size_t comp_cap = 0; 47| 9.45k| static uint8_t* decomp_buf = NULL; 48| 9.45k| static size_t decomp_cap = 0; 49| | 50| | /* ------------------------------------------------------------------ */ 51| | /* Phase 1: Compress with seekable=1 */ 52| | /* ------------------------------------------------------------------ */ 53| 9.45k| const uint64_t bound64 = zxc_compress_bound(size); 54| 9.45k| if (bound64 == 0 || bound64 > SIZE_MAX) return 0; ------------------ | Branch (54:9): [True: 0, False: 9.45k] | Branch (54:25): [True: 0, False: 9.45k] ------------------ 55| 9.45k| const size_t bound = (size_t)bound64; 56| 9.45k| if (bound > comp_cap) { ------------------ | Branch (56:9): [True: 5.12k, False: 4.33k] ------------------ 57| 5.12k| void* new_buf = realloc(comp_buf, bound); 58| 5.12k| if (!new_buf) return 0; ------------------ | Branch (58:13): [True: 0, False: 5.12k] ------------------ 59| 5.12k| comp_buf = (uint8_t*)new_buf; 60| 5.12k| comp_cap = bound; 61| 5.12k| } 62| | 63| 9.45k| zxc_compress_opts_t copts = { 64| 9.45k| .level = level, 65| 9.45k| .checksum_enabled = use_checksum, 66| 9.45k| .seekable = 1, 67| 9.45k| }; 68| 9.45k| const int64_t csize = zxc_compress(data, size, comp_buf, bound, &copts); 69| 9.45k| if (csize < 0) return 0; ------------------ | Branch (69:9): [True: 0, False: 9.45k] ------------------ 70| | 71| | /* ------------------------------------------------------------------ */ 72| | /* Phase 2: Open seekable handle */ 73| | /* ------------------------------------------------------------------ */ 74| 9.45k| zxc_seekable* s = zxc_seekable_open(comp_buf, (size_t)csize); 75| 9.45k| if (!s) return 0; ------------------ | Branch (75:9): [True: 0, False: 9.45k] ------------------ 76| | 77| | /* ------------------------------------------------------------------ */ 78| | /* Phase 3: Exercise metadata getters */ 79| | /* ------------------------------------------------------------------ */ 80| 9.45k| const uint32_t num_blocks = zxc_seekable_get_num_blocks(s); 81| 9.45k| const uint64_t total_decomp = zxc_seekable_get_decompressed_size(s); 82| 9.45k| assert(total_decomp == size); ------------------ | Branch (82:5): [True: 0, False: 9.45k] | Branch (82:5): [True: 9.45k, False: 0] ------------------ 83| | 84| 20.1k| for (uint32_t i = 0; i < num_blocks; i++) { ------------------ | Branch (84:26): [True: 10.6k, False: 9.45k] ------------------ 85| 10.6k| const uint32_t csz = zxc_seekable_get_block_comp_size(s, i); 86| 10.6k| const uint32_t dsz = zxc_seekable_get_block_decomp_size(s, i); 87| 10.6k| assert(csz > 0); ------------------ | Branch (87:9): [True: 0, False: 10.6k] | Branch (87:9): [True: 10.6k, False: 0] ------------------ 88| 10.6k| assert(dsz > 0); ------------------ | Branch (88:9): [True: 0, False: 10.6k] | Branch (88:9): [True: 10.6k, False: 0] ------------------ 89| 10.6k| (void)csz; 90| 10.6k| (void)dsz; 91| 10.6k| } 92| | /* Out-of-range access should return 0 */ 93| 9.45k| assert(zxc_seekable_get_block_comp_size(s, num_blocks) == 0); ------------------ | Branch (93:5): [True: 0, False: 9.45k] | Branch (93:5): [True: 9.45k, False: 0] ------------------ 94| 9.45k| assert(zxc_seekable_get_block_decomp_size(s, num_blocks) == 0); ------------------ | Branch (94:5): [True: 0, False: 9.45k] | Branch (94:5): [True: 9.45k, False: 0] ------------------ 95| | 96| | /* ------------------------------------------------------------------ */ 97| | /* Phase 4: Full decompression via seekable range */ 98| | /* ------------------------------------------------------------------ */ 99| 9.45k| if (size > decomp_cap) { ------------------ | Branch (99:9): [True: 5.12k, False: 4.33k] ------------------ 100| 5.12k| void* new_buf = realloc(decomp_buf, size); 101| 5.12k| if (!new_buf) { ------------------ | Branch (101:13): [True: 0, False: 5.12k] ------------------ 102| 0| zxc_seekable_free(s); 103| 0| return 0; 104| 0| } 105| 5.12k| decomp_buf = (uint8_t*)new_buf; 106| 5.12k| decomp_cap = size; 107| 5.12k| } 108| | 109| 9.45k| int64_t dec_result; 110| 9.45k| if (use_mt && size > 4096) { ------------------ | Branch (110:9): [True: 4.77k, False: 4.68k] | Branch (110:19): [True: 1.72k, False: 3.04k] ------------------ 111| 1.72k| dec_result = zxc_seekable_decompress_range_mt(s, decomp_buf, size, 0, size, 2); 112| 7.72k| } else { 113| 7.72k| dec_result = zxc_seekable_decompress_range(s, decomp_buf, size, 0, size); 114| 7.72k| } 115| 9.45k| assert(dec_result == (int64_t)size); ------------------ | Branch (115:5): [True: 0, False: 9.45k] | Branch (115:5): [True: 9.45k, False: 0] ------------------ 116| 9.45k| (void)dec_result; 117| 9.45k| assert(memcmp(data, decomp_buf, size) == 0); ------------------ | Branch (117:5): [True: 0, False: 9.45k] | Branch (117:5): [True: 9.45k, False: 0] ------------------ 118| | 119| | /* ------------------------------------------------------------------ */ 120| | /* Phase 5: Partial range decompression (sub-block extraction) */ 121| | /* ------------------------------------------------------------------ */ 122| 9.45k| if (size >= 4) { ------------------ | Branch (122:9): [True: 9.43k, False: 25] ------------------ 123| | /* Extract a range from the middle */ 124| 9.43k| const size_t off = size / 4; 125| 9.43k| const size_t len = size / 2; 126| 9.43k| dec_result = zxc_seekable_decompress_range(s, decomp_buf, len, off, len); 127| 9.43k| assert(dec_result == (int64_t)len); ------------------ | Branch (127:9): [True: 0, False: 9.43k] | Branch (127:9): [True: 9.43k, False: 0] ------------------ 128| 9.43k| assert(memcmp(data + off, decomp_buf, len) == 0); ------------------ | Branch (128:9): [True: 0, False: 9.43k] | Branch (128:9): [True: 9.43k, False: 0] ------------------ 129| 9.43k| } 130| | 131| | /* ------------------------------------------------------------------ */ 132| | /* Phase 6: Edge cases */ 133| | /* ------------------------------------------------------------------ */ 134| | /* Zero-length read */ 135| 9.45k| dec_result = zxc_seekable_decompress_range(s, decomp_buf, size, 0, 0); 136| 9.45k| assert(dec_result == 0); ------------------ | Branch (136:5): [True: 0, False: 9.45k] | Branch (136:5): [True: 9.45k, False: 0] ------------------ 137| | 138| | /* Out-of-bounds read */ 139| 9.45k| dec_result = zxc_seekable_decompress_range(s, decomp_buf, size, total_decomp, 1); 140| 9.45k| assert(dec_result < 0); ------------------ | Branch (140:5): [True: 0, False: 9.45k] | Branch (140:5): [True: 9.45k, False: 0] ------------------ 141| | 142| | /* NULL handle */ 143| 9.45k| assert(zxc_seekable_get_num_blocks(NULL) == 0); ------------------ | Branch (143:5): [True: 0, False: 9.45k] | Branch (143:5): [True: 9.45k, False: 0] ------------------ 144| 9.45k| assert(zxc_seekable_get_decompressed_size(NULL) == 0); ------------------ | Branch (144:5): [True: 0, False: 9.45k] | Branch (144:5): [True: 9.45k, False: 0] ------------------ 145| | 146| | /* ------------------------------------------------------------------ */ 147| | /* Cleanup */ 148| | /* ------------------------------------------------------------------ */ 149| 9.45k| zxc_seekable_free(s); 150| | 151| | /* ------------------------------------------------------------------ */ 152| | /* Phase 7: Fuzz the parser with raw data (malformed seekable archives) */ 153| | /* ------------------------------------------------------------------ */ 154| 9.45k| zxc_seekable* s2 = zxc_seekable_open(raw_data, raw_size); 155| 9.45k| if (s2) { ------------------ | Branch (155:9): [True: 0, False: 9.45k] ------------------ 156| | /* If it parsed, try a read - should not crash */ 157| 0| const uint64_t td = zxc_seekable_get_decompressed_size(s2); 158| 0| if (td > 0 && td <= FUZZ_SEEKABLE_MAX_INPUT) { ------------------ | | 26| 0|#define FUZZ_SEEKABLE_MAX_INPUT (4 << 20) /* 4 MiB */ ------------------ | Branch (158:13): [True: 0, False: 0] | Branch (158:23): [True: 0, False: 0] ------------------ 159| 0| uint8_t* tmp = (uint8_t*)malloc((size_t)td); 160| 0| if (tmp) { ------------------ | Branch (160:17): [True: 0, False: 0] ------------------ 161| 0| zxc_seekable_decompress_range(s2, tmp, (size_t)td, 0, (size_t)td); 162| 0| free(tmp); 163| 0| } 164| 0| } 165| 0| zxc_seekable_free(s2); 166| 0| } 167| | 168| 9.45k| return 0; 169| 9.45k|}