zxc_aligned_malloc: 36| 8.79k|void* zxc_aligned_malloc(const size_t size, const size_t alignment) { 37| |#if defined(_WIN32) 38| | return _aligned_malloc(size, alignment); 39| |#else 40| 8.79k| void* ptr = NULL; 41| 8.79k| if (posix_memalign(&ptr, alignment, size) != 0) return NULL; ------------------ | Branch (41:9): [True: 0, False: 8.79k] ------------------ 42| 8.79k| return ptr; 43| 8.79k|#endif 44| 8.79k|} zxc_aligned_free: 51| 8.79k|void zxc_aligned_free(void* ptr) { 52| |#if defined(_WIN32) 53| | _aligned_free(ptr); 54| |#else 55| 8.79k| free(ptr); 56| 8.79k|#endif 57| 8.79k|} zxc_cctx_init_in_workspace: 298| 6.48k| const int defer_entropy_scratch) { 299| 6.48k| if (UNLIKELY(!ctx || !workspace || chunk_size == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 25.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 6.48k] | | | Branch (151:42): [True: 0, False: 6.48k] | | | Branch (151:42): [True: 0, False: 6.48k] | | | Branch (151:42): [True: 0, False: 6.48k] | | ------------------ ------------------ 300| | 301| 6.48k| const zxc_cctx_layout_t layout = 302| 6.48k| compute_cctx_layout(chunk_size, mode, level, dict_size, defer_entropy_scratch); 303| 6.48k| if (UNLIKELY(workspace_size < layout.total)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 6.48k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 6.48k] | | ------------------ ------------------ 304| | 305| 6.48k| ZXC_MEMSET(ctx, 0, sizeof(zxc_cctx_t)); ------------------ | | 178| 6.48k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 306| 6.48k| ctx->checksum_enabled = checksum_enabled; 307| 6.48k| ctx->chunk_size = chunk_size; 308| 6.48k| const uint32_t offset_bits = zxc_log2_u32((uint32_t)chunk_size); 309| 6.48k| ctx->offset_bits = offset_bits; 310| 6.48k| ctx->offset_mask = (uint32_t)((1ULL << offset_bits) - 1); 311| 6.48k| ctx->max_epoch = (uint32_t)(1ULL << (32 - offset_bits)); 312| | 313| | /* memory_block stays NULL on the static-init path so zxc_cctx_free does 314| | * not try to free the caller's workspace. Sub-buffer pointers carry the 315| | * partition; ownership is implicit (the caller owns @p workspace). */ 316| 6.48k| uint8_t* const mem = (uint8_t*)workspace; 317| | 318| | /* Dictionary concat scratch (both modes); init owns dict_size now so callers 319| | * no longer assign ctx->dict_size after init. */ 320| 6.48k| ctx->dict_size = dict_size; 321| 6.48k| if (dict_size > 0) { ------------------ | Branch (321:9): [True: 0, False: 6.48k] ------------------ 322| 0| ctx->dict_buffer = mem + layout.off_dict; 323| 0| ctx->dict_buffer_cap = layout.sz_dict; 324| 0| ctx->dict_huf = (zxc_dict_huf_state_t*)(void*)(mem + layout.off_dict_huf); 325| 0| } 326| | 327| 6.48k| if (mode == 0) { ------------------ | Branch (327:9): [True: 6.48k, False: 0] ------------------ 328| 6.48k| ctx->work_buf = mem + layout.off_work; 329| 6.48k| ctx->work_buf_cap = chunk_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 349| 6.48k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 6.48k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 330| 6.48k| ctx->lit_buffer = mem + layout.off_lit_dctx; 331| 6.48k| ctx->lit_buffer_cap = chunk_size + ZXC_PAD_SIZE; ------------------ | | 337| 6.48k|#define ZXC_PAD_SIZE 32 ------------------ 332| 6.48k| if (layout.sz_pivco_dctx) { ------------------ | Branch (332:13): [True: 0, False: 6.48k] ------------------ 333| 0| ctx->tok_buffer = mem + layout.off_tok_dctx; 334| 0| ctx->tok_buffer_cap = layout.sz_tok_dctx; 335| 0| ctx->pivco_scratch = mem + layout.off_pivco_dctx; 336| 0| ctx->pivco_scratch_cap = layout.sz_pivco_dctx; 337| 0| } 338| 6.48k| return ZXC_OK; 339| 6.48k| } 340| | 341| 0| ctx->hash_table = (uint32_t*)(mem + layout.off_hash_pos); 342| 0| ctx->hash_tags = mem + layout.off_hash_tags; 343| 0| ctx->chain_table = (uint16_t*)(mem + layout.off_chain); 344| 0| ctx->buf_sequences = (uint32_t*)(mem + layout.off_seq_union); 345| 0| ctx->buf_offsets = (uint16_t*)(mem + layout.off_seq_union); 346| 0| ctx->buf_tokens = mem + layout.off_seq_union + layout.max_seq * sizeof(uint16_t); 347| 0| ctx->buf_extras = mem + layout.off_extras; 348| 0| ctx->literals = mem + layout.off_lit_cctx; 349| 0| if (layout.sz_opt) { ------------------ | Branch (349:9): [True: 0, False: 0] ------------------ 350| 0| ctx->opt_scratch = mem + layout.off_opt; 351| 0| ctx->opt_scratch_cap = layout.sz_opt; 352| 0| } 353| | 354| 0| ctx->compression_level = level; 355| 0| ctx->epoch = 1; 356| | 357| 0| ZXC_MEMSET(ctx->hash_table, 0, layout.sz_hash_pos); ------------------ | | 178| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 358| 0| ZXC_MEMSET(ctx->hash_tags, 0, layout.sz_hash_tags); ------------------ | | 178| 0|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 359| 0| return ZXC_OK; 360| 6.48k|} zxc_cctx_init: 381| 6.48k| const int level, const int checksum_enabled, const size_t dict_size) { 382| 6.48k| if (UNLIKELY(chunk_size == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 6.48k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 6.48k] | | ------------------ ------------------ 383| | /* Heap contexts defer the decode-side entropy scratch to the first entropy 384| | * section; static workspaces must have it already, they may never allocate. */ 385| 6.48k| const int defer_entropy = (mode == 0); 386| 6.48k| const size_t total = 387| 6.48k| compute_cctx_layout(chunk_size, mode, level, dict_size, defer_entropy).total; 388| 6.48k| if (UNLIKELY(total == 0)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 6.48k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 6.48k] | | ------------------ ------------------ 389| | 390| 6.48k| uint8_t* const mem = (uint8_t*)ZXC_ALIGNED_MALLOC(total, ZXC_CACHE_LINE_SIZE); ------------------ | | 110| 6.48k|#define ZXC_ALIGNED_MALLOC(size, alignment) zxc_aligned_malloc(size, alignment) ------------------ 391| 6.48k| if (UNLIKELY(!mem)) return ZXC_ERROR_MEMORY; ------------------ | | 151| 6.48k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 6.48k] | | ------------------ ------------------ 392| | 393| 6.48k| const int rc = zxc_cctx_init_in_workspace(ctx, mem, total, chunk_size, mode, level, 394| 6.48k| checksum_enabled, dict_size, defer_entropy); 395| 6.48k| if (UNLIKELY(rc != ZXC_OK)) { ------------------ | | 151| 6.48k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 6.48k] | | ------------------ ------------------ 396| | // LCOV_EXCL_START 397| 0| ZXC_ALIGNED_FREE(mem); ------------------ | | 116| 0|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 398| 0| return rc; 399| | // LCOV_EXCL_STOP 400| 0| } 401| | /* Library-owned buffer: record the allocation so zxc_cctx_free frees it. */ 402| 6.48k| ctx->memory_block = mem; 403| 6.48k| return ZXC_OK; 404| 6.48k|} zxc_cctx_alloc_entropy_scratch: 414| 2.30k|int zxc_cctx_alloc_entropy_scratch(zxc_cctx_t* ctx) { 415| 2.30k| if (LIKELY(ctx->pivco_scratch != NULL)) return ZXC_OK; ------------------ | | 145| 2.30k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 0, False: 2.30k] | | ------------------ ------------------ 416| | 417| 2.30k| size_t sz_tok = 0; 418| 2.30k| size_t sz_pivco = 0; 419| 2.30k| zxc_dctx_entropy_sizes(ctx->chunk_size, &sz_tok, &sz_pivco); 420| 2.30k| const size_t total = ZXC_ALIGN_CL(sz_tok) + ZXC_ALIGN_CL(sz_pivco); ------------------ | | 355| 2.30k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 2.30k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 2.30k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 2.30k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 2.30k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ const size_t total = ZXC_ALIGN_CL(sz_tok) + ZXC_ALIGN_CL(sz_pivco); ------------------ | | 355| 2.30k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 2.30k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 2.30k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 2.30k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 2.30k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 421| | 422| 2.30k| uint8_t* const mem = (uint8_t*)ZXC_ALIGNED_MALLOC(total, ZXC_CACHE_LINE_SIZE); ------------------ | | 110| 2.30k|#define ZXC_ALIGNED_MALLOC(size, alignment) zxc_aligned_malloc(size, alignment) ------------------ 423| 2.30k| if (UNLIKELY(!mem)) return ZXC_ERROR_MEMORY; // LCOV_EXCL_LINE ------------------ | | 151| 2.30k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 2.30k] | | ------------------ ------------------ 424| | 425| 2.30k| ctx->entropy_block = mem; 426| 2.30k| ctx->tok_buffer = mem; 427| 2.30k| ctx->tok_buffer_cap = sz_tok; 428| 2.30k| ctx->pivco_scratch = mem + ZXC_ALIGN_CL(sz_tok); ------------------ | | 355| 2.30k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 2.30k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 2.30k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 2.30k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 2.30k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 429| 2.30k| ctx->pivco_scratch_cap = sz_pivco; 430| 2.30k| return ZXC_OK; 431| 2.30k|} zxc_cctx_free: 441| 6.48k|void zxc_cctx_free(zxc_cctx_t* ctx) { 442| 6.48k| if (ctx->memory_block) { ------------------ | Branch (442:9): [True: 6.48k, False: 0] ------------------ 443| 6.48k| ZXC_ALIGNED_FREE(ctx->memory_block); ------------------ | | 116| 6.48k|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 444| 6.48k| ctx->memory_block = NULL; 445| 6.48k| } 446| 6.48k| if (ctx->entropy_block) { ------------------ | Branch (446:9): [True: 2.30k, False: 4.18k] ------------------ 447| 2.30k| ZXC_ALIGNED_FREE(ctx->entropy_block); ------------------ | | 116| 2.30k|#define ZXC_ALIGNED_FREE(ptr) zxc_aligned_free(ptr) ------------------ 448| 2.30k| ctx->entropy_block = NULL; 449| 2.30k| } 450| | 451| 6.48k| ctx->lit_buffer = NULL; 452| 6.48k| ctx->hash_table = NULL; 453| 6.48k| ctx->hash_tags = NULL; 454| 6.48k| ctx->chain_table = NULL; 455| 6.48k| ctx->buf_sequences = NULL; 456| 6.48k| ctx->buf_tokens = NULL; 457| 6.48k| ctx->buf_offsets = NULL; 458| 6.48k| ctx->buf_extras = NULL; 459| 6.48k| ctx->literals = NULL; 460| 6.48k| ctx->work_buf = NULL; 461| 6.48k| ctx->tok_buffer = NULL; 462| 6.48k| ctx->pivco_scratch = NULL; 463| 6.48k| ctx->opt_scratch = NULL; 464| 6.48k| ctx->dict_buffer = NULL; 465| 6.48k| ctx->dict_huf = NULL; 466| | 467| 6.48k| ctx->epoch = 0; 468| 6.48k| ctx->lit_buffer_cap = 0; 469| 6.48k| ctx->work_buf_cap = 0; 470| 6.48k| ctx->tok_buffer_cap = 0; 471| 6.48k| ctx->pivco_scratch_cap = 0; 472| 6.48k| ctx->opt_scratch_cap = 0; 473| 6.48k| ctx->dict_buffer_cap = 0; 474| 6.48k| ctx->dict_size = 0; 475| 6.48k| ctx->dict_huf_tree_ok = 0; 476| | ctx->lit_freq_acc = NULL; 477| 6.48k|} zxc_cctx_attach_dict_huf: 492| 6.47k|int zxc_cctx_attach_dict_huf(zxc_cctx_t* RESTRICT ctx, const uint8_t* RESTRICT lengths) { 493| 6.47k| if (UNLIKELY(!ctx)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 6.47k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 6.47k] | | ------------------ ------------------ 494| 6.47k| ctx->dict_huf_tree_ok = 0; 495| 6.47k| if (lengths == NULL) return ZXC_OK; ------------------ | Branch (495:9): [True: 6.47k, False: 0] ------------------ 496| | 497| | /* Empty (all-zero) table from a low-entropy corpus: treat it as "no shared table". */ 498| 0| int empty = 1; 499| 0| for (size_t i = 0; i < ZXC_HUF_TABLE_SIZE; i++) { ------------------ | | 83| 0|#define ZXC_HUF_TABLE_SIZE 128 ------------------ | Branch (499:24): [True: 0, False: 0] ------------------ 500| 0| if (lengths[i]) { ------------------ | Branch (500:13): [True: 0, False: 0] ------------------ 501| 0| empty = 0; 502| 0| break; 503| 0| } 504| 0| } 505| 0| if (UNLIKELY(empty || ctx->dict_huf == NULL)) return ZXC_OK; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 506| | 507| | /* Tree-at-attach: unpack + build the PivCo tree, codes and decoder tables 508| | * once here; the per-block encode/estimate/decode paths reuse them via 509| | * the context. */ 510| 0| const int rc = zxc_huf_dict_tree_build(lengths, &ctx->dict_huf->tree, ctx->dict_huf->codes, 511| 0| ctx->dict_huf->code_len, &ctx->dict_huf->dec); 512| 0| if (UNLIKELY(rc != ZXC_OK)) return rc; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 513| 0| ctx->dict_huf_tree_ok = 1; 514| 0| return ZXC_OK; 515| 0|} zxc_read_file_header: 580| 13.1k| uint32_t* RESTRICT out_dict_id) { 581| 13.1k| if (UNLIKELY(src_size < ZXC_FILE_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 13.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 13.1k] | | ------------------ ------------------ 582| 13.1k| if (UNLIKELY(zxc_le32(src) != ZXC_MAGIC_WORD)) return ZXC_ERROR_BAD_MAGIC; ------------------ | | 151| 13.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 57, False: 13.0k] | | ------------------ ------------------ 583| 13.0k| if (UNLIKELY(src[4] != ZXC_FILE_FORMAT_VERSION)) return ZXC_ERROR_BAD_VERSION; ------------------ | | 151| 13.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 16, False: 13.0k] | | ------------------ ------------------ 584| | 585| 13.0k| uint8_t temp[ZXC_FILE_HEADER_SIZE]; 586| 13.0k| ZXC_MEMCPY(temp, src, ZXC_FILE_HEADER_SIZE); ------------------ | | 173| 13.0k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 587| | // Zero out CRC bytes (14-15) before hash check 588| 13.0k| temp[14] = 0; 589| 13.0k| temp[15] = 0; 590| | // Header CRC16 (integrity), then the checksum-algorithm id in flags bits 0-3 591| | // (only 0 = RapidHash is defined). CRC is checked first via short-circuit. 592| 13.0k| if (UNLIKELY(zxc_le16(src + 14) != zxc_hash16(temp) || ------------------ | | 151| 26.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 60, False: 12.9k] | | | Branch (151:42): [True: 52, False: 13.0k] | | | Branch (151:42): [True: 8, False: 12.9k] | | ------------------ ------------------ 593| 13.0k| (src[6] & 0x0FU) != ZXC_CHECKSUM_RAPIDHASH)) 594| 60| return ZXC_ERROR_BAD_HEADER; 595| | 596| 12.9k| if (out_block_size) { ------------------ | Branch (596:9): [True: 12.9k, False: 0] ------------------ 597| 12.9k| const uint8_t code = src[5]; 598| 12.9k| if (UNLIKELY(code < ZXC_BLOCK_SIZE_MIN_LOG2 || code > ZXC_BLOCK_SIZE_MAX_LOG2)) ------------------ | | 151| 25.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 16, False: 12.9k] | | | Branch (151:42): [True: 6, False: 12.9k] | | | Branch (151:42): [True: 10, False: 12.9k] | | ------------------ ------------------ 599| 16| return ZXC_ERROR_BAD_BLOCK_SIZE; 600| | // Exponent encoding: block_size = 2^code (4 KB - 2 MB) 601| 12.9k| *out_block_size = (size_t)1U << code; 602| 12.9k| } 603| 12.9k| if (out_has_checksum) *out_has_checksum = (src[6] & ZXC_FILE_FLAG_HAS_CHECKSUM) ? 1 : 0; ------------------ | | 365| 12.9k|#define ZXC_FILE_FLAG_HAS_CHECKSUM 0x80U ------------------ | Branch (603:9): [True: 12.9k, False: 0] | Branch (603:47): [True: 2.29k, False: 10.6k] ------------------ 604| 12.9k| if (out_dict_id) *out_dict_id = (src[6] & ZXC_FILE_FLAG_HAS_DICTIONARY) ? zxc_le32(src + 7) : 0; ------------------ | | 367| 12.9k|#define ZXC_FILE_FLAG_HAS_DICTIONARY 0x40U ------------------ | Branch (604:9): [True: 12.9k, False: 0] | Branch (604:37): [True: 26, False: 12.9k] ------------------ 605| | 606| 12.9k| return ZXC_OK; 607| 12.9k|} zxc_read_block_header: 643| 49.3k| zxc_block_header_t* RESTRICT bh) { 644| 49.3k| if (UNLIKELY(src_size < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 49.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 709, False: 48.6k] | | ------------------ ------------------ 645| | 646| 48.6k| uint8_t temp[ZXC_BLOCK_HEADER_SIZE]; 647| 48.6k| ZXC_MEMCPY(temp, src, ZXC_BLOCK_HEADER_SIZE); ------------------ | | 173| 48.6k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 648| 48.6k| temp[7] = 0; // Zero out checksum byte before hashing 649| 48.6k| if (UNLIKELY(src[7] != zxc_hash8(temp))) return ZXC_ERROR_BAD_HEADER; ------------------ | | 151| 48.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 732, False: 47.9k] | | ------------------ ------------------ 650| | 651| 47.9k| bh->block_type = src[0]; 652| 47.9k| bh->block_flags = 0; // Flags not used currently 653| 47.9k| bh->reserved = src[2]; 654| 47.9k| bh->comp_size = zxc_le32(src + 3); 655| 47.9k| bh->header_crc = src[7]; 656| | 657| 47.9k| return ZXC_OK; 658| 48.6k|} zxc_read_glo_header_and_desc: 733| 64.2k| zxc_section_desc_t desc[ZXC_GLO_SECTIONS]) { 734| 64.2k| const size_t needed = 735| 64.2k| ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 401| 64.2k|#define ZXC_GLO_HEADER_BINARY_SIZE 16 ------------------ ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 419| 64.2k|#define ZXC_GLO_SECTIONS 4 ------------------ ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 64.2k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 736| | 737| 64.2k| if (UNLIKELY(len < needed)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 64.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 12, False: 64.2k] | | ------------------ ------------------ 738| | 739| 64.2k| gh->n_sequences = zxc_le32(src); 740| 64.2k| gh->n_literals = zxc_le32(src + 4); 741| 64.2k| gh->enc_lit = src[8]; 742| 64.2k| gh->enc_litlen = src[9]; 743| 64.2k| gh->enc_mlen = src[10]; 744| 64.2k| gh->enc_off = src[11]; 745| | 746| 64.2k| const uint8_t* p = src + ZXC_GLO_HEADER_BINARY_SIZE; ------------------ | | 401| 64.2k|#define ZXC_GLO_HEADER_BINARY_SIZE 16 ------------------ 747| | 748| 321k| for (int i = 0; i < ZXC_GLO_SECTIONS; i++) { ------------------ | | 419| 321k|#define ZXC_GLO_SECTIONS 4 ------------------ | Branch (748:21): [True: 257k, False: 64.2k] ------------------ 749| 257k| desc[i].sizes = zxc_le64(p); 750| 257k| p += ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 257k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 751| 257k| } 752| 64.2k| return ZXC_OK; 753| 64.2k|} zxc_read_ghi_header_and_desc: 802| 2.39k| zxc_section_desc_t desc[ZXC_GHI_SECTIONS]) { 803| 2.39k| const size_t needed = 804| 2.39k| ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 403| 2.39k|#define ZXC_GHI_HEADER_BINARY_SIZE 16 ------------------ ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 421| 2.39k|#define ZXC_GHI_SECTIONS 3 ------------------ ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 2.39k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 805| | 806| 2.39k| if (UNLIKELY(len < needed)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 2.39k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 13, False: 2.38k] | | ------------------ ------------------ 807| | 808| 2.38k| gh->n_sequences = zxc_le32(src); 809| 2.38k| gh->n_literals = zxc_le32(src + 4); 810| 2.38k| gh->enc_lit = src[8]; 811| 2.38k| gh->enc_litlen = src[9]; 812| 2.38k| gh->enc_mlen = src[10]; 813| 2.38k| gh->enc_off = src[11]; 814| | 815| 2.38k| const uint8_t* p = src + ZXC_GHI_HEADER_BINARY_SIZE; ------------------ | | 403| 2.38k|#define ZXC_GHI_HEADER_BINARY_SIZE 16 ------------------ 816| | 817| 9.52k| for (int i = 0; i < ZXC_GHI_SECTIONS; i++) { ------------------ | | 421| 9.52k|#define ZXC_GHI_SECTIONS 3 ------------------ | Branch (817:21): [True: 7.14k, False: 2.38k] ------------------ 818| 7.14k| desc[i].sizes = zxc_le64(p); 819| 7.14k| p += ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 7.14k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 820| 7.14k| } 821| 2.38k| return ZXC_OK; 822| 2.39k|} zxc_common.c:compute_cctx_layout: 163| 12.9k| const int defer_entropy_scratch) { 164| 12.9k| zxc_cctx_layout_t layout = {0}; 165| | 166| 12.9k| const size_t max_seq = zxc_cctx_max_seq(chunk_size); 167| | 168| 12.9k| if (mode == 0) { ------------------ | Branch (168:9): [True: 12.9k, False: 0] ------------------ 169| | /* Decompress: work_buf + lit_buffer, padded for wild-copy overshoot and 170| | * sized worst-case. lit_buffer is provisioned at every level - the decoder 171| | * cannot predict a block's literal encoding (RAW / RLE / HUFFMAN). */ 172| 12.9k| const size_t sz_work = chunk_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 349| 12.9k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 12.9k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 173| 12.9k| const size_t sz_lit = chunk_size + ZXC_PAD_SIZE; ------------------ | | 337| 12.9k|#define ZXC_PAD_SIZE 32 ------------------ 174| | 175| 12.9k| layout.off_work = layout.total; 176| 12.9k| layout.total += ZXC_ALIGN_CL(sz_work); ------------------ | | 355| 12.9k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 12.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 12.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 12.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 12.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 177| 12.9k| layout.off_lit_dctx = layout.total; 178| 12.9k| layout.total += ZXC_ALIGN_CL(sz_lit); ------------------ | | 355| 12.9k|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 12.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 12.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 12.9k|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 12.9k|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 179| | /* Token-section decode scratch (level-7 GLO Huffman-codes the tokens) + 180| | * PivCo ping-pong scratch. enc_lit/enc_litlen are unpredictable, so static 181| | * workspaces provision both up front (no-alloc contract); heap contexts 182| | * defer them to the first entropy section, sparing L1-5 archives ~1.2x 183| | * chunk_size. See zxc_cctx_alloc_entropy_scratch. */ 184| 12.9k| if (!defer_entropy_scratch) { ------------------ | Branch (184:13): [True: 0, False: 12.9k] ------------------ 185| 0| size_t sz_tok = 0; 186| 0| size_t sz_pivco = 0; 187| 0| zxc_dctx_entropy_sizes(chunk_size, &sz_tok, &sz_pivco); 188| 0| layout.sz_tok_dctx = sz_tok; 189| 0| layout.off_tok_dctx = layout.total; 190| 0| layout.total += ZXC_ALIGN_CL(layout.sz_tok_dctx); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 191| 0| layout.sz_pivco_dctx = sz_pivco; 192| 0| layout.off_pivco_dctx = layout.total; 193| 0| layout.total += ZXC_ALIGN_CL(layout.sz_pivco_dctx); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 194| 0| } 195| 12.9k| } else { 196| | /* Compress: 6 partitions + optional opt_scratch at level >= ZXC_LEVEL_DENSITY. */ 197| 0| const uint32_t offset_bits = zxc_log2_u32((uint32_t)chunk_size); 198| 0| layout.max_seq = max_seq; 199| 0| layout.sz_hash_pos = ZXC_LZ_HASH_SIZE * sizeof(uint32_t); ------------------ | | 502| 0|#define ZXC_LZ_HASH_SIZE (1U << ZXC_LZ_HASH_BITS) | | ------------------ | | | | 496| 0|#define ZXC_LZ_HASH_BITS 15 | | ------------------ ------------------ 200| 0| layout.sz_hash_tags = ZXC_LZ_HASH_SIZE * sizeof(uint8_t); ------------------ | | 502| 0|#define ZXC_LZ_HASH_SIZE (1U << ZXC_LZ_HASH_BITS) | | ------------------ | | | | 496| 0|#define ZXC_LZ_HASH_BITS 15 | | ------------------ ------------------ 201| 0| const size_t sz_chain = ZXC_LZ_WINDOW_SIZE * sizeof(uint16_t); ------------------ | | 504| 0|#define ZXC_LZ_WINDOW_SIZE (1U << 16) ------------------ 202| | /* buf_sequences (GHI, level <= ZXC_LEVEL_FAST) aliases buf_offsets + buf_tokens (GLO, 203| | * level >= ZXC_LEVEL_DEFAULT). Mutually exclusive per block; sized for the larger. */ 204| 0| const size_t sz_seq_union = layout.max_seq * sizeof(uint32_t); 205| 0| const size_t vbyte_len = (offset_bits + 6) / 7; 206| 0| const size_t sz_extras = layout.max_seq * 2 * vbyte_len; 207| 0| const size_t sz_lit = chunk_size + ZXC_PAD_SIZE; ------------------ | | 337| 0|#define ZXC_PAD_SIZE 32 ------------------ 208| | 209| | /* opt_scratch (level >= ZXC_LEVEL_DENSITY): the optimal parser's DP arrays, 210| | * reused transiently as package-merge scratch by the code-length builder, 211| | * so sized to the larger demand. Keep in sync with zxc_estimate_cctx_size() 212| | * and its consumer in zxc_compress.c. */ 213| 0| if (level >= ZXC_LEVEL_DENSITY) { ------------------ | Branch (213:13): [True: 0, False: 0] ------------------ 214| 0| const size_t sz_dp = ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint32_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 215| 0| const size_t sz_pl = ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint16_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 216| 0| const size_t sz_po = ZXC_ALIGN_CL((chunk_size + 1) * sizeof(uint16_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 217| 0| const size_t n_bm_words = ZXC_BITMAP_WORDS(chunk_size + 1); ------------------ | | 362| 0|#define ZXC_BITMAP_WORDS(n_bits) (((n_bits) + 63) / 64) ------------------ 218| 0| const size_t sz_bm = ZXC_ALIGN_CL(n_bm_words * sizeof(uint64_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 219| 0| const size_t dp_needed = sz_dp + sz_pl + sz_po + sz_bm; 220| 0| layout.sz_opt = 221| 0| (dp_needed > ZXC_HUF_BUILD_SCRATCH_SIZE) ? dp_needed : ZXC_HUF_BUILD_SCRATCH_SIZE; ------------------ | | 841| 0| ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 834| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 589| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 842| 0| sizeof(zxc_huf_pm_item_t) + \ | | 843| 0| 8U + (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * sizeof(int) + 8U + \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | 844| 0| (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 834| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 589| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 845| 0| sizeof(zxc_huf_pm_frame_t)) ------------------ (dp_needed > ZXC_HUF_BUILD_SCRATCH_SIZE) ? dp_needed : ZXC_HUF_BUILD_SCRATCH_SIZE; ------------------ | | 841| 0| ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | ((size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 834| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 589| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 842| 0| sizeof(zxc_huf_pm_item_t) + \ | | 843| 0| 8U + (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * sizeof(int) + 8U + \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | 844| 0| (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 582| 0|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 | | ------------------ | | (size_t)ZXC_HUF_MAX_CODE_LEN_ULTRA * (size_t)ZXC_HUF_PM_LEVEL_BOUND * \ | | ------------------ | | | | 834| 0|#define ZXC_HUF_PM_LEVEL_BOUND (2 * ZXC_HUF_NUM_SYMBOLS) | | | | ------------------ | | | | | | 589| 0|#define ZXC_HUF_NUM_SYMBOLS 256 | | | | ------------------ | | ------------------ | | 845| 0| sizeof(zxc_huf_pm_frame_t)) ------------------ | Branch (221:17): [True: 0, False: 0] ------------------ 222| 0| } 223| | 224| 0| layout.off_hash_pos = layout.total; 225| 0| layout.total += ZXC_ALIGN_CL(layout.sz_hash_pos); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 226| 0| layout.off_hash_tags = layout.total; 227| 0| layout.total += ZXC_ALIGN_CL(layout.sz_hash_tags); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 228| 0| layout.off_chain = layout.total; 229| 0| layout.total += ZXC_ALIGN_CL(sz_chain); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 230| 0| layout.off_seq_union = layout.total; 231| 0| layout.total += ZXC_ALIGN_CL(sz_seq_union); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 232| 0| layout.off_extras = layout.total; 233| 0| layout.total += ZXC_ALIGN_CL(sz_extras); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 234| 0| layout.off_lit_cctx = layout.total; 235| 0| layout.total += ZXC_ALIGN_CL(sz_lit); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 236| | /* opt_scratch is appended last so it is absent for levels 1..5 (zero 237| | * waste on the common path) and only inflates the workspace at level 6. */ 238| 0| if (layout.sz_opt) { ------------------ | Branch (238:13): [True: 0, False: 0] ------------------ 239| 0| layout.off_opt = layout.total; 240| 0| layout.total += ZXC_ALIGN_CL(layout.sz_opt); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 241| 0| } 242| 0| } 243| | 244| | /* [dict | data] concat scratch (dict only). Compress chunk_size already 245| | * spans [dict | block]; decompress prepends dict to a (chunk + PAD) region. */ 246| 12.9k| if (dict_size > 0) { ------------------ | Branch (246:9): [True: 0, False: 12.9k] ------------------ 247| 0| layout.sz_dict = (mode == 1) ? (chunk_size + ZXC_DECOMPRESS_TAIL_PAD) ------------------ | | 349| 0|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 0|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ | Branch (247:26): [True: 0, False: 0] ------------------ 248| 0| : (dict_size + chunk_size + ZXC_DECOMPRESS_TAIL_PAD); ------------------ | | 349| 0|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 0|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 249| 0| layout.off_dict = layout.total; 250| 0| layout.total += ZXC_ALIGN_CL(layout.sz_dict); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 251| 0| layout.off_dict_huf = layout.total; 252| 0| layout.total += ZXC_ALIGN_CL(sizeof(zxc_dict_huf_state_t)); ------------------ | | 355| 0|#define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ | | #define ZXC_ALIGN_CL(x) (((x) + ZXC_ALIGNMENT_MASK) & ~(size_t)ZXC_ALIGNMENT_MASK) | | ------------------ | | | | 353| 0|#define ZXC_ALIGNMENT_MASK (ZXC_CACHE_LINE_SIZE - 1) | | | | ------------------ | | | | | | 351| 0|#define ZXC_CACHE_LINE_SIZE 64 | | | | ------------------ | | ------------------ ------------------ 253| 0| } 254| 12.9k| return layout; 255| 12.9k|} zxc_common.c:zxc_cctx_max_seq: 144| 15.2k|static ZXC_ALWAYS_INLINE size_t zxc_cctx_max_seq(const size_t chunk_size) { 145| 15.2k| return chunk_size / ZXC_LZ_MIN_MATCH_LEN + 16; ------------------ | | 508| 15.2k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 146| 15.2k|} zxc_common.c:zxc_dctx_entropy_sizes: 156| 2.30k| size_t* RESTRICT sz_pivco) { 157| 2.30k| *sz_tok = zxc_cctx_max_seq(chunk_size) + ZXC_PAD_SIZE; ------------------ | | 337| 2.30k|#define ZXC_PAD_SIZE 32 ------------------ 158| 2.30k| *sz_pivco = chunk_size + ZXC_PIVCO_SCRATCH_PAD; ------------------ | | 1598| 2.30k|#define ZXC_PIVCO_SCRATCH_PAD 32 ------------------ 159| 2.30k|} zxc_decompress_chunk_wrapper_default: 1532| 47.7k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 1533| 47.7k| return zxc_decompress_chunk_wrapper_body(ctx, src, src_sz, dst, dst_cap, 0, 0); 1534| 47.7k|} zxc_decompress.c:zxc_decompress_chunk_wrapper_body: 1468| 47.7k| uint8_t* RESTRICT dst, const size_t dst_cap, const int has_dict, const int safe) { 1469| 47.7k| if (UNLIKELY(src_sz < ZXC_BLOCK_HEADER_SIZE)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 47.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 47.7k] | | ------------------ ------------------ 1470| | 1471| 47.7k| const uint8_t type = src[0]; 1472| 47.7k| const uint32_t comp_sz = zxc_le32(src + 3); 1473| 47.7k| const int has_crc = ctx->checksum_enabled; 1474| | 1475| | // Check bounds: Header + Body + Checksum(if any) 1476| 47.7k| const size_t expected_sz = 1477| 47.7k| (size_t)ZXC_BLOCK_HEADER_SIZE + comp_sz + (has_crc ? ZXC_BLOCK_CHECKSUM_SIZE : 0); ------------------ | | 397| 47.7k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ (size_t)ZXC_BLOCK_HEADER_SIZE + comp_sz + (has_crc ? ZXC_BLOCK_CHECKSUM_SIZE : 0); ------------------ | | 399| 0|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (1477:52): [True: 0, False: 47.7k] ------------------ 1478| 47.7k| if (UNLIKELY(src_sz < expected_sz)) return ZXC_ERROR_SRC_TOO_SMALL; ------------------ | | 151| 47.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 240, False: 47.5k] | | ------------------ ------------------ 1479| | 1480| 47.5k| const uint8_t* data = src + ZXC_BLOCK_HEADER_SIZE; ------------------ | | 397| 47.5k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1481| | 1482| 47.5k| if (has_crc) { ------------------ | Branch (1482:9): [True: 0, False: 47.5k] ------------------ 1483| 0| const uint32_t stored = zxc_le32(data + comp_sz); 1484| 0| const uint32_t calc = zxc_checksum(data, comp_sz, ZXC_CHECKSUM_RAPIDHASH); ------------------ | | 424| 0|#define ZXC_CHECKSUM_RAPIDHASH 0 ------------------ 1485| 0| if (UNLIKELY(stored != calc)) return ZXC_ERROR_BAD_CHECKSUM; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1486| 0| } 1487| | 1488| 47.5k| int decoded_sz = ZXC_ERROR_BAD_BLOCK_TYPE; 1489| | 1490| 47.5k| switch (type) { 1491| 42.5k| case ZXC_BLOCK_GLO: ------------------ | Branch (1491:9): [True: 42.5k, False: 4.97k] ------------------ 1492| 42.5k| decoded_sz = safe ? zxc_decode_block_glo_safe(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1492:26): [True: 0, False: 42.5k] ------------------ 1493| 42.5k| : has_dict ? zxc_decode_block_glo_dict(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1493:28): [True: 0, False: 42.5k] ------------------ 1494| 42.5k| : zxc_decode_block_glo(ctx, data, comp_sz, dst, dst_cap); 1495| 42.5k| break; 1496| 2.39k| case ZXC_BLOCK_GHI: ------------------ | Branch (1496:9): [True: 2.39k, False: 45.1k] ------------------ 1497| 2.39k| decoded_sz = safe ? zxc_decode_block_ghi_safe(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1497:26): [True: 0, False: 2.39k] ------------------ 1498| 2.39k| : has_dict ? zxc_decode_block_ghi_dict(ctx, data, comp_sz, dst, dst_cap) ------------------ | Branch (1498:28): [True: 0, False: 2.39k] ------------------ 1499| 2.39k| : zxc_decode_block_ghi(ctx, data, comp_sz, dst, dst_cap); 1500| 2.39k| break; 1501| 2.56k| case ZXC_BLOCK_RAW: ------------------ | Branch (1501:9): [True: 2.56k, False: 44.9k] ------------------ 1502| | // For RAW blocks, comp_sz == raw_sz (uncompressed data stored as-is) 1503| 2.56k| if (UNLIKELY(comp_sz > dst_cap)) return ZXC_ERROR_DST_TOO_SMALL; ------------------ | | 151| 2.56k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1, False: 2.56k] | | ------------------ ------------------ 1504| 2.56k| ZXC_MEMCPY(dst, data, comp_sz); ------------------ | | 173| 2.56k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1505| 2.56k| decoded_sz = (int)comp_sz; 1506| 2.56k| break; 1507| 0| case ZXC_BLOCK_EOF: ------------------ | Branch (1507:9): [True: 0, False: 47.5k] ------------------ 1508| | // EOF should be handled by the dispatcher, not here 1509| 0| return ZXC_ERROR_CORRUPT_DATA; 1510| 10| default: ------------------ | Branch (1510:9): [True: 10, False: 47.4k] ------------------ 1511| 10| return ZXC_ERROR_BAD_BLOCK_TYPE; 1512| 47.5k| } 1513| | 1514| 47.4k| return decoded_sz; 1515| 47.5k|} zxc_decompress.c:zxc_decode_block_glo_impl: 689| 64.2k| const int has_dict, const int tok_entropy) { 690| 64.2k| zxc_gnr_header_t gh; 691| | 692| | /* Constant 0 when !has_dict, so `d_floor` folds to `dst`. */ 693| 64.2k| const size_t dict_size = has_dict ? ctx->dict_size : 0; ------------------ | Branch (693:30): [True: 0, False: 64.2k] ------------------ 694| 64.2k| zxc_section_desc_t desc[ZXC_GLO_SECTIONS]; 695| | 696| 64.2k| if (UNLIKELY(zxc_read_glo_header_and_desc(src, src_size, &gh, desc) != ZXC_OK)) ------------------ | | 151| 64.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 12, False: 64.2k] | | ------------------ ------------------ 697| 12| return ZXC_ERROR_BAD_HEADER; 698| | 699| | /* Entropy-coded tokens (level 7): tail-call the dedicated instantiation 700| | * before any section work - it restarts from the header, so only the parse 701| | * above is duplicated. Flags are constant per instantiation, so exactly one 702| | * call survives and t_ptr keeps a single provenance below. */ 703| 64.2k| if (!tok_entropy && UNLIKELY(gh.enc_litlen == ZXC_SECTION_ENCODING_HUFFMAN)) { ------------------ | | 151| 42.5k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 21.7k, False: 20.7k] | | ------------------ ------------------ | Branch (703:9): [True: 42.5k, False: 21.7k] ------------------ 704| 21.7k| if (safe) return zxc_decode_block_glo_entropy_safe(ctx, src, src_size, dst, dst_capacity); ------------------ | Branch (704:13): [True: 0, False: 21.7k] ------------------ 705| 21.7k| if (has_dict) ------------------ | Branch (705:13): [True: 0, False: 21.7k] ------------------ 706| 0| return zxc_decode_block_glo_entropy_dict(ctx, src, src_size, dst, dst_capacity); 707| 21.7k| return zxc_decode_block_glo_entropy(ctx, src, src_size, dst, dst_capacity); 708| 21.7k| } 709| | 710| 42.5k| const uint8_t* p_data = 711| 42.5k| src + ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 401| 42.5k|#define ZXC_GLO_HEADER_BINARY_SIZE 16 ------------------ src + ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 419| 42.5k|#define ZXC_GLO_SECTIONS 4 ------------------ src + ZXC_GLO_HEADER_BINARY_SIZE + ZXC_GLO_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 42.5k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 712| 42.5k| const uint8_t* p_curr = p_data; 713| | 714| | // --- Literal Stream Setup --- 715| 42.5k| const uint8_t* l_ptr; 716| 42.5k| const uint8_t* l_end; 717| 42.5k| uint8_t* rle_buf = NULL; 718| | 719| 42.5k| size_t lit_stream_size = (size_t)(desc[0].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 42.5k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 720| | /* Decoded size of an encoded literal section (RAW ignores it). */ 721| 42.5k| const size_t required_size = (size_t)(desc[0].sizes >> 32); 722| | 723| 42.5k| if (gh.enc_lit == ZXC_SECTION_ENCODING_HUFFMAN || ------------------ | Branch (723:9): [True: 2.46k, False: 40.0k] ------------------ 724| 40.0k| gh.enc_lit == ZXC_SECTION_ENCODING_HUFFMAN_DICT) { ------------------ | Branch (724:9): [True: 293, False: 39.7k] ------------------ 725| 2.75k| if (UNLIKELY(lit_stream_size > (size_t)(src + src_size - p_curr))) ------------------ | | 151| 2.75k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 96, False: 2.65k] | | ------------------ ------------------ 726| 96| return ZXC_ERROR_CORRUPT_DATA; 727| 2.65k| if (required_size == 0) { ------------------ | Branch (727:13): [True: 434, False: 2.22k] ------------------ 728| 434| l_ptr = p_curr; 729| 434| l_end = p_curr; 730| 2.22k| } else { 731| 2.22k| if (UNLIKELY(required_size > dst_capacity || required_size > SIZE_MAX - ZXC_PAD_SIZE)) ------------------ | | 151| 4.27k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 171, False: 2.05k] | | | Branch (151:42): [True: 171, False: 2.05k] | | | Branch (151:42): [True: 0, False: 2.05k] | | ------------------ ------------------ 732| 171| return ZXC_ERROR_DST_TOO_SMALL; 733| 2.05k| const int rc = 734| 2.05k| (gh.enc_lit == ZXC_SECTION_ENCODING_HUFFMAN) ------------------ | Branch (734:17): [True: 2.05k, False: 3] ------------------ 735| 2.05k| ? zxc_decode_lit_pivco(ctx, p_curr, lit_stream_size, required_size) 736| 2.05k| : zxc_decode_lit_pivco_dict(ctx, p_curr, lit_stream_size, required_size); 737| 2.05k| if (UNLIKELY(rc != ZXC_OK)) return rc; ------------------ | | 151| 2.05k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 138, False: 1.91k] | | ------------------ ------------------ 738| 1.91k| l_ptr = ctx->lit_buffer; 739| 1.91k| l_end = ctx->lit_buffer + required_size; 740| 1.91k| } 741| 39.7k| } else if (gh.enc_lit == ZXC_SECTION_ENCODING_RLE) { ------------------ | Branch (741:16): [True: 34.3k, False: 5.43k] ------------------ 742| 34.3k| if (required_size > 0) { ------------------ | Branch (742:13): [True: 34.0k, False: 268] ------------------ 743| 34.0k| if (UNLIKELY(required_size > dst_capacity || required_size > SIZE_MAX - ZXC_PAD_SIZE)) ------------------ | | 151| 67.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 160, False: 33.9k] | | | Branch (151:42): [True: 160, False: 33.9k] | | | Branch (151:42): [True: 0, False: 33.9k] | | ------------------ ------------------ 744| 160| return ZXC_ERROR_DST_TOO_SMALL; 745| 33.9k| const size_t alloc_size = required_size + ZXC_PAD_SIZE; ------------------ | | 337| 33.9k|#define ZXC_PAD_SIZE 32 ------------------ 746| | 747| | /* lit_buffer is pre-allocated to chunk_size + ZXC_PAD_SIZE by 748| | * zxc_cctx_init (mode == 0).*/ 749| 33.9k| if (UNLIKELY(ctx->lit_buffer_cap < alloc_size)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 33.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 6, False: 33.8k] | | ------------------ ------------------ 750| | 751| 33.8k| rle_buf = ctx->lit_buffer; 752| 33.8k| if (UNLIKELY(!rle_buf || lit_stream_size > (size_t)(src + src_size - p_curr))) ------------------ | | 151| 67.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 134, False: 33.7k] | | | Branch (151:42): [True: 0, False: 33.8k] | | | Branch (151:42): [True: 134, False: 33.7k] | | ------------------ ------------------ 753| 134| return ZXC_ERROR_CORRUPT_DATA; 754| | 755| 33.7k| const uint8_t* r_ptr = p_curr; 756| 33.7k| const uint8_t* r_end = r_ptr + lit_stream_size; 757| 33.7k| uint8_t* w_ptr = rle_buf; 758| 33.7k| const uint8_t* const w_end = rle_buf + required_size; 759| | 760| 211k| while (r_ptr < r_end && w_ptr < w_end) { ------------------ | Branch (760:20): [True: 209k, False: 1.87k] | Branch (760:37): [True: 177k, False: 31.8k] ------------------ 761| 177k| uint8_t token = *r_ptr++; 762| 177k| if (LIKELY(!(token & ZXC_LIT_RLE_FLAG))) { ------------------ | | 145| 177k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 120k, False: 57.0k] | | ------------------ ------------------ 763| | // Raw copy (most common path): use ZXC_PAD_SIZE-byte wild copies 764| | // token is 7-bit (0-127), so len is 1-128 bytes 765| 120k| const uint32_t len = (uint32_t)token + 1; 766| 120k| if (UNLIKELY(w_ptr + len > w_end || r_ptr + len > r_end)) ------------------ | | 151| 241k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 60, False: 120k] | | | Branch (151:42): [True: 9, False: 120k] | | | Branch (151:42): [True: 51, False: 120k] | | ------------------ ------------------ 767| 60| return ZXC_ERROR_CORRUPT_DATA; 768| | 769| | // Destination has ZXC_PAD_SIZE bytes of safe overrun space. 770| | // Source may not - check before wild copy. 771| | // Fast path: source has ZXC_PAD_SIZE-byte read headroom (most common) 772| 120k| if (LIKELY(r_ptr + ZXC_PAD_SIZE <= r_end)) { ------------------ | | 145| 120k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 1.89k, False: 118k] | | ------------------ ------------------ 773| | // Single 32-byte copy covers len <= ZXC_PAD_SIZE (most tokens) 774| 1.89k| zxc_copy32(w_ptr, r_ptr); 775| | 776| 1.89k| if (UNLIKELY(len > ZXC_PAD_SIZE)) { ------------------ | | 151| 1.89k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 216, False: 1.67k] | | ------------------ ------------------ 777| | // Unroll: max len=128, so max 4 copies total 778| | // Use unconditional stores with overlap - faster than branches 779| 216| if (len <= 2 * ZXC_PAD_SIZE) { ------------------ | | 337| 216|#define ZXC_PAD_SIZE 32 ------------------ | Branch (779:33): [True: 115, False: 101] ------------------ 780| 115| zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); ------------------ | | 337| 115|#define ZXC_PAD_SIZE 32 ------------------ zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); ------------------ | | 337| 115|#define ZXC_PAD_SIZE 32 ------------------ 781| 115| } else if (len <= 3 * ZXC_PAD_SIZE) { ------------------ | | 337| 101|#define ZXC_PAD_SIZE 32 ------------------ | Branch (781:40): [True: 53, False: 48] ------------------ 782| 53| zxc_copy32(w_ptr + ZXC_PAD_SIZE, r_ptr + ZXC_PAD_SIZE); ------------------ | | 337| 53|#define ZXC_PAD_SIZE 32 ------------------ zxc_copy32(w_ptr + ZXC_PAD_SIZE, r_ptr + ZXC_PAD_SIZE); ------------------ | | 337| 53|#define ZXC_PAD_SIZE 32 ------------------ 783| 53| zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); ------------------ | | 337| 53|#define ZXC_PAD_SIZE 32 ------------------ zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); ------------------ | | 337| 53|#define ZXC_PAD_SIZE 32 ------------------ 784| 53| } else { 785| 48| zxc_copy32(w_ptr + ZXC_PAD_SIZE, r_ptr + ZXC_PAD_SIZE); ------------------ | | 337| 48|#define ZXC_PAD_SIZE 32 ------------------ zxc_copy32(w_ptr + ZXC_PAD_SIZE, r_ptr + ZXC_PAD_SIZE); ------------------ | | 337| 48|#define ZXC_PAD_SIZE 32 ------------------ 786| 48| zxc_copy32(w_ptr + 2 * ZXC_PAD_SIZE, r_ptr + 2 * ZXC_PAD_SIZE); ------------------ | | 337| 48|#define ZXC_PAD_SIZE 32 ------------------ zxc_copy32(w_ptr + 2 * ZXC_PAD_SIZE, r_ptr + 2 * ZXC_PAD_SIZE); ------------------ | | 337| 48|#define ZXC_PAD_SIZE 32 ------------------ 787| 48| zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); ------------------ | | 337| 48|#define ZXC_PAD_SIZE 32 ------------------ zxc_copy32(w_ptr + len - ZXC_PAD_SIZE, r_ptr + len - ZXC_PAD_SIZE); ------------------ | | 337| 48|#define ZXC_PAD_SIZE 32 ------------------ 788| 48| } 789| 216| } 790| 118k| } else { 791| | // Near end of source: safe copy (rare cold path) 792| 118k| ZXC_MEMCPY(w_ptr, r_ptr, len); ------------------ | | 173| 118k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 793| 118k| } 794| | 795| 120k| w_ptr += len; 796| 120k| r_ptr += len; 797| 120k| } else { 798| | // RLE run: fill with single byte 799| 57.0k| const uint32_t len = (token & ZXC_LIT_LEN_MASK) + 4; ------------------ | | 481| 57.0k|#define ZXC_LIT_LEN_MASK (ZXC_LIT_RLE_FLAG - 1) | | ------------------ | | | | 479| 57.0k|#define ZXC_LIT_RLE_FLAG 0x80U | | ------------------ ------------------ 800| 57.0k| if (UNLIKELY(w_ptr + len > w_end || r_ptr >= r_end)) ------------------ | | 151| 114k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 23, False: 57.0k] | | | Branch (151:42): [True: 15, False: 57.0k] | | | Branch (151:42): [True: 8, False: 57.0k] | | ------------------ ------------------ 801| 23| return ZXC_ERROR_CORRUPT_DATA; 802| 57.0k| ZXC_MEMSET(w_ptr, *r_ptr++, len); ------------------ | | 178| 57.0k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 803| 57.0k| w_ptr += len; 804| 57.0k| } 805| 177k| } 806| 33.6k| if (UNLIKELY(w_ptr != w_end)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 33.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 42, False: 33.6k] | | ------------------ ------------------ 807| 33.6k| l_ptr = rle_buf; 808| 33.6k| l_end = rle_buf + required_size; 809| 33.6k| } else { 810| 268| l_ptr = p_curr; 811| 268| l_end = p_curr; 812| 268| } 813| 34.3k| } else if (gh.enc_lit == ZXC_SECTION_ENCODING_RAW) { ------------------ | Branch (813:16): [True: 5.41k, False: 27] ------------------ 814| 5.41k| l_ptr = p_curr; 815| 5.41k| l_end = p_curr + lit_stream_size; 816| 5.41k| } else { 817| 27| return ZXC_ERROR_CORRUPT_DATA; 818| 27| } 819| | 820| 41.6k| p_curr += lit_stream_size; 821| | 822| | // --- Stream Pointers & Validation --- 823| 41.6k| const size_t sz_tokens = (size_t)(desc[1].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 41.6k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 824| 41.6k| const size_t sz_offsets = (size_t)(desc[2].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 41.6k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 825| 41.6k| const size_t sz_extras = (size_t)(desc[3].sizes & ZXC_SECTION_SIZE_MASK); ------------------ | | 417| 41.6k|#define ZXC_SECTION_SIZE_MASK 0xFFFFFFFFU ------------------ 826| | 827| | // Validate stream sizes match sequence count (early rejection of malformed data) 828| 41.6k| const uint64_t expected_off_size = 829| 41.6k| GLO_OFF8 ? (uint64_t)gh.n_sequences : (uint64_t)gh.n_sequences * 2; ------------------ | | 436| 41.6k|#define GLO_OFF8 (gh.enc_off == 1) | | ------------------ | | | Branch (436:18): [True: 32.0k, False: 9.57k] | | ------------------ ------------------ 830| | 831| | /* Offsets/extras follow the on-disk token SECTION; sz_tokens is its size 832| | * (== n_sequences when RAW, the Huffman payload size when enc_litlen set). */ 833| 41.6k| const uint8_t* o_ptr = p_curr + sz_tokens; 834| 41.6k| const uint8_t* e_ptr = o_ptr + sz_offsets; 835| 41.6k| const uint8_t* const e_end = e_ptr + sz_extras; // For vbyte overflow detection 836| | 837| | // Validate streams don't overflow source buffer + match sequence count. 838| 41.6k| if (UNLIKELY((e_end != src + src_size) || (uint64_t)sz_offsets < expected_off_size)) ------------------ | | 151| 83.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 249, False: 41.4k] | | | Branch (151:42): [True: 192, False: 41.4k] | | | Branch (151:42): [True: 57, False: 41.4k] | | ------------------ ------------------ 839| 249| return ZXC_ERROR_CORRUPT_DATA; 840| | 841| 41.4k| if (UNLIKELY(gh.enc_lit == ZXC_SECTION_ENCODING_RAW && ------------------ | | 151| 46.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 541, False: 40.8k] | | | Branch (151:42): [True: 5.36k, False: 36.0k] | | | Branch (151:42): [True: 541, False: 4.82k] | | ------------------ ------------------ 842| 41.4k| (size_t)(src + src_size - l_end) < ZXC_PAD_SIZE)) { 843| 541| const int rc = zxc_stage_raw_literals(ctx, &l_ptr, &l_end); 844| 541| if (UNLIKELY(rc != ZXC_OK)) return rc; ------------------ | | 151| 541|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 541] | | ------------------ ------------------ 845| 541| } 846| | 847| 41.4k| const uint8_t* RESTRICT t_ptr; 848| 41.4k| if (!tok_entropy) { ------------------ | Branch (848:9): [True: 20.1k, False: 21.2k] ------------------ 849| | /* enc_litlen == 2 was re-routed right after the header parse. */ 850| 20.1k| if (UNLIKELY(sz_tokens < gh.n_sequences)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 20.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 4, False: 20.1k] | | ------------------ ------------------ 851| 20.1k| t_ptr = p_curr; 852| 21.2k| } else { 853| 21.2k| if (UNLIKELY(gh.enc_litlen != ZXC_SECTION_ENCODING_HUFFMAN)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 21.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 21.2k] | | ------------------ ------------------ 854| 21.2k| const int rc = zxc_decode_tok_pivco(ctx, p_curr, sz_tokens, gh.n_sequences); 855| 21.2k| if (UNLIKELY(rc != ZXC_OK)) return rc; ------------------ | | 151| 21.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 65, False: 21.2k] | | ------------------ ------------------ 856| 21.2k| t_ptr = ctx->tok_buffer; 857| 21.2k| } 858| | 859| 41.3k| uint8_t* d_ptr = dst; 860| 41.3k| const uint8_t* const d_end = dst + dst_capacity; 861| | // Lowest address a match may reach: the dictionary prefix if any, else dst. 862| 41.3k| const uint8_t* const d_floor = dst - dict_size; 863| | // Destination safe margin for 4x loop: max output without varint extension. 864| | // ll_max = 14, ml_max = 14 + 5 = 19, per-seq = 33, 4x = 132. 865| | // Add ZXC_PAD_SIZE (32) for the wild zxc_copy32 overshoot + 4 safety = 168. 866| 41.3k| const uint8_t* const d_end_safe = d_end - (132 + ZXC_PAD_SIZE + 4); ------------------ | | 337| 41.3k|#define ZXC_PAD_SIZE 32 ------------------ 867| | 868| | // Literal margin for the 4x loops: without a varint, ll <= 14 per sequence, 869| | // so 4 * 14 = 56. Past that margin only the cold varint path checks l_ptr. 870| 41.3k| const size_t glo_sz_lit = (size_t)(l_end - l_ptr); 871| 41.3k| const size_t glo_margin_4x = 4 * (ZXC_TOKEN_LL_MASK - 1); // 56 ------------------ | | 454| 41.3k|#define ZXC_TOKEN_LL_MASK ((1U << ZXC_TOKEN_LIT_BITS) - 1) | | ------------------ | | | | 450| 41.3k|#define ZXC_TOKEN_LIT_BITS 4 | | ------------------ ------------------ 872| 41.3k| const size_t glo_margin_1x = ZXC_TOKEN_LL_MASK - 1; // 14 ------------------ | | 454| 41.3k|#define ZXC_TOKEN_LL_MASK ((1U << ZXC_TOKEN_LIT_BITS) - 1) | | ------------------ | | | | 450| 41.3k|#define ZXC_TOKEN_LIT_BITS 4 | | ------------------ ------------------ 873| 41.3k| const uint8_t* const l_end_safe_4x = 874| 41.3k| (glo_sz_lit > glo_margin_4x) ? l_end - glo_margin_4x : l_ptr; ------------------ | Branch (874:9): [True: 34.9k, False: 6.42k] ------------------ 875| 41.3k| const uint8_t* const l_end_safe_1x = 876| 41.3k| (glo_sz_lit > glo_margin_1x) ? l_end - glo_margin_1x : l_ptr; ------------------ | Branch (876:9): [True: 36.9k, False: 4.36k] ------------------ 877| | 878| 41.3k| uint32_t n_seq = gh.n_sequences; 879| | 880| | // Offsets only need checking until the output is wider than the widest offset 881| | // the format can encode (256 for 1-byte offsets, 65536 for 2-byte); past that 882| | // none can reach below d_floor. A dictionary counts as already written, so a 883| | // large one puts d_bounds at dst and skips the SAFE loop. 884| 41.3k| const size_t off_span = GLO_OFF8 ? (1U << 8) : (1U << 16); ------------------ | | 436| 41.3k|#define GLO_OFF8 (gh.enc_off == 1) | | ------------------ | | | Branch (436:18): [True: 32.0k, False: 9.32k] | | ------------------ ------------------ 885| 41.3k| const size_t span = (dict_size >= off_span) ? 0 : off_span - dict_size; ------------------ | Branch (885:25): [True: 0, False: 41.3k] ------------------ 886| 41.3k| const uint8_t* const d_bounds = (span >= dst_capacity) ? d_end : dst + span; ------------------ | Branch (886:37): [True: 4.10k, False: 37.2k] ------------------ 887| | 888| | // --- SAFE Loop: offset validation until d_bounds (4x unroll) --- 889| 41.3k| if (safe) { ------------------ | Branch (889:9): [True: 0, False: 41.3k] ------------------ 890| | /* SAFE variant: save per-batch state so overflow can rollback. */ 891| 0| while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (891:16): [True: 0, False: 0] | Branch (891:30): [True: 0, False: 0] | Branch (891:52): [True: 0, False: 0] | Branch (891:77): [True: 0, False: 0] ------------------ 892| 0| const uint8_t* const t_save = t_ptr; 893| 0| const uint8_t* const o_save = o_ptr; 894| 0| const uint8_t* const e_save = e_ptr; 895| 0| uint8_t* const d_save = d_ptr; 896| 0| const uint8_t* const l_save = l_ptr; 897| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x); ------------------ | | 479| 0| do { \ | | 480| 0| uint32_t tokens = zxc_le32(t_ptr); \ | | 481| 0| t_ptr += sizeof(uint32_t); \ | | 482| 0| uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 483| 0| off4 = ZXC_LZ_OFFSET_BIAS; \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 484| 0| if (GLO_OFF8) { \ | | ------------------ | | | | 436| 0|#define GLO_OFF8 (gh.enc_off == 1) | | | | ------------------ | | | | | Branch (436:18): [True: 0, False: 0] | | | | ------------------ | | ------------------ | | 485| 0| uint32_t offsets = zxc_le32(o_ptr); \ | | 486| 0| o_ptr += sizeof(uint32_t); \ | | 487| 0| off1 += offsets & 0xFF; \ | | 488| 0| off2 += (offsets >> 8) & 0xFF; \ | | 489| 0| off3 += (offsets >> 16) & 0xFF; \ | | 490| 0| off4 += (offsets >> 24) & 0xFF; \ | | 491| 0| } else { \ | | 492| 0| uint64_t offsets = zxc_le64(o_ptr); \ | | 493| 0| o_ptr += sizeof(uint64_t); \ | | 494| 0| off1 += (uint32_t)(offsets & 0xFFFF); \ | | 495| 0| off2 += (uint32_t)((offsets >> 16) & 0xFFFF); \ | | 496| 0| off3 += (uint32_t)((offsets >> 32) & 0xFFFF); \ | | 497| 0| off4 += (uint32_t)((offsets >> 48) & 0xFFFF); \ | | 498| 0| } \ | | 499| 0| DECODE_GLO_SEQ((tokens & 0x0F0) >> 4, (tokens & 0x00F), off1, \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 897| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 500| 0| ((tokens >> 12) & 0xF) + ((tokens >> 20) & 0xF) + (tokens >> 28), 3U, \ | | 501| 0| DECODE, ON_FAIL); \ | | 502| 0| DECODE_GLO_SEQ((tokens & 0x0F000) >> 12, (tokens & 0x00F00) >> 8, off2, \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 897| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 503| 0| ((tokens >> 20) & 0xF) + (tokens >> 28), 2U, DECODE, ON_FAIL); \ | | 504| 0| DECODE_GLO_SEQ((tokens & 0x0F00000) >> 20, (tokens & 0x00F0000) >> 16, off3, \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 897| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 505| 0| (tokens >> 28), 1U, DECODE, ON_FAIL); \ | | 506| 0| DECODE_GLO_SEQ((tokens >> 28), (tokens >> 24) & 0x0F, off4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 897| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 507| 0| n_seq -= 4; \ | | 508| 0| } while (0) | | ------------------ | | | Branch (508:14): [Folded, False: 0] | | ------------------ ------------------ 898| 0| continue; 899| | 900| 0| rollback_safe_4x: 901| 0| t_ptr = t_save; 902| 0| o_ptr = o_save; 903| 0| e_ptr = e_save; 904| 0| d_ptr = d_save; 905| 0| l_ptr = l_save; 906| 0| break; 907| 0| } 908| 41.3k| } else { 909| 154k| while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (909:16): [True: 144k, False: 10.1k] | Branch (909:30): [True: 143k, False: 16] | Branch (909:52): [True: 140k, False: 3.46k] | Branch (909:77): [True: 113k, False: 27.3k] ------------------ 910| 113k| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW); ------------------ | | 479| 113k| do { \ | | 480| 113k| uint32_t tokens = zxc_le32(t_ptr); \ | | 481| 113k| t_ptr += sizeof(uint32_t); \ | | 482| 113k| uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 113k|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 113k|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 113k|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 483| 113k| off4 = ZXC_LZ_OFFSET_BIAS; \ | | ------------------ | | | | 527| 113k|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 484| 113k| if (GLO_OFF8) { \ | | ------------------ | | | | 436| 113k|#define GLO_OFF8 (gh.enc_off == 1) | | | | ------------------ | | | | | Branch (436:18): [True: 96.2k, False: 16.9k] | | | | ------------------ | | ------------------ | | 485| 96.2k| uint32_t offsets = zxc_le32(o_ptr); \ | | 486| 96.2k| o_ptr += sizeof(uint32_t); \ | | 487| 96.2k| off1 += offsets & 0xFF; \ | | 488| 96.2k| off2 += (offsets >> 8) & 0xFF; \ | | 489| 96.2k| off3 += (offsets >> 16) & 0xFF; \ | | 490| 96.2k| off4 += (offsets >> 24) & 0xFF; \ | | 491| 96.2k| } else { \ | | 492| 16.9k| uint64_t offsets = zxc_le64(o_ptr); \ | | 493| 16.9k| o_ptr += sizeof(uint64_t); \ | | 494| 16.9k| off1 += (uint32_t)(offsets & 0xFFFF); \ | | 495| 16.9k| off2 += (uint32_t)((offsets >> 16) & 0xFFFF); \ | | 496| 16.9k| off3 += (uint32_t)((offsets >> 32) & 0xFFFF); \ | | 497| 16.9k| off4 += (uint32_t)((offsets >> 48) & 0xFFFF); \ | | 498| 16.9k| } \ | | 499| 113k| DECODE_GLO_SEQ((tokens & 0x0F0) >> 4, (tokens & 0x00F), off1, \ | | ------------------ | | | | 453| 113k| do { \ | | | | 454| 113k| uint64_t ll = (LL); \ | | | | 455| 113k| uint64_t ml = (ML); \ | | | | 456| 113k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 113k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 10.9k, False: 102k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 10.9k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 10.9k| const uint64_t reserve = (RESERVE); \ | | | | 459| 10.9k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 21.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 33, False: 10.9k] | | | | | | | Branch (151:42): [True: 33, False: 10.9k] | | | | | | | Branch (151:42): [True: 0, False: 10.9k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 10.9k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 10.9k| ON_FAIL; \ | | | | 462| 10.9k| } \ | | | | 463| 113k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 113k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 36.9k, False: 76.1k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 36.9k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 36.9k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 36.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 12, False: 36.9k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 36.9k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 36.9k| (size_t)(d_end - d_ptr))) \ | | | | 468| 36.9k| ON_FAIL; \ | | | | 469| 36.9k| } \ | | | | 470| 113k| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 113k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 113k| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 910| 113k| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 405| 113k| do { \ | | | | | | | | 406| 113k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 113k| l_ptr += ll; \ | | | | | | | | 408| 113k| d_ptr += ll; \ | | | | | | | | 409| 113k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 113k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 38, False: 113k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 113k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 113k| d_ptr += ml; \ | | | | | | | | 412| 113k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 113k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 113k| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 113k] | | | | ------------------ | | ------------------ | | 500| 113k| ((tokens >> 12) & 0xF) + ((tokens >> 20) & 0xF) + (tokens >> 28), 3U, \ | | 501| 113k| DECODE, ON_FAIL); \ | | 502| 113k| DECODE_GLO_SEQ((tokens & 0x0F000) >> 12, (tokens & 0x00F00) >> 8, off2, \ | | ------------------ | | | | 453| 113k| do { \ | | | | 454| 113k| uint64_t ll = (LL); \ | | | | 455| 113k| uint64_t ml = (ML); \ | | | | 456| 113k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 113k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 14.1k, False: 98.9k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 14.1k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 14.1k| const uint64_t reserve = (RESERVE); \ | | | | 459| 14.1k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 28.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 41, False: 14.1k] | | | | | | | Branch (151:42): [True: 41, False: 14.1k] | | | | | | | Branch (151:42): [True: 0, False: 14.1k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 14.1k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 14.1k| ON_FAIL; \ | | | | 462| 14.1k| } \ | | | | 463| 113k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 113k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 37.5k, False: 75.5k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 37.5k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 37.5k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 37.5k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 43, False: 37.4k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 37.5k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 37.5k| (size_t)(d_end - d_ptr))) \ | | | | 468| 37.5k| ON_FAIL; \ | | | | 469| 37.5k| } \ | | | | 470| 113k| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 113k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 113k| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 910| 113k| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 405| 113k| do { \ | | | | | | | | 406| 113k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 113k| l_ptr += ll; \ | | | | | | | | 408| 113k| d_ptr += ll; \ | | | | | | | | 409| 113k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 113k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 28, False: 112k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 113k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 112k| d_ptr += ml; \ | | | | | | | | 412| 112k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 112k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 113k| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 112k] | | | | ------------------ | | ------------------ | | 503| 113k| ((tokens >> 20) & 0xF) + (tokens >> 28), 2U, DECODE, ON_FAIL); \ | | 504| 113k| DECODE_GLO_SEQ((tokens & 0x0F00000) >> 20, (tokens & 0x00F0000) >> 16, off3, \ | | ------------------ | | | | 453| 112k| do { \ | | | | 454| 112k| uint64_t ll = (LL); \ | | | | 455| 112k| uint64_t ml = (ML); \ | | | | 456| 112k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 112k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 14.5k, False: 98.4k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 14.5k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 14.5k| const uint64_t reserve = (RESERVE); \ | | | | 459| 14.5k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 29.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 33, False: 14.4k] | | | | | | | Branch (151:42): [True: 33, False: 14.4k] | | | | | | | Branch (151:42): [True: 0, False: 14.4k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 14.5k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 14.5k| ON_FAIL; \ | | | | 462| 14.5k| } \ | | | | 463| 112k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 112k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 42.8k, False: 70.1k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 42.8k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 42.8k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 42.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 32, False: 42.7k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 42.8k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 42.8k| (size_t)(d_end - d_ptr))) \ | | | | 468| 42.8k| ON_FAIL; \ | | | | 469| 42.8k| } \ | | | | 470| 112k| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 112k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 112k| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 910| 112k| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 405| 112k| do { \ | | | | | | | | 406| 112k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 112k| l_ptr += ll; \ | | | | | | | | 408| 112k| d_ptr += ll; \ | | | | | | | | 409| 112k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 112k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 33, False: 112k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 112k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 112k| d_ptr += ml; \ | | | | | | | | 412| 112k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 112k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 112k| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 112k] | | | | ------------------ | | ------------------ | | 505| 112k| (tokens >> 28), 1U, DECODE, ON_FAIL); \ | | 506| 112k| DECODE_GLO_SEQ((tokens >> 28), (tokens >> 24) & 0x0F, off4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 453| 112k| do { \ | | | | 454| 112k| uint64_t ll = (LL); \ | | | | 455| 112k| uint64_t ml = (ML); \ | | | | 456| 112k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 112k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 8.25k, False: 104k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 8.25k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 8.25k| const uint64_t reserve = (RESERVE); \ | | | | 459| 8.25k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 16.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 47, False: 8.21k] | | | | | | | Branch (151:42): [True: 40, False: 8.21k] | | | | | | | Branch (151:42): [True: 7, False: 8.21k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 8.25k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 8.25k| ON_FAIL; \ | | | | 462| 8.25k| } \ | | | | 463| 112k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 112k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 43.5k, False: 69.2k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 43.5k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 43.5k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 43.5k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 32, False: 43.5k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 43.5k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 43.5k| (size_t)(d_end - d_ptr))) \ | | | | 468| 43.5k| ON_FAIL; \ | | | | 469| 43.5k| } \ | | | | 470| 112k| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 112k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 112k| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 910| 112k| DECODE_GLO_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 405| 112k| do { \ | | | | | | | | 406| 112k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 112k| l_ptr += ll; \ | | | | | | | | 408| 112k| d_ptr += ll; \ | | | | | | | | 409| 112k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 112k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 33, False: 112k] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 112k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 112k| d_ptr += ml; \ | | | | | | | | 412| 112k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 112k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 112k| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 112k] | | | | ------------------ | | ------------------ | | 507| 112k| n_seq -= 4; \ | | 508| 112k| } while (0) | | ------------------ | | | Branch (508:14): [Folded, False: 112k] | | ------------------ ------------------ 911| 113k| } 912| 41.3k| } 913| | 914| | // --- FAST Loop: After threshold, no offset validation needed (4x unroll) --- 915| 40.9k| if (safe) { ------------------ | Branch (915:9): [True: 0, False: 40.9k] ------------------ 916| 0| while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x) { ------------------ | Branch (916:16): [True: 0, False: 0] | Branch (916:30): [True: 0, False: 0] | Branch (916:52): [True: 0, False: 0] ------------------ 917| 0| const uint8_t* const t_save = t_ptr; 918| 0| const uint8_t* const o_save = o_ptr; 919| 0| const uint8_t* const e_save = e_ptr; 920| 0| uint8_t* const d_save = d_ptr; 921| 0| const uint8_t* const l_save = l_ptr; 922| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x); ------------------ | | 479| 0| do { \ | | 480| 0| uint32_t tokens = zxc_le32(t_ptr); \ | | 481| 0| t_ptr += sizeof(uint32_t); \ | | 482| 0| uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 483| 0| off4 = ZXC_LZ_OFFSET_BIAS; \ | | ------------------ | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 484| 0| if (GLO_OFF8) { \ | | ------------------ | | | | 436| 0|#define GLO_OFF8 (gh.enc_off == 1) | | | | ------------------ | | | | | Branch (436:18): [True: 0, False: 0] | | | | ------------------ | | ------------------ | | 485| 0| uint32_t offsets = zxc_le32(o_ptr); \ | | 486| 0| o_ptr += sizeof(uint32_t); \ | | 487| 0| off1 += offsets & 0xFF; \ | | 488| 0| off2 += (offsets >> 8) & 0xFF; \ | | 489| 0| off3 += (offsets >> 16) & 0xFF; \ | | 490| 0| off4 += (offsets >> 24) & 0xFF; \ | | 491| 0| } else { \ | | 492| 0| uint64_t offsets = zxc_le64(o_ptr); \ | | 493| 0| o_ptr += sizeof(uint64_t); \ | | 494| 0| off1 += (uint32_t)(offsets & 0xFFFF); \ | | 495| 0| off2 += (uint32_t)((offsets >> 16) & 0xFFFF); \ | | 496| 0| off3 += (uint32_t)((offsets >> 32) & 0xFFFF); \ | | 497| 0| off4 += (uint32_t)((offsets >> 48) & 0xFFFF); \ | | 498| 0| } \ | | 499| 0| DECODE_GLO_SEQ((tokens & 0x0F0) >> 4, (tokens & 0x00F), off1, \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 922| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x); | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 500| 0| ((tokens >> 12) & 0xF) + ((tokens >> 20) & 0xF) + (tokens >> 28), 3U, \ | | 501| 0| DECODE, ON_FAIL); \ | | 502| 0| DECODE_GLO_SEQ((tokens & 0x0F000) >> 12, (tokens & 0x00F00) >> 8, off2, \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 922| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x); | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 503| 0| ((tokens >> 20) & 0xF) + (tokens >> 28), 2U, DECODE, ON_FAIL); \ | | 504| 0| DECODE_GLO_SEQ((tokens & 0x0F00000) >> 20, (tokens & 0x00F0000) >> 16, off3, \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 922| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x); | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 505| 0| (tokens >> 28), 1U, DECODE, ON_FAIL); \ | | 506| 0| DECODE_GLO_SEQ((tokens >> 28), (tokens >> 24) & 0x0F, off4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 453| 0| do { \ | | | | 454| 0| uint64_t ll = (LL); \ | | | | 455| 0| uint64_t ml = (ML); \ | | | | 456| 0| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 457| 0| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 0| const uint64_t reserve = (RESERVE); \ | | | | 459| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 460| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 0| ON_FAIL; \ | | | | 462| 0| } \ | | | | 463| 0| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 464| 0| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 0| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 466| 0| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 0| (size_t)(d_end - d_ptr))) \ | | | | 468| 0| ON_FAIL; \ | | | | 469| 0| } \ | | | | 470| 0| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 0| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 922| 0| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x); | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 0| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 507| 0| n_seq -= 4; \ | | 508| 0| } while (0) | | ------------------ | | | Branch (508:14): [Folded, False: 0] | | ------------------ ------------------ 923| 0| continue; 924| | 925| 0| rollback_fast_4x: 926| 0| t_ptr = t_save; 927| 0| o_ptr = o_save; 928| 0| e_ptr = e_save; 929| 0| d_ptr = d_save; 930| 0| l_ptr = l_save; 931| 0| break; 932| 0| } 933| 40.9k| } else { 934| 150k| while (n_seq >= 4 && d_ptr < d_end_safe && l_ptr < l_end_safe_4x) { ------------------ | Branch (934:16): [True: 119k, False: 30.6k] | Branch (934:30): [True: 119k, False: 232] | Branch (934:52): [True: 109k, False: 9.65k] ------------------ 935| 109k| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW); ------------------ | | 479| 109k| do { \ | | 480| 109k| uint32_t tokens = zxc_le32(t_ptr); \ | | 481| 109k| t_ptr += sizeof(uint32_t); \ | | 482| 109k| uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 109k|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 109k|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | uint32_t off1 = ZXC_LZ_OFFSET_BIAS, off2 = ZXC_LZ_OFFSET_BIAS, off3 = ZXC_LZ_OFFSET_BIAS, \ | | ------------------ | | | | 527| 109k|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 483| 109k| off4 = ZXC_LZ_OFFSET_BIAS; \ | | ------------------ | | | | 527| 109k|#define ZXC_LZ_OFFSET_BIAS 1 | | ------------------ | | 484| 109k| if (GLO_OFF8) { \ | | ------------------ | | | | 436| 109k|#define GLO_OFF8 (gh.enc_off == 1) | | | | ------------------ | | | | | Branch (436:18): [True: 108k, False: 957] | | | | ------------------ | | ------------------ | | 485| 108k| uint32_t offsets = zxc_le32(o_ptr); \ | | 486| 108k| o_ptr += sizeof(uint32_t); \ | | 487| 108k| off1 += offsets & 0xFF; \ | | 488| 108k| off2 += (offsets >> 8) & 0xFF; \ | | 489| 108k| off3 += (offsets >> 16) & 0xFF; \ | | 490| 108k| off4 += (offsets >> 24) & 0xFF; \ | | 491| 108k| } else { \ | | 492| 957| uint64_t offsets = zxc_le64(o_ptr); \ | | 493| 957| o_ptr += sizeof(uint64_t); \ | | 494| 957| off1 += (uint32_t)(offsets & 0xFFFF); \ | | 495| 957| off2 += (uint32_t)((offsets >> 16) & 0xFFFF); \ | | 496| 957| off3 += (uint32_t)((offsets >> 32) & 0xFFFF); \ | | 497| 957| off4 += (uint32_t)((offsets >> 48) & 0xFFFF); \ | | 498| 957| } \ | | 499| 109k| DECODE_GLO_SEQ((tokens & 0x0F0) >> 4, (tokens & 0x00F), off1, \ | | ------------------ | | | | 453| 109k| do { \ | | | | 454| 109k| uint64_t ll = (LL); \ | | | | 455| 109k| uint64_t ml = (ML); \ | | | | 456| 109k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 109k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 13.8k, False: 95.7k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 13.8k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 13.8k| const uint64_t reserve = (RESERVE); \ | | | | 459| 13.8k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 27.5k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 60, False: 13.7k] | | | | | | | Branch (151:42): [True: 59, False: 13.7k] | | | | | | | Branch (151:42): [True: 1, False: 13.7k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 13.8k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 13.8k| ON_FAIL; \ | | | | 462| 13.8k| } \ | | | | 463| 109k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 109k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 52.5k, False: 56.9k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 52.5k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 52.5k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 52.5k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 51, False: 52.4k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 52.5k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 52.5k| (size_t)(d_end - d_ptr))) \ | | | | 468| 52.5k| ON_FAIL; \ | | | | 469| 52.5k| } \ | | | | 470| 109k| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 109k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 109k| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 935| 109k| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 417| 109k| do { \ | | | | | | | | 418| 109k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 109k| l_ptr += ll; \ | | | | | | | | 420| 109k| d_ptr += ll; \ | | | | | | | | 421| 109k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 109k| d_ptr += ml; \ | | | | | | | | 423| 109k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 109k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 109k| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 109k] | | | | ------------------ | | ------------------ | | 500| 109k| ((tokens >> 12) & 0xF) + ((tokens >> 20) & 0xF) + (tokens >> 28), 3U, \ | | 501| 109k| DECODE, ON_FAIL); \ | | 502| 109k| DECODE_GLO_SEQ((tokens & 0x0F000) >> 12, (tokens & 0x00F00) >> 8, off2, \ | | ------------------ | | | | 453| 109k| do { \ | | | | 454| 109k| uint64_t ll = (LL); \ | | | | 455| 109k| uint64_t ml = (ML); \ | | | | 456| 109k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 109k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 8.77k, False: 100k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 8.77k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 8.77k| const uint64_t reserve = (RESERVE); \ | | | | 459| 8.77k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 17.5k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 59, False: 8.71k] | | | | | | | Branch (151:42): [True: 51, False: 8.72k] | | | | | | | Branch (151:42): [True: 8, False: 8.71k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 8.77k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 8.77k| ON_FAIL; \ | | | | 462| 8.77k| } \ | | | | 463| 109k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 109k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 53.0k, False: 56.3k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 53.0k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 53.0k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 53.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 53, False: 53.0k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 53.0k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 53.0k| (size_t)(d_end - d_ptr))) \ | | | | 468| 53.0k| ON_FAIL; \ | | | | 469| 53.0k| } \ | | | | 470| 109k| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 109k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 109k| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 935| 109k| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 417| 109k| do { \ | | | | | | | | 418| 109k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 109k| l_ptr += ll; \ | | | | | | | | 420| 109k| d_ptr += ll; \ | | | | | | | | 421| 109k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 109k| d_ptr += ml; \ | | | | | | | | 423| 109k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 109k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 109k| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 109k] | | | | ------------------ | | ------------------ | | 503| 109k| ((tokens >> 20) & 0xF) + (tokens >> 28), 2U, DECODE, ON_FAIL); \ | | 504| 109k| DECODE_GLO_SEQ((tokens & 0x0F00000) >> 20, (tokens & 0x00F0000) >> 16, off3, \ | | ------------------ | | | | 453| 109k| do { \ | | | | 454| 109k| uint64_t ll = (LL); \ | | | | 455| 109k| uint64_t ml = (ML); \ | | | | 456| 109k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 109k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 16.4k, False: 92.8k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 16.4k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 16.4k| const uint64_t reserve = (RESERVE); \ | | | | 459| 16.4k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 32.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 54, False: 16.4k] | | | | | | | Branch (151:42): [True: 53, False: 16.4k] | | | | | | | Branch (151:42): [True: 1, False: 16.4k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 16.4k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 16.4k| ON_FAIL; \ | | | | 462| 16.4k| } \ | | | | 463| 109k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 109k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 56.2k, False: 53.0k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 56.2k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 56.2k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 56.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 50, False: 56.2k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 56.2k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 56.2k| (size_t)(d_end - d_ptr))) \ | | | | 468| 56.2k| ON_FAIL; \ | | | | 469| 56.2k| } \ | | | | 470| 109k| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 109k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 109k| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 935| 109k| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 417| 109k| do { \ | | | | | | | | 418| 109k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 109k| l_ptr += ll; \ | | | | | | | | 420| 109k| d_ptr += ll; \ | | | | | | | | 421| 109k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 109k| d_ptr += ml; \ | | | | | | | | 423| 109k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 109k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 109k| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 109k] | | | | ------------------ | | ------------------ | | 505| 109k| (tokens >> 28), 1U, DECODE, ON_FAIL); \ | | 506| 109k| DECODE_GLO_SEQ((tokens >> 28), (tokens >> 24) & 0x0F, off4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 453| 109k| do { \ | | | | 454| 109k| uint64_t ll = (LL); \ | | | | 455| 109k| uint64_t ml = (ML); \ | | | | 456| 109k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 109k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 11.3k, False: 97.9k] | | | | | | ------------------ | | | | ------------------ | | | | 457| 11.3k| ll += zxc_read_varint(&e_ptr, e_end); \ | | | | 458| 11.3k| const uint64_t reserve = (RESERVE); \ | | | | 459| 11.3k| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 22.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 72, False: 11.2k] | | | | | | | Branch (151:42): [True: 66, False: 11.2k] | | | | | | | Branch (151:42): [True: 6, False: 11.2k] | | | | | | ------------------ | | | | ------------------ | | | | 460| 11.3k| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 461| 11.3k| ON_FAIL; \ | | | | 462| 11.3k| } \ | | | | 463| 109k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 109k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 56.8k, False: 52.2k] | | | | | | ------------------ | | | | ------------------ | | | | 464| 56.8k| ml += zxc_read_varint(&e_ptr, e_end); \ | | | | 465| 56.8k| if (UNLIKELY(ll + ml + ZXC_LZ_MIN_MATCH_LEN + \ | | | | ------------------ | | | | | | 151| 56.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 44, False: 56.8k] | | | | | | ------------------ | | | | ------------------ | | | | 466| 56.8k| (N_REM) * ZXC_GLO_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | 467| 56.8k| (size_t)(d_end - d_ptr))) \ | | | | 468| 56.8k| ON_FAIL; \ | | | | 469| 56.8k| } \ | | | | 470| 109k| ml += ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 109k|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 471| 109k| DECODE(ll, ml, OFF); \ | | | | ------------------ | | | | | | 935| 109k| DECODE_GLO_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW); | | | | | | ------------------ | | | | | | | | 417| 109k| do { \ | | | | | | | | 418| 109k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 109k| l_ptr += ll; \ | | | | | | | | 420| 109k| d_ptr += ll; \ | | | | | | | | 421| 109k| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 109k| d_ptr += ml; \ | | | | | | | | 423| 109k| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 109k] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 472| 109k| } while (0) | | | | ------------------ | | | | | Branch (472:14): [Folded, False: 109k] | | | | ------------------ | | ------------------ | | 507| 109k| n_seq -= 4; \ | | 508| 109k| } while (0) | | ------------------ | | | Branch (508:14): [Folded, False: 109k] | | ------------------ ------------------ 936| 109k| } 937| 40.9k| } 938| | 939| | // Validate vbyte reads didn't overflow 940| 40.4k| if (UNLIKELY(e_ptr > e_end)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 40.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 40.4k] | | ------------------ ------------------ 941| | 942| | // --- Remaining 1 sequence (Fast Path) --- 943| 177k| while (n_seq > 0 && d_ptr < d_end_safe && l_ptr < l_end_safe_1x) { ------------------ | Branch (943:12): [True: 145k, False: 31.5k] | Branch (943:25): [True: 145k, False: 530] | Branch (943:47): [True: 136k, False: 8.22k] ------------------ 944| | // Save pointers before reading (in case we need to fall back to Safe Path) 945| 136k| const uint8_t* t_save = t_ptr; 946| 136k| const uint8_t* o_save = o_ptr; 947| 136k| const uint8_t* e_save = e_ptr; 948| | 949| 136k| uint8_t token = *t_ptr++; 950| 136k| uint64_t ll = token >> ZXC_TOKEN_LIT_BITS; ------------------ | | 450| 136k|#define ZXC_TOKEN_LIT_BITS 4 ------------------ 951| 136k| uint64_t ml = token & ZXC_TOKEN_ML_MASK; ------------------ | | 456| 136k|#define ZXC_TOKEN_ML_MASK ((1U << ZXC_TOKEN_ML_BITS) - 1) | | ------------------ | | | | 452| 136k|#define ZXC_TOKEN_ML_BITS 4 | | ------------------ ------------------ 952| 136k| uint32_t offset = ZXC_LZ_OFFSET_BIAS; ------------------ | | 527| 136k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 953| 136k| if (GLO_OFF8) { ------------------ | | 436| 136k|#define GLO_OFF8 (gh.enc_off == 1) | | ------------------ | | | Branch (436:18): [True: 126k, False: 10.5k] | | ------------------ ------------------ 954| 126k| offset += *o_ptr++; // 1-byte offset (biased) 955| 126k| } else { 956| 10.5k| offset += zxc_le16(o_ptr); // 2-byte offset (biased) 957| 10.5k| o_ptr += sizeof(uint16_t); 958| 10.5k| } 959| | 960| 136k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) { ------------------ | | 151| 136k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 15.5k, False: 121k] | | ------------------ ------------------ 961| 15.5k| ll += zxc_read_varint(&e_ptr, e_end); 962| 15.5k| if (UNLIKELY(l_ptr + ll > l_end)) { ------------------ | | 151| 15.5k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 116, False: 15.4k] | | ------------------ ------------------ 963| 116| t_ptr = t_save; 964| 116| o_ptr = o_save; 965| 116| e_ptr = e_save; 966| 116| break; 967| 116| } 968| 15.5k| } 969| 136k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) ml += zxc_read_varint(&e_ptr, e_end); ------------------ | | 151| 136k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 38.5k, False: 98.1k] | | ------------------ ------------------ 970| 136k| ml += ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 136k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 971| | 972| | // Check bounds before wild copies - if too close to end, fall back to Safe Path 973| 136k| if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) { ------------------ | | 151| 136k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 61, False: 136k] | | ------------------ ------------------ 974| 61| t_ptr = t_save; 975| 61| o_ptr = o_save; 976| 61| e_ptr = e_save; 977| 61| break; 978| 61| } 979| | 980| 136k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); 981| 136k| l_ptr += ll; 982| 136k| d_ptr += ll; 983| | 984| 136k| if (UNLIKELY(d_ptr < d_bounds && (size_t)(d_ptr - d_floor) < offset)) ------------------ | | 151| 153k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 52, False: 136k] | | | Branch (151:42): [True: 16.4k, False: 120k] | | | Branch (151:42): [True: 52, False: 16.3k] | | ------------------ ------------------ 985| 52| return ZXC_ERROR_BAD_OFFSET; 986| | 987| | /* The loop entry check guarantees ll + ml + ZXC_PAD_SIZE bytes of 988| | * headroom, so the wild-copy ladder (incl. overlap/fill runs) is safe. */ 989| 136k| zxc_decode_copy_match(d_ptr, offset, ml); 990| 136k| d_ptr += ml; 991| 136k| n_seq--; 992| 136k| } 993| | 994| | // --- Safe Path for Remaining Sequences --- 995| 72.2k| while (n_seq > 0) { ------------------ | Branch (995:12): [True: 32.6k, False: 39.6k] ------------------ 996| 32.6k| uint8_t token = *t_ptr++; 997| 32.6k| uint64_t ll = token >> ZXC_TOKEN_LIT_BITS; ------------------ | | 450| 32.6k|#define ZXC_TOKEN_LIT_BITS 4 ------------------ 998| 32.6k| uint64_t ml = token & ZXC_TOKEN_ML_MASK; ------------------ | | 456| 32.6k|#define ZXC_TOKEN_ML_MASK ((1U << ZXC_TOKEN_ML_BITS) - 1) | | ------------------ | | | | 452| 32.6k|#define ZXC_TOKEN_ML_BITS 4 | | ------------------ ------------------ 999| 32.6k| uint32_t offset = ZXC_LZ_OFFSET_BIAS; ------------------ | | 527| 32.6k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1000| 32.6k| if (GLO_OFF8) { ------------------ | | 436| 32.6k|#define GLO_OFF8 (gh.enc_off == 1) | | ------------------ | | | Branch (436:18): [True: 26.1k, False: 6.45k] | | ------------------ ------------------ 1001| 26.1k| offset += *o_ptr++; // 1-byte offset (biased) 1002| 26.1k| } else { 1003| 6.45k| offset += zxc_le16(o_ptr); // 2-byte offset (biased) 1004| 6.45k| o_ptr += sizeof(uint16_t); 1005| 6.45k| } 1006| | 1007| 32.6k| if (UNLIKELY(ll == ZXC_TOKEN_LL_MASK)) ll += zxc_read_varint(&e_ptr, e_end); ------------------ | | 151| 32.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 961, False: 31.6k] | | ------------------ ------------------ 1008| 32.6k| if (UNLIKELY(ml == ZXC_TOKEN_ML_MASK)) ml += zxc_read_varint(&e_ptr, e_end); ------------------ | | 151| 32.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 14.5k, False: 18.0k] | | ------------------ ------------------ 1009| 32.6k| ml += ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 32.6k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1010| | 1011| 32.6k| if (UNLIKELY(ll + ml > (size_t)(d_end - d_ptr) || l_ptr + ll > l_end)) ------------------ | | 151| 65.0k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 786, False: 31.8k] | | | Branch (151:42): [True: 166, False: 32.4k] | | | Branch (151:42): [True: 620, False: 31.8k] | | ------------------ ------------------ 1012| 786| return ZXC_ERROR_OVERFLOW; 1013| 31.8k| ZXC_MEMCPY(d_ptr, l_ptr, ll); ------------------ | | 173| 31.8k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1014| 31.8k| l_ptr += ll; 1015| 31.8k| d_ptr += ll; 1016| | 1017| 31.8k| if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET; ------------------ | | 151| 31.8k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 33, False: 31.8k] | | ------------------ ------------------ 1018| 31.8k| const uint8_t* match_src = d_ptr - offset; 1019| | 1020| 31.8k| zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml); 1021| 31.8k| d_ptr += ml; 1022| 31.8k| n_seq--; 1023| 31.8k| } 1024| | 1025| | // --- Trailing Literals --- 1026| | // Copy remaining literals from source stream (literal exhaustion) 1027| 39.6k| if (UNLIKELY(l_ptr > l_end)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 39.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 39.6k] | | ------------------ ------------------ 1028| 39.6k| const size_t remaining_literals = (size_t)(l_end - l_ptr); 1029| 39.6k| if (UNLIKELY(remaining_literals > (size_t)(d_end - d_ptr))) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 39.6k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 20, False: 39.6k] | | ------------------ ------------------ 1030| 39.6k| ZXC_MEMCPY(d_ptr, l_ptr, remaining_literals); ------------------ | | 173| 39.6k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1031| 39.6k| d_ptr += remaining_literals; 1032| | 1033| 39.6k| return (int)(d_ptr - dst); 1034| 39.6k|} zxc_decompress.c:zxc_decode_block_glo_entropy: 1299| 21.7k| const size_t dst_capacity) { 1300| 21.7k| return zxc_decode_block_glo_impl(ctx, src, src_size, dst, dst_capacity, 0, 0, 1); 1301| 21.7k|} zxc_decompress.c:zxc_decode_lit_pivco: 609| 2.05k| const size_t required_size) { 610| 2.05k| const int arc = zxc_ensure_entropy_scratch(ctx); 611| 2.05k| if (UNLIKELY(arc != ZXC_OK)) return arc; ------------------ | | 151| 2.05k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 2.05k] | | ------------------ ------------------ 612| 2.05k| if (UNLIKELY(ctx->lit_buffer_cap < required_size + ZXC_PAD_SIZE || ------------------ | | 151| 4.09k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 2, False: 2.04k] | | | Branch (151:42): [True: 2, False: 2.04k] | | | Branch (151:42): [True: 0, False: 2.04k] | | ------------------ ------------------ 613| 2.05k| ctx->pivco_scratch_cap < required_size + ZXC_PIVCO_SCRATCH_PAD)) 614| 2| return ZXC_ERROR_CORRUPT_DATA; 615| 2.04k| return zxc_huf_decode_section(payload, psize, ctx->lit_buffer, required_size, ------------------ | | 31| 2.04k|#define zxc_huf_decode_section ZXC_CAT(zxc_huf_decode_section, ZXC_FUNCTION_SUFFIX) | | ------------------ | | | | 25| 2.04k|#define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) | | | | ------------------ | | | | | | 24| 2.04k|#define ZXC_CAT_IMPL(x, y) x##y | | | | ------------------ | | ------------------ ------------------ 616| 2.04k| ctx->pivco_scratch); 617| 2.05k|} zxc_decompress.c:zxc_ensure_entropy_scratch: 571| 23.3k|static ZXC_NOINLINE ZXC_COLD int zxc_ensure_entropy_scratch(const zxc_cctx_t* RESTRICT ctx) { 572| 23.3k| if (LIKELY(ctx->pivco_scratch != NULL)) return ZXC_OK; ------------------ | | 145| 23.3k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 21.0k, False: 2.30k] | | ------------------ ------------------ 573| 2.30k| return zxc_cctx_alloc_entropy_scratch((zxc_cctx_t*)(uintptr_t)ctx); 574| 23.3k|} zxc_decompress.c:zxc_decode_lit_pivco_dict: 622| 3| const size_t required_size) { 623| 3| if (UNLIKELY(!ctx->dict_huf_tree_ok)) return ZXC_ERROR_DICT_REQUIRED; ------------------ | | 151| 3|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 3, False: 0] | | ------------------ ------------------ 624| 0| const int arc = zxc_ensure_entropy_scratch(ctx); 625| 0| if (UNLIKELY(arc != ZXC_OK)) return arc; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 626| 0| if (UNLIKELY(ctx->lit_buffer_cap < required_size + ZXC_PAD_SIZE || ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 627| 0| ctx->pivco_scratch_cap < required_size + ZXC_PIVCO_SCRATCH_PAD)) 628| 0| return ZXC_ERROR_CORRUPT_DATA; 629| 0| return zxc_huf_decode_section_dict(payload, psize, ctx->lit_buffer, required_size, ------------------ | | 32| 0|#define zxc_huf_decode_section_dict ZXC_CAT(zxc_huf_decode_section_dict, ZXC_FUNCTION_SUFFIX) | | ------------------ | | | | 25| 0|#define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) | | | | ------------------ | | | | | | 24| 0|#define ZXC_CAT_IMPL(x, y) x##y | | | | ------------------ | | ------------------ ------------------ 630| 0| &ctx->dict_huf->tree, &ctx->dict_huf->dec, 631| 0| ctx->pivco_scratch); 632| 0|} zxc_decompress.c:zxc_stage_raw_literals: 594| 2.59k| const uint8_t** RESTRICT l_end) { 595| 2.59k| const size_t lit_size = (size_t)(*l_end - *l_ptr); 596| 2.59k| uint8_t* const stage = ctx->lit_buffer; 597| 2.59k| if (UNLIKELY(!stage || ctx->lit_buffer_cap < lit_size + ZXC_PAD_SIZE)) ------------------ | | 151| 5.19k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 2.59k] | | | Branch (151:42): [True: 0, False: 2.59k] | | | Branch (151:42): [True: 0, False: 2.59k] | | ------------------ ------------------ 598| 0| return ZXC_ERROR_CORRUPT_DATA; 599| | 600| 2.59k| ZXC_MEMCPY(stage, *l_ptr, lit_size); ------------------ | | 173| 2.59k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 601| 2.59k| *l_ptr = stage; 602| 2.59k| *l_end = stage + lit_size; 603| 2.59k| return ZXC_OK; 604| 2.59k|} zxc_decompress.c:zxc_decode_tok_pivco: 636| 21.2k| const size_t psize, const size_t n_tok) { 637| 21.2k| const int arc = zxc_ensure_entropy_scratch(ctx); 638| 21.2k| if (UNLIKELY(arc != ZXC_OK)) return arc; ------------------ | | 151| 21.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 21.2k] | | ------------------ ------------------ 639| 21.2k| if (UNLIKELY(n_tok + ZXC_PAD_SIZE > ctx->tok_buffer_cap || ------------------ | | 151| 42.5k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 21.2k] | | | Branch (151:42): [True: 0, False: 21.2k] | | | Branch (151:42): [True: 0, False: 21.2k] | | ------------------ ------------------ 640| 21.2k| n_tok + ZXC_PIVCO_SCRATCH_PAD > ctx->pivco_scratch_cap)) 641| 0| return ZXC_ERROR_CORRUPT_DATA; 642| 21.2k| return zxc_huf_decode_section(payload, psize, ctx->tok_buffer, n_tok, ctx->pivco_scratch); ------------------ | | 31| 21.2k|#define zxc_huf_decode_section ZXC_CAT(zxc_huf_decode_section, ZXC_FUNCTION_SUFFIX) | | ------------------ | | | | 25| 21.2k|#define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) | | | | ------------------ | | | | | | 24| 21.2k|#define ZXC_CAT_IMPL(x, y) x##y | | | | ------------------ | | ------------------ ------------------ 643| 21.2k|} zxc_decompress.c:zxc_read_varint: 54| 548k|static ZXC_ALWAYS_INLINE uint32_t zxc_read_varint(const uint8_t** ptr, const uint8_t* end) { 55| 548k| const uint8_t* p = *ptr; 56| 548k| if (UNLIKELY(p >= end)) return 0; ------------------ | | 151| 548k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 217k, False: 331k] | | ------------------ ------------------ 57| | 58| 331k| const uint32_t b0 = p[0]; 59| | 60| | // 1 Byte: 0xxxxxxx (7 bits) -> val < 128 (2^7) 61| 331k| if (LIKELY(b0 < 0x80)) { ------------------ | | 145| 331k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 243k, False: 87.7k] | | ------------------ ------------------ 62| 243k| *ptr = p + 1; 63| 243k| return b0; 64| 243k| } 65| | 66| | // 2 Bytes: 10xxxxxx xxxxxxxx (14 bits) -> val < 16384 (2^14) 67| 87.7k| if (LIKELY(b0 < 0xC0)) { ------------------ | | 145| 87.7k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 54.4k, False: 33.2k] | | ------------------ ------------------ 68| 54.4k| if (UNLIKELY(p + 1 >= end)) { ------------------ | | 151| 54.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 6.78k, False: 47.6k] | | ------------------ ------------------ 69| 6.78k| *ptr = end; 70| 6.78k| return 0; 71| 6.78k| } 72| 47.6k| *ptr = p + 2; 73| 47.6k| return (b0 & 0x3F) | ((uint32_t)p[1] << 6); 74| 54.4k| } 75| | 76| | // 3 Bytes: 110xxxxx xxxxxxxx xxxxxxxx (21 bits) -> val < 2^21. The longest 77| | // a legitimate varint can be: values are (ll - MASK) or (ml - MASK), always 78| | // strictly below block_size_max = 2^21. 79| 33.2k| if (LIKELY(b0 < 0xE0)) { ------------------ | | 145| 33.2k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 19.3k, False: 13.9k] | | ------------------ ------------------ 80| 19.3k| if (UNLIKELY(p + 2 >= end)) { ------------------ | | 151| 19.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.30k, False: 18.0k] | | ------------------ ------------------ 81| 1.30k| *ptr = end; 82| 1.30k| return 0; 83| 1.30k| } 84| 18.0k| *ptr = p + 3; 85| 18.0k| return (b0 & 0x1F) | ((uint32_t)p[1] << 5) | ((uint32_t)p[2] << 13); 86| 19.3k| } 87| | 88| | // extra encoding: out-of-spec for the current format, reject. 89| 13.9k| *ptr = end; 90| 13.9k| return 0; 91| 33.2k|} zxc_decompress.c:zxc_decode_copy_literals: 297| 1.03M| const uint64_t ll) { 298| 1.03M| zxc_copy32(dst, src); 299| 1.03M| if (UNLIKELY(ll > ZXC_PAD_SIZE)) { ------------------ | | 151| 1.03M|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 9.39k, False: 1.02M] | | ------------------ ------------------ 300| 9.39k| dst += ZXC_PAD_SIZE; ------------------ | | 337| 9.39k|#define ZXC_PAD_SIZE 32 ------------------ 301| 9.39k| src += ZXC_PAD_SIZE; ------------------ | | 337| 9.39k|#define ZXC_PAD_SIZE 32 ------------------ 302| 9.39k| size_t rem = ll - ZXC_PAD_SIZE; ------------------ | | 337| 9.39k|#define ZXC_PAD_SIZE 32 ------------------ 303| 14.4k| while (rem > ZXC_PAD_SIZE) { ------------------ | | 337| 14.4k|#define ZXC_PAD_SIZE 32 ------------------ | Branch (303:16): [True: 5.02k, False: 9.39k] ------------------ 304| 5.02k| zxc_copy32(dst, src); 305| 5.02k| dst += ZXC_PAD_SIZE; ------------------ | | 337| 5.02k|#define ZXC_PAD_SIZE 32 ------------------ 306| 5.02k| src += ZXC_PAD_SIZE; ------------------ | | 337| 5.02k|#define ZXC_PAD_SIZE 32 ------------------ 307| 5.02k| rem -= ZXC_PAD_SIZE; ------------------ | | 337| 5.02k|#define ZXC_PAD_SIZE 32 ------------------ 308| 5.02k| } 309| 9.39k| zxc_copy32(dst, src); 310| 9.39k| } 311| 1.03M|} zxc_decompress.c:zxc_decode_copy_match: 333| 1.03M| const uint64_t ml) { 334| 1.03M| const uint8_t* match_src = d_ptr - off; 335| 1.03M| if (LIKELY(off >= ZXC_PAD_SIZE)) { ------------------ | | 145| 1.03M|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 215k, False: 815k] | | ------------------ ------------------ 336| 215k| zxc_copy32(d_ptr, match_src); 337| 215k| if (UNLIKELY(ml > ZXC_PAD_SIZE)) { ------------------ | | 151| 215k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 44.6k, False: 170k] | | ------------------ ------------------ 338| 44.6k| uint8_t* out = d_ptr + ZXC_PAD_SIZE; ------------------ | | 337| 44.6k|#define ZXC_PAD_SIZE 32 ------------------ 339| 44.6k| const uint8_t* ref = match_src + ZXC_PAD_SIZE; ------------------ | | 337| 44.6k|#define ZXC_PAD_SIZE 32 ------------------ 340| 44.6k| size_t rem = ml - ZXC_PAD_SIZE; ------------------ | | 337| 44.6k|#define ZXC_PAD_SIZE 32 ------------------ 341| 11.0M| while (rem > ZXC_PAD_SIZE) { ------------------ | | 337| 11.0M|#define ZXC_PAD_SIZE 32 ------------------ | Branch (341:20): [True: 10.9M, False: 44.6k] ------------------ 342| 10.9M| zxc_copy32(out, ref); 343| 10.9M| out += ZXC_PAD_SIZE; ------------------ | | 337| 10.9M|#define ZXC_PAD_SIZE 32 ------------------ 344| 10.9M| ref += ZXC_PAD_SIZE; ------------------ | | 337| 10.9M|#define ZXC_PAD_SIZE 32 ------------------ 345| 10.9M| rem -= ZXC_PAD_SIZE; ------------------ | | 337| 10.9M|#define ZXC_PAD_SIZE 32 ------------------ 346| 10.9M| } 347| 44.6k| zxc_copy32(out, ref); 348| 44.6k| } 349| 815k| } else if (off >= (ZXC_PAD_SIZE / 2)) { ------------------ | | 337| 815k|#define ZXC_PAD_SIZE 32 ------------------ | Branch (349:16): [True: 181k, False: 633k] ------------------ 350| 181k| zxc_copy16(d_ptr, match_src); 351| 181k| if (UNLIKELY(ml > (ZXC_PAD_SIZE / 2))) { ------------------ | | 151| 181k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 112k, False: 69.2k] | | ------------------ ------------------ 352| 112k| uint8_t* out = d_ptr + (ZXC_PAD_SIZE / 2); ------------------ | | 337| 112k|#define ZXC_PAD_SIZE 32 ------------------ 353| 112k| const uint8_t* ref = match_src + (ZXC_PAD_SIZE / 2); ------------------ | | 337| 112k|#define ZXC_PAD_SIZE 32 ------------------ 354| 112k| size_t rem = ml - (ZXC_PAD_SIZE / 2); ------------------ | | 337| 112k|#define ZXC_PAD_SIZE 32 ------------------ 355| 14.7M| while (rem > (ZXC_PAD_SIZE / 2)) { ------------------ | | 337| 14.7M|#define ZXC_PAD_SIZE 32 ------------------ | Branch (355:20): [True: 14.6M, False: 112k] ------------------ 356| 14.6M| zxc_copy16(out, ref); 357| 14.6M| out += (ZXC_PAD_SIZE / 2); ------------------ | | 337| 14.6M|#define ZXC_PAD_SIZE 32 ------------------ 358| 14.6M| ref += (ZXC_PAD_SIZE / 2); ------------------ | | 337| 14.6M|#define ZXC_PAD_SIZE 32 ------------------ 359| 14.6M| rem -= (ZXC_PAD_SIZE / 2); ------------------ | | 337| 14.6M|#define ZXC_PAD_SIZE 32 ------------------ 360| 14.6M| } 361| 112k| zxc_copy16(out, ref); 362| 112k| } 363| 633k| } else if (off == 1) { ------------------ | Branch (363:16): [True: 450k, False: 182k] ------------------ 364| 450k| zxc_decode_fill_run(d_ptr, match_src[0], ml); 365| 450k| } else { 366| 182k| zxc_decode_copy_overlap_run(d_ptr, off, ml); 367| 182k| } 368| 1.03M|} zxc_decompress.c:zxc_decode_fill_run: 222| 450k| const uint64_t ml) { 223| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 224| | const __m256i v = _mm256_set1_epi8((char)byte); 225| | _mm256_storeu_si256((__m256i*)dst, v); 226| | if (UNLIKELY(ml > ZXC_PAD_SIZE)) { 227| | uint8_t* out = dst + ZXC_PAD_SIZE; 228| | size_t rem = ml - ZXC_PAD_SIZE; 229| | while (rem > ZXC_PAD_SIZE) { 230| | _mm256_storeu_si256((__m256i*)out, v); 231| | out += ZXC_PAD_SIZE; 232| | rem -= ZXC_PAD_SIZE; 233| | } 234| | _mm256_storeu_si256((__m256i*)out, v); 235| | } 236| |#elif defined(ZXC_USE_SSE2) 237| | const __m128i v = _mm_set1_epi8((char)byte); 238| 450k| _mm_storeu_si128((__m128i*)dst, v); 239| 450k| _mm_storeu_si128((__m128i*)(dst + 16), v); 240| 450k| if (UNLIKELY(ml > ZXC_PAD_SIZE)) { ------------------ | | 151| 450k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 58.5k, False: 392k] | | ------------------ ------------------ 241| 58.5k| uint8_t* out = dst + ZXC_PAD_SIZE; ------------------ | | 337| 58.5k|#define ZXC_PAD_SIZE 32 ------------------ 242| 58.5k| size_t rem = ml - ZXC_PAD_SIZE; ------------------ | | 337| 58.5k|#define ZXC_PAD_SIZE 32 ------------------ 243| 12.8M| while (rem > ZXC_PAD_SIZE) { ------------------ | | 337| 12.8M|#define ZXC_PAD_SIZE 32 ------------------ | Branch (243:16): [True: 12.7M, False: 58.5k] ------------------ 244| 12.7M| _mm_storeu_si128((__m128i*)out, v); 245| 12.7M| _mm_storeu_si128((__m128i*)(out + 16), v); 246| 12.7M| out += ZXC_PAD_SIZE; ------------------ | | 337| 12.7M|#define ZXC_PAD_SIZE 32 ------------------ 247| 12.7M| rem -= ZXC_PAD_SIZE; ------------------ | | 337| 12.7M|#define ZXC_PAD_SIZE 32 ------------------ 248| 12.7M| } 249| 58.5k| _mm_storeu_si128((__m128i*)out, v); 250| 58.5k| _mm_storeu_si128((__m128i*)(out + 16), v); 251| 58.5k| } 252| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 253| | const uint8x16_t v = vdupq_n_u8(byte); 254| | vst1q_u8(dst, v); 255| | vst1q_u8(dst + 16, v); 256| | if (UNLIKELY(ml > ZXC_PAD_SIZE)) { 257| | uint8_t* out = dst + ZXC_PAD_SIZE; 258| | size_t rem = ml - ZXC_PAD_SIZE; 259| | while (rem > ZXC_PAD_SIZE) { 260| | vst1q_u8(out, v); 261| | vst1q_u8(out + 16, v); 262| | out += ZXC_PAD_SIZE; 263| | rem -= ZXC_PAD_SIZE; 264| | } 265| | vst1q_u8(out, v); 266| | vst1q_u8(out + 16, v); 267| | } 268| |#else 269| | ZXC_MEMSET(dst, byte, ml); 270| |#endif 271| 450k|} zxc_decompress.c:zxc_decode_copy_overlap_run: 157| 182k| const uint64_t ml) { 158| 182k| const size_t stride = zxc_overlap_strides[off]; 159| 182k| size_t copied = 0; 160| |#if defined(ZXC_USE_NEON64) 161| | const uint8x16_t mask = vld1q_u8(zxc_overlap_masks[off]); 162| | const uint8x16_t pat = vqtbl1q_u8(vld1q_u8(dst - off), mask); 163| | do { 164| | vst1q_u8(dst + copied, pat); 165| | copied += stride; 166| | } while (copied < ml); 167| | 168| |#elif defined(ZXC_USE_NEON32) 169| | uint8x8x2_t src_tbl; 170| | src_tbl.val[0] = vld1_u8(dst - off); 171| | src_tbl.val[1] = vld1_u8(dst - off + 8); 172| | const uint8x8_t pat_lo = vtbl2_u8(src_tbl, vld1_u8(zxc_overlap_masks[off])); 173| | const uint8x8_t pat_hi = vtbl2_u8(src_tbl, vld1_u8(zxc_overlap_masks[off] + 8)); 174| | do { 175| | vst1_u8(dst + copied, pat_lo); 176| | vst1_u8(dst + copied + 8, pat_hi); 177| | copied += stride; 178| | } while (copied < ml); 179| | 180| |#elif defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 181| | const __m128i mask = _mm_load_si128((const __m128i*)zxc_overlap_masks[off]); 182| | const __m128i src_data = _mm_loadu_si128((const __m128i*)(dst - off)); 183| | const __m128i pat = _mm_shuffle_epi8(src_data, mask); 184| | do { 185| | _mm_storeu_si128((__m128i*)(dst + copied), pat); 186| | copied += stride; 187| | } while (copied < ml); 188| | 189| |#else 190| | // SSE2-only tier and non-SIMD builds: no PSHUFB, build the pattern with a 191| | // wrap counter (no per-byte modulo), then store it via zxc_copy16. 192| 182k| const uint8_t* src = dst - off; 193| 182k| uint8_t pat[16]; 194| 182k| uint32_t k = 0; 195| 3.11M| for (size_t i = 0; i < 16; i++) { ------------------ | Branch (195:24): [True: 2.92M, False: 182k] ------------------ 196| 2.92M| pat[i] = src[k]; 197| 2.92M| if (++k == off) k = 0; ------------------ | Branch (197:13): [True: 621k, False: 2.30M] ------------------ 198| 2.92M| } 199| 2.03M| do { 200| 2.03M| zxc_copy16(dst + copied, pat); 201| 2.03M| copied += stride; 202| 2.03M| } while (copied < ml); ------------------ | Branch (202:14): [True: 1.85M, False: 182k] ------------------ 203| 182k|#endif 204| 182k|} zxc_decompress.c:zxc_decode_copy_match_exact: 387| 34.2k| const size_t off, const size_t ml) { 388| 34.2k| if (off >= ml) { ------------------ | Branch (388:9): [True: 13.3k, False: 20.8k] ------------------ 389| 13.3k| ZXC_MEMCPY(d_ptr, match_src, ml); ------------------ | | 173| 13.3k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 390| 20.8k| } else if (ml < 16) { ------------------ | Branch (390:16): [True: 9.39k, False: 11.4k] ------------------ 391| 79.0k| for (size_t i = 0; i < ml; i++) d_ptr[i] = match_src[i]; ------------------ | Branch (391:28): [True: 69.6k, False: 9.39k] ------------------ 392| 11.4k| } else { 393| 11.4k| size_t n = off; 394| 11.4k| ZXC_MEMCPY(d_ptr, match_src, n); ------------------ | | 173| 11.4k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 395| 55.9k| while (n <= ml - n) { ------------------ | Branch (395:16): [True: 44.4k, False: 11.4k] ------------------ 396| 44.4k| ZXC_MEMCPY(d_ptr + n, d_ptr, n); ------------------ | | 173| 44.4k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 397| 44.4k| n <<= 1; 398| 44.4k| } 399| 11.4k| ZXC_MEMCPY(d_ptr + n, d_ptr, ml - n); ------------------ | | 173| 11.4k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 400| 11.4k| } 401| 34.2k|} zxc_decompress.c:zxc_decode_block_glo: 1334| 42.5k| const size_t dst_capacity) { 1335| 42.5k| return zxc_decode_block_glo_impl(ctx, src, src_size, dst, dst_capacity, 0, 0, 0); 1336| 42.5k|} zxc_decompress.c:zxc_decode_block_ghi_impl: 1060| 2.39k| const int has_dict) { 1061| 2.39k| zxc_gnr_header_t gh; 1062| | 1063| | /* 0 when !has_dict (safe path) -> folds `d_floor` to `dst`. */ 1064| 2.39k| const size_t dict_size = has_dict ? ctx->dict_size : 0; ------------------ | Branch (1064:30): [True: 0, False: 2.39k] ------------------ 1065| 2.39k| zxc_section_desc_t desc[ZXC_GHI_SECTIONS]; 1066| | 1067| 2.39k| if (UNLIKELY(zxc_read_ghi_header_and_desc(src, src_size, &gh, desc) != ZXC_OK)) ------------------ | | 151| 2.39k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 13, False: 2.38k] | | ------------------ ------------------ 1068| 13| return ZXC_ERROR_BAD_HEADER; 1069| | 1070| 2.38k| const uint8_t* p_curr = 1071| 2.38k| src + ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 403| 2.38k|#define ZXC_GHI_HEADER_BINARY_SIZE 16 ------------------ src + ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 421| 2.38k|#define ZXC_GHI_SECTIONS 3 ------------------ src + ZXC_GHI_HEADER_BINARY_SIZE + ZXC_GHI_SECTIONS * ZXC_SECTION_DESC_BINARY_SIZE; ------------------ | | 415| 2.38k|#define ZXC_SECTION_DESC_BINARY_SIZE 8 ------------------ 1072| | 1073| | // --- Stream Pointers & Validation --- 1074| 2.38k| const size_t sz_lit = (uint32_t)desc[0].sizes; 1075| 2.38k| const size_t sz_seqs = (uint32_t)desc[1].sizes; 1076| 2.38k| const size_t sz_exts = (uint32_t)desc[2].sizes; 1077| 2.38k| const uint8_t* l_ptr = p_curr; 1078| 2.38k| const uint8_t* l_end = l_ptr + sz_lit; 1079| 2.38k| p_curr += sz_lit; 1080| | 1081| 2.38k| const uint8_t* seq_ptr = p_curr; 1082| 2.38k| const uint8_t* extras_ptr = p_curr + sz_seqs; 1083| 2.38k| const uint8_t* const extras_end = extras_ptr + sz_exts; 1084| | 1085| | // Validate streams don't overflow source buffer + 1086| | // Validate sequence stream size matches sequence count 1087| 2.38k| if (UNLIKELY((extras_end != src + src_size) || ------------------ | | 151| 4.75k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 35, False: 2.34k] | | | Branch (151:42): [True: 5, False: 2.37k] | | | Branch (151:42): [True: 30, False: 2.34k] | | ------------------ ------------------ 1088| 2.38k| ((uint64_t)sz_seqs < (uint64_t)gh.n_sequences * 4))) 1089| 35| return ZXC_ERROR_CORRUPT_DATA; 1090| | 1091| 2.34k| if (UNLIKELY((size_t)(src + src_size - l_end) < ZXC_PAD_SIZE)) { ------------------ | | 151| 2.34k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 2.05k, False: 289] | | ------------------ ------------------ 1092| 2.05k| const int rc = zxc_stage_raw_literals(ctx, &l_ptr, &l_end); 1093| 2.05k| if (UNLIKELY(rc != ZXC_OK)) return rc; ------------------ | | 151| 2.05k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 2.05k] | | ------------------ ------------------ 1094| 2.05k| } 1095| | 1096| 2.34k| uint8_t* d_ptr = dst; 1097| 2.34k| const uint8_t* const d_end = dst + dst_capacity; 1098| | // Lowest address a match may reach: the dictionary prefix if any, else dst. 1099| 2.34k| const uint8_t* const d_floor = dst - dict_size; 1100| 2.34k| const uint8_t* const d_end_safe = d_end - (ZXC_PAD_SIZE * 4); // 128 ------------------ | | 337| 2.34k|#define ZXC_PAD_SIZE 32 ------------------ 1101| | // Safety margin for 4x unrolled loop: 4 * (ZXC_SEQ_LL_MASK LL + 1102| | // ZXC_SEQ_ML_MASK+ZXC_LZ_MIN_MATCH_LEN ML) + ZXC_PAD_SIZE Pad = 4 x (255 + 255 + 5) + 32 = 2092 1103| 2.34k| const uint8_t* const d_end_fast = d_end - ZXC_DECOMPRESS_TAIL_PAD; // 2112 ------------------ | | 349| 2.34k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 2.34k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 1104| | 1105| | // Literal margin for the GHI loops: without a varint, ll <= 254 per sequence, 1106| | // so 4 * 254 = 1016. Past that only the cold varint path checks l_ptr. 1107| 2.34k| const size_t ghi_margin_4x = 4 * (ZXC_SEQ_LL_MASK - 1); // 1016 ------------------ | | 469| 2.34k|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 463| 2.34k|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ 1108| 2.34k| const size_t ghi_margin_1x = ZXC_SEQ_LL_MASK - 1; // 254 ------------------ | | 469| 2.34k|#define ZXC_SEQ_LL_MASK ((1U << ZXC_SEQ_LL_BITS) - 1) | | ------------------ | | | | 463| 2.34k|#define ZXC_SEQ_LL_BITS 8 | | ------------------ ------------------ 1109| 2.34k| const uint8_t* const l_end_safe_4x = (sz_lit > ghi_margin_4x) ? l_end - ghi_margin_4x : l_ptr; ------------------ | Branch (1109:42): [True: 0, False: 2.34k] ------------------ 1110| 2.34k| const uint8_t* const l_end_safe_1x = (sz_lit > ghi_margin_1x) ? l_end - ghi_margin_1x : l_ptr; ------------------ | Branch (1110:42): [True: 110, False: 2.23k] ------------------ 1111| | 1112| 2.34k| uint32_t n_seq = gh.n_sequences; 1113| | 1114| | // Same handover as GLO, but the GHI offset is inline in the sequence word and 1115| | // always 16 bits -- enc_off is not a width here, so the span is fixed. 1116| 2.34k| const size_t off_span = 1U << 16; 1117| 2.34k| const size_t span = (dict_size >= off_span) ? 0 : off_span - dict_size; ------------------ | Branch (1117:25): [True: 0, False: 2.34k] ------------------ 1118| 2.34k| const uint8_t* const d_bounds = (span >= dst_capacity) ? d_end : dst + span; ------------------ | Branch (1118:37): [True: 0, False: 2.34k] ------------------ 1119| | 1120| | // --- SAFE loop: validate offsets until d_bounds (4x unroll) --- 1121| 2.34k| if (safe) { ------------------ | Branch (1121:9): [True: 0, False: 2.34k] ------------------ 1122| | /* SAFE variant: save per-batch state so an OVERFLOW can rollback and 1123| | * hand over to the 1x loop / Safe Path. Wild writes already committed 1124| | * are deterministically overwritten when the 1x loop replays. */ 1125| 0| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (1125:16): [True: 0, False: 0] | Branch (1125:30): [True: 0, False: 0] | Branch (1125:52): [True: 0, False: 0] | Branch (1125:77): [True: 0, False: 0] ------------------ 1126| 0| const uint8_t* const t_save = seq_ptr; 1127| 0| const uint8_t* const e_save = extras_ptr; 1128| 0| uint8_t* const d_save = d_ptr; 1129| 0| const uint8_t* const l_save = l_ptr; 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); ------------------ | | 543| 0| do { \ | | 544| 0| uint32_t s1 = zxc_le32(seq_ptr); \ | | 545| 0| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 546| 0| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 547| 0| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 548| 0| seq_ptr += 4 * sizeof(uint32_t); \ | | 549| 0| PREFETCH; \ | | 550| 0| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 551| 0| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 552| 0| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 553| 0| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1130| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, goto rollback_safe_4x, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 554| 0| n_seq -= 4; \ | | 555| 0| } while (0) | | ------------------ | | | Branch (555:14): [Folded, False: 0] | | ------------------ ------------------ 1131| 0| continue; 1132| | 1133| 0| rollback_safe_4x: 1134| 0| seq_ptr = t_save; 1135| 0| extras_ptr = e_save; 1136| 0| d_ptr = d_save; 1137| 0| l_ptr = l_save; 1138| 0| break; 1139| 0| } 1140| 2.34k| } else { 1141| 2.34k| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x && d_ptr < d_bounds) { ------------------ | Branch (1141:16): [True: 236, False: 2.10k] | Branch (1141:30): [True: 236, False: 0] | Branch (1141:52): [True: 0, False: 236] | Branch (1141:77): [True: 0, False: 0] ------------------ 1142| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); ------------------ | | 543| 0| do { \ | | 544| 0| uint32_t s1 = zxc_le32(seq_ptr); \ | | 545| 0| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 546| 0| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 547| 0| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 548| 0| seq_ptr += 4 * sizeof(uint32_t); \ | | 549| 0| PREFETCH; \ | | 550| 0| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1142| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 551| 0| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1142| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 552| 0| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1142| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 553| 0| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1142| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_SAFE, return ZXC_ERROR_OVERFLOW, (void)0); | | | | | | ------------------ | | | | | | | | 405| 0| do { \ | | | | | | | | 406| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 407| 0| l_ptr += ll; \ | | | | | | | | 408| 0| d_ptr += ll; \ | | | | | | | | 409| 0| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | | | | | | | ------------------ | | | | | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | | | 410| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 411| 0| d_ptr += ml; \ | | | | | | | | 412| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (412:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 554| 0| n_seq -= 4; \ | | 555| 0| } while (0) | | ------------------ | | | Branch (555:14): [Folded, False: 0] | | ------------------ ------------------ 1143| 0| } 1144| 2.34k| } 1145| | 1146| | // --- SAFE Loop tail: remaining sequences with offset validation (1x) --- 1147| 9.11k| while (n_seq > 0 && d_ptr < d_end_safe && d_ptr < d_bounds) { ------------------ | Branch (1147:12): [True: 7.03k, False: 2.08k] | Branch (1147:25): [True: 7.02k, False: 10] | Branch (1147:47): [True: 6.98k, False: 41] ------------------ 1148| 6.98k| uint32_t seq = zxc_le32(seq_ptr); 1149| 6.98k| seq_ptr += sizeof(uint32_t); 1150| | 1151| 6.98k| uint64_t ll = seq >> 24; 1152| 6.98k| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 6.98k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 111, False: 6.87k] | | ------------------ ------------------ 1153| | 1154| 6.98k| uint32_t m_bits = (seq >> 16) & 0xFF; 1155| 6.98k| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 6.98k|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1156| 6.98k| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 6.98k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 886, False: 6.09k] | | ------------------ ------------------ 1157| | 1158| 6.98k| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 527| 6.98k|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1159| | 1160| | // Strict bounds check: sequence must fit, AND wild copies must not overshoot 1161| | // Check both destination (d_ptr) and source literal stream (l_ptr) 1162| 6.98k| if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr) || ------------------ | | 151| 13.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 2.26k, False: 4.71k] | | | Branch (151:42): [True: 37, False: 6.94k] | | | Branch (151:42): [True: 2.22k, False: 4.71k] | | ------------------ ------------------ 1163| 6.98k| ll + ZXC_PAD_SIZE > (size_t)(l_end - l_ptr))) { 1164| | // Fallback to exact copy (slow but safe) 1165| 2.26k| if (UNLIKELY(d_ptr + ll > d_end || l_ptr + ll > l_end)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 4.50k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 152, False: 2.11k] | | | Branch (151:42): [True: 21, False: 2.24k] | | | Branch (151:42): [True: 131, False: 2.11k] | | ------------------ ------------------ 1166| 2.11k| ZXC_MEMCPY(d_ptr, l_ptr, ll); ------------------ | | 173| 2.11k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1167| 2.11k| l_ptr += ll; 1168| 2.11k| d_ptr += ll; 1169| | 1170| 2.11k| if (UNLIKELY(d_ptr + ml > d_end)) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 2.11k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 3, False: 2.11k] | | ------------------ ------------------ 1171| 2.11k| if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET; ------------------ | | 151| 2.11k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 17, False: 2.09k] | | ------------------ ------------------ 1172| 2.09k| const uint8_t* match_src = d_ptr - offset; 1173| | 1174| 2.09k| zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml); 1175| 2.09k| d_ptr += ml; 1176| 4.71k| } else { 1177| 4.71k| DECODE_SEQ_SAFE(ll, ml, offset); ------------------ | | 405| 4.71k| do { \ | | 406| 4.71k| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | 407| 4.71k| l_ptr += ll; \ | | 408| 4.71k| d_ptr += ll; \ | | 409| 4.71k| if (UNLIKELY((size_t)(d_ptr - d_floor) < (off))) return ZXC_ERROR_BAD_OFFSET; \ | | ------------------ | | | | 151| 4.71k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | ------------------ | | | | | Branch (151:21): [True: 38, False: 4.68k] | | | | ------------------ | | ------------------ | | 410| 4.71k| zxc_decode_copy_match(d_ptr, off, ml); \ | | 411| 4.68k| d_ptr += ml; \ | | 412| 4.68k| } while (0) | | ------------------ | | | Branch (412:14): [Folded, False: 4.68k] | | ------------------ ------------------ 1178| 4.71k| } 1179| 6.77k| n_seq--; 1180| 6.77k| } 1181| | 1182| | // --- FAST Loop: After threshold, check large margin to avoid individual bounds checks --- 1183| 2.13k| if (safe) { ------------------ | Branch (1183:9): [True: 0, False: 2.13k] ------------------ 1184| 0| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) { ------------------ | Branch (1184:16): [True: 0, False: 0] | Branch (1184:30): [True: 0, False: 0] | Branch (1184:52): [True: 0, False: 0] ------------------ 1185| 0| const uint8_t* const t_save = seq_ptr; 1186| 0| const uint8_t* const e_save = extras_ptr; 1187| 0| uint8_t* const d_save = d_ptr; 1188| 0| const uint8_t* const l_save = l_ptr; 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, ------------------ | | 543| 0| do { \ | | 544| 0| uint32_t s1 = zxc_le32(seq_ptr); \ | | 545| 0| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 546| 0| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 547| 0| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 548| 0| seq_ptr += 4 * sizeof(uint32_t); \ | | 549| 0| PREFETCH; \ | | 550| 0| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 551| 0| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 552| 0| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 553| 0| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1189| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, goto rollback_fast_4x, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 554| 0| n_seq -= 4; \ | | 555| 0| } while (0) | | ------------------ | | | Branch (555:14): [Folded, False: 0] | | ------------------ ------------------ 1190| 0| ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE)); 1191| 0| continue; 1192| | 1193| 0| rollback_fast_4x: 1194| 0| seq_ptr = t_save; 1195| 0| extras_ptr = e_save; 1196| 0| d_ptr = d_save; 1197| 0| l_ptr = l_save; 1198| 0| break; 1199| 0| } 1200| 2.13k| } else { 1201| 2.13k| while (n_seq >= 4 && d_ptr < d_end_fast && l_ptr < l_end_safe_4x) { ------------------ | Branch (1201:16): [True: 32, False: 2.10k] | Branch (1201:30): [True: 2, False: 30] | Branch (1201:52): [True: 0, False: 2] ------------------ 1202| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, ------------------ | | 543| 0| do { \ | | 544| 0| uint32_t s1 = zxc_le32(seq_ptr); \ | | 545| 0| uint32_t s2 = zxc_le32(seq_ptr + sizeof(uint32_t)); \ | | 546| 0| uint32_t s3 = zxc_le32(seq_ptr + 2 * sizeof(uint32_t)); \ | | 547| 0| uint32_t s4 = zxc_le32(seq_ptr + 3 * sizeof(uint32_t)); \ | | 548| 0| seq_ptr += 4 * sizeof(uint32_t); \ | | 549| 0| PREFETCH; \ | | 550| 0| DECODE_GHI_SEQ(s1, (s2 >> 24) + (s3 >> 24) + (s4 >> 24), 3U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1202| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 551| 0| DECODE_GHI_SEQ(s2, (s3 >> 24) + (s4 >> 24), 2U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1202| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 552| 0| DECODE_GHI_SEQ(s3, (s4 >> 24), 1U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1202| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 553| 0| DECODE_GHI_SEQ(s4, 0, 0U, DECODE, ON_FAIL); \ | | ------------------ | | | | 516| 0| do { \ | | | | 517| 0| uint64_t ll = (S) >> 24; \ | | | | 518| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 519| 0| ll += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 520| 0| const uint64_t reserve = (RESERVE); \ | | | | 521| 0| if (UNLIKELY(ll + reserve > (size_t)(l_end - l_ptr) || \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | | Branch (151:42): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 522| 0| ll + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr))) \ | | | | 523| 0| ON_FAIL; \ | | | | 524| 0| } \ | | | | 525| 0| const uint32_t mb = ((S) >> 16) & 0xFF; \ | | | | 526| 0| uint64_t ml = mb + ZXC_LZ_MIN_MATCH_LEN; \ | | | | ------------------ | | | | | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 | | | | ------------------ | | | | 527| 0| if (UNLIKELY(mb == ZXC_SEQ_ML_MASK)) { \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 528| 0| ml += zxc_read_varint(&extras_ptr, extras_end); \ | | | | 529| 0| if (UNLIKELY(ll + ml + (N_REM) * ZXC_GHI_MAX_INLINE_OUT_PER_SEQ + ZXC_PAD_SIZE > \ | | | | ------------------ | | | | | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | | | | | ------------------ | | | | | | | Branch (151:21): [True: 0, False: 0] | | | | | | ------------------ | | | | ------------------ | | | | 530| 0| (size_t)(d_end - d_ptr))) \ | | | | 531| 0| ON_FAIL; \ | | | | 532| 0| } \ | | | | 533| 0| const uint32_t off = ((S) & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; \ | | | | ------------------ | | | | | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 | | | | ------------------ | | | | 534| 0| DECODE(ll, ml, off); \ | | | | ------------------ | | | | | | 1202| 0| DECODE_GHI_BATCH_4X(DECODE_SEQ_FAST, return ZXC_ERROR_OVERFLOW, | | | | | | ------------------ | | | | | | | | 417| 0| do { \ | | | | | | | | 418| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); \ | | | | | | | | 419| 0| l_ptr += ll; \ | | | | | | | | 420| 0| d_ptr += ll; \ | | | | | | | | 421| 0| zxc_decode_copy_match(d_ptr, off, ml); \ | | | | | | | | 422| 0| d_ptr += ml; \ | | | | | | | | 423| 0| } while (0) | | | | | | | | ------------------ | | | | | | | | | Branch (423:14): [Folded, False: 0] | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 535| 0| } while (0) | | | | ------------------ | | | | | Branch (535:14): [Folded, False: 0] | | | | ------------------ | | ------------------ | | 554| 0| n_seq -= 4; \ | | 555| 0| } while (0) | | ------------------ | | | Branch (555:14): [Folded, False: 0] | | ------------------ ------------------ 1203| 0| ZXC_PREFETCH_READ(l_ptr + ZXC_CACHE_LINE_SIZE)); 1204| 0| } 1205| 2.13k| } 1206| | 1207| | // --- Remaining 1 sequence (Fast Path) --- 1208| 2.13k| while (n_seq > 0 && d_ptr < d_end_safe && l_ptr < l_end_safe_1x) { ------------------ | Branch (1208:12): [True: 51, False: 2.08k] | Branch (1208:25): [True: 41, False: 10] | Branch (1208:47): [True: 0, False: 41] ------------------ 1209| | // Save state for fallback 1210| 0| const uint8_t* seq_save = seq_ptr; 1211| 0| const uint8_t* ext_save = extras_ptr; 1212| | 1213| 0| const uint32_t seq = zxc_le32(seq_ptr); 1214| 0| seq_ptr += sizeof(uint32_t); 1215| | 1216| 0| uint64_t ll = seq >> 24; 1217| 0| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) { ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1218| 0| ll += zxc_read_varint(&extras_ptr, extras_end); 1219| 0| if (UNLIKELY(l_ptr + ll > l_end)) { ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1220| 0| seq_ptr = seq_save; 1221| 0| extras_ptr = ext_save; 1222| 0| break; 1223| 0| } 1224| 0| } 1225| | 1226| 0| uint32_t m_bits = (seq >> 16) & 0xFF; 1227| 0| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 0|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1228| 0| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 1229| | 1230| | // Strict bounds checks (including wild copy overrun safety) 1231| 0| if (UNLIKELY(ll + ml + ZXC_PAD_SIZE > (size_t)(d_end - d_ptr) || ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1232| 0| ll + ZXC_PAD_SIZE > (size_t)(l_end - l_ptr))) { 1233| 0| seq_ptr = seq_save; 1234| 0| extras_ptr = ext_save; 1235| 0| break; 1236| 0| } 1237| 0| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 527| 0|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1238| | 1239| 0| zxc_decode_copy_literals(d_ptr, l_ptr, ll); 1240| 0| l_ptr += ll; 1241| 0| d_ptr += ll; 1242| | 1243| 0| if (UNLIKELY(d_ptr < d_bounds && (size_t)(d_ptr - d_floor) < offset)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 1244| 0| return ZXC_ERROR_BAD_OFFSET; 1245| | 1246| | /* The loop entry check guarantees ll + ml + ZXC_PAD_SIZE bytes of 1247| | * headroom, so the wild-copy ladder (incl. overlap/fill runs) is safe. */ 1248| 0| zxc_decode_copy_match(d_ptr, offset, ml); 1249| 0| d_ptr += ml; 1250| 0| n_seq--; 1251| 0| } 1252| | 1253| | // --- Safe Path for Remaining Sequences --- 1254| 2.44k| while (n_seq > 0) { ------------------ | Branch (1254:12): [True: 344, False: 2.09k] ------------------ 1255| 344| uint32_t seq = zxc_le32(seq_ptr); 1256| 344| seq_ptr += sizeof(uint32_t); 1257| | 1258| 344| uint64_t ll = seq >> 24; 1259| 344| if (UNLIKELY(ll == ZXC_SEQ_LL_MASK)) ll += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 344|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 19, False: 325] | | ------------------ ------------------ 1260| | 1261| 344| uint32_t m_bits = (seq >> 16) & 0xFF; 1262| 344| uint64_t ml = m_bits + ZXC_LZ_MIN_MATCH_LEN; ------------------ | | 508| 344|#define ZXC_LZ_MIN_MATCH_LEN 5 ------------------ 1263| 344| if (UNLIKELY(m_bits == ZXC_SEQ_ML_MASK)) ml += zxc_read_varint(&extras_ptr, extras_end); ------------------ | | 151| 344|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 68, False: 276] | | ------------------ ------------------ 1264| 344| uint32_t offset = (seq & 0xFFFF) + ZXC_LZ_OFFSET_BIAS; ------------------ | | 527| 344|#define ZXC_LZ_OFFSET_BIAS 1 ------------------ 1265| | 1266| 344| if (UNLIKELY(ll + ml > (size_t)(d_end - d_ptr) || l_ptr + ll > l_end)) ------------------ | | 151| 660|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 39, False: 305] | | | Branch (151:42): [True: 28, False: 316] | | | Branch (151:42): [True: 11, False: 305] | | ------------------ ------------------ 1267| 39| return ZXC_ERROR_OVERFLOW; 1268| 305| ZXC_MEMCPY(d_ptr, l_ptr, ll); ------------------ | | 173| 305|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1269| 305| l_ptr += ll; 1270| 305| d_ptr += ll; 1271| | 1272| 305| if (UNLIKELY((size_t)(d_ptr - d_floor) < offset)) return ZXC_ERROR_BAD_OFFSET; ------------------ | | 151| 305|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 305] | | ------------------ ------------------ 1273| 305| const uint8_t* match_src = d_ptr - offset; 1274| | 1275| 305| zxc_decode_copy_match_exact(d_ptr, match_src, offset, ml); 1276| 305| d_ptr += ml; 1277| 305| n_seq--; 1278| 305| } 1279| | 1280| | // --- Trailing Literals --- 1281| | // Copy remaining literals from source stream (literal exhaustion) 1282| 2.09k| if (UNLIKELY(l_ptr > l_end)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 2.09k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 2.09k] | | ------------------ ------------------ 1283| 2.09k| const size_t remaining_literals = (size_t)(l_end - l_ptr); 1284| 2.09k| if (UNLIKELY(remaining_literals > (size_t)(d_end - d_ptr))) return ZXC_ERROR_OVERFLOW; ------------------ | | 151| 2.09k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 5, False: 2.09k] | | ------------------ ------------------ 1285| 2.09k| ZXC_MEMCPY(d_ptr, l_ptr, remaining_literals); ------------------ | | 173| 2.09k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1286| 2.09k| d_ptr += remaining_literals; 1287| | 1288| 2.09k| return (int)(d_ptr - dst); 1289| 2.09k|} zxc_decompress.c:zxc_decode_block_ghi: 1352| 2.39k| const size_t dst_capacity) { 1353| 2.39k| return zxc_decode_block_ghi_impl(ctx, src, src_size, dst, dst_capacity, 0, 0); 1354| 2.39k|} zxc_decompress_chunk_wrapper: 488| 47.7k| const size_t src_sz, uint8_t* RESTRICT dst, const size_t dst_cap) { 489| | /* dict_size is constant for a stream; this per-block branch (outside the decode 490| | * loop) routes to the dict variant only when a dictionary is active, so the 491| | * no-dict path runs the dict-free chunk wrapper (identical codegen to main). */ 492| 47.7k|#if ZXC_USE_C11_ATOMICS 493| 47.7k| const zxc_decompress_func_t func = atomic_load_explicit( 494| 47.7k| ctx->dict_size ? &zxc_decompress_dict_ptr : &zxc_decompress_ptr, memory_order_acquire); ------------------ | Branch (494:9): [True: 0, False: 47.7k] ------------------ 495| |#else 496| | const zxc_decompress_func_t func = 497| | ctx->dict_size ? zxc_decompress_dict_ptr : zxc_decompress_ptr; 498| |#endif 499| 47.7k| if (UNLIKELY(!func)) return zxc_decompress_dispatch_init(ctx, src, src_sz, dst, dst_cap); ------------------ | | 151| 47.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1, False: 47.7k] | | ------------------ ------------------ 500| 47.7k| return func(ctx, src, src_sz, dst, dst_cap); 501| 47.7k|} zxc_decompress: 855| 6.60k| const size_t dst_capacity, const zxc_decompress_opts_t* opts) { 856| 6.60k| if (UNLIKELY(!src || src_size < ZXC_FILE_HEADER_SIZE + ZXC_FILE_FOOTER_SIZE || ------------------ | | 151| 32.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 9, False: 6.59k] | | | Branch (151:42): [True: 0, False: 6.60k] | | | Branch (151:42): [True: 9, False: 6.59k] | | | Branch (151:42): [True: 0, False: 6.59k] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 857| 6.60k| (!dst && dst_capacity != 0))) 858| 9| return ZXC_ERROR_NULL_INPUT; 859| | 860| 6.59k| if (UNLIKELY(!dst || dst_capacity == 0)) { ------------------ | | 151| 13.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 6.59k] | | | Branch (151:42): [True: 0, False: 6.59k] | | | Branch (151:42): [True: 0, False: 6.59k] | | ------------------ ------------------ 861| | /* Empty-frame case (stored size == 0). */ 862| 0| if (UNLIKELY(zxc_le32(src) != ZXC_MAGIC_WORD)) return ZXC_ERROR_NULL_INPUT; ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 863| 0| const uint8_t* footer = (const uint8_t*)src + src_size - ZXC_FILE_FOOTER_SIZE; ------------------ | | 105| 0|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 864| 0| return (zxc_le64(footer) == 0) ? 0 : (int64_t)ZXC_ERROR_DST_TOO_SMALL; ------------------ | Branch (864:16): [True: 0, False: 0] ------------------ 865| 0| } 866| | 867| 6.59k| return zxc_decompress_frame((const uint8_t*)src, src_size, (uint8_t*)dst, dst_capacity, opts); 868| 6.59k|} zxc_get_decompressed_size: 1164| 6.60k|uint64_t zxc_get_decompressed_size(const void* src, const size_t src_size) { 1165| 6.60k| if (UNLIKELY(src_size < ZXC_FILE_HEADER_SIZE + ZXC_FILE_FOOTER_SIZE)) return 0; ------------------ | | 151| 6.60k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 9, False: 6.59k] | | ------------------ ------------------ 1166| | 1167| 6.59k| const uint8_t* const p = (const uint8_t*)src; 1168| 6.59k| if (UNLIKELY(zxc_le32(p) != ZXC_MAGIC_WORD)) return 0; ------------------ | | 151| 6.59k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 57, False: 6.53k] | | ------------------ ------------------ 1169| | 1170| 6.53k| size_t chunk_size = 0; 1171| 6.53k| int has_cs = 0; 1172| 6.53k| uint32_t did = 0; 1173| 6.53k| if (UNLIKELY(zxc_read_file_header(p, src_size, &chunk_size, &has_cs, &did) != ZXC_OK)) return 0; ------------------ | | 151| 6.53k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 46, False: 6.48k] | | ------------------ ------------------ 1174| | 1175| 6.48k| const uint8_t* const footer = p + src_size - ZXC_FILE_FOOTER_SIZE; ------------------ | | 105| 6.48k|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 1176| 6.48k| const uint64_t dsize = zxc_le64(footer); 1177| | 1178| | /* Plausibility: at most src_size / ZXC_BLOCK_HEADER_SIZE blocks, each 1179| | * decoding to at most chunk_size. The division form keeps the compare 1180| | * overflow-free - a ceil would wrap on a forged dsize near UINT64_MAX. */ 1181| 6.48k| const uint64_t blocks_needed = 1182| 6.48k| chunk_size ? dsize / (uint64_t)chunk_size + (dsize % (uint64_t)chunk_size != 0) : 0; ------------------ | Branch (1182:9): [True: 6.48k, False: 0] ------------------ 1183| 6.48k| if (UNLIKELY(blocks_needed > (uint64_t)(src_size / ZXC_BLOCK_HEADER_SIZE))) return 0; ------------------ | | 151| 6.48k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 5.76k, False: 722] | | ------------------ ------------------ 1184| | 1185| 722| return dsize; 1186| 6.48k|} zxc_dispatch.c:zxc_decompress_dispatch_init: 319| 1| const size_t dst_cap) { 320| 1| const zxc_cpu_feature_t cpu = zxc_detect_cpu_features(); 321| 1| zxc_decompress_func_t zxc_decompress_ptr_local = NULL; 322| 1| zxc_decompress_func_t zxc_decompress_dict_ptr_local = NULL; 323| | 324| |#ifndef ZXC_ONLY_DEFAULT 325| |#if defined(__x86_64__) || defined(_M_X64) 326| | if (cpu == ZXC_CPU_AVX512) { 327| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_avx512; 328| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_avx512; 329| | } else if (cpu == ZXC_CPU_AVX2) { 330| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_avx2; 331| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_avx2; 332| | } else { 333| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; 334| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_default; 335| | } 336| |#elif defined(__arm__) || defined(_M_ARM) 337| | // 32-bit ARM: the only arch with a real runtime NEON probe (getauxval). 338| | // cppcheck-suppress knownConditionTrueFalse 339| | if (cpu == ZXC_CPU_NEON) { 340| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_neon32; 341| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_neon32; 342| | } else { 343| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; 344| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_default; 345| | } 346| |#else 347| | (void)cpu; 348| | zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; 349| | zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_default; 350| |#endif 351| |#else 352| 1| (void)cpu; 353| 1| zxc_decompress_ptr_local = zxc_decompress_chunk_wrapper_default; 354| 1| zxc_decompress_dict_ptr_local = zxc_decompress_chunk_wrapper_dict_default; 355| 1|#endif 356| | 357| 1|#if ZXC_USE_C11_ATOMICS 358| 1| atomic_store_explicit(&zxc_decompress_ptr, zxc_decompress_ptr_local, memory_order_release); 359| 1| atomic_store_explicit(&zxc_decompress_dict_ptr, zxc_decompress_dict_ptr_local, 360| 1| memory_order_release); 361| |#else 362| | zxc_decompress_ptr = zxc_decompress_ptr_local; 363| | zxc_decompress_dict_ptr = zxc_decompress_dict_ptr_local; 364| |#endif 365| 1| return (ctx->dict_size ? zxc_decompress_dict_ptr_local : zxc_decompress_ptr_local)( ------------------ | Branch (365:13): [True: 0, False: 1] ------------------ 366| 1| ctx, src, src_sz, dst, dst_cap); 367| 1|} zxc_dispatch.c:zxc_detect_cpu_features: 206| 1|static zxc_cpu_feature_t zxc_detect_cpu_features(void) { 207| 1|#ifdef ZXC_ONLY_DEFAULT 208| 1| return ZXC_CPU_GENERIC; 209| |#else 210| | zxc_cpu_feature_t features = ZXC_CPU_GENERIC; 211| | 212| |#if defined(__x86_64__) || defined(_M_X64) 213| | // AVX2/AVX512 need OS-enabled YMM/ZMM state: gate on OSXSAVE + XGETBV/XCR0, 214| | // not CPUID alone (else a VEX/EVEX op faults #UD when the OS hasn't enabled it). 215| | uint32_t regs[4]; 216| | int sse2 = 0; 217| | int avx2 = 0; 218| | int avx512 = 0; 219| | int bmi_lzcnt = 0; 220| | 221| | zxc_cpuid(0, 0, regs); 222| | const uint32_t max_leaf = regs[0]; 223| | 224| | zxc_cpuid(1, 0, regs); 225| | if (regs[3] & (1U << 26)) sse2 = 1; // CPUID.1:EDX[26] 226| | if ((regs[2] & (1U << 27)) && max_leaf >= 7) { // OSXSAVE, and leaf 7 is real 227| | const uint64_t xcr0 = zxc_xgetbv0(); 228| | if ((xcr0 & 0x6) == 0x6) { // SSE+YMM enabled 229| | zxc_cpuid(7, 0, regs); 230| | const int bmi1 = (regs[1] >> 3) & 1; // CPUID.7.0:EBX[3] 231| | const int bmi2 = (regs[1] >> 8) & 1; // CPUID.7.0:EBX[8] 232| | if (regs[1] & (1U << 5)) avx2 = 1; 233| | // AVX512 also needs XCR0[5..7] (opmask/ZMM) 234| | if ((regs[1] & (1U << 16)) && (regs[1] & (1U << 30)) && (regs[2] & (1U << 6)) && 235| | (xcr0 & 0xE0) == 0xE0) 236| | avx512 = 1; /* AVX512 tier = F+BW+VBMI2 (variant built with -mavx512vbmi2) */ 237| | // The AVX2/AVX512 variants are compiled with BMI1/BMI2/LZCNT enabled, 238| | // so both gates must prove those bits too. LZCNT (ABM) lives in 239| | // CPUID.80000001H:ECX[5]; that leaf is architectural on x86-64. 240| | zxc_cpuid(0x80000001U, 0, regs); 241| | bmi_lzcnt = bmi1 && bmi2 && ((regs[2] >> 5) & 1); 242| | } 243| | } 244| | 245| | if (avx512 && bmi_lzcnt) { 246| | features = ZXC_CPU_AVX512; 247| | } else if (avx2 && bmi_lzcnt) { 248| | features = ZXC_CPU_AVX2; 249| | } else if (sse2) { 250| | features = ZXC_CPU_SSE2; 251| | } 252| | 253| |#elif defined(__aarch64__) || defined(_M_ARM64) 254| | // ARM64 usually guarantees NEON 255| | features = ZXC_CPU_NEON; 256| | 257| |#elif defined(__arm__) || defined(_M_ARM) 258| | // ARM32 Runtime detection for Linux 259| |#if defined(__linux__) 260| | const unsigned long hwcaps = getauxval(AT_HWCAP); 261| | if (hwcaps & HWCAP_NEON) { 262| | features = ZXC_CPU_NEON; 263| | } 264| |#else 265| |// Fallback for non-Linux: rely on compiler flags. 266| |// If compiled with -mfpu=neon, we assume target supports it. 267| |// Otherwise, safe default is GENERIC. 268| |#if defined(__ARM_NEON) 269| | features = ZXC_CPU_NEON; 270| |#endif 271| |#endif 272| |#endif 273| | 274| | return features; 275| |#endif 276| 1|} zxc_dispatch.c:zxc_decompress_frame: 871| 6.59k| const size_t dst_capacity, const zxc_decompress_opts_t* opts) { 872| 6.59k| const int checksum_enabled = opts ? opts->checksum_enabled : 0; ------------------ | Branch (872:34): [True: 0, False: 6.59k] ------------------ 873| 6.59k| const uint8_t* dict = opts ? (const uint8_t*)opts->dict : NULL; ------------------ | Branch (873:27): [True: 0, False: 6.59k] ------------------ 874| 6.59k| const size_t dict_size = (opts && opts->dict) ? opts->dict_size : 0; ------------------ | Branch (874:31): [True: 0, False: 6.59k] | Branch (874:39): [True: 0, False: 0] ------------------ 875| 6.59k| const uint8_t* dict_huf = (opts && opts->dict) ? (const uint8_t*)opts->dict_huf : NULL; ------------------ | Branch (875:32): [True: 0, False: 6.59k] | Branch (875:40): [True: 0, False: 0] ------------------ 876| | 877| 6.59k| const uint8_t* ip = src; 878| 6.59k| const uint8_t* ip_end = ip + src_size; 879| 6.59k| uint8_t* op = dst; 880| 6.59k| const uint8_t* op_start = op; 881| 6.59k| const uint8_t* op_end = op + dst_capacity; 882| 6.59k| size_t runtime_chunk_size = 0; 883| 6.59k| zxc_cctx_t ctx; 884| | 885| 6.59k| int file_has_checksums = 0; 886| 6.59k| uint32_t header_dict_id = 0; 887| 6.59k| if (UNLIKELY(zxc_read_file_header(ip, src_size, &runtime_chunk_size, &file_has_checksums, ------------------ | | 151| 20.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 103, False: 6.48k] | | | Branch (151:42): [True: 1.14k, False: 5.34k] | | | Branch (151:42): [True: 0, False: 1.14k] | | | Branch (151:42): [True: 103, False: 6.48k] | | | Branch (151:42): [True: 0, False: 6.48k] | | ------------------ ------------------ 888| 6.59k| &header_dict_id) != ZXC_OK || 889| 6.59k| zxc_cctx_init(&ctx, runtime_chunk_size, 0, 0, 890| 6.59k| file_has_checksums && checksum_enabled, dict_size) != ZXC_OK)) { 891| 103| return ZXC_ERROR_BAD_HEADER; 892| 103| } 893| | 894| | /* Dictionary validation */ 895| 6.48k| if (header_dict_id != 0) { ------------------ | Branch (895:9): [True: 13, False: 6.47k] ------------------ 896| 13| if (UNLIKELY(!dict || dict_size == 0)) { ------------------ | | 151| 13|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 13, False: 0] | | | Branch (151:42): [True: 13, False: 0] | | | Branch (151:42): [True: 0, False: 0] | | ------------------ ------------------ 897| 13| zxc_cctx_free(&ctx); 898| 13| return ZXC_ERROR_DICT_REQUIRED; 899| 13| } 900| 0| if (UNLIKELY(zxc_dict_id(dict, dict_size, dict_huf) != header_dict_id)) { ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 901| 0| zxc_cctx_free(&ctx); 902| 0| return ZXC_ERROR_DICT_MISMATCH; 903| 0| } 904| 0| } 905| 6.47k| if (UNLIKELY(zxc_cctx_attach_dict_huf(&ctx, dict_huf) != ZXC_OK)) { ------------------ | | 151| 6.47k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 6.47k] | | ------------------ ------------------ 906| | // LCOV_EXCL_START 907| 0| zxc_cctx_free(&ctx); 908| 0| return ZXC_ERROR_CORRUPT_DATA; 909| | // LCOV_EXCL_STOP 910| 0| } 911| | 912| 6.47k| ip += ZXC_FILE_HEADER_SIZE; ------------------ | | 103| 6.47k|#define ZXC_FILE_HEADER_SIZE 16 ------------------ 913| | 914| 6.47k| const size_t work_sz = runtime_chunk_size + ZXC_DECOMPRESS_TAIL_PAD; ------------------ | | 349| 6.47k|#define ZXC_DECOMPRESS_TAIL_PAD (ZXC_PAD_SIZE * 66) | | ------------------ | | | | 337| 6.47k|#define ZXC_PAD_SIZE 32 | | ------------------ ------------------ 915| | 916| | /* Dict decode buffer: [dict_content | decode_space + PAD], carved into the 917| | * cctx workspace (NULL when no dictionary is active). */ 918| 6.47k| uint8_t* const dict_dec = ctx.dict_buffer; 919| 6.47k| if (dict_dec) ZXC_MEMCPY(dict_dec, dict, dict_size); ------------------ | | 173| 0|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ | Branch (919:9): [True: 0, False: 6.47k] ------------------ 920| | 921| | // Block decompression loop 922| 6.47k| uint32_t global_hash = 0; 923| | 924| 50.7k| while (ip < ip_end) { ------------------ | Branch (924:12): [True: 49.3k, False: 1.34k] ------------------ 925| 49.3k| const size_t rem_src = (size_t)(ip_end - ip); 926| 49.3k| zxc_block_header_t bh; 927| | // Read the block header to determine the compressed size 928| 49.3k| if (UNLIKELY(zxc_read_block_header(ip, rem_src, &bh) != ZXC_OK)) { ------------------ | | 151| 49.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1.44k, False: 47.9k] | | ------------------ ------------------ 929| 1.44k| zxc_cctx_free(&ctx); 930| 1.44k| return ZXC_ERROR_BAD_HEADER; 931| 1.44k| } 932| | 933| | // Handle EOF block separately (not a real chunk to decompress) 934| 47.9k| if (UNLIKELY(bh.block_type == ZXC_BLOCK_EOF)) { ------------------ | | 151| 47.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 179, False: 47.7k] | | ------------------ ------------------ 935| | // EOF carries no payload; a non-zero comp_size is a malformed header. 936| 179| if (UNLIKELY(bh.comp_size != 0)) { ------------------ | | 151| 179|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 58, False: 121] | | ------------------ ------------------ 937| 58| zxc_cctx_free(&ctx); 938| 58| return ZXC_ERROR_BAD_HEADER; 939| 58| } 940| | // Footer is always the last ZXC_FILE_FOOTER_SIZE bytes of the source, 941| | // even when a seek table is inserted between EOF block and footer. 942| | // LCOV_EXCL_START 943| 121| if (UNLIKELY(src_size < ZXC_FILE_FOOTER_SIZE)) { ------------------ | | 151| 121|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 121] | | ------------------ ------------------ 944| 0| zxc_cctx_free(&ctx); 945| 0| return ZXC_ERROR_SRC_TOO_SMALL; 946| 0| } 947| | // LCOV_EXCL_STOP 948| 121| const uint8_t* const footer = src + src_size - ZXC_FILE_FOOTER_SIZE; ------------------ | | 105| 121|#define ZXC_FILE_FOOTER_SIZE 12 ------------------ 949| | 950| | // Validate source size matches what we decompressed 951| 121| const uint64_t stored_size = zxc_le64(footer); 952| 121| if (UNLIKELY(stored_size != (uint64_t)(op - op_start))) { ------------------ | | 151| 121|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 120, False: 1] | | ------------------ ------------------ 953| 120| zxc_cctx_free(&ctx); 954| 120| return ZXC_ERROR_CORRUPT_DATA; 955| 120| } 956| | 957| | // Validate global checksum if enabled and file has checksums 958| 1| if (checksum_enabled && file_has_checksums) { ------------------ | Branch (958:17): [True: 0, False: 1] | Branch (958:37): [True: 0, False: 0] ------------------ 959| 0| const uint32_t stored_hash = zxc_le32(footer + sizeof(uint64_t)); 960| 0| if (UNLIKELY(stored_hash != global_hash)) { ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 961| 0| zxc_cctx_free(&ctx); 962| 0| return ZXC_ERROR_BAD_CHECKSUM; 963| 0| } 964| 0| } 965| 1| break; // EOF reached, exit loop 966| 1| } 967| | 968| 47.7k| int res; 969| 47.7k| const size_t rem_cap = (size_t)(op_end - op); 970| 47.7k| if (dict_dec) { ------------------ | Branch (970:13): [True: 0, False: 47.7k] ------------------ 971| | /* Dict path: decode into bounce buffer with dict prefix so match 972| | * copies that reference dict content resolve naturally. */ 973| 0| res = zxc_decompress_chunk_wrapper(&ctx, ip, rem_src, dict_dec + dict_size, work_sz); 974| 0| if (LIKELY(res > 0)) { ------------------ | | 145| 0|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 0, False: 0] | | ------------------ ------------------ 975| 0| if (UNLIKELY((size_t)res > rem_cap)) { ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 976| | // LCOV_EXCL_START 977| 0| zxc_cctx_free(&ctx); 978| 0| return ZXC_ERROR_DST_TOO_SMALL; 979| | // LCOV_EXCL_STOP 980| 0| } 981| 0| ZXC_MEMCPY(op, dict_dec + dict_size, (size_t)res); ------------------ | | 173| 0|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 982| 0| } 983| 47.7k| } else if (LIKELY(rem_cap >= work_sz)) { ------------------ | | 145| 47.7k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 45.6k, False: 2.05k] | | ------------------ ------------------ 984| | // Fast path: decode directly into dst. Cap dst_cap to chunk_size + PAD 985| 45.6k| res = zxc_decompress_chunk_wrapper(&ctx, ip, rem_src, op, work_sz); 986| 45.6k| } else { 987| | // Safe path: decode into bounce buffer, then copy exact result. 988| 2.05k| res = zxc_decompress_chunk_wrapper(&ctx, ip, rem_src, ctx.work_buf, ctx.work_buf_cap); 989| 2.05k| if (LIKELY(res > 0)) { ------------------ | | 145| 2.05k|#define LIKELY(x) (__builtin_expect(!!(x), 1)) | | ------------------ | | | Branch (145:19): [True: 1.29k, False: 754] | | ------------------ ------------------ 990| | // LCOV_EXCL_START 991| 1.29k| if (UNLIKELY((size_t)res > rem_cap)) { ------------------ | | 151| 1.29k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 35, False: 1.26k] | | ------------------ ------------------ 992| 35| zxc_cctx_free(&ctx); 993| 35| return ZXC_ERROR_DST_TOO_SMALL; 994| 35| } 995| | // LCOV_EXCL_STOP 996| 1.26k| ZXC_MEMCPY(op, ctx.work_buf, (size_t)res); ------------------ | | 173| 1.26k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 997| 1.26k| } 998| 2.05k| } 999| 47.7k| if (UNLIKELY(res < 0)) { ------------------ | | 151| 47.7k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 3.47k, False: 44.2k] | | ------------------ ------------------ 1000| 3.47k| zxc_cctx_free(&ctx); 1001| 3.47k| return res; 1002| 3.47k| } 1003| | 1004| | // Update global hash from block checksum 1005| 44.2k| if (checksum_enabled && file_has_checksums) { ------------------ | Branch (1005:13): [True: 0, False: 44.2k] | Branch (1005:33): [True: 0, False: 0] ------------------ 1006| 0| const uint32_t block_hash = zxc_le32(ip + ZXC_BLOCK_HEADER_SIZE + bh.comp_size); ------------------ | | 397| 0|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1007| 0| global_hash = zxc_hash_combine_rotate(global_hash, block_hash); 1008| 0| } 1009| | 1010| 44.2k| ip += ZXC_BLOCK_HEADER_SIZE + bh.comp_size + ------------------ | | 397| 44.2k|#define ZXC_BLOCK_HEADER_SIZE 8 ------------------ 1011| 44.2k| (file_has_checksums ? ZXC_BLOCK_CHECKSUM_SIZE : 0); ------------------ | | 399| 426|#define ZXC_BLOCK_CHECKSUM_SIZE 4 ------------------ | Branch (1011:16): [True: 426, False: 43.8k] ------------------ 1012| 44.2k| op += res; 1013| 44.2k| } 1014| | 1015| 1.34k| zxc_cctx_free(&ctx); 1016| 1.34k| return (int64_t)(op - op_start); 1017| 6.47k|} zxc_huf_unpack_lengths_default: 1001| 23.2k|int zxc_huf_unpack_lengths(const uint8_t* RESTRICT in, uint8_t* RESTRICT code_len) { 1002| 23.2k| int max_len = 0; 1003| 23.2k| int n_present = 0; 1004| 3.00M| for (int i = 0; i < ZXC_HUF_NUM_SYMBOLS; i += 2) { ------------------ | | 589| 3.00M|#define ZXC_HUF_NUM_SYMBOLS 256 ------------------ | Branch (1004:21): [True: 2.97M, False: 23.2k] ------------------ 1005| 2.97M| const uint8_t b = in[i >> 1]; 1006| 2.97M| const uint8_t lo = b & 0x0F; 1007| 2.97M| const uint8_t hi = (uint8_t)(b >> 4); 1008| 2.97M| code_len[i] = lo; 1009| 2.97M| code_len[i + 1] = hi; 1010| 2.97M| if (lo > max_len) max_len = lo; ------------------ | Branch (1010:13): [True: 4.58k, False: 2.97M] ------------------ 1011| 2.97M| if (hi > max_len) max_len = hi; ------------------ | Branch (1011:13): [True: 22.8k, False: 2.95M] ------------------ 1012| 2.97M| if (lo) n_present++; ------------------ | Branch (1012:13): [True: 51.3k, False: 2.92M] ------------------ 1013| 2.97M| if (hi) n_present++; ------------------ | Branch (1013:13): [True: 83.8k, False: 2.89M] ------------------ 1014| 2.97M| } 1015| 23.2k| if (UNLIKELY(max_len > ZXC_HUF_MAX_CODE_LEN_ULTRA || n_present == 0)) ------------------ | | 151| 46.4k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 61, False: 23.2k] | | | Branch (151:42): [True: 60, False: 23.2k] | | | Branch (151:42): [True: 1, False: 23.2k] | | ------------------ ------------------ 1016| 61| return ZXC_ERROR_CORRUPT_DATA; 1017| 23.2k| return ZXC_OK; 1018| 23.2k|} zxc_huf_decode_section_default: 2298| 23.3k| uint8_t* RESTRICT dst, const size_t n, uint8_t* RESTRICT scratch) { 2299| 23.3k| if (UNLIKELY(payload_size < (size_t)ZXC_HUF_TABLE_SIZE)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 23.3k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 55, False: 23.2k] | | ------------------ ------------------ 2300| 23.2k| uint8_t code_len[ZXC_HUF_NUM_SYMBOLS]; 2301| 23.2k| const int rc = zxc_huf_unpack_lengths(payload, code_len); ------------------ | | 46| 23.2k|#define zxc_huf_unpack_lengths ZXC_CAT(zxc_huf_unpack_lengths, ZXC_FUNCTION_SUFFIX) | | ------------------ | | | | 43| 23.2k|#define ZXC_CAT(x, y) ZXC_CAT_IMPL(x, y) | | | | ------------------ | | | | | | 42| 23.2k|#define ZXC_CAT_IMPL(x, y) x##y | | | | ------------------ | | ------------------ ------------------ 2302| 23.2k| if (UNLIKELY(rc != ZXC_OK)) return rc; ------------------ | | 151| 23.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 61, False: 23.2k] | | ------------------ ------------------ 2303| 23.2k| zxc_pivco_tree_t t; 2304| 23.2k| if (UNLIKELY(zxc_pivco_tree_build(code_len, &t, NULL) != 0)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 23.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 25, False: 23.1k] | | ------------------ ------------------ 2305| 23.1k| return zxc_pivco_decode_core(payload + ZXC_HUF_TABLE_SIZE, payload_size - ZXC_HUF_TABLE_SIZE, ------------------ | | 83| 23.1k|#define ZXC_HUF_TABLE_SIZE 128 ------------------ return zxc_pivco_decode_core(payload + ZXC_HUF_TABLE_SIZE, payload_size - ZXC_HUF_TABLE_SIZE, ------------------ | | 83| 23.1k|#define ZXC_HUF_TABLE_SIZE 128 ------------------ 2306| | dst, n, &t, NULL, scratch); 2307| 23.2k|} zxc_huffman.c:zxc_pivco_tree_build: 1085| 23.2k| uint32_t* RESTRICT codes) { 1086| 23.2k| uint32_t bl_count[ZXC_HUF_MAX_CODE_LEN_ULTRA + 1] = {0}; 1087| 23.2k| int n_present = 0; 1088| 5.96M| for (int s = 0; s < ZXC_HUF_NUM_SYMBOLS; s++) { ------------------ | | 589| 5.96M|#define ZXC_HUF_NUM_SYMBOLS 256 ------------------ | Branch (1088:21): [True: 5.94M, False: 23.2k] ------------------ 1089| 5.94M| const int l = code_len[s]; 1090| 5.94M| if (l == 0) continue; ------------------ | Branch (1090:13): [True: 5.80M, False: 131k] ------------------ 1091| 131k| if (UNLIKELY(l > ZXC_HUF_MAX_CODE_LEN_ULTRA)) return -1; ------------------ | | 151| 131k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 131k] | | ------------------ ------------------ 1092| 131k| bl_count[l]++; 1093| 131k| n_present++; 1094| 131k| } 1095| 23.2k| if (UNLIKELY(n_present == 0)) return -1; ------------------ | | 151| 23.2k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 23.2k] | | ------------------ ------------------ 1096| | 1097| 23.2k| if (n_present >= 2) { ------------------ | Branch (1097:9): [True: 19.9k, False: 3.24k] ------------------ 1098| 19.9k| uint32_t kraft = 0; 1099| 239k| for (int l = 1; l <= ZXC_HUF_MAX_CODE_LEN_ULTRA; l++) ------------------ | | 582| 239k|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 ------------------ | Branch (1099:25): [True: 219k, False: 19.9k] ------------------ 1100| 219k| kraft += bl_count[l] << (ZXC_HUF_MAX_CODE_LEN_ULTRA - l); ------------------ | | 582| 219k|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 ------------------ 1101| 19.9k| if (UNLIKELY(kraft != (1U << ZXC_HUF_MAX_CODE_LEN_ULTRA))) return -1; ------------------ | | 151| 19.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 24, False: 19.9k] | | ------------------ ------------------ 1102| 19.9k| } else { 1103| | /* Degenerate single-symbol table: the format pins the lone symbol at code 1104| | * length exactly 1, so reject longer unary chains. */ 1105| 3.24k| if (UNLIKELY(bl_count[1] != 1)) return -1; ------------------ | | 151| 3.24k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1, False: 3.24k] | | ------------------ ------------------ 1106| 3.24k| } 1107| | 1108| 23.1k| uint32_t next_code[ZXC_HUF_MAX_CODE_LEN_ULTRA + 2] = {0}; 1109| 23.1k| uint32_t code = 0; 1110| 278k| for (int l = 1; l <= ZXC_HUF_MAX_CODE_LEN_ULTRA; l++) { ------------------ | | 582| 278k|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 ------------------ | Branch (1110:21): [True: 254k, False: 23.1k] ------------------ 1111| 254k| code = (code + bl_count[l - 1]) << 1; 1112| 254k| next_code[l] = code; 1113| 254k| } 1114| | 1115| 23.1k| t->n_nodes = 1; /* root */ 1116| 23.1k| t->nd[0].child[0] = -1; 1117| 23.1k| t->nd[0].child[1] = -1; 1118| 23.1k| t->nd[0].sym = -1; 1119| 23.1k| int max_depth = 0; 1120| | 1121| 5.95M| for (int s = 0; s < ZXC_HUF_NUM_SYMBOLS; s++) { ------------------ | | 589| 5.95M|#define ZXC_HUF_NUM_SYMBOLS 256 ------------------ | Branch (1121:21): [True: 5.93M, False: 23.1k] ------------------ 1122| 5.93M| const int l = code_len[s]; 1123| 5.93M| if (l == 0) continue; ------------------ | Branch (1123:13): [True: 5.80M, False: 129k] ------------------ 1124| 129k| const uint32_t c = next_code[l]++; 1125| 129k| if (UNLIKELY(c >> l)) return -1; /* code space overflow */ ------------------ | | 151| 129k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 129k] | | ------------------ ------------------ 1126| 129k| if (codes) codes[s] = c; ------------------ | Branch (1126:13): [True: 0, False: 129k] ------------------ 1127| 129k| int cur = 0; 1128| 763k| for (int d = l - 1; d >= 0; d--) { ------------------ | Branch (1128:29): [True: 634k, False: 129k] ------------------ 1129| 634k| if (UNLIKELY(t->nd[cur].sym >= 0)) return -1; /* prefix collision */ ------------------ | | 151| 634k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 634k] | | ------------------ ------------------ 1130| 634k| const int bit = (int)((c >> d) & 1U); 1131| 634k| int nxt = t->nd[cur].child[bit]; 1132| 634k| if (nxt < 0) { ------------------ | Branch (1132:17): [True: 215k, False: 418k] ------------------ 1133| 215k| if (UNLIKELY(t->n_nodes >= ZXC_PIVCO_MAX_NODES)) return -1; ------------------ | | 151| 215k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 215k] | | ------------------ ------------------ 1134| 215k| nxt = t->n_nodes++; 1135| 215k| t->nd[nxt].child[0] = -1; 1136| 215k| t->nd[nxt].child[1] = -1; 1137| 215k| t->nd[nxt].sym = -1; 1138| 215k| t->nd[cur].child[bit] = (int16_t)nxt; 1139| 215k| } 1140| 634k| cur = nxt; 1141| 634k| } 1142| 129k| if (UNLIKELY(t->nd[cur].child[0] >= 0 || t->nd[cur].child[1] >= 0)) return -1; ------------------ | | 151| 258k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 129k] | | | Branch (151:42): [True: 0, False: 129k] | | | Branch (151:42): [True: 0, False: 129k] | | ------------------ ------------------ 1143| 129k| t->nd[cur].sym = (int16_t)s; 1144| 129k| if (l > max_depth) max_depth = l; ------------------ | Branch (1144:13): [True: 27.2k, False: 102k] ------------------ 1145| 129k| } 1146| 23.1k| t->max_depth = max_depth; 1147| | 1148| | /* BFS order (parents before children, left before right): this is both 1149| | * the wire order of the node bit runs and the property that makes each 1150| | * parent's children CONTIGUOUS in the next level's sequence buffer. */ 1151| 23.1k| int head = 0; 1152| 23.1k| int tail = 0; 1153| 23.1k| t->bfs[tail++] = 0; 1154| 23.1k| int depth_end = 1; /* index in bfs where the current depth ends */ 1155| 23.1k| int depth = 0; 1156| 23.1k| t->lvl_start[0] = 0; 1157| 261k| while (head < tail) { ------------------ | Branch (1157:12): [True: 238k, False: 23.1k] ------------------ 1158| 238k| if (head == depth_end) { ------------------ | Branch (1158:13): [True: 35.0k, False: 203k] ------------------ 1159| 35.0k| depth++; 1160| 35.0k| t->lvl_start[depth] = (int16_t)head; 1161| 35.0k| depth_end = tail; 1162| 35.0k| } 1163| 238k| const int nid = t->bfs[head++]; 1164| 716k| for (int b = 0; b < 2; b++) { ------------------ | Branch (1164:25): [True: 477k, False: 238k] ------------------ 1165| 477k| const int ch = t->nd[nid].child[b]; 1166| 477k| if (ch >= 0) t->bfs[tail++] = (int16_t)ch; ------------------ | Branch (1166:17): [True: 215k, False: 261k] ------------------ 1167| 477k| } 1168| 238k| } 1169| 46.3k| for (int d = depth + 1; d <= max_depth + 1; d++) t->lvl_start[d] = (int16_t)tail; ------------------ | Branch (1169:29): [True: 23.1k, False: 23.1k] ------------------ 1170| | 1171| | /* Flat-subtree detection: min/max leaf depth per node in one reverse-BFS 1172| | * sweep, then maximality by masking descendants of the first flat node on 1173| | * each root-to-leaf path. */ 1174| 23.1k| { 1175| 23.1k| int8_t mn[ZXC_PIVCO_MAX_NODES]; 1176| 23.1k| int8_t mx[ZXC_PIVCO_MAX_NODES]; 1177| 261k| for (int i = t->n_nodes - 1; i >= 0; i--) { ------------------ | Branch (1177:38): [True: 238k, False: 23.1k] ------------------ 1178| 238k| const int nid = t->bfs[i]; 1179| 238k| const zxc_pivco_node_t* nd = &t->nd[nid]; 1180| 238k| if (nd->sym >= 0) { ------------------ | Branch (1180:17): [True: 129k, False: 109k] ------------------ 1181| 129k| mn[nid] = 0; 1182| 129k| mx[nid] = 0; 1183| 129k| } else if (nd->child[0] >= 0 && nd->child[1] >= 0) { ------------------ | Branch (1183:24): [True: 109k, False: 0] | Branch (1183:45): [True: 106k, False: 3.24k] ------------------ 1184| 106k| const int a = mn[nd->child[0]]; 1185| 106k| const int b = mn[nd->child[1]]; 1186| 106k| const int c = mx[nd->child[0]]; 1187| 106k| const int d2 = mx[nd->child[1]]; 1188| 106k| mn[nid] = (int8_t)(1 + (a < b ? a : b)); ------------------ | Branch (1188:41): [True: 10.8k, False: 95.3k] ------------------ 1189| 106k| mx[nid] = (int8_t)(1 + (c > d2 ? c : d2)); ------------------ | Branch (1189:41): [True: 0, False: 106k] ------------------ 1190| 106k| } else { /* single-child (degenerate) : never flat */ 1191| 3.24k| mn[nid] = 0; 1192| 3.24k| mx[nid] = (int8_t)ZXC_HUF_MAX_CODE_LEN_ULTRA; ------------------ | | 582| 3.24k|#define ZXC_HUF_MAX_CODE_LEN_ULTRA 11 ------------------ 1193| 3.24k| } 1194| 238k| } 1195| 261k| for (int i = 0; i < t->n_nodes; i++) { ------------------ | Branch (1195:25): [True: 238k, False: 23.1k] ------------------ 1196| 238k| const int nid = t->bfs[i]; 1197| 238k| t->flat_d[nid] = 0; 1198| 238k| if (i == 0) t->covered[nid] = 0; /* root */ ------------------ | Branch (1198:17): [True: 23.1k, False: 215k] ------------------ 1199| | /* Any complete subtree unpacks flat, beating the merge cascade 1200| | * (FORMAT RULE, both sides): D = 2-6 have SIMD unpackers, D >= 7 the 1201| | * scalar one. (D = 1 is a leaf pair, handled on the merge path.) */ 1202| 238k| if (!t->covered[nid] && t->nd[nid].sym < 0 && mn[nid] == mx[nid] && mn[nid] >= 2) ------------------ | Branch (1202:17): [True: 98.3k, False: 140k] | Branch (1202:37): [True: 45.0k, False: 53.2k] | Branch (1202:59): [True: 28.8k, False: 16.1k] | Branch (1202:81): [True: 5.84k, False: 23.0k] ------------------ 1203| 5.84k| t->flat_d[nid] = (uint8_t)mn[nid]; 1204| 238k| const int ch0 = t->nd[nid].child[0]; 1205| 238k| const int ch1 = t->nd[nid].child[1]; 1206| 238k| const uint8_t cov = (uint8_t)(t->covered[nid] || t->flat_d[nid]); ------------------ | Branch (1206:43): [True: 140k, False: 98.3k] | Branch (1206:62): [True: 5.84k, False: 92.4k] ------------------ 1207| 238k| if (ch0 >= 0) t->covered[ch0] = cov; ------------------ | Branch (1207:17): [True: 109k, False: 129k] ------------------ 1208| 238k| if (ch1 >= 0) t->covered[ch1] = cov; ------------------ | Branch (1208:17): [True: 106k, False: 132k] ------------------ 1209| 238k| } 1210| 23.1k| } 1211| 23.1k| return 0; 1212| 23.1k|} zxc_huffman.c:zxc_pivco_run_bytes: 1257| 44.7k|static ZXC_ALWAYS_INLINE size_t zxc_pivco_run_bytes(const uint32_t count, const uint8_t flat_d) { 1258| 44.7k| return flat_d ? ((size_t)count * flat_d + 7) / 8 : ((size_t)count + 7) / 8; ------------------ | Branch (1258:12): [True: 5.73k, False: 38.9k] ------------------ 1259| 44.7k|} zxc_huffman.c:zxc_pivco_decode_core: 2125| 23.1k| uint8_t* RESTRICT scratch) { 2126| 23.1k| if (UNLIKELY(n == 0)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 23.1k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 1, False: 23.1k] | | ------------------ ------------------ 2127| | 2128| | /* Pass 1: node counts + bit-run pointers, straight from the wire order. */ 2129| 23.1k| uint32_t count[ZXC_PIVCO_MAX_NODES]; 2130| 23.1k| const uint8_t* bit_ptr[ZXC_PIVCO_MAX_NODES]; 2131| 23.1k| count[0] = (uint32_t)n; 2132| 23.1k| const uint8_t* p = payload; 2133| 23.1k| const uint8_t* const pend = payload + payload_size; 2134| 252k| for (int i = 0; i < t->n_nodes; i++) { ------------------ | Branch (2134:21): [True: 229k, False: 23.1k] ------------------ 2135| 229k| const int nid = t->bfs[i]; 2136| 229k| const zxc_pivco_node_t* nd = &t->nd[nid]; 2137| 229k| if (t->covered[nid] || nd->sym >= 0) continue; ------------------ | Branch (2137:13): [True: 131k, False: 97.8k] | Branch (2137:32): [True: 53.1k, False: 44.7k] ------------------ 2138| 44.7k| const uint32_t c = count[nid]; 2139| 44.7k| if (t->flat_d[nid]) { ------------------ | Branch (2139:13): [True: 5.73k, False: 38.9k] ------------------ 2140| | /* Packed-code run: no partition below, nothing to popcount. */ 2141| 5.73k| const size_t fbytes = zxc_pivco_run_bytes(c, t->flat_d[nid]); 2142| 5.73k| if (UNLIKELY((size_t)(pend - p) < fbytes)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 5.73k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 18, False: 5.71k] | | ------------------ ------------------ 2143| 5.71k| bit_ptr[nid] = p; 2144| 5.71k| p += fbytes; 2145| 5.71k| continue; 2146| 5.73k| } 2147| 38.9k| const size_t nbytes = zxc_pivco_run_bytes(c, 0); /* merge node: c partition bits */ 2148| 38.9k| if (UNLIKELY((size_t)(pend - p) < nbytes)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 38.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 28, False: 38.9k] | | ------------------ ------------------ 2149| 38.9k| bit_ptr[nid] = p; 2150| | /* popcount of the c valid bits = the right child's count. All but the 2151| | * last byte counted fast, the last one with its padding masked - if the 2152| | * 8-byte loop swallowed it unmasked, padding bits would inflate `ones`. */ 2153| 38.9k| uint32_t ones = 0; 2154| 38.9k| if (nbytes) { ------------------ | Branch (2154:13): [True: 34.5k, False: 4.44k] ------------------ 2155| 34.5k| const size_t full = nbytes - 1; 2156| 34.5k| size_t k = 0; 2157| 37.3k| for (; k + 8 <= full; k += 8) { ------------------ | Branch (2157:20): [True: 2.86k, False: 34.5k] ------------------ 2158| 2.86k| uint64_t w; 2159| 2.86k| ZXC_MEMCPY(&w, p + k, 8); ------------------ | | 173| 2.86k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 2160| 2.86k| ones += (uint32_t)zxc_pivco_popcnt64(w); 2161| 2.86k| } 2162| 95.6k| for (; k < full; k++) ones += (uint32_t)zxc_pivco_popcnt32(p[k]); ------------------ | Branch (2162:20): [True: 61.0k, False: 34.5k] ------------------ 2163| 34.5k| uint8_t last = p[full]; 2164| 34.5k| if (c & 7U) last &= (uint8_t)((1U << (c & 7U)) - 1U); ------------------ | Branch (2164:17): [True: 28.5k, False: 5.92k] ------------------ 2165| 34.5k| ones += (uint32_t)zxc_pivco_popcnt32(last); 2166| 34.5k| } 2167| 38.9k| p += nbytes; 2168| 38.9k| if (UNLIKELY(ones > c)) return ZXC_ERROR_CORRUPT_DATA; ------------------ | | 151| 38.9k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 38.9k] | | ------------------ ------------------ 2169| 38.9k| const int ch0 = nd->child[0]; 2170| 38.9k| const int ch1 = nd->child[1]; 2171| 38.9k| if (ch1 >= 0) ------------------ | Branch (2171:13): [True: 35.7k, False: 3.23k] ------------------ 2172| 35.7k| count[ch1] = ones; 2173| 3.23k| else if (UNLIKELY(ones != 0)) ------------------ | | 151| 3.23k|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 10, False: 3.22k] | | ------------------ ------------------ 2174| 10| return ZXC_ERROR_CORRUPT_DATA; 2175| 38.9k| if (ch0 >= 0) ------------------ | Branch (2175:13): [True: 38.9k, False: 0] ------------------ 2176| 38.9k| count[ch0] = c - ones; 2177| 0| else if (UNLIKELY(c - ones != 0)) ------------------ | | 151| 0|#define UNLIKELY(x) (__builtin_expect(!!(x), 0)) | | ------------------ | | | Branch (151:21): [True: 0, False: 0] | | ------------------ ------------------ 2178| 0| return ZXC_ERROR_CORRUPT_DATA; 2179| 38.9k| } 2180| | 2181| | /* Per-level sequence offsets (BFS order => children contiguous). */ 2182| 23.1k| uint32_t seq_off[ZXC_PIVCO_MAX_NODES]; 2183| 80.9k| for (int d = 0; d <= t->max_depth; d++) { ------------------ | Branch (2183:21): [True: 57.8k, False: 23.1k] ------------------ 2184| 57.8k| uint32_t off = 0; 2185| 286k| for (int i = t->lvl_start[d]; i < t->lvl_start[d + 1]; i++) { ------------------ | Branch (2185:39): [True: 228k, False: 57.8k] ------------------ 2186| 228k| const int nid = t->bfs[i]; 2187| 228k| if (t->covered[nid]) continue; /* not materialised anywhere */ ------------------ | Branch (2187:17): [True: 130k, False: 97.5k] ------------------ 2188| 97.5k| seq_off[nid] = off; 2189| 97.5k| off += count[nid]; 2190| 97.5k| } 2191| 57.8k| } 2192| | 2193| | /* Leaf-pair parents emit both runs directly from their bits (XOR-blend), 2194| | * so their children never need materialising: flag them for skipping. 2195| | * A dictionary tree carries these flags precomputed in aux. */ 2196| 23.1k| uint8_t skip_local[ZXC_PIVCO_MAX_NODES]; 2197| 23.1k| const uint8_t* skip; 2198| 23.1k| if (aux) { ------------------ | Branch (2198:9): [True: 0, False: 23.1k] ------------------ 2199| 0| skip = aux->skip; 2200| 23.1k| } else { 2201| 23.1k| ZXC_MEMSET(skip_local, 0, sizeof(skip_local)); ------------------ | | 178| 23.1k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 2202| 251k| for (int i = 0; i < t->n_nodes; i++) { ------------------ | Branch (2202:25): [True: 228k, False: 23.1k] ------------------ 2203| 228k| const zxc_pivco_node_t* nd = &t->nd[t->bfs[i]]; 2204| 228k| if (nd->sym >= 0) continue; ------------------ | Branch (2204:17): [True: 124k, False: 104k] ------------------ 2205| 104k| const int ch0 = nd->child[0]; 2206| 104k| const int ch1 = nd->child[1]; 2207| 104k| if (ch0 >= 0 && ch1 >= 0 && t->nd[ch0].sym >= 0 && t->nd[ch1].sym >= 0) { ------------------ | Branch (2207:17): [True: 104k, False: 0] | Branch (2207:29): [True: 100k, False: 3.22k] | Branch (2207:41): [True: 62.4k, False: 38.5k] | Branch (2207:64): [True: 58.4k, False: 3.93k] ------------------ 2208| 58.4k| skip_local[ch0] = 1; 2209| 58.4k| skip_local[ch1] = 1; 2210| 58.4k| } 2211| 104k| } 2212| 23.1k| skip = skip_local; 2213| 23.1k| } 2214| | 2215| | /* Pass 2: bottom-up level reconstruction. */ 2216| 80.9k| for (int d = t->max_depth; d >= 0; d--) { ------------------ | Branch (2216:32): [True: 57.8k, False: 23.1k] ------------------ 2217| 57.8k| uint8_t* const buf_d = (d & 1) ? scratch : dst; ------------------ | Branch (2217:32): [True: 28.3k, False: 29.5k] ------------------ 2218| 57.8k| const uint8_t* const buf_c = (d & 1) ? dst : scratch; /* children at d+1 (read-only) */ ------------------ | Branch (2218:38): [True: 28.3k, False: 29.5k] ------------------ 2219| 286k| for (int i = t->lvl_start[d]; i < t->lvl_start[d + 1]; i++) { ------------------ | Branch (2219:39): [True: 228k, False: 57.8k] ------------------ 2220| 228k| const int nid = t->bfs[i]; 2221| 228k| if (t->covered[nid]) continue; /* lives inside a flat root's run */ ------------------ | Branch (2221:17): [True: 130k, False: 97.5k] ------------------ 2222| 97.5k| const zxc_pivco_node_t* nd = &t->nd[nid]; 2223| 97.5k| const uint32_t c = count[nid]; 2224| 97.5k| if (c == 0 || skip[nid]) continue; ------------------ | Branch (2224:17): [True: 17.7k, False: 79.8k] | Branch (2224:27): [True: 36.1k, False: 43.6k] ------------------ 2225| 43.6k| if (nd->sym >= 0) { ------------------ | Branch (2225:17): [True: 5.81k, False: 37.8k] ------------------ 2226| 5.81k| ZXC_MEMSET(buf_d + seq_off[nid], (uint8_t)nd->sym, c); ------------------ | | 178| 5.81k|#define ZXC_MEMSET(dst, val, n) __builtin_memset(dst, val, n) ------------------ 2227| 37.8k| } else if (t->flat_d[nid]) { ------------------ | Branch (2227:24): [True: 3.45k, False: 34.4k] ------------------ 2228| | /* Packed-path -> symbol table (complete subtree of depth D: 2229| | * 2^D leaves): precomputed at attach for a dictionary tree, 2230| | * else built here, then unpack the code run directly. */ 2231| 3.45k| const int D = t->flat_d[nid]; 2232| 3.45k| uint8_t c2s_local[1U << ZXC_HUF_MAX_CODE_LEN_ULTRA]; 2233| 3.45k| const uint8_t* c2s; 2234| 3.45k| if (aux) { ------------------ | Branch (2234:21): [True: 0, False: 3.45k] ------------------ 2235| 0| c2s = aux->c2s_pool + aux->c2s_off[nid]; 2236| 3.45k| } else { 2237| 3.45k| int16_t stk_n[ZXC_HUF_MAX_CODE_LEN_ULTRA + 1]; 2238| 3.45k| uint16_t stk_p[ZXC_HUF_MAX_CODE_LEN_ULTRA + 1]; 2239| 3.45k| uint8_t stk_l[ZXC_HUF_MAX_CODE_LEN_ULTRA + 1]; 2240| 3.45k| int sp = 0; 2241| 3.45k| stk_n[0] = (int16_t)nid; 2242| 3.45k| stk_p[0] = 0; 2243| 3.45k| stk_l[0] = 0; 2244| 95.4k| while (sp >= 0) { ------------------ | Branch (2244:28): [True: 91.9k, False: 3.45k] ------------------ 2245| 91.9k| const int cn = stk_n[sp]; 2246| 91.9k| const uint32_t cp = stk_p[sp]; 2247| 91.9k| const int cl = stk_l[sp]; 2248| 91.9k| sp--; 2249| 91.9k| if (t->nd[cn].sym >= 0) { ------------------ | Branch (2249:29): [True: 47.7k, False: 44.2k] ------------------ 2250| 47.7k| c2s_local[cp] = (uint8_t)t->nd[cn].sym; 2251| 47.7k| continue; 2252| 47.7k| } 2253| 44.2k| sp++; 2254| 44.2k| stk_n[sp] = t->nd[cn].child[0]; 2255| 44.2k| stk_p[sp] = (uint16_t)cp; 2256| 44.2k| stk_l[sp] = (uint8_t)(cl + 1); 2257| 44.2k| sp++; 2258| 44.2k| stk_n[sp] = t->nd[cn].child[1]; 2259| 44.2k| stk_p[sp] = (uint16_t)(cp | (1U << cl)); 2260| 44.2k| stk_l[sp] = (uint8_t)(cl + 1); 2261| 44.2k| } 2262| 3.45k| c2s = c2s_local; 2263| 3.45k| } 2264| 3.45k| zxc_pivco_unpack_flat(buf_d + seq_off[nid], c, D, bit_ptr[nid], c2s); 2265| 34.4k| } else { 2266| 34.4k| const int ch0 = nd->child[0]; 2267| 34.4k| const int ch1 = nd->child[1]; 2268| 34.4k| if (ch0 >= 0 && ch1 >= 0 && t->nd[ch0].sym >= 0 && t->nd[ch1].sym >= 0) { ------------------ | Branch (2268:21): [True: 34.4k, False: 0] | Branch (2268:33): [True: 31.1k, False: 3.22k] | Branch (2268:45): [True: 23.9k, False: 7.29k] | Branch (2268:68): [True: 20.8k, False: 3.08k] ------------------ 2269| 20.8k| zxc_pivco_emit_leaf_pair(buf_d + seq_off[nid], c, (uint8_t)t->nd[ch0].sym, 2270| 20.8k| (uint8_t)t->nd[ch1].sym, bit_ptr[nid]); 2271| 20.8k| continue; 2272| 20.8k| } 2273| 13.6k| const uint32_t nl = ch0 >= 0 ? count[ch0] : 0; ------------------ | Branch (2273:37): [True: 13.6k, False: 0] ------------------ 2274| 13.6k| const uint32_t src_off = ch0 >= 0 ? seq_off[ch0] : seq_off[nd->child[1]]; ------------------ | Branch (2274:42): [True: 13.6k, False: 0] ------------------ 2275| 13.6k| zxc_pivco_merge(buf_d + seq_off[nid], buf_c + src_off, nl, c - nl, bit_ptr[nid]); 2276| 13.6k| } 2277| 43.6k| } 2278| 57.8k| } 2279| 23.1k| return ZXC_OK; 2280| 23.1k|} zxc_huffman.c:zxc_pivco_popcnt64: 1064| 2.86k|static ZXC_ALWAYS_INLINE int zxc_pivco_popcnt64(const uint64_t v) { 1065| 2.86k|#if defined(__GNUC__) || defined(__clang__) 1066| 2.86k| return __builtin_popcountll(v); 1067| |#else 1068| | return zxc_pivco_popcnt32((uint32_t)v) + zxc_pivco_popcnt32((uint32_t)(v >> 32)); 1069| |#endif 1070| 2.86k|} zxc_huffman.c:zxc_pivco_popcnt32: 1052| 129k|static ZXC_ALWAYS_INLINE int zxc_pivco_popcnt32(const uint32_t v) { 1053| 129k|#if defined(__GNUC__) || defined(__clang__) 1054| 129k| return __builtin_popcount(v); 1055| |#else 1056| | /* Portable SWAR popcount for MSVC */ 1057| | uint32_t x = v - ((v >> 1) & 0x55555555U); 1058| | x = (x & 0x33333333U) + ((x >> 2) & 0x33333333U); 1059| | x = (x + (x >> 4)) & 0x0F0F0F0FU; 1060| | return (int)((x * 0x01010101U) >> 24); 1061| |#endif 1062| 129k|} zxc_huffman.c:zxc_pivco_unpack_flat: 1656| 3.45k| const uint8_t* RESTRICT bits, const uint8_t* RESTRICT c2s) { 1657| 3.45k| size_t i = 0; 1658| |#if defined(ZXC_USE_NEON64) 1659| | if (D == 4) { 1660| | const uint8x16_t vc2s = vld1q_u8(c2s); /* 16 entries */ 1661| | static const uint8_t rep2[16] = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7}; 1662| | static const int8_t sh4[16] = {0, -4, 0, -4, 0, -4, 0, -4, 0, -4, 0, -4, 0, -4, 0, -4}; 1663| | const uint8x16_t vrep = vld1q_u8(rep2); 1664| | const int8x16_t vsh = vld1q_s8(sh4); 1665| | const uint8x16_t vmask = vdupq_n_u8(0x0F); 1666| | while (i + 16 <= n) { 1667| | uint8x16_t raw = vdupq_n_u8(0); 1668| | raw = vreinterpretq_u8_u64( 1669| | vsetq_lane_u64(zxc_le64(bits + ((i * 4) >> 3)), vreinterpretq_u64_u8(raw), 0)); 1670| | const uint8x16_t rep = vqtbl1q_u8(raw, vrep); 1671| | const uint8x16_t codes = vandq_u8(vshlq_u8(rep, vsh), vmask); 1672| | vst1q_u8(out + i, vqtbl1q_u8(vc2s, codes)); 1673| | i += 16; 1674| | } 1675| | } else if (D == 2) { 1676| | uint8_t c2s16[16]; 1677| | for (int k = 0; k < 16; k++) c2s16[k] = c2s[k & 3]; 1678| | const uint8x16_t vc2s = vld1q_u8(c2s16); 1679| | static const uint8_t rep4[16] = {0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3}; 1680| | static const int8_t sh2[16] = {0, -2, -4, -6, 0, -2, -4, -6, 0, -2, -4, -6, 0, -2, -4, -6}; 1681| | const uint8x16_t vrep = vld1q_u8(rep4); 1682| | const int8x16_t vsh = vld1q_s8(sh2); 1683| | const uint8x16_t vmask = vdupq_n_u8(0x03); 1684| | while (i + 16 <= n) { 1685| | uint8x16_t raw = vdupq_n_u8(0); 1686| | raw = vreinterpretq_u8_u32( 1687| | vsetq_lane_u32(zxc_le32(bits + ((i * 2) >> 3)), vreinterpretq_u32_u8(raw), 0)); 1688| | const uint8x16_t rep = vqtbl1q_u8(raw, vrep); 1689| | const uint8x16_t codes = vandq_u8(vshlq_u8(rep, vsh), vmask); 1690| | vst1q_u8(out + i, vqtbl1q_u8(vc2s, codes)); 1691| | i += 16; 1692| | } 1693| | } else if (D == 3) { 1694| | /* Odd D straddle bytes: build a 16-bit window {byte_j,byte_j+1} per lane 1695| | * (byte_j = Di>>3), right-shift by Di&7, mask D bits. 16/step, two u16x8 1696| | * halves sharing the shift vector. D=3: 6 source bytes. */ 1697| | uint8_t c2s16[16]; 1698| | for (int k = 0; k < 16; k++) c2s16[k] = c2s[k & 7]; 1699| | const uint8x16_t vc2s = vld1q_u8(c2s16); /* 8 entries, padded to 16 */ 1700| | static const uint8_t idxA[16] = {0, 1, 0, 1, 0, 1, 1, 2, 1, 2, 1, 2, 2, 3, 2, 3}; 1701| | static const uint8_t idxB[16] = {3, 4, 3, 4, 3, 4, 4, 5, 4, 5, 4, 5, 5, 6, 5, 6}; 1702| | static const int16_t sh8[8] = {0, -3, -6, -1, -4, -7, -2, -5}; 1703| | const uint8x16_t vidxA = vld1q_u8(idxA); 1704| | const uint8x16_t vidxB = vld1q_u8(idxB); 1705| | const int16x8_t vsh = vld1q_s16(sh8); 1706| | const uint16x8_t vmask = vdupq_n_u16(0x07); 1707| | while (i + 16 <= n) { 1708| | const size_t off = (i * 3) >> 3; 1709| | uint32_t w0; 1710| | uint16_t w1; 1711| | ZXC_MEMCPY(&w0, bits + off, 4); 1712| | ZXC_MEMCPY(&w1, bits + off + 4, 2); /* exactly 6 bytes read */ 1713| | uint8x16_t raw = vdupq_n_u8(0); 1714| | raw = vreinterpretq_u8_u32(vsetq_lane_u32(w0, vreinterpretq_u32_u8(raw), 0)); 1715| | raw = vreinterpretq_u8_u16(vsetq_lane_u16(w1, vreinterpretq_u16_u8(raw), 2)); 1716| | const uint16x8_t winA = vreinterpretq_u16_u8(vqtbl1q_u8(raw, vidxA)); 1717| | const uint16x8_t winB = vreinterpretq_u16_u8(vqtbl1q_u8(raw, vidxB)); 1718| | const uint8x8_t cA = vmovn_u16(vandq_u16(vshlq_u16(winA, vsh), vmask)); 1719| | const uint8x8_t cB = vmovn_u16(vandq_u16(vshlq_u16(winB, vsh), vmask)); 1720| | vst1q_u8(out + i, vqtbl1q_u8(vc2s, vcombine_u8(cA, cB))); 1721| | i += 16; 1722| | } 1723| | } else if (D == 5) { 1724| | /* Same window scheme as D=3; c2s has 32 entries (vqtbl2q). 10 bytes. */ 1725| | uint8x16x2_t vc2s; 1726| | vc2s.val[0] = vld1q_u8(c2s); 1727| | vc2s.val[1] = vld1q_u8(c2s + 16); 1728| | static const uint8_t idxA[16] = {0, 1, 0, 1, 1, 2, 1, 2, 2, 3, 3, 4, 3, 4, 4, 5}; 1729| | static const uint8_t idxB[16] = {5, 6, 5, 6, 6, 7, 6, 7, 7, 8, 8, 9, 8, 9, 9, 10}; 1730| | static const int16_t sh8[8] = {0, -5, -2, -7, -4, -1, -6, -3}; 1731| | const uint8x16_t vidxA = vld1q_u8(idxA); 1732| | const uint8x16_t vidxB = vld1q_u8(idxB); 1733| | const int16x8_t vsh = vld1q_s16(sh8); 1734| | const uint16x8_t vmask = vdupq_n_u16(0x1F); 1735| | while (i + 16 <= n) { 1736| | const size_t off = (i * 5) >> 3; 1737| | uint64_t w0; 1738| | uint16_t w1; 1739| | ZXC_MEMCPY(&w0, bits + off, 8); 1740| | ZXC_MEMCPY(&w1, bits + off + 8, 2); 1741| | uint8x16_t raw = vdupq_n_u8(0); 1742| | raw = vreinterpretq_u8_u64(vsetq_lane_u64(w0, vreinterpretq_u64_u8(raw), 0)); 1743| | raw = vreinterpretq_u8_u16(vsetq_lane_u16(w1, vreinterpretq_u16_u8(raw), 4)); 1744| | const uint16x8_t winA = vreinterpretq_u16_u8(vqtbl1q_u8(raw, vidxA)); 1745| | const uint16x8_t winB = vreinterpretq_u16_u8(vqtbl1q_u8(raw, vidxB)); 1746| | const uint8x8_t cA = vmovn_u16(vandq_u16(vshlq_u16(winA, vsh), vmask)); 1747| | const uint8x8_t cB = vmovn_u16(vandq_u16(vshlq_u16(winB, vsh), vmask)); 1748| | vst1q_u8(out + i, vqtbl2q_u8(vc2s, vcombine_u8(cA, cB))); 1749| | i += 16; 1750| | } 1751| | } else if (D == 6) { 1752| | /* Same window scheme as D=3; c2s has 64 entries (vqtbl4q). 12 bytes. */ 1753| | uint8x16x4_t vc2s; 1754| | vc2s.val[0] = vld1q_u8(c2s); 1755| | vc2s.val[1] = vld1q_u8(c2s + 16); 1756| | vc2s.val[2] = vld1q_u8(c2s + 32); 1757| | vc2s.val[3] = vld1q_u8(c2s + 48); 1758| | static const uint8_t idxA[16] = {0, 1, 0, 1, 1, 2, 2, 3, 3, 4, 3, 4, 4, 5, 5, 6}; 1759| | static const uint8_t idxB[16] = {6, 7, 6, 7, 7, 8, 8, 9, 9, 10, 9, 10, 10, 11, 11, 12}; 1760| | static const int16_t sh8[8] = {0, -6, -4, -2, 0, -6, -4, -2}; 1761| | const uint8x16_t vidxA = vld1q_u8(idxA); 1762| | const uint8x16_t vidxB = vld1q_u8(idxB); 1763| | const int16x8_t vsh = vld1q_s16(sh8); 1764| | const uint16x8_t vmask = vdupq_n_u16(0x3F); 1765| | while (i + 16 <= n) { 1766| | const size_t off = (i * 6) >> 3; 1767| | uint64_t w0; 1768| | uint32_t w1; 1769| | ZXC_MEMCPY(&w0, bits + off, 8); 1770| | ZXC_MEMCPY(&w1, bits + off + 8, 4); 1771| | uint8x16_t raw = vdupq_n_u8(0); 1772| | raw = vreinterpretq_u8_u64(vsetq_lane_u64(w0, vreinterpretq_u64_u8(raw), 0)); 1773| | raw = vreinterpretq_u8_u32(vsetq_lane_u32(w1, vreinterpretq_u32_u8(raw), 2)); 1774| | const uint16x8_t winA = vreinterpretq_u16_u8(vqtbl1q_u8(raw, vidxA)); 1775| | const uint16x8_t winB = vreinterpretq_u16_u8(vqtbl1q_u8(raw, vidxB)); 1776| | const uint8x8_t cA = vmovn_u16(vandq_u16(vshlq_u16(winA, vsh), vmask)); 1777| | const uint8x8_t cB = vmovn_u16(vandq_u16(vshlq_u16(winB, vsh), vmask)); 1778| | vst1q_u8(out + i, vqtbl4q_u8(vc2s, vcombine_u8(cA, cB))); 1779| | i += 16; 1780| | } 1781| | } 1782| |#elif defined(ZXC_USE_NEON32) 1783| | /* d-register mirrors of the AArch64 kernels: 8 codes per step, c2s lookup 1784| | * via VTBL (16-entry table = two d-registers). */ 1785| | if (D == 4) { 1786| | uint8x8x2_t vc2s; 1787| | vc2s.val[0] = vld1_u8(c2s); 1788| | vc2s.val[1] = vld1_u8(c2s + 8); 1789| | static const uint8_t rep2[8] = {0, 0, 1, 1, 2, 2, 3, 3}; 1790| | static const int8_t sh4[8] = {0, -4, 0, -4, 0, -4, 0, -4}; 1791| | const uint8x8_t vrep = vld1_u8(rep2); 1792| | const int8x8_t vsh = vld1_s8(sh4); 1793| | const uint8x8_t vmask = vdup_n_u8(0x0F); 1794| | while (i + 8 <= n) { 1795| | uint32_t w; 1796| | ZXC_MEMCPY(&w, bits + ((i * 4) >> 3), 4); 1797| | const uint8x8_t raw = vreinterpret_u8_u32(vdup_n_u32(w)); 1798| | const uint8x8_t rep = vtbl1_u8(raw, vrep); 1799| | const uint8x8_t codes = vand_u8(vshl_u8(rep, vsh), vmask); 1800| | vst1_u8(out + i, vtbl2_u8(vc2s, codes)); 1801| | i += 8; 1802| | } 1803| | } else if (D == 2) { 1804| | uint8_t c2s8[8]; 1805| | for (int k = 0; k < 8; k++) c2s8[k] = c2s[k & 3]; 1806| | const uint8x8_t vc2s = vld1_u8(c2s8); 1807| | static const uint8_t rep4[8] = {0, 0, 0, 0, 1, 1, 1, 1}; 1808| | static const int8_t sh2[8] = {0, -2, -4, -6, 0, -2, -4, -6}; 1809| | const uint8x8_t vrep = vld1_u8(rep4); 1810| | const int8x8_t vsh = vld1_s8(sh2); 1811| | const uint8x8_t vmask = vdup_n_u8(0x03); 1812| | while (i + 8 <= n) { 1813| | const uint16_t w = (uint16_t)((uint16_t)bits[(i * 2) >> 3] | 1814| | ((uint16_t)bits[((i * 2) >> 3) + 1] << 8)); 1815| | const uint8x8_t raw = vreinterpret_u8_u16(vdup_n_u16(w)); 1816| | const uint8x8_t rep = vtbl1_u8(raw, vrep); 1817| | const uint8x8_t codes = vand_u8(vshl_u8(rep, vsh), vmask); 1818| | vst1_u8(out + i, vtbl1_u8(vc2s, codes)); 1819| | i += 8; 1820| | } 1821| | } else if (D == 3) { 1822| | /* 8 codes/step. 3-bit codes straddle bytes, so build a 16-bit window 1823| | * {byte_j, byte_j+1} per lane via two VTBLs, then extract bits [s,s+3) 1824| | * by multiply-by-2^(13-s) then >>13 - ARMv7 has no per-lane u16 variable 1825| | * shift. 8 codes fit in 3 source bytes. */ 1826| | const uint8x8_t vc2s = vld1_u8(c2s); /* 8 entries (2^3) */ 1827| | static const uint8_t idxlo[8] = {0, 1, 0, 1, 0, 1, 1, 2}; 1828| | static const uint8_t idxhi[8] = {1, 2, 1, 2, 2, 3, 2, 3}; 1829| | static const uint16_t mul8[8] = {8192, 1024, 128, 4096, 512, 64, 2048, 256}; 1830| | const uint8x8_t vidxlo = vld1_u8(idxlo); 1831| | const uint8x8_t vidxhi = vld1_u8(idxhi); 1832| | const uint16x8_t vmul = vld1q_u16(mul8); 1833| | const uint16x8_t vmask = vdupq_n_u16(0x07); 1834| | while (i + 8 <= n) { 1835| | const size_t off = (i * 3) >> 3; 1836| | const uint32_t w = (uint32_t)bits[off] | ((uint32_t)bits[off + 1] << 8) | 1837| | ((uint32_t)bits[off + 2] << 16); /* exactly 3 bytes */ 1838| | const uint8x8_t raw = vreinterpret_u8_u32(vdup_n_u32(w)); 1839| | const uint8x8_t wlo = vtbl1_u8(raw, vidxlo); 1840| | const uint8x8_t whi = vtbl1_u8(raw, vidxhi); 1841| | const uint16x8_t win = vreinterpretq_u16_u8(vcombine_u8(wlo, whi)); 1842| | const uint16x8_t codes16 = vandq_u16(vshrq_n_u16(vmulq_u16(win, vmul), 13), vmask); 1843| | vst1_u8(out + i, vtbl1_u8(vc2s, vmovn_u16(codes16))); 1844| | i += 8; 1845| | } 1846| | } else if (D == 5) { 1847| | /* Same window scheme as D=3; c2s has 32 entries (VTBL4). 5 source bytes. */ 1848| | uint8x8x4_t vc2s; 1849| | vc2s.val[0] = vld1_u8(c2s); 1850| | vc2s.val[1] = vld1_u8(c2s + 8); 1851| | vc2s.val[2] = vld1_u8(c2s + 16); 1852| | vc2s.val[3] = vld1_u8(c2s + 24); 1853| | static const uint8_t idxlo[8] = {0, 1, 0, 1, 1, 2, 1, 2}; 1854| | static const uint8_t idxhi[8] = {2, 3, 3, 4, 3, 4, 4, 5}; 1855| | static const uint16_t mul8[8] = {2048, 64, 512, 16, 128, 1024, 32, 256}; 1856| | const uint8x8_t vidxlo = vld1_u8(idxlo); 1857| | const uint8x8_t vidxhi = vld1_u8(idxhi); 1858| | const uint16x8_t vmul = vld1q_u16(mul8); 1859| | const uint16x8_t vmask = vdupq_n_u16(0x1F); 1860| | while (i + 8 <= n) { 1861| | uint64_t w = 0; 1862| | ZXC_MEMCPY(&w, bits + ((i * 5) >> 3), 5); 1863| | const uint8x8_t raw = vcreate_u8(w); 1864| | const uint16x8_t win = 1865| | vreinterpretq_u16_u8(vcombine_u8(vtbl1_u8(raw, vidxlo), vtbl1_u8(raw, vidxhi))); 1866| | const uint8x8_t codes = 1867| | vmovn_u16(vandq_u16(vshrq_n_u16(vmulq_u16(win, vmul), 11), vmask)); 1868| | vst1_u8(out + i, vtbl4_u8(vc2s, codes)); 1869| | i += 8; 1870| | } 1871| | } else if (D == 6) { 1872| | /* Same window scheme; c2s has 64 entries: two VTBL4 halves, select by 1873| | * code bit 5. 6 source bytes. */ 1874| | uint8x8x4_t lo, hi; 1875| | lo.val[0] = vld1_u8(c2s); 1876| | lo.val[1] = vld1_u8(c2s + 8); 1877| | lo.val[2] = vld1_u8(c2s + 16); 1878| | lo.val[3] = vld1_u8(c2s + 24); 1879| | hi.val[0] = vld1_u8(c2s + 32); 1880| | hi.val[1] = vld1_u8(c2s + 40); 1881| | hi.val[2] = vld1_u8(c2s + 48); 1882| | hi.val[3] = vld1_u8(c2s + 56); 1883| | static const uint8_t idxlo[8] = {0, 1, 0, 1, 1, 2, 2, 3}; 1884| | static const uint8_t idxhi[8] = {3, 4, 3, 4, 4, 5, 5, 6}; 1885| | static const uint16_t mul8[8] = {1024, 16, 64, 256, 1024, 16, 64, 256}; 1886| | const uint8x8_t vidxlo = vld1_u8(idxlo); 1887| | const uint8x8_t vidxhi = vld1_u8(idxhi); 1888| | const uint16x8_t vmul = vld1q_u16(mul8); 1889| | const uint16x8_t vmask = vdupq_n_u16(0x3F); 1890| | while (i + 8 <= n) { 1891| | uint64_t w = 0; 1892| | ZXC_MEMCPY(&w, bits + ((i * 6) >> 3), 6); 1893| | const uint8x8_t raw = vcreate_u8(w); 1894| | const uint16x8_t win = 1895| | vreinterpretq_u16_u8(vcombine_u8(vtbl1_u8(raw, vidxlo), vtbl1_u8(raw, vidxhi))); 1896| | const uint8x8_t codes = 1897| | vmovn_u16(vandq_u16(vshrq_n_u16(vmulq_u16(win, vmul), 10), vmask)); 1898| | const uint8x8_t sel = 1899| | vbsl_u8(vcge_u8(codes, vdup_n_u8(32)), vtbl4_u8(hi, vsub_u8(codes, vdup_n_u8(32))), 1900| | vtbl4_u8(lo, codes)); 1901| | vst1_u8(out + i, sel); 1902| | i += 8; 1903| | } 1904| | } 1905| |#elif defined(ZXC_USE_AVX512) || defined(ZXC_USE_AVX2) || \ 1906| | (defined(ZXC_USE_SSE2) && defined(__SSE4_1__)) 1907| | if (D == 4) { 1908| | const __m128i vc2s = _mm_loadu_si128((const __m128i*)(const void*)c2s); 1909| | const __m128i vrep = _mm_setr_epi8(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7); 1910| | const __m128i vmask = _mm_set1_epi8(0x0F); 1911| | /* even lanes keep low nibble, odd lanes take the high nibble: shift the 1912| | * replicated bytes right by 4 via a 16-bit shift + odd-lane select. */ 1913| | const __m128i vodd = _mm_set1_epi16(0x0F00 - 0x0F00 + (short)0xFF00); 1914| | while (i + 16 <= n) { 1915| | const __m128i raw = 1916| | _mm_loadl_epi64((const __m128i*)(const void*)(bits + ((i * 4) >> 3))); 1917| | const __m128i rep = _mm_shuffle_epi8(raw, vrep); 1918| | const __m128i hi = _mm_and_si128(_mm_srli_epi16(rep, 4), _mm_set1_epi8(0x0F)); 1919| | const __m128i lo = _mm_and_si128(rep, vmask); 1920| | const __m128i codes = _mm_blendv_epi8(lo, hi, vodd); 1921| | _mm_storeu_si128((__m128i*)(void*)(out + i), _mm_shuffle_epi8(vc2s, codes)); 1922| | i += 16; 1923| | } 1924| | } else if (D == 2) { 1925| | /* 16 codes from 4 bytes. SSE has no per-lane byte shift, so shift by 1926| | * multiply: (b << (6 - 2j)) >> 6 keeps bits [2j+1:2j]. Bytes spread to 1927| | * u16 lanes, times {64,16,4,1}, shifted, then packed back. */ 1928| | uint8_t c2s16[16]; 1929| | for (int k = 0; k < 16; k++) c2s16[k] = c2s[k & 3]; 1930| | const __m128i vc2s = _mm_loadu_si128((const __m128i*)(const void*)c2s16); 1931| | const __m128i vspreadA = 1932| | _mm_setr_epi8(0, (char)0x80, 0, (char)0x80, 0, (char)0x80, 0, (char)0x80, 1, (char)0x80, 1933| | 1, (char)0x80, 1, (char)0x80, 1, (char)0x80); 1934| | const __m128i vspreadB = 1935| | _mm_setr_epi8(2, (char)0x80, 2, (char)0x80, 2, (char)0x80, 2, (char)0x80, 3, (char)0x80, 1936| | 3, (char)0x80, 3, (char)0x80, 3, (char)0x80); 1937| | const __m128i vmul = _mm_setr_epi16(64, 16, 4, 1, 64, 16, 4, 1); 1938| | while (i + 16 <= n) { 1939| | const __m128i raw = _mm_cvtsi32_si128( 1940| | (int)((uint32_t)bits[(i * 2) >> 3] | ((uint32_t)bits[((i * 2) >> 3) + 1] << 8) | 1941| | ((uint32_t)bits[((i * 2) >> 3) + 2] << 16) | 1942| | ((uint32_t)bits[((i * 2) >> 3) + 3] << 24))); 1943| | const __m128i a16 = 1944| | _mm_srli_epi16(_mm_mullo_epi16(_mm_shuffle_epi8(raw, vspreadA), vmul), 6); 1945| | const __m128i b16 = 1946| | _mm_srli_epi16(_mm_mullo_epi16(_mm_shuffle_epi8(raw, vspreadB), vmul), 6); 1947| | const __m128i codes = _mm_and_si128(_mm_packus_epi16(a16, b16), _mm_set1_epi8(3)); 1948| | _mm_storeu_si128((__m128i*)(void*)(out + i), _mm_shuffle_epi8(vc2s, codes)); 1949| | i += 16; 1950| | } 1951| | } else if (D == 3) { 1952| | /* 16-bit window {byte_j,byte_j+1} per lane (byte_j = Di>>3); SSE has no 1953| | * per-lane u16 shift, so extract bits [s,s+D) of window w by w*2^(16-D-s) 1954| | * (field lands in the top D bits) then >>(16-D); s_j = Dj&7. D=3: 6 bytes. */ 1955| | uint8_t c2s16[16]; 1956| | for (int k = 0; k < 16; k++) c2s16[k] = c2s[k & 7]; 1957| | const __m128i vc2s = _mm_loadu_si128((const __m128i*)(const void*)c2s16); 1958| | const __m128i shufA = _mm_setr_epi8(0, 1, 0, 1, 0, 1, 1, 2, 1, 2, 1, 2, 2, 3, 2, 3); 1959| | const __m128i shufB = _mm_setr_epi8(3, 4, 3, 4, 3, 4, 4, 5, 4, 5, 4, 5, 5, 6, 5, 6); 1960| | const __m128i vmul = _mm_setr_epi16(8192, 1024, 128, 4096, 512, 64, 2048, 256); 1961| | while (i + 16 <= n) { 1962| | const size_t off = (i * 3) >> 3; 1963| | uint32_t w0; 1964| | uint16_t w1; 1965| | ZXC_MEMCPY(&w0, bits + off, 4); 1966| | ZXC_MEMCPY(&w1, bits + off + 4, 2); /* exactly 6 bytes read */ 1967| | const __m128i raw = _mm_insert_epi16(_mm_cvtsi32_si128((int)w0), w1, 2); 1968| | const __m128i cA = 1969| | _mm_srli_epi16(_mm_mullo_epi16(_mm_shuffle_epi8(raw, shufA), vmul), 13); 1970| | const __m128i cB = 1971| | _mm_srli_epi16(_mm_mullo_epi16(_mm_shuffle_epi8(raw, shufB), vmul), 13); 1972| | const __m128i codes = _mm_and_si128(_mm_packus_epi16(cA, cB), _mm_set1_epi8(7)); 1973| | _mm_storeu_si128((__m128i*)(void*)(out + i), _mm_shuffle_epi8(vc2s, codes)); 1974| | i += 16; 1975| | } 1976| | } else if (D == 5) { 1977| | /* c2s has 32 entries: pshufb does 16, so select lo/hi by code bit 4. */ 1978| | const __m128i vlo = _mm_loadu_si128((const __m128i*)(const void*)c2s); 1979| | const __m128i vhi = _mm_loadu_si128((const __m128i*)(const void*)(c2s + 16)); 1980| | const __m128i shufA = _mm_setr_epi8(0, 1, 0, 1, 1, 2, 1, 2, 2, 3, 3, 4, 3, 4, 4, 5); 1981| | const __m128i shufB = _mm_setr_epi8(5, 6, 5, 6, 6, 7, 6, 7, 7, 8, 8, 9, 8, 9, 9, 10); 1982| | const __m128i vmul = _mm_setr_epi16(2048, 64, 512, 16, 128, 1024, 32, 256); 1983| | while (i + 16 <= n) { 1984| | const size_t off = (i * 5) >> 3; 1985| | uint16_t w1; 1986| | ZXC_MEMCPY(&w1, bits + off + 8, 2); /* 10 source bytes */ 1987| | const __m128i raw = 1988| | _mm_insert_epi16(_mm_loadl_epi64((const __m128i*)(const void*)(bits + off)), w1, 4); 1989| | const __m128i cA = 1990| | _mm_srli_epi16(_mm_mullo_epi16(_mm_shuffle_epi8(raw, shufA), vmul), 11); 1991| | const __m128i cB = 1992| | _mm_srli_epi16(_mm_mullo_epi16(_mm_shuffle_epi8(raw, shufB), vmul), 11); 1993| | const __m128i codes = _mm_and_si128(_mm_packus_epi16(cA, cB), _mm_set1_epi8(0x1F)); 1994| | const __m128i sel = 1995| | _mm_blendv_epi8(_mm_shuffle_epi8(vlo, codes), _mm_shuffle_epi8(vhi, codes), 1996| | _mm_slli_epi16(codes, 3)); 1997| | _mm_storeu_si128((__m128i*)(void*)(out + i), sel); 1998| | i += 16; 1999| | } 2000| | } else if (D == 6) { 2001| | /* c2s has 64 entries: 4 pshufb sub-tables, select by code bits 4-5. */ 2002| | const __m128i t0 = _mm_loadu_si128((const __m128i*)(const void*)c2s); 2003| | const __m128i t1 = _mm_loadu_si128((const __m128i*)(const void*)(c2s + 16)); 2004| | const __m128i t2 = _mm_loadu_si128((const __m128i*)(const void*)(c2s + 32)); 2005| | const __m128i t3 = _mm_loadu_si128((const __m128i*)(const void*)(c2s + 48)); 2006| | const __m128i shufA = _mm_setr_epi8(0, 1, 0, 1, 1, 2, 2, 3, 3, 4, 3, 4, 4, 5, 5, 6); 2007| | const __m128i shufB = _mm_setr_epi8(6, 7, 6, 7, 7, 8, 8, 9, 9, 10, 9, 10, 10, 11, 11, 12); 2008| | const __m128i vmul = _mm_setr_epi16(1024, 16, 64, 256, 1024, 16, 64, 256); 2009| | while (i + 16 <= n) { 2010| | const size_t off = (i * 6) >> 3; 2011| | uint32_t w1; 2012| | ZXC_MEMCPY(&w1, bits + off + 8, 4); /* 12 source bytes */ 2013| | const __m128i raw = _mm_insert_epi32( 2014| | _mm_loadl_epi64((const __m128i*)(const void*)(bits + off)), (int)w1, 2); 2015| | const __m128i cA = 2016| | _mm_srli_epi16(_mm_mullo_epi16(_mm_shuffle_epi8(raw, shufA), vmul), 10); 2017| | const __m128i cB = 2018| | _mm_srli_epi16(_mm_mullo_epi16(_mm_shuffle_epi8(raw, shufB), vmul), 10); 2019| | const __m128i codes = _mm_and_si128(_mm_packus_epi16(cA, cB), _mm_set1_epi8(0x3F)); 2020| | const __m128i m4 = _mm_slli_epi16(codes, 3); 2021| | const __m128i lo = 2022| | _mm_blendv_epi8(_mm_shuffle_epi8(t0, codes), _mm_shuffle_epi8(t1, codes), m4); 2023| | const __m128i hi = 2024| | _mm_blendv_epi8(_mm_shuffle_epi8(t2, codes), _mm_shuffle_epi8(t3, codes), m4); 2025| | _mm_storeu_si128((__m128i*)(void*)(out + i), 2026| | _mm_blendv_epi8(lo, hi, _mm_slli_epi16(codes, 2))); 2027| | i += 16; 2028| | } 2029| | } 2030| |#endif 2031| | /* Generic scalar bit-reader (any D). */ 2032| 3.45k| { 2033| 3.45k| uint64_t bitpos = i * (uint64_t)D; 2034| 3.45k| const uint32_t m = (1U << D) - 1U; 2035| 3.45k| const size_t fbytes = (size_t)(((uint64_t)n * (uint64_t)D + 7U) >> 3); 2036| 116k| while (i < n) { ------------------ | Branch (2036:16): [True: 113k, False: 3.45k] ------------------ 2037| 113k| const size_t byte = (size_t)(bitpos >> 3); 2038| 113k| uint32_t w = bits[byte]; 2039| 113k| if (byte + 1 < fbytes) w |= (uint32_t)bits[byte + 1] << 8; ------------------ | Branch (2039:17): [True: 108k, False: 5.12k] ------------------ 2040| 113k| if (D > 8 && byte + 2 < fbytes) w |= (uint32_t)bits[byte + 2] << 16; ------------------ | Branch (2040:17): [True: 0, False: 113k] | Branch (2040:26): [True: 0, False: 0] ------------------ 2041| 113k| out[i++] = c2s[(w >> (bitpos & 7U)) & m]; 2042| 113k| bitpos += (uint64_t)D; 2043| 113k| } 2044| 3.45k| } 2045| 3.45k|} zxc_huffman.c:zxc_pivco_emit_leaf_pair: 2057| 20.8k| const uint8_t* RESTRICT bits) { 2058| 20.8k| size_t i = 0; 2059| |#if defined(ZXC_USE_NEON64) 2060| | const uint8x16_t vsym0 = vdupq_n_u8(sym0); 2061| | const uint8x16_t vdelta = vdupq_n_u8((uint8_t)(sym0 ^ sym1)); 2062| | static const uint8_t rep_idx[16] = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1}; 2063| | static const uint8_t bit_sel[16] = {1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128}; 2064| | const uint8x16_t vrep = vld1q_u8(rep_idx); 2065| | const uint8x16_t vsel = vld1q_u8(bit_sel); 2066| | while (i + 16 <= n) { 2067| | uint8x16_t ctrl = vdupq_n_u8(0); 2068| | ctrl = vsetq_lane_u8(bits[i >> 3], ctrl, 0); 2069| | ctrl = vsetq_lane_u8(bits[(i >> 3) + 1], ctrl, 1); 2070| | const uint8x16_t rep = vqtbl1q_u8(ctrl, vrep); 2071| | const uint8x16_t mask = vtstq_u8(rep, vsel); 2072| | vst1q_u8(out + i, veorq_u8(vsym0, vandq_u8(vdelta, mask))); 2073| | i += 16; 2074| | } 2075| |#elif defined(ZXC_USE_NEON32) 2076| | const uint8x8_t vsym0 = vdup_n_u8(sym0); 2077| | const uint8x8_t vdelta = vdup_n_u8((uint8_t)(sym0 ^ sym1)); 2078| | static const uint8_t bit_sel8[8] = {1, 2, 4, 8, 16, 32, 64, 128}; 2079| | const uint8x8_t vsel = vld1_u8(bit_sel8); 2080| | while (i + 8 <= n) { 2081| | const uint8x8_t mask = vtst_u8(vdup_n_u8(bits[i >> 3]), vsel); 2082| | vst1_u8(out + i, veor_u8(vsym0, vand_u8(vdelta, mask))); 2083| | i += 8; 2084| | } 2085| |#elif defined(ZXC_USE_AVX512) || defined(ZXC_USE_AVX2) || \ 2086| | (defined(ZXC_USE_SSE2) && defined(__SSSE3__)) 2087| | const __m128i vsym0 = _mm_set1_epi8((char)sym0); 2088| | const __m128i vdelta = _mm_set1_epi8((char)(uint8_t)(sym0 ^ sym1)); 2089| | const __m128i vrep = _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1); 2090| | const __m128i vsel = 2091| | _mm_setr_epi8(1, 2, 4, 8, 16, 32, 64, (char)128, 1, 2, 4, 8, 16, 32, 64, (char)128); 2092| | while (i + 16 <= n) { 2093| | const __m128i ctrl = _mm_cvtsi32_si128(bits[i >> 3] | ((int)bits[(i >> 3) + 1] << 8)); 2094| | const __m128i rep = _mm_shuffle_epi8(ctrl, vrep); 2095| | /* lanes where the selected bit is set -> 0xFF */ 2096| | const __m128i mask = _mm_cmpeq_epi8(_mm_and_si128(rep, vsel), vsel); 2097| | _mm_storeu_si128((__m128i*)(void*)(out + i), 2098| | _mm_xor_si128(vsym0, _mm_and_si128(vdelta, mask))); 2099| | i += 16; 2100| | } 2101| |#endif 2102| 533k| while (i < n) { ------------------ | Branch (2102:12): [True: 512k, False: 20.8k] ------------------ 2103| 512k| const int bit = (bits[i >> 3] >> (i & 7)) & 1; 2104| 512k| out[i++] = bit ? sym1 : sym0; ------------------ | Branch (2104:20): [True: 165k, False: 346k] ------------------ 2105| 512k| } 2106| 20.8k|} zxc_huffman.c:zxc_pivco_merge: 1507| 13.6k| const uint8_t* RESTRICT bits) { 1508| 13.6k| const uint8_t* L = src; 1509| 13.6k| const uint8_t* R = src + nl; 1510| 13.6k| const size_t n = nl + nr; 1511| 13.6k| size_t i = 0; 1512| 13.6k| size_t lp = 0; 1513| 13.6k| size_t rp = 0; 1514| |#if defined(ZXC_USE_NEON64) 1515| | /* 16 outputs per step: two-register TBL over {L[0..15], R[0..15]} with the 1516| | * signed-index tables (see zxc_pivco_tables.h). */ 1517| | while (i + 16 <= n) { 1518| | const uint8_t b0 = bits[i >> 3]; 1519| | const uint8_t b1 = bits[(i >> 3) + 1]; 1520| | uint8x16x2_t tb; 1521| | tb.val[0] = vld1q_u8(L + lp); 1522| | tb.val[1] = vld1q_u8(R + rp); 1523| | const int pc0 = zxc_pivco_popcnt32(b0); 1524| | const uint8x8_t ia = vld1_u8(zxc_pivco_idxa_u8[b0]); 1525| | const uint8x8_t ib = vld1_u8(zxc_pivco_idxb_pre[pc0][b1]); 1526| | vst1q_u8(out + i, vqtbl2q_u8(tb, vcombine_u8(ia, ib))); 1527| | const int pc = pc0 + zxc_pivco_popcnt32(b1); 1528| | rp += (size_t)pc; 1529| | lp += (size_t)(16 - pc); 1530| | i += 16; 1531| | } 1532| |#else /* x86 tiers */ 1533| |#if defined(ZXC_USE_AVX512) && defined(__AVX512VBMI2__) 1534| | /* 64 outputs per step: the merge IS a pair of byte expands - L's bytes fill 1535| | * the control word's 0-bit lanes, R's the 1-bit lanes. Masked loads keep the 1536| | * speculative reads fault-safe without extra buffer slack. */ 1537| | while (i + 64 <= n) { 1538| | uint64_t ctrl; 1539| | ZXC_MEMCPY(&ctrl, bits + (i >> 3), 8); 1540| | const int pc = zxc_pivco_popcnt64(ctrl); 1541| | const __mmask64 lmask = (pc == 0) ? ~(__mmask64)0 : (((__mmask64)1 << (64 - pc)) - 1U); 1542| | const __mmask64 rmask = (pc == 64) ? ~(__mmask64)0 : (((__mmask64)1 << pc) - 1U); 1543| | const __m512i vl = _mm512_maskz_loadu_epi8(lmask, (const void*)(L + lp)); 1544| | const __m512i vr = _mm512_maskz_loadu_epi8(rmask, (const void*)(R + rp)); 1545| | const __m512i outl = _mm512_maskz_expand_epi8((__mmask64)~ctrl, vl); 1546| | const __m512i outv = _mm512_mask_expand_epi8(outl, (__mmask64)ctrl, vr); 1547| | _mm512_storeu_si512((void*)(out + i), outv); 1548| | rp += (size_t)pc; 1549| | lp += (size_t)(64 - pc); 1550| | i += 64; 1551| | } 1552| |#endif 1553| |#if defined(ZXC_USE_AVX512) || defined(ZXC_USE_AVX2) || \ 1554| | (defined(ZXC_USE_SSE2) && defined(__SSSE3__)) 1555| | /* 16 outputs/step: pshufb(L, ix+0x70) | pshufb(R, ix-16) selects L[ix] or 1556| | * R[ix-16] (out-of-range lanes zero out); 32/step unrolls it twice. */ 1557| | while (i + 32 <= n) { 1558| | const uint8_t* cb = bits + (i >> 3); 1559| | const int pcA0 = zxc_pivco_popcnt32(cb[0]); 1560| | const int pcA = pcA0 + zxc_pivco_popcnt32(cb[1]); 1561| | const int pcB0 = zxc_pivco_popcnt32(cb[2]); 1562| | const int pcB = pcB0 + zxc_pivco_popcnt32(cb[3]); 1563| | const size_t lpB = lp + (size_t)(16 - pcA); 1564| | const size_t rpB = rp + (size_t)pcA; 1565| | const __m128i vlA = _mm_loadu_si128((const __m128i*)(const void*)(L + lp)); 1566| | const __m128i vrA = _mm_loadu_si128((const __m128i*)(const void*)(R + rp)); 1567| | const __m128i vlB = _mm_loadu_si128((const __m128i*)(const void*)(L + lpB)); 1568| | const __m128i vrB = _mm_loadu_si128((const __m128i*)(const void*)(R + rpB)); 1569| | const __m128i iaA = _mm_loadl_epi64((const __m128i*)(const void*)zxc_pivco_idxa_u8[cb[0]]); 1570| | const __m128i ibA = 1571| | _mm_loadl_epi64((const __m128i*)(const void*)zxc_pivco_idxb_pre[pcA0][cb[1]]); 1572| | const __m128i iaB = _mm_loadl_epi64((const __m128i*)(const void*)zxc_pivco_idxa_u8[cb[2]]); 1573| | const __m128i ibB = 1574| | _mm_loadl_epi64((const __m128i*)(const void*)zxc_pivco_idxb_pre[pcB0][cb[3]]); 1575| | const __m128i ixA = _mm_unpacklo_epi64(iaA, ibA); 1576| | const __m128i ixB = _mm_unpacklo_epi64(iaB, ibB); 1577| | const __m128i selA = 1578| | _mm_or_si128(_mm_shuffle_epi8(vlA, _mm_add_epi8(ixA, _mm_set1_epi8(0x70))), 1579| | _mm_shuffle_epi8(vrA, _mm_sub_epi8(ixA, _mm_set1_epi8(16)))); 1580| | const __m128i selB = 1581| | _mm_or_si128(_mm_shuffle_epi8(vlB, _mm_add_epi8(ixB, _mm_set1_epi8(0x70))), 1582| | _mm_shuffle_epi8(vrB, _mm_sub_epi8(ixB, _mm_set1_epi8(16)))); 1583| | _mm_storeu_si128((__m128i*)(void*)(out + i), selA); 1584| | _mm_storeu_si128((__m128i*)(void*)(out + i + 16), selB); 1585| | lp = lpB + (size_t)(16 - pcB); 1586| | rp = rpB + (size_t)pcB; 1587| | i += 32; 1588| | } 1589| | while (i + 16 <= n) { 1590| | const uint8_t b0 = bits[i >> 3]; 1591| | const uint8_t b1 = bits[(i >> 3) + 1]; 1592| | const __m128i vl = _mm_loadu_si128((const __m128i*)(const void*)(L + lp)); 1593| | const __m128i vr = _mm_loadu_si128((const __m128i*)(const void*)(R + rp)); 1594| | const int pc0 = zxc_pivco_popcnt32(b0); 1595| | const __m128i ia = _mm_loadl_epi64((const __m128i*)(const void*)zxc_pivco_idxa_u8[b0]); 1596| | const __m128i ib = 1597| | _mm_loadl_epi64((const __m128i*)(const void*)zxc_pivco_idxb_pre[pc0][b1]); 1598| | const __m128i ix = _mm_unpacklo_epi64(ia, ib); 1599| | const __m128i sell = _mm_shuffle_epi8(vl, _mm_add_epi8(ix, _mm_set1_epi8(0x70))); 1600| | const __m128i selr = _mm_shuffle_epi8(vr, _mm_sub_epi8(ix, _mm_set1_epi8(16))); 1601| | _mm_storeu_si128((__m128i*)(void*)(out + i), _mm_or_si128(sell, selr)); 1602| | const int pc = pc0 + zxc_pivco_popcnt32(b1); 1603| | rp += (size_t)pc; 1604| | lp += (size_t)(16 - pc); 1605| | i += 16; 1606| | } 1607| |#elif defined(ZXC_USE_NEON32) 1608| | /* 8 outputs per step: four-register VTBL over {L[0..7], -, R[0..7], -}. The 1609| | * shared index tables address a 32-lane view, and an 8-output step only 1610| | * touches lanes 0..7 and 16..23, so the two spare d-registers stay zero. */ 1611| | while (i + 8 <= n) { 1612| | const uint8_t b = bits[i >> 3]; 1613| | uint8x8x4_t tb; 1614| | tb.val[0] = vld1_u8(L + lp); 1615| | tb.val[1] = vdup_n_u8(0); 1616| | tb.val[2] = vld1_u8(R + rp); 1617| | tb.val[3] = vdup_n_u8(0); 1618| | vst1_u8(out + i, vtbl4_u8(tb, vld1_u8(zxc_pivco_idxa_u8[b]))); 1619| | const int pc = zxc_pivco_popcnt32(b); 1620| | rp += (size_t)pc; 1621| | lp += (size_t)(8 - pc); 1622| | i += 8; 1623| | } 1624| |#endif 1625| 13.6k|#endif /* x86 tiers */ 1626| | /* Portable 8-output path: byte selects from a 24-byte scratch laid out 1627| | * as the low half of the 32-lane view the shared index tables assume 1628| | * (L at offset 0, R at offset 16). */ 1629| 47.4k| while (i + 8 <= n) { ------------------ | Branch (1629:12): [True: 33.8k, False: 13.6k] ------------------ 1630| 33.8k| const uint8_t b = bits[i >> 3]; 1631| 33.8k| uint8_t comb[24]; 1632| 33.8k| ZXC_MEMCPY(comb, L + lp, 8); ------------------ | | 173| 33.8k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1633| 33.8k| ZXC_MEMCPY(comb + 16, R + rp, 8); ------------------ | | 173| 33.8k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1634| 33.8k| const uint8_t* ix = zxc_pivco_idxa_u8[b]; 1635| 304k| for (int j = 0; j < 8; j++) out[i + (size_t)j] = comb[ix[j]]; ------------------ | Branch (1635:25): [True: 270k, False: 33.8k] ------------------ 1636| 33.8k| const int pc = zxc_pivco_popcnt32(b); 1637| 33.8k| rp += (size_t)pc; 1638| 33.8k| lp += (size_t)(8 - pc); 1639| 33.8k| i += 8; 1640| 33.8k| } 1641| 53.1k| while (i < n) { ------------------ | Branch (1641:12): [True: 39.5k, False: 13.6k] ------------------ 1642| 39.5k| const int bit = (bits[i >> 3] >> (i & 7)) & 1; 1643| 39.5k| out[i++] = bit ? R[rp++] : L[lp++]; ------------------ | Branch (1643:20): [True: 13.2k, False: 26.2k] ------------------ 1644| 39.5k| } 1645| 13.6k|} zxc_common.c:zxc_log2_u32: 856| 6.48k|static ZXC_ALWAYS_INLINE uint32_t zxc_log2_u32(const uint32_t v) { 857| |#ifdef _MSC_VER 858| | unsigned long index; 859| | return (v == 0) ? 0 : (_BitScanReverse(&index, v) ? index : 0); 860| |#else 861| 6.48k| return (v == 0) ? 0 : (uint32_t)(31 - __builtin_clz(v)); ------------------ | Branch (861:12): [True: 0, False: 6.48k] ------------------ 862| 6.48k|#endif 863| 6.48k|} zxc_common.c:zxc_hash16: 1219| 13.0k|static ZXC_ALWAYS_INLINE uint16_t zxc_hash16(const uint8_t* p) { 1220| 13.0k| const uint64_t v1 = zxc_le64(p); 1221| 13.0k| const uint64_t v2 = zxc_le64(p + 8); 1222| 13.0k| uint64_t h = v1 ^ v2 ^ ZXC_HASH_PRIME2; ------------------ | | 559| 13.0k|#define ZXC_HASH_PRIME2 0xD2D84A61D2D84A61ULL ------------------ 1223| 13.0k| h ^= h << 13; 1224| 13.0k| h ^= h >> 7; 1225| 13.0k| h ^= h << 17; 1226| 13.0k| const uint32_t res = (uint32_t)((h >> 32) ^ h); 1227| 13.0k| return (uint16_t)((res >> 16) ^ res); 1228| 13.0k|} zxc_common.c:zxc_le32: 1094| 194k|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1095| 194k| uint32_t v; 1096| 194k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 194k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1097| |#ifdef ZXC_BIG_ENDIAN 1098| | return ZXC_BSWAP32(v); 1099| |#else 1100| 194k| return v; 1101| 194k|#endif 1102| 194k|} zxc_common.c:zxc_le16: 1074| 13.0k|static ZXC_ALWAYS_INLINE uint16_t zxc_le16(const void* p) { 1075| 13.0k| uint16_t v; 1076| 13.0k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 13.0k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1077| |#ifdef ZXC_BIG_ENDIAN 1078| | return ZXC_BSWAP16(v); 1079| |#else 1080| 13.0k| return v; 1081| 13.0k|#endif 1082| 13.0k|} zxc_common.c:zxc_hash8: 1200| 48.6k|static ZXC_ALWAYS_INLINE uint8_t zxc_hash8(const uint8_t* p) { 1201| 48.6k| const uint64_t v = zxc_le64(p); 1202| 48.6k| uint64_t h = v ^ ZXC_HASH_PRIME1; ------------------ | | 557| 48.6k|#define ZXC_HASH_PRIME1 0x9E3779B97F4A7C15ULL ------------------ 1203| 48.6k| h ^= h << 13; 1204| 48.6k| h ^= h >> 7; 1205| 48.6k| h ^= h << 17; 1206| 48.6k| return (uint8_t)((h >> 32) ^ h); 1207| 48.6k|} zxc_common.c:zxc_le64: 1114| 338k|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1115| 338k| uint64_t v; 1116| 338k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 338k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1117| |#ifdef ZXC_BIG_ENDIAN 1118| | return ZXC_BSWAP64(v); 1119| |#else 1120| 338k| return v; 1121| 338k|#endif 1122| 338k|} zxc_decompress.c:zxc_le32: 1094| 482k|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1095| 482k| uint32_t v; 1096| 482k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 482k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1097| |#ifdef ZXC_BIG_ENDIAN 1098| | return ZXC_BSWAP32(v); 1099| |#else 1100| 482k| return v; 1101| 482k|#endif 1102| 482k|} zxc_decompress.c:zxc_copy32: 1258| 12.2M|static ZXC_ALWAYS_INLINE void zxc_copy32(void* dst, const void* src) { 1259| |#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) 1260| | // AVX2/AVX512: Single 256-bit (32 byte) unaligned load/store 1261| | _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src)); 1262| |#elif defined(ZXC_USE_SSE2) 1263| | // SSE2: Two 128-bit (16 byte) unaligned load/stores (no 256-bit regs) 1264| 12.2M| _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); 1265| 12.2M| _mm_storeu_si128((__m128i*)((uint8_t*)dst + 16), 1266| 12.2M| _mm_loadu_si128((const __m128i*)((const uint8_t*)src + 16))); 1267| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1268| | // NEON: Two 128-bit (16 byte) unaligned load/stores 1269| | vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); 1270| | vst1q_u8((uint8_t*)dst + 16, vld1q_u8((const uint8_t*)src + 16)); 1271| |#else 1272| | ZXC_MEMCPY(dst, src, 32); 1273| |#endif 1274| 12.2M|} zxc_decompress.c:zxc_le64: 1114| 17.8k|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1115| 17.8k| uint64_t v; 1116| 17.8k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 17.8k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1117| |#ifdef ZXC_BIG_ENDIAN 1118| | return ZXC_BSWAP64(v); 1119| |#else 1120| 17.8k| return v; 1121| 17.8k|#endif 1122| 17.8k|} zxc_decompress.c:zxc_copy16: 1239| 16.9M|static ZXC_ALWAYS_INLINE void zxc_copy16(void* dst, const void* src) { 1240| 16.9M|#if defined(ZXC_USE_AVX2) || defined(ZXC_USE_AVX512) || defined(ZXC_USE_SSE2) 1241| | // x86 SSE2/AVX2/AVX512: Single 128-bit unaligned load/store 1242| 16.9M| _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); 1243| |#elif defined(ZXC_USE_NEON64) || defined(ZXC_USE_NEON32) 1244| | vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); 1245| |#else 1246| | ZXC_MEMCPY(dst, src, 16); 1247| |#endif 1248| 16.9M|} zxc_decompress.c:zxc_le16: 1074| 17.0k|static ZXC_ALWAYS_INLINE uint16_t zxc_le16(const void* p) { 1075| 17.0k| uint16_t v; 1076| 17.0k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 17.0k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1077| |#ifdef ZXC_BIG_ENDIAN 1078| | return ZXC_BSWAP16(v); 1079| |#else 1080| 17.0k| return v; 1081| 17.0k|#endif 1082| 17.0k|} zxc_dispatch.c:zxc_le32: 1094| 6.59k|static ZXC_ALWAYS_INLINE uint32_t zxc_le32(const void* p) { 1095| 6.59k| uint32_t v; 1096| 6.59k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 6.59k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1097| |#ifdef ZXC_BIG_ENDIAN 1098| | return ZXC_BSWAP32(v); 1099| |#else 1100| 6.59k| return v; 1101| 6.59k|#endif 1102| 6.59k|} zxc_dispatch.c:zxc_le64: 1114| 6.61k|static ZXC_ALWAYS_INLINE uint64_t zxc_le64(const void* p) { 1115| 6.61k| uint64_t v; 1116| 6.61k| ZXC_MEMCPY(&v, p, sizeof(v)); ------------------ | | 173| 6.61k|#define ZXC_MEMCPY(dst, src, n) __builtin_memcpy(dst, src, n) ------------------ 1117| |#ifdef ZXC_BIG_ENDIAN 1118| | return ZXC_BSWAP64(v); 1119| |#else 1120| 6.61k| return v; 1121| 6.61k|#endif 1122| 6.61k|} LLVMFuzzerTestOneInput: 31| 6.60k|int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { 32| 6.60k| static uint8_t out_buf[4 << 20]; /* 4 MiB */ 33| 6.60k| size_t out_capacity = sizeof(out_buf); 34| | 35| 6.60k| uint64_t expected_size = zxc_get_decompressed_size(data, size); 36| | 37| 6.60k| if (expected_size > 0 && expected_size <= out_capacity) out_capacity = (size_t)expected_size; ------------------ | Branch (37:9): [True: 415, False: 6.18k] | Branch (37:30): [True: 376, False: 39] ------------------ 38| | 39| 6.60k| zxc_decompress(data, size, out_buf, out_capacity, 0); 40| | 41| 6.60k| return 0; 42| 6.60k|}