/src/zlib-ng/insert_string_tpl.h
Line | Count | Source |
1 | | /* insert_string_tpl.h -- Private insert_string functions shared with more than |
2 | | * one insert string implementation |
3 | | * |
4 | | * Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler |
5 | | * |
6 | | * Copyright (C) 2013 Intel Corporation. All rights reserved. |
7 | | * Authors: |
8 | | * Wajdi Feghali <wajdi.k.feghali@intel.com> |
9 | | * Jim Guilford <james.guilford@intel.com> |
10 | | * Vinodh Gopal <vinodh.gopal@intel.com> |
11 | | * Erdinc Ozturk <erdinc.ozturk@intel.com> |
12 | | * Jim Kukunas <james.t.kukunas@linux.intel.com> |
13 | | * |
14 | | * Portions are Copyright (C) 2016 12Sided Technology, LLC. |
15 | | * Author: |
16 | | * Phil Vachon <pvachon@12sidedtech.com> |
17 | | * |
18 | | * For conditions of distribution and use, see copyright notice in zlib.h |
19 | | * |
20 | | */ |
21 | | |
22 | | #ifndef HASH_CALC_READ |
23 | | # if BYTE_ORDER == LITTLE_ENDIAN |
24 | | # define HASH_CALC_READ \ |
25 | 1.10G | val = zng_memread_4(strstart); |
26 | | # else |
27 | | # define HASH_CALC_READ \ |
28 | | val = ZSWAP32(zng_memread_4(strstart)); |
29 | | # endif |
30 | | #endif |
31 | | |
32 | | /* =========================================================================== |
33 | | * Update a hash value with the given input byte |
34 | | * IN assertion: all calls to UPDATE_HASH are made with consecutive |
35 | | * input characters, so that a running hash key can be computed from the |
36 | | * previous key instead of complete recalculation each time. |
37 | | */ |
38 | 45.1M | Z_FORCEINLINE static uint32_t UPDATE_HASH(uint32_t h, uint32_t val) { |
39 | 45.1M | HASH_CALC(h, val); |
40 | 45.1M | return h & HASH_CALC_MASK; |
41 | 45.1M | } deflate.c:update_hash_roll Line | Count | Source | 38 | 28.3k | Z_FORCEINLINE static uint32_t UPDATE_HASH(uint32_t h, uint32_t val) { | 39 | 28.3k | HASH_CALC(h, val); | 40 | 28.3k | return h & HASH_CALC_MASK; | 41 | 28.3k | } |
Unexecuted instantiation: deflate.c:update_hash Unexecuted instantiation: deflate_fast.c:update_hash Unexecuted instantiation: deflate_fast.c:update_hash_roll Unexecuted instantiation: deflate_medium.c:update_hash Unexecuted instantiation: deflate_medium.c:update_hash_roll Unexecuted instantiation: deflate_quick.c:update_hash Unexecuted instantiation: deflate_quick.c:update_hash_roll Unexecuted instantiation: deflate_slow.c:update_hash Unexecuted instantiation: deflate_slow.c:update_hash_roll Unexecuted instantiation: insert_string.c:update_hash Unexecuted instantiation: insert_string.c:update_hash_roll Unexecuted instantiation: compare256_sse2.c:update_hash_roll Unexecuted instantiation: compare256_sse2.c:update_hash compare256_avx2.c:update_hash_roll Line | Count | Source | 38 | 45.1M | Z_FORCEINLINE static uint32_t UPDATE_HASH(uint32_t h, uint32_t val) { | 39 | 45.1M | HASH_CALC(h, val); | 40 | 45.1M | return h & HASH_CALC_MASK; | 41 | 45.1M | } |
Unexecuted instantiation: compare256_avx2.c:update_hash Unexecuted instantiation: compare256_avx512.c:update_hash_roll Unexecuted instantiation: compare256_avx512.c:update_hash |
42 | | |
43 | | /* =========================================================================== |
44 | | * Quick insert string str in the dictionary using a pre-read value and set match_head |
45 | | * to the previous head of the hash chain (the most recent string with same hash key). |
46 | | * Return the previous length of the hash chain. |
47 | | */ |
48 | 184M | Z_FORCEINLINE static uint32_t QUICK_INSERT_VALUE(deflate_state *const s, uint32_t str, uint32_t val) { |
49 | 184M | uint32_t hm, head; |
50 | | |
51 | 184M | HASH_CALC_VAR_INIT; |
52 | 184M | HASH_CALC(HASH_CALC_VAR, val); |
53 | 184M | HASH_CALC_VAR &= HASH_CALC_MASK; |
54 | 184M | hm = HASH_CALC_VAR; |
55 | | |
56 | 184M | head = s->head[hm]; |
57 | 184M | if (LIKELY(head != str)) { |
58 | 184M | s->prev[str & W_MASK(s)] = (Pos)head; |
59 | 184M | s->head[hm] = (Pos)str; |
60 | 184M | } |
61 | 184M | return head; |
62 | 184M | } Unexecuted instantiation: deflate.c:quick_insert_value Unexecuted instantiation: deflate.c:quick_insert_value_roll deflate_fast.c:quick_insert_value Line | Count | Source | 48 | 16.7M | Z_FORCEINLINE static uint32_t QUICK_INSERT_VALUE(deflate_state *const s, uint32_t str, uint32_t val) { | 49 | 16.7M | uint32_t hm, head; | 50 | | | 51 | 16.7M | HASH_CALC_VAR_INIT; | 52 | 16.7M | HASH_CALC(HASH_CALC_VAR, val); | 53 | 16.7M | HASH_CALC_VAR &= HASH_CALC_MASK; | 54 | 16.7M | hm = HASH_CALC_VAR; | 55 | | | 56 | 16.7M | head = s->head[hm]; | 57 | 16.7M | if (LIKELY(head != str)) { | 58 | 16.6M | s->prev[str & W_MASK(s)] = (Pos)head; | 59 | 16.6M | s->head[hm] = (Pos)str; | 60 | 16.6M | } | 61 | 16.7M | return head; | 62 | 16.7M | } |
Unexecuted instantiation: deflate_fast.c:quick_insert_value_roll Unexecuted instantiation: deflate_medium.c:quick_insert_value Unexecuted instantiation: deflate_medium.c:quick_insert_value_roll deflate_quick.c:quick_insert_value Line | Count | Source | 48 | 167M | Z_FORCEINLINE static uint32_t QUICK_INSERT_VALUE(deflate_state *const s, uint32_t str, uint32_t val) { | 49 | 167M | uint32_t hm, head; | 50 | | | 51 | 167M | HASH_CALC_VAR_INIT; | 52 | 167M | HASH_CALC(HASH_CALC_VAR, val); | 53 | 167M | HASH_CALC_VAR &= HASH_CALC_MASK; | 54 | 167M | hm = HASH_CALC_VAR; | 55 | | | 56 | 167M | head = s->head[hm]; | 57 | 167M | if (LIKELY(head != str)) { | 58 | 167M | s->prev[str & W_MASK(s)] = (Pos)head; | 59 | 167M | s->head[hm] = (Pos)str; | 60 | 167M | } | 61 | 167M | return head; | 62 | 167M | } |
Unexecuted instantiation: deflate_quick.c:quick_insert_value_roll Unexecuted instantiation: deflate_slow.c:quick_insert_value Unexecuted instantiation: deflate_slow.c:quick_insert_value_roll Unexecuted instantiation: insert_string.c:quick_insert_value Unexecuted instantiation: insert_string.c:quick_insert_value_roll Unexecuted instantiation: compare256_sse2.c:quick_insert_value Unexecuted instantiation: compare256_sse2.c:quick_insert_value_roll Unexecuted instantiation: compare256_avx2.c:quick_insert_value Unexecuted instantiation: compare256_avx2.c:quick_insert_value_roll Unexecuted instantiation: compare256_avx512.c:quick_insert_value Unexecuted instantiation: compare256_avx512.c:quick_insert_value_roll |
63 | | |
64 | | /* =========================================================================== |
65 | | * Quick insert string str in the dictionary and set match_head to the previous head |
66 | | * of the hash chain (the most recent string with same hash key). Return |
67 | | * the previous length of the hash chain. |
68 | | */ |
69 | 888M | Z_FORCEINLINE static uint32_t QUICK_INSERT_STRING(deflate_state *const s, uint32_t str) { |
70 | 888M | uint8_t *strstart = s->window + str + HASH_CALC_OFFSET; |
71 | 888M | uint32_t val, hm, head; |
72 | | |
73 | 888M | HASH_CALC_VAR_INIT; |
74 | 888M | HASH_CALC_READ; |
75 | 888M | HASH_CALC(HASH_CALC_VAR, val); |
76 | 888M | HASH_CALC_VAR &= HASH_CALC_MASK; |
77 | 888M | hm = HASH_CALC_VAR; |
78 | | |
79 | 888M | head = s->head[hm]; |
80 | 888M | if (LIKELY(head != str)) { |
81 | 733M | s->prev[str & W_MASK(s)] = (Pos)head; |
82 | 733M | s->head[hm] = (Pos)str; |
83 | 733M | } |
84 | 888M | return head; |
85 | 888M | } deflate.c:quick_insert_string Line | Count | Source | 69 | 155M | Z_FORCEINLINE static uint32_t QUICK_INSERT_STRING(deflate_state *const s, uint32_t str) { | 70 | 155M | uint8_t *strstart = s->window + str + HASH_CALC_OFFSET; | 71 | 155M | uint32_t val, hm, head; | 72 | | | 73 | 155M | HASH_CALC_VAR_INIT; | 74 | 155M | HASH_CALC_READ; | 75 | 155M | HASH_CALC(HASH_CALC_VAR, val); | 76 | 155M | HASH_CALC_VAR &= HASH_CALC_MASK; | 77 | 155M | hm = HASH_CALC_VAR; | 78 | | | 79 | 155M | head = s->head[hm]; | 80 | 155M | if (LIKELY(head != str)) { | 81 | 97.8k | s->prev[str & W_MASK(s)] = (Pos)head; | 82 | 97.8k | s->head[hm] = (Pos)str; | 83 | 97.8k | } | 84 | 155M | return head; | 85 | 155M | } |
Unexecuted instantiation: deflate.c:quick_insert_string_roll deflate_fast.c:quick_insert_string Line | Count | Source | 69 | 587k | Z_FORCEINLINE static uint32_t QUICK_INSERT_STRING(deflate_state *const s, uint32_t str) { | 70 | 587k | uint8_t *strstart = s->window + str + HASH_CALC_OFFSET; | 71 | 587k | uint32_t val, hm, head; | 72 | | | 73 | 587k | HASH_CALC_VAR_INIT; | 74 | 587k | HASH_CALC_READ; | 75 | 587k | HASH_CALC(HASH_CALC_VAR, val); | 76 | 587k | HASH_CALC_VAR &= HASH_CALC_MASK; | 77 | 587k | hm = HASH_CALC_VAR; | 78 | | | 79 | 587k | head = s->head[hm]; | 80 | 587k | if (LIKELY(head != str)) { | 81 | 587k | s->prev[str & W_MASK(s)] = (Pos)head; | 82 | 587k | s->head[hm] = (Pos)str; | 83 | 587k | } | 84 | 587k | return head; | 85 | 587k | } |
Unexecuted instantiation: deflate_fast.c:quick_insert_string_roll deflate_medium.c:quick_insert_string Line | Count | Source | 69 | 548M | Z_FORCEINLINE static uint32_t QUICK_INSERT_STRING(deflate_state *const s, uint32_t str) { | 70 | 548M | uint8_t *strstart = s->window + str + HASH_CALC_OFFSET; | 71 | 548M | uint32_t val, hm, head; | 72 | | | 73 | 548M | HASH_CALC_VAR_INIT; | 74 | 548M | HASH_CALC_READ; | 75 | 548M | HASH_CALC(HASH_CALC_VAR, val); | 76 | 548M | HASH_CALC_VAR &= HASH_CALC_MASK; | 77 | 548M | hm = HASH_CALC_VAR; | 78 | | | 79 | 548M | head = s->head[hm]; | 80 | 548M | if (LIKELY(head != str)) { | 81 | 548M | s->prev[str & W_MASK(s)] = (Pos)head; | 82 | 548M | s->head[hm] = (Pos)str; | 83 | 548M | } | 84 | 548M | return head; | 85 | 548M | } |
Unexecuted instantiation: deflate_medium.c:quick_insert_string_roll Unexecuted instantiation: deflate_quick.c:quick_insert_string Unexecuted instantiation: deflate_quick.c:quick_insert_string_roll deflate_slow.c:quick_insert_string_roll Line | Count | Source | 69 | 47.4M | Z_FORCEINLINE static uint32_t QUICK_INSERT_STRING(deflate_state *const s, uint32_t str) { | 70 | 47.4M | uint8_t *strstart = s->window + str + HASH_CALC_OFFSET; | 71 | 47.4M | uint32_t val, hm, head; | 72 | | | 73 | 47.4M | HASH_CALC_VAR_INIT; | 74 | 47.4M | HASH_CALC_READ; | 75 | 47.4M | HASH_CALC(HASH_CALC_VAR, val); | 76 | 47.4M | HASH_CALC_VAR &= HASH_CALC_MASK; | 77 | 47.4M | hm = HASH_CALC_VAR; | 78 | | | 79 | 47.4M | head = s->head[hm]; | 80 | 47.4M | if (LIKELY(head != str)) { | 81 | 47.4M | s->prev[str & W_MASK(s)] = (Pos)head; | 82 | 47.4M | s->head[hm] = (Pos)str; | 83 | 47.4M | } | 84 | 47.4M | return head; | 85 | 47.4M | } |
deflate_slow.c:quick_insert_string Line | Count | Source | 69 | 136M | Z_FORCEINLINE static uint32_t QUICK_INSERT_STRING(deflate_state *const s, uint32_t str) { | 70 | 136M | uint8_t *strstart = s->window + str + HASH_CALC_OFFSET; | 71 | 136M | uint32_t val, hm, head; | 72 | | | 73 | 136M | HASH_CALC_VAR_INIT; | 74 | 136M | HASH_CALC_READ; | 75 | 136M | HASH_CALC(HASH_CALC_VAR, val); | 76 | 136M | HASH_CALC_VAR &= HASH_CALC_MASK; | 77 | 136M | hm = HASH_CALC_VAR; | 78 | | | 79 | 136M | head = s->head[hm]; | 80 | 136M | if (LIKELY(head != str)) { | 81 | 136M | s->prev[str & W_MASK(s)] = (Pos)head; | 82 | 136M | s->head[hm] = (Pos)str; | 83 | 136M | } | 84 | 136M | return head; | 85 | 136M | } |
Unexecuted instantiation: insert_string.c:quick_insert_string Unexecuted instantiation: insert_string.c:quick_insert_string_roll Unexecuted instantiation: compare256_sse2.c:quick_insert_string Unexecuted instantiation: compare256_sse2.c:quick_insert_string_roll Unexecuted instantiation: compare256_avx2.c:quick_insert_string Unexecuted instantiation: compare256_avx2.c:quick_insert_string_roll Unexecuted instantiation: compare256_avx512.c:quick_insert_string Unexecuted instantiation: compare256_avx512.c:quick_insert_string_roll |
86 | | |
87 | | /* =========================================================================== |
88 | | * Insert string str in the dictionary and set match_head to the previous head |
89 | | * of the hash chain (the most recent string with same hash key). Return |
90 | | * the previous length of the hash chain. |
91 | | * IN assertion: all calls to INSERT_STRING are made with consecutive |
92 | | * input characters and the first STD_MIN_MATCH bytes of str are valid |
93 | | * (except for the last STD_MIN_MATCH-1 bytes of the input file). |
94 | | */ |
95 | 28.5M | Z_FORCEINLINE static void INSERT_STRING(deflate_state *const s, uint32_t str, uint32_t count) { |
96 | 28.5M | uint8_t *strstart = s->window + str + HASH_CALC_OFFSET; |
97 | 28.5M | uint8_t *strend = strstart + count; |
98 | | |
99 | | /* Local pointers to avoid indirection */ |
100 | 28.5M | Pos *headp = s->head; |
101 | 28.5M | Pos *prevp = s->prev; |
102 | 28.5M | const unsigned int w_mask = W_MASK(s); |
103 | | |
104 | 734M | for (uint32_t idx = str; strstart < strend; idx++, strstart++) { |
105 | 705M | uint32_t val, hm, head; |
106 | | |
107 | 705M | HASH_CALC_VAR_INIT; |
108 | 705M | HASH_CALC_READ; |
109 | 705M | HASH_CALC(HASH_CALC_VAR, val); |
110 | 705M | HASH_CALC_VAR &= HASH_CALC_MASK; |
111 | 705M | hm = HASH_CALC_VAR; |
112 | | |
113 | 705M | head = headp[hm]; |
114 | 705M | if (LIKELY(head != idx)) { |
115 | 705M | prevp[idx & w_mask] = (Pos)head; |
116 | 705M | headp[hm] = (Pos)idx; |
117 | 705M | } |
118 | 705M | } |
119 | 28.5M | } Unexecuted instantiation: deflate.c:insert_string_static Unexecuted instantiation: deflate.c:insert_string_roll_static deflate_fast.c:insert_string_static Line | Count | Source | 95 | 422k | Z_FORCEINLINE static void INSERT_STRING(deflate_state *const s, uint32_t str, uint32_t count) { | 96 | 422k | uint8_t *strstart = s->window + str + HASH_CALC_OFFSET; | 97 | 422k | uint8_t *strend = strstart + count; | 98 | | | 99 | | /* Local pointers to avoid indirection */ | 100 | 422k | Pos *headp = s->head; | 101 | 422k | Pos *prevp = s->prev; | 102 | 422k | const unsigned int w_mask = W_MASK(s); | 103 | | | 104 | 1.68M | for (uint32_t idx = str; strstart < strend; idx++, strstart++) { | 105 | 1.26M | uint32_t val, hm, head; | 106 | | | 107 | 1.26M | HASH_CALC_VAR_INIT; | 108 | 1.26M | HASH_CALC_READ; | 109 | 1.26M | HASH_CALC(HASH_CALC_VAR, val); | 110 | 1.26M | HASH_CALC_VAR &= HASH_CALC_MASK; | 111 | 1.26M | hm = HASH_CALC_VAR; | 112 | | | 113 | 1.26M | head = headp[hm]; | 114 | 1.26M | if (LIKELY(head != idx)) { | 115 | 1.26M | prevp[idx & w_mask] = (Pos)head; | 116 | 1.26M | headp[hm] = (Pos)idx; | 117 | 1.26M | } | 118 | 1.26M | } | 119 | 422k | } |
Unexecuted instantiation: deflate_fast.c:insert_string_roll_static Unexecuted instantiation: deflate_medium.c:insert_string_static Unexecuted instantiation: deflate_medium.c:insert_string_roll_static Unexecuted instantiation: deflate_quick.c:insert_string_static Unexecuted instantiation: deflate_quick.c:insert_string_roll_static Unexecuted instantiation: deflate_slow.c:insert_string_static Unexecuted instantiation: deflate_slow.c:insert_string_roll_static insert_string.c:insert_string_static Line | Count | Source | 95 | 20.6M | Z_FORCEINLINE static void INSERT_STRING(deflate_state *const s, uint32_t str, uint32_t count) { | 96 | 20.6M | uint8_t *strstart = s->window + str + HASH_CALC_OFFSET; | 97 | 20.6M | uint8_t *strend = strstart + count; | 98 | | | 99 | | /* Local pointers to avoid indirection */ | 100 | 20.6M | Pos *headp = s->head; | 101 | 20.6M | Pos *prevp = s->prev; | 102 | 20.6M | const unsigned int w_mask = W_MASK(s); | 103 | | | 104 | 279M | for (uint32_t idx = str; strstart < strend; idx++, strstart++) { | 105 | 258M | uint32_t val, hm, head; | 106 | | | 107 | 258M | HASH_CALC_VAR_INIT; | 108 | 258M | HASH_CALC_READ; | 109 | 258M | HASH_CALC(HASH_CALC_VAR, val); | 110 | 258M | HASH_CALC_VAR &= HASH_CALC_MASK; | 111 | 258M | hm = HASH_CALC_VAR; | 112 | | | 113 | 258M | head = headp[hm]; | 114 | 258M | if (LIKELY(head != idx)) { | 115 | 258M | prevp[idx & w_mask] = (Pos)head; | 116 | 258M | headp[hm] = (Pos)idx; | 117 | 258M | } | 118 | 258M | } | 119 | 20.6M | } |
insert_string.c:insert_string_roll_static Line | Count | Source | 95 | 7.46M | Z_FORCEINLINE static void INSERT_STRING(deflate_state *const s, uint32_t str, uint32_t count) { | 96 | 7.46M | uint8_t *strstart = s->window + str + HASH_CALC_OFFSET; | 97 | 7.46M | uint8_t *strend = strstart + count; | 98 | | | 99 | | /* Local pointers to avoid indirection */ | 100 | 7.46M | Pos *headp = s->head; | 101 | 7.46M | Pos *prevp = s->prev; | 102 | 7.46M | const unsigned int w_mask = W_MASK(s); | 103 | | | 104 | 452M | for (uint32_t idx = str; strstart < strend; idx++, strstart++) { | 105 | 445M | uint32_t val, hm, head; | 106 | | | 107 | 445M | HASH_CALC_VAR_INIT; | 108 | 445M | HASH_CALC_READ; | 109 | 445M | HASH_CALC(HASH_CALC_VAR, val); | 110 | 445M | HASH_CALC_VAR &= HASH_CALC_MASK; | 111 | 445M | hm = HASH_CALC_VAR; | 112 | | | 113 | 445M | head = headp[hm]; | 114 | 445M | if (LIKELY(head != idx)) { | 115 | 445M | prevp[idx & w_mask] = (Pos)head; | 116 | 445M | headp[hm] = (Pos)idx; | 117 | 445M | } | 118 | 445M | } | 119 | 7.46M | } |
Unexecuted instantiation: compare256_sse2.c:insert_string_static Unexecuted instantiation: compare256_sse2.c:insert_string_roll_static Unexecuted instantiation: compare256_avx2.c:insert_string_static Unexecuted instantiation: compare256_avx2.c:insert_string_roll_static Unexecuted instantiation: compare256_avx512.c:insert_string_static Unexecuted instantiation: compare256_avx512.c:insert_string_roll_static |