/src/libwebp/src/utils/bit_reader_inl_utils.h
Line | Count | Source |
1 | | // Copyright 2014 Google Inc. All Rights Reserved. |
2 | | // |
3 | | // Use of this source code is governed by a BSD-style license |
4 | | // that can be found in the COPYING file in the root of the source |
5 | | // tree. An additional intellectual property rights grant can be found |
6 | | // in the file PATENTS. All contributing project authors may |
7 | | // be found in the AUTHORS file in the root of the source tree. |
8 | | // ----------------------------------------------------------------------------- |
9 | | // |
10 | | // Specific inlined methods for boolean decoder [VP8GetBit() ...] |
11 | | // This file should be included by the .c sources that actually need to call |
12 | | // these methods. |
13 | | // |
14 | | // Author: Skal (pascal.massimino@gmail.com) |
15 | | |
16 | | #ifndef WEBP_UTILS_BIT_READER_INL_UTILS_H_ |
17 | | #define WEBP_UTILS_BIT_READER_INL_UTILS_H_ |
18 | | |
19 | | #ifdef HAVE_CONFIG_H |
20 | | #include "src/webp/config.h" |
21 | | #endif |
22 | | |
23 | | #include <assert.h> |
24 | | #include <string.h> // for memcpy |
25 | | |
26 | | #include "src/dsp/cpu.h" |
27 | | #include "src/dsp/dsp.h" |
28 | | #include "src/utils/bit_reader_utils.h" |
29 | | #include "src/utils/bounds_safety.h" |
30 | | #include "src/utils/endian_inl_utils.h" |
31 | | #include "src/utils/utils.h" |
32 | | #include "src/webp/types.h" |
33 | | |
34 | | WEBP_ASSUME_UNSAFE_INDEXABLE_ABI |
35 | | |
36 | | #ifdef __cplusplus |
37 | | extern "C" { |
38 | | #endif |
39 | | |
40 | | //------------------------------------------------------------------------------ |
41 | | // Derived type lbit_t = natural type for memory I/O |
42 | | |
43 | | #if (BITS > 32) |
44 | | typedef uint64_t lbit_t; |
45 | | #elif (BITS > 16) |
46 | | typedef uint32_t lbit_t; |
47 | | #elif (BITS > 8) |
48 | | typedef uint16_t lbit_t; |
49 | | #else |
50 | | typedef uint8_t lbit_t; |
51 | | #endif |
52 | | |
53 | | extern const uint8_t kVP8Log2Range[128]; |
54 | | extern const uint8_t kVP8NewRange[128]; |
55 | | |
56 | | // special case for the tail byte-reading |
57 | | void VP8LoadFinalBytes(VP8BitReader* const br); |
58 | | |
59 | | //------------------------------------------------------------------------------ |
60 | | // Inlined critical functions |
61 | | |
62 | | // makes sure br->value has at least BITS bits worth of data |
63 | | static WEBP_UBSAN_IGNORE_UNDEF WEBP_INLINE void VP8LoadNewBytes( |
64 | 1.28M | VP8BitReader* WEBP_RESTRICT const br) { |
65 | 1.28M | assert(br != NULL && br->buf != NULL); |
66 | | // Read 'BITS' bits at a time if possible. |
67 | 1.28M | if (br->buf < br->buf_max) { |
68 | | // convert memory type to register type (with some zero'ing!) |
69 | 36.5k | bit_t bits; |
70 | | #if defined(WEBP_USE_MIPS32) |
71 | | // This is needed because of un-aligned read. |
72 | | lbit_t in_bits; |
73 | | lbit_t* p_buf = (lbit_t*)br->buf; |
74 | | __asm__ volatile( |
75 | | ".set push \n\t" |
76 | | ".set at \n\t" |
77 | | ".set macro \n\t" |
78 | | "ulw %[in_bits], 0(%[p_buf]) \n\t" |
79 | | ".set pop \n\t" |
80 | | : [in_bits] "=r"(in_bits) |
81 | | : [p_buf] "r"(p_buf) |
82 | | : "memory", "at"); |
83 | | #else |
84 | 36.5k | lbit_t in_bits; |
85 | 36.5k | WEBP_UNSAFE_MEMCPY(&in_bits, br->buf, sizeof(in_bits)); |
86 | 36.5k | #endif |
87 | 36.5k | br->buf += BITS >> 3; |
88 | 36.5k | WEBP_SELF_ASSIGN(br->buf_end); |
89 | 36.5k | #if !defined(WORDS_BIGENDIAN) |
90 | 36.5k | #if (BITS > 32) |
91 | 36.5k | bits = BSwap64(in_bits); |
92 | 36.5k | bits >>= 64 - BITS; |
93 | | #elif (BITS >= 24) |
94 | | bits = BSwap32(in_bits); |
95 | | bits >>= (32 - BITS); |
96 | | #elif (BITS == 16) |
97 | | bits = BSwap16(in_bits); |
98 | | #else // BITS == 8 |
99 | | bits = (bit_t)in_bits; |
100 | | #endif // BITS > 32 |
101 | | #else // WORDS_BIGENDIAN |
102 | | bits = (bit_t)in_bits; |
103 | | if (BITS != 8 * sizeof(bit_t)) bits >>= (8 * sizeof(bit_t) - BITS); |
104 | | #endif |
105 | 36.5k | br->value = bits | (br->value << BITS); |
106 | 36.5k | br->bits += BITS; |
107 | 1.24M | } else { |
108 | 1.24M | VP8LoadFinalBytes(br); // no need to be inlined |
109 | 1.24M | } |
110 | 1.28M | } vp8_dec.c:VP8LoadNewBytes Line | Count | Source | 64 | 410k | VP8BitReader* WEBP_RESTRICT const br) { | 65 | 410k | assert(br != NULL && br->buf != NULL); | 66 | | // Read 'BITS' bits at a time if possible. | 67 | 410k | if (br->buf < br->buf_max) { | 68 | | // convert memory type to register type (with some zero'ing!) | 69 | 18.3k | bit_t bits; | 70 | | #if defined(WEBP_USE_MIPS32) | 71 | | // This is needed because of un-aligned read. | 72 | | lbit_t in_bits; | 73 | | lbit_t* p_buf = (lbit_t*)br->buf; | 74 | | __asm__ volatile( | 75 | | ".set push \n\t" | 76 | | ".set at \n\t" | 77 | | ".set macro \n\t" | 78 | | "ulw %[in_bits], 0(%[p_buf]) \n\t" | 79 | | ".set pop \n\t" | 80 | | : [in_bits] "=r"(in_bits) | 81 | | : [p_buf] "r"(p_buf) | 82 | | : "memory", "at"); | 83 | | #else | 84 | 18.3k | lbit_t in_bits; | 85 | 18.3k | WEBP_UNSAFE_MEMCPY(&in_bits, br->buf, sizeof(in_bits)); | 86 | 18.3k | #endif | 87 | 18.3k | br->buf += BITS >> 3; | 88 | 18.3k | WEBP_SELF_ASSIGN(br->buf_end); | 89 | 18.3k | #if !defined(WORDS_BIGENDIAN) | 90 | 18.3k | #if (BITS > 32) | 91 | 18.3k | bits = BSwap64(in_bits); | 92 | 18.3k | bits >>= 64 - BITS; | 93 | | #elif (BITS >= 24) | 94 | | bits = BSwap32(in_bits); | 95 | | bits >>= (32 - BITS); | 96 | | #elif (BITS == 16) | 97 | | bits = BSwap16(in_bits); | 98 | | #else // BITS == 8 | 99 | | bits = (bit_t)in_bits; | 100 | | #endif // BITS > 32 | 101 | | #else // WORDS_BIGENDIAN | 102 | | bits = (bit_t)in_bits; | 103 | | if (BITS != 8 * sizeof(bit_t)) bits >>= (8 * sizeof(bit_t) - BITS); | 104 | | #endif | 105 | 18.3k | br->value = bits | (br->value << BITS); | 106 | 18.3k | br->bits += BITS; | 107 | 391k | } else { | 108 | 391k | VP8LoadFinalBytes(br); // no need to be inlined | 109 | 391k | } | 110 | 410k | } |
tree_dec.c:VP8LoadNewBytes Line | Count | Source | 64 | 669k | VP8BitReader* WEBP_RESTRICT const br) { | 65 | 669k | assert(br != NULL && br->buf != NULL); | 66 | | // Read 'BITS' bits at a time if possible. | 67 | 669k | if (br->buf < br->buf_max) { | 68 | | // convert memory type to register type (with some zero'ing!) | 69 | 11.3k | bit_t bits; | 70 | | #if defined(WEBP_USE_MIPS32) | 71 | | // This is needed because of un-aligned read. | 72 | | lbit_t in_bits; | 73 | | lbit_t* p_buf = (lbit_t*)br->buf; | 74 | | __asm__ volatile( | 75 | | ".set push \n\t" | 76 | | ".set at \n\t" | 77 | | ".set macro \n\t" | 78 | | "ulw %[in_bits], 0(%[p_buf]) \n\t" | 79 | | ".set pop \n\t" | 80 | | : [in_bits] "=r"(in_bits) | 81 | | : [p_buf] "r"(p_buf) | 82 | | : "memory", "at"); | 83 | | #else | 84 | 11.3k | lbit_t in_bits; | 85 | 11.3k | WEBP_UNSAFE_MEMCPY(&in_bits, br->buf, sizeof(in_bits)); | 86 | 11.3k | #endif | 87 | 11.3k | br->buf += BITS >> 3; | 88 | 11.3k | WEBP_SELF_ASSIGN(br->buf_end); | 89 | 11.3k | #if !defined(WORDS_BIGENDIAN) | 90 | 11.3k | #if (BITS > 32) | 91 | 11.3k | bits = BSwap64(in_bits); | 92 | 11.3k | bits >>= 64 - BITS; | 93 | | #elif (BITS >= 24) | 94 | | bits = BSwap32(in_bits); | 95 | | bits >>= (32 - BITS); | 96 | | #elif (BITS == 16) | 97 | | bits = BSwap16(in_bits); | 98 | | #else // BITS == 8 | 99 | | bits = (bit_t)in_bits; | 100 | | #endif // BITS > 32 | 101 | | #else // WORDS_BIGENDIAN | 102 | | bits = (bit_t)in_bits; | 103 | | if (BITS != 8 * sizeof(bit_t)) bits >>= (8 * sizeof(bit_t) - BITS); | 104 | | #endif | 105 | 11.3k | br->value = bits | (br->value << BITS); | 106 | 11.3k | br->bits += BITS; | 107 | 658k | } else { | 108 | 658k | VP8LoadFinalBytes(br); // no need to be inlined | 109 | 658k | } | 110 | 669k | } |
bit_reader_utils.c:VP8LoadNewBytes Line | Count | Source | 64 | 204k | VP8BitReader* WEBP_RESTRICT const br) { | 65 | 204k | assert(br != NULL && br->buf != NULL); | 66 | | // Read 'BITS' bits at a time if possible. | 67 | 204k | if (br->buf < br->buf_max) { | 68 | | // convert memory type to register type (with some zero'ing!) | 69 | 6.86k | bit_t bits; | 70 | | #if defined(WEBP_USE_MIPS32) | 71 | | // This is needed because of un-aligned read. | 72 | | lbit_t in_bits; | 73 | | lbit_t* p_buf = (lbit_t*)br->buf; | 74 | | __asm__ volatile( | 75 | | ".set push \n\t" | 76 | | ".set at \n\t" | 77 | | ".set macro \n\t" | 78 | | "ulw %[in_bits], 0(%[p_buf]) \n\t" | 79 | | ".set pop \n\t" | 80 | | : [in_bits] "=r"(in_bits) | 81 | | : [p_buf] "r"(p_buf) | 82 | | : "memory", "at"); | 83 | | #else | 84 | 6.86k | lbit_t in_bits; | 85 | 6.86k | WEBP_UNSAFE_MEMCPY(&in_bits, br->buf, sizeof(in_bits)); | 86 | 6.86k | #endif | 87 | 6.86k | br->buf += BITS >> 3; | 88 | 6.86k | WEBP_SELF_ASSIGN(br->buf_end); | 89 | 6.86k | #if !defined(WORDS_BIGENDIAN) | 90 | 6.86k | #if (BITS > 32) | 91 | 6.86k | bits = BSwap64(in_bits); | 92 | 6.86k | bits >>= 64 - BITS; | 93 | | #elif (BITS >= 24) | 94 | | bits = BSwap32(in_bits); | 95 | | bits >>= (32 - BITS); | 96 | | #elif (BITS == 16) | 97 | | bits = BSwap16(in_bits); | 98 | | #else // BITS == 8 | 99 | | bits = (bit_t)in_bits; | 100 | | #endif // BITS > 32 | 101 | | #else // WORDS_BIGENDIAN | 102 | | bits = (bit_t)in_bits; | 103 | | if (BITS != 8 * sizeof(bit_t)) bits >>= (8 * sizeof(bit_t) - BITS); | 104 | | #endif | 105 | 6.86k | br->value = bits | (br->value << BITS); | 106 | 6.86k | br->bits += BITS; | 107 | 197k | } else { | 108 | 197k | VP8LoadFinalBytes(br); // no need to be inlined | 109 | 197k | } | 110 | 204k | } |
|
111 | | |
112 | | // Read a bit with proba 'prob'. Speed-critical function! |
113 | | static WEBP_INLINE int VP8GetBit(VP8BitReader* WEBP_RESTRICT const br, int prob, |
114 | 7.87M | const char label[]) { |
115 | | // Don't move this declaration! It makes a big speed difference to store |
116 | | // 'range' *before* calling VP8LoadNewBytes(), even if this function doesn't |
117 | | // alter br->range value. |
118 | 7.87M | range_t range = br->range; |
119 | 7.87M | if (br->bits < 0) { |
120 | 1.25M | VP8LoadNewBytes(br); |
121 | 1.25M | } |
122 | 7.87M | { |
123 | 7.87M | const int pos = br->bits; |
124 | 7.87M | const range_t split = (range * prob) >> 8; |
125 | 7.87M | const range_t value = (range_t)(br->value >> pos); |
126 | 7.87M | const int bit = (value > split); |
127 | 7.87M | if (bit) { |
128 | 1.68M | range -= split; |
129 | 1.68M | br->value -= (bit_t)(split + 1) << pos; |
130 | 6.19M | } else { |
131 | 6.19M | range = split + 1; |
132 | 6.19M | } |
133 | 7.87M | { |
134 | 7.87M | const int shift = 7 ^ BitsLog2Floor(range); |
135 | 7.87M | range <<= shift; |
136 | 7.87M | br->bits -= shift; |
137 | 7.87M | } |
138 | 7.87M | br->range = range - 1; |
139 | 7.87M | BT_TRACK(br); |
140 | 7.87M | return bit; |
141 | 7.87M | } |
142 | 7.87M | } Line | Count | Source | 114 | 2.19M | const char label[]) { | 115 | | // Don't move this declaration! It makes a big speed difference to store | 116 | | // 'range' *before* calling VP8LoadNewBytes(), even if this function doesn't | 117 | | // alter br->range value. | 118 | 2.19M | range_t range = br->range; | 119 | 2.19M | if (br->bits < 0) { | 120 | 387k | VP8LoadNewBytes(br); | 121 | 387k | } | 122 | 2.19M | { | 123 | 2.19M | const int pos = br->bits; | 124 | 2.19M | const range_t split = (range * prob) >> 8; | 125 | 2.19M | const range_t value = (range_t)(br->value >> pos); | 126 | 2.19M | const int bit = (value > split); | 127 | 2.19M | if (bit) { | 128 | 839k | range -= split; | 129 | 839k | br->value -= (bit_t)(split + 1) << pos; | 130 | 1.36M | } else { | 131 | 1.36M | range = split + 1; | 132 | 1.36M | } | 133 | 2.19M | { | 134 | 2.19M | const int shift = 7 ^ BitsLog2Floor(range); | 135 | 2.19M | range <<= shift; | 136 | 2.19M | br->bits -= shift; | 137 | 2.19M | } | 138 | 2.19M | br->range = range - 1; | 139 | 2.19M | BT_TRACK(br); | 140 | 2.19M | return bit; | 141 | 2.19M | } | 142 | 2.19M | } |
Line | Count | Source | 114 | 5.28M | const char label[]) { | 115 | | // Don't move this declaration! It makes a big speed difference to store | 116 | | // 'range' *before* calling VP8LoadNewBytes(), even if this function doesn't | 117 | | // alter br->range value. | 118 | 5.28M | range_t range = br->range; | 119 | 5.28M | if (br->bits < 0) { | 120 | 669k | VP8LoadNewBytes(br); | 121 | 669k | } | 122 | 5.28M | { | 123 | 5.28M | const int pos = br->bits; | 124 | 5.28M | const range_t split = (range * prob) >> 8; | 125 | 5.28M | const range_t value = (range_t)(br->value >> pos); | 126 | 5.28M | const int bit = (value > split); | 127 | 5.28M | if (bit) { | 128 | 541k | range -= split; | 129 | 541k | br->value -= (bit_t)(split + 1) << pos; | 130 | 4.74M | } else { | 131 | 4.74M | range = split + 1; | 132 | 4.74M | } | 133 | 5.28M | { | 134 | 5.28M | const int shift = 7 ^ BitsLog2Floor(range); | 135 | 5.28M | range <<= shift; | 136 | 5.28M | br->bits -= shift; | 137 | 5.28M | } | 138 | 5.28M | br->range = range - 1; | 139 | 5.28M | BT_TRACK(br); | 140 | 5.28M | return bit; | 141 | 5.28M | } | 142 | 5.28M | } |
bit_reader_utils.c:VP8GetBit Line | Count | Source | 114 | 395k | const char label[]) { | 115 | | // Don't move this declaration! It makes a big speed difference to store | 116 | | // 'range' *before* calling VP8LoadNewBytes(), even if this function doesn't | 117 | | // alter br->range value. | 118 | 395k | range_t range = br->range; | 119 | 395k | if (br->bits < 0) { | 120 | 199k | VP8LoadNewBytes(br); | 121 | 199k | } | 122 | 395k | { | 123 | 395k | const int pos = br->bits; | 124 | 395k | const range_t split = (range * prob) >> 8; | 125 | 395k | const range_t value = (range_t)(br->value >> pos); | 126 | 395k | const int bit = (value > split); | 127 | 395k | if (bit) { | 128 | 300k | range -= split; | 129 | 300k | br->value -= (bit_t)(split + 1) << pos; | 130 | 300k | } else { | 131 | 94.9k | range = split + 1; | 132 | 94.9k | } | 133 | 395k | { | 134 | 395k | const int shift = 7 ^ BitsLog2Floor(range); | 135 | 395k | range <<= shift; | 136 | 395k | br->bits -= shift; | 137 | 395k | } | 138 | 395k | br->range = range - 1; | 139 | 395k | BT_TRACK(br); | 140 | 395k | return bit; | 141 | 395k | } | 142 | 395k | } |
|
143 | | |
144 | | // simplified version of VP8GetBit() for prob=0x80 (note shift is always 1 here) |
145 | | static WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW WEBP_INLINE int VP8GetSigned( |
146 | 185k | VP8BitReader* WEBP_RESTRICT const br, int v, const char label[]) { |
147 | 185k | if (br->bits < 0) { |
148 | 22.5k | VP8LoadNewBytes(br); |
149 | 22.5k | } |
150 | 185k | { |
151 | 185k | const int pos = br->bits; |
152 | 185k | const range_t split = br->range >> 1; |
153 | 185k | const range_t value = (range_t)(br->value >> pos); |
154 | 185k | const int32_t mask = (int32_t)(split - value) >> 31; // -1 or 0 |
155 | 185k | br->bits -= 1; |
156 | 185k | br->range += (range_t)mask; |
157 | 185k | br->range |= 1; |
158 | 185k | br->value -= (bit_t)((split + 1) & (uint32_t)mask) << pos; |
159 | 185k | BT_TRACK(br); |
160 | 185k | return (v ^ mask) - mask; |
161 | 185k | } |
162 | 185k | } Line | Count | Source | 146 | 185k | VP8BitReader* WEBP_RESTRICT const br, int v, const char label[]) { | 147 | 185k | if (br->bits < 0) { | 148 | 22.5k | VP8LoadNewBytes(br); | 149 | 22.5k | } | 150 | 185k | { | 151 | 185k | const int pos = br->bits; | 152 | 185k | const range_t split = br->range >> 1; | 153 | 185k | const range_t value = (range_t)(br->value >> pos); | 154 | 185k | const int32_t mask = (int32_t)(split - value) >> 31; // -1 or 0 | 155 | 185k | br->bits -= 1; | 156 | 185k | br->range += (range_t)mask; | 157 | 185k | br->range |= 1; | 158 | 185k | br->value -= (bit_t)((split + 1) & (uint32_t)mask) << pos; | 159 | 185k | BT_TRACK(br); | 160 | 185k | return (v ^ mask) - mask; | 161 | 185k | } | 162 | 185k | } |
Unexecuted instantiation: tree_dec.c:VP8GetSigned Unexecuted instantiation: bit_reader_utils.c:VP8GetSigned |
163 | | |
164 | | static WEBP_INLINE int VP8GetBitAlt(VP8BitReader* WEBP_RESTRICT const br, |
165 | 0 | int prob, const char label[]) { |
166 | | // Don't move this declaration! It makes a big speed difference to store |
167 | | // 'range' *before* calling VP8LoadNewBytes(), even if this function doesn't |
168 | | // alter br->range value. |
169 | 0 | range_t range = br->range; |
170 | 0 | if (br->bits < 0) { |
171 | 0 | VP8LoadNewBytes(br); |
172 | 0 | } |
173 | 0 | { |
174 | 0 | const int pos = br->bits; |
175 | 0 | const range_t split = (range * prob) >> 8; |
176 | 0 | const range_t value = (range_t)(br->value >> pos); |
177 | 0 | int bit; // Don't use 'const int bit = (value > split);", it's slower. |
178 | 0 | if (value > split) { |
179 | 0 | range -= split + 1; |
180 | 0 | br->value -= (bit_t)(split + 1) << pos; |
181 | 0 | bit = 1; |
182 | 0 | } else { |
183 | 0 | range = split; |
184 | 0 | bit = 0; |
185 | 0 | } |
186 | 0 | if (range <= (range_t)0x7e) { |
187 | 0 | const int shift = kVP8Log2Range[range]; |
188 | 0 | range = kVP8NewRange[range]; |
189 | 0 | br->bits -= shift; |
190 | 0 | } |
191 | 0 | br->range = range; |
192 | 0 | BT_TRACK(br); |
193 | 0 | return bit; |
194 | 0 | } |
195 | 0 | } Unexecuted instantiation: vp8_dec.c:VP8GetBitAlt Unexecuted instantiation: tree_dec.c:VP8GetBitAlt Unexecuted instantiation: bit_reader_utils.c:VP8GetBitAlt |
196 | | |
197 | | #ifdef __cplusplus |
198 | | } // extern "C" |
199 | | #endif |
200 | | |
201 | | #endif // WEBP_UTILS_BIT_READER_INL_UTILS_H_ |