/src/rust-brotli/src/enc/literal_cost.rs
Line | Count | Source |
1 | | use core::cmp::min; |
2 | | |
3 | | use super::util::{floatX, FastLog2f64}; |
4 | | use crate::enc::utf8_util::is_mostly_utf8; |
5 | | |
6 | | static kMinUTF8Ratio: floatX = 0.75; |
7 | | |
8 | 0 | fn UTF8Position(last: usize, c: usize, clamp: usize) -> usize { |
9 | 0 | if c < 128usize { |
10 | 0 | 0usize |
11 | 0 | } else if c >= 192usize { |
12 | 0 | min(1usize, clamp) |
13 | 0 | } else if last < 0xe0usize { |
14 | 0 | 0usize |
15 | | } else { |
16 | 0 | min(2usize, clamp) |
17 | | } |
18 | 0 | } |
19 | | |
20 | 0 | fn DecideMultiByteStatsLevel(pos: usize, len: usize, mask: usize, data: &[u8]) -> usize { |
21 | 0 | let mut counts = [0usize; 3]; |
22 | 0 | let mut max_utf8: usize = 1; |
23 | 0 | let mut last_c: usize = 0usize; |
24 | | let mut i: usize; |
25 | 0 | i = 0usize; |
26 | 0 | while i < len { |
27 | 0 | { |
28 | 0 | let c: usize = data[(pos.wrapping_add(i) & mask)] as usize; |
29 | 0 | { |
30 | 0 | let _rhs = 1; |
31 | 0 | let _lhs = &mut counts[UTF8Position(last_c, c, 2usize)]; |
32 | 0 | *_lhs = (*_lhs).wrapping_add(_rhs as usize); |
33 | 0 | } |
34 | 0 | last_c = c; |
35 | 0 | } |
36 | 0 | i = i.wrapping_add(1); |
37 | 0 | } |
38 | 0 | if counts[2] < 500usize { |
39 | 0 | max_utf8 = 1; |
40 | 0 | } |
41 | 0 | if counts[1].wrapping_add(counts[2]) < 25usize { |
42 | 0 | max_utf8 = 0usize; |
43 | 0 | } |
44 | 0 | max_utf8 |
45 | 0 | } |
46 | | |
47 | 0 | fn EstimateBitCostsForLiteralsUTF8( |
48 | 0 | pos: usize, |
49 | 0 | len: usize, |
50 | 0 | mask: usize, |
51 | 0 | data: &[u8], |
52 | 0 | cost: &mut [floatX], |
53 | 0 | ) { |
54 | 0 | let max_utf8: usize = DecideMultiByteStatsLevel(pos, len, mask, data); |
55 | 0 | let mut histogram = [[0usize; 256]; 3]; |
56 | 0 | let window_half: usize = 495usize; |
57 | 0 | let in_window: usize = min(window_half, len); |
58 | 0 | let mut in_window_utf8 = [0usize; 3]; |
59 | | let mut i: usize; |
60 | | { |
61 | 0 | let mut last_c: usize = 0usize; |
62 | 0 | let mut utf8_pos: usize = 0usize; |
63 | 0 | i = 0usize; |
64 | 0 | while i < in_window { |
65 | 0 | { |
66 | 0 | let c: usize = data[(pos.wrapping_add(i) & mask)] as usize; |
67 | 0 | { |
68 | 0 | let _rhs = 1; |
69 | 0 | let _lhs = &mut histogram[utf8_pos][c]; |
70 | 0 | *_lhs = (*_lhs).wrapping_add(_rhs as usize); |
71 | 0 | } |
72 | 0 | { |
73 | 0 | let _rhs = 1; |
74 | 0 | let _lhs = &mut in_window_utf8[utf8_pos]; |
75 | 0 | *_lhs = (*_lhs).wrapping_add(_rhs as usize); |
76 | 0 | } |
77 | 0 | utf8_pos = UTF8Position(last_c, c, max_utf8); |
78 | 0 | last_c = c; |
79 | 0 | } |
80 | 0 | i = i.wrapping_add(1); |
81 | 0 | } |
82 | | } |
83 | 0 | i = 0usize; |
84 | 0 | while i < len { |
85 | | { |
86 | 0 | if i >= window_half { |
87 | 0 | let c: usize = (if i < window_half.wrapping_add(1) { |
88 | 0 | 0i32 |
89 | | } else { |
90 | 0 | data[(pos |
91 | 0 | .wrapping_add(i) |
92 | 0 | .wrapping_sub(window_half) |
93 | 0 | .wrapping_sub(1) |
94 | 0 | & mask)] as i32 |
95 | | }) as usize; |
96 | 0 | let last_c: usize = (if i < window_half.wrapping_add(2) { |
97 | 0 | 0i32 |
98 | | } else { |
99 | 0 | data[(pos |
100 | 0 | .wrapping_add(i) |
101 | 0 | .wrapping_sub(window_half) |
102 | 0 | .wrapping_sub(2) |
103 | 0 | & mask)] as i32 |
104 | | }) as usize; |
105 | 0 | let utf8_pos2: usize = UTF8Position(last_c, c, max_utf8); |
106 | 0 | { |
107 | 0 | let _rhs = 1; |
108 | 0 | let _lhs = &mut histogram[utf8_pos2] |
109 | 0 | [data[(pos.wrapping_add(i).wrapping_sub(window_half) & mask)] as usize]; |
110 | 0 | *_lhs = (*_lhs).wrapping_sub(_rhs as usize); |
111 | 0 | } |
112 | 0 | { |
113 | 0 | let _rhs = 1; |
114 | 0 | let _lhs = &mut in_window_utf8[utf8_pos2]; |
115 | 0 | *_lhs = (*_lhs).wrapping_sub(_rhs as usize); |
116 | 0 | } |
117 | 0 | } |
118 | 0 | if i.wrapping_add(window_half) < len { |
119 | 0 | let c: usize = data[(pos |
120 | 0 | .wrapping_add(i) |
121 | 0 | .wrapping_add(window_half) |
122 | 0 | .wrapping_sub(1) |
123 | 0 | & mask)] as usize; |
124 | 0 | let last_c: usize = data[(pos |
125 | 0 | .wrapping_add(i) |
126 | 0 | .wrapping_add(window_half) |
127 | 0 | .wrapping_sub(2) |
128 | 0 | & mask)] as usize; |
129 | 0 | let utf8_pos2: usize = UTF8Position(last_c, c, max_utf8); |
130 | 0 | { |
131 | 0 | let _rhs = 1; |
132 | 0 | let _lhs = &mut histogram[utf8_pos2] |
133 | 0 | [data[(pos.wrapping_add(i).wrapping_add(window_half) & mask)] as usize]; |
134 | 0 | *_lhs = (*_lhs).wrapping_add(_rhs as usize); |
135 | 0 | } |
136 | 0 | { |
137 | 0 | let _rhs = 1; |
138 | 0 | let _lhs = &mut in_window_utf8[utf8_pos2]; |
139 | 0 | *_lhs = (*_lhs).wrapping_add(_rhs as usize); |
140 | 0 | } |
141 | 0 | } |
142 | | { |
143 | 0 | let c: usize = (if i < 1 { |
144 | 0 | 0i32 |
145 | | } else { |
146 | 0 | data[(pos.wrapping_add(i).wrapping_sub(1) & mask)] as i32 |
147 | | }) as usize; |
148 | 0 | let last_c: usize = (if i < 2usize { |
149 | 0 | 0i32 |
150 | | } else { |
151 | 0 | data[(pos.wrapping_add(i).wrapping_sub(2) & mask)] as i32 |
152 | | }) as usize; |
153 | 0 | let utf8_pos: usize = UTF8Position(last_c, c, max_utf8); |
154 | 0 | let masked_pos: usize = pos.wrapping_add(i) & mask; |
155 | 0 | let mut histo: usize = histogram[utf8_pos][data[masked_pos] as usize]; |
156 | | //precision is vital here: lets keep double precision |
157 | | let mut lit_cost: f64; |
158 | 0 | if histo == 0usize { |
159 | 0 | histo = 1; |
160 | 0 | } |
161 | 0 | lit_cost = FastLog2f64(in_window_utf8[utf8_pos] as u64) as f64 |
162 | 0 | - FastLog2f64(histo as u64) as f64; |
163 | 0 | lit_cost += 0.02905; |
164 | 0 | if lit_cost < 1.0 { |
165 | 0 | lit_cost *= 0.5; |
166 | 0 | lit_cost += 0.5; |
167 | 0 | } |
168 | 0 | if i < 2000usize { |
169 | 0 | lit_cost += (0.7 - (2000usize).wrapping_sub(i) as (f64) / 2000.0 * 0.35); |
170 | 0 | } |
171 | 0 | cost[i] = lit_cost as floatX; |
172 | | } |
173 | | } |
174 | 0 | i = i.wrapping_add(1); |
175 | | } |
176 | 0 | } |
177 | | |
178 | 0 | pub fn BrotliEstimateBitCostsForLiterals( |
179 | 0 | pos: usize, |
180 | 0 | len: usize, |
181 | 0 | mask: usize, |
182 | 0 | data: &[u8], |
183 | 0 | cost: &mut [floatX], |
184 | 0 | ) { |
185 | 0 | if is_mostly_utf8(data, pos, mask, len, kMinUTF8Ratio) { |
186 | 0 | EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost); |
187 | 0 | } else { |
188 | 0 | let mut histogram: [usize; 256] = [0; 256]; |
189 | | |
190 | 0 | let window_half: usize = 2000usize; |
191 | 0 | let mut in_window: usize = min(window_half, len); |
192 | | let mut i: usize; |
193 | 0 | for i in 0usize..in_window { |
194 | 0 | let _rhs = 1; |
195 | 0 | let _lhs = &mut histogram[data[(pos.wrapping_add(i) & mask)] as usize]; |
196 | 0 | *_lhs = (*_lhs).wrapping_add(_rhs as usize); |
197 | 0 | } |
198 | 0 | i = 0usize; |
199 | 0 | while i < len { |
200 | | { |
201 | | let mut histo: usize; |
202 | 0 | if i >= window_half { |
203 | 0 | { |
204 | 0 | let _rhs = 1; |
205 | 0 | let _lhs = &mut histogram |
206 | 0 | [data[(pos.wrapping_add(i).wrapping_sub(window_half) & mask)] as usize]; |
207 | 0 | *_lhs = (*_lhs).wrapping_sub(_rhs as usize); |
208 | 0 | } |
209 | 0 | in_window = in_window.wrapping_sub(1); |
210 | 0 | } |
211 | 0 | if i.wrapping_add(window_half) < len { |
212 | 0 | { |
213 | 0 | let _rhs = 1; |
214 | 0 | let _lhs = &mut histogram |
215 | 0 | [data[(pos.wrapping_add(i).wrapping_add(window_half) & mask)] as usize]; |
216 | 0 | *_lhs = (*_lhs).wrapping_add(_rhs as usize); |
217 | 0 | } |
218 | 0 | in_window = in_window.wrapping_add(1); |
219 | 0 | } |
220 | 0 | histo = histogram[data[(pos.wrapping_add(i) & mask)] as usize]; |
221 | 0 | if histo == 0usize { |
222 | 0 | histo = 1; |
223 | 0 | } |
224 | | { |
225 | | //precision is vital here: lets keep double precision |
226 | 0 | let mut lit_cost: f64 = |
227 | 0 | FastLog2f64(in_window as u64) as f64 - FastLog2f64(histo as u64) as f64; |
228 | 0 | lit_cost += 0.029; |
229 | 0 | if lit_cost < 1.0 { |
230 | 0 | lit_cost *= 0.5; |
231 | 0 | lit_cost += 0.5; |
232 | 0 | } |
233 | 0 | cost[i] = lit_cost as floatX; |
234 | | } |
235 | | } |
236 | 0 | i = i.wrapping_add(1); |
237 | | } |
238 | | } |
239 | 0 | } |