/src/xz/src/liblzma/lzma/lzma_encoder.c
Line | Count | Source |
1 | | // SPDX-License-Identifier: 0BSD |
2 | | |
3 | | /////////////////////////////////////////////////////////////////////////////// |
4 | | // |
5 | | /// \file lzma_encoder.c |
6 | | /// \brief LZMA encoder |
7 | | /// |
8 | | // Authors: Igor Pavlov |
9 | | // Lasse Collin |
10 | | // |
11 | | /////////////////////////////////////////////////////////////////////////////// |
12 | | |
13 | | #include "lzma2_encoder.h" |
14 | | #include "lzma_encoder_private.h" |
15 | | #include "fastpos.h" |
16 | | |
17 | | |
18 | | ///////////// |
19 | | // Literal // |
20 | | ///////////// |
21 | | |
22 | | static inline void |
23 | | literal_matched(lzma_range_encoder *rc, probability *subcoder, |
24 | | uint32_t match_byte, uint32_t symbol) |
25 | 0 | { |
26 | 0 | uint32_t offset = 0x100; |
27 | 0 | symbol += UINT32_C(1) << 8; |
28 | |
|
29 | 0 | do { |
30 | 0 | match_byte <<= 1; |
31 | 0 | const uint32_t match_bit = match_byte & offset; |
32 | 0 | const uint32_t subcoder_index |
33 | 0 | = offset + match_bit + (symbol >> 8); |
34 | 0 | const uint32_t bit = (symbol >> 7) & 1; |
35 | 0 | rc_bit(rc, &subcoder[subcoder_index], bit); |
36 | |
|
37 | 0 | symbol <<= 1; |
38 | 0 | offset &= ~(match_byte ^ symbol); |
39 | |
|
40 | 0 | } while (symbol < (UINT32_C(1) << 16)); |
41 | 0 | } |
42 | | |
43 | | |
44 | | static inline void |
45 | | literal(lzma_lzma1_encoder *coder, lzma_mf *mf, uint32_t position) |
46 | 0 | { |
47 | | // Locate the literal byte to be encoded and the subcoder. |
48 | 0 | const uint8_t cur_byte = mf->buffer[ |
49 | 0 | mf->read_pos - mf->read_ahead]; |
50 | 0 | probability *subcoder = literal_subcoder(coder->literal, |
51 | 0 | coder->literal_context_bits, coder->literal_mask, |
52 | 0 | position, mf->buffer[mf->read_pos - mf->read_ahead - 1]); |
53 | |
|
54 | 0 | if (is_literal_state(coder->state)) { |
55 | | // Previous LZMA-symbol was a literal. Encode a normal |
56 | | // literal without a match byte. |
57 | 0 | update_literal_normal(coder->state); |
58 | 0 | rc_bittree(&coder->rc, subcoder, 8, cur_byte); |
59 | 0 | } else { |
60 | | // Previous LZMA-symbol was a match. Use the last byte of |
61 | | // the match as a "match byte". That is, compare the bits |
62 | | // of the current literal and the match byte. |
63 | 0 | update_literal_matched(coder->state); |
64 | 0 | const uint8_t match_byte = mf->buffer[ |
65 | 0 | mf->read_pos - coder->reps[0] - 1 |
66 | 0 | - mf->read_ahead]; |
67 | 0 | literal_matched(&coder->rc, subcoder, match_byte, cur_byte); |
68 | 0 | } |
69 | 0 | } |
70 | | |
71 | | |
72 | | ////////////////// |
73 | | // Match length // |
74 | | ////////////////// |
75 | | |
76 | | static void |
77 | | length_update_prices(lzma_length_encoder *lc, const uint32_t pos_state) |
78 | 0 | { |
79 | 0 | const uint32_t table_size = lc->table_size; |
80 | 0 | lc->counters[pos_state] = table_size; |
81 | |
|
82 | 0 | const uint32_t a0 = rc_bit_0_price(lc->choice); |
83 | 0 | const uint32_t a1 = rc_bit_1_price(lc->choice); |
84 | 0 | const uint32_t b0 = a1 + rc_bit_0_price(lc->choice2); |
85 | 0 | const uint32_t b1 = a1 + rc_bit_1_price(lc->choice2); |
86 | 0 | uint32_t *const prices = lc->prices[pos_state]; |
87 | |
|
88 | 0 | uint32_t i; |
89 | 0 | for (i = 0; i < table_size && i < LEN_LOW_SYMBOLS; ++i) |
90 | 0 | prices[i] = a0 + rc_bittree_price(lc->low[pos_state], |
91 | 0 | LEN_LOW_BITS, i); |
92 | |
|
93 | 0 | for (; i < table_size && i < LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; ++i) |
94 | 0 | prices[i] = b0 + rc_bittree_price(lc->mid[pos_state], |
95 | 0 | LEN_MID_BITS, i - LEN_LOW_SYMBOLS); |
96 | |
|
97 | 0 | for (; i < table_size; ++i) |
98 | 0 | prices[i] = b1 + rc_bittree_price(lc->high, LEN_HIGH_BITS, |
99 | 0 | i - LEN_LOW_SYMBOLS - LEN_MID_SYMBOLS); |
100 | |
|
101 | 0 | return; |
102 | 0 | } |
103 | | |
104 | | |
105 | | static inline void |
106 | | length(lzma_range_encoder *rc, lzma_length_encoder *lc, |
107 | | const uint32_t pos_state, uint32_t len, const bool fast_mode) |
108 | 0 | { |
109 | 0 | assert(len <= MATCH_LEN_MAX); |
110 | 0 | len -= MATCH_LEN_MIN; |
111 | |
|
112 | 0 | if (len < LEN_LOW_SYMBOLS) { |
113 | 0 | rc_bit(rc, &lc->choice, 0); |
114 | 0 | rc_bittree(rc, lc->low[pos_state], LEN_LOW_BITS, len); |
115 | 0 | } else { |
116 | 0 | rc_bit(rc, &lc->choice, 1); |
117 | 0 | len -= LEN_LOW_SYMBOLS; |
118 | |
|
119 | 0 | if (len < LEN_MID_SYMBOLS) { |
120 | 0 | rc_bit(rc, &lc->choice2, 0); |
121 | 0 | rc_bittree(rc, lc->mid[pos_state], LEN_MID_BITS, len); |
122 | 0 | } else { |
123 | 0 | rc_bit(rc, &lc->choice2, 1); |
124 | 0 | len -= LEN_MID_SYMBOLS; |
125 | 0 | rc_bittree(rc, lc->high, LEN_HIGH_BITS, len); |
126 | 0 | } |
127 | 0 | } |
128 | | |
129 | | // Only getoptimum uses the prices so don't update the table when |
130 | | // in fast mode. |
131 | 0 | if (!fast_mode) |
132 | 0 | if (--lc->counters[pos_state] == 0) |
133 | 0 | length_update_prices(lc, pos_state); |
134 | 0 | } |
135 | | |
136 | | |
137 | | /////////// |
138 | | // Match // |
139 | | /////////// |
140 | | |
141 | | static inline void |
142 | | match(lzma_lzma1_encoder *coder, const uint32_t pos_state, |
143 | | const uint32_t distance, const uint32_t len) |
144 | 0 | { |
145 | 0 | update_match(coder->state); |
146 | |
|
147 | 0 | length(&coder->rc, &coder->match_len_encoder, pos_state, len, |
148 | 0 | coder->fast_mode); |
149 | |
|
150 | 0 | const uint32_t dist_slot = get_dist_slot(distance); |
151 | 0 | const uint32_t dist_state = get_dist_state(len); |
152 | 0 | rc_bittree(&coder->rc, coder->dist_slot[dist_state], |
153 | 0 | DIST_SLOT_BITS, dist_slot); |
154 | |
|
155 | 0 | if (dist_slot >= DIST_MODEL_START) { |
156 | 0 | const uint32_t footer_bits = (dist_slot >> 1) - 1; |
157 | 0 | const uint32_t base = (2 | (dist_slot & 1)) << footer_bits; |
158 | 0 | const uint32_t dist_reduced = distance - base; |
159 | |
|
160 | 0 | if (dist_slot < DIST_MODEL_END) { |
161 | | // Careful here: base - dist_slot - 1 can be -1, but |
162 | | // rc_bittree_reverse starts at probs[1], not probs[0]. |
163 | 0 | rc_bittree_reverse(&coder->rc, |
164 | 0 | coder->dist_special + base - dist_slot - 1, |
165 | 0 | footer_bits, dist_reduced); |
166 | 0 | } else { |
167 | 0 | rc_direct(&coder->rc, dist_reduced >> ALIGN_BITS, |
168 | 0 | footer_bits - ALIGN_BITS); |
169 | 0 | rc_bittree_reverse( |
170 | 0 | &coder->rc, coder->dist_align, |
171 | 0 | ALIGN_BITS, dist_reduced & ALIGN_MASK); |
172 | 0 | ++coder->align_price_count; |
173 | 0 | } |
174 | 0 | } |
175 | |
|
176 | 0 | coder->reps[3] = coder->reps[2]; |
177 | 0 | coder->reps[2] = coder->reps[1]; |
178 | 0 | coder->reps[1] = coder->reps[0]; |
179 | 0 | coder->reps[0] = distance; |
180 | 0 | ++coder->match_price_count; |
181 | 0 | } |
182 | | |
183 | | |
184 | | //////////////////// |
185 | | // Repeated match // |
186 | | //////////////////// |
187 | | |
188 | | static inline void |
189 | | rep_match(lzma_lzma1_encoder *coder, const uint32_t pos_state, |
190 | | const uint32_t rep, const uint32_t len) |
191 | 0 | { |
192 | 0 | if (rep == 0) { |
193 | 0 | rc_bit(&coder->rc, &coder->is_rep0[coder->state], 0); |
194 | 0 | rc_bit(&coder->rc, |
195 | 0 | &coder->is_rep0_long[coder->state][pos_state], |
196 | 0 | len != 1); |
197 | 0 | } else { |
198 | 0 | const uint32_t distance = coder->reps[rep]; |
199 | 0 | rc_bit(&coder->rc, &coder->is_rep0[coder->state], 1); |
200 | |
|
201 | 0 | if (rep == 1) { |
202 | 0 | rc_bit(&coder->rc, &coder->is_rep1[coder->state], 0); |
203 | 0 | } else { |
204 | 0 | rc_bit(&coder->rc, &coder->is_rep1[coder->state], 1); |
205 | 0 | rc_bit(&coder->rc, &coder->is_rep2[coder->state], |
206 | 0 | rep - 2); |
207 | |
|
208 | 0 | if (rep == 3) |
209 | 0 | coder->reps[3] = coder->reps[2]; |
210 | |
|
211 | 0 | coder->reps[2] = coder->reps[1]; |
212 | 0 | } |
213 | |
|
214 | 0 | coder->reps[1] = coder->reps[0]; |
215 | 0 | coder->reps[0] = distance; |
216 | 0 | } |
217 | |
|
218 | 0 | if (len == 1) { |
219 | 0 | update_short_rep(coder->state); |
220 | 0 | } else { |
221 | 0 | length(&coder->rc, &coder->rep_len_encoder, pos_state, len, |
222 | 0 | coder->fast_mode); |
223 | 0 | update_long_rep(coder->state); |
224 | 0 | } |
225 | 0 | } |
226 | | |
227 | | |
228 | | ////////// |
229 | | // Main // |
230 | | ////////// |
231 | | |
232 | | static void |
233 | | encode_symbol(lzma_lzma1_encoder *coder, lzma_mf *mf, |
234 | | uint32_t back, uint32_t len, uint32_t position) |
235 | 0 | { |
236 | 0 | const uint32_t pos_state = position & coder->pos_mask; |
237 | |
|
238 | 0 | if (back == UINT32_MAX) { |
239 | | // Literal i.e. eight-bit byte |
240 | 0 | assert(len == 1); |
241 | 0 | rc_bit(&coder->rc, |
242 | 0 | &coder->is_match[coder->state][pos_state], 0); |
243 | 0 | literal(coder, mf, position); |
244 | 0 | } else { |
245 | | // Some type of match |
246 | 0 | rc_bit(&coder->rc, |
247 | 0 | &coder->is_match[coder->state][pos_state], 1); |
248 | |
|
249 | 0 | if (back < REPS) { |
250 | | // It's a repeated match i.e. the same distance |
251 | | // has been used earlier. |
252 | 0 | rc_bit(&coder->rc, &coder->is_rep[coder->state], 1); |
253 | 0 | rep_match(coder, pos_state, back, len); |
254 | 0 | } else { |
255 | | // Normal match |
256 | 0 | rc_bit(&coder->rc, &coder->is_rep[coder->state], 0); |
257 | 0 | match(coder, pos_state, back - REPS, len); |
258 | 0 | } |
259 | 0 | } |
260 | |
|
261 | 0 | assert(mf->read_ahead >= len); |
262 | 0 | mf->read_ahead -= len; |
263 | 0 | } |
264 | | |
265 | | |
266 | | static bool |
267 | | encode_init(lzma_lzma1_encoder *coder, lzma_mf *mf) |
268 | 0 | { |
269 | 0 | assert(mf_position(mf) == 0); |
270 | 0 | assert(coder->uncomp_size == 0); |
271 | |
|
272 | 0 | if (mf->read_pos == mf->read_limit) { |
273 | 0 | if (mf->action == LZMA_RUN) |
274 | 0 | return false; // We cannot do anything. |
275 | | |
276 | | // We are finishing (we cannot get here when flushing). |
277 | 0 | assert(mf->write_pos == mf->read_pos); |
278 | 0 | assert(mf->action == LZMA_FINISH); |
279 | 0 | } else { |
280 | | // Do the actual initialization. The first LZMA symbol must |
281 | | // always be a literal. |
282 | 0 | mf_skip(mf, 1); |
283 | 0 | mf->read_ahead = 0; |
284 | 0 | rc_bit(&coder->rc, &coder->is_match[0][0], 0); |
285 | 0 | rc_bittree(&coder->rc, coder->literal + 0, 8, mf->buffer[0]); |
286 | 0 | ++coder->uncomp_size; |
287 | 0 | } |
288 | | |
289 | | // Initialization is done (except if empty file). |
290 | 0 | coder->is_initialized = true; |
291 | |
|
292 | 0 | return true; |
293 | 0 | } |
294 | | |
295 | | |
296 | | static void |
297 | | encode_eopm(lzma_lzma1_encoder *coder, uint32_t position) |
298 | 0 | { |
299 | 0 | const uint32_t pos_state = position & coder->pos_mask; |
300 | 0 | rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1); |
301 | 0 | rc_bit(&coder->rc, &coder->is_rep[coder->state], 0); |
302 | 0 | match(coder, pos_state, UINT32_MAX, MATCH_LEN_MIN); |
303 | 0 | } |
304 | | |
305 | | |
306 | | /// Number of bytes that a single encoding loop in lzma_lzma_encode() can |
307 | | /// consume from the dictionary. This limit comes from lzma_lzma_optimum() |
308 | | /// and may need to be updated if that function is significantly modified. |
309 | 0 | #define LOOP_INPUT_MAX (OPTS + 1) |
310 | | |
311 | | |
312 | | extern lzma_ret |
313 | | lzma_lzma_encode(lzma_lzma1_encoder *restrict coder, lzma_mf *restrict mf, |
314 | | uint8_t *restrict out, size_t *restrict out_pos, |
315 | | size_t out_size, uint32_t limit) |
316 | 0 | { |
317 | | // Initialize the stream if no data has been encoded yet. |
318 | 0 | if (!coder->is_initialized && !encode_init(coder, mf)) |
319 | 0 | return LZMA_OK; |
320 | | |
321 | | // Encode pending output bytes from the range encoder. |
322 | | // At the start of the stream, encode_init() encodes one literal. |
323 | | // Later there can be pending output only with LZMA1 because LZMA2 |
324 | | // ensures that there is always enough output space. Thus when using |
325 | | // LZMA2, rc_encode() calls in this function will always return false. |
326 | 0 | if (rc_encode(&coder->rc, out, out_pos, out_size)) { |
327 | | // We don't get here with LZMA2. |
328 | 0 | assert(limit == UINT32_MAX); |
329 | 0 | return LZMA_OK; |
330 | 0 | } |
331 | | |
332 | | // If the range encoder was flushed in an earlier call to this |
333 | | // function but there wasn't enough output buffer space, those |
334 | | // bytes would have now been encoded by the above rc_encode() call |
335 | | // and the stream has now been finished. This can only happen with |
336 | | // LZMA1 as LZMA2 always provides enough output buffer space. |
337 | 0 | if (coder->is_flushed) { |
338 | 0 | assert(limit == UINT32_MAX); |
339 | 0 | return LZMA_STREAM_END; |
340 | 0 | } |
341 | | |
342 | 0 | while (true) { |
343 | | // With LZMA2 we need to take care that compressed size of |
344 | | // a chunk doesn't get too big. |
345 | | // FIXME? Check if this could be improved. |
346 | 0 | if (limit != UINT32_MAX |
347 | 0 | && (mf->read_pos - mf->read_ahead >= limit |
348 | 0 | || *out_pos + rc_pending(&coder->rc) |
349 | 0 | >= LZMA2_CHUNK_MAX |
350 | 0 | - LOOP_INPUT_MAX)) |
351 | 0 | break; |
352 | | |
353 | | // Check that there is some input to process. |
354 | 0 | if (mf->read_pos >= mf->read_limit) { |
355 | 0 | if (mf->action == LZMA_RUN) |
356 | 0 | return LZMA_OK; |
357 | | |
358 | 0 | if (mf->read_ahead == 0) |
359 | 0 | break; |
360 | 0 | } |
361 | | |
362 | | // Get optimal match (repeat position and length). |
363 | | // Value ranges for pos: |
364 | | // - [0, REPS): repeated match |
365 | | // - [REPS, UINT32_MAX): |
366 | | // match at (pos - REPS) |
367 | | // - UINT32_MAX: not a match but a literal |
368 | | // Value ranges for len: |
369 | | // - [MATCH_LEN_MIN, MATCH_LEN_MAX] |
370 | 0 | uint32_t len; |
371 | 0 | uint32_t back; |
372 | |
|
373 | 0 | if (coder->fast_mode) |
374 | 0 | lzma_lzma_optimum_fast(coder, mf, &back, &len); |
375 | 0 | else |
376 | 0 | lzma_lzma_optimum_normal(coder, mf, &back, &len, |
377 | 0 | (uint32_t)(coder->uncomp_size)); |
378 | |
|
379 | 0 | encode_symbol(coder, mf, back, len, |
380 | 0 | (uint32_t)(coder->uncomp_size)); |
381 | | |
382 | | // If output size limiting is active (out_limit != 0), check |
383 | | // if encoding this LZMA symbol would make the output size |
384 | | // exceed the specified limit. |
385 | 0 | if (coder->out_limit != 0 && rc_encode_dummy( |
386 | 0 | &coder->rc, coder->out_limit)) { |
387 | | // The most recent LZMA symbol would make the output |
388 | | // too big. Throw it away. |
389 | 0 | rc_forget(&coder->rc); |
390 | | |
391 | | // FIXME: Tell the LZ layer to not read more input as |
392 | | // it would be waste of time. This doesn't matter if |
393 | | // output-size-limited encoding is done with a single |
394 | | // call though. |
395 | |
|
396 | 0 | break; |
397 | 0 | } |
398 | | |
399 | | // This symbol will be encoded so update the uncompressed size. |
400 | 0 | coder->uncomp_size += len; |
401 | | |
402 | | // Encode the LZMA symbol. |
403 | 0 | if (rc_encode(&coder->rc, out, out_pos, out_size)) { |
404 | | // Once again, this can only happen with LZMA1. |
405 | 0 | assert(limit == UINT32_MAX); |
406 | 0 | return LZMA_OK; |
407 | 0 | } |
408 | 0 | } |
409 | | |
410 | | // Make the uncompressed size available to the application. |
411 | 0 | if (coder->uncomp_size_ptr != NULL) |
412 | 0 | *coder->uncomp_size_ptr = coder->uncomp_size; |
413 | | |
414 | | // LZMA2 doesn't use EOPM at LZMA level. |
415 | | // |
416 | | // Plain LZMA streams without EOPM aren't supported except when |
417 | | // output size limiting is enabled. |
418 | 0 | if (coder->use_eopm) |
419 | 0 | encode_eopm(coder, (uint32_t)(coder->uncomp_size)); |
420 | | |
421 | | // Flush the remaining bytes from the range encoder. |
422 | 0 | rc_flush(&coder->rc); |
423 | | |
424 | | // Copy the remaining bytes to the output buffer. If there |
425 | | // isn't enough output space, we will copy out the remaining |
426 | | // bytes on the next call to this function. |
427 | 0 | if (rc_encode(&coder->rc, out, out_pos, out_size)) { |
428 | | // This cannot happen with LZMA2. |
429 | 0 | assert(limit == UINT32_MAX); |
430 | |
|
431 | 0 | coder->is_flushed = true; |
432 | 0 | return LZMA_OK; |
433 | 0 | } |
434 | | |
435 | 0 | return LZMA_STREAM_END; |
436 | 0 | } |
437 | | |
438 | | |
439 | | static lzma_ret |
440 | | lzma_encode(void *coder, lzma_mf *restrict mf, |
441 | | uint8_t *restrict out, size_t *restrict out_pos, |
442 | | size_t out_size) |
443 | 0 | { |
444 | | // Plain LZMA has no support for sync-flushing. |
445 | 0 | if (unlikely(mf->action == LZMA_SYNC_FLUSH)) |
446 | 0 | return LZMA_OPTIONS_ERROR; |
447 | | |
448 | 0 | return lzma_lzma_encode(coder, mf, out, out_pos, out_size, UINT32_MAX); |
449 | 0 | } |
450 | | |
451 | | |
452 | | static lzma_ret |
453 | | lzma_lzma_set_out_limit( |
454 | | void *coder_ptr, uint64_t *uncomp_size, uint64_t out_limit) |
455 | 0 | { |
456 | | // Minimum output size is 5 bytes but that cannot hold any output |
457 | | // so we use 6 bytes. |
458 | 0 | if (out_limit < 6) |
459 | 0 | return LZMA_BUF_ERROR; |
460 | | |
461 | 0 | lzma_lzma1_encoder *coder = coder_ptr; |
462 | 0 | coder->out_limit = out_limit; |
463 | 0 | coder->uncomp_size_ptr = uncomp_size; |
464 | 0 | coder->use_eopm = false; |
465 | 0 | return LZMA_OK; |
466 | 0 | } |
467 | | |
468 | | |
469 | | //////////////////// |
470 | | // Initialization // |
471 | | //////////////////// |
472 | | |
473 | | static bool |
474 | | is_options_valid(const lzma_options_lzma *options) |
475 | 0 | { |
476 | | // Validate some of the options. LZ encoder validates nice_len too |
477 | | // but we need a valid value here earlier. |
478 | 0 | return is_lclppb_valid(options) |
479 | 0 | && options->nice_len >= MATCH_LEN_MIN |
480 | 0 | && options->nice_len <= MATCH_LEN_MAX |
481 | 0 | && (options->mode == LZMA_MODE_FAST |
482 | 0 | || options->mode == LZMA_MODE_NORMAL); |
483 | 0 | } |
484 | | |
485 | | |
486 | | static void |
487 | | set_lz_options(lzma_lz_options *lz_options, const lzma_options_lzma *options) |
488 | 0 | { |
489 | | // LZ encoder initialization does the validation for these so we |
490 | | // don't need to validate here. |
491 | 0 | lz_options->before_size = OPTS; |
492 | 0 | lz_options->dict_size = options->dict_size; |
493 | 0 | lz_options->after_size = LOOP_INPUT_MAX; |
494 | 0 | lz_options->match_len_max = MATCH_LEN_MAX; |
495 | 0 | lz_options->nice_len = my_max(mf_get_hash_bytes(options->mf), |
496 | 0 | options->nice_len); |
497 | 0 | lz_options->match_finder = options->mf; |
498 | 0 | lz_options->depth = options->depth; |
499 | 0 | lz_options->preset_dict = options->preset_dict; |
500 | 0 | lz_options->preset_dict_size = options->preset_dict_size; |
501 | 0 | return; |
502 | 0 | } |
503 | | |
504 | | |
505 | | static void |
506 | | length_encoder_reset(lzma_length_encoder *lencoder, |
507 | | const uint32_t num_pos_states, const bool fast_mode) |
508 | 0 | { |
509 | 0 | bit_reset(lencoder->choice); |
510 | 0 | bit_reset(lencoder->choice2); |
511 | |
|
512 | 0 | for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { |
513 | 0 | bittree_reset(lencoder->low[pos_state], LEN_LOW_BITS); |
514 | 0 | bittree_reset(lencoder->mid[pos_state], LEN_MID_BITS); |
515 | 0 | } |
516 | |
|
517 | 0 | bittree_reset(lencoder->high, LEN_HIGH_BITS); |
518 | |
|
519 | 0 | if (!fast_mode) |
520 | 0 | for (uint32_t pos_state = 0; pos_state < num_pos_states; |
521 | 0 | ++pos_state) |
522 | 0 | length_update_prices(lencoder, pos_state); |
523 | |
|
524 | 0 | return; |
525 | 0 | } |
526 | | |
527 | | |
528 | | extern lzma_ret |
529 | | lzma_lzma_encoder_reset(lzma_lzma1_encoder *coder, |
530 | | const lzma_options_lzma *options) |
531 | 0 | { |
532 | 0 | if (!is_options_valid(options)) |
533 | 0 | return LZMA_OPTIONS_ERROR; |
534 | | |
535 | 0 | coder->pos_mask = (1U << options->pb) - 1; |
536 | 0 | coder->literal_context_bits = options->lc; |
537 | 0 | coder->literal_mask = literal_mask_calc(options->lc, options->lp); |
538 | | |
539 | | // Range coder |
540 | 0 | rc_reset(&coder->rc); |
541 | | |
542 | | // State |
543 | 0 | coder->state = STATE_LIT_LIT; |
544 | 0 | for (size_t i = 0; i < REPS; ++i) |
545 | 0 | coder->reps[i] = 0; |
546 | |
|
547 | 0 | literal_init(coder->literal, options->lc, options->lp); |
548 | | |
549 | | // Bit encoders |
550 | 0 | for (size_t i = 0; i < STATES; ++i) { |
551 | 0 | for (size_t j = 0; j <= coder->pos_mask; ++j) { |
552 | 0 | bit_reset(coder->is_match[i][j]); |
553 | 0 | bit_reset(coder->is_rep0_long[i][j]); |
554 | 0 | } |
555 | |
|
556 | 0 | bit_reset(coder->is_rep[i]); |
557 | 0 | bit_reset(coder->is_rep0[i]); |
558 | 0 | bit_reset(coder->is_rep1[i]); |
559 | 0 | bit_reset(coder->is_rep2[i]); |
560 | 0 | } |
561 | |
|
562 | 0 | for (size_t i = 0; i < FULL_DISTANCES - DIST_MODEL_END; ++i) |
563 | 0 | bit_reset(coder->dist_special[i]); |
564 | | |
565 | | // Bit tree encoders |
566 | 0 | for (size_t i = 0; i < DIST_STATES; ++i) |
567 | 0 | bittree_reset(coder->dist_slot[i], DIST_SLOT_BITS); |
568 | |
|
569 | 0 | bittree_reset(coder->dist_align, ALIGN_BITS); |
570 | | |
571 | | // Length encoders |
572 | 0 | length_encoder_reset(&coder->match_len_encoder, |
573 | 0 | 1U << options->pb, coder->fast_mode); |
574 | |
|
575 | 0 | length_encoder_reset(&coder->rep_len_encoder, |
576 | 0 | 1U << options->pb, coder->fast_mode); |
577 | | |
578 | | // Price counts are incremented every time appropriate probabilities |
579 | | // are changed. price counts are set to zero when the price tables |
580 | | // are updated, which is done when the appropriate price counts have |
581 | | // big enough value, and lzma_mf.read_ahead == 0 which happens at |
582 | | // least every OPTS (a few thousand) possible price count increments. |
583 | | // |
584 | | // By resetting price counts to UINT32_MAX / 2, we make sure that the |
585 | | // price tables will be initialized before they will be used (since |
586 | | // the value is definitely big enough), and that it is OK to increment |
587 | | // price counts without risk of integer overflow (since UINT32_MAX / 2 |
588 | | // is small enough). The current code doesn't increment price counts |
589 | | // before initializing price tables, but it maybe done in future if |
590 | | // we add support for saving the state between LZMA2 chunks. |
591 | 0 | coder->match_price_count = UINT32_MAX / 2; |
592 | 0 | coder->align_price_count = UINT32_MAX / 2; |
593 | |
|
594 | 0 | coder->opts_end_index = 0; |
595 | 0 | coder->opts_current_index = 0; |
596 | |
|
597 | 0 | return LZMA_OK; |
598 | 0 | } |
599 | | |
600 | | |
601 | | extern lzma_ret |
602 | | lzma_lzma_encoder_create(void **coder_ptr, const lzma_allocator *allocator, |
603 | | lzma_vli id, const lzma_options_lzma *options, |
604 | | lzma_lz_options *lz_options) |
605 | 0 | { |
606 | 0 | assert(id == LZMA_FILTER_LZMA1 || id == LZMA_FILTER_LZMA1EXT |
607 | 0 | || id == LZMA_FILTER_LZMA2); |
608 | | |
609 | | // Allocate lzma_lzma1_encoder if it wasn't already allocated. |
610 | 0 | if (*coder_ptr == NULL) { |
611 | 0 | *coder_ptr = lzma_alloc(sizeof(lzma_lzma1_encoder), allocator); |
612 | 0 | if (*coder_ptr == NULL) |
613 | 0 | return LZMA_MEM_ERROR; |
614 | 0 | } |
615 | | |
616 | 0 | lzma_lzma1_encoder *coder = *coder_ptr; |
617 | | |
618 | | // Set compression mode. Note that we haven't validated the options |
619 | | // yet. Invalid options will get rejected by lzma_lzma_encoder_reset() |
620 | | // call at the end of this function. |
621 | 0 | switch (options->mode) { |
622 | 0 | case LZMA_MODE_FAST: |
623 | 0 | coder->fast_mode = true; |
624 | 0 | break; |
625 | | |
626 | 0 | case LZMA_MODE_NORMAL: { |
627 | 0 | coder->fast_mode = false; |
628 | | |
629 | | // Set dist_table_size. |
630 | | // Round the dictionary size up to next 2^n. |
631 | | // |
632 | | // Currently the maximum encoder dictionary size |
633 | | // is 1.5 GiB due to lz_encoder.c and here we need |
634 | | // to be below 2 GiB to make the rounded up value |
635 | | // fit in an uint32_t and avoid an infinite while-loop |
636 | | // (and undefined behavior due to a too large shift). |
637 | | // So do the same check as in LZ encoder, |
638 | | // limiting to 1.5 GiB. |
639 | 0 | if (options->dict_size > (UINT32_C(1) << 30) |
640 | 0 | + (UINT32_C(1) << 29)) |
641 | 0 | return LZMA_OPTIONS_ERROR; |
642 | | |
643 | 0 | uint32_t log_size = 0; |
644 | 0 | while ((UINT32_C(1) << log_size) < options->dict_size) |
645 | 0 | ++log_size; |
646 | |
|
647 | 0 | coder->dist_table_size = log_size * 2; |
648 | | |
649 | | // Length encoders' price table size |
650 | 0 | const uint32_t nice_len = my_max( |
651 | 0 | mf_get_hash_bytes(options->mf), |
652 | 0 | options->nice_len); |
653 | |
|
654 | 0 | coder->match_len_encoder.table_size |
655 | 0 | = nice_len + 1 - MATCH_LEN_MIN; |
656 | 0 | coder->rep_len_encoder.table_size |
657 | 0 | = nice_len + 1 - MATCH_LEN_MIN; |
658 | 0 | break; |
659 | 0 | } |
660 | | |
661 | 0 | default: |
662 | 0 | return LZMA_OPTIONS_ERROR; |
663 | 0 | } |
664 | | |
665 | | // We don't need to write the first byte as literal if there is |
666 | | // a non-empty preset dictionary. encode_init() wouldn't even work |
667 | | // if there is a non-empty preset dictionary, because encode_init() |
668 | | // assumes that position is zero and previous byte is also zero. |
669 | 0 | coder->is_initialized = options->preset_dict != NULL |
670 | 0 | && options->preset_dict_size > 0; |
671 | 0 | coder->is_flushed = false; |
672 | 0 | coder->uncomp_size = 0; |
673 | 0 | coder->uncomp_size_ptr = NULL; |
674 | | |
675 | | // Output size limiting is disabled by default. |
676 | 0 | coder->out_limit = 0; |
677 | | |
678 | | // Determine if end marker is wanted: |
679 | | // - It is never used with LZMA2. |
680 | | // - It is always used with LZMA_FILTER_LZMA1 (unless |
681 | | // lzma_lzma_set_out_limit() is called later). |
682 | | // - LZMA_FILTER_LZMA1EXT has a flag for it in the options. |
683 | 0 | coder->use_eopm = (id == LZMA_FILTER_LZMA1); |
684 | 0 | if (id == LZMA_FILTER_LZMA1EXT) { |
685 | | // Check if unsupported flags are present. |
686 | 0 | if (options->ext_flags & ~LZMA_LZMA1EXT_ALLOW_EOPM) |
687 | 0 | return LZMA_OPTIONS_ERROR; |
688 | | |
689 | 0 | coder->use_eopm = (options->ext_flags |
690 | 0 | & LZMA_LZMA1EXT_ALLOW_EOPM) != 0; |
691 | | |
692 | | // TODO? As long as there are no filters that change the size |
693 | | // of the data, it is enough to look at lzma_stream.total_in |
694 | | // after encoding has been finished to know the uncompressed |
695 | | // size of the LZMA1 stream. But in the future there could be |
696 | | // filters that change the size of the data and then total_in |
697 | | // doesn't work as the LZMA1 stream size might be different |
698 | | // due to another filter in the chain. The problem is simple |
699 | | // to solve: Add another flag to ext_flags and then set |
700 | | // coder->uncomp_size_ptr to the address stored in |
701 | | // lzma_options_lzma.reserved_ptr2 (or _ptr1). |
702 | 0 | } |
703 | | |
704 | 0 | set_lz_options(lz_options, options); |
705 | |
|
706 | 0 | return lzma_lzma_encoder_reset(coder, options); |
707 | 0 | } |
708 | | |
709 | | |
710 | | static lzma_ret |
711 | | lzma_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator, |
712 | | lzma_vli id, const void *options, lzma_lz_options *lz_options) |
713 | 0 | { |
714 | 0 | if (options == NULL) |
715 | 0 | return LZMA_PROG_ERROR; |
716 | | |
717 | 0 | lz->code = &lzma_encode; |
718 | 0 | lz->set_out_limit = &lzma_lzma_set_out_limit; |
719 | 0 | return lzma_lzma_encoder_create( |
720 | 0 | &lz->coder, allocator, id, options, lz_options); |
721 | 0 | } |
722 | | |
723 | | |
724 | | extern lzma_ret |
725 | | lzma_lzma_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, |
726 | | const lzma_filter_info *filters) |
727 | 0 | { |
728 | 0 | return lzma_lz_encoder_init( |
729 | 0 | next, allocator, filters, &lzma_encoder_init); |
730 | 0 | } |
731 | | |
732 | | |
733 | | extern uint64_t |
734 | | lzma_lzma_encoder_memusage(const void *options) |
735 | 0 | { |
736 | 0 | if (!is_options_valid(options)) |
737 | 0 | return UINT64_MAX; |
738 | | |
739 | 0 | lzma_lz_options lz_options; |
740 | 0 | set_lz_options(&lz_options, options); |
741 | |
|
742 | 0 | const uint64_t lz_memusage = lzma_lz_encoder_memusage(&lz_options); |
743 | 0 | if (lz_memusage == UINT64_MAX) |
744 | 0 | return UINT64_MAX; |
745 | | |
746 | 0 | return (uint64_t)(sizeof(lzma_lzma1_encoder)) + lz_memusage; |
747 | 0 | } |
748 | | |
749 | | |
750 | | extern bool |
751 | | lzma_lzma_lclppb_encode(const lzma_options_lzma *options, uint8_t *byte) |
752 | 0 | { |
753 | 0 | if (!is_lclppb_valid(options)) |
754 | 0 | return true; |
755 | | |
756 | 0 | *byte = (options->pb * 5 + options->lp) * 9 + options->lc; |
757 | 0 | assert(*byte <= (4 * 5 + 4) * 9 + 8); |
758 | |
|
759 | 0 | return false; |
760 | 0 | } |
761 | | |
762 | | |
763 | | #ifdef HAVE_ENCODER_LZMA1 |
764 | | extern lzma_ret |
765 | | lzma_lzma_props_encode(const void *options, uint8_t *out) |
766 | 0 | { |
767 | 0 | if (options == NULL) |
768 | 0 | return LZMA_PROG_ERROR; |
769 | | |
770 | 0 | const lzma_options_lzma *const opt = options; |
771 | |
|
772 | 0 | if (lzma_lzma_lclppb_encode(opt, out)) |
773 | 0 | return LZMA_PROG_ERROR; |
774 | | |
775 | 0 | write32le(out + 1, opt->dict_size); |
776 | |
|
777 | 0 | return LZMA_OK; |
778 | 0 | } |
779 | | #endif |
780 | | |
781 | | |
782 | | extern LZMA_API(lzma_bool) |
783 | | lzma_mode_is_supported(lzma_mode mode) |
784 | 0 | { |
785 | 0 | return mode == LZMA_MODE_FAST || mode == LZMA_MODE_NORMAL; |
786 | 0 | } |