/rust/registry/src/index.crates.io-1949cf8c6b5b557f/zstd-0.14.3/src/stream/raw.rs
Line | Count | Source |
1 | | //! Raw in-memory stream compression/decompression. |
2 | | //! |
3 | | //! This module defines a `Decoder` and an `Encoder` to decode/encode streams |
4 | | //! of data using buffers. |
5 | | //! |
6 | | //! They are mostly thin wrappers around `zstd_safe::{DCtx, CCtx}`. |
7 | | use std::io; |
8 | | |
9 | | pub use zstd_safe::{CParameter, DParameter, InBuffer, OutBuffer, WriteBuf}; |
10 | | |
11 | | use crate::dict::{DecoderDictionary, EncoderDictionary}; |
12 | | use crate::map_error_code; |
13 | | |
14 | | /// Represents an abstract compression/decompression operation. |
15 | | /// |
16 | | /// This trait covers both `Encoder` and `Decoder`. |
17 | | pub trait Operation { |
18 | | /// Performs a single step of this operation. |
19 | | /// |
20 | | /// Should return a hint for the next input size. |
21 | | /// |
22 | | /// If the result is `Ok(0)`, it may indicate that a frame was just |
23 | | /// finished. |
24 | | fn run<C: WriteBuf + ?Sized>( |
25 | | &mut self, |
26 | | input: &mut InBuffer<'_>, |
27 | | output: &mut OutBuffer<'_, C>, |
28 | | ) -> io::Result<usize>; |
29 | | |
30 | | /// Performs a single step of this operation. |
31 | | /// |
32 | | /// This is a comvenience wrapper around `Operation::run` if you don't |
33 | | /// want to deal with `InBuffer`/`OutBuffer`. |
34 | 0 | fn run_on_buffers( |
35 | 0 | &mut self, |
36 | 0 | input: &[u8], |
37 | 0 | output: &mut [u8], |
38 | 0 | ) -> io::Result<Status> { |
39 | 0 | let mut input = InBuffer::around(input); |
40 | 0 | let mut output = OutBuffer::around(output); |
41 | | |
42 | 0 | let remaining = self.run(&mut input, &mut output)?; |
43 | | |
44 | 0 | Ok(Status { |
45 | 0 | remaining, |
46 | 0 | bytes_read: input.pos(), |
47 | 0 | bytes_written: output.pos(), |
48 | 0 | }) |
49 | 0 | } |
50 | | |
51 | | /// Flushes any internal buffer, if any. |
52 | | /// |
53 | | /// Returns the number of bytes still in the buffer. |
54 | | /// To flush entirely, keep calling until it returns `Ok(0)`. |
55 | 0 | fn flush<C: WriteBuf + ?Sized>( |
56 | 0 | &mut self, |
57 | 0 | output: &mut OutBuffer<'_, C>, |
58 | 0 | ) -> io::Result<usize> { |
59 | 0 | let _ = output; |
60 | 0 | Ok(0) |
61 | 0 | } |
62 | | |
63 | | /// Prepares the operation for a new frame. |
64 | | /// |
65 | | /// This is hopefully cheaper than creating a new operation. |
66 | 0 | fn reinit(&mut self) -> io::Result<()> { |
67 | 0 | Ok(()) |
68 | 0 | } |
69 | | |
70 | | /// Finishes the operation, writing any footer if necessary. |
71 | | /// |
72 | | /// Returns the number of bytes still to write. |
73 | | /// |
74 | | /// Keep calling this method until it returns `Ok(0)`, |
75 | | /// and then don't ever call this method. |
76 | 0 | fn finish<C: WriteBuf + ?Sized>( |
77 | 0 | &mut self, |
78 | 0 | output: &mut OutBuffer<'_, C>, |
79 | 0 | finished_frame: bool, |
80 | 0 | ) -> io::Result<usize> { |
81 | 0 | let _ = output; |
82 | 0 | let _ = finished_frame; |
83 | 0 | Ok(0) |
84 | 0 | } |
85 | | } |
86 | | |
87 | | /// Dummy operation that just copies its input to the output. |
88 | | pub struct NoOp; |
89 | | |
90 | | impl Operation for NoOp { |
91 | 0 | fn run<C: WriteBuf + ?Sized>( |
92 | 0 | &mut self, |
93 | 0 | input: &mut InBuffer<'_>, |
94 | 0 | output: &mut OutBuffer<'_, C>, |
95 | 0 | ) -> io::Result<usize> { |
96 | | // Skip the prelude |
97 | 0 | let src = &input.src[input.pos..]; |
98 | | // Safe because `output.pos() <= output.capacity()`. |
99 | 0 | let output_pos = output.pos(); |
100 | 0 | let dst = unsafe { output.as_mut_ptr().add(output_pos) }; |
101 | | |
102 | | // Ignore anything past the end |
103 | 0 | let len = usize::min(src.len(), output.capacity() - output_pos); |
104 | 0 | let src = &src[..len]; |
105 | | |
106 | | // Safe because: |
107 | | // * `len` is less than either of the two lengths |
108 | | // * `src` and `dst` do not overlap because we have `&mut` to each. |
109 | 0 | unsafe { std::ptr::copy_nonoverlapping(src.as_ptr(), dst, len) }; |
110 | 0 | input.set_pos(input.pos() + len); |
111 | 0 | unsafe { output.set_pos(output_pos + len) }; |
112 | | |
113 | 0 | Ok(0) |
114 | 0 | } |
115 | | } |
116 | | |
117 | | /// Describes the result of an operation. |
118 | | pub struct Status { |
119 | | /// Number of bytes expected for next input. |
120 | | /// |
121 | | /// * If `remaining = 0`, then we are at the end of a frame. |
122 | | /// * If `remaining > 0`, then it's just a hint for how much there is still |
123 | | /// to read. |
124 | | pub remaining: usize, |
125 | | |
126 | | /// Number of bytes read from the input. |
127 | | pub bytes_read: usize, |
128 | | |
129 | | /// Number of bytes written to the output. |
130 | | pub bytes_written: usize, |
131 | | } |
132 | | |
133 | | /// An in-memory decoder for streams of data. |
134 | | pub struct Decoder<'a> { |
135 | | context: MaybeOwnedDCtx<'a>, |
136 | | } |
137 | | |
138 | 0 | fn create_dctx<'a>() -> io::Result<zstd_safe::DCtx<'a>> { |
139 | 0 | zstd_safe::DCtx::try_create().ok_or_else(|| { |
140 | 0 | io::Error::new( |
141 | 0 | io::ErrorKind::Other, |
142 | | "zstd could not allocate a decompression context", |
143 | | ) |
144 | 0 | }) |
145 | 0 | } |
146 | | |
147 | | impl Decoder<'static> { |
148 | | /// Creates a new decoder. |
149 | 0 | pub fn new() -> io::Result<Self> { |
150 | 0 | Self::with_dictionary(&[]) |
151 | 0 | } |
152 | | |
153 | | /// Creates a new decoder initialized with the given dictionary. |
154 | 0 | pub fn with_dictionary(dictionary: &[u8]) -> io::Result<Self> { |
155 | 0 | let mut context = create_dctx()?; |
156 | 0 | context.init().map_err(map_error_code)?; |
157 | 0 | context |
158 | 0 | .load_dictionary(dictionary) |
159 | 0 | .map_err(map_error_code)?; |
160 | 0 | Ok(Decoder { |
161 | 0 | context: MaybeOwnedDCtx::Owned(context), |
162 | 0 | }) |
163 | 0 | } |
164 | | } |
165 | | |
166 | | impl<'a> Decoder<'a> { |
167 | | /// Creates a new decoder which employs the provided context for deserialization. |
168 | | /// |
169 | | /// The context's session is reset first: anything left over from a frame |
170 | | /// that a previous user of the context did not finish is discarded. |
171 | | /// Parameters and dictionary are kept. |
172 | 0 | pub fn with_context(context: &'a mut zstd_safe::DCtx<'static>) -> Self { |
173 | | // Otherwise we would start by handing out what is left of the |
174 | | // previous decoder's output. Resetting only the session cannot fail. |
175 | 0 | let _ = context.reset(zstd_safe::ResetDirective::SessionOnly); |
176 | | |
177 | 0 | Self { |
178 | 0 | context: MaybeOwnedDCtx::Borrowed(context), |
179 | 0 | } |
180 | 0 | } |
181 | | |
182 | | /// Creates a new decoder, using an existing `DecoderDictionary`. |
183 | 0 | pub fn with_prepared_dictionary<'b>( |
184 | 0 | dictionary: &'a DecoderDictionary<'b>, |
185 | 0 | ) -> io::Result<Self> |
186 | 0 | where |
187 | 0 | 'b: 'a, |
188 | | { |
189 | 0 | let mut context = create_dctx()?; |
190 | 0 | context |
191 | 0 | .ref_ddict(dictionary.as_ddict()) |
192 | 0 | .map_err(map_error_code)?; |
193 | 0 | Ok(Decoder { |
194 | 0 | context: MaybeOwnedDCtx::Owned(context), |
195 | 0 | }) |
196 | 0 | } |
197 | | |
198 | | /// Creates a new decoder, using a ref prefix |
199 | 0 | pub fn with_ref_prefix<'b>(ref_prefix: &'b [u8]) -> io::Result<Self> |
200 | 0 | where |
201 | 0 | 'b: 'a, |
202 | | { |
203 | 0 | let mut context = create_dctx()?; |
204 | 0 | context.ref_prefix(ref_prefix).map_err(map_error_code)?; |
205 | 0 | Ok(Decoder { |
206 | 0 | context: MaybeOwnedDCtx::Owned(context), |
207 | 0 | }) |
208 | 0 | } |
209 | | |
210 | | /// Sets a decompression parameter for this decoder. |
211 | 0 | pub fn set_parameter(&mut self, parameter: DParameter) -> io::Result<()> { |
212 | 0 | match &mut self.context { |
213 | 0 | MaybeOwnedDCtx::Owned(x) => x.set_parameter(parameter), |
214 | 0 | MaybeOwnedDCtx::Borrowed(x) => x.set_parameter(parameter), |
215 | | } |
216 | 0 | .map_err(map_error_code)?; |
217 | 0 | Ok(()) |
218 | 0 | } |
219 | | } |
220 | | |
221 | | impl Operation for Decoder<'_> { |
222 | 0 | fn run<C: WriteBuf + ?Sized>( |
223 | 0 | &mut self, |
224 | 0 | input: &mut InBuffer<'_>, |
225 | 0 | output: &mut OutBuffer<'_, C>, |
226 | 0 | ) -> io::Result<usize> { |
227 | 0 | match &mut self.context { |
228 | 0 | MaybeOwnedDCtx::Owned(x) => x.decompress_stream(output, input), |
229 | 0 | MaybeOwnedDCtx::Borrowed(x) => x.decompress_stream(output, input), |
230 | | } |
231 | 0 | .map_err(map_error_code) |
232 | 0 | } |
233 | | |
234 | 0 | fn flush<C: WriteBuf + ?Sized>( |
235 | 0 | &mut self, |
236 | 0 | output: &mut OutBuffer<'_, C>, |
237 | 0 | ) -> io::Result<usize> { |
238 | | // To flush, we just offer no additional input. |
239 | 0 | self.run(&mut InBuffer::around(&[]), output)?; |
240 | | |
241 | | // We don't _know_ how much (decompressed data) there is still in buffer. |
242 | 0 | if output.pos() < output.capacity() { |
243 | | // We only know when there's none (the output buffer is not full). |
244 | 0 | Ok(0) |
245 | | } else { |
246 | | // Otherwise, pretend there's still "1 byte" remaining. |
247 | 0 | Ok(1) |
248 | | } |
249 | 0 | } |
250 | | |
251 | 0 | fn reinit(&mut self) -> io::Result<()> { |
252 | 0 | match &mut self.context { |
253 | 0 | MaybeOwnedDCtx::Owned(x) => { |
254 | 0 | x.reset(zstd_safe::ResetDirective::SessionOnly) |
255 | | } |
256 | 0 | MaybeOwnedDCtx::Borrowed(x) => { |
257 | 0 | x.reset(zstd_safe::ResetDirective::SessionOnly) |
258 | | } |
259 | | } |
260 | 0 | .map_err(map_error_code)?; |
261 | 0 | Ok(()) |
262 | 0 | } |
263 | | |
264 | 0 | fn finish<C: WriteBuf + ?Sized>( |
265 | 0 | &mut self, |
266 | 0 | _output: &mut OutBuffer<'_, C>, |
267 | 0 | finished_frame: bool, |
268 | 0 | ) -> io::Result<usize> { |
269 | 0 | if finished_frame { |
270 | 0 | Ok(0) |
271 | | } else { |
272 | 0 | Err(io::Error::new( |
273 | 0 | io::ErrorKind::UnexpectedEof, |
274 | 0 | "incomplete frame", |
275 | 0 | )) |
276 | | } |
277 | 0 | } |
278 | | } |
279 | | |
280 | | /// An in-memory encoder for streams of data. |
281 | | pub struct Encoder<'a> { |
282 | | context: MaybeOwnedCCtx<'a>, |
283 | | } |
284 | | |
285 | | impl Encoder<'static> { |
286 | | /// Creates a new encoder. |
287 | 0 | pub fn new(level: i32) -> io::Result<Self> { |
288 | 0 | Self::with_dictionary(level, &[]) |
289 | 0 | } |
290 | | |
291 | | /// Creates a new encoder initialized with the given dictionary. |
292 | 0 | pub fn with_dictionary(level: i32, dictionary: &[u8]) -> io::Result<Self> { |
293 | 0 | let mut context = zstd_safe::CCtx::create(); |
294 | | |
295 | 0 | context |
296 | 0 | .set_parameter(CParameter::CompressionLevel(level)) |
297 | 0 | .map_err(map_error_code)?; |
298 | | |
299 | 0 | context |
300 | 0 | .load_dictionary(dictionary) |
301 | 0 | .map_err(map_error_code)?; |
302 | | |
303 | 0 | Ok(Encoder { |
304 | 0 | context: MaybeOwnedCCtx::Owned(context), |
305 | 0 | }) |
306 | 0 | } |
307 | | } |
308 | | |
309 | | impl<'a> Encoder<'a> { |
310 | | /// Creates a new encoder that uses the provided context for serialization. |
311 | | /// |
312 | | /// The context's session is reset first: anything left over from a frame |
313 | | /// that a previous user of the context did not finish is discarded, and |
314 | | /// so is the pledged source size (set it on the encoder instead). |
315 | | /// Parameters and dictionary are kept. |
316 | 0 | pub fn with_context(context: &'a mut zstd_safe::CCtx<'static>) -> Self { |
317 | | // Otherwise the input buffered by the previous encoder would become |
318 | | // the start of our frame. Resetting only the session cannot fail. |
319 | 0 | let _ = context.reset(zstd_safe::ResetDirective::SessionOnly); |
320 | | |
321 | 0 | Self { |
322 | 0 | context: MaybeOwnedCCtx::Borrowed(context), |
323 | 0 | } |
324 | 0 | } |
325 | | |
326 | | /// Creates a new encoder using an existing `EncoderDictionary`. |
327 | 0 | pub fn with_prepared_dictionary<'b>( |
328 | 0 | dictionary: &'a EncoderDictionary<'b>, |
329 | 0 | ) -> io::Result<Self> |
330 | 0 | where |
331 | 0 | 'b: 'a, |
332 | | { |
333 | 0 | let mut context = zstd_safe::CCtx::create(); |
334 | 0 | context |
335 | 0 | .ref_cdict(dictionary.as_cdict()) |
336 | 0 | .map_err(map_error_code)?; |
337 | 0 | Ok(Encoder { |
338 | 0 | context: MaybeOwnedCCtx::Owned(context), |
339 | 0 | }) |
340 | 0 | } |
341 | | |
342 | | /// Creates a new encoder initialized with the given ref prefix. |
343 | 0 | pub fn with_ref_prefix<'b>( |
344 | 0 | level: i32, |
345 | 0 | ref_prefix: &'b [u8], |
346 | 0 | ) -> io::Result<Self> |
347 | 0 | where |
348 | 0 | 'b: 'a, |
349 | | { |
350 | 0 | let mut context = zstd_safe::CCtx::create(); |
351 | | |
352 | 0 | context |
353 | 0 | .set_parameter(CParameter::CompressionLevel(level)) |
354 | 0 | .map_err(map_error_code)?; |
355 | | |
356 | 0 | context.ref_prefix(ref_prefix).map_err(map_error_code)?; |
357 | | |
358 | 0 | Ok(Encoder { |
359 | 0 | context: MaybeOwnedCCtx::Owned(context), |
360 | 0 | }) |
361 | 0 | } |
362 | | |
363 | | /// Sets a compression parameter for this encoder. |
364 | 0 | pub fn set_parameter(&mut self, parameter: CParameter) -> io::Result<()> { |
365 | 0 | match &mut self.context { |
366 | 0 | MaybeOwnedCCtx::Owned(x) => x.set_parameter(parameter), |
367 | 0 | MaybeOwnedCCtx::Borrowed(x) => x.set_parameter(parameter), |
368 | | } |
369 | 0 | .map_err(map_error_code)?; |
370 | 0 | Ok(()) |
371 | 0 | } |
372 | | |
373 | | /// Sets the size of the input expected by zstd. |
374 | | /// |
375 | | /// May affect compression ratio. |
376 | | /// |
377 | | /// It is an error to give an incorrect size (an error _will_ be returned when closing the |
378 | | /// stream). |
379 | | /// |
380 | | /// If `None` is given, it assume the size is not known (default behaviour). |
381 | 0 | pub fn set_pledged_src_size( |
382 | 0 | &mut self, |
383 | 0 | pledged_src_size: Option<u64>, |
384 | 0 | ) -> io::Result<()> { |
385 | 0 | match &mut self.context { |
386 | 0 | MaybeOwnedCCtx::Owned(x) => { |
387 | 0 | x.set_pledged_src_size(pledged_src_size) |
388 | | } |
389 | 0 | MaybeOwnedCCtx::Borrowed(x) => { |
390 | 0 | x.set_pledged_src_size(pledged_src_size) |
391 | | } |
392 | | } |
393 | 0 | .map_err(map_error_code)?; |
394 | 0 | Ok(()) |
395 | 0 | } |
396 | | } |
397 | | |
398 | | impl<'a> Operation for Encoder<'a> { |
399 | 0 | fn run<C: WriteBuf + ?Sized>( |
400 | 0 | &mut self, |
401 | 0 | input: &mut InBuffer<'_>, |
402 | 0 | output: &mut OutBuffer<'_, C>, |
403 | 0 | ) -> io::Result<usize> { |
404 | 0 | match &mut self.context { |
405 | 0 | MaybeOwnedCCtx::Owned(x) => x.compress_stream(output, input), |
406 | 0 | MaybeOwnedCCtx::Borrowed(x) => x.compress_stream(output, input), |
407 | | } |
408 | 0 | .map_err(map_error_code) |
409 | 0 | } |
410 | | |
411 | 0 | fn flush<C: WriteBuf + ?Sized>( |
412 | 0 | &mut self, |
413 | 0 | output: &mut OutBuffer<'_, C>, |
414 | 0 | ) -> io::Result<usize> { |
415 | 0 | match &mut self.context { |
416 | 0 | MaybeOwnedCCtx::Owned(x) => x.flush_stream(output), |
417 | 0 | MaybeOwnedCCtx::Borrowed(x) => x.flush_stream(output), |
418 | | } |
419 | 0 | .map_err(map_error_code) |
420 | 0 | } |
421 | | |
422 | 0 | fn finish<C: WriteBuf + ?Sized>( |
423 | 0 | &mut self, |
424 | 0 | output: &mut OutBuffer<'_, C>, |
425 | 0 | _finished_frame: bool, |
426 | 0 | ) -> io::Result<usize> { |
427 | 0 | match &mut self.context { |
428 | 0 | MaybeOwnedCCtx::Owned(x) => x.end_stream(output), |
429 | 0 | MaybeOwnedCCtx::Borrowed(x) => x.end_stream(output), |
430 | | } |
431 | 0 | .map_err(map_error_code) |
432 | 0 | } |
433 | | |
434 | 0 | fn reinit(&mut self) -> io::Result<()> { |
435 | 0 | match &mut self.context { |
436 | 0 | MaybeOwnedCCtx::Owned(x) => { |
437 | 0 | x.reset(zstd_safe::ResetDirective::SessionOnly) |
438 | | } |
439 | 0 | MaybeOwnedCCtx::Borrowed(x) => { |
440 | 0 | x.reset(zstd_safe::ResetDirective::SessionOnly) |
441 | | } |
442 | | } |
443 | 0 | .map_err(map_error_code)?; |
444 | 0 | Ok(()) |
445 | 0 | } |
446 | | } |
447 | | |
448 | | enum MaybeOwnedCCtx<'a> { |
449 | | Owned(zstd_safe::CCtx<'a>), |
450 | | Borrowed(&'a mut zstd_safe::CCtx<'static>), |
451 | | } |
452 | | |
453 | | enum MaybeOwnedDCtx<'a> { |
454 | | Owned(zstd_safe::DCtx<'a>), |
455 | | Borrowed(&'a mut zstd_safe::DCtx<'static>), |
456 | | } |
457 | | |
458 | | #[cfg(test)] |
459 | | mod tests { |
460 | | |
461 | | // This requires impl for [u8; N] which is currently behind a feature. |
462 | | #[cfg(feature = "arrays")] |
463 | | #[test] |
464 | | fn test_cycle() { |
465 | | use super::{Decoder, Encoder, InBuffer, Operation, OutBuffer}; |
466 | | |
467 | | let mut encoder = Encoder::new(1).unwrap(); |
468 | | let mut decoder = Decoder::new().unwrap(); |
469 | | |
470 | | // Step 1: compress |
471 | | let mut input = InBuffer::around(b"AbcdefAbcdefabcdef"); |
472 | | |
473 | | let mut output = [0u8; 128]; |
474 | | let mut output = OutBuffer::around(&mut output); |
475 | | |
476 | | loop { |
477 | | encoder.run(&mut input, &mut output).unwrap(); |
478 | | |
479 | | if input.pos == input.src.len() { |
480 | | break; |
481 | | } |
482 | | } |
483 | | encoder.finish(&mut output, true).unwrap(); |
484 | | |
485 | | let initial_data = input.src; |
486 | | |
487 | | // Step 2: decompress |
488 | | let mut input = InBuffer::around(output.as_slice()); |
489 | | let mut output = [0u8; 128]; |
490 | | let mut output = OutBuffer::around(&mut output); |
491 | | |
492 | | loop { |
493 | | decoder.run(&mut input, &mut output).unwrap(); |
494 | | |
495 | | if input.pos == input.src.len() { |
496 | | break; |
497 | | } |
498 | | } |
499 | | |
500 | | assert_eq!(initial_data, output.as_slice()); |
501 | | } |
502 | | } |