Coverage Report

Created: 2026-11-11 06:34

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/rust/registry/src/index.crates.io-1949cf8c6b5b557f/zstd-0.14.3/src/stream/raw.rs
Line
Count
Source
1
//! Raw in-memory stream compression/decompression.
2
//!
3
//! This module defines a `Decoder` and an `Encoder` to decode/encode streams
4
//! of data using buffers.
5
//!
6
//! They are mostly thin wrappers around `zstd_safe::{DCtx, CCtx}`.
7
use std::io;
8
9
pub use zstd_safe::{CParameter, DParameter, InBuffer, OutBuffer, WriteBuf};
10
11
use crate::dict::{DecoderDictionary, EncoderDictionary};
12
use crate::map_error_code;
13
14
/// Represents an abstract compression/decompression operation.
15
///
16
/// This trait covers both `Encoder` and `Decoder`.
17
pub trait Operation {
18
    /// Performs a single step of this operation.
19
    ///
20
    /// Should return a hint for the next input size.
21
    ///
22
    /// If the result is `Ok(0)`, it may indicate that a frame was just
23
    /// finished.
24
    fn run<C: WriteBuf + ?Sized>(
25
        &mut self,
26
        input: &mut InBuffer<'_>,
27
        output: &mut OutBuffer<'_, C>,
28
    ) -> io::Result<usize>;
29
30
    /// Performs a single step of this operation.
31
    ///
32
    /// This is a comvenience wrapper around `Operation::run` if you don't
33
    /// want to deal with `InBuffer`/`OutBuffer`.
34
0
    fn run_on_buffers(
35
0
        &mut self,
36
0
        input: &[u8],
37
0
        output: &mut [u8],
38
0
    ) -> io::Result<Status> {
39
0
        let mut input = InBuffer::around(input);
40
0
        let mut output = OutBuffer::around(output);
41
42
0
        let remaining = self.run(&mut input, &mut output)?;
43
44
0
        Ok(Status {
45
0
            remaining,
46
0
            bytes_read: input.pos(),
47
0
            bytes_written: output.pos(),
48
0
        })
49
0
    }
50
51
    /// Flushes any internal buffer, if any.
52
    ///
53
    /// Returns the number of bytes still in the buffer.
54
    /// To flush entirely, keep calling until it returns `Ok(0)`.
55
0
    fn flush<C: WriteBuf + ?Sized>(
56
0
        &mut self,
57
0
        output: &mut OutBuffer<'_, C>,
58
0
    ) -> io::Result<usize> {
59
0
        let _ = output;
60
0
        Ok(0)
61
0
    }
62
63
    /// Prepares the operation for a new frame.
64
    ///
65
    /// This is hopefully cheaper than creating a new operation.
66
0
    fn reinit(&mut self) -> io::Result<()> {
67
0
        Ok(())
68
0
    }
69
70
    /// Finishes the operation, writing any footer if necessary.
71
    ///
72
    /// Returns the number of bytes still to write.
73
    ///
74
    /// Keep calling this method until it returns `Ok(0)`,
75
    /// and then don't ever call this method.
76
0
    fn finish<C: WriteBuf + ?Sized>(
77
0
        &mut self,
78
0
        output: &mut OutBuffer<'_, C>,
79
0
        finished_frame: bool,
80
0
    ) -> io::Result<usize> {
81
0
        let _ = output;
82
0
        let _ = finished_frame;
83
0
        Ok(0)
84
0
    }
85
}
86
87
/// Dummy operation that just copies its input to the output.
88
pub struct NoOp;
89
90
impl Operation for NoOp {
91
0
    fn run<C: WriteBuf + ?Sized>(
92
0
        &mut self,
93
0
        input: &mut InBuffer<'_>,
94
0
        output: &mut OutBuffer<'_, C>,
95
0
    ) -> io::Result<usize> {
96
        // Skip the prelude
97
0
        let src = &input.src[input.pos..];
98
        // Safe because `output.pos() <= output.capacity()`.
99
0
        let output_pos = output.pos();
100
0
        let dst = unsafe { output.as_mut_ptr().add(output_pos) };
101
102
        // Ignore anything past the end
103
0
        let len = usize::min(src.len(), output.capacity() - output_pos);
104
0
        let src = &src[..len];
105
106
        // Safe because:
107
        // * `len` is less than either of the two lengths
108
        // * `src` and `dst` do not overlap because we have `&mut` to each.
109
0
        unsafe { std::ptr::copy_nonoverlapping(src.as_ptr(), dst, len) };
110
0
        input.set_pos(input.pos() + len);
111
0
        unsafe { output.set_pos(output_pos + len) };
112
113
0
        Ok(0)
114
0
    }
115
}
116
117
/// Describes the result of an operation.
118
pub struct Status {
119
    /// Number of bytes expected for next input.
120
    ///
121
    /// * If `remaining = 0`, then we are at the end of a frame.
122
    /// * If `remaining > 0`, then it's just a hint for how much there is still
123
    ///   to read.
124
    pub remaining: usize,
125
126
    /// Number of bytes read from the input.
127
    pub bytes_read: usize,
128
129
    /// Number of bytes written to the output.
130
    pub bytes_written: usize,
131
}
132
133
/// An in-memory decoder for streams of data.
134
pub struct Decoder<'a> {
135
    context: MaybeOwnedDCtx<'a>,
136
}
137
138
0
fn create_dctx<'a>() -> io::Result<zstd_safe::DCtx<'a>> {
139
0
    zstd_safe::DCtx::try_create().ok_or_else(|| {
140
0
        io::Error::new(
141
0
            io::ErrorKind::Other,
142
            "zstd could not allocate a decompression context",
143
        )
144
0
    })
145
0
}
146
147
impl Decoder<'static> {
148
    /// Creates a new decoder.
149
0
    pub fn new() -> io::Result<Self> {
150
0
        Self::with_dictionary(&[])
151
0
    }
152
153
    /// Creates a new decoder initialized with the given dictionary.
154
0
    pub fn with_dictionary(dictionary: &[u8]) -> io::Result<Self> {
155
0
        let mut context = create_dctx()?;
156
0
        context.init().map_err(map_error_code)?;
157
0
        context
158
0
            .load_dictionary(dictionary)
159
0
            .map_err(map_error_code)?;
160
0
        Ok(Decoder {
161
0
            context: MaybeOwnedDCtx::Owned(context),
162
0
        })
163
0
    }
164
}
165
166
impl<'a> Decoder<'a> {
167
    /// Creates a new decoder which employs the provided context for deserialization.
168
    ///
169
    /// The context's session is reset first: anything left over from a frame
170
    /// that a previous user of the context did not finish is discarded.
171
    /// Parameters and dictionary are kept.
172
0
    pub fn with_context(context: &'a mut zstd_safe::DCtx<'static>) -> Self {
173
        // Otherwise we would start by handing out what is left of the
174
        // previous decoder's output. Resetting only the session cannot fail.
175
0
        let _ = context.reset(zstd_safe::ResetDirective::SessionOnly);
176
177
0
        Self {
178
0
            context: MaybeOwnedDCtx::Borrowed(context),
179
0
        }
180
0
    }
181
182
    /// Creates a new decoder, using an existing `DecoderDictionary`.
183
0
    pub fn with_prepared_dictionary<'b>(
184
0
        dictionary: &'a DecoderDictionary<'b>,
185
0
    ) -> io::Result<Self>
186
0
    where
187
0
        'b: 'a,
188
    {
189
0
        let mut context = create_dctx()?;
190
0
        context
191
0
            .ref_ddict(dictionary.as_ddict())
192
0
            .map_err(map_error_code)?;
193
0
        Ok(Decoder {
194
0
            context: MaybeOwnedDCtx::Owned(context),
195
0
        })
196
0
    }
197
198
    /// Creates a new decoder, using a ref prefix
199
0
    pub fn with_ref_prefix<'b>(ref_prefix: &'b [u8]) -> io::Result<Self>
200
0
    where
201
0
        'b: 'a,
202
    {
203
0
        let mut context = create_dctx()?;
204
0
        context.ref_prefix(ref_prefix).map_err(map_error_code)?;
205
0
        Ok(Decoder {
206
0
            context: MaybeOwnedDCtx::Owned(context),
207
0
        })
208
0
    }
209
210
    /// Sets a decompression parameter for this decoder.
211
0
    pub fn set_parameter(&mut self, parameter: DParameter) -> io::Result<()> {
212
0
        match &mut self.context {
213
0
            MaybeOwnedDCtx::Owned(x) => x.set_parameter(parameter),
214
0
            MaybeOwnedDCtx::Borrowed(x) => x.set_parameter(parameter),
215
        }
216
0
        .map_err(map_error_code)?;
217
0
        Ok(())
218
0
    }
219
}
220
221
impl Operation for Decoder<'_> {
222
0
    fn run<C: WriteBuf + ?Sized>(
223
0
        &mut self,
224
0
        input: &mut InBuffer<'_>,
225
0
        output: &mut OutBuffer<'_, C>,
226
0
    ) -> io::Result<usize> {
227
0
        match &mut self.context {
228
0
            MaybeOwnedDCtx::Owned(x) => x.decompress_stream(output, input),
229
0
            MaybeOwnedDCtx::Borrowed(x) => x.decompress_stream(output, input),
230
        }
231
0
        .map_err(map_error_code)
232
0
    }
233
234
0
    fn flush<C: WriteBuf + ?Sized>(
235
0
        &mut self,
236
0
        output: &mut OutBuffer<'_, C>,
237
0
    ) -> io::Result<usize> {
238
        // To flush, we just offer no additional input.
239
0
        self.run(&mut InBuffer::around(&[]), output)?;
240
241
        // We don't _know_ how much (decompressed data) there is still in buffer.
242
0
        if output.pos() < output.capacity() {
243
            // We only know when there's none (the output buffer is not full).
244
0
            Ok(0)
245
        } else {
246
            // Otherwise, pretend there's still "1 byte" remaining.
247
0
            Ok(1)
248
        }
249
0
    }
250
251
0
    fn reinit(&mut self) -> io::Result<()> {
252
0
        match &mut self.context {
253
0
            MaybeOwnedDCtx::Owned(x) => {
254
0
                x.reset(zstd_safe::ResetDirective::SessionOnly)
255
            }
256
0
            MaybeOwnedDCtx::Borrowed(x) => {
257
0
                x.reset(zstd_safe::ResetDirective::SessionOnly)
258
            }
259
        }
260
0
        .map_err(map_error_code)?;
261
0
        Ok(())
262
0
    }
263
264
0
    fn finish<C: WriteBuf + ?Sized>(
265
0
        &mut self,
266
0
        _output: &mut OutBuffer<'_, C>,
267
0
        finished_frame: bool,
268
0
    ) -> io::Result<usize> {
269
0
        if finished_frame {
270
0
            Ok(0)
271
        } else {
272
0
            Err(io::Error::new(
273
0
                io::ErrorKind::UnexpectedEof,
274
0
                "incomplete frame",
275
0
            ))
276
        }
277
0
    }
278
}
279
280
/// An in-memory encoder for streams of data.
281
pub struct Encoder<'a> {
282
    context: MaybeOwnedCCtx<'a>,
283
}
284
285
impl Encoder<'static> {
286
    /// Creates a new encoder.
287
0
    pub fn new(level: i32) -> io::Result<Self> {
288
0
        Self::with_dictionary(level, &[])
289
0
    }
290
291
    /// Creates a new encoder initialized with the given dictionary.
292
0
    pub fn with_dictionary(level: i32, dictionary: &[u8]) -> io::Result<Self> {
293
0
        let mut context = zstd_safe::CCtx::create();
294
295
0
        context
296
0
            .set_parameter(CParameter::CompressionLevel(level))
297
0
            .map_err(map_error_code)?;
298
299
0
        context
300
0
            .load_dictionary(dictionary)
301
0
            .map_err(map_error_code)?;
302
303
0
        Ok(Encoder {
304
0
            context: MaybeOwnedCCtx::Owned(context),
305
0
        })
306
0
    }
307
}
308
309
impl<'a> Encoder<'a> {
310
    /// Creates a new encoder that uses the provided context for serialization.
311
    ///
312
    /// The context's session is reset first: anything left over from a frame
313
    /// that a previous user of the context did not finish is discarded, and
314
    /// so is the pledged source size (set it on the encoder instead).
315
    /// Parameters and dictionary are kept.
316
0
    pub fn with_context(context: &'a mut zstd_safe::CCtx<'static>) -> Self {
317
        // Otherwise the input buffered by the previous encoder would become
318
        // the start of our frame. Resetting only the session cannot fail.
319
0
        let _ = context.reset(zstd_safe::ResetDirective::SessionOnly);
320
321
0
        Self {
322
0
            context: MaybeOwnedCCtx::Borrowed(context),
323
0
        }
324
0
    }
325
326
    /// Creates a new encoder using an existing `EncoderDictionary`.
327
0
    pub fn with_prepared_dictionary<'b>(
328
0
        dictionary: &'a EncoderDictionary<'b>,
329
0
    ) -> io::Result<Self>
330
0
    where
331
0
        'b: 'a,
332
    {
333
0
        let mut context = zstd_safe::CCtx::create();
334
0
        context
335
0
            .ref_cdict(dictionary.as_cdict())
336
0
            .map_err(map_error_code)?;
337
0
        Ok(Encoder {
338
0
            context: MaybeOwnedCCtx::Owned(context),
339
0
        })
340
0
    }
341
342
    /// Creates a new encoder initialized with the given ref prefix.
343
0
    pub fn with_ref_prefix<'b>(
344
0
        level: i32,
345
0
        ref_prefix: &'b [u8],
346
0
    ) -> io::Result<Self>
347
0
    where
348
0
        'b: 'a,
349
    {
350
0
        let mut context = zstd_safe::CCtx::create();
351
352
0
        context
353
0
            .set_parameter(CParameter::CompressionLevel(level))
354
0
            .map_err(map_error_code)?;
355
356
0
        context.ref_prefix(ref_prefix).map_err(map_error_code)?;
357
358
0
        Ok(Encoder {
359
0
            context: MaybeOwnedCCtx::Owned(context),
360
0
        })
361
0
    }
362
363
    /// Sets a compression parameter for this encoder.
364
0
    pub fn set_parameter(&mut self, parameter: CParameter) -> io::Result<()> {
365
0
        match &mut self.context {
366
0
            MaybeOwnedCCtx::Owned(x) => x.set_parameter(parameter),
367
0
            MaybeOwnedCCtx::Borrowed(x) => x.set_parameter(parameter),
368
        }
369
0
        .map_err(map_error_code)?;
370
0
        Ok(())
371
0
    }
372
373
    /// Sets the size of the input expected by zstd.
374
    ///
375
    /// May affect compression ratio.
376
    ///
377
    /// It is an error to give an incorrect size (an error _will_ be returned when closing the
378
    /// stream).
379
    ///
380
    /// If `None` is given, it assume the size is not known (default behaviour).
381
0
    pub fn set_pledged_src_size(
382
0
        &mut self,
383
0
        pledged_src_size: Option<u64>,
384
0
    ) -> io::Result<()> {
385
0
        match &mut self.context {
386
0
            MaybeOwnedCCtx::Owned(x) => {
387
0
                x.set_pledged_src_size(pledged_src_size)
388
            }
389
0
            MaybeOwnedCCtx::Borrowed(x) => {
390
0
                x.set_pledged_src_size(pledged_src_size)
391
            }
392
        }
393
0
        .map_err(map_error_code)?;
394
0
        Ok(())
395
0
    }
396
}
397
398
impl<'a> Operation for Encoder<'a> {
399
0
    fn run<C: WriteBuf + ?Sized>(
400
0
        &mut self,
401
0
        input: &mut InBuffer<'_>,
402
0
        output: &mut OutBuffer<'_, C>,
403
0
    ) -> io::Result<usize> {
404
0
        match &mut self.context {
405
0
            MaybeOwnedCCtx::Owned(x) => x.compress_stream(output, input),
406
0
            MaybeOwnedCCtx::Borrowed(x) => x.compress_stream(output, input),
407
        }
408
0
        .map_err(map_error_code)
409
0
    }
410
411
0
    fn flush<C: WriteBuf + ?Sized>(
412
0
        &mut self,
413
0
        output: &mut OutBuffer<'_, C>,
414
0
    ) -> io::Result<usize> {
415
0
        match &mut self.context {
416
0
            MaybeOwnedCCtx::Owned(x) => x.flush_stream(output),
417
0
            MaybeOwnedCCtx::Borrowed(x) => x.flush_stream(output),
418
        }
419
0
        .map_err(map_error_code)
420
0
    }
421
422
0
    fn finish<C: WriteBuf + ?Sized>(
423
0
        &mut self,
424
0
        output: &mut OutBuffer<'_, C>,
425
0
        _finished_frame: bool,
426
0
    ) -> io::Result<usize> {
427
0
        match &mut self.context {
428
0
            MaybeOwnedCCtx::Owned(x) => x.end_stream(output),
429
0
            MaybeOwnedCCtx::Borrowed(x) => x.end_stream(output),
430
        }
431
0
        .map_err(map_error_code)
432
0
    }
433
434
0
    fn reinit(&mut self) -> io::Result<()> {
435
0
        match &mut self.context {
436
0
            MaybeOwnedCCtx::Owned(x) => {
437
0
                x.reset(zstd_safe::ResetDirective::SessionOnly)
438
            }
439
0
            MaybeOwnedCCtx::Borrowed(x) => {
440
0
                x.reset(zstd_safe::ResetDirective::SessionOnly)
441
            }
442
        }
443
0
        .map_err(map_error_code)?;
444
0
        Ok(())
445
0
    }
446
}
447
448
enum MaybeOwnedCCtx<'a> {
449
    Owned(zstd_safe::CCtx<'a>),
450
    Borrowed(&'a mut zstd_safe::CCtx<'static>),
451
}
452
453
enum MaybeOwnedDCtx<'a> {
454
    Owned(zstd_safe::DCtx<'a>),
455
    Borrowed(&'a mut zstd_safe::DCtx<'static>),
456
}
457
458
#[cfg(test)]
459
mod tests {
460
461
    // This requires impl for [u8; N] which is currently behind a feature.
462
    #[cfg(feature = "arrays")]
463
    #[test]
464
    fn test_cycle() {
465
        use super::{Decoder, Encoder, InBuffer, Operation, OutBuffer};
466
467
        let mut encoder = Encoder::new(1).unwrap();
468
        let mut decoder = Decoder::new().unwrap();
469
470
        // Step 1: compress
471
        let mut input = InBuffer::around(b"AbcdefAbcdefabcdef");
472
473
        let mut output = [0u8; 128];
474
        let mut output = OutBuffer::around(&mut output);
475
476
        loop {
477
            encoder.run(&mut input, &mut output).unwrap();
478
479
            if input.pos == input.src.len() {
480
                break;
481
            }
482
        }
483
        encoder.finish(&mut output, true).unwrap();
484
485
        let initial_data = input.src;
486
487
        // Step 2: decompress
488
        let mut input = InBuffer::around(output.as_slice());
489
        let mut output = [0u8; 128];
490
        let mut output = OutBuffer::around(&mut output);
491
492
        loop {
493
            decoder.run(&mut input, &mut output).unwrap();
494
495
            if input.pos == input.src.len() {
496
                break;
497
            }
498
        }
499
500
        assert_eq!(initial_data, output.as_slice());
501
    }
502
}