/rust/registry/src/index.crates.io-1949cf8c6b5b557f/lz4-1.28.1/src/block/mod.rs
Line | Count | Source |
1 | | //! This module provides access to the block mode functions of the lz4 C library. |
2 | | //! It somehow resembles the [Python-lz4](http://python-lz4.readthedocs.io/en/stable/lz4.block.html) api, |
3 | | //! but using Rust's Option type, the function parameters have been a little simplified. |
4 | | //! As does python-lz4, this module supports prepending the compressed buffer with a u32 value |
5 | | //! representing the size of the original, uncompressed data. |
6 | | //! |
7 | | //! # Examples |
8 | | //! ``` |
9 | | //! |
10 | | //! use lz4::block::{compress,decompress}; |
11 | | //! |
12 | | //! let v = vec![0u8; 1024]; |
13 | | //! |
14 | | //! let comp_with_prefix = compress(&v, None, true).unwrap(); |
15 | | //! let comp_wo_prefix = compress(&v, None, false).unwrap(); |
16 | | //! |
17 | | //! assert_eq!(v, decompress(&comp_with_prefix, None).unwrap()); |
18 | | //! assert_eq!(v, decompress(&comp_wo_prefix, Some(1024)).unwrap()); |
19 | | //! ``` |
20 | | |
21 | | use super::c_char; |
22 | | use super::liblz4::*; |
23 | | use std::io::{Error, ErrorKind, Result}; |
24 | | |
25 | | /// Represents the compression mode do be used. |
26 | | #[derive(Clone, Copy, Debug)] |
27 | | pub enum CompressionMode { |
28 | | /// High compression with compression parameter |
29 | | HIGHCOMPRESSION(i32), |
30 | | /// Fast compression with acceleration paramet |
31 | | FAST(i32), |
32 | | /// Default compression |
33 | | DEFAULT, |
34 | | } |
35 | | |
36 | | impl Default for CompressionMode { |
37 | 0 | fn default() -> Self { |
38 | 0 | CompressionMode::DEFAULT |
39 | 0 | } |
40 | | } |
41 | | |
42 | | /// Returns the size of the buffer that is guaranteed to hold the result of |
43 | | /// compressing `uncompressed_size` bytes of in data. Returns std::io::Error |
44 | | /// with ErrorKind::InvalidInput if input data is too long to be compressed by lz4. |
45 | 0 | pub fn compress_bound(uncompressed_size: usize) -> Result<usize> { |
46 | | // 0 iff src too large |
47 | 0 | let compress_bound: i32 = unsafe { LZ4_compressBound(uncompressed_size as i32) }; |
48 | | |
49 | 0 | if uncompressed_size > (i32::max_value() as usize) || compress_bound <= 0 { |
50 | 0 | return Err(Error::new( |
51 | 0 | ErrorKind::InvalidInput, |
52 | 0 | "Compression input too long.", |
53 | 0 | )); |
54 | 0 | } |
55 | | |
56 | 0 | Ok(compress_bound as usize) |
57 | 0 | } |
58 | | |
59 | | /// Compresses the full src buffer using the specified CompressionMode, where None and Some(Default) |
60 | | /// are treated equally. If prepend_size is set, the source length will be prepended to the output |
61 | | /// buffer. |
62 | | /// |
63 | | /// |
64 | | /// # Errors |
65 | | /// Returns std::io::Error with ErrorKind::InvalidInput if the src buffer is too long. |
66 | | /// Returns std::io::Error with ErrorKind::Other if the compression failed inside the C library. If |
67 | | /// this happens, the C api was not able to provide more information about the cause. |
68 | 0 | pub fn compress(src: &[u8], mode: Option<CompressionMode>, prepend_size: bool) -> Result<Vec<u8>> { |
69 | | // 0 iff src too large |
70 | 0 | let compress_bound: i32 = unsafe { LZ4_compressBound(src.len() as i32) }; |
71 | | |
72 | 0 | if src.len() > (i32::max_value() as usize) || compress_bound <= 0 { |
73 | 0 | return Err(Error::new( |
74 | 0 | ErrorKind::InvalidInput, |
75 | 0 | "Compression input too long.", |
76 | 0 | )); |
77 | 0 | } |
78 | | |
79 | 0 | let mut compressed: Vec<u8> = vec![ |
80 | | 0; |
81 | 0 | (if prepend_size { |
82 | 0 | compress_bound + 4 |
83 | | } else { |
84 | 0 | compress_bound |
85 | | }) as usize |
86 | | ]; |
87 | | |
88 | 0 | let dec_size = compress_to_buffer(src, mode, prepend_size, &mut compressed)?; |
89 | 0 | compressed.truncate(dec_size as usize); |
90 | 0 | Ok(compressed) |
91 | 0 | } |
92 | | |
93 | | /// Compresses the full `src` buffer using the specified CompressionMode, where None and Some(Default) |
94 | | /// are treated equally, writing compressed bytes to `buffer`. |
95 | | /// |
96 | | /// # Errors |
97 | | /// Returns std::io::Error with ErrorKind::InvalidInput if the src buffer is too long. |
98 | | /// The buffer cannot be larger than `i32::MAX`. |
99 | | /// Returns std::io::Error with ErrorKind::Other if the compression data does not fit in `buffer`. |
100 | 0 | pub fn compress_to_buffer( |
101 | 0 | src: &[u8], |
102 | 0 | mode: Option<CompressionMode>, |
103 | 0 | prepend_size: bool, |
104 | 0 | buffer: &mut [u8], |
105 | 0 | ) -> Result<usize> { |
106 | | // check that src isn't too big for lz4 |
107 | 0 | let max_len: i32 = unsafe { LZ4_compressBound(src.len() as i32) }; |
108 | | |
109 | 0 | if src.len() > (i32::max_value() as usize) || max_len <= 0 { |
110 | 0 | return Err(Error::new( |
111 | 0 | ErrorKind::InvalidInput, |
112 | 0 | "Compression input too long.", |
113 | 0 | )); |
114 | 0 | } |
115 | | |
116 | | let dec_size; |
117 | | { |
118 | 0 | let dst_buf = if prepend_size { |
119 | 0 | let size = src.len() as u32; |
120 | 0 | buffer[0] = size as u8; |
121 | 0 | buffer[1] = (size >> 8) as u8; |
122 | 0 | buffer[2] = (size >> 16) as u8; |
123 | 0 | buffer[3] = (size >> 24) as u8; |
124 | 0 | &mut buffer[4..] |
125 | | } else { |
126 | 0 | buffer |
127 | | }; |
128 | | |
129 | 0 | let buf_len = dst_buf.len() as i32; |
130 | | |
131 | 0 | dec_size = match mode { |
132 | 0 | Some(CompressionMode::HIGHCOMPRESSION(level)) => unsafe { |
133 | 0 | LZ4_compress_HC( |
134 | 0 | src.as_ptr() as *const c_char, |
135 | 0 | dst_buf.as_mut_ptr() as *mut c_char, |
136 | 0 | src.len() as i32, |
137 | 0 | buf_len, |
138 | 0 | level, |
139 | | ) |
140 | | }, |
141 | 0 | Some(CompressionMode::FAST(accel)) => unsafe { |
142 | 0 | LZ4_compress_fast( |
143 | 0 | src.as_ptr() as *const c_char, |
144 | 0 | dst_buf.as_mut_ptr() as *mut c_char, |
145 | 0 | src.len() as i32, |
146 | 0 | buf_len, |
147 | 0 | accel, |
148 | | ) |
149 | | }, |
150 | | _ => unsafe { |
151 | 0 | LZ4_compress_default( |
152 | 0 | src.as_ptr() as *const c_char, |
153 | 0 | dst_buf.as_mut_ptr() as *mut c_char, |
154 | 0 | src.len() as i32, |
155 | 0 | buf_len, |
156 | | ) |
157 | | }, |
158 | | }; |
159 | | } |
160 | 0 | if dec_size <= 0 { |
161 | 0 | return Err(Error::new(ErrorKind::Other, "Compression failed")); |
162 | 0 | } |
163 | | |
164 | 0 | let written_size = if prepend_size { dec_size + 4 } else { dec_size }; |
165 | | |
166 | 0 | Ok(written_size as usize) |
167 | 0 | } |
168 | | |
169 | 0 | fn get_decompressed_size(src: &[u8], uncompressed_size: Option<i32>) -> Result<usize> { |
170 | | let size; |
171 | | |
172 | 0 | if let Some(s) = uncompressed_size { |
173 | 0 | size = s; |
174 | 0 | } else { |
175 | 0 | if src.len() < 4 { |
176 | 0 | return Err(Error::new( |
177 | 0 | ErrorKind::InvalidInput, |
178 | 0 | "Source buffer must at least contain size prefix.", |
179 | 0 | )); |
180 | 0 | } |
181 | 0 | size = |
182 | 0 | (src[0] as i32) | (src[1] as i32) << 8 | (src[2] as i32) << 16 | (src[3] as i32) << 24; |
183 | | } |
184 | | |
185 | 0 | if size < 0 { |
186 | 0 | return Err(Error::new( |
187 | 0 | ErrorKind::InvalidInput, |
188 | 0 | if uncompressed_size.is_some() { |
189 | 0 | "Size parameter must not be negative." |
190 | | } else { |
191 | 0 | "Parsed size prefix in buffer must not be negative." |
192 | | }, |
193 | | )); |
194 | 0 | } |
195 | | |
196 | 0 | if unsafe { LZ4_compressBound(size) } <= 0 { |
197 | 0 | return Err(Error::new( |
198 | 0 | ErrorKind::InvalidInput, |
199 | 0 | "Given size parameter is too big", |
200 | 0 | )); |
201 | 0 | } |
202 | | |
203 | 0 | Ok(size as usize) |
204 | 0 | } |
205 | | |
206 | | /// Decompresses the src buffer. If uncompressed_size is None, the source length will be read from |
207 | | /// the start of the input buffer. |
208 | | /// |
209 | | /// |
210 | | /// # Errors |
211 | | /// Returns std::io::Error with ErrorKind::InvalidInput if the src buffer is too short, the |
212 | | /// provided (or parsed) uncompressed_size is to large or negative. |
213 | | /// Returns std::io::Error with ErrorKind::InvalidData if the decompression failed inside the C |
214 | | /// library. This is most likely due to malformed input. |
215 | | /// |
216 | 0 | pub fn decompress(src: &[u8], uncompressed_size: Option<i32>) -> Result<Vec<u8>> { |
217 | 0 | let size = get_decompressed_size(src, uncompressed_size)?; |
218 | | |
219 | 0 | let mut buffer = vec![0u8; size]; |
220 | | |
221 | 0 | let sz = decompress_to_buffer(src, uncompressed_size, &mut buffer)?; |
222 | 0 | buffer.truncate(sz); |
223 | 0 | Ok(buffer) |
224 | 0 | } |
225 | | |
226 | 0 | pub fn decompress_to_buffer( |
227 | 0 | mut src: &[u8], |
228 | 0 | uncompressed_size: Option<i32>, |
229 | 0 | buffer: &mut [u8], |
230 | 0 | ) -> Result<usize> { |
231 | | let size; |
232 | | |
233 | 0 | if let Some(s) = uncompressed_size { |
234 | 0 | size = s; |
235 | 0 | } else { |
236 | 0 | if src.len() < 4 { |
237 | 0 | return Err(Error::new( |
238 | 0 | ErrorKind::InvalidInput, |
239 | 0 | "Source buffer must at least contain size prefix.", |
240 | 0 | )); |
241 | 0 | } |
242 | 0 | size = |
243 | 0 | (src[0] as i32) | (src[1] as i32) << 8 | (src[2] as i32) << 16 | (src[3] as i32) << 24; |
244 | | |
245 | 0 | src = &src[4..]; |
246 | | } |
247 | | |
248 | 0 | if size < 0 { |
249 | 0 | return Err(Error::new( |
250 | 0 | ErrorKind::InvalidInput, |
251 | 0 | if uncompressed_size.is_some() { |
252 | 0 | "Size parameter must not be negative." |
253 | | } else { |
254 | 0 | "Parsed size prefix in buffer must not be negative." |
255 | | }, |
256 | | )); |
257 | 0 | } |
258 | | |
259 | 0 | if unsafe { LZ4_compressBound(size) } <= 0 { |
260 | 0 | return Err(Error::new( |
261 | 0 | ErrorKind::InvalidInput, |
262 | 0 | "Given size parameter is too big", |
263 | 0 | )); |
264 | 0 | } |
265 | | |
266 | 0 | if size as usize > buffer.len() { |
267 | 0 | return Err(Error::new( |
268 | 0 | ErrorKind::InvalidInput, |
269 | 0 | "buffer isn't large enough to hold decompressed data", |
270 | 0 | )); |
271 | 0 | } |
272 | | |
273 | 0 | let dec_bytes = unsafe { |
274 | 0 | LZ4_decompress_safe( |
275 | 0 | src.as_ptr() as *const c_char, |
276 | 0 | buffer.as_mut_ptr() as *mut c_char, |
277 | 0 | src.len() as i32, |
278 | 0 | size, |
279 | | ) |
280 | | }; |
281 | | |
282 | 0 | if dec_bytes < 0 { |
283 | 0 | return Err(Error::new( |
284 | 0 | ErrorKind::InvalidData, |
285 | 0 | "Decompression failed. Input invalid or too long?", |
286 | 0 | )); |
287 | 0 | } |
288 | | |
289 | 0 | Ok(dec_bytes as usize) |
290 | 0 | } |
291 | | |
292 | | #[cfg(test)] |
293 | | mod test { |
294 | | use crate::block::{compress, decompress, decompress_to_buffer, CompressionMode}; |
295 | | |
296 | | use super::compress_to_buffer; |
297 | | |
298 | | /// This test will fail unless the buffer created by decompress is correctly truncated |
299 | | #[test] |
300 | | fn decompress_truncate_test() { |
301 | | let src = "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111".as_bytes(); |
302 | | let rs_compressed = compress(src, None, false).unwrap(); |
303 | | let rs_compressed_rs_uncompressed = |
304 | | decompress(&rs_compressed, Some((src.len() as i32) * 256)).unwrap(); |
305 | | |
306 | | // compare the uncompressed result from rust |
307 | | assert_eq!(rs_compressed_rs_uncompressed, src,); |
308 | | } |
309 | | |
310 | | #[test] |
311 | | fn test_compression_without_prefix() { |
312 | | let size = 65536; |
313 | | let mut to_compress = Vec::with_capacity(size); |
314 | | for i in 0..size { |
315 | | to_compress.push(i as u8); |
316 | | } |
317 | | let mut v: Vec<Vec<u8>> = vec![]; |
318 | | for i in 1..100 { |
319 | | v.push(compress(&to_compress, Some(CompressionMode::FAST(i)), false).unwrap()); |
320 | | } |
321 | | |
322 | | // 12 is max high compression parameter |
323 | | for i in 1..12 { |
324 | | v.push( |
325 | | compress( |
326 | | &to_compress, |
327 | | Some(CompressionMode::HIGHCOMPRESSION(i)), |
328 | | false, |
329 | | ) |
330 | | .unwrap(), |
331 | | ); |
332 | | } |
333 | | |
334 | | v.push(compress(&to_compress, None, false).unwrap()); |
335 | | |
336 | | for val in v { |
337 | | assert_eq!( |
338 | | decompress(&val, Some(to_compress.len() as i32)).unwrap(), |
339 | | to_compress |
340 | | ); |
341 | | } |
342 | | } |
343 | | |
344 | | #[test] |
345 | | fn test_compression_with_prefix() { |
346 | | let size = 65536; |
347 | | let mut to_compress = Vec::with_capacity(size); |
348 | | for i in 0..size { |
349 | | to_compress.push(i as u8); |
350 | | } |
351 | | let mut v: Vec<Vec<u8>> = vec![]; |
352 | | for i in 1..100 { |
353 | | v.push(compress(&to_compress, Some(CompressionMode::FAST(i)), true).unwrap()); |
354 | | } |
355 | | |
356 | | // 12 is max high compression parameter |
357 | | for i in 1..12 { |
358 | | v.push( |
359 | | compress( |
360 | | &to_compress, |
361 | | Some(CompressionMode::HIGHCOMPRESSION(i)), |
362 | | true, |
363 | | ) |
364 | | .unwrap(), |
365 | | ); |
366 | | } |
367 | | |
368 | | v.push(compress(&to_compress, None, true).unwrap()); |
369 | | |
370 | | for val in v { |
371 | | assert_eq!(decompress(&val, None).unwrap(), to_compress); |
372 | | } |
373 | | } |
374 | | |
375 | | #[test] |
376 | | fn test_compress_to_buffer() { |
377 | | let data = b"qfn3489fqASFqvegrwe$%344thI,..kmTMN3 g{P}wefwf2fv2443ef3443RT[]rete$7-80956GRWbefvw@fVGrwGB24tggrm%&*I@!"; |
378 | | |
379 | | // test what happens when not enough space is available. |
380 | | let mut small_buf = vec![0; 5]; |
381 | | let r = compress_to_buffer(&data[..], None, true, &mut small_buf); |
382 | | assert!(r.is_err()); |
383 | | |
384 | | let mut big_buf = vec![0; 1000]; |
385 | | let r = compress_to_buffer(&data[..], None, true, &mut big_buf).unwrap(); |
386 | | assert_eq!(big_buf[r], 0); |
387 | | |
388 | | let mut dec_buf = vec![0u8; data.len() + 1]; |
389 | | let dec_bytes = decompress_to_buffer(&big_buf[..r], None, &mut dec_buf).unwrap(); |
390 | | assert_eq!(&dec_buf[..dec_bytes], &data[..]); |
391 | | } |
392 | | #[test] |
393 | | fn test_decompression_with_prefix() { |
394 | | let compressed: [u8; 250] = [ |
395 | | 0, 188, 0, 0, 255, 32, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 116, 101, 115, |
396 | | 116, 32, 115, 116, 114, 105, 110, 103, 32, 99, 111, 109, 112, 114, 101, 115, 115, 101, |
397 | | 100, 32, 98, 121, 32, 112, 121, 116, 104, 111, 110, 45, 108, 122, 52, 32, 47, 0, 255, |
398 | | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
399 | | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
400 | | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
401 | | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
402 | | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
403 | | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
404 | | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
405 | | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
406 | | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
407 | | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
408 | | 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, |
409 | | 117, 80, 45, 108, 122, 52, 32, |
410 | | ]; |
411 | | |
412 | | let mut reference: String = String::new(); |
413 | | for _ in 0..1024 { |
414 | | reference += "this is a test string compressed by python-lz4 "; |
415 | | } |
416 | | |
417 | | assert_eq!(decompress(&compressed, None).unwrap(), reference.as_bytes()) |
418 | | } |
419 | | |
420 | | #[test] |
421 | | fn test_empty_compress() { |
422 | | use crate::block::{compress, decompress}; |
423 | | let v = vec![0u8; 0]; |
424 | | let comp_with_prefix = compress(&v, None, true).unwrap(); |
425 | | dbg!(&comp_with_prefix); |
426 | | assert_eq!(v, decompress(&comp_with_prefix, None).unwrap()); |
427 | | } |
428 | | } |