Line | Count | Source |
1 | | // Copyright 2005 and onwards Google Inc. |
2 | | // |
3 | | // Redistribution and use in source and binary forms, with or without |
4 | | // modification, are permitted provided that the following conditions are |
5 | | // met: |
6 | | // |
7 | | // * Redistributions of source code must retain the above copyright |
8 | | // notice, this list of conditions and the following disclaimer. |
9 | | // * Redistributions in binary form must reproduce the above |
10 | | // copyright notice, this list of conditions and the following disclaimer |
11 | | // in the documentation and/or other materials provided with the |
12 | | // distribution. |
13 | | // * Neither the name of Google Inc. nor the names of its |
14 | | // contributors may be used to endorse or promote products derived from |
15 | | // this software without specific prior written permission. |
16 | | // |
17 | | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
18 | | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
19 | | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
20 | | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
21 | | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
22 | | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
23 | | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
24 | | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
25 | | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
26 | | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
27 | | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
28 | | // |
29 | | // A light-weight compression algorithm. It is designed for speed of |
30 | | // compression and decompression, rather than for the utmost in space |
31 | | // savings. |
32 | | // |
33 | | // For getting better compression ratios when you are compressing data |
34 | | // with long repeated sequences or compressing data that is similar to |
35 | | // other data, while still compressing fast, you might look at first |
36 | | // using BMDiff and then compressing the output of BMDiff with |
37 | | // Snappy. |
38 | | |
39 | | #ifndef THIRD_PARTY_SNAPPY_SNAPPY_H__ |
40 | | #define THIRD_PARTY_SNAPPY_SNAPPY_H__ |
41 | | |
42 | | #include <stddef.h> |
43 | | #include <stdint.h> |
44 | | |
45 | | #include <string> |
46 | | |
47 | | #include "snappy-stubs-public.h" |
48 | | |
49 | | namespace snappy { |
50 | | class Source; |
51 | | class Sink; |
52 | | |
53 | | namespace internal { |
54 | | class WorkingMemory; |
55 | | } // end namespace internal |
56 | | |
57 | | struct CompressionOptions { |
58 | | // Compression level. |
59 | | // Level 1 is the fastest |
60 | | // Level 2 is a little slower but provides better compression. Level 2 is |
61 | | // **EXPERIMENTAL** for the time being. It might happen that we decide to |
62 | | // fall back to level 1 in the future. |
63 | | // Levels 3+ are currently not supported. We plan to support levels up to |
64 | | // 9 in the future. |
65 | | // If you played with other compression algorithms, level 1 is equivalent to |
66 | | // fast mode (level 1) of LZ4, level 2 is equivalent to LZ4's level 2 mode |
67 | | // and compresses somewhere around zstd:-3 and zstd:-2 but generally with |
68 | | // faster decompression speeds than snappy:1 and zstd:-3. |
69 | | int level = DefaultCompressionLevel(); |
70 | | |
71 | 0 | constexpr CompressionOptions() = default; |
72 | | constexpr CompressionOptions(int compression_level) |
73 | 5.29k | : level(compression_level) {} |
74 | | |
75 | 2.64k | static constexpr int MinCompressionLevel() { return 1; } |
76 | 7.94k | static constexpr int MaxCompressionLevel() { return 2; } |
77 | 0 | static constexpr int DefaultCompressionLevel() { return 1; } |
78 | | }; |
79 | | |
80 | | // Scratch memory for compression, reusable across compressions. Callers that |
81 | | // compress frequently, or that need to avoid large heap allocations can |
82 | | // allocate a CompressionContext once and pass it to Compress()/RawCompress() |
83 | | // to reuse the working memory across calls. |
84 | | // |
85 | | // The context is sized for the largest block and works for inputs of any |
86 | | // size. A context may be used by any number of sequential compressions, but |
87 | | // must not be used from multiple threads concurrently. A moved-from context |
88 | | // may only be destroyed or assigned to. |
89 | | class CompressionContext { |
90 | | public: |
91 | | // Allocates the working memory on the heap. |
92 | | CompressionContext(); |
93 | | |
94 | | // Constructs a context whose working memory is placed in the |
95 | | // caller-provided "workspace" instead of being heap-allocated; the |
96 | | // library performs no allocation at all. |
97 | | // |
98 | | // REQUIRES: "workspace" points to at least "workspace_size" bytes with |
99 | | // "workspace_size >= WorkspaceSize()", is suitably aligned for any |
100 | | // object type (as if returned by malloc), and outlives "*this". |
101 | | CompressionContext(void* workspace, size_t workspace_size); |
102 | | |
103 | | ~CompressionContext(); |
104 | | |
105 | | CompressionContext(CompressionContext&& other) noexcept; |
106 | | CompressionContext& operator=(CompressionContext&& other) noexcept; |
107 | | |
108 | | CompressionContext(const CompressionContext&) = delete; |
109 | | CompressionContext& operator=(const CompressionContext&) = delete; |
110 | | |
111 | | // The workspace size required by the non-allocating constructor above. |
112 | | static size_t WorkspaceSize(); |
113 | | |
114 | | private: |
115 | | friend size_t Compress(Source* reader, Sink* writer, |
116 | | CompressionOptions options, CompressionContext* ctx); |
117 | | |
118 | | // Destroys the working memory as appropriate for how it was created |
119 | | // (delete if heap-allocated, in-place destruction if placement-constructed |
120 | | // in a caller-provided workspace). |
121 | | void Reset(); |
122 | | |
123 | | internal::WorkingMemory* working_memory_; |
124 | | bool owns_working_memory_; |
125 | | }; |
126 | | |
127 | | // ------------------------------------------------------------------------ |
128 | | // Generic compression/decompression routines. |
129 | | // ------------------------------------------------------------------------ |
130 | | |
131 | | // Compress the bytes read from "*reader" and append to "*writer". Return the |
132 | | // number of bytes written. |
133 | | // First version is to preserve ABI. |
134 | | size_t Compress(Source* reader, Sink* writer); |
135 | | size_t Compress(Source* reader, Sink* writer, |
136 | | CompressionOptions options); |
137 | | |
138 | | // Same as the above, but uses the working memory of "*ctx" instead of |
139 | | // allocating it internally. See CompressionContext. |
140 | | size_t Compress(Source* reader, Sink* writer, CompressionOptions options, |
141 | | CompressionContext* ctx); |
142 | | |
143 | | // Find the uncompressed length of the given stream, as given by the header. |
144 | | // Note that the true length could deviate from this; the stream could e.g. |
145 | | // be truncated. |
146 | | // |
147 | | // Also note that this leaves "*source" in a state that is unsuitable for |
148 | | // further operations, such as RawUncompress(). You will need to rewind |
149 | | // or recreate the source yourself before attempting any further calls. |
150 | | bool GetUncompressedLength(Source* source, uint32_t* result); |
151 | | |
152 | | // ------------------------------------------------------------------------ |
153 | | // Higher-level string based routines (should be sufficient for most users) |
154 | | // ------------------------------------------------------------------------ |
155 | | |
156 | | // Sets "*compressed" to the compressed version of "input[0..input_length-1]". |
157 | | // Original contents of *compressed are lost. |
158 | | // |
159 | | // REQUIRES: "input[]" is not an alias of "*compressed". |
160 | | // First version is to preserve ABI. |
161 | | size_t Compress(const char* input, size_t input_length, |
162 | | std::string* compressed); |
163 | | size_t Compress(const char* input, size_t input_length, |
164 | | std::string* compressed, CompressionOptions options); |
165 | | |
166 | | // Same as `Compress` above but taking an `iovec` array as input. Note that |
167 | | // this function preprocesses the inputs to compute the sum of |
168 | | // `iov[0..iov_cnt-1].iov_len` before reading. To avoid this, use |
169 | | // `RawCompressFromIOVec` below. |
170 | | // First version is to preserve ABI. |
171 | | size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt, |
172 | | std::string* compressed); |
173 | | size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt, |
174 | | std::string* compressed, |
175 | | CompressionOptions options); |
176 | | |
177 | | // Decompresses "compressed[0..compressed_length-1]" to "*uncompressed". |
178 | | // Original contents of "*uncompressed" are lost. |
179 | | // |
180 | | // REQUIRES: "compressed[]" is not an alias of "*uncompressed". |
181 | | // |
182 | | // returns false if the message is corrupted and could not be decompressed |
183 | | bool Uncompress(const char* compressed, size_t compressed_length, |
184 | | std::string* uncompressed); |
185 | | |
186 | | // Decompresses "compressed" to "*uncompressed". |
187 | | // |
188 | | // returns false if the message is corrupted and could not be decompressed |
189 | | bool Uncompress(Source* compressed, Sink* uncompressed); |
190 | | |
191 | | // This routine uncompresses as much of the "compressed" as possible |
192 | | // into sink. It returns the number of valid bytes added to sink |
193 | | // (extra invalid bytes may have been added due to errors; the caller |
194 | | // should ignore those). The emitted data typically has length |
195 | | // GetUncompressedLength(), but may be shorter if an error is |
196 | | // encountered. |
197 | | size_t UncompressAsMuchAsPossible(Source* compressed, Sink* uncompressed); |
198 | | |
199 | | // ------------------------------------------------------------------------ |
200 | | // Lower-level character array based routines. May be useful for |
201 | | // efficiency reasons in certain circumstances. |
202 | | // ------------------------------------------------------------------------ |
203 | | |
204 | | // REQUIRES: "compressed" must point to an area of memory that is at |
205 | | // least "MaxCompressedLength(input_length)" bytes in length. |
206 | | // |
207 | | // Takes the data stored in "input[0..input_length]" and stores |
208 | | // it in the array pointed to by "compressed". |
209 | | // |
210 | | // "*compressed_length" is set to the length of the compressed output. |
211 | | // |
212 | | // Example: |
213 | | // char* output = new char[snappy::MaxCompressedLength(input_length)]; |
214 | | // size_t output_length; |
215 | | // RawCompress(input, input_length, output, &output_length); |
216 | | // ... Process(output, output_length) ... |
217 | | // delete [] output; |
218 | | // First version is to preserve ABI. |
219 | | void RawCompress(const char* input, size_t input_length, char* compressed, |
220 | | size_t* compressed_length); |
221 | | void RawCompress(const char* input, size_t input_length, char* compressed, |
222 | | size_t* compressed_length, CompressionOptions options); |
223 | | |
224 | | // Same as the above, but uses the working memory of "*ctx" instead of |
225 | | // allocating it internally. See CompressionContext. |
226 | | void RawCompress(const char* input, size_t input_length, char* compressed, |
227 | | size_t* compressed_length, CompressionOptions options, |
228 | | CompressionContext* ctx); |
229 | | |
230 | | // Same as `RawCompress` above but taking an `iovec` array as input. Note that |
231 | | // `uncompressed_length` is the total number of bytes to be read from the |
232 | | // elements of `iov` (_not_ the number of elements in `iov`). |
233 | | // First version is to preserve ABI. |
234 | | void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length, |
235 | | char* compressed, size_t* compressed_length); |
236 | | void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length, |
237 | | char* compressed, size_t* compressed_length, |
238 | | CompressionOptions options); |
239 | | |
240 | | // Given data in "compressed[0..compressed_length-1]" generated by |
241 | | // calling the Snappy::Compress routine, this routine |
242 | | // stores the uncompressed data to |
243 | | // uncompressed[0..GetUncompressedLength(compressed)-1] |
244 | | // returns false if the message is corrupted and could not be decrypted |
245 | | bool RawUncompress(const char* compressed, size_t compressed_length, |
246 | | char* uncompressed); |
247 | | |
248 | | // Given data from the byte source 'compressed' generated by calling |
249 | | // the Snappy::Compress routine, this routine stores the uncompressed |
250 | | // data to |
251 | | // uncompressed[0..GetUncompressedLength(compressed,compressed_length)-1] |
252 | | // returns false if the message is corrupted and could not be decrypted |
253 | | bool RawUncompress(Source* compressed, char* uncompressed); |
254 | | |
255 | | // Given data in "compressed[0..compressed_length-1]" generated by |
256 | | // calling the Snappy::Compress routine, this routine |
257 | | // stores the uncompressed data to the iovec "iov". The number of physical |
258 | | // buffers in "iov" is given by iov_cnt and their cumulative size |
259 | | // must be at least GetUncompressedLength(compressed). The individual buffers |
260 | | // in "iov" must not overlap with each other. |
261 | | // |
262 | | // returns false if the message is corrupted and could not be decrypted |
263 | | bool RawUncompressToIOVec(const char* compressed, size_t compressed_length, |
264 | | const struct iovec* iov, size_t iov_cnt); |
265 | | |
266 | | // Given data from the byte source 'compressed' generated by calling |
267 | | // the Snappy::Compress routine, this routine stores the uncompressed |
268 | | // data to the iovec "iov". The number of physical |
269 | | // buffers in "iov" is given by iov_cnt and their cumulative size |
270 | | // must be at least GetUncompressedLength(compressed). The individual buffers |
271 | | // in "iov" must not overlap with each other. |
272 | | // |
273 | | // returns false if the message is corrupted and could not be decrypted |
274 | | bool RawUncompressToIOVec(Source* compressed, const struct iovec* iov, |
275 | | size_t iov_cnt); |
276 | | |
277 | | // Returns the maximal size of the compressed representation of |
278 | | // input data that is "source_bytes" bytes in length; |
279 | | size_t MaxCompressedLength(size_t source_bytes); |
280 | | |
281 | | // REQUIRES: "compressed[]" was produced by RawCompress() or Compress() |
282 | | // Returns true and stores the length of the uncompressed data in |
283 | | // *result normally. Returns false on parsing error. |
284 | | // This operation takes O(1) time. |
285 | | bool GetUncompressedLength(const char* compressed, size_t compressed_length, |
286 | | size_t* result); |
287 | | |
288 | | // Returns true iff the contents of "compressed[]" can be uncompressed |
289 | | // successfully. Does not return the uncompressed data. Takes |
290 | | // time proportional to compressed_length, but is usually at least |
291 | | // a factor of four faster than actual decompression. |
292 | | bool IsValidCompressedBuffer(const char* compressed, |
293 | | size_t compressed_length); |
294 | | |
295 | | // Returns true iff the contents of "compressed" can be uncompressed |
296 | | // successfully. Does not return the uncompressed data. Takes |
297 | | // time proportional to *compressed length, but is usually at least |
298 | | // a factor of four faster than actual decompression. |
299 | | // On success, consumes all of *compressed. On failure, consumes an |
300 | | // unspecified prefix of *compressed. |
301 | | bool IsValidCompressed(Source* compressed); |
302 | | |
303 | | // The size of a compression block. Note that many parts of the compression |
304 | | // code assumes that kBlockSize <= 65536; in particular, the hash table |
305 | | // can only store 16-bit offsets, and EmitCopy() also assumes the offset |
306 | | // is 65535 bytes or less. Note also that if you change this, it will |
307 | | // affect the framing format (see framing_format.txt). |
308 | | // |
309 | | // Note that there might be older data around that is compressed with larger |
310 | | // block sizes, so the decompression code should not rely on the |
311 | | // non-existence of long backreferences. |
312 | | static constexpr int kBlockLog = 16; |
313 | | static constexpr size_t kBlockSize = 1 << kBlockLog; |
314 | | |
315 | | static constexpr int kMinHashTableBits = 8; |
316 | | static constexpr size_t kMinHashTableSize = 1 << kMinHashTableBits; |
317 | | |
318 | | static constexpr int kMaxHashTableBits = 15; |
319 | | static constexpr size_t kMaxHashTableSize = 1 << kMaxHashTableBits; |
320 | | } // end namespace snappy |
321 | | |
322 | | #endif // THIRD_PARTY_SNAPPY_SNAPPY_H__ |