Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pyzstd/__init__.py: 34%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1try:
2 # Import C implementation
3 from ._c import (
4 CParameter,
5 DParameter,
6 EndlessZstdDecompressor,
7 PYZSTD_CONFIG,
8 RichMemZstdCompressor,
9 Strategy,
10 ZstdCompressor,
11 ZstdDecompressor,
12 ZstdDict,
13 ZstdError,
14 ZstdFileReader as _ZstdFileReader,
15 ZstdFileWriter as _ZstdFileWriter,
16 _ZSTD_CStreamSizes,
17 _ZSTD_DStreamSizes,
18 _finalize_dict,
19 _train_dict,
20 compress_stream as _compress_stream,
21 compressionLevel_values,
22 decompress,
23 decompress_stream as _decompress_stream,
24 get_frame_info,
25 get_frame_size,
26 zstd_version,
27 zstd_version_info
28 )
29except ImportError:
30 try:
31 # Import CFFI implementation
32 from ._cffi import (
33 CParameter,
34 DParameter,
35 EndlessZstdDecompressor,
36 PYZSTD_CONFIG,
37 RichMemZstdCompressor,
38 Strategy,
39 ZstdCompressor,
40 ZstdDecompressor,
41 ZstdDict,
42 ZstdError,
43 ZstdFileReader as _ZstdFileReader,
44 ZstdFileWriter as _ZstdFileWriter,
45 _ZSTD_CStreamSizes,
46 _ZSTD_DStreamSizes,
47 _finalize_dict,
48 _train_dict,
49 compress_stream as _compress_stream,
50 compressionLevel_values,
51 decompress,
52 decompress_stream as _decompress_stream,
53 get_frame_info,
54 get_frame_size,
55 zstd_version,
56 zstd_version_info
57 )
58 except ImportError:
59 raise ImportError(
60 "\n\npyzstd module: Can't import compiled .so/.pyd file.\n"
61 "1, If pyzstd module is dynamically linked to zstd library: Make sure\n"
62 " not to remove zstd library, and the run-time zstd library's version\n"
63 " can't be lower than that at compile-time; On Windows, the directory\n"
64 " that has libzstd.dll should be added by os.add_dll_directory() function.\n"
65 "2, Please install pyzstd module through pip, to ensure that compiled\n"
66 " .so/.pyd file matches the architecture/OS/Python.\n")
67from ._zstdfile import ZstdFile, open
68from ._seekable_zstdfile import SeekableFormatError, SeekableZstdFile
70from functools import wraps
72try:
73 from warnings import deprecated
74except ImportError:
75 from typing_extensions import deprecated
78__version__ = '0.17.0'
80__doc__ = '''\
81Python bindings to Zstandard (zstd) compression library, the API style is
82similar to Python's bz2/lzma/zlib modules.
84Command line interface of this module: python -m pyzstd --help
86Documentation: https://pyzstd.readthedocs.io
87GitHub: https://github.com/Rogdham/pyzstd
88PyPI: https://pypi.org/project/pyzstd'''
90__all__ = ('ZstdCompressor', 'RichMemZstdCompressor',
91 'ZstdDecompressor', 'EndlessZstdDecompressor',
92 'CParameter', 'DParameter', 'Strategy', 'ZstdError',
93 'compress', 'richmem_compress', 'decompress',
94 'compress_stream', 'decompress_stream',
95 'ZstdDict', 'train_dict', 'finalize_dict',
96 'get_frame_info', 'get_frame_size', 'ZstdFile', 'open',
97 'zstd_version', 'zstd_version_info',
98 'zstd_support_multithread', 'compressionLevel_values',
99 'SeekableZstdFile', 'SeekableFormatError')
102zstd_support_multithread = (CParameter.nbWorkers.bounds() != (0, 0))
105def compress(data, level_or_option=None, zstd_dict=None):
106 """Compress a block of data, return a bytes object.
108 Compressing b'' will get an empty content frame (9 bytes or more).
110 Parameters
111 data: A bytes-like object, data to be compressed.
112 level_or_option: When it's an int object, it represents compression level.
113 When it's a dict object, it contains advanced compression
114 parameters.
115 zstd_dict: A ZstdDict object, pre-trained dictionary for compression.
116 """
117 comp = ZstdCompressor(level_or_option, zstd_dict)
118 return comp.compress(data, ZstdCompressor.FLUSH_FRAME)
121def richmem_compress(data, level_or_option=None, zstd_dict=None):
122 """Compress a block of data, return a bytes object.
124 Use rich memory mode, it's faster than compress() in some cases, but
125 allocates more memory.
127 Compressing b'' will get an empty content frame (9 bytes or more).
129 Parameters
130 data: A bytes-like object, data to be compressed.
131 level_or_option: When it's an int object, it represents compression level.
132 When it's a dict object, it contains advanced compression
133 parameters.
134 zstd_dict: A ZstdDict object, pre-trained dictionary for compression.
135 """
136 comp = RichMemZstdCompressor(level_or_option, zstd_dict)
137 return comp.compress(data)
140def _nbytes(dat):
141 if isinstance(dat, (bytes, bytearray)):
142 return len(dat)
143 with memoryview(dat) as mv:
144 return mv.nbytes
147def train_dict(samples, dict_size):
148 """Train a zstd dictionary, return a ZstdDict object.
150 Parameters
151 samples: An iterable of samples, a sample is a bytes-like object
152 represents a file.
153 dict_size: The dictionary's maximum size, in bytes.
154 """
155 # Check argument's type
156 if not isinstance(dict_size, int):
157 raise TypeError('dict_size argument should be an int object.')
159 # Prepare data
160 chunks = []
161 chunk_sizes = []
162 for chunk in samples:
163 chunks.append(chunk)
164 chunk_sizes.append(_nbytes(chunk))
166 chunks = b''.join(chunks)
167 if not chunks:
168 raise ValueError("The samples are empty content, can't train dictionary.")
170 # samples_bytes: samples be stored concatenated in a single flat buffer.
171 # samples_size_list: a list of each sample's size.
172 # dict_size: size of the dictionary, in bytes.
173 dict_content = _train_dict(chunks, chunk_sizes, dict_size)
175 return ZstdDict(dict_content)
178def finalize_dict(zstd_dict, samples, dict_size, level):
179 """Finalize a zstd dictionary, return a ZstdDict object.
181 Given a custom content as a basis for dictionary, and a set of samples,
182 finalize dictionary by adding headers and statistics according to the zstd
183 dictionary format.
185 You may compose an effective dictionary content by hand, which is used as
186 basis dictionary, and use some samples to finalize a dictionary. The basis
187 dictionary can be a "raw content" dictionary, see is_raw parameter in
188 ZstdDict.__init__ method.
190 Parameters
191 zstd_dict: A ZstdDict object, basis dictionary.
192 samples: An iterable of samples, a sample is a bytes-like object
193 represents a file.
194 dict_size: The dictionary's maximum size, in bytes.
195 level: The compression level expected to use in production. The
196 statistics for each compression level differ, so tuning the
197 dictionary for the compression level can help quite a bit.
198 """
199 if zstd_version_info < (1, 4, 5):
200 msg = ("This function only available when the underlying zstd "
201 "library's version is greater than or equal to v1.4.5, "
202 "the current underlying zstd library's version is v%s.") % zstd_version
203 raise NotImplementedError(msg)
205 # Check arguments' type
206 if not isinstance(zstd_dict, ZstdDict):
207 raise TypeError('zstd_dict argument should be a ZstdDict object.')
208 if not isinstance(dict_size, int):
209 raise TypeError('dict_size argument should be an int object.')
210 if not isinstance(level, int):
211 raise TypeError('level argument should be an int object.')
213 # Prepare data
214 chunks = []
215 chunk_sizes = []
216 for chunk in samples:
217 chunks.append(chunk)
218 chunk_sizes.append(_nbytes(chunk))
220 chunks = b''.join(chunks)
221 if not chunks:
222 raise ValueError("The samples are empty content, can't finalize dictionary.")
224 # custom_dict_bytes: existing dictionary.
225 # samples_bytes: samples be stored concatenated in a single flat buffer.
226 # samples_size_list: a list of each sample's size.
227 # dict_size: maximal size of the dictionary, in bytes.
228 # compression_level: compression level expected to use in production.
229 dict_content = _finalize_dict(zstd_dict.dict_content,
230 chunks, chunk_sizes,
231 dict_size, level)
233 return ZstdDict(dict_content)
236@wraps(_compress_stream)
237@deprecated("See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives to pyzstd.compress_stream")
238def compress_stream(*args, **kwargs):
239 return _compress_stream(*args, **kwargs)
241@wraps(_decompress_stream)
242@deprecated("See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives to pyzstd.decompress_stream")
243def decompress_stream(*args, **kwargs):
244 return _decompress_stream(*args, **kwargs)