Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pyzstd/__init__.py: 39%

1try: 

2 # Import C implementation 

3 from ._c import ( 

4 CParameter, 

5 DParameter, 

6 EndlessZstdDecompressor, 

7 PYZSTD_CONFIG, 

8 RichMemZstdCompressor as _RichMemZstdCompressor, 

9 Strategy, 

10 ZstdCompressor, 

11 ZstdDecompressor, 

12 ZstdDict, 

13 ZstdError, 

14 ZstdFileReader as _ZstdFileReader, 

15 ZstdFileWriter as _ZstdFileWriter, 

16 _ZSTD_CStreamSizes, 

17 _ZSTD_DStreamSizes, 

18 _finalize_dict, 

19 _train_dict, 

20 compress_stream as _compress_stream, 

21 compressionLevel_values, 

22 decompress, 

23 decompress_stream as _decompress_stream, 

24 get_frame_info, 

25 get_frame_size, 

26 zstd_version, 

27 zstd_version_info 

28 ) 

29except ImportError: 

30 try: 

31 # Import CFFI implementation 

32 from ._cffi import ( 

33 CParameter, 

34 DParameter, 

35 EndlessZstdDecompressor, 

36 PYZSTD_CONFIG, 

37 RichMemZstdCompressor as _RichMemZstdCompressor, 

38 Strategy, 

39 ZstdCompressor, 

40 ZstdDecompressor, 

41 ZstdDict, 

42 ZstdError, 

43 ZstdFileReader as _ZstdFileReader, 

44 ZstdFileWriter as _ZstdFileWriter, 

45 _ZSTD_CStreamSizes, 

46 _ZSTD_DStreamSizes, 

47 _finalize_dict, 

48 _train_dict, 

49 compress_stream as _compress_stream, 

50 compressionLevel_values, 

51 decompress, 

52 decompress_stream as _decompress_stream, 

53 get_frame_info, 

54 get_frame_size, 

55 zstd_version, 

56 zstd_version_info 

57 ) 

58 except ImportError: 

59 raise ImportError( 

60 "\n\npyzstd module: Can't import compiled .so/.pyd file.\n" 

61 "1, If pyzstd module is dynamically linked to zstd library: Make sure\n" 

62 " not to remove zstd library, and the run-time zstd library's version\n" 

63 " can't be lower than that at compile-time; On Windows, the directory\n" 

64 " that has libzstd.dll should be added by os.add_dll_directory() function.\n" 

65 "2, Please install pyzstd module through pip, to ensure that compiled\n" 

66 " .so/.pyd file matches the architecture/OS/Python.\n") 

67from ._zstdfile import ZstdFile, open 

68from ._seekable_zstdfile import SeekableFormatError, SeekableZstdFile 

69 

70from functools import wraps 

71 

72try: 

73 from warnings import deprecated 

74except ImportError: 

75 from typing_extensions import deprecated 

76 

77 

78__version__ = '0.18.0' 

79 

80__doc__ = '''\ 

81Python bindings to Zstandard (zstd) compression library, the API style is 

82similar to Python's bz2/lzma/zlib modules. 

83 

84Command line interface of this module: python -m pyzstd --help 

85 

86Documentation: https://pyzstd.readthedocs.io 

87GitHub: https://github.com/Rogdham/pyzstd 

88PyPI: https://pypi.org/project/pyzstd''' 

89 

90__all__ = ('ZstdCompressor', 'RichMemZstdCompressor', 

91 'ZstdDecompressor', 'EndlessZstdDecompressor', 

92 'CParameter', 'DParameter', 'Strategy', 'ZstdError', 

93 'compress', 'richmem_compress', 'decompress', 

94 'compress_stream', 'decompress_stream', 

95 'ZstdDict', 'train_dict', 'finalize_dict', 

96 'get_frame_info', 'get_frame_size', 'ZstdFile', 'open', 

97 'zstd_version', 'zstd_version_info', 

98 'zstd_support_multithread', 'compressionLevel_values', 

99 'SeekableZstdFile', 'SeekableFormatError') 

100 

101 

102zstd_support_multithread = (CParameter.nbWorkers.bounds() != (0, 0)) 

103 

104 

105def compress(data, level_or_option=None, zstd_dict=None): 

106 """Compress a block of data, return a bytes object. 

107 

108 Compressing b'' will get an empty content frame (9 bytes or more). 

109 

110 Parameters 

111 data: A bytes-like object, data to be compressed. 

112 level_or_option: When it's an int object, it represents compression level. 

113 When it's a dict object, it contains advanced compression 

114 parameters. 

115 zstd_dict: A ZstdDict object, pre-trained dictionary for compression. 

116 """ 

117 comp = ZstdCompressor(level_or_option, zstd_dict) 

118 return comp.compress(data, ZstdCompressor.FLUSH_FRAME) 

119 

120 

121@deprecated("See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives to pyzstd.richmem_compress") 

122def richmem_compress(data, level_or_option=None, zstd_dict=None): 

123 """Compress a block of data, return a bytes object. 

124 

125 Use rich memory mode, it's faster than compress() in some cases, but 

126 allocates more memory. 

127 

128 Compressing b'' will get an empty content frame (9 bytes or more). 

129 

130 Parameters 

131 data: A bytes-like object, data to be compressed. 

132 level_or_option: When it's an int object, it represents compression level. 

133 When it's a dict object, it contains advanced compression 

134 parameters. 

135 zstd_dict: A ZstdDict object, pre-trained dictionary for compression. 

136 """ 

137 comp = _RichMemZstdCompressor(level_or_option, zstd_dict) 

138 return comp.compress(data) 

139 

140 

141def _nbytes(dat): 

142 if isinstance(dat, (bytes, bytearray)): 

143 return len(dat) 

144 with memoryview(dat) as mv: 

145 return mv.nbytes 

146 

147 

148def train_dict(samples, dict_size): 

149 """Train a zstd dictionary, return a ZstdDict object. 

150 

151 Parameters 

152 samples: An iterable of samples, a sample is a bytes-like object 

153 represents a file. 

154 dict_size: The dictionary's maximum size, in bytes. 

155 """ 

156 # Check argument's type 

157 if not isinstance(dict_size, int): 

158 raise TypeError('dict_size argument should be an int object.') 

159 

160 # Prepare data 

161 chunks = [] 

162 chunk_sizes = [] 

163 for chunk in samples: 

164 chunks.append(chunk) 

165 chunk_sizes.append(_nbytes(chunk)) 

166 

167 chunks = b''.join(chunks) 

168 if not chunks: 

169 raise ValueError("The samples are empty content, can't train dictionary.") 

170 

171 # samples_bytes: samples be stored concatenated in a single flat buffer. 

172 # samples_size_list: a list of each sample's size. 

173 # dict_size: size of the dictionary, in bytes. 

174 dict_content = _train_dict(chunks, chunk_sizes, dict_size) 

175 

176 return ZstdDict(dict_content) 

177 

178 

179def finalize_dict(zstd_dict, samples, dict_size, level): 

180 """Finalize a zstd dictionary, return a ZstdDict object. 

181 

182 Given a custom content as a basis for dictionary, and a set of samples, 

183 finalize dictionary by adding headers and statistics according to the zstd 

184 dictionary format. 

185 

186 You may compose an effective dictionary content by hand, which is used as 

187 basis dictionary, and use some samples to finalize a dictionary. The basis 

188 dictionary can be a "raw content" dictionary, see is_raw parameter in 

189 ZstdDict.__init__ method. 

190 

191 Parameters 

192 zstd_dict: A ZstdDict object, basis dictionary. 

193 samples: An iterable of samples, a sample is a bytes-like object 

194 represents a file. 

195 dict_size: The dictionary's maximum size, in bytes. 

196 level: The compression level expected to use in production. The 

197 statistics for each compression level differ, so tuning the 

198 dictionary for the compression level can help quite a bit. 

199 """ 

200 if zstd_version_info < (1, 4, 5): 

201 msg = ("This function only available when the underlying zstd " 

202 "library's version is greater than or equal to v1.4.5, " 

203 "the current underlying zstd library's version is v%s.") % zstd_version 

204 raise NotImplementedError(msg) 

205 

206 # Check arguments' type 

207 if not isinstance(zstd_dict, ZstdDict): 

208 raise TypeError('zstd_dict argument should be a ZstdDict object.') 

209 if not isinstance(dict_size, int): 

210 raise TypeError('dict_size argument should be an int object.') 

211 if not isinstance(level, int): 

212 raise TypeError('level argument should be an int object.') 

213 

214 # Prepare data 

215 chunks = [] 

216 chunk_sizes = [] 

217 for chunk in samples: 

218 chunks.append(chunk) 

219 chunk_sizes.append(_nbytes(chunk)) 

220 

221 chunks = b''.join(chunks) 

222 if not chunks: 

223 raise ValueError("The samples are empty content, can't finalize dictionary.") 

224 

225 # custom_dict_bytes: existing dictionary. 

226 # samples_bytes: samples be stored concatenated in a single flat buffer. 

227 # samples_size_list: a list of each sample's size. 

228 # dict_size: maximal size of the dictionary, in bytes. 

229 # compression_level: compression level expected to use in production. 

230 dict_content = _finalize_dict(zstd_dict.dict_content, 

231 chunks, chunk_sizes, 

232 dict_size, level) 

233 

234 return ZstdDict(dict_content) 

235 

236 

237@wraps(_compress_stream) 

238@deprecated("See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives to pyzstd.compress_stream") 

239def compress_stream(*args, **kwargs): 

240 return _compress_stream(*args, **kwargs) 

241 

242@wraps(_decompress_stream) 

243@deprecated("See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives to pyzstd.decompress_stream") 

244def decompress_stream(*args, **kwargs): 

245 return _decompress_stream(*args, **kwargs) 

246 

247@deprecated("See https://pyzstd.readthedocs.io/en/stable/deprecated.html for alternatives to pyzstd.RichMemZstdCompressor") 

248class RichMemZstdCompressor(_RichMemZstdCompressor): 

249 pass 

250 

251RichMemZstdCompressor.__doc__ = _RichMemZstdCompressor.__doc__