/src/c-blosc2/include/blosc2.h
Line | Count | Source |
1 | | /********************************************************************* |
2 | | Blosc - Blocked Shuffling and Compression Library |
3 | | |
4 | | Copyright (c) 2021 Blosc Development Team <blosc@blosc.org> |
5 | | https://blosc.org |
6 | | License: BSD 3-Clause (see LICENSE.txt) |
7 | | |
8 | | See LICENSE.txt for details about copyright and rights to use. |
9 | | **********************************************************************/ |
10 | | |
11 | | /********************************************************************* |
12 | | @file blosc2.h |
13 | | @brief Blosc2 header file. |
14 | | |
15 | | This file contains Blosc2 public API and the structures needed to use it. |
16 | | @author Blosc Development Team <blosc@blosc.org> |
17 | | **********************************************************************/ |
18 | | |
19 | | #ifndef BLOSC_BLOSC2_H |
20 | | #define BLOSC_BLOSC2_H |
21 | | |
22 | | #include "blosc2/blosc2-export.h" |
23 | | #include "blosc2/blosc2-common.h" |
24 | | #include "blosc2/blosc2-stdio.h" |
25 | | |
26 | | #if defined(_WIN32) && !defined(__MINGW32__) |
27 | | #include <windows.h> |
28 | | #include <malloc.h> |
29 | | #include <process.h> |
30 | | #define getpid _getpid |
31 | | #endif |
32 | | |
33 | | #include <limits.h> |
34 | | #include <stdlib.h> |
35 | | #include <stdint.h> |
36 | | #include <stdbool.h> |
37 | | #include <stdio.h> |
38 | | |
39 | | #ifdef __cplusplus |
40 | | extern "C" { |
41 | | #endif |
42 | | |
43 | | // For compatibility with the Blosc 1.x series |
44 | | #ifdef BLOSC1_COMPAT |
45 | | // Blosc2 symbols that should be accessible from Blosc 1.x API |
46 | | #define BLOSC_VERSION_MAJOR BLOSC2_VERSION_MAJOR |
47 | | #define BLOSC_VERSION_MINOR BLOSC2_VERSION_MINOR |
48 | | #define BLOSC_VERSION_RELEASE BLOSC2_VERSION_RELEASE |
49 | | #define BLOSC_VERSION_STRING BLOSC2_VERSION_STRING |
50 | | #define BLOSC_VERSION_DATE BLOSC2_VERSION_DATE |
51 | | #define BLOSC_MAX_OVERHEAD BLOSC2_MAX_OVERHEAD |
52 | | #define BLOSC_MAX_BUFFERSIZE BLOSC2_MAX_BUFFERSIZE |
53 | | |
54 | | // API that changed to blosc1_ prefix |
55 | | #define blosc_compress blosc1_compress |
56 | | #define blosc_decompress blosc1_decompress |
57 | | #define blosc_getitem blosc1_getitem |
58 | | #define blosc_get_compressor blosc1_get_compressor |
59 | | #define blosc_set_compressor blosc1_set_compressor |
60 | | #define blosc_cbuffer_sizes blosc1_cbuffer_sizes |
61 | | #define blosc_cbuffer_validate blosc1_cbuffer_validate |
62 | | #define blosc_cbuffer_metainfo blosc1_cbuffer_metainfo |
63 | | #define blosc_get_blocksize blosc1_get_blocksize |
64 | | #define blosc_set_blocksize blosc1_set_blocksize |
65 | | #define blosc_set_splitmode blosc1_set_splitmode |
66 | | |
67 | | // API that changed to blosc2_ prefix |
68 | | #define blosc_init blosc2_init |
69 | | #define blosc_destroy blosc2_destroy |
70 | | #define blosc_free_resources blosc2_free_resources |
71 | | #define blosc_get_nthreads blosc2_get_nthreads |
72 | | #define blosc_set_nthreads blosc2_set_nthreads |
73 | | #define blosc_compcode_to_compname blosc2_compcode_to_compname |
74 | | #define blosc_compname_to_compcode blosc2_compname_to_compcode |
75 | | #define blosc_list_compressors blosc2_list_compressors |
76 | | #define blosc_get_version_string blosc2_get_version_string |
77 | | #define blosc_get_complib_info blosc2_get_complib_info |
78 | | #define blosc_cbuffer_versions blosc2_cbuffer_versions |
79 | | #define blosc_cbuffer_complib blosc2_cbuffer_complib |
80 | | #endif |
81 | | |
82 | | |
83 | | /* Version numbers */ |
84 | | #define BLOSC2_VERSION_MAJOR 3 /* for major interface/format changes */ |
85 | | #define BLOSC2_VERSION_MINOR 7 /* for minor interface/format changes */ |
86 | | #define BLOSC2_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ |
87 | | |
88 | 0 | #define BLOSC2_VERSION_STRING "3.7.1.dev" /* string version. Sync with above! */ |
89 | | #define BLOSC2_VERSION_DATE "$Date:: 2026-11-18 #$" /* date version year-month-day */ |
90 | | |
91 | | |
92 | | /* Tracing macros */ |
93 | 339k | #define BLOSC_TRACE_ERROR(msg, ...) BLOSC_TRACE(error, msg, ##__VA_ARGS__) |
94 | 720 | #define BLOSC_TRACE_WARNING(msg, ...) BLOSC_TRACE(warning, msg, ##__VA_ARGS__) |
95 | 216 | #define BLOSC_TRACE_INFO(msg, ...) BLOSC_TRACE(info, msg, ##__VA_ARGS__) |
96 | | #define BLOSC_TRACE(cat, msg, ...) \ |
97 | 340k | do { \ |
98 | 340k | const char *__e = getenv("BLOSC_TRACE"); \ |
99 | 341k | if (!__e) { break; } \ |
100 | 18.4E | fprintf(stderr, "[%s] - " msg " (%s:%d)\n", #cat, ##__VA_ARGS__, __FILE__, __LINE__); \ |
101 | 18.4E | } while(0) |
102 | | |
103 | | #define BLOSC_ERROR_NULL(pointer, rc) \ |
104 | 15.8M | do { \ |
105 | 15.8M | if ((pointer) == NULL) { \ |
106 | 64 | BLOSC_TRACE_ERROR("Pointer is null"); \ |
107 | 64 | return (rc); \ |
108 | 64 | } \ |
109 | 15.8M | } while (0) |
110 | | #define BLOSC_ERROR(rc) \ |
111 | 3.39M | do { \ |
112 | 3.39M | int rc_ = (rc); \ |
113 | 3.39M | if (rc_ < BLOSC2_ERROR_SUCCESS) { \ |
114 | 7.34k | char *error_msg = print_error(rc_); \ |
115 | 7.34k | BLOSC_TRACE_ERROR("%s", error_msg); \ |
116 | 7.34k | return rc_; \ |
117 | 7.34k | } \ |
118 | 3.39M | } while (0) |
119 | | |
120 | | #define BLOSC_INFO(msg, ...) \ |
121 | 3.38M | do { \ |
122 | 3.38M | const char *__e = getenv("BLOSC_INFO"); \ |
123 | 3.38M | if (!__e) { break; } \ |
124 | 3.38M | fprintf(stderr, "[INFO] - " msg "\n", ##__VA_ARGS__); \ |
125 | 0 | } while(0) |
126 | | |
127 | | |
128 | | /* The VERSION_FORMAT symbols below should be just 1-byte long */ |
129 | | enum { |
130 | | /* Blosc format version, starting at 1 |
131 | | 1 -> Blosc pre-1.0 |
132 | | 2 -> Blosc 1.x stable series |
133 | | 3 -> Blosc 2-alpha.x series |
134 | | 4 -> Blosc 2.x beta.1 series |
135 | | 5 -> Blosc 2.x stable series |
136 | | */ |
137 | | BLOSC1_VERSION_FORMAT_PRE1 = 1, |
138 | | BLOSC1_VERSION_FORMAT = 2, |
139 | | BLOSC2_VERSION_FORMAT_ALPHA = 3, |
140 | | BLOSC2_VERSION_FORMAT_BETA1 = 4, |
141 | | BLOSC2_VERSION_FORMAT_STABLE = 5, |
142 | | BLOSC2_VERSION_FORMAT_VL_BLOCKS = 6, |
143 | | /* Highest chunk format version supported by this library. */ |
144 | | BLOSC2_VERSION_FORMAT = BLOSC2_VERSION_FORMAT_VL_BLOCKS, |
145 | | }; |
146 | | |
147 | | |
148 | | /* The FRAME_FORMAT_VERSION symbols below should be just 4-bit long */ |
149 | | enum { |
150 | | /* Blosc format version |
151 | | * 1 -> First version (introduced in beta.2) |
152 | | * 2 -> Second version (introduced in rc.1) |
153 | | * |
154 | | */ |
155 | | BLOSC2_VERSION_FRAME_FORMAT_BETA2 = 1, // for 2.0.0-beta2 and after |
156 | | BLOSC2_VERSION_FRAME_FORMAT_RC1 = 2, // for 2.0.0-rc1 and after |
157 | | BLOSC2_VERSION_FRAME_FORMAT_VL_BLOCKS = 3, |
158 | | /* Highest cframe format version supported by this library. */ |
159 | | BLOSC2_VERSION_FRAME_FORMAT = BLOSC2_VERSION_FRAME_FORMAT_VL_BLOCKS, |
160 | | }; |
161 | | |
162 | | |
163 | | //!< Struct for storing data from instrumentation of codecs |
164 | | // This can be flexible because it is typically used mainly for development |
165 | | typedef struct { |
166 | | float cratio; |
167 | | float cspeed; |
168 | | float filter_speed; |
169 | | //float memory; |
170 | | //float power; |
171 | | uint8_t flags[4]; |
172 | | } blosc2_instr; |
173 | | |
174 | | |
175 | | enum { |
176 | | #ifndef BLOSC_H |
177 | | BLOSC_MIN_HEADER_LENGTH = 16, |
178 | | //!< Minimum header length (Blosc1) |
179 | | #endif // BLOSC_H |
180 | | BLOSC_EXTENDED_HEADER_LENGTH = 32, |
181 | | //!< Extended header length (Blosc2, see README_HEADER) |
182 | | BLOSC2_MAX_OVERHEAD = BLOSC_EXTENDED_HEADER_LENGTH, |
183 | | //!< The maximum overhead during compression in bytes. This equals |
184 | | //!< to @ref BLOSC_EXTENDED_HEADER_LENGTH now, but can be higher in future |
185 | | //!< implementations. |
186 | | BLOSC2_MAX_BUFFERSIZE = (INT_MAX - BLOSC2_MAX_OVERHEAD), |
187 | | //!< Maximum source buffer size to be compressed |
188 | | #ifndef BLOSC_H |
189 | | BLOSC_MAX_TYPESIZE = UINT8_MAX, |
190 | | //!< Maximum typesize before considering source buffer as a stream of bytes. |
191 | | //!< Cannot be larger than 255. |
192 | | #endif // BLOSC_H |
193 | | BLOSC_MIN_BUFFERSIZE = 32, |
194 | | //!< Minimum buffer size to be compressed. |
195 | | }; |
196 | | |
197 | | enum { |
198 | | BLOSC2_DEFINED_TUNER_START = 0, |
199 | | BLOSC2_DEFINED_TUNER_STOP = 31, |
200 | | //!< Blosc-defined tuners must be between 0 - 31. |
201 | | BLOSC2_GLOBAL_REGISTERED_TUNER_START = 32, |
202 | | BLOSC2_GLOBAL_REGISTERED_TUNER_STOP = 159, |
203 | | //!< Blosc-registered tuners must be between 31 - 159. |
204 | | BLOSC2_GLOBAL_REGISTERED_TUNERS = 0, |
205 | | //!< Number of Blosc-registered tuners at the moment. |
206 | | BLOSC2_USER_REGISTERED_TUNER_START = 160, |
207 | | BLOSC2_USER_REGISTERED_TUNER_STOP = 255, |
208 | | //!< User-defined tuners must be between 160 - 255. |
209 | | }; |
210 | | |
211 | | /** |
212 | | * @brief Codes for the different tuners shipped with Blosc |
213 | | */ |
214 | | enum { |
215 | | BLOSC_STUNE = 0, |
216 | | BLOSC_LAST_TUNER = 1, |
217 | | //!< Determine the last tuner defined by Blosc. |
218 | | BLOSC_LAST_REGISTERED_TUNE = BLOSC2_GLOBAL_REGISTERED_TUNER_START + BLOSC2_GLOBAL_REGISTERED_TUNERS - 1, |
219 | | //!< Determine the last registered tuner. It is used to check if a tuner is registered or not. |
220 | | }; |
221 | | |
222 | | enum { |
223 | | BLOSC2_DEFINED_FILTERS_START = 0, |
224 | | BLOSC2_DEFINED_FILTERS_STOP = 31, |
225 | | //!< Blosc-defined filters must be between 0 - 31. |
226 | | BLOSC2_GLOBAL_REGISTERED_FILTERS_START = 32, |
227 | | BLOSC2_GLOBAL_REGISTERED_FILTERS_STOP = 159, |
228 | | //!< Blosc-registered filters must be between 32 - 159. |
229 | | BLOSC2_GLOBAL_REGISTERED_FILTERS = 5, |
230 | | //!< Number of Blosc-registered filters at the moment. |
231 | | BLOSC2_USER_REGISTERED_FILTERS_START = 160, |
232 | | BLOSC2_USER_REGISTERED_FILTERS_STOP = 255, |
233 | | //!< User-defined filters must be between 128 - 255. |
234 | | BLOSC2_MAX_FILTERS = 6, |
235 | | //!< Maximum number of filters in the filter pipeline |
236 | | BLOSC2_MAX_UDFILTERS = 16, |
237 | | //!< Maximum number of filters that a user can register. |
238 | | }; |
239 | | |
240 | | |
241 | | /** |
242 | | * @brief Codes for filters. |
243 | | * |
244 | | * @sa #blosc1_compress |
245 | | */ |
246 | | enum { |
247 | | #ifndef BLOSC_H |
248 | | BLOSC_NOSHUFFLE = 0, |
249 | | //!< No shuffle (for compatibility with Blosc1). |
250 | | BLOSC_NOFILTER = 0, |
251 | | //!< No filter. |
252 | | BLOSC_SHUFFLE = 1, |
253 | | //!< Byte-wise shuffle. `filters_meta` is the number of bytestreams to shuffle from the input. If 0, defaults to typesize. |
254 | | BLOSC_BITSHUFFLE = 2, |
255 | | //!< Bit-wise shuffle. `filters_meta` does not have any effect. |
256 | | #endif // BLOSC_H |
257 | | BLOSC_DELTA = 3, |
258 | | //!< Delta filter (bitwise XOR relative to reference). `filters_meta` does not have any effect. |
259 | | BLOSC_TRUNC_PREC = 4, |
260 | | //!< Truncate mantissa precision. |
261 | | //!< Positive values in `filters_meta` will keep bits; negative values will zero bits. |
262 | | BLOSC_LAST_FILTER = 5, |
263 | | //!< sentinel |
264 | | BLOSC_LAST_REGISTERED_FILTER = BLOSC2_GLOBAL_REGISTERED_FILTERS_START + BLOSC2_GLOBAL_REGISTERED_FILTERS - 1, |
265 | | //!< Determine the last registered filter. It is used to check if a filter is registered or not. |
266 | | }; |
267 | | |
268 | | /** |
269 | | * @brief Codes for internal flags (see blosc1_cbuffer_metainfo) |
270 | | */ |
271 | | enum { |
272 | | #ifndef BLOSC_H |
273 | | BLOSC_DOSHUFFLE = 0x1, //!< byte-wise shuffle |
274 | | BLOSC_MEMCPYED = 0x2, //!< plain copy |
275 | | BLOSC_DOBITSHUFFLE = 0x4, //!< bit-wise shuffle |
276 | | #endif // BLOSC_H |
277 | | BLOSC_DODELTA = 0x8, //!< delta coding |
278 | | }; |
279 | | |
280 | | /** |
281 | | * @brief Codes for new internal flags in Blosc2 |
282 | | */ |
283 | | enum { |
284 | | BLOSC2_USEDICT = 0x1, //!< use dictionaries with codec |
285 | | BLOSC2_BIGENDIAN = 0x2, //!< data is in big-endian ordering |
286 | | BLOSC2_INSTR_CODEC = 0x80, //!< codec is instrumented (mainly for development) |
287 | | }; |
288 | | |
289 | | /** |
290 | | * @brief Codes for flags in the secondary Blosc2 header byte. |
291 | | */ |
292 | | enum { |
293 | | BLOSC2_VL_BLOCKS = 0x1, //!< chunk uses variable-length blocks |
294 | | }; |
295 | | |
296 | | /** |
297 | | * @brief Values for different Blosc2 capabilities |
298 | | */ |
299 | | enum { |
300 | | BLOSC2_MAXDICTSIZE = 32 * 1024, //!< maximum size for compression dicts |
301 | | BLOSC2_MINUSEFULDICT = 256, //!< minimum dict size considered useful; smaller → fall back to plain compression |
302 | | BLOSC2_MAXBLOCKSIZE = 536866816, //!< maximum size for blocks |
303 | | BLOSC2_MAXTYPESIZE = BLOSC2_MAXBLOCKSIZE, //!< maximum size for types |
304 | | }; |
305 | | |
306 | | |
307 | | enum { |
308 | | BLOSC2_DEFINED_CODECS_START = 0, |
309 | | BLOSC2_DEFINED_CODECS_STOP = 31, |
310 | | //!< Blosc-defined codecs must be between 0 - 31. |
311 | | BLOSC2_GLOBAL_REGISTERED_CODECS_START = 32, |
312 | | BLOSC2_GLOBAL_REGISTERED_CODECS_STOP = 159, |
313 | | //!< Blosc-registered codecs must be between 31 - 159. |
314 | | BLOSC2_GLOBAL_REGISTERED_CODECS = 5, |
315 | | //!< Number of Blosc-registered codecs at the moment. |
316 | | BLOSC2_USER_REGISTERED_CODECS_START = 160, |
317 | | BLOSC2_USER_REGISTERED_CODECS_STOP = 255, |
318 | | //!< User-defined codecs must be between 160 - 255. |
319 | | }; |
320 | | |
321 | | /** |
322 | | * @brief Codes for the different compressors shipped with Blosc |
323 | | */ |
324 | | enum { |
325 | | #ifndef BLOSC_H |
326 | | BLOSC_BLOSCLZ = 0, |
327 | | BLOSC_LZ4 = 1, |
328 | | BLOSC_LZ4HC = 2, |
329 | | BLOSC_ZLIB = 4, |
330 | | BLOSC_ZSTD = 5, |
331 | | #endif // BLOSC_H |
332 | | BLOSC_LAST_CODEC = 6, |
333 | | //!< Determine the last codec defined by Blosc. |
334 | | BLOSC_LAST_REGISTERED_CODEC = BLOSC2_GLOBAL_REGISTERED_CODECS_START + BLOSC2_GLOBAL_REGISTERED_CODECS - 1, |
335 | | //!< Determine the last registered codec. It is used to check if a codec is registered or not. |
336 | | }; |
337 | | |
338 | | |
339 | | // Names for the different compressors shipped with Blosc |
340 | | |
341 | | #ifndef BLOSC_H |
342 | 21.8k | #define BLOSC_BLOSCLZ_COMPNAME "blosclz" |
343 | 16.7k | #define BLOSC_LZ4_COMPNAME "lz4" |
344 | 15.9k | #define BLOSC_LZ4HC_COMPNAME "lz4hc" |
345 | 12.7k | #define BLOSC_ZLIB_COMPNAME "zlib" |
346 | 9.26k | #define BLOSC_ZSTD_COMPNAME "zstd" |
347 | | #endif // BLOSC_H |
348 | | |
349 | | /** |
350 | | * @brief Codes for compression libraries shipped with Blosc (code must be < 8) |
351 | | */ |
352 | | enum { |
353 | | #ifndef BLOSC_H |
354 | | BLOSC_BLOSCLZ_LIB = 0, |
355 | | BLOSC_LZ4_LIB = 1, |
356 | | BLOSC_ZLIB_LIB = 3, |
357 | | BLOSC_ZSTD_LIB = 4, |
358 | | #endif // BLOSC_H |
359 | | BLOSC_UDCODEC_LIB = 6, |
360 | | BLOSC_SCHUNK_LIB = 7, //!< compressor library in super-chunk header |
361 | | }; |
362 | | |
363 | | /** |
364 | | * @brief Names for the different compression libraries shipped with Blosc |
365 | | */ |
366 | | #ifndef BLOSC_H |
367 | 0 | #define BLOSC_BLOSCLZ_LIBNAME "BloscLZ" |
368 | 0 | #define BLOSC_LZ4_LIBNAME "LZ4" |
369 | 0 | #define BLOSC_ZLIB_LIBNAME "Zlib" |
370 | 0 | #define BLOSC_ZSTD_LIBNAME "Zstd" |
371 | | #endif // BLOSC_H |
372 | | |
373 | | /** |
374 | | * @brief The codes for compressor formats shipped with Blosc |
375 | | */ |
376 | | enum { |
377 | | #ifndef BLOSC_H |
378 | | BLOSC_BLOSCLZ_FORMAT = BLOSC_BLOSCLZ_LIB, |
379 | | BLOSC_LZ4_FORMAT = BLOSC_LZ4_LIB, |
380 | | //!< LZ4HC and LZ4 share the same format |
381 | | BLOSC_LZ4HC_FORMAT = BLOSC_LZ4_LIB, |
382 | | BLOSC_ZLIB_FORMAT = BLOSC_ZLIB_LIB, |
383 | | BLOSC_ZSTD_FORMAT = BLOSC_ZSTD_LIB, |
384 | | #endif // BLOSC_H |
385 | | BLOSC_UDCODEC_FORMAT = BLOSC_UDCODEC_LIB, |
386 | | }; |
387 | | |
388 | | /** |
389 | | * @brief The version formats for compressors shipped with Blosc. |
390 | | * All versions here starts at 1 |
391 | | */ |
392 | | enum { |
393 | | #ifndef BLOSC_H |
394 | | BLOSC_BLOSCLZ_VERSION_FORMAT = 1, |
395 | | BLOSC_LZ4_VERSION_FORMAT = 1, |
396 | | BLOSC_LZ4HC_VERSION_FORMAT = 1, /* LZ4HC and LZ4 share the same format */ |
397 | | BLOSC_ZLIB_VERSION_FORMAT = 1, |
398 | | BLOSC_ZSTD_VERSION_FORMAT = 1, |
399 | | #endif // BLOSC_H |
400 | | BLOSC_UDCODEC_VERSION_FORMAT = 1, |
401 | | }; |
402 | | |
403 | | /** |
404 | | * @brief Split mode for blocks. |
405 | | * NEVER and ALWAYS are for experimenting with compression ratio. |
406 | | * AUTO for nearly optimal behaviour (based on heuristics). |
407 | | * FORWARD_COMPAT provides best forward compatibility (default). |
408 | | */ |
409 | | #ifndef BLOSC_H |
410 | | enum { |
411 | | BLOSC_ALWAYS_SPLIT = 1, |
412 | | BLOSC_NEVER_SPLIT = 2, |
413 | | BLOSC_AUTO_SPLIT = 3, |
414 | | BLOSC_FORWARD_COMPAT_SPLIT = 4, |
415 | | }; |
416 | | #endif // BLOSC_H |
417 | | |
418 | | /** |
419 | | * @brief Offsets for fields in Blosc2 chunk header. |
420 | | */ |
421 | | enum { |
422 | | BLOSC2_CHUNK_VERSION = 0x0, //!< the version for the chunk format |
423 | | BLOSC2_CHUNK_VERSIONLZ = 0x1, //!< the version for the format of internal codec |
424 | | BLOSC2_CHUNK_FLAGS = 0x2, //!< flags and codec info |
425 | | BLOSC2_CHUNK_TYPESIZE = 0x3, //!< (uint8) the number of bytes of the atomic type |
426 | | BLOSC2_CHUNK_NBYTES = 0x4, //!< (int32) uncompressed size of the buffer (this header is not included) |
427 | | BLOSC2_CHUNK_BLOCKSIZE = 0x8, //!< (int32) size of internal blocks |
428 | | BLOSC2_CHUNK_CBYTES = 0xc, //!< (int32) compressed size of the buffer (including this header) |
429 | | BLOSC2_CHUNK_FILTER_CODES = 0x10, //!< the codecs for the filter pipeline (1 byte per code) |
430 | | BLOSC2_CHUNK_FILTER_META = 0x18, //!< meta info for the filter pipeline (1 byte per code) |
431 | | BLOSC2_CHUNK_BLOSC2_FLAGS2 = 0x1e,//!< second flags byte specific for Blosc2 functionality |
432 | | BLOSC2_CHUNK_BLOSC2_FLAGS = 0x1F, //!< flags specific for Blosc2 functionality |
433 | | }; |
434 | | |
435 | | /** |
436 | | * @brief Run lengths for special values for chunks/frames |
437 | | */ |
438 | | enum { |
439 | | BLOSC2_NO_SPECIAL = 0x0, //!< no special value |
440 | | BLOSC2_SPECIAL_ZERO = 0x1, //!< zero special value |
441 | | BLOSC2_SPECIAL_NAN = 0x2, //!< NaN special value |
442 | | BLOSC2_SPECIAL_VALUE = 0x3, //!< repeated special value |
443 | | BLOSC2_SPECIAL_UNINIT = 0x4, //!< non initialized values |
444 | | BLOSC2_SPECIAL_LASTID = 0x4, //!< last valid ID for special value (update this adequately) |
445 | | BLOSC2_SPECIAL_MASK = 0x7 //!< special value mask (prev IDs cannot be larger than this) |
446 | | }; |
447 | | |
448 | | /** |
449 | | * @brief Error codes |
450 | | * Each time an error code is added here, its corresponding message error should be added in |
451 | | * print_error() |
452 | | */ |
453 | | enum { |
454 | | BLOSC2_ERROR_SUCCESS = 0, //<! Success |
455 | | BLOSC2_ERROR_FAILURE = -1, //<! Generic failure |
456 | | BLOSC2_ERROR_STREAM = -2, //<! Bad stream |
457 | | BLOSC2_ERROR_DATA = -3, //<! Invalid data |
458 | | BLOSC2_ERROR_MEMORY_ALLOC = -4, //<! Memory alloc/realloc failure |
459 | | BLOSC2_ERROR_READ_BUFFER = -5, //!< Not enough space to read |
460 | | BLOSC2_ERROR_WRITE_BUFFER = -6, //!< Not enough space to write |
461 | | BLOSC2_ERROR_CODEC_SUPPORT = -7, //!< Codec not supported |
462 | | BLOSC2_ERROR_CODEC_PARAM = -8, //!< Invalid parameter supplied to codec |
463 | | BLOSC2_ERROR_CODEC_DICT = -9, //!< Codec dictionary error |
464 | | BLOSC2_ERROR_VERSION_SUPPORT = -10, //!< Version not supported |
465 | | BLOSC2_ERROR_INVALID_HEADER = -11, //!< Invalid value in header |
466 | | BLOSC2_ERROR_INVALID_PARAM = -12, //!< Invalid parameter supplied to function |
467 | | BLOSC2_ERROR_FILE_READ = -13, //!< File read failure |
468 | | BLOSC2_ERROR_FILE_WRITE = -14, //!< File write failure |
469 | | BLOSC2_ERROR_FILE_OPEN = -15, //!< File open failure |
470 | | BLOSC2_ERROR_NOT_FOUND = -16, //!< Not found |
471 | | BLOSC2_ERROR_RUN_LENGTH = -17, //!< Bad run length encoding |
472 | | BLOSC2_ERROR_FILTER_PIPELINE = -18, //!< Filter pipeline error |
473 | | BLOSC2_ERROR_CHUNK_INSERT = -19, //!< Chunk insert failure |
474 | | BLOSC2_ERROR_CHUNK_APPEND = -20, //!< Chunk append failure |
475 | | BLOSC2_ERROR_CHUNK_UPDATE = -21, //!< Chunk update failure |
476 | | BLOSC2_ERROR_2GB_LIMIT = -22, //!< Sizes larger than 2gb not supported |
477 | | BLOSC2_ERROR_SCHUNK_COPY = -23, //!< Super-chunk copy failure |
478 | | BLOSC2_ERROR_FRAME_TYPE = -24, //!< Wrong type for frame |
479 | | BLOSC2_ERROR_FILE_TRUNCATE = -25, //!< File truncate failure |
480 | | BLOSC2_ERROR_THREAD_CREATE = -26, //!< Thread or thread context creation failure |
481 | | BLOSC2_ERROR_POSTFILTER = -27, //!< Postfilter failure |
482 | | BLOSC2_ERROR_FRAME_SPECIAL = -28, //!< Special frame failure |
483 | | BLOSC2_ERROR_SCHUNK_SPECIAL = -29, //!< Special super-chunk failure |
484 | | BLOSC2_ERROR_PLUGIN_IO = -30, //!< IO plugin error |
485 | | BLOSC2_ERROR_FILE_REMOVE = -31, //!< Remove file failure |
486 | | BLOSC2_ERROR_NULL_POINTER = -32, //!< Pointer is null |
487 | | BLOSC2_ERROR_INVALID_INDEX = -33, //!< Invalid index |
488 | | BLOSC2_ERROR_METALAYER_NOT_FOUND = -34, //!< Metalayer has not been found |
489 | | BLOSC2_ERROR_MAX_BUFSIZE_EXCEEDED = -35, //!< Max buffer size exceeded |
490 | | BLOSC2_ERROR_TUNER = -36, //!< Tuner failure |
491 | | BLOSC2_ERROR_LOCK = -37, //!< Frame lock failure |
492 | | }; |
493 | | |
494 | | |
495 | | #ifdef __GNUC__ |
496 | | #define BLOSC_ATTRIBUTE_UNUSED __attribute__((unused)) |
497 | | #else |
498 | | #define BLOSC_ATTRIBUTE_UNUSED |
499 | | #endif |
500 | | |
501 | | /** |
502 | | * @brief Get the string representation of an error code. |
503 | | * |
504 | | * The returned string is a static string, so it should not be modified or freed. |
505 | | * For unknown error codes, it returns "Unknown error". |
506 | | * |
507 | | * @param error_code The error code to get the string representation for. |
508 | | * @return A pointer to a static string representing the error code. |
509 | | */ |
510 | | BLOSC_EXPORT const char *blosc2_error_string(int error_code); |
511 | | |
512 | | /** |
513 | | * @brief Get the string representation of an error code. |
514 | | * |
515 | | * This function is identical to #blosc2_error_string, but with a legacy interface. |
516 | | * The returned string is a static read only string, so it should not be modified or freed, |
517 | | * although it is not const. |
518 | | * |
519 | | * Prefer using #blosc2_error_string instead. |
520 | | * |
521 | | * @param rc The error code to get the string representation for. |
522 | | * @return A pointer to a static string representing the error code. |
523 | | */ |
524 | | static char *print_error(int rc) BLOSC_ATTRIBUTE_UNUSED; |
525 | 7.34k | static char *print_error(int rc) { |
526 | | // TODO: remove this function, use blosc2_error_string only |
527 | 7.34k | return (char *) blosc2_error_string(rc); |
528 | 7.34k | } Unexecuted instantiation: fuzz_compress_frame.c:print_error Line | Count | Source | 525 | 295 | static char *print_error(int rc) { | 526 | | // TODO: remove this function, use blosc2_error_string only | 527 | 295 | return (char *) blosc2_error_string(rc); | 528 | 295 | } |
Unexecuted instantiation: blosclz.c:print_error Unexecuted instantiation: schunk.c:print_error Unexecuted instantiation: frame.c:print_error Unexecuted instantiation: prefetch.c:print_error Unexecuted instantiation: stune.c:print_error Unexecuted instantiation: trunc-prec.c:print_error Unexecuted instantiation: timestamp.c:print_error Unexecuted instantiation: sframe.c:print_error Unexecuted instantiation: blosc2-stdio.c:print_error Line | Count | Source | 525 | 6.93k | static char *print_error(int rc) { | 526 | | // TODO: remove this function, use blosc2_error_string only | 527 | 6.93k | return (char *) blosc2_error_string(rc); | 528 | 6.93k | } |
Unexecuted instantiation: b2nd_utils.c:print_error Unexecuted instantiation: shuffle.c:print_error Unexecuted instantiation: shuffle-bytedelta-ssse3.c:print_error Unexecuted instantiation: blosc2-zfp.c:print_error Unexecuted instantiation: codecs-registry.c:print_error Unexecuted instantiation: tuners-registry.c:print_error Unexecuted instantiation: filters-registry.c:print_error Unexecuted instantiation: ndlz.c:print_error Unexecuted instantiation: ndlz4x4.c:print_error Unexecuted instantiation: ndlz8x8.c:print_error Unexecuted instantiation: ndcell.c:print_error Unexecuted instantiation: ndmean.c:print_error Line | Count | Source | 525 | 113 | static char *print_error(int rc) { | 526 | | // TODO: remove this function, use blosc2_error_string only | 527 | 113 | return (char *) blosc2_error_string(rc); | 528 | 113 | } |
Unexecuted instantiation: int_trunc.c:print_error Unexecuted instantiation: fuzz_plugins.c:print_error Unexecuted instantiation: fuzz_decompress_frame.c:print_error Unexecuted instantiation: fuzz_compress_chunk.c:print_error Unexecuted instantiation: fuzz_schunk_ops.c:print_error Unexecuted instantiation: fuzz_decompress_chunk.c:print_error Unexecuted instantiation: fuzz_b2nd.c:print_error |
529 | | |
530 | | |
531 | | /** |
532 | | * @brief Initialize the Blosc library environment. |
533 | | * |
534 | | * You must call this previous to any other Blosc call, unless you want |
535 | | * Blosc to be used simultaneously in a multi-threaded environment, in |
536 | | * which case you can use the #blosc2_compress_ctx #blosc2_decompress_ctx pair. |
537 | | * |
538 | | * The **BLOSC_PREFETCH** environment variable, set to a number of threads, turns on read-ahead for frames on disk |
539 | | * here, and **BLOSC_PREFETCH_POPULATE** says for which memory-mapped frames it populates the mapping (see |
540 | | * #blosc2_set_prefetch). |
541 | | * |
542 | | * @sa #blosc2_destroy |
543 | | */ |
544 | | BLOSC_EXPORT void blosc2_init(void); |
545 | | |
546 | | |
547 | | /** |
548 | | * @brief Destroy the Blosc library environment. |
549 | | * |
550 | | * You must call this after to you are done with all the Blosc calls, |
551 | | * unless you have not used blosc2_init() before. |
552 | | * |
553 | | * @sa #blosc2_init |
554 | | */ |
555 | | BLOSC_EXPORT void blosc2_destroy(void); |
556 | | |
557 | | |
558 | | /** |
559 | | * @brief Compress a block of data in the @p src buffer and returns the size of |
560 | | * compressed block. |
561 | | * |
562 | | * @remark Compression is memory safe and guaranteed not to write @p dest |
563 | | * more than what is specified in @p destsize. |
564 | | * There is not a minimum for @p src buffer size @p nbytes. |
565 | | * Equivalent to #blosc2_compress. |
566 | | * |
567 | | * @warning The @p src buffer and the @p dest buffer can not overlap. |
568 | | * |
569 | | * @param clevel The desired compression level and must be a number |
570 | | * between 0 (no compression) and 9 (maximum compression). |
571 | | * @param doshuffle Specifies whether the shuffle compression preconditioner |
572 | | * should be applied or not. #BLOSC_NOFILTER means not applying filters, |
573 | | * #BLOSC_SHUFFLE means applying shuffle at a byte level and |
574 | | * #BLOSC_BITSHUFFLE at a bit level (slower but *may* achieve better |
575 | | * compression). |
576 | | * @param typesize Is the number of bytes for the atomic type in binary |
577 | | * @p src buffer. This is mainly useful for the shuffle preconditioner. |
578 | | * For implementation reasons, only a 1 < typesize < 256 will allow the |
579 | | * shuffle filter to work. When typesize is not in this range, shuffle |
580 | | * will be silently disabled. |
581 | | * @param nbytes The number of bytes to compress in the @p src buffer. |
582 | | * @param src The buffer containing the data to compress. |
583 | | * @param dest The buffer where the compressed data will be put, |
584 | | * must have at least the size of @p destsize. |
585 | | * @param destsize The size of the dest buffer. Blosc |
586 | | * guarantees that if you set @p destsize to, at least, |
587 | | * (@p nbytes + #BLOSC2_MAX_OVERHEAD), the compression will always succeed. |
588 | | * |
589 | | * @return The number of bytes compressed. |
590 | | * If @p src buffer cannot be compressed into @p destsize, the return |
591 | | * value is zero and you should discard the contents of the @p dest |
592 | | * buffer. A negative return value means that an internal error happened. This |
593 | | * should never happen. If you see this, please report it back |
594 | | * together with the buffer data causing this and compression settings. |
595 | | * |
596 | | * |
597 | | * @par Environment variables |
598 | | * @parblock |
599 | | * |
600 | | * This function honors different environment variables to control |
601 | | * internal parameters without the need of doing that programmatically. |
602 | | * Here are the ones supported: |
603 | | * |
604 | | * * **BLOSC_CLEVEL=(INTEGER)**: This will overwrite the @p clevel parameter |
605 | | * before the compression process starts. |
606 | | * |
607 | | * * **BLOSC_SHUFFLE=[NOSHUFFLE | SHUFFLE | BITSHUFFLE]**: This will |
608 | | * overwrite the @p doshuffle parameter before the compression process |
609 | | * starts. |
610 | | * |
611 | | * * **BLOSC_DELTA=(1|0)**: This will call #blosc2_set_delta() before the |
612 | | * compression process starts. |
613 | | * |
614 | | * * **BLOSC_TYPESIZE=(INTEGER)**: This will overwrite the @p typesize |
615 | | * parameter before the compression process starts. |
616 | | * |
617 | | * * **BLOSC_COMPRESSOR=[BLOSCLZ | LZ4 | LZ4HC | ZLIB | ZSTD]**: |
618 | | * This will call #blosc1_set_compressor before the compression process starts. |
619 | | * |
620 | | * * **BLOSC_NTHREADS=(INTEGER)**: This will call |
621 | | * #blosc2_set_nthreads before the compression process starts. |
622 | | * |
623 | | * * **BLOSC_SPLITMODE=(ALWAYS | NEVER | AUTO | FORWARD_COMPAT)**: |
624 | | * This will call #blosc1_set_splitmode() before the compression process starts. |
625 | | * |
626 | | * * **BLOSC_BLOCKSIZE=(INTEGER)**: This will call |
627 | | * #blosc1_set_blocksize before the compression process starts. |
628 | | * *NOTE:* The *blocksize* is a critical parameter with |
629 | | * important restrictions in the allowed values, so use this with care. |
630 | | * |
631 | | * * **BLOSC_NOLOCK=(ANY VALUE)**: This will call #blosc2_compress_ctx under |
632 | | * the hood, with the *compressor*, *blocksize* and |
633 | | * *numinternalthreads* parameters set to the same as the last calls to |
634 | | * #blosc1_set_compressor, #blosc1_set_blocksize and |
635 | | * #blosc2_set_nthreads. *BLOSC_CLEVEL*, *BLOSC_SHUFFLE*, *BLOSC_DELTA* and |
636 | | * *BLOSC_TYPESIZE* environment vars will also be honored. |
637 | | * |
638 | | * * **BLOSC_BLOSC1_COMPAT=(ANY VALUE)**: The chunk is written in the Blosc1 format, |
639 | | * as if #blosc2_ctx_set_blosc1_format() had been called, so that C-Blosc 1.x can |
640 | | * decode it. Also honored together with *BLOSC_NOLOCK*. |
641 | | * |
642 | | * @endparblock |
643 | | * |
644 | | * @sa #blosc1_decompress |
645 | | */ |
646 | | BLOSC_EXPORT int blosc1_compress(int clevel, int doshuffle, size_t typesize, |
647 | | size_t nbytes, const void* src, void* dest, |
648 | | size_t destsize); |
649 | | |
650 | | |
651 | | /** |
652 | | * @brief Decompress a block of compressed data in @p src, put the result in |
653 | | * @p dest and returns the size of the decompressed block. |
654 | | * |
655 | | * @warning The @p src buffer and the @p dest buffer can not overlap. |
656 | | * |
657 | | * @remark Decompression is memory safe and guaranteed not to write the @p dest |
658 | | * buffer more than what is specified in @p destsize. |
659 | | * Similar to #blosc2_decompress. |
660 | | * |
661 | | * @remark In case you want to keep under control the number of bytes read from |
662 | | * source, you can call #blosc1_cbuffer_sizes first to check whether the |
663 | | * @p nbytes (i.e. the number of bytes to be read from @p src buffer by this |
664 | | * function) in the compressed buffer is ok with you. |
665 | | * |
666 | | * @param src The buffer to be decompressed. |
667 | | * @param dest The buffer where the decompressed data will be put. |
668 | | * @param destsize The size of the @p dest buffer. |
669 | | * |
670 | | * @return The number of bytes decompressed. |
671 | | * If an error occurs, e.g. the compressed data is corrupted or the |
672 | | * output buffer is not large enough, then a negative value |
673 | | * will be returned instead. |
674 | | * |
675 | | * @par Environment variables |
676 | | * @parblock |
677 | | * This function honors different environment variables to control |
678 | | * internal parameters without the need of doing that programmatically. |
679 | | * Here are the ones supported: |
680 | | * |
681 | | * * **BLOSC_NTHREADS=(INTEGER)**: This will call |
682 | | * #blosc2_set_nthreads before the proper decompression |
683 | | * process starts. |
684 | | * |
685 | | * * **BLOSC_NOLOCK=(ANY VALUE)**: This will call #blosc2_decompress_ctx |
686 | | * under the hood, with the *numinternalthreads* parameter set to the |
687 | | * same value as the last call to #blosc2_set_nthreads. |
688 | | * |
689 | | * @endparblock |
690 | | * |
691 | | * @sa #blosc1_compress |
692 | | */ |
693 | | BLOSC_EXPORT int blosc1_decompress(const void* src, void* dest, size_t destsize); |
694 | | |
695 | | |
696 | | /** |
697 | | * @brief Get @p nitems (of @p typesize size) in @p src buffer starting in @p start. |
698 | | * The items are returned in @p dest buffer, which has to have enough |
699 | | * space for storing all items. |
700 | | * |
701 | | * @remark The function #blosc2_getitem is a more complete and secure version. |
702 | | * |
703 | | * @param src The compressed buffer from data will be decompressed. |
704 | | * @param start The position of the first item (of @p typesize size) from where data |
705 | | * will be retrieved. |
706 | | * @param nitems The number of items (of @p typesize size) that will be retrieved. |
707 | | * @param dest The buffer where the decompressed data retrieved will be put. |
708 | | * |
709 | | * @return The number of bytes copied to @p dest or a negative value if |
710 | | * some error happens. |
711 | | */ |
712 | | BLOSC_EXPORT int blosc1_getitem(const void* src, int start, int nitems, void* dest); |
713 | | |
714 | | /** |
715 | | * @brief Get @p nitems (of @p typesize size) in @p src buffer starting in @p start. |
716 | | * The items are returned in @p dest buffer. The dest buffer should have enough space |
717 | | * for storing all items. This function is a more secure version of #blosc1_getitem. |
718 | | * |
719 | | * @param src The compressed buffer holding the data to be retrieved. |
720 | | * @param srcsize Size of the compressed buffer. |
721 | | * @param start The position of the first item (of @p typesize size) from where data |
722 | | * will be retrieved. |
723 | | * @param nitems The number of items (of @p typesize size) that will be retrieved. |
724 | | * @param dest The buffer where the retrieved data will be stored decompressed. |
725 | | * @param destsize Size of the buffer where retrieved data will be stored. |
726 | | * |
727 | | * @return The number of bytes copied to @p dest or a negative value if |
728 | | * some error happens. |
729 | | */ |
730 | | BLOSC_EXPORT int blosc2_getitem(const void* src, int32_t srcsize, int start, int nitems, |
731 | | void* dest, int32_t destsize); |
732 | | |
733 | | /** |
734 | | Pointer to a callback function that executes `dojob(jobdata + i*jobdata_elsize)` for `i = 0 to numjobs-1`, |
735 | | possibly in parallel threads (but not returning until all `dojob` calls have returned). This allows the |
736 | | caller to provide a custom threading backend as an alternative to the default Blosc-managed threads. |
737 | | `callback_data` is passed through from `blosc2_set_threads_callback`. |
738 | | */ |
739 | | typedef void (*blosc_threads_callback)(void *callback_data, void (*dojob)(void *), int numjobs, size_t jobdata_elsize, void *jobdata); |
740 | | |
741 | | /** |
742 | | * @brief Set the threading backend for parallel compression/decompression to use @p callback to execute work |
743 | | * instead of using the Blosc-managed threads. The @p callback_data argument is passed through to the callback. |
744 | | |
745 | | * @param callback: the callback to use. Passing `NULL` uses the default Blosc threading backend. |
746 | | * @param callback_data: the callback data. |
747 | | * |
748 | | * @warning This function is *not* thread-safe and should be called |
749 | | * before any other Blosc function: it affects all Blosc contexts. |
750 | | * @sa https://github.com/Blosc/c-blosc2/pull/81 |
751 | | */ |
752 | | BLOSC_EXPORT void blosc2_set_threads_callback(blosc_threads_callback callback, void *callback_data); |
753 | | |
754 | | |
755 | | /** |
756 | | * @brief Returns the current number of threads that are used for |
757 | | * compression/decompression. |
758 | | */ |
759 | | BLOSC_EXPORT int16_t blosc2_get_nthreads(void); |
760 | | |
761 | | |
762 | | /** |
763 | | * @brief Initialize a pool of threads for compression/decompression. If |
764 | | * @p nthreads is 1, then the serial version is chosen and a possible |
765 | | * previous existing pool is released (#blosc2_free_resources ends the |
766 | | * pools that are no longer used). If this is not called, @p nthreads |
767 | | * is set to 1 internally. The threads are per process; a child process |
768 | | * created with fork() starts its own threads on first use (see |
769 | | * README_THREADED.rst). |
770 | | * |
771 | | * @param nthreads The number of threads to use. |
772 | | * |
773 | | * @return The previous number of threads. |
774 | | */ |
775 | | BLOSC_EXPORT int16_t blosc2_set_nthreads(int16_t nthreads); |
776 | | |
777 | | |
778 | | /** |
779 | | * @brief Default for the @p max_ahead parameter of #blosc2_set_prefetch. |
780 | | */ |
781 | 0 | #define BLOSC2_PREFETCH_MAX_AHEAD_DEFAULT ((int64_t)64 * 1024 * 1024) |
782 | | |
783 | | /** |
784 | | * @brief Most read-ahead threads #blosc2_set_prefetch accepts. |
785 | | */ |
786 | 0 | #define BLOSC2_PREFETCH_MAX_NTHREADS 256 |
787 | | |
788 | | /** |
789 | | * @brief Default for the @p piece_size parameter of #blosc2_set_prefetch. |
790 | | */ |
791 | 0 | #define BLOSC2_PREFETCH_PIECE_SIZE_DEFAULT (1024 * 1024) |
792 | | |
793 | | /** |
794 | | * @brief Smallest @p piece_size #blosc2_set_prefetch accepts. |
795 | | */ |
796 | 0 | #define BLOSC2_PREFETCH_MIN_PIECE_SIZE (64 * 1024) |
797 | | |
798 | | /** |
799 | | * @brief Read ahead when reading slices of arrays and super-chunks stored in contiguous frames on disk with the |
800 | | * default filesystem or the memory-mapped I/O: #b2nd_get_slice_cbuffer, #b2nd_get_slice, |
801 | | * #b2nd_get_orthogonal_selection and #blosc2_schunk_get_slice_buffer. |
802 | | * |
803 | | * Without read-ahead, a slice reads the blocks it needs one after the other, each with at most the kernel's read-ahead |
804 | | * window per read; on a network file system every such read is a round trip. With read-ahead, the chunks the slice |
805 | | * needs (and that are not in the page cache yet) are read by up to @p nthreads background threads in pieces of at most |
806 | | * @p piece_size bytes, in the order the slice decompresses them and at most @p max_ahead bytes ahead of it, and each |
807 | | * chunk is decompressed as soon as its bytes are in. Of a chunk that a slice (#b2nd_get_slice_cbuffer, |
808 | | * #b2nd_get_slice) covers only in part, only the blocks it needs are read, after the chunk's header (which says where |
809 | | * they are; one round trip more when it is not cached). The data goes to the page cache (memory-mapped reads stay |
810 | | * zero-copy); pages that are cached already are not read again (asked with cachestat() on Linux >= 6.5, else |
811 | | * mincore()). On Linux the kernel answers that only for files the process owns or may write to (before 6.5: only |
812 | | * files it owns); other files are read again even when cached, which is correct but costs a copy. |
813 | | * |
814 | | * Opening a frame file gets its start and its end (header, trailer and chunk index) in one round trip, also with |
815 | | * read-ahead off; with read-ahead on, a file of up to 1 MiB (from the frame's offset on) is asked for whole. |
816 | | * |
817 | | * Reads that are not cached get faster on any storage, most on high-latency ones; 4 threads and the default piece |
818 | | * size give most of the speed. The threads mostly wait for the storage, but the kernel spends CPU on every byte they |
819 | | * read; README_THREADED.rst has which settings to use when. |
820 | | * |
821 | | * On GPFS (IBM Storage Scale) read() does not fill the page cache but a cache of GPFS's own, so for a frame read |
822 | | * through the mmap I/O (#BLOSC2_IO_FILESYSTEM_MMAP) the threads populate its mapping there instead |
823 | | * (madvise(MADV_POPULATE_READ), Linux >= 5.14), cached pages included: the slice then finds the pages mapped and does |
824 | | * not fault on them (builds with glibc >= 2.28 recognize GPFS). The **BLOSC_PREFETCH_POPULATE** environment |
825 | | * variable, read at #blosc2_init, does this for the memory-mapped frames on every file system when set to 1, and for |
826 | | * none when set to 0. |
827 | | * |
828 | | * How many threads: each slice being read gets up to @p nthreads read-ahead threads, and slices that threads of the |
829 | | * process read at the same time get theirs each. So a process runs up to @p nthreads times the number of slices it |
830 | | * reads at once, at most #BLOSC2_PREFETCH_MAX_NTHREADS in all: a program that reads with one thread has @p nthreads |
831 | | * of them, one that reads with 8 threads at once up to 8 x @p nthreads. Decompression threads (`nthreads` of |
832 | | * #blosc2_dparams) are separate and do not count: they decompress a chunk while the read-ahead threads fetch the next |
833 | | * ones. Read-ahead threads are started as needed and run until #blosc2_destroy (lowering the number does not stop |
834 | | * them), each keeping a buffer as large as the largest piece it has read. |
835 | | * |
836 | | * Off by default; the **BLOSC_PREFETCH** environment variable, set to a number of threads, turns it on at |
837 | | * #blosc2_init. The setting and the threads are per process; a child process created with fork() starts its own |
838 | | * threads on first use. Supported on POSIX systems; elsewhere (Windows) this has no effect. |
839 | | * |
840 | | * @param nthreads Read-ahead threads per slice being read (see above), at most #BLOSC2_PREFETCH_MAX_NTHREADS; 0 turns |
841 | | * read-ahead off. |
842 | | * @param max_ahead Most bytes read ahead of the chunk being decompressed (0 for the default, |
843 | | * #BLOSC2_PREFETCH_MAX_AHEAD_DEFAULT). |
844 | | * @param piece_size Largest read handed to one thread, in bytes (0 for the default, |
845 | | * #BLOSC2_PREFETCH_PIECE_SIZE_DEFAULT; at least #BLOSC2_PREFETCH_MIN_PIECE_SIZE). |
846 | | * |
847 | | * @return The previous number of threads, or a negative error code. |
848 | | */ |
849 | | BLOSC_EXPORT int blosc2_set_prefetch(int nthreads, int64_t max_ahead, int64_t piece_size); |
850 | | |
851 | | |
852 | | /** |
853 | | * @brief Get the current compressor that is used for compression. |
854 | | * |
855 | | * @return The string identifying the compressor being used. |
856 | | */ |
857 | | BLOSC_EXPORT const char* blosc1_get_compressor(void); |
858 | | |
859 | | |
860 | | /** |
861 | | * @brief Select the compressor to be used. The supported ones are "blosclz", |
862 | | * "lz4", "lz4hc", "zlib" and "ztsd". If this function is not |
863 | | * called, then "blosclz" will be used. |
864 | | * |
865 | | * @param compname The name identifier of the compressor to be set. |
866 | | * |
867 | | * @return The code for the compressor (>=0). In case the compressor |
868 | | * is not recognized, or there is not support for it in this build, |
869 | | * it returns a -1. |
870 | | */ |
871 | | BLOSC_EXPORT int blosc1_set_compressor(const char* compname); |
872 | | |
873 | | |
874 | | /** |
875 | | * @brief Select the delta coding filter to be used. |
876 | | * |
877 | | * @param dodelta A value >0 will activate the delta filter. |
878 | | * If 0, it will be de-activated |
879 | | * |
880 | | * This call should always succeed. |
881 | | */ |
882 | | BLOSC_EXPORT void blosc2_set_delta(int dodelta); |
883 | | |
884 | | |
885 | | /** |
886 | | * @brief Get the compressor name associated with the compressor code. |
887 | | * |
888 | | * @param compcode The code identifying the compressor |
889 | | * @param compname The pointer to a string where the compressor name will be put. |
890 | | * |
891 | | * @return The compressor code. If the compressor code is not recognized, |
892 | | * or there is not support for it in this build, -1 is returned. |
893 | | */ |
894 | | BLOSC_EXPORT int blosc2_compcode_to_compname(int compcode, const char** compname); |
895 | | |
896 | | |
897 | | /** |
898 | | * @brief Get the compressor code associated with the compressor name. |
899 | | * |
900 | | * @param compname The string containing the compressor name. |
901 | | * |
902 | | * @return The compressor code. If the compressor name is not recognized, |
903 | | * or there is not support for it in this build, -1 is returned instead. |
904 | | */ |
905 | | BLOSC_EXPORT int blosc2_compname_to_compcode(const char* compname); |
906 | | |
907 | | |
908 | | /** |
909 | | * @brief Get a list of compressors supported in the current build. |
910 | | * |
911 | | * @return The comma separated string with the list of compressor names |
912 | | * supported. |
913 | | * |
914 | | * This function does not leak, so you should not free() the returned |
915 | | * list. |
916 | | * |
917 | | * This function should always succeed. |
918 | | */ |
919 | | BLOSC_EXPORT const char* blosc2_list_compressors(void); |
920 | | |
921 | | |
922 | | /** |
923 | | * @brief Get the version of Blosc in string format. |
924 | | * |
925 | | * @return The string with the current Blosc version. |
926 | | * Useful for dynamic libraries. |
927 | | */ |
928 | | BLOSC_EXPORT const char* blosc2_get_version_string(void); |
929 | | |
930 | | |
931 | | /** |
932 | | * @brief Get info from compression libraries included in the current build. |
933 | | * |
934 | | * @param compname The compressor name that you want info from. |
935 | | * @param complib The pointer to a string where the |
936 | | * compression library name, if available, will be put. |
937 | | * @param version The pointer to a string where the |
938 | | * compression library version, if available, will be put. |
939 | | * |
940 | | * @warning You are in charge of the @p complib and @p version strings, |
941 | | * you should free() them so as to avoid leaks. |
942 | | * |
943 | | * @return The code for the compression library (>=0). If it is not supported, |
944 | | * this function returns -1. |
945 | | */ |
946 | | BLOSC_EXPORT int blosc2_get_complib_info(const char* compname, char** complib, |
947 | | char** version); |
948 | | |
949 | | |
950 | | /** |
951 | | * @brief Free possible memory temporaries and thread resources. Use this |
952 | | * when you are not going to use Blosc for a long while. |
953 | | * |
954 | | * Blosc's own thread pools (not on Windows) that no context uses any more |
955 | | * are kept, a few of them with the temporaries of their threads, so that |
956 | | * contexts created later with the same number of threads do not start new |
957 | | * threads; this function ends them. |
958 | | * |
959 | | * @return A 0 if succeeds, in case of problems releasing the resources, |
960 | | * it returns a negative number. |
961 | | */ |
962 | | BLOSC_EXPORT int blosc2_free_resources(void); |
963 | | |
964 | | |
965 | | /** |
966 | | * @brief Get information about a compressed buffer, namely the number of |
967 | | * uncompressed bytes (@p nbytes) and compressed (@p cbytes). It also |
968 | | * returns the @p blocksize (which is used internally for doing the |
969 | | * compression by blocks). |
970 | | * |
971 | | * @remark Equivalent to function #blosc2_cbuffer_sizes. |
972 | | * |
973 | | * @param cbuffer The buffer of compressed data. |
974 | | * @param nbytes The pointer where the number of uncompressed bytes will be put. |
975 | | * @param cbytes The pointer where the number of compressed bytes will be put. |
976 | | * @param blocksize The pointer where the block size will be put. |
977 | | * |
978 | | * You only need to pass the first BLOSC_MIN_HEADER_LENGTH bytes of a |
979 | | * compressed buffer for this call to work. |
980 | | * |
981 | | * This function should always succeed. |
982 | | */ |
983 | | BLOSC_EXPORT void blosc1_cbuffer_sizes(const void* cbuffer, size_t* nbytes, |
984 | | size_t* cbytes, size_t* blocksize); |
985 | | /** |
986 | | * @brief Get information about a compressed buffer, namely the number of |
987 | | * uncompressed bytes (@p nbytes) and compressed (@p cbytes). It also |
988 | | * returns the @p blocksize (which is used internally for doing the |
989 | | * compression by blocks). |
990 | | * |
991 | | * @param cbuffer The buffer of compressed data. |
992 | | * @param nbytes The pointer where the number of uncompressed bytes will be put. |
993 | | * @param cbytes The pointer where the number of compressed bytes stored in the |
994 | | * chunk header will be put. |
995 | | * @param blocksize The pointer where the block size will be put. |
996 | | * |
997 | | * @note: if any of the nbytes, cbytes or blocksize is NULL, it will not be returned. |
998 | | * |
999 | | * You only need to pass the first BLOSC_MIN_HEADER_LENGTH bytes of a |
1000 | | * compressed buffer for this call to work. |
1001 | | * |
1002 | | * @note For lazy chunks returned by #blosc2_schunk_get_lazychunk, the returned |
1003 | | * @p cbytes value is still the compressed size encoded in the chunk header, |
1004 | | * not the full size of the lazy proxy buffer. When passing a lazy chunk to |
1005 | | * #blosc2_decompress_ctx or #blosc2_getitem_ctx, use the size returned by |
1006 | | * #blosc2_schunk_get_lazychunk as @p srcsize. |
1007 | | * |
1008 | | * @return On failure, returns negative value. |
1009 | | */ |
1010 | | BLOSC_EXPORT int blosc2_cbuffer_sizes(const void* cbuffer, int32_t* nbytes, |
1011 | | int32_t* cbytes, int32_t* blocksize); |
1012 | | |
1013 | | /** |
1014 | | * @brief Checks that the compressed buffer starting at @p cbuffer of length @p cbytes |
1015 | | * may contain valid blosc compressed data, and that it is safe to call |
1016 | | * blosc1_decompress/blosc1_getitem. |
1017 | | * On success, returns 0 and sets @p nbytes to the size of the uncompressed data. |
1018 | | * This does not guarantee that the decompression function won't return an error, |
1019 | | * but does guarantee that it is safe to attempt decompression. |
1020 | | * |
1021 | | * @param cbuffer The buffer of compressed data. |
1022 | | * @param cbytes The number of compressed bytes. |
1023 | | * @param nbytes The pointer where the number of uncompressed bytes will be put. |
1024 | | * |
1025 | | * @return On failure, returns negative value. |
1026 | | */ |
1027 | | BLOSC_EXPORT int blosc1_cbuffer_validate(const void* cbuffer, size_t cbytes, |
1028 | | size_t* nbytes); |
1029 | | |
1030 | | /** |
1031 | | * @brief Get information about a compressed buffer, namely the type size |
1032 | | * (@p typesize), as well as some internal @p flags. |
1033 | | * |
1034 | | * @param cbuffer The buffer of compressed data. |
1035 | | * @param typesize The pointer where the type size will be put. |
1036 | | * @param flags The pointer of the integer where the additional info is encoded. |
1037 | | * The @p flags is a set of bits, where the currently used ones are: |
1038 | | * * bit 0: whether the shuffle filter has been applied or not |
1039 | | * * bit 1: whether the internal buffer is a pure memcpy or not |
1040 | | * * bit 2: whether the bitshuffle filter has been applied or not |
1041 | | * * bit 3: whether the delta coding filter has been applied or not |
1042 | | * |
1043 | | * You can use the @p BLOSC_DOSHUFFLE, @p BLOSC_DOBITSHUFFLE, @p BLOSC_DODELTA |
1044 | | * and @p BLOSC_MEMCPYED symbols for extracting the interesting bits |
1045 | | * (e.g. @p flags & @p BLOSC_DOSHUFFLE says whether the buffer is byte-shuffled |
1046 | | * or not). |
1047 | | * |
1048 | | * This function should always succeed. |
1049 | | */ |
1050 | | BLOSC_EXPORT void blosc1_cbuffer_metainfo(const void* cbuffer, size_t* typesize, |
1051 | | int* flags); |
1052 | | |
1053 | | |
1054 | | /** |
1055 | | * @brief Get information about a compressed buffer, namely the internal |
1056 | | * Blosc format version (@p version) and the format for the internal |
1057 | | * Lempel-Ziv compressor used (@p versionlz). |
1058 | | * |
1059 | | * @param cbuffer The buffer of compressed data. |
1060 | | * @param version The pointer where the Blosc format version will be put. |
1061 | | * @param versionlz The pointer where the Lempel-Ziv version will be put. |
1062 | | * |
1063 | | * This function should always succeed. |
1064 | | */ |
1065 | | BLOSC_EXPORT void blosc2_cbuffer_versions(const void* cbuffer, int* version, |
1066 | | int* versionlz); |
1067 | | |
1068 | | |
1069 | | /** |
1070 | | * @brief Get the compressor library/format used in a compressed buffer. |
1071 | | * |
1072 | | * @param cbuffer The buffer of compressed data. |
1073 | | * |
1074 | | * @return The string identifying the compressor library/format used. |
1075 | | * |
1076 | | * This function should always succeed. |
1077 | | */ |
1078 | | BLOSC_EXPORT const char* blosc2_cbuffer_complib(const void* cbuffer); |
1079 | | |
1080 | | /********************************************************************* |
1081 | | Structures and functions related with user-defined input/output. |
1082 | | *********************************************************************/ |
1083 | | |
1084 | | enum { |
1085 | | BLOSC2_IO_FILESYSTEM = 0, |
1086 | | BLOSC2_IO_FILESYSTEM_MMAP = 1, |
1087 | | BLOSC_IO_LAST_BLOSC_DEFINED = 2, // sentinel |
1088 | | BLOSC_IO_LAST_REGISTERED = 32, // sentinel |
1089 | | }; |
1090 | | |
1091 | | enum { |
1092 | | BLOSC2_IO_BLOSC_DEFINED = 32, |
1093 | | BLOSC2_IO_REGISTERED = 160, |
1094 | | BLOSC2_IO_USER_DEFINED = 256 |
1095 | | }; |
1096 | | |
1097 | | typedef void* (*blosc2_open_cb)(const char *urlpath, const char *mode, void *params); |
1098 | | typedef int (*blosc2_close_cb)(void *stream); |
1099 | | typedef int64_t (*blosc2_size_cb)(void *stream); |
1100 | | typedef int64_t (*blosc2_write_cb)(const void *ptr, int64_t size, int64_t nitems, int64_t position, void *stream); |
1101 | | typedef int64_t (*blosc2_read_cb)(void **ptr, int64_t size, int64_t nitems, int64_t position, void *stream); |
1102 | | typedef int (*blosc2_truncate_cb)(void *stream, int64_t size); |
1103 | | typedef int (*blosc2_destroy_cb)(void *params); |
1104 | | |
1105 | | |
1106 | | /* |
1107 | | * Input/Output callbacks. |
1108 | | */ |
1109 | | typedef struct { |
1110 | | uint8_t id; |
1111 | | //!< The IO identifier. |
1112 | | char* name; |
1113 | | //!< The IO name. |
1114 | | bool is_allocation_necessary; |
1115 | | //!< If true, the caller needs to allocate data for the read function (ptr argument). If false, the read function |
1116 | | //!< stores in the ptr argument the address of memory that it owns (the caller must not free it), or NULL when |
1117 | | //!< it cannot read the requested bytes: check the returned count before using it. |
1118 | | blosc2_open_cb open; |
1119 | | //!< The IO open callback. |
1120 | | blosc2_close_cb close; |
1121 | | //!< The IO close callback. |
1122 | | blosc2_size_cb size; |
1123 | | //!< The IO size callback. |
1124 | | blosc2_write_cb write; |
1125 | | //!< The IO write callback. |
1126 | | blosc2_read_cb read; |
1127 | | //!< The IO read callback. |
1128 | | blosc2_truncate_cb truncate; |
1129 | | //!< The IO truncate callback. |
1130 | | blosc2_destroy_cb destroy; |
1131 | | //!< The IO destroy callback (called in the end when finished with the schunk). |
1132 | | } blosc2_io_cb; |
1133 | | |
1134 | | |
1135 | | /* |
1136 | | * Input/Output parameters. |
1137 | | */ |
1138 | | typedef struct { |
1139 | | uint8_t id; |
1140 | | const char *name; |
1141 | | //!< The IO identifier. |
1142 | | void *params; |
1143 | | //!< The IO parameters. |
1144 | | } blosc2_io; |
1145 | | |
1146 | | static const blosc2_io BLOSC2_IO_DEFAULTS = { |
1147 | | /* .id = */ BLOSC2_IO_FILESYSTEM, |
1148 | | /* .name = */ "filesystem", |
1149 | | /* .params = */ NULL, |
1150 | | }; |
1151 | | |
1152 | | |
1153 | | /** |
1154 | | * @brief Register a user-defined input/output callbacks in Blosc. |
1155 | | * |
1156 | | * @param io The callbacks API to register. |
1157 | | * |
1158 | | * @return 0 if succeeds. Else a negative code is returned. |
1159 | | */ |
1160 | | BLOSC_EXPORT int blosc2_register_io_cb(const blosc2_io_cb *io); |
1161 | | |
1162 | | /** |
1163 | | * @brief Get a user-defined input/output callback in Blosc. |
1164 | | * |
1165 | | * @param id The id of the callback to get. |
1166 | | * |
1167 | | * @return A pointer containing the desired callback if success. Else a NULL pointer is returned. |
1168 | | */ |
1169 | | BLOSC_EXPORT blosc2_io_cb *blosc2_get_io_cb(uint8_t id); |
1170 | | |
1171 | | /********************************************************************* |
1172 | | Structures and functions related with contexts. |
1173 | | *********************************************************************/ |
1174 | | |
1175 | | typedef struct blosc2_context_s blosc2_context; /* opaque type */ |
1176 | | |
1177 | | typedef struct { |
1178 | | int (*init)(void * config, blosc2_context* cctx, blosc2_context* dctx); |
1179 | | //!< Initialize tuner. Keep in mind dctx may be NULL. This should memcpy the cctx->tuner_params. |
1180 | | int (*next_blocksize)(blosc2_context * context); |
1181 | | //!< Only compute the next blocksize. Only it is executed if tuner is not initialized. |
1182 | | int (*next_cparams)(blosc2_context * context); |
1183 | | //!< Compute the next cparams. Only is executed if tuner is initialized. |
1184 | | int (*update)(blosc2_context * context, double ctime); |
1185 | | //!< Update the tuner parameters. |
1186 | | int (*free)(blosc2_context * context); |
1187 | | //!< Free the tuner. |
1188 | | int id; |
1189 | | //!< The tuner id |
1190 | | char *name; |
1191 | | //!< The tuner name |
1192 | | } blosc2_tuner; |
1193 | | |
1194 | | |
1195 | | /** |
1196 | | * @brief Register locally a user-defined tuner in Blosc. |
1197 | | * |
1198 | | * @param tuner The tuner to register. |
1199 | | * |
1200 | | * @return 0 if succeeds. Else a negative code is returned. |
1201 | | */ |
1202 | | BLOSC_EXPORT int blosc2_register_tuner(blosc2_tuner *tuner); |
1203 | | |
1204 | | |
1205 | | /** |
1206 | | * @brief The parameters for a prefilter function. |
1207 | | * |
1208 | | */ |
1209 | | typedef struct { |
1210 | | void *user_data; // user-provided info (optional) |
1211 | | const uint8_t *input; // the input buffer |
1212 | | uint8_t *output; // the output buffer |
1213 | | int32_t output_size; // the output size (in bytes) |
1214 | | int32_t output_typesize; // the output typesize |
1215 | | int32_t output_offset; // offset to reach the start of the output buffer |
1216 | | int64_t nchunk; // the current nchunk in associated schunk (if exists; if not -1) |
1217 | | int32_t nblock; // the current nblock in associated chunk |
1218 | | int32_t tid; // thread id |
1219 | | uint8_t *ttmp; // a temporary that is able to hold several blocks for the output and is private for each thread |
1220 | | size_t ttmp_nbytes; // the size of the temporary in bytes |
1221 | | blosc2_context *ctx; // the compression context |
1222 | | bool output_is_disposable; // whether the output buffer is disposable |
1223 | | } blosc2_prefilter_params; |
1224 | | |
1225 | | /** |
1226 | | * @brief The parameters for a postfilter function. |
1227 | | * |
1228 | | */ |
1229 | | typedef struct { |
1230 | | void *user_data; // user-provided info (optional) |
1231 | | const uint8_t *input; // the input buffer |
1232 | | uint8_t *output; // the output buffer |
1233 | | int32_t size; // the input size (in bytes) |
1234 | | int32_t typesize; // the input typesize |
1235 | | int32_t offset; // offset to reach the start of the input buffer |
1236 | | int64_t nchunk; // the current nchunk in associated schunk (if exists; if not -1) |
1237 | | int32_t nblock; // the current nblock in associated chunk |
1238 | | int32_t tid; // thread id |
1239 | | uint8_t *ttmp; // a temporary that is able to hold several blocks for the output and is private for each thread |
1240 | | size_t ttmp_nbytes; // the size of the temporary in bytes |
1241 | | blosc2_context *ctx; // the decompression context |
1242 | | } blosc2_postfilter_params; |
1243 | | |
1244 | | /** |
1245 | | * @brief The type of the prefilter function. |
1246 | | * |
1247 | | * If the function call is successful, the return value should be 0; else, a negative value. |
1248 | | */ |
1249 | | typedef int (*blosc2_prefilter_fn)(blosc2_prefilter_params* params); |
1250 | | |
1251 | | /** |
1252 | | * @brief The type of the postfilter function. |
1253 | | * |
1254 | | * If the function call is successful, the return value should be 0; else, a negative value. |
1255 | | */ |
1256 | | typedef int (*blosc2_postfilter_fn)(blosc2_postfilter_params* params); |
1257 | | |
1258 | | /** |
1259 | | * @brief The parameters for creating a context for compression purposes. |
1260 | | * |
1261 | | * In parenthesis it is shown the default value used internally when a 0 |
1262 | | * (zero) in the fields of the struct is passed to a function. |
1263 | | */ |
1264 | | typedef struct { |
1265 | | uint8_t compcode; |
1266 | | //!< The compressor codec. |
1267 | | uint8_t compcode_meta; |
1268 | | //!< The metadata for the compressor codec. |
1269 | | uint8_t clevel; |
1270 | | //!< The compression level (5). |
1271 | | int use_dict; |
1272 | | //!< Use dicts or not when compressing. Only ZSTD and LZ4/LZ4HC support this; |
1273 | | //!< other codecs will return an error if this is set. |
1274 | | int32_t typesize; |
1275 | | //!< The type size (8). |
1276 | | int16_t nthreads; |
1277 | | //!< The number of threads to use internally (1). |
1278 | | int32_t blocksize; |
1279 | | //!< The requested size of the compressed blocks (0 means automatic). |
1280 | | int32_t splitmode; |
1281 | | //!< Whether the blocks should be split or not. |
1282 | | void* schunk; |
1283 | | //!< The associated schunk, if any (NULL). |
1284 | | uint8_t filters[BLOSC2_MAX_FILTERS]; |
1285 | | //!< The (sequence of) filters. |
1286 | | uint8_t filters_meta[BLOSC2_MAX_FILTERS]; |
1287 | | //!< The metadata for filters. |
1288 | | blosc2_prefilter_fn prefilter; |
1289 | | //!< The prefilter function. |
1290 | | blosc2_prefilter_params *preparams; |
1291 | | //!< The prefilter parameters. |
1292 | | void *tuner_params; |
1293 | | //!< Tune configuration. |
1294 | | int tuner_id; |
1295 | | //!< The tuner id. |
1296 | | bool instr_codec; |
1297 | | //!< Whether the codec is instrumented or not |
1298 | | void *codec_params; |
1299 | | //!< User defined parameters for the codec |
1300 | | void *filter_params[BLOSC2_MAX_FILTERS]; |
1301 | | //!< User defined parameters for the filters |
1302 | | } blosc2_cparams; |
1303 | | |
1304 | | /** |
1305 | | * @brief Default struct for compression params meant for user initialization. |
1306 | | */ |
1307 | | static const blosc2_cparams BLOSC2_CPARAMS_DEFAULTS = { |
1308 | | BLOSC_BLOSCLZ, 0, 5, 0, 8, 1, 0, |
1309 | | BLOSC_FORWARD_COMPAT_SPLIT, NULL, |
1310 | | {0, 0, 0, 0, 0, BLOSC_SHUFFLE}, |
1311 | | {0, 0, 0, 0, 0, 0}, |
1312 | | NULL, NULL, NULL, 0, 0, |
1313 | | NULL, {NULL, NULL, NULL, NULL, NULL, NULL} |
1314 | | }; |
1315 | | |
1316 | | |
1317 | | /** |
1318 | | @brief The parameters for creating a context for decompression purposes. |
1319 | | |
1320 | | In parenthesis it is shown the default value used internally when a 0 |
1321 | | (zero) in the fields of the struct is passed to a function. |
1322 | | */ |
1323 | | typedef struct { |
1324 | | int16_t nthreads; |
1325 | | //!< The number of threads to use internally (1). |
1326 | | void* schunk; |
1327 | | //!< The associated schunk, if any (NULL). |
1328 | | blosc2_postfilter_fn postfilter; |
1329 | | //!< The postfilter function. |
1330 | | blosc2_postfilter_params *postparams; |
1331 | | //!< The postfilter parameters. |
1332 | | int32_t typesize; |
1333 | | //!< The type size (8). |
1334 | | } blosc2_dparams; |
1335 | | |
1336 | | /** |
1337 | | * @brief Default struct for decompression params meant for user initialization. |
1338 | | */ |
1339 | | static const blosc2_dparams BLOSC2_DPARAMS_DEFAULTS = {1, NULL, NULL, NULL, 8}; |
1340 | | |
1341 | | |
1342 | | /** |
1343 | | * @brief Create a context for @a *_ctx() compression functions. |
1344 | | * |
1345 | | * @param cparams The blosc2_cparams struct with the compression parameters. |
1346 | | * |
1347 | | * @return A pointer to the new context. NULL is returned if this fails. |
1348 | | * |
1349 | | * @note This supports the same environment variables than #blosc2_compress |
1350 | | * for overriding the programmatic compression values. |
1351 | | * |
1352 | | * @sa #blosc2_compress |
1353 | | */ |
1354 | | BLOSC_EXPORT blosc2_context* blosc2_create_cctx(blosc2_cparams cparams); |
1355 | | |
1356 | | /** |
1357 | | * @brief Opt a compression context's prefilter out of output zero-initialization. |
1358 | | * @since 3.7.0 |
1359 | | * |
1360 | | * With @p enabled true, the prefilter must initialize every byte of |
1361 | | * `params->output[0..params->output_size)` on each successful call whose output |
1362 | | * is retained, including padding and skipped blocks. It must not rely on the |
1363 | | * output's previous contents. This does not make the output disposable. |
1364 | | * |
1365 | | * The default is false, preserving zero-initialization for existing callbacks. |
1366 | | * Configure this only while the context is idle. The setting is context-local: |
1367 | | * it is not copied by blosc2_ctx_get_cparams() and must be reapplied after |
1368 | | * recreating a context. Disabling it restores the default behavior. |
1369 | | * |
1370 | | * @return 0 on success; BLOSC2_ERROR_INVALID_PARAM for a NULL or decompression |
1371 | | * context, or when enabling the setting without a prefilter. |
1372 | | */ |
1373 | | BLOSC_EXPORT int blosc2_set_prefilter_full_write(blosc2_context *ctx, bool enabled); |
1374 | | |
1375 | | /** |
1376 | | * @brief Create a context for *_ctx() decompression functions. |
1377 | | * |
1378 | | * @param dparams The blosc2_dparams struct with the decompression parameters. |
1379 | | * |
1380 | | * @return A pointer to the new context. NULL is returned if this fails. |
1381 | | * |
1382 | | * @note This supports the same environment variables than #blosc2_decompress |
1383 | | * for overriding the programmatic decompression values. |
1384 | | * |
1385 | | * @sa #blosc2_decompress |
1386 | | * |
1387 | | */ |
1388 | | BLOSC_EXPORT blosc2_context* blosc2_create_dctx(blosc2_dparams dparams); |
1389 | | |
1390 | | /** |
1391 | | * @brief Free the resources associated with a context. |
1392 | | * |
1393 | | * @param context The context to free. |
1394 | | * |
1395 | | * This function should always succeed and is valid for contexts meant for |
1396 | | * both compression and decompression. |
1397 | | */ |
1398 | | BLOSC_EXPORT void blosc2_free_ctx(blosc2_context* context); |
1399 | | |
1400 | | /** |
1401 | | * @brief Create a @p cparams associated to a context. |
1402 | | * |
1403 | | * @param ctx The context from where to extract the compression parameters. |
1404 | | * @param cparams The pointer where the compression params will be stored. |
1405 | | * |
1406 | | * @return 0 if succeeds. Else a negative code is returned. |
1407 | | */ |
1408 | | BLOSC_EXPORT int blosc2_ctx_get_cparams(blosc2_context *ctx, blosc2_cparams *cparams); |
1409 | | |
1410 | | /** |
1411 | | * @brief Create a @p dparams associated to a context. |
1412 | | * |
1413 | | * @param ctx The context from where to extract the decompression parameters. |
1414 | | * @param dparams The pointer where the decompression params will be stored. |
1415 | | * |
1416 | | * @return 0 if succeeds. Else a negative code is returned. |
1417 | | */ |
1418 | | BLOSC_EXPORT int blosc2_ctx_get_dparams(blosc2_context *ctx, blosc2_dparams *dparams); |
1419 | | |
1420 | | /** |
1421 | | * @brief Set a maskout so as to avoid decompressing specified blocks. |
1422 | | * |
1423 | | * @param ctx The decompression context to update. |
1424 | | * |
1425 | | * @param maskout The boolean mask for the blocks where decompression |
1426 | | * is to be avoided. |
1427 | | * |
1428 | | * @remark The maskout is valid for contexts *only* meant for decompressing |
1429 | | * a chunk via #blosc2_decompress_ctx. Once a call to #blosc2_decompress_ctx |
1430 | | * is done, this mask is reset so that next call to #blosc2_decompress_ctx |
1431 | | * will decompress the whole chunk. |
1432 | | * |
1433 | | * @param nblocks The number of blocks in maskout above. |
1434 | | * |
1435 | | * @return If success, a 0 is returned. An error is signaled with a negative int. |
1436 | | * |
1437 | | */ |
1438 | | BLOSC_EXPORT int blosc2_set_maskout(blosc2_context *ctx, bool *maskout, int nblocks); |
1439 | | |
1440 | | /** |
1441 | | * @brief Compress a block of data in the @p src buffer and returns the size of |
1442 | | * compressed block. |
1443 | | * |
1444 | | * @remark Compression is memory safe and guaranteed not to write @p dest |
1445 | | * more than what is specified in @p destsize. |
1446 | | * There is not a minimum for @p src buffer size @p nbytes. |
1447 | | * |
1448 | | * @warning The @p src buffer and the @p dest buffer can not overlap. |
1449 | | * |
1450 | | * @param clevel The desired compression level and must be a number |
1451 | | * between 0 (no compression) and 9 (maximum compression). |
1452 | | * @param doshuffle Specifies whether the shuffle compression preconditioner |
1453 | | * should be applied or not. #BLOSC_NOFILTER means not applying filters, |
1454 | | * #BLOSC_SHUFFLE means applying shuffle at a byte level and |
1455 | | * #BLOSC_BITSHUFFLE at a bit level (slower but *may* achieve better |
1456 | | * compression). |
1457 | | * @param typesize Is the number of bytes for the atomic type in binary |
1458 | | * @p src buffer. This is mainly useful for the shuffle preconditioner. |
1459 | | * For implementation reasons, only a 1 < typesize < 256 will allow the |
1460 | | * shuffle filter to work. When typesize is not in this range, shuffle |
1461 | | * will be silently disabled. |
1462 | | * @param src The buffer containing the data to compress. |
1463 | | * @param srcsize The number of bytes to compress in the @p src buffer. |
1464 | | * @param dest The buffer where the compressed data will be put, |
1465 | | * must have at least the size of @p destsize. |
1466 | | * @param destsize The size of the dest buffer. Blosc |
1467 | | * guarantees that if you set @p destsize to, at least, |
1468 | | * (@p nbytes + #BLOSC2_MAX_OVERHEAD), the compression will always succeed. |
1469 | | * |
1470 | | * @return The number of bytes compressed. |
1471 | | * If @p src buffer cannot be compressed into @p destsize, the return |
1472 | | * value is zero and you should discard the contents of the @p dest |
1473 | | * buffer. A negative return value means that either a parameter is not correct |
1474 | | * or that an internal error happened. Set the BLOSC_TRACE environment variable |
1475 | | * for getting more info on what is happening. If the error is not related with |
1476 | | * wrong params, please report it back together with the buffer data causing this, |
1477 | | * as well as the compression params used. |
1478 | | */ |
1479 | | /* |
1480 | | * Environment variables |
1481 | | * _____________________ |
1482 | | * |
1483 | | * *blosc2_compress()* honors different environment variables to control |
1484 | | * internal parameters without the need of doing that programmatically. |
1485 | | * Here are the ones supported: |
1486 | | * |
1487 | | * **BLOSC_CLEVEL=(INTEGER)**: This will overwrite the @p clevel parameter |
1488 | | * before the compression process starts. |
1489 | | * |
1490 | | * **BLOSC_SHUFFLE=[NOSHUFFLE | SHUFFLE | BITSHUFFLE]**: This will |
1491 | | * overwrite the *doshuffle* parameter before the compression process |
1492 | | * starts. |
1493 | | * |
1494 | | * **BLOSC_DELTA=(1|0)**: This will call *blosc2_set_delta()^* before the |
1495 | | * compression process starts. |
1496 | | * |
1497 | | * **BLOSC_TYPESIZE=(INTEGER)**: This will overwrite the *typesize* |
1498 | | * parameter before the compression process starts. |
1499 | | * |
1500 | | * **BLOSC_COMPRESSOR=[BLOSCLZ | LZ4 | LZ4HC | ZLIB | ZSTD]**: |
1501 | | * This will call #blosc_set_compressor before the compression process starts. |
1502 | | * |
1503 | | * **BLOSC_NTHREADS=(INTEGER)**: This will call |
1504 | | * #blosc_set_nthreads before the compression process |
1505 | | * starts. |
1506 | | * |
1507 | | * **BLOSC_SPLITMODE=(ALWAYS | NEVER | AUTO | FORWARD_COMPAT)**: |
1508 | | * This will call #blosc1_set_splitmode() before the compression process starts. |
1509 | | * |
1510 | | * **BLOSC_BLOCKSIZE=(INTEGER)**: This will call |
1511 | | * #blosc_set_blocksize before the compression process starts. |
1512 | | * *NOTE:* The blocksize is a critical parameter with |
1513 | | * important restrictions in the allowed values, so use this with care. |
1514 | | * |
1515 | | * **BLOSC_NOLOCK=(ANY VALUE)**: This will call #blosc2_compress_ctx under |
1516 | | * the hood, with the *compressor*, *blocksize* and |
1517 | | * *numinternalthreads* parameters set to the same as the last calls to |
1518 | | * #blosc1_set_compressor, #blosc1_set_blocksize and |
1519 | | * #blosc2_set_nthreads. *BLOSC_CLEVEL*, *BLOSC_SHUFFLE*, *BLOSC_DELTA* and |
1520 | | * *BLOSC_TYPESIZE* environment vars will also be honored. |
1521 | | * |
1522 | | */ |
1523 | | BLOSC_EXPORT int blosc2_compress(int clevel, int doshuffle, int32_t typesize, |
1524 | | const void* src, int32_t srcsize, void* dest, |
1525 | | int32_t destsize); |
1526 | | |
1527 | | |
1528 | | /** |
1529 | | * @brief Decompress a block of compressed data in @p src, put the result in |
1530 | | * @p dest and returns the size of the decompressed block. |
1531 | | * |
1532 | | * @warning The @p src buffer and the @p dest buffer can not overlap. |
1533 | | * |
1534 | | * @remark Decompression is memory safe and guaranteed not to write the @p dest |
1535 | | * buffer more than what is specified in @p destsize. |
1536 | | * |
1537 | | * @remark In case you want to keep under control the number of bytes read from |
1538 | | * source, you can call #blosc1_cbuffer_sizes first to check whether the |
1539 | | * @p nbytes (i.e. the number of bytes to be read from @p src buffer by this |
1540 | | * function) in the compressed buffer is ok with you. |
1541 | | * |
1542 | | * @param src The buffer to be decompressed. |
1543 | | * @param srcsize The size of the buffer to be decompressed. |
1544 | | * @param dest The buffer where the decompressed data will be put. |
1545 | | * @param destsize The size of the @p dest buffer. |
1546 | | * |
1547 | | * @return The number of bytes decompressed. |
1548 | | * If an error occurs, e.g. the compressed data is corrupted or the |
1549 | | * output buffer is not large enough, then a negative value |
1550 | | * will be returned instead. |
1551 | | */ |
1552 | | /* |
1553 | | * Environment variables |
1554 | | * _____________________ |
1555 | | * |
1556 | | * *blosc1_decompress* honors different environment variables to control |
1557 | | * internal parameters without the need of doing that programmatically. |
1558 | | * Here are the ones supported: |
1559 | | * |
1560 | | * **BLOSC_NTHREADS=(INTEGER)**: This will call |
1561 | | * *blosc_set_nthreads(BLOSC_NTHREADS)* before the proper decompression |
1562 | | * process starts. |
1563 | | * |
1564 | | * **BLOSC_NOLOCK=(ANY VALUE)**: This will call *blosc2_decompress_ctx* |
1565 | | * under the hood, with the *numinternalthreads* parameter set to the |
1566 | | * same value as the last call to *blosc2_set_nthreads*. |
1567 | | * |
1568 | | */ |
1569 | | BLOSC_EXPORT int blosc2_decompress(const void* src, int32_t srcsize, |
1570 | | void* dest, int32_t destsize); |
1571 | | |
1572 | | /** |
1573 | | * @brief Context interface to Blosc compression. This does not require a call |
1574 | | * to #blosc2_init and can be called from multithreaded applications |
1575 | | * without the global lock being used, so allowing Blosc be executed |
1576 | | * simultaneously in those scenarios. |
1577 | | * |
1578 | | * @param context A blosc2_context struct with the different compression params. |
1579 | | * @param src The buffer containing the data to be compressed. |
1580 | | * @param srcsize The number of bytes to be compressed from the @p src buffer. |
1581 | | * @param dest The buffer where the compressed data will be put. |
1582 | | * @param destsize The size in bytes of the @p dest buffer. |
1583 | | * |
1584 | | * @return The number of bytes compressed. |
1585 | | * If @p src buffer cannot be compressed into @p destsize, the return |
1586 | | * value is zero and you should discard the contents of the @p dest |
1587 | | * buffer. A negative return value means that an internal error happened. |
1588 | | * It could happen that context is not meant for compression (which is stated in stderr). |
1589 | | * Otherwise, please report it back together with the buffer data causing this |
1590 | | * and compression settings. |
1591 | | */ |
1592 | | BLOSC_EXPORT int blosc2_compress_ctx( |
1593 | | blosc2_context* context, const void* src, int32_t srcsize, void* dest, |
1594 | | int32_t destsize); |
1595 | | |
1596 | | /** |
1597 | | * @brief Context interface to Blosc compression for chunks with variable-length blocks. |
1598 | | * |
1599 | | * @param context A blosc2_context struct with the different compression params. |
1600 | | * @param srcs A list of pointers, one per block. |
1601 | | * @param srcsizes A list of uncompressed sizes, one per block. |
1602 | | * @param nblocks The number of blocks in the chunk. |
1603 | | * @param dest The buffer where the compressed chunk will be put. |
1604 | | * @param destsize The size in bytes of the @p dest buffer. |
1605 | | * |
1606 | | * @return The number of bytes compressed, or a negative error code. |
1607 | | */ |
1608 | | BLOSC_EXPORT int blosc2_vlcompress_ctx( |
1609 | | blosc2_context* context, const void* const* srcs, const int32_t* srcsizes, |
1610 | | int32_t nblocks, void* dest, int32_t destsize); |
1611 | | |
1612 | | |
1613 | | /** |
1614 | | * @brief Context interface to Blosc decompression. This does not require a |
1615 | | * call to #blosc2_init and can be called from multithreaded |
1616 | | * applications without the global lock being used, so allowing Blosc |
1617 | | * be executed simultaneously in those scenarios. |
1618 | | * |
1619 | | * @param context The blosc2_context struct with the different compression params. |
1620 | | * @param src The buffer of compressed data. |
1621 | | * @param srcsize The length of buffer of compressed data. If @p src is a lazy |
1622 | | * chunk returned by #blosc2_schunk_get_lazychunk, pass the size returned by |
1623 | | * #blosc2_schunk_get_lazychunk here rather than the @p cbytes value reported by |
1624 | | * #blosc2_cbuffer_sizes. |
1625 | | * @param dest The buffer where the decompressed data will be put. |
1626 | | * @param destsize The size in bytes of the @p dest buffer. |
1627 | | * |
1628 | | * @warning The @p src buffer and the @p dest buffer can not overlap. |
1629 | | * |
1630 | | * @remark Decompression is memory safe and guaranteed not to write the @p dest |
1631 | | * buffer more than what is specified in @p destsize. |
1632 | | * |
1633 | | * @remark In case you want to keep under control the number of bytes read from |
1634 | | * source, you can call #blosc1_cbuffer_sizes first to check the @p nbytes |
1635 | | * (i.e. the number of bytes to be read from @p src buffer by this function) |
1636 | | * in the compressed buffer. |
1637 | | * |
1638 | | * @remark If #blosc2_set_maskout is called prior to this function, its |
1639 | | * @p block_maskout parameter will be honored for just *one single* shot; |
1640 | | * i.e. the maskout in context will be automatically reset to NULL, so |
1641 | | * mask won't be used next time (unless #blosc2_set_maskout is called again). |
1642 | | * |
1643 | | * @return The number of bytes decompressed (i.e. the maskout blocks are not |
1644 | | * counted). If an error occurs, e.g. the compressed data is corrupted, |
1645 | | * @p destsize is not large enough or context is not meant for decompression, |
1646 | | * then a negative value will be returned instead. |
1647 | | */ |
1648 | | BLOSC_EXPORT int blosc2_decompress_ctx(blosc2_context* context, const void* src, |
1649 | | int32_t srcsize, void* dest, int32_t destsize); |
1650 | | |
1651 | | /** |
1652 | | * @brief Context interface to Blosc decompression for chunks with variable-length blocks. |
1653 | | * |
1654 | | * @param context The blosc2_context struct with the different decompression params. |
1655 | | * @param src The buffer of compressed data. |
1656 | | * @param srcsize The length of buffer of compressed data. |
1657 | | * @param dests On output, one newly allocated buffer per block. |
1658 | | * @param destsizes On output, the uncompressed sizes of the blocks. |
1659 | | * @param maxblocks The number of entries available in @p dests and @p destsizes. |
1660 | | * |
1661 | | * @return The number of blocks decompressed, or a negative error code. |
1662 | | */ |
1663 | | BLOSC_EXPORT int blosc2_vldecompress_ctx(blosc2_context* context, const void* src, |
1664 | | int32_t srcsize, void** dests, |
1665 | | int32_t* destsizes, int32_t maxblocks); |
1666 | | |
1667 | | /** |
1668 | | * @brief Return the number of variable-length blocks stored in a VL-block chunk. |
1669 | | * |
1670 | | * This is a header-only query: it reads only the chunk header and does not |
1671 | | * allocate or decompress anything. |
1672 | | * |
1673 | | * @param src The buffer of compressed data. Must carry the #BLOSC2_VL_BLOCKS flag. |
1674 | | * @param srcsize The length of the compressed data buffer. |
1675 | | * @param nblocks On success, the number of VL blocks in the chunk. |
1676 | | * |
1677 | | * @return 0 on success, or a negative error code. |
1678 | | * Returns #BLOSC2_ERROR_INVALID_PARAM if the chunk does not use VL blocks. |
1679 | | */ |
1680 | | BLOSC_EXPORT int blosc2_vlchunk_get_nblocks(const void* src, int32_t srcsize, |
1681 | | int32_t* nblocks); |
1682 | | |
1683 | | /** |
1684 | | * @brief Decompress a single variable-length block from a VL-block chunk. |
1685 | | * |
1686 | | * Only the requested block is decompressed and allocated; all other blocks |
1687 | | * in the chunk are untouched. |
1688 | | * |
1689 | | * @param context A decompression context (#blosc2_context created with |
1690 | | * #blosc2_create_dctx). If @p src is a lazy chunk (obtained via |
1691 | | * #blosc2_schunk_get_lazychunk), the caller must set @c context->schunk |
1692 | | * to the owning super-chunk (which must have an associated frame) before |
1693 | | * calling this function; the frame is used to read block data from disk. |
1694 | | * @param src The buffer of compressed data. Must carry the #BLOSC2_VL_BLOCKS flag. |
1695 | | * May be a fully in-memory chunk or a lazy chunk proxy. |
1696 | | * @param srcsize The length of the compressed data buffer. |
1697 | | * @param nblock Zero-based index of the block to decompress. |
1698 | | * @param dest On success, points to a newly allocated buffer containing the |
1699 | | * decompressed block. The caller is responsible for freeing this buffer with |
1700 | | * @c free(). |
1701 | | * @param destsize On success, the uncompressed byte size of the block. |
1702 | | * |
1703 | | * @return The uncompressed byte size of the block on success, or a negative |
1704 | | * error code. Returns #BLOSC2_ERROR_INVALID_PARAM if the chunk does not use |
1705 | | * VL blocks or if @p nblock is out of range. |
1706 | | */ |
1707 | | BLOSC_EXPORT int blosc2_vldecompress_block_ctx(blosc2_context* context, |
1708 | | const void* src, int32_t srcsize, |
1709 | | int32_t nblock, uint8_t** dest, |
1710 | | int32_t* destsize); |
1711 | | |
1712 | | /** |
1713 | | * @brief Query the uncompressed size of one VL block, without allocating |
1714 | | * payload, dictionary, or decompression scratch buffers. |
1715 | | * |
1716 | | * @param context A decompression context. For lazy chunks it must be associated |
1717 | | * with the owning SChunk through blosc2_dparams.schunk. |
1718 | | * @param src A compressed VL chunk, or a lazy chunk. |
1719 | | * @param srcsize Actual source buffer length (the size returned by |
1720 | | * #blosc2_schunk_get_lazychunk for lazy chunks, not header cbytes). |
1721 | | * @param nblock Zero-based block index. |
1722 | | * @param nbytes Receives the selected block size on success; unchanged on error. |
1723 | | * @return 0 on success, a negative error code on failure. |
1724 | | * |
1725 | | * Lazy queries read only the selected four-byte size prefix. This is an |
1726 | | * admission query, not full payload validation; decompression revalidates the |
1727 | | * size. Source buffers, contexts and backing storage must not be modified |
1728 | | * concurrently. The context is not reinitialized by this query. |
1729 | | */ |
1730 | | BLOSC_EXPORT int blosc2_vlblock_get_nbytes_ctx(blosc2_context* context, |
1731 | | const void* src, int32_t srcsize, |
1732 | | int32_t nblock, int32_t* nbytes); |
1733 | | |
1734 | | /** |
1735 | | * @brief Decompress one VL block into a caller-owned buffer. |
1736 | | * |
1737 | | * Parameters have the same meaning as #blosc2_vlblock_get_nbytes_ctx. |
1738 | | * @param dest Caller-provided output buffer, which must not overlap src. |
1739 | | * @param destsize Capacity of dest in bytes. |
1740 | | * @return Bytes decompressed, or a negative error code. An insufficient |
1741 | | * destination returns BLOSC2_ERROR_WRITE_BUFFER before dictionary or scratch |
1742 | | * allocation. The output buffer may be partially modified on other errors. |
1743 | | * |
1744 | | * Never writes beyond destsize. Decompression scratch is sized from the |
1745 | | * selected admitted block, not other blocks. Lazy spans are checked against |
1746 | | * their chunk and owning frame before reading. This is not a total-memory |
1747 | | * budget: block tables, compressed input, dictionaries, and codec internals |
1748 | | * have separate allocations. No output buffer is allocated by this function. |
1749 | | */ |
1750 | | BLOSC_EXPORT int blosc2_vldecompress_block_into_ctx(blosc2_context* context, |
1751 | | const void* src, int32_t srcsize, |
1752 | | int32_t nblock, void* dest, |
1753 | | int32_t destsize); |
1754 | | |
1755 | | /** |
1756 | | * @brief Create a chunk made of zeros. |
1757 | | * |
1758 | | * @param cparams The compression parameters. |
1759 | | * @param nbytes The size (in bytes) of the chunk. |
1760 | | * @param dest The buffer where the data chunk will be put. |
1761 | | * @param destsize The size (in bytes) of the @p dest buffer; |
1762 | | * must be BLOSC_EXTENDED_HEADER_LENGTH at least. |
1763 | | * |
1764 | | * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH). |
1765 | | * If negative, there has been an error and @p dest is unusable. |
1766 | | * */ |
1767 | | BLOSC_EXPORT int blosc2_chunk_zeros(blosc2_cparams cparams, int32_t nbytes, |
1768 | | void* dest, int32_t destsize); |
1769 | | |
1770 | | |
1771 | | /** |
1772 | | * @brief Create a chunk made of nans. |
1773 | | * |
1774 | | * @param cparams The compression parameters; |
1775 | | * only 4 bytes (float) and 8 bytes (double) are supported. |
1776 | | * @param nbytes The size (in bytes) of the chunk. |
1777 | | * @param dest The buffer where the data chunk will be put. |
1778 | | * @param destsize The size (in bytes) of the @p dest buffer; |
1779 | | * must be BLOSC_EXTENDED_HEADER_LENGTH at least. |
1780 | | * |
1781 | | * @note Whether the NaNs are floats or doubles will be given by the typesize. |
1782 | | * |
1783 | | * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH). |
1784 | | * If negative, there has been an error and @p dest is unusable. |
1785 | | * */ |
1786 | | BLOSC_EXPORT int blosc2_chunk_nans(blosc2_cparams cparams, int32_t nbytes, |
1787 | | void* dest, int32_t destsize); |
1788 | | |
1789 | | |
1790 | | /** |
1791 | | * @brief Create a chunk made of repeated values. |
1792 | | * |
1793 | | * @param cparams The compression parameters. |
1794 | | * @param nbytes The size (in bytes) of the chunk. |
1795 | | * @param dest The buffer where the data chunk will be put. |
1796 | | * @param destsize The size (in bytes) of the @p dest buffer. |
1797 | | * @param repeatval A pointer to the repeated value (little endian). |
1798 | | * The size of the value is given by @p cparams.typesize param. |
1799 | | * |
1800 | | * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH + typesize). |
1801 | | * If negative, there has been an error and @p dest is unusable. |
1802 | | * */ |
1803 | | BLOSC_EXPORT int blosc2_chunk_repeatval(blosc2_cparams cparams, int32_t nbytes, |
1804 | | void* dest, int32_t destsize, const void* repeatval); |
1805 | | |
1806 | | |
1807 | | /** |
1808 | | * @brief Create a chunk made of uninitialized values. |
1809 | | * |
1810 | | * @param cparams The compression parameters. |
1811 | | * @param nbytes The size (in bytes) of the chunk. |
1812 | | * @param dest The buffer where the data chunk will be put. |
1813 | | * @param destsize The size (in bytes) of the @p dest buffer; |
1814 | | * must be BLOSC_EXTENDED_HEADER_LENGTH at least. |
1815 | | * |
1816 | | * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH). |
1817 | | * If negative, there has been an error and @p dest is unusable. |
1818 | | * */ |
1819 | | BLOSC_EXPORT int blosc2_chunk_uninit(blosc2_cparams cparams, int32_t nbytes, |
1820 | | void* dest, int32_t destsize); |
1821 | | |
1822 | | |
1823 | | /** |
1824 | | * @brief Context interface counterpart for #blosc1_getitem. |
1825 | | * |
1826 | | * @param context Context pointer. |
1827 | | * @param src The compressed buffer from data will be decompressed. |
1828 | | * @param srcsize Compressed buffer length. If @p src is a lazy chunk returned by |
1829 | | * #blosc2_schunk_get_lazychunk, pass the size returned by |
1830 | | * #blosc2_schunk_get_lazychunk here rather than the @p cbytes value reported by |
1831 | | * #blosc2_cbuffer_sizes. |
1832 | | * @param start The position of the first item (of @p typesize size) from where data |
1833 | | * will be retrieved. |
1834 | | * @param nitems The number of items (of @p typesize size) that will be retrieved. |
1835 | | * @param dest The buffer where the decompressed data retrieved will be put. |
1836 | | * @param destsize Output buffer length. |
1837 | | * |
1838 | | * @warning @p start and @p nitems are counted in the typesize that the *chunk* |
1839 | | * records, which is not the typesize the data was created with when the latter |
1840 | | * exceeds #BLOSC_MAX_TYPESIZE. Such buffers are compressed as a plain stream of |
1841 | | * bytes with a stored typesize of 1, so for them one "item" here is one byte. |
1842 | | * Code that does not control the typesize (a library reading whatever its caller |
1843 | | * supplied) should use #blosc2_getitem_bytes_ctx instead, which is unambiguous |
1844 | | * at any typesize. |
1845 | | * |
1846 | | * @return The number of bytes copied to @p dest or a negative value if |
1847 | | * some error happens. |
1848 | | */ |
1849 | | BLOSC_EXPORT int blosc2_getitem_ctx(blosc2_context* context, const void* src, |
1850 | | int32_t srcsize, int start, int nitems, void* dest, |
1851 | | int32_t destsize); |
1852 | | |
1853 | | |
1854 | | /** |
1855 | | * @brief Retrieve a byte range out of a compressed buffer. |
1856 | | * |
1857 | | * This is the counterpart of #blosc2_getitem_ctx that counts in bytes rather |
1858 | | * than in items. Because a buffer whose typesize exceeds #BLOSC_MAX_TYPESIZE is |
1859 | | * stored with a typesize of 1, the unit of #blosc2_getitem_ctx silently changes |
1860 | | * with the typesize; bytes do not. Prefer this entry point in generic code that |
1861 | | * does not pick the typesize itself, and #blosc2_getitem_ctx where the typesize |
1862 | | * is known and at most #BLOSC_MAX_TYPESIZE, as counting in items reads better |
1863 | | * there. |
1864 | | * |
1865 | | * @param context Context pointer. |
1866 | | * @param src The compressed buffer from data will be decompressed. |
1867 | | * @param srcsize Compressed buffer length. If @p src is a lazy chunk returned by |
1868 | | * #blosc2_schunk_get_lazychunk, pass the size returned by |
1869 | | * #blosc2_schunk_get_lazychunk here rather than the @p cbytes value reported by |
1870 | | * #blosc2_cbuffer_sizes. |
1871 | | * @param start The position, counted in bytes into the *uncompressed* buffer, of |
1872 | | * the first byte from where data will be retrieved. |
1873 | | * @param nbytes The number of bytes that will be retrieved. |
1874 | | * @param dest The buffer where the decompressed data retrieved will be put. |
1875 | | * @param destsize Output buffer length. |
1876 | | * |
1877 | | * @remark @p start and @p nbytes must both be multiples of the typesize stored |
1878 | | * in the chunk, or #BLOSC2_ERROR_INVALID_PARAM is returned. This constraint is |
1879 | | * vacuous for typesizes above #BLOSC_MAX_TYPESIZE (the stored typesize is 1), and |
1880 | | * is met automatically by any caller deriving byte offsets from the real |
1881 | | * typesize, so it costs nothing in the cases this function exists for. |
1882 | | * |
1883 | | * @return The number of bytes copied to @p dest or a negative value if |
1884 | | * some error happens. |
1885 | | */ |
1886 | | BLOSC_EXPORT int blosc2_getitem_bytes_ctx(blosc2_context* context, const void* src, |
1887 | | int32_t srcsize, int32_t start, int32_t nbytes, |
1888 | | void* dest, int32_t destsize); |
1889 | | |
1890 | | |
1891 | | /********************************************************************* |
1892 | | Super-chunk related structures and functions. |
1893 | | *********************************************************************/ |
1894 | | |
1895 | 2.22M | #define BLOSC2_MAX_METALAYERS 16 |
1896 | 64.0k | #define BLOSC2_METALAYER_NAME_MAXLEN 31 |
1897 | | |
1898 | | // Allow for a reasonable number of vl metalayers |
1899 | | // max is 64 * 1024 due to msgpack map 16 in frame |
1900 | | // mem usage 8 * 1024 entries for blosc2_schunk.vlmetalayers[] is 64 KB |
1901 | 3.13k | #define BLOSC2_MAX_VLMETALAYERS (8 * 1024) |
1902 | | #define BLOSC2_VLMETALAYERS_NAME_MAXLEN BLOSC2_METALAYER_NAME_MAXLEN |
1903 | | |
1904 | | /** |
1905 | | * @brief This struct is meant for holding storage parameters for a |
1906 | | * for a blosc2 container, allowing to specify, for example, how to interpret |
1907 | | * the contents included in the schunk. |
1908 | | */ |
1909 | | typedef struct { |
1910 | | bool contiguous; |
1911 | | //!< Whether the chunks are contiguous or sparse. |
1912 | | char* urlpath; |
1913 | | //!< The path for persistent storage. If NULL, that means in-memory. |
1914 | | blosc2_cparams* cparams; |
1915 | | //!< The compression params when creating a schunk. |
1916 | | //!< If NULL, sensible defaults are used depending on the context. |
1917 | | blosc2_dparams* dparams; |
1918 | | //!< The decompression params when creating a schunk. |
1919 | | //!< If NULL, sensible defaults are used depending on the context. |
1920 | | blosc2_io *io; |
1921 | | //!< Input/output backend. |
1922 | | } blosc2_storage; |
1923 | | |
1924 | | /** |
1925 | | * @brief Default struct for #blosc2_storage meant for user initialization. |
1926 | | */ |
1927 | | static const blosc2_storage BLOSC2_STORAGE_DEFAULTS = {false, NULL, NULL, NULL, NULL}; |
1928 | | |
1929 | | /** |
1930 | | * @brief Get default struct for compression params meant for user initialization. |
1931 | | */ |
1932 | | BLOSC_EXPORT blosc2_cparams blosc2_get_blosc2_cparams_defaults(void); |
1933 | | |
1934 | | /** |
1935 | | * @brief Get default struct for decompression params meant for user initialization. |
1936 | | */ |
1937 | | BLOSC_EXPORT blosc2_dparams blosc2_get_blosc2_dparams_defaults(void); |
1938 | | |
1939 | | /** |
1940 | | * @brief Get default struct for #blosc2_storage meant for user initialization. |
1941 | | */ |
1942 | | BLOSC_EXPORT blosc2_storage blosc2_get_blosc2_storage_defaults(void); |
1943 | | |
1944 | | /** |
1945 | | * @brief Get default struct for #blosc2_io meant for user initialization. |
1946 | | */ |
1947 | | BLOSC_EXPORT blosc2_io blosc2_get_blosc2_io_defaults(void); |
1948 | | |
1949 | | /** |
1950 | | * @brief Get default struct for #blosc2_stdio_mmap meant for user initialization. |
1951 | | */ |
1952 | | BLOSC_EXPORT blosc2_stdio_mmap blosc2_get_blosc2_stdio_mmap_defaults(void); |
1953 | | |
1954 | | typedef struct blosc2_frame_s blosc2_frame; /* opaque type */ |
1955 | | |
1956 | | /** |
1957 | | * @brief This struct is meant to store metadata information inside |
1958 | | * a #blosc2_schunk, allowing to specify, for example, how to interpret |
1959 | | * the contents included in the schunk. |
1960 | | */ |
1961 | | typedef struct blosc2_metalayer { |
1962 | | char* name; //!< The metalayer identifier for Blosc client (e.g. Blosc2 NDim). |
1963 | | uint8_t* content; //!< The serialized (msgpack preferably) content of the metalayer. |
1964 | | int32_t content_len; //!< The length in bytes of the content. |
1965 | | } blosc2_metalayer; |
1966 | | |
1967 | | /** |
1968 | | * @brief This struct is the standard container for Blosc 2 compressed data. |
1969 | | * |
1970 | | * This is essentially a container for Blosc 1 chunks of compressed data, |
1971 | | * and it allows to overcome the 32-bit limitation in Blosc 1. Optionally, |
1972 | | * a #blosc2_frame can be attached so as to store the compressed chunks contiguously. |
1973 | | */ |
1974 | | typedef struct blosc2_schunk { |
1975 | | uint8_t version; |
1976 | | uint8_t compcode; |
1977 | | //!< The default compressor. Each chunk can override this. |
1978 | | uint8_t compcode_meta; |
1979 | | //!< The default compressor metadata. Each chunk can override this. |
1980 | | uint8_t clevel; |
1981 | | //!< The compression level and other compress params. |
1982 | | uint8_t splitmode; |
1983 | | //!< The split mode. |
1984 | | int32_t typesize; |
1985 | | //!< The type size. |
1986 | | int32_t blocksize; |
1987 | | //!< The requested size of the compressed blocks (0; meaning automatic). |
1988 | | int32_t chunksize; |
1989 | | //!< Size of each chunk. 0 if not a fixed chunksize. |
1990 | | uint8_t flags2; |
1991 | | //!< Secondary chunk-format flags shared by all chunks in the schunk. 0 when empty. |
1992 | | uint8_t use_dict; |
1993 | | //!< Whether to use a dictionary for codec compression (1) or not (0). |
1994 | | uint8_t filters[BLOSC2_MAX_FILTERS]; |
1995 | | //!< The (sequence of) filters. 8-bit per filter. |
1996 | | uint8_t filters_meta[BLOSC2_MAX_FILTERS]; |
1997 | | //!< Metadata for filters. 8-bit per meta-slot. |
1998 | | int64_t nchunks; |
1999 | | //!< Number of chunks in super-chunk. |
2000 | | int64_t current_nchunk; |
2001 | | //!< The current chunk that is being accessed |
2002 | | int64_t nbytes; |
2003 | | //!< The data size (uncompressed). |
2004 | | int64_t cbytes; |
2005 | | //!< The data size + chunks header size (compressed). |
2006 | | uint8_t** data; |
2007 | | //!< Pointer to chunk data pointers buffer. |
2008 | | size_t data_len; |
2009 | | //!< Length of the chunk data pointers buffer. |
2010 | | blosc2_storage* storage; |
2011 | | //!< Pointer to storage info. |
2012 | | blosc2_frame* frame; |
2013 | | //!< Pointer to frame used as store for chunks. |
2014 | | //!<uint8_t* ctx; |
2015 | | //!< Context for the thread holder. NULL if not acquired. |
2016 | | blosc2_context* cctx; |
2017 | | //!< Context for compression |
2018 | | blosc2_context* dctx; |
2019 | | //!< Context for decompression. |
2020 | | struct blosc2_metalayer *metalayers[BLOSC2_MAX_METALAYERS]; |
2021 | | //!< The array of metalayers. |
2022 | | uint16_t nmetalayers; |
2023 | | //!< The number of metalayers in the super-chunk |
2024 | | struct blosc2_metalayer *vlmetalayers[BLOSC2_MAX_VLMETALAYERS]; |
2025 | | //<! The array of variable-length metalayers. |
2026 | | int16_t nvlmetalayers; |
2027 | | //!< The number of variable-length metalayers. |
2028 | | void *tuner_params; |
2029 | | //!< Tune configuration. |
2030 | | int tuner_id; |
2031 | | //<! Id for tuner |
2032 | | int8_t ndim; |
2033 | | //<! The ndim (mainly for ZFP usage) |
2034 | | int64_t *blockshape; |
2035 | | //<! The blockshape (mainly for ZFP usage) |
2036 | | bool view; |
2037 | | //<! Whether the schunk is a view or not. |
2038 | | int64_t change_tick; |
2039 | | //!< Bumped whenever the (vl)metalayers change: by local mutations and by the |
2040 | | //!< refresh that follows a mutation made through another handle on the same |
2041 | | //!< on-disk frame. Upper layers compare it against a cached value to know |
2042 | | //!< whether their deserialized view of the metalayers is still current. |
2043 | | } blosc2_schunk; |
2044 | | |
2045 | | |
2046 | | /** |
2047 | | * @brief Create a new super-chunk. |
2048 | | * |
2049 | | * @param storage The storage properties. |
2050 | | * |
2051 | | * @remark In case that storage.urlpath is not NULL, the data is stored |
2052 | | * on-disk. If the data file(s) exist, they are *overwritten*. |
2053 | | * |
2054 | | * @return The new super-chunk. |
2055 | | */ |
2056 | | BLOSC_EXPORT blosc2_schunk* blosc2_schunk_new(blosc2_storage *storage); |
2057 | | |
2058 | | /** |
2059 | | * Create a copy of a super-chunk. |
2060 | | * |
2061 | | * @param schunk The super-chunk to be copied. |
2062 | | * @param storage The storage properties. |
2063 | | * |
2064 | | * @remark If @p schunk has a postfilter, the copy holds the data as read from |
2065 | | * @p schunk, i.e. the output of the postfilter (except with variable-length |
2066 | | * blocks, whose chunks are always copied as they are). The copy does not |
2067 | | * inherit the postfilter: it reads that output as it is, unless |
2068 | | * @p storage sets a postfilter in its dparams, which then runs on it again. |
2069 | | * To serialize the chunks as they are stored instead, see |
2070 | | * blosc2_schunk_to_buffer(). |
2071 | | * |
2072 | | * @return The new super-chunk. |
2073 | | */ |
2074 | | BLOSC_EXPORT blosc2_schunk* blosc2_schunk_copy(blosc2_schunk *schunk, blosc2_storage *storage); |
2075 | | |
2076 | | /** |
2077 | | * @brief Create a super-chunk out of a contiguous frame buffer. |
2078 | | * |
2079 | | * @param cframe The buffer of the in-memory frame. |
2080 | | * @param copy Whether the super-chunk should make a copy of |
2081 | | * the @p cframe data or not. The copy will be made to an internal |
2082 | | * sparse frame. |
2083 | | * |
2084 | | * @remark If copy is false, the @p cframe buffer passed will be owned |
2085 | | * by the super-chunk and will be automatically freed when |
2086 | | * blosc2_schunk_free() is called. If the user frees it after the |
2087 | | * opening, bad things will happen. Don't do that (or set @p copy). |
2088 | | * |
2089 | | * @param len The length of the buffer (in bytes). |
2090 | | * |
2091 | | * @return The new super-chunk. |
2092 | | */ |
2093 | | BLOSC_EXPORT blosc2_schunk* blosc2_schunk_from_buffer(uint8_t *cframe, int64_t len, bool copy); |
2094 | | |
2095 | | /** |
2096 | | * @brief Set the private `avoid_cframe_free` field in a frame. |
2097 | | * |
2098 | | * @param schunk The super-chunk referencing the frame. |
2099 | | * @param avoid_cframe_free The value to set in the blosc2_frame_s structure. |
2100 | | * |
2101 | | * @warning If you set it to `true` you will be responsible of freeing it. |
2102 | | */ |
2103 | | BLOSC_EXPORT void blosc2_schunk_avoid_cframe_free(blosc2_schunk *schunk, bool avoid_cframe_free); |
2104 | | |
2105 | | /** |
2106 | | * @brief Take the exclusive frame lock of a super-chunk and hold it across |
2107 | | * several operations. |
2108 | | * |
2109 | | * When the (opt-in) file locking is enabled on the handle (see the `locking` |
2110 | | * member of #blosc2_stdio_params, or the BLOSC_LOCKING environment variable), |
2111 | | * every schunk operation locks and unlocks the on-disk frame individually, so |
2112 | | * a sequence of operations is not atomic as a whole. Bracketing the sequence |
2113 | | * between blosc2_schunk_lock() and blosc2_schunk_unlock() makes it atomic |
2114 | | * against other handles and processes: the operations inside nest on the |
2115 | | * already-held lock instead of re-acquiring it. |
2116 | | * |
2117 | | * Everything inside the bracket is serialized exclusively — including plain |
2118 | | * reads through other locked handles — so keep brackets short. The bracket |
2119 | | * is re-entrant on the same handle. When locking is not enabled on the |
2120 | | * handle (or the super-chunk is not disk-based), this is a no-op returning |
2121 | | * success, so callers do not need to check whether locking is active. |
2122 | | * |
2123 | | * @param schunk The super-chunk. |
2124 | | * |
2125 | | * @return 0 on success; a negative error code (e.g. BLOSC2_ERROR_LOCK) |
2126 | | * otherwise. |
2127 | | */ |
2128 | | BLOSC_EXPORT int blosc2_schunk_lock(blosc2_schunk *schunk); |
2129 | | |
2130 | | /** |
2131 | | * @brief Release the frame lock taken with blosc2_schunk_lock(). |
2132 | | * |
2133 | | * @param schunk The super-chunk. |
2134 | | * |
2135 | | * @return 0 on success; a negative error code otherwise. Like |
2136 | | * blosc2_schunk_lock(), a no-op when locking is not enabled. |
2137 | | */ |
2138 | | BLOSC_EXPORT int blosc2_schunk_unlock(blosc2_schunk *schunk); |
2139 | | |
2140 | | /** |
2141 | | * @brief Start a sequence of chunk updates of a super-chunk. |
2142 | | * |
2143 | | * A contiguous frame keeps its chunks back to back, so an update that changes |
2144 | | * the compressed size of a chunk moves everything stored behind it: updating |
2145 | | * all n chunks one by one moves ~n^2/2 chunks. Between |
2146 | | * blosc2_schunk_begin_updates() and blosc2_schunk_end_updates(), such an |
2147 | | * update moves nothing else. The chunk goes into its old place when it fits |
2148 | | * there (together with the free space right before and after it), else to the |
2149 | | * end of the frame, and the space left over stays free until |
2150 | | * blosc2_schunk_end_updates() closes all of it in one pass, in which every |
2151 | | * chunk moves at most once. The frame is valid at every point in between, |
2152 | | * just larger, and `cbytes` includes the free space until the end. |
2153 | | * |
2154 | | * Writing functions that update several chunks (b2nd_set_slice_cbuffer(), |
2155 | | * b2nd_set_orthogonal_selection(), b2nd_resize(), |
2156 | | * blosc2_schunk_set_slice_buffer()) do this by themselves; use it to bracket |
2157 | | * a sequence of such calls or of blosc2_schunk_update_chunk() calls. |
2158 | | * |
2159 | | * It also takes the exclusive frame lock, as blosc2_schunk_lock() does, until |
2160 | | * blosc2_schunk_end_updates(). Brackets nest. For super-chunks that are not |
2161 | | * contiguous frames this is just the lock (a no-op without locking). A frame |
2162 | | * whose bracket is never closed stays valid; its free space just stays, like |
2163 | | * the space of deleted chunks. |
2164 | | * |
2165 | | * @param schunk The super-chunk. |
2166 | | * |
2167 | | * @return 0 on success; a negative error code otherwise (then nothing has started). |
2168 | | */ |
2169 | | BLOSC_EXPORT int blosc2_schunk_begin_updates(blosc2_schunk *schunk); |
2170 | | |
2171 | | /** |
2172 | | * @brief End a sequence of chunk updates started with blosc2_schunk_begin_updates(). |
2173 | | * |
2174 | | * The outermost call closes the free space that the updates left in a |
2175 | | * contiguous frame (moving each chunk at most once; a frame on disk is cut to |
2176 | | * its new length) and releases the lock. |
2177 | | * |
2178 | | * @param schunk The super-chunk. |
2179 | | * |
2180 | | * @return 0 on success; a negative error code otherwise. The lock is released |
2181 | | * either way. |
2182 | | */ |
2183 | | BLOSC_EXPORT int blosc2_schunk_end_updates(blosc2_schunk *schunk); |
2184 | | |
2185 | | /** |
2186 | | * @brief Re-sync the cached counters (nchunks, nbytes, cbytes) and |
2187 | | * metalayers of a disk-based super-chunk when another handle — possibly in |
2188 | | * another process — has changed it. |
2189 | | * |
2190 | | * This gives a deterministic sync point for readers in a single-writer, |
2191 | | * multiple-readers (SWMR) workflow: after calling it, the counters in |
2192 | | * @p schunk reflect the current on-disk state. Data-access functions |
2193 | | * already do this implicitly, so calling it is only needed to observe |
2194 | | * changes without touching data (e.g. when polling). |
2195 | | * |
2196 | | * A no-op for super-chunks not backed by an on-disk frame. |
2197 | | * |
2198 | | * @param schunk The super-chunk to refresh. |
2199 | | * |
2200 | | * @return 1 if the counters were re-synced, 0 if they were already current, |
2201 | | * or a negative error code. |
2202 | | */ |
2203 | | BLOSC_EXPORT int blosc2_schunk_refresh(blosc2_schunk *schunk); |
2204 | | |
2205 | | /** |
2206 | | * @brief Open an existing super-chunk that is on-disk (frame). No in-memory copy is made. |
2207 | | * |
2208 | | * @param urlpath The file name. |
2209 | | * |
2210 | | * @return The new super-chunk. NULL if not found or not in frame format. |
2211 | | */ |
2212 | | BLOSC_EXPORT blosc2_schunk* blosc2_schunk_open(const char* urlpath); |
2213 | | |
2214 | | /** |
2215 | | * @brief Open an existing super-chunk that is on-disk (frame). No in-memory copy is made. |
2216 | | * |
2217 | | * @param urlpath The file name. |
2218 | | * |
2219 | | * @param offset The frame offset. |
2220 | | * |
2221 | | * @return The new super-chunk. NULL if not found or not in frame format. |
2222 | | */ |
2223 | | BLOSC_EXPORT blosc2_schunk* blosc2_schunk_open_offset(const char* urlpath, int64_t offset); |
2224 | | |
2225 | | /** |
2226 | | * @brief Open an existing super-chunk (no copy is made) using a user-defined I/O interface. |
2227 | | * |
2228 | | * @param urlpath The file name. |
2229 | | * |
2230 | | * @param udio The user-defined I/O interface. |
2231 | | * |
2232 | | * @return The new super-chunk. |
2233 | | */ |
2234 | | BLOSC_EXPORT blosc2_schunk* blosc2_schunk_open_udio(const char* urlpath, const blosc2_io *udio); |
2235 | | |
2236 | | /** |
2237 | | * @brief Open an existing super-chunk (no copy is made) using a user-defined I/O interface. |
2238 | | * |
2239 | | * @param urlpath The file name. |
2240 | | * |
2241 | | * @param offset The frame offset. |
2242 | | * |
2243 | | * @param udio The user-defined I/O interface. |
2244 | | * |
2245 | | * @return The new super-chunk. |
2246 | | */ |
2247 | | BLOSC_EXPORT blosc2_schunk* blosc2_schunk_open_offset_udio(const char* urlpath, int64_t offset, const blosc2_io *udio); |
2248 | | |
2249 | | /** |
2250 | | * @brief Convert a super-chunk into a contiguous frame buffer. |
2251 | | * |
2252 | | * @param schunk The super-chunk to convert. |
2253 | | * @param cframe The address of the destination buffer (output). |
2254 | | * @param needs_free The pointer to a boolean indicating if it is the user's |
2255 | | * responsibility to free the resulting @p cframe buffer or not. |
2256 | | * |
2257 | | * @remark The frame holds the chunks as they are stored, whatever the storage of |
2258 | | * @p schunk: a postfilter of @p schunk is not applied (unlike in |
2259 | | * blosc2_schunk_copy()), and it is not part of the frame either, so a |
2260 | | * super-chunk made from the frame reads the stored values until the same |
2261 | | * postfilter is set on it again. For a frame with the postfiltered values, |
2262 | | * serialize a blosc2_schunk_copy() of @p schunk. The same holds for |
2263 | | * blosc2_schunk_to_buffer_alloc(), blosc2_schunk_to_file() and |
2264 | | * blosc2_schunk_append_file(). |
2265 | | * |
2266 | | * @return If successful, return the size of the (frame) @p cframe buffer. |
2267 | | * Else, a negative value. |
2268 | | */ |
2269 | | BLOSC_EXPORT int64_t blosc2_schunk_to_buffer(blosc2_schunk* schunk, uint8_t** cframe, bool* needs_free); |
2270 | | |
2271 | | /** |
2272 | | * @brief Convert a super-chunk into a contiguous frame in a buffer provided by the caller. |
2273 | | * |
2274 | | * The frame is the same that blosc2_schunk_to_buffer() returns, but it goes into |
2275 | | * memory that the caller owns, e.g. the final destination of the serialized |
2276 | | * super-chunk, which saves copying it there afterwards. Unless the super-chunk |
2277 | | * already is a frame in memory, the frame is moreover written in place instead |
2278 | | * of being built first. |
2279 | | * |
2280 | | * @param schunk The super-chunk to convert. |
2281 | | * @param alloc Called once with the exact length of the frame; it returns a buffer |
2282 | | * of (at least) that many bytes, or NULL to give up. |
2283 | | * @param user_data Passed on to @p alloc. |
2284 | | * |
2285 | | * @remark Like blosc2_schunk_to_buffer(), it writes the chunks as they are |
2286 | | * stored, without applying a postfilter of @p schunk. |
2287 | | * |
2288 | | * @return If successful, return the size of the frame written into the buffer. |
2289 | | * Else (also when @p alloc returns NULL), a negative value. |
2290 | | */ |
2291 | | BLOSC_EXPORT int64_t blosc2_schunk_to_buffer_alloc(blosc2_schunk* schunk, |
2292 | | uint8_t* (*alloc)(int64_t len, void* user_data), |
2293 | | void* user_data); |
2294 | | |
2295 | | /** |
2296 | | * @brief Store a super-chunk into a file. |
2297 | | * |
2298 | | * @param schunk The super-chunk to write. |
2299 | | * @param urlpath The path for persistent storage. |
2300 | | * |
2301 | | * @remark The file holds the same frame that blosc2_schunk_to_buffer() |
2302 | | * returns: the chunks as they are stored, without applying a postfilter of |
2303 | | * @p schunk. |
2304 | | * |
2305 | | * @return If successful, return the size of the (fileframe) in @p urlpath. |
2306 | | * Else, a negative value. |
2307 | | */ |
2308 | | BLOSC_EXPORT int64_t blosc2_schunk_to_file(blosc2_schunk* schunk, const char* urlpath); |
2309 | | |
2310 | | /** |
2311 | | * @brief Append a super-chunk into a file. |
2312 | | * |
2313 | | * @param schunk The super-chunk to write. |
2314 | | * @param urlpath The path for persistent storage. |
2315 | | * |
2316 | | * @remark It appends the same frame that blosc2_schunk_to_buffer() returns: |
2317 | | * the chunks as they are stored, without applying a postfilter of @p schunk. |
2318 | | * |
2319 | | * @return If successful, return the offset where @p schunk has been appended in @p urlpath. |
2320 | | * Else, a negative value. |
2321 | | */ |
2322 | | BLOSC_EXPORT int64_t blosc2_schunk_append_file(blosc2_schunk* schunk, const char* urlpath); |
2323 | | |
2324 | | /** |
2325 | | * @brief Release resources from a super-chunk. |
2326 | | * |
2327 | | * @param schunk The super-chunk to be freed. |
2328 | | * |
2329 | | * @remark All the memory resources attached to the super-chunk are freed. |
2330 | | * If the super-chunk is on-disk, the data continues there for a later |
2331 | | * re-opening. |
2332 | | * |
2333 | | * @return 0 if success. |
2334 | | */ |
2335 | | BLOSC_EXPORT int blosc2_schunk_free(blosc2_schunk *schunk); |
2336 | | |
2337 | | /** |
2338 | | * @brief Append an existing @p chunk to a super-chunk. |
2339 | | * |
2340 | | * @param schunk The super-chunk where the chunk will be appended. |
2341 | | * @param chunk The @p chunk to append. If @p copy is true, an internal copy is made, so |
2342 | | * @p chunk can be reused or freed if desired. If @p copy is false, ownership is transferred |
2343 | | * to the super-chunk on success; on error, the caller retains ownership. |
2344 | | * @param copy Whether the chunk should be copied internally or can be used as-is. |
2345 | | * |
2346 | | * @return The number of chunks in super-chunk. If some problem is |
2347 | | * detected, this number will be negative. |
2348 | | */ |
2349 | | BLOSC_EXPORT int64_t blosc2_schunk_append_chunk(blosc2_schunk *schunk, uint8_t *chunk, bool copy); |
2350 | | |
2351 | | /** |
2352 | | * @brief Update a chunk at a specific position in a super-chunk. |
2353 | | * |
2354 | | * @param schunk The super-chunk where the chunk will be updated. |
2355 | | * @param nchunk The position where the chunk will be updated. |
2356 | | * @param chunk The new @p chunk. If @p copy is true, an internal copy is made, so |
2357 | | * @p chunk can be reused or freed if desired. If @p copy is false, ownership is transferred |
2358 | | * to the super-chunk on success; on error, the caller retains ownership. |
2359 | | * @param copy Whether the chunk should be copied internally or can be used as-is. |
2360 | | * |
2361 | | * @return The number of chunks in super-chunk. If some problem is |
2362 | | * detected, this number will be negative. |
2363 | | */ |
2364 | | BLOSC_EXPORT int64_t blosc2_schunk_update_chunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t *chunk, bool copy); |
2365 | | |
2366 | | /** |
2367 | | * @brief Insert a chunk at a specific position in a super-chunk. |
2368 | | * |
2369 | | * @param schunk The super-chunk where the chunk will be inserted. |
2370 | | * @param nchunk The position where the chunk will be inserted. |
2371 | | * @param chunk The @p chunk to insert. If @p copy is true, an internal copy is made, so |
2372 | | * @p chunk can be reused or freed if desired. If @p copy is false, ownership is transferred |
2373 | | * to the super-chunk on success; on error, the caller retains ownership. |
2374 | | * @param copy Whether the chunk should be copied internally or can be used as-is. |
2375 | | * |
2376 | | * @return The number of chunks in super-chunk. If some problem is |
2377 | | * detected, this number will be negative. |
2378 | | */ |
2379 | | BLOSC_EXPORT int64_t blosc2_schunk_insert_chunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t *chunk, bool copy); |
2380 | | |
2381 | | /** |
2382 | | * @brief Delete a chunk at a specific position in a super-chunk. |
2383 | | * |
2384 | | * @param schunk The super-chunk where the chunk will be deleted. |
2385 | | * @param nchunk The position where the chunk will be deleted. |
2386 | | * |
2387 | | * @return The number of chunks in super-chunk. If some problem is |
2388 | | * detected, this number will be negative. |
2389 | | */ |
2390 | | BLOSC_EXPORT int64_t blosc2_schunk_delete_chunk(blosc2_schunk *schunk, int64_t nchunk); |
2391 | | |
2392 | | /** |
2393 | | * @brief Append a @p src data buffer to a super-chunk. |
2394 | | * |
2395 | | * @param schunk The super-chunk where data will be appended. |
2396 | | * @param src The buffer of data to compress. |
2397 | | * @param nbytes The size of the @p src buffer. |
2398 | | * |
2399 | | * @return The number of chunks in super-chunk. If some problem is |
2400 | | * detected, this number will be negative. |
2401 | | */ |
2402 | | BLOSC_EXPORT int64_t blosc2_schunk_append_buffer(blosc2_schunk *schunk, const void *src, int32_t nbytes); |
2403 | | |
2404 | | /** |
2405 | | * @brief Compress a @p src data buffer and insert it as a new chunk at a specific |
2406 | | * position in a super-chunk. |
2407 | | * |
2408 | | * Same as compressing @p src with the super-chunk's context and inserting the |
2409 | | * result with blosc2_schunk_insert_chunk(), but frames are written from a |
2410 | | * compression buffer that the calling thread reuses, not a new one per chunk. |
2411 | | * |
2412 | | * @param schunk The super-chunk where the chunk will be inserted. |
2413 | | * @param nchunk The position where the chunk will be inserted. |
2414 | | * @param src The buffer of data to compress. |
2415 | | * @param nbytes The size of the @p src buffer. |
2416 | | * |
2417 | | * @return The number of chunks in super-chunk. If some problem is |
2418 | | * detected, this number will be negative. |
2419 | | */ |
2420 | | BLOSC_EXPORT int64_t blosc2_schunk_insert_buffer(blosc2_schunk *schunk, int64_t nchunk, |
2421 | | const void *src, int32_t nbytes); |
2422 | | |
2423 | | /** |
2424 | | * @brief Compress a @p src data buffer and replace the chunk at a specific |
2425 | | * position in a super-chunk with it. |
2426 | | * |
2427 | | * Same as compressing @p src with the super-chunk's context and storing the |
2428 | | * result with blosc2_schunk_update_chunk(), but frames are written from a |
2429 | | * compression buffer that the calling thread reuses, not a new one per chunk. |
2430 | | * |
2431 | | * @param schunk The super-chunk where the chunk will be updated. |
2432 | | * @param nchunk The position of the chunk to update. |
2433 | | * @param src The buffer of data to compress. |
2434 | | * @param nbytes The size of the @p src buffer. |
2435 | | * |
2436 | | * @return The number of chunks in super-chunk. If some problem is |
2437 | | * detected, this number will be negative. |
2438 | | */ |
2439 | | BLOSC_EXPORT int64_t blosc2_schunk_update_buffer(blosc2_schunk *schunk, int64_t nchunk, |
2440 | | const void *src, int32_t nbytes); |
2441 | | |
2442 | | /** |
2443 | | * @brief Decompress and return the @p nchunk chunk of a super-chunk. |
2444 | | * |
2445 | | * If the chunk is uncompressed successfully, it is put in the @p *dest |
2446 | | * pointer. |
2447 | | * |
2448 | | * @param schunk The super-chunk from where the chunk will be decompressed. |
2449 | | * @param nchunk The chunk to be decompressed (0 indexed). |
2450 | | * @param dest The buffer where the decompressed data will be put. |
2451 | | * @param nbytes The size of the area pointed by @p *dest. |
2452 | | * |
2453 | | * @warning You must make sure that you have enough space to store the |
2454 | | * uncompressed data. |
2455 | | * |
2456 | | * @return The size of the decompressed chunk or 0 if it is non-initialized. If some problem is |
2457 | | * detected, a negative code is returned instead. |
2458 | | */ |
2459 | | BLOSC_EXPORT int blosc2_schunk_decompress_chunk(blosc2_schunk *schunk, int64_t nchunk, void *dest, int32_t nbytes); |
2460 | | |
2461 | | /** |
2462 | | * @brief Return a compressed chunk that is part of a super-chunk in the @p chunk parameter. |
2463 | | * |
2464 | | * @param schunk The super-chunk from where to extract a chunk. |
2465 | | * @param nchunk The chunk to be extracted (0 indexed). |
2466 | | * @param chunk The pointer to the chunk of compressed data. |
2467 | | * @param needs_free The pointer to a boolean indicating if it is the user's |
2468 | | * responsibility to free the chunk returned or not. |
2469 | | * |
2470 | | * @warning If the super-chunk is backed by a frame that is disk-based, a buffer is allocated for the |
2471 | | * (compressed) chunk, and hence a free is needed. |
2472 | | * You can check whether the chunk requires a free with the @p needs_free parameter. |
2473 | | * If the chunk does not need a free, it means that a pointer to the location in the super-chunk |
2474 | | * (or the backing in-memory frame) is returned in the @p chunk parameter. |
2475 | | * |
2476 | | * @note If the frame on disk has been modified through another handle (or process), |
2477 | | * the handle re-syncs its view automatically on the next access. A concurrent |
2478 | | * (non-serialized) write can still surface as #BLOSC2_ERROR_FILE_READ; retry or |
2479 | | * reopen the super-chunk in that case. Writers must be serialized externally. |
2480 | | * |
2481 | | * @return The size of the (compressed) chunk or 0 if it is non-initialized. If some problem is |
2482 | | * detected, a negative code is returned instead. |
2483 | | */ |
2484 | | BLOSC_EXPORT int blosc2_schunk_get_chunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t **chunk, |
2485 | | bool *needs_free); |
2486 | | |
2487 | | /** |
2488 | | * @brief Return a (lazy) compressed chunk that is part of a super-chunk in the @p chunk parameter. |
2489 | | * |
2490 | | * @param schunk The super-chunk from where to extract a chunk. |
2491 | | * @param nchunk The chunk to be extracted (0 indexed). |
2492 | | * @param chunk The pointer to the (lazy) chunk of compressed data. |
2493 | | * @param needs_free The pointer to a boolean indicating if it is the user's |
2494 | | * responsibility to free the chunk returned or not. |
2495 | | * |
2496 | | * @note For disk-based frames, a lazy chunk is always returned. |
2497 | | * |
2498 | | * @warning Currently, a lazy chunk can only be used by #blosc2_decompress_ctx and #blosc2_getitem_ctx. |
2499 | | * |
2500 | | * @note The return value is the effective size of the returned buffer. For lazy |
2501 | | * chunks, use this return value as @p srcsize when calling |
2502 | | * #blosc2_decompress_ctx or #blosc2_getitem_ctx. This value can be larger than |
2503 | | * the @p cbytes reported by #blosc2_cbuffer_sizes because lazy chunks carry |
2504 | | * extra trailer metadata. |
2505 | | * |
2506 | | * @warning If the super-chunk is backed by a frame that is disk-based, a buffer is allocated for the |
2507 | | * (compressed) chunk, and hence a free is needed. |
2508 | | * You can check whether requires a free with the @p needs_free parameter. |
2509 | | * If the chunk does not need a free, it means that a pointer to the location in the super-chunk |
2510 | | * (or the backing in-memory frame) is returned in the @p chunk parameter. In this case the returned |
2511 | | * chunk is not lazy. |
2512 | | * |
2513 | | * @note If the frame on disk has been modified through another handle (or process), |
2514 | | * the handle re-syncs its view automatically on the next access. A concurrent |
2515 | | * (non-serialized) write can still surface as #BLOSC2_ERROR_FILE_READ; retry or |
2516 | | * reopen the super-chunk in that case. Writers must be serialized externally. |
2517 | | * |
2518 | | * @return The size of the returned chunk buffer, or 0 if it is non-initialized. |
2519 | | * If some problem is detected, a negative code is returned instead. For regular |
2520 | | * chunks this matches the compressed chunk size. Note that a lazy chunk is |
2521 | | * somewhat larger than a regular chunk because of the trailer section (for |
2522 | | * details see `README_CHUNK_FORMAT.rst`). |
2523 | | */ |
2524 | | BLOSC_EXPORT int blosc2_schunk_get_lazychunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t **chunk, |
2525 | | bool *needs_free); |
2526 | | |
2527 | | /** |
2528 | | * @brief Where a chunk of a super-chunk stored in a contiguous frame on disk is in its file. |
2529 | | * |
2530 | | * This lets callers read (or ask the operating system to read ahead) the bytes of the chunks they will need, |
2531 | | * e.g. in parallel, without reading any chunk to find them: the answer comes from the frame header and the chunk |
2532 | | * index, which opening the frame has read already (and which stay in the page cache). |
2533 | | * |
2534 | | * @param schunk The super-chunk. |
2535 | | * @param nchunk The chunk. |
2536 | | * @param offset Position of its first byte in the file. |
2537 | | * @param nbytes Its size in the file: the distance to the next chunk, which is the compressed size for chunks |
2538 | | * written in order and otherwise at most the largest possible chunk size. 0 for chunks that the index codes as a |
2539 | | * special value (zeros, NaNs, uninitialized), which have no bytes in the file. |
2540 | | * |
2541 | | * @return 0 on success, or a negative error code (e.g. for super-chunks in memory or in sparse frames). |
2542 | | */ |
2543 | | BLOSC_EXPORT int blosc2_schunk_get_chunk_range(blosc2_schunk *schunk, int64_t nchunk, int64_t *offset, |
2544 | | int64_t *nbytes); |
2545 | | |
2546 | | /** |
2547 | | * @brief Decompress a single variable-length block from a schunk chunk. |
2548 | | * |
2549 | | * Convenience wrapper around #blosc2_vldecompress_block_ctx that fetches the |
2550 | | * compressed chunk from @p schunk and decompresses only the requested VL block. |
2551 | | * Only the requested block is allocated and decompressed; all other blocks in |
2552 | | * the chunk are untouched. |
2553 | | * |
2554 | | * @param schunk The super-chunk from which to read. |
2555 | | * @param nchunk Zero-based index of the chunk inside the super-chunk. |
2556 | | * @param nblock Zero-based index of the VL block inside the chunk. |
2557 | | * @param dest On success, points to a newly allocated buffer containing the |
2558 | | * decompressed block. The caller is responsible for freeing it with @c free(). |
2559 | | * @param destsize On success, the uncompressed byte size of the block. |
2560 | | * |
2561 | | * @return The uncompressed byte size of the block on success, or a negative |
2562 | | * error code. Returns #BLOSC2_ERROR_INVALID_PARAM if the chunk does not use |
2563 | | * VL blocks or if @p nblock is out of range. |
2564 | | */ |
2565 | | BLOSC_EXPORT int blosc2_schunk_get_vlblock(blosc2_schunk *schunk, int64_t nchunk, |
2566 | | int32_t nblock, |
2567 | | uint8_t **dest, int32_t *destsize); |
2568 | | |
2569 | | /** |
2570 | | * @brief Fill buffer with a schunk slice. |
2571 | | * |
2572 | | * @param schunk The super-chunk from where to extract a slice. |
2573 | | * @param start Index (0-based) where the slice begins. |
2574 | | * @param stop The first index (0-based) that is not in the selected slice. |
2575 | | * @param buffer The buffer where the data will be stored. |
2576 | | * |
2577 | | * @warning You must make sure that you have enough space in buffer to store the |
2578 | | * uncompressed data. |
2579 | | * |
2580 | | * @return An error code. |
2581 | | */ |
2582 | | BLOSC_EXPORT int blosc2_schunk_get_slice_buffer(blosc2_schunk *schunk, int64_t start, int64_t stop, void *buffer); |
2583 | | |
2584 | | /** |
2585 | | * @brief Fill buffer with elements at the given coordinates from a schunk. |
2586 | | * |
2587 | | * @param schunk The super-chunk from where to extract elements. |
2588 | | * @param ncoords The number of coordinates. |
2589 | | * @param coords The coordinates of the elements to be extracted. |
2590 | | * @param buffer The buffer where the data will be stored. |
2591 | | * |
2592 | | * @warning You must make sure that you have enough space in buffer to store the |
2593 | | * uncompressed data. |
2594 | | * |
2595 | | * @return An error code. |
2596 | | */ |
2597 | | BLOSC_EXPORT int blosc2_schunk_get_sparse_buffer(blosc2_schunk *schunk, int64_t ncoords, |
2598 | | const int64_t* coords, void *buffer); |
2599 | | |
2600 | | /** |
2601 | | * @brief Update a schunk slice from buffer. |
2602 | | * |
2603 | | * @param schunk The super-chunk where to set the slice. |
2604 | | * @param start Index (0-based) where the slice begins. |
2605 | | * @param stop The first index (0-based) that is not in the selected slice. |
2606 | | * @param buffer The buffer containing the data to set. |
2607 | | * |
2608 | | * |
2609 | | * @return An error code. |
2610 | | */ |
2611 | | BLOSC_EXPORT int blosc2_schunk_set_slice_buffer(blosc2_schunk *schunk, int64_t start, int64_t stop, void *buffer); |
2612 | | |
2613 | | /** |
2614 | | * @brief Return the @p cparams associated to a super-chunk. |
2615 | | * |
2616 | | * @param schunk The super-chunk from where to extract the compression parameters. |
2617 | | * @param cparams The pointer where the compression params will be returned. |
2618 | | * |
2619 | | * @warning A new struct is allocated, and the user should free it after use. |
2620 | | * |
2621 | | * @return 0 if succeeds. Else a negative code is returned. |
2622 | | */ |
2623 | | BLOSC_EXPORT int blosc2_schunk_get_cparams(blosc2_schunk *schunk, blosc2_cparams **cparams); |
2624 | | |
2625 | | /** |
2626 | | * @brief Return the @p dparams struct associated to a super-chunk. |
2627 | | * |
2628 | | * @param schunk The super-chunk from where to extract the decompression parameters. |
2629 | | * @param dparams The pointer where the decompression params will be returned. |
2630 | | * |
2631 | | * @warning A new struct is allocated, and the user should free it after use. |
2632 | | * |
2633 | | * @return 0 if succeeds. Else a negative code is returned. |
2634 | | */ |
2635 | | BLOSC_EXPORT int blosc2_schunk_get_dparams(blosc2_schunk *schunk, blosc2_dparams **dparams); |
2636 | | |
2637 | | /** |
2638 | | * @brief Reorder the chunk offsets of an existing super-chunk. |
2639 | | * |
2640 | | * @param schunk The super-chunk whose chunk offsets are to be reordered. |
2641 | | * @param offsets_order The new order of the chunk offsets. |
2642 | | * |
2643 | | * @return 0 if succeeds. Else a negative code is returned. |
2644 | | */ |
2645 | | BLOSC_EXPORT int blosc2_schunk_reorder_offsets(blosc2_schunk *schunk, int64_t *offsets_order); |
2646 | | |
2647 | | /** |
2648 | | * @brief Get the length (in bytes) of the internal frame of the super-chunk. |
2649 | | * |
2650 | | * @param schunk The super-chunk. |
2651 | | * |
2652 | | * @return The length (in bytes) of the internal frame. |
2653 | | * If there is not an internal frame, an estimate of the length is provided. |
2654 | | */ |
2655 | | BLOSC_EXPORT int64_t blosc2_schunk_frame_len(blosc2_schunk* schunk); |
2656 | | |
2657 | | /** |
2658 | | * @brief Quickly fill an empty frame with special values (zeros, NaNs, uninit). |
2659 | | * |
2660 | | * @param schunk The super-chunk to be filled. This must be empty initially. |
2661 | | * @param nitems The number of items to fill. |
2662 | | * @param special_value The special value to use for filling. The only values |
2663 | | * supported for now are BLOSC2_SPECIAL_ZERO, BLOSC2_SPECIAL_NAN and BLOSC2_SPECIAL_UNINIT. |
2664 | | * @param chunksize The chunksize for the chunks that are to be added to the super-chunk. |
2665 | | * |
2666 | | * @return The total number of chunks that have been added to the super-chunk. |
2667 | | * If there is an error, a negative value is returned. |
2668 | | */ |
2669 | | BLOSC_EXPORT int64_t blosc2_schunk_fill_special(blosc2_schunk* schunk, int64_t nitems, |
2670 | | int special_value, int32_t chunksize); |
2671 | | |
2672 | | |
2673 | | /********************************************************************* |
2674 | | Functions related with fixed-length metalayers. |
2675 | | *********************************************************************/ |
2676 | | |
2677 | | /** |
2678 | | * @brief Find whether the schunk has a metalayer or not. |
2679 | | * |
2680 | | * @param schunk The super-chunk from which the metalayer will be checked. |
2681 | | * @param name The name of the metalayer to be checked. |
2682 | | * |
2683 | | * @return If successful, return the index of the metalayer. Else, return a negative value. |
2684 | | */ |
2685 | 64.0k | static inline int blosc2_meta_exists(blosc2_schunk *schunk, const char *name) { |
2686 | 64.0k | if (strlen(name) > BLOSC2_METALAYER_NAME_MAXLEN) { |
2687 | 0 | BLOSC_TRACE_ERROR("Metalayers cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN); |
2688 | 0 | return BLOSC2_ERROR_INVALID_PARAM; |
2689 | 0 | } |
2690 | | |
2691 | 64.0k | if (schunk == NULL) { |
2692 | 0 | BLOSC_TRACE_ERROR("Schunk must not be NUll."); |
2693 | 0 | return BLOSC2_ERROR_INVALID_PARAM; |
2694 | 0 | } |
2695 | | |
2696 | 70.3k | for (int nmetalayer = 0; nmetalayer < schunk->nmetalayers; nmetalayer++) { |
2697 | 30.5k | if (strcmp(name, schunk->metalayers[nmetalayer]->name) == 0) { |
2698 | 24.1k | return nmetalayer; |
2699 | 24.1k | } |
2700 | 30.5k | } |
2701 | 39.8k | return BLOSC2_ERROR_NOT_FOUND; |
2702 | 64.0k | } Unexecuted instantiation: fuzz_compress_frame.c:blosc2_meta_exists Unexecuted instantiation: blosc2.c:blosc2_meta_exists Unexecuted instantiation: blosclz.c:blosc2_meta_exists schunk.c:blosc2_meta_exists Line | Count | Source | 2685 | 23.6k | static inline int blosc2_meta_exists(blosc2_schunk *schunk, const char *name) { | 2686 | 23.6k | if (strlen(name) > BLOSC2_METALAYER_NAME_MAXLEN) { | 2687 | 0 | BLOSC_TRACE_ERROR("Metalayers cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN); | 2688 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2689 | 0 | } | 2690 | | | 2691 | 23.6k | if (schunk == NULL) { | 2692 | 0 | BLOSC_TRACE_ERROR("Schunk must not be NUll."); | 2693 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2694 | 0 | } | 2695 | | | 2696 | 25.1k | for (int nmetalayer = 0; nmetalayer < schunk->nmetalayers; nmetalayer++) { | 2697 | 4.04k | if (strcmp(name, schunk->metalayers[nmetalayer]->name) == 0) { | 2698 | 2.52k | return nmetalayer; | 2699 | 2.52k | } | 2700 | 4.04k | } | 2701 | 21.1k | return BLOSC2_ERROR_NOT_FOUND; | 2702 | 23.6k | } |
Unexecuted instantiation: frame.c:blosc2_meta_exists Unexecuted instantiation: prefetch.c:blosc2_meta_exists Unexecuted instantiation: stune.c:blosc2_meta_exists Unexecuted instantiation: trunc-prec.c:blosc2_meta_exists Unexecuted instantiation: timestamp.c:blosc2_meta_exists Unexecuted instantiation: sframe.c:blosc2_meta_exists Unexecuted instantiation: blosc2-stdio.c:blosc2_meta_exists b2nd.c:blosc2_meta_exists Line | Count | Source | 2685 | 9.03k | static inline int blosc2_meta_exists(blosc2_schunk *schunk, const char *name) { | 2686 | 9.03k | if (strlen(name) > BLOSC2_METALAYER_NAME_MAXLEN) { | 2687 | 0 | BLOSC_TRACE_ERROR("Metalayers cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN); | 2688 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2689 | 0 | } | 2690 | | | 2691 | 9.03k | if (schunk == NULL) { | 2692 | 0 | BLOSC_TRACE_ERROR("Schunk must not be NUll."); | 2693 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2694 | 0 | } | 2695 | | | 2696 | 9.03k | for (int nmetalayer = 0; nmetalayer < schunk->nmetalayers; nmetalayer++) { | 2697 | 9.03k | if (strcmp(name, schunk->metalayers[nmetalayer]->name) == 0) { | 2698 | 9.03k | return nmetalayer; | 2699 | 9.03k | } | 2700 | 9.03k | } | 2701 | 4 | return BLOSC2_ERROR_NOT_FOUND; | 2702 | 9.03k | } |
Unexecuted instantiation: b2nd_utils.c:blosc2_meta_exists Unexecuted instantiation: shuffle.c:blosc2_meta_exists Unexecuted instantiation: shuffle-bytedelta-ssse3.c:blosc2_meta_exists blosc2-zfp.c:blosc2_meta_exists Line | Count | Source | 2685 | 3.07k | static inline int blosc2_meta_exists(blosc2_schunk *schunk, const char *name) { | 2686 | 3.07k | if (strlen(name) > BLOSC2_METALAYER_NAME_MAXLEN) { | 2687 | 0 | BLOSC_TRACE_ERROR("Metalayers cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN); | 2688 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2689 | 0 | } | 2690 | | | 2691 | 3.07k | if (schunk == NULL) { | 2692 | 0 | BLOSC_TRACE_ERROR("Schunk must not be NUll."); | 2693 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2694 | 0 | } | 2695 | | | 2696 | 4.71k | for (int nmetalayer = 0; nmetalayer < schunk->nmetalayers; nmetalayer++) { | 2697 | 4.39k | if (strcmp(name, schunk->metalayers[nmetalayer]->name) == 0) { | 2698 | 2.75k | return nmetalayer; | 2699 | 2.75k | } | 2700 | 4.39k | } | 2701 | 321 | return BLOSC2_ERROR_NOT_FOUND; | 2702 | 3.07k | } |
Unexecuted instantiation: codecs-registry.c:blosc2_meta_exists Unexecuted instantiation: tuners-registry.c:blosc2_meta_exists Unexecuted instantiation: filters-registry.c:blosc2_meta_exists Unexecuted instantiation: ndlz.c:blosc2_meta_exists Unexecuted instantiation: ndlz4x4.c:blosc2_meta_exists ndlz8x8.c:blosc2_meta_exists Line | Count | Source | 2685 | 1 | static inline int blosc2_meta_exists(blosc2_schunk *schunk, const char *name) { | 2686 | 1 | if (strlen(name) > BLOSC2_METALAYER_NAME_MAXLEN) { | 2687 | 0 | BLOSC_TRACE_ERROR("Metalayers cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN); | 2688 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2689 | 0 | } | 2690 | | | 2691 | 1 | if (schunk == NULL) { | 2692 | 0 | BLOSC_TRACE_ERROR("Schunk must not be NUll."); | 2693 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2694 | 0 | } | 2695 | | | 2696 | 1 | for (int nmetalayer = 0; nmetalayer < schunk->nmetalayers; nmetalayer++) { | 2697 | 1 | if (strcmp(name, schunk->metalayers[nmetalayer]->name) == 0) { | 2698 | 1 | return nmetalayer; | 2699 | 1 | } | 2700 | 1 | } | 2701 | 0 | return BLOSC2_ERROR_NOT_FOUND; | 2702 | 1 | } |
ndcell.c:blosc2_meta_exists Line | Count | Source | 2685 | 1.47k | static inline int blosc2_meta_exists(blosc2_schunk *schunk, const char *name) { | 2686 | 1.47k | if (strlen(name) > BLOSC2_METALAYER_NAME_MAXLEN) { | 2687 | 0 | BLOSC_TRACE_ERROR("Metalayers cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN); | 2688 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2689 | 0 | } | 2690 | | | 2691 | 1.47k | if (schunk == NULL) { | 2692 | 0 | BLOSC_TRACE_ERROR("Schunk must not be NUll."); | 2693 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2694 | 0 | } | 2695 | | | 2696 | 1.62k | for (int nmetalayer = 0; nmetalayer < schunk->nmetalayers; nmetalayer++) { | 2697 | 1.42k | if (strcmp(name, schunk->metalayers[nmetalayer]->name) == 0) { | 2698 | 1.26k | return nmetalayer; | 2699 | 1.26k | } | 2700 | 1.42k | } | 2701 | 206 | return BLOSC2_ERROR_NOT_FOUND; | 2702 | 1.47k | } |
ndmean.c:blosc2_meta_exists Line | Count | Source | 2685 | 2.68k | static inline int blosc2_meta_exists(blosc2_schunk *schunk, const char *name) { | 2686 | 2.68k | if (strlen(name) > BLOSC2_METALAYER_NAME_MAXLEN) { | 2687 | 0 | BLOSC_TRACE_ERROR("Metalayers cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN); | 2688 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2689 | 0 | } | 2690 | | | 2691 | 2.68k | if (schunk == NULL) { | 2692 | 0 | BLOSC_TRACE_ERROR("Schunk must not be NUll."); | 2693 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2694 | 0 | } | 2695 | | | 2696 | 2.87k | for (int nmetalayer = 0; nmetalayer < schunk->nmetalayers; nmetalayer++) { | 2697 | 2.68k | if (strcmp(name, schunk->metalayers[nmetalayer]->name) == 0) { | 2698 | 2.49k | return nmetalayer; | 2699 | 2.49k | } | 2700 | 2.68k | } | 2701 | 189 | return BLOSC2_ERROR_NOT_FOUND; | 2702 | 2.68k | } |
Unexecuted instantiation: bytedelta.c:blosc2_meta_exists Unexecuted instantiation: int_trunc.c:blosc2_meta_exists Unexecuted instantiation: fuzz_plugins.c:blosc2_meta_exists Unexecuted instantiation: fuzz_decompress_frame.c:blosc2_meta_exists Unexecuted instantiation: fuzz_compress_chunk.c:blosc2_meta_exists fuzz_schunk_ops.c:blosc2_meta_exists Line | Count | Source | 2685 | 24.1k | static inline int blosc2_meta_exists(blosc2_schunk *schunk, const char *name) { | 2686 | 24.1k | if (strlen(name) > BLOSC2_METALAYER_NAME_MAXLEN) { | 2687 | 0 | BLOSC_TRACE_ERROR("Metalayers cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN); | 2688 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2689 | 0 | } | 2690 | | | 2691 | 24.1k | if (schunk == NULL) { | 2692 | 0 | BLOSC_TRACE_ERROR("Schunk must not be NUll."); | 2693 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2694 | 0 | } | 2695 | | | 2696 | 26.9k | for (int nmetalayer = 0; nmetalayer < schunk->nmetalayers; nmetalayer++) { | 2697 | 8.92k | if (strcmp(name, schunk->metalayers[nmetalayer]->name) == 0) { | 2698 | 6.08k | return nmetalayer; | 2699 | 6.08k | } | 2700 | 8.92k | } | 2701 | 18.0k | return BLOSC2_ERROR_NOT_FOUND; | 2702 | 24.1k | } |
Unexecuted instantiation: fuzz_decompress_chunk.c:blosc2_meta_exists Unexecuted instantiation: fuzz_b2nd.c:blosc2_meta_exists |
2703 | | |
2704 | | /** |
2705 | | * @brief Add content into a new metalayer. |
2706 | | * |
2707 | | * @param schunk The super-chunk to which the metalayer should be added. |
2708 | | * @param name The name of the metalayer. |
2709 | | * @param content The content of the metalayer. |
2710 | | * @param content_len The length of the content. |
2711 | | * |
2712 | | * @return If successful, the index of the new metalayer. Else, return a negative value. |
2713 | | */ |
2714 | | BLOSC_EXPORT int blosc2_meta_add(blosc2_schunk *schunk, const char *name, uint8_t *content, |
2715 | | int32_t content_len); |
2716 | | |
2717 | | /** |
2718 | | * @brief Update the content of an existing metalayer. |
2719 | | * |
2720 | | * @param schunk The frame containing the metalayer. |
2721 | | * @param name The name of the metalayer to be updated. |
2722 | | * @param content The new content of the metalayer. |
2723 | | * @param content_len The length of the content. |
2724 | | * |
2725 | | * @note Contrarily to #blosc2_meta_add the updates to metalayers |
2726 | | * are automatically serialized into a possible attached frame. |
2727 | | * |
2728 | | * @return If successful, the index of the metalayer. Else, return a negative value. |
2729 | | */ |
2730 | | BLOSC_EXPORT int blosc2_meta_update(blosc2_schunk *schunk, const char *name, uint8_t *content, |
2731 | | int32_t content_len); |
2732 | | |
2733 | | /** |
2734 | | * @brief Get the content out of a metalayer. |
2735 | | * |
2736 | | * @param schunk The frame containing the metalayer. |
2737 | | * @param name The name of the metalayer. |
2738 | | * @param content The pointer where the content will be put. |
2739 | | * @param content_len The length of the content. |
2740 | | * |
2741 | | * @warning The @p **content receives a malloc'ed copy of the content. |
2742 | | * The user is responsible of freeing it. |
2743 | | * |
2744 | | * @note This function is inlined so that external codec/filter plugins (like |
2745 | | * blosc2_grok) can use it without linking against libblosc2. This avoids |
2746 | | * pulling all of libblosc2's symbols (e.g. internal ZFP, Zstd) into the |
2747 | | * global namespace at load time, which would otherwise shadow symbols from |
2748 | | * other libraries that need differently-configured builds of the same |
2749 | | * dependencies. |
2750 | | * |
2751 | | * @return If successful, the index of the new metalayer. Else, return a negative value. |
2752 | | */ |
2753 | | static inline int blosc2_meta_get(blosc2_schunk *schunk, const char *name, uint8_t **content, |
2754 | 18.1k | int32_t *content_len) { |
2755 | 18.1k | if (schunk == NULL || name == NULL || content == NULL || content_len == NULL) { |
2756 | 106 | BLOSC_TRACE_ERROR("Invalid parameters."); |
2757 | 106 | return BLOSC2_ERROR_INVALID_PARAM; |
2758 | 106 | } |
2759 | | |
2760 | 18.0k | int nmetalayer = blosc2_meta_exists(schunk, name); |
2761 | 18.0k | if (nmetalayer < 0) { |
2762 | 720 | BLOSC_TRACE_WARNING("Metalayer \"%s\" not found.", name); |
2763 | 720 | return nmetalayer; |
2764 | 720 | } |
2765 | 17.2k | int32_t len = schunk->metalayers[nmetalayer]->content_len; |
2766 | 17.2k | if (len < 0) { |
2767 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content length %d.", name, len); |
2768 | 0 | return BLOSC2_ERROR_DATA; |
2769 | 0 | } |
2770 | 17.2k | *content_len = len; |
2771 | 17.2k | if (len == 0) { |
2772 | 0 | *content = NULL; |
2773 | 0 | return nmetalayer; |
2774 | 0 | } |
2775 | 17.2k | *content = (uint8_t*)malloc((size_t)len); |
2776 | 17.2k | if (*content == NULL) { |
2777 | 0 | BLOSC_TRACE_ERROR("Unable to allocate metalayer content buffer."); |
2778 | 0 | *content_len = 0; |
2779 | 0 | return BLOSC2_ERROR_MEMORY_ALLOC; |
2780 | 0 | } |
2781 | 17.2k | if (len > 0 && schunk->metalayers[nmetalayer]->content == NULL) { |
2782 | 0 | free(*content); |
2783 | 0 | *content = NULL; |
2784 | 0 | *content_len = 0; |
2785 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content pointer.", name); |
2786 | 0 | return BLOSC2_ERROR_DATA; |
2787 | 0 | } |
2788 | 17.2k | memcpy(*content, schunk->metalayers[nmetalayer]->content, (size_t)len); |
2789 | 17.2k | return nmetalayer; |
2790 | 17.2k | } Unexecuted instantiation: fuzz_compress_frame.c:blosc2_meta_get Unexecuted instantiation: blosc2.c:blosc2_meta_get Unexecuted instantiation: blosclz.c:blosc2_meta_get Unexecuted instantiation: schunk.c:blosc2_meta_get Unexecuted instantiation: frame.c:blosc2_meta_get Unexecuted instantiation: prefetch.c:blosc2_meta_get Unexecuted instantiation: stune.c:blosc2_meta_get Unexecuted instantiation: trunc-prec.c:blosc2_meta_get Unexecuted instantiation: timestamp.c:blosc2_meta_get Unexecuted instantiation: sframe.c:blosc2_meta_get Unexecuted instantiation: blosc2-stdio.c:blosc2_meta_get Line | Count | Source | 2754 | 7.73k | int32_t *content_len) { | 2755 | 7.73k | if (schunk == NULL || name == NULL || content == NULL || content_len == NULL) { | 2756 | 0 | BLOSC_TRACE_ERROR("Invalid parameters."); | 2757 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2758 | 0 | } | 2759 | | | 2760 | 7.73k | int nmetalayer = blosc2_meta_exists(schunk, name); | 2761 | 7.73k | if (nmetalayer < 0) { | 2762 | 4 | BLOSC_TRACE_WARNING("Metalayer \"%s\" not found.", name); | 2763 | 4 | return nmetalayer; | 2764 | 4 | } | 2765 | 7.72k | int32_t len = schunk->metalayers[nmetalayer]->content_len; | 2766 | 7.72k | if (len < 0) { | 2767 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content length %d.", name, len); | 2768 | 0 | return BLOSC2_ERROR_DATA; | 2769 | 0 | } | 2770 | 7.72k | *content_len = len; | 2771 | 7.72k | if (len == 0) { | 2772 | 0 | *content = NULL; | 2773 | 0 | return nmetalayer; | 2774 | 0 | } | 2775 | 7.72k | *content = (uint8_t*)malloc((size_t)len); | 2776 | 7.72k | if (*content == NULL) { | 2777 | 0 | BLOSC_TRACE_ERROR("Unable to allocate metalayer content buffer."); | 2778 | 0 | *content_len = 0; | 2779 | 0 | return BLOSC2_ERROR_MEMORY_ALLOC; | 2780 | 0 | } | 2781 | 7.72k | if (len > 0 && schunk->metalayers[nmetalayer]->content == NULL) { | 2782 | 0 | free(*content); | 2783 | 0 | *content = NULL; | 2784 | 0 | *content_len = 0; | 2785 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content pointer.", name); | 2786 | 0 | return BLOSC2_ERROR_DATA; | 2787 | 0 | } | 2788 | 7.72k | memcpy(*content, schunk->metalayers[nmetalayer]->content, (size_t)len); | 2789 | 7.72k | return nmetalayer; | 2790 | 7.72k | } |
Unexecuted instantiation: b2nd_utils.c:blosc2_meta_get Unexecuted instantiation: shuffle.c:blosc2_meta_get Unexecuted instantiation: shuffle-bytedelta-ssse3.c:blosc2_meta_get blosc2-zfp.c:blosc2_meta_get Line | Count | Source | 2754 | 3.07k | int32_t *content_len) { | 2755 | 3.07k | if (schunk == NULL || name == NULL || content == NULL || content_len == NULL) { | 2756 | 0 | BLOSC_TRACE_ERROR("Invalid parameters."); | 2757 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2758 | 0 | } | 2759 | | | 2760 | 3.07k | int nmetalayer = blosc2_meta_exists(schunk, name); | 2761 | 3.07k | if (nmetalayer < 0) { | 2762 | 321 | BLOSC_TRACE_WARNING("Metalayer \"%s\" not found.", name); | 2763 | 321 | return nmetalayer; | 2764 | 321 | } | 2765 | 2.75k | int32_t len = schunk->metalayers[nmetalayer]->content_len; | 2766 | 2.75k | if (len < 0) { | 2767 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content length %d.", name, len); | 2768 | 0 | return BLOSC2_ERROR_DATA; | 2769 | 0 | } | 2770 | 2.75k | *content_len = len; | 2771 | 2.75k | if (len == 0) { | 2772 | 0 | *content = NULL; | 2773 | 0 | return nmetalayer; | 2774 | 0 | } | 2775 | 2.75k | *content = (uint8_t*)malloc((size_t)len); | 2776 | 2.75k | if (*content == NULL) { | 2777 | 0 | BLOSC_TRACE_ERROR("Unable to allocate metalayer content buffer."); | 2778 | 0 | *content_len = 0; | 2779 | 0 | return BLOSC2_ERROR_MEMORY_ALLOC; | 2780 | 0 | } | 2781 | 2.75k | if (len > 0 && schunk->metalayers[nmetalayer]->content == NULL) { | 2782 | 0 | free(*content); | 2783 | 0 | *content = NULL; | 2784 | 0 | *content_len = 0; | 2785 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content pointer.", name); | 2786 | 0 | return BLOSC2_ERROR_DATA; | 2787 | 0 | } | 2788 | 2.75k | memcpy(*content, schunk->metalayers[nmetalayer]->content, (size_t)len); | 2789 | 2.75k | return nmetalayer; | 2790 | 2.75k | } |
Unexecuted instantiation: codecs-registry.c:blosc2_meta_get Unexecuted instantiation: tuners-registry.c:blosc2_meta_get Unexecuted instantiation: filters-registry.c:blosc2_meta_get Unexecuted instantiation: ndlz.c:blosc2_meta_get Unexecuted instantiation: ndlz4x4.c:blosc2_meta_get ndlz8x8.c:blosc2_meta_get Line | Count | Source | 2754 | 1 | int32_t *content_len) { | 2755 | 1 | if (schunk == NULL || name == NULL || content == NULL || content_len == NULL) { | 2756 | 0 | BLOSC_TRACE_ERROR("Invalid parameters."); | 2757 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2758 | 0 | } | 2759 | | | 2760 | 1 | int nmetalayer = blosc2_meta_exists(schunk, name); | 2761 | 1 | if (nmetalayer < 0) { | 2762 | 0 | BLOSC_TRACE_WARNING("Metalayer \"%s\" not found.", name); | 2763 | 0 | return nmetalayer; | 2764 | 0 | } | 2765 | 1 | int32_t len = schunk->metalayers[nmetalayer]->content_len; | 2766 | 1 | if (len < 0) { | 2767 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content length %d.", name, len); | 2768 | 0 | return BLOSC2_ERROR_DATA; | 2769 | 0 | } | 2770 | 1 | *content_len = len; | 2771 | 1 | if (len == 0) { | 2772 | 0 | *content = NULL; | 2773 | 0 | return nmetalayer; | 2774 | 0 | } | 2775 | 1 | *content = (uint8_t*)malloc((size_t)len); | 2776 | 1 | if (*content == NULL) { | 2777 | 0 | BLOSC_TRACE_ERROR("Unable to allocate metalayer content buffer."); | 2778 | 0 | *content_len = 0; | 2779 | 0 | return BLOSC2_ERROR_MEMORY_ALLOC; | 2780 | 0 | } | 2781 | 1 | if (len > 0 && schunk->metalayers[nmetalayer]->content == NULL) { | 2782 | 0 | free(*content); | 2783 | 0 | *content = NULL; | 2784 | 0 | *content_len = 0; | 2785 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content pointer.", name); | 2786 | 0 | return BLOSC2_ERROR_DATA; | 2787 | 0 | } | 2788 | 1 | memcpy(*content, schunk->metalayers[nmetalayer]->content, (size_t)len); | 2789 | 1 | return nmetalayer; | 2790 | 1 | } |
Line | Count | Source | 2754 | 1.53k | int32_t *content_len) { | 2755 | 1.53k | if (schunk == NULL || name == NULL || content == NULL || content_len == NULL) { | 2756 | 55 | BLOSC_TRACE_ERROR("Invalid parameters."); | 2757 | 55 | return BLOSC2_ERROR_INVALID_PARAM; | 2758 | 55 | } | 2759 | | | 2760 | 1.47k | int nmetalayer = blosc2_meta_exists(schunk, name); | 2761 | 1.47k | if (nmetalayer < 0) { | 2762 | 206 | BLOSC_TRACE_WARNING("Metalayer \"%s\" not found.", name); | 2763 | 206 | return nmetalayer; | 2764 | 206 | } | 2765 | 1.26k | int32_t len = schunk->metalayers[nmetalayer]->content_len; | 2766 | 1.26k | if (len < 0) { | 2767 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content length %d.", name, len); | 2768 | 0 | return BLOSC2_ERROR_DATA; | 2769 | 0 | } | 2770 | 1.26k | *content_len = len; | 2771 | 1.26k | if (len == 0) { | 2772 | 0 | *content = NULL; | 2773 | 0 | return nmetalayer; | 2774 | 0 | } | 2775 | 1.26k | *content = (uint8_t*)malloc((size_t)len); | 2776 | 1.26k | if (*content == NULL) { | 2777 | 0 | BLOSC_TRACE_ERROR("Unable to allocate metalayer content buffer."); | 2778 | 0 | *content_len = 0; | 2779 | 0 | return BLOSC2_ERROR_MEMORY_ALLOC; | 2780 | 0 | } | 2781 | 1.26k | if (len > 0 && schunk->metalayers[nmetalayer]->content == NULL) { | 2782 | 0 | free(*content); | 2783 | 0 | *content = NULL; | 2784 | 0 | *content_len = 0; | 2785 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content pointer.", name); | 2786 | 0 | return BLOSC2_ERROR_DATA; | 2787 | 0 | } | 2788 | 1.26k | memcpy(*content, schunk->metalayers[nmetalayer]->content, (size_t)len); | 2789 | 1.26k | return nmetalayer; | 2790 | 1.26k | } |
Line | Count | Source | 2754 | 2.73k | int32_t *content_len) { | 2755 | 2.73k | if (schunk == NULL || name == NULL || content == NULL || content_len == NULL) { | 2756 | 51 | BLOSC_TRACE_ERROR("Invalid parameters."); | 2757 | 51 | return BLOSC2_ERROR_INVALID_PARAM; | 2758 | 51 | } | 2759 | | | 2760 | 2.68k | int nmetalayer = blosc2_meta_exists(schunk, name); | 2761 | 2.68k | if (nmetalayer < 0) { | 2762 | 189 | BLOSC_TRACE_WARNING("Metalayer \"%s\" not found.", name); | 2763 | 189 | return nmetalayer; | 2764 | 189 | } | 2765 | 2.49k | int32_t len = schunk->metalayers[nmetalayer]->content_len; | 2766 | 2.49k | if (len < 0) { | 2767 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content length %d.", name, len); | 2768 | 0 | return BLOSC2_ERROR_DATA; | 2769 | 0 | } | 2770 | 2.49k | *content_len = len; | 2771 | 2.49k | if (len == 0) { | 2772 | 0 | *content = NULL; | 2773 | 0 | return nmetalayer; | 2774 | 0 | } | 2775 | 2.49k | *content = (uint8_t*)malloc((size_t)len); | 2776 | 2.49k | if (*content == NULL) { | 2777 | 0 | BLOSC_TRACE_ERROR("Unable to allocate metalayer content buffer."); | 2778 | 0 | *content_len = 0; | 2779 | 0 | return BLOSC2_ERROR_MEMORY_ALLOC; | 2780 | 0 | } | 2781 | 2.49k | if (len > 0 && schunk->metalayers[nmetalayer]->content == NULL) { | 2782 | 0 | free(*content); | 2783 | 0 | *content = NULL; | 2784 | 0 | *content_len = 0; | 2785 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content pointer.", name); | 2786 | 0 | return BLOSC2_ERROR_DATA; | 2787 | 0 | } | 2788 | 2.49k | memcpy(*content, schunk->metalayers[nmetalayer]->content, (size_t)len); | 2789 | 2.49k | return nmetalayer; | 2790 | 2.49k | } |
Unexecuted instantiation: bytedelta.c:blosc2_meta_get Unexecuted instantiation: int_trunc.c:blosc2_meta_get Unexecuted instantiation: fuzz_plugins.c:blosc2_meta_get Unexecuted instantiation: fuzz_decompress_frame.c:blosc2_meta_get Unexecuted instantiation: fuzz_compress_chunk.c:blosc2_meta_get fuzz_schunk_ops.c:blosc2_meta_get Line | Count | Source | 2754 | 3.04k | int32_t *content_len) { | 2755 | 3.04k | if (schunk == NULL || name == NULL || content == NULL || content_len == NULL) { | 2756 | 0 | BLOSC_TRACE_ERROR("Invalid parameters."); | 2757 | 0 | return BLOSC2_ERROR_INVALID_PARAM; | 2758 | 0 | } | 2759 | | | 2760 | 3.04k | int nmetalayer = blosc2_meta_exists(schunk, name); | 2761 | 3.04k | if (nmetalayer < 0) { | 2762 | 0 | BLOSC_TRACE_WARNING("Metalayer \"%s\" not found.", name); | 2763 | 0 | return nmetalayer; | 2764 | 0 | } | 2765 | 3.04k | int32_t len = schunk->metalayers[nmetalayer]->content_len; | 2766 | 3.04k | if (len < 0) { | 2767 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content length %d.", name, len); | 2768 | 0 | return BLOSC2_ERROR_DATA; | 2769 | 0 | } | 2770 | 3.04k | *content_len = len; | 2771 | 3.04k | if (len == 0) { | 2772 | 0 | *content = NULL; | 2773 | 0 | return nmetalayer; | 2774 | 0 | } | 2775 | 3.04k | *content = (uint8_t*)malloc((size_t)len); | 2776 | 3.04k | if (*content == NULL) { | 2777 | 0 | BLOSC_TRACE_ERROR("Unable to allocate metalayer content buffer."); | 2778 | 0 | *content_len = 0; | 2779 | 0 | return BLOSC2_ERROR_MEMORY_ALLOC; | 2780 | 0 | } | 2781 | 3.04k | if (len > 0 && schunk->metalayers[nmetalayer]->content == NULL) { | 2782 | 0 | free(*content); | 2783 | 0 | *content = NULL; | 2784 | 0 | *content_len = 0; | 2785 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" has corrupted content pointer.", name); | 2786 | 0 | return BLOSC2_ERROR_DATA; | 2787 | 0 | } | 2788 | 3.04k | memcpy(*content, schunk->metalayers[nmetalayer]->content, (size_t)len); | 2789 | 3.04k | return nmetalayer; | 2790 | 3.04k | } |
Unexecuted instantiation: fuzz_decompress_chunk.c:blosc2_meta_get Unexecuted instantiation: fuzz_b2nd.c:blosc2_meta_get |
2791 | | |
2792 | | |
2793 | | /********************************************************************* |
2794 | | Variable-length metalayers functions. |
2795 | | *********************************************************************/ |
2796 | | |
2797 | | /** |
2798 | | * @brief Find whether the schunk has a variable-length metalayer or not. |
2799 | | * |
2800 | | * For on-disk super-chunks this also re-syncs the cached variable-length |
2801 | | * metalayers when another handle (or process) rewrote the frame, so a |
2802 | | * metalayer added or deleted elsewhere is reflected in the answer. |
2803 | | * |
2804 | | * @param schunk The super-chunk from which the variable-length metalayer will be checked. |
2805 | | * @param name The name of the variable-length metalayer to be checked. |
2806 | | * |
2807 | | * @return If successful, return the index of the variable-length metalayer. Else, return a negative value. |
2808 | | */ |
2809 | | BLOSC_EXPORT int blosc2_vlmeta_exists(blosc2_schunk *schunk, const char *name); |
2810 | | |
2811 | | /** |
2812 | | * @brief Add content into a new variable-length metalayer. |
2813 | | * |
2814 | | * @param schunk The super-chunk to which the variable-length metalayer should be added. |
2815 | | * @param name The name of the variable-length metalayer. |
2816 | | * @param content The content to be added. |
2817 | | * @param content_len The length of the content. |
2818 | | * @param cparams The parameters for compressing the variable-length metalayer content. If NULL, |
2819 | | * the `BLOSC2_CPARAMS_DEFAULTS` will be used. Their prefilter, if any, is not applied (the content |
2820 | | * is not chunk data). |
2821 | | * |
2822 | | * @return If successful, the index of the new variable-length metalayer. Else, return a negative value. |
2823 | | */ |
2824 | | BLOSC_EXPORT int blosc2_vlmeta_add(blosc2_schunk *schunk, const char *name, |
2825 | | uint8_t *content, int32_t content_len, |
2826 | | blosc2_cparams *cparams); |
2827 | | |
2828 | | /** |
2829 | | * @brief Update the content of an existing variable-length metalayer. |
2830 | | * |
2831 | | * @param schunk The super-chunk containing the variable-length metalayer. |
2832 | | * @param name The name of the variable-length metalayer to be updated. |
2833 | | * @param content The new content of the variable-length metalayer. |
2834 | | * @param content_len The length of the content. |
2835 | | * @param cparams The parameters for compressing the variable-length metalayer content. If NULL, |
2836 | | * the `BLOSC2_CPARAMS_DEFAULTS` will be used. Their prefilter, if any, is not applied (the content |
2837 | | * is not chunk data). |
2838 | | * |
2839 | | * @return If successful, the index of the variable-length metalayer. Else, return a negative value. |
2840 | | */ |
2841 | | BLOSC_EXPORT int blosc2_vlmeta_update(blosc2_schunk *schunk, const char *name, |
2842 | | uint8_t *content, int32_t content_len, |
2843 | | blosc2_cparams *cparams); |
2844 | | |
2845 | | /** |
2846 | | * @brief Get the content out of a variable-length metalayer. |
2847 | | * |
2848 | | * @param schunk The super-chunk containing the variable-length metalayer. |
2849 | | * @param name The name of the variable-length metalayer. |
2850 | | * @param content The pointer where the content will be put. |
2851 | | * @param content_len The pointer where the length of the content will be put. |
2852 | | * |
2853 | | * @warning The @p **content receives a malloc'ed copy of the content. |
2854 | | * The user is responsible of freeing it. |
2855 | | * |
2856 | | * @note For disk-based super-chunks, this checks whether another handle has |
2857 | | * rewritten the frame and, if so, re-reads the cached metalayers first, so |
2858 | | * updates made through other handles (or processes) become visible. |
2859 | | * |
2860 | | * @note The postfilter of the super-chunk is not applied (the content is not chunk data). |
2861 | | * |
2862 | | * @return If successful, the index of the new variable-length metalayer. Else, return a negative value. |
2863 | | */ |
2864 | | BLOSC_EXPORT int blosc2_vlmeta_get(blosc2_schunk *schunk, const char *name, |
2865 | | uint8_t **content, int32_t *content_len); |
2866 | | |
2867 | | /** |
2868 | | * @brief Delete the variable-length metalayer from the super-chunk. |
2869 | | * |
2870 | | * @param schunk The super-chunk containing the variable-length metalayer. |
2871 | | * @param name The name of the variable-length metalayer. |
2872 | | * |
2873 | | * @return If successful, the number of the variable-length metalayers in the super-chunk. Else, return a negative value. |
2874 | | */ |
2875 | | BLOSC_EXPORT int blosc2_vlmeta_delete(blosc2_schunk *schunk, const char *name); |
2876 | | |
2877 | | /** |
2878 | | * @brief Get a list of all the variable-length metalayer names. |
2879 | | * |
2880 | | * @param schunk The super-chunk containing the variable-length metalayers. |
2881 | | * @param names The pointer to a char** to store the name pointers. This should |
2882 | | * be of size *schunk->nvlmetalayers * sizeof(char*). |
2883 | | * |
2884 | | * @return The number of the variable-length metalayers in the super-chunk. |
2885 | | * This cannot fail unless the user does not pass a @p names which is large enough to |
2886 | | * keep pointers to all names, in which case funny things (seg faults and such) will happen. |
2887 | | */ |
2888 | | BLOSC_EXPORT int blosc2_vlmeta_get_names(blosc2_schunk *schunk, char **names); |
2889 | | |
2890 | | |
2891 | | /********************************************************************* |
2892 | | Time measurement utilities. |
2893 | | *********************************************************************/ |
2894 | | |
2895 | | #if defined(_WIN32) |
2896 | | /* For QueryPerformanceCounter(), etc. */ |
2897 | | #include <windows.h> |
2898 | | #elif defined(__MACH__) && defined(__APPLE__) |
2899 | | #include <mach/clock.h> |
2900 | | #include <mach/mach.h> |
2901 | | #include <time.h> |
2902 | | #elif defined(__unix__) |
2903 | | #if defined(__linux__) |
2904 | | #include <time.h> |
2905 | | #else |
2906 | | #include <sys/time.h> |
2907 | | #endif |
2908 | | #else |
2909 | | #error Unable to detect platform. |
2910 | | #endif |
2911 | | |
2912 | | /* The type of timestamp used on this system. */ |
2913 | | #if defined(_WIN32) |
2914 | | typedef LARGE_INTEGER blosc_timestamp_t; |
2915 | | #else |
2916 | | typedef struct timespec blosc_timestamp_t; |
2917 | | #endif |
2918 | | |
2919 | | /* |
2920 | | * @brief Set a timestamp. |
2921 | | * |
2922 | | * @param timestamp |
2923 | | * |
2924 | | */ |
2925 | | BLOSC_EXPORT void blosc_set_timestamp(blosc_timestamp_t* timestamp); |
2926 | | |
2927 | | /* |
2928 | | * @brief Get the nanoseconds between 2 timestamps. |
2929 | | * |
2930 | | * @param start_time |
2931 | | * @param end_time |
2932 | | * |
2933 | | * @return The nanoseconds between start_time and end_time. |
2934 | | */ |
2935 | | BLOSC_EXPORT double blosc_elapsed_nsecs(blosc_timestamp_t start_time, |
2936 | | blosc_timestamp_t end_time); |
2937 | | |
2938 | | /* |
2939 | | * @brief Get the seconds between 2 timestamps. |
2940 | | * |
2941 | | * @param start_time |
2942 | | * @param end_time |
2943 | | * |
2944 | | * @return The seconds between start_time and end_time. |
2945 | | */ |
2946 | | BLOSC_EXPORT double blosc_elapsed_secs(blosc_timestamp_t start_time, |
2947 | | blosc_timestamp_t end_time); |
2948 | | |
2949 | | |
2950 | | /********************************************************************* |
2951 | | Low-level functions follows. Use them only if you are an expert! |
2952 | | *********************************************************************/ |
2953 | | |
2954 | | /** |
2955 | | * @brief Get the internal blocksize to be used during compression. 0 means |
2956 | | * that an automatic blocksize is computed internally. |
2957 | | * |
2958 | | * @return The size in bytes of the internal block size. |
2959 | | */ |
2960 | | BLOSC_EXPORT int blosc1_get_blocksize(void); |
2961 | | |
2962 | | /** |
2963 | | * @brief Force the use of a specific blocksize. If 0, an automatic |
2964 | | * blocksize will be used (the default). |
2965 | | * |
2966 | | * @warning The blocksize is a critical parameter with important |
2967 | | * restrictions in the allowed values, so use this with care. |
2968 | | */ |
2969 | | BLOSC_EXPORT void blosc1_set_blocksize(size_t blocksize); |
2970 | | |
2971 | | |
2972 | | /** |
2973 | | * @brief Set the split mode. |
2974 | | |
2975 | | * @param splitmode It can take the next values: |
2976 | | * BLOSC_FORWARD_COMPAT_SPLIT |
2977 | | * BLOSC_AUTO_SPLIT |
2978 | | * BLOSC_NEVER_SPLIT |
2979 | | * BLOSC_ALWAYS_SPLIT |
2980 | | * |
2981 | | * BLOSC_FORWARD_COMPAT offers reasonably forward compatibility, |
2982 | | * BLOSC_AUTO_SPLIT is for nearly optimal results (based on heuristics), |
2983 | | * BLOSC_NEVER_SPLIT and BLOSC_ALWAYS_SPLIT are for the user experimenting |
2984 | | * when trying to get best compression ratios and/or speed. |
2985 | | * |
2986 | | * If not called, the default mode is BLOSC_FORWARD_COMPAT_SPLIT. |
2987 | | * |
2988 | | * This function should always succeed. |
2989 | | */ |
2990 | | BLOSC_EXPORT void blosc1_set_splitmode(int splitmode); |
2991 | | |
2992 | | /** |
2993 | | * @brief Make a compression context write Blosc1 chunks. |
2994 | | * |
2995 | | * The chunks use format version 2 and the 16-byte header, so that they can also be |
2996 | | * decoded by C-Blosc 1.x. They are not byte-identical to what C-Blosc 1.x would write. |
2997 | | * Contexts do not write Blosc1 chunks by default. |
2998 | | * |
2999 | | * The chunk format has room for a plain shuffle or bitshuffle filter, in whichever slot of |
3000 | | * the filter pipeline it is, and for the blosclz, lz4, lz4hc, zlib and zstd codecs. A |
3001 | | * compression call on a context that has any other filter or more than one filter, another |
3002 | | * codec, a dictionary or variable-length blocks fails with an error. A prefilter is applied |
3003 | | * as usual: the chunk records nothing about it, so a reader gets the prefiltered data. |
3004 | | * |
3005 | | * @param context A compression context. |
3006 | | * @param enable Whether to write Blosc1 chunks. |
3007 | | * |
3008 | | * @return 0 on success, a negative error code otherwise. |
3009 | | * |
3010 | | * @sa #blosc2_ctx_get_blosc1_format |
3011 | | */ |
3012 | | BLOSC_EXPORT int blosc2_ctx_set_blosc1_format(blosc2_context* context, bool enable); |
3013 | | |
3014 | | /** |
3015 | | * @brief Query whether a compression context writes Blosc1 chunks. |
3016 | | * |
3017 | | * @param context A compression context. |
3018 | | * |
3019 | | * @return true if the context is set to write Blosc1 chunks, false if it is not or if |
3020 | | * @p context is NULL. |
3021 | | * |
3022 | | * @sa #blosc2_ctx_set_blosc1_format |
3023 | | */ |
3024 | | BLOSC_EXPORT bool blosc2_ctx_get_blosc1_format(const blosc2_context* context); |
3025 | | |
3026 | | |
3027 | | /** |
3028 | | * @brief Get the offsets of a frame in a super-chunk. |
3029 | | * |
3030 | | * @param schunk The super-chunk containing the frame. |
3031 | | * |
3032 | | * @return If successful, return a pointer to a buffer of the decompressed offsets. |
3033 | | * The number of offsets is equal to schunk->nchunks; the user is |
3034 | | * responsible to free this buffer. Else, return a NULL value. |
3035 | | */ |
3036 | | BLOSC_EXPORT int64_t* blosc2_frame_get_offsets(blosc2_schunk *schunk); |
3037 | | |
3038 | | |
3039 | | /********************************************************************* |
3040 | | Structures and functions related with compression codecs. |
3041 | | *********************************************************************/ |
3042 | | |
3043 | | typedef int (* blosc2_codec_encoder_cb) (const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, |
3044 | | uint8_t meta, blosc2_cparams *cparams, const void* chunk); |
3045 | | typedef int (* blosc2_codec_decoder_cb) (const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, |
3046 | | uint8_t meta, blosc2_dparams *dparams, const void* chunk); |
3047 | | |
3048 | | typedef struct { |
3049 | | uint8_t compcode; |
3050 | | //!< The codec identifier. |
3051 | | char *compname; |
3052 | | //!< The codec name. |
3053 | | uint8_t complib; |
3054 | | //!< The codec library format. |
3055 | | uint8_t version; |
3056 | | //!< The codec version. |
3057 | | blosc2_codec_encoder_cb encoder; |
3058 | | //!< The codec encoder that is used during compression. |
3059 | | blosc2_codec_decoder_cb decoder; |
3060 | | //!< The codec decoder that is used during decompression. |
3061 | | // int (*free)(void* codec_params); |
3062 | | // //!< Free the codec_params stored in blosc2_context. |
3063 | | } blosc2_codec; |
3064 | | |
3065 | | /** |
3066 | | * @brief Register locally a user-defined codec in Blosc. |
3067 | | * |
3068 | | * @param codec The codec to register. |
3069 | | * |
3070 | | * @return 0 if succeeds. Else a negative code is returned. |
3071 | | */ |
3072 | | BLOSC_EXPORT int blosc2_register_codec(blosc2_codec *codec); |
3073 | | |
3074 | | |
3075 | | /********************************************************************* |
3076 | | Structures and functions related with filters plugins. |
3077 | | *********************************************************************/ |
3078 | | |
3079 | | typedef int (* blosc2_filter_forward_cb) (const uint8_t *, uint8_t *, int32_t, uint8_t, blosc2_cparams *, |
3080 | | uint8_t); |
3081 | | typedef int (* blosc2_filter_backward_cb) (const uint8_t *, uint8_t *, int32_t, uint8_t, blosc2_dparams *, |
3082 | | uint8_t); |
3083 | | |
3084 | | /** |
3085 | | * @brief The parameters for a user-defined filter. |
3086 | | */ |
3087 | | typedef struct { |
3088 | | uint8_t id; |
3089 | | //!< The filter identifier. |
3090 | | char * name; |
3091 | | //!< The filter name. |
3092 | | uint8_t version; |
3093 | | //!< The filter version. |
3094 | | blosc2_filter_forward_cb forward; |
3095 | | //!< The filter function that is used during compression. |
3096 | | blosc2_filter_backward_cb backward; |
3097 | | //!< The filter function that is used during decompression. |
3098 | | } blosc2_filter; |
3099 | | |
3100 | | /** |
3101 | | * @brief Register locally a user-defined filter in Blosc. |
3102 | | * |
3103 | | * @param filter The filter to register. |
3104 | | * |
3105 | | * @return 0 if succeeds. Else a negative code is returned. |
3106 | | */ |
3107 | | BLOSC_EXPORT int blosc2_register_filter(blosc2_filter *filter); |
3108 | | |
3109 | | /********************************************************************* |
3110 | | Directory utilities. |
3111 | | *********************************************************************/ |
3112 | | |
3113 | | /* |
3114 | | * @brief Remove a directory and its files. |
3115 | | * |
3116 | | * @param path The directory to remove. |
3117 | | * |
3118 | | * @return 0 if succeeds. Else a negative code is returned. |
3119 | | */ |
3120 | | BLOSC_EXPORT int blosc2_remove_dir(const char *path); |
3121 | | |
3122 | | /* |
3123 | | * @brief Remove a file or a directory given by path. |
3124 | | * |
3125 | | * @param path The file or directory to remove. |
3126 | | * |
3127 | | * @return 0 if succeeds. Else a negative code is returned. |
3128 | | */ |
3129 | | BLOSC_EXPORT int blosc2_remove_urlpath(const char *path); |
3130 | | |
3131 | | /* |
3132 | | * @brief Rename a file or a directory given by old_urlpath to new_path. |
3133 | | * |
3134 | | * @param old_urlpath The original path to the directory or file. |
3135 | | * @param new_path The new path to the directory or file. |
3136 | | * |
3137 | | * @return 0 if succeeds. Else a negative code is returned. |
3138 | | */ |
3139 | | BLOSC_EXPORT int blosc2_rename_urlpath(char* old_urlpath, char* new_path); |
3140 | | |
3141 | | |
3142 | | /********************************************************************* |
3143 | | Index utilities. |
3144 | | *********************************************************************/ |
3145 | | |
3146 | | /* |
3147 | | * @brief Convert a sequential index into a multidimensional index |
3148 | | * |
3149 | | * This function assume ndim <= B2ND_MAX_DIM. |
3150 | | */ |
3151 | | BLOSC_EXPORT void blosc2_unidim_to_multidim(uint8_t ndim, int64_t *shape, int64_t i, int64_t *index); |
3152 | | |
3153 | | /* |
3154 | | * @brief Convert a multidimensional index into a sequential index |
3155 | | */ |
3156 | | BLOSC_EXPORT void blosc2_multidim_to_unidim(const int64_t *index, int8_t ndim, const int64_t *strides, int64_t *i); |
3157 | | |
3158 | | /* |
3159 | | * @brief Get the unidimensional chunk indexes needed to get a slice of a schunk or a b2nd array |
3160 | | * |
3161 | | * @param schunk The super-chunk (of b2nd array or not). |
3162 | | * @param start Index (0-based if it is a schunk) where the slice begins. |
3163 | | * @param stop The first index (0-based if it is a schunk) that is not in the selected slice. |
3164 | | * @param chunks_idx The pointer to the buffer where the indexes will be written. It is the user responsibility |
3165 | | * to free the buffer. |
3166 | | * |
3167 | | * @return The number of chunks needed to get the slice. If some problem is |
3168 | | * detected, a negative code is returned instead. |
3169 | | */ |
3170 | | BLOSC_EXPORT int64_t blosc2_get_slice_nchunks(blosc2_schunk* schunk, int64_t *start, int64_t *stop, int64_t **chunks_idx); |
3171 | | |
3172 | | |
3173 | | /********************************************************************* |
3174 | | Raw shuffle functions. |
3175 | | *********************************************************************/ |
3176 | | |
3177 | | /** |
3178 | | * @brief Applies the shuffle operation to a block of data in @p src, and puts the result in @p dest. |
3179 | | * |
3180 | | * @warning The @p src buffer and the @p dest buffer can not overlap. |
3181 | | * |
3182 | | * @param typesize Is the number of bytes for the atomic type in binary @p src buffer. Only 1 < typesize |
3183 | | * < 256 is allowed. |
3184 | | * @param blocksize The size of the block. |
3185 | | * @param src The source buffer to be shuffled. |
3186 | | * @param dest The destination buffer where the shuffled data will be written. |
3187 | | * |
3188 | | * @return @p blocksize on success or a negative value if some error happens (mainly an invalid parameter). |
3189 | | */ |
3190 | | BLOSC_EXPORT int32_t blosc2_shuffle(const int32_t typesize, const int32_t blocksize, const void* src, |
3191 | | void* dest); |
3192 | | |
3193 | | /** |
3194 | | * @brief Applies the inverse shuffle operation to a block of data in @p src, and puts the result in @p dest. |
3195 | | * |
3196 | | * @warning The @p src buffer and the @p dest buffer can not overlap. |
3197 | | * |
3198 | | * @param typesize Is the number of bytes for the atomic type in binary @p src buffer. Only 1 < typesize |
3199 | | * < 256 is allowed. |
3200 | | * @param blocksize The size of the block. |
3201 | | * @param src The source buffer to be unshuffled. |
3202 | | * @param dest The destination buffer where the unshuffled data will be written. |
3203 | | * |
3204 | | * @return @p blocksize on success or a negative value if some error happens (mainly an invalid parameter). |
3205 | | */ |
3206 | | BLOSC_EXPORT int32_t blosc2_unshuffle(const int32_t typesize, const int32_t blocksize, const void* src, |
3207 | | void* dest); |
3208 | | |
3209 | | /** |
3210 | | * @brief Applies the bitshuffle operation to a block of data in @p src, and puts the result in @p dest. |
3211 | | * |
3212 | | * @warning The @p src buffer and the @p dest buffer can not overlap. |
3213 | | * |
3214 | | * @remark The function will shuffle the maximum amount of elements that can be divided by 8, and copy the |
3215 | | * rest to the destination buffer unchanged. There are @p blocksize / @p typesize elements. |
3216 | | * |
3217 | | * @param typesize Is the number of bytes for the atomic type in binary @p src buffer. Only 1 < typesize |
3218 | | * < 256 is allowed. |
3219 | | * @param blocksize The size of the block. |
3220 | | * @param src The source buffer to be shuffled. |
3221 | | * @param dest The destination buffer where the shuffled data will be written. |
3222 | | * |
3223 | | * @return @p blocksize on success or a negative value if some error happens (mainly an invalid parameter). |
3224 | | */ |
3225 | | BLOSC_EXPORT int32_t blosc2_bitshuffle(const int32_t typesize, const int32_t blocksize, const void* src, |
3226 | | void* dest); |
3227 | | |
3228 | | /** |
3229 | | * @brief Applies the inverse bitshuffle operation to a block of data in @p src, and puts the result in @p dest. |
3230 | | * |
3231 | | * @warning The @p src buffer and the @p dest buffer can not overlap. |
3232 | | * |
3233 | | * @remark The function will shuffle the maximum amount of elements that can be divided by 8, and copy the |
3234 | | * rest to the destination buffer unchanged. There are @p blocksize / @p typesize elements. |
3235 | | * |
3236 | | * @param typesize Is the number of bytes for the atomic type in binary @p src buffer. Only 1 < typesize |
3237 | | * < 256 is allowed. |
3238 | | * @param blocksize The size of the block. |
3239 | | * @param src The source buffer to be unshuffled. |
3240 | | * @param dest The destination buffer where the unshuffled data will be written. |
3241 | | * |
3242 | | * @return @p blocksize on success or a negative value if some error happens (mainly an invalid parameter). |
3243 | | */ |
3244 | | BLOSC_EXPORT int32_t blosc2_bitunshuffle(const int32_t typesize, const int32_t blocksize, const void* src, |
3245 | | void* dest); |
3246 | | |
3247 | | |
3248 | | /********************************************************************* |
3249 | | Private functions, these are here for convenience, |
3250 | | and are not meant to be included in public docs |
3251 | | *********************************************************************/ |
3252 | | |
3253 | | // Private function needed in b2nd.h for deserializing meta |
3254 | 339k | static inline void swap_store(void *dest, const void *pa, int size) { |
3255 | 339k | uint8_t *pa_ = (uint8_t *) pa; |
3256 | 339k | uint8_t *pa2_ = (uint8_t*)malloc((size_t) size); |
3257 | 339k | int i = 1; /* for big/little endian detection */ |
3258 | 339k | char *p = (char *) &i; |
3259 | | |
3260 | 339k | if (p[0] == 1) { |
3261 | | /* little endian */ |
3262 | 339k | switch (size) { |
3263 | 106k | case 8: |
3264 | 106k | pa2_[0] = pa_[7]; |
3265 | 106k | pa2_[1] = pa_[6]; |
3266 | 106k | pa2_[2] = pa_[5]; |
3267 | 106k | pa2_[3] = pa_[4]; |
3268 | 106k | pa2_[4] = pa_[3]; |
3269 | 106k | pa2_[5] = pa_[2]; |
3270 | 106k | pa2_[6] = pa_[1]; |
3271 | 106k | pa2_[7] = pa_[0]; |
3272 | 106k | break; |
3273 | 232k | case 4: |
3274 | 232k | pa2_[0] = pa_[3]; |
3275 | 232k | pa2_[1] = pa_[2]; |
3276 | 232k | pa2_[2] = pa_[1]; |
3277 | 232k | pa2_[3] = pa_[0]; |
3278 | 232k | break; |
3279 | 0 | case 2: |
3280 | 0 | pa2_[0] = pa_[1]; |
3281 | 0 | pa2_[1] = pa_[0]; |
3282 | 0 | break; |
3283 | 0 | case 1: |
3284 | 0 | pa2_[0] = pa_[0]; |
3285 | 0 | break; |
3286 | 0 | default: |
3287 | 0 | fprintf(stderr, "Unhandled nitems: %d\n", size); |
3288 | 339k | } |
3289 | 339k | } else { |
3290 | | /* big endian: native byte order is already big-endian, just copy */ |
3291 | 0 | memcpy(pa2_, pa_, size); |
3292 | 0 | } |
3293 | 339k | memcpy(dest, pa2_, size); |
3294 | 339k | free(pa2_); |
3295 | 339k | } Unexecuted instantiation: fuzz_compress_frame.c:swap_store Unexecuted instantiation: blosc2.c:swap_store Unexecuted instantiation: blosclz.c:swap_store Unexecuted instantiation: schunk.c:swap_store Unexecuted instantiation: frame.c:swap_store Unexecuted instantiation: prefetch.c:swap_store Unexecuted instantiation: stune.c:swap_store Unexecuted instantiation: trunc-prec.c:swap_store Unexecuted instantiation: timestamp.c:swap_store Unexecuted instantiation: sframe.c:swap_store Unexecuted instantiation: blosc2-stdio.c:swap_store Line | Count | Source | 3254 | 336k | static inline void swap_store(void *dest, const void *pa, int size) { | 3255 | 336k | uint8_t *pa_ = (uint8_t *) pa; | 3256 | 336k | uint8_t *pa2_ = (uint8_t*)malloc((size_t) size); | 3257 | 336k | int i = 1; /* for big/little endian detection */ | 3258 | 336k | char *p = (char *) &i; | 3259 | | | 3260 | 336k | if (p[0] == 1) { | 3261 | | /* little endian */ | 3262 | 336k | switch (size) { | 3263 | 105k | case 8: | 3264 | 105k | pa2_[0] = pa_[7]; | 3265 | 105k | pa2_[1] = pa_[6]; | 3266 | 105k | pa2_[2] = pa_[5]; | 3267 | 105k | pa2_[3] = pa_[4]; | 3268 | 105k | pa2_[4] = pa_[3]; | 3269 | 105k | pa2_[5] = pa_[2]; | 3270 | 105k | pa2_[6] = pa_[1]; | 3271 | 105k | pa2_[7] = pa_[0]; | 3272 | 105k | break; | 3273 | 230k | case 4: | 3274 | 230k | pa2_[0] = pa_[3]; | 3275 | 230k | pa2_[1] = pa_[2]; | 3276 | 230k | pa2_[2] = pa_[1]; | 3277 | 230k | pa2_[3] = pa_[0]; | 3278 | 230k | break; | 3279 | 0 | case 2: | 3280 | 0 | pa2_[0] = pa_[1]; | 3281 | 0 | pa2_[1] = pa_[0]; | 3282 | 0 | break; | 3283 | 0 | case 1: | 3284 | 0 | pa2_[0] = pa_[0]; | 3285 | 0 | break; | 3286 | 0 | default: | 3287 | 0 | fprintf(stderr, "Unhandled nitems: %d\n", size); | 3288 | 336k | } | 3289 | 336k | } else { | 3290 | | /* big endian: native byte order is already big-endian, just copy */ | 3291 | 0 | memcpy(pa2_, pa_, size); | 3292 | 0 | } | 3293 | 336k | memcpy(dest, pa2_, size); | 3294 | 336k | free(pa2_); | 3295 | 336k | } |
Unexecuted instantiation: b2nd_utils.c:swap_store Unexecuted instantiation: shuffle.c:swap_store Unexecuted instantiation: shuffle-bytedelta-ssse3.c:swap_store Line | Count | Source | 3254 | 3.12k | static inline void swap_store(void *dest, const void *pa, int size) { | 3255 | 3.12k | uint8_t *pa_ = (uint8_t *) pa; | 3256 | 3.12k | uint8_t *pa2_ = (uint8_t*)malloc((size_t) size); | 3257 | 3.12k | int i = 1; /* for big/little endian detection */ | 3258 | 3.12k | char *p = (char *) &i; | 3259 | | | 3260 | 3.12k | if (p[0] == 1) { | 3261 | | /* little endian */ | 3262 | 3.12k | switch (size) { | 3263 | 1.15k | case 8: | 3264 | 1.15k | pa2_[0] = pa_[7]; | 3265 | 1.15k | pa2_[1] = pa_[6]; | 3266 | 1.15k | pa2_[2] = pa_[5]; | 3267 | 1.15k | pa2_[3] = pa_[4]; | 3268 | 1.15k | pa2_[4] = pa_[3]; | 3269 | 1.15k | pa2_[5] = pa_[2]; | 3270 | 1.15k | pa2_[6] = pa_[1]; | 3271 | 1.15k | pa2_[7] = pa_[0]; | 3272 | 1.15k | break; | 3273 | 1.96k | case 4: | 3274 | 1.96k | pa2_[0] = pa_[3]; | 3275 | 1.96k | pa2_[1] = pa_[2]; | 3276 | 1.96k | pa2_[2] = pa_[1]; | 3277 | 1.96k | pa2_[3] = pa_[0]; | 3278 | 1.96k | break; | 3279 | 0 | case 2: | 3280 | 0 | pa2_[0] = pa_[1]; | 3281 | 0 | pa2_[1] = pa_[0]; | 3282 | 0 | break; | 3283 | 0 | case 1: | 3284 | 0 | pa2_[0] = pa_[0]; | 3285 | 0 | break; | 3286 | 0 | default: | 3287 | 0 | fprintf(stderr, "Unhandled nitems: %d\n", size); | 3288 | 3.12k | } | 3289 | 3.12k | } else { | 3290 | | /* big endian: native byte order is already big-endian, just copy */ | 3291 | 0 | memcpy(pa2_, pa_, size); | 3292 | 0 | } | 3293 | 3.12k | memcpy(dest, pa2_, size); | 3294 | 3.12k | free(pa2_); | 3295 | 3.12k | } |
Unexecuted instantiation: codecs-registry.c:swap_store Unexecuted instantiation: tuners-registry.c:swap_store Unexecuted instantiation: filters-registry.c:swap_store Unexecuted instantiation: ndlz.c:swap_store Unexecuted instantiation: ndlz4x4.c:swap_store Unexecuted instantiation: ndlz8x8.c:swap_store Unexecuted instantiation: ndcell.c:swap_store Unexecuted instantiation: ndmean.c:swap_store Unexecuted instantiation: bytedelta.c:swap_store Unexecuted instantiation: int_trunc.c:swap_store Unexecuted instantiation: fuzz_plugins.c:swap_store Unexecuted instantiation: fuzz_decompress_frame.c:swap_store Unexecuted instantiation: fuzz_compress_chunk.c:swap_store Unexecuted instantiation: fuzz_schunk_ops.c:swap_store Unexecuted instantiation: fuzz_decompress_chunk.c:swap_store Unexecuted instantiation: fuzz_b2nd.c:swap_store |
3296 | | |
3297 | | #ifdef __cplusplus |
3298 | | } |
3299 | | #endif |
3300 | | |
3301 | | #endif /* BLOSC_BLOSC2_H */ |