/src/c-blosc2/blosc/schunk.c
Line | Count | Source (jump to first uncovered line) |
1 | | /********************************************************************* |
2 | | Blosc - Blocked Shuffling and Compression Library |
3 | | |
4 | | Copyright (c) 2021 The Blosc Development Team <blosc@blosc.org> |
5 | | https://blosc.org |
6 | | License: BSD 3-Clause (see LICENSE.txt) |
7 | | |
8 | | See LICENSE.txt for details about copyright and rights to use. |
9 | | **********************************************************************/ |
10 | | |
11 | | #include "frame.h" |
12 | | #include "stune.h" |
13 | | #include "blosc-private.h" |
14 | | #include "blosc2/tuners-registry.h" |
15 | | #include "blosc2.h" |
16 | | |
17 | | #if defined(_WIN32) |
18 | | #include <windows.h> |
19 | | #include <direct.h> |
20 | | #include <malloc.h> |
21 | | #define mkdir(D, M) _mkdir(D) |
22 | | #endif /* _WIN32 */ |
23 | | |
24 | | #include <sys/stat.h> |
25 | | |
26 | | #include <inttypes.h> |
27 | | #include <stdio.h> |
28 | | #include <stdint.h> |
29 | | #include <stdlib.h> |
30 | | #include <string.h> |
31 | | |
32 | | /* If C11 is supported, use it's built-in aligned allocation. */ |
33 | | #if __STDC_VERSION__ >= 201112L |
34 | | #include <stdalign.h> |
35 | | #endif |
36 | | |
37 | | |
38 | | /* Get the cparams associated with a super-chunk */ |
39 | 5 | int blosc2_schunk_get_cparams(blosc2_schunk *schunk, blosc2_cparams **cparams) { |
40 | 5 | *cparams = calloc(1, sizeof(blosc2_cparams)); |
41 | 5 | (*cparams)->schunk = schunk; |
42 | 35 | for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) { |
43 | 30 | (*cparams)->filters[i] = schunk->filters[i]; |
44 | 30 | (*cparams)->filters_meta[i] = schunk->filters_meta[i]; |
45 | 30 | } |
46 | 5 | (*cparams)->compcode = schunk->compcode; |
47 | 5 | (*cparams)->compcode_meta = schunk->compcode_meta; |
48 | 5 | (*cparams)->clevel = schunk->clevel; |
49 | 5 | (*cparams)->typesize = schunk->typesize; |
50 | 5 | (*cparams)->blocksize = schunk->blocksize; |
51 | 5 | (*cparams)->splitmode = schunk->splitmode; |
52 | 5 | if (schunk->cctx == NULL) { |
53 | 5 | (*cparams)->nthreads = blosc2_get_nthreads(); |
54 | 5 | } |
55 | 0 | else { |
56 | 0 | (*cparams)->nthreads = (int16_t)schunk->cctx->nthreads; |
57 | 0 | } |
58 | 5 | return 0; |
59 | 5 | } |
60 | | |
61 | | |
62 | | /* Get the dparams associated with a super-chunk */ |
63 | 5 | int blosc2_schunk_get_dparams(blosc2_schunk *schunk, blosc2_dparams **dparams) { |
64 | 5 | *dparams = calloc(1, sizeof(blosc2_dparams)); |
65 | 5 | (*dparams)->schunk = schunk; |
66 | 5 | if (schunk->dctx == NULL) { |
67 | 5 | (*dparams)->nthreads = blosc2_get_nthreads(); |
68 | 5 | } |
69 | 0 | else { |
70 | 0 | (*dparams)->nthreads = schunk->dctx->nthreads; |
71 | 0 | } |
72 | 5 | return 0; |
73 | 5 | } |
74 | | |
75 | | |
76 | 2.40k | int update_schunk_properties(struct blosc2_schunk* schunk) { |
77 | 2.40k | blosc2_cparams* cparams = schunk->storage->cparams; |
78 | 2.40k | blosc2_dparams* dparams = schunk->storage->dparams; |
79 | | |
80 | 16.8k | for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) { |
81 | 14.4k | schunk->filters[i] = cparams->filters[i]; |
82 | 14.4k | schunk->filters_meta[i] = cparams->filters_meta[i]; |
83 | 14.4k | } |
84 | 2.40k | schunk->compcode = cparams->compcode; |
85 | 2.40k | schunk->compcode_meta = cparams->compcode_meta; |
86 | 2.40k | schunk->clevel = cparams->clevel; |
87 | 2.40k | schunk->splitmode = cparams->splitmode; |
88 | 2.40k | schunk->typesize = cparams->typesize; |
89 | 2.40k | schunk->blocksize = cparams->blocksize; |
90 | 2.40k | schunk->chunksize = -1; |
91 | 2.40k | schunk->tuner_params = cparams->tuner_params; |
92 | 2.40k | schunk->tuner_id = cparams->tuner_id; |
93 | 2.40k | if (cparams->tuner_id == BLOSC_BTUNE) { |
94 | 0 | cparams->use_dict = 0; |
95 | 0 | } |
96 | | /* The compression context */ |
97 | 2.40k | if (schunk->cctx != NULL) { |
98 | 0 | blosc2_free_ctx(schunk->cctx); |
99 | 0 | } |
100 | 2.40k | cparams->schunk = schunk; |
101 | 2.40k | schunk->cctx = blosc2_create_cctx(*cparams); |
102 | 2.40k | if (schunk->cctx == NULL) { |
103 | 0 | BLOSC_TRACE_ERROR("Could not create compression ctx"); |
104 | 0 | return BLOSC2_ERROR_NULL_POINTER; |
105 | 0 | } |
106 | | |
107 | | /* The decompression context */ |
108 | 2.40k | if (schunk->dctx != NULL) { |
109 | 0 | blosc2_free_ctx(schunk->dctx); |
110 | 0 | } |
111 | 2.40k | dparams->schunk = schunk; |
112 | 2.40k | schunk->dctx = blosc2_create_dctx(*dparams); |
113 | 2.40k | if (schunk->dctx == NULL) { |
114 | 0 | BLOSC_TRACE_ERROR("Could not create decompression ctx"); |
115 | 0 | return BLOSC2_ERROR_NULL_POINTER; |
116 | 0 | } |
117 | | |
118 | 2.40k | return BLOSC2_ERROR_SUCCESS; |
119 | 2.40k | } |
120 | | |
121 | | |
122 | 0 | static bool file_exists (char *filename) { |
123 | 0 | struct stat buffer; |
124 | 0 | return (stat (filename, &buffer) == 0); |
125 | 0 | } |
126 | | |
127 | | |
128 | | /* Create a new super-chunk */ |
129 | 2.40k | blosc2_schunk* blosc2_schunk_new(blosc2_storage *storage) { |
130 | 2.40k | blosc2_schunk* schunk = calloc(1, sizeof(blosc2_schunk)); |
131 | 2.40k | schunk->version = 0; /* pre-first version */ |
132 | | |
133 | | // Get the storage with proper defaults |
134 | 2.40k | schunk->storage = get_new_storage(storage, &BLOSC2_CPARAMS_DEFAULTS, &BLOSC2_DPARAMS_DEFAULTS, &BLOSC2_IO_DEFAULTS); |
135 | | // Update the (local variable) storage |
136 | 2.40k | storage = schunk->storage; |
137 | | |
138 | 2.40k | char* tradeoff = getenv("BTUNE_TRADEOFF"); |
139 | 2.40k | if (tradeoff != NULL) { |
140 | | // If BTUNE_TRADEOFF passed, automatically use btune |
141 | 0 | storage->cparams->tuner_id = BLOSC_BTUNE; |
142 | 0 | } |
143 | | |
144 | | // ...and update internal properties |
145 | 2.40k | if (update_schunk_properties(schunk) < 0) { |
146 | 0 | BLOSC_TRACE_ERROR("Error when updating schunk properties"); |
147 | 0 | return NULL; |
148 | 0 | } |
149 | | |
150 | 2.40k | if (!storage->contiguous && storage->urlpath != NULL){ |
151 | 0 | char* urlpath; |
152 | 0 | char last_char = storage->urlpath[strlen(storage->urlpath) - 1]; |
153 | 0 | urlpath = malloc(strlen(storage->urlpath) + 1); |
154 | 0 | strcpy(urlpath, storage->urlpath); |
155 | 0 | if (last_char == '\\' || last_char == '/') { |
156 | 0 | urlpath[strlen(storage->urlpath) - 1] = '\0'; |
157 | 0 | } |
158 | | // Create directory |
159 | 0 | if (mkdir(urlpath, 0777) == -1) { |
160 | 0 | BLOSC_TRACE_ERROR("Error during the creation of the directory, maybe it already exists."); |
161 | 0 | return NULL; |
162 | 0 | } |
163 | | // We want a sparse (directory) frame as storage |
164 | 0 | blosc2_frame_s* frame = frame_new(urlpath); |
165 | 0 | free(urlpath); |
166 | 0 | frame->sframe = true; |
167 | | // Initialize frame (basically, encode the header) |
168 | 0 | frame->schunk = schunk; |
169 | 0 | int64_t frame_len = frame_from_schunk(schunk, frame); |
170 | 0 | if (frame_len < 0) { |
171 | 0 | BLOSC_TRACE_ERROR("Error during the conversion of schunk to frame."); |
172 | 0 | return NULL; |
173 | 0 | } |
174 | 0 | schunk->frame = (blosc2_frame*)frame; |
175 | 0 | } |
176 | 2.40k | if (storage->contiguous){ |
177 | | // We want a contiguous frame as storage |
178 | 0 | if (storage->urlpath != NULL) { |
179 | 0 | if (file_exists(storage->urlpath)) { |
180 | 0 | BLOSC_TRACE_ERROR("You are trying to overwrite an existing frame. Remove it first!"); |
181 | 0 | return NULL; |
182 | 0 | } |
183 | 0 | } |
184 | 0 | blosc2_frame_s* frame = frame_new(storage->urlpath); |
185 | 0 | frame->sframe = false; |
186 | | // Initialize frame (basically, encode the header) |
187 | 0 | frame->schunk = schunk; |
188 | 0 | int64_t frame_len = frame_from_schunk(schunk, frame); |
189 | 0 | if (frame_len < 0) { |
190 | 0 | BLOSC_TRACE_ERROR("Error during the conversion of schunk to frame."); |
191 | 0 | return NULL; |
192 | 0 | } |
193 | 0 | schunk->frame = (blosc2_frame*)frame; |
194 | 0 | } |
195 | | |
196 | 2.40k | return schunk; |
197 | 2.40k | } |
198 | | |
199 | | |
200 | | /* Create a copy of a super-chunk */ |
201 | 0 | blosc2_schunk* blosc2_schunk_copy(blosc2_schunk *schunk, blosc2_storage *storage) { |
202 | 0 | if (schunk == NULL) { |
203 | 0 | BLOSC_TRACE_ERROR("Can not copy a NULL `schunk`."); |
204 | 0 | return NULL; |
205 | 0 | } |
206 | | |
207 | | // Check if cparams are equals |
208 | 0 | bool cparams_equal = true; |
209 | 0 | blosc2_cparams cparams = {0}; |
210 | 0 | if (storage->cparams == NULL) { |
211 | | // When cparams are not specified, just use the same of schunk |
212 | 0 | cparams.typesize = schunk->cctx->typesize; |
213 | 0 | cparams.clevel = schunk->cctx->clevel; |
214 | 0 | cparams.compcode = schunk->cctx->compcode; |
215 | 0 | cparams.compcode_meta = schunk->cctx->compcode_meta; |
216 | 0 | cparams.splitmode = schunk->cctx->splitmode; |
217 | 0 | cparams.use_dict = schunk->cctx->use_dict; |
218 | 0 | cparams.blocksize = schunk->cctx->blocksize; |
219 | 0 | memcpy(cparams.filters, schunk->cctx->filters, BLOSC2_MAX_FILTERS); |
220 | 0 | memcpy(cparams.filters_meta, schunk->cctx->filters_meta, BLOSC2_MAX_FILTERS); |
221 | 0 | storage->cparams = &cparams; |
222 | 0 | } |
223 | 0 | else { |
224 | 0 | cparams = *storage->cparams; |
225 | 0 | } |
226 | 0 | if (cparams.blocksize == 0) { |
227 | | // TODO: blocksize should be read from schunk->blocksize |
228 | | // For this, it should be updated during the first append |
229 | | // (or change API to make this a property during schunk creation). |
230 | 0 | cparams.blocksize = schunk->cctx->blocksize; |
231 | 0 | } |
232 | |
|
233 | 0 | if (cparams.typesize != schunk->cctx->typesize || |
234 | 0 | cparams.clevel != schunk->cctx->clevel || |
235 | 0 | cparams.compcode != schunk->cctx->compcode || |
236 | 0 | cparams.use_dict != schunk->cctx->use_dict || |
237 | 0 | cparams.blocksize != schunk->cctx->blocksize || |
238 | | // In case of prefilters or postfilters, force their execution. |
239 | 0 | schunk->cctx->prefilter != NULL || |
240 | 0 | schunk->dctx->postfilter != NULL) { |
241 | 0 | cparams_equal = false; |
242 | 0 | } |
243 | 0 | for (int i = 0; i < BLOSC2_MAX_FILTERS; ++i) { |
244 | 0 | if (cparams.filters[i] != schunk->cctx->filters[i] || |
245 | 0 | cparams.filters_meta[i] != schunk->cctx->filters_meta[i]) { |
246 | 0 | cparams_equal = false; |
247 | 0 | } |
248 | 0 | } |
249 | | |
250 | | // Create new schunk |
251 | 0 | blosc2_schunk *new_schunk = blosc2_schunk_new(storage); |
252 | 0 | if (new_schunk == NULL) { |
253 | 0 | BLOSC_TRACE_ERROR("Can not create a new schunk"); |
254 | 0 | return NULL; |
255 | 0 | } |
256 | | |
257 | | // Copy metalayers |
258 | 0 | for (int nmeta = 0; nmeta < schunk->nmetalayers; ++nmeta) { |
259 | 0 | blosc2_metalayer *meta = schunk->metalayers[nmeta]; |
260 | 0 | if (blosc2_meta_add(new_schunk, meta->name, meta->content, meta->content_len) < 0) { |
261 | 0 | BLOSC_TRACE_ERROR("Can not add %s `metalayer`.", meta->name); |
262 | 0 | return NULL; |
263 | 0 | } |
264 | 0 | } |
265 | | |
266 | | // Copy chunks |
267 | 0 | if (cparams_equal) { |
268 | 0 | for (int nchunk = 0; nchunk < schunk->nchunks; ++nchunk) { |
269 | 0 | uint8_t *chunk; |
270 | 0 | bool needs_free; |
271 | 0 | if (blosc2_schunk_get_chunk(schunk, nchunk, &chunk, &needs_free) < 0) { |
272 | 0 | BLOSC_TRACE_ERROR("Can not get the `chunk` %d.", nchunk); |
273 | 0 | return NULL; |
274 | 0 | } |
275 | 0 | if (blosc2_schunk_append_chunk(new_schunk, chunk, !needs_free) < 0) { |
276 | 0 | BLOSC_TRACE_ERROR("Can not append the `chunk` into super-chunk."); |
277 | 0 | return NULL; |
278 | 0 | } |
279 | 0 | } |
280 | 0 | } else { |
281 | 0 | int32_t chunksize = schunk->chunksize == -1 ? 0 : schunk->chunksize; |
282 | 0 | uint8_t *buffer = malloc(chunksize); |
283 | 0 | for (int nchunk = 0; nchunk < schunk->nchunks; ++nchunk) { |
284 | 0 | if (blosc2_schunk_decompress_chunk(schunk, nchunk, buffer, schunk->chunksize) < 0) { |
285 | 0 | BLOSC_TRACE_ERROR("Can not decompress the `chunk` %d.", nchunk); |
286 | 0 | return NULL; |
287 | 0 | } |
288 | 0 | if (blosc2_schunk_append_buffer(new_schunk, buffer, schunk->chunksize) < 0) { |
289 | 0 | BLOSC_TRACE_ERROR("Can not append the `buffer` into super-chunk."); |
290 | 0 | return NULL; |
291 | 0 | } |
292 | 0 | } |
293 | 0 | free(buffer); |
294 | 0 | } |
295 | | |
296 | | // Copy vlmetalayers |
297 | 0 | for (int nmeta = 0; nmeta < schunk->nvlmetalayers; ++nmeta) { |
298 | 0 | uint8_t *content; |
299 | 0 | int32_t content_len; |
300 | 0 | char* name = schunk->vlmetalayers[nmeta]->name; |
301 | 0 | if (blosc2_vlmeta_get(schunk, name, &content, &content_len) < 0) { |
302 | 0 | BLOSC_TRACE_ERROR("Can not get %s `vlmetalayer`.", name); |
303 | 0 | } |
304 | 0 | if (blosc2_vlmeta_add(new_schunk, name, content, content_len, NULL) < 0) { |
305 | 0 | BLOSC_TRACE_ERROR("Can not add %s `vlmetalayer`.", name); |
306 | 0 | return NULL; |
307 | 0 | } |
308 | 0 | free(content); |
309 | 0 | } |
310 | 0 | return new_schunk; |
311 | 0 | } |
312 | | |
313 | | |
314 | | /* Open an existing super-chunk that is on-disk (no copy is made). */ |
315 | 0 | blosc2_schunk* blosc2_schunk_open_udio(const char* urlpath, const blosc2_io *udio) { |
316 | 0 | if (urlpath == NULL) { |
317 | 0 | BLOSC_TRACE_ERROR("You need to supply a urlpath."); |
318 | 0 | return NULL; |
319 | 0 | } |
320 | | |
321 | 0 | blosc2_frame_s* frame = frame_from_file_offset(urlpath, udio, 0); |
322 | 0 | if (frame == NULL) { |
323 | 0 | return NULL; |
324 | 0 | } |
325 | 0 | blosc2_schunk* schunk = frame_to_schunk(frame, false, udio); |
326 | | |
327 | | // Set the storage with proper defaults |
328 | 0 | size_t pathlen = strlen(urlpath); |
329 | 0 | schunk->storage->urlpath = malloc(pathlen + 1); |
330 | 0 | strcpy(schunk->storage->urlpath, urlpath); |
331 | 0 | schunk->storage->contiguous = !frame->sframe; |
332 | |
|
333 | 0 | return schunk; |
334 | 0 | } |
335 | | |
336 | 0 | blosc2_schunk* blosc2_schunk_open(const char* urlpath) { |
337 | 0 | return blosc2_schunk_open_udio(urlpath, &BLOSC2_IO_DEFAULTS); |
338 | 0 | } |
339 | | |
340 | 0 | blosc2_schunk* blosc2_schunk_open_offset(const char* urlpath, int64_t offset) { |
341 | 0 | if (urlpath == NULL) { |
342 | 0 | BLOSC_TRACE_ERROR("You need to supply a urlpath."); |
343 | 0 | return NULL; |
344 | 0 | } |
345 | | |
346 | 0 | blosc2_frame_s* frame = frame_from_file_offset(urlpath, &BLOSC2_IO_DEFAULTS, offset); |
347 | 0 | if (frame == NULL) { |
348 | 0 | return NULL; |
349 | 0 | } |
350 | 0 | blosc2_schunk* schunk = frame_to_schunk(frame, false, &BLOSC2_IO_DEFAULTS); |
351 | | |
352 | | // Set the storage with proper defaults |
353 | 0 | size_t pathlen = strlen(urlpath); |
354 | 0 | schunk->storage->urlpath = malloc(pathlen + 1); |
355 | 0 | strcpy(schunk->storage->urlpath, urlpath); |
356 | 0 | schunk->storage->contiguous = !frame->sframe; |
357 | |
|
358 | 0 | return schunk; |
359 | 0 | } |
360 | | |
361 | 0 | int64_t blosc2_schunk_to_buffer(blosc2_schunk* schunk, uint8_t** dest, bool* needs_free) { |
362 | 0 | blosc2_frame_s* frame; |
363 | 0 | int64_t cframe_len; |
364 | | |
365 | | // Initialize defaults in case of errors |
366 | 0 | *dest = NULL; |
367 | 0 | *needs_free = false; |
368 | |
|
369 | 0 | if ((schunk->storage->contiguous == true) && (schunk->storage->urlpath == NULL)) { |
370 | 0 | frame = (blosc2_frame_s*)(schunk->frame); |
371 | 0 | *dest = frame->cframe; |
372 | 0 | cframe_len = frame->len; |
373 | 0 | *needs_free = false; |
374 | 0 | } |
375 | 0 | else { |
376 | | // Copy to a contiguous storage |
377 | 0 | blosc2_storage frame_storage = {.contiguous=true}; |
378 | 0 | blosc2_schunk* schunk_copy = blosc2_schunk_copy(schunk, &frame_storage); |
379 | 0 | if (schunk_copy == NULL) { |
380 | 0 | BLOSC_TRACE_ERROR("Error during the conversion of schunk to buffer."); |
381 | 0 | return BLOSC2_ERROR_SCHUNK_COPY; |
382 | 0 | } |
383 | 0 | frame = (blosc2_frame_s*)(schunk_copy->frame); |
384 | 0 | *dest = frame->cframe; |
385 | 0 | cframe_len = frame->len; |
386 | 0 | *needs_free = true; |
387 | 0 | frame->avoid_cframe_free = true; |
388 | 0 | blosc2_schunk_free(schunk_copy); |
389 | 0 | } |
390 | | |
391 | 0 | return cframe_len; |
392 | |
|
393 | 0 | } |
394 | | |
395 | | |
396 | | /* Write an in-memory frame out to a file. */ |
397 | 0 | int64_t frame_to_file(blosc2_frame_s* frame, const char* urlpath) { |
398 | 0 | blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); |
399 | 0 | if (io_cb == NULL) { |
400 | 0 | BLOSC_TRACE_ERROR("Error getting the input/output API"); |
401 | 0 | return BLOSC2_ERROR_PLUGIN_IO; |
402 | 0 | } |
403 | 0 | void* fp = io_cb->open(urlpath, "wb", frame->schunk->storage->io); |
404 | 0 | int64_t nitems = io_cb->write(frame->cframe, frame->len, 1, fp); |
405 | 0 | io_cb->close(fp); |
406 | 0 | return nitems * frame->len; |
407 | 0 | } |
408 | | |
409 | | |
410 | | /* Append an in-memory frame to a file. */ |
411 | 0 | int64_t append_frame_to_file(blosc2_frame_s* frame, const char* urlpath) { |
412 | 0 | blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id); |
413 | 0 | if (io_cb == NULL) { |
414 | 0 | BLOSC_TRACE_ERROR("Error getting the input/output API"); |
415 | 0 | return BLOSC2_ERROR_PLUGIN_IO; |
416 | 0 | } |
417 | 0 | void* fp = io_cb->open(urlpath, "ab", frame->schunk->storage->io); |
418 | 0 | int64_t offset; |
419 | |
|
420 | | # if (UNIX) |
421 | | offset = io_cb->tell(fp); |
422 | | # else |
423 | 0 | io_cb->seek(fp, 0, SEEK_END); |
424 | 0 | offset = io_cb->tell(fp); |
425 | 0 | # endif |
426 | |
|
427 | 0 | io_cb->write(frame->cframe, frame->len, 1, fp); |
428 | 0 | io_cb->close(fp); |
429 | 0 | return offset; |
430 | 0 | } |
431 | | |
432 | | |
433 | | /* Write super-chunk out to a file. */ |
434 | 0 | int64_t blosc2_schunk_to_file(blosc2_schunk* schunk, const char* urlpath) { |
435 | 0 | if (urlpath == NULL) { |
436 | 0 | BLOSC_TRACE_ERROR("urlpath cannot be NULL"); |
437 | 0 | return BLOSC2_ERROR_INVALID_PARAM; |
438 | 0 | } |
439 | | |
440 | | // Accelerated path for in-memory frames |
441 | 0 | if (schunk->storage->contiguous && schunk->storage->urlpath == NULL) { |
442 | 0 | int64_t len = frame_to_file((blosc2_frame_s*)(schunk->frame), urlpath); |
443 | 0 | if (len <= 0) { |
444 | 0 | BLOSC_TRACE_ERROR("Error writing to file"); |
445 | 0 | return len; |
446 | 0 | } |
447 | 0 | return len; |
448 | 0 | } |
449 | | |
450 | | // Copy to a contiguous file |
451 | 0 | blosc2_storage frame_storage = {.contiguous=true, .urlpath=(char*)urlpath}; |
452 | 0 | blosc2_schunk* schunk_copy = blosc2_schunk_copy(schunk, &frame_storage); |
453 | 0 | if (schunk_copy == NULL) { |
454 | 0 | BLOSC_TRACE_ERROR("Error during the conversion of schunk to buffer."); |
455 | 0 | return BLOSC2_ERROR_SCHUNK_COPY; |
456 | 0 | } |
457 | 0 | blosc2_frame_s* frame = (blosc2_frame_s*)(schunk_copy->frame); |
458 | 0 | int64_t frame_len = frame->len; |
459 | 0 | blosc2_schunk_free(schunk_copy); |
460 | 0 | return frame_len; |
461 | 0 | } |
462 | | |
463 | | |
464 | | /* Append a super-chunk to a file. */ |
465 | 0 | int64_t blosc2_schunk_append_file(blosc2_schunk* schunk, const char* urlpath) { |
466 | 0 | if (urlpath == NULL) { |
467 | 0 | BLOSC_TRACE_ERROR("urlpath cannot be NULL"); |
468 | 0 | return BLOSC2_ERROR_INVALID_PARAM; |
469 | 0 | } |
470 | | |
471 | | // Accelerated path for in-memory frames |
472 | 0 | if (schunk->storage->contiguous && schunk->storage->urlpath == NULL) { |
473 | 0 | int64_t offset = append_frame_to_file((blosc2_frame_s*)(schunk->frame), urlpath); |
474 | 0 | if (offset <= 0) { |
475 | 0 | BLOSC_TRACE_ERROR("Error writing to file"); |
476 | 0 | return offset; |
477 | 0 | } |
478 | 0 | return offset; |
479 | 0 | } |
480 | | |
481 | | // Copy to a contiguous file |
482 | 0 | blosc2_storage frame_storage = {.contiguous=true, .urlpath=NULL}; |
483 | 0 | blosc2_schunk* schunk_copy = blosc2_schunk_copy(schunk, &frame_storage); |
484 | 0 | if (schunk_copy == NULL) { |
485 | 0 | BLOSC_TRACE_ERROR("Error during the conversion of schunk to buffer."); |
486 | 0 | return BLOSC2_ERROR_SCHUNK_COPY; |
487 | 0 | } |
488 | 0 | blosc2_frame_s* frame = (blosc2_frame_s*)(schunk_copy->frame); |
489 | 0 | int64_t offset = append_frame_to_file(frame, urlpath); |
490 | 0 | blosc2_schunk_free(schunk_copy); |
491 | 0 | return offset; |
492 | 0 | } |
493 | | |
494 | | |
495 | | /* Free all memory from a super-chunk. */ |
496 | 2.41k | int blosc2_schunk_free(blosc2_schunk *schunk) { |
497 | 2.41k | if (schunk->data != NULL) { |
498 | 50.5k | for (int i = 0; i < schunk->nchunks; i++) { |
499 | 48.1k | free(schunk->data[i]); |
500 | 48.1k | } |
501 | 2.40k | free(schunk->data); |
502 | 2.40k | } |
503 | 2.41k | if (schunk->cctx != NULL) |
504 | 2.41k | blosc2_free_ctx(schunk->cctx); |
505 | 2.41k | if (schunk->dctx != NULL) |
506 | 2.41k | blosc2_free_ctx(schunk->dctx); |
507 | 2.41k | if (schunk->blockshape != NULL) |
508 | 0 | free(schunk->blockshape); |
509 | | |
510 | 2.41k | if (schunk->nmetalayers > 0) { |
511 | 0 | for (int i = 0; i < schunk->nmetalayers; i++) { |
512 | 0 | if (schunk->metalayers[i] != NULL) { |
513 | 0 | if (schunk->metalayers[i]->name != NULL) |
514 | 0 | free(schunk->metalayers[i]->name); |
515 | 0 | if (schunk->metalayers[i]->content != NULL) |
516 | 0 | free(schunk->metalayers[i]->content); |
517 | 0 | free(schunk->metalayers[i]); |
518 | 0 | } |
519 | 0 | } |
520 | 0 | schunk->nmetalayers = 0; |
521 | 0 | } |
522 | | |
523 | 2.41k | if (schunk->storage != NULL) { |
524 | 2.41k | if (schunk->storage->urlpath != NULL) { |
525 | 0 | free(schunk->storage->urlpath); |
526 | 0 | } |
527 | 2.41k | free(schunk->storage->cparams); |
528 | 2.41k | free(schunk->storage->dparams); |
529 | 2.41k | free(schunk->storage->io); |
530 | 2.41k | free(schunk->storage); |
531 | 2.41k | } |
532 | | |
533 | 2.41k | if (schunk->frame != NULL) { |
534 | 5 | frame_free((blosc2_frame_s *) schunk->frame); |
535 | 5 | } |
536 | | |
537 | 2.41k | if (schunk->nvlmetalayers > 0) { |
538 | 10 | for (int i = 0; i < schunk->nvlmetalayers; ++i) { |
539 | 5 | if (schunk->vlmetalayers[i] != NULL) { |
540 | 5 | if (schunk->vlmetalayers[i]->name != NULL) |
541 | 5 | free(schunk->vlmetalayers[i]->name); |
542 | 5 | if (schunk->vlmetalayers[i]->content != NULL) |
543 | 5 | free(schunk->vlmetalayers[i]->content); |
544 | 5 | free(schunk->vlmetalayers[i]); |
545 | 5 | } |
546 | 5 | } |
547 | 5 | } |
548 | | |
549 | 2.41k | free(schunk); |
550 | | |
551 | 2.41k | return 0; |
552 | 2.41k | } |
553 | | |
554 | | |
555 | | /* Create a super-chunk out of a contiguous frame buffer */ |
556 | 10 | blosc2_schunk* blosc2_schunk_from_buffer(uint8_t *cframe, int64_t len, bool copy) { |
557 | 10 | blosc2_frame_s* frame = frame_from_cframe(cframe, len, false); |
558 | 10 | if (frame == NULL) { |
559 | 5 | return NULL; |
560 | 5 | } |
561 | | // Check that the buffer actually comes from a cframe |
562 | 5 | char *magic_number = (char *)cframe; |
563 | 5 | magic_number += FRAME_HEADER_MAGIC; |
564 | 5 | if (strcmp(magic_number, "b2frame\0") != 0) { |
565 | 0 | return NULL; |
566 | 0 | } |
567 | 5 | blosc2_schunk* schunk = frame_to_schunk(frame, copy, &BLOSC2_IO_DEFAULTS); |
568 | 5 | if (schunk && copy) { |
569 | | // Super-chunk has its own copy of frame |
570 | 0 | frame_free(frame); |
571 | 0 | } |
572 | 5 | return schunk; |
573 | 5 | } |
574 | | |
575 | | |
576 | | /* Create a super-chunk out of a contiguous frame buffer */ |
577 | 0 | void blosc2_schunk_avoid_cframe_free(blosc2_schunk *schunk, bool avoid_cframe_free) { |
578 | 0 | blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; |
579 | 0 | frame_avoid_cframe_free(frame, avoid_cframe_free); |
580 | 0 | } |
581 | | |
582 | | |
583 | | /* Fill an empty frame with special values (fast path). */ |
584 | | int64_t blosc2_schunk_fill_special(blosc2_schunk* schunk, int64_t nitems, int special_value, |
585 | 0 | int32_t chunksize) { |
586 | 0 | if (nitems == 0) { |
587 | 0 | return 0; |
588 | 0 | } |
589 | | |
590 | 0 | int32_t typesize = schunk->typesize; |
591 | |
|
592 | 0 | if ((nitems * typesize / chunksize) > INT_MAX) { |
593 | 0 | BLOSC_TRACE_ERROR("nitems is too large. Try increasing the chunksize."); |
594 | 0 | return BLOSC2_ERROR_SCHUNK_SPECIAL; |
595 | 0 | } |
596 | | |
597 | 0 | if ((schunk->nbytes > 0) || (schunk->cbytes > 0)) { |
598 | 0 | BLOSC_TRACE_ERROR("Filling with special values only works on empty super-chunks"); |
599 | 0 | return BLOSC2_ERROR_FRAME_SPECIAL; |
600 | 0 | } |
601 | | |
602 | | // Compute the number of chunks and the length of the offsets chunk |
603 | 0 | int32_t chunkitems = chunksize / typesize; |
604 | 0 | int64_t nchunks = nitems / chunkitems; |
605 | 0 | int32_t leftover_items = (int32_t)(nitems % chunkitems); |
606 | |
|
607 | 0 | if (schunk->frame == NULL) { |
608 | | // Build the special chunks |
609 | 0 | int32_t leftover_size = leftover_items * typesize; |
610 | 0 | void* chunk = malloc(BLOSC_EXTENDED_HEADER_LENGTH); |
611 | 0 | void* chunk2 = malloc(BLOSC_EXTENDED_HEADER_LENGTH); |
612 | 0 | blosc2_cparams* cparams; |
613 | 0 | blosc2_schunk_get_cparams(schunk, &cparams); |
614 | 0 | int csize, csize2; |
615 | 0 | switch (special_value) { |
616 | 0 | case BLOSC2_SPECIAL_ZERO: |
617 | 0 | csize = blosc2_chunk_zeros(*cparams, chunksize, chunk, BLOSC_EXTENDED_HEADER_LENGTH); |
618 | 0 | csize2 = blosc2_chunk_zeros(*cparams, leftover_size, chunk2, BLOSC_EXTENDED_HEADER_LENGTH); |
619 | 0 | break; |
620 | 0 | case BLOSC2_SPECIAL_UNINIT: |
621 | 0 | csize = blosc2_chunk_uninit(*cparams, chunksize, chunk, BLOSC_EXTENDED_HEADER_LENGTH); |
622 | 0 | csize2 = blosc2_chunk_uninit(*cparams, leftover_size, chunk2, BLOSC_EXTENDED_HEADER_LENGTH); |
623 | 0 | break; |
624 | 0 | case BLOSC2_SPECIAL_NAN: |
625 | 0 | csize = blosc2_chunk_nans(*cparams, chunksize, chunk, BLOSC_EXTENDED_HEADER_LENGTH); |
626 | 0 | csize2 = blosc2_chunk_nans(*cparams, leftover_size, chunk2, BLOSC_EXTENDED_HEADER_LENGTH); |
627 | 0 | break; |
628 | 0 | default: |
629 | 0 | BLOSC_TRACE_ERROR("Only zeros, NaNs or non-initialized values are supported."); |
630 | 0 | return BLOSC2_ERROR_SCHUNK_SPECIAL; |
631 | 0 | } |
632 | 0 | free(cparams); |
633 | 0 | if (csize < 0 || csize2 < 0) { |
634 | 0 | BLOSC_TRACE_ERROR("Error creating special chunks."); |
635 | 0 | return BLOSC2_ERROR_SCHUNK_SPECIAL; |
636 | 0 | } |
637 | | |
638 | 0 | for (int nchunk = 0; nchunk < nchunks; nchunk++) { |
639 | 0 | int64_t nchunk_ = blosc2_schunk_append_chunk(schunk, chunk, true); |
640 | 0 | if (nchunk_ != nchunk + 1) { |
641 | 0 | BLOSC_TRACE_ERROR("Error appending special chunks."); |
642 | 0 | return BLOSC2_ERROR_SCHUNK_SPECIAL; |
643 | 0 | } |
644 | 0 | } |
645 | | |
646 | 0 | if (leftover_items) { |
647 | 0 | int64_t nchunk_ = blosc2_schunk_append_chunk(schunk, chunk2, true); |
648 | 0 | if (nchunk_ != nchunks + 1) { |
649 | 0 | BLOSC_TRACE_ERROR("Error appending last special chunk."); |
650 | 0 | return BLOSC2_ERROR_SCHUNK_SPECIAL; |
651 | 0 | } |
652 | 0 | } |
653 | 0 | free(chunk); |
654 | 0 | free(chunk2); |
655 | 0 | } |
656 | 0 | else { |
657 | | /* Fill an empty frame with special values (fast path). */ |
658 | 0 | blosc2_frame_s *frame = (blosc2_frame_s *) schunk->frame; |
659 | | /* Update counters (necessary for the frame_fill_special() logic) */ |
660 | 0 | if (leftover_items) { |
661 | 0 | nchunks += 1; |
662 | 0 | } |
663 | 0 | schunk->chunksize = chunksize; |
664 | 0 | schunk->nchunks = nchunks; |
665 | 0 | schunk->nbytes = nitems * typesize; |
666 | 0 | int64_t frame_len = frame_fill_special(frame, nitems, special_value, chunksize, schunk); |
667 | 0 | if (frame_len < 0) { |
668 | 0 | BLOSC_TRACE_ERROR("Error creating special frame."); |
669 | 0 | return frame_len; |
670 | 0 | } |
671 | 0 | } |
672 | | |
673 | 0 | return schunk->nchunks; |
674 | 0 | } |
675 | | |
676 | | /* Append an existing chunk into a super-chunk. */ |
677 | 48.1k | int64_t blosc2_schunk_append_chunk(blosc2_schunk *schunk, uint8_t *chunk, bool copy) { |
678 | 48.1k | int32_t chunk_nbytes; |
679 | 48.1k | int32_t chunk_cbytes; |
680 | 48.1k | int64_t nchunks = schunk->nchunks; |
681 | | |
682 | 48.1k | int rc = blosc2_cbuffer_sizes(chunk, &chunk_nbytes, &chunk_cbytes, NULL); |
683 | 48.1k | if (rc < 0) { |
684 | 0 | return rc; |
685 | 0 | } |
686 | | |
687 | 48.1k | if (schunk->chunksize == -1) { |
688 | 2.40k | schunk->chunksize = chunk_nbytes; // The super-chunk is initialized now |
689 | 2.40k | } |
690 | 48.1k | if (chunk_nbytes > schunk->chunksize) { |
691 | 0 | BLOSC_TRACE_ERROR("Appending chunks that have different lengths in the same schunk " |
692 | 0 | "is not supported yet: %d > %d.", chunk_nbytes, schunk->chunksize); |
693 | 0 | return BLOSC2_ERROR_CHUNK_APPEND; |
694 | 0 | } |
695 | | |
696 | | /* Update counters */ |
697 | 48.1k | schunk->current_nchunk = nchunks; |
698 | 48.1k | schunk->nchunks = nchunks + 1; |
699 | 48.1k | schunk->nbytes += chunk_nbytes; |
700 | 48.1k | if (schunk->frame == NULL) { |
701 | 48.1k | schunk->cbytes += chunk_cbytes; |
702 | 48.1k | } else { |
703 | | // A frame |
704 | 0 | int special_value = (chunk[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK; |
705 | 0 | switch (special_value) { |
706 | 0 | case BLOSC2_SPECIAL_ZERO: |
707 | 0 | case BLOSC2_SPECIAL_NAN: |
708 | 0 | case BLOSC2_SPECIAL_UNINIT: |
709 | 0 | schunk->cbytes += 0; |
710 | 0 | break; |
711 | 0 | default: |
712 | 0 | schunk->cbytes += chunk_cbytes; |
713 | 0 | } |
714 | 0 | } |
715 | | |
716 | 48.1k | if (copy) { |
717 | | // Make a copy of the chunk |
718 | 0 | uint8_t *chunk_copy = malloc(chunk_cbytes); |
719 | 0 | memcpy(chunk_copy, chunk, chunk_cbytes); |
720 | 0 | chunk = chunk_copy; |
721 | 0 | } |
722 | | |
723 | | // Update super-chunk or frame |
724 | 48.1k | blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; |
725 | 48.1k | if (frame == NULL) { |
726 | | // Check that we are not appending a small chunk after another small chunk |
727 | 48.1k | if ((schunk->nchunks > 1) && (chunk_nbytes < schunk->chunksize)) { |
728 | 391 | uint8_t* last_chunk = schunk->data[nchunks - 1]; |
729 | 391 | int32_t last_nbytes; |
730 | 391 | rc = blosc2_cbuffer_sizes(last_chunk, &last_nbytes, NULL, NULL); |
731 | 391 | if (rc < 0) { |
732 | 0 | return rc; |
733 | 0 | } |
734 | 391 | if ((last_nbytes < schunk->chunksize) && (chunk_nbytes < schunk->chunksize)) { |
735 | 0 | BLOSC_TRACE_ERROR( |
736 | 0 | "Appending two consecutive chunks with a chunksize smaller than the schunk chunksize " |
737 | 0 | "is not allowed yet: %d != %d.", chunk_nbytes, schunk->chunksize); |
738 | 0 | return BLOSC2_ERROR_CHUNK_APPEND; |
739 | 0 | } |
740 | 391 | } |
741 | | |
742 | 48.1k | if (!copy && (chunk_cbytes < chunk_nbytes)) { |
743 | | // We still want to do a shrink of the chunk |
744 | 32.5k | chunk = realloc(chunk, chunk_cbytes); |
745 | 32.5k | } |
746 | | |
747 | | /* Make space for appending the copy of the chunk and do it */ |
748 | 48.1k | if ((nchunks + 1) * sizeof(void *) > schunk->data_len) { |
749 | | // Extend the data pointer by one memory page (4k) |
750 | 2.44k | schunk->data_len += 4096; // must be a multiple of sizeof(void*) |
751 | 2.44k | schunk->data = realloc(schunk->data, schunk->data_len); |
752 | 2.44k | } |
753 | 48.1k | schunk->data[nchunks] = chunk; |
754 | 48.1k | } |
755 | 0 | else { |
756 | 0 | if (frame_append_chunk(frame, chunk, schunk) == NULL) { |
757 | 0 | BLOSC_TRACE_ERROR("Problems appending a chunk."); |
758 | 0 | return BLOSC2_ERROR_CHUNK_APPEND; |
759 | 0 | } |
760 | 0 | } |
761 | 48.1k | return schunk->nchunks; |
762 | 48.1k | } |
763 | | |
764 | | |
765 | | /* Insert an existing @p chunk in a specified position on a super-chunk */ |
766 | 0 | int64_t blosc2_schunk_insert_chunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t *chunk, bool copy) { |
767 | 0 | int32_t chunk_nbytes; |
768 | 0 | int32_t chunk_cbytes; |
769 | 0 | int64_t nchunks = schunk->nchunks; |
770 | |
|
771 | 0 | int rc = blosc2_cbuffer_sizes(chunk, &chunk_nbytes, &chunk_cbytes, NULL); |
772 | 0 | if (rc < 0) { |
773 | 0 | return rc; |
774 | 0 | } |
775 | | |
776 | 0 | if (schunk->chunksize == -1) { |
777 | 0 | schunk->chunksize = chunk_nbytes; // The super-chunk is initialized now |
778 | 0 | } |
779 | |
|
780 | 0 | if (chunk_nbytes > schunk->chunksize) { |
781 | 0 | BLOSC_TRACE_ERROR("Inserting chunks that have different lengths in the same schunk " |
782 | 0 | "is not supported yet: %d > %d.", chunk_nbytes, schunk->chunksize); |
783 | 0 | return BLOSC2_ERROR_CHUNK_INSERT; |
784 | 0 | } |
785 | | |
786 | | /* Update counters */ |
787 | 0 | schunk->current_nchunk = nchunk; |
788 | 0 | schunk->nchunks = nchunks + 1; |
789 | 0 | schunk->nbytes += chunk_nbytes; |
790 | 0 | if (schunk->frame == NULL) { |
791 | 0 | schunk->cbytes += chunk_cbytes; |
792 | 0 | } else { |
793 | | // A frame |
794 | 0 | int special_value = (chunk[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK; |
795 | 0 | switch (special_value) { |
796 | 0 | case BLOSC2_SPECIAL_ZERO: |
797 | 0 | case BLOSC2_SPECIAL_NAN: |
798 | 0 | case BLOSC2_SPECIAL_UNINIT: |
799 | 0 | schunk->cbytes += 0; |
800 | 0 | break; |
801 | 0 | default: |
802 | 0 | schunk->cbytes += chunk_cbytes; |
803 | 0 | } |
804 | 0 | } |
805 | | |
806 | 0 | if (copy) { |
807 | | // Make a copy of the chunk |
808 | 0 | uint8_t *chunk_copy = malloc(chunk_cbytes); |
809 | 0 | memcpy(chunk_copy, chunk, chunk_cbytes); |
810 | 0 | chunk = chunk_copy; |
811 | 0 | } |
812 | | |
813 | | // Update super-chunk or frame |
814 | 0 | blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; |
815 | 0 | if (frame == NULL) { |
816 | | // Check that we are not appending a small chunk after another small chunk |
817 | 0 | if ((schunk->nchunks > 0) && (chunk_nbytes < schunk->chunksize)) { |
818 | 0 | uint8_t* last_chunk = schunk->data[nchunks - 1]; |
819 | 0 | int32_t last_nbytes; |
820 | 0 | rc = blosc2_cbuffer_sizes(last_chunk, &last_nbytes, NULL, NULL); |
821 | 0 | if (rc < 0) { |
822 | 0 | return rc; |
823 | 0 | } |
824 | 0 | if ((last_nbytes < schunk->chunksize) && (chunk_nbytes < schunk->chunksize)) { |
825 | 0 | BLOSC_TRACE_ERROR("Appending two consecutive chunks with a chunksize smaller " |
826 | 0 | "than the schunk chunksize is not allowed yet: %d != %d", |
827 | 0 | chunk_nbytes, schunk->chunksize); |
828 | 0 | return BLOSC2_ERROR_CHUNK_APPEND; |
829 | 0 | } |
830 | 0 | } |
831 | | |
832 | 0 | if (!copy && (chunk_cbytes < chunk_nbytes)) { |
833 | | // We still want to do a shrink of the chunk |
834 | 0 | chunk = realloc(chunk, chunk_cbytes); |
835 | 0 | } |
836 | | |
837 | | // Make space for appending the copy of the chunk and do it |
838 | 0 | if ((nchunks + 1) * sizeof(void *) > schunk->data_len) { |
839 | | // Extend the data pointer by one memory page (4k) |
840 | 0 | schunk->data_len += 4096; // must be a multiple of sizeof(void*) |
841 | 0 | schunk->data = realloc(schunk->data, schunk->data_len); |
842 | 0 | } |
843 | | |
844 | | // Reorder the offsets and insert the new chunk |
845 | 0 | for (int64_t i = nchunks; i > nchunk; --i) { |
846 | 0 | schunk->data[i] = schunk->data[i-1]; |
847 | 0 | } |
848 | 0 | schunk->data[nchunk] = chunk; |
849 | 0 | } |
850 | | |
851 | 0 | else { |
852 | 0 | if (frame_insert_chunk(frame, nchunk, chunk, schunk) == NULL) { |
853 | 0 | BLOSC_TRACE_ERROR("Problems inserting a chunk in a frame."); |
854 | 0 | return BLOSC2_ERROR_CHUNK_INSERT; |
855 | 0 | } |
856 | 0 | } |
857 | 0 | return schunk->nchunks; |
858 | 0 | } |
859 | | |
860 | | |
861 | 0 | int64_t blosc2_schunk_update_chunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t *chunk, bool copy) { |
862 | 0 | int32_t chunk_nbytes; |
863 | 0 | int32_t chunk_cbytes; |
864 | |
|
865 | 0 | int rc = blosc2_cbuffer_sizes(chunk, &chunk_nbytes, &chunk_cbytes, NULL); |
866 | 0 | if (rc < 0) { |
867 | 0 | return rc; |
868 | 0 | } |
869 | | |
870 | 0 | if (schunk->chunksize == -1) { |
871 | 0 | schunk->chunksize = chunk_nbytes; // The super-chunk is initialized now |
872 | 0 | } |
873 | |
|
874 | 0 | if (schunk->chunksize != 0 && (chunk_nbytes > schunk->chunksize || |
875 | 0 | (chunk_nbytes < schunk->chunksize && nchunk != schunk->nchunks - 1))) { |
876 | 0 | BLOSC_TRACE_ERROR("Updating chunks that have different lengths in the same schunk " |
877 | 0 | "is not supported yet (unless it's the last one and smaller):" |
878 | 0 | " %d > %d.", chunk_nbytes, schunk->chunksize); |
879 | 0 | return BLOSC2_ERROR_CHUNK_UPDATE; |
880 | 0 | } |
881 | | |
882 | 0 | bool needs_free; |
883 | 0 | uint8_t *chunk_old; |
884 | 0 | int err = blosc2_schunk_get_chunk(schunk, nchunk, &chunk_old, &needs_free); |
885 | 0 | if (err < 0) { |
886 | 0 | BLOSC_TRACE_ERROR("%" PRId64 " chunk can not be obtained from schunk.", nchunk); |
887 | 0 | return -1; |
888 | 0 | } |
889 | 0 | int32_t chunk_nbytes_old = 0; |
890 | 0 | int32_t chunk_cbytes_old = 0; |
891 | 0 | schunk->current_nchunk = nchunk; |
892 | |
|
893 | 0 | if (chunk_old != 0) { |
894 | 0 | rc = blosc2_cbuffer_sizes(chunk_old, &chunk_nbytes_old, &chunk_cbytes_old, NULL); |
895 | 0 | if (rc < 0) { |
896 | 0 | return rc; |
897 | 0 | } |
898 | 0 | if (chunk_cbytes_old == BLOSC2_MAX_OVERHEAD) { |
899 | 0 | chunk_cbytes_old = 0; |
900 | 0 | } |
901 | 0 | } |
902 | 0 | if (needs_free) { |
903 | 0 | free(chunk_old); |
904 | 0 | } |
905 | |
|
906 | 0 | if (copy) { |
907 | | // Make a copy of the chunk |
908 | 0 | uint8_t *chunk_copy = malloc(chunk_cbytes); |
909 | 0 | memcpy(chunk_copy, chunk, chunk_cbytes); |
910 | 0 | chunk = chunk_copy; |
911 | 0 | } |
912 | |
|
913 | 0 | blosc2_frame_s* frame = (blosc2_frame_s*)(schunk->frame); |
914 | 0 | if (schunk->frame == NULL) { |
915 | | /* Update counters */ |
916 | 0 | schunk->nbytes += chunk_nbytes; |
917 | 0 | schunk->nbytes -= chunk_nbytes_old; |
918 | 0 | schunk->cbytes += chunk_cbytes; |
919 | 0 | schunk->cbytes -= chunk_cbytes_old; |
920 | 0 | } else { |
921 | | // A frame |
922 | 0 | int special_value = (chunk[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK; |
923 | 0 | switch (special_value) { |
924 | 0 | case BLOSC2_SPECIAL_ZERO: |
925 | 0 | case BLOSC2_SPECIAL_NAN: |
926 | 0 | case BLOSC2_SPECIAL_UNINIT: |
927 | 0 | schunk->nbytes += chunk_nbytes; |
928 | 0 | schunk->nbytes -= chunk_nbytes_old; |
929 | 0 | if (frame->sframe) { |
930 | 0 | schunk->cbytes -= chunk_cbytes_old; |
931 | 0 | } |
932 | 0 | break; |
933 | 0 | default: |
934 | | /* Update counters */ |
935 | 0 | schunk->nbytes += chunk_nbytes; |
936 | 0 | schunk->nbytes -= chunk_nbytes_old; |
937 | 0 | schunk->cbytes += chunk_cbytes; |
938 | 0 | if (frame->sframe) { |
939 | 0 | schunk->cbytes -= chunk_cbytes_old; |
940 | 0 | } |
941 | 0 | else { |
942 | 0 | if (chunk_cbytes_old >= chunk_cbytes) { |
943 | 0 | schunk->cbytes -= chunk_cbytes; |
944 | 0 | } |
945 | 0 | } |
946 | 0 | } |
947 | 0 | } |
948 | | |
949 | | // Update super-chunk or frame |
950 | 0 | if (schunk->frame == NULL) { |
951 | 0 | if (!copy && (chunk_cbytes < chunk_nbytes)) { |
952 | | // We still want to do a shrink of the chunk |
953 | 0 | chunk = realloc(chunk, chunk_cbytes); |
954 | 0 | } |
955 | | |
956 | | // Free old chunk and add reference to new chunk |
957 | 0 | if (schunk->data[nchunk] != 0) { |
958 | 0 | free(schunk->data[nchunk]); |
959 | 0 | } |
960 | 0 | schunk->data[nchunk] = chunk; |
961 | 0 | } |
962 | 0 | else { |
963 | 0 | if (frame_update_chunk(frame, nchunk, chunk, schunk) == NULL) { |
964 | 0 | BLOSC_TRACE_ERROR("Problems updating a chunk in a frame."); |
965 | 0 | return BLOSC2_ERROR_CHUNK_UPDATE; |
966 | 0 | } |
967 | 0 | } |
968 | | |
969 | 0 | return schunk->nchunks; |
970 | 0 | } |
971 | | |
972 | 0 | int64_t blosc2_schunk_delete_chunk(blosc2_schunk *schunk, int64_t nchunk) { |
973 | 0 | int rc; |
974 | 0 | if (schunk->nchunks < nchunk) { |
975 | 0 | BLOSC_TRACE_ERROR("The schunk has not enough chunks (%" PRId64 ")!", schunk->nchunks); |
976 | 0 | } |
977 | | |
978 | 0 | bool needs_free; |
979 | 0 | uint8_t *chunk_old; |
980 | 0 | int err = blosc2_schunk_get_chunk(schunk, nchunk, &chunk_old, &needs_free); |
981 | 0 | if (err < 0) { |
982 | 0 | BLOSC_TRACE_ERROR("%" PRId64 "chunk can not be obtained from schunk.", nchunk); |
983 | 0 | return -1; |
984 | 0 | } |
985 | 0 | int32_t chunk_nbytes_old = 0; |
986 | 0 | int32_t chunk_cbytes_old = 0; |
987 | 0 | schunk->current_nchunk = nchunk; |
988 | |
|
989 | 0 | if (chunk_old != 0) { |
990 | 0 | rc = blosc2_cbuffer_sizes(chunk_old, &chunk_nbytes_old, &chunk_cbytes_old, NULL); |
991 | 0 | if (rc < 0) { |
992 | 0 | return rc; |
993 | 0 | } |
994 | 0 | if (chunk_cbytes_old == BLOSC2_MAX_OVERHEAD) { |
995 | 0 | chunk_cbytes_old = 0; |
996 | 0 | } |
997 | 0 | } |
998 | 0 | if (needs_free) { |
999 | 0 | free(chunk_old); |
1000 | 0 | } |
1001 | |
|
1002 | 0 | blosc2_frame_s* frame = (blosc2_frame_s*)(schunk->frame); |
1003 | 0 | schunk->nchunks -= 1; |
1004 | 0 | if (schunk->frame == NULL) { |
1005 | | /* Update counters */ |
1006 | 0 | schunk->nbytes -= chunk_nbytes_old; |
1007 | 0 | schunk->cbytes -= chunk_cbytes_old; |
1008 | 0 | } else { |
1009 | | // A frame |
1010 | 0 | schunk->nbytes -= chunk_nbytes_old; |
1011 | 0 | if (frame->sframe) { |
1012 | 0 | schunk->cbytes -= chunk_cbytes_old; |
1013 | 0 | } |
1014 | 0 | } |
1015 | | |
1016 | | // Update super-chunk or frame |
1017 | 0 | if (schunk->frame == NULL) { |
1018 | | // Free old chunk |
1019 | 0 | if (schunk->data[nchunk] != 0) { |
1020 | 0 | free(schunk->data[nchunk]); |
1021 | 0 | } |
1022 | | // Reorder the offsets and insert the new chunk |
1023 | 0 | for (int64_t i = nchunk; i < schunk->nchunks; i++) { |
1024 | 0 | schunk->data[i] = schunk->data[i + 1]; |
1025 | 0 | } |
1026 | 0 | schunk->data[schunk->nchunks] = NULL; |
1027 | |
|
1028 | 0 | } |
1029 | 0 | else { |
1030 | 0 | if (frame_delete_chunk(frame, nchunk, schunk) == NULL) { |
1031 | 0 | BLOSC_TRACE_ERROR("Problems deleting a chunk in a frame."); |
1032 | 0 | return BLOSC2_ERROR_CHUNK_UPDATE; |
1033 | 0 | } |
1034 | 0 | } |
1035 | 0 | return schunk->nchunks; |
1036 | 0 | } |
1037 | | |
1038 | | |
1039 | | /* Append a data buffer to a super-chunk. */ |
1040 | 48.1k | int64_t blosc2_schunk_append_buffer(blosc2_schunk *schunk, void *src, int32_t nbytes) { |
1041 | 48.1k | uint8_t* chunk = malloc(nbytes + BLOSC2_MAX_OVERHEAD); |
1042 | 48.1k | schunk->current_nchunk = schunk->nchunks; |
1043 | | /* Compress the src buffer using super-chunk context */ |
1044 | 48.1k | int cbytes = blosc2_compress_ctx(schunk->cctx, src, nbytes, chunk, |
1045 | 48.1k | nbytes + BLOSC2_MAX_OVERHEAD); |
1046 | 48.1k | if (cbytes < 0) { |
1047 | 0 | free(chunk); |
1048 | 0 | return cbytes; |
1049 | 0 | } |
1050 | | // We don't need a copy of the chunk, as it will be shrunk if necessary |
1051 | 48.1k | int64_t nchunks = blosc2_schunk_append_chunk(schunk, chunk, false); |
1052 | 48.1k | if (nchunks < 0) { |
1053 | 0 | BLOSC_TRACE_ERROR("Error appending a buffer in super-chunk"); |
1054 | 0 | return nchunks; |
1055 | 0 | } |
1056 | | |
1057 | 48.1k | return nchunks; |
1058 | 48.1k | } |
1059 | | |
1060 | | |
1061 | | /* Decompress and return a chunk that is part of a super-chunk. */ |
1062 | | int blosc2_schunk_decompress_chunk(blosc2_schunk *schunk, int64_t nchunk, |
1063 | 15.9k | void *dest, int32_t nbytes) { |
1064 | 15.9k | int32_t chunk_nbytes; |
1065 | 15.9k | int32_t chunk_cbytes; |
1066 | 15.9k | int chunksize; |
1067 | 15.9k | int rc; |
1068 | 15.9k | blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; |
1069 | | |
1070 | 15.9k | schunk->current_nchunk = nchunk; |
1071 | 15.9k | if (frame == NULL) { |
1072 | 15.9k | if (nchunk >= schunk->nchunks) { |
1073 | 0 | BLOSC_TRACE_ERROR("nchunk ('%" PRId64 "') exceeds the number of chunks " |
1074 | 0 | "('%" PRId64 "') in super-chunk.", nchunk, schunk->nchunks); |
1075 | 0 | return BLOSC2_ERROR_INVALID_PARAM; |
1076 | 0 | } |
1077 | 15.9k | uint8_t* src = schunk->data[nchunk]; |
1078 | 15.9k | if (src == 0) { |
1079 | 0 | return 0; |
1080 | 0 | } |
1081 | | |
1082 | 15.9k | rc = blosc2_cbuffer_sizes(src, &chunk_nbytes, &chunk_cbytes, NULL); |
1083 | 15.9k | if (rc < 0) { |
1084 | 0 | return rc; |
1085 | 0 | } |
1086 | | |
1087 | 15.9k | if (nbytes < chunk_nbytes) { |
1088 | 391 | BLOSC_TRACE_ERROR("Buffer size is too small for the decompressed buffer " |
1089 | 0 | "('%d' bytes, but '%d' are needed).", nbytes, chunk_nbytes); |
1090 | 0 | return BLOSC2_ERROR_INVALID_PARAM; |
1091 | 391 | } |
1092 | | |
1093 | 15.5k | chunksize = blosc2_decompress_ctx(schunk->dctx, src, chunk_cbytes, dest, nbytes); |
1094 | 15.5k | if (chunksize < 0 || chunksize != chunk_nbytes) { |
1095 | 0 | BLOSC_TRACE_ERROR("Error in decompressing chunk."); |
1096 | 0 | if (chunksize < 0) |
1097 | 0 | return chunksize; |
1098 | 0 | return BLOSC2_ERROR_FAILURE; |
1099 | 0 | } |
1100 | 15.5k | } else { |
1101 | 21 | chunksize = frame_decompress_chunk(schunk->dctx, frame, nchunk, dest, nbytes); |
1102 | 21 | if (chunksize < 0) { |
1103 | 4 | return chunksize; |
1104 | 4 | } |
1105 | 21 | } |
1106 | 15.5k | return chunksize; |
1107 | 15.9k | } |
1108 | | |
1109 | | |
1110 | | /* Return a compressed chunk that is part of a super-chunk in the `chunk` parameter. |
1111 | | * If the super-chunk is backed by a frame that is disk-based, a buffer is allocated for the |
1112 | | * (compressed) chunk, and hence a free is needed. You can check if the chunk requires a free |
1113 | | * with the `needs_free` parameter. |
1114 | | * If the chunk does not need a free, it means that a pointer to the location in the super-chunk |
1115 | | * (or the backing in-memory frame) is returned in the `chunk` parameter. |
1116 | | * |
1117 | | * The size of the (compressed) chunk is returned. If some problem is detected, a negative code |
1118 | | * is returned instead. |
1119 | | */ |
1120 | 0 | int blosc2_schunk_get_chunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t **chunk, bool *needs_free) { |
1121 | 0 | if (schunk->dctx->threads_started > 1) { |
1122 | 0 | pthread_mutex_lock(&schunk->dctx->nchunk_mutex); |
1123 | 0 | schunk->current_nchunk = nchunk; |
1124 | 0 | pthread_mutex_unlock(&schunk->dctx->nchunk_mutex); |
1125 | 0 | } |
1126 | 0 | else { |
1127 | 0 | schunk->current_nchunk = nchunk; |
1128 | 0 | } |
1129 | 0 | blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; |
1130 | 0 | if (frame != NULL) { |
1131 | 0 | return frame_get_chunk(frame, nchunk, chunk, needs_free); |
1132 | 0 | } |
1133 | | |
1134 | 0 | if (nchunk >= schunk->nchunks) { |
1135 | 0 | BLOSC_TRACE_ERROR("nchunk ('%" PRId64 "') exceeds the number of chunks " |
1136 | 0 | "('%" PRId64 "') in schunk.", nchunk, schunk->nchunks); |
1137 | 0 | return BLOSC2_ERROR_INVALID_PARAM; |
1138 | 0 | } |
1139 | | |
1140 | 0 | *chunk = schunk->data[nchunk]; |
1141 | 0 | if (*chunk == 0) { |
1142 | 0 | *needs_free = 0; |
1143 | 0 | return 0; |
1144 | 0 | } |
1145 | | |
1146 | 0 | *needs_free = false; |
1147 | 0 | int32_t chunk_cbytes; |
1148 | 0 | int rc = blosc2_cbuffer_sizes(*chunk, NULL, &chunk_cbytes, NULL); |
1149 | 0 | if (rc < 0) { |
1150 | 0 | return rc; |
1151 | 0 | } |
1152 | 0 | return (int)chunk_cbytes; |
1153 | 0 | } |
1154 | | |
1155 | | |
1156 | | /* Return a compressed chunk that is part of a super-chunk in the `chunk` parameter. |
1157 | | * If the super-chunk is backed by a frame that is disk-based, a buffer is allocated for the |
1158 | | * (compressed) chunk, and hence a free is needed. You can check if the chunk requires a free |
1159 | | * with the `needs_free` parameter. |
1160 | | * If the chunk does not need a free, it means that a pointer to the location in the super-chunk |
1161 | | * (or the backing in-memory frame) is returned in the `chunk` parameter. |
1162 | | * |
1163 | | * The size of the (compressed) chunk is returned. If some problem is detected, a negative code |
1164 | | * is returned instead. |
1165 | | */ |
1166 | 0 | int blosc2_schunk_get_lazychunk(blosc2_schunk *schunk, int64_t nchunk, uint8_t **chunk, bool *needs_free) { |
1167 | 0 | if (schunk->dctx->threads_started > 1) { |
1168 | 0 | pthread_mutex_lock(&schunk->dctx->nchunk_mutex); |
1169 | 0 | schunk->current_nchunk = nchunk; |
1170 | 0 | pthread_mutex_unlock(&schunk->dctx->nchunk_mutex); |
1171 | 0 | } |
1172 | 0 | else { |
1173 | 0 | schunk->current_nchunk = nchunk; |
1174 | 0 | } |
1175 | 0 | blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; |
1176 | 0 | if (schunk->frame != NULL) { |
1177 | 0 | return frame_get_lazychunk(frame, nchunk, chunk, needs_free); |
1178 | 0 | } |
1179 | | |
1180 | 0 | if (nchunk >= schunk->nchunks) { |
1181 | 0 | BLOSC_TRACE_ERROR("nchunk ('%" PRId64 "') exceeds the number of chunks " |
1182 | 0 | "('%" PRId64 "') in schunk.", nchunk, schunk->nchunks); |
1183 | 0 | return BLOSC2_ERROR_INVALID_PARAM; |
1184 | 0 | } |
1185 | | |
1186 | 0 | *chunk = schunk->data[nchunk]; |
1187 | 0 | if (*chunk == 0) { |
1188 | 0 | *needs_free = 0; |
1189 | 0 | return 0; |
1190 | 0 | } |
1191 | | |
1192 | 0 | *needs_free = false; |
1193 | 0 | int32_t chunk_cbytes; |
1194 | 0 | int rc = blosc2_cbuffer_sizes(*chunk, NULL, &chunk_cbytes, NULL); |
1195 | 0 | if (rc < 0) { |
1196 | 0 | return rc; |
1197 | 0 | } |
1198 | 0 | return (int)chunk_cbytes; |
1199 | 0 | } |
1200 | | |
1201 | | |
1202 | 0 | int blosc2_schunk_get_slice_buffer(blosc2_schunk *schunk, int64_t start, int64_t stop, void *buffer) { |
1203 | 0 | int64_t byte_start = start * schunk->typesize; |
1204 | 0 | int64_t byte_stop = stop * schunk->typesize; |
1205 | 0 | int64_t nchunk_start = byte_start / schunk->chunksize; |
1206 | 0 | int32_t chunk_start = (int32_t) (byte_start % schunk->chunksize); // 0 indexed |
1207 | 0 | int32_t chunk_stop; // 0 indexed |
1208 | 0 | if (byte_stop >= (nchunk_start + 1) * schunk->chunksize) { |
1209 | 0 | chunk_stop = schunk->chunksize; |
1210 | 0 | } |
1211 | 0 | else { |
1212 | 0 | chunk_stop = (int32_t) (byte_stop % schunk->chunksize); |
1213 | 0 | } |
1214 | |
|
1215 | 0 | uint8_t *dst_ptr = (uint8_t *) buffer; |
1216 | 0 | bool needs_free; |
1217 | 0 | uint8_t *chunk; |
1218 | 0 | int32_t cbytes; |
1219 | 0 | int64_t nchunk = nchunk_start; |
1220 | 0 | int64_t nbytes_read = 0; |
1221 | 0 | int32_t nbytes; |
1222 | 0 | int32_t chunksize = schunk->chunksize; |
1223 | |
|
1224 | 0 | while (nbytes_read < ((stop - start) * schunk->typesize)) { |
1225 | 0 | cbytes = blosc2_schunk_get_lazychunk(schunk, nchunk, &chunk, &needs_free); |
1226 | 0 | if (cbytes < 0) { |
1227 | 0 | BLOSC_TRACE_ERROR("Cannot get lazychunk ('%" PRId64 "').", nchunk); |
1228 | 0 | return BLOSC2_ERROR_FAILURE; |
1229 | 0 | } |
1230 | 0 | int32_t blocksize = sw32_(chunk + BLOSC2_CHUNK_BLOCKSIZE); |
1231 | |
|
1232 | 0 | int32_t nblock_start = (int32_t) (chunk_start / blocksize); |
1233 | 0 | int32_t nblock_stop = (int32_t) ((chunk_stop - 1) / blocksize); |
1234 | 0 | if (nchunk == (schunk->nchunks - 1) && schunk->nbytes % schunk->chunksize != 0) { |
1235 | 0 | chunksize = schunk->nbytes % schunk->chunksize; |
1236 | 0 | } |
1237 | 0 | int32_t nblocks = chunksize / blocksize; |
1238 | 0 | if (chunksize % blocksize != 0) { |
1239 | 0 | nblocks++; |
1240 | 0 | } |
1241 | |
|
1242 | 0 | if (chunk_start == 0 && chunk_stop == chunksize) { |
1243 | | // Avoid memcpy |
1244 | 0 | nbytes = blosc2_decompress_ctx(schunk->dctx, chunk, cbytes, dst_ptr, chunksize); |
1245 | 0 | if (nbytes < 0) { |
1246 | 0 | BLOSC_TRACE_ERROR("Cannot decompress chunk ('%" PRId64 "').", nchunk); |
1247 | 0 | return BLOSC2_ERROR_FAILURE; |
1248 | 0 | } |
1249 | 0 | } |
1250 | 0 | else { |
1251 | | // After extensive timing I have not been able to see lots of situations where |
1252 | | // a maskout read is better than a getitem one. Disabling for now. |
1253 | | // if (nblock_start != nblock_stop) { |
1254 | 0 | if (false) { |
1255 | 0 | uint8_t *data = malloc(chunksize); |
1256 | | /* We have more than 1 block to read, so use a masked read */ |
1257 | 0 | bool *block_maskout = calloc(nblocks, 1); |
1258 | 0 | for (int32_t nblock = 0; nblock < nblocks; nblock++) { |
1259 | 0 | if ((nblock < nblock_start) || (nblock > nblock_stop)) { |
1260 | 0 | block_maskout[nblock] = true; |
1261 | 0 | } |
1262 | 0 | } |
1263 | 0 | if (blosc2_set_maskout(schunk->dctx, block_maskout, nblocks) < 0) { |
1264 | 0 | BLOSC_TRACE_ERROR("Cannot set maskout"); |
1265 | 0 | return BLOSC2_ERROR_FAILURE; |
1266 | 0 | } |
1267 | | |
1268 | 0 | nbytes = blosc2_decompress_ctx(schunk->dctx, chunk, cbytes, data, chunksize); |
1269 | 0 | if (nbytes < 0) { |
1270 | 0 | BLOSC_TRACE_ERROR("Cannot decompress chunk ('%" PRId64 "').", nchunk); |
1271 | 0 | return BLOSC2_ERROR_FAILURE; |
1272 | 0 | } |
1273 | 0 | nbytes = chunk_stop - chunk_start; |
1274 | 0 | memcpy(dst_ptr, &data[chunk_start], nbytes); |
1275 | 0 | free(block_maskout); |
1276 | 0 | free(data); |
1277 | 0 | } |
1278 | 0 | else { |
1279 | | /* Less than 1 block to read; use a getitem call */ |
1280 | 0 | nbytes = blosc2_getitem_ctx(schunk->dctx, chunk, cbytes, (int32_t) (chunk_start / schunk->typesize), |
1281 | 0 | (chunk_stop - chunk_start) / schunk->typesize, dst_ptr, chunksize); |
1282 | 0 | if (nbytes < 0) { |
1283 | 0 | BLOSC_TRACE_ERROR("Cannot get item from ('%" PRId64 "') chunk.", nchunk); |
1284 | 0 | return BLOSC2_ERROR_FAILURE; |
1285 | 0 | } |
1286 | 0 | } |
1287 | 0 | } |
1288 | | |
1289 | 0 | dst_ptr += nbytes; |
1290 | 0 | nbytes_read += nbytes; |
1291 | 0 | nchunk++; |
1292 | |
|
1293 | 0 | if (needs_free) { |
1294 | 0 | free(chunk); |
1295 | 0 | } |
1296 | 0 | chunk_start = 0; |
1297 | 0 | if (byte_stop >= (nchunk + 1) * chunksize) { |
1298 | 0 | chunk_stop = chunksize; |
1299 | 0 | } |
1300 | 0 | else { |
1301 | 0 | chunk_stop = (int32_t)(byte_stop % chunksize); |
1302 | 0 | } |
1303 | 0 | } |
1304 | | |
1305 | 0 | return BLOSC2_ERROR_SUCCESS; |
1306 | 0 | } |
1307 | | |
1308 | | |
1309 | 0 | int blosc2_schunk_set_slice_buffer(blosc2_schunk *schunk, int64_t start, int64_t stop, void *buffer) { |
1310 | 0 | int64_t byte_start = start * schunk->typesize; |
1311 | 0 | int64_t byte_stop = stop * schunk->typesize; |
1312 | 0 | int64_t nchunk_start = byte_start / schunk->chunksize; |
1313 | 0 | int32_t chunk_start = (int32_t) (byte_start % schunk->chunksize); |
1314 | 0 | int32_t chunk_stop; |
1315 | 0 | if (byte_stop >= (nchunk_start + 1) * schunk->chunksize) { |
1316 | 0 | chunk_stop = schunk->chunksize; |
1317 | 0 | } |
1318 | 0 | else { |
1319 | 0 | chunk_stop = (int32_t) (byte_stop % schunk->chunksize); |
1320 | 0 | } |
1321 | |
|
1322 | 0 | uint8_t *src_ptr = (uint8_t *) buffer; |
1323 | 0 | int64_t nchunk = nchunk_start; |
1324 | 0 | int64_t nbytes_written = 0; |
1325 | 0 | int32_t nbytes; |
1326 | 0 | uint8_t *data = malloc(schunk->chunksize); |
1327 | 0 | int64_t nchunks; |
1328 | 0 | int32_t chunksize = schunk->chunksize; |
1329 | |
|
1330 | 0 | while (nbytes_written < ((stop - start) * schunk->typesize)) { |
1331 | 0 | if (chunk_start == 0 && |
1332 | 0 | (chunk_stop == schunk->chunksize || chunk_stop == schunk->nbytes % schunk->chunksize)) { |
1333 | 0 | if (chunk_stop == schunk->nbytes % schunk->chunksize) { |
1334 | 0 | chunksize = chunk_stop; |
1335 | 0 | } |
1336 | 0 | uint8_t *chunk = malloc(chunksize + BLOSC2_MAX_OVERHEAD); |
1337 | 0 | if (blosc2_compress_ctx(schunk->cctx, src_ptr, chunksize, chunk, chunksize + BLOSC2_MAX_OVERHEAD) < 0) { |
1338 | 0 | BLOSC_TRACE_ERROR("Cannot compress data of chunk ('%" PRId64 "').", nchunk); |
1339 | 0 | return BLOSC2_ERROR_FAILURE; |
1340 | 0 | } |
1341 | 0 | nchunks = blosc2_schunk_update_chunk(schunk, nchunk, chunk, false); |
1342 | 0 | if (nchunks != schunk->nchunks) { |
1343 | 0 | BLOSC_TRACE_ERROR("Cannot update chunk ('%" PRId64 "').", nchunk); |
1344 | 0 | return BLOSC2_ERROR_CHUNK_UPDATE; |
1345 | 0 | } |
1346 | 0 | } |
1347 | 0 | else { |
1348 | 0 | nbytes = blosc2_schunk_decompress_chunk(schunk, nchunk, data, schunk->chunksize); |
1349 | 0 | if (nbytes < 0) { |
1350 | 0 | BLOSC_TRACE_ERROR("Cannot decompress chunk ('%" PRId64 "').", nchunk); |
1351 | 0 | return BLOSC2_ERROR_FAILURE; |
1352 | 0 | } |
1353 | 0 | memcpy(&data[chunk_start], src_ptr, chunk_stop - chunk_start); |
1354 | 0 | uint8_t *chunk = malloc(nbytes + BLOSC2_MAX_OVERHEAD); |
1355 | 0 | if (blosc2_compress_ctx(schunk->cctx, data, nbytes, chunk, nbytes + BLOSC2_MAX_OVERHEAD) < 0) { |
1356 | 0 | BLOSC_TRACE_ERROR("Cannot compress data of chunk ('%" PRId64 "').", nchunk); |
1357 | 0 | return BLOSC2_ERROR_FAILURE; |
1358 | 0 | } |
1359 | 0 | nchunks = blosc2_schunk_update_chunk(schunk, nchunk, chunk, false); |
1360 | 0 | if (nchunks != schunk->nchunks) { |
1361 | 0 | BLOSC_TRACE_ERROR("Cannot update chunk ('%" PRId64 "').", nchunk); |
1362 | 0 | return BLOSC2_ERROR_CHUNK_UPDATE; |
1363 | 0 | } |
1364 | 0 | } |
1365 | 0 | nchunk++; |
1366 | 0 | nbytes_written += chunk_stop - chunk_start; |
1367 | 0 | src_ptr += chunk_stop - chunk_start; |
1368 | 0 | chunk_start = 0; |
1369 | 0 | if (byte_stop >= (nchunk + 1) * schunk->chunksize) { |
1370 | 0 | chunk_stop = schunk->chunksize; |
1371 | 0 | } |
1372 | 0 | else { |
1373 | 0 | chunk_stop = (int32_t) (byte_stop % schunk->chunksize); |
1374 | 0 | } |
1375 | 0 | } |
1376 | 0 | free(data); |
1377 | |
|
1378 | 0 | return BLOSC2_ERROR_SUCCESS; |
1379 | 0 | } |
1380 | | |
1381 | | |
1382 | | /* Reorder the chunk offsets of an existing super-chunk. */ |
1383 | 0 | int blosc2_schunk_reorder_offsets(blosc2_schunk *schunk, int64_t *offsets_order) { |
1384 | | // Check that the offsets order are correct |
1385 | 0 | bool *index_check = (bool *) calloc(schunk->nchunks, sizeof(bool)); |
1386 | 0 | for (int i = 0; i < schunk->nchunks; ++i) { |
1387 | 0 | int64_t index = offsets_order[i]; |
1388 | 0 | if (index >= schunk->nchunks) { |
1389 | 0 | BLOSC_TRACE_ERROR("Index is bigger than the number of chunks."); |
1390 | 0 | free(index_check); |
1391 | 0 | return BLOSC2_ERROR_DATA; |
1392 | 0 | } |
1393 | 0 | if (index_check[index] == false) { |
1394 | 0 | index_check[index] = true; |
1395 | 0 | } else { |
1396 | 0 | BLOSC_TRACE_ERROR("Index is yet used."); |
1397 | 0 | free(index_check); |
1398 | 0 | return BLOSC2_ERROR_DATA; |
1399 | 0 | } |
1400 | 0 | } |
1401 | 0 | free(index_check); |
1402 | |
|
1403 | 0 | blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; |
1404 | 0 | if (frame != NULL) { |
1405 | 0 | return frame_reorder_offsets(frame, offsets_order, schunk); |
1406 | 0 | } |
1407 | 0 | uint8_t **offsets = schunk->data; |
1408 | | |
1409 | | // Make a copy of the chunk offsets and reorder it |
1410 | 0 | uint8_t **offsets_copy = malloc(schunk->data_len); |
1411 | 0 | memcpy(offsets_copy, offsets, schunk->data_len); |
1412 | |
|
1413 | 0 | for (int i = 0; i < schunk->nchunks; ++i) { |
1414 | 0 | offsets[i] = offsets_copy[offsets_order[i]]; |
1415 | 0 | } |
1416 | 0 | free(offsets_copy); |
1417 | |
|
1418 | 0 | return 0; |
1419 | 0 | } |
1420 | | |
1421 | | |
1422 | | // Get the length (in bytes) of the internal frame of the super-chunk |
1423 | 0 | int64_t blosc2_schunk_frame_len(blosc2_schunk* schunk) { |
1424 | 0 | int64_t len; |
1425 | 0 | blosc2_frame_s* frame_s = (blosc2_frame_s*)(schunk->frame); |
1426 | 0 | if (frame_s != NULL) { |
1427 | 0 | len = frame_s->len; |
1428 | 0 | } |
1429 | 0 | else { |
1430 | | // No attached frame, but we can still come with an estimate |
1431 | 0 | len = (int64_t) (schunk->cbytes + schunk->nchunks * sizeof(int64_t)); |
1432 | 0 | } |
1433 | |
|
1434 | 0 | return len; |
1435 | 0 | } |
1436 | | |
1437 | | |
1438 | | /** |
1439 | | * @brief Flush metalayers content into a possible attached frame. |
1440 | | * |
1441 | | * @param schunk The super-chunk to which the flush should be applied. |
1442 | | * |
1443 | | * @return If successful, a 0 is returned. Else, return a negative value. |
1444 | | */ |
1445 | | // Initially, this was a public function, but as it is really meant to be used only |
1446 | | // in the schunk_add_metalayer(), I decided to convert it into private and call it |
1447 | | // implicitly instead of requiring the user to do so. The only drawback is that |
1448 | | // each add operation requires a complete frame re-build, but as users should need |
1449 | | // very few metalayers, this overhead should be negligible in practice. |
1450 | 0 | int metalayer_flush(blosc2_schunk* schunk) { |
1451 | 0 | int rc = BLOSC2_ERROR_SUCCESS; |
1452 | 0 | blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; |
1453 | 0 | if (frame == NULL) { |
1454 | 0 | return rc; |
1455 | 0 | } |
1456 | 0 | rc = frame_update_header(frame, schunk, true); |
1457 | 0 | if (rc < 0) { |
1458 | 0 | BLOSC_TRACE_ERROR("Unable to update metalayers into frame."); |
1459 | 0 | return rc; |
1460 | 0 | } |
1461 | 0 | rc = frame_update_trailer(frame, schunk); |
1462 | 0 | if (rc < 0) { |
1463 | 0 | BLOSC_TRACE_ERROR("Unable to update trailer into frame."); |
1464 | 0 | return rc; |
1465 | 0 | } |
1466 | 0 | return rc; |
1467 | 0 | } |
1468 | | |
1469 | | |
1470 | | /* Add content into a new metalayer. |
1471 | | * |
1472 | | * If successful, return the index of the new metalayer. Else, return a negative value. |
1473 | | */ |
1474 | 0 | int blosc2_meta_add(blosc2_schunk *schunk, const char *name, uint8_t *content, int32_t content_len) { |
1475 | 0 | int nmetalayer = blosc2_meta_exists(schunk, name); |
1476 | 0 | if (nmetalayer >= 0) { |
1477 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" already exists.", name); |
1478 | 0 | return BLOSC2_ERROR_INVALID_PARAM; |
1479 | 0 | } |
1480 | | |
1481 | | // Add the metalayer |
1482 | 0 | blosc2_metalayer *metalayer = malloc(sizeof(blosc2_metalayer)); |
1483 | 0 | char* name_ = malloc(strlen(name) + 1); |
1484 | 0 | strcpy(name_, name); |
1485 | 0 | metalayer->name = name_; |
1486 | 0 | uint8_t* content_buf = malloc((size_t)content_len); |
1487 | 0 | memcpy(content_buf, content, content_len); |
1488 | 0 | metalayer->content = content_buf; |
1489 | 0 | metalayer->content_len = content_len; |
1490 | 0 | schunk->metalayers[schunk->nmetalayers] = metalayer; |
1491 | 0 | schunk->nmetalayers += 1; |
1492 | |
|
1493 | 0 | int rc = metalayer_flush(schunk); |
1494 | 0 | if (rc < 0) { |
1495 | 0 | return rc; |
1496 | 0 | } |
1497 | | |
1498 | 0 | return schunk->nmetalayers - 1; |
1499 | 0 | } |
1500 | | |
1501 | | |
1502 | | /* Update the content of an existing metalayer. |
1503 | | * |
1504 | | * If successful, return the index of the new metalayer. Else, return a negative value. |
1505 | | */ |
1506 | 0 | int blosc2_meta_update(blosc2_schunk *schunk, const char *name, uint8_t *content, int32_t content_len) { |
1507 | 0 | int nmetalayer = blosc2_meta_exists(schunk, name); |
1508 | 0 | if (nmetalayer < 0) { |
1509 | 0 | BLOSC_TRACE_ERROR("Metalayer \"%s\" not found.", name); |
1510 | 0 | return nmetalayer; |
1511 | 0 | } |
1512 | | |
1513 | 0 | blosc2_metalayer *metalayer = schunk->metalayers[nmetalayer]; |
1514 | 0 | if (content_len > metalayer->content_len) { |
1515 | 0 | BLOSC_TRACE_ERROR("`content_len` cannot exceed the existing size of %d bytes.", metalayer->content_len); |
1516 | 0 | return nmetalayer; |
1517 | 0 | } |
1518 | | |
1519 | | // Update the contents of the metalayer |
1520 | 0 | memcpy(metalayer->content, content, content_len); |
1521 | | |
1522 | | // Update the metalayers in frame (as size has not changed, we don't need to update the trailer) |
1523 | 0 | blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; |
1524 | 0 | if (frame != NULL) { |
1525 | 0 | int rc = frame_update_header(frame, schunk, false); |
1526 | 0 | if (rc < 0) { |
1527 | 0 | BLOSC_TRACE_ERROR("Unable to update meta info from frame."); |
1528 | 0 | return rc; |
1529 | 0 | } |
1530 | 0 | } |
1531 | | |
1532 | 0 | return nmetalayer; |
1533 | 0 | } |
1534 | | |
1535 | | |
1536 | | /* Find whether the schunk has a variable-length metalayer or not. |
1537 | | * |
1538 | | * If successful, return the index of the variable-length metalayer. Else, return a negative value. |
1539 | | */ |
1540 | 0 | int blosc2_vlmeta_exists(blosc2_schunk *schunk, const char *name) { |
1541 | 0 | if (strlen(name) > BLOSC2_METALAYER_NAME_MAXLEN) { |
1542 | 0 | BLOSC_TRACE_ERROR("Variable-length metalayer names cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN); |
1543 | 0 | return BLOSC2_ERROR_INVALID_PARAM; |
1544 | 0 | } |
1545 | | |
1546 | 0 | for (int nvlmetalayer = 0; nvlmetalayer < schunk->nvlmetalayers; nvlmetalayer++) { |
1547 | 0 | if (strcmp(name, schunk->vlmetalayers[nvlmetalayer]->name) == 0) { |
1548 | 0 | return nvlmetalayer; |
1549 | 0 | } |
1550 | 0 | } |
1551 | 0 | return BLOSC2_ERROR_NOT_FOUND; |
1552 | 0 | } |
1553 | | |
1554 | 0 | int vlmetalayer_flush(blosc2_schunk* schunk) { |
1555 | 0 | int rc = BLOSC2_ERROR_SUCCESS; |
1556 | 0 | blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame; |
1557 | 0 | if (frame == NULL) { |
1558 | 0 | return rc; |
1559 | 0 | } |
1560 | 0 | rc = frame_update_header(frame, schunk, false); |
1561 | 0 | if (rc < 0) { |
1562 | 0 | BLOSC_TRACE_ERROR("Unable to update metalayers into frame."); |
1563 | 0 | return rc; |
1564 | 0 | } |
1565 | 0 | rc = frame_update_trailer(frame, schunk); |
1566 | 0 | if (rc < 0) { |
1567 | 0 | BLOSC_TRACE_ERROR("Unable to update trailer into frame."); |
1568 | 0 | return rc; |
1569 | 0 | } |
1570 | 0 | return rc; |
1571 | 0 | } |
1572 | | |
1573 | | /* Add content into a new variable-length metalayer. |
1574 | | * |
1575 | | * If successful, return the index of the new variable-length metalayer. Else, return a negative value. |
1576 | | */ |
1577 | | int blosc2_vlmeta_add(blosc2_schunk *schunk, const char *name, uint8_t *content, int32_t content_len, |
1578 | 0 | blosc2_cparams *cparams) { |
1579 | 0 | int nvlmetalayer = blosc2_vlmeta_exists(schunk, name); |
1580 | 0 | if (nvlmetalayer >= 0) { |
1581 | 0 | BLOSC_TRACE_ERROR("Variable-length metalayer \"%s\" already exists.", name); |
1582 | 0 | return BLOSC2_ERROR_INVALID_PARAM; |
1583 | 0 | } |
1584 | | |
1585 | | // Add the vlmetalayer |
1586 | 0 | blosc2_metalayer *vlmetalayer = malloc(sizeof(blosc2_metalayer)); |
1587 | 0 | vlmetalayer->name = strdup(name); |
1588 | 0 | uint8_t* content_buf = malloc((size_t) content_len + BLOSC2_MAX_OVERHEAD); |
1589 | |
|
1590 | 0 | blosc2_context *cctx; |
1591 | 0 | if (cparams != NULL) { |
1592 | 0 | cctx = blosc2_create_cctx(*cparams); |
1593 | 0 | } else { |
1594 | 0 | cctx = blosc2_create_cctx(BLOSC2_CPARAMS_DEFAULTS); |
1595 | 0 | } |
1596 | 0 | if (cctx == NULL) { |
1597 | 0 | BLOSC_TRACE_ERROR("Error while creating the compression context"); |
1598 | 0 | return BLOSC2_ERROR_NULL_POINTER; |
1599 | 0 | } |
1600 | | |
1601 | 0 | int csize = blosc2_compress_ctx(cctx, content, content_len, content_buf, content_len + BLOSC2_MAX_OVERHEAD); |
1602 | 0 | if (csize < 0) { |
1603 | 0 | BLOSC_TRACE_ERROR("Can not compress the `%s` variable-length metalayer.", name); |
1604 | 0 | return csize; |
1605 | 0 | } |
1606 | 0 | blosc2_free_ctx(cctx); |
1607 | |
|
1608 | 0 | vlmetalayer->content = realloc(content_buf, csize); |
1609 | 0 | vlmetalayer->content_len = csize; |
1610 | 0 | schunk->vlmetalayers[schunk->nvlmetalayers] = vlmetalayer; |
1611 | 0 | schunk->nvlmetalayers += 1; |
1612 | | |
1613 | | // Propagate to frames |
1614 | 0 | int rc = vlmetalayer_flush(schunk); |
1615 | 0 | if (rc < 0) { |
1616 | 0 | BLOSC_TRACE_ERROR("Can not propagate de `%s` variable-length metalayer to a frame.", name); |
1617 | 0 | return rc; |
1618 | 0 | } |
1619 | | |
1620 | 0 | return schunk->nvlmetalayers - 1; |
1621 | 0 | } |
1622 | | |
1623 | | |
1624 | | int blosc2_vlmeta_get(blosc2_schunk *schunk, const char *name, uint8_t **content, |
1625 | 0 | int32_t *content_len) { |
1626 | 0 | int nvlmetalayer = blosc2_vlmeta_exists(schunk, name); |
1627 | 0 | if (nvlmetalayer < 0) { |
1628 | 0 | BLOSC_TRACE_ERROR("User metalayer \"%s\" not found.", name); |
1629 | 0 | return nvlmetalayer; |
1630 | 0 | } |
1631 | 0 | blosc2_metalayer *meta = schunk->vlmetalayers[nvlmetalayer]; |
1632 | 0 | int32_t nbytes, cbytes; |
1633 | 0 | blosc2_cbuffer_sizes(meta->content, &nbytes, &cbytes, NULL); |
1634 | 0 | if (cbytes != meta->content_len) { |
1635 | 0 | BLOSC_TRACE_ERROR("User metalayer \"%s\" is corrupted.", meta->name); |
1636 | 0 | return BLOSC2_ERROR_DATA; |
1637 | 0 | } |
1638 | 0 | *content_len = nbytes; |
1639 | 0 | *content = malloc((size_t) nbytes); |
1640 | 0 | blosc2_context *dctx = blosc2_create_dctx(*schunk->storage->dparams); |
1641 | 0 | if (dctx == NULL) { |
1642 | 0 | BLOSC_TRACE_ERROR("Error while creating the decompression context"); |
1643 | 0 | return BLOSC2_ERROR_NULL_POINTER; |
1644 | 0 | } |
1645 | 0 | int nbytes_ = blosc2_decompress_ctx(dctx, meta->content, meta->content_len, *content, nbytes); |
1646 | 0 | blosc2_free_ctx(dctx); |
1647 | 0 | if (nbytes_ != nbytes) { |
1648 | 0 | BLOSC_TRACE_ERROR("User metalayer \"%s\" is corrupted.", meta->name); |
1649 | 0 | return BLOSC2_ERROR_READ_BUFFER; |
1650 | 0 | } |
1651 | 0 | return nvlmetalayer; |
1652 | 0 | } |
1653 | | |
1654 | | int blosc2_vlmeta_update(blosc2_schunk *schunk, const char *name, uint8_t *content, int32_t content_len, |
1655 | 0 | blosc2_cparams *cparams) { |
1656 | 0 | int nvlmetalayer = blosc2_vlmeta_exists(schunk, name); |
1657 | 0 | if (nvlmetalayer < 0) { |
1658 | 0 | BLOSC_TRACE_ERROR("User vlmetalayer \"%s\" not found.", name); |
1659 | 0 | return nvlmetalayer; |
1660 | 0 | } |
1661 | | |
1662 | 0 | blosc2_metalayer *vlmetalayer = schunk->vlmetalayers[nvlmetalayer]; |
1663 | 0 | free(vlmetalayer->content); |
1664 | 0 | uint8_t* content_buf = malloc((size_t) content_len + BLOSC2_MAX_OVERHEAD); |
1665 | |
|
1666 | 0 | blosc2_context *cctx; |
1667 | 0 | if (cparams != NULL) { |
1668 | 0 | cctx = blosc2_create_cctx(*cparams); |
1669 | 0 | } else { |
1670 | 0 | cctx = blosc2_create_cctx(BLOSC2_CPARAMS_DEFAULTS); |
1671 | 0 | } |
1672 | 0 | if (cctx == NULL) { |
1673 | 0 | BLOSC_TRACE_ERROR("Error while creating the compression context"); |
1674 | 0 | return BLOSC2_ERROR_NULL_POINTER; |
1675 | 0 | } |
1676 | | |
1677 | 0 | int csize = blosc2_compress_ctx(cctx, content, content_len, content_buf, content_len + BLOSC2_MAX_OVERHEAD); |
1678 | 0 | if (csize < 0) { |
1679 | 0 | BLOSC_TRACE_ERROR("Can not compress the `%s` variable-length metalayer.", name); |
1680 | 0 | return csize; |
1681 | 0 | } |
1682 | 0 | blosc2_free_ctx(cctx); |
1683 | |
|
1684 | 0 | vlmetalayer->content = realloc(content_buf, csize); |
1685 | 0 | vlmetalayer->content_len = csize; |
1686 | | |
1687 | | // Propagate to frames |
1688 | 0 | int rc = vlmetalayer_flush(schunk); |
1689 | 0 | if (rc < 0) { |
1690 | 0 | BLOSC_TRACE_ERROR("Can not propagate de `%s` variable-length metalayer to a frame.", name); |
1691 | 0 | return rc; |
1692 | 0 | } |
1693 | | |
1694 | 0 | return nvlmetalayer; |
1695 | 0 | } |
1696 | | |
1697 | 0 | int blosc2_vlmeta_delete(blosc2_schunk *schunk, const char *name) { |
1698 | 0 | int nvlmetalayer = blosc2_vlmeta_exists(schunk, name); |
1699 | 0 | if (nvlmetalayer < 0) { |
1700 | 0 | BLOSC_TRACE_ERROR("User vlmetalayer \"%s\" not found.", name); |
1701 | 0 | return nvlmetalayer; |
1702 | 0 | } |
1703 | | |
1704 | 0 | blosc2_metalayer *vlmetalayer = schunk->vlmetalayers[nvlmetalayer]; |
1705 | 0 | for (int i = nvlmetalayer; i < (schunk->nvlmetalayers - 1); i++) { |
1706 | 0 | schunk->vlmetalayers[i] = schunk->vlmetalayers[i + 1]; |
1707 | 0 | } |
1708 | 0 | free(vlmetalayer->content); |
1709 | 0 | schunk->nvlmetalayers--; |
1710 | | |
1711 | | // Propagate to frames |
1712 | 0 | int rc = vlmetalayer_flush(schunk); |
1713 | 0 | if (rc < 0) { |
1714 | 0 | BLOSC_TRACE_ERROR("Can not propagate de `%s` variable-length metalayer to a frame.", name); |
1715 | 0 | return rc; |
1716 | 0 | } |
1717 | | |
1718 | 0 | return schunk->nvlmetalayers; |
1719 | 0 | } |
1720 | | |
1721 | | |
1722 | 0 | int blosc2_vlmeta_get_names(blosc2_schunk *schunk, char **names) { |
1723 | 0 | int16_t nvlmetalayers = schunk->nvlmetalayers; |
1724 | |
|
1725 | 0 | for (int i = 0; i < nvlmetalayers; ++i) { |
1726 | 0 | names[i] = schunk->vlmetalayers[i]->name; |
1727 | 0 | } |
1728 | |
|
1729 | 0 | return nvlmetalayers; |
1730 | 0 | } |