1 /*********************************************************************
2 Blosc - Blocked Shuffling and Compression Library
3
4 Copyright (C) 2021 The Blosc Developers <blosc@blosc.org>
5 https://blosc.org
6 License: BSD 3-Clause (see LICENSE.txt)
7
8 See LICENSE.txt for details about copyright and rights to use.
9 **********************************************************************/
10
11
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <string.h>
15 #include <sys/stat.h>
16 #include "blosc2.h"
17 #include "blosc-private.h"
18 #include "frame.h"
19 #include "stune.h"
20
21 #if defined(_WIN32)
22 #include <windows.h>
23 #include <direct.h>
24 #include <malloc.h>
25
26 #define mkdir(D, M) _mkdir(D)
27
28 /* stdint.h only available in VS2010 (VC++ 16.0) and newer */
29 #if defined(_MSC_VER) && _MSC_VER < 1600
30 #include "win32/stdint-windows.h"
31 #else
32 #include <stdint.h>
33 #endif
34
35 #endif /* _WIN32 */
36
37
38 /* If C11 is supported, use it's built-in aligned allocation. */
39 #if __STDC_VERSION__ >= 201112L
40 #include <stdalign.h>
41 #endif
42
43
44 /* Get the cparams associated with a super-chunk */
blosc2_schunk_get_cparams(blosc2_schunk * schunk,blosc2_cparams ** cparams)45 int blosc2_schunk_get_cparams(blosc2_schunk *schunk, blosc2_cparams **cparams) {
46 *cparams = calloc(sizeof(blosc2_cparams), 1);
47 (*cparams)->schunk = schunk;
48 for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) {
49 (*cparams)->filters[i] = schunk->filters[i];
50 (*cparams)->filters_meta[i] = schunk->filters_meta[i];
51 }
52 (*cparams)->compcode = schunk->compcode;
53 (*cparams)->compcode_meta = schunk->compcode_meta;
54 (*cparams)->clevel = schunk->clevel;
55 (*cparams)->typesize = schunk->typesize;
56 (*cparams)->blocksize = schunk->blocksize;
57 if (schunk->cctx == NULL) {
58 (*cparams)->nthreads = BLOSC2_CPARAMS_DEFAULTS.nthreads;
59 }
60 else {
61 (*cparams)->nthreads = (int16_t)schunk->cctx->nthreads;
62 }
63 return 0;
64 }
65
66
67 /* Get the dparams associated with a super-chunk */
blosc2_schunk_get_dparams(blosc2_schunk * schunk,blosc2_dparams ** dparams)68 int blosc2_schunk_get_dparams(blosc2_schunk *schunk, blosc2_dparams **dparams) {
69 *dparams = calloc(sizeof(blosc2_dparams), 1);
70 (*dparams)->schunk = schunk;
71 if (schunk->dctx == NULL) {
72 (*dparams)->nthreads = BLOSC2_DPARAMS_DEFAULTS.nthreads;
73 }
74 else {
75 (*dparams)->nthreads = schunk->dctx->nthreads;
76 }
77 return 0;
78 }
79
80
update_schunk_properties(struct blosc2_schunk * schunk)81 void update_schunk_properties(struct blosc2_schunk* schunk) {
82 blosc2_cparams* cparams = schunk->storage->cparams;
83 blosc2_dparams* dparams = schunk->storage->dparams;
84
85 for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) {
86 schunk->filters[i] = cparams->filters[i];
87 schunk->filters_meta[i] = cparams->filters_meta[i];
88 }
89 schunk->compcode = cparams->compcode;
90 schunk->compcode_meta = cparams->compcode_meta;
91 schunk->clevel = cparams->clevel;
92 schunk->typesize = cparams->typesize;
93 schunk->blocksize = cparams->blocksize;
94 schunk->chunksize = -1;
95
96 /* The compression context */
97 if (schunk->cctx != NULL) {
98 blosc2_free_ctx(schunk->cctx);
99 }
100 cparams->schunk = schunk;
101 schunk->cctx = blosc2_create_cctx(*cparams);
102
103 /* The decompression context */
104 if (schunk->dctx != NULL) {
105 blosc2_free_ctx(schunk->dctx);
106 }
107 dparams->schunk = schunk;
108 schunk->dctx = blosc2_create_dctx(*dparams);
109 }
110
111
file_exists(char * filename)112 static bool file_exists (char *filename) {
113 struct stat buffer;
114 return (stat (filename, &buffer) == 0);
115 }
116
117
118 /* Create a new super-chunk */
blosc2_schunk_new(blosc2_storage * storage)119 blosc2_schunk* blosc2_schunk_new(blosc2_storage *storage) {
120 blosc2_schunk* schunk = calloc(1, sizeof(blosc2_schunk));
121 schunk->version = 0; /* pre-first version */
122
123 // Get the storage with proper defaults
124 schunk->storage = get_new_storage(storage, &BLOSC2_CPARAMS_DEFAULTS, &BLOSC2_DPARAMS_DEFAULTS, &BLOSC2_IO_DEFAULTS);
125 // Update the (local variable) storage
126 storage = schunk->storage;
127
128 schunk->udbtune = malloc(sizeof(blosc2_btune));
129 if (schunk->storage->cparams->udbtune == NULL) {
130 memcpy(schunk->udbtune, &BTUNE_DEFAULTS, sizeof(blosc2_btune));
131 } else {
132 memcpy(schunk->udbtune, schunk->storage->cparams->udbtune, sizeof(blosc2_btune));
133 }
134 schunk->storage->cparams->udbtune = schunk->udbtune;
135
136 // ...and update internal properties
137 update_schunk_properties(schunk);
138
139 schunk->cctx->udbtune->btune_init(schunk->udbtune->btune_config, schunk->cctx, schunk->dctx);
140
141 if (!storage->contiguous && storage->urlpath != NULL){
142 char* urlpath;
143 char last_char = storage->urlpath[strlen(storage->urlpath) - 1];
144 urlpath = malloc(strlen(storage->urlpath) + 1);
145 strcpy(urlpath, storage->urlpath);
146 if (last_char == '\\' || last_char == '/') {
147 urlpath[strlen(storage->urlpath) - 1] = '\0';
148 }
149 // Create directory
150 if (mkdir(urlpath, 0777) == -1) {
151 BLOSC_TRACE_ERROR("Error during the creation of the directory, maybe it already exists.");
152 return NULL;
153 }
154 // We want a sparse (directory) frame as storage
155 blosc2_frame_s* frame = frame_new(urlpath);
156 free(urlpath);
157 frame->sframe = true;
158 // Initialize frame (basically, encode the header)
159 frame->schunk = schunk;
160 int64_t frame_len = frame_from_schunk(schunk, frame);
161 if (frame_len < 0) {
162 BLOSC_TRACE_ERROR("Error during the conversion of schunk to frame.");
163 return NULL;
164 }
165 schunk->frame = (blosc2_frame*)frame;
166 }
167 if (storage->contiguous){
168 // We want a contiguous frame as storage
169 if (storage->urlpath != NULL) {
170 if (file_exists(storage->urlpath)) {
171 BLOSC_TRACE_ERROR("You are trying to ovewrite an existing frame. Remove it first!");
172 return NULL;
173 }
174 }
175 blosc2_frame_s* frame = frame_new(storage->urlpath);
176 frame->sframe = false;
177 // Initialize frame (basically, encode the header)
178 frame->schunk = schunk;
179 int64_t frame_len = frame_from_schunk(schunk, frame);
180 if (frame_len < 0) {
181 BLOSC_TRACE_ERROR("Error during the conversion of schunk to frame.");
182 return NULL;
183 }
184 schunk->frame = (blosc2_frame*)frame;
185 }
186
187 return schunk;
188 }
189
190
191 /* Create a copy of a super-chunk */
blosc2_schunk_copy(blosc2_schunk * schunk,blosc2_storage * storage)192 blosc2_schunk* blosc2_schunk_copy(blosc2_schunk *schunk, blosc2_storage *storage) {
193 if (schunk == NULL) {
194 BLOSC_TRACE_ERROR("Can not copy a NULL `schunk`.");
195 return NULL;
196 }
197
198 // Check if cparams are equals
199 bool cparams_equal = true;
200 blosc2_cparams cparams = {0};
201 if (storage->cparams == NULL) {
202 // When cparams are not specified, just use the same of schunk
203 cparams.typesize = schunk->cctx->typesize;
204 cparams.clevel = schunk->cctx->clevel;
205 cparams.compcode = schunk->cctx->compcode;
206 cparams.compcode_meta = schunk->cctx->compcode_meta;
207 cparams.use_dict = schunk->cctx->use_dict;
208 cparams.blocksize = schunk->cctx->blocksize;
209 memcpy(cparams.filters, schunk->cctx->filters, BLOSC2_MAX_FILTERS);
210 memcpy(cparams.filters_meta, schunk->cctx->filters_meta, BLOSC2_MAX_FILTERS);
211 storage->cparams = &cparams;
212 }
213 else {
214 cparams = *storage->cparams;
215 }
216 if (cparams.blocksize == 0) {
217 // TODO: blocksize should be read from schunk->blocksize
218 // For this, it should be updated during the first append
219 // (or change API to make this a property during schunk creation).
220 cparams.blocksize = schunk->cctx->blocksize;
221 }
222
223 if (cparams.typesize != schunk->cctx->typesize ||
224 cparams.clevel != schunk->cctx->clevel ||
225 cparams.compcode != schunk->cctx->compcode ||
226 cparams.use_dict != schunk->cctx->use_dict ||
227 cparams.blocksize != schunk->cctx->blocksize ||
228 // In case of prefilters or postfilters, force their execution.
229 schunk->cctx->prefilter != NULL ||
230 schunk->dctx->postfilter != NULL) {
231 cparams_equal = false;
232 }
233 for (int i = 0; i < BLOSC2_MAX_FILTERS; ++i) {
234 if (cparams.filters[i] != schunk->cctx->filters[i] ||
235 cparams.filters_meta[i] != schunk->cctx->filters_meta[i]) {
236 cparams_equal = false;
237 }
238 }
239
240 // Create new schunk
241 blosc2_schunk *new_schunk = blosc2_schunk_new(storage);
242 if (new_schunk == NULL) {
243 BLOSC_TRACE_ERROR("Can not create a new schunk");
244 return NULL;
245 }
246
247 // Copy metalayers
248 for (int nmeta = 0; nmeta < schunk->nmetalayers; ++nmeta) {
249 blosc2_metalayer *meta = schunk->metalayers[nmeta];
250 if (blosc2_meta_add(new_schunk, meta->name, meta->content, meta->content_len) < 0) {
251 BLOSC_TRACE_ERROR("Can not add %s `metalayer`.", meta->name);
252 return NULL;
253 }
254 }
255
256 // Copy chunks
257 if (cparams_equal) {
258 for (int nchunk = 0; nchunk < schunk->nchunks; ++nchunk) {
259 uint8_t *chunk;
260 bool needs_free;
261 if (blosc2_schunk_get_chunk(schunk, nchunk, &chunk, &needs_free) < 0) {
262 BLOSC_TRACE_ERROR("Can not get the `chunk` %d.", nchunk);
263 return NULL;
264 }
265 if (blosc2_schunk_append_chunk(new_schunk, chunk, !needs_free) < 0) {
266 BLOSC_TRACE_ERROR("Can not append the `chunk` into super-chunk.");
267 return NULL;
268 }
269 }
270 } else {
271 int32_t chunksize = schunk->chunksize == -1 ? 0 : schunk->chunksize;
272 uint8_t *buffer = malloc(chunksize);
273 for (int nchunk = 0; nchunk < schunk->nchunks; ++nchunk) {
274 if (blosc2_schunk_decompress_chunk(schunk, nchunk, buffer, schunk->chunksize) < 0) {
275 BLOSC_TRACE_ERROR("Can not decompress the `chunk` %d.", nchunk);
276 return NULL;
277 }
278 if (blosc2_schunk_append_buffer(new_schunk, buffer, schunk->chunksize) < 0) {
279 BLOSC_TRACE_ERROR("Can not append the `buffer` into super-chunk.");
280 return NULL;
281 }
282 }
283 free(buffer);
284 }
285
286 // Copy vlmetalayers
287 for (int nmeta = 0; nmeta < schunk->nvlmetalayers; ++nmeta) {
288 uint8_t *content;
289 uint32_t content_len;
290 char* name = schunk->vlmetalayers[nmeta]->name;
291 if (blosc2_vlmeta_get(schunk, name, &content, &content_len) < 0) {
292 BLOSC_TRACE_ERROR("Can not get %s `vlmetalayer`.", name);
293 }
294 if (blosc2_vlmeta_add(new_schunk, name, content, content_len, NULL) < 0) {
295 BLOSC_TRACE_ERROR("Can not add %s `vlmetalayer`.", name);
296 return NULL;
297 }
298 free(content);
299 }
300 return new_schunk;
301 }
302
303
304 /* Open an existing super-chunk that is on-disk (no copy is made). */
blosc2_schunk_open_udio(const char * urlpath,const blosc2_io * udio)305 blosc2_schunk* blosc2_schunk_open_udio(const char* urlpath, const blosc2_io *udio) {
306 if (urlpath == NULL) {
307 BLOSC_TRACE_ERROR("You need to supply a urlpath.");
308 return NULL;
309 }
310
311 blosc2_frame_s* frame = frame_from_file(urlpath, udio);
312 blosc2_schunk* schunk = frame_to_schunk(frame, false, udio);
313
314 // Set the storage with proper defaults
315 size_t pathlen = strlen(urlpath);
316 schunk->storage->urlpath = malloc(pathlen + 1);
317 strcpy(schunk->storage->urlpath, urlpath);
318 schunk->storage->contiguous = !frame->sframe;
319
320 return schunk;
321 }
322
blosc2_schunk_open(const char * urlpath)323 blosc2_schunk* blosc2_schunk_open(const char* urlpath) {
324 return blosc2_schunk_open_udio(urlpath, &BLOSC2_IO_DEFAULTS);
325 }
326
blosc2_schunk_to_buffer(blosc2_schunk * schunk,uint8_t ** dest,bool * needs_free)327 int64_t blosc2_schunk_to_buffer(blosc2_schunk* schunk, uint8_t** dest, bool* needs_free) {
328 blosc2_frame_s* frame;
329 int64_t cframe_len;
330
331 // Initialize defaults in case of errors
332 *dest = NULL;
333 *needs_free = false;
334
335 if ((schunk->storage->contiguous == true) && (schunk->storage->urlpath == NULL)) {
336 frame = (blosc2_frame_s*)(schunk->frame);
337 *dest = frame->cframe;
338 cframe_len = frame->len;
339 *needs_free = false;
340 }
341 else {
342 // Copy to a contiguous storage
343 blosc2_storage frame_storage = {.contiguous=true};
344 blosc2_schunk* schunk_copy = blosc2_schunk_copy(schunk, &frame_storage);
345 if (schunk_copy == NULL) {
346 BLOSC_TRACE_ERROR("Error during the conversion of schunk to buffer.");
347 return BLOSC2_ERROR_SCHUNK_COPY;
348 }
349 frame = (blosc2_frame_s*)(schunk_copy->frame);
350 *dest = frame->cframe;
351 cframe_len = frame->len;
352 *needs_free = true;
353 frame->avoid_cframe_free = true;
354 blosc2_schunk_free(schunk_copy);
355 }
356
357 return cframe_len;
358
359 }
360
361
362 /* Write an in-memory frame out to a file. */
frame_to_file(blosc2_frame_s * frame,const char * urlpath)363 int64_t frame_to_file(blosc2_frame_s* frame, const char* urlpath) {
364 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
365 if (io_cb == NULL) {
366 BLOSC_TRACE_ERROR("Error getting the input/output API");
367 return BLOSC2_ERROR_PLUGIN_IO;
368 }
369 void* fp = io_cb->open(urlpath, "wb", frame->schunk->storage->io);
370 int64_t nitems = io_cb->write(frame->cframe, frame->len, 1, fp);
371 io_cb->close(fp);
372 return nitems * (size_t)frame->len;
373 }
374
375
376 /* Write super-chunk out to a file. */
blosc2_schunk_to_file(blosc2_schunk * schunk,const char * urlpath)377 int64_t blosc2_schunk_to_file(blosc2_schunk* schunk, const char* urlpath) {
378 if (urlpath == NULL) {
379 BLOSC_TRACE_ERROR("urlpath cannot be NULL");
380 return BLOSC2_ERROR_INVALID_PARAM;
381 }
382
383 // Accelerated path for in-memory frames
384 if (schunk->storage->contiguous && schunk->storage->urlpath == NULL) {
385 int64_t len = frame_to_file((blosc2_frame_s*)(schunk->frame), urlpath);
386 if (len <= 0) {
387 BLOSC_TRACE_ERROR("Error writing to file");
388 return len;
389 }
390 return len;
391 }
392
393 // Copy to a contiguous file
394 blosc2_storage frame_storage = {.contiguous=true, .urlpath=(char*)urlpath};
395 blosc2_schunk* schunk_copy = blosc2_schunk_copy(schunk, &frame_storage);
396 if (schunk_copy == NULL) {
397 BLOSC_TRACE_ERROR("Error during the conversion of schunk to buffer.");
398 return BLOSC2_ERROR_SCHUNK_COPY;
399 }
400 blosc2_frame_s* frame = (blosc2_frame_s*)(schunk_copy->frame);
401 int64_t frame_len = frame->len;
402 blosc2_schunk_free(schunk_copy);
403 return frame_len;
404 }
405
406
407 /* Free all memory from a super-chunk. */
blosc2_schunk_free(blosc2_schunk * schunk)408 int blosc2_schunk_free(blosc2_schunk *schunk) {
409 if (schunk->data != NULL) {
410 for (int i = 0; i < schunk->nchunks; i++) {
411 free(schunk->data[i]);
412 }
413 free(schunk->data);
414 }
415 if (schunk->cctx != NULL)
416 blosc2_free_ctx(schunk->cctx);
417 if (schunk->dctx != NULL)
418 blosc2_free_ctx(schunk->dctx);
419
420 if (schunk->nmetalayers > 0) {
421 for (int i = 0; i < schunk->nmetalayers; i++) {
422 if (schunk->metalayers[i] != NULL) {
423 if (schunk->metalayers[i]->name != NULL)
424 free(schunk->metalayers[i]->name);
425 if (schunk->metalayers[i]->content != NULL)
426 free(schunk->metalayers[i]->content);
427 free(schunk->metalayers[i]);
428 }
429 }
430 schunk->nmetalayers = 0;
431 }
432
433 if (schunk->storage != NULL) {
434 if (schunk->storage->urlpath != NULL) {
435 free(schunk->storage->urlpath);
436 }
437 free(schunk->storage->cparams);
438 free(schunk->storage->dparams);
439 free(schunk->storage->io);
440 free(schunk->storage);
441 }
442
443 if (schunk->frame != NULL) {
444 frame_free((blosc2_frame_s *) schunk->frame);
445 }
446
447 if (schunk->nvlmetalayers > 0) {
448 for (int i = 0; i < schunk->nvlmetalayers; ++i) {
449 if (schunk->vlmetalayers[i] != NULL) {
450 if (schunk->vlmetalayers[i]->name != NULL)
451 free(schunk->vlmetalayers[i]->name);
452 if (schunk->vlmetalayers[i]->content != NULL)
453 free(schunk->vlmetalayers[i]->content);
454 free(schunk->vlmetalayers[i]);
455 }
456 }
457 }
458
459 if (schunk->udbtune != NULL) {
460 free(schunk->udbtune);
461 }
462 free(schunk);
463
464 return 0;
465 }
466
467
468 /* Create a super-chunk out of a contiguous frame buffer */
blosc2_schunk_from_buffer(uint8_t * cframe,int64_t len,bool copy)469 blosc2_schunk* blosc2_schunk_from_buffer(uint8_t *cframe, int64_t len, bool copy) {
470 blosc2_frame_s* frame = frame_from_cframe(cframe, len, false);
471 if (frame == NULL) {
472 return NULL;
473 }
474 blosc2_schunk* schunk = frame_to_schunk(frame, copy, &BLOSC2_IO_DEFAULTS);
475 if (schunk && copy) {
476 // Super-chunk has its own copy of frame
477 frame_free(frame);
478 }
479 return schunk;
480 }
481
482 /* Fill an empty frame with special values (fast path). */
blosc2_schunk_fill_special(blosc2_schunk * schunk,int64_t nitems,int special_value,int32_t chunksize)483 int blosc2_schunk_fill_special(blosc2_schunk* schunk, int64_t nitems, int special_value,
484 int32_t chunksize) {
485 if (nitems == 0) {
486 return 0;
487 }
488
489 int32_t typesize = schunk->typesize;
490
491 if ((nitems * typesize / chunksize) > INT_MAX) {
492 BLOSC_TRACE_ERROR("nitems is too large. Try increasing the chunksize.");
493 return BLOSC2_ERROR_SCHUNK_SPECIAL;
494 }
495
496 if ((schunk->nbytes > 0) || (schunk->cbytes > 0)) {
497 BLOSC_TRACE_ERROR("Filling with special values only works on empty super-chunks");
498 return BLOSC2_ERROR_FRAME_SPECIAL;
499 }
500
501 // Compute the number of chunks and the length of the offsets chunk
502 int32_t chunkitems = chunksize / typesize;
503 int32_t nchunks = (int32_t)(nitems / chunkitems);
504 int32_t leftover_items = (int32_t)(nitems % chunkitems);
505
506 if (schunk->frame == NULL) {
507 // Build the special chunks
508 int32_t leftover_size = leftover_items * typesize;
509 void* chunk = malloc(BLOSC_EXTENDED_HEADER_LENGTH);
510 void* chunk2 = malloc(BLOSC_EXTENDED_HEADER_LENGTH);
511 blosc2_cparams* cparams;
512 blosc2_schunk_get_cparams(schunk, &cparams);
513 int csize, csize2;
514 switch (special_value) {
515 case BLOSC2_SPECIAL_ZERO:
516 csize = blosc2_chunk_zeros(*cparams, chunksize, chunk, BLOSC_EXTENDED_HEADER_LENGTH);
517 csize2 = blosc2_chunk_zeros(*cparams, leftover_size, chunk2, BLOSC_EXTENDED_HEADER_LENGTH);
518 break;
519 case BLOSC2_SPECIAL_UNINIT:
520 csize = blosc2_chunk_uninit(*cparams, chunksize, chunk, BLOSC_EXTENDED_HEADER_LENGTH);
521 csize2 = blosc2_chunk_uninit(*cparams, leftover_size, chunk2, BLOSC_EXTENDED_HEADER_LENGTH);
522 break;
523 case BLOSC2_SPECIAL_NAN:
524 csize = blosc2_chunk_nans(*cparams, chunksize, chunk, BLOSC_EXTENDED_HEADER_LENGTH);
525 csize2 = blosc2_chunk_nans(*cparams, leftover_size, chunk2, BLOSC_EXTENDED_HEADER_LENGTH);
526 break;
527 default:
528 BLOSC_TRACE_ERROR("Only zeros, NaNs or non-initialized values are supported.");
529 return BLOSC2_ERROR_SCHUNK_SPECIAL;
530 }
531 free(cparams);
532 if (csize < 0 || csize2 < 0) {
533 BLOSC_TRACE_ERROR("Error creating special chunks.");
534 return BLOSC2_ERROR_SCHUNK_SPECIAL;
535 }
536
537 for (int nchunk = 0; nchunk < nchunks; nchunk++) {
538 int nchunk_ = blosc2_schunk_append_chunk(schunk, chunk, true);
539 if (nchunk_ != nchunk + 1) {
540 BLOSC_TRACE_ERROR("Error appending special chunks.");
541 return BLOSC2_ERROR_SCHUNK_SPECIAL;
542 }
543 }
544
545 if (leftover_items) {
546 int nchunk_ = blosc2_schunk_append_chunk(schunk, chunk2, true);
547 if (nchunk_ != nchunks + 1) {
548 BLOSC_TRACE_ERROR("Error appending last special chunk.");
549 return BLOSC2_ERROR_SCHUNK_SPECIAL;
550 }
551 }
552 free(chunk);
553 free(chunk2);
554 }
555 else {
556 /* Fill an empty frame with special values (fast path). */
557 blosc2_frame_s *frame = (blosc2_frame_s *) schunk->frame;
558 /* Update counters (necessary for the frame_fill_special() logic) */
559 if (leftover_items) {
560 nchunks += 1;
561 }
562 schunk->chunksize = chunksize;
563 schunk->nchunks = nchunks;
564 schunk->nbytes = nitems * typesize;
565 int64_t frame_len = frame_fill_special(frame, nitems, special_value, chunksize, schunk);
566 if (frame_len < 0) {
567 BLOSC_TRACE_ERROR("Error creating special frame.");
568 return frame_len;
569 }
570 }
571
572 return schunk->nchunks;
573 }
574
575 /* Append an existing chunk into a super-chunk. */
blosc2_schunk_append_chunk(blosc2_schunk * schunk,uint8_t * chunk,bool copy)576 int blosc2_schunk_append_chunk(blosc2_schunk *schunk, uint8_t *chunk, bool copy) {
577 int32_t chunk_nbytes;
578 int32_t chunk_cbytes;
579 int32_t nchunks = schunk->nchunks;
580
581 int rc = blosc2_cbuffer_sizes(chunk, &chunk_nbytes, &chunk_cbytes, NULL);
582 if (rc < 0) {
583 return rc;
584 }
585
586 if (schunk->chunksize == -1) {
587 schunk->chunksize = chunk_nbytes; // The super-chunk is initialized now
588 }
589 if (chunk_nbytes > (size_t)schunk->chunksize) {
590 BLOSC_TRACE_ERROR("Appending chunks that have different lengths in the same schunk "
591 "is not supported yet: %d > %d.", chunk_nbytes, schunk->chunksize);
592 return BLOSC2_ERROR_CHUNK_APPEND;
593 }
594
595 /* Update counters */
596 schunk->nchunks = nchunks + 1;
597 schunk->nbytes += chunk_nbytes;
598 if (schunk->frame == NULL) {
599 schunk->cbytes += chunk_cbytes;
600 } else {
601 // A frame
602 int special_value = (chunk[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK;
603 switch (special_value) {
604 case BLOSC2_SPECIAL_ZERO:
605 case BLOSC2_SPECIAL_NAN:
606 case BLOSC2_SPECIAL_UNINIT:
607 schunk->cbytes += 0;
608 break;
609 default:
610 schunk->cbytes += chunk_cbytes;
611 }
612 }
613
614 if (copy) {
615 // Make a copy of the chunk
616 uint8_t *chunk_copy = malloc(chunk_cbytes);
617 memcpy(chunk_copy, chunk, chunk_cbytes);
618 chunk = chunk_copy;
619 }
620
621 // Update super-chunk or frame
622 blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame;
623 if (frame == NULL) {
624 // Check that we are not appending a small chunk after another small chunk
625 if ((schunk->nchunks > 0) && (chunk_nbytes < (size_t)schunk->chunksize)) {
626 uint8_t* last_chunk = schunk->data[nchunks - 1];
627 int32_t last_nbytes;
628 rc = blosc2_cbuffer_sizes(last_chunk, &last_nbytes, NULL, NULL);
629 if (rc < 0) {
630 return rc;
631 }
632 if ((last_nbytes < schunk->chunksize) && (chunk_nbytes < (size_t)schunk->chunksize)) {
633 BLOSC_TRACE_ERROR(
634 "Appending two consecutive chunks with a chunksize smaller than the schunk chunksize "
635 "is not allowed yet: %d != %d.", chunk_nbytes, schunk->chunksize);
636 return BLOSC2_ERROR_CHUNK_APPEND;
637 }
638 }
639
640 if (!copy && (chunk_cbytes < chunk_nbytes)) {
641 // We still want to do a shrink of the chunk
642 chunk = realloc(chunk, chunk_cbytes);
643 }
644
645 /* Make space for appending the copy of the chunk and do it */
646 if ((nchunks + 1) * sizeof(void *) > schunk->data_len) {
647 // Extend the data pointer by one memory page (4k)
648 schunk->data_len += 4096; // must be a multiple of sizeof(void*)
649 schunk->data = realloc(schunk->data, schunk->data_len);
650 }
651 schunk->data[nchunks] = chunk;
652 }
653 else {
654 if (frame_append_chunk(frame, chunk, schunk) == NULL) {
655 BLOSC_TRACE_ERROR("Problems appending a chunk.");
656 return BLOSC2_ERROR_CHUNK_APPEND;
657 }
658 }
659 return schunk->nchunks;
660 }
661
662
663 /* Insert an existing @p chunk in a specified position on a super-chunk */
blosc2_schunk_insert_chunk(blosc2_schunk * schunk,int nchunk,uint8_t * chunk,bool copy)664 int blosc2_schunk_insert_chunk(blosc2_schunk *schunk, int nchunk, uint8_t *chunk, bool copy) {
665 int32_t chunk_nbytes;
666 int32_t chunk_cbytes;
667 int32_t nchunks = schunk->nchunks;
668
669 int rc = blosc2_cbuffer_sizes(chunk, &chunk_nbytes, &chunk_cbytes, NULL);
670 if (rc < 0) {
671 return rc;
672 }
673
674 if (schunk->chunksize == -1) {
675 schunk->chunksize = chunk_nbytes; // The super-chunk is initialized now
676 }
677
678 if (chunk_nbytes > schunk->chunksize) {
679 BLOSC_TRACE_ERROR("Inserting chunks that have different lengths in the same schunk "
680 "is not supported yet: %d > %d.", chunk_nbytes, schunk->chunksize);
681 return BLOSC2_ERROR_CHUNK_INSERT;
682 }
683
684 /* Update counters */
685 schunk->nchunks = nchunks + 1;
686 schunk->nbytes += chunk_nbytes;
687 if (schunk->frame == NULL) {
688 schunk->cbytes += chunk_cbytes;
689 } else {
690 // A frame
691 int special_value = (chunk[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK;
692 switch (special_value) {
693 case BLOSC2_SPECIAL_ZERO:
694 case BLOSC2_SPECIAL_NAN:
695 case BLOSC2_SPECIAL_UNINIT:
696 schunk->cbytes += 0;
697 break;
698 default:
699 schunk->cbytes += chunk_cbytes;
700 }
701 }
702
703 if (copy) {
704 // Make a copy of the chunk
705 uint8_t *chunk_copy = malloc(chunk_cbytes);
706 memcpy(chunk_copy, chunk, chunk_cbytes);
707 chunk = chunk_copy;
708 }
709
710 // Update super-chunk or frame
711 blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame;
712 if (frame == NULL) {
713 // Check that we are not appending a small chunk after another small chunk
714 if ((schunk->nchunks > 0) && (chunk_nbytes < schunk->chunksize)) {
715 uint8_t* last_chunk = schunk->data[nchunks - 1];
716 int32_t last_nbytes;
717 rc = blosc2_cbuffer_sizes(last_chunk, &last_nbytes, NULL, NULL);
718 if (rc < 0) {
719 return rc;
720 }
721 if ((last_nbytes < schunk->chunksize) && (chunk_nbytes < schunk->chunksize)) {
722 BLOSC_TRACE_ERROR("Appending two consecutive chunks with a chunksize smaller "
723 "than the schunk chunksize is not allowed yet: %d != %d",
724 chunk_nbytes, schunk->chunksize);
725 return BLOSC2_ERROR_CHUNK_APPEND;
726 }
727 }
728
729 if (!copy && (chunk_cbytes < chunk_nbytes)) {
730 // We still want to do a shrink of the chunk
731 chunk = realloc(chunk, chunk_cbytes);
732 }
733
734 // Make space for appending the copy of the chunk and do it
735 if ((nchunks + 1) * sizeof(void *) > schunk->data_len) {
736 // Extend the data pointer by one memory page (4k)
737 schunk->data_len += 4096; // must be a multiple of sizeof(void*)
738 schunk->data = realloc(schunk->data, schunk->data_len);
739 }
740
741 // Reorder the offsets and insert the new chunk
742 for (int i = nchunks; i > nchunk; --i) {
743 schunk->data[i] = schunk->data[i-1];
744 }
745 schunk->data[nchunk] = chunk;
746 }
747
748 else {
749 if (frame_insert_chunk(frame, nchunk, chunk, schunk) == NULL) {
750 BLOSC_TRACE_ERROR("Problems inserting a chunk in a frame.");
751 return BLOSC2_ERROR_CHUNK_INSERT;
752 }
753 }
754 return schunk->nchunks;
755 }
756
757
blosc2_schunk_update_chunk(blosc2_schunk * schunk,int nchunk,uint8_t * chunk,bool copy)758 int blosc2_schunk_update_chunk(blosc2_schunk *schunk, int nchunk, uint8_t *chunk, bool copy) {
759 int32_t chunk_nbytes;
760 int32_t chunk_cbytes;
761
762 int rc = blosc2_cbuffer_sizes(chunk, &chunk_nbytes, &chunk_cbytes, NULL);
763 if (rc < 0) {
764 return rc;
765 }
766
767 if (schunk->chunksize == -1) {
768 schunk->chunksize = chunk_nbytes; // The super-chunk is initialized now
769 }
770
771 if ((schunk->chunksize != 0) && (chunk_nbytes != schunk->chunksize)) {
772 BLOSC_TRACE_ERROR("Inserting chunks that have different lengths in the same schunk "
773 "is not supported yet: %d > %d.", chunk_nbytes, schunk->chunksize);
774 return BLOSC2_ERROR_CHUNK_INSERT;
775 }
776
777 bool needs_free;
778 uint8_t *chunk_old;
779 int err = blosc2_schunk_get_chunk(schunk, nchunk, &chunk_old, &needs_free);
780 if (err < 0) {
781 BLOSC_TRACE_ERROR("%d chunk can not be obtained from schunk.", nchunk);
782 return -1;
783 }
784 int32_t chunk_nbytes_old = 0;
785 int32_t chunk_cbytes_old = 0;
786
787 if (chunk_old != 0) {
788 rc = blosc2_cbuffer_sizes(chunk_old, &chunk_nbytes_old, &chunk_cbytes_old, NULL);
789 if (rc < 0) {
790 return rc;
791 }
792 if (chunk_cbytes_old == BLOSC_MAX_OVERHEAD) {
793 chunk_cbytes_old = 0;
794 }
795 }
796 if (needs_free) {
797 free(chunk_old);
798 }
799
800 if (copy) {
801 // Make a copy of the chunk
802 uint8_t *chunk_copy = malloc(chunk_cbytes);
803 memcpy(chunk_copy, chunk, chunk_cbytes);
804 chunk = chunk_copy;
805 }
806
807 blosc2_frame_s* frame = (blosc2_frame_s*)(schunk->frame);
808 if (schunk->frame == NULL) {
809 /* Update counters */
810 schunk->nbytes += chunk_nbytes;
811 schunk->nbytes -= chunk_nbytes_old;
812 schunk->cbytes += chunk_cbytes;
813 schunk->cbytes -= chunk_cbytes_old;
814 } else {
815 // A frame
816 int special_value = (chunk[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK;
817 switch (special_value) {
818 case BLOSC2_SPECIAL_ZERO:
819 case BLOSC2_SPECIAL_NAN:
820 case BLOSC2_SPECIAL_UNINIT:
821 schunk->nbytes += chunk_nbytes;
822 schunk->nbytes -= chunk_nbytes_old;
823 if (frame->sframe) {
824 schunk->cbytes -= chunk_cbytes_old;
825 }
826 break;
827 default:
828 /* Update counters */
829 schunk->nbytes += chunk_nbytes;
830 schunk->nbytes -= chunk_nbytes_old;
831 schunk->cbytes += chunk_cbytes;
832 if (frame->sframe) {
833 schunk->cbytes -= chunk_cbytes_old;
834 }
835 else {
836 if (chunk_cbytes_old >= chunk_cbytes) {
837 schunk->cbytes -= chunk_cbytes;
838 }
839 }
840 }
841 }
842
843 // Update super-chunk or frame
844 if (schunk->frame == NULL) {
845 if (!copy && (chunk_cbytes < chunk_nbytes)) {
846 // We still want to do a shrink of the chunk
847 chunk = realloc(chunk, chunk_cbytes);
848 }
849
850 // Free old chunk and add reference to new chunk
851 if (schunk->data[nchunk] != 0) {
852 free(schunk->data[nchunk]);
853 }
854 schunk->data[nchunk] = chunk;
855 }
856 else {
857 if (frame_update_chunk(frame, nchunk, chunk, schunk) == NULL) {
858 BLOSC_TRACE_ERROR("Problems updating a chunk in a frame.");
859 return BLOSC2_ERROR_CHUNK_UPDATE;
860 }
861 }
862
863 return schunk->nchunks;
864 }
865
blosc2_schunk_delete_chunk(blosc2_schunk * schunk,int nchunk)866 int blosc2_schunk_delete_chunk(blosc2_schunk *schunk, int nchunk) {
867 int rc;
868 if (schunk->nchunks < nchunk) {
869 BLOSC_TRACE_ERROR("The schunk has not enough chunks (%d)!", schunk->nchunks);
870 }
871
872 bool needs_free;
873 uint8_t *chunk_old;
874 int err = blosc2_schunk_get_chunk(schunk, nchunk, &chunk_old, &needs_free);
875 if (err < 0) {
876 BLOSC_TRACE_ERROR("%d chunk can not be obtained from schunk.", nchunk);
877 return -1;
878 }
879 int32_t chunk_nbytes_old = 0;
880 int32_t chunk_cbytes_old = 0;
881
882 if (chunk_old != 0) {
883 rc = blosc2_cbuffer_sizes(chunk_old, &chunk_nbytes_old, &chunk_cbytes_old, NULL);
884 if (rc < 0) {
885 return rc;
886 }
887 if (chunk_cbytes_old == BLOSC_MAX_OVERHEAD) {
888 chunk_cbytes_old = 0;
889 }
890 }
891 if (needs_free) {
892 free(chunk_old);
893 }
894
895 blosc2_frame_s* frame = (blosc2_frame_s*)(schunk->frame);
896 schunk->nchunks -= 1;
897 if (schunk->frame == NULL) {
898 /* Update counters */
899 schunk->nbytes -= chunk_nbytes_old;
900 schunk->cbytes -= chunk_cbytes_old;
901 } else {
902 // A frame
903 schunk->nbytes -= chunk_nbytes_old;
904 if (frame->sframe) {
905 schunk->cbytes -= chunk_cbytes_old;
906 }
907 }
908
909 // Update super-chunk or frame
910 if (schunk->frame == NULL) {
911 // Free old chunk
912 if (schunk->data[nchunk] != 0) {
913 free(schunk->data[nchunk]);
914 }
915 // Reorder the offsets and insert the new chunk
916 for (int i = nchunk; i < schunk->nchunks; i++) {
917 schunk->data[i] = schunk->data[i + 1];
918 }
919 schunk->data[schunk->nchunks] = NULL;
920
921 }
922 else {
923 if (frame_delete_chunk(frame, nchunk, schunk) == NULL) {
924 BLOSC_TRACE_ERROR("Problems deleting a chunk in a frame.");
925 return BLOSC2_ERROR_CHUNK_UPDATE;
926 }
927 }
928 return schunk->nchunks;
929 }
930
931
932 /* Append a data buffer to a super-chunk. */
blosc2_schunk_append_buffer(blosc2_schunk * schunk,void * src,int32_t nbytes)933 int blosc2_schunk_append_buffer(blosc2_schunk *schunk, void *src, int32_t nbytes) {
934 uint8_t* chunk = malloc(nbytes + BLOSC_MAX_OVERHEAD);
935 /* Compress the src buffer using super-chunk context */
936 int cbytes = blosc2_compress_ctx(schunk->cctx, src, nbytes, chunk,
937 nbytes + BLOSC_MAX_OVERHEAD);
938 if (cbytes < 0) {
939 free(chunk);
940 return cbytes;
941 }
942 // We don't need a copy of the chunk, as it will be shrinked if necessary
943 int nchunks = blosc2_schunk_append_chunk(schunk, chunk, false);
944 if (nchunks < 0) {
945 BLOSC_TRACE_ERROR("Error appending a buffer in super-chunk");
946 return nchunks;
947 }
948
949 return nchunks;
950 }
951
952 /* Decompress and return a chunk that is part of a super-chunk. */
blosc2_schunk_decompress_chunk(blosc2_schunk * schunk,int nchunk,void * dest,int32_t nbytes)953 int blosc2_schunk_decompress_chunk(blosc2_schunk *schunk, int nchunk,
954 void *dest, int32_t nbytes) {
955 int32_t chunk_nbytes;
956 int32_t chunk_cbytes;
957 int chunksize;
958 int rc;
959 blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame;
960
961 if (frame == NULL) {
962 if (nchunk >= schunk->nchunks) {
963 BLOSC_TRACE_ERROR("nchunk ('%d') exceeds the number of chunks "
964 "('%d') in super-chunk.", nchunk, schunk->nchunks);
965 return BLOSC2_ERROR_INVALID_PARAM;
966 }
967 uint8_t* src = schunk->data[nchunk];
968 if (src == 0) {
969 return 0;
970 }
971
972 rc = blosc2_cbuffer_sizes(src, &chunk_nbytes, &chunk_cbytes, NULL);
973 if (rc < 0) {
974 return rc;
975 }
976
977 if (nbytes < chunk_nbytes) {
978 BLOSC_TRACE_ERROR("Buffer size is too small for the decompressed buffer "
979 "('%d' bytes, but '%d' are needed).", nbytes, chunk_nbytes);
980 return BLOSC2_ERROR_INVALID_PARAM;
981 }
982
983 chunksize = blosc2_decompress_ctx(schunk->dctx, src, chunk_cbytes, dest, nbytes);
984 if (chunksize < 0 || chunksize != chunk_nbytes) {
985 BLOSC_TRACE_ERROR("Error in decompressing chunk.");
986 if (chunksize < 0)
987 return chunksize;
988 return BLOSC2_ERROR_FAILURE;
989 }
990 } else {
991 chunksize = frame_decompress_chunk(schunk->dctx, frame, nchunk, dest, nbytes);
992 if (chunksize < 0) {
993 return chunksize;
994 }
995 }
996 return chunksize;
997 }
998
999 /* Return a compressed chunk that is part of a super-chunk in the `chunk` parameter.
1000 * If the super-chunk is backed by a frame that is disk-based, a buffer is allocated for the
1001 * (compressed) chunk, and hence a free is needed. You can check if the chunk requires a free
1002 * with the `needs_free` parameter.
1003 * If the chunk does not need a free, it means that a pointer to the location in the super-chunk
1004 * (or the backing in-memory frame) is returned in the `chunk` parameter.
1005 *
1006 * The size of the (compressed) chunk is returned. If some problem is detected, a negative code
1007 * is returned instead.
1008 */
blosc2_schunk_get_chunk(blosc2_schunk * schunk,int nchunk,uint8_t ** chunk,bool * needs_free)1009 int blosc2_schunk_get_chunk(blosc2_schunk *schunk, int nchunk, uint8_t **chunk, bool *needs_free) {
1010 blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame;
1011 if (frame != NULL) {
1012 return frame_get_chunk(frame, nchunk, chunk, needs_free);
1013 }
1014
1015 if (nchunk >= schunk->nchunks) {
1016 BLOSC_TRACE_ERROR("nchunk ('%d') exceeds the number of chunks "
1017 "('%d') in schunk.", nchunk, schunk->nchunks);
1018 return BLOSC2_ERROR_INVALID_PARAM;
1019 }
1020
1021 *chunk = schunk->data[nchunk];
1022 if (*chunk == 0) {
1023 *needs_free = 0;
1024 return 0;
1025 }
1026
1027 *needs_free = false;
1028 int32_t chunk_cbytes;
1029 int rc = blosc2_cbuffer_sizes(*chunk, NULL, &chunk_cbytes, NULL);
1030 if (rc < 0) {
1031 return rc;
1032 }
1033 return (int)chunk_cbytes;
1034 }
1035
1036
1037 /* Return a compressed chunk that is part of a super-chunk in the `chunk` parameter.
1038 * If the super-chunk is backed by a frame that is disk-based, a buffer is allocated for the
1039 * (compressed) chunk, and hence a free is needed. You can check if the chunk requires a free
1040 * with the `needs_free` parameter.
1041 * If the chunk does not need a free, it means that a pointer to the location in the super-chunk
1042 * (or the backing in-memory frame) is returned in the `chunk` parameter.
1043 *
1044 * The size of the (compressed) chunk is returned. If some problem is detected, a negative code
1045 * is returned instead.
1046 */
blosc2_schunk_get_lazychunk(blosc2_schunk * schunk,int nchunk,uint8_t ** chunk,bool * needs_free)1047 int blosc2_schunk_get_lazychunk(blosc2_schunk *schunk, int nchunk, uint8_t **chunk, bool *needs_free) {
1048 blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame;
1049 if (schunk->frame != NULL) {
1050 return frame_get_lazychunk(frame, nchunk, chunk, needs_free);
1051 }
1052
1053 if (nchunk >= schunk->nchunks) {
1054 BLOSC_TRACE_ERROR("nchunk ('%d') exceeds the number of chunks "
1055 "('%d') in schunk.", nchunk, schunk->nchunks);
1056 return BLOSC2_ERROR_INVALID_PARAM;
1057 }
1058
1059 *chunk = schunk->data[nchunk];
1060 if (*chunk == 0) {
1061 *needs_free = 0;
1062 return 0;
1063 }
1064
1065 *needs_free = false;
1066 int32_t chunk_cbytes;
1067 int rc = blosc2_cbuffer_sizes(*chunk, NULL, &chunk_cbytes, NULL);
1068 if (rc < 0) {
1069 return rc;
1070 }
1071 return (int)chunk_cbytes;
1072 }
1073
1074
1075 /* Find whether the schunk has a metalayer or not.
1076 *
1077 * If successful, return the index of the metalayer. Else, return a negative value.
1078 */
blosc2_meta_exists(blosc2_schunk * schunk,const char * name)1079 int blosc2_meta_exists(blosc2_schunk *schunk, const char *name) {
1080 if (strlen(name) > BLOSC2_METALAYER_NAME_MAXLEN) {
1081 BLOSC_TRACE_ERROR("Metalayers cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN);
1082 return BLOSC2_ERROR_INVALID_PARAM;
1083 }
1084
1085 if (schunk == NULL) {
1086 BLOSC_TRACE_ERROR("Schunk must not be NUll.");
1087 return BLOSC2_ERROR_INVALID_PARAM;
1088 }
1089
1090 for (int nmetalayer = 0; nmetalayer < schunk->nmetalayers; nmetalayer++) {
1091 if (strcmp(name, schunk->metalayers[nmetalayer]->name) == 0) {
1092 return nmetalayer;
1093 }
1094 }
1095 return BLOSC2_ERROR_NOT_FOUND;
1096 }
1097
1098 /* Reorder the chunk offsets of an existing super-chunk. */
blosc2_schunk_reorder_offsets(blosc2_schunk * schunk,int * offsets_order)1099 int blosc2_schunk_reorder_offsets(blosc2_schunk *schunk, int *offsets_order) {
1100 // Check that the offsets order are correct
1101 bool *index_check = (bool *) calloc(schunk->nchunks, sizeof(bool));
1102 for (int i = 0; i < schunk->nchunks; ++i) {
1103 int index = offsets_order[i];
1104 if (index >= schunk->nchunks) {
1105 BLOSC_TRACE_ERROR("Index is bigger than the number of chunks.");
1106 free(index_check);
1107 return BLOSC2_ERROR_DATA;
1108 }
1109 if (index_check[index] == false) {
1110 index_check[index] = true;
1111 } else {
1112 BLOSC_TRACE_ERROR("Index is yet used.");
1113 free(index_check);
1114 return BLOSC2_ERROR_DATA;
1115 }
1116 }
1117 free(index_check);
1118
1119 blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame;
1120 if (frame != NULL) {
1121 return frame_reorder_offsets(frame, offsets_order, schunk);
1122 }
1123 uint8_t **offsets = schunk->data;
1124
1125 // Make a copy of the chunk offsets and reorder it
1126 uint8_t **offsets_copy = malloc(schunk->data_len);
1127 memcpy(offsets_copy, offsets, schunk->data_len);
1128
1129 for (int i = 0; i < schunk->nchunks; ++i) {
1130 offsets[i] = offsets_copy[offsets_order[i]];
1131 }
1132 free(offsets_copy);
1133
1134 return 0;
1135 }
1136
1137
1138 // Get the length (in bytes) of the internal frame of the super-chunk
blosc2_schunk_frame_len(blosc2_schunk * schunk)1139 int64_t blosc2_schunk_frame_len(blosc2_schunk* schunk) {
1140 int64_t len;
1141 blosc2_frame_s* frame_s = (blosc2_frame_s*)(schunk->frame);
1142 if (frame_s != NULL) {
1143 len = frame_s->len;
1144 }
1145 else {
1146 // No attached frame, but we can still come with an estimate
1147 len = schunk->cbytes + schunk->nchunks * sizeof(int64_t);
1148 }
1149
1150 return len;
1151 }
1152
1153
1154 /**
1155 * @brief Flush metalayers content into a possible attached frame.
1156 *
1157 * @param schunk The super-chunk to which the flush should be applied.
1158 *
1159 * @return If successful, a 0 is returned. Else, return a negative value.
1160 */
1161 // Initially, this was a public function, but as it is really meant to be used only
1162 // in the schunk_add_metalayer(), I decided to convert it into private and call it
1163 // implicitly instead of requiring the user to do so. The only drawback is that
1164 // each add operation requires a complete frame re-build, but as users should need
1165 // very few metalayers, this overhead should be negligible in practice.
metalayer_flush(blosc2_schunk * schunk)1166 int metalayer_flush(blosc2_schunk* schunk) {
1167 int rc = BLOSC2_ERROR_SUCCESS;
1168 blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame;
1169 if (frame == NULL) {
1170 return rc;
1171 }
1172 rc = frame_update_header(frame, schunk, true);
1173 if (rc < 0) {
1174 BLOSC_TRACE_ERROR("Unable to update metalayers into frame.");
1175 return rc;
1176 }
1177 rc = frame_update_trailer(frame, schunk);
1178 if (rc < 0) {
1179 BLOSC_TRACE_ERROR("Unable to update trailer into frame.");
1180 return rc;
1181 }
1182 return rc;
1183 }
1184
1185
1186 /* Add content into a new metalayer.
1187 *
1188 * If successful, return the index of the new metalayer. Else, return a negative value.
1189 */
blosc2_meta_add(blosc2_schunk * schunk,const char * name,uint8_t * content,uint32_t content_len)1190 int blosc2_meta_add(blosc2_schunk *schunk, const char *name, uint8_t *content, uint32_t content_len) {
1191 int nmetalayer = blosc2_meta_exists(schunk, name);
1192 if (nmetalayer >= 0) {
1193 BLOSC_TRACE_ERROR("Metalayer \"%s\" already exists.", name);
1194 return BLOSC2_ERROR_INVALID_PARAM;
1195 }
1196
1197 // Add the metalayer
1198 blosc2_metalayer *metalayer = malloc(sizeof(blosc2_metalayer));
1199 char* name_ = malloc(strlen(name) + 1);
1200 strcpy(name_, name);
1201 metalayer->name = name_;
1202 uint8_t* content_buf = malloc((size_t)content_len);
1203 memcpy(content_buf, content, content_len);
1204 metalayer->content = content_buf;
1205 metalayer->content_len = content_len;
1206 schunk->metalayers[schunk->nmetalayers] = metalayer;
1207 schunk->nmetalayers += 1;
1208
1209 int rc = metalayer_flush(schunk);
1210 if (rc < 0) {
1211 return rc;
1212 }
1213
1214 return schunk->nmetalayers - 1;
1215 }
1216
1217
1218 /* Update the content of an existing metalayer.
1219 *
1220 * If successful, return the index of the new metalayer. Else, return a negative value.
1221 */
blosc2_meta_update(blosc2_schunk * schunk,const char * name,uint8_t * content,uint32_t content_len)1222 int blosc2_meta_update(blosc2_schunk *schunk, const char *name, uint8_t *content, uint32_t content_len) {
1223 int nmetalayer = blosc2_meta_exists(schunk, name);
1224 if (nmetalayer < 0) {
1225 BLOSC_TRACE_ERROR("Metalayer \"%s\" not found.", name);
1226 return nmetalayer;
1227 }
1228
1229 blosc2_metalayer *metalayer = schunk->metalayers[nmetalayer];
1230 if (content_len > (uint32_t)metalayer->content_len) {
1231 BLOSC_TRACE_ERROR("`content_len` cannot exceed the existing size of %d bytes.", metalayer->content_len);
1232 return nmetalayer;
1233 }
1234
1235 // Update the contents of the metalayer
1236 memcpy(metalayer->content, content, content_len);
1237
1238 // Update the metalayers in frame (as size has not changed, we don't need to update the trailer)
1239 blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame;
1240 if (frame != NULL) {
1241 int rc = frame_update_header(frame, schunk, false);
1242 if (rc < 0) {
1243 BLOSC_TRACE_ERROR("Unable to update meta info from frame.");
1244 return rc;
1245 }
1246 }
1247
1248 return nmetalayer;
1249 }
1250
1251
1252 /* Get the content out of a metalayer.
1253 *
1254 * The `**content` receives a malloc'ed copy of the content. The user is responsible of freeing it.
1255 *
1256 * If successful, return the index of the new metalayer. Else, return a negative value.
1257 */
blosc2_meta_get(blosc2_schunk * schunk,const char * name,uint8_t ** content,uint32_t * content_len)1258 int blosc2_meta_get(blosc2_schunk *schunk, const char *name, uint8_t **content,
1259 uint32_t *content_len) {
1260 int nmetalayer = blosc2_meta_exists(schunk, name);
1261 if (nmetalayer < 0) {
1262 BLOSC_TRACE_ERROR("Metalayer \"%s\" not found.", name);
1263 return nmetalayer;
1264 }
1265 *content_len = (uint32_t)schunk->metalayers[nmetalayer]->content_len;
1266 *content = malloc((size_t)*content_len);
1267 memcpy(*content, schunk->metalayers[nmetalayer]->content, (size_t)*content_len);
1268 return nmetalayer;
1269 }
1270
1271 /* Find whether the schunk has a variable-length metalayer or not.
1272 *
1273 * If successful, return the index of the variable-length metalayer. Else, return a negative value.
1274 */
blosc2_vlmeta_exists(blosc2_schunk * schunk,const char * name)1275 int blosc2_vlmeta_exists(blosc2_schunk *schunk, const char *name) {
1276 if (strlen(name) > BLOSC2_METALAYER_NAME_MAXLEN) {
1277 BLOSC_TRACE_ERROR("Variable-length metalayer names cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN);
1278 return BLOSC2_ERROR_INVALID_PARAM;
1279 }
1280
1281 for (int nvlmetalayer = 0; nvlmetalayer < schunk->nvlmetalayers; nvlmetalayer++) {
1282 if (strcmp(name, schunk->vlmetalayers[nvlmetalayer]->name) == 0) {
1283 return nvlmetalayer;
1284 }
1285 }
1286 return BLOSC2_ERROR_NOT_FOUND;
1287 }
1288
vlmetalayer_flush(blosc2_schunk * schunk)1289 int vlmetalayer_flush(blosc2_schunk* schunk) {
1290 int rc = BLOSC2_ERROR_SUCCESS;
1291 blosc2_frame_s* frame = (blosc2_frame_s*)schunk->frame;
1292 if (frame == NULL) {
1293 return rc;
1294 }
1295 rc = frame_update_header(frame, schunk, false);
1296 if (rc < 0) {
1297 BLOSC_TRACE_ERROR("Unable to update metalayers into frame.");
1298 return rc;
1299 }
1300 rc = frame_update_trailer(frame, schunk);
1301 if (rc < 0) {
1302 BLOSC_TRACE_ERROR("Unable to update trailer into frame.");
1303 return rc;
1304 }
1305 return rc;
1306 }
1307
1308 /* Add content into a new variable-length metalayer.
1309 *
1310 * If successful, return the index of the new variable-length metalayer. Else, return a negative value.
1311 */
blosc2_vlmeta_add(blosc2_schunk * schunk,const char * name,uint8_t * content,uint32_t content_len,blosc2_cparams * cparams)1312 int blosc2_vlmeta_add(blosc2_schunk *schunk, const char *name, uint8_t *content, uint32_t content_len,
1313 blosc2_cparams *cparams) {
1314 int nvlmetalayer = blosc2_vlmeta_exists(schunk, name);
1315 if (nvlmetalayer >= 0) {
1316 BLOSC_TRACE_ERROR("Variable-length metalayer \"%s\" already exists.", name);
1317 return BLOSC2_ERROR_INVALID_PARAM;
1318 }
1319
1320 // Add the vlmetalayer
1321 blosc2_metalayer *vlmetalayer = malloc(sizeof(blosc2_metalayer));
1322 vlmetalayer->name = strdup(name);
1323 uint8_t* content_buf = malloc((size_t) content_len + BLOSC_MAX_OVERHEAD);
1324
1325 blosc2_context *cctx;
1326 if (cparams != NULL) {
1327 cctx = blosc2_create_cctx(*cparams);
1328 } else {
1329 cctx = blosc2_create_cctx(BLOSC2_CPARAMS_DEFAULTS);
1330 }
1331
1332 int csize = blosc2_compress_ctx(cctx, content, content_len, content_buf, content_len + BLOSC_MAX_OVERHEAD);
1333 if (csize < 0) {
1334 BLOSC_TRACE_ERROR("Can not compress the `%s` variable-length metalayer.", name);
1335 return csize;
1336 }
1337 blosc2_free_ctx(cctx);
1338
1339 vlmetalayer->content = realloc(content_buf, csize);
1340 vlmetalayer->content_len = csize;
1341 schunk->vlmetalayers[schunk->nvlmetalayers] = vlmetalayer;
1342 schunk->nvlmetalayers += 1;
1343
1344 // Propagate to frames
1345 int rc = vlmetalayer_flush(schunk);
1346 if (rc < 0) {
1347 BLOSC_TRACE_ERROR("Can not propagate de `%s` variable-length metalayer to a frame.", name);
1348 return rc;
1349 }
1350
1351 return schunk->nvlmetalayers - 1;
1352 }
1353
1354
blosc2_vlmeta_get(blosc2_schunk * schunk,const char * name,uint8_t ** content,uint32_t * content_len)1355 int blosc2_vlmeta_get(blosc2_schunk *schunk, const char *name, uint8_t **content,
1356 uint32_t *content_len) {
1357 int nvlmetalayer = blosc2_vlmeta_exists(schunk, name);
1358 if (nvlmetalayer < 0) {
1359 BLOSC_TRACE_ERROR("User metalayer \"%s\" not found.", name);
1360 return nvlmetalayer;
1361 }
1362 blosc2_metalayer *meta = schunk->vlmetalayers[nvlmetalayer];
1363 int32_t nbytes, cbytes;
1364 blosc2_cbuffer_sizes(meta->content, &nbytes, &cbytes, NULL);
1365 if (cbytes != meta->content_len) {
1366 BLOSC_TRACE_ERROR("User metalayer \"%s\" is corrupted.", meta->name);
1367 return BLOSC2_ERROR_DATA;
1368 }
1369 *content_len = nbytes;
1370 *content = malloc((size_t) nbytes);
1371 blosc2_context *dctx = blosc2_create_dctx(*schunk->storage->dparams);
1372 int nbytes_ = blosc2_decompress_ctx(dctx, meta->content, meta->content_len, *content, nbytes);
1373 blosc2_free_ctx(dctx);
1374 if (nbytes_ != nbytes) {
1375 BLOSC_TRACE_ERROR("User metalayer \"%s\" is corrupted.", meta->name);
1376 return BLOSC2_ERROR_READ_BUFFER;
1377 }
1378 return nvlmetalayer;
1379 }
1380
blosc2_vlmeta_update(blosc2_schunk * schunk,const char * name,uint8_t * content,uint32_t content_len,blosc2_cparams * cparams)1381 int blosc2_vlmeta_update(blosc2_schunk *schunk, const char *name, uint8_t *content, uint32_t content_len,
1382 blosc2_cparams *cparams) {
1383 int nvlmetalayer = blosc2_vlmeta_exists(schunk, name);
1384 if (nvlmetalayer < 0) {
1385 BLOSC_TRACE_ERROR("User vlmetalayer \"%s\" not found.", name);
1386 return nvlmetalayer;
1387 }
1388
1389 blosc2_metalayer *vlmetalayer = schunk->vlmetalayers[nvlmetalayer];
1390 free(vlmetalayer->content);
1391 uint8_t* content_buf = malloc((size_t) content_len + BLOSC_MAX_OVERHEAD);
1392
1393 blosc2_context *cctx;
1394 if (cparams != NULL) {
1395 cctx = blosc2_create_cctx(*cparams);
1396 } else {
1397 cctx = blosc2_create_cctx(BLOSC2_CPARAMS_DEFAULTS);
1398 }
1399
1400 int csize = blosc2_compress_ctx(cctx, content, content_len, content_buf, content_len + BLOSC_MAX_OVERHEAD);
1401 if (csize < 0) {
1402 BLOSC_TRACE_ERROR("Can not compress the `%s` variable-length metalayer.", name);
1403 return csize;
1404 }
1405 blosc2_free_ctx(cctx);
1406
1407 vlmetalayer->content = realloc(content_buf, csize);
1408 vlmetalayer->content_len = csize;
1409
1410 // Propagate to frames
1411 int rc = vlmetalayer_flush(schunk);
1412 if (rc < 0) {
1413 BLOSC_TRACE_ERROR("Can not propagate de `%s` variable-length metalayer to a frame.", name);
1414 return rc;
1415 }
1416
1417 return nvlmetalayer;
1418 }
1419
blosc2_vlmeta_delete(blosc2_schunk * schunk,const char * name)1420 int blosc2_vlmeta_delete(blosc2_schunk *schunk, const char *name) {
1421 int nvlmetalayer = blosc2_vlmeta_exists(schunk, name);
1422 if (nvlmetalayer < 0) {
1423 BLOSC_TRACE_ERROR("User vlmetalayer \"%s\" not found.", name);
1424 return nvlmetalayer;
1425 }
1426
1427 blosc2_metalayer *vlmetalayer = schunk->vlmetalayers[nvlmetalayer];
1428 for (int i = nvlmetalayer; i < (schunk->nvlmetalayers - 1); i++) {
1429 schunk->vlmetalayers[i] = schunk->vlmetalayers[i + 1];
1430 }
1431 free(vlmetalayer->content);
1432 schunk->nvlmetalayers--;
1433
1434 // Propagate to frames
1435 int rc = vlmetalayer_flush(schunk);
1436 if (rc < 0) {
1437 BLOSC_TRACE_ERROR("Can not propagate de `%s` variable-length metalayer to a frame.", name);
1438 return rc;
1439 }
1440
1441 return schunk->nvlmetalayers;
1442 }
1443