1 /*********************************************************************
2 Blosc - Blocked Shuffling and Compression Library
3
4 Copyright (C) 2021 The Blosc Developers <blosc@blosc.org>
5 https://blosc.org
6 License: BSD 3-Clause (see LICENSE.txt)
7
8 See LICENSE.txt for details about copyright and rights to use.
9 **********************************************************************/
10
11 #include <stdio.h>
12 #include <stdlib.h>
13 #include <string.h>
14 #include <stdbool.h>
15 #include <sys/stat.h>
16 #include "blosc2.h"
17 #include "blosc-private.h"
18 #include "context.h"
19 #include "frame.h"
20 #include "sframe.h"
21 #include <inttypes.h>
22
23 #if defined(_WIN32)
24 #include <windows.h>
25 #include <malloc.h>
26
27 /* stdint.h only available in VS2010 (VC++ 16.0) and newer */
28 #if defined(_MSC_VER) && _MSC_VER < 1600
29 #include "win32/stdint-windows.h"
30 #else
31 #include <stdint.h>
32 #endif
33
34 #endif /* _WIN32 */
35
36 /* If C11 is supported, use it's built-in aligned allocation. */
37 #if __STDC_VERSION__ >= 201112L
38 #include <stdalign.h>
39 #endif
40
41
42 /* Create a new (empty) frame */
frame_new(const char * urlpath)43 blosc2_frame_s* frame_new(const char* urlpath) {
44 blosc2_frame_s* new_frame = calloc(1, sizeof(blosc2_frame_s));
45 if (urlpath != NULL) {
46 char* new_urlpath = malloc(strlen(urlpath) + 1); // + 1 for the trailing NULL
47 new_frame->urlpath = strcpy(new_urlpath, urlpath);
48 }
49 return new_frame;
50 }
51
52
53 /* Free memory from a frame. */
frame_free(blosc2_frame_s * frame)54 int frame_free(blosc2_frame_s* frame) {
55
56 if (frame->cframe != NULL && !frame->avoid_cframe_free) {
57 free(frame->cframe);
58 }
59
60 if (frame->coffsets != NULL) {
61 free(frame->coffsets);
62 }
63
64 if (frame->urlpath != NULL) {
65 free(frame->urlpath);
66 }
67
68 free(frame);
69
70 return 0;
71 }
72
73
new_header_frame(blosc2_schunk * schunk,blosc2_frame_s * frame)74 void *new_header_frame(blosc2_schunk *schunk, blosc2_frame_s *frame) {
75 if (frame == NULL) {
76 return NULL;
77 }
78 uint8_t* h2 = calloc(FRAME_HEADER_MINLEN, 1);
79 uint8_t* h2p = h2;
80
81 // The msgpack header starts here
82 *h2p = 0x90; // fixarray...
83 *h2p += 14; // ...with 13 elements
84 h2p += 1;
85
86 // Magic number
87 *h2p = 0xa0 + 8; // str with 8 elements
88 h2p += 1;
89 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
90 return NULL;
91 }
92 strcpy((char*)h2p, "b2frame");
93 h2p += 8;
94
95 // Header size
96 *h2p = 0xd2; // int32
97 h2p += 1 + 4;
98 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
99 return NULL;
100 }
101
102 // Total frame size
103 *h2p = 0xcf; // uint64
104 // Fill it with frame->len which is known *after* the creation of the frame (e.g. when updating the header)
105 int64_t flen = frame->len;
106 to_big(h2 + FRAME_LEN, &flen, sizeof(flen));
107 h2p += 1 + 8;
108 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
109 return NULL;
110 }
111
112 // Flags
113 *h2p = 0xa0 + 4; // str with 4 elements
114 h2p += 1;
115 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
116 return NULL;
117 }
118 // General flags
119 *h2p = BLOSC2_VERSION_FRAME_FORMAT; // version
120 *h2p += 0x10; // 64-bit offsets. We only support this for now.
121 h2p += 1;
122 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
123 return NULL;
124 }
125
126 // Frame type
127 // We only support contiguous and sparse directories frames currently
128 *h2p = frame->sframe ? 1 : 0;
129 h2p += 1;
130 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
131 return NULL;
132 }
133
134 // Codec flags
135 *h2p = schunk->compcode;
136 if (schunk->compcode >= BLOSC_LAST_CODEC) {
137 *h2p = BLOSC_UDCODEC_FORMAT;
138 }
139 *h2p += (schunk->clevel) << 4u; // clevel
140 h2p += 1;
141 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
142 return NULL;
143 }
144
145 // Reserved flags
146 *h2p = 0;
147 h2p += 1;
148 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
149 return NULL;
150 }
151
152 // Uncompressed size
153 *h2p = 0xd3; // int64
154 h2p += 1;
155 int64_t nbytes = schunk->nbytes;
156 to_big(h2p, &nbytes, sizeof(nbytes));
157 h2p += 8;
158 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
159 return NULL;
160 }
161
162 // Compressed size
163 *h2p = 0xd3; // int64
164 h2p += 1;
165 int64_t cbytes = schunk->cbytes;
166 to_big(h2p, &cbytes, sizeof(cbytes));
167 h2p += 8;
168 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
169 return NULL;
170 }
171
172 // Type size
173 *h2p = 0xd2; // int32
174 h2p += 1;
175 int32_t typesize = schunk->typesize;
176 to_big(h2p, &typesize, sizeof(typesize));
177 h2p += 4;
178 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
179 return NULL;
180 }
181
182 // Block size
183 *h2p = 0xd2; // int32
184 h2p += 1;
185 int32_t blocksize = schunk->blocksize;
186 to_big(h2p, &blocksize, sizeof(blocksize));
187 h2p += 4;
188 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
189 return NULL;
190 }
191
192 // Chunk size
193 *h2p = 0xd2; // int32
194 h2p += 1;
195 int32_t chunksize = schunk->chunksize;
196 to_big(h2p, &chunksize, sizeof(chunksize));
197 h2p += 4;
198 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
199 return NULL;
200 }
201
202 // Number of threads for compression
203 *h2p = 0xd1; // int16
204 h2p += 1;
205 int16_t nthreads = (int16_t)schunk->cctx->nthreads;
206 to_big(h2p, &nthreads, sizeof(nthreads));
207 h2p += 2;
208 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
209 return NULL;
210 }
211
212 // Number of threads for decompression
213 *h2p = 0xd1; // int16
214 h2p += 1;
215 nthreads = (int16_t)schunk->dctx->nthreads;
216 to_big(h2p, &nthreads, sizeof(nthreads));
217 h2p += 2;
218 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
219 return NULL;
220 }
221
222 // The boolean for variable-length metalayers
223 *h2p = (schunk->nvlmetalayers > 0) ? (uint8_t)0xc3 : (uint8_t)0xc2;
224 h2p += 1;
225 if (h2p - h2 >= FRAME_HEADER_MINLEN) {
226 return NULL;
227 }
228
229 // The space for FRAME_FILTER_PIPELINE
230 *h2p = 0xd8; // fixext 16
231 h2p += 1;
232 if (BLOSC2_MAX_FILTERS > FRAME_FILTER_PIPELINE_MAX) {
233 return NULL;
234 }
235 // Store the filter pipeline in header
236 uint8_t* mp_filters = h2 + FRAME_FILTER_PIPELINE + 1;
237 uint8_t* mp_meta = h2 + FRAME_FILTER_PIPELINE + 1 + FRAME_FILTER_PIPELINE_MAX;
238 int nfilters = 0;
239 for (int i = 0; i < BLOSC2_MAX_FILTERS; i++) {
240 mp_filters[i] = schunk->filters[i];
241 mp_meta[i] = schunk->filters_meta[i];
242 }
243 *h2p = (uint8_t) BLOSC2_MAX_FILTERS;
244 h2p += 1;
245 h2p += 16;
246
247 // User-defined codec and codec metadata
248 uint8_t* udcodec = h2 + FRAME_UDCODEC;
249 *udcodec = schunk->compcode;
250 uint8_t* codec_meta = h2 + FRAME_CODEC_META;
251 *codec_meta = schunk->compcode_meta;
252
253 if (h2p - h2 != FRAME_HEADER_MINLEN) {
254 return NULL;
255 }
256
257 int32_t hsize = FRAME_HEADER_MINLEN;
258
259 // Now, deal with metalayers
260 int16_t nmetalayers = schunk->nmetalayers;
261 if (nmetalayers < 0 || nmetalayers > BLOSC2_MAX_METALAYERS) {
262 return NULL;
263 }
264
265 // Make space for the header of metalayers (array marker, size, map of offsets)
266 h2 = realloc(h2, (size_t)hsize + 1 + 1 + 2 + 1 + 2);
267 h2p = h2 + hsize;
268
269 // The msgpack header for the metalayers (array_marker, size, map of offsets, list of metalayers)
270 *h2p = 0x90 + 3; // array with 3 elements
271 h2p += 1;
272
273 // Size for the map (index) of offsets, including this uint16 size (to be filled out later on)
274 *h2p = 0xcd; // uint16
275 h2p += 1 + 2;
276
277 // Map (index) of offsets for optional metalayers
278 *h2p = 0xde; // map 16 with N keys
279 h2p += 1;
280 to_big(h2p, &nmetalayers, sizeof(nmetalayers));
281 h2p += sizeof(nmetalayers);
282 int32_t current_header_len = (int32_t)(h2p - h2);
283 int32_t *offtooff = malloc(nmetalayers * sizeof(int32_t));
284 for (int nmetalayer = 0; nmetalayer < nmetalayers; nmetalayer++) {
285 if (frame == NULL) {
286 return NULL;
287 }
288 blosc2_metalayer *metalayer = schunk->metalayers[nmetalayer];
289 uint8_t namelen = (uint8_t) strlen(metalayer->name);
290 h2 = realloc(h2, (size_t)current_header_len + 1 + namelen + 1 + 4);
291 h2p = h2 + current_header_len;
292 // Store the metalayer
293 if (namelen >= (1U << 5U)) { // metalayer strings cannot be longer than 32 bytes
294 free(offtooff);
295 return NULL;
296 }
297 *h2p = (uint8_t)0xa0 + namelen; // str
298 h2p += 1;
299 memcpy(h2p, metalayer->name, namelen);
300 h2p += namelen;
301 // Space for storing the offset for the value of this metalayer
302 *h2p = 0xd2; // int32
303 h2p += 1;
304 offtooff[nmetalayer] = (int32_t)(h2p - h2);
305 h2p += 4;
306 current_header_len += 1 + namelen + 1 + 4;
307 }
308 int32_t hsize2 = (int32_t)(h2p - h2);
309 if (hsize2 != current_header_len) { // sanity check
310 return NULL;
311 }
312
313 // Map size + int16 size
314 if ((uint32_t) (hsize2 - hsize) >= (1U << 16U)) {
315 return NULL;
316 }
317 uint16_t map_size = (uint16_t) (hsize2 - hsize);
318 to_big(h2 + FRAME_IDX_SIZE, &map_size, sizeof(map_size));
319
320 // Make space for an (empty) array
321 hsize = (int32_t)(h2p - h2);
322 h2 = realloc(h2, (size_t)hsize + 2 + 1 + 2);
323 h2p = h2 + hsize;
324
325 // Now, store the values in an array
326 *h2p = 0xdc; // array 16 with N elements
327 h2p += 1;
328 to_big(h2p, &nmetalayers, sizeof(nmetalayers));
329 h2p += sizeof(nmetalayers);
330 current_header_len = (int32_t)(h2p - h2);
331 for (int nmetalayer = 0; nmetalayer < nmetalayers; nmetalayer++) {
332 if (frame == NULL) {
333 return NULL;
334 }
335 blosc2_metalayer *metalayer = schunk->metalayers[nmetalayer];
336 h2 = realloc(h2, (size_t)current_header_len + 1 + 4 + metalayer->content_len);
337 h2p = h2 + current_header_len;
338 // Store the serialized contents for this metalayer
339 *h2p = 0xc6; // bin32
340 h2p += 1;
341 to_big(h2p, &(metalayer->content_len), sizeof(metalayer->content_len));
342 h2p += 4;
343 memcpy(h2p, metalayer->content, metalayer->content_len); // buffer, no need to swap
344 h2p += metalayer->content_len;
345 // Update the offset now that we know it
346 to_big(h2 + offtooff[nmetalayer], ¤t_header_len, sizeof(current_header_len));
347 current_header_len += 1 + 4 + metalayer->content_len;
348 }
349 free(offtooff);
350 hsize = (int32_t)(h2p - h2);
351 if (hsize != current_header_len) { // sanity check
352 return NULL;
353 }
354
355 // Set the length of the whole header now that we know it
356 to_big(h2 + FRAME_HEADER_LEN, &hsize, sizeof(hsize));
357
358 return h2;
359 }
360
361
get_header_info(blosc2_frame_s * frame,int32_t * header_len,int64_t * frame_len,int64_t * nbytes,int64_t * cbytes,int32_t * blocksize,int32_t * chunksize,int32_t * nchunks,int32_t * typesize,uint8_t * compcode,uint8_t * compcode_meta,uint8_t * clevel,uint8_t * filters,uint8_t * filters_meta,const blosc2_io * io)362 int get_header_info(blosc2_frame_s *frame, int32_t *header_len, int64_t *frame_len, int64_t *nbytes, int64_t *cbytes,
363 int32_t *blocksize, int32_t *chunksize, int32_t *nchunks, int32_t *typesize, uint8_t *compcode,
364 uint8_t *compcode_meta, uint8_t *clevel, uint8_t *filters, uint8_t *filters_meta, const blosc2_io *io) {
365 uint8_t* framep = frame->cframe;
366 uint8_t header[FRAME_HEADER_MINLEN];
367
368 blosc2_io_cb *io_cb = blosc2_get_io_cb(io->id);
369 if (io_cb == NULL) {
370 BLOSC_TRACE_ERROR("Error getting the input/output API");
371 return BLOSC2_ERROR_PLUGIN_IO;
372 }
373
374 if (frame->len <= 0) {
375 return BLOSC2_ERROR_READ_BUFFER;
376 }
377
378 if (frame->cframe == NULL) {
379 int64_t rbytes = 0;
380 void* fp = NULL;
381 if (frame->sframe) {
382 fp = sframe_open_index(frame->urlpath, "rb",
383 io);
384 }
385 else {
386 fp = io_cb->open(frame->urlpath, "rb", io->params);
387 }
388 if (fp != NULL) {
389 rbytes = io_cb->read(header, 1, FRAME_HEADER_MINLEN, fp);
390 io_cb->close(fp);
391 }
392 (void) rbytes;
393 if (rbytes != FRAME_HEADER_MINLEN) {
394 return BLOSC2_ERROR_FILE_READ;
395 }
396 framep = header;
397 }
398
399 // Consistency check for frame type
400 uint8_t frame_type = framep[FRAME_TYPE];
401 if (frame->sframe) {
402 if (frame_type != FRAME_DIRECTORY_TYPE) {
403 return BLOSC2_ERROR_FRAME_TYPE;
404 }
405 } else {
406 if (frame_type != FRAME_CONTIGUOUS_TYPE) {
407 return BLOSC2_ERROR_FRAME_TYPE;
408 }
409 }
410
411 // Fetch some internal lengths
412 from_big(header_len, framep + FRAME_HEADER_LEN, sizeof(*header_len));
413 if (*header_len < FRAME_HEADER_MINLEN) {
414 BLOSC_TRACE_ERROR("Header length is zero or smaller than min allowed.");
415 return BLOSC2_ERROR_INVALID_HEADER;
416 }
417 from_big(frame_len, framep + FRAME_LEN, sizeof(*frame_len));
418 if (*header_len > *frame_len) {
419 BLOSC_TRACE_ERROR("Header length exceeds length of the frame.");
420 return BLOSC2_ERROR_INVALID_HEADER;
421 }
422 from_big(nbytes, framep + FRAME_NBYTES, sizeof(*nbytes));
423 from_big(cbytes, framep + FRAME_CBYTES, sizeof(*cbytes));
424 from_big(blocksize, framep + FRAME_BLOCKSIZE, sizeof(*blocksize));
425 if (chunksize != NULL) {
426 from_big(chunksize, framep + FRAME_CHUNKSIZE, sizeof(*chunksize));
427 }
428 if (typesize != NULL) {
429 from_big(typesize, framep + FRAME_TYPESIZE, sizeof(*typesize));
430 if (*typesize <= 0 || *typesize > BLOSC_MAX_TYPESIZE) {
431 BLOSC_TRACE_ERROR("`typesize` is zero or greater than max allowed.");
432 return BLOSC2_ERROR_INVALID_HEADER;
433 }
434 }
435
436 // Codecs
437 uint8_t frame_codecs = framep[FRAME_CODECS];
438 if (clevel != NULL) {
439 *clevel = frame_codecs >> 4u;
440 }
441 if (compcode != NULL) {
442 *compcode = frame_codecs & 0xFu;
443 if (*compcode == BLOSC_UDCODEC_FORMAT) {
444 from_big(compcode, framep + FRAME_UDCODEC, sizeof(*compcode));
445 }
446 }
447
448
449 if (compcode_meta != NULL) {
450 from_big(compcode_meta, framep + FRAME_CODEC_META, sizeof(*compcode_meta));
451 }
452
453 // Filters
454 if (filters != NULL && filters_meta != NULL) {
455 uint8_t nfilters = framep[FRAME_FILTER_PIPELINE];
456 if (nfilters > BLOSC2_MAX_FILTERS) {
457 BLOSC_TRACE_ERROR("The number of filters in frame header are too large for Blosc2.");
458 return BLOSC2_ERROR_INVALID_HEADER;
459 }
460 uint8_t *filters_ = framep + FRAME_FILTER_PIPELINE + 1;
461 uint8_t *filters_meta_ = framep + FRAME_FILTER_PIPELINE + 1 + FRAME_FILTER_PIPELINE_MAX;
462 for (int i = 0; i < nfilters; i++) {
463 filters[i] = filters_[i];
464 filters_meta[i] = filters_meta_[i];
465 }
466 }
467
468 if (*nbytes > 0 && *chunksize > 0) {
469 // We can compute the number of chunks only when the frame has actual data
470 *nchunks = (int32_t) (*nbytes / *chunksize);
471 if (*nbytes % *chunksize > 0) {
472 if (*nchunks == INT32_MAX) {
473 BLOSC_TRACE_ERROR("Number of chunks exceeds maximum allowed.");
474 return BLOSC2_ERROR_INVALID_HEADER;
475 }
476 *nchunks += 1;
477 }
478
479 // Sanity check for compressed sizes
480 if ((*cbytes < 0) || ((int64_t)*nchunks * *chunksize < *nbytes)) {
481 BLOSC_TRACE_ERROR("Invalid compressed size in frame header.");
482 return BLOSC2_ERROR_INVALID_HEADER;
483 }
484 } else {
485 *nchunks = 0;
486 }
487
488 return 0;
489 }
490
491
get_trailer_offset(blosc2_frame_s * frame,int32_t header_len,bool has_coffsets)492 int64_t get_trailer_offset(blosc2_frame_s *frame, int32_t header_len, bool has_coffsets) {
493 if (!has_coffsets) {
494 // No data chunks yet
495 return header_len;
496 }
497 return frame->len - frame->trailer_len;
498 }
499
500
501 // Update the length in the header
update_frame_len(blosc2_frame_s * frame,int64_t len)502 int update_frame_len(blosc2_frame_s* frame, int64_t len) {
503 int rc = 1;
504 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
505 if (io_cb == NULL) {
506 BLOSC_TRACE_ERROR("Error getting the input/output API");
507 return BLOSC2_ERROR_PLUGIN_IO;
508 }
509
510 if (frame->cframe != NULL) {
511 to_big(frame->cframe + FRAME_LEN, &len, sizeof(int64_t));
512 }
513 else {
514 void* fp = NULL;
515 if (frame->sframe) {
516 fp = sframe_open_index(frame->urlpath, "rb+",
517 frame->schunk->storage->io);
518 }
519 else {
520 fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params);
521 }
522 io_cb->seek(fp, FRAME_LEN, SEEK_SET);
523 int64_t swap_len;
524 to_big(&swap_len, &len, sizeof(int64_t));
525 int64_t wbytes = io_cb->write(&swap_len, 1, sizeof(int64_t), fp);
526 io_cb->close(fp);
527 if (wbytes != sizeof(int64_t)) {
528 BLOSC_TRACE_ERROR("Cannot write the frame length in header.");
529 return BLOSC2_ERROR_FILE_WRITE;
530 }
531 }
532 return rc;
533 }
534
535
frame_update_trailer(blosc2_frame_s * frame,blosc2_schunk * schunk)536 int frame_update_trailer(blosc2_frame_s* frame, blosc2_schunk* schunk) {
537 if (frame != NULL && frame->len == 0) {
538 BLOSC_TRACE_ERROR("The trailer cannot be updated on empty frames.");
539 }
540
541 // Create the trailer in msgpack (see the frame format document)
542 uint32_t trailer_len = FRAME_TRAILER_MINLEN;
543 uint8_t* trailer = (uint8_t*)calloc((size_t)trailer_len, 1);
544 uint8_t* ptrailer = trailer;
545 *ptrailer = 0x90 + 4; // fixarray with 4 elements
546 ptrailer += 1;
547 // Trailer format version
548 *ptrailer = FRAME_TRAILER_VERSION;
549 ptrailer += 1;
550
551 int32_t current_trailer_len = (int32_t)(ptrailer - trailer);
552
553 // Now, deal with variable-length metalayers
554 int16_t nvlmetalayers = schunk->nvlmetalayers;
555 if (nvlmetalayers < 0 || nvlmetalayers > BLOSC2_MAX_METALAYERS) {
556 return -1;
557 }
558
559 // Make space for the header of metalayers (array marker, size, map of offsets)
560 trailer = realloc(trailer, (size_t) current_trailer_len + 1 + 1 + 2 + 1 + 2);
561 ptrailer = trailer + current_trailer_len;
562
563 // The msgpack header for the metalayers (array_marker, size, map of offsets, list of metalayers)
564 *ptrailer = 0x90 + 3; // array with 3 elements
565 ptrailer += 1;
566
567 int32_t tsize = (ptrailer - trailer);
568
569 // Size for the map (index) of metalayer offsets, including this uint16 size (to be filled out later on)
570 *ptrailer = 0xcd; // uint16
571 ptrailer += 1 + 2;
572
573 // Map (index) of offsets for optional metalayers
574 *ptrailer = 0xde; // map 16 with N keys
575 ptrailer += 1;
576 to_big(ptrailer, &nvlmetalayers, sizeof(nvlmetalayers));
577 ptrailer += sizeof(nvlmetalayers);
578 current_trailer_len = (int32_t)(ptrailer - trailer);
579 int32_t *offtodata = malloc(nvlmetalayers * sizeof(int32_t));
580 for (int nvlmetalayer = 0; nvlmetalayer < nvlmetalayers; nvlmetalayer++) {
581 if (frame == NULL) {
582 return -1;
583 }
584 blosc2_metalayer *vlmetalayer = schunk->vlmetalayers[nvlmetalayer];
585 uint8_t name_len = (uint8_t) strlen(vlmetalayer->name);
586 trailer = realloc(trailer, (size_t)current_trailer_len + 1 + name_len + 1 + 4);
587 ptrailer = trailer + current_trailer_len;
588 // Store the vlmetalayer
589 if (name_len >= (1U << 5U)) { // metalayer strings cannot be longer than 32 bytes
590 free(offtodata);
591 return -1;
592 }
593 *ptrailer = (uint8_t)0xa0 + name_len; // str
594 ptrailer += 1;
595 memcpy(ptrailer, vlmetalayer->name, name_len);
596 ptrailer += name_len;
597 // Space for storing the offset for the value of this vlmetalayer
598 *ptrailer = 0xd2; // int32
599 ptrailer += 1;
600 offtodata[nvlmetalayer] = (int32_t)(ptrailer - trailer);
601 ptrailer += 4;
602 current_trailer_len += 1 + name_len + 1 + 4;
603 }
604 int32_t tsize2 = (int32_t)(ptrailer - trailer);
605 if (tsize2 != current_trailer_len) { // sanity check
606 return -1;
607 }
608
609 // Map size + int16 size
610 if ((uint32_t) (tsize2 - tsize) >= (1U << 16U)) {
611 return -1;
612 }
613 uint16_t map_size = (uint16_t) (tsize2 - tsize);
614 to_big(trailer + 4, &map_size, sizeof(map_size));
615
616 // Make space for an (empty) array
617 tsize = (int32_t)(ptrailer - trailer);
618 trailer = realloc(trailer, (size_t) tsize + 2 + 1 + 2);
619 ptrailer = trailer + tsize;
620
621 // Now, store the values in an array
622 *ptrailer = 0xdc; // array 16 with N elements
623 ptrailer += 1;
624 to_big(ptrailer, &nvlmetalayers, sizeof(nvlmetalayers));
625 ptrailer += sizeof(nvlmetalayers);
626 current_trailer_len = (int32_t)(ptrailer - trailer);
627 for (int nvlmetalayer = 0; nvlmetalayer < nvlmetalayers; nvlmetalayer++) {
628 if (frame == NULL) {
629 return -1;
630 }
631 blosc2_metalayer *vlmetalayer = schunk->vlmetalayers[nvlmetalayer];
632 trailer = realloc(trailer, (size_t)current_trailer_len + 1 + 4 + vlmetalayer->content_len);
633 ptrailer = trailer + current_trailer_len;
634 // Store the serialized contents for this vlmetalayer
635 *ptrailer = 0xc6; // bin32
636 ptrailer += 1;
637 to_big(ptrailer, &(vlmetalayer->content_len), sizeof(vlmetalayer->content_len));
638 ptrailer += 4;
639 memcpy(ptrailer, vlmetalayer->content, vlmetalayer->content_len); // buffer, no need to swap
640 ptrailer += vlmetalayer->content_len;
641 // Update the offset now that we know it
642 to_big(trailer + offtodata[nvlmetalayer], ¤t_trailer_len, sizeof(current_trailer_len));
643 current_trailer_len += 1 + 4 + vlmetalayer->content_len;
644 }
645 free(offtodata);
646 tsize = (int32_t)(ptrailer - trailer);
647 if (tsize != current_trailer_len) { // sanity check
648 return -1;
649 }
650
651 trailer = realloc(trailer, (size_t)current_trailer_len + 23);
652 ptrailer = trailer + current_trailer_len;
653 trailer_len = (ptrailer - trailer) + 23;
654
655 // Trailer length
656 *ptrailer = 0xce; // uint32
657 ptrailer += 1;
658 to_big(ptrailer, &trailer_len, sizeof(uint32_t));
659 ptrailer += sizeof(uint32_t);
660 // Up to 16 bytes for frame fingerprint (using XXH3 included in https://github.com/Cyan4973/xxHash)
661 // Maybe someone would need 256-bit in the future, but for the time being 128-bit seems like a good tradeoff
662 *ptrailer = 0xd8; // fixext 16
663 ptrailer += 1;
664 *ptrailer = 0; // fingerprint type: 0 -> no fp; 1 -> 32-bit; 2 -> 64-bit; 3 -> 128-bit
665 ptrailer += 1;
666
667 // Remove call to memset when we compute an actual fingerprint
668 memset(ptrailer, 0, 16);
669 // Uncomment call to memcpy when we compute an actual fingerprint
670 // memcpy(ptrailer, xxh3_fingerprint, sizeof(xxh3_fingerprint));
671 ptrailer += 16;
672
673 // Sanity check
674 if (ptrailer - trailer != trailer_len) {
675 return BLOSC2_ERROR_DATA;
676 }
677
678 int32_t header_len;
679 int64_t frame_len;
680 int64_t nbytes;
681 int64_t cbytes;
682 int32_t blocksize;
683 int32_t chunksize;
684 int32_t nchunks;
685 int ret = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes,
686 &blocksize, &chunksize, &nchunks,
687 NULL, NULL, NULL, NULL, NULL, NULL,
688 frame->schunk->storage->io);
689 if (ret < 0) {
690 BLOSC_TRACE_ERROR("Unable to get meta info from frame.");
691 return ret;
692 }
693
694 int64_t trailer_offset = get_trailer_offset(frame, header_len, nbytes > 0);
695
696 if (trailer_offset < BLOSC_EXTENDED_HEADER_LENGTH) {
697 BLOSC_TRACE_ERROR("Unable to get trailer offset in frame.");
698 return BLOSC2_ERROR_READ_BUFFER;
699 }
700
701 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
702 if (io_cb == NULL) {
703 BLOSC_TRACE_ERROR("Error getting the input/output API");
704 return BLOSC2_ERROR_PLUGIN_IO;
705 }
706 // Update the trailer. As there are no internal offsets to the trailer section,
707 // and it is always at the end of the frame, we can just write (or overwrite) it
708 // at the end of the frame.
709 if (frame->cframe != NULL) {
710 frame->cframe = realloc(frame->cframe, (size_t)(trailer_offset + trailer_len));
711 if (frame->cframe == NULL) {
712 BLOSC_TRACE_ERROR("Cannot realloc space for the frame.");
713 return BLOSC2_ERROR_MEMORY_ALLOC;
714 }
715 memcpy(frame->cframe + trailer_offset, trailer, trailer_len);
716 }
717 else {
718 void* fp = NULL;
719 if (frame->sframe) {
720 fp = sframe_open_index(frame->urlpath, "rb+",
721 frame->schunk->storage->io);
722 }
723 else {
724 fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params);
725 }
726 if (fp == NULL) {
727 BLOSC_TRACE_ERROR("Cannot open the frame for reading and writing.");
728 return BLOSC2_ERROR_FILE_OPEN;
729 }
730 io_cb->seek(fp, trailer_offset, SEEK_SET);
731 int64_t wbytes = io_cb->write(trailer, 1, trailer_len, fp);
732 if (wbytes != (size_t)trailer_len) {
733 BLOSC_TRACE_ERROR("Cannot write the trailer length in trailer.");
734 return BLOSC2_ERROR_FILE_WRITE;
735 }
736 if (io_cb->truncate(fp, trailer_offset + trailer_len) != 0) {
737 BLOSC_TRACE_ERROR("Cannot truncate the frame.");
738 return BLOSC2_ERROR_FILE_TRUNCATE;
739 }
740 io_cb->close(fp);
741
742 }
743 free(trailer);
744
745 int rc = update_frame_len(frame, trailer_offset + trailer_len);
746 if (rc < 0) {
747 return rc;
748 }
749 frame->len = trailer_offset + trailer_len;
750 frame->trailer_len = trailer_len;
751
752 return 1;
753 }
754
755
756 // Remove a file:/// prefix
757 // This is a temporary workaround for allowing to use proper URLs for local files/dirs
normalize_urlpath(const char * urlpath)758 static char* normalize_urlpath(const char* urlpath) {
759 char* localpath = strstr(urlpath, "file:///");
760 if (localpath == urlpath) {
761 // There is a file:/// prefix. Get rid of it.
762 localpath += strlen("file:///");
763 }
764 else {
765 localpath = (char*)urlpath;
766 }
767 return localpath;
768 }
769
770
771 /* Initialize a frame out of a file */
frame_from_file(const char * urlpath,const blosc2_io * io)772 blosc2_frame_s* frame_from_file(const char* urlpath, const blosc2_io *io) {
773 // Get the length of the frame
774 uint8_t header[FRAME_HEADER_MINLEN];
775 uint8_t trailer[FRAME_TRAILER_MINLEN];
776
777 void* fp = NULL;
778 bool sframe = false;
779 struct stat path_stat;
780
781 urlpath = normalize_urlpath(urlpath);
782
783 if(stat(urlpath, &path_stat) < 0) {
784 BLOSC_TRACE_ERROR("Cannot get information about the path %s.", urlpath);
785 return NULL;
786 }
787
788 blosc2_io_cb *io_cb = blosc2_get_io_cb(io->id);
789 if (io_cb == NULL) {
790 BLOSC_TRACE_ERROR("Error getting the input/output API");
791 return NULL;
792 }
793
794 char* urlpath_cpy;
795 if (path_stat.st_mode & S_IFDIR) {
796 urlpath_cpy = malloc(strlen(urlpath) + 1);
797 strcpy(urlpath_cpy, urlpath);
798 char last_char = urlpath[strlen(urlpath) - 1];
799 if (last_char == '\\' || last_char == '/') {
800 urlpath_cpy[strlen(urlpath) - 1] = '\0';
801 }
802 else {
803 }
804 fp = sframe_open_index(urlpath_cpy, "rb", io);
805 sframe = true;
806 }
807 else {
808 urlpath_cpy = malloc(strlen(urlpath) + 1);
809 strcpy(urlpath_cpy, urlpath);
810 fp = io_cb->open(urlpath, "rb", io->params);
811 }
812 int64_t rbytes = io_cb->read(header, 1, FRAME_HEADER_MINLEN, fp);
813 if (rbytes != FRAME_HEADER_MINLEN) {
814 BLOSC_TRACE_ERROR("Cannot read from file '%s'.", urlpath);
815 io_cb->close(fp);
816 free(urlpath_cpy);
817 return NULL;
818 }
819 int64_t frame_len;
820 to_big(&frame_len, header + FRAME_LEN, sizeof(frame_len));
821
822 blosc2_frame_s* frame = calloc(1, sizeof(blosc2_frame_s));
823 frame->urlpath = urlpath_cpy;
824 frame->len = frame_len;
825 frame->sframe = sframe;
826
827 // Now, the trailer length
828 io_cb->seek(fp, frame_len - FRAME_TRAILER_MINLEN, SEEK_SET);
829 rbytes = io_cb->read(trailer, 1, FRAME_TRAILER_MINLEN, fp);
830 io_cb->close(fp);
831 if (rbytes != FRAME_TRAILER_MINLEN) {
832 BLOSC_TRACE_ERROR("Cannot read from file '%s'.", urlpath);
833 free(urlpath_cpy);
834 free(frame);
835 return NULL;
836 }
837 int trailer_offset = FRAME_TRAILER_MINLEN - FRAME_TRAILER_LEN_OFFSET;
838 if (trailer[trailer_offset - 1] != 0xce) {
839 free(urlpath_cpy);
840 free(frame);
841 return NULL;
842 }
843 uint32_t trailer_len;
844 to_big(&trailer_len, trailer + trailer_offset, sizeof(trailer_len));
845 frame->trailer_len = trailer_len;
846
847 return frame;
848 }
849
850
851 /* Initialize a frame out of a contiguous frame buffer */
frame_from_cframe(uint8_t * cframe,int64_t len,bool copy)852 blosc2_frame_s* frame_from_cframe(uint8_t *cframe, int64_t len, bool copy) {
853 // Get the length of the frame
854 const uint8_t* header = cframe;
855 int64_t frame_len;
856 if (len < FRAME_HEADER_MINLEN) {
857 return NULL;
858 }
859
860 from_big(&frame_len, header + FRAME_LEN, sizeof(frame_len));
861 if (frame_len != len) { // sanity check
862 return NULL;
863 }
864
865 blosc2_frame_s* frame = calloc(1, sizeof(blosc2_frame_s));
866 frame->len = frame_len;
867
868 // Now, the trailer length
869 const uint8_t* trailer = cframe + frame_len - FRAME_TRAILER_MINLEN;
870 int trailer_offset = FRAME_TRAILER_MINLEN - FRAME_TRAILER_LEN_OFFSET;
871 if (trailer[trailer_offset - 1] != 0xce) {
872 free(frame);
873 return NULL;
874 }
875 uint32_t trailer_len;
876 from_big(&trailer_len, trailer + trailer_offset, sizeof(trailer_len));
877 frame->trailer_len = trailer_len;
878
879 if (copy) {
880 frame->cframe = malloc((size_t)len);
881 memcpy(frame->cframe, cframe, (size_t)len);
882 }
883 else {
884 frame->cframe = cframe;
885 frame->avoid_cframe_free = true;
886 }
887
888 return frame;
889 }
890
891
892 /* Create a frame out of a super-chunk. */
frame_from_schunk(blosc2_schunk * schunk,blosc2_frame_s * frame)893 int64_t frame_from_schunk(blosc2_schunk *schunk, blosc2_frame_s *frame) {
894 int32_t nchunks = schunk->nchunks;
895 int64_t cbytes = schunk->cbytes;
896 int32_t chunk_cbytes;
897 int32_t chunk_nbytes;
898 void* fp = NULL;
899 int rc;
900
901 uint8_t* h2 = new_header_frame(schunk, frame);
902 if (h2 == NULL) {
903 return BLOSC2_ERROR_DATA;
904 }
905 uint32_t h2len;
906 from_big(&h2len, h2 + FRAME_HEADER_LEN, sizeof(h2len));
907 // Build the offsets chunk
908 int32_t chunksize = -1;
909 int32_t off_cbytes = 0;
910 uint64_t coffset = 0;
911 int32_t off_nbytes = nchunks * sizeof(int64_t);
912 uint64_t* data_tmp = malloc(off_nbytes);
913 bool needs_free = false;
914 for (int i = 0; i < nchunks; i++) {
915 uint8_t* data_chunk;
916 data_chunk = schunk->data[i];
917 rc = blosc2_cbuffer_sizes(data_chunk, &chunk_nbytes, &chunk_cbytes, NULL);
918 if (rc < 0) {
919 return rc;
920 }
921 data_tmp[i] = coffset;
922 coffset += chunk_cbytes;
923 int32_t chunksize_ = chunk_nbytes;
924 if (i == 0) {
925 chunksize = chunksize_;
926 }
927 else if (chunksize != chunksize_) {
928 // Variable size // TODO: update flags for this (or do not use them at all)
929 chunksize = 0;
930 }
931 if (needs_free) {
932 free(data_chunk);
933 }
934 }
935 if ((int64_t)coffset != cbytes) {
936 free(data_tmp);
937 return BLOSC2_ERROR_DATA;
938 }
939 uint8_t *off_chunk = NULL;
940 if (nchunks > 0) {
941 // Compress the chunk of offsets
942 off_chunk = malloc(off_nbytes + BLOSC_MAX_OVERHEAD);
943 blosc2_context *cctx = blosc2_create_cctx(BLOSC2_CPARAMS_DEFAULTS);
944 cctx->typesize = sizeof(int64_t);
945 off_cbytes = blosc2_compress_ctx(cctx, data_tmp, off_nbytes, off_chunk,
946 off_nbytes + BLOSC_MAX_OVERHEAD);
947 blosc2_free_ctx(cctx);
948 if (off_cbytes < 0) {
949 free(off_chunk);
950 free(h2);
951 return off_cbytes;
952 }
953 }
954 else {
955 off_cbytes = 0;
956 }
957 free(data_tmp);
958
959 // Now that we know them, fill the chunksize and frame length in header
960 to_big(h2 + FRAME_CHUNKSIZE, &chunksize, sizeof(chunksize));
961 frame->len = h2len + cbytes + off_cbytes + FRAME_TRAILER_MINLEN;
962 if (frame->sframe) {
963 frame->len = h2len + off_cbytes + FRAME_TRAILER_MINLEN;
964 }
965 int64_t tbytes = frame->len;
966 to_big(h2 + FRAME_LEN, &tbytes, sizeof(tbytes));
967
968 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
969 if (io_cb == NULL) {
970 BLOSC_TRACE_ERROR("Error getting the input/output API");
971 return BLOSC2_ERROR_PLUGIN_IO;
972 }
973
974 // Create the frame and put the header at the beginning
975 if (frame->urlpath == NULL) {
976 frame->cframe = malloc((size_t)frame->len);
977 memcpy(frame->cframe, h2, h2len);
978 }
979 else {
980 if (frame->sframe) {
981 fp = sframe_open_index(frame->urlpath, "wb",
982 frame->schunk->storage->io);
983 }
984 else {
985 fp = io_cb->open(frame->urlpath, "wb", frame->schunk->storage->io->params);
986 }
987 io_cb->write(h2, h2len, 1, fp);
988 }
989 free(h2);
990
991 // Fill the frame with the actual data chunks
992 if (!frame->sframe) {
993 coffset = 0;
994 for (int i = 0; i < nchunks; i++) {
995 uint8_t* data_chunk = schunk->data[i];
996 rc = blosc2_cbuffer_sizes(data_chunk, NULL, &chunk_cbytes, NULL);
997 if (rc < 0) {
998 return rc;
999 }
1000 if (frame->urlpath == NULL) {
1001 memcpy(frame->cframe + h2len + coffset, data_chunk, (size_t)chunk_cbytes);
1002 } else {
1003 io_cb->write(data_chunk, chunk_cbytes, 1, fp);
1004 }
1005 coffset += chunk_cbytes;
1006 }
1007 if ((int64_t)coffset != cbytes) {
1008 return BLOSC2_ERROR_FAILURE;
1009 }
1010 }
1011
1012 // Copy the offsets chunk at the end of the frame
1013 if (frame->urlpath == NULL) {
1014 memcpy(frame->cframe + h2len + cbytes, off_chunk, off_cbytes);
1015 }
1016 else {
1017 io_cb->write(off_chunk, (size_t)off_cbytes, 1, fp);
1018 io_cb->close(fp);
1019 }
1020 free(off_chunk);
1021 rc = frame_update_trailer(frame, schunk);
1022 if (rc < 0) {
1023 return rc;
1024 }
1025
1026 return frame->len;
1027 }
1028
1029
1030 // Get the compressed data offsets
get_coffsets(blosc2_frame_s * frame,int32_t header_len,int64_t cbytes,int32_t nchunks,int32_t * off_cbytes)1031 uint8_t* get_coffsets(blosc2_frame_s *frame, int32_t header_len, int64_t cbytes,
1032 int32_t nchunks, int32_t *off_cbytes) {
1033 int32_t chunk_cbytes;
1034 int rc;
1035
1036 if (frame->coffsets != NULL) {
1037 if (off_cbytes != NULL) {
1038 rc = blosc2_cbuffer_sizes(frame->coffsets, NULL, &chunk_cbytes, NULL);
1039 if (rc < 0) {
1040 return NULL;
1041 }
1042 *off_cbytes = (int32_t)chunk_cbytes;
1043 }
1044 return frame->coffsets;
1045 }
1046 if (frame->cframe != NULL) {
1047 int64_t off_pos = header_len;
1048 if (cbytes < INT64_MAX - header_len) {
1049 off_pos += cbytes;
1050 }
1051 // Check that there is enough room to read Blosc header
1052 if (off_pos < 0 || off_pos > INT64_MAX - BLOSC_EXTENDED_HEADER_LENGTH ||
1053 off_pos + BLOSC_EXTENDED_HEADER_LENGTH > frame->len) {
1054 BLOSC_TRACE_ERROR("Cannot read the offsets outside of frame boundary.");
1055 return NULL;
1056 }
1057 // For in-memory frames, the coffset is just one pointer away
1058 uint8_t* off_start = frame->cframe + off_pos;
1059 if (off_cbytes != NULL) {
1060 int32_t chunk_nbytes;
1061 int32_t chunk_blocksize;
1062 rc = blosc2_cbuffer_sizes(off_start, &chunk_nbytes, &chunk_cbytes, &chunk_blocksize);
1063 if (rc < 0) {
1064 return NULL;
1065 }
1066 *off_cbytes = (int32_t)chunk_cbytes;
1067 if (*off_cbytes < 0 || off_pos + *off_cbytes > frame->len) {
1068 BLOSC_TRACE_ERROR("Cannot read the cbytes outside of frame boundary.");
1069 return NULL;
1070 }
1071 if (chunk_nbytes != nchunks * sizeof(int64_t)) {
1072 BLOSC_TRACE_ERROR("The number of chunks in offset idx "
1073 "does not match the ones in the header frame.");
1074 return NULL;
1075 }
1076
1077 }
1078 return off_start;
1079 }
1080
1081 int64_t trailer_offset = get_trailer_offset(frame, header_len, true);
1082
1083 if (trailer_offset < BLOSC_EXTENDED_HEADER_LENGTH || trailer_offset + FRAME_TRAILER_MINLEN > frame->len) {
1084 BLOSC_TRACE_ERROR("Cannot read the trailer out of the frame.");
1085 return NULL;
1086 }
1087
1088 int32_t coffsets_cbytes;
1089 if (frame->sframe) {
1090 coffsets_cbytes = (int32_t)(trailer_offset - (header_len + 0));
1091 }
1092 else {
1093 coffsets_cbytes = (int32_t)(trailer_offset - (header_len + cbytes));
1094 }
1095
1096 if (off_cbytes != NULL) {
1097 *off_cbytes = coffsets_cbytes;
1098 }
1099
1100 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
1101 if (io_cb == NULL) {
1102 BLOSC_TRACE_ERROR("Error getting the input/output API");
1103 return NULL;
1104 }
1105
1106 void* fp = NULL;
1107 uint8_t* coffsets = malloc((size_t)coffsets_cbytes);
1108 if (frame->sframe) {
1109 fp = sframe_open_index(frame->urlpath, "rb",
1110 frame->schunk->storage->io);
1111 io_cb->seek(fp, header_len + 0, SEEK_SET);
1112 }
1113 else {
1114 fp = io_cb->open(frame->urlpath, "rb", frame->schunk->storage->io->params);
1115 io_cb->seek(fp, header_len + cbytes, SEEK_SET);
1116 }
1117 int64_t rbytes = io_cb->read(coffsets, 1, (size_t)coffsets_cbytes, fp);
1118 io_cb->close(fp);
1119 if (rbytes != coffsets_cbytes) {
1120 BLOSC_TRACE_ERROR("Cannot read the offsets out of the frame.");
1121 free(coffsets);
1122 return NULL;
1123 }
1124 frame->coffsets = coffsets;
1125 return coffsets;
1126 }
1127
1128
frame_update_header(blosc2_frame_s * frame,blosc2_schunk * schunk,bool new)1129 int frame_update_header(blosc2_frame_s* frame, blosc2_schunk* schunk, bool new) {
1130 uint8_t* framep = frame->cframe;
1131 uint8_t header[FRAME_HEADER_MINLEN];
1132
1133 if (frame->len <= 0) {
1134 return BLOSC2_ERROR_INVALID_PARAM;
1135 }
1136
1137 if (new && schunk->cbytes > 0) {
1138 BLOSC_TRACE_ERROR("New metalayers cannot be added after actual data "
1139 "has been appended.");
1140 return BLOSC2_ERROR_INVALID_PARAM;
1141 }
1142
1143 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
1144 if (io_cb == NULL) {
1145 BLOSC_TRACE_ERROR("Error getting the input/output API");
1146 return BLOSC2_ERROR_PLUGIN_IO;
1147 }
1148
1149 if (frame->cframe == NULL) {
1150 int64_t rbytes = 0;
1151 void* fp = NULL;
1152 if (frame->sframe) {
1153 fp = sframe_open_index(frame->urlpath, "rb+",
1154 frame->schunk->storage->io);
1155 }
1156 else {
1157 fp = io_cb->open(frame->urlpath, "rb", frame->schunk->storage->io->params);
1158 }
1159 if (fp != NULL) {
1160 rbytes = io_cb->read(header, 1, FRAME_HEADER_MINLEN, fp);
1161 io_cb->close(fp);
1162 }
1163 (void) rbytes;
1164 if (rbytes != FRAME_HEADER_MINLEN) {
1165 return BLOSC2_ERROR_FILE_WRITE;
1166 }
1167 framep = header;
1168 }
1169 uint32_t prev_h2len;
1170 from_big(&prev_h2len, framep + FRAME_HEADER_LEN, sizeof(prev_h2len));
1171
1172 // Build a new header
1173 uint8_t* h2 = new_header_frame(schunk, frame);
1174 uint32_t h2len;
1175 from_big(&h2len, h2 + FRAME_HEADER_LEN, sizeof(h2len));
1176
1177 // The frame length is outdated when adding a new metalayer, so update it
1178 if (new) {
1179 int64_t frame_len = h2len; // at adding time, we only have to worry of the header for now
1180 to_big(h2 + FRAME_LEN, &frame_len, sizeof(frame_len));
1181 frame->len = frame_len;
1182 }
1183
1184 if (!new && prev_h2len != h2len) {
1185 BLOSC_TRACE_ERROR("The new metalayer sizes should be equal the existing ones.");
1186 return BLOSC2_ERROR_DATA;
1187 }
1188
1189 void* fp = NULL;
1190 if (frame->cframe == NULL) {
1191 // Write updated header down to file
1192 if (frame->sframe) {
1193 fp = sframe_open_index(frame->urlpath, "rb+",
1194 frame->schunk->storage->io);
1195 }
1196 else {
1197 fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params);
1198 }
1199 if (fp != NULL) {
1200 io_cb->write(h2, h2len, 1, fp);
1201 io_cb->close(fp);
1202 }
1203 }
1204 else {
1205 if (new) {
1206 frame->cframe = realloc(frame->cframe, h2len);
1207 }
1208 memcpy(frame->cframe, h2, h2len);
1209 }
1210 free(h2);
1211
1212 return 1;
1213 }
1214
1215
get_meta_from_header(blosc2_frame_s * frame,blosc2_schunk * schunk,uint8_t * header,int32_t header_len)1216 static int get_meta_from_header(blosc2_frame_s* frame, blosc2_schunk* schunk, uint8_t* header,
1217 int32_t header_len) {
1218 int64_t header_pos = FRAME_IDX_SIZE;
1219
1220 // Get the size for the index of metalayers
1221 uint16_t idx_size;
1222 header_pos += sizeof(idx_size);
1223 if (header_len < header_pos) {
1224 return BLOSC2_ERROR_READ_BUFFER;
1225 }
1226 from_big(&idx_size, header + FRAME_IDX_SIZE, sizeof(idx_size));
1227
1228 // Get the actual index of metalayers
1229 uint8_t* metalayers_idx = header + FRAME_IDX_SIZE + 2;
1230 header_pos += 1;
1231 if (header_len < header_pos) {
1232 return BLOSC2_ERROR_READ_BUFFER;
1233 }
1234 if (metalayers_idx[0] != 0xde) { // sanity check
1235 return BLOSC2_ERROR_DATA;
1236 }
1237 uint8_t* idxp = metalayers_idx + 1;
1238 uint16_t nmetalayers;
1239 header_pos += sizeof(nmetalayers);
1240 if (header_len < header_pos) {
1241 return BLOSC2_ERROR_READ_BUFFER;
1242 }
1243 from_big(&nmetalayers, idxp, sizeof(uint16_t));
1244 idxp += 2;
1245 if (nmetalayers < 0 || nmetalayers > BLOSC2_MAX_METALAYERS) {
1246 return BLOSC2_ERROR_DATA;
1247 }
1248 schunk->nmetalayers = nmetalayers;
1249
1250 // Populate the metalayers and its serialized values
1251 for (int nmetalayer = 0; nmetalayer < nmetalayers; nmetalayer++) {
1252 header_pos += 1;
1253 if (header_len < header_pos) {
1254 return BLOSC2_ERROR_READ_BUFFER;
1255 }
1256 if ((*idxp & 0xe0u) != 0xa0u) { // sanity check
1257 return BLOSC2_ERROR_DATA;
1258 }
1259 blosc2_metalayer* metalayer = calloc(sizeof(blosc2_metalayer), 1);
1260 schunk->metalayers[nmetalayer] = metalayer;
1261
1262 // Populate the metalayer string
1263 int8_t nslen = *idxp & (uint8_t)0x1F;
1264 idxp += 1;
1265 header_pos += nslen;
1266 if (header_len < header_pos) {
1267 return BLOSC2_ERROR_READ_BUFFER;
1268 }
1269 char* ns = malloc((size_t)nslen + 1);
1270 memcpy(ns, idxp, nslen);
1271 ns[nslen] = '\0';
1272 idxp += nslen;
1273 metalayer->name = ns;
1274
1275 // Populate the serialized value for this metalayer
1276 // Get the offset
1277 header_pos += 1;
1278 if (header_len < header_pos) {
1279 return BLOSC2_ERROR_READ_BUFFER;
1280 }
1281 if ((*idxp & 0xffu) != 0xd2u) { // sanity check
1282 return BLOSC2_ERROR_DATA;
1283 }
1284 idxp += 1;
1285 int32_t offset;
1286 header_pos += sizeof(offset);
1287 if (header_len < header_pos) {
1288 return BLOSC2_ERROR_READ_BUFFER;
1289 }
1290 from_big(&offset, idxp, sizeof(offset));
1291 idxp += 4;
1292 if (offset < 0 || offset >= header_len) {
1293 // Offset is less than zero or exceeds header length
1294 return BLOSC2_ERROR_DATA;
1295 }
1296 // Go to offset and see if we have the correct marker
1297 uint8_t* content_marker = header + offset;
1298 if (header_len < (size_t)offset + 1 + 4) {
1299 return BLOSC2_ERROR_READ_BUFFER;
1300 }
1301 if (*content_marker != 0xc6) {
1302 return BLOSC2_ERROR_DATA;
1303 }
1304
1305 // Read the size of the content
1306 int32_t content_len;
1307 from_big(&content_len, content_marker + 1, sizeof(content_len));
1308 if (content_len < 0) {
1309 return BLOSC2_ERROR_DATA;
1310 }
1311 metalayer->content_len = content_len;
1312
1313 // Finally, read the content
1314 if (header_len < (size_t)offset + 1 + 4 + content_len) {
1315 return BLOSC2_ERROR_READ_BUFFER;
1316 }
1317 char* content = malloc((size_t)content_len);
1318 memcpy(content, content_marker + 1 + 4, (size_t)content_len);
1319 metalayer->content = (uint8_t*)content;
1320 }
1321
1322 return 1;
1323 }
1324
frame_get_metalayers(blosc2_frame_s * frame,blosc2_schunk * schunk)1325 int frame_get_metalayers(blosc2_frame_s* frame, blosc2_schunk* schunk) {
1326 int32_t header_len;
1327 int64_t frame_len;
1328 int64_t nbytes;
1329 int64_t cbytes;
1330 int32_t blocksize;
1331 int32_t chunksize;
1332 int32_t nchunks;
1333 int ret = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes,
1334 &blocksize, &chunksize, &nchunks,
1335 NULL, NULL, NULL, NULL, NULL, NULL,
1336 schunk->storage->io);
1337 if (ret < 0) {
1338 BLOSC_TRACE_ERROR("Unable to get the header info from frame.");
1339 return ret;
1340 }
1341
1342 // Get the header
1343 uint8_t* header = NULL;
1344 if (frame->cframe != NULL) {
1345 header = frame->cframe;
1346 } else {
1347 int64_t rbytes = 0;
1348 header = malloc(header_len);
1349 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
1350 if (io_cb == NULL) {
1351 BLOSC_TRACE_ERROR("Error getting the input/output API");
1352 return BLOSC2_ERROR_PLUGIN_IO;
1353 }
1354
1355 void* fp = NULL;
1356 if (frame->sframe) {
1357 fp = sframe_open_index(frame->urlpath, "rb",
1358 frame->schunk->storage->io);
1359 }
1360 else {
1361 fp = io_cb->open(frame->urlpath, "rb", frame->schunk->storage->io->params);
1362 }
1363 if (fp != NULL) {
1364 rbytes = io_cb->read(header, 1, header_len, fp);
1365 io_cb->close(fp);
1366 }
1367 if (rbytes != (size_t) header_len) {
1368 BLOSC_TRACE_ERROR("Cannot access the header out of the frame.");
1369 free(header);
1370 return BLOSC2_ERROR_FILE_READ;
1371 }
1372 }
1373
1374 ret = get_meta_from_header(frame, schunk, header, header_len);
1375
1376 if (frame->cframe == NULL) {
1377 free(header);
1378 }
1379
1380 return ret;
1381 }
1382
get_vlmeta_from_trailer(blosc2_frame_s * frame,blosc2_schunk * schunk,uint8_t * trailer,int32_t trailer_len)1383 static int get_vlmeta_from_trailer(blosc2_frame_s* frame, blosc2_schunk* schunk, uint8_t* trailer,
1384 int32_t trailer_len) {
1385
1386 int64_t trailer_pos = FRAME_TRAILER_VLMETALAYERS + 2;
1387 uint8_t* idxp = trailer + trailer_pos;
1388
1389 // Get the size for the index of metalayers
1390 trailer_pos += 2;
1391 if (trailer_len < trailer_pos) {
1392 return BLOSC2_ERROR_READ_BUFFER;
1393 }
1394 uint16_t idx_size;
1395 from_big(&idx_size, idxp, sizeof(idx_size));
1396 idxp += 2;
1397
1398 trailer_pos += 1;
1399 // Get the actual index of metalayers
1400 if (trailer_len < trailer_pos) {
1401 return BLOSC2_ERROR_READ_BUFFER;
1402 }
1403 if (idxp[0] != 0xde) { // sanity check
1404 return BLOSC2_ERROR_DATA;
1405 }
1406 idxp += 1;
1407
1408 uint16_t nmetalayers;
1409 trailer_pos += sizeof(nmetalayers);
1410 if (trailer_len < trailer_pos) {
1411 return BLOSC2_ERROR_READ_BUFFER;
1412 }
1413 from_big(&nmetalayers, idxp, sizeof(uint16_t));
1414 idxp += 2;
1415 if (nmetalayers < 0 || nmetalayers > BLOSC2_MAX_VLMETALAYERS) {
1416 return BLOSC2_ERROR_DATA;
1417 }
1418 schunk->nvlmetalayers = nmetalayers;
1419
1420 // Populate the metalayers and its serialized values
1421 for (int nmetalayer = 0; nmetalayer < nmetalayers; nmetalayer++) {
1422 trailer_pos += 1;
1423 if (trailer_len < trailer_pos) {
1424 return BLOSC2_ERROR_READ_BUFFER;
1425 }
1426 if ((*idxp & 0xe0u) != 0xa0u) { // sanity check
1427 return BLOSC2_ERROR_DATA;
1428 }
1429 blosc2_metalayer* metalayer = calloc(sizeof(blosc2_metalayer), 1);
1430 schunk->vlmetalayers[nmetalayer] = metalayer;
1431
1432 // Populate the metalayer string
1433 int8_t nslen = *idxp & (uint8_t)0x1F;
1434 idxp += 1;
1435 trailer_pos += nslen;
1436 if (trailer_len < trailer_pos) {
1437 return BLOSC2_ERROR_READ_BUFFER;
1438 }
1439 char* ns = malloc((size_t)nslen + 1);
1440 memcpy(ns, idxp, nslen);
1441 ns[nslen] = '\0';
1442 idxp += nslen;
1443 metalayer->name = ns;
1444
1445 // Populate the serialized value for this metalayer
1446 // Get the offset
1447 trailer_pos += 1;
1448 if (trailer_len < trailer_pos) {
1449 return BLOSC2_ERROR_READ_BUFFER;
1450 }
1451 if ((*idxp & 0xffu) != 0xd2u) { // sanity check
1452 return BLOSC2_ERROR_DATA;
1453 }
1454 idxp += 1;
1455 int32_t offset;
1456 trailer_pos += sizeof(offset);
1457 if (trailer_len < trailer_pos) {
1458 return BLOSC2_ERROR_READ_BUFFER;
1459 }
1460 from_big(&offset, idxp, sizeof(offset));
1461 idxp += 4;
1462 if (offset < 0 || offset >= trailer_len) {
1463 // Offset is less than zero or exceeds trailer length
1464 return BLOSC2_ERROR_DATA;
1465 }
1466 // Go to offset and see if we have the correct marker
1467 uint8_t* content_marker = trailer + offset;
1468 if (trailer_len < (size_t)offset + 1 + 4) {
1469 return BLOSC2_ERROR_READ_BUFFER;
1470 }
1471 if (*content_marker != 0xc6) {
1472 return BLOSC2_ERROR_DATA;
1473 }
1474
1475 // Read the size of the content
1476 int32_t content_len;
1477 from_big(&content_len, content_marker + 1, sizeof(content_len));
1478 if (content_len < 0) {
1479 return BLOSC2_ERROR_DATA;
1480 }
1481 metalayer->content_len = content_len;
1482
1483 // Finally, read the content
1484 if (trailer_len < (size_t)offset + 1 + 4 + content_len) {
1485 return BLOSC2_ERROR_READ_BUFFER;
1486 }
1487 char* content = malloc((size_t)content_len);
1488 memcpy(content, content_marker + 1 + 4, (size_t)content_len);
1489 metalayer->content = (uint8_t*)content;
1490 }
1491 return 1;
1492 }
1493
frame_get_vlmetalayers(blosc2_frame_s * frame,blosc2_schunk * schunk)1494 int frame_get_vlmetalayers(blosc2_frame_s* frame, blosc2_schunk* schunk) {
1495 int32_t header_len;
1496 int64_t frame_len;
1497 int64_t nbytes;
1498 int64_t cbytes;
1499 int32_t blocksize;
1500 int32_t chunksize;
1501 int32_t nchunks;
1502 int ret = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes,
1503 &blocksize, &chunksize, &nchunks,
1504 NULL, NULL, NULL, NULL, NULL, NULL,
1505 schunk->storage->io);
1506 if (ret < 0) {
1507 BLOSC_TRACE_ERROR("Unable to get the trailer info from frame.");
1508 return ret;
1509 }
1510
1511 int64_t trailer_offset = get_trailer_offset(frame, header_len, nbytes > 0);
1512 int32_t trailer_len = frame->trailer_len;
1513
1514 if (trailer_offset < BLOSC_EXTENDED_HEADER_LENGTH || trailer_offset + trailer_len > frame->len) {
1515 BLOSC_TRACE_ERROR("Cannot access the trailer out of the frame.");
1516 return BLOSC2_ERROR_READ_BUFFER;
1517 }
1518
1519 // Get the trailer
1520 uint8_t* trailer = NULL;
1521 if (frame->cframe != NULL) {
1522 trailer = frame->cframe + trailer_offset;
1523 } else {
1524 int64_t rbytes = 0;
1525 trailer = malloc(trailer_len);
1526
1527 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
1528 if (io_cb == NULL) {
1529 BLOSC_TRACE_ERROR("Error getting the input/output API");
1530 return BLOSC2_ERROR_PLUGIN_IO;
1531 }
1532
1533 void* fp = NULL;
1534 if (frame->sframe) {
1535 char* eframe_name = malloc(strlen(frame->urlpath) + strlen("/chunks.b2frame") + 1);
1536 sprintf(eframe_name, "%s/chunks.b2frame", frame->urlpath);
1537 fp = io_cb->open(eframe_name, "rb", frame->schunk->storage->io->params);
1538 free(eframe_name);
1539 }
1540 else {
1541 fp = io_cb->open(frame->urlpath, "rb", frame->schunk->storage->io->params);
1542 }
1543 if (fp != NULL) {
1544 io_cb->seek(fp, trailer_offset, SEEK_SET);
1545 rbytes = io_cb->read(trailer, 1, trailer_len, fp);
1546 io_cb->close(fp);
1547 }
1548 if (rbytes != (size_t) trailer_len) {
1549 BLOSC_TRACE_ERROR("Cannot access the trailer out of the fileframe.");
1550 free(trailer);
1551 return BLOSC2_ERROR_FILE_READ;
1552 }
1553 }
1554
1555 ret = get_vlmeta_from_trailer(frame, schunk, trailer, trailer_len);
1556
1557 if (frame->cframe == NULL) {
1558 free(trailer);
1559 }
1560
1561 return ret;
1562 }
1563
1564
get_new_storage(const blosc2_storage * storage,const blosc2_cparams * cdefaults,const blosc2_dparams * ddefaults,const blosc2_io * iodefaults)1565 blosc2_storage* get_new_storage(const blosc2_storage* storage,
1566 const blosc2_cparams* cdefaults,
1567 const blosc2_dparams* ddefaults,
1568 const blosc2_io* iodefaults) {
1569
1570 blosc2_storage* new_storage = (blosc2_storage*)calloc(1, sizeof(blosc2_storage));
1571 memcpy(new_storage, storage, sizeof(blosc2_storage));
1572 if (storage->urlpath != NULL) {
1573 char* urlpath = normalize_urlpath(storage->urlpath);
1574 new_storage->urlpath = malloc(strlen(urlpath) + 1);
1575 strcpy(new_storage->urlpath, urlpath);
1576 }
1577
1578 // cparams
1579 blosc2_cparams* cparams = malloc(sizeof(blosc2_cparams));
1580 if (storage->cparams != NULL) {
1581 memcpy(cparams, storage->cparams, sizeof(blosc2_cparams));
1582 } else {
1583 memcpy(cparams, cdefaults, sizeof(blosc2_cparams));
1584 }
1585 new_storage->cparams = cparams;
1586
1587 // dparams
1588 blosc2_dparams* dparams = malloc(sizeof(blosc2_dparams));
1589 if (storage->dparams != NULL) {
1590 memcpy(dparams, storage->dparams, sizeof(blosc2_dparams));
1591 }
1592 else {
1593 memcpy(dparams, ddefaults, sizeof(blosc2_dparams));
1594 }
1595 new_storage->dparams = dparams;
1596
1597 // iodefaults
1598 blosc2_io* udio = malloc(sizeof(blosc2_io));
1599 if (storage->io != NULL) {
1600 memcpy(udio, storage->io, sizeof(blosc2_io));
1601 }
1602 else {
1603 memcpy(udio, iodefaults, sizeof(blosc2_io));
1604 }
1605 new_storage->io = udio;
1606
1607 return new_storage;
1608 }
1609
1610
1611 /* Get a super-chunk out of a frame */
frame_to_schunk(blosc2_frame_s * frame,bool copy,const blosc2_io * udio)1612 blosc2_schunk* frame_to_schunk(blosc2_frame_s* frame, bool copy, const blosc2_io *udio) {
1613 int32_t header_len;
1614 int64_t frame_len;
1615 int rc;
1616 blosc2_schunk* schunk = calloc(1, sizeof(blosc2_schunk));
1617 schunk->frame = (blosc2_frame*)frame;
1618 frame->schunk = schunk;
1619
1620 rc = get_header_info(frame, &header_len, &frame_len, &schunk->nbytes,
1621 &schunk->cbytes, &schunk->blocksize,
1622 &schunk->chunksize, &schunk->nchunks, &schunk->typesize,
1623 &schunk->compcode, &schunk->compcode_meta, &schunk->clevel, schunk->filters,
1624 schunk->filters_meta, udio);
1625 if (rc < 0) {
1626 BLOSC_TRACE_ERROR("Unable to get meta info from frame.");
1627 blosc2_schunk_free(schunk);
1628 return NULL;
1629 }
1630 int32_t nchunks = schunk->nchunks;
1631 int64_t nbytes = schunk->nbytes;
1632 (void) nbytes;
1633 int64_t cbytes = schunk->cbytes;
1634
1635 // Compression and decompression contexts
1636 blosc2_cparams *cparams;
1637 blosc2_schunk_get_cparams(schunk, &cparams);
1638 schunk->cctx = blosc2_create_cctx(*cparams);
1639 blosc2_dparams *dparams;
1640 blosc2_schunk_get_dparams(schunk, &dparams);
1641 schunk->dctx = blosc2_create_dctx(*dparams);
1642 blosc2_storage storage = {.contiguous = copy ? false : true};
1643 schunk->storage = get_new_storage(&storage, cparams, dparams, udio);
1644 free(cparams);
1645 free(dparams);
1646 if (!copy) {
1647 goto out;
1648 }
1649
1650 // We are not attached to a frame anymore
1651 schunk->frame = NULL;
1652 frame->schunk = NULL;
1653
1654 if (nchunks == 0) {
1655 goto out;
1656 }
1657
1658 // Get the compressed offsets
1659 int32_t coffsets_cbytes = 0;
1660 uint8_t* coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &coffsets_cbytes);
1661 if (coffsets == NULL) {
1662 blosc2_schunk_free(schunk);
1663 BLOSC_TRACE_ERROR("Cannot get the offsets for the frame.");
1664 return NULL;
1665 }
1666
1667 // Decompress offsets
1668 blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS;
1669 blosc2_context *dctx = blosc2_create_dctx(off_dparams);
1670 int64_t* offsets = (int64_t *) malloc((size_t)nchunks * sizeof(int64_t));
1671 int32_t off_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes,
1672 offsets, nchunks * sizeof(int64_t));
1673 blosc2_free_ctx(dctx);
1674 if (off_nbytes < 0) {
1675 free(offsets);
1676 blosc2_schunk_free(schunk);
1677 BLOSC_TRACE_ERROR("Cannot decompress the offsets chunk.");
1678 return NULL;
1679 }
1680
1681 // We want the contiguous schunk, so create the actual data chunks (and, while doing this,
1682 // get a guess at the blocksize used in this frame)
1683 int64_t acc_nbytes = 0;
1684 int64_t acc_cbytes = 0;
1685 int32_t blocksize = 0;
1686 int32_t chunk_nbytes;
1687 int32_t chunk_cbytes;
1688 int32_t chunk_blocksize;
1689 size_t prev_alloc = BLOSC_EXTENDED_HEADER_LENGTH;
1690 uint8_t* data_chunk = NULL;
1691 bool needs_free = false;
1692 const blosc2_io_cb *io_cb = blosc2_get_io_cb(udio->id);
1693 if (io_cb == NULL) {
1694 blosc2_schunk_free(schunk);
1695 BLOSC_TRACE_ERROR("Error getting the input/output API");
1696 return NULL;
1697 }
1698
1699 void* fp = NULL;
1700 if (frame->cframe == NULL) {
1701 data_chunk = malloc((size_t)prev_alloc);
1702 needs_free = true;
1703 if (!frame->sframe) {
1704 // If not the chunks won't be in the frame
1705 fp = io_cb->open(frame->urlpath, "rb", udio->params);
1706 if (fp == NULL) {
1707 rc = BLOSC2_ERROR_FILE_OPEN;
1708 goto end;
1709 }
1710 }
1711 }
1712 schunk->data = malloc(nchunks * sizeof(void*));
1713 for (int i = 0; i < nchunks; i++) {
1714 if (frame->cframe != NULL) {
1715 data_chunk = frame->cframe + header_len + offsets[i];
1716 needs_free = false;
1717 rc = blosc2_cbuffer_sizes(data_chunk, NULL, &chunk_cbytes, NULL);
1718 if (rc < 0) {
1719 break;
1720 }
1721 }
1722 else {
1723 int64_t rbytes;
1724 if (frame->sframe) {
1725 if (needs_free) {
1726 free(data_chunk);
1727 }
1728 rbytes = frame_get_lazychunk(frame, offsets[i], &data_chunk, &needs_free);
1729 }
1730 else {
1731 io_cb->seek(fp, header_len + offsets[i], SEEK_SET);
1732 rbytes = io_cb->read(data_chunk, 1, BLOSC_EXTENDED_HEADER_LENGTH, fp);
1733 }
1734 if (rbytes != BLOSC_EXTENDED_HEADER_LENGTH) {
1735 rc = BLOSC2_ERROR_READ_BUFFER;
1736 break;
1737 }
1738 rc = blosc2_cbuffer_sizes(data_chunk, NULL, &chunk_cbytes, NULL);
1739 if (rc < 0) {
1740 break;
1741 }
1742 if (chunk_cbytes > prev_alloc) {
1743 data_chunk = realloc(data_chunk, chunk_cbytes);
1744 prev_alloc = chunk_cbytes;
1745 }
1746 if (!frame->sframe) {
1747 io_cb->seek(fp, header_len + offsets[i], SEEK_SET);
1748 rbytes = io_cb->read(data_chunk, 1, chunk_cbytes, fp);
1749 if (rbytes != chunk_cbytes) {
1750 rc = BLOSC2_ERROR_READ_BUFFER;
1751 break;
1752 }
1753 }
1754 }
1755 uint8_t* new_chunk = malloc(chunk_cbytes);
1756 memcpy(new_chunk, data_chunk, chunk_cbytes);
1757 schunk->data[i] = new_chunk;
1758 rc = blosc2_cbuffer_sizes(data_chunk, &chunk_nbytes, NULL, &chunk_blocksize);
1759 if (rc < 0) {
1760 break;
1761 }
1762 acc_nbytes += chunk_nbytes;
1763 acc_cbytes += chunk_cbytes;
1764 if (i == 0) {
1765 blocksize = chunk_blocksize;
1766 }
1767 else if (blocksize != chunk_blocksize) {
1768 // Blocksize varies
1769 blocksize = 0;
1770 }
1771 }
1772
1773 end:
1774 if (needs_free) {
1775 free(data_chunk);
1776 }
1777 if (frame->cframe == NULL) {
1778 if (!frame->sframe) {
1779 io_cb->close(fp);
1780 }
1781 }
1782 free(offsets);
1783
1784 if (rc < 0 || acc_nbytes != nbytes || acc_cbytes != cbytes) {
1785 blosc2_schunk_free(schunk);
1786 return NULL;
1787 }
1788
1789 schunk->blocksize = blocksize;
1790
1791 out:
1792 rc = frame_get_metalayers(frame, schunk);
1793 if (rc < 0) {
1794 blosc2_schunk_free(schunk);
1795 BLOSC_TRACE_ERROR("Cannot access the metalayers.");
1796 return NULL;
1797 }
1798
1799 rc = frame_get_vlmetalayers(frame, schunk);
1800 if (rc < 0) {
1801 blosc2_schunk_free(schunk);
1802 BLOSC_TRACE_ERROR("Cannot access the vlmetalayers.");
1803 return NULL;
1804 }
1805
1806 return schunk;
1807 }
1808
1809
1810 struct csize_idx {
1811 int32_t val;
1812 int32_t idx;
1813 };
1814
1815 // Helper function for qsorting block offsets
sort_offset(const void * a,const void * b)1816 int sort_offset(const void* a, const void* b) {
1817 int32_t a_ = ((struct csize_idx*)a)->val;
1818 int32_t b_ = ((struct csize_idx*)b)->val;
1819 return a_ - b_;
1820 }
1821
1822
get_coffset(blosc2_frame_s * frame,int32_t header_len,int64_t cbytes,int32_t nchunk,int32_t nchunks,int64_t * offset)1823 int get_coffset(blosc2_frame_s* frame, int32_t header_len, int64_t cbytes,
1824 int32_t nchunk, int32_t nchunks, int64_t *offset) {
1825 int32_t off_cbytes;
1826 // Get the offset to nchunk
1827 uint8_t *coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &off_cbytes);
1828 if (coffsets == NULL) {
1829 BLOSC_TRACE_ERROR("Cannot get the offset for chunk %d for the frame.", nchunk);
1830 return BLOSC2_ERROR_DATA;
1831 }
1832
1833 // Get the 64-bit offset
1834 int rc = blosc2_getitem(coffsets, off_cbytes, nchunk, 1, offset, (int32_t)sizeof(int64_t));
1835 if (rc < 0) {
1836 BLOSC_TRACE_ERROR("Problems retrieving a chunk offset.");
1837 } else if (!frame->sframe && *offset > frame->len) {
1838 BLOSC_TRACE_ERROR("Cannot read chunk %d outside of frame boundary.", nchunk);
1839 rc = BLOSC2_ERROR_READ_BUFFER;
1840 }
1841
1842 return rc;
1843 }
1844
1845
1846 // Detect and return a chunk with special values in offsets (only zeros, NaNs and non initialized)
frame_special_chunk(int64_t special_value,int32_t nbytes,int32_t typesize,int32_t blocksize,uint8_t ** chunk,int32_t cbytes,bool * needs_free)1847 int frame_special_chunk(int64_t special_value, int32_t nbytes, int32_t typesize, int32_t blocksize,
1848 uint8_t** chunk, int32_t cbytes, bool *needs_free) {
1849 int rc = 0;
1850 *chunk = malloc(cbytes);
1851 *needs_free = true;
1852
1853 // Detect the kind of special value
1854 uint64_t zeros_mask = (uint64_t) BLOSC2_SPECIAL_ZERO << (8 * 7); // chunk of zeros
1855 uint64_t nans_mask = (uint64_t) BLOSC2_SPECIAL_NAN << (8 * 7); // chunk of NaNs
1856 uint64_t uninit_mask = (uint64_t) BLOSC2_SPECIAL_UNINIT << (8 * 7); // chunk of uninit values
1857
1858 blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
1859 cparams.typesize = typesize;
1860 cparams.blocksize = blocksize;
1861 if (special_value & zeros_mask) {
1862 rc = blosc2_chunk_zeros(cparams, nbytes, *chunk, cbytes);
1863 if (rc < 0) {
1864 BLOSC_TRACE_ERROR("Error creating a zero chunk");
1865 }
1866 }
1867 else if (special_value & uninit_mask) {
1868 rc = blosc2_chunk_uninit(cparams, nbytes, *chunk, cbytes);
1869 if (rc < 0) {
1870 BLOSC_TRACE_ERROR("Error creating a non initialized chunk");
1871 }
1872 }
1873 else if (special_value & nans_mask) {
1874 rc = blosc2_chunk_nans(cparams, nbytes, *chunk, cbytes);
1875 if (rc < 0) {
1876 BLOSC_TRACE_ERROR("Error creating a nan chunk");
1877 }
1878 }
1879 else {
1880 BLOSC_TRACE_ERROR("Special value not recognized: %" PRId64 "", special_value);
1881 rc = BLOSC2_ERROR_DATA;
1882 }
1883
1884 if (rc < 0) {
1885 free(*chunk);
1886 *needs_free = false;
1887 *chunk = NULL;
1888 }
1889
1890 return rc;
1891 }
1892
1893
1894 /* Return a compressed chunk that is part of a frame in the `chunk` parameter.
1895 * If the frame is disk-based, a buffer is allocated for the (compressed) chunk,
1896 * and hence a free is needed. You can check if the chunk requires a free with the `needs_free`
1897 * parameter.
1898 * If the chunk does not need a free, it means that a pointer to the location in frame is returned
1899 * in the `chunk` parameter.
1900 *
1901 * The size of the (compressed) chunk is returned. If some problem is detected, a negative code
1902 * is returned instead.
1903 */
frame_get_chunk(blosc2_frame_s * frame,int nchunk,uint8_t ** chunk,bool * needs_free)1904 int frame_get_chunk(blosc2_frame_s *frame, int nchunk, uint8_t **chunk, bool *needs_free) {
1905 int32_t header_len;
1906 int64_t frame_len;
1907 int64_t nbytes;
1908 int64_t cbytes;
1909 int32_t blocksize;
1910 int32_t chunksize;
1911 int32_t nchunks;
1912 int32_t typesize;
1913 int64_t offset;
1914 int32_t chunk_cbytes;
1915 int rc;
1916
1917 *chunk = NULL;
1918 *needs_free = false;
1919 rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes,
1920 &blocksize, &chunksize, &nchunks,
1921 &typesize, NULL, NULL, NULL, NULL, NULL,
1922 frame->schunk->storage->io);
1923 if (rc < 0) {
1924 BLOSC_TRACE_ERROR("Unable to get meta info from frame.");
1925 return rc;
1926 }
1927
1928 if (nchunk >= nchunks) {
1929 BLOSC_TRACE_ERROR("nchunk ('%d') exceeds the number of chunks "
1930 "('%d') in frame.", nchunk, nchunks);
1931 return BLOSC2_ERROR_INVALID_PARAM;
1932 }
1933
1934 // Get the offset to nchunk
1935 rc = get_coffset(frame, header_len, cbytes, nchunk, nchunks, &offset);
1936 if (rc < 0) {
1937 BLOSC_TRACE_ERROR("Unable to get offset to chunk %d.", nchunk);
1938 return rc;
1939 }
1940
1941 if (offset < 0) {
1942 // Special value
1943 chunk_cbytes = BLOSC_EXTENDED_HEADER_LENGTH;
1944 int32_t chunksize_ = chunksize;
1945 if ((nchunk == nchunks - 1) && (nbytes % chunksize)) {
1946 // Last chunk is incomplete. Compute its actual size.
1947 chunksize_ = nbytes % chunksize;
1948 }
1949 rc = frame_special_chunk(offset, chunksize_, typesize, blocksize, chunk, chunk_cbytes, needs_free);
1950 if (rc < 0) {
1951 return rc;
1952 }
1953 goto end;
1954 }
1955
1956 if (frame->sframe) {
1957 // Sparse on-disk
1958 nchunk = offset;
1959 return sframe_get_chunk(frame, nchunk, chunk, needs_free);
1960 }
1961
1962 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
1963 if (io_cb == NULL) {
1964 BLOSC_TRACE_ERROR("Error getting the input/output API");
1965 return BLOSC2_ERROR_PLUGIN_IO;
1966 }
1967
1968 if (frame->cframe == NULL) {
1969 uint8_t header[BLOSC_EXTENDED_HEADER_LENGTH];
1970 void* fp = io_cb->open(frame->urlpath, "rb", frame->schunk->storage->io->params);
1971 io_cb->seek(fp, header_len + offset, SEEK_SET);
1972 int64_t rbytes = io_cb->read(header, 1, sizeof(header), fp);
1973 if (rbytes != sizeof(header)) {
1974 BLOSC_TRACE_ERROR("Cannot read the cbytes for chunk in the frame.");
1975 io_cb->close(fp);
1976 return BLOSC2_ERROR_FILE_READ;
1977 }
1978 rc = blosc2_cbuffer_sizes(header, NULL, &chunk_cbytes, NULL);
1979 if (rc < 0) {
1980 BLOSC_TRACE_ERROR("Cannot read the cbytes for chunk in the frame.");
1981 io_cb->close(fp);
1982 return rc;
1983 }
1984 *chunk = malloc(chunk_cbytes);
1985 io_cb->seek(fp, header_len + offset, SEEK_SET);
1986 rbytes = io_cb->read(*chunk, 1, chunk_cbytes, fp);
1987 io_cb->close(fp);
1988 if (rbytes != chunk_cbytes) {
1989 BLOSC_TRACE_ERROR("Cannot read the chunk out of the frame.");
1990 return BLOSC2_ERROR_FILE_READ;
1991 }
1992 *needs_free = true;
1993 } else {
1994 // The chunk is in memory and just one pointer away
1995 *chunk = frame->cframe + header_len + offset;
1996 rc = blosc2_cbuffer_sizes(*chunk, NULL, &chunk_cbytes, NULL);
1997 if (rc < 0) {
1998 return rc;
1999 }
2000 }
2001
2002 end:
2003 return (int32_t)chunk_cbytes;
2004 }
2005
2006
2007 /* Return a compressed chunk that is part of a frame in the `chunk` parameter.
2008 * If the frame is disk-based, a buffer is allocated for the (lazy) chunk,
2009 * and hence a free is needed. You can check if the chunk requires a free with the `needs_free`
2010 * parameter.
2011 * If the chunk does not need a free, it means that the frame is in memory and that just a
2012 * pointer to the location of the chunk in memory is returned.
2013 *
2014 * The size of the (compressed, potentially lazy) chunk is returned. If some problem is detected,
2015 * a negative code is returned instead.
2016 */
frame_get_lazychunk(blosc2_frame_s * frame,int nchunk,uint8_t ** chunk,bool * needs_free)2017 int frame_get_lazychunk(blosc2_frame_s *frame, int nchunk, uint8_t **chunk, bool *needs_free) {
2018 int32_t header_len;
2019 int64_t frame_len;
2020 int64_t nbytes;
2021 int64_t cbytes;
2022 int32_t blocksize;
2023 int32_t chunksize;
2024 int32_t nchunks;
2025 int32_t typesize;
2026 int32_t lazychunk_cbytes;
2027 int64_t offset;
2028 void* fp = NULL;
2029
2030 *chunk = NULL;
2031 *needs_free = false;
2032 int rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes,
2033 &blocksize, &chunksize, &nchunks,
2034 &typesize, NULL, NULL, NULL, NULL, NULL,
2035 frame->schunk->storage->io);
2036 if (rc < 0) {
2037 BLOSC_TRACE_ERROR("Unable to get meta info from frame.");
2038 return rc;
2039 }
2040
2041 if (nchunk >= nchunks) {
2042 BLOSC_TRACE_ERROR("nchunk ('%d') exceeds the number of chunks "
2043 "('%d') in frame.", nchunk, nchunks);
2044 return BLOSC2_ERROR_INVALID_PARAM;
2045 }
2046
2047 // Get the offset to nchunk
2048 rc = get_coffset(frame, header_len, cbytes, nchunk, nchunks, &offset);
2049 if (rc < 0) {
2050 BLOSC_TRACE_ERROR("Unable to get offset to chunk %d.", nchunk);
2051 return rc;
2052 }
2053
2054 if (offset < 0) {
2055 // Special value
2056 lazychunk_cbytes = BLOSC_EXTENDED_HEADER_LENGTH;
2057 int32_t chunksize_ = chunksize;
2058 if ((nchunk == nchunks - 1) && (nbytes % chunksize)) {
2059 // Last chunk is incomplete. Compute its actual size.
2060 chunksize_ = nbytes % chunksize;
2061 }
2062 rc = frame_special_chunk(offset, chunksize_, typesize, blocksize, chunk,
2063 (int32_t)lazychunk_cbytes, needs_free);
2064 goto end;
2065 }
2066
2067 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
2068 if (io_cb == NULL) {
2069 BLOSC_TRACE_ERROR("Error getting the input/output API");
2070 rc = BLOSC2_ERROR_PLUGIN_IO;
2071 goto end;
2072 }
2073
2074 if (frame->cframe == NULL) {
2075 // TODO: make this portable across different endianness
2076 // Get info for building a lazy chunk
2077 int32_t chunk_nbytes;
2078 int32_t chunk_cbytes;
2079 int32_t chunk_blocksize;
2080 uint8_t header[BLOSC_EXTENDED_HEADER_LENGTH];
2081 if (frame->sframe) {
2082 // The chunk is not in the frame
2083 fp = sframe_open_chunk(frame->urlpath, offset, "rb",
2084 frame->schunk->storage->io);
2085 }
2086 else {
2087 fp = io_cb->open(frame->urlpath, "rb", frame->schunk->storage->io->params);
2088 io_cb->seek(fp, header_len + offset, SEEK_SET);
2089 }
2090 int64_t rbytes = io_cb->read(header, 1, BLOSC_EXTENDED_HEADER_LENGTH, fp);
2091 if (rbytes != BLOSC_EXTENDED_HEADER_LENGTH) {
2092 BLOSC_TRACE_ERROR("Cannot read the header for chunk in the frame.");
2093 rc = BLOSC2_ERROR_FILE_READ;
2094 goto end;
2095 }
2096 rc = blosc2_cbuffer_sizes(header, &chunk_nbytes, &chunk_cbytes, &chunk_blocksize);
2097 if (rc < 0) {
2098 goto end;
2099 }
2100 size_t nblocks = chunk_nbytes / chunk_blocksize;
2101 size_t leftover_block = chunk_nbytes % chunk_blocksize;
2102 nblocks = leftover_block ? nblocks + 1 : nblocks;
2103 // Allocate space for the lazy chunk
2104 size_t trailer_len;
2105 int32_t special_type = (header[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK;
2106 int memcpyed = header[BLOSC2_CHUNK_FLAGS] & (uint8_t) BLOSC_MEMCPYED;
2107
2108 size_t trailer_offset = BLOSC_EXTENDED_HEADER_LENGTH;
2109 size_t streams_offset = BLOSC_EXTENDED_HEADER_LENGTH;
2110 if (special_type == 0) {
2111 // Regular values have offsets for blocks
2112 trailer_offset += nblocks * sizeof(int32_t);
2113 if (memcpyed) {
2114 streams_offset += 0;
2115 } else {
2116 streams_offset += nblocks * sizeof(int32_t);
2117 }
2118 trailer_len = sizeof(int32_t) + sizeof(int64_t) + nblocks * sizeof(int32_t);
2119 lazychunk_cbytes = trailer_offset + trailer_len;
2120 }
2121 else if (special_type == BLOSC2_SPECIAL_VALUE) {
2122 trailer_offset += typesize;
2123 streams_offset += typesize;
2124 trailer_len = 0;
2125 lazychunk_cbytes = trailer_offset + trailer_len;
2126 }
2127 else {
2128 rc = BLOSC2_ERROR_INVALID_HEADER;
2129 goto end;
2130 }
2131 *chunk = malloc(lazychunk_cbytes);
2132 *needs_free = true;
2133
2134 // Read just the full header and bstarts section too (lazy partial length)
2135 if (frame->sframe) {
2136 io_cb->seek(fp, 0, SEEK_SET);
2137 }
2138 else {
2139 io_cb->seek(fp, header_len + offset, SEEK_SET);
2140 }
2141
2142 rbytes = io_cb->read(*chunk, 1, streams_offset, fp);
2143 if (rbytes != streams_offset) {
2144 BLOSC_TRACE_ERROR("Cannot read the (lazy) chunk out of the frame.");
2145 rc = BLOSC2_ERROR_FILE_READ;
2146 goto end;
2147 }
2148 if (special_type == BLOSC2_SPECIAL_VALUE) {
2149 // Value runlen is not returning a lazy chunk. We are done.
2150 goto end;
2151 }
2152
2153 // Mark chunk as lazy
2154 uint8_t* blosc2_flags = *chunk + BLOSC2_CHUNK_BLOSC2_FLAGS;
2155 *blosc2_flags |= 0x08U;
2156
2157 // Add the trailer (currently, nchunk + offset + block_csizes)
2158 if (frame->sframe) {
2159 *(int32_t*)(*chunk + trailer_offset) = (int32_t)offset; // offset is nchunk for sframes
2160 *(int64_t*)(*chunk + trailer_offset + sizeof(int32_t)) = offset;
2161 }
2162 else {
2163 *(int32_t*)(*chunk + trailer_offset) = nchunk;
2164 *(int64_t*)(*chunk + trailer_offset + sizeof(int32_t)) = header_len + offset;
2165 }
2166
2167 int32_t* block_csizes = malloc(nblocks * sizeof(int32_t));
2168
2169 if (memcpyed) {
2170 // When memcpyed the blocksizes are trivial to compute
2171 for (int i = 0; i < (int)nblocks - 1; i++) {
2172 block_csizes[i] = (int)chunk_blocksize;
2173 }
2174 // The last block could be incomplete, mainly due to the fact that the block size is not divisible
2175 // by the typesize
2176 block_csizes[nblocks - 1] = (int)(leftover_block ? leftover_block : chunk_blocksize);
2177 }
2178 else {
2179 // In regular, compressed chunks, we need to sort the bstarts (they can be out
2180 // of order because of multi-threading), and get a reverse index too.
2181 memcpy(block_csizes, *chunk + BLOSC_EXTENDED_HEADER_LENGTH, nblocks * sizeof(int32_t));
2182 // Helper structure to keep track of original indexes
2183 struct csize_idx *csize_idx = malloc(nblocks * sizeof(struct csize_idx));
2184 for (int n = 0; n < (int)nblocks; n++) {
2185 csize_idx[n].val = block_csizes[n];
2186 csize_idx[n].idx = n;
2187 }
2188 qsort(csize_idx, nblocks, sizeof(struct csize_idx), &sort_offset);
2189 // Compute the actual csizes
2190 int idx;
2191 for (int n = 0; n < (int)nblocks - 1; n++) {
2192 idx = csize_idx[n].idx;
2193 block_csizes[idx] = csize_idx[n + 1].val - csize_idx[n].val;
2194 }
2195 idx = csize_idx[nblocks - 1].idx;
2196 block_csizes[idx] = (int)chunk_cbytes - csize_idx[nblocks - 1].val;
2197 free(csize_idx);
2198 }
2199 // Copy the csizes at the end of the trailer
2200 void *trailer_csizes = *chunk + lazychunk_cbytes - nblocks * sizeof(int32_t);
2201 memcpy(trailer_csizes, block_csizes, nblocks * sizeof(int32_t));
2202 free(block_csizes);
2203 } else {
2204 // The chunk is in memory and just one pointer away
2205 int64_t chunk_header_offset = header_len + offset;
2206 int64_t chunk_cbytes_offset = chunk_header_offset + BLOSC_MIN_HEADER_LENGTH;
2207
2208 *chunk = frame->cframe + chunk_header_offset;
2209
2210 if (chunk_cbytes_offset > frame->len) {
2211 BLOSC_TRACE_ERROR("Cannot read the header for chunk in the (contiguous) frame.");
2212 rc = BLOSC2_ERROR_READ_BUFFER;
2213 } else {
2214 rc = blosc2_cbuffer_sizes(*chunk, NULL, &lazychunk_cbytes, NULL);
2215 if (rc && chunk_cbytes_offset + lazychunk_cbytes > frame_len) {
2216 BLOSC_TRACE_ERROR("Compressed bytes exceed beyond frame length.");
2217 rc = BLOSC2_ERROR_READ_BUFFER;
2218 }
2219 }
2220 }
2221
2222 end:
2223 if (fp != NULL) {
2224 io_cb->close(fp);
2225 }
2226 if (rc < 0) {
2227 if (*needs_free) {
2228 free(*chunk);
2229 *chunk = NULL;
2230 *needs_free = false;
2231 }
2232 return rc;
2233 }
2234
2235 return (int)lazychunk_cbytes;
2236 }
2237
2238
2239 /* Fill an empty frame with special values (fast path). */
frame_fill_special(blosc2_frame_s * frame,int64_t nitems,int special_value,int32_t chunksize,blosc2_schunk * schunk)2240 int frame_fill_special(blosc2_frame_s* frame, int64_t nitems, int special_value,
2241 int32_t chunksize, blosc2_schunk* schunk) {
2242 int32_t header_len;
2243 int64_t frame_len;
2244 int64_t nbytes;
2245 int64_t cbytes;
2246 int32_t blocksize;
2247 int32_t typesize;
2248 int32_t nchunks;
2249
2250 int rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, &blocksize, NULL,
2251 &nchunks, &typesize, NULL, NULL, NULL, NULL, NULL,
2252 schunk->storage->io);
2253 if (rc < 0) {
2254 BLOSC_TRACE_ERROR("Unable to get meta info from frame.");
2255 return BLOSC2_ERROR_DATA;
2256 }
2257
2258 if (nitems == 0) {
2259 return frame_len;
2260 }
2261
2262 if ((nitems / chunksize) > INT_MAX) {
2263 BLOSC_TRACE_ERROR("nitems is too large. Try increasing the chunksize.");
2264 return BLOSC2_ERROR_FRAME_SPECIAL;
2265 }
2266
2267 if ((nbytes > 0) || (cbytes > 0)) {
2268 BLOSC_TRACE_ERROR("Filling with special values only works on empty frames");
2269 return BLOSC2_ERROR_FRAME_SPECIAL;
2270 }
2271
2272 // Compute the number of chunks and the length of the offsets chunk
2273 int32_t chunkitems = chunksize / typesize;
2274 nchunks = (int32_t)(nitems / chunkitems);
2275 int32_t leftover_items = (int32_t)(nitems % chunkitems);
2276 if (leftover_items) {
2277 nchunks += 1;
2278 }
2279
2280 blosc2_cparams* cparams;
2281 blosc2_schunk_get_cparams(schunk, &cparams);
2282
2283 // Build the offsets with a special chunk
2284 int new_off_cbytes = BLOSC_EXTENDED_HEADER_LENGTH + sizeof(int64_t);
2285 uint8_t* off_chunk = malloc(new_off_cbytes);
2286 uint64_t offset_value = ((uint64_t)1 << 63);
2287 uint8_t* sample_chunk = malloc(BLOSC_EXTENDED_HEADER_LENGTH);
2288 int csize;
2289 switch (special_value) {
2290 case BLOSC2_SPECIAL_ZERO:
2291 offset_value += (uint64_t) BLOSC2_SPECIAL_ZERO << (8 * 7);
2292 csize = blosc2_chunk_zeros(*cparams, chunksize, sample_chunk, BLOSC_EXTENDED_HEADER_LENGTH);
2293 break;
2294 case BLOSC2_SPECIAL_UNINIT:
2295 offset_value += (uint64_t) BLOSC2_SPECIAL_UNINIT << (8 * 7);
2296 csize = blosc2_chunk_uninit(*cparams, chunksize, sample_chunk, BLOSC_EXTENDED_HEADER_LENGTH);
2297 break;
2298 case BLOSC2_SPECIAL_NAN:
2299 offset_value += (uint64_t)BLOSC2_SPECIAL_NAN << (8 * 7);
2300 csize = blosc2_chunk_nans(*cparams, chunksize, sample_chunk, BLOSC_EXTENDED_HEADER_LENGTH);
2301 break;
2302 default:
2303 BLOSC_TRACE_ERROR("Only zeros, NaNs or non-initialized values are supported.");
2304 return BLOSC2_ERROR_FRAME_SPECIAL;
2305 }
2306 if (csize < 0) {
2307 BLOSC_TRACE_ERROR("Error creating sample chunk");
2308 return BLOSC2_ERROR_FRAME_SPECIAL;
2309 }
2310 cparams->typesize = sizeof(int64_t); // change it to offsets typesize
2311 // cparams->blocksize = 0; // automatic blocksize
2312 cparams->blocksize = 8 * 2 * 1024; // based on experiments with create_frame.c bench
2313 cparams->clevel = 5;
2314 cparams->compcode = BLOSC_BLOSCLZ;
2315 int32_t special_nbytes = nchunks * sizeof(int64_t);
2316 rc = blosc2_chunk_repeatval(*cparams, special_nbytes, off_chunk, new_off_cbytes, &offset_value);
2317 free(cparams);
2318 if (rc < 0) {
2319 BLOSC_TRACE_ERROR("Error creating a special offsets chunk");
2320 return BLOSC2_ERROR_DATA;
2321 }
2322
2323 // Get the blocksize associated to the sample chunk
2324 blosc2_cbuffer_sizes(sample_chunk, NULL, NULL, &blocksize);
2325 free(sample_chunk);
2326 // and use it for the super-chunk
2327 schunk->blocksize = blocksize;
2328 // schunk->blocksize = 0; // for experimenting with automatic blocksize
2329
2330 // We have the new offsets; update the frame.
2331 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
2332 if (io_cb == NULL) {
2333 BLOSC_TRACE_ERROR("Error getting the input/output API");
2334 return BLOSC2_ERROR_PLUGIN_IO;
2335 }
2336
2337 int64_t new_frame_len = header_len + new_off_cbytes + frame->trailer_len;
2338 void* fp = NULL;
2339 if (frame->cframe != NULL) {
2340 uint8_t* framep = frame->cframe;
2341 /* Make space for the new chunk and copy it */
2342 frame->cframe = framep = realloc(framep, (size_t)new_frame_len);
2343 if (framep == NULL) {
2344 BLOSC_TRACE_ERROR("Cannot realloc space for the frame.");
2345 return BLOSC2_ERROR_FRAME_SPECIAL;
2346 }
2347 /* Copy the offsets */
2348 memcpy(framep + header_len, off_chunk, (size_t)new_off_cbytes);
2349 }
2350 else {
2351 size_t wbytes;
2352 if (frame->sframe) {
2353 // Update the offsets chunk in the chunks frame
2354 fp = sframe_open_index(frame->urlpath, "rb+", frame->schunk->storage->io);
2355 io_cb->seek(fp, header_len, SEEK_SET);
2356 }
2357 else {
2358 // Regular frame
2359 fp = io_cb->open(frame->urlpath, "rb+", schunk->storage->io->params);
2360 io_cb->seek(fp, header_len + cbytes, SEEK_SET);
2361 }
2362 wbytes = io_cb->write(off_chunk, 1, (size_t)new_off_cbytes, fp); // the new offsets
2363 io_cb->close(fp);
2364 if (wbytes != (size_t)new_off_cbytes) {
2365 BLOSC_TRACE_ERROR("Cannot write the offsets to frame.");
2366 return BLOSC2_ERROR_FRAME_SPECIAL;
2367 }
2368 }
2369
2370 // Invalidate the cache for chunk offsets
2371 if (frame->coffsets != NULL) {
2372 free(frame->coffsets);
2373 frame->coffsets = NULL;
2374 }
2375 free(off_chunk);
2376
2377 frame->len = new_frame_len;
2378 rc = frame_update_header(frame, schunk, false);
2379 if (rc < 0) {
2380 return BLOSC2_ERROR_FRAME_SPECIAL;
2381 }
2382
2383 rc = frame_update_trailer(frame, schunk);
2384 if (rc < 0) {
2385 return BLOSC2_ERROR_FRAME_SPECIAL;
2386 }
2387
2388 return frame->len;
2389 }
2390
2391
2392 /* Append an existing chunk into a frame. */
frame_append_chunk(blosc2_frame_s * frame,void * chunk,blosc2_schunk * schunk)2393 void* frame_append_chunk(blosc2_frame_s* frame, void* chunk, blosc2_schunk* schunk) {
2394 int8_t* chunk_ = chunk;
2395 int32_t header_len;
2396 int64_t frame_len;
2397 int64_t nbytes;
2398 int64_t cbytes;
2399 int32_t blocksize;
2400 int32_t chunksize;
2401 int32_t nchunks;
2402 int rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes, &blocksize, &chunksize,
2403 &nchunks, NULL, NULL, NULL, NULL, NULL, NULL,
2404 frame->schunk->storage->io);
2405 if (rc < 0) {
2406 BLOSC_TRACE_ERROR("Unable to get meta info from frame.");
2407 return NULL;
2408 }
2409
2410 /* The uncompressed and compressed sizes start at byte 4 and 12 */
2411 int32_t chunk_nbytes;
2412 int32_t chunk_cbytes;
2413 rc = blosc2_cbuffer_sizes(chunk, &chunk_nbytes, &chunk_cbytes, NULL);
2414 if (rc < 0) {
2415 return NULL;
2416 }
2417
2418 if ((nchunks > 0) && (chunk_nbytes > (size_t)chunksize)) {
2419 BLOSC_TRACE_ERROR("Appending chunks with a larger chunksize than frame is "
2420 "not allowed yet %d != %d.", chunk_nbytes, chunksize);
2421 return NULL;
2422 }
2423
2424 // Check that we are not appending a small chunk after another small chunk
2425 int32_t chunk_nbytes_last;
2426 if (chunksize == 0 && (nchunks > 0) && (chunk_nbytes < (size_t)chunksize)) {
2427 uint8_t* last_chunk;
2428 bool needs_free;
2429 rc = frame_get_lazychunk(frame, nchunks - 1, &last_chunk, &needs_free);
2430 if (rc < 0) {
2431 BLOSC_TRACE_ERROR("Cannot get the last chunk (in position %d).", nchunks - 1);
2432 } else {
2433 rc = blosc2_cbuffer_sizes(last_chunk, &chunk_nbytes_last, NULL, NULL);
2434 }
2435 if (needs_free) {
2436 free(last_chunk);
2437 }
2438 if (rc < 0) {
2439 return NULL;
2440 }
2441 if ((chunk_nbytes_last < (size_t)chunksize) && (nbytes < (size_t)chunksize)) {
2442 BLOSC_TRACE_ERROR("Appending two consecutive chunks with a chunksize smaller "
2443 "than the frame chunksize is not allowed yet: %d != %d.",
2444 chunk_nbytes, chunksize);
2445 return NULL;
2446 }
2447 }
2448
2449 // Get the current offsets and add one more
2450 int32_t off_nbytes = (nchunks + 1) * sizeof(int64_t);
2451 int64_t* offsets = (int64_t *) malloc((size_t)off_nbytes);
2452 if (nchunks > 0) {
2453 int32_t coffsets_cbytes;
2454 uint8_t *coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &coffsets_cbytes);
2455 if (coffsets == NULL) {
2456 BLOSC_TRACE_ERROR("Cannot get the offsets for the frame.");
2457 free(offsets);
2458 return NULL;
2459 }
2460 if (coffsets_cbytes == 0) {
2461 coffsets_cbytes = (int32_t)cbytes;
2462 }
2463
2464 // Decompress offsets
2465 blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS;
2466 blosc2_context *dctx = blosc2_create_dctx(off_dparams);
2467 int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets,
2468 nchunks * sizeof(int64_t));
2469 blosc2_free_ctx(dctx);
2470 if (prev_nbytes < 0) {
2471 free(offsets);
2472 BLOSC_TRACE_ERROR("Cannot decompress the offsets chunk.");
2473 return NULL;
2474 }
2475 }
2476
2477 // Add the new offset
2478 int64_t sframe_chunk_id = -1;
2479 int special_value = (chunk_[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK;
2480 uint64_t offset_value = ((uint64_t)1 << 63);
2481 switch (special_value) {
2482 case BLOSC2_SPECIAL_ZERO:
2483 // Zero chunk. Code it in a special way.
2484 offset_value += (uint64_t) BLOSC2_SPECIAL_ZERO << (8 * 7); // chunk of zeros
2485 to_little(offsets + nchunks, &offset_value, sizeof(uint64_t));
2486 chunk_cbytes = 0; // we don't need to store the chunk
2487 break;
2488 case BLOSC2_SPECIAL_UNINIT:
2489 // Non initizalized values chunk. Code it in a special way.
2490 offset_value += (uint64_t) BLOSC2_SPECIAL_UNINIT << (8 * 7); // chunk of uninit values
2491 to_little(offsets + nchunks, &offset_value, sizeof(uint64_t));
2492 chunk_cbytes = 0; // we don't need to store the chunk
2493 break;
2494 case BLOSC2_SPECIAL_NAN:
2495 // NaN chunk. Code it in a special way.
2496 offset_value += (uint64_t)BLOSC2_SPECIAL_NAN << (8 * 7); // chunk of NANs
2497 to_little(offsets + nchunks, &offset_value, sizeof(uint64_t));
2498 chunk_cbytes = 0; // we don't need to store the chunk
2499 break;
2500 default:
2501 if (frame->sframe) {
2502 // Compute the sframe_chunk_id value
2503 for (int i = 0; i < nchunks; ++i) {
2504 if (offsets[i] > sframe_chunk_id) {
2505 sframe_chunk_id = offsets[i];
2506 }
2507 }
2508 offsets[nchunks] = ++sframe_chunk_id;
2509 }
2510 else {
2511 offsets[nchunks] = cbytes;
2512 }
2513 }
2514
2515 // Re-compress the offsets again
2516 blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
2517 cparams.splitmode = BLOSC_NEVER_SPLIT;
2518 cparams.typesize = sizeof(int64_t);
2519 cparams.blocksize = 16 * 1024; // based on experiments with create_frame.c bench
2520 cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs
2521 cparams.compcode = BLOSC_BLOSCLZ;
2522 blosc2_context* cctx = blosc2_create_cctx(cparams);
2523 void* off_chunk = malloc((size_t)off_nbytes + BLOSC_MAX_OVERHEAD);
2524 int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes,
2525 off_chunk, off_nbytes + BLOSC_MAX_OVERHEAD);
2526 blosc2_free_ctx(cctx);
2527 free(offsets);
2528 if (new_off_cbytes < 0) {
2529 free(off_chunk);
2530 return NULL;
2531 }
2532 // printf("%f\n", (double) off_nbytes / new_off_cbytes);
2533
2534 int64_t new_cbytes = cbytes + chunk_cbytes;
2535 int64_t new_frame_len;
2536 if (frame->sframe) {
2537 new_frame_len = header_len + 0 + new_off_cbytes + frame->trailer_len;
2538 }
2539 else {
2540 new_frame_len = header_len + new_cbytes + new_off_cbytes + frame->trailer_len;
2541 }
2542
2543 void* fp = NULL;
2544 if (frame->cframe != NULL) {
2545 uint8_t* framep = frame->cframe;
2546 /* Make space for the new chunk and copy it */
2547 frame->cframe = framep = realloc(framep, (size_t)new_frame_len);
2548 if (framep == NULL) {
2549 BLOSC_TRACE_ERROR("Cannot realloc space for the frame.");
2550 return NULL;
2551 }
2552 /* Copy the chunk */
2553 memcpy(framep + header_len + cbytes, chunk, (size_t)chunk_cbytes);
2554 /* Copy the offsets */
2555 memcpy(framep + header_len + new_cbytes, off_chunk, (size_t)new_off_cbytes);
2556 }
2557 else {
2558 int64_t wbytes;
2559 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
2560 if (io_cb == NULL) {
2561 BLOSC_TRACE_ERROR("Error getting the input/output API");
2562 return NULL;
2563 }
2564
2565 if (frame->sframe) {
2566 // Update the offsets chunk in the chunks frame
2567 if (chunk_cbytes != 0) {
2568 if (sframe_chunk_id < 0) {
2569 BLOSC_TRACE_ERROR("The chunk id (%" PRId64 ") is not correct", sframe_chunk_id);
2570 return NULL;
2571 }
2572 if (sframe_create_chunk(frame, chunk, sframe_chunk_id, chunk_cbytes) == NULL) {
2573 BLOSC_TRACE_ERROR("Cannot write the full chunk.");
2574 return NULL;
2575 }
2576 }
2577 fp = sframe_open_index(frame->urlpath, "rb+",
2578 frame->schunk->storage->io);
2579 io_cb->seek(fp, header_len, SEEK_SET);
2580 }
2581 else {
2582 // Regular frame
2583 fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params);
2584 io_cb->seek(fp, header_len + cbytes, SEEK_SET);
2585 wbytes = io_cb->write(chunk, 1, chunk_cbytes, fp); // the new chunk
2586 if (wbytes != (size_t)chunk_cbytes) {
2587 BLOSC_TRACE_ERROR("Cannot write the full chunk to frame.");
2588 io_cb->close(fp);
2589 return NULL;
2590 }
2591 }
2592 wbytes = io_cb->write(off_chunk, 1, new_off_cbytes, fp); // the new offsets
2593 io_cb->close(fp);
2594 if (wbytes != (size_t)new_off_cbytes) {
2595 BLOSC_TRACE_ERROR("Cannot write the offsets to frame.");
2596 return NULL;
2597 }
2598 }
2599 // Invalidate the cache for chunk offsets
2600 if (frame->coffsets != NULL) {
2601 free(frame->coffsets);
2602 frame->coffsets = NULL;
2603 }
2604 free(chunk); // chunk has always to be a copy when reaching here...
2605 free(off_chunk);
2606
2607 frame->len = new_frame_len;
2608 rc = frame_update_header(frame, schunk, false);
2609 if (rc < 0) {
2610 return NULL;
2611 }
2612
2613 rc = frame_update_trailer(frame, schunk);
2614 if (rc < 0) {
2615 return NULL;
2616 }
2617
2618 return frame;
2619 }
2620
2621
frame_insert_chunk(blosc2_frame_s * frame,int nchunk,void * chunk,blosc2_schunk * schunk)2622 void* frame_insert_chunk(blosc2_frame_s* frame, int nchunk, void* chunk, blosc2_schunk* schunk) {
2623 uint8_t* chunk_ = chunk;
2624 int32_t header_len;
2625 int64_t frame_len;
2626 int64_t nbytes;
2627 int64_t cbytes;
2628 int32_t blocksize;
2629 int32_t chunksize;
2630 int32_t nchunks;
2631 int rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes,
2632 &blocksize, &chunksize, &nchunks,
2633 NULL, NULL, NULL, NULL, NULL, NULL,
2634 frame->schunk->storage->io);
2635 if (rc < 0) {
2636 BLOSC_TRACE_ERROR("Unable to get meta info from frame.");
2637 return NULL;
2638 }
2639 int32_t chunk_cbytes;
2640 rc = blosc2_cbuffer_sizes(chunk_, NULL, &chunk_cbytes, NULL);
2641 if (rc < 0) {
2642 return NULL;
2643 }
2644
2645 // Get the current offsets
2646 int32_t off_nbytes = (nchunks + 1) * sizeof(int64_t);
2647 int64_t* offsets = (int64_t *) malloc((size_t)off_nbytes);
2648 if (nchunks > 0) {
2649 int32_t coffsets_cbytes = 0;
2650 uint8_t *coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &coffsets_cbytes);
2651 if (coffsets == NULL) {
2652 BLOSC_TRACE_ERROR("Cannot get the offsets for the frame.");
2653 return NULL;
2654 }
2655 if (coffsets_cbytes == 0) {
2656 coffsets_cbytes = (int32_t)cbytes;
2657 }
2658
2659 // Decompress offsets
2660 blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS;
2661 blosc2_context *dctx = blosc2_create_dctx(off_dparams);
2662 int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, nchunks * sizeof(int64_t));
2663 blosc2_free_ctx(dctx);
2664 if (prev_nbytes < 0) {
2665 free(offsets);
2666 BLOSC_TRACE_ERROR("Cannot decompress the offsets chunk.");
2667 return NULL;
2668 }
2669 }
2670
2671 // TODO: Improvement: Check if new chunk is smaller than previous one
2672
2673 // Add the new offset
2674 int64_t sframe_chunk_id = -1;
2675 int special_value = (chunk_[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK;
2676 uint64_t offset_value = ((uint64_t)1 << 63);
2677 switch (special_value) {
2678 case BLOSC2_SPECIAL_ZERO:
2679 // Zero chunk. Code it in a special way.
2680 offset_value += (uint64_t)BLOSC2_SPECIAL_ZERO << (8 * 7); // indicate a chunk of zeros
2681 for (int i = nchunks; i > nchunk; i--) {
2682 offsets[i] = offsets[i - 1];
2683 }
2684 to_little(offsets + nchunk, &offset_value, sizeof(uint64_t));
2685 chunk_cbytes = 0; // we don't need to store the chunk
2686 break;
2687 case BLOSC2_SPECIAL_UNINIT:
2688 // Non initizalized values chunk. Code it in a special way.
2689 offset_value += (uint64_t) BLOSC2_SPECIAL_UNINIT << (8 * 7); // chunk of uninit values
2690 for (int i = nchunks; i > nchunk; i--) {
2691 offsets[i] = offsets[i - 1];
2692 }
2693 to_little(offsets + nchunk, &offset_value, sizeof(uint64_t));
2694 chunk_cbytes = 0; // we don't need to store the chunk
2695 break;
2696 case BLOSC2_SPECIAL_NAN:
2697 // NaN chunk. Code it in a special way.
2698 offset_value += (uint64_t)BLOSC2_SPECIAL_NAN << (8 * 7); // indicate a chunk of NANs
2699 for (int i = nchunks; i > nchunk; i--) {
2700 offsets[i] = offsets[i - 1];
2701 }
2702 to_little(offsets + nchunk, &offset_value, sizeof(uint64_t));
2703 chunk_cbytes = 0; // we don't need to store the chunk
2704 break;
2705 default:
2706 // Add the new offset
2707 for (int i = nchunks; i > nchunk; i--) {
2708 offsets[i] = offsets[i - 1];
2709 }
2710 if (frame->sframe) {
2711 for (int i = 0; i < nchunks; ++i) {
2712 if (offsets[i] > sframe_chunk_id) {
2713 sframe_chunk_id = offsets[i];
2714 }
2715 }
2716 offsets[nchunk] = ++sframe_chunk_id;
2717 }
2718 else {
2719 offsets[nchunk] = cbytes;
2720 }
2721 }
2722
2723 // Re-compress the offsets again
2724 blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
2725 cparams.splitmode = BLOSC_NEVER_SPLIT;
2726 cparams.typesize = sizeof(int64_t);
2727 cparams.blocksize = 16 * 1024; // based on experiments with create_frame.c bench
2728 cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs
2729 cparams.compcode = BLOSC_BLOSCLZ;
2730 blosc2_context* cctx = blosc2_create_cctx(cparams);
2731 void* off_chunk = malloc((size_t)off_nbytes + BLOSC_MAX_OVERHEAD);
2732 int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes,
2733 off_chunk, off_nbytes + BLOSC_MAX_OVERHEAD);
2734 blosc2_free_ctx(cctx);
2735
2736 free(offsets);
2737 if (new_off_cbytes < 0) {
2738 free(off_chunk);
2739 return NULL;
2740 }
2741
2742 int64_t new_cbytes = cbytes + chunk_cbytes;
2743
2744 int64_t new_frame_len;
2745 if (frame->sframe) {
2746 new_frame_len = header_len + 0 + new_off_cbytes + frame->trailer_len;
2747 }
2748 else {
2749 new_frame_len = header_len + new_cbytes + new_off_cbytes + frame->trailer_len;
2750 }
2751
2752 // Add the chunk and update meta
2753 void* fp = NULL;
2754 if (frame->cframe != NULL) {
2755 uint8_t* framep = frame->cframe;
2756 /* Make space for the new chunk and copy it */
2757 frame->cframe = framep = realloc(framep, (size_t)new_frame_len);
2758 if (framep == NULL) {
2759 BLOSC_TRACE_ERROR("Cannot realloc space for the frame.");
2760 return NULL;
2761 }
2762 /* Copy the chunk */
2763 memcpy(framep + header_len + cbytes, chunk, (size_t)chunk_cbytes);
2764 /* Copy the offsets */
2765 memcpy(framep + header_len + new_cbytes, off_chunk, (size_t)new_off_cbytes);
2766 } else {
2767 int64_t wbytes;
2768
2769 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
2770 if (io_cb == NULL) {
2771 BLOSC_TRACE_ERROR("Error getting the input/output API");
2772 return NULL;
2773 }
2774
2775 if (frame->sframe) {
2776 if (chunk_cbytes != 0) {
2777 if (sframe_chunk_id < 0) {
2778 BLOSC_TRACE_ERROR("The chunk id (%" PRId64 ") is not correct", sframe_chunk_id);
2779 return NULL;
2780 }
2781 if (sframe_create_chunk(frame, chunk, sframe_chunk_id, chunk_cbytes) == NULL) {
2782 BLOSC_TRACE_ERROR("Cannot write the full chunk.");
2783 return NULL;
2784 }
2785 }
2786 // Update the offsets chunk in the chunks frame
2787 fp = sframe_open_index(frame->urlpath, "rb+",
2788 frame->schunk->storage->io);
2789 io_cb->seek(fp, header_len + 0, SEEK_SET);
2790 }
2791 else {
2792 // Regular frame
2793 fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params);
2794 io_cb->seek(fp, header_len + cbytes, SEEK_SET);
2795 wbytes = io_cb->write(chunk, 1, chunk_cbytes, fp); // the new chunk
2796 if (wbytes != (size_t)chunk_cbytes) {
2797 BLOSC_TRACE_ERROR("Cannot write the full chunk to frame.");
2798 io_cb->close(fp);
2799 return NULL;
2800 }
2801 }
2802 wbytes = io_cb->write(off_chunk, 1, new_off_cbytes, fp); // the new offsets
2803 io_cb->close(fp);
2804 if (wbytes != (size_t)new_off_cbytes) {
2805 BLOSC_TRACE_ERROR("Cannot write the offsets to frame.");
2806 return NULL;
2807 }
2808 // Invalidate the cache for chunk offsets
2809 if (frame->coffsets != NULL) {
2810 free(frame->coffsets);
2811 frame->coffsets = NULL;
2812 }
2813 }
2814 free(chunk); // chunk has always to be a copy when reaching here...
2815 free(off_chunk);
2816
2817 frame->len = new_frame_len;
2818 rc = frame_update_header(frame, schunk, false);
2819 if (rc < 0) {
2820 return NULL;
2821 }
2822
2823 rc = frame_update_trailer(frame, schunk);
2824 if (rc < 0) {
2825 return NULL;
2826 }
2827
2828 return frame;
2829 }
2830
2831
frame_update_chunk(blosc2_frame_s * frame,int nchunk,void * chunk,blosc2_schunk * schunk)2832 void* frame_update_chunk(blosc2_frame_s* frame, int nchunk, void* chunk, blosc2_schunk* schunk) {
2833 uint8_t *chunk_ = (uint8_t *) chunk;
2834 int32_t header_len;
2835 int64_t frame_len;
2836 int64_t nbytes;
2837 int64_t cbytes;
2838 int32_t blocksize;
2839 int32_t chunksize;
2840 int32_t nchunks;
2841 int rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes,
2842 &blocksize, &chunksize, &nchunks,
2843 NULL, NULL, NULL, NULL, NULL, NULL,
2844 frame->schunk->storage->io);
2845 if (rc < 0) {
2846 BLOSC_TRACE_ERROR("Unable to get meta info from frame.");
2847 return NULL;
2848 }
2849 if (nchunk >= nchunks) {
2850 BLOSC_TRACE_ERROR("The chunk must already exist.");
2851 return NULL;
2852 }
2853
2854 int32_t chunk_cbytes;
2855 rc = blosc2_cbuffer_sizes(chunk, NULL, &chunk_cbytes, NULL);
2856 if (rc < 0) {
2857 return NULL;
2858 }
2859
2860 // Get the current offsets
2861 int32_t off_nbytes = nchunks * sizeof(int64_t);
2862 int64_t* offsets = (int64_t *) malloc((size_t)off_nbytes);
2863 if (nchunks > 0) {
2864 int32_t coffsets_cbytes = 0;
2865 uint8_t *coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &coffsets_cbytes);
2866 if (coffsets == NULL) {
2867 BLOSC_TRACE_ERROR("Cannot get the offsets for the frame.");
2868 return NULL;
2869 }
2870 if (coffsets_cbytes == 0) {
2871 coffsets_cbytes = (int32_t)cbytes;
2872 }
2873
2874 // Decompress offsets
2875 blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS;
2876 blosc2_context *dctx = blosc2_create_dctx(off_dparams);
2877 int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, nchunks * sizeof(int64_t));
2878 blosc2_free_ctx(dctx);
2879 if (prev_nbytes < 0) {
2880 free(offsets);
2881 BLOSC_TRACE_ERROR("Cannot decompress the offsets chunk.");
2882 return NULL;
2883 }
2884 }
2885 int32_t cbytes_old;
2886 int32_t old_offset;
2887 if (!frame->sframe) {
2888 // See how big would be the space
2889 old_offset = offsets[nchunk];
2890 bool needs_free;
2891 uint8_t *chunk_old;
2892 int err = blosc2_schunk_get_chunk(schunk, nchunk, &chunk_old, &needs_free);
2893 if (err < 0) {
2894 BLOSC_TRACE_ERROR("%d chunk can not be obtained from schunk.", nchunk);
2895 return NULL;
2896 }
2897
2898 if (chunk_old == NULL) {
2899 cbytes_old = 0;
2900 }
2901 else {
2902 cbytes_old = sw32_(chunk_old + BLOSC2_CHUNK_CBYTES);
2903 if (cbytes_old == BLOSC_MAX_OVERHEAD) {
2904 cbytes_old = 0;
2905 }
2906 }
2907 if (needs_free) {
2908 free(chunk_old);
2909 }
2910 }
2911
2912 // Add the new offset
2913 int special_value = (chunk_[BLOSC2_CHUNK_BLOSC2_FLAGS] >> 4) & BLOSC2_SPECIAL_MASK;
2914 uint64_t offset_value = ((uint64_t)1 << 63);
2915 switch (special_value) {
2916 case BLOSC2_SPECIAL_ZERO:
2917 // Zero chunk. Code it in a special way.
2918 offset_value += (uint64_t)BLOSC2_SPECIAL_ZERO << (8 * 7); // indicate a chunk of zeros
2919 to_little(offsets + nchunk, &offset_value, sizeof(uint64_t));
2920 chunk_cbytes = 0; // we don't need to store the chunk
2921 break;
2922 case BLOSC2_SPECIAL_UNINIT:
2923 // Non initizalized values chunk. Code it in a special way.
2924 offset_value += (uint64_t)BLOSC2_SPECIAL_UNINIT << (8 * 7); // indicate a chunk of uninit values
2925 to_little(offsets + nchunk, &offset_value, sizeof(uint64_t));
2926 chunk_cbytes = 0; // we don't need to store the chunk
2927 break;
2928 case BLOSC2_SPECIAL_NAN:
2929 // NaN chunk. Code it in a special way.
2930 offset_value += (uint64_t)BLOSC2_SPECIAL_NAN << (8 * 7); // indicate a chunk of NANs
2931 to_little(offsets + nchunk, &offset_value, sizeof(uint64_t));
2932 chunk_cbytes = 0; // we don't need to store the chunk
2933 break;
2934 default:
2935 if (frame->sframe) {
2936 // In case there was a reorder
2937 offsets[nchunk] = nchunk;
2938 }
2939 else {
2940 // Add the new offset
2941 offsets[nchunk] = cbytes;
2942 }
2943 }
2944
2945 if (!frame->sframe && chunk_cbytes != 0 && cbytes_old >= chunk_cbytes) {
2946 offsets[nchunk] = old_offset;
2947 cbytes = old_offset;
2948 }
2949 // Re-compress the offsets again
2950 blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
2951 cparams.splitmode = BLOSC_NEVER_SPLIT;
2952 cparams.typesize = sizeof(int64_t);
2953 cparams.blocksize = 16 * 1024; // based on experiments with create_frame.c bench
2954 cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs
2955 cparams.compcode = BLOSC_BLOSCLZ;
2956 blosc2_context* cctx = blosc2_create_cctx(cparams);
2957 void* off_chunk = malloc((size_t)off_nbytes + BLOSC_MAX_OVERHEAD);
2958 int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes,
2959 off_chunk, off_nbytes + BLOSC_MAX_OVERHEAD);
2960 blosc2_free_ctx(cctx);
2961
2962 free(offsets);
2963 if (new_off_cbytes < 0) {
2964 free(off_chunk);
2965 return NULL;
2966 }
2967
2968 int64_t new_cbytes = schunk->cbytes;
2969 int64_t new_frame_len;
2970 if (frame->sframe) {
2971 // The chunk is not stored in the frame
2972 new_frame_len = header_len + 0 + new_off_cbytes + frame->trailer_len;
2973 }
2974 else {
2975 new_frame_len = header_len + new_cbytes + new_off_cbytes + frame->trailer_len;
2976 }
2977
2978 void* fp = NULL;
2979 if (frame->cframe != NULL) {
2980 uint8_t* framep = frame->cframe;
2981 /* Make space for the new chunk and copy it */
2982 frame->cframe = framep = realloc(framep, (size_t)new_frame_len);
2983 if (framep == NULL) {
2984 BLOSC_TRACE_ERROR("Cannot realloc space for the frame.");
2985 return NULL;
2986 }
2987 /* Copy the chunk */
2988 memcpy(framep + header_len + cbytes, chunk, (size_t)chunk_cbytes);
2989 /* Copy the offsets */
2990 memcpy(framep + header_len + new_cbytes, off_chunk, (size_t)new_off_cbytes);
2991 } else {
2992 int64_t wbytes;
2993
2994 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
2995 if (io_cb == NULL) {
2996 BLOSC_TRACE_ERROR("Error getting the input/output API");
2997 return NULL;
2998 }
2999
3000 if (frame->sframe) {
3001 if (chunk_cbytes) {
3002 if (sframe_create_chunk(frame, chunk, nchunk, chunk_cbytes) == NULL) {
3003 BLOSC_TRACE_ERROR("Cannot write the full chunk.");
3004 return NULL;
3005 }
3006 }
3007 // Update the offsets chunk in the chunks frame
3008 fp = sframe_open_index(frame->urlpath, "rb+",
3009 frame->schunk->storage->io);
3010 io_cb->seek(fp, header_len + 0, SEEK_SET);
3011 }
3012 else {
3013 // Regular frame
3014 fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params);
3015 io_cb->seek(fp, header_len + cbytes, SEEK_SET);
3016 wbytes = io_cb->write(chunk, 1, chunk_cbytes, fp); // the new chunk
3017 if (wbytes != (size_t)chunk_cbytes) {
3018 BLOSC_TRACE_ERROR("Cannot write the full chunk to frame.");
3019 io_cb->close(fp);
3020 return NULL;
3021 }
3022 io_cb->seek(fp, header_len + new_cbytes, SEEK_SET);
3023 }
3024 wbytes = io_cb->write(off_chunk, 1, new_off_cbytes, fp); // the new offsets
3025 io_cb->close(fp);
3026 if (wbytes != (size_t)new_off_cbytes) {
3027 BLOSC_TRACE_ERROR("Cannot write the offsets to frame.");
3028 return NULL;
3029 }
3030 // Invalidate the cache for chunk offsets
3031 if (frame->coffsets != NULL) {
3032 free(frame->coffsets);
3033 frame->coffsets = NULL;
3034 }
3035 }
3036 free(chunk); // chunk has always to be a copy when reaching here...
3037 free(off_chunk);
3038
3039 frame->len = new_frame_len;
3040 rc = frame_update_header(frame, schunk, false);
3041 if (rc < 0) {
3042 return NULL;
3043 }
3044
3045 rc = frame_update_trailer(frame, schunk);
3046 if (rc < 0) {
3047 return NULL;
3048 }
3049
3050 return frame;
3051 }
3052
3053
frame_delete_chunk(blosc2_frame_s * frame,int nchunk,blosc2_schunk * schunk)3054 void* frame_delete_chunk(blosc2_frame_s* frame, int nchunk, blosc2_schunk* schunk) {
3055 int32_t header_len;
3056 int64_t frame_len;
3057 int64_t nbytes;
3058 int64_t cbytes;
3059 int32_t blocksize;
3060 int32_t chunksize;
3061 int32_t nchunks;
3062 int rc = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes,
3063 &blocksize, &chunksize, &nchunks,
3064 NULL, NULL, NULL, NULL, NULL, NULL, frame->schunk->storage->io);
3065 if (rc < 0) {
3066 BLOSC_TRACE_ERROR("Unable to get meta info from frame.");
3067 return NULL;
3068 }
3069
3070 // Get the current offsets
3071 int32_t off_nbytes = (nchunks) * sizeof(int64_t);
3072 int64_t* offsets = (int64_t *) malloc((size_t)off_nbytes);
3073 if (nchunks > 0) {
3074 int32_t coffsets_cbytes = 0;
3075 uint8_t *coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &coffsets_cbytes);
3076 if (coffsets == NULL) {
3077 BLOSC_TRACE_ERROR("Cannot get the offsets for the frame.");
3078 return NULL;
3079 }
3080 if (coffsets_cbytes == 0) {
3081 coffsets_cbytes = (int32_t)cbytes;
3082 }
3083
3084 // Decompress offsets
3085 blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS;
3086 blosc2_context *dctx = blosc2_create_dctx(off_dparams);
3087 int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes, offsets, nchunks * sizeof(int64_t));
3088 blosc2_free_ctx(dctx);
3089 if (prev_nbytes < 0) {
3090 free(offsets);
3091 BLOSC_TRACE_ERROR("Cannot decompress the offsets chunk.");
3092 return NULL;
3093 }
3094 }
3095
3096 // Delete the new offset
3097 for (int i = nchunk; i < nchunks - 1; i++) {
3098 offsets[i] = offsets[i + 1];
3099 }
3100 offsets[nchunks - 1] = 0;
3101
3102 // Re-compress the offsets again
3103 blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
3104 cparams.splitmode = BLOSC_NEVER_SPLIT;
3105 cparams.typesize = sizeof(int64_t);
3106 cparams.blocksize = 16 * 1024; // based on experiments with create_frame.c bench
3107 cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs
3108 cparams.compcode = BLOSC_BLOSCLZ;
3109 blosc2_context* cctx = blosc2_create_cctx(cparams);
3110 void* off_chunk = malloc((size_t)off_nbytes + BLOSC_MAX_OVERHEAD);
3111 int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes - sizeof(int64_t),
3112 off_chunk, off_nbytes + BLOSC_MAX_OVERHEAD);
3113 blosc2_free_ctx(cctx);
3114
3115 free(offsets);
3116 if (new_off_cbytes < 0) {
3117 free(off_chunk);
3118 return NULL;
3119 }
3120
3121 int64_t new_cbytes = cbytes;
3122
3123 int64_t new_frame_len;
3124 if (frame->sframe) {
3125 new_frame_len = header_len + 0 + new_off_cbytes + frame->trailer_len;
3126 }
3127 else {
3128 new_frame_len = header_len + new_cbytes + new_off_cbytes + frame->trailer_len;
3129 }
3130
3131 // Add the chunk and update meta
3132 FILE* fp = NULL;
3133 if (frame->cframe != NULL) {
3134 uint8_t* framep = frame->cframe;
3135 /* Make space for the new chunk and copy it */
3136 frame->cframe = framep = realloc(framep, (size_t)new_frame_len);
3137 if (framep == NULL) {
3138 BLOSC_TRACE_ERROR("Cannot realloc space for the frame.");
3139 return NULL;
3140 }
3141 /* Copy the offsets */
3142 memcpy(framep + header_len + new_cbytes, off_chunk, (size_t)new_off_cbytes);
3143 } else {
3144 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
3145 if (io_cb == NULL) {
3146 BLOSC_TRACE_ERROR("Error getting the input/output API");
3147 return NULL;
3148 }
3149
3150 size_t wbytes;
3151 if (frame->sframe) {
3152 int64_t offset;
3153 rc = get_coffset(frame, header_len, cbytes, nchunk, nchunks, &offset);
3154 if (rc < 0) {
3155 BLOSC_TRACE_ERROR("Unable to get offset to chunk %d.", nchunk);
3156 return NULL;
3157 }
3158 if (offset >= 0){
3159 // Remove the chunk file only if it is not a special value chunk
3160 int err = sframe_delete_chunk(frame->urlpath, offset);
3161 if (err != 0) {
3162 BLOSC_TRACE_ERROR("Unable to delete chunk!");
3163 return NULL;
3164 }
3165 }
3166 // Update the offsets chunk in the chunks frame
3167 fp = sframe_open_index(frame->urlpath, "rb+", frame->schunk->storage->io);
3168 io_cb->seek(fp, header_len + 0, SEEK_SET);
3169 }
3170 else {
3171 // Regular frame
3172 fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io);
3173 io_cb->seek(fp, header_len + cbytes, SEEK_SET);
3174 }
3175 wbytes = io_cb->write(off_chunk, 1, (size_t)new_off_cbytes, fp); // the new offsets
3176 io_cb->close(fp);
3177 if (wbytes != (size_t)new_off_cbytes) {
3178 BLOSC_TRACE_ERROR("Cannot write the offsets to frame.");
3179 return NULL;
3180 }
3181 // Invalidate the cache for chunk offsets
3182 if (frame->coffsets != NULL) {
3183 free(frame->coffsets);
3184 frame->coffsets = NULL;
3185 }
3186 }
3187 free(off_chunk);
3188
3189 frame->len = new_frame_len;
3190 rc = frame_update_header(frame, schunk, false);
3191 if (rc < 0) {
3192 return NULL;
3193 }
3194
3195 rc = frame_update_trailer(frame, schunk);
3196 if (rc < 0) {
3197 return NULL;
3198 }
3199
3200 return frame;
3201 }
3202
3203
frame_reorder_offsets(blosc2_frame_s * frame,const int * offsets_order,blosc2_schunk * schunk)3204 int frame_reorder_offsets(blosc2_frame_s* frame, const int* offsets_order, blosc2_schunk* schunk) {
3205 // Get header info
3206 int32_t header_len;
3207 int64_t frame_len;
3208 int64_t nbytes;
3209 int64_t cbytes;
3210 int32_t blocksize;
3211 int32_t chunksize;
3212 int32_t nchunks;
3213 int ret = get_header_info(frame, &header_len, &frame_len, &nbytes, &cbytes,
3214 &blocksize, &chunksize, &nchunks,
3215 NULL, NULL, NULL, NULL, NULL, NULL,
3216 frame->schunk->storage->io);
3217 if (ret < 0) {
3218 BLOSC_TRACE_ERROR("Cannot get the header info for the frame.");
3219 return ret;
3220 }
3221
3222 // Get the current offsets and add one more
3223 int32_t off_nbytes = nchunks * sizeof(int64_t);
3224 int64_t* offsets = (int64_t *) malloc((size_t)off_nbytes);
3225
3226 int32_t coffsets_cbytes = 0;
3227 uint8_t *coffsets = get_coffsets(frame, header_len, cbytes, nchunks, &coffsets_cbytes);
3228 if (coffsets == NULL) {
3229 BLOSC_TRACE_ERROR("Cannot get the offsets for the frame.");
3230 free(offsets);
3231 return BLOSC2_ERROR_DATA;
3232 }
3233
3234 // Decompress offsets
3235 blosc2_dparams off_dparams = BLOSC2_DPARAMS_DEFAULTS;
3236 blosc2_context *dctx = blosc2_create_dctx(off_dparams);
3237 int32_t prev_nbytes = blosc2_decompress_ctx(dctx, coffsets, coffsets_cbytes,
3238 offsets, off_nbytes);
3239 blosc2_free_ctx(dctx);
3240 if (prev_nbytes < 0) {
3241 free(offsets);
3242 BLOSC_TRACE_ERROR("Cannot decompress the offsets chunk.");
3243 return prev_nbytes;
3244 }
3245
3246 // Make a copy of the chunk offsets and reorder it
3247 int64_t *offsets_copy = malloc(prev_nbytes);
3248 memcpy(offsets_copy, offsets, prev_nbytes);
3249
3250 for (int i = 0; i < nchunks; ++i) {
3251 offsets[i] = offsets_copy[offsets_order[i]];
3252 }
3253 free(offsets_copy);
3254
3255 // Re-compress the offsets again
3256 blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
3257 cparams.splitmode = BLOSC_NEVER_SPLIT;
3258 cparams.typesize = sizeof(int64_t);
3259 cparams.blocksize = 16 * 1024; // based on experiments with create_frame.c bench
3260 cparams.nthreads = 4; // 4 threads seems a decent default for nowadays CPUs
3261 cparams.compcode = BLOSC_BLOSCLZ;
3262 blosc2_context* cctx = blosc2_create_cctx(cparams);
3263 void* off_chunk = malloc((size_t)off_nbytes + BLOSC_MAX_OVERHEAD);
3264 int32_t new_off_cbytes = blosc2_compress_ctx(cctx, offsets, off_nbytes,
3265 off_chunk, off_nbytes + BLOSC_MAX_OVERHEAD);
3266 blosc2_free_ctx(cctx);
3267
3268 if (new_off_cbytes < 0) {
3269 free(offsets);
3270 free(off_chunk);
3271 return new_off_cbytes;
3272 }
3273 free(offsets);
3274 int64_t new_frame_len;
3275 if (frame->sframe) {
3276 // The chunks are not in the frame
3277 new_frame_len = header_len + 0 + new_off_cbytes + frame->trailer_len;
3278 }
3279 else {
3280 new_frame_len = header_len + cbytes + new_off_cbytes + frame->trailer_len;
3281 }
3282
3283 if (frame->cframe != NULL) {
3284 uint8_t* framep = frame->cframe;
3285 /* Make space for the new chunk and copy it */
3286 frame->cframe = framep = realloc(framep, (size_t)new_frame_len);
3287 if (framep == NULL) {
3288 BLOSC_TRACE_ERROR("Cannot realloc space for the frame.");
3289 return BLOSC2_ERROR_MEMORY_ALLOC;
3290 }
3291 /* Copy the offsets */
3292 memcpy(framep + header_len + cbytes, off_chunk, (size_t)new_off_cbytes);
3293 }
3294 else {
3295 void* fp = NULL;
3296
3297 blosc2_io_cb *io_cb = blosc2_get_io_cb(frame->schunk->storage->io->id);
3298 if (io_cb == NULL) {
3299 BLOSC_TRACE_ERROR("Error getting the input/output API");
3300 return BLOSC2_ERROR_PLUGIN_IO;
3301 }
3302
3303 if (frame->sframe) {
3304 // Update the offsets chunk in the chunks frame
3305 fp = sframe_open_index(frame->urlpath, "rb+",
3306 frame->schunk->storage->io);
3307 io_cb->seek(fp, header_len + 0, SEEK_SET);
3308 }
3309 else {
3310 // Regular frame
3311 fp = io_cb->open(frame->urlpath, "rb+", frame->schunk->storage->io->params);
3312 io_cb->seek(fp, header_len + cbytes, SEEK_SET);
3313 }
3314 int64_t wbytes = io_cb->write(off_chunk, 1, new_off_cbytes, fp); // the new offsets
3315 io_cb->close(fp);
3316 if (wbytes != (size_t)new_off_cbytes) {
3317 BLOSC_TRACE_ERROR("Cannot write the offsets to frame.");
3318 return BLOSC2_ERROR_FILE_WRITE;
3319 }
3320 }
3321
3322 // Invalidate the cache for chunk offsets
3323 if (frame->coffsets != NULL) {
3324 free(frame->coffsets);
3325 frame->coffsets = NULL;
3326 }
3327 free(off_chunk);
3328
3329 frame->len = new_frame_len;
3330 int rc = frame_update_header(frame, schunk, false);
3331 if (rc < 0) {
3332 return rc;
3333 }
3334
3335 rc = frame_update_trailer(frame, schunk);
3336 if (rc < 0) {
3337 return rc;
3338 }
3339
3340 return 0;
3341 }
3342
3343
3344 /* Decompress and return a chunk that is part of a frame. */
frame_decompress_chunk(blosc2_context * dctx,blosc2_frame_s * frame,int nchunk,void * dest,int32_t nbytes)3345 int frame_decompress_chunk(blosc2_context *dctx, blosc2_frame_s* frame, int nchunk, void *dest, int32_t nbytes) {
3346 uint8_t* src;
3347 bool needs_free;
3348 int32_t chunk_nbytes;
3349 int32_t chunk_cbytes;
3350 int rc;
3351
3352 // Use a lazychunk here in order to do a potential parallel read.
3353 rc = frame_get_lazychunk(frame, nchunk, &src, &needs_free);
3354 if (rc < 0) {
3355 BLOSC_TRACE_ERROR("Cannot get the chunk in position %d.", nchunk);
3356 goto end;
3357 }
3358 chunk_cbytes = rc;
3359 if (chunk_cbytes < (signed)sizeof(int32_t)) {
3360 /* Not enough input to read `nbytes` */
3361 rc = BLOSC2_ERROR_READ_BUFFER;
3362 }
3363
3364 rc = blosc2_cbuffer_sizes(src, &chunk_nbytes, &chunk_cbytes, NULL);
3365 if (rc < 0) {
3366 goto end;
3367 }
3368
3369 /* Create a buffer for destination */
3370 if (chunk_nbytes > (size_t)nbytes) {
3371 BLOSC_TRACE_ERROR("Not enough space for decompressing in dest.");
3372 rc = BLOSC2_ERROR_WRITE_BUFFER;
3373 goto end;
3374 }
3375 /* And decompress it */
3376 dctx->header_overhead = BLOSC_EXTENDED_HEADER_LENGTH;
3377 int chunksize = rc = blosc2_decompress_ctx(dctx, src, chunk_cbytes, dest, nbytes);
3378 if (chunksize < 0 || chunksize != chunk_nbytes) {
3379 BLOSC_TRACE_ERROR("Error in decompressing chunk.");
3380 if (chunksize >= 0)
3381 rc = BLOSC2_ERROR_FAILURE;
3382 }
3383 end:
3384 if (needs_free) {
3385 free(src);
3386 }
3387 return rc;
3388 }
3389