1 /********************************************************************* 2 Blosc - Blocked Shuffling and Compression Library 3 4 Copyright (C) 2021 The Blosc Developers <blosc@blosc.org> 5 https://blosc.org 6 License: BSD 3-Clause (see LICENSE.txt) 7 8 See LICENSE.txt for details about copyright and rights to use. 9 **********************************************************************/ 10 11 /********************************************************************* 12 @file blosc.h 13 @brief Blosc header file. 14 15 This file contains Blosc public API and the structures needed to use it. 16 @author The Blosc Developers <blosc@blosc.org> 17 **********************************************************************/ 18 19 20 #ifndef BLOSC_H 21 #define BLOSC_H 22 23 #include <limits.h> 24 #include <stdlib.h> 25 #include <stdint.h> 26 #include <stdbool.h> 27 #include <stdio.h> 28 #include "blosc2/blosc2-export.h" 29 #include "blosc2/blosc2-common.h" 30 #include "blosc2/blosc2-stdio.h" 31 32 #if defined(_WIN32) && !defined(__MINGW32__) 33 #include <windows.h> 34 #include <malloc.h> 35 36 #include <process.h> 37 #define getpid _getpid 38 #endif 39 40 #ifdef __cplusplus 41 extern "C" { 42 #endif 43 44 45 /* Version numbers */ 46 #define BLOSC_VERSION_MAJOR 2 /* for major interface/format changes */ 47 #define BLOSC_VERSION_MINOR 0 /* for minor interface/format changes */ 48 #define BLOSC_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */ 49 50 #define BLOSC_VERSION_STRING "2.0.4" /* string version. Sync with above! */ 51 #define BLOSC_VERSION_DATE "$Date:: 2021-10-02 #$" /* date version */ 52 53 54 /* Tracing macros */ 55 #define BLOSC_TRACE_ERROR(msg, ...) BLOSC_TRACE(error, msg, ##__VA_ARGS__) 56 #define BLOSC_TRACE_WARNING(msg, ...) BLOSC_TRACE(warning, msg, ##__VA_ARGS__) 57 #define BLOSC_TRACE(cat, msg, ...) \ 58 do { \ 59 const char *__e = getenv("BLOSC_TRACE"); \ 60 if (!__e) { break; } \ 61 fprintf(stderr, "[%s] - " msg " (%s:%d)\n", #cat, ##__VA_ARGS__, __FILE__, __LINE__); \ 62 } while(0) 63 64 65 /* The VERSION_FORMAT symbols below should be just 1-byte long */ 66 enum { 67 /* Blosc format version, starting at 1 68 1 -> Blosc pre-1.0 69 2 -> Blosc 1.x stable series 70 3 -> Blosc 2-alpha.x series 71 4 -> Blosc 2.x beta.1 series 72 5 -> Blosc 2.x stable series 73 */ 74 BLOSC_VERSION_FORMAT_PRE1 = 1, 75 BLOSC1_VERSION_FORMAT = 2, 76 BLOSC2_VERSION_FORMAT_ALPHA = 3, 77 BLOSC2_VERSION_FORMAT_BETA1 = 4, 78 BLOSC2_VERSION_FORMAT_STABLE = 5, 79 BLOSC_VERSION_FORMAT = BLOSC2_VERSION_FORMAT_STABLE, 80 }; 81 82 83 /* The FRAME_FORMAT_VERSION symbols below should be just 4-bit long */ 84 enum { 85 /* Blosc format version 86 * 1 -> First version (introduced in beta.2) 87 * 2 -> Second version (introduced in rc.1) 88 * 89 */ 90 BLOSC2_VERSION_FRAME_FORMAT_BETA2 = 1, // for 2.0.0-beta2 and after 91 BLOSC2_VERSION_FRAME_FORMAT_RC1 = 2, // for 2.0.0-rc1 and after 92 BLOSC2_VERSION_FRAME_FORMAT = BLOSC2_VERSION_FRAME_FORMAT_RC1, 93 }; 94 95 enum { 96 BLOSC_MIN_HEADER_LENGTH = 16, 97 //!< Minimum header length (Blosc1) 98 BLOSC_EXTENDED_HEADER_LENGTH = 32, 99 //!< Extended header length (Blosc2, see README_HEADER) 100 BLOSC_MAX_OVERHEAD = BLOSC_EXTENDED_HEADER_LENGTH, 101 //!< The maximum overhead during compression in bytes. This equals 102 //!< to @ref BLOSC_EXTENDED_HEADER_LENGTH now, but can be higher in future 103 //!< implementations. 104 BLOSC_MAX_BUFFERSIZE = (INT_MAX - BLOSC_MAX_OVERHEAD), 105 //!< Maximum source buffer size to be compressed 106 BLOSC_MAX_TYPESIZE = 255, 107 //!< Maximum typesize before considering source buffer as a stream of bytes. 108 //!< Cannot be larger than 255. 109 BLOSC_MIN_BUFFERSIZE = 128, 110 //!< Minimum buffer size to be compressed. Cannot be smaller than 66. 111 }; 112 113 114 enum { 115 BLOSC2_DEFINED_FILTERS_START = 0, 116 BLOSC2_DEFINED_FILTERS_STOP = 31, 117 //!< Blosc-defined filters must be between 0 - 31. 118 BLOSC2_GLOBAL_REGISTERED_FILTERS_START = 32, 119 BLOSC2_GLOBAL_REGISTERED_FILTERS_STOP = 159, 120 //!< Blosc-registered filters must be between 32 - 159. 121 BLOSC2_GLOBAL_REGISTERED_FILTERS = 2, 122 //!< Number of Blosc-registered filters at the moment. 123 BLOSC2_USER_REGISTERED_FILTERS_START = 128, 124 BLOSC2_USER_REGISTERED_FILTERS_STOP = 255, 125 //!< User-defined filters must be between 128 - 255. 126 BLOSC2_MAX_FILTERS = 6, 127 //!< Maximum number of filters in the filter pipeline 128 BLOSC2_MAX_UDFILTERS = 16, 129 //!< Maximum number of filters that a user can register. 130 }; 131 132 133 /** 134 * @brief Codes for filters. 135 * 136 * @sa #blosc_compress 137 */ 138 enum { 139 BLOSC_NOSHUFFLE = 0, //!< No shuffle (for compatibility with Blosc1). 140 BLOSC_NOFILTER = 0, //!< No filter. 141 BLOSC_SHUFFLE = 1, //!< Byte-wise shuffle. 142 BLOSC_BITSHUFFLE = 2, //!< Bit-wise shuffle. 143 BLOSC_DELTA = 3, //!< Delta filter. 144 BLOSC_TRUNC_PREC = 4, //!< Truncate precision filter. 145 BLOSC_LAST_FILTER = 5, //!< sentinel 146 BLOSC_LAST_REGISTERED_FILTER = BLOSC2_GLOBAL_REGISTERED_FILTERS_START + BLOSC2_GLOBAL_REGISTERED_FILTERS - 1, 147 //!< Determine the last registered filter. It is used to check if a filter is registered or not. 148 }; 149 150 /** 151 * @brief Codes for internal flags (see blosc_cbuffer_metainfo) 152 */ 153 enum { 154 BLOSC_DOSHUFFLE = 0x1, //!< byte-wise shuffle 155 BLOSC_MEMCPYED = 0x2, //!< plain copy 156 BLOSC_DOBITSHUFFLE = 0x4, //!< bit-wise shuffle 157 BLOSC_DODELTA = 0x8, //!< delta coding 158 }; 159 160 /** 161 * @brief Codes for new internal flags in Blosc2 162 */ 163 enum { 164 BLOSC2_USEDICT = 0x1, //!< use dictionaries with codec 165 BLOSC2_BIGENDIAN = 0x2, //!< data is in big-endian ordering 166 }; 167 168 /** 169 * @brief Values for different Blosc2 capabilities 170 */ 171 enum { 172 BLOSC2_MAXDICTSIZE = 128 * 1024, //!< maximum size for compression dicts 173 BLOSC2_MAXBLOCKSIZE = 536866816 //!< maximum size for blocks 174 }; 175 176 177 enum { 178 BLOSC2_DEFINED_CODECS_START = 0, 179 BLOSC2_DEFINED_CODECS_STOP = 31, 180 //!< Blosc-defined codecs must be between 0 - 31. 181 BLOSC2_GLOBAL_REGISTERED_CODECS_START = 32, 182 BLOSC2_GLOBAL_REGISTERED_CODECS_STOP = 159, 183 //!< Blosc-registered codecs must be between 31 - 159. 184 BLOSC2_GLOBAL_REGISTERED_CODECS = 1, 185 //!< Number of Blosc-registered codecs at the moment. 186 BLOSC2_USER_REGISTERED_CODECS_START = 160, 187 BLOSC2_USER_REGISTERED_CODECS_STOP = 255, 188 //!< User-defined codecs must be between 160 - 255. 189 }; 190 191 /** 192 * @brief Codes for the different compressors shipped with Blosc 193 */ 194 enum { 195 BLOSC_BLOSCLZ = 0, 196 BLOSC_LZ4 = 1, 197 BLOSC_LZ4HC = 2, 198 BLOSC_ZLIB = 4, 199 BLOSC_ZSTD = 5, 200 BLOSC_LAST_CODEC = 6, 201 //!< Determine the last codec defined by Blosc. 202 BLOSC_LAST_REGISTERED_CODEC = BLOSC2_GLOBAL_REGISTERED_CODECS_START + BLOSC2_GLOBAL_REGISTERED_CODECS - 1, 203 //!< Determine the last registered codec. It is used to check if a codec is registered or not. 204 }; 205 206 207 // Names for the different compressors shipped with Blosc 208 209 #define BLOSC_BLOSCLZ_COMPNAME "blosclz" 210 #define BLOSC_LZ4_COMPNAME "lz4" 211 #define BLOSC_LZ4HC_COMPNAME "lz4hc" 212 #define BLOSC_ZLIB_COMPNAME "zlib" 213 #define BLOSC_ZSTD_COMPNAME "zstd" 214 215 /** 216 * @brief Codes for compression libraries shipped with Blosc (code must be < 8) 217 */ 218 enum { 219 BLOSC_BLOSCLZ_LIB = 0, 220 BLOSC_LZ4_LIB = 1, 221 BLOSC_ZLIB_LIB = 3, 222 BLOSC_ZSTD_LIB = 4, 223 BLOSC_UDCODEC_LIB = 6, 224 BLOSC_SCHUNK_LIB = 7, //!< compressor library in super-chunk header 225 }; 226 227 /** 228 * @brief Names for the different compression libraries shipped with Blosc 229 */ 230 #define BLOSC_BLOSCLZ_LIBNAME "BloscLZ" 231 #define BLOSC_LZ4_LIBNAME "LZ4" 232 #define BLOSC_ZLIB_LIBNAME "Zlib" 233 #define BLOSC_ZSTD_LIBNAME "Zstd" 234 235 /** 236 * @brief The codes for compressor formats shipped with Blosc 237 */ 238 enum { 239 BLOSC_BLOSCLZ_FORMAT = BLOSC_BLOSCLZ_LIB, 240 BLOSC_LZ4_FORMAT = BLOSC_LZ4_LIB, 241 //!< LZ4HC and LZ4 share the same format 242 BLOSC_LZ4HC_FORMAT = BLOSC_LZ4_LIB, 243 BLOSC_ZLIB_FORMAT = BLOSC_ZLIB_LIB, 244 BLOSC_ZSTD_FORMAT = BLOSC_ZSTD_LIB, 245 BLOSC_UDCODEC_FORMAT = BLOSC_UDCODEC_LIB, 246 }; 247 248 /** 249 * @brief The version formats for compressors shipped with Blosc. 250 * All versions here starts at 1 251 */ 252 enum { 253 BLOSC_BLOSCLZ_VERSION_FORMAT = 1, 254 BLOSC_LZ4_VERSION_FORMAT = 1, 255 BLOSC_LZ4HC_VERSION_FORMAT = 1, /* LZ4HC and LZ4 share the same format */ 256 BLOSC_ZLIB_VERSION_FORMAT = 1, 257 BLOSC_ZSTD_VERSION_FORMAT = 1, 258 BLOSC_UDCODEC_VERSION_FORMAT = 1, 259 }; 260 261 /** 262 * @brief Split mode for blocks. 263 * NEVER and ALWAYS are for experimenting with compression ratio. 264 * AUTO for nearly optimal behaviour (based on heuristics). 265 * FORWARD_COMPAT provides best forward compatibility (default). 266 */ 267 enum { 268 BLOSC_ALWAYS_SPLIT = 1, 269 BLOSC_NEVER_SPLIT = 2, 270 BLOSC_AUTO_SPLIT = 3, 271 BLOSC_FORWARD_COMPAT_SPLIT = 4, 272 }; 273 274 /** 275 * @brief Offsets for fields in Blosc2 chunk header. 276 */ 277 enum { 278 BLOSC2_CHUNK_VERSION = 0x0, //!< the version for the chunk format 279 BLOSC2_CHUNK_VERSIONLZ = 0x1, //!< the version for the format of internal codec 280 BLOSC2_CHUNK_FLAGS = 0x2, //!< flags and codec info 281 BLOSC2_CHUNK_TYPESIZE = 0x3, //!< (uint8) the number of bytes of the atomic type 282 BLOSC2_CHUNK_NBYTES = 0x4, //!< (int32) uncompressed size of the buffer (this header is not included) 283 BLOSC2_CHUNK_BLOCKSIZE = 0x8, //!< (int32) size of internal blocks 284 BLOSC2_CHUNK_CBYTES = 0xc, //!< (int32) compressed size of the buffer (including this header) 285 BLOSC2_CHUNK_FILTER_CODES = 0x10, //!< the codecs for the filter pipeline (1 byte per code) 286 BLOSC2_CHUNK_FILTER_META = 0x18, //!< meta info for the filter pipeline (1 byte per code) 287 BLOSC2_CHUNK_BLOSC2_FLAGS = 0x1F, //!< flags specific for Blosc2 functionality 288 }; 289 290 /** 291 * @brief Run lengths for special values for chunks/frames 292 */ 293 enum { 294 BLOSC2_NO_SPECIAL = 0x0, //!< no special value 295 BLOSC2_SPECIAL_ZERO = 0x1, //!< zero special value 296 BLOSC2_SPECIAL_NAN = 0x2, //!< NaN special value 297 BLOSC2_SPECIAL_VALUE = 0x3, //!< generic special value 298 BLOSC2_SPECIAL_UNINIT = 0x4, //!< non initialized values 299 BLOSC2_SPECIAL_LASTID = 0x4, //!< last valid ID for special value (update this adequately) 300 BLOSC2_SPECIAL_MASK = 0x7 //!< special value mask (prev IDs cannot be larger than this) 301 }; 302 303 /** 304 * @brief Error codes 305 */ 306 enum { 307 BLOSC2_ERROR_SUCCESS = 0, //<! Success 308 BLOSC2_ERROR_FAILURE = -1, //<! Generic failure 309 BLOSC2_ERROR_STREAM = 2, //<! Bad stream 310 BLOSC2_ERROR_DATA = -3, //<! Invalid data 311 BLOSC2_ERROR_MEMORY_ALLOC = -4, //<! Memory alloc/realloc failure 312 BLOSC2_ERROR_READ_BUFFER = -5, //!< Not enough space to read 313 BLOSC2_ERROR_WRITE_BUFFER = -6, //!< Not enough space to write 314 BLOSC2_ERROR_CODEC_SUPPORT = -7, //!< Codec not supported 315 BLOSC2_ERROR_CODEC_PARAM = -8, //!< Invalid parameter supplied to codec 316 BLOSC2_ERROR_CODEC_DICT = -9, //!< Codec dictionary error 317 BLOSC2_ERROR_VERSION_SUPPORT = -10, //!< Version not supported 318 BLOSC2_ERROR_INVALID_HEADER = -11, //!< Invalid value in header 319 BLOSC2_ERROR_INVALID_PARAM = -12, //!< Invalid parameter supplied to function 320 BLOSC2_ERROR_FILE_READ = -13, //!< File read failure 321 BLOSC2_ERROR_FILE_WRITE = -14, //!< File write failure 322 BLOSC2_ERROR_FILE_OPEN = -15, //!< File open failure 323 BLOSC2_ERROR_NOT_FOUND = -16, //!< Not found 324 BLOSC2_ERROR_RUN_LENGTH = -17, //!< Bad run length encoding 325 BLOSC2_ERROR_FILTER_PIPELINE = -18, //!< Filter pipeline error 326 BLOSC2_ERROR_CHUNK_INSERT = -19, //!< Chunk insert failure 327 BLOSC2_ERROR_CHUNK_APPEND = -20, //!< Chunk append failure 328 BLOSC2_ERROR_CHUNK_UPDATE = -21, //!< Chunk update failure 329 BLOSC2_ERROR_2GB_LIMIT = -22, //!< Sizes larger than 2gb not supported 330 BLOSC2_ERROR_SCHUNK_COPY = -23, //!< Super-chunk copy failure 331 BLOSC2_ERROR_FRAME_TYPE = -24, //!< Wrong type for frame 332 BLOSC2_ERROR_FILE_TRUNCATE = -25, //!< File truncate failure 333 BLOSC2_ERROR_THREAD_CREATE = -26, //!< Thread or thread context creation failure 334 BLOSC2_ERROR_POSTFILTER = -27, //!< Postfilter failure 335 BLOSC2_ERROR_FRAME_SPECIAL = -28, //!< Special frame failure 336 BLOSC2_ERROR_SCHUNK_SPECIAL = -29, //!< Special super-chunk failure 337 BLOSC2_ERROR_PLUGIN_IO = -30, //!< IO plugin error 338 BLOSC2_ERROR_FILE_REMOVE = -31, //!< Remove file failure 339 }; 340 341 /** 342 * @brief Initialize the Blosc library environment. 343 * 344 * You must call this previous to any other Blosc call, unless you want 345 * Blosc to be used simultaneously in a multi-threaded environment, in 346 * which case you can use the #blosc2_compress_ctx #blosc2_decompress_ctx pair. 347 * 348 * @sa #blosc_destroy 349 */ 350 BLOSC_EXPORT void blosc_init(void); 351 352 353 /** 354 * @brief Destroy the Blosc library environment. 355 * 356 * You must call this after to you are done with all the Blosc calls, 357 * unless you have not used blosc_init() before. 358 * 359 * @sa #blosc_init 360 */ 361 BLOSC_EXPORT void blosc_destroy(void); 362 363 364 /** 365 * @brief Compress a block of data in the @p src buffer and returns the size of 366 * compressed block. 367 * 368 * @remark Compression is memory safe and guaranteed not to write @p dest 369 * more than what is specified in @p destsize. 370 * There is not a minimum for @p src buffer size @p nbytes. 371 * 372 * @warning The @p src buffer and the @p dest buffer can not overlap. 373 * 374 * @param clevel The desired compression level and must be a number 375 * between 0 (no compression) and 9 (maximum compression). 376 * @param doshuffle Specifies whether the shuffle compression preconditioner 377 * should be applied or not. #BLOSC_NOFILTER means not applying filters, 378 * #BLOSC_SHUFFLE means applying shuffle at a byte level and 379 * #BLOSC_BITSHUFFLE at a bit level (slower but *may* achieve better 380 * compression). 381 * @param typesize Is the number of bytes for the atomic type in binary 382 * @p src buffer. This is mainly useful for the shuffle preconditioner. 383 * For implementation reasons, only a 1 < typesize < 256 will allow the 384 * shuffle filter to work. When typesize is not in this range, shuffle 385 * will be silently disabled. 386 * @param nbytes The number of bytes to compress in the @p src buffer. 387 * @param src The buffer containing the data to compress. 388 * @param dest The buffer where the compressed data will be put, 389 * must have at least the size of @p destsize. 390 * @param destsize The size of the dest buffer. Blosc 391 * guarantees that if you set @p destsize to, at least, 392 * (@p nbytes + #BLOSC_MAX_OVERHEAD), the compression will always succeed. 393 * 394 * @return The number of bytes compressed. 395 * If @p src buffer cannot be compressed into @p destsize, the return 396 * value is zero and you should discard the contents of the @p dest 397 * buffer. A negative return value means that an internal error happened. This 398 * should never happen. If you see this, please report it back 399 * together with the buffer data causing this and compression settings. 400 * 401 * 402 * @par Environment variables 403 * @parblock 404 * 405 * This function honors different environment variables to control 406 * internal parameters without the need of doing that programatically. 407 * Here are the ones supported: 408 * 409 * * **BLOSC_CLEVEL=(INTEGER)**: This will overwrite the @p clevel parameter 410 * before the compression process starts. 411 * 412 * * **BLOSC_SHUFFLE=[NOSHUFFLE | SHUFFLE | BITSHUFFLE]**: This will 413 * overwrite the @p doshuffle parameter before the compression process 414 * starts. 415 * 416 * * **BLOSC_DELTA=(1|0)**: This will call #blosc_set_delta() before the 417 * compression process starts. 418 * 419 * * **BLOSC_TYPESIZE=(INTEGER)**: This will overwrite the @p typesize 420 * parameter before the compression process starts. 421 * 422 * * **BLOSC_COMPRESSOR=[BLOSCLZ | LZ4 | LZ4HC | SNAPPY | ZLIB | ZSTD]**: 423 * This will call #blosc_set_compressor(BLOSC_COMPRESSOR) before the 424 * compression process starts. 425 * 426 * * **BLOSC_NTHREADS=(INTEGER)**: This will call 427 * #blosc_set_nthreads(BLOSC_NTHREADS) before the compression process 428 * starts. 429 * 430 * * **BLOSC_BLOCKSIZE=(INTEGER)**: This will call 431 * #blosc_set_blocksize(BLOSC_BLOCKSIZE) before the compression process 432 * starts. *NOTE:* The *blocksize* is a critical parameter with 433 * important restrictions in the allowed values, so use this with care. 434 * 435 * * **BLOSC_NOLOCK=(ANY VALUE)**: This will call *blosc2_compress_ctx()* under 436 * the hood, with the *compressor*, *blocksize* and 437 * *numinternalthreads* parameters set to the same as the last calls to 438 * #blosc_set_compressor, #blosc_set_blocksize and 439 * #blosc_set_nthreads. *BLOSC_CLEVEL*, *BLOSC_SHUFFLE*, *BLOSC_DELTA* and 440 * *BLOSC_TYPESIZE* environment vars will also be honored. 441 * 442 * @endparblock 443 * 444 * @sa blosc_decompress 445 */ 446 BLOSC_EXPORT int blosc_compress(int clevel, int doshuffle, size_t typesize, 447 size_t nbytes, const void* src, void* dest, 448 size_t destsize); 449 450 451 /** 452 * @brief Decompress a block of compressed data in @p src, put the result in 453 * @p dest and returns the size of the decompressed block. 454 * 455 * @warning The @p src buffer and the @p dest buffer can not overlap. 456 * 457 * @remark Decompression is memory safe and guaranteed not to write the @p dest 458 * buffer more than what is specified in @p destsize. 459 * 460 * @remark In case you want to keep under control the number of bytes read from 461 * source, you can call #blosc_cbuffer_sizes first to check whether the 462 * @p nbytes (i.e. the number of bytes to be read from @p src buffer by this 463 * function) in the compressed buffer is ok with you. 464 * 465 * @param src The buffer to be decompressed. 466 * @param dest The buffer where the decompressed data will be put. 467 * @param destsize The size of the @p dest buffer. 468 * 469 * @return The number of bytes decompressed. 470 * If an error occurs, e.g. the compressed data is corrupted or the 471 * output buffer is not large enough, then a negative value 472 * will be returned instead. 473 * 474 * @par Environment variables 475 * @parblock 476 * This function honors different environment variables to control 477 * internal parameters without the need of doing that programatically. 478 * Here are the ones supported: 479 * 480 * * **BLOSC_NTHREADS=(INTEGER)**: This will call 481 * #blosc_set_nthreads(BLOSC_NTHREADS) before the proper decompression 482 * process starts. 483 * 484 * * **BLOSC_NOLOCK=(ANY VALUE)**: This will call #blosc2_decompress_ctx 485 * under the hood, with the *numinternalthreads* parameter set to the 486 * same value as the last call to #blosc_set_nthreads. 487 * 488 * @endparblock 489 * 490 * @sa blosc_compress 491 */ 492 BLOSC_EXPORT int blosc_decompress(const void* src, void* dest, size_t destsize); 493 494 495 /** 496 * @brief Get @p nitems (of @p typesize size) in @p src buffer starting in @p start. 497 * The items are returned in @p dest buffer, which has to have enough 498 * space for storing all items. 499 * 500 * @param src The compressed buffer from data will be decompressed. 501 * @param start The position of the first item (of @p typesize size) from where data 502 * will be retrieved. 503 * @param nitems The number of items (of @p typesize size) that will be retrieved. 504 * @param dest The buffer where the decompressed data retrieved will be put. 505 * 506 * @return The number of bytes copied to @p dest or a negative value if 507 * some error happens. 508 */ 509 BLOSC_EXPORT int blosc_getitem(const void* src, int start, int nitems, void* dest); 510 511 /** 512 * @brief Get @p nitems (of @p typesize size) in @p src buffer starting in @p start. 513 * The items are returned in @p dest buffer. The dest buffer should have enough space 514 * for storing all items. This function is a more secure version of #blosc_getitem. 515 * 516 * @param src The compressed buffer holding the data to be retrieved. 517 * @param srcsize Size of the compressed buffer. 518 * @param start The position of the first item (of @p typesize size) from where data 519 * will be retrieved. 520 * @param nitems The number of items (of @p typesize size) that will be retrieved. 521 * @param dest The buffer where the retrieved data will be stored decompressed. 522 * @param destsize Size of the buffer where retrieved data will be stored. 523 * 524 * @return The number of bytes copied to @p dest or a negative value if 525 * some error happens. 526 */ 527 BLOSC_EXPORT int blosc2_getitem(const void* src, int32_t srcsize, int start, int nitems, 528 void* dest, int32_t destsize); 529 530 /** 531 Pointer to a callback function that executes `dojob(jobdata + i*jobdata_elsize)` for `i = 0 to numjobs-1`, 532 possibly in parallel threads (but not returning until all `dojob` calls have returned). This allows the 533 caller to provide a custom threading backend as an alternative to the default Blosc-managed threads. 534 `callback_data` is passed through from `blosc_set_threads_callback`. 535 */ 536 typedef void (*blosc_threads_callback)(void *callback_data, void (*dojob)(void *), int numjobs, size_t jobdata_elsize, void *jobdata); 537 538 /** 539 Set the threading backend for parallel compression/decompression to use `callback` to execute work 540 instead of using the Blosc-managed threads. This function is *not* thread-safe and should be called 541 before any other Blosc function: it affects all Blosc contexts. Passing `NULL` uses the default 542 Blosc threading backend. The `callback_data` argument is passed through to the callback. 543 */ 544 BLOSC_EXPORT void blosc_set_threads_callback(blosc_threads_callback callback, void *callback_data); 545 546 547 /** 548 * @brief Returns the current number of threads that are used for 549 * compression/decompression. 550 */ 551 BLOSC_EXPORT int16_t blosc_get_nthreads(void); 552 553 554 /** 555 * @brief Initialize a pool of threads for compression/decompression. If 556 * @p nthreads is 1, then the serial version is chosen and a possible 557 * previous existing pool is ended. If this is not called, @p nthreads 558 * is set to 1 internally. 559 * 560 * @param nthreads The number of threads to use. 561 * 562 * @return The previous number of threads. 563 */ 564 BLOSC_EXPORT int16_t blosc_set_nthreads(int16_t nthreads); 565 566 567 /** 568 * @brief Get the current compressor that is used for compression. 569 * 570 * @return The string identifying the compressor being used. 571 */ 572 BLOSC_EXPORT const char* blosc_get_compressor(void); 573 574 575 /** 576 * @brief Select the compressor to be used. The supported ones are "blosclz", 577 * "lz4", "lz4hc", "zlib" and "ztsd". If this function is not 578 * called, then "blosclz" will be used. 579 * 580 * @param compname The name identifier of the compressor to be set. 581 * 582 * @return The code for the compressor (>=0). In case the compressor 583 * is not recognized, or there is not support for it in this build, 584 * it returns a -1. 585 */ 586 BLOSC_EXPORT int blosc_set_compressor(const char* compname); 587 588 589 /** 590 * @brief Select the delta coding filter to be used. 591 * 592 * @param dodelta A value >0 will activate the delta filter. 593 * If 0, it will be de-activated 594 * 595 * This call should always succeed. 596 */ 597 BLOSC_EXPORT void blosc_set_delta(int dodelta); 598 599 600 /** 601 * @brief Get the compressor name associated with the compressor code. 602 * 603 * @param compcode The code identifying the compressor 604 * @param compname The pointer to a string where the compressor name will be put. 605 * 606 * @return The compressor code. If the compressor code is not recognized, 607 * or there is not support for it in this build, -1 is returned. 608 */ 609 BLOSC_EXPORT int blosc_compcode_to_compname(int compcode, const char** compname); 610 611 612 /** 613 * @brief Get the compressor code associated with the compressor name. 614 * 615 * @param compname The string containing the compressor name. 616 * 617 * @return The compressor code. If the compressor name is not recognized, 618 * or there is not support for it in this build, -1 is returned instead. 619 */ 620 BLOSC_EXPORT int blosc_compname_to_compcode(const char* compname); 621 622 623 /** 624 * @brief Get a list of compressors supported in the current build. 625 * 626 * @return The comma separated string with the list of compressor names 627 * supported. 628 * 629 * This function does not leak, so you should not free() the returned 630 * list. 631 * 632 * This function should always succeed. 633 */ 634 BLOSC_EXPORT const char* blosc_list_compressors(void); 635 636 637 /** 638 * @brief Get the version of Blosc in string format. 639 * 640 * @return The string with the current Blosc version. 641 * Useful for dynamic libraries. 642 */ 643 BLOSC_EXPORT const char* blosc_get_version_string(void); 644 645 646 /** 647 * @brief Get info from compression libraries included in the current build. 648 * 649 * @param compname The compressor name that you want info from. 650 * @param complib The pointer to a string where the 651 * compression library name, if available, will be put. 652 * @param version The pointer to a string where the 653 * compression library version, if available, will be put. 654 * 655 * @warning You are in charge of the @p complib and @p version strings, 656 * you should free() them so as to avoid leaks. 657 * 658 * @return The code for the compression library (>=0). If it is not supported, 659 * this function returns -1. 660 */ 661 BLOSC_EXPORT int blosc_get_complib_info(const char* compname, char** complib, 662 char** version); 663 664 665 /** 666 * @brief Free possible memory temporaries and thread resources. Use this 667 * when you are not going to use Blosc for a long while. 668 * 669 * @return A 0 if succeeds, in case of problems releasing the resources, 670 * it returns a negative number. 671 */ 672 BLOSC_EXPORT int blosc_free_resources(void); 673 674 675 /** 676 * @brief Get information about a compressed buffer, namely the number of 677 * uncompressed bytes (@p nbytes) and compressed (@p cbytes). It also 678 * returns the @p blocksize (which is used internally for doing the 679 * compression by blocks). 680 * 681 * @param cbuffer The buffer of compressed data. 682 * @param nbytes The pointer where the number of uncompressed bytes will be put. 683 * @param cbytes The pointer where the number of compressed bytes will be put. 684 * @param blocksize The pointer where the block size will be put. 685 * 686 * You only need to pass the first BLOSC_EXTENDED_HEADER_LENGTH bytes of a 687 * compressed buffer for this call to work. 688 * 689 * This function should always succeed. 690 */ 691 BLOSC_EXPORT void blosc_cbuffer_sizes(const void* cbuffer, size_t* nbytes, 692 size_t* cbytes, size_t* blocksize); 693 /** 694 * @brief Get information about a compressed buffer, namely the number of 695 * uncompressed bytes (@p nbytes) and compressed (@p cbytes). It also 696 * returns the @p blocksize (which is used internally for doing the 697 * compression by blocks). 698 * 699 * @param cbuffer The buffer of compressed data. 700 * @param nbytes The pointer where the number of uncompressed bytes will be put. 701 * @param cbytes The pointer where the number of compressed bytes will be put. 702 * @param blocksize The pointer where the block size will be put. 703 * 704 * You only need to pass the first BLOSC_EXTENDED_HEADER_LENGTH bytes of a 705 * compressed buffer for this call to work. 706 * 707 * @return On failure, returns negative value. 708 */ 709 BLOSC_EXPORT int blosc2_cbuffer_sizes(const void* cbuffer, int32_t* nbytes, 710 int32_t* cbytes, int32_t* blocksize); 711 712 /** 713 * @brief Checks that the compressed buffer starting at @cbuffer of length @cbytes may 714 * contain valid blosc compressed data, and that it is safe to call 715 * blosc_decompress/blosc_decompress_ctx/blosc_getitem. 716 * On success, returns 0 and sets @nbytes to the size of the uncompressed data. 717 * This does not guarantee that the decompression function won't return an error, 718 * but does guarantee that it is safe to attempt decompression. 719 * 720 * @param cbuffer The buffer of compressed data. 721 * @param cbytes The number of compressed bytes. 722 * @param nbytes The pointer where the number of uncompressed bytes will be put. 723 * 724 * @return On failure, returns negative value. 725 */ 726 BLOSC_EXPORT int blosc_cbuffer_validate(const void* cbuffer, size_t cbytes, 727 size_t* nbytes); 728 729 /** 730 * @brief Get information about a compressed buffer, namely the type size 731 * (@p typesize), as well as some internal @p flags. 732 * 733 * @param cbuffer The buffer of compressed data. 734 * @param typesize The pointer where the type size will be put. 735 * @param flags The pointer of the integer where the additional info is encoded. 736 * The @p flags is a set of bits, where the currently used ones are: 737 * * bit 0: whether the shuffle filter has been applied or not 738 * * bit 1: whether the internal buffer is a pure memcpy or not 739 * * bit 2: whether the bitshuffle filter has been applied or not 740 * * bit 3: whether the delta coding filter has been applied or not 741 * 742 * You can use the @p BLOSC_DOSHUFFLE, @p BLOSC_DOBITSHUFFLE, @p BLOSC_DODELTA 743 * and @p BLOSC_MEMCPYED symbols for extracting the interesting bits 744 * (e.g. @p flags & @p BLOSC_DOSHUFFLE says whether the buffer is byte-shuffled 745 * or not). 746 * 747 * This function should always succeed. 748 */ 749 BLOSC_EXPORT void blosc_cbuffer_metainfo(const void* cbuffer, size_t* typesize, 750 int* flags); 751 752 753 /** 754 * @brief Get information about a compressed buffer, namely the internal 755 * Blosc format version (@p version) and the format for the internal 756 * Lempel-Ziv compressor used (@p versionlz). 757 * 758 * @param cbuffer The buffer of compressed data. 759 * @param version The pointer where the Blosc format version will be put. 760 * @param versionlz The pointer where the Lempel-Ziv version will be put. 761 * 762 * This function should always succeed. 763 */ 764 BLOSC_EXPORT void blosc_cbuffer_versions(const void* cbuffer, int* version, 765 int* versionlz); 766 767 768 /** 769 * @brief Get the compressor library/format used in a compressed buffer. 770 * 771 * @param cbuffer The buffer of compressed data. 772 * 773 * @return The string identifying the compressor library/format used. 774 * 775 * This function should always succeed. 776 */ 777 BLOSC_EXPORT const char* blosc_cbuffer_complib(const void* cbuffer); 778 779 /********************************************************************* 780 Structures and functions related with user-defined input/output. 781 *********************************************************************/ 782 783 enum { 784 BLOSC2_IO_FILESYSTEM = 0, 785 BLOSC_IO_LAST_BLOSC_DEFINED = 1, // sentinel 786 BLOSC_IO_LAST_REGISTERED = 32, // sentinel 787 }; 788 789 enum { 790 BLOSC2_IO_BLOSC_DEFINED = 32, 791 BLOSC2_IO_REGISTERED = 160, 792 BLOSC2_IO_USER_DEFINED = 256 793 }; 794 795 typedef void* (*blosc2_open_cb)(const char *urlpath, const char *mode, void *params); 796 typedef int (*blosc2_close_cb)(void *stream); 797 typedef int64_t (*blosc2_tell_cb)(void *stream); 798 typedef int (*blosc2_seek_cb)(void *stream, int64_t offset, int whence); 799 typedef int64_t (*blosc2_write_cb)(const void *ptr, int64_t size, int64_t nitems, void *stream); 800 typedef int64_t (*blosc2_read_cb)(void *ptr, int64_t size, int64_t nitems, void *stream); 801 typedef int (*blosc2_truncate_cb)(void *stream, int64_t size); 802 803 804 /* 805 * Input/Ouput callbacks. 806 */ 807 typedef struct { 808 uint8_t id; 809 //!< The IO identifier. 810 blosc2_open_cb open; 811 //!< The IO open callback. 812 blosc2_close_cb close; 813 //!< The IO close callback. 814 blosc2_tell_cb tell; 815 //!< The IO tell callback. 816 blosc2_seek_cb seek; 817 //!< The IO seek callback. 818 blosc2_write_cb write; 819 //!< The IO write callback. 820 blosc2_read_cb read; 821 //!< The IO read callback. 822 blosc2_truncate_cb truncate; 823 //!< The IO truncate callback. 824 } blosc2_io_cb; 825 826 827 /* 828 * Input/Output parameters. 829 */ 830 typedef struct { 831 uint8_t id; 832 //!< The IO identifier. 833 void *params; 834 //!< The IO parameters. 835 } blosc2_io; 836 837 static const blosc2_io_cb BLOSC2_IO_CB_DEFAULTS = { 838 .id = BLOSC2_IO_FILESYSTEM, 839 .open = (blosc2_open_cb) blosc2_stdio_open, 840 .close = (blosc2_close_cb) blosc2_stdio_close, 841 .tell = (blosc2_tell_cb) blosc2_stdio_tell, 842 .seek = (blosc2_seek_cb) blosc2_stdio_seek, 843 .write = (blosc2_write_cb) blosc2_stdio_write, 844 .read = (blosc2_read_cb) blosc2_stdio_read, 845 .truncate = (blosc2_truncate_cb) blosc2_stdio_truncate, 846 }; 847 848 static const blosc2_io BLOSC2_IO_DEFAULTS = { 849 .id = BLOSC2_IO_FILESYSTEM, 850 .params = NULL, 851 }; 852 853 854 /** 855 * @brief Register a user-defined input/output callbacks in Blosc. 856 * 857 * @param filter The callbacks API to register. 858 * 859 * @return 0 if succeeds. Else a negative code is returned. 860 */ 861 BLOSC_EXPORT int blosc2_register_io_cb(const blosc2_io_cb *io); 862 863 BLOSC_EXPORT blosc2_io_cb *blosc2_get_io_cb(uint8_t id); 864 865 /********************************************************************* 866 Structures and functions related with contexts. 867 *********************************************************************/ 868 869 typedef struct blosc2_context_s blosc2_context; /* opaque type */ 870 871 typedef struct { 872 void (*btune_init)(void * config, blosc2_context* cctx, blosc2_context* dctx); 873 //!< Initialize BTune. 874 void (*btune_next_blocksize)(blosc2_context * context); 875 //!< Only compute the next blocksize. Only it is executed if BTune is not initialized. 876 void (*btune_next_cparams)(blosc2_context * context); 877 //!< Compute the next cparams. Only is executed if BTune is initialized. 878 void (*btune_update)(blosc2_context * context, double ctime); 879 //!< Update the BTune parameters. 880 void (*btune_free)(blosc2_context * context); 881 //!< Free the BTune. 882 void *btune_config; 883 //!> BTune configuration. 884 }blosc2_btune; 885 886 887 /** 888 * @brief The parameters for a prefilter function. 889 * 890 */ 891 typedef struct { 892 void *user_data; // user-provided info (optional) 893 const uint8_t *in; // the input buffer 894 uint8_t *out; // the output buffer 895 int32_t out_size; // the output size (in bytes) 896 int32_t out_typesize; // the output typesize 897 int32_t out_offset; // offset to reach the start of the output buffer 898 int32_t tid; // thread id 899 uint8_t *ttmp; // a temporary that is able to hold several blocks for the output and is private for each thread 900 size_t ttmp_nbytes; // the size of the temporary in bytes 901 blosc2_context *ctx; // the compression context 902 } blosc2_prefilter_params; 903 904 /** 905 * @brief The parameters for a postfilter function. 906 * 907 */ 908 typedef struct { 909 void *user_data; // user-provided info (optional) 910 const uint8_t *in; // the input buffer 911 uint8_t *out; // the output buffer 912 int32_t size; // the input size (in bytes) 913 int32_t typesize; // the input typesize 914 int32_t offset; // offset to reach the start of the input buffer 915 int32_t tid; // thread id 916 uint8_t *ttmp; // a temporary that is able to hold several blocks for the output and is private for each thread 917 size_t ttmp_nbytes; // the size of the temporary in bytes 918 blosc2_context *ctx; // the decompression context 919 } blosc2_postfilter_params; 920 921 /** 922 * @brief The type of the prefilter function. 923 * 924 * If the function call is successful, the return value should be 0; else, a negative value. 925 */ 926 typedef int (*blosc2_prefilter_fn)(blosc2_prefilter_params* params); 927 928 /** 929 * @brief The type of the postfilter function. 930 * 931 * If the function call is successful, the return value should be 0; else, a negative value. 932 */ 933 typedef int (*blosc2_postfilter_fn)(blosc2_postfilter_params* params); 934 935 /** 936 * @brief The parameters for creating a context for compression purposes. 937 * 938 * In parenthesis it is shown the default value used internally when a 0 939 * (zero) in the fields of the struct is passed to a function. 940 */ 941 typedef struct { 942 uint8_t compcode; 943 //!< The compressor codec. 944 uint8_t compcode_meta; 945 //!< The metadata for the compressor codec. 946 uint8_t clevel; 947 //!< The compression level (5). 948 int use_dict; 949 //!< Use dicts or not when compressing (only for ZSTD). 950 int32_t typesize; 951 //!< The type size (8). 952 int16_t nthreads; 953 //!< The number of threads to use internally (1). 954 int32_t blocksize; 955 //!< The requested size of the compressed blocks (0; meaning automatic). 956 int32_t splitmode; 957 //!< Whether the blocks should be split or not. 958 void* schunk; 959 //!< The associated schunk, if any (NULL). 960 uint8_t filters[BLOSC2_MAX_FILTERS]; 961 //!< The (sequence of) filters. 962 uint8_t filters_meta[BLOSC2_MAX_FILTERS]; 963 //!< The metadata for filters. 964 blosc2_prefilter_fn prefilter; 965 //!< The prefilter function. 966 blosc2_prefilter_params *preparams; 967 //!< The prefilter parameters. 968 blosc2_btune *udbtune; 969 //!< The user-defined BTune parameters. 970 } blosc2_cparams; 971 972 /** 973 * @brief Default struct for compression params meant for user initialization. 974 */ 975 static const blosc2_cparams BLOSC2_CPARAMS_DEFAULTS = { 976 BLOSC_BLOSCLZ, 0, 5, 0, 8, 1, 0, BLOSC_FORWARD_COMPAT_SPLIT, 977 NULL, {0, 0, 0, 0, 0, BLOSC_SHUFFLE}, {0, 0, 0, 0, 0, 0}, 978 NULL, NULL, NULL}; 979 980 981 /** 982 @brief The parameters for creating a context for decompression purposes. 983 984 In parenthesis it is shown the default value used internally when a 0 985 (zero) in the fields of the struct is passed to a function. 986 */ 987 typedef struct { 988 int16_t nthreads; 989 //!< The number of threads to use internally (1). 990 void* schunk; 991 //!< The associated schunk, if any (NULL). 992 blosc2_postfilter_fn postfilter; 993 //!< The postfilter function. 994 blosc2_postfilter_params *postparams; 995 //!< The postfilter parameters. 996 } blosc2_dparams; 997 998 /** 999 * @brief Default struct for decompression params meant for user initialization. 1000 */ 1001 static const blosc2_dparams BLOSC2_DPARAMS_DEFAULTS = {1, NULL, NULL, NULL}; 1002 1003 /** 1004 * @brief Create a context for @a *_ctx() compression functions. 1005 * 1006 * @param cparams The blosc2_cparams struct with the compression parameters. 1007 * 1008 * @return A pointer to the new context. NULL is returned if this fails. 1009 */ 1010 BLOSC_EXPORT blosc2_context* blosc2_create_cctx(blosc2_cparams cparams); 1011 1012 /** 1013 * @brief Create a context for *_ctx() decompression functions. 1014 * 1015 * @param dparams The blosc2_dparams struct with the decompression parameters. 1016 * 1017 * @return A pointer to the new context. NULL is returned if this fails. 1018 */ 1019 BLOSC_EXPORT blosc2_context* blosc2_create_dctx(blosc2_dparams dparams); 1020 1021 /** 1022 * @brief Free the resources associated with a context. 1023 * 1024 * @param context The context to free. 1025 * 1026 * This function should always succeed and is valid for contexts meant for 1027 * both compression and decompression. 1028 */ 1029 BLOSC_EXPORT void blosc2_free_ctx(blosc2_context* context); 1030 1031 /** 1032 * @brief Create a @p cparams associated to a context. 1033 * 1034 * @param schunk The context from where to extract the compression parameters. 1035 * @param cparams The pointer where the compression params will be stored. 1036 * 1037 * @return 0 if succeeds. Else a negative code is returned. 1038 */ 1039 BLOSC_EXPORT int blosc2_ctx_get_cparams(blosc2_context *ctx, blosc2_cparams *cparams); 1040 1041 /** 1042 * @brief Create a @p dparams associated to a context. 1043 * 1044 * @param schunk The context from where to extract the decompression parameters. 1045 * @param dparams The pointer where the decompression params will be stored. 1046 * 1047 * @return 0 if succeeds. Else a negative code is returned. 1048 */ 1049 BLOSC_EXPORT int blosc2_ctx_get_dparams(blosc2_context *ctx, blosc2_dparams *dparams); 1050 1051 /** 1052 * @brief Set a maskout so as to avoid decompressing specified blocks. 1053 * 1054 * @param ctx The decompression context to update. 1055 * 1056 * @param maskout The boolean mask for the blocks where decompression 1057 * is to be avoided. 1058 * 1059 * @remark The maskout is valid for contexts *only* meant for decompressing 1060 * a chunk via #blosc2_decompress_ctx. Once a call to #blosc2_decompress_ctx 1061 * is done, this mask is reset so that next call to #blosc2_decompress_ctx 1062 * will decompress the whole chunk. 1063 * 1064 * @param nblocks The number of blocks in maskout above. 1065 * 1066 * @return If success, a 0 values is returned. An error is signaled with a 1067 * negative int. 1068 * 1069 */ 1070 BLOSC_EXPORT int blosc2_set_maskout(blosc2_context *ctx, bool *maskout, int nblocks); 1071 1072 /** 1073 * @brief Compress a block of data in the @p src buffer and returns the size of 1074 * compressed block. 1075 * 1076 * @remark Compression is memory safe and guaranteed not to write @p dest 1077 * more than what is specified in @p destsize. 1078 * There is not a minimum for @p src buffer size @p nbytes. 1079 * 1080 * @warning The @p src buffer and the @p dest buffer can not overlap. 1081 * 1082 * @param clevel The desired compression level and must be a number 1083 * between 0 (no compression) and 9 (maximum compression). 1084 * @param doshuffle Specifies whether the shuffle compression preconditioner 1085 * should be applied or not. #BLOSC_NOFILTER means not applying filters, 1086 * #BLOSC_SHUFFLE means applying shuffle at a byte level and 1087 * #BLOSC_BITSHUFFLE at a bit level (slower but *may* achieve better 1088 * compression). 1089 * @param typesize Is the number of bytes for the atomic type in binary 1090 * @p src buffer. This is mainly useful for the shuffle preconditioner. 1091 * For implementation reasons, only a 1 < typesize < 256 will allow the 1092 * shuffle filter to work. When typesize is not in this range, shuffle 1093 * will be silently disabled. 1094 * @param src The buffer containing the data to compress. 1095 * @param srcsize The number of bytes to compress in the @p src buffer. 1096 * @param dest The buffer where the compressed data will be put, 1097 * must have at least the size of @p destsize. 1098 * @param destsize The size of the dest buffer. Blosc 1099 * guarantees that if you set @p destsize to, at least, 1100 * (@p nbytes + #BLOSC_MAX_OVERHEAD), the compression will always succeed. 1101 * 1102 * @return The number of bytes compressed. 1103 * If @p src buffer cannot be compressed into @p destsize, the return 1104 * value is zero and you should discard the contents of the @p dest 1105 * buffer. A negative return value means that an internal error happened. This 1106 * should never happen. If you see this, please report it back 1107 * together with the buffer data causing this and compression settings. 1108 */ 1109 /* 1110 * Environment variables 1111 * _____________________ 1112 * 1113 * *blosc_compress()* honors different environment variables to control 1114 * internal parameters without the need of doing that programatically. 1115 * Here are the ones supported: 1116 * 1117 * **BLOSC_CLEVEL=(INTEGER)**: This will overwrite the @p clevel parameter 1118 * before the compression process starts. 1119 * 1120 * **BLOSC_SHUFFLE=[NOSHUFFLE | SHUFFLE | BITSHUFFLE]**: This will 1121 * overwrite the *doshuffle* parameter before the compression process 1122 * starts. 1123 * 1124 * **BLOSC_DELTA=(1|0)**: This will call *blosc_set_delta()^* before the 1125 * compression process starts. 1126 * 1127 * **BLOSC_TYPESIZE=(INTEGER)**: This will overwrite the *typesize* 1128 * parameter before the compression process starts. 1129 * 1130 * **BLOSC_COMPRESSOR=[BLOSCLZ | LZ4 | LZ4HC | ZLIB | ZSTD]**: 1131 * This will call *blosc_set_compressor(BLOSC_COMPRESSOR)* before the 1132 * compression process starts. 1133 * 1134 * **BLOSC_NTHREADS=(INTEGER)**: This will call 1135 * *blosc_set_nthreads(BLOSC_NTHREADS)* before the compression process 1136 * starts. 1137 * 1138 * **BLOSC_BLOCKSIZE=(INTEGER)**: This will call 1139 * *blosc_set_blocksize(BLOSC_BLOCKSIZE)* before the compression process 1140 * starts. *NOTE:* The blocksize is a critical parameter with 1141 * important restrictions in the allowed values, so use this with care. 1142 * 1143 * **BLOSC_NOLOCK=(ANY VALUE)**: This will call *blosc2_compress_ctx()* under 1144 * the hood, with the *compressor*, *blocksize* and 1145 * *numinternalthreads* parameters set to the same as the last calls to 1146 * *blosc_set_compressor*, *blosc_set_blocksize* and 1147 * *blosc_set_nthreads*. *BLOSC_CLEVEL*, *BLOSC_SHUFFLE*, *BLOSC_DELTA* and 1148 * *BLOSC_TYPESIZE* environment vars will also be honored. 1149 * 1150 */ 1151 BLOSC_EXPORT int blosc2_compress(int clevel, int doshuffle, int32_t typesize, 1152 const void* src, int32_t srcsize, void* dest, 1153 int32_t destsize); 1154 1155 1156 /** 1157 * @brief Decompress a block of compressed data in @p src, put the result in 1158 * @p dest and returns the size of the decompressed block. 1159 * 1160 * @warning The @p src buffer and the @p dest buffer can not overlap. 1161 * 1162 * @remark Decompression is memory safe and guaranteed not to write the @p dest 1163 * buffer more than what is specified in @p destsize. 1164 * 1165 * @remark In case you want to keep under control the number of bytes read from 1166 * source, you can call #blosc_cbuffer_sizes first to check whether the 1167 * @p nbytes (i.e. the number of bytes to be read from @p src buffer by this 1168 * function) in the compressed buffer is ok with you. 1169 * 1170 * @param src The buffer to be decompressed. 1171 * @param srcsize The size of the buffer to be decompressed. 1172 * @param dest The buffer where the decompressed data will be put. 1173 * @param destsize The size of the @p dest buffer. 1174 * 1175 * @return The number of bytes decompressed. 1176 * If an error occurs, e.g. the compressed data is corrupted or the 1177 * output buffer is not large enough, then a negative value 1178 * will be returned instead. 1179 */ 1180 /* 1181 * Environment variables 1182 * _____________________ 1183 * 1184 * *blosc_decompress* honors different environment variables to control 1185 * internal parameters without the need of doing that programatically. 1186 * Here are the ones supported: 1187 * 1188 * **BLOSC_NTHREADS=(INTEGER)**: This will call 1189 * *blosc_set_nthreads(BLOSC_NTHREADS)* before the proper decompression 1190 * process starts. 1191 * 1192 * **BLOSC_NOLOCK=(ANY VALUE)**: This will call *blosc2_decompress_ctx* 1193 * under the hood, with the *numinternalthreads* parameter set to the 1194 * same value as the last call to *blosc_set_nthreads*. 1195 * 1196 */ 1197 BLOSC_EXPORT int blosc2_decompress(const void* src, int32_t srcsize, 1198 void* dest, int32_t destsize); 1199 1200 /** 1201 * @brief Context interface to Blosc compression. This does not require a call 1202 * to #blosc_init and can be called from multithreaded applications 1203 * without the global lock being used, so allowing Blosc be executed 1204 * simultaneously in those scenarios. 1205 * 1206 * @param context A blosc2_context struct with the different compression params. 1207 * @param src The buffer containing the data to be compressed. 1208 * @param srcsize The number of bytes to be compressed from the @p src buffer. 1209 * @param dest The buffer where the compressed data will be put. 1210 * @param destsize The size in bytes of the @p dest buffer. 1211 * 1212 * @return The number of bytes compressed. 1213 * If @p src buffer cannot be compressed into @p destsize, the return 1214 * value is zero and you should discard the contents of the @p dest 1215 * buffer. A negative return value means that an internal error happened. 1216 * It could happen that context is not meant for compression (which is stated in stderr). 1217 * Otherwise, please report it back together with the buffer data causing this 1218 * and compression settings. 1219 */ 1220 BLOSC_EXPORT int blosc2_compress_ctx( 1221 blosc2_context* context, const void* src, int32_t srcsize, void* dest, 1222 int32_t destsize); 1223 1224 1225 /** 1226 * @brief Context interface to Blosc decompression. This does not require a 1227 * call to #blosc_init and can be called from multithreaded 1228 * applications without the global lock being used, so allowing Blosc 1229 * be executed simultaneously in those scenarios. 1230 * 1231 * @param context The blosc2_context struct with the different compression params. 1232 * @param src The buffer of compressed data. 1233 * @param srcsize The length of buffer of compressed data. 1234 * @param dest The buffer where the decompressed data will be put. 1235 * @param destsize The size in bytes of the @p dest buffer. 1236 * 1237 * @warning The @p src buffer and the @p dest buffer can not overlap. 1238 * 1239 * @remark Decompression is memory safe and guaranteed not to write the @p dest 1240 * buffer more than what is specified in @p destsize. 1241 * 1242 * @remark In case you want to keep under control the number of bytes read from 1243 * source, you can call #blosc_cbuffer_sizes first to check the @p nbytes 1244 * (i.e. the number of bytes to be read from @p src buffer by this function) 1245 * in the compressed buffer. 1246 * 1247 * @remark If #blosc2_set_maskout is called prior to this function, its 1248 * @p block_maskout parameter will be honored for just *one single* shot; 1249 * i.e. the maskout in context will be automatically reset to NULL, so 1250 * mask won't be used next time (unless #blosc2_set_maskout is called again). 1251 * 1252 * @return The number of bytes decompressed (i.e. the maskout blocks are not 1253 * counted). If an error occurs, e.g. the compressed data is corrupted, 1254 * @p destsize is not large enough or context is not meant for decompression, 1255 * then a negative value will be returned instead. 1256 */ 1257 BLOSC_EXPORT int blosc2_decompress_ctx(blosc2_context* context, const void* src, 1258 int32_t srcsize, void* dest, int32_t destsize); 1259 1260 /** 1261 * @brief Create a chunk made of zeros. 1262 * 1263 * @param cparams The compression parameters. 1264 * @param nbytes The size (in bytes) of the chunk. 1265 * @param dest The buffer where the data chunk will be put. 1266 * @param destsize The size (in bytes) of the @p dest buffer; 1267 * must be BLOSC_EXTENDED_HEADER_LENGTH at least. 1268 * 1269 * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH). 1270 * If negative, there has been an error and @dest is unusable. 1271 * */ 1272 BLOSC_EXPORT int blosc2_chunk_zeros(blosc2_cparams cparams, size_t nbytes, 1273 void* dest, size_t destsize); 1274 1275 1276 /** 1277 * @brief Create a chunk made of nans. 1278 * 1279 * @param cparams The compression parameters; 1280 * only 4 bytes (float) and 8 bytes (double) are supported. 1281 * @param nbytes The size (in bytes) of the chunk. 1282 * @param dest The buffer where the data chunk will be put. 1283 * @param destsize The size (in bytes) of the @p dest buffer; 1284 * must be BLOSC_EXTENDED_HEADER_LENGTH at least. 1285 * 1286 * @note Whether the NaNs are floats or doubles will be given by the typesize. 1287 * 1288 * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH). 1289 * If negative, there has been an error and @dest is unusable. 1290 * */ 1291 BLOSC_EXPORT int blosc2_chunk_nans(blosc2_cparams cparams, size_t nbytes, 1292 void* dest, size_t destsize); 1293 1294 1295 /** 1296 * @brief Create a chunk made of repeated values. 1297 * 1298 * @param cparams The compression parameters. 1299 * @param nbytes The size (in bytes) of the chunk. 1300 * @param dest The buffer where the data chunk will be put. 1301 * @param destsize The size (in bytes) of the @p dest buffer. 1302 * @param repeatval A pointer to the repeated value (little endian). 1303 * The size of the value is given by @p cparams.typesize param. 1304 * 1305 * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH + typesize). 1306 * If negative, there has been an error and @dest is unusable. 1307 * */ 1308 BLOSC_EXPORT int blosc2_chunk_repeatval(blosc2_cparams cparams, size_t nbytes, 1309 void* dest, size_t destsize, void* repeatval); 1310 1311 1312 /** 1313 * @brief Create a chunk made of uninitialized values. 1314 * 1315 * @param cparams The compression parameters. 1316 * @param nbytes The size (in bytes) of the chunk. 1317 * @param dest The buffer where the data chunk will be put. 1318 * @param destsize The size (in bytes) of the @p dest buffer; 1319 * must be BLOSC_EXTENDED_HEADER_LENGTH at least. 1320 * 1321 * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH). 1322 * If negative, there has been an error and @dest is unusable. 1323 * */ 1324 BLOSC_EXPORT int blosc2_chunk_uninit(blosc2_cparams cparams, size_t nbytes, 1325 void* dest, size_t destsize); 1326 1327 1328 /** 1329 * @brief Context interface counterpart for #blosc_getitem. 1330 * 1331 * It uses many of the same parameters as blosc_getitem() function with 1332 * a few additions. 1333 * 1334 * @param context Context pointer. 1335 * @param srcsize Compressed buffer length. 1336 * @param destsize Output buffer length. 1337 * 1338 * @return The number of bytes copied to @p dest or a negative value if 1339 * some error happens. 1340 */ 1341 BLOSC_EXPORT int blosc2_getitem_ctx(blosc2_context* context, const void* src, 1342 int32_t srcsize, int start, int nitems, void* dest, 1343 int32_t destsize); 1344 1345 1346 /********************************************************************* 1347 Super-chunk related structures and functions. 1348 *********************************************************************/ 1349 1350 #define BLOSC2_MAX_METALAYERS 16 1351 #define BLOSC2_METALAYER_NAME_MAXLEN 31 1352 1353 // Allow for a reasonable number of vl metalayers 1354 // max is 64 * 1024 due to msgpack map 16 in frame 1355 // mem usage 8 * 1024 entries for blosc2_schunk.vlmetalayers[] is 64 KB 1356 #define BLOSC2_MAX_VLMETALAYERS (8 * 1024) 1357 #define BLOSC2_VLMETALAYERS_NAME_MAXLEN BLOSC2_METALAYER_NAME_MAXLEN 1358 1359 /** 1360 * @brief This struct is meant for holding storage parameters for a 1361 * for a blosc2 container, allowing to specify, for example, how to interpret 1362 * the contents included in the schunk. 1363 */ 1364 typedef struct { 1365 bool contiguous; 1366 //!< Whether the chunks are contiguous or sparse. 1367 char* urlpath; 1368 //!< The path for persistent storage. If NULL, that means in-memory. 1369 blosc2_cparams* cparams; 1370 //!< The compression params when creating a schunk. 1371 //!< If NULL, sensible defaults are used depending on the context. 1372 blosc2_dparams* dparams; 1373 //!< The decompression params when creating a schunk. 1374 //!< If NULL, sensible defaults are used depending on the context. 1375 blosc2_io *io; 1376 //!< Input/output backend. 1377 } blosc2_storage; 1378 1379 /** 1380 * @brief Default struct for #blosc2_storage meant for user initialization. 1381 */ 1382 static const blosc2_storage BLOSC2_STORAGE_DEFAULTS = {false, NULL, NULL, NULL, NULL}; 1383 1384 typedef struct blosc2_frame_s blosc2_frame; /* opaque type */ 1385 1386 /** 1387 * @brief This struct is meant to store metadata information inside 1388 * a #blosc2_schunk, allowing to specify, for example, how to interpret 1389 * the contents included in the schunk. 1390 */ 1391 typedef struct blosc2_metalayer { 1392 char* name; //!< The metalayer identifier for Blosc client (e.g. Caterva). 1393 uint8_t* content; //!< The serialized (msgpack preferably) content of the metalayer. 1394 int32_t content_len; //!< The length in bytes of the content. 1395 } blosc2_metalayer; 1396 1397 /** 1398 * @brief This struct is the standard container for Blosc 2 compressed data. 1399 * 1400 * This is essentially a container for Blosc 1 chunks of compressed data, 1401 * and it allows to overcome the 32-bit limitation in Blosc 1. Optionally, 1402 * a #blosc2_frame can be attached so as to store the compressed chunks contiguously. 1403 */ 1404 typedef struct blosc2_schunk { 1405 uint8_t version; 1406 uint8_t compcode; 1407 //!< The default compressor. Each chunk can override this. 1408 uint8_t compcode_meta; 1409 //!< The default compressor metadata. Each chunk can override this. 1410 uint8_t clevel; 1411 //!< The compression level and other compress params. 1412 int32_t typesize; 1413 //!< The type size. 1414 int32_t blocksize; 1415 //!< The requested size of the compressed blocks (0; meaning automatic). 1416 int32_t chunksize; 1417 //!< Size of each chunk. 0 if not a fixed chunksize. 1418 uint8_t filters[BLOSC2_MAX_FILTERS]; 1419 //!< The (sequence of) filters. 8-bit per filter. 1420 uint8_t filters_meta[BLOSC2_MAX_FILTERS]; 1421 //!< Metadata for filters. 8-bit per meta-slot. 1422 int32_t nchunks; 1423 //!< Number of chunks in super-chunk. 1424 int64_t nbytes; 1425 //!< The data size + metadata size + header size (uncompressed). 1426 int64_t cbytes; 1427 //!< The data size + metadata size + header size (compressed). 1428 uint8_t** data; 1429 //!< Pointer to chunk data pointers buffer. 1430 size_t data_len; 1431 //!< Length of the chunk data pointers buffer. 1432 blosc2_storage* storage; 1433 //!< Pointer to storage info. 1434 blosc2_frame* frame; 1435 //!< Pointer to frame used as store for chunks. 1436 //!<uint8_t* ctx; 1437 //!< Context for the thread holder. NULL if not acquired. 1438 blosc2_context* cctx; 1439 //!< Context for compression 1440 blosc2_context* dctx; 1441 //!< Context for decompression. 1442 struct blosc2_metalayer *metalayers[BLOSC2_MAX_METALAYERS]; 1443 //!< The array of metalayers. 1444 int16_t nmetalayers; 1445 //!< The number of metalayers in the super-chunk 1446 struct blosc2_metalayer *vlmetalayers[BLOSC2_MAX_VLMETALAYERS]; 1447 //<! The array of variable-length metalayers. 1448 int16_t nvlmetalayers; 1449 //!< The number of variable-length metalayers. 1450 blosc2_btune *udbtune; 1451 } blosc2_schunk; 1452 1453 1454 /** 1455 * @brief Create a new super-chunk. 1456 * 1457 * @param storage The storage properties. 1458 * 1459 * @remark In case that storage.urlpath is not NULL, the data is stored 1460 * on-disk. If the data file(s) exist, they are *overwritten*. 1461 * 1462 * @return The new super-chunk. 1463 */ 1464 BLOSC_EXPORT blosc2_schunk* blosc2_schunk_new(blosc2_storage *storage); 1465 1466 /** 1467 * Create a copy of a super-chunk. 1468 * 1469 * @param schunk The super-chunk to be copied. 1470 * @param storage The storage properties. 1471 * 1472 * @return The new super-chunk. 1473 */ 1474 BLOSC_EXPORT blosc2_schunk* blosc2_schunk_copy(blosc2_schunk *schunk, blosc2_storage *storage); 1475 1476 /** 1477 * @brief Create a super-chunk out of a contiguous frame buffer. 1478 * 1479 * @param cframe The buffer of the in-memory frame. 1480 * @param copy Whether the super-chunk should make a copy of 1481 * the @p cframe data or not. The copy will be made to an internal 1482 * sparse frame. 1483 * 1484 * @remark If copy is false, the @p cframe buffer passed will be owned 1485 * by the super-chunk and will be automatically freed when 1486 * blosc2_schunk_free() is called. If the user frees it after the 1487 * opening, bad things will happen. Don't do that (or set @p copy). 1488 * 1489 * @param len The length of the buffer (in bytes). 1490 * 1491 * @return The new super-chunk. 1492 */ 1493 BLOSC_EXPORT blosc2_schunk* blosc2_schunk_from_buffer(uint8_t *cframe, int64_t len, bool copy); 1494 1495 /** 1496 * @brief Open an existing super-chunk that is on-disk (no copy is made). 1497 * 1498 * @param storage The storage properties of the source. 1499 * 1500 * @remark The storage.urlpath must be not NULL and it should exist on-disk. 1501 * New data or metadata can be appended or updated. 1502 * 1503 * @return The new super-chunk. 1504 */ 1505 BLOSC_EXPORT blosc2_schunk* blosc2_schunk_open(const char* urlpath); 1506 1507 /** 1508 * @brief Open an existing super-chunk (no copy is made) using a user-defined I/O interface. 1509 * 1510 * @param storage The storage properties of the source. 1511 * 1512 * @param udio The user-defined I/O interface. 1513 * 1514 * @remark The storage.urlpath must be not NULL and it should exist on-disk. 1515 * New data or metadata can be appended or updated. 1516 * 1517 * @return The new super-chunk. 1518 */ 1519 BLOSC_EXPORT blosc2_schunk* blosc2_schunk_open_udio(const char* urlpath, const blosc2_io *udio); 1520 1521 /* @brief Convert a super-chunk into a contiguous frame buffer. 1522 * 1523 * @param schunk The super-chunk to convert. 1524 * @param cframe The address of the destination buffer (output). 1525 * @param needs_free The pointer to a boolean indicating if it is the user's 1526 * responsibility to free the chunk returned or not. 1527 * 1528 * @note The user is responsible to free the @p cframe buffer (not always required). 1529 * You can check whether the cframe requires a free with the @p needs_free parameter. 1530 * 1531 * @return If successful, return the size of the (frame) @p cframe buffer. 1532 * Else, a negative value. 1533 */ 1534 BLOSC_EXPORT int64_t blosc2_schunk_to_buffer(blosc2_schunk* schunk, uint8_t** cframe, bool* needs_free); 1535 1536 /* @brief Store a super-chunk into a file. 1537 * 1538 * @param schunk The super-chunk to write. 1539 * @param urlpath The path for persistent storage. 1540 * 1541 * @return If successful, return the size of the (fileframe) in @p urlpath. 1542 * Else, a negative value. 1543 */ 1544 BLOSC_EXPORT int64_t blosc2_schunk_to_file(blosc2_schunk* schunk, const char* urlpath); 1545 1546 1547 /** 1548 * @brief Release resources from a super-chunk. 1549 * 1550 * @param schunk The super-chunk to be freed. 1551 * 1552 * @remark All the memory resources attached to the super-frame are freed. 1553 * If the super-chunk is on-disk, the data continues there for a later 1554 * re-opening. 1555 * 1556 * @return 0 if success. 1557 */ 1558 BLOSC_EXPORT int blosc2_schunk_free(blosc2_schunk *schunk); 1559 1560 /** 1561 * @brief Append an existing @p chunk to a super-chunk. 1562 * 1563 * @param schunk The super-chunk where the chunk will be appended. 1564 * @param chunk The @p chunk to append. An internal copy is made, so @p chunk can be reused or 1565 * freed if desired. 1566 * @param copy Whether the chunk should be copied internally or can be used as-is. 1567 * 1568 * @return The number of chunks in super-chunk. If some problem is 1569 * detected, this number will be negative. 1570 */ 1571 BLOSC_EXPORT int blosc2_schunk_append_chunk(blosc2_schunk *schunk, uint8_t *chunk, bool copy); 1572 1573 /** 1574 * @brief Update a chunk at a specific position in a super-chunk. 1575 * 1576 * @param schunk The super-chunk where the chunk will be updated. 1577 * @param nchunk The position where the chunk will be updated. 1578 * @param chunk The new @p chunk. If an internal copy is made, the @p chunk can be reused or 1579 * freed if desired. 1580 * @param copy Whether the chunk should be copied internally or can be used as-is. 1581 * 1582 * @return The number of chunks in super-chunk. If some problem is 1583 * detected, this number will be negative. 1584 */ 1585 BLOSC_EXPORT int blosc2_schunk_update_chunk(blosc2_schunk *schunk, int nchunk, uint8_t *chunk, bool copy); 1586 1587 /** 1588 * @brief Insert a chunk at a specific position in a super-chunk. 1589 * 1590 * @param schunk The super-chunk where the chunk will be appended. 1591 * @param nchunk The position where the chunk will be inserted. 1592 * @param chunk The @p chunk to insert. If an internal copy is made, the @p chunk can be reused or 1593 * freed if desired. 1594 * @param copy Whether the chunk should be copied internally or can be used as-is. 1595 * 1596 * @return The number of chunks in super-chunk. If some problem is 1597 * detected, this number will be negative. 1598 */ 1599 BLOSC_EXPORT int blosc2_schunk_insert_chunk(blosc2_schunk *schunk, int nchunk, uint8_t *chunk, bool copy); 1600 1601 /** 1602 * @brief Delete a chunk at a specific position in a super-chunk. 1603 * 1604 * @param schunk The super-chunk where the chunk will be deleted. 1605 * @param nchunk The position where the chunk will be deleted. 1606 * 1607 * @return The number of chunks in super-chunk. If some problem is 1608 * detected, this number will be negative. 1609 */ 1610 BLOSC_EXPORT int blosc2_schunk_delete_chunk(blosc2_schunk *schunk, int nchunk); 1611 1612 /** 1613 * @brief Append a @p src data buffer to a super-chunk. 1614 * 1615 * @param schunk The super-chunk where data will be appended. 1616 * @param src The buffer of data to compress. 1617 * @param nbytes The size of the @p src buffer. 1618 * 1619 * @return The number of chunks in super-chunk. If some problem is 1620 * detected, this number will be negative. 1621 */ 1622 BLOSC_EXPORT int blosc2_schunk_append_buffer(blosc2_schunk *schunk, void *src, int32_t nbytes); 1623 1624 /** 1625 * @brief Decompress and return the @p nchunk chunk of a super-chunk. 1626 * 1627 * If the chunk is uncompressed successfully, it is put in the @p *dest 1628 * pointer. 1629 * 1630 * @param schunk The super-chunk from where the chunk will be decompressed. 1631 * @param nchunk The chunk to be decompressed (0 indexed). 1632 * @param dest The buffer where the decompressed data will be put. 1633 * @param nbytes The size of the area pointed by @p *dest. 1634 * 1635 * @warning You must make sure that you have space enough to store the 1636 * uncompressed data. 1637 * 1638 * @return The size of the decompressed chunk or 0 if it is non-initialized. If some problem is 1639 * detected, a negative code is returned instead. 1640 */ 1641 BLOSC_EXPORT int blosc2_schunk_decompress_chunk(blosc2_schunk *schunk, int nchunk, void *dest, int32_t nbytes); 1642 1643 /** 1644 * @brief Return a compressed chunk that is part of a super-chunk in the @p chunk parameter. 1645 * 1646 * @param schunk The super-chunk from where to extract a chunk. 1647 * @param nchunk The chunk to be extracted (0 indexed). 1648 * @param chunk The pointer to the chunk of compressed data. 1649 * @param needs_free The pointer to a boolean indicating if it is the user's 1650 * responsibility to free the chunk returned or not. 1651 * 1652 * @warning If the super-chunk is backed by a frame that is disk-based, a buffer is allocated for the 1653 * (compressed) chunk, and hence a free is needed. 1654 * You can check if the chunk requires a free with the @p needs_free parameter. 1655 * If the chunk does not need a free, it means that a pointer to the location in the super-chunk 1656 * (or the backing in-memory frame) is returned in the @p chunk parameter. 1657 * 1658 * @return The size of the (compressed) chunk or 0 if it is non-initialized. If some problem is 1659 * detected, a negative code is returned instead. 1660 */ 1661 BLOSC_EXPORT int blosc2_schunk_get_chunk(blosc2_schunk *schunk, int nchunk, uint8_t **chunk, 1662 bool *needs_free); 1663 1664 /** 1665 * @brief Return a (lazy) compressed chunk that is part of a super-chunk in the @p chunk parameter. 1666 * 1667 * @param schunk The super-chunk from where to extract a chunk. 1668 * @param nchunk The chunk to be extracted (0 indexed). 1669 * @param chunk The pointer to the (lazy) chunk of compressed data. 1670 * @param needs_free The pointer to a boolean indicating if it is the user's 1671 * responsibility to free the chunk returned or not. 1672 * 1673 * @note For disk-based frames, a lazy chunk is always returned. 1674 * 1675 * @warning Currently, a lazy chunk can only be used by #blosc2_decompress_ctx and #blosc2_getitem_ctx. 1676 * 1677 * @warning If the super-chunk is backed by a frame that is disk-based, a buffer is allocated for the 1678 * (compressed) chunk, and hence a free is needed. 1679 * You can check if the chunk requires a free with the @p needs_free parameter. 1680 * If the chunk does not need a free, it means that a pointer to the location in the super-chunk 1681 * (or the backing in-memory frame) is returned in the @p chunk parameter. In this case the returned 1682 * chunk is not lazy. 1683 * 1684 * @return The size of the (compressed) chunk or 0 if it is non-initialized. If some problem is 1685 * detected, a negative code is returned instead. Note that a lazy chunk is somewhat larger than 1686 * a regular chunk because of the trailer section (for details see `README_CHUNK_FORMAT.rst`). 1687 */ 1688 BLOSC_EXPORT int blosc2_schunk_get_lazychunk(blosc2_schunk *schunk, int nchunk, uint8_t **chunk, 1689 bool *needs_free); 1690 1691 /** 1692 * @brief Return the @p cparams associated to a super-chunk. 1693 * 1694 * @param schunk The super-chunk from where to extract the compression parameters. 1695 * @param cparams The pointer where the compression params will be returned. 1696 * 1697 * @warning A new struct is allocated, and the user should free it after use. 1698 * 1699 * @return 0 if succeeds. Else a negative code is returned. 1700 */ 1701 BLOSC_EXPORT int blosc2_schunk_get_cparams(blosc2_schunk *schunk, blosc2_cparams **cparams); 1702 1703 /** 1704 * @brief Return the @p dparams struct associated to a super-chunk. 1705 * 1706 * @param schunk The super-chunk from where to extract the decompression parameters. 1707 * @param dparams The pointer where the decompression params will be returned. 1708 * 1709 * @warning A new struct is allocated, and the user should free it after use. 1710 * 1711 * @return 0 if succeeds. Else a negative code is returned. 1712 */ 1713 BLOSC_EXPORT int blosc2_schunk_get_dparams(blosc2_schunk *schunk, blosc2_dparams **dparams); 1714 1715 /** 1716 * @brief Reorder the chunk offsets of an existing super-chunk. 1717 * 1718 * @param schunk The super-chunk whose chunk offsets are to be reordered. 1719 * @param offsets_order The new order of the chunk offsets. 1720 * 1721 * @return 0 if suceeds. Else a negative code is returned. 1722 */ 1723 BLOSC_EXPORT int blosc2_schunk_reorder_offsets(blosc2_schunk *schunk, int *offsets_order); 1724 1725 /** 1726 * @brief Get the length (in bytes) of the internal frame of the super-chunk. 1727 * 1728 * @param schunk The super-chunk. 1729 * 1730 * @return The length (in bytes) of the internal frame. 1731 * If there is not an internal frame, an estimate of the length is provided. 1732 */ 1733 BLOSC_EXPORT int64_t blosc2_schunk_frame_len(blosc2_schunk* schunk); 1734 1735 /** 1736 * @brief Quickly fill an empty frame with special values (zeros, NaNs, uninit). 1737 * 1738 * @param schunk The super-chunk to be filled. This must be empty initially. 1739 * @param nitems The number of items to fill. 1740 * @param special_value The special value to use for filling. The only values 1741 * supported for now are BLOSC2_SPECIAL_ZERO, BLOSC2_SPECIAL_NAN and BLOSC2_SPECIAL_UNINIT. 1742 * @param chunksize The chunksize for the chunks that are to be added to the super-chunk. 1743 * 1744 * @return The total number of chunks that have been added to the super-chunk. 1745 * If there is an error, a negative value is returned. 1746 */ 1747 BLOSC_EXPORT int blosc2_schunk_fill_special(blosc2_schunk* schunk, int64_t nitems, 1748 int special_value, int32_t chunksize); 1749 1750 1751 /********************************************************************* 1752 Functions related with fixed-length metalayers. 1753 *********************************************************************/ 1754 1755 /** 1756 * @brief Find whether the schunk has a metalayer or not. 1757 * 1758 * @param schunk The super-chunk from which the metalayer will be checked. 1759 * @param name The name of the metalayer to be checked. 1760 * 1761 * @return If successful, return the index of the metalayer. Else, return a negative value. 1762 */ 1763 BLOSC_EXPORT int blosc2_meta_exists(blosc2_schunk *schunk, const char *name); 1764 1765 /** 1766 * @brief Add content into a new metalayer. 1767 * 1768 * @param schunk The super-chunk to which the metalayer should be added. 1769 * @param name The name of the metalayer. 1770 * @param content The content of the metalayer. 1771 * @param content_len The length of the content. 1772 * 1773 * @return If successful, the index of the new metalayer. Else, return a negative value. 1774 */ 1775 BLOSC_EXPORT int blosc2_meta_add(blosc2_schunk *schunk, const char *name, uint8_t *content, 1776 uint32_t content_len); 1777 1778 /** 1779 * @brief Update the content of an existing metalayer. 1780 * 1781 * @param schunk The frame containing the metalayer. 1782 * @param name The name of the metalayer to be updated. 1783 * @param content The new content of the metalayer. 1784 * @param content_len The length of the content. 1785 * 1786 * @note Contrarily to #blosc2_meta_add the updates to metalayers 1787 * are automatically serialized into a possible attached frame. 1788 * 1789 * @return If successful, the index of the metalayer. Else, return a negative value. 1790 */ 1791 BLOSC_EXPORT int blosc2_meta_update(blosc2_schunk *schunk, const char *name, uint8_t *content, 1792 uint32_t content_len); 1793 1794 /** 1795 * @brief Get the content out of a metalayer. 1796 * 1797 * @param schunk The frame containing the metalayer. 1798 * @param name The name of the metalayer. 1799 * @param content The pointer where the content will be put. 1800 * @param content_len The length of the content. 1801 * 1802 * @warning The @p **content receives a malloc'ed copy of the content. 1803 * The user is responsible of freeing it. 1804 * 1805 * @return If successful, the index of the new metalayer. Else, return a negative value. 1806 */ 1807 BLOSC_EXPORT int blosc2_meta_get(blosc2_schunk *schunk, const char *name, uint8_t **content, 1808 uint32_t *content_len); 1809 1810 1811 /********************************************************************* 1812 Variable-length metalayers functions. 1813 *********************************************************************/ 1814 1815 /** 1816 * @brief Find whether the schunk has a variable-length metalayer or not. 1817 * 1818 * @param schunk The super-chunk from which the variable-length metalayer will be checked. 1819 * @param name The name of the variable-length metalayer to be checked. 1820 * 1821 * @return If successful, return the index of the variable-length metalayer. Else, return a negative value. 1822 */ 1823 BLOSC_EXPORT int blosc2_vlmeta_exists(blosc2_schunk *schunk, const char *name); 1824 1825 /** 1826 * @brief Add content into a new variable-length metalayer. 1827 * 1828 * @param schunk The super-chunk to which the variable-length metalayer should be added. 1829 * @param name The name of the variable-length metalayer. 1830 * @param content The content to be added. 1831 * @param content_len The length of the content. 1832 * @param cparams The parameters for compressing the variable-length metalayer content. If NULL, 1833 * the `BLOSC2_CPARAMS_DEFAULTS` will be used. 1834 * 1835 * @return If successful, the index of the new variable-length metalayer. Else, return a negative value. 1836 */ 1837 BLOSC_EXPORT int blosc2_vlmeta_add(blosc2_schunk *schunk, const char *name, 1838 uint8_t *content, uint32_t content_len, 1839 blosc2_cparams *cparams); 1840 1841 /** 1842 * @brief Update the content of an existing variable-length metalayer. 1843 * 1844 * @param schunk The super-chunk containing the variable-length metalayer. 1845 * @param name The name of the variable-length metalayer to be updated. 1846 * @param content The new content of the variable-length metalayer. 1847 * @param content_len The length of the content. 1848 * @param cparams The parameters for compressing the variable-length metalayer content. If NULL, 1849 * the `BLOSC2_CPARAMS_DEFAULTS` will be used. 1850 * 1851 * @return If successful, the index of the variable-length metalayer. Else, return a negative value. 1852 */ 1853 BLOSC_EXPORT int blosc2_vlmeta_update(blosc2_schunk *schunk, const char *name, 1854 uint8_t *content, uint32_t content_len, 1855 blosc2_cparams *cparams); 1856 1857 /** 1858 * @brief Get the content out of a variable-length metalayer. 1859 * 1860 * @param schunk The super-chunk containing the variable-length metalayer. 1861 * @param name The name of the variable-length metalayer. 1862 * @param content The pointer where the content will be put. 1863 * @param content_len The pointer where the length of the content will be put. 1864 * 1865 * @warning The @p **content receives a malloc'ed copy of the content. 1866 * The user is responsible of freeing it. 1867 * 1868 * @return If successful, the index of the new variable-length metalayer. Else, return a negative value. 1869 */ 1870 BLOSC_EXPORT int blosc2_vlmeta_get(blosc2_schunk *schunk, const char *name, 1871 uint8_t **content, uint32_t *content_len); 1872 1873 /** 1874 * @brief Delete the variable-length metalayer from the super-chunk. 1875 * 1876 * @param schunk The super-chunk containing the variable-length metalayer. 1877 * @param name The name of the variable-length metalayer. 1878 * 1879 * @return If successful, the number of the variable-length metalayers in the super-chunk. Else, return a negative value. 1880 */ 1881 BLOSC_EXPORT int blosc2_vlmeta_delete(blosc2_schunk *schunk, const char *name); 1882 1883 1884 /********************************************************************* 1885 Time measurement utilities. 1886 *********************************************************************/ 1887 1888 #if defined(_WIN32) 1889 /* For QueryPerformanceCounter(), etc. */ 1890 #include <windows.h> 1891 #elif defined(__MACH__) 1892 #include <mach/clock.h> 1893 #include <mach/mach.h> 1894 #include <time.h> 1895 #elif defined(__unix__) 1896 #if defined(__linux__) 1897 #include <time.h> 1898 #else 1899 #include <sys/time.h> 1900 #endif 1901 #else 1902 #error Unable to detect platform. 1903 #endif 1904 1905 /* The type of timestamp used on this system. */ 1906 #if defined(_WIN32) 1907 #define blosc_timestamp_t LARGE_INTEGER 1908 #else 1909 #define blosc_timestamp_t struct timespec 1910 #endif 1911 1912 /* 1913 * Set a timestamp. 1914 */ 1915 BLOSC_EXPORT void blosc_set_timestamp(blosc_timestamp_t* timestamp); 1916 1917 /* 1918 * Return the nanoseconds between 2 timestamps. 1919 */ 1920 BLOSC_EXPORT double blosc_elapsed_nsecs(blosc_timestamp_t start_time, 1921 blosc_timestamp_t end_time); 1922 1923 /* 1924 * Return the seconds between 2 timestamps. 1925 */ 1926 BLOSC_EXPORT double blosc_elapsed_secs(blosc_timestamp_t start_time, 1927 blosc_timestamp_t end_time); 1928 1929 1930 /********************************************************************* 1931 Low-level functions follows. Use them only if you are an expert! 1932 *********************************************************************/ 1933 1934 /** 1935 * @brief Get the internal blocksize to be used during compression. 0 means 1936 * that an automatic blocksize is computed internally. 1937 * 1938 * @return The size in bytes of the internal block size. 1939 */ 1940 BLOSC_EXPORT int blosc_get_blocksize(void); 1941 1942 /** 1943 * @brief Force the use of a specific blocksize. If 0, an automatic 1944 * blocksize will be used (the default). 1945 * 1946 * @warning The blocksize is a critical parameter with important 1947 * restrictions in the allowed values, so use this with care. 1948 */ 1949 BLOSC_EXPORT void blosc_set_blocksize(size_t blocksize); 1950 1951 /** 1952 * @brief Set pointer to super-chunk. If NULL, no super-chunk will be 1953 * available (the default). 1954 */ 1955 BLOSC_EXPORT void blosc_set_schunk(blosc2_schunk* schunk); 1956 1957 /********************************************************************* 1958 Structures and functions related with compression codecs. 1959 *********************************************************************/ 1960 1961 typedef int (* blosc2_codec_encoder_cb) (const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, uint8_t meta, blosc2_cparams *cparams); 1962 typedef int (* blosc2_codec_decoder_cb) (const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, uint8_t meta, blosc2_dparams *dparams); 1963 1964 typedef struct { 1965 uint8_t compcode; 1966 //!< The codec identifier. 1967 char *compname; 1968 //!< The codec name. 1969 uint8_t complib; 1970 //!< The codec library format. 1971 uint8_t compver; 1972 //!< The codec version. 1973 blosc2_codec_encoder_cb encoder; 1974 //!< The codec encoder that is used during compression. 1975 blosc2_codec_decoder_cb decoder; 1976 //!< The codec decoder that is used during decompression. 1977 } blosc2_codec; 1978 1979 /** 1980 * @brief Register locally a user-defined codec in Blosc. 1981 * 1982 * @param codec The codec to register. 1983 * 1984 * @return 0 if succeeds. Else a negative code is returned. 1985 */ 1986 BLOSC_EXPORT int blosc2_register_codec(blosc2_codec *codec); 1987 1988 1989 /********************************************************************* 1990 Structures and functions related with filters plugins. 1991 *********************************************************************/ 1992 1993 typedef int (* blosc2_filter_forward_cb) (const uint8_t *, uint8_t *, int32_t, uint8_t, blosc2_cparams *); 1994 typedef int (* blosc2_filter_backward_cb) (const uint8_t *, uint8_t *, int32_t, uint8_t, blosc2_dparams *); 1995 1996 /** 1997 * @brief The parameters for a user-defined filter. 1998 */ 1999 typedef struct { 2000 uint8_t id; 2001 //!< The filter identifier. 2002 blosc2_filter_forward_cb forward; 2003 //!< The filter function that is used during compression. 2004 blosc2_filter_backward_cb backward; 2005 //!< The filter function that is used during decompression. 2006 } blosc2_filter; 2007 2008 /** 2009 * @brief Register locally a user-defined filter in Blosc. 2010 * 2011 * @param filter The filter to register. 2012 * 2013 * @return 0 if succeeds. Else a negative code is returned. 2014 */ 2015 BLOSC_EXPORT int blosc2_register_filter(blosc2_filter *filter); 2016 2017 2018 /********************************************************************* 2019 Directory utilities. 2020 *********************************************************************/ 2021 2022 /* 2023 * Remove a directory and its files. 2024 */ 2025 BLOSC_EXPORT int blosc2_remove_dir(const char *path); 2026 2027 /* 2028 * Remove a file or a directory given by path. 2029 */ 2030 BLOSC_EXPORT int blosc2_remove_urlpath(const char *path); 2031 2032 /* 2033 * Rename a file or a directory given by old_urlpath to new_path. 2034 */ 2035 BLOSC_EXPORT int blosc2_rename_urlpath(char* old_urlpath, char* new_path); 2036 2037 #ifdef __cplusplus 2038 } 2039 #endif 2040 2041 2042 #endif /* BLOSC_H */ 2043