1 /*
2 Copyright (C) 2021 The Blosc Developers <blosc@blosc.org>
3 https://blosc.org
4 License: BSD 3-Clause (see LICENSE.txt)
5
6 Example program demonstrating frames going bigger than 2 GB.
7
8 To compile this program:
9
10 $ gcc -O frame_big.c -o frame_big -lblosc2
11
12 To run:
13
14 $ ./frame_big
15 Blosc version info: 2.0.0-beta.4.dev ($Date:: 2019-09-02 #$)
16 Compression ratio: 4577.6 MB -> 169.8 MB (27.0x)
17 Time for append data to a schunk backed by a fileframe: 2.61 s, 1750.8 MB/s
18 Successful roundtrip data <-> schunk (frame-backed) !
19
20 */
21
22 #include <stdio.h>
23 #include <assert.h>
24 #include <blosc2.h>
25
26 #define KB 1024.
27 #define MB (1024*KB)
28 #define GB (1024*MB)
29
30 #define CHUNKSIZE (1000 * 1000)
31 #define NCHUNKS 1200 // > 4 GB int32 frame
32 #define NTHREADS 4
33
34
main(void)35 int main(void) {
36 blosc_init();
37
38 static int32_t data[CHUNKSIZE];
39 static int32_t data_dest[CHUNKSIZE];
40 size_t isize = CHUNKSIZE * sizeof(int32_t);
41 blosc_timestamp_t last, current;
42
43 printf("Blosc version info: %s (%s)\n",
44 BLOSC_VERSION_STRING, BLOSC_VERSION_DATE);
45
46 // Compression and decompression parameters
47 blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
48 cparams.typesize = sizeof(int32_t);
49 cparams.clevel = 9;
50 cparams.nthreads = NTHREADS;
51 blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
52 dparams.nthreads = NTHREADS;
53
54 /* Create a new super-chunk backed by a fileframe */
55 char* urlpath = "frame_big.b2frame";
56 blosc2_storage storage = {.contiguous=true, .urlpath=urlpath,
57 .cparams=&cparams, .dparams=&dparams};
58 remove(urlpath);
59 blosc2_schunk* schunk = blosc2_schunk_new(&storage);
60
61 blosc_set_timestamp(&last);
62 for (int nchunk = 0; nchunk < NCHUNKS; nchunk++) {
63 for (int i = 0; i < CHUNKSIZE; i++) {
64 data[i] = i * nchunk;
65 }
66 int nchunks = blosc2_schunk_append_buffer(schunk, data, isize);
67 assert(nchunks == nchunk + 1);
68 }
69 /* Gather some info */
70 int64_t nbytes = schunk->nbytes;
71 int64_t cbytes = schunk->cbytes;
72 blosc_set_timestamp(¤t);
73 double ttotal = blosc_elapsed_secs(last, current);
74 printf("Compression ratio: %.1f MB -> %.1f MB (%.1fx)\n",
75 nbytes / MB, cbytes / MB, (1. * nbytes) / cbytes);
76 printf("Time for append data to a schunk backed by a fileframe: %.3g s, %.1f MB/s\n",
77 ttotal, nbytes / (ttotal * MB));
78
79 /* Retrieve and decompress the chunks from the super-chunks and compare values */
80 for (int nchunk = 0; nchunk < NCHUNKS; nchunk++) {
81 int32_t dsize = blosc2_schunk_decompress_chunk(schunk, nchunk, data_dest, isize);
82 if (dsize < 0) {
83 printf("Decompression error in schunk. Error code: %d\n", dsize);
84 return dsize;
85 }
86 /* Check integrity of the last chunk */
87 for (int i = 0; i < CHUNKSIZE; i++) {
88 assert (data_dest[i] == i * nchunk);
89 }
90 }
91
92 printf("Successful roundtrip data <-> schunk (frame-backed) !\n");
93
94 /* Free resources */
95 blosc2_schunk_free(schunk);
96
97 blosc_destroy();
98
99 return 0;
100 }
101