1 /* This is part of the netCDF package.
2 Copyright 2011 University Corporation for Atmospheric Research/Unidata
3 See COPYRIGHT file for conditions of use.
4
5 Test netcdf-4 chunking.
6 */
7
8 #include <nc_tests.h>
9 #include "err_macros.h"
10
11 #define FILE_NAME "tst_chunks2.nc"
12 #define MAX_WASTE 25.0
13 #define NUM_RANDOM_TESTS 3
14 #define NDIMS3 3
15
16 /* Calculate the waste of the chunking. A waste of 10% means the
17 * chunked data is 10% larget then the unchunked data. */
18 static int
calculate_waste(int ndims,size_t * dimlen,size_t * chunksize,float * waste)19 calculate_waste(int ndims, size_t *dimlen, size_t *chunksize, float *waste)
20 {
21 int d;
22 float chunked = 1, unchunked = 1;
23 size_t *num_chunks;
24 size_t chunk_size = 1;
25
26 assert(waste && dimlen && chunksize && ndims);
27 if (!(num_chunks = calloc(ndims, sizeof(size_t)))) ERR;
28
29 #ifdef PRINT_CHUNK_WASTE_REPORT
30 printf("\n");
31 #endif
32 /* Caclulate the total space taken up by the chunked data. */
33 for (d = 0; d < ndims; d++)
34 {
35 /* How many chunks along this dimension are required to hold all the data? */
36 for (num_chunks[d] = 0; (num_chunks[d] * chunksize[d]) < (dimlen[d] ? dimlen[d] : 1);
37 num_chunks[d]++)
38 ;
39 chunked *= (num_chunks[d] * chunksize[d]);
40 }
41
42 /* Calculate the minimum space required for this data
43 * (i.e. unchunked) or one record of it. */
44 for (d = 0; d < ndims; d++)
45 unchunked *= (dimlen[d] ? dimlen[d] : 1);
46
47 #ifdef PRINT_CHUNK_WASTE_REPORT
48 printf("size for unchunked %g elements; size for chunked %g elements\n",
49 unchunked, chunked);
50 #endif
51
52 /* Percent of the chunked file that is wasted space. */
53 *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0;
54
55 #ifdef PRINT_CHUNK_WASTE_REPORT
56 printf("\ndimlen\tchunksize\tnum_chunks\n");
57 #endif
58 for (d = 0; d < ndims; d++)
59 {
60 #ifdef PRINT_CHUNK_WASTE_REPORT
61 printf("%ld\t%ld\t\t%ld\n", (long int)dimlen[d], (long int)chunksize[d],
62 (long int)num_chunks[d]);
63 #endif
64 chunk_size *= chunksize[d];
65 }
66 #ifdef PRINT_CHUNK_WASTE_REPORT
67 printf("size of chunk: %ld elements; wasted space: %2.2f percent\n",
68 (long int)chunk_size, *waste);
69 #endif
70
71 free(num_chunks);
72 return 0;
73 }
74
75 int
main(int argc,char ** argv)76 main(int argc, char **argv)
77 {
78 printf("\n*** Testing netcdf-4 variable chunking.\n");
79 printf("**** testing default chunksizes...");
80 {
81 #define NDIMS3 3
82 #define NUM_VARS 1
83 #define Y_NAME "y"
84 #define X_NAME "x"
85 #define Z_NAME "z"
86 #define VAR_NAME_JOE "joe"
87 #define XDIM_LEN 2
88 #define YDIM_LEN 5
89 #define ZDIM_LEN 3000
90
91 int varid, ncid, dims[NDIMS3], dims_in[NDIMS3];
92 int ndims, nvars, ngatts, unlimdimid, natts;
93 char name_in[NC_MAX_NAME + 1];
94 nc_type type_in;
95 size_t len_in[NDIMS3];
96 int storage = 0;
97 size_t chunksizes[NDIMS3];
98 float waste = 0;
99
100 /* Create a file with 3D var, turn on chunking, but don't provide chunksizes. */
101 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
102 if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR;
103 if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR;
104 if (nc_def_dim(ncid, Z_NAME, ZDIM_LEN, &dims[2])) ERR;
105 if (nc_def_var(ncid, VAR_NAME_JOE, NC_FLOAT, NDIMS3, dims, &varid)) ERR;
106 if (nc_def_var_chunking(ncid, 0, NC_CHUNKED, NULL)) ERR;
107
108 /* Check it out. */
109 if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
110 if (nvars != NUM_VARS || ndims != NDIMS3 || ngatts != 0 || unlimdimid != -1) ERR;
111 if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR;
112 if (strcmp(name_in, VAR_NAME_JOE) || type_in != NC_FLOAT || ndims != NDIMS3 ||
113 dims_in[0] != dims[0] || dims_in[1] != dims[1] || dims_in[2] != dims[2] || natts != 0) ERR;
114 if (nc_inq_dim(ncid, 0, name_in, &len_in[0])) ERR;
115 if (strcmp(name_in, X_NAME) || len_in[0] != XDIM_LEN) ERR;
116 if (nc_inq_dim(ncid, 1, name_in, &len_in[1])) ERR;
117 if (strcmp(name_in, Y_NAME) || len_in[1] != YDIM_LEN) ERR;
118 if (nc_inq_dim(ncid, 2, name_in, &len_in[2])) ERR;
119 if (strcmp(name_in, Z_NAME) || len_in[2] != ZDIM_LEN) ERR;
120 if (nc_inq_var_chunking(ncid, 0, &storage, chunksizes)) ERR;
121 if (storage != NC_CHUNKED) ERR;
122 if (nc_close(ncid)) ERR;
123
124 /* Open the file and check again. */
125 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
126 if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
127 if (nvars != NUM_VARS || ndims != NDIMS3 || ngatts != 0 || unlimdimid != -1) ERR;
128 if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR;
129 if (strcmp(name_in, VAR_NAME_JOE) || type_in != NC_FLOAT || ndims != NDIMS3 ||
130 dims_in[0] != dims[0] || dims_in[1] != dims[1] || dims_in[2] != dims[2] || natts != 0) ERR;
131 if (nc_inq_dim(ncid, 0, name_in, &len_in[0])) ERR;
132 if (strcmp(name_in, X_NAME) || len_in[0] != XDIM_LEN) ERR;
133 if (nc_inq_dim(ncid, 1, name_in, &len_in[1])) ERR;
134 if (strcmp(name_in, Y_NAME) || len_in[1] != YDIM_LEN) ERR;
135 if (nc_inq_dim(ncid, 2, name_in, &len_in[2])) ERR;
136 if (strcmp(name_in, Z_NAME) || len_in[2] != ZDIM_LEN) ERR;
137 if (nc_inq_var_chunking(ncid, 0, &storage, chunksizes)) ERR;
138 if (storage != NC_CHUNKED) ERR;
139 if (calculate_waste(NDIMS3, len_in, chunksizes, &waste)) ERR;
140 /*if (waste > MAX_WASTE) ERR;*/
141 if (nc_close(ncid)) ERR;
142 }
143 SUMMARIZE_ERR;
144 printf("**** testing default chunksizes some more for a 3D var...");
145 {
146 #define NDIMS3 3
147 #define VAR_NAME "op-amp"
148
149 int varid, ncid;
150 int dimids[NDIMS3];
151 size_t dim_len[NDIMS3] = {1, 11, 152750};
152
153 int storage = 0;
154 size_t chunksizes[NDIMS3];
155 int d;
156 char dim_name[NC_MAX_NAME + 1];
157 float waste;
158
159 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
160
161 /* Create a few dimensions. */
162 for (d = 0; d < NDIMS3; d++)
163 {
164 sprintf(dim_name, "dim_%d", d);
165 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
166 }
167
168 /* Define a var with these dimensions, and turn on chunking. */
169 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
170 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
171
172 /* Check how default chunking worked. */
173 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
174 if (storage != NC_CHUNKED) ERR;
175 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
176 /* if (waste > MAX_WASTE) ERR;*/
177
178 if (nc_close(ncid)) ERR;
179
180 /* Open the file and check. */
181 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
182 if (nc_close(ncid)) ERR;
183 }
184 SUMMARIZE_ERR;
185 printf("**** testing default chunksizes even more for a 3D var...");
186 {
187 int varid, ncid;
188 int dimids[NDIMS3];
189 size_t dim_len[NDIMS3] = {1804289383, 846930886, 1681692777};
190
191 int storage = 0;
192 size_t chunksizes[NDIMS3];
193 int d;
194 char dim_name[NC_MAX_NAME + 1];
195 float waste;
196
197 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
198
199 /* Create a few dimensions. */
200 for (d = 0; d < NDIMS3; d++)
201 {
202 sprintf(dim_name, "dim_%d", d);
203 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
204 }
205
206 /* Define a var with these dimensions, and turn on chunking. */
207 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
208 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
209
210 /* Check how default chunking worked. */
211 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
212 if (storage != NC_CHUNKED) ERR;
213 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
214 /* if (waste > MAX_WASTE) ERR;*/
215
216 if (nc_close(ncid)) ERR;
217
218 /* Open the file and check. */
219 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
220 if (nc_close(ncid)) ERR;
221 }
222 SUMMARIZE_ERR;
223 printf("**** testing default chunksizes even even more for a 3D var...");
224 {
225 int varid, ncid;
226 int dimids[NDIMS3];
227 size_t dim_len[NDIMS3] = {1714636915, 1957747793, 424238335};
228
229 int storage = 0;
230 size_t chunksizes[NDIMS3];
231 int d;
232 char dim_name[NC_MAX_NAME + 1];
233 float waste;
234
235 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
236
237 /* Create a few dimensions. */
238 for (d = 0; d < NDIMS3; d++)
239 {
240 sprintf(dim_name, "dim_%d", d);
241 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
242 }
243
244 /* Define a var with these dimensions, and turn on chunking. */
245 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
246 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
247
248 /* Check how default chunking worked. */
249 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
250 if (storage != NC_CHUNKED) ERR;
251 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
252 /* if (waste > MAX_WASTE) ERR;*/
253
254 if (nc_close(ncid)) ERR;
255
256 /* Open the file and check. */
257 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
258 if (nc_close(ncid)) ERR;
259 }
260 SUMMARIZE_ERR;
261 printf("**** testing default chunksizes some more for a 3D var...");
262 {
263 #define NDIMS3 3
264 #define VAR_NAME "op-amp"
265
266 int varid, ncid;
267 int dimids[NDIMS3];
268 size_t dim_len[NDIMS3] = {1967513926, 1365180540, 426};
269
270 int storage = 0;
271 size_t chunksizes[NDIMS3];
272 int d;
273 char dim_name[NC_MAX_NAME + 1];
274 float waste;
275
276 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
277
278 /* Create a few dimensions. */
279 for (d = 0; d < NDIMS3; d++)
280 {
281 sprintf(dim_name, "dim_%d", d);
282 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
283 }
284
285 /* Define a var with these dimensions, and turn on chunking. */
286 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
287 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
288
289 /* Check how default chunking worked. */
290 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
291 if (storage != NC_CHUNKED) ERR;
292 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
293 /* if (waste > MAX_WASTE) ERR;*/
294
295 if (nc_close(ncid)) ERR;
296
297 /* Open the file and check. */
298 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
299 if (nc_close(ncid)) ERR;
300 }
301 SUMMARIZE_ERR;
302 printf("**** testing default chunksizes for very large 3D var...");
303 {
304 #define NDIMS3 3
305
306 int varid, ncid;
307 int dimids[NDIMS3];
308 size_t dim_len[NDIMS3] = {1804289383, 846930886, 1681692777};
309
310 int storage = 0;
311 size_t chunksizes[NDIMS3];
312 int d;
313 char dim_name[NC_MAX_NAME + 1];
314 float waste;
315
316 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
317
318 /* Create a few dimensions. */
319 for (d = 0; d < NDIMS3; d++)
320 {
321 sprintf(dim_name, "dim_%d", d);
322 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
323 }
324
325 /* Define a var with these dimensions, and turn on chunking. */
326 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
327 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
328
329 /* Check how default chunking worked. */
330 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
331 if (storage != NC_CHUNKED) ERR;
332 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
333 /* if (waste > MAX_WASTE) ERR;*/
334
335 if (nc_close(ncid)) ERR;
336
337 /* Open the file and check. */
338 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
339 if (nc_close(ncid)) ERR;
340 }
341 SUMMARIZE_ERR;
342 printf("**** testing default chunksizes some randomly sized 3D vars...");
343 {
344 #define NDIMS3 3
345
346 int varid, ncid;
347 int dimids[NDIMS3];
348 size_t dim_len[NDIMS3];
349 int storage = 0;
350 size_t chunksizes[NDIMS3];
351 int d, t;
352 char dim_name[NC_MAX_NAME + 1];
353 float waste;
354
355 for (t = 0; t < NUM_RANDOM_TESTS; t++)
356 {
357 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
358
359 /* Create a few dimensions. */
360 for (d = 0; d < NDIMS3; d++)
361 {
362 dim_len[d] = rand();
363 sprintf(dim_name, "dim_%d", d);
364 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
365 }
366
367 /* Define a var with these dimensions, and turn on chunking. */
368 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
369 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
370
371 /* Check how well default chunking worked. */
372 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
373 if (storage != NC_CHUNKED) ERR;
374 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
375 if (waste > MAX_WASTE) ERR;
376
377 if (nc_close(ncid)) ERR;
378 }
379 }
380 SUMMARIZE_ERR;
381 printf("**** testing default chunksizes some randomly sized 3D vars, with one small dimension...");
382 {
383 int varid, ncid;
384 int dimids[NDIMS3];
385 size_t dim_len[NDIMS3];
386 int storage = 0;
387 size_t chunksizes[NDIMS3];
388 int d, t;
389 char dim_name[NC_MAX_NAME + 1];
390 float waste;
391
392 for (t = 0; t < NUM_RANDOM_TESTS; t++)
393 {
394 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
395
396 dim_len[0] = rand();
397 dim_len[1] = rand();
398 dim_len[2] = rand() % 1000;
399 /* Create a few dimensions. */
400 for (d = 0; d < NDIMS3; d++)
401 {
402 sprintf(dim_name, "dim_%d", d);
403 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
404 }
405
406 /* Define a var with these dimensions, and turn on chunking. */
407 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
408 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
409
410 /* Check how well default chunking worked. */
411 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
412 if (storage != NC_CHUNKED) ERR;
413 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
414 if (waste > MAX_WASTE) ERR;
415
416 if (nc_close(ncid)) ERR;
417 }
418 }
419 SUMMARIZE_ERR;
420 printf("**** testing default chunksizes some randomly sized 3D vars, with two small dimensions...");
421 {
422 int varid, ncid;
423 int dimids[NDIMS3];
424 size_t dim_len[NDIMS3];
425 int storage = 0;
426 size_t chunksizes[NDIMS3];
427 int d, t;
428 char dim_name[NC_MAX_NAME + 1];
429 float waste;
430
431 for (t = 0; t < NUM_RANDOM_TESTS; t++)
432 {
433 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
434
435 dim_len[0] = rand();
436 dim_len[1] = rand() % 1000;
437 dim_len[2] = rand() % 1000;
438 /* Create a few dimensions. */
439 for (d = 0; d < NDIMS3; d++)
440 {
441 sprintf(dim_name, "dim_%d", d);
442 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
443 }
444
445 /* Define a var with these dimensions, and turn on chunking. */
446 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
447 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
448
449 /* Check how well default chunking worked. */
450 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
451 if (storage != NC_CHUNKED) ERR;
452 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
453 if (waste > MAX_WASTE) ERR;
454
455 if (nc_close(ncid)) ERR;
456 }
457 }
458 SUMMARIZE_ERR;
459 FINAL_RESULTS;
460 }
461