1 /* This is part of the netCDF package.
2 Copyright 2005 University Corporation for Atmospheric Research/Unidata
3 See COPYRIGHT file for conditions of use.
4
5 Test netcdf-4 variables.
6 Ed Hartnett
7 */
8
9 #include <nc_tests.h>
10 #include "err_macros.h"
11
12 #define FILE_NAME "tst_chunks.nc"
13 #define NDIMS1 1
14 #define D_SMALL "small_dim"
15 #define D_SMALL_LEN 16
16 #define D_MEDIUM "medium_dim"
17 #define D_MEDIUM_LEN 65546
18 #define D_LARGE "large_dim"
19 #define D_LARGE_LEN 1048586
20 #define V_SMALL "small_var"
21 #define V_MEDIUM "medium_var"
22 #define V_LARGE "large_var"
23
24 int
main(int argc,char ** argv)25 main(int argc, char **argv)
26 {
27 printf("\n*** Testing netcdf-4 variable chunking.\n");
28 printf("**** testing that fixed vars with filter end up being chunked, with good sizes...");
29 {
30
31 int ncid;
32 int nvars, ndims, ngatts, unlimdimid;
33 int contig;
34 int ndims_in, natts_in, dimids_in;
35 int small_dimid, medium_dimid, large_dimid;
36 int small_varid, medium_varid, large_varid;
37 char var_name_in[NC_MAX_NAME + 1];
38 size_t chunksize_in[NDIMS1];
39 nc_type xtype_in;
40
41 /* Create a netcdf-4 file with three dimensions. */
42 if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
43 if (nc_def_dim(ncid, D_SMALL, D_SMALL_LEN, &small_dimid)) ERR;
44 if (nc_def_dim(ncid, D_MEDIUM, D_MEDIUM_LEN, &medium_dimid)) ERR;
45 if (nc_def_dim(ncid, D_LARGE, D_LARGE_LEN, &large_dimid)) ERR;
46
47 /* Add three vars, with filters to force chunking. */
48 if (nc_def_var(ncid, V_SMALL, NC_INT64, NDIMS1, &small_dimid, &small_varid)) ERR;
49 if (nc_def_var_deflate(ncid, small_varid, 0, 1, 4)) ERR;
50 if (nc_def_var(ncid, V_MEDIUM, NC_INT64, NDIMS1, &medium_dimid, &medium_varid)) ERR;
51 if (nc_def_var_deflate(ncid, medium_varid, 1, 0, 0)) ERR;
52 if (nc_def_var(ncid, V_LARGE, NC_INT64, NDIMS1, &large_dimid, &large_varid)) ERR;
53 if (nc_def_var_fletcher32(ncid, large_varid, 1)) ERR;
54 if (nc_close(ncid)) ERR;
55
56 /* Open the file and check. */
57 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
58 if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
59 if (nvars != 3 || ndims != 3 || ngatts != 0 || unlimdimid != -1) ERR;
60 if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims_in, &dimids_in, &natts_in)) ERR;
61 if (strcmp(var_name_in, V_SMALL) || xtype_in != NC_INT64 || ndims_in != 1 ||
62 natts_in != 0) ERR;
63
64 /* Make sure chunking sizes are what we expect. */
65 if (nc_inq_var_chunking(ncid, small_varid, &contig, chunksize_in)) ERR;
66 if (contig || chunksize_in[0] != D_SMALL_LEN) ERR;
67 if (nc_inq_var_chunking(ncid, medium_varid, &contig, chunksize_in)) ERR;
68 if (contig || chunksize_in[0] * sizeof(long long) > DEFAULT_CHUNK_SIZE) ERR;
69 if (nc_inq_var_chunking(ncid, large_varid, &contig, chunksize_in)) ERR;
70 if (contig || chunksize_in[0] * sizeof(long long) > DEFAULT_CHUNK_SIZE) ERR;
71
72 if (nc_close(ncid)) ERR;
73 }
74 SUMMARIZE_ERR;
75 printf("**** testing default chunksizes...");
76 {
77 int nvars, ndims, ngatts, unlimdimid;
78 int contig;
79 #define NUM_DIM 4
80 #define NUM_TYPE 2
81 int ncid;
82 int dim_len[NUM_DIM] = {NC_UNLIMITED, 100, 1000, 2000};
83 size_t chunksize_in[NUM_DIM];
84 int type_id[NUM_TYPE] = {NC_BYTE, NC_INT};
85 int dimid[NUM_DIM], varid[NUM_TYPE];
86 char dim_name[NC_MAX_NAME + 1], var_name[NC_MAX_NAME + 1];
87 int d, t;
88
89 /* Create a netcdf-4 file with NUM_DIM dimensions. */
90 if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
91 for (d = 0; d < NUM_DIM; d++)
92 {
93 sprintf(dim_name, "dim_%d", dim_len[d]);
94 #ifdef PRINT_DEFAULT_CHUNKSIZE_TABLE
95 printf("creating dim %s\n", dim_name);
96 #endif
97 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimid[d])) ERR;
98 }
99
100 for (t = 0; t < NUM_TYPE; t++)
101 {
102 sprintf(var_name, "var_%d", type_id[t]);
103 if (nc_def_var(ncid, var_name, type_id[t], NUM_DIM, dimid, &varid[t])) ERR;
104 if (nc_inq_var_chunking(ncid, varid[t], &contig, chunksize_in)) ERR;
105 #ifdef PRINT_DEFAULT_CHUNKSIZE_TABLE
106 printf("chunksizes for %d x %d x %d x %d var: %d x %d x %d x %d (=%d)\n",
107 dim_len[0], dim_len[1], dim_len[2], dim_len[3],
108 (int)chunksize_in[0], (int)chunksize_in[1], (int)chunksize_in[2],
109 (int)chunksize_in[3],
110 (int)(chunksize_in[0] * chunksize_in[1] * chunksize_in[2] * chunksize_in[3]));
111 #endif
112 }
113
114 if (nc_close(ncid)) ERR;
115
116 /* Open the file and check. */
117 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
118 if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
119 if (nvars != NUM_TYPE || ndims != NUM_DIM || ngatts != 0 || unlimdimid != 0) ERR;
120
121 for (t = 0; t < NUM_TYPE; t++)
122 {
123 sprintf(var_name, "var_%d", type_id[t]);
124 if (nc_inq_var_chunking(ncid, varid[t], &contig, chunksize_in)) ERR;
125 if (contig) ERR;
126 #ifdef PRINT_DEFAULT_CHUNKSIZE_TABLE
127 printf("chunksizes for %d x %d x %d x %d var: %d x %d x %d x %d (=%d)\n",
128 dim_len[0], dim_len[1], dim_len[2], dim_len[3],
129 (int)chunksize_in[0], (int)chunksize_in[1], (int)chunksize_in[2],
130 (int)chunksize_in[3],
131 (int)(chunksize_in[0] * chunksize_in[1] * chunksize_in[2] * chunksize_in[3]));
132 #endif
133 }
134
135 if (nc_close(ncid)) ERR;
136 }
137 SUMMARIZE_ERR;
138 printf("**** testing that chunking works on classic mode files...");
139 {
140 #define D_SMALL_LEN2 66
141 int ncid;
142 int nvars, ndims, ngatts, unlimdimid;
143 int contig;
144 int ndims_in, natts_in, dimids_in;
145 int small_dimid, medium_dimid, large_dimid;
146 int small_varid, medium_varid, large_varid;
147 char var_name_in[NC_MAX_NAME + 1];
148 size_t chunks[1], chunksize_in;
149 nc_type xtype_in;
150
151 /* Create a netcdf-4 file with three dimensions. */
152 if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
153 if (nc_def_dim(ncid, D_SMALL, D_SMALL_LEN2, &small_dimid)) ERR;
154 if (nc_def_dim(ncid, D_MEDIUM, D_MEDIUM_LEN, &medium_dimid)) ERR;
155 if (nc_def_dim(ncid, D_LARGE, D_LARGE_LEN, &large_dimid)) ERR;
156
157 /* Add three vars. */
158 if (nc_def_var(ncid, V_SMALL, NC_INT64, NDIMS1, &small_dimid, &small_varid)) ERR;
159 if (nc_def_var_chunking(ncid, small_varid, 1, NULL)) ERR;
160
161 if (nc_def_var(ncid, V_MEDIUM, NC_INT64, NDIMS1, &medium_dimid, &medium_varid)) ERR;
162 chunks[0] = D_MEDIUM_LEN / 100;
163 if (nc_def_var_chunking(ncid, medium_varid, 0, chunks)) ERR;
164 if (nc_def_var_deflate(ncid, medium_varid, 1, 0, 0)) ERR;
165
166 if (nc_def_var(ncid, V_LARGE, NC_INT64, NDIMS1, &large_dimid, &large_varid)) ERR;
167 chunks[0] = D_LARGE_LEN / 1000;
168 if (nc_def_var_chunking(ncid, large_varid, 0, chunks)) ERR;
169 if (nc_def_var_fletcher32(ncid, large_varid, 1)) ERR;
170 if (nc_close(ncid)) ERR;
171
172 /* Open the file and check. */
173 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
174 if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
175 if (nvars != 3 || ndims != 3 || ngatts != 0 || unlimdimid != -1) ERR;
176 if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims_in, &dimids_in, &natts_in)) ERR;
177 if (strcmp(var_name_in, V_SMALL) || xtype_in != NC_INT64 || ndims_in != 1 ||
178 natts_in != 0) ERR;
179
180 /* Make sure chunking settings are what we expect. */
181 if (nc_inq_var_chunking(ncid, small_varid, &contig, &chunksize_in)) ERR;
182 if (!contig) ERR;
183 if (nc_inq_var_chunking(ncid, medium_varid, &contig, &chunksize_in)) ERR;
184 if (contig || chunksize_in != D_MEDIUM_LEN / 100) ERR;
185 if (nc_inq_var_chunking(ncid, large_varid, &contig, &chunksize_in)) ERR;
186 if (contig || chunksize_in != D_LARGE_LEN / 1000) ERR;
187
188 if (nc_close(ncid)) ERR;
189 }
190 SUMMARIZE_ERR;
191 printf("**** testing many chunking and contiguous variables...");
192 {
193 #define NDIMS_3 3
194 #define NUM_PLANS 30
195 #define D_SNEAKINESS "sneakiness"
196 #define D_SNEAKINESS_LEN 5
197 #define D_CLEVERNESS "clevernesss"
198 #define D_CLEVERNESS_LEN 3
199 #define D_EFFECTIVENESS "effectiveness"
200 #define D_EFFECTIVENESS_LEN 2
201
202 int ncid, dimids[NDIMS_3], varid[NUM_PLANS];
203 size_t chunksize[NDIMS_3] = {D_SNEAKINESS_LEN, D_CLEVERNESS_LEN,
204 D_EFFECTIVENESS_LEN};
205 char plan_name[NC_MAX_NAME + 1];
206 int contig;
207 size_t chunksize_in[NDIMS_3];
208 int i, j;
209
210 /* Create a netcdf-4 file with three dimensions. */
211 if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
212 if (nc_def_dim(ncid, D_SNEAKINESS, D_SNEAKINESS_LEN, &dimids[0])) ERR;
213 if (nc_def_dim(ncid, D_CLEVERNESS, D_CLEVERNESS_LEN, &dimids[1])) ERR;
214 if (nc_def_dim(ncid, D_EFFECTIVENESS, D_EFFECTIVENESS_LEN, &dimids[2])) ERR;
215
216 /* Oh that tricky Cardinal Richelieu, he had many plans! */
217 for (i = 0; i < NUM_PLANS; i++)
218 {
219 sprintf(plan_name, "Richelieu_sneaky_plan_%d", i);
220 if (nc_def_var(ncid, plan_name, i % (NC_STRING - 1) + 1, NDIMS_3,
221 dimids, &varid[i])) ERR;
222 if (i % 2 && nc_def_var_chunking(ncid, varid[i], 0, chunksize)) ERR;
223 }
224
225 /* Check the chunking. */
226 for (i = 0; i < NUM_PLANS; i++)
227 {
228 if (nc_inq_var_chunking(ncid, varid[i], &contig, chunksize_in)) ERR;
229 if (i % 2)
230 {
231 for (j = 0; j < NDIMS_3; j++)
232 if (chunksize_in[j] != chunksize[j]) ERR;
233 }
234 else
235 if (!contig) ERR;
236 }
237 if (nc_close(ncid)) ERR;
238
239 /* Open the file and check. */
240 if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
241 /* Check the chunking. */
242 for (i = 0; i < NUM_PLANS; i++)
243 {
244 if (nc_inq_var_chunking(ncid, varid[i], &contig, chunksize_in)) ERR;
245 if (i % 2)
246 {
247 for (j = 0; j < NDIMS_3; j++)
248 if (chunksize_in[j] != chunksize[j]) ERR;
249 }
250 else
251 if (!contig) ERR;
252 }
253 if (nc_close(ncid)) ERR;
254 }
255 SUMMARIZE_ERR;
256 printf("**** testing that too large chunksizes fail...");
257 {
258 #define D_SMALL_LEN2 66
259 int stat = NC_NOERR;
260 int ncid;
261 int small_dimid;
262 int small_varid;
263 size_t chunks[1];
264
265 /* Create a netcdf-4 file with three dimensions. */
266 if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
267 if (nc_def_dim(ncid, D_SMALL, D_SMALL_LEN2, &small_dimid)) ERR;
268
269 /* Add one var. */
270 if (nc_def_var(ncid, V_SMALL, NC_INT64, NDIMS1, &small_dimid, &small_varid)) ERR;
271
272 /* Attempt to set too large chunksizes */
273 chunks[0] = D_SMALL_LEN2 + 1;
274 stat = nc_def_var_chunking(ncid, small_varid, NC_CHUNKED, chunks);
275 if(stat != NC_EBADCHUNK) {
276 printf("Return code is '%s', expected NC_BADCHUNK",nc_strerror(stat));
277 ERR;
278 }
279 /* try again with proper chunksize */
280 chunks[0] = D_SMALL_LEN2;
281 stat = nc_def_var_chunking(ncid, small_varid, NC_CHUNKED, chunks);
282 if(stat != NC_NOERR) {
283 printf("Return code is '%s', expected NC_NOERR",nc_strerror(stat));
284 ERR;
285 }
286 if (nc_abort(ncid)) ERR;
287 }
288 SUMMARIZE_ERR;
289 printf("**** testing cache size smaller than chunk size...");
290 {
291 #define NDIM2 2
292 #define DIM_X_LEN 10000
293 #define DIM_Y_LEN 10000
294 #define DIM_NAME_X_CACHE_CHUNK "Height"
295 #define DIM_NAME_Y_CACHE_CHUNK "Width"
296 #define VAR_NAME_CACHE_CHUNK "House_Size"
297 #define VAR_NAME_CACHE_CHUNK_2 "Boat_Size"
298 #define VAR_NAME_CACHE_CHUNK_3 "Deck_Size"
299
300 int ncid;
301 int dimid[NDIM2];
302 int varid, varid2, varid3;
303 size_t chunks[NDIM2] = {100, 100};
304 size_t chunks_big[NDIM2] = {DIM_X_LEN, DIM_Y_LEN};
305 size_t chunks_in[NDIM2];
306 int contiguous;
307 size_t cache_size = 16;
308 size_t cache_nelems = 1;
309 float cache_preemption = 0.5;
310 size_t cache_size_in;
311 size_t cache_nelems_in;
312 float cache_preemption_in;
313
314 /* Create a netcdf-4 file with two dimensions. */
315 if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
316 if (nc_def_dim(ncid, DIM_NAME_X_CACHE_CHUNK, DIM_X_LEN, &dimid[0])) ERR;
317 if (nc_def_dim(ncid, DIM_NAME_Y_CACHE_CHUNK, DIM_Y_LEN, &dimid[1])) ERR;
318
319 /* Add vars. */
320 if (nc_def_var(ncid, VAR_NAME_CACHE_CHUNK, NC_INT64, NDIM2, dimid, &varid)) ERR;
321 if (nc_def_var(ncid, VAR_NAME_CACHE_CHUNK_2, NC_INT64, NDIM2, dimid, &varid2)) ERR;
322 if (nc_def_var(ncid, VAR_NAME_CACHE_CHUNK_3, NC_INT64, NDIM2, dimid, &varid3)) ERR;
323
324 /* Set the var cache. */
325 if (nc_set_var_chunk_cache(ncid, varid, cache_size, cache_nelems,
326 cache_preemption)) ERR;
327
328 /* Set the chunking. */
329 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, chunks)) ERR;
330 if (nc_inq_var_chunking(ncid, varid, &contiguous, chunks_in)) ERR;
331 if (contiguous || chunks_in[0] != chunks[0] || chunks_in[1] != chunks[1]) ERR;
332 if (nc_def_var_chunking(ncid, varid2, NC_CHUNKED, chunks)) ERR;
333 if (nc_inq_var_chunking(ncid, varid2, &contiguous, chunks_in)) ERR;
334 if (contiguous || chunks_in[0] != chunks[0] || chunks_in[1] != chunks[1]) ERR;
335 if (nc_def_var_chunking(ncid, varid3, NC_CHUNKED, chunks_big)) ERR;
336 if (nc_inq_var_chunking(ncid, varid3, &contiguous, chunks_in)) ERR;
337 if (contiguous || chunks_in[0] != chunks_big[0] || chunks_in[1] != chunks_big[1]) ERR;
338
339 /* Get the var cache values. */
340 if (nc_get_var_chunk_cache(ncid, varid, &cache_size_in, &cache_nelems_in,
341 &cache_preemption_in)) ERR;
342 if (cache_size_in != cache_size || cache_nelems_in != cache_nelems ||
343 cache_preemption_in != cache_preemption) ERR;
344 if (nc_get_var_chunk_cache(ncid, varid2, &cache_size_in, &cache_nelems_in,
345 &cache_preemption_in)) ERR;
346 if (cache_size_in != CHUNK_CACHE_SIZE || cache_nelems_in != CHUNK_CACHE_NELEMS ||
347 cache_preemption_in != CHUNK_CACHE_PREEMPTION) ERR;
348
349 /* The cache_size has been increased due to larger chunksizes
350 * for varid3. */
351 if (nc_get_var_chunk_cache(ncid, varid3, &cache_size_in, &cache_nelems_in,
352 &cache_preemption_in)) ERR;
353 if (cache_nelems_in != CHUNK_CACHE_NELEMS ||
354 cache_preemption_in != CHUNK_CACHE_PREEMPTION) ERR;
355 /* printf("cache_size_in %ld\n", cache_size_in); */
356 #ifndef USE_PARALLEL
357 /* THe cache size does not change under parallel. Not sure why. */
358 if (cache_size_in <= CHUNK_CACHE_SIZE) ERR;
359 #endif
360
361 /* Close the file. */
362 if (nc_close(ncid)) ERR;
363
364 /* Reopen the file. */
365 if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
366
367 /* Close the file. */
368 if (nc_close(ncid)) ERR;
369 }
370 SUMMARIZE_ERR;
371 FINAL_RESULTS;
372 }
373