1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2 * Copyright by The HDF Group. *
3 * Copyright by the Board of Trustees of the University of Illinois. *
4 * All rights reserved. *
5 * *
6 * This file is part of HDF5. The full HDF5 copyright notice, including *
7 * terms governing use, modification, and redistribution, is contained in *
8 * the COPYING file, which can be found at the root of the source code *
9 * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
10 * If you do not have access to either file, you may request a copy from *
11 * help@hdfgroup.org. *
12 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
13
14 /*
15 * Programmer: Robb Matzke <matzke@llnl.gov>
16 * Monday, September 28, 1998
17 *
18 * Purpose: Creates a chunked dataset and measures the storage overhead.
19 */
20
21 /* See H5private.h for how to include headers */
22 #undef NDEBUG
23 #include "hdf5.h"
24 #include "H5private.h"
25
26 #ifdef H5_STDC_HEADERS
27 # include <ctype.h>
28 # include <fcntl.h>
29 # include <stdlib.h>
30 # include <sys/stat.h>
31 # include <string.h>
32 #endif
33
34 #ifdef H5_HAVE_IO_H
35 # include <io.h>
36 #endif
37
38 #ifdef H5_HAVE_UNISTD_H
39 # include <sys/types.h>
40 # include <unistd.h>
41 #endif
42
43 /* Solaris Studio defines attribute, but for the attributes we need */
44 #if !defined(H5_HAVE_ATTRIBUTE) || defined __cplusplus || defined(__SUNPRO_C)
45 # undef __attribute__
46 # define __attribute__(X) /*void*/
47 # define H5_ATTR_UNUSED /*void*/
48 #else
49 # define H5_ATTR_UNUSED __attribute__((unused))
50 #endif
51
52 #define FILE_NAME_1 "overhead.h5"
53 #ifndef FALSE
54 #define FALSE 0
55 #endif /* FALSE */
56 #ifndef TRUE
57 #define TRUE 1
58 #endif /* TRUE */
59
60 typedef enum fill_t {
61 FILL_ALL,
62 FILL_FORWARD,
63 FILL_REVERSE,
64 FILL_INWARD,
65 FILL_OUTWARD,
66 FILL_RANDOM
67 } fill_t;
68
69
70 /*-------------------------------------------------------------------------
71 * Function: usage
72 *
73 * Purpose: Prints a usage message and exits.
74 *
75 * Return: never returns
76 *
77 * Programmer: Robb Matzke
78 * Wednesday, September 30, 1998
79 *
80 * Modifications:
81 *
82 *-------------------------------------------------------------------------
83 */
84 static void
usage(const char * prog)85 usage(const char *prog)
86 {
87 HDfprintf(stderr, "usage: %s [STYLE|cache] [LEFT [MIDDLE [RIGHT]]]\n",
88 prog);
89 HDfprintf(stderr, "\
90 STYLE is the order that the dataset is filled and should be one of:\n\
91 forward -- Fill the dataset from lowest address to highest\n\
92 address. This style tests the right split ratio.\n\
93 reverse -- Fill the dataset from highest address to lowest\n\
94 address. This is the reverse order of `forward' and\n\
95 tests the left split ratio.\n\
96 inward -- Fill beginning at both the lowest and highest\n\
97 addresses and work in toward the center of the\n\
98 dataset. This tests the middle split ratio.\n\
99 outward -- Start at the center of the dataset and work outward\n\
100 toward the lowest and highest addresses. This tests\n\
101 both left and right split ratios.\n\
102 random -- Write the chunks of the dataset in random order. This\n\
103 tests all split ratios.\n\
104 If no fill style is specified then all fill styles are tried and a\n\
105 single value is printed for each one.\n\
106 \n\
107 If the word `cache' is used instead of a fill style then the raw data\n\
108 cache is enabled. It is not possible to enable the raw data cache when\n\
109 a specific fill style is used because H5Fflush() is called after each\n\
110 chunk is written in order to calculate overhead during the test. If\n\
111 the cache is enabled then chunks are written to disk in different orders\n\
112 than the actual H5Dwrite() calls in the test due to collisions and the\n\
113 resulting B-tree will be split differently.\n\
114 \n\
115 LEFT, MIDDLE, and RIGHT are the ratios to use for splitting and should\n\
116 be values between zero and one, inclusive.\n");
117 exit(1);
118 }
119
120
121 /*-------------------------------------------------------------------------
122 * Function: cleanup
123 *
124 * Purpose: Removes test files
125 *
126 * Return: void
127 *
128 * Programmer: Robb Matzke
129 * Thursday, June 4, 1998
130 *
131 * Modifications:
132 *
133 *-------------------------------------------------------------------------
134 */
135 static void
cleanup(void)136 cleanup (void)
137 {
138 if (!getenv ("HDF5_NOCLEANUP")) {
139 remove (FILE_NAME_1);
140 }
141 }
142
143
144 /*-------------------------------------------------------------------------
145 * Function: display_error_cb
146 *
147 * Purpose: Displays the error stack after printing "*FAILED*".
148 *
149 * Return: Success: 0
150 *
151 * Failure: -1
152 *
153 * Programmer: Robb Matzke
154 * Wednesday, March 4, 1998
155 *
156 * Modifications:
157 *
158 *-------------------------------------------------------------------------
159 */
160 static herr_t
display_error_cb(hid_t estack,void H5_ATTR_UNUSED * client_data)161 display_error_cb (hid_t estack, void H5_ATTR_UNUSED *client_data)
162 {
163 puts ("*FAILED*");
164 H5Eprint2(estack, stdout);
165
166 return 0;
167 }
168
169
170 /*-------------------------------------------------------------------------
171 * Function: test
172 *
173 * Purpose: The guts of the test
174 *
175 * Return: Success: 0
176 *
177 * Failure: number of errors
178 *
179 * Programmer: Robb Matzke
180 * Wednesday, September 30, 1998
181 *
182 * Modifications:
183 *
184 *-------------------------------------------------------------------------
185 */
186 static int
test(fill_t fill_style,const double splits[],hbool_t verbose,hbool_t use_rdcc)187 test(fill_t fill_style, const double splits[],
188 hbool_t verbose, hbool_t use_rdcc)
189 {
190 hid_t file = (-1), fapl = (-1), dcpl = (-1), xfer = (-1), mspace = (-1), fspace = (-1), dset = (-1);
191 hsize_t ch_size[1] = {1}; /*chunk size */
192 hsize_t cur_size[1] = {1000}; /*current dataset size */
193 hsize_t max_size[1] = {H5S_UNLIMITED}; /*maximum dataset size */
194 hsize_t hs_start[1]; /*hyperslab start offset*/
195 hsize_t hs_count[1] = {1}; /*hyperslab nelmts */
196 int fd = (-1); /*h5 file direct */
197 int *had = NULL; /*for random filling */
198 const char *sname=NULL; /*fill style nam */
199 int mdc_nelmts; /*num meta objs to cache*/
200 hsize_t i, k;
201 int j;
202 h5_stat_t sb;
203
204 if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto error;
205 if(!use_rdcc) {
206 if(H5Pget_cache(fapl, &mdc_nelmts, NULL, NULL, NULL) < 0) goto error;
207 if(H5Pset_cache(fapl, mdc_nelmts, 0, 0, 0.0F) < 0) goto error;
208 }
209 if((file = H5Fcreate(FILE_NAME_1, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) goto error;
210 if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto error;
211 if(H5Pset_chunk(dcpl, 1, ch_size) < 0) goto error;
212 if((xfer = H5Pcreate(H5P_DATASET_XFER)) < 0) goto error;
213 if(H5Pset_btree_ratios(xfer, splits[0], splits[1], splits[2]) < 0) goto error;
214 if((fspace = H5Screate_simple(1, cur_size, max_size)) < 0) goto error;
215 if((mspace = H5Screate_simple(1, ch_size, ch_size)) < 0) goto error;
216 if((dset = H5Dcreate2(file, "chunked", H5T_NATIVE_INT,
217 fspace, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error;
218 if ((fd=HDopen(FILE_NAME_1, O_RDONLY, 0666)) < 0) goto error;
219
220 if(FILL_RANDOM==fill_style)
221 had = (int *)calloc((size_t)cur_size[0], sizeof(int));
222
223 for (i=1; i<=cur_size[0]; i++) {
224
225 /* Decide which chunk to write to */
226 switch (fill_style) {
227 case FILL_FORWARD:
228 hs_start[0] = i-1;
229 break;
230 case FILL_REVERSE:
231 hs_start[0] = cur_size[0]-i;
232 break;
233 case FILL_INWARD:
234 hs_start[0] = i%2 ? i/2 : cur_size[0]-i/2;
235 break;
236 case FILL_OUTWARD:
237 k = (cur_size[0] - i) + 1;
238 hs_start[0] = k % 2 ? (k / 2) : (hsize_t)((hssize_t)cur_size[0] - (hssize_t)(k / 2));
239 break;
240 case FILL_RANDOM:
241 for (j=HDrand()%(int)cur_size[0]; had[j]; j=(j+1)%(int)cur_size[0])
242 /*void*/;
243 hs_start[0] = (hsize_t)j;
244 had[j] = 1;
245 break;
246 case FILL_ALL:
247 abort();
248 default:
249 /* unknown request */
250 HDfprintf(stderr, "Unknown fill style\n");
251 goto error;
252 break;
253 }
254
255 /* Write the chunk */
256 if (H5Sselect_hyperslab(fspace, H5S_SELECT_SET, hs_start, NULL,
257 hs_count, NULL) < 0) goto error;
258 if (H5Dwrite(dset, H5T_NATIVE_INT, mspace, fspace, xfer, &i) < 0) {
259 goto error;
260 }
261
262 /* Determine overhead */
263 if (verbose) {
264 if (H5Fflush(file, H5F_SCOPE_LOCAL) < 0) goto error;
265 if (HDfstat(fd, &sb) < 0) goto error;
266 printf("%4lu %8.3f ***\n",
267 (unsigned long)i,
268 (double)(sb.st_size - (HDoff_t)(i * sizeof(int))) / (double)i);
269 }
270 }
271
272 if(had) {
273 free(had);
274 had = NULL;
275 } /* end if */
276
277 H5Dclose(dset);
278 H5Sclose(mspace);
279 H5Sclose(fspace);
280 H5Pclose(dcpl);
281 H5Pclose(xfer);
282 H5Fclose(file);
283
284 if (!verbose) {
285 switch (fill_style) {
286 case FILL_FORWARD:
287 sname = "forward";
288 break;
289 case FILL_REVERSE:
290 sname = "reverse";
291 break;
292 case FILL_INWARD:
293 sname = "inward";
294 break;
295 case FILL_OUTWARD:
296 sname = "outward";
297 break;
298 case FILL_RANDOM:
299 sname = "random";
300 break;
301 case FILL_ALL:
302 abort();
303 default:
304 /* unknown request */
305 HDfprintf(stderr, "Unknown fill style\n");
306 goto error;
307 break;
308 }
309
310 if (HDfstat(fd, &sb) < 0) goto error;
311 printf("%-7s %8.3f\n", sname,
312 (double)(sb.st_size - (HDoff_t)(cur_size[0] * sizeof(int))) /
313 (double)cur_size[0]);
314 }
315 HDclose(fd);
316
317 return 0;
318
319 error:
320 H5Dclose(dset);
321 H5Sclose(mspace);
322 H5Sclose(fspace);
323 H5Pclose(dcpl);
324 H5Pclose(xfer);
325 H5Fclose(file);
326 if(had)
327 free(had);
328 HDclose(fd);
329 return 1;
330 }
331
332
333 /*-------------------------------------------------------------------------
334 * Function: main
335 *
336 * Purpose:
337 *
338 * Return: Success: zero
339 *
340 * Failure: non-zero
341 *
342 * Programmer: Robb Matzke
343 * Monday, September 28, 1998
344 *
345 * Modifications:
346 *
347 *-------------------------------------------------------------------------
348 */
349 int
main(int argc,char * argv[])350 main(int argc, char *argv[])
351 {
352 hid_t xfer;
353 fill_t fill_style = FILL_ALL;
354 hbool_t use_cache = FALSE;
355 double splits[3];
356 int i, j, nerrors=0;
357
358 /* Default split ratios */
359 H5Eset_auto2(H5E_DEFAULT, display_error_cb, NULL);
360
361 if((xfer = H5Pcreate(H5P_DATASET_XFER)) < 0) goto error;
362 if(H5Pget_btree_ratios(xfer, splits+0, splits+1, splits+2) < 0) goto error;
363 if(H5Pclose(xfer) < 0) goto error;
364
365 /* Parse command-line options */
366 for(i = 1, j = 0; i < argc; i++) {
367 if (!strcmp(argv[i], "forward")) {
368 fill_style = FILL_FORWARD;
369 } else if (!strcmp(argv[i], "reverse")) {
370 fill_style = FILL_REVERSE;
371 } else if (!strcmp(argv[i], "inward")) {
372 fill_style = FILL_INWARD;
373 } else if (!strcmp(argv[i], "outward")) {
374 fill_style = FILL_OUTWARD;
375 } else if (!strcmp(argv[i], "random")) {
376 fill_style = FILL_RANDOM;
377 } else if (!strcmp(argv[i], "cache")) {
378 use_cache = TRUE;
379 } else if (j<3 && (isdigit(argv[i][0]) || '.'==argv[i][0])) {
380 splits[j++] = strtod(argv[i], NULL);
381 } else {
382 usage(argv[0]);
383 }
384 }
385
386 if (FILL_ALL==fill_style) {
387 printf("%-7s %8s\n", "Style", "Bytes/Chunk");
388 printf("%-7s %8s\n", "-----", "-----------");
389 nerrors += test(FILL_FORWARD, splits, FALSE, use_cache);
390 nerrors += test(FILL_REVERSE, splits, FALSE, use_cache);
391 nerrors += test(FILL_INWARD, splits, FALSE, use_cache);
392 nerrors += test(FILL_OUTWARD, splits, FALSE, use_cache);
393 nerrors += test(FILL_RANDOM, splits, FALSE, use_cache);
394 }
395 else {
396 if (use_cache) usage(argv[0]);
397 nerrors += test(fill_style, splits, TRUE, FALSE);
398 }
399 if (nerrors>0) goto error;
400 cleanup();
401 return 0;
402
403 error:
404 HDfprintf(stderr, "*** ERRORS DETECTED ***\n");
405 return 1;
406 }
407