1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2 * Copyright by The HDF Group. *
3 * Copyright by the Board of Trustees of the University of Illinois. *
4 * All rights reserved. *
5 * *
6 * This file is part of HDF. The full HDF copyright notice, including *
7 * terms governing use, modification, and redistribution, is contained in *
8 * the COPYING file, which can be found at the root of the source code *
9 * distribution tree, or in https://support.hdfgroup.org/ftp/HDF/releases/. *
10 * If you do not have access to either file, you may request a copy from *
11 * help@hdfgroup.org. *
12 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
13
14 /*
15 FILE - buffer.c
16 Test HDF buffered data I/O routines
17
18 DESIGN
19 - Create a new data element and get a benchmark time for reading it in
20 various ways.
21 - Buffer the element and get times for the buffered element.
22 - Make a new external element and get benchmark times
23 - Buffer the external element and get times for the buffered element.
24 - Make a new compressed element and get benchmark times
25 - Buffer the compressed element and get times for the buffered element.
26 - Make a new linked block element and get benchmark times
27 - Buffer the linked block element and get times for the buffered element.
28
29 BUGS/LIMITATIONS
30
31 EXPORTED ROUTINES
32
33 AUTHOR
34 Quincey Koziol
35
36 MODIFICATION HISTORY
37 12/11/98 - Wrote tests
38 12/20/06 - Compute the wallclock time and use the environment
39 variable or macro HDF4_TESTPREFIX to add prefix to the
40 testing files names. Also, it allows to specify the number
41 of elements in buffer at the command line.
42
43 */
44
45 /* $Id$ */
46
47 #define TESTMASTER
48
49 #include "hdf.h"
50 #ifdef H4_HAVE_SYS_TIME_H
51 #include <sys/time.h>
52 #endif
53 #include "tutils.h"
54 #include "hfile.h"
55
56 #define TESTFILE_NAME "tbuffer.hdf"
57 #define EXTFILE_NAME "tbuffer.dat"
58
59 /* Size of data elements to create */
60 #ifdef __CRAY_XT3__
61 /* Use a smaller test size as small unbuffered IO is expensive in XT3. */
62 /* This runs much faster in Lustre file system like /scratchN. elemsize 1000 */
63 /* takes 3 minutes to run in /scratch3. 2006/6/20, -AKC- */
64 #define ELEMSIZE 1000
65 #else
66 #define ELEMSIZE 16384
67 #endif
68
69 /* define aliases for random number generation */
70 #define RAND rand
71 #define SEED(a) srand((unsigned)(a))
72
73 /* Tag to use for creating the test elements */
74 #define BUFF_TAG 1000
75
76 /* Number of tests */
77 #define NUM_TESTS 4
78
79 /* Factor for converting seconds to microseconds */
80 #define FACTOR 1000000
81
82 /* Number of timing tests to run */
83 /* 0 - read/write entire buffer in one I/O operation */
84 /* 1 - read/write entire buffer one byte at a time forwards */
85 /* 2 - read/write entire buffer one byte at a time every other byte forwards */
86 /* 3 - read/write entire buffer one byte at a time backwards */
87 /* 4 - read/write entire buffer one byte at a time every other byte backwards */
88 #define NUM_TIMINGS 5
89 long read_time[NUM_TESTS][2]; /* 0 is unbuffered, 1 is buffered */
90 long write_time[NUM_TESTS][2]; /* 0 is unbuffered, 1 is buffered */
91
92 int32 elemsize; /* Actual number of elements in buffer */
93
94 /* I/O buffers */
95 uint8 *out_buf; /* Buffer for writing data */
96 uint8 *in_buf; /* Buffer for reading data */
97
98 /* local function prototypes */
99 static void init_buffer(void);
100 static void usage(void);
101 static char *fixname(const char *base_name, char *fullname, size_t size);
102 static long read_test(int32 aid);
103 static long write_test(int32 aid,intn num_timings);
104
105 /* Initialize output buffer */
106 static void
init_buffer(void)107 init_buffer(void)
108 {
109 intn j;
110
111 SEED(time(NULL));
112 for (j = 0; j < elemsize; j++)
113 {
114 out_buf[j] = (uint8) RAND();
115 } /* end for */
116 } /* init_buffers() */
117
118 static void
usage(void)119 usage(void)
120 {
121 printf("\nUsage: buffer [elemsize] \n\n");
122 printf("where elemsize is the number of elements in buffer (default: 1000 in Cray, 16384 in other platforms)\n");
123 printf("\n");
124 } /* end usage() */
125
126
127 /*
128 Creates a file name from a file base name like 'test' and return it through
129 the FULLNAME (at most SIZE characters counting the null terminator). The
130 full name is created by prepending the contents of HDF4_TESTPREFIX
131 (separated from the base name by a slash). Returns NULL if BASENAME or
132 FULLNAME is the null pointer or if FULLNAME isn't large enough for the
133 result.
134 */
fixname(const char * base_name,char * fullname,size_t size)135 static char *fixname(const char *base_name, char *fullname, size_t size)
136 {
137 const char *prefix = NULL;
138 char *ptr, last = '\0';
139 size_t i, j;
140
141 if (!base_name || !fullname || size<1)
142 return NULL;
143
144 memset(fullname, 0, size);
145
146 /* First use the environment variable, then try the constant */
147 prefix = getenv("HDF4_TESTPREFIX");
148
149 #ifdef HDF4_TESTPREFIX
150 if (!prefix)
151 prefix = HDF4_TESTPREFIX;
152 #endif
153
154 /* Prepend the prefix value to the base name */
155 if (prefix && *prefix) {
156 if (snprintf(fullname, size, "%s/%s", prefix, base_name)==(int)size)
157 /* Buffer is too small */
158 return NULL;
159 } /* end if */
160 else {
161 if (strlen(base_name) >= size)
162 /* Buffer is too small */
163 return NULL;
164 else
165 strcpy(fullname, base_name);
166 } /* end else*/
167
168 /* Remove any double slashes in the filename */
169 for (ptr = fullname, i = j = 0; ptr && i < size; i++, ptr++) {
170 if (*ptr != '/' || last != '/')
171 fullname[j++] = *ptr;
172 last = *ptr;
173 } /* end for */
174 return fullname;
175
176 } /* end fixname() */
177
178
179 static long
read_test(int32 aid)180 read_test(int32 aid)
181 {
182 struct timeval start_time, end_time; /* timing counts */
183 long acc_time;
184 int32 ret;
185 intn i; /* local counting index */
186 intn timing; /* Which timing test we are on */
187 intn err_count; /* number of incorrect array positions */
188
189 acc_time=0;
190 for(timing=0; timing<NUM_TIMINGS; timing++) {
191
192 /* Seek to beginning of element */
193 ret=Hseek(aid,0,DF_START);
194 CHECK(ret, FAIL, "Hseek");
195
196 switch(timing) {
197 case 0: /* Read entire buffer in one I/O operation */
198 gettimeofday(&start_time,(struct timezone*)0);
199 ret=Hread(aid,elemsize,in_buf);
200 VERIFY(ret, elemsize, "Hread");
201 gettimeofday(&end_time,(struct timezone*)0);
202 break;
203
204 case 1: /* Read entire buffer one byte at a time forwards */
205 gettimeofday(&start_time,(struct timezone*)0);
206 for(i=0; i<elemsize; i++) {
207 /* Seek to correct location within element */
208 ret=Hseek(aid,i,DF_START);
209 CHECK(ret, FAIL, "Hseek");
210
211 ret=Hread(aid,1,&in_buf[i]);
212 VERIFY(ret, 1, "Hread");
213 } /* end for */
214 gettimeofday(&end_time,(struct timezone*)0);
215 break;
216
217 case 2: /* Read entire buffer one byte at a time every one byte forwards */
218 gettimeofday(&start_time,(struct timezone*)0);
219 for(i=0; i<elemsize; i+=2) {
220 /* Seek to correct location within element */
221 ret=Hseek(aid,i,DF_START);
222 CHECK(ret, FAIL, "Hseek");
223
224 ret=Hread(aid,1,&in_buf[i]);
225 VERIFY(ret, 1, "Hread");
226 } /* end for */
227 for(i=1; i<elemsize; i+=2) {
228 /* Seek to correct location within element */
229 ret=Hseek(aid,i,DF_START);
230 CHECK(ret, FAIL, "Hseek");
231
232 ret=Hread(aid,1,&in_buf[i]);
233 VERIFY(ret, 1, "Hread");
234 } /* end for */
235 gettimeofday(&end_time,(struct timezone*)0);
236 break;
237
238 case 3: /* Read entire buffer one byte at a time backwards */
239 gettimeofday(&start_time,(struct timezone*)0);
240 for(i=elemsize-1; i>=0; i--) {
241 /* Seek to correct location within element */
242 ret=Hseek(aid,i,DF_START);
243 CHECK(ret, FAIL, "Hseek");
244
245 ret=Hread(aid,1,&in_buf[i]);
246 VERIFY(ret, 1, "Hread");
247 } /* end for */
248 gettimeofday(&end_time,(struct timezone*)0);
249 break;
250
251 case 4: /* Read entire buffer one byte at a time every one byte backwards */
252 gettimeofday(&start_time,(struct timezone*)0);
253 for(i=elemsize-1; i>=0; i-=2) {
254 /* Seek to correct location within element */
255 ret=Hseek(aid,i,DF_START);
256 CHECK(ret, FAIL, "Hseek");
257
258 ret=Hread(aid,1,&in_buf[i]);
259 VERIFY(ret, 1, "Hread");
260 } /* end for */
261 for(i=elemsize-2; i>=0; i-=2) {
262 /* Seek to correct location within element */
263 ret=Hseek(aid,i,DF_START);
264 CHECK(ret, FAIL, "Hseek");
265
266 ret=Hread(aid,1,&in_buf[i]);
267 VERIFY(ret, 1, "Hread");
268 } /* end for */
269 gettimeofday(&end_time,(struct timezone*)0);
270 break;
271 } /* end switch */
272
273 /* Verify buffer contents */
274 for(err_count=0,i=0; i<elemsize; i++) {
275 if(out_buf[i]!=in_buf[i]) {
276 printf("Position (%d) read in is (%d), should be (%d)\n",i,(int)in_buf[i],(int)out_buf[i]);
277 num_errs++;
278 err_count++;
279 if(err_count>10)
280 break;
281 } /* end if */
282 } /* end for */
283
284 /* Clear input buffer */
285 HDmemset(in_buf,0,elemsize);
286
287 /* Increment the total I/O time */
288 acc_time+=(end_time.tv_sec-start_time.tv_sec)*FACTOR+
289 (end_time.tv_usec-start_time.tv_usec);
290 } /* end for */
291
292 return(acc_time);
293 } /* end read_test() */
294
295 static long
write_test(int32 aid,intn num_timings)296 write_test(int32 aid,intn num_timings)
297 {
298 struct timeval start_time, end_time; /* timing counts */
299 long acc_time;
300 int32 ret;
301 intn i; /* local counting index */
302 intn timing; /* Which timing test we are on */
303
304 acc_time=0;
305 for(timing=0; timing<num_timings; timing++) {
306
307 /* Refresh output buffer with new values */
308 init_buffer();
309
310 /* Seek to beginning of element */
311 ret=Hseek(aid,0,DF_START);
312 CHECK(ret, FAIL, "Hseek");
313
314 switch(timing) {
315 case 0: /* Write entire buffer in one I/O operation */
316 gettimeofday(&start_time,(struct timezone*)0);
317 ret=Hwrite(aid,elemsize,out_buf);
318 VERIFY(ret, elemsize, "Hwrite");
319 gettimeofday(&end_time,(struct timezone*)0);
320 break;
321
322 case 1: /* Write entire buffer one byte at a time forwards */
323 gettimeofday(&start_time,(struct timezone*)0);
324 for(i=0; i<elemsize; i++) {
325 /* Seek to correct location within element */
326 ret=Hseek(aid,i,DF_START);
327 CHECK(ret, FAIL, "Hseek");
328
329 ret=Hwrite(aid,1,&out_buf[i]);
330 VERIFY(ret, 1, "Hwrite");
331 } /* end for */
332 gettimeofday(&end_time,(struct timezone*)0);
333 break;
334
335 case 2: /* Write entire buffer one byte at a time every one byte forwards */
336 gettimeofday(&start_time,(struct timezone*)0);
337 for(i=0; i<elemsize; i+=2) {
338 /* Seek to correct location within element */
339 ret=Hseek(aid,i,DF_START);
340 CHECK(ret, FAIL, "Hseek");
341
342 ret=Hwrite(aid,1,&out_buf[i]);
343 VERIFY(ret, 1, "Hwrite");
344 } /* end for */
345 for(i=1; i<elemsize; i+=2) {
346 /* Seek to correct location within element */
347 ret=Hseek(aid,i,DF_START);
348 CHECK(ret, FAIL, "Hseek");
349
350 ret=Hwrite(aid,1,&out_buf[i]);
351 VERIFY(ret, 1, "Hwrite");
352 } /* end for */
353 gettimeofday(&end_time,(struct timezone*)0);
354 break;
355
356 case 3: /* Write entire buffer one byte at a time backwards */
357 gettimeofday(&start_time,(struct timezone*)0);
358 for(i=elemsize-1; i>=0; i--) {
359 /* Seek to correct location within element */
360 ret=Hseek(aid,i,DF_START);
361 CHECK(ret, FAIL, "Hseek");
362
363 ret=Hwrite(aid,1,&out_buf[i]);
364 VERIFY(ret, 1, "Hwrite");
365 } /* end for */
366 gettimeofday(&end_time,(struct timezone*)0);
367 break;
368
369 case 4: /* Write entire buffer one byte at a time every one byte backwards */
370 gettimeofday(&start_time,(struct timezone*)0);
371 for(i=elemsize-1; i>=0; i-=2) {
372 /* Seek to correct location within element */
373 ret=Hseek(aid,i,DF_START);
374 CHECK(ret, FAIL, "Hseek");
375
376 ret=Hwrite(aid,1,&out_buf[i]);
377 VERIFY(ret, 1, "Hwrite");
378 } /* end for */
379 for(i=elemsize-2; i>=0; i-=2) {
380 /* Seek to correct location within element */
381 ret=Hseek(aid,i,DF_START);
382 CHECK(ret, FAIL, "Hseek");
383
384 ret=Hwrite(aid,1,&out_buf[i]);
385 VERIFY(ret, 1, "Hwrite");
386 } /* end for */
387 gettimeofday(&end_time,(struct timezone*)0);
388 break;
389 } /* end switch */
390
391 /* Seek to beginning of element */
392 ret=Hseek(aid,0,DF_START);
393 CHECK(ret, FAIL, "Hseek");
394
395 /* Read buffer contents */
396 ret=Hread(aid,elemsize,in_buf);
397 VERIFY(ret, elemsize, "Hread");
398
399 /* Verify buffer contents */
400 for(i=0; i<elemsize; i++) {
401 if(out_buf[i]!=in_buf[i]) {
402 printf("Position (%d) read in is (%d), should be (%d)\n",i,(int)in_buf[i],(int)out_buf[i]);
403 num_errs++;
404 break;
405 } /* end if */
406 } /* end for */
407
408
409 /* Clear input buffer */
410 HDmemset(in_buf,0,elemsize);
411
412 /* Increment the total I/O time */
413 acc_time+=(end_time.tv_sec-start_time.tv_sec)*FACTOR+
414 (end_time.tv_usec-start_time.tv_usec);
415 } /* end for */
416
417 return(acc_time);
418 } /* end read_test() */
419
420 int
main(int argc,char * argv[])421 main(int argc, char *argv[])
422 {
423 model_info m_info;
424 comp_info c_info;
425 uint16 ref_num; /* reference number of the data written out */
426 int32 fid; /* file ID of HDF file for testing */
427 int32 aid; /* AID of element to test */
428 intn test_num;
429 int32 ret;
430 char hfilename[32];
431 char extfilename[32];
432 int CleanUp = 1;
433 int Cache = 1;
434 uint32 lmajor, lminor, lrelease;
435 char lstring[81];
436
437 /* Un-buffer the stdout and stderr */
438 setbuf(stderr, NULL);
439 setbuf(stdout, NULL);
440
441 if (argc>2) {
442 usage();
443 exit(1);
444 }
445 else
446 elemsize = (argc==2)? (int32)atol(argv[1]):(int32)ELEMSIZE;
447
448 if (elemsize<=0) {
449 usage();
450 exit(1);
451 }
452
453 out_buf = HDmalloc(elemsize * sizeof(uint8));
454 in_buf = HDmalloc(elemsize * sizeof(uint8));
455
456 Verbosity = 4; /* Default Verbosity is Low */
457
458 Hgetlibversion(&lmajor, &lminor, &lrelease, lstring);
459
460 printf("Built with HDF Library Version: %u.%u.%u, %s\n\n", (unsigned) lmajor,
461 (unsigned) lminor, (unsigned) lrelease, lstring);
462
463 MESSAGE(6, printf("Starting buffered element test (elemsize=%d)\n", elemsize);
464 )
465
466 if(Cache) /* turn on caching, unless we were instucted not to */
467 Hcache(CACHE_ALL_FILES,TRUE);
468
469 /* fill the buffer with interesting data to compress */
470 init_buffer();
471
472 fixname(TESTFILE_NAME, hfilename, sizeof hfilename);
473
474 /* open the HDF file */
475 fid = Hopen(hfilename, DFACC_ALL, 0);
476 CHECK(fid, FAIL, "Hopen");
477
478 /* Cycle through the different testing element types */
479 /* Performing timings on each type of buffer and record results for output */
480 /* if verbosity level is high enough */
481 for (test_num = 0; test_num < NUM_TESTS; test_num++)
482 {
483 /* Get a new reference number */
484 ref_num=Htagnewref(fid,BUFF_TAG);
485 CHECK(ref_num, 0, "Htagnewref");
486
487 /* Create the data element to perform the tests on */
488 switch(test_num) {
489 case 0: /* create plain data element */
490 aid=Hstartaccess(fid,BUFF_TAG,ref_num,DFACC_RDWR);
491 CHECK(aid, FAIL, "Hstartaccess");
492 break;
493
494 case 1: /* create external data element */
495 fixname(EXTFILE_NAME, extfilename, sizeof extfilename);
496 aid=HXcreate(fid,BUFF_TAG,ref_num,extfilename,0,elemsize);
497 CHECK(aid, FAIL, "HXcreate");
498 break;
499
500 case 2: /* create compressed data element */
501 c_info.deflate.level=9;
502 aid=HCcreate(fid,BUFF_TAG,ref_num,COMP_MODEL_STDIO,&m_info,COMP_CODE_DEFLATE,&c_info);
503 CHECK(aid, FAIL, "HCcreate");
504 break;
505
506 case 3: /* create linked-block data element */
507 aid=HLcreate(fid,BUFF_TAG,ref_num,HDF_APPENDABLE_BLOCK_LEN,HDF_APPENDABLE_BLOCK_NUM);
508 CHECK(aid, FAIL, "HLcreate");
509 break;
510
511 } /* end switch */
512
513 /* Write the initial data to the data element */
514 ret=Hwrite(aid,elemsize,out_buf);
515 VERIFY(ret, elemsize, "Hwrite");
516
517 /* Perform read timing tests on un-buffered data element */
518 read_time[test_num][0]=read_test(aid);
519
520 /* Perform write timing tests on un-buffered data element */
521 /* Just write un-buffered compressed data in one block */
522 write_time[test_num][0]=write_test(aid,(test_num==2 ? 1 : NUM_TIMINGS));
523
524 /* Convert element to a buffered element */
525 ret=HBconvert(aid);
526 CHECK(ret, FAIL, "HBconvert");
527
528 /* Perform read timing tests on buffered data element */
529 read_time[test_num][1]=read_test(aid);
530
531 /* Perform write timing tests on un-buffered data element */
532 write_time[test_num][1]=write_test(aid,NUM_TIMINGS);
533
534 /* Close data element */
535 ret=Hendaccess(aid);
536 CHECK(ret, FAIL, "Hendaccess");
537
538 MESSAGE(3, {
539 printf("Unbuffered read time=%f seconds\n",((float)read_time[test_num][0]/FACTOR));
540 printf("Unbuffered write time=%f seconds\n",((float)write_time[test_num][0]/FACTOR));
541 printf("Buffered read time=%f seconds\n",((float)read_time[test_num][1]/FACTOR));
542 printf("Buffered write time=%f seconds\n",((float)write_time[test_num][1]/FACTOR));
543 }
544 )
545
546 } /* end for */
547
548 /* close the HDF file */
549 ret = Hclose(fid);
550 CHECK(ret, FAIL, "Hclose");
551
552 /* Clean up files created */
553 if (CleanUp) {
554 remove(extfilename);
555 remove(hfilename);
556 }
557
558 HDfree(out_buf);
559 HDfree(in_buf);
560
561 MESSAGE(6, printf("Finished buffered element test\n");
562 )
563 exit(num_errs);
564 return num_errs;
565 } /* end main() */
566