1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2 * Copyright by The HDF Group. *
3 * Copyright by the Board of Trustees of the University of Illinois. *
4 * All rights reserved. *
5 * *
6 * This file is part of HDF5. The full HDF5 copyright notice, including *
7 * terms governing use, modification, and redistribution, is contained in *
8 * the COPYING file, which can be found at the root of the source code *
9 * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
10 * If you do not have access to either file, you may request a copy from *
11 * help@hdfgroup.org. *
12 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
13 #include "hdf5.h"
14 #include "H5private.h"
15
16 /* The HDF5 test files */
17 const char *FILENAME[] = {
18 "h5clear_sec2_v3.h5", /* 0 -- sec2 file with superblock version 3 */
19 "h5clear_log_v3.h5", /* 1 -- log file with superblock veresion 3 */
20 "h5clear_sec2_v0.h5", /* 2 -- sec2 file with superblock version 0 */
21 "h5clear_sec2_v2.h5" /* 3 -- sec2 file with superblock version 2 */
22 };
23
24 const char *FILENAME_ENHANCE[] = {
25 "h5clear_fsm_persist_equal.h5", /* 0: persisting free-space, stored EOA = actual EOF */
26 "h5clear_fsm_persist_greater.h5", /* 1: persisting free-space, stored EOA > actual EOF */
27 "h5clear_fsm_persist_less.h5", /* 2: persisting free-space, stored EOA < actual EOF */
28 "h5clear_fsm_persist_user_equal.h5", /* 3: user block, persisting free-space, stored EOA = actual EOF */
29 "h5clear_fsm_persist_user_greater.h5", /* 4: user block, persisting free-space, stored EOA > actual EOF */
30 "h5clear_fsm_persist_user_less.h5", /* 5: user block, persisting free-space, stored EOA < actual EOF */
31 "h5clear_status_noclose.h5", /* 6 -- v3 superblock, nonzero status_flags, no flush, exit,
32 stored EOA < actual EOF */
33 "h5clear_fsm_persist_noclose.h5" /* 7 -- persisting free-space, no flush, exit, stored EOA < actual EOF */
34 };
35
36 #define KB 1024U
37
38 #define CACHE_IMAGE_FILE "h5clear_mdc_image.h5"
39 #define DSET "DSET"
40 #define DATASET "dset"
41 #define NUM_ELMTS 100
42 #define USERBLOCK 512
43
44 /*-------------------------------------------------------------------------
45 * Function: gen_cache_image_file
46 *
47 * Purpose: To create a file with cache image feature enabled.
48 *
49 * Return: Success: 0
50 * Failure: 1
51 *
52 * Programmer: Vailin Choi; March 2017
53 *
54 *-------------------------------------------------------------------------
55 */
56 static int
gen_cache_image_file(const char * fname)57 gen_cache_image_file(const char *fname)
58 {
59 hid_t fid = -1; /* File ID */
60 hid_t did = -1, sid = -1; /* Dataset ID, dataspace ID */
61 hid_t fapl = -1; /* File access property list */
62 hid_t dcpl = -1; /* Dataset creation property list */
63 hsize_t dims[2]; /* Dimension sizes */
64 hsize_t chunks[2]; /* Chunked dimension sizes */
65 int buf[50][100]; /* Buffer for data to write */
66 int i, j; /* Local index variables */
67 H5AC_cache_image_config_t cache_image_config = /* Cache image input configuration */
68 { H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION,
69 TRUE, FALSE,
70 H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE};
71
72 /* Create a copy of file access property list */
73 if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
74 goto error;
75
76 /* Enable latest format in fapl */
77 if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
78 goto error;
79
80 /* Enable metadata cache image in fapl */
81 if(H5Pset_mdc_image_config(fapl, &cache_image_config) < 0)
82 goto error;
83
84 /* Create the file */
85 if((fid = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
86 goto error;
87
88 /* Create dataspace */
89 dims[0] = 50;
90 dims[1] = 100;
91 if((sid = H5Screate_simple(2, dims, NULL)) < 0)
92 goto error;
93
94 /* Initialize buffer for writing to dataset */
95 for(i = 0; i < 50; i++)
96 for(j = 0; j < 100; j++)
97 buf[i][j] = i * j;
98
99 /* Set up to create a chunked dataset */
100 if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
101 goto error;
102 chunks[0] = 5;
103 chunks[1] = 10;
104 if(H5Pset_chunk(dcpl, 2, chunks) < 0)
105 goto error;
106 if((did = H5Dcreate2(fid, DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
107 goto error;
108
109 /* Write to the dataset */
110 if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0)
111 goto error;
112
113 /* Closing */
114 if(H5Dclose(did) < 0)
115 goto error;
116 if(H5Pclose(dcpl) < 0)
117 goto error;
118 if(H5Pclose(fapl) < 0)
119 goto error;
120 if(H5Sclose(sid) < 0)
121 goto error;
122 if(H5Fclose(fid) < 0)
123 goto error;
124 return 0;
125
126 error:
127 H5E_BEGIN_TRY {
128 H5Pclose(dcpl);
129 H5Sclose(sid);
130 H5Dclose(did);
131 H5Fclose(fid);
132 H5Pclose(fapl);
133 H5Pclose(dcpl);
134 } H5E_END_TRY;
135 return 1;
136 } /* gen_cache_image_file() */
137
138 /*-------------------------------------------------------------------------
139 * Function: gen_enhance_files
140 *
141 * Purpose: To create the first 6 files in FILENAME_ENHANCE[]:
142 * (0) FILENAME_ENHANCE[0]: "h5clear_fsm_persist_equal.h5"
143 * (1) FILENAME_ENHANCE[1]: "h5clear_fsm_persist_greater.h5"
144 * (2) FILENAME_ENHANCE[2]: "h5clear_fsm_persist_less.h5"
145 * (3) FILENAME_ENHANCE[3]: "h5clear_user_fsm_persist_equal.h5"
146 * (4) FILENAME_ENHANCE[4]: "h5clear_user_fsm_persist_greater.h5"
147 * (5) FILENAME_ENHANCE[5]: "h5clear_user_fsm_persist_less.h5"
148 * After creating the files for #1, #2, #4 #5, write invalid EOA
149 * value to the location where the EOA is stored in the superblock.
150 * Also modify the chksum in the superblock due to this change.
151 *
152 * The first call to this routine (without user block) will generate
153 * the first 3 files.
154 * The second call to this routine (with user block) will generate
155 * the last 3 files.
156 *
157 * Return: Success: 0
158 * Failure: 1
159 *
160 * Programmer: Vailin Choi; March 2017
161 *
162 *-------------------------------------------------------------------------
163 */
164 static int
gen_enhance_files(hbool_t user)165 gen_enhance_files(hbool_t user)
166 {
167 hid_t fid = -1; /* File ID */
168 hid_t fcpl = -1; /* File creation property list */
169 hid_t sid = -1; /* Dataspace ID */
170 hid_t did = -1; /* Dataset ID */
171 hsize_t dim[1]; /* Dimension sizes */
172 int data[NUM_ELMTS]; /* Buffer for data */
173 int fd = -1; /* The file descriptor ID */
174 int64_t eoa; /* The EOA value */
175 uint32_t chksum; /* The chksum value */
176 int i = 0 , j = 0, u = 0; /* Local index variable */
177
178 /* Get a copy of the default file creation property */
179 if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
180 goto error;
181
182 /* Check to see if user block will be added */
183 if(user) {
184 if(H5Pset_userblock(fcpl, (hsize_t)USERBLOCK) < 0)
185 goto error;
186 u = 3;
187 }
188
189 /* Set file space strategy and persisting free-space */
190 if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0)
191 goto error;
192
193 /*
194 * Create the file, then write invalid EOA to the file.
195 */
196 for(i = 0+u; i < 3+u; i++) {
197
198 /* Create the file with the file space strategy and persisting free-space */
199 if((fid = H5Fcreate(FILENAME_ENHANCE[i], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0)
200 goto error;
201
202 /* Create the dataset */
203 dim[0] = NUM_ELMTS;
204 if((sid = H5Screate_simple(1, dim, NULL)) < 0)
205 goto error;
206 if((did = H5Dcreate2(fid, DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
207 goto error;
208
209 for(j = 0; j < NUM_ELMTS; j++)
210 data[j] = j;
211
212 /* Write the dataset */
213 if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0)
214 goto error;
215
216 /* Closing */
217 if(H5Dclose(did) < 0)
218 goto error;
219 if(H5Sclose(sid) < 0)
220 goto error;
221 if(H5Fclose(fid) < 0)
222 goto error;
223
224 /*
225 * No further action for:
226 * --FILENAME_ENHANCE[0]: "h5clear_fsm_persist_equal.h5"
227 * --FILENAME_ENHANCE[3]: "h5clear_fsm_persist_user_equal.h5",
228 */
229 if(!(i % 3))
230 continue;
231 /*
232 * For the following files:
233 * --FILENAME_ENHANCE[1]: "h5clear_fsm_persist_greater.h5"
234 * --FILENAME_ENHANCE[2]: "h5clear_fsm_persist_less.h5"
235 * --FILENAME_ENHANCE[4]: "h5clear_fsm_persist_greater.h5"
236 * --FILENAME_ENHANCE[5]: "h5clear_fsm_persist_less.h5"
237 *
238 * Write invalid value to the location for stored eoa and
239 * update the chksum value.
240 */
241 /* Open the file */
242 if((fd = open(FILENAME_ENHANCE[i], O_RDWR, 0663)) < 0)
243 goto error;
244
245 switch(i) {
246 case 1: /* stored EOA is > EOF */
247 eoa = 3048;
248 chksum = 268376587;
249 break;
250
251 case 2: /* stored EOA is < EOF */
252 eoa = 512;
253 chksum = 372920305;
254 break;
255
256 case 4: /* with userblock, stored EOA > EOF */
257 eoa = 4000;
258 chksum = 4168810027;
259 break;
260
261 case 5: /* with userblock, stored EOA < EOF */
262 eoa = 3000;
263 chksum = 3716054346;
264 break;
265
266 default:
267 break;
268 }
269
270 /* location of "end of file address" */
271 if(lseek(fd, (off_t)(28+(user?USERBLOCK:0)), SEEK_SET) < 0)
272 goto error;
273
274 /* Write the bad eoa value to the file */
275 if(write(fd, &eoa, sizeof(eoa)) < 0)
276 goto error;
277
278 /* location of "superblock checksum" */
279 if(lseek(fd, (off_t)(44+(user?USERBLOCK:0)), SEEK_SET) < 0)
280 goto error;
281
282 /* Write the chksum value to the file */
283 if(write(fd, &chksum, sizeof(chksum)) < 0)
284 goto error;
285
286 /* Close the file */
287 if(close(fd) < 0)
288 goto error;
289
290 } /* end for */
291
292 /* Close the property list */
293 if(H5Pclose(fcpl) < 0)
294 goto error;
295
296 return 0;
297
298 error:
299 H5E_BEGIN_TRY {
300 H5Sclose(sid);
301 H5Dclose(did);
302 H5Fclose(fid);
303 H5Pclose(fcpl);
304 } H5E_END_TRY;
305 return 1;
306 } /* gen_enhance_files() */
307
308 /*-------------------------------------------------------------------------
309 * Function: main
310 *
311 * Purpose: Generate test files used by h5clear.
312 *
313 * (A) gen_cache_image_file():
314 * --generate a file with cache image feature
315 * --"h5clear_mdc_image.h5"
316 * (B) gen_enhance_files():
317 * --generate the first 6 files in FILENAME_ENHANCE[]:
318 * (0) "h5clear_fsm_persist_equal.h5"
319 * (1) "h5clear_fsm_persist_greater.h5"
320 * (2) "h5clear_fsm_persist_less.h5"
321 * (3) "h5clear_fsm_persist_user_equal.h5"
322 * (4) "h5clear_fsm_persist_user_greater.h5"
323 * (5) "h5clear_fsm_persist_user_less.h5"
324 *
325 * (C) Generate the following FILENAME[] files in main():
326 * (0a) "h5clear_sec2_v3.h5"
327 * (0b) "latest_h5clear_sec2_v3.h5"
328 * (1a) "h5clear_log_v3.h5",
329 * (1b) "latest_h5clear_log_v3.h5"
330 * (2) "h5clear_sec2_v0.h5"
331 * (3) "h5clear_sec2_v2.h5"
332 *
333 * These HDF5 files are created with non-zero status_flags in
334 * the superblock via flushing and exiting without closing the
335 * library.
336 * Due to file locking, status_flags in the superblock will be
337 * nonzero after H5Fcreate. The library will clear status_flags
338 * on file closing.
339 * This program, after "H5Fcreate" the files, exits without
340 * going through library closing. Thus, status_flags for these
341 * files are not cleared.
342 * The library will check consistency of status_flags when
343 * opening a file with superblock >= v3 and will return error
344 * accordingly.
345 * The library will not check status_flags when opening a file
346 * with < v3 superblock.
347 * These files are used by "h5clear" to see if the tool clears
348 * status_flags properly so users can open the files afterwards.
349 *
350 * (D) Generate the last two files in FILENAME_ENHANCE[] in main():
351 * (6) "h5clear_status_noclose.h5",
352 * (7) "h5clear_fsm_persist_noclose.h5"
353 *
354 * Return: Success: 0
355 * Failure: 1
356 *
357 * Programmer: Vailin Choi; July 2013
358 *
359 *-------------------------------------------------------------------------
360 */
361 int
main(void)362 main(void)
363 {
364 hid_t fid = -1; /* File ID */
365 hid_t fcpl = -1; /* File creation property list */
366 hid_t fapl = -1, new_fapl = -1; /* File access property lists */
367 char fname[512]; /* File name */
368 unsigned new_format; /* To use latest library format or not */
369 hid_t sid = -1; /* Dataspace ID */
370 hid_t did = -1; /* Dataset ID */
371 hsize_t dim[1]; /* Dimension sizes */
372 int data[NUM_ELMTS]; /* Buffer for data */
373 int i; /* Local index variables */
374
375 /* Generate a file with cache image feature enabled */
376 if(gen_cache_image_file(CACHE_IMAGE_FILE) < 0)
377 goto error;
378
379 /* Generate the first 6 files in FILENAME_ENHANCE[] */
380 if(gen_enhance_files(FALSE) < 0)
381 goto error;
382 if(gen_enhance_files(TRUE) < 0)
383 goto error;
384
385 /*
386 * Generate files in FILENAME[]
387 */
388 /* Create a copy of the file access property list */
389 if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
390 goto error;
391
392 /* Copy the file access property list */
393 if((new_fapl = H5Pcopy(fapl)) < 0)
394 goto error;
395 /* Set to latest library format */
396 if(H5Pset_libver_bounds(new_fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
397 goto error;
398
399 /*
400 * Files created within this for loop will have v3 superblock and nonzero status_flags
401 * --FILENAME[0]: "h5clear_sec2_v3.h5", "latest_h5clear_sec2_v3.h5"
402 * --FILENAME[1]: "h5clear_log_v3.h5", "latest_h5clear_log_v3.h5"
403 */
404 for(new_format = FALSE; new_format <= TRUE; new_format++) {
405 hid_t fapl2, my_fapl; /* File access property lists */
406
407 /* Set to use the appropriate file access property list */
408 if(new_format)
409 fapl2 = new_fapl;
410 else
411 fapl2 = fapl;
412 /*
413 * Create a sec2 file
414 */
415 if((my_fapl = H5Pcopy(fapl2)) < 0)
416 goto error;
417 /* Create the file */
418 HDsprintf(fname, "%s%s", new_format? "latest_":"", FILENAME[0]);
419 if((fid = H5Fcreate(fname, H5F_ACC_TRUNC | (new_format ? 0 : H5F_ACC_SWMR_WRITE), H5P_DEFAULT, my_fapl)) < 0)
420 goto error;
421
422 /* Flush the file */
423 if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0)
424 goto error;
425
426 /* Close the property list */
427 if(H5Pclose(my_fapl) < 0)
428 goto error;
429
430 /*
431 * Create a log file
432 */
433 /* Create a copy of file access property list */
434 if((my_fapl = H5Pcopy(fapl2)) < 0)
435 goto error;
436
437 /* Setup the fapl for the log driver */
438 if(H5Pset_fapl_log(my_fapl, "append.log", (unsigned long long)H5FD_LOG_ALL, (size_t)(4 * KB)) < 0)
439 goto error;
440
441 /* Create the file */
442 HDsprintf(fname, "%s%s", new_format? "latest_":"", FILENAME[1]);
443 if((fid = H5Fcreate(fname, H5F_ACC_TRUNC | (new_format ? 0 : H5F_ACC_SWMR_WRITE), H5P_DEFAULT, my_fapl)) < 0)
444 goto error;
445
446 /* Flush the file */
447 if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0)
448 goto error;
449
450 /* Close the property list */
451 if(H5Pclose(my_fapl) < 0)
452 goto error;
453
454 } /* end for */
455
456 /*
457 * Create a sec2 file with v0 superblock but nonzero status_flags:
458 * FILENAME[2]: "h5clear_sec2_v0.h5"
459 */
460 if((fid = H5Fcreate(FILENAME[2], H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
461 goto error;
462
463 /* Flush the file */
464 if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0)
465 goto error;
466
467
468 /*
469 * Create a sec2 file with v2 superblock but nonzero status_flags:
470 * FILENAME[3]: "h5clear_sec2_v2.h5"
471 */
472 if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
473 goto error;
474 if(H5Pset_shared_mesg_nindexes(fcpl, 1) < 0)
475 goto error;
476 if(H5Pset_shared_mesg_index(fcpl, 0, H5O_SHMESG_DTYPE_FLAG, 50) < 0)
477 goto error;
478
479 if((fid = H5Fcreate(FILENAME[3], H5F_ACC_TRUNC, fcpl, fapl)) < 0)
480 goto error;
481
482 /* Flush the file */
483 if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0)
484 goto error;
485
486
487 /* Close the property lists */
488 if(H5Pclose(fapl) < 0)
489 goto error;
490 if(H5Pclose(new_fapl) < 0)
491 goto error;
492 if(H5Pclose(fcpl) < 0)
493 goto error;
494
495 /*
496 * Create the last two files in FILENAME_ENHANCE[]:
497 * --FILENAME_ENHANCE[6]: h5clear_status_noclose.h5
498 * --FILENAME_ENHANCE[7]: h5clear_fsm_persist_noclose.h5
499 */
500 /*
501 * FILENAME_ENHANCE[6]: h5clear_status_noclose.h5
502 * --stored EOA < actual EOF
503 * --version 3 superblock
504 * --nonzero status_flags
505 * --does not persist free-space
506 * --does not flush the file, just exit without closing file:
507 * --this file is similar to the user-suppplied test file attached with HDFFV-10347
508 */
509 if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
510 goto error;
511
512 /* Set to latest format */
513 if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
514 goto error;
515
516 /* Create file with SWMR-write access */
517 if((fid = H5Fcreate(FILENAME_ENHANCE[6], H5F_ACC_TRUNC|H5F_ACC_SWMR_WRITE, H5P_DEFAULT, fapl)) < 0)
518 goto error;
519
520 /* Create the dataset */
521 dim[0] = NUM_ELMTS;
522 if((sid = H5Screate_simple(1, dim, NULL)) < 0)
523 goto error;
524 if((did = H5Dcreate2(fid, DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
525 goto error;
526
527 for(i = 0; i < NUM_ELMTS; i++)
528 data[i] = i;
529
530 /* Write the dataset */
531 if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0)
532 goto error;
533
534 /* Closing */
535 if(H5Dclose(did) < 0)
536 goto error;
537 if(H5Sclose(sid) < 0)
538 goto error;
539 if(H5Pclose(fapl) < 0)
540 goto error;
541
542 /* Does not flush and does not close the file */
543
544
545 /*
546 * FILENAME_ENHANCE[7]: h5clear_fsm_persist_noclose.h5
547 * --stored EOA < actual EOF
548 * --persisting free-space
549 * --undefined fsinfo.eoa_pre_fsm_fsalloc
550 * --undefined fsinfo.fs_addr
551 * --does not flush the file, just exit without closing
552 */
553 if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
554 goto error;
555
556 /* Set file space strategy and persisting free-space */
557 if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0)
558 goto error;
559
560 /* Create the file with the set file space info */
561 if((fid = H5Fcreate(FILENAME_ENHANCE[7], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0)
562 goto error;
563
564 /* Create the dataset */
565 dim[0] = NUM_ELMTS;
566 if((sid = H5Screate_simple(1, dim, NULL)) < 0)
567 goto error;
568 if((did = H5Dcreate2(fid, DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
569 goto error;
570
571 for(i = 0; i < NUM_ELMTS; i++)
572 data[i] = i;
573
574 /* Write the dataset */
575 if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0)
576 goto error;
577
578 /* Closing */
579 if(H5Dclose(did) < 0)
580 goto error;
581 if(H5Sclose(sid) < 0)
582 goto error;
583 if(H5Pclose(fcpl) < 0)
584 goto error;
585
586 /* Does not flush and does not close the file */
587
588
589 fflush(stdout);
590 fflush(stderr);
591
592 /* Not going through library closing by calling _exit(0) with success */
593 HD_exit(0);
594
595 error:
596
597 /* Exit with failure */
598 HD_exit(1);
599 }
600