1 /**
2 * @file
3 * @internal Add provenance info for netcdf-4 files.
4 *
5 * Copyright 2018, UCAR/Unidata See netcdf/COPYRIGHT file for copying
6 * and redistribution conditions.
7 * @author Dennis Heimbigner
8 */
9
10 #include "config.h"
11 #include "nc4internal.h"
12 #include "hdf5internal.h"
13 #include "nc_provenance.h"
14 #include "nclist.h"
15 #include "ncbytes.h"
16
17 /* Provide a hack to suppress the writing of _NCProperties attribute.
18 This is for creating a file without _NCProperties for testing purposes.
19 */
20 #undef SUPPRESSNCPROPS
21
22 /* Various Constants */
23 #define NCPROPS_MAX_NAME 1024 /* max key name size */
24 #define NCPROPS_MAX_VALUE 1024 /* max value size */
25 #define HDF5_MAX_NAME 1024 /**< HDF5 max name. */
26
27 #define ESCAPECHARS "\\=|,"
28
29 /** @internal Check NetCDF return code. */
30 #define NCHECK(expr) {if((expr)!=NC_NOERR) {goto done;}}
31
32 /** @internal Check HDF5 return code. */
33 #define HCHECK(expr) {if((expr)<0) {ncstat = NC_EHDFERR; goto done;}}
34
35 static int NC4_read_ncproperties(NC_FILE_INFO_T* h5, char** propstring);
36 static int NC4_write_ncproperties(NC_FILE_INFO_T* h5);
37
38 static int globalpropinitialized = 0;
39 static NC4_Provenance globalprovenance;
40
41 /**
42 * @internal Initialize default provenance info
43 * This will only be used for newly created files
44 * or for opened files that do not contain an _NCProperties
45 * attribute.
46 *
47 * @return ::NC_NOERR No error.
48 * @author Dennis Heimbigner
49 */
50 int
NC4_provenance_init(void)51 NC4_provenance_init(void)
52 {
53 int stat = NC_NOERR;
54 char* name = NULL;
55 char* value = NULL;
56 unsigned major,minor,release;
57 NCbytes* buffer = NULL; /* for constructing the global _NCProperties */
58 char printbuf[1024];
59 const char* p = NULL;
60
61 if(globalpropinitialized)
62 return stat;
63
64 /* Build _NCProperties info */
65
66 /* Initialize globalpropinfo */
67 memset((void*)&globalprovenance,0,sizeof(NC4_Provenance));
68 globalprovenance.version = NCPROPS_VERSION;
69
70 buffer = ncbytesnew();
71
72 /* Insert version as first entry */
73 ncbytescat(buffer,NCPVERSION);
74 ncbytescat(buffer,"=");
75
76 snprintf(printbuf,sizeof(printbuf),"%d",globalprovenance.version);
77 ncbytescat(buffer,printbuf);
78
79 /* Insert the netcdf version */
80 ncbytesappend(buffer,NCPROPSSEP2);
81 ncbytescat(buffer,NCPNCLIB2);
82 ncbytescat(buffer,"=");
83 ncbytescat(buffer,PACKAGE_VERSION);
84
85 /* Insert the HDF5 as underlying storage format library */
86 ncbytesappend(buffer,NCPROPSSEP2);
87 ncbytescat(buffer,NCPHDF5LIB2);
88 ncbytescat(buffer,"=");
89 if((stat = NC4_hdf5get_libversion(&major,&minor,&release))) goto done;
90 snprintf(printbuf,sizeof(printbuf),"%1u.%1u.%1u",major,minor,release);
91 ncbytescat(buffer,printbuf);
92
93 #ifdef NCPROPERTIES_EXTRA
94 /* Add any extra fields */
95 p = NCPROPERTIES_EXTRA;
96 if(p[0] == NCPROPSSEP2) p++; /* If leading separator */
97 ncbytesappend(buffer,NCPROPSSEP2);
98 ncbytescat(buffer,p);
99 #endif
100 ncbytesnull(buffer);
101 globalprovenance.ncproperties = ncbytesextract(buffer);
102
103 done:
104 ncbytesfree(buffer);
105 if(name != NULL) free(name);
106 if(value != NULL) free(value);
107 if(stat == NC_NOERR)
108 globalpropinitialized = 1; /* avoid repeating it */
109 return stat;
110 }
111
112 /**
113 * @internal finalize default provenance info
114 *
115 * @return ::NC_NOERR No error.
116 * @author Dennis Heimbigner
117 */
118 int
NC4_provenance_finalize(void)119 NC4_provenance_finalize(void)
120 {
121 return NC4_clear_provenance(&globalprovenance);
122 }
123
124 /**
125 * @internal
126 *
127 * Construct the provenance information for a newly created file.
128 * Note that creation of the _NCProperties attribute is deferred
129 * to the sync_netcdf4_file function.
130 *
131 * @param file Pointer to file object.
132 *
133 * @return ::NC_NOERR No error.
134 * [Note: other errors are reported via LOG()]
135 * @author Dennis Heimbigner
136 */
137 int
NC4_new_provenance(NC_FILE_INFO_T * file)138 NC4_new_provenance(NC_FILE_INFO_T* file)
139 {
140 int ncstat = NC_NOERR;
141 NC4_Provenance* provenance = NULL;
142 int superblock = -1;
143
144 LOG((5, "%s: ncid 0x%x", __func__, file->root_grp->hdr.id));
145
146 assert(file->provenance.ncproperties == NULL); /* not yet defined */
147
148 provenance = &file->provenance;
149 memset(provenance,0,sizeof(NC4_Provenance)); /* make sure */
150
151 /* Set the version */
152 provenance->version = globalprovenance.version;
153
154 /* Set the superblock number */
155 if((ncstat = NC4_hdf5get_superblock(file,&superblock))) goto done;
156 provenance->superblockversion = superblock;
157
158 if(globalprovenance.ncproperties != NULL) {
159 if((provenance->ncproperties = strdup(globalprovenance.ncproperties)) == NULL)
160 {ncstat = NC_ENOMEM; goto done;}
161 }
162
163 done:
164 if(ncstat) {
165 LOG((0,"Could not create _NCProperties attribute"));
166 }
167 return NC_NOERR;
168 }
169
170 /**
171 * @internal
172 *
173 * Construct the provenance information for an existing file.
174 *
175 * @param file Pointer to file object.
176 *
177 * @return ::NC_NOERR No error.
178 * [Note: other errors are reported via LOG()]
179 * @author Dennis Heimbigner
180 */
181 int
NC4_read_provenance(NC_FILE_INFO_T * file)182 NC4_read_provenance(NC_FILE_INFO_T* file)
183 {
184 int ncstat = NC_NOERR;
185 NC4_Provenance* provenance = NULL;
186 int superblock = -1;
187 char* propstring = NULL;
188
189 LOG((5, "%s: ncid 0x%x", __func__, file->root_grp->hdr.id));
190
191 assert(file->provenance.version == 0); /* not yet defined */
192
193 provenance = &file->provenance;
194 memset(provenance,0,sizeof(NC4_Provenance)); /* make sure */
195
196 /* Set the superblock number */
197 if((ncstat = NC4_hdf5get_superblock(file,&superblock))) goto done;
198 provenance->superblockversion = superblock;
199
200 /* Read the _NCProperties value from the file */
201 /* We do not return a size and assume the size is that upto the
202 first nul character */
203 if((ncstat = NC4_read_ncproperties(file,&propstring))) goto done;
204 provenance->ncproperties = propstring;
205 propstring = NULL;
206
207 done:
208 nullfree(propstring);
209 if(ncstat) {
210 LOG((0,"Could not create _NCProperties attribute"));
211 }
212 return NC_NOERR;
213 }
214
215 /**
216 * @internal
217 *
218 * Add the provenance information to a newly created file.
219 *
220 * @param file Pointer to file object.
221 *
222 * @return ::NC_NOERR No error.
223 * [Note: other errors are reported via LOG()]
224 * @author Dennis Heimbigner
225 */
226 int
NC4_write_provenance(NC_FILE_INFO_T * file)227 NC4_write_provenance(NC_FILE_INFO_T* file)
228 {
229 int ncstat = NC_NOERR;
230 if((ncstat = NC4_write_ncproperties(file)))
231 goto done;
232 done:
233 return ncstat;
234 }
235
236 /* HDF5 Specific attribute read/write of _NCProperties */
237 static int
NC4_read_ncproperties(NC_FILE_INFO_T * h5,char ** propstring)238 NC4_read_ncproperties(NC_FILE_INFO_T* h5, char** propstring)
239 {
240 int retval = NC_NOERR;
241 hid_t hdf5grpid = -1;
242 hid_t attid = -1;
243 hid_t aspace = -1;
244 hid_t atype = -1;
245 hid_t ntype = -1;
246 char* text = NULL;
247 H5T_class_t t_class;
248 hsize_t size;
249
250 LOG((5, "%s", __func__));
251
252 hdf5grpid = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid;
253
254 if(H5Aexists(hdf5grpid,NCPROPS) <= 0) { /* Does not exist */
255 /* File did not contain a _NCProperties attribute; leave empty */
256 goto done;
257 }
258
259 /* NCPROPS Attribute exists, make sure it is legitimate */
260 attid = H5Aopen_name(hdf5grpid, NCPROPS);
261 assert(attid > 0);
262 aspace = H5Aget_space(attid);
263 atype = H5Aget_type(attid);
264 /* Verify atype and size */
265 t_class = H5Tget_class(atype);
266 if(t_class != H5T_STRING)
267 {retval = NC_EINVAL; goto done;}
268 size = H5Tget_size(atype);
269 if(size == 0)
270 {retval = NC_EINVAL; goto done;}
271 text = (char*)malloc(1+(size_t)size);
272 if(text == NULL)
273 {retval = NC_ENOMEM; goto done;}
274 if((ntype = H5Tget_native_type(atype, H5T_DIR_DEFAULT)) < 0)
275 {retval = NC_EHDFERR; goto done;}
276 if((H5Aread(attid, ntype, text)) < 0)
277 {retval = NC_EHDFERR; goto done;}
278 /* Make sure its null terminated */
279 text[(size_t)size] = '\0';
280 if(propstring) {*propstring = text; text = NULL;}
281
282 done:
283 if(text != NULL) free(text);
284 /* Close out the HDF5 objects */
285 if(attid > 0 && H5Aclose(attid) < 0) retval = NC_EHDFERR;
286 if(aspace > 0 && H5Sclose(aspace) < 0) retval = NC_EHDFERR;
287 if(atype > 0 && H5Tclose(atype) < 0) retval = NC_EHDFERR;
288 if(ntype > 0 && H5Tclose(ntype) < 0) retval = NC_EHDFERR;
289
290 /* For certain errors, actually fail, else log that attribute was invalid and ignore */
291 if(retval != NC_NOERR) {
292 if(retval != NC_ENOMEM && retval != NC_EHDFERR) {
293 LOG((0,"Invalid _NCProperties attribute: ignored"));
294 retval = NC_NOERR;
295 }
296 }
297 return retval;
298 }
299
300 static int
NC4_write_ncproperties(NC_FILE_INFO_T * h5)301 NC4_write_ncproperties(NC_FILE_INFO_T* h5)
302 {
303 #ifdef SUPPRESSNCPROPERTY
304 return NC_NOERR;
305 #else /*!SUPPRESSNCPROPERTY*/
306 int retval = NC_NOERR;
307 hid_t hdf5grpid = -1;
308 hid_t attid = -1;
309 hid_t aspace = -1;
310 hid_t atype = -1;
311 size_t len = 0;
312 NC4_Provenance* prov = &h5->provenance;
313
314 LOG((5, "%s", __func__));
315
316 /* If the file is read-only, return an error. */
317 if (h5->no_write)
318 {retval = NC_EPERM; goto done;}
319
320 hdf5grpid = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid;
321
322 if(H5Aexists(hdf5grpid,NCPROPS) > 0) /* Already exists, no overwrite */
323 goto done;
324
325 /* Build the property if we have legit value */
326 if(prov->ncproperties != NULL) {
327 /* Build the HDF5 string type */
328 if ((atype = H5Tcopy(H5T_C_S1)) < 0)
329 {retval = NC_EHDFERR; goto done;}
330 if (H5Tset_strpad(atype, H5T_STR_NULLTERM) < 0)
331 {retval = NC_EHDFERR; goto done;}
332 if(H5Tset_cset(atype, H5T_CSET_ASCII) < 0)
333 {retval = NC_EHDFERR; goto done;}
334 len = strlen(prov->ncproperties);
335 if(H5Tset_size(atype, len) < 0)
336 {retval = NC_EFILEMETA; goto done;}
337 /* Create NCPROPS attribute */
338 if((aspace = H5Screate(H5S_SCALAR)) < 0)
339 {retval = NC_EFILEMETA; goto done;}
340 if ((attid = H5Acreate(hdf5grpid, NCPROPS, atype, aspace, H5P_DEFAULT)) < 0)
341 {retval = NC_EFILEMETA; goto done;}
342 if (H5Awrite(attid, atype, prov->ncproperties) < 0)
343 {retval = NC_EFILEMETA; goto done;}
344 /* Verify */
345 #if 0
346 {
347 hid_t spacev, typev;
348 hsize_t dsize, tsize;
349 typev = H5Aget_type(attid);
350 spacev = H5Aget_space(attid);
351 dsize = H5Aget_storage_size(attid);
352 tsize = H5Tget_size(typev);
353 fprintf(stderr,"dsize=%lu tsize=%lu\n",(unsigned long)dsize,(unsigned long)tsize);
354 }
355 #endif
356 }
357
358 done:
359 /* Close out the HDF5 objects */
360 if(attid > 0 && H5Aclose(attid) < 0) retval = NC_EHDFERR;
361 if(aspace > 0 && H5Sclose(aspace) < 0) retval = NC_EHDFERR;
362 if(atype > 0 && H5Tclose(atype) < 0) retval = NC_EHDFERR;
363
364 /* For certain errors, actually fail, else log that attribute was invalid and ignore */
365 switch (retval) {
366 case NC_ENOMEM:
367 case NC_EHDFERR:
368 case NC_EPERM:
369 case NC_EFILEMETA:
370 case NC_NOERR:
371 break;
372 default:
373 LOG((0,"Invalid _NCProperties attribute"));
374 retval = NC_NOERR;
375 break;
376 }
377 return retval;
378 #endif /*!SUPPRESSNCPROPERTY*/
379 }
380
381 /**************************************************/
382 /* Utilities */
383
384 /* Debugging */
385
386 void
ncprintprovenance(NC4_Provenance * info)387 ncprintprovenance(NC4_Provenance* info)
388 {
389 fprintf(stderr,"[%p] version=%d superblockversion=%d ncproperties=|%s|\n",
390 info,
391 info->version,
392 info->superblockversion,
393 (info->ncproperties==NULL?"":info->ncproperties));
394 }
395
396 /**
397 * @internal
398 *
399 * Clear the NCPROVENANCE object; do not free it
400 * @param prov Pointer to provenance object
401 *
402 * @return ::NC_NOERR No error.
403 * @author Dennis Heimbigner
404 */
405 int
NC4_clear_provenance(NC4_Provenance * prov)406 NC4_clear_provenance(NC4_Provenance* prov)
407 {
408 LOG((5, "%s", __func__));
409
410 if(prov == NULL) return NC_NOERR;
411 nullfree(prov->ncproperties);
412 memset(prov,0,sizeof(NC4_Provenance));
413 return NC_NOERR;
414 }
415
416 #if 0
417 /* Unused functions */
418
419 /**
420 * @internal Parse file properties.
421 *
422 * @param text0 Text properties.
423 * @param pairs list of parsed (key,value) pairs
424 *
425 * @return ::NC_NOERR No error.
426 * @author Dennis Heimbigner
427 */
428 static int
429 properties_parse(const char* text0, NClist* pairs)
430 {
431 int ret = NC_NOERR;
432 char* p;
433 char* q;
434 char* text = NULL;
435
436 if(text0 == NULL || strlen(text0) == 0)
437 goto done;
438
439 text = strdup(text0);
440 if(text == NULL) return NC_ENOMEM;
441
442 /* For back compatibility with version 1, translate '|' -> ',' */
443 for(p=text;*p;p++) {
444 if(*p == NCPROPSSEP1)
445 *p = NCPROPSSEP2;
446 }
447
448 /* Walk and fill in ncinfo */
449 p = text;
450 while(*p) {
451 char* name = p;
452 char* value = NULL;
453 char* next = NULL;
454
455 /* Delimit whole (key,value) pair */
456 q = locate(p,NCPROPSSEP2);
457 if(*q != '\0') /* Never go beyond the final nul term */
458 *q++ = '\0';
459 next = q;
460 /* split key and value */
461 q = locate(p,'=');
462 name = p;
463 *q++ = '\0';
464 value = q;
465 /* Set up p for next iteration */
466 p = next;
467 nclistpush(pairs,strdup(name));
468 nclistpush(pairs,strdup(value));
469 }
470 done:
471 if(text) free(text);
472 return ret;
473 }
474
475 /* Locate a specific character and return its pointer
476 or EOS if not found
477 take \ escapes into account */
478 static char*
479 locate(char* p, char tag)
480 {
481 char* next;
482 int c;
483 assert(p != NULL);
484 for(next = p;(c = *next);next++) {
485 if(c == tag)
486 return next;
487 else if(c == '\\' && next[1] != '\0')
488 next++; /* skip escaped char */
489 }
490 return next; /* not found */
491 }
492
493 /* Utility to transfer a string to a buffer with escaping */
494 static void
495 escapify(NCbytes* buffer, const char* s)
496 {
497 const char* p;
498 for(p=s;*p;p++) {
499 if(strchr(ESCAPECHARS,*p) != NULL)
500 ncbytesappend(buffer,'\\');
501 ncbytesappend(buffer,*p);
502 }
503 }
504
505 /**
506 * @internal Build _NCProperties attribute value.
507 *
508 * Convert a NCPROPINFO instance to a single string.
509 * Will always convert to current format
510 *
511 * @param version
512 * @param list Properties list
513 * @param spropp Pointer that gets properties string.
514 * @return ::NC_NOERR No error.
515 * @return ::NC_EINVAL failed.
516 * @author Dennis Heimbigner
517 */
518 static int
519 build_propstring(int version, NClist* list, char** spropp)
520 {
521 int stat = NC_NOERR;
522 int i;
523 NCbytes* buffer = NULL;
524 char sversion[64];
525
526 LOG((5, "%s version=%d", __func__, version));
527
528 if(spropp != NULL) *spropp = NULL;
529
530 if(version == 0 || version > NCPROPS_VERSION) /* unknown case */
531 goto done;
532 if(list == NULL)
533 {stat = NC_EINVAL; goto done;}
534
535 if((buffer = ncbytesnew()) == NULL)
536 {stat = NC_ENOMEM; goto done;}
537
538 /* start with version */
539 ncbytescat(buffer,NCPVERSION);
540 ncbytesappend(buffer,'=');
541 /* Use current version */
542 snprintf(sversion,sizeof(sversion),"%d",NCPROPS_VERSION);
543 ncbytescat(buffer,sversion);
544
545 for(i=0;i<nclistlength(list);i+=2) {
546 char* value, *name;
547 name = nclistget(list,i);
548 if(name == NULL) continue;
549 value = nclistget(list,i+1);
550 ncbytesappend(buffer,NCPROPSSEP2); /* terminate last entry */
551 escapify(buffer,name);
552 ncbytesappend(buffer,'=');
553 escapify(buffer,value);
554 }
555 /* Force null termination */
556 ncbytesnull(buffer);
557 if(spropp) *spropp = ncbytesextract(buffer);
558
559 done:
560 if(buffer != NULL) ncbytesfree(buffer);
561 return stat;
562 }
563
564 static int
565 properties_getversion(const char* propstring, int* versionp)
566 {
567 int ncstat = NC_NOERR;
568 int version = 0;
569 /* propstring should begin with "version=dddd" */
570 if(propstring == NULL || strlen(propstring) < strlen("version=") + strlen("1"))
571 {ncstat = NC_EINVAL; goto done;} /* illegal version */
572 if(memcmp(propstring,"version=",strlen("version=")) != 0)
573 {ncstat = NC_EINVAL; goto done;} /* illegal version */
574 propstring += strlen("version=");
575 /* get version */
576 version = atoi(propstring);
577 if(version < 0)
578 {ncstat = NC_EINVAL; goto done;} /* illegal version */
579 if(versionp) *versionp = version;
580 done:
581 return ncstat;
582 }
583
584 /**
585 * @internal
586 *
587 * Construct the parsed provenance information
588 *
589 * @param prov Pointer to provenance object
590 *
591 * @return ::NC_NOERR No error.
592 * @return ::NC_ENOMEM
593 * @return ::NC_EINVAL
594 * @author Dennis Heimbigner
595 */
596 static int
597 parse_provenance(NC4_Provenance* prov)
598 {
599 int ncstat = NC_NOERR;
600 char *name = NULL;
601 char *value = NULL;
602 int version = 0;
603 NClist* list = NULL;
604
605 LOG((5, "%s: prov 0x%x", __func__, prov));
606
607 if(prov->ncproperty == NULL || strlen(prov->ncproperty) < strlen("version="))
608 {ncstat = NC_EINVAL; goto done;}
609 if((list = nclistnew()) == NULL)
610 {ncstat = NC_ENOMEM; goto done;}
611
612 /* Do we understand the version? */
613 if(prov->version > 0 && prov->version <= NCPROPS_VERSION) {/* recognized version */
614 if((ncstat=properties_parse(prov->ncproperty,list)))
615 goto done;
616 /* Remove version pair from properties list*/
617 if(nclistlength(list) < 2)
618 {ncstat = NC_EINVAL; goto done;} /* bad _NCProperties attribute */
619 /* Throw away the purported version=... */
620 nclistremove(list,0); /* version key */
621 nclistremove(list,0); /* version value */
622
623 /* Now, rebuild to the latest version */
624 switch (version) {
625 default: break; /* do nothing */
626 case 1: {
627 int i;
628 for(i=0;i<nclistlength(list);i+=2) {
629 char* newname = NULL;
630 name = nclistget(list,i);
631 if(name == NULL) continue; /* ignore */
632 if(strcmp(name,NCPNCLIB1) == 0)
633 newname = NCPNCLIB2; /* change name */
634 else if(strcmp(name,NCPHDF5LIB1) == 0)
635 newname = NCPHDF5LIB2;
636 else continue; /* ignore */
637 /* Do any rename */
638 nclistset(list,i,strdup(newname));
639 if(name) {free(name); name = NULL;}
640 }
641 } break;
642 } /*switch*/
643 }
644 prov->properties = list;
645 list = NULL;
646
647 done:
648 nclistfreeall(list);
649 if(name != NULL) free(name);
650 if(value != NULL) free(value);
651 return ncstat;
652 }
653
654 /**
655 * @internal
656 *
657 * Clear and Free the NC4_Provenance object
658 * @param prov Pointer to provenance object
659 *
660 * @return ::NC_NOERR No error.
661 * @author Dennis Heimbigner
662 */
663 static int
664 NC4_free_provenance(NC4_Provenance* prov)
665 {
666 LOG((5, "%s", __func__));
667
668 if(prov == NULL) return NC_NOERR;
669 NC4_clear_provenance(prov);
670 free(prov);
671 return NC_NOERR;
672 }
673
674 /* Utility to copy contents of the dfalt into an NCPROPINFO object */
675 static int
676 propinfo_default(NC4_Properties* dst, const NC4_Properties* dfalt)
677 {
678 int i;
679 if(dst->properties == NULL) {
680 dst->properties = nclistnew();
681 if(dst->properties == NULL) return NC_ENOMEM;
682 }
683 dst->version = dfalt->version;
684 for(i=0;i<nclistlength(dfalt->properties);i++) {
685 char* s = nclistget(dfalt->properties,i);
686 s = strdup(s);
687 if(s == NULL) return NC_ENOMEM;
688 nclistpush(dst->properties,s);
689 }
690 return NC_NOERR;
691 }
692
693 #endif /*0*/
694