1 /* Copyright 2003-2018, University Corporation for Atmospheric
2 * Research. See COPYRIGHT file for copying and redistribution
3 * conditions. */
4 /**
5 * @file
6 * @internal This file contains functions that are used in file
7 * opens.
8 *
9 * @author Ed Hartnett
10 */
11
12 #include "config.h"
13 #include "hdf5internal.h"
14 #include "ncrc.h"
15
16 #define NUM_TYPES 12 /**< Number of netCDF atomic types. */
17 #define CD_NELEMS_ZLIB 1 /**< Number of parameters needed for ZLIB filter. */
18
19 /** @internal Native HDF5 constants for atomic types. For performance,
20 * fill this array only the first time, and keep it in global memory
21 * for each further use. */
22 static hid_t h5_native_type_constant_g[NUM_TYPES];
23
24 /** @internal NetCDF atomic type names. */
25 static const char nc_type_name_g[NUM_TYPES][NC_MAX_NAME + 1] = {"char", "byte", "short",
26 "int", "float", "double", "ubyte",
27 "ushort", "uint", "int64",
28 "uint64", "string"};
29
30 /** @internal NetCDF atomic types. */
31 static const nc_type nc_type_constant_g[NUM_TYPES] = {NC_CHAR, NC_BYTE, NC_SHORT,
32 NC_INT, NC_FLOAT, NC_DOUBLE, NC_UBYTE,
33 NC_USHORT, NC_UINT, NC_INT64,
34 NC_UINT64, NC_STRING};
35
36 /** @internal NetCDF atomic type sizes. */
37 static const int nc_type_size_g[NUM_TYPES] = {sizeof(char), sizeof(char), sizeof(short),
38 sizeof(int), sizeof(float), sizeof(double), sizeof(unsigned char),
39 sizeof(unsigned short), sizeof(unsigned int), sizeof(long long),
40 sizeof(unsigned long long), sizeof(char *)};
41
42 /** @internal These flags may not be set for open mode. */
43 static const int ILLEGAL_OPEN_FLAGS = (NC_MMAP);
44
45 /* From hdf5file.c. */
46 extern size_t nc4_chunk_cache_size;
47 extern size_t nc4_chunk_cache_nelems;
48 extern float nc4_chunk_cache_preemption;
49
50 /* From nc4mem.c */
51 extern int NC4_open_image_file(NC_FILE_INFO_T* h5);
52
53 /* Defined later in this file. */
54 static int nc4_rec_read_metadata(NC_GRP_INFO_T *grp);
55
56 /**
57 * @internal Struct to track information about objects in a group, for
58 * nc4_rec_read_metadata()
59 */
60 typedef struct NC4_rec_read_metadata_obj_info
61 {
62 hid_t oid; /* HDF5 object ID */
63 char oname[NC_MAX_NAME + 1]; /* Name of object */
64 H5G_stat_t statbuf; /* Information about the object */
65 struct NC4_rec_read_metadata_obj_info *next; /* Pointer to next node in list */
66 } NC4_rec_read_metadata_obj_info_t;
67
68 /**
69 * @internal User data struct for call to H5Literate() in
70 * nc4_rec_read_metadata(). Tracks the groups, named datatypes and
71 * datasets in the group, for later use.
72 */
73 typedef struct NC4_rec_read_metadata_ud
74 {
75 NClist* grps; /* NClist<NC4_rec_read_metadata_obj_info_t*> */
76 NC_GRP_INFO_T *grp; /* Pointer to parent group */
77 } NC4_rec_read_metadata_ud_t;
78
79 /* Custom iteration callback data */
80 typedef struct {
81 NC_GRP_INFO_T *grp;
82 NC_VAR_INFO_T *var;
83 } att_iter_info;
84
85 /**
86 * @internal Given an HDF5 type, set a pointer to netcdf type_info
87 * struct, either an existing one (for user-defined types) or a newly
88 * created one.
89 *
90 * @param h5 Pointer to HDF5 file info struct.
91 * @param datasetid HDF5 dataset ID.
92 * @param type_info Pointer to pointer that gets type info struct.
93 *
94 * @return ::NC_NOERR No error.
95 * @return ::NC_EBADID Bad ncid.
96 * @return ::NC_EHDFERR HDF5 returned error.
97 * @return ::NC_EBADTYPID Type not found.
98 * @author Ed Hartnett
99 */
100 static int
get_type_info2(NC_FILE_INFO_T * h5,hid_t datasetid,NC_TYPE_INFO_T ** type_info)101 get_type_info2(NC_FILE_INFO_T *h5, hid_t datasetid,
102 NC_TYPE_INFO_T **type_info)
103 {
104 htri_t is_str, equal = 0;
105 H5T_class_t class;
106 hid_t native_typeid, hdf_typeid;
107 H5T_order_t order;
108 int t;
109
110 assert(h5 && type_info);
111
112 /* Because these N5T_NATIVE_* constants are actually function calls
113 * (!) in H5Tpublic.h, I can't initialize this array in the usual
114 * way, because at least some C compilers (like Irix) complain
115 * about calling functions when defining constants. So I have to do
116 * it like this. Note that there's no native types for char or
117 * string. Those are handled later. */
118 if (!h5_native_type_constant_g[1])
119 {
120 h5_native_type_constant_g[1] = H5T_NATIVE_SCHAR;
121 h5_native_type_constant_g[2] = H5T_NATIVE_SHORT;
122 h5_native_type_constant_g[3] = H5T_NATIVE_INT;
123 h5_native_type_constant_g[4] = H5T_NATIVE_FLOAT;
124 h5_native_type_constant_g[5] = H5T_NATIVE_DOUBLE;
125 h5_native_type_constant_g[6] = H5T_NATIVE_UCHAR;
126 h5_native_type_constant_g[7] = H5T_NATIVE_USHORT;
127 h5_native_type_constant_g[8] = H5T_NATIVE_UINT;
128 h5_native_type_constant_g[9] = H5T_NATIVE_LLONG;
129 h5_native_type_constant_g[10] = H5T_NATIVE_ULLONG;
130 }
131
132 /* Get the HDF5 typeid - we'll need it later. */
133 if ((hdf_typeid = H5Dget_type(datasetid)) < 0)
134 return NC_EHDFERR;
135
136 /* Get the native typeid. Will be equivalent to hdf_typeid when
137 * creating but not necessarily when reading, a variable. */
138 if ((native_typeid = H5Tget_native_type(hdf_typeid, H5T_DIR_DEFAULT)) < 0)
139 return NC_EHDFERR;
140
141 /* Is this type an integer, string, compound, or what? */
142 if ((class = H5Tget_class(native_typeid)) < 0)
143 return NC_EHDFERR;
144
145 /* Is this an atomic type? */
146 if (class == H5T_STRING || class == H5T_INTEGER || class == H5T_FLOAT)
147 {
148 /* Allocate a phony NC_TYPE_INFO_T struct to hold type info. */
149 if (!(*type_info = calloc(1, sizeof(NC_TYPE_INFO_T))))
150 return NC_ENOMEM;
151
152 /* H5Tequal doesn't work with H5T_C_S1 for some reason. But
153 * H5Tget_class will return H5T_STRING if this is a string. */
154 if (class == H5T_STRING)
155 {
156 if ((is_str = H5Tis_variable_str(native_typeid)) < 0)
157 return NC_EHDFERR;
158 /* Make sure fixed-len strings will work like variable-len
159 * strings */
160 if (is_str || H5Tget_size(hdf_typeid) > 1)
161 {
162 /* Set a class for the type */
163 t = NUM_TYPES - 1;
164 (*type_info)->nc_type_class = NC_STRING;
165 }
166 else
167 {
168 /* Set a class for the type */
169 t = 0;
170 (*type_info)->nc_type_class = NC_CHAR;
171 }
172 }
173 else
174 {
175 for (t = 1; t < NUM_TYPES - 1; t++)
176 {
177 if ((equal = H5Tequal(native_typeid,
178 h5_native_type_constant_g[t])) < 0)
179 return NC_EHDFERR;
180 if (equal)
181 break;
182 }
183
184 /* Find out about endianness. As of HDF 1.8.6, this works
185 * with all data types Not just H5T_INTEGER. See
186 * https://www.hdfgroup.org/HDF5/doc/RM/RM_H5T.html#Datatype-GetOrder */
187 if((order = H5Tget_order(hdf_typeid)) < 0)
188 return NC_EHDFERR;
189
190 if(order == H5T_ORDER_LE)
191 (*type_info)->endianness = NC_ENDIAN_LITTLE;
192 else if(order == H5T_ORDER_BE)
193 (*type_info)->endianness = NC_ENDIAN_BIG;
194 else
195 return NC_EBADTYPE;
196
197 if(class == H5T_INTEGER)
198 (*type_info)->nc_type_class = NC_INT;
199 else
200 (*type_info)->nc_type_class = NC_FLOAT;
201 }
202 (*type_info)->hdr.id = nc_type_constant_g[t];
203 (*type_info)->size = nc_type_size_g[t];
204 if (!((*type_info)->hdr.name = strdup(nc_type_name_g[t])))
205 return NC_ENOMEM;
206 (*type_info)->hdf_typeid = hdf_typeid;
207 (*type_info)->native_hdf_typeid = native_typeid;
208 return NC_NOERR;
209 }
210 else
211 {
212 NC_TYPE_INFO_T *type;
213
214 /* This is a user-defined type. */
215 if((type = nc4_rec_find_hdf_type(h5, native_typeid)))
216 *type_info = type;
217
218 /* The type entry in the array of user-defined types already has
219 * an open data typeid (and native typeid), so close the ones we
220 * opened above. */
221 if (H5Tclose(native_typeid) < 0)
222 return NC_EHDFERR;
223 if (H5Tclose(hdf_typeid) < 0)
224 return NC_EHDFERR;
225
226 if (type)
227 return NC_NOERR;
228 }
229
230 return NC_EBADTYPID;
231 }
232
233 /**
234 * @internal This function reads the coordinates attribute used for
235 * multi-dimensional coordinates.
236 *
237 * @param grp Group info pointer.
238 * @param var Var info pointer.
239 *
240 * @return NC_NOERR No error.
241 * @author Ed Hartnett
242 */
243 static int
read_coord_dimids(NC_GRP_INFO_T * grp,NC_VAR_INFO_T * var)244 read_coord_dimids(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
245 {
246 hid_t coord_att_typeid = -1, coord_attid = -1, spaceid = -1;
247 hssize_t npoints;
248 int retval = NC_NOERR;
249 int d;
250
251 /* There is a hidden attribute telling us the ids of the
252 * dimensions that apply to this multi-dimensional coordinate
253 * variable. Read it. */
254 if ((coord_attid = H5Aopen_name(var->hdf_datasetid, COORDINATES)) < 0)
255 BAIL(NC_EATTMETA);
256
257 if ((coord_att_typeid = H5Aget_type(coord_attid)) < 0)
258 BAIL(NC_EATTMETA);
259
260 /* How many dimensions are there? */
261 if ((spaceid = H5Aget_space(coord_attid)) < 0)
262 BAIL(NC_EATTMETA);
263 if ((npoints = H5Sget_simple_extent_npoints(spaceid)) < 0)
264 BAIL(NC_EATTMETA);
265
266 /* Check that the number of points is the same as the number of
267 * dimensions for the variable. */
268 if (npoints != var->ndims)
269 BAIL(NC_EATTMETA);
270
271 if (H5Aread(coord_attid, coord_att_typeid, var->dimids) < 0)
272 BAIL(NC_EATTMETA);
273 LOG((4, "dimscale %s is multidimensional and has coords", var->hdr.name));
274
275 /* Update var->dim field based on the var->dimids. Ok if does not
276 * find a dim at this time, but if found set it. */
277 for (d = 0; d < var->ndims; d++)
278 nc4_find_dim(grp, var->dimids[d], &var->dim[d], NULL);
279
280 exit:
281 if (spaceid >= 0 && H5Sclose(spaceid) < 0)
282 BAIL2(NC_EHDFERR);
283 if (coord_att_typeid >= 0 && H5Tclose(coord_att_typeid) < 0)
284 BAIL2(NC_EHDFERR);
285 if (coord_attid >= 0 && H5Aclose(coord_attid) < 0)
286 BAIL2(NC_EHDFERR);
287 return retval;
288 }
289
290 /**
291 * @internal This function is called when reading a file's metadata
292 * for each dimension scale attached to a variable.
293 *
294 * @param did HDF5 ID for dimscale.
295 * @param dim
296 * @param dsid
297 * @param dimscale_hdf5_objids
298 *
299 * @return 0 for success, -1 for error.
300 * @author Ed Hartnett
301 */
302 static herr_t
dimscale_visitor(hid_t did,unsigned dim,hid_t dsid,void * dimscale_hdf5_objids)303 dimscale_visitor(hid_t did, unsigned dim, hid_t dsid,
304 void *dimscale_hdf5_objids)
305 {
306 H5G_stat_t statbuf;
307
308 /* Get more info on the dimscale object.*/
309 if (H5Gget_objinfo(dsid, ".", 1, &statbuf) < 0)
310 return -1;
311
312 /* Pass this information back to caller. */
313 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).fileno[0] = statbuf.fileno[0];
314 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).fileno[1] = statbuf.fileno[1];
315 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).objno[0] = statbuf.objno[0];
316 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).objno[1] = statbuf.objno[1];
317 return 0;
318 }
319
320 /**
321 * @internal Check for the attribute that indicates that netcdf
322 * classic model is in use.
323 *
324 * @param root_grp pointer to the group info for the root group of the
325 * @param is_classic store 1 if this is a classic file.
326 * file.
327 *
328 * @return NC_NOERR No error.
329 * @author Ed Hartnett
330 */
331 static int
check_for_classic_model(NC_GRP_INFO_T * root_grp,int * is_classic)332 check_for_classic_model(NC_GRP_INFO_T *root_grp, int *is_classic)
333 {
334 htri_t attr_exists = -1;
335 hid_t grpid;
336
337 /* Check inputs. */
338 assert(root_grp && root_grp->format_grp_info && !root_grp->parent
339 && is_classic);
340
341 /* Get the HDF5 group id. */
342 grpid = ((NC_HDF5_GRP_INFO_T *)(root_grp->format_grp_info))->hdf_grpid;
343
344 /* If this attribute exists in the root group, then classic model
345 * is in effect. */
346 if ((attr_exists = H5Aexists(grpid, NC3_STRICT_ATT_NAME)) < 0)
347 return NC_EHDFERR;
348 *is_classic = attr_exists ? 1 : 0;
349
350 return NC_NOERR;
351 }
352
353 /**
354 * @internal Open a netcdf-4 file. Things have already been kicked off
355 * in ncfunc.c in nc_open, but here the netCDF-4 part of opening a
356 * file is handled.
357 *
358 * @param path The file name of the new file.
359 * @param mode The open mode flag.
360 * @param parameters File parameters.
361 * @param nc Pointer to NC file info.
362 *
363 * @return ::NC_NOERR No error.
364 * @author Ed Hartnett, Dennis Heimbigner
365 */
366 static int
nc4_open_file(const char * path,int mode,void * parameters,NC * nc)367 nc4_open_file(const char *path, int mode, void* parameters, NC *nc)
368 {
369 hid_t fapl_id = H5P_DEFAULT;
370 int retval;
371 unsigned flags;
372 NC_FILE_INFO_T *nc4_info = NULL;
373 int is_classic;
374
375 #ifdef USE_PARALLEL4
376 NC_MPI_INFO* mpiinfo = NULL;
377 int comm_duped = 0; /* Whether the MPI Communicator was duplicated */
378 int info_duped = 0; /* Whether the MPI Info object was duplicated */
379 #endif
380
381 LOG((3, "%s: path %s mode %d", __func__, path, mode));
382 assert(path && nc);
383
384 flags = (mode & NC_WRITE) ? H5F_ACC_RDWR : H5F_ACC_RDONLY;
385
386 /* Add necessary structs to hold netcdf-4 file data. */
387 if ((retval = nc4_nc4f_list_add(nc, path, mode)))
388 BAIL(retval);
389 nc4_info = NC4_DATA(nc);
390 assert(nc4_info && nc4_info->root_grp);
391
392 /* Add struct to hold HDF5-specific file metadata. */
393 if (!(nc4_info->format_file_info = calloc(1, sizeof(NC_HDF5_FILE_INFO_T))))
394 BAIL(NC_ENOMEM);
395
396 /* Add struct to hold HDF5-specific group info. */
397 if (!(nc4_info->root_grp->format_grp_info = calloc(1, sizeof(NC_HDF5_GRP_INFO_T))))
398 return NC_ENOMEM;
399
400 nc4_info->mem.inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY);
401 nc4_info->mem.diskless = ((mode & NC_DISKLESS) == NC_DISKLESS);
402 nc4_info->mem.persist = ((mode & NC_PERSIST) == NC_PERSIST);
403 /* Does the mode specify that this file is read-only? */
404 if ((mode & NC_WRITE) == 0)
405 nc4_info->no_write = NC_TRUE;
406
407 if(nc4_info->mem.inmemory && nc4_info->mem.diskless)
408 BAIL(NC_EINTERNAL);
409
410 #ifdef USE_PARALLEL4
411 mpiinfo = (NC_MPI_INFO*)parameters; /* assume, may be changed if inmemory is true */
412 #endif /* !USE_PARALLEL4 */
413
414 /* Need this access plist to control how HDF5 handles open objects
415 * on file close. (Setting H5F_CLOSE_SEMI will cause H5Fclose to
416 * fail if there are any open objects in the file). */
417 if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
418 BAIL(NC_EHDFERR);
419
420 if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI) < 0)
421 BAIL(NC_EHDFERR);
422
423 #ifdef USE_PARALLEL4
424 if (!(mode & (NC_INMEMORY | NC_DISKLESS)) && mpiinfo != NULL) {
425 /* If this is a parallel file create, set up the file creation
426 * property list.
427 */
428 nc4_info->parallel = NC_TRUE;
429 LOG((4, "opening parallel file with MPI/IO"));
430 if (H5Pset_fapl_mpio(fapl_id, mpiinfo->comm, mpiinfo->info) < 0)
431 BAIL(NC_EPARINIT);
432
433 /* Keep copies of the MPI Comm & Info objects */
434 if (MPI_SUCCESS != MPI_Comm_dup(mpiinfo->comm, &nc4_info->comm))
435 BAIL(NC_EMPI);
436 comm_duped++;
437 if (MPI_INFO_NULL != mpiinfo->info)
438 {
439 if (MPI_SUCCESS != MPI_Info_dup(mpiinfo->info, &nc4_info->info))
440 BAIL(NC_EMPI);
441 info_duped++;
442 }
443 else
444 {
445 /* No dup, just copy it. */
446 nc4_info->info = mpiinfo->info;
447 }
448 }
449
450 #ifdef HDF5_HAS_COLL_METADATA_OPS
451 if (H5Pset_all_coll_metadata_ops(fapl_id, 1) < 0)
452 BAIL(NC_EPARINIT);
453 #endif
454
455 #else /* only set cache for non-parallel. */
456 if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
457 nc4_chunk_cache_preemption) < 0)
458 BAIL(NC_EHDFERR);
459 LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f",
460 __func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems,
461 nc4_chunk_cache_preemption));
462 #endif /* USE_PARALLEL4 */
463
464 /* Process NC_INMEMORY */
465 if(nc4_info->mem.inmemory) {
466 NC_memio* memio;
467 /* validate */
468 if(parameters == NULL)
469 BAIL(NC_EINMEMORY);
470 memio = (NC_memio*)parameters;
471 if(memio->memory == NULL || memio->size == 0)
472 BAIL(NC_EINMEMORY);
473 /* initialize h5->mem */
474 nc4_info->mem.memio = *memio;
475 /* Is the incoming memory locked? */
476 nc4_info->mem.locked = (nc4_info->mem.memio.flags & NC_MEMIO_LOCKED) == NC_MEMIO_LOCKED;
477 /* As a safeguard, if not locked and not read-only,
478 then we must take control of the incoming memory */
479 if(!nc4_info->mem.locked && !nc4_info->no_write) {
480 memio->memory = NULL; /* take control */
481 memio->size = 0;
482 }
483 retval = NC4_open_image_file(nc4_info);
484 if(retval)
485 BAIL(NC_EHDFERR);
486 }
487 else
488 if(nc4_info->mem.diskless) { /* Process NC_DISKLESS */
489 NC_HDF5_FILE_INFO_T *hdf5_info;
490 size_t min_incr = 65536; /* Minimum buffer increment */
491 /* Configure FAPL to use the core file driver */
492 if (H5Pset_fapl_core(fapl_id, min_incr, (nc4_info->mem.persist?1:0)) < 0)
493 BAIL(NC_EHDFERR);
494 hdf5_info = (NC_HDF5_FILE_INFO_T *)nc4_info->format_file_info;
495 /* Open the HDF5 file. */
496 if ((hdf5_info->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
497 BAIL(NC_EHDFERR);
498 }
499 else
500 {
501 NC_HDF5_FILE_INFO_T *hdf5_info;
502 hdf5_info = (NC_HDF5_FILE_INFO_T *)nc4_info->format_file_info;
503
504 /* Open the HDF5 file. */
505 if ((hdf5_info->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
506 BAIL(NC_EHDFERR);
507 }
508
509 /* Now read in all the metadata. Some types and dimscale
510 * information may be difficult to resolve here, if, for example, a
511 * dataset of user-defined type is encountered before the
512 * definition of that type. */
513 if ((retval = nc4_rec_read_metadata(nc4_info->root_grp)))
514 BAIL(retval);
515
516 /* Check for classic model attribute. */
517 if ((retval = check_for_classic_model(nc4_info->root_grp, &is_classic)))
518 BAIL(retval);
519 if (is_classic)
520 nc4_info->cmode |= NC_CLASSIC_MODEL;
521
522 /* See if this file contained _NCPROPERTIES,
523 * and if yes, process it, if no, then fake it.
524 */
525 if(nc4_info->root_grp != NULL) {
526 /* Since _NCProperties does not exist as an NC_ATT_INFO object,
527 * we need to check using the HDF5 API
528 */
529 if((retval = NC4_read_ncproperties(nc4_info)))
530 BAIL(retval);
531 }
532
533 if ((retval = check_for_classic_model(nc4_info->root_grp, &is_classic)))
534 BAIL(retval);
535 if (is_classic)
536 nc4_info->cmode |= NC_CLASSIC_MODEL;
537
538 /* Now figure out which netCDF dims are indicated by the dimscale
539 * information. */
540 if ((retval = nc4_rec_match_dimscales(nc4_info->root_grp)))
541 BAIL(retval);
542
543 #ifdef LOGGING
544 /* This will print out the names, types, lens, etc of the vars and
545 atts in the file, if the logging level is 2 or greater. */
546 log_metadata_nc(nc4_info);
547 #endif
548
549 /* Close the property list. */
550 if (H5Pclose(fapl_id) < 0)
551 BAIL(NC_EHDFERR);
552
553 return NC_NOERR;
554
555 exit:
556 #ifdef USE_PARALLEL4
557 if (comm_duped) MPI_Comm_free(&nc4_info->comm);
558 if (info_duped) MPI_Info_free(&nc4_info->info);
559 #endif
560
561 if (fapl_id > 0 && fapl_id != H5P_DEFAULT)
562 H5Pclose(fapl_id);
563 if (nc4_info)
564 nc4_close_hdf5_file(nc4_info, 1, 0); /* treat like abort*/
565 return retval;
566 }
567
568 /**
569 * @internal Open a netCDF-4 file.
570 *
571 * @param path The file name of the new file.
572 * @param mode The open mode flag.
573 * @param basepe Ignored by this function.
574 * @param chunksizehintp Ignored by this function.
575 * @param parameters pointer to struct holding extra data (e.g. for parallel I/O)
576 * layer. Ignored if NULL.
577 * @param dispatch Pointer to the dispatch table for this file.
578 * @param nc_file Pointer to an instance of NC.
579 *
580 * @return ::NC_NOERR No error.
581 * @return ::NC_EINVAL Invalid inputs.
582 * @author Ed Hartnett
583 */
584 int
NC4_open(const char * path,int mode,int basepe,size_t * chunksizehintp,void * parameters,NC_Dispatch * dispatch,NC * nc_file)585 NC4_open(const char *path, int mode, int basepe, size_t *chunksizehintp,
586 void *parameters, NC_Dispatch *dispatch, NC *nc_file)
587 {
588 assert(nc_file && path && dispatch && nc_file &&
589 nc_file->model == NC_FORMATX_NC4);
590
591 LOG((1, "%s: path %s mode %d params %x",
592 __func__, path, mode, parameters));
593
594 /* Check the mode for validity */
595 if (mode & ILLEGAL_OPEN_FLAGS)
596 return NC_EINVAL;
597
598 if((mode & NC_DISKLESS) && (mode & NC_INMEMORY))
599 return NC_EINVAL;
600
601 /* If this is our first file, initialize HDF5. */
602 if (!nc4_hdf5_initialized)
603 nc4_hdf5_initialize();
604
605 #ifdef LOGGING
606 /* If nc logging level has changed, see if we need to turn on
607 * HDF5's error messages. */
608 hdf5_set_log_level();
609 #endif /* LOGGING */
610
611 nc_file->int_ncid = nc_file->ext_ncid;
612
613 /* Open the file. */
614 return nc4_open_file(path, mode, parameters, nc_file);
615 }
616
617 /**
618 * @internal This function is called by read_dataset(), (which is called
619 * by nc4_rec_read_metadata()) when a netCDF variable is found in the
620 * file. This function reads in all the metadata about the var,
621 * including the attributes.
622 *
623 * @param grp Pointer to group info struct.
624 * @param datasetid HDF5 dataset ID.
625 * @param obj_name Name of the HDF5 object to read.
626 * @param ndims Number of dimensions.
627 * @param dim
628 *
629 * @return ::NC_NOERR No error.
630 * @return ::NC_EBADID Bad ncid.
631 * @return ::NC_EHDFERR HDF5 returned error.
632 * @author Ed Hartnett
633 */
634 static int
read_var(NC_GRP_INFO_T * grp,hid_t datasetid,const char * obj_name,size_t ndims,NC_DIM_INFO_T * dim)635 read_var(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name,
636 size_t ndims, NC_DIM_INFO_T *dim)
637 {
638 NC_VAR_INFO_T *var = NULL;
639 hid_t access_pid = 0;
640 int incr_id_rc = 0; /* Whether dataset ID's ref count has been incremented */
641 int d;
642 H5Z_filter_t filter;
643 int num_filters;
644 unsigned int cd_values_zip[CD_NELEMS_ZLIB];
645 size_t cd_nelems = CD_NELEMS_ZLIB;
646 hid_t propid = 0;
647 H5D_fill_value_t fill_status;
648 H5D_layout_t layout;
649 hsize_t chunksize[NC_MAX_VAR_DIMS] = {0};
650 int retval = NC_NOERR;
651 double rdcc_w0;
652 int f;
653 char* finalname = NULL;
654
655 assert(obj_name && grp);
656 LOG((4, "%s: obj_name %s", __func__, obj_name));
657
658 /* Check for a weird case: a non-coordinate variable that has the
659 * same name as a dimension. It's legal in netcdf, and requires
660 * that the HDF5 dataset name be changed. */
661 if (strlen(obj_name) > strlen(NON_COORD_PREPEND) &&
662 !strncmp(obj_name, NON_COORD_PREPEND, strlen(NON_COORD_PREPEND)))
663 {
664 /* Allocate space for the name. */
665 if(finalname) {free(finalname); finalname = NULL;}
666 if (!(finalname = malloc(((strlen(obj_name) -
667 strlen(NON_COORD_PREPEND))+ 1) * sizeof(char))))
668 BAIL(NC_ENOMEM);
669 strcpy(finalname, &obj_name[strlen(NON_COORD_PREPEND)]);
670 } else
671 finalname = strdup(obj_name);
672
673 /* Add a variable to the end of the group's var list. */
674 if ((retval = nc4_var_list_add(grp,finalname,ndims,&var)))
675 BAIL(retval);
676
677 /* Fill in what we already know. */
678 var->hdf_datasetid = datasetid;
679 H5Iinc_ref(var->hdf_datasetid); /* Increment number of objects using ID */
680 incr_id_rc++; /* Indicate that we've incremented the ref. count (for errors) */
681 var->created = NC_TRUE;
682
683 /* Get the current chunk cache settings. */
684 if ((access_pid = H5Dget_access_plist(datasetid)) < 0)
685 BAIL(NC_EVARMETA);
686
687 /* Learn about current chunk cache settings. */
688 if ((H5Pget_chunk_cache(access_pid, &(var->chunk_cache_nelems),
689 &(var->chunk_cache_size), &rdcc_w0)) < 0)
690 BAIL(NC_EHDFERR);
691 var->chunk_cache_preemption = rdcc_w0;
692
693 /* Find out what filters are applied to this HDF5 dataset,
694 * fletcher32, deflate, and/or shuffle. All other filters are
695 * just dumped */
696 if ((propid = H5Dget_create_plist(datasetid)) < 0)
697 BAIL(NC_EHDFERR);
698
699 /* Get the chunking info for non-scalar vars. */
700 if ((layout = H5Pget_layout(propid)) < -1)
701 BAIL(NC_EHDFERR);
702 if (layout == H5D_CHUNKED)
703 {
704 if (H5Pget_chunk(propid, NC_MAX_VAR_DIMS, chunksize) < 0)
705 BAIL(NC_EHDFERR);
706 if (!(var->chunksizes = malloc(var->ndims * sizeof(size_t))))
707 BAIL(NC_ENOMEM);
708 for (d = 0; d < var->ndims; d++)
709 var->chunksizes[d] = chunksize[d];
710 }
711 else if (layout == H5D_CONTIGUOUS || layout == H5D_COMPACT)
712 var->contiguous = NC_TRUE;
713
714 /* The possible values of filter (which is just an int) can be
715 * found in H5Zpublic.h. */
716 if ((num_filters = H5Pget_nfilters(propid)) < 0)
717 BAIL(NC_EHDFERR);
718 for (f = 0; f < num_filters; f++)
719 {
720 if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems,
721 cd_values_zip, 0, NULL, NULL)) < 0)
722 BAIL(NC_EHDFERR);
723 switch (filter)
724 {
725 case H5Z_FILTER_SHUFFLE:
726 var->shuffle = NC_TRUE;
727 break;
728
729 case H5Z_FILTER_FLETCHER32:
730 var->fletcher32 = NC_TRUE;
731 break;
732
733 case H5Z_FILTER_DEFLATE:
734 var->deflate = NC_TRUE;
735 if (cd_nelems != CD_NELEMS_ZLIB ||
736 cd_values_zip[0] > NC_MAX_DEFLATE_LEVEL)
737 BAIL(NC_EHDFERR);
738 var->deflate_level = cd_values_zip[0];
739 break;
740
741 case H5Z_FILTER_SZIP:
742 /* Szip is tricky because the filter code expands the set of parameters from 2 to 4
743 and changes some of the parameter values */
744 var->filterid = filter;
745 if(cd_nelems == 0)
746 var->params = NULL;
747 else {
748 /* We have to re-read the parameters based on actual nparams,
749 which in the case of szip, differs from users original nparams */
750 var->params = (unsigned int*)calloc(1,sizeof(unsigned int)*cd_nelems);
751 if(var->params == NULL)
752 BAIL(NC_ENOMEM);
753 if((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems,
754 var->params, 0, NULL, NULL)) < 0)
755 BAIL(NC_EHDFERR);
756 /* fix up the parameters and the #params */
757 var->nparams = cd_nelems;
758 }
759 break;
760
761 default:
762 var->filterid = filter;
763 var->nparams = cd_nelems;
764 if(cd_nelems == 0)
765 var->params = NULL;
766 else {
767 /* We have to re-read the parameters based on actual nparams */
768 var->params = (unsigned int*)calloc(1,sizeof(unsigned int)*var->nparams);
769 if(var->params == NULL)
770 BAIL(NC_ENOMEM);
771 if((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems,
772 var->params, 0, NULL, NULL)) < 0)
773 BAIL(NC_EHDFERR);
774 }
775 break;
776 }
777 }
778
779 /* Learn all about the type of this variable. */
780 if ((retval = get_type_info2(grp->nc4_info, datasetid,
781 &var->type_info)))
782 BAIL(retval);
783
784 /* Indicate that the variable has a pointer to the type */
785 var->type_info->rc++;
786
787 /* Is there a fill value associated with this dataset? */
788 if (H5Pfill_value_defined(propid, &fill_status) < 0)
789 BAIL(NC_EHDFERR);
790
791 /* Get the fill value, if there is one defined. */
792 if (fill_status == H5D_FILL_VALUE_USER_DEFINED)
793 {
794 /* Allocate space to hold the fill value. */
795 if (!var->fill_value)
796 {
797 if (var->type_info->nc_type_class == NC_VLEN)
798 {
799 if (!(var->fill_value = malloc(sizeof(nc_vlen_t))))
800 BAIL(NC_ENOMEM);
801 }
802 else if (var->type_info->nc_type_class == NC_STRING)
803 {
804 if (!(var->fill_value = malloc(sizeof(char *))))
805 BAIL(NC_ENOMEM);
806 }
807 else
808 {
809 assert(var->type_info->size);
810 if (!(var->fill_value = malloc(var->type_info->size)))
811 BAIL(NC_ENOMEM);
812 }
813 }
814
815 /* Get the fill value from the HDF5 property lust. */
816 if (H5Pget_fill_value(propid, var->type_info->native_hdf_typeid,
817 var->fill_value) < 0)
818 BAIL(NC_EHDFERR);
819 }
820 else
821 var->no_fill = NC_TRUE;
822
823 /* If it's a scale, mark it as such. */
824 if (dim)
825 {
826 assert(ndims);
827 var->dimscale = NC_TRUE;
828 if (var->ndims > 1)
829 {
830 if ((retval = read_coord_dimids(grp, var)))
831 BAIL(retval);
832 }
833 else
834 {
835 /* sanity check */
836 assert(0 == strcmp(var->hdr.name, dim->hdr.name));
837
838 var->dimids[0] = dim->hdr.id;
839 var->dim[0] = dim;
840 }
841 dim->coord_var = var;
842 }
843 /* If this is not a scale, but has scales, iterate
844 * through them. (i.e. this is a variable that is not a
845 * coordinate variable) */
846 else
847 {
848 int num_scales = 0;
849
850 /* Find out how many scales are attached to this
851 * dataset. H5DSget_num_scales returns an error if there are no
852 * scales, so convert a negative return value to zero. */
853 num_scales = H5DSget_num_scales(datasetid, 0);
854 if (num_scales < 0)
855 num_scales = 0;
856
857 if (num_scales && ndims)
858 {
859 /* Allocate space to remember whether the dimscale has been
860 * attached for each dimension. */
861 if (!(var->dimscale_attached = calloc(ndims, sizeof(nc_bool_t))))
862 BAIL(NC_ENOMEM);
863
864 /* Store id information allowing us to match hdf5
865 * dimscales to netcdf dimensions. */
866 if (!(var->dimscale_hdf5_objids = malloc(ndims *
867 sizeof(struct hdf5_objid))))
868 BAIL(NC_ENOMEM);
869 for (d = 0; d < var->ndims; d++)
870 {
871 if (H5DSiterate_scales(var->hdf_datasetid, d, NULL, dimscale_visitor,
872 &(var->dimscale_hdf5_objids[d])) < 0)
873 BAIL(NC_EHDFERR);
874 var->dimscale_attached[d] = NC_TRUE;
875 }
876 }
877 }
878
879 /* Read variable attributes. */
880 var->atts_not_read = 1;
881
882 /* Is this a deflated variable with a chunksize greater than the
883 * current cache size? */
884 if ((retval = nc4_adjust_var_cache(grp, var)))
885 BAIL(retval);
886
887 exit:
888 if(finalname) free(finalname);
889 if (retval)
890 {
891 if (incr_id_rc && H5Idec_ref(datasetid) < 0)
892 BAIL2(NC_EHDFERR);
893 if (var != NULL) {
894 nc4_var_list_del(grp,var);
895 }
896 }
897 if (access_pid && H5Pclose(access_pid) < 0)
898 BAIL2(NC_EHDFERR);
899 if (propid > 0 && H5Pclose(propid) < 0)
900 BAIL2(NC_EHDFERR);
901 return retval;
902 }
903
904 /**
905 * @internal Given an HDF5 type, set a pointer to netcdf type.
906 *
907 * @param h5 Pointer to HDF5 file info struct.
908 * @param native_typeid HDF5 type ID.
909 * @param xtype Pointer that gets netCDF type.
910 *
911 * @return ::NC_NOERR No error.
912 * @return ::NC_EBADID Bad ncid.
913 * @return ::NC_EHDFERR HDF5 returned error.
914 * @return ::NC_EBADTYPID Type not found.
915 * @author Ed Hartnett
916 */
917 static int
get_netcdf_type(NC_FILE_INFO_T * h5,hid_t native_typeid,nc_type * xtype)918 get_netcdf_type(NC_FILE_INFO_T *h5, hid_t native_typeid,
919 nc_type *xtype)
920 {
921 NC_TYPE_INFO_T *type;
922 H5T_class_t class;
923 htri_t is_str, equal = 0;
924
925 assert(h5 && xtype);
926
927 if ((class = H5Tget_class(native_typeid)) < 0)
928 return NC_EHDFERR;
929
930 /* H5Tequal doesn't work with H5T_C_S1 for some reason. But
931 * H5Tget_class will return H5T_STRING if this is a string. */
932 if (class == H5T_STRING)
933 {
934 if ((is_str = H5Tis_variable_str(native_typeid)) < 0)
935 return NC_EHDFERR;
936 if (is_str)
937 *xtype = NC_STRING;
938 else
939 *xtype = NC_CHAR;
940 return NC_NOERR;
941 }
942 else if (class == H5T_INTEGER || class == H5T_FLOAT)
943 {
944 /* For integers and floats, we don't have to worry about
945 * endianness if we compare native types. */
946 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_SCHAR)) < 0)
947 return NC_EHDFERR;
948 if (equal)
949 {
950 *xtype = NC_BYTE;
951 return NC_NOERR;
952 }
953 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_SHORT)) < 0)
954 return NC_EHDFERR;
955 if (equal)
956 {
957 *xtype = NC_SHORT;
958 return NC_NOERR;
959 }
960 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_INT)) < 0)
961 return NC_EHDFERR;
962 if (equal)
963 {
964 *xtype = NC_INT;
965 return NC_NOERR;
966 }
967 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_FLOAT)) < 0)
968 return NC_EHDFERR;
969 if (equal)
970 {
971 *xtype = NC_FLOAT;
972 return NC_NOERR;
973 }
974 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_DOUBLE)) < 0)
975 return NC_EHDFERR;
976 if (equal)
977 {
978 *xtype = NC_DOUBLE;
979 return NC_NOERR;
980 }
981 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_UCHAR)) < 0)
982 return NC_EHDFERR;
983 if (equal)
984 {
985 *xtype = NC_UBYTE;
986 return NC_NOERR;
987 }
988 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_USHORT)) < 0)
989 return NC_EHDFERR;
990 if (equal)
991 {
992 *xtype = NC_USHORT;
993 return NC_NOERR;
994 }
995 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_UINT)) < 0)
996 return NC_EHDFERR;
997 if (equal)
998 {
999 *xtype = NC_UINT;
1000 return NC_NOERR;
1001 }
1002 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_LLONG)) < 0)
1003 return NC_EHDFERR;
1004 if (equal)
1005 {
1006 *xtype = NC_INT64;
1007 return NC_NOERR;
1008 }
1009 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_ULLONG)) < 0)
1010 return NC_EHDFERR;
1011 if (equal)
1012 {
1013 *xtype = NC_UINT64;
1014 return NC_NOERR;
1015 }
1016 }
1017
1018 /* Maybe we already know about this type. */
1019 if (!equal)
1020 if((type = nc4_rec_find_hdf_type(h5, native_typeid)))
1021 {
1022 *xtype = type->hdr.id;
1023 return NC_NOERR;
1024 }
1025
1026 *xtype = NC_NAT;
1027 return NC_EBADTYPID;
1028 }
1029
1030 /**
1031 * @internal Read an attribute. This is called by
1032 * att_read_callbk().
1033 *
1034 * @param grp Pointer to group info struct.
1035 * @param attid Attribute ID.
1036 * @param att Pointer that gets att info struct.
1037 *
1038 * @return ::NC_NOERR No error.
1039 * @return ::NC_EHDFERR HDF5 returned error.
1040 * @author Ed Hartnett
1041 */
1042 static int
read_hdf5_att(NC_GRP_INFO_T * grp,hid_t attid,NC_ATT_INFO_T * att)1043 read_hdf5_att(NC_GRP_INFO_T *grp, hid_t attid, NC_ATT_INFO_T *att)
1044 {
1045 NC_HDF5_ATT_INFO_T *hdf5_att;
1046 hid_t spaceid = 0, file_typeid = 0;
1047 hsize_t dims[1] = {0}; /* netcdf attributes always 1-D. */
1048 size_t type_size;
1049 int att_ndims;
1050 hssize_t att_npoints;
1051 H5T_class_t att_class;
1052 int fixed_len_string = 0;
1053 size_t fixed_size = 0;
1054 int retval = NC_NOERR;
1055
1056 assert(att && att->hdr.name && att->format_att_info);
1057 LOG((5, "%s: att->hdr.id %d att->hdr.name %s att->nc_typeid %d att->len %d",
1058 __func__, att->hdr.id, att->hdr.name, (int)att->nc_typeid, att->len));
1059
1060 /* Get HDF5-sepecific info stuct for this attribute. */
1061 hdf5_att = (NC_HDF5_ATT_INFO_T *)att->format_att_info;
1062
1063 /* Get type of attribute in file. */
1064 if ((file_typeid = H5Aget_type(attid)) < 0)
1065 return NC_EATTMETA;
1066 if ((hdf5_att->native_hdf_typeid = H5Tget_native_type(file_typeid,
1067 H5T_DIR_DEFAULT)) < 0)
1068 BAIL(NC_EHDFERR);
1069 if ((att_class = H5Tget_class(hdf5_att->native_hdf_typeid)) < 0)
1070 BAIL(NC_EATTMETA);
1071 if (att_class == H5T_STRING &&
1072 !H5Tis_variable_str(hdf5_att->native_hdf_typeid))
1073 {
1074 fixed_len_string++;
1075 if (!(fixed_size = H5Tget_size(hdf5_att->native_hdf_typeid)))
1076 BAIL(NC_EATTMETA);
1077 }
1078 if ((retval = get_netcdf_type(grp->nc4_info, hdf5_att->native_hdf_typeid,
1079 &(att->nc_typeid))))
1080 BAIL(retval);
1081
1082
1083 /* Get len. */
1084 if ((spaceid = H5Aget_space(attid)) < 0)
1085 BAIL(NC_EATTMETA);
1086 if ((att_ndims = H5Sget_simple_extent_ndims(spaceid)) < 0)
1087 BAIL(NC_EATTMETA);
1088 if ((att_npoints = H5Sget_simple_extent_npoints(spaceid)) < 0)
1089 BAIL(NC_EATTMETA);
1090
1091 /* If both att_ndims and att_npoints are zero, then this is a
1092 * zero length att. */
1093 if (att_ndims == 0 && att_npoints == 0)
1094 dims[0] = 0;
1095 else if (att->nc_typeid == NC_STRING)
1096 dims[0] = att_npoints;
1097 else if (att->nc_typeid == NC_CHAR)
1098 {
1099 /* NC_CHAR attributes are written as a scalar in HDF5, of type
1100 * H5T_C_S1, of variable length. */
1101 if (att_ndims == 0)
1102 {
1103 if (!(dims[0] = H5Tget_size(file_typeid)))
1104 BAIL(NC_EATTMETA);
1105 }
1106 else
1107 {
1108 /* This is really a string type! */
1109 att->nc_typeid = NC_STRING;
1110 dims[0] = att_npoints;
1111 }
1112 }
1113 else
1114 {
1115 H5S_class_t space_class;
1116
1117 /* All netcdf attributes are scalar or 1-D only. */
1118 if (att_ndims > 1)
1119 BAIL(NC_EATTMETA);
1120
1121 /* Check class of HDF5 dataspace */
1122 if ((space_class = H5Sget_simple_extent_type(spaceid)) < 0)
1123 BAIL(NC_EATTMETA);
1124
1125 /* Check for NULL HDF5 dataspace class (should be weeded out
1126 * earlier) */
1127 if (H5S_NULL == space_class)
1128 BAIL(NC_EATTMETA);
1129
1130 /* check for SCALAR HDF5 dataspace class */
1131 if (H5S_SCALAR == space_class)
1132 dims[0] = 1;
1133 else /* Must be "simple" dataspace */
1134 {
1135 /* Read the size of this attribute. */
1136 if (H5Sget_simple_extent_dims(spaceid, dims, NULL) < 0)
1137 BAIL(NC_EATTMETA);
1138 }
1139 }
1140
1141 /* Tell the user what the length if this attribute is. */
1142 att->len = dims[0];
1143
1144 /* Allocate some memory if the len is not zero, and read the
1145 attribute. */
1146 if (dims[0])
1147 {
1148 if ((retval = nc4_get_typelen_mem(grp->nc4_info, att->nc_typeid,
1149 &type_size)))
1150 return retval;
1151 if (att_class == H5T_VLEN)
1152 {
1153 if (!(att->vldata = malloc((unsigned int)(att->len * sizeof(hvl_t)))))
1154 BAIL(NC_ENOMEM);
1155 if (H5Aread(attid, hdf5_att->native_hdf_typeid, att->vldata) < 0)
1156 BAIL(NC_EATTMETA);
1157 }
1158 else if (att->nc_typeid == NC_STRING)
1159 {
1160 if (!(att->stdata = calloc(att->len, sizeof(char *))))
1161 BAIL(NC_ENOMEM);
1162 /* For a fixed length HDF5 string, the read requires
1163 * contiguous memory. Meanwhile, the netCDF API requires that
1164 * nc_free_string be called on string arrays, which would not
1165 * work if one contiguous memory block were used. So here I
1166 * convert the contiguous block of strings into an array of
1167 * malloced strings (each string with its own malloc). Then I
1168 * copy the data and free the contiguous memory. This
1169 * involves copying the data, which is bad, but this only
1170 * occurs for fixed length string attributes, and presumably
1171 * these are small. (And netCDF-4 does not create them - it
1172 * always uses variable length strings. */
1173 if (fixed_len_string)
1174 {
1175 int i;
1176 char *contig_buf, *cur;
1177
1178 /* Alloc space for the contiguous memory read. */
1179 if (!(contig_buf = malloc(att->len * fixed_size * sizeof(char))))
1180 BAIL(NC_ENOMEM);
1181
1182 /* Read the fixed-len strings as one big block. */
1183 if (H5Aread(attid, hdf5_att->native_hdf_typeid, contig_buf) < 0) {
1184 free(contig_buf);
1185 BAIL(NC_EATTMETA);
1186 }
1187
1188 /* Copy strings, one at a time, into their new home. Alloc
1189 space for each string. The user will later free this
1190 space with nc_free_string. */
1191 cur = contig_buf;
1192 for (i = 0; i < att->len; i++)
1193 {
1194 if (!(att->stdata[i] = malloc(fixed_size))) {
1195 free(contig_buf);
1196 BAIL(NC_ENOMEM);
1197 }
1198 strncpy(att->stdata[i], cur, fixed_size);
1199 cur += fixed_size;
1200 }
1201
1202 /* Free contiguous memory buffer. */
1203 free(contig_buf);
1204 }
1205 else
1206 {
1207 /* Read variable-length string atts. */
1208 if (H5Aread(attid, hdf5_att->native_hdf_typeid, att->stdata) < 0)
1209 BAIL(NC_EATTMETA);
1210 }
1211 }
1212 else
1213 {
1214 if (!(att->data = malloc((unsigned int)(att->len * type_size))))
1215 BAIL(NC_ENOMEM);
1216 if (H5Aread(attid, hdf5_att->native_hdf_typeid, att->data) < 0)
1217 BAIL(NC_EATTMETA);
1218 }
1219 }
1220
1221 if (H5Tclose(file_typeid) < 0)
1222 BAIL(NC_EHDFERR);
1223 if (H5Sclose(spaceid) < 0)
1224 return NC_EHDFERR;
1225
1226 return NC_NOERR;
1227
1228 exit:
1229 if (H5Tclose(file_typeid) < 0)
1230 BAIL2(NC_EHDFERR);
1231 if (spaceid > 0 && H5Sclose(spaceid) < 0)
1232 BAIL2(NC_EHDFERR);
1233 return retval;
1234 }
1235
1236 /**
1237 * @internal Wrap HDF5 allocated memory free operations
1238 *
1239 * @param memory Pointer to memory to be freed.
1240 *
1241 * @return ::NC_NOERR No error.
1242 * @author Dennis Heimbigner
1243 */
1244 static void
hdf5free(void * memory)1245 hdf5free(void* memory)
1246 {
1247 #ifndef JNA
1248 /* On Windows using the microsoft runtime, it is an error
1249 for one library to free memory allocated by a different library.*/
1250 #ifdef HDF5_HAS_H5FREE
1251 if(memory != NULL) H5free_memory(memory);
1252 #else
1253 #ifndef _MSC_VER
1254 if(memory != NULL) free(memory);
1255 #endif
1256 #endif
1257 #endif
1258 }
1259
1260 /**
1261 * @internal Read information about a user defined type from the HDF5
1262 * file, and stash it in the group's list of types.
1263 *
1264 * @param grp Pointer to group info struct.
1265 * @param hdf_typeid HDF5 type ID.
1266 * @param type_name Pointer that gets the type name.
1267 *
1268 * @return ::NC_NOERR No error.
1269 * @return ::NC_EBADID Bad ncid.
1270 * @return ::NC_EHDFERR HDF5 returned error.
1271 * @return ::NC_EBADTYPID Type not found.
1272 * @author Ed Hartnett
1273 */
1274 static int
read_type(NC_GRP_INFO_T * grp,hid_t hdf_typeid,char * type_name)1275 read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name)
1276 {
1277 NC_TYPE_INFO_T *type;
1278 H5T_class_t class;
1279 hid_t native_typeid;
1280 size_t type_size;
1281 int retval = NC_NOERR;
1282 int nmembers;
1283
1284 assert(grp && type_name);
1285
1286 LOG((4, "%s: type_name %s grp->hdr.name %s", __func__, type_name,
1287 grp->hdr.name));
1288
1289 /* What is the native type for this platform? */
1290 if ((native_typeid = H5Tget_native_type(hdf_typeid, H5T_DIR_DEFAULT)) < 0)
1291 return NC_EHDFERR;
1292
1293 /* What is the size of this type on this platform. */
1294 if (!(type_size = H5Tget_size(native_typeid)))
1295 return NC_EHDFERR;
1296 LOG((5, "type_size %d", type_size));
1297
1298 /* Add to the list for this new type, and get a local pointer to it. */
1299 if ((retval = nc4_type_list_add(grp, type_size, type_name, &type)))
1300 return retval;
1301
1302 /* Remember common info about this type. */
1303 type->committed = NC_TRUE;
1304 type->hdf_typeid = hdf_typeid;
1305 H5Iinc_ref(type->hdf_typeid); /* Increment number of objects using ID */
1306 type->native_hdf_typeid = native_typeid;
1307
1308 /* What is the class of this type, compound, vlen, etc. */
1309 if ((class = H5Tget_class(hdf_typeid)) < 0)
1310 return NC_EHDFERR;
1311 switch (class)
1312 {
1313 case H5T_STRING:
1314 type->nc_type_class = NC_STRING;
1315 break;
1316
1317 case H5T_COMPOUND:
1318 {
1319 int nmembers;
1320 unsigned int m;
1321 char* member_name = NULL;
1322 #ifdef JNA
1323 char jna[1001];
1324 #endif
1325
1326 type->nc_type_class = NC_COMPOUND;
1327
1328 if ((nmembers = H5Tget_nmembers(hdf_typeid)) < 0)
1329 return NC_EHDFERR;
1330 LOG((5, "compound type has %d members", nmembers));
1331 type->u.c.field = nclistnew();
1332 nclistsetalloc(type->u.c.field,nmembers);
1333
1334 for (m = 0; m < nmembers; m++)
1335 {
1336 hid_t member_hdf_typeid;
1337 hid_t member_native_typeid;
1338 size_t member_offset;
1339 H5T_class_t mem_class;
1340 nc_type member_xtype;
1341
1342 /* Get the typeid and native typeid of this member of the
1343 * compound type. */
1344 if ((member_hdf_typeid = H5Tget_member_type(type->native_hdf_typeid, m)) < 0)
1345 return NC_EHDFERR;
1346
1347 if ((member_native_typeid = H5Tget_native_type(member_hdf_typeid,
1348 H5T_DIR_DEFAULT)) < 0)
1349 return NC_EHDFERR;
1350
1351 /* Get the name of the member.*/
1352 member_name = H5Tget_member_name(type->native_hdf_typeid, m);
1353 if (!member_name || strlen(member_name) > NC_MAX_NAME) {
1354 retval = NC_EBADNAME;
1355 break;
1356 }
1357 #ifdef JNA
1358 else {
1359 strncpy(jna,member_name,1000);
1360 member_name = jna;
1361 }
1362 #endif
1363
1364 /* Offset in bytes on *this* platform. */
1365 member_offset = H5Tget_member_offset(type->native_hdf_typeid, m);
1366
1367 /* Get dimensional data if this member is an array of something. */
1368 if ((mem_class = H5Tget_class(member_hdf_typeid)) < 0)
1369 return NC_EHDFERR;
1370 if (mem_class == H5T_ARRAY)
1371 {
1372 int ndims, dim_size[NC_MAX_VAR_DIMS];
1373 hsize_t dims[NC_MAX_VAR_DIMS];
1374 int d;
1375
1376 if ((ndims = H5Tget_array_ndims(member_hdf_typeid)) < 0) {
1377 retval = NC_EHDFERR;
1378 break;
1379 }
1380 if (H5Tget_array_dims(member_hdf_typeid, dims, NULL) != ndims) {
1381 retval = NC_EHDFERR;
1382 break;
1383 }
1384 for (d = 0; d < ndims; d++)
1385 dim_size[d] = dims[d];
1386
1387 /* What is the netCDF typeid of this member? */
1388 if ((retval = get_netcdf_type(grp->nc4_info, H5Tget_super(member_hdf_typeid),
1389 &member_xtype)))
1390 break;
1391
1392 /* Add this member to our list of fields in this compound type. */
1393 if ((retval = nc4_field_list_add(type, member_name,
1394 member_offset, H5Tget_super(member_hdf_typeid),
1395 H5Tget_super(member_native_typeid),
1396 member_xtype, ndims, dim_size)))
1397 break;
1398 }
1399 else
1400 {
1401 /* What is the netCDF typeid of this member? */
1402 if ((retval = get_netcdf_type(grp->nc4_info, member_native_typeid,
1403 &member_xtype)))
1404 break;
1405
1406 /* Add this member to our list of fields in this compound type. */
1407 if ((retval = nc4_field_list_add(type, member_name,
1408 member_offset, member_hdf_typeid, member_native_typeid,
1409 member_xtype, 0, NULL)))
1410 break;
1411 }
1412
1413 hdf5free(member_name);
1414 member_name = NULL;
1415 }
1416 hdf5free(member_name);
1417 member_name = NULL;
1418 if(retval) /* error exit from loop */
1419 return retval;
1420 }
1421 break;
1422
1423 case H5T_VLEN:
1424 {
1425 htri_t ret;
1426
1427 /* For conveninence we allow user to pass vlens of strings
1428 * with null terminated strings. This means strings are
1429 * treated slightly differently by the API, although they are
1430 * really just VLENs of characters. */
1431 if ((ret = H5Tis_variable_str(hdf_typeid)) < 0)
1432 return NC_EHDFERR;
1433 if (ret)
1434 type->nc_type_class = NC_STRING;
1435 else
1436 {
1437 hid_t base_hdf_typeid;
1438 nc_type base_nc_type = NC_NAT;
1439
1440 type->nc_type_class = NC_VLEN;
1441
1442 /* Find the base type of this vlen (i.e. what is this a
1443 * vlen of?) */
1444 if (!(base_hdf_typeid = H5Tget_super(native_typeid)))
1445 return NC_EHDFERR;
1446
1447 /* What size is this type? */
1448 if (!(type_size = H5Tget_size(base_hdf_typeid)))
1449 return NC_EHDFERR;
1450
1451 /* What is the netcdf corresponding type. */
1452 if ((retval = get_netcdf_type(grp->nc4_info, base_hdf_typeid,
1453 &base_nc_type)))
1454 return retval;
1455 LOG((5, "base_hdf_typeid 0x%x type_size %d base_nc_type %d",
1456 base_hdf_typeid, type_size, base_nc_type));
1457
1458 /* Remember the base types for this vlen */
1459 type->u.v.base_nc_typeid = base_nc_type;
1460 type->u.v.base_hdf_typeid = base_hdf_typeid;
1461 }
1462 }
1463 break;
1464
1465 case H5T_OPAQUE:
1466 type->nc_type_class = NC_OPAQUE;
1467 break;
1468
1469 case H5T_ENUM:
1470 {
1471 hid_t base_hdf_typeid;
1472 nc_type base_nc_type = NC_NAT;
1473 void *value;
1474 int i;
1475 char *member_name = NULL;
1476 #ifdef JNA
1477 char jna[1001];
1478 #endif
1479
1480 type->nc_type_class = NC_ENUM;
1481
1482 /* Find the base type of this enum (i.e. what is this a
1483 * enum of?) */
1484 if (!(base_hdf_typeid = H5Tget_super(hdf_typeid)))
1485 return NC_EHDFERR;
1486 /* What size is this type? */
1487 if (!(type_size = H5Tget_size(base_hdf_typeid)))
1488 return NC_EHDFERR;
1489 /* What is the netcdf corresponding type. */
1490 if ((retval = get_netcdf_type(grp->nc4_info, base_hdf_typeid,
1491 &base_nc_type)))
1492 return retval;
1493 LOG((5, "base_hdf_typeid 0x%x type_size %d base_nc_type %d",
1494 base_hdf_typeid, type_size, base_nc_type));
1495
1496 /* Remember the base types for this enum */
1497 type->u.e.base_nc_typeid = base_nc_type;
1498 type->u.e.base_hdf_typeid = base_hdf_typeid;
1499
1500 /* Find out how many member are in the enum. */
1501 if ((nmembers = H5Tget_nmembers(hdf_typeid)) < 0)
1502 return NC_EHDFERR;
1503 type->u.e.enum_member = nclistnew();
1504 nclistsetalloc(type->u.e.enum_member,nmembers);
1505
1506 /* Allocate space for one value. */
1507 if (!(value = calloc(1, type_size)))
1508 return NC_ENOMEM;
1509
1510 /* Read each name and value defined in the enum. */
1511 for (i = 0; i < nmembers; i++)
1512 {
1513 /* Get the name and value from HDF5. */
1514 if (!(member_name = H5Tget_member_name(hdf_typeid, i)))
1515 {
1516 retval = NC_EHDFERR;
1517 break;
1518 }
1519 #ifdef JNA
1520 strncpy(jna,member_name,1000);
1521 member_name = jna;
1522 #endif
1523
1524 if (strlen(member_name) > NC_MAX_NAME)
1525 {
1526 retval = NC_EBADNAME;
1527 break;
1528 }
1529 if (H5Tget_member_value(hdf_typeid, i, value) < 0)
1530 {
1531 retval = NC_EHDFERR;
1532 break;
1533 }
1534
1535 /* Insert new field into this type's list of fields. */
1536 if ((retval = nc4_enum_member_add(type, type->size,
1537 member_name, value)))
1538 {
1539 break;
1540 }
1541
1542 hdf5free(member_name);
1543 member_name = NULL;
1544 }
1545 hdf5free(member_name);
1546 member_name = NULL;
1547 if(value) free(value);
1548 if(retval) /* error exit from loop */
1549 return retval;
1550 }
1551 break;
1552
1553 default:
1554 LOG((0, "unknown class"));
1555 return NC_EBADCLASS;
1556 }
1557 return retval;
1558 }
1559
1560 /**
1561 * @internal Callback function for reading attributes. This is used
1562 * for both global and variable attributes.
1563 *
1564 * @param loc_id HDF5 attribute ID.
1565 * @param att_name Name of the attrigute.
1566 * @param ainfo HDF5 info struct for attribute.
1567 * @param att_data Pointer to an att_iter_info struct, which contains
1568 * pointers to the NC_GRP_INFO_T and (for variable attributes) the
1569 * NC_VAR_INFO_T. For global atts the var pointer is NULL.
1570 *
1571 * @return ::NC_NOERR No error. Iteration continues.
1572 * @return ::-1 Error. Stop iteration.
1573 * @author Ed Hartnett
1574 */
1575 static herr_t
att_read_callbk(hid_t loc_id,const char * att_name,const H5A_info_t * ainfo,void * att_data)1576 att_read_callbk(hid_t loc_id, const char *att_name, const H5A_info_t *ainfo,
1577 void *att_data)
1578 {
1579
1580 hid_t attid = 0;
1581 NC_ATT_INFO_T *att;
1582 NCindex *list;
1583 att_iter_info *att_info = (att_iter_info *)att_data;
1584 int retval = NC_NOERR;
1585
1586 /* Determin what list is being added to. */
1587 list = att_info->var ? att_info->var->att : att_info->grp->att;
1588
1589 /* This may be an attribute telling us that strict netcdf-3 rules
1590 * are in effect. If so, we will make note of the fact, but not add
1591 * this attribute to the metadata. It's not a user attribute, but
1592 * an internal netcdf-4 one. */
1593 if (!strcmp(att_name, NC3_STRICT_ATT_NAME))
1594 {
1595 /* Only relevant for groups, not vars. */
1596 if (!att_info->var)
1597 att_info->grp->nc4_info->cmode |= NC_CLASSIC_MODEL;
1598 return NC_NOERR;
1599 }
1600
1601 /* Should we ignore this attribute? */
1602 if (NC_findreserved(att_name))
1603 return NC_NOERR;
1604
1605 /* Add to the end of the list of atts for this var. */
1606 if ((retval = nc4_att_list_add(list, att_name, &att)))
1607 BAIL(-1);
1608
1609 /* Allocate storage for the HDF5 specific att info. */
1610 if (!(att->format_att_info = calloc(1, sizeof(NC_HDF5_ATT_INFO_T))))
1611 BAIL(-1);
1612
1613 /* Open the att by name. */
1614 if ((attid = H5Aopen(loc_id, att_name, H5P_DEFAULT)) < 0)
1615 BAIL(-1);
1616 LOG((4, "%s:: att_name %s", __func__, att_name));
1617
1618 /* Read the rest of the info about the att,
1619 * including its values. */
1620 if ((retval = read_hdf5_att(att_info->grp, attid, att)))
1621 BAIL(retval);
1622
1623 if (att)
1624 att->created = NC_TRUE;
1625
1626 exit:
1627 if (retval == NC_EBADTYPID)
1628 {
1629 /* NC_EBADTYPID will be normally converted to NC_NOERR so that
1630 the parent iterator does not fail. */
1631 retval = nc4_att_list_del(list, att);
1632 att = NULL;
1633 }
1634 if (attid > 0 && H5Aclose(attid) < 0)
1635 retval = -1;
1636
1637 /* Since this is a HDF5 iterator callback, return -1 for any error
1638 * to stop iteration. */
1639 if (retval)
1640 retval = -1;
1641 return retval;
1642 }
1643
1644 /**
1645 * @internal This function reads all the attributes of a variable or
1646 * the global attributes of a group.
1647 *
1648 * @param grp Pointer to the group info.
1649 * @param var Pointer to the var info. NULL for global att reads.
1650 *
1651 * @return ::NC_NOERR No error.
1652 * @return ::NC_EATTMETA Some error occured reading attributes.
1653 * @author Ed Hartnett
1654 */
1655 int
nc4_read_atts(NC_GRP_INFO_T * grp,NC_VAR_INFO_T * var)1656 nc4_read_atts(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
1657 {
1658 att_iter_info att_info; /* Custom iteration information */
1659 hid_t locid; /* HDF5 location to read atts from. */
1660
1661 /* Check inputs. */
1662 assert(grp);
1663
1664 /* Assign var and grp in struct. (var may be NULL). */
1665 att_info.var = var;
1666 att_info.grp = grp;
1667
1668 /* Determine where to read from in the HDF5 file. */
1669 locid = var ? var->hdf_datasetid :
1670 ((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid;
1671
1672 /* Now read all the attributes at this location, ignoring special
1673 * netCDF hidden attributes. */
1674 if ((H5Aiterate2(locid, H5_INDEX_CRT_ORDER, H5_ITER_INC, NULL,
1675 att_read_callbk, &att_info)) < 0)
1676 return NC_EATTMETA;
1677
1678 /* Remember that we have read the atts for this var or group. */
1679 if (var)
1680 var->atts_not_read = 0;
1681 else
1682 grp->atts_not_read = 0;
1683
1684 return NC_NOERR;
1685 }
1686
1687 /**
1688 * @internal This function is called by read_dataset when a dimension
1689 * scale dataset is encountered. It reads in the dimension data
1690 * (creating a new NC_DIM_INFO_T object), and also checks to see if
1691 * this is a dimension without a variable - that is, a coordinate
1692 * dimension which does not have any coordinate data.
1693 *
1694 * @param grp Pointer to group info struct.
1695 * @param datasetid The HDF5 dataset ID.
1696 * @param obj_name
1697 * @param statbuf
1698 * @param scale_size Size of dimension scale.
1699 * @param max_scale_size Maximum size of dim scale.
1700 * @param dim Pointer to pointer that gets new dim info struct.
1701 *
1702 * @returns ::NC_NOERR No error.
1703 * @return ::NC_EHDFERR HDF5 returned error.
1704 * @author Ed Hartnett
1705 */
1706 static int
read_scale(NC_GRP_INFO_T * grp,hid_t datasetid,const char * obj_name,const H5G_stat_t * statbuf,hsize_t scale_size,hsize_t max_scale_size,NC_DIM_INFO_T ** dim)1707 read_scale(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name,
1708 const H5G_stat_t *statbuf, hsize_t scale_size, hsize_t max_scale_size,
1709 NC_DIM_INFO_T **dim)
1710 {
1711 NC_DIM_INFO_T *new_dim; /* Dimension added to group */
1712 NC_HDF5_DIM_INFO_T *new_hdf5_dim; /* HDF5-specific dim info. */
1713 char dimscale_name_att[NC_MAX_NAME + 1] = ""; /* Dimscale name, for checking if dim without var */
1714 htri_t attr_exists = -1; /* Flag indicating hidden attribute exists */
1715 hid_t attid = -1; /* ID of hidden attribute (to store dim ID) */
1716 int dimscale_created = 0; /* Remember if a dimension was created (for error recovery) */
1717 short initial_next_dimid = grp->nc4_info->next_dimid;/* Retain for error recovery */
1718 int retval;
1719 size_t len = 0;
1720 int too_long = NC_FALSE;
1721 int assigned_id = -1;
1722
1723 /* Does this dataset have a hidden attribute that tells us its
1724 * dimid? If so, read it. */
1725 if ((attr_exists = H5Aexists(datasetid, NC_DIMID_ATT_NAME)) < 0)
1726 BAIL(NC_EHDFERR);
1727 if (attr_exists)
1728 {
1729 if ((attid = H5Aopen_name(datasetid, NC_DIMID_ATT_NAME)) < 0)
1730 BAIL(NC_EHDFERR);
1731
1732 if (H5Aread(attid, H5T_NATIVE_INT, &assigned_id) < 0)
1733 BAIL(NC_EHDFERR);
1734
1735 /* Check if scale's dimid should impact the group's next dimid */
1736 if (assigned_id >= grp->nc4_info->next_dimid)
1737 grp->nc4_info->next_dimid = assigned_id + 1;
1738 }
1739
1740 if (SIZEOF_SIZE_T < 8 && scale_size > NC_MAX_UINT)
1741 {
1742 len = NC_MAX_UINT;
1743 too_long = NC_TRUE;
1744 }
1745 else
1746 len = scale_size;
1747
1748 /* Create the dimension for this scale. */
1749 if ((retval = nc4_dim_list_add(grp, obj_name, len, assigned_id, &new_dim)))
1750 BAIL(retval);
1751
1752 /* Create struct for HDF5-specific dim info. */
1753 if (!(new_dim->format_dim_info = calloc(1, sizeof(NC_HDF5_DIM_INFO_T))))
1754 BAIL(NC_ENOMEM);
1755 new_hdf5_dim = (NC_HDF5_DIM_INFO_T *)new_dim->format_dim_info;
1756
1757 new_dim->too_long = too_long;
1758
1759 dimscale_created++;
1760
1761 new_hdf5_dim->hdf5_objid.fileno[0] = statbuf->fileno[0];
1762 new_hdf5_dim->hdf5_objid.fileno[1] = statbuf->fileno[1];
1763 new_hdf5_dim->hdf5_objid.objno[0] = statbuf->objno[0];
1764 new_hdf5_dim->hdf5_objid.objno[1] = statbuf->objno[1];
1765
1766 /* If the dimscale has an unlimited dimension, then this dimension
1767 * is unlimited. */
1768 if (max_scale_size == H5S_UNLIMITED)
1769 new_dim->unlimited = NC_TRUE;
1770
1771 /* If the scale name is set to DIM_WITHOUT_VARIABLE, then this is a
1772 * dimension, but not a variable. (If get_scale_name returns an
1773 * error, just move on, there's no NAME.) */
1774 if (H5DSget_scale_name(datasetid, dimscale_name_att, NC_MAX_NAME) >= 0)
1775 {
1776 if (!strncmp(dimscale_name_att, DIM_WITHOUT_VARIABLE,
1777 strlen(DIM_WITHOUT_VARIABLE)))
1778 {
1779 if (new_dim->unlimited)
1780 {
1781 size_t len = 0, *lenp = &len;
1782
1783 if ((retval = nc4_find_dim_len(grp, new_dim->hdr.id, &lenp)))
1784 BAIL(retval);
1785 new_dim->len = *lenp;
1786 }
1787
1788 /* Hold open the dataset, since the dimension doesn't have a
1789 * coordinate variable */
1790 new_hdf5_dim->hdf_dimscaleid = datasetid;
1791 H5Iinc_ref(new_hdf5_dim->hdf_dimscaleid); /* Increment number of objects using ID */
1792 }
1793 }
1794
1795 /* Set the dimension created */
1796 *dim = new_dim;
1797
1798 exit:
1799 /* Close the hidden attribute, if it was opened (error, or no
1800 * error) */
1801 if (attid > 0 && H5Aclose(attid) < 0)
1802 BAIL2(NC_EHDFERR);
1803
1804 /* On error, undo any dimscale creation */
1805 if (retval < 0 && dimscale_created)
1806 {
1807 /* free the dimension */
1808 if ((retval = nc4_dim_list_del(grp, new_dim)))
1809 BAIL2(retval);
1810
1811 /* Reset the group's information */
1812 grp->nc4_info->next_dimid = initial_next_dimid;
1813 }
1814
1815 return retval;
1816 }
1817
1818 /**
1819 * @internal This function is called when nc4_rec_read_metadata
1820 * encounters an HDF5 dataset when reading a file.
1821 *
1822 * @param grp Pointer to group info struct.
1823 * @param datasetid HDF5 dataset ID.
1824 * @param obj_name Object name.
1825 * @param statbuf HDF5 status buffer.
1826 *
1827 * @return ::NC_NOERR No error.
1828 * @return ::NC_EBADID Bad ncid.
1829 * @return ::NC_EHDFERR HDF5 returned error.
1830 * @author Ed Hartnett
1831 */
1832 static int
read_dataset(NC_GRP_INFO_T * grp,hid_t datasetid,const char * obj_name,const H5G_stat_t * statbuf)1833 read_dataset(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name,
1834 const H5G_stat_t *statbuf)
1835 {
1836 NC_DIM_INFO_T *dim = NULL; /* Dimension created for scales */
1837 NC_HDF5_DIM_INFO_T *hdf5_dim;
1838 hid_t spaceid = 0;
1839 int ndims;
1840 htri_t is_scale;
1841 int retval = NC_NOERR;
1842
1843 /* Get the dimension information for this dataset. */
1844 if ((spaceid = H5Dget_space(datasetid)) < 0)
1845 BAIL(NC_EHDFERR);
1846 if ((ndims = H5Sget_simple_extent_ndims(spaceid)) < 0)
1847 BAIL(NC_EHDFERR);
1848
1849 /* Is this a dimscale? */
1850 if ((is_scale = H5DSis_scale(datasetid)) < 0)
1851 BAIL(NC_EHDFERR);
1852 if (is_scale)
1853 {
1854 hsize_t dims[H5S_MAX_RANK];
1855 hsize_t max_dims[H5S_MAX_RANK];
1856
1857 /* Query the scale's size & max. size */
1858 if (H5Sget_simple_extent_dims(spaceid, dims, max_dims) < 0)
1859 BAIL(NC_EHDFERR);
1860
1861 /* Read the scale information. */
1862 if ((retval = read_scale(grp, datasetid, obj_name, statbuf, dims[0],
1863 max_dims[0], &dim)))
1864 BAIL(retval);
1865 hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1866 }
1867
1868 /* Add a var to the linked list, and get its metadata,
1869 * unless this is one of those funny dimscales that are a
1870 * dimension in netCDF but not a variable. (Spooky!) */
1871 if (!dim || (dim && !hdf5_dim->hdf_dimscaleid))
1872 if ((retval = read_var(grp, datasetid, obj_name, ndims, dim)))
1873 BAIL(retval);
1874
1875 exit:
1876 if (spaceid && H5Sclose(spaceid) <0)
1877 BAIL2(retval);
1878
1879 return retval;
1880 }
1881
1882 /**
1883 * @internal Add callback function to list.
1884 *
1885 * @param udata - the callback state
1886 * @param oinfo The object info.
1887 *
1888 * @return ::NC_NOERR No error.
1889 * @return ::NC_ENOMEM Out of memory.
1890 * @author Ed Hartnett
1891 */
1892 static int
nc4_rec_read_metadata_cb_list_add(NC4_rec_read_metadata_ud_t * udata,const NC4_rec_read_metadata_obj_info_t * oinfo)1893 nc4_rec_read_metadata_cb_list_add(NC4_rec_read_metadata_ud_t* udata,
1894 const NC4_rec_read_metadata_obj_info_t *oinfo)
1895 {
1896 NC4_rec_read_metadata_obj_info_t *new_oinfo; /* Pointer to info for object */
1897
1898 /* Allocate memory for the object's info */
1899 if (!(new_oinfo = calloc(1, sizeof(*new_oinfo))))
1900 return NC_ENOMEM;
1901
1902 /* Make a copy of the object's info */
1903 memcpy(new_oinfo, oinfo, sizeof(*oinfo));
1904
1905 nclistpush(udata->grps,new_oinfo);
1906 return (NC_NOERR);
1907 }
1908
1909 /**
1910 * @internal Callback function called from nc4_rec_read_metadata().
1911 *
1912 * @param grpid HDF5 group ID.
1913 * @param name Name of object.
1914 * @param info Info struct for object.
1915 * @param _op_data Pointer to data.
1916 *
1917 * @return ::NC_NOERR No error.
1918 * @return H5_ITER_ERROR HDF5 error.
1919 * @author Ed Hartnett
1920 */
1921 static int
nc4_rec_read_metadata_cb(hid_t grpid,const char * name,const H5L_info_t * info,void * _op_data)1922 nc4_rec_read_metadata_cb(hid_t grpid, const char *name, const H5L_info_t *info,
1923 void *_op_data)
1924 {
1925 /* Pointer to user data for callback */
1926 NC4_rec_read_metadata_ud_t *udata = (NC4_rec_read_metadata_ud_t *)_op_data;
1927 NC4_rec_read_metadata_obj_info_t oinfo; /* Pointer to info for object */
1928 int retval = H5_ITER_CONT;
1929
1930 /* Open this critter. */
1931 if ((oinfo.oid = H5Oopen(grpid, name, H5P_DEFAULT)) < 0)
1932 BAIL(H5_ITER_ERROR);
1933
1934 /* Get info about the object.*/
1935 if (H5Gget_objinfo(oinfo.oid, ".", 1, &oinfo.statbuf) < 0)
1936 BAIL(H5_ITER_ERROR);
1937
1938 strncpy(oinfo.oname, name, NC_MAX_NAME);
1939
1940 /* Add object to list, for later */
1941 switch(oinfo.statbuf.type)
1942 {
1943 case H5G_GROUP:
1944 LOG((3, "found group %s", oinfo.oname));
1945
1946 /* Defer descending into child group immediately, so that the
1947 * types in the current group can be processed and be ready for
1948 * use by vars in the child group(s). */
1949 if (nc4_rec_read_metadata_cb_list_add(udata, &oinfo))
1950 BAIL(H5_ITER_ERROR);
1951 break;
1952
1953 case H5G_DATASET:
1954 LOG((3, "found dataset %s", oinfo.oname));
1955
1956 /* Learn all about this dataset, which may be a dimscale
1957 * (i.e. dimension metadata), or real data. */
1958 if ((retval = read_dataset(udata->grp, oinfo.oid, oinfo.oname,
1959 &oinfo.statbuf)))
1960 {
1961 /* Allow NC_EBADTYPID to transparently skip over datasets
1962 * which have a datatype that netCDF-4 doesn't undertand
1963 * (currently), but break out of iteration for other
1964 * errors. */
1965 if (retval != NC_EBADTYPID)
1966 BAIL(H5_ITER_ERROR);
1967 else
1968 retval = H5_ITER_CONT;
1969 }
1970
1971 /* Close the object */
1972 if (H5Oclose(oinfo.oid) < 0)
1973 BAIL(H5_ITER_ERROR);
1974 break;
1975
1976 case H5G_TYPE:
1977 LOG((3, "found datatype %s", oinfo.oname));
1978
1979 /* Process the named datatype */
1980 if (read_type(udata->grp, oinfo.oid, oinfo.oname))
1981 BAIL(H5_ITER_ERROR);
1982
1983 /* Close the object */
1984 if (H5Oclose(oinfo.oid) < 0)
1985 BAIL(H5_ITER_ERROR);
1986 break;
1987
1988 default:
1989 LOG((0, "Unknown object class %d in %s!", oinfo.statbuf.type, __func__));
1990 BAIL(H5_ITER_ERROR);
1991 }
1992
1993 exit:
1994 if (retval)
1995 {
1996 if (oinfo.oid > 0 && H5Oclose(oinfo.oid) < 0)
1997 BAIL2(H5_ITER_ERROR);
1998 }
1999
2000 return (retval);
2001 }
2002
2003 /**
2004 * @internal This is the main function to recursively read all the
2005 * metadata for the file. The links in the 'grp' are iterated over
2006 * and added to the file's metadata information. Note that child
2007 * groups are not immediately processed, but are deferred until all
2008 * the other links in the group are handled (so that vars in the child
2009 * groups are guaranteed to have types that they use in a parent group
2010 * in place).
2011 *
2012 * @param grp Pointer to a group.
2013 *
2014 * @return ::NC_NOERR No error.
2015 * @author Ed Hartnett
2016 */
2017 static int
nc4_rec_read_metadata(NC_GRP_INFO_T * grp)2018 nc4_rec_read_metadata(NC_GRP_INFO_T *grp)
2019 {
2020 NC_HDF5_GRP_INFO_T *hdf5_grp;
2021 NC4_rec_read_metadata_ud_t udata; /* User data for iteration */
2022 NC4_rec_read_metadata_obj_info_t *oinfo; /* Pointer to info for object */
2023 hsize_t idx = 0;
2024 hid_t pid = 0;
2025 unsigned crt_order_flags = 0;
2026 H5_index_t iter_index;
2027 int i, retval = NC_NOERR; /* everything worked! */
2028
2029 assert(grp && grp->hdr.name && grp->format_grp_info);
2030 LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
2031
2032 /* Get HDF5-specific group info. */
2033 hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
2034
2035 /* Portably initialize user data for later */
2036 memset(&udata, 0, sizeof(udata));
2037
2038 /* Open this HDF5 group and retain its grpid. It will remain open
2039 * with HDF5 until this file is nc_closed. */
2040 if (!hdf5_grp->hdf_grpid)
2041 {
2042 if (grp->parent)
2043 {
2044 NC_HDF5_GRP_INFO_T *parent_hdf5_grp;
2045 parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info;
2046
2047 if ((hdf5_grp->hdf_grpid = H5Gopen2(parent_hdf5_grp->hdf_grpid,
2048 grp->hdr.name, H5P_DEFAULT)) < 0)
2049 BAIL(NC_EHDFERR);
2050 }
2051 else
2052 {
2053 NC_HDF5_FILE_INFO_T *hdf5_info;
2054 hdf5_info = (NC_HDF5_FILE_INFO_T *)grp->nc4_info->format_file_info;
2055
2056 if ((hdf5_grp->hdf_grpid = H5Gopen2(hdf5_info->hdfid, "/",
2057 H5P_DEFAULT)) < 0)
2058 BAIL(NC_EHDFERR);
2059 }
2060 }
2061 assert(hdf5_grp->hdf_grpid > 0);
2062
2063 /* Get the group creation flags, to check for creation ordering */
2064 pid = H5Gget_create_plist(hdf5_grp->hdf_grpid);
2065 H5Pget_link_creation_order(pid, &crt_order_flags);
2066 if (H5Pclose(pid) < 0)
2067 BAIL(NC_EHDFERR);
2068
2069 /* Set the iteration index to use */
2070 if (crt_order_flags & H5P_CRT_ORDER_TRACKED)
2071 iter_index = H5_INDEX_CRT_ORDER;
2072 else
2073 {
2074 NC_FILE_INFO_T *h5 = grp->nc4_info;
2075
2076 /* Without creation ordering, file must be read-only. */
2077 if (!h5->no_write)
2078 BAIL(NC_ECANTWRITE);
2079
2080 iter_index = H5_INDEX_NAME;
2081 }
2082
2083 /* Set user data for iteration */
2084 udata.grp = grp;
2085 udata.grps = nclistnew();
2086
2087 /* Iterate over links in this group, building lists for the types,
2088 * datasets and groups encountered. */
2089 if (H5Literate(hdf5_grp->hdf_grpid, iter_index, H5_ITER_INC, &idx,
2090 nc4_rec_read_metadata_cb, (void *)&udata) < 0)
2091 BAIL(NC_EHDFERR);
2092
2093 /* Process the child groups found. (Deferred until now, so that the
2094 * types in the current group get processed and are available for
2095 * vars in the child group(s).) */
2096 for (i = 0; i < nclistlength(udata.grps); i++)
2097 {
2098 NC_GRP_INFO_T *child_grp;
2099 oinfo = (NC4_rec_read_metadata_obj_info_t*)nclistget(udata.grps, i);
2100
2101 /* Add group to file's hierarchy */
2102 if ((retval = nc4_grp_list_add(grp->nc4_info, grp, oinfo->oname,
2103 &child_grp)))
2104 BAIL(retval);
2105
2106 /* Allocate storage for HDF5-specific group info. */
2107 if (!(child_grp->format_grp_info = calloc(1, sizeof(NC_HDF5_GRP_INFO_T))))
2108 return NC_ENOMEM;
2109
2110 /* Recursively read the child group's metadata */
2111 if ((retval = nc4_rec_read_metadata(child_grp)))
2112 BAIL(retval);
2113
2114 /* Close the object */
2115 if (H5Oclose(oinfo->oid) < 0)
2116 BAIL(NC_EHDFERR);
2117 }
2118
2119 /* Defer the reading of global atts until someone asks for one. */
2120 grp->atts_not_read = 1;
2121
2122 /* when exiting define mode, mark all variable written */
2123 for (i = 0; i < ncindexsize(grp->vars); i++)
2124 {
2125 NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i);
2126 assert(var);
2127 var->written_to = NC_TRUE;
2128 }
2129
2130 exit:
2131 /* Clean up local information, if anything remains */
2132 for (i = 0; i < nclistlength(udata.grps); i++)
2133 {
2134 oinfo = (NC4_rec_read_metadata_obj_info_t*)nclistget(udata.grps,i);
2135 if (retval)
2136 {
2137 /* Close the object */
2138 if (H5Oclose(oinfo->oid) < 0)
2139 BAIL2(NC_EHDFERR);
2140 }
2141 free(oinfo);
2142 }
2143 nclistfree(udata.grps);
2144 udata.grps = NULL;
2145
2146 return retval;
2147 }
2148