1 /******************************************************************************
2 *
3 * Project: HDF5 read Driver
4 * Author: Even Rouault <even.rouault at spatialys.com>
5 *
6 ******************************************************************************
7 * Copyright (c) 2019, Even Rouault <even.rouault at spatialys.com>
8 *
9 * Permission is hereby granted, free of charge, to any person obtaining a
10 * copy of this software and associated documentation files (the "Software"),
11 * to deal in the Software without restriction, including without limitation
12 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 * and/or sell copies of the Software, and to permit persons to whom the
14 * Software is furnished to do so, subject to the following conditions:
15 *
16 * The above copyright notice and this permission notice shall be included
17 * in all copies or substantial portions of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 * DEALINGS IN THE SOFTWARE.
26 ****************************************************************************/
27
28 #include "hdf5dataset.h"
29
30 #include <algorithm>
31 #include <set>
32 #include <utility>
33
34 namespace GDAL
35 {
36
37 /************************************************************************/
38 /* HDF5Group */
39 /************************************************************************/
40
41 class HDF5Group final: public GDALGroup
42 {
43 std::shared_ptr<HDF5SharedResources> m_poShared;
44 hid_t m_hGroup;
45 std::set<std::pair<unsigned long, unsigned long>> m_oSetParentIds{};
46 mutable std::vector<std::string> m_osListSubGroups{};
47 mutable std::vector<std::string> m_osListArrays{};
48 mutable std::vector<std::shared_ptr<GDALAttribute>> m_oListAttributes{};
49 mutable bool m_bShowAllAttributes = false;
50 mutable bool m_bGotDims = false;
51 mutable std::vector<std::shared_ptr<GDALDimension>> m_cachedDims{};
52
53 static herr_t GetGroupNamesCallback( hid_t hGroup, const char *pszObjName,
54 void* );
55
56 static herr_t GetArrayNamesCallback( hid_t hGroup, const char *pszObjName,
57 void* );
58
59 static herr_t GetAttributesCallback( hid_t hGroup, const char *pszObjName,
60 void* );
61
62 public:
HDF5Group(const std::string & osParentName,const std::string & osName,const std::shared_ptr<HDF5SharedResources> & poShared,const std::set<std::pair<unsigned long,unsigned long>> & oSetParentIds,hid_t hGroup,unsigned long objIds[2])63 HDF5Group(const std::string& osParentName,
64 const std::string& osName,
65 const std::shared_ptr<HDF5SharedResources>& poShared,
66 const std::set<std::pair<unsigned long, unsigned long>>& oSetParentIds,
67 hid_t hGroup,
68 unsigned long objIds[2]):
69 GDALGroup(osParentName, osName),
70 m_poShared(poShared),
71 m_hGroup(hGroup),
72 m_oSetParentIds(oSetParentIds)
73 {
74 m_oSetParentIds.insert(
75 std::pair<unsigned long, unsigned long>(objIds[0], objIds[1]));
76 }
77
~HDF5Group()78 ~HDF5Group()
79 {
80 H5Gclose(m_hGroup);
81 }
82
83 std::vector<std::shared_ptr<GDALDimension>> GetDimensions(CSLConstList papszOptions = nullptr) const override;
84
85 std::vector<std::string> GetGroupNames(CSLConstList papszOptions) const override;
86 std::shared_ptr<GDALGroup> OpenGroup(const std::string& osName, CSLConstList) const override;
87
88 std::vector<std::string> GetMDArrayNames(CSLConstList papszOptions) const override;
89 std::shared_ptr<GDALMDArray> OpenMDArray(const std::string& osName,
90 CSLConstList papszOptions) const override;
91
92 std::vector<std::shared_ptr<GDALAttribute>> GetAttributes(CSLConstList papszOptions = nullptr) const override;
93 };
94
95 /************************************************************************/
96 /* HDF5Dimension */
97 /************************************************************************/
98
99 class HDF5Dimension final: public GDALDimension
100 {
101 std::string m_osGroupFullname;
102 std::shared_ptr<HDF5SharedResources> m_poShared;
103
104 public:
HDF5Dimension(const std::string & osParentName,const std::string & osName,const std::string & osType,const std::string & osDirection,GUInt64 nSize,const std::shared_ptr<HDF5SharedResources> & poShared)105 HDF5Dimension(const std::string& osParentName,
106 const std::string& osName,
107 const std::string& osType,
108 const std::string& osDirection,
109 GUInt64 nSize,
110 const std::shared_ptr<HDF5SharedResources>& poShared):
111 GDALDimension(osParentName, osName, osType, osDirection, nSize),
112 m_osGroupFullname(osParentName),
113 m_poShared(poShared)
114 {
115 }
116
117 std::shared_ptr<GDALMDArray> GetIndexingVariable() const override;
118 };
119
120 /************************************************************************/
121 /* BuildDataType() */
122 /************************************************************************/
123
BuildDataType(hid_t hDataType,bool & bHasVLen,bool & bNonNativeDataType,const std::vector<std::pair<std::string,hid_t>> & oTypes)124 static GDALExtendedDataType BuildDataType(hid_t hDataType, bool& bHasVLen, bool& bNonNativeDataType,
125 const std::vector<std::pair<std::string, hid_t>>& oTypes)
126 {
127 const auto klass = H5Tget_class(hDataType);
128 GDALDataType eDT = ::HDF5Dataset::GetDataType(hDataType);
129 if( H5Tequal(H5T_NATIVE_SCHAR, hDataType) )
130 {
131 bNonNativeDataType = true;
132 return GDALExtendedDataType::Create(GDT_Int16);
133 }
134 else if( H5Tequal(H5T_NATIVE_LLONG, hDataType) )
135 {
136 bNonNativeDataType = true;
137 return GDALExtendedDataType::Create(GDT_Float64);
138 }
139 else if( H5Tequal(H5T_NATIVE_ULLONG, hDataType) )
140 {
141 bNonNativeDataType = true;
142 return GDALExtendedDataType::Create(GDT_Float64);
143 }
144 else if( eDT != GDT_Unknown )
145 return GDALExtendedDataType::Create(eDT);
146 else if (klass == H5T_STRING )
147 {
148 if( H5Tis_variable_str(hDataType) )
149 bHasVLen = true;
150 return GDALExtendedDataType::CreateString();
151 }
152 else if (klass == H5T_COMPOUND )
153 {
154 const unsigned nMembers = H5Tget_nmembers(hDataType);
155 std::vector<std::unique_ptr<GDALEDTComponent>> components;
156 size_t nOffset = 0;
157 for(unsigned i = 0; i < nMembers; i++ )
158 {
159 char* pszName = H5Tget_member_name(hDataType, i);
160 if( !pszName )
161 return GDALExtendedDataType::Create(GDT_Unknown);
162 CPLString osCompName(pszName);
163 H5free_memory(pszName);
164 const auto hMemberType = H5Tget_member_type(hDataType, i);
165 if( hMemberType < 0 )
166 return GDALExtendedDataType::Create(GDT_Unknown);
167 const hid_t hNativeMemberType =
168 H5Tget_native_type(hMemberType, H5T_DIR_ASCEND);
169 auto memberDT = BuildDataType(hNativeMemberType, bHasVLen, bNonNativeDataType, oTypes);
170 H5Tclose(hNativeMemberType);
171 H5Tclose(hMemberType);
172 if( memberDT.GetClass() == GEDTC_NUMERIC && memberDT.GetNumericDataType() == GDT_Unknown )
173 return GDALExtendedDataType::Create(GDT_Unknown);
174 if( (nOffset % memberDT.GetSize()) != 0 )
175 nOffset += memberDT.GetSize() - (nOffset % memberDT.GetSize());
176 if( nOffset != H5Tget_member_offset(hDataType, i) )
177 bNonNativeDataType = true;
178 components.emplace_back(std::unique_ptr<GDALEDTComponent>(
179 new GDALEDTComponent(osCompName, nOffset, memberDT)));
180 nOffset += memberDT.GetSize();
181 }
182 if( !components.empty() && (nOffset % components[0]->GetType().GetSize()) != 0 )
183 nOffset += components[0]->GetType().GetSize() - (nOffset % components[0]->GetType().GetSize());
184 if( nOffset != H5Tget_size(hDataType) )
185 bNonNativeDataType = true;
186 std::string osName("unnamed");
187 for( const auto& oPair: oTypes )
188 {
189 const auto hPairNativeType = H5Tget_native_type(oPair.second, H5T_DIR_ASCEND);
190 const auto matches = H5Tequal(hPairNativeType, hDataType);
191 H5Tclose(hPairNativeType);
192 if( matches )
193 {
194 osName = oPair.first;
195 break;
196 }
197 }
198 return GDALExtendedDataType::Create(osName,
199 nOffset,
200 std::move(components));
201 }
202 else if (klass == H5T_ENUM )
203 {
204 const auto hParent = H5Tget_super(hDataType);
205 const hid_t hNativeParent =
206 H5Tget_native_type(hParent, H5T_DIR_ASCEND);
207 auto ret(BuildDataType(hNativeParent, bHasVLen, bNonNativeDataType, oTypes));
208 H5Tclose(hNativeParent);
209 H5Tclose(hParent);
210 return ret;
211 }
212 else
213 {
214 return GDALExtendedDataType::Create(GDT_Unknown);
215 }
216 }
217
218 /************************************************************************/
219 /* GetDataTypesInGroup() */
220 /************************************************************************/
221
GetDataTypesInGroup(hid_t hHDF5,const std::string & osGroupFullName,std::vector<std::pair<std::string,hid_t>> & oTypes)222 static void GetDataTypesInGroup(hid_t hHDF5,
223 const std::string& osGroupFullName,
224 std::vector<std::pair<std::string, hid_t>>& oTypes)
225 {
226 struct Callback
227 {
228 static herr_t f(hid_t hGroup, const char *pszObjName, void* user_data)
229 {
230 std::vector<std::pair<std::string, hid_t>>* poTypes =
231 static_cast<std::vector<std::pair<std::string, hid_t>>*>(user_data);
232 H5G_stat_t oStatbuf;
233
234 if( H5Gget_objinfo(hGroup, pszObjName, FALSE, &oStatbuf) < 0 )
235 return -1;
236
237 if( oStatbuf.type == H5G_TYPE )
238 {
239 poTypes->push_back(std::pair<std::string, hid_t>(
240 pszObjName, H5Topen(hGroup, pszObjName)));
241 }
242
243 return 0;
244 }
245 };
246 H5Giterate(hHDF5, osGroupFullName.c_str(), nullptr,
247 &(Callback::f), &oTypes);
248 }
249
250 /************************************************************************/
251 /* HDF5Array */
252 /************************************************************************/
253
254 class HDF5Array final: public GDALMDArray
255 {
256 std::string m_osGroupFullname;
257 std::shared_ptr<HDF5SharedResources> m_poShared;
258 hid_t m_hArray;
259 hid_t m_hDataSpace;
260 std::vector<std::shared_ptr<GDALDimension>> m_dims{};
261 GDALExtendedDataType m_dt = GDALExtendedDataType::Create(GDT_Unknown);
262 hid_t m_hNativeDT = H5I_INVALID_HID;
263 mutable std::vector<std::shared_ptr<GDALAttribute>> m_oListAttributes{};
264 mutable bool m_bShowAllAttributes = false;
265 bool m_bHasVLenMember = false;
266 bool m_bHasNonNativeDataType = false;
267 mutable std::vector<GByte> m_abyNoData{};
268 mutable std::string m_osUnit{};
269 mutable bool m_bHasDimensionList = false;
270 mutable bool m_bHasDimensionLabels = false;
271 haddr_t m_nOffset;
272
273 HDF5Array(const std::string& osParentName,
274 const std::string& osName,
275 const std::shared_ptr<HDF5SharedResources>& poShared,
276 hid_t hArray,
277 const HDF5Group* poGroup,
278 bool bSkipFullDimensionInstantiation);
279
280 void InstantiateDimensions(const std::string& osParentName,
281 const HDF5Group* poGroup);
282
283 bool ReadSlow(const GUInt64* arrayStartIdx,
284 const size_t* count,
285 const GInt64* arrayStep,
286 const GPtrDiff_t* bufferStride,
287 const GDALExtendedDataType& bufferDataType,
288 void* pDstBuffer) const;
289
290 static herr_t GetAttributesCallback( hid_t hArray, const char *pszObjName,
291 void* );
292
293 protected:
294
295 bool IRead(const GUInt64* arrayStartIdx,
296 const size_t* count,
297 const GInt64* arrayStep,
298 const GPtrDiff_t* bufferStride,
299 const GDALExtendedDataType& bufferDataType,
300 void* pDstBuffer) const override;
301
302 public:
303 ~HDF5Array();
304
Create(const std::string & osParentName,const std::string & osName,const std::shared_ptr<HDF5SharedResources> & poShared,hid_t hArray,const HDF5Group * poGroup,bool bSkipFullDimensionInstantiation)305 static std::shared_ptr<HDF5Array> Create(
306 const std::string& osParentName,
307 const std::string& osName,
308 const std::shared_ptr<HDF5SharedResources>& poShared,
309 hid_t hArray,
310 const HDF5Group* poGroup,
311 bool bSkipFullDimensionInstantiation)
312 {
313 auto ar(std::shared_ptr<HDF5Array>(new HDF5Array(
314 osParentName, osName, poShared, hArray, poGroup, bSkipFullDimensionInstantiation)));
315 if( ar->m_dt.GetClass() == GEDTC_NUMERIC &&
316 ar->m_dt.GetNumericDataType() == GDT_Unknown )
317 {
318 return nullptr;
319 }
320 ar->SetSelf(ar);
321 return ar;
322 }
323
IsWritable() const324 bool IsWritable() const override { return !m_poShared->IsReadOnly(); }
325
GetDimensions() const326 const std::vector<std::shared_ptr<GDALDimension>>& GetDimensions() const override { return m_dims; }
327
GetDataType() const328 const GDALExtendedDataType &GetDataType() const override { return m_dt; }
329
330 std::shared_ptr<GDALAttribute> GetAttribute(const std::string& osName) const override;
331
332 std::vector<std::shared_ptr<GDALAttribute>> GetAttributes(CSLConstList papszOptions = nullptr) const override;
333
GetRawNoDataValue() const334 const void* GetRawNoDataValue() const override
335 {
336 return m_abyNoData.empty() ? nullptr : m_abyNoData.data();
337 }
338
GetUnit() const339 const std::string& GetUnit() const override
340 {
341 return m_osUnit;
342 }
343
GetFileOffset() const344 haddr_t GetFileOffset() const { return m_nOffset; }
345 };
346
347 /************************************************************************/
348 /* HDF5Attribute */
349 /************************************************************************/
350
351 class HDF5Attribute final: public GDALAttribute
352 {
353 std::shared_ptr<HDF5SharedResources> m_poShared;
354 hid_t m_hAttribute;
355 hid_t m_hDataSpace;
356 std::vector<std::shared_ptr<GDALDimension>> m_dims{};
357 GDALExtendedDataType m_dt = GDALExtendedDataType::Create(GDT_Unknown);
358 hid_t m_hNativeDT = H5I_INVALID_HID;
359 size_t m_nElements = 1;
360 bool m_bHasVLenMember = false;
361 bool m_bHasNonNativeDataType = false;
362
HDF5Attribute(const std::string & osGroupFullName,const std::string & osParentName,const std::string & osName,const std::shared_ptr<HDF5SharedResources> & poShared,hid_t hAttribute)363 HDF5Attribute(const std::string& osGroupFullName,
364 const std::string& osParentName,
365 const std::string& osName,
366 const std::shared_ptr<HDF5SharedResources>& poShared,
367 hid_t hAttribute):
368 GDALAbstractMDArray(osParentName, osName),
369 GDALAttribute(osParentName, osName),
370 m_poShared(poShared),
371 m_hAttribute(hAttribute),
372 m_hDataSpace(H5Aget_space(hAttribute))
373 {
374 const int nDims = H5Sget_simple_extent_ndims(m_hDataSpace);
375 std::vector<hsize_t> anDimSizes(nDims);
376 if( nDims )
377 {
378 H5Sget_simple_extent_dims(m_hDataSpace, &anDimSizes[0], nullptr);
379 for( int i = 0; i < nDims; ++i )
380 {
381 m_nElements *= static_cast<size_t>(anDimSizes[i]);
382 if( nDims == 1 && m_nElements == 1 )
383 {
384 // Expose 1-dim of size 1 as scalar
385 break;
386 }
387 m_dims.emplace_back(std::make_shared<GDALDimension>(
388 std::string(),
389 CPLSPrintf("dim%d", i),
390 std::string(),
391 std::string(),
392 anDimSizes[i]));
393 }
394 }
395
396 const hid_t hDataType = H5Aget_type(hAttribute);
397 m_hNativeDT = H5Tget_native_type(hDataType, H5T_DIR_ASCEND);
398 H5Tclose(hDataType);
399
400 std::vector<std::pair<std::string, hid_t>> oTypes;
401 if( !osGroupFullName.empty() &&
402 H5Tget_class(m_hNativeDT) == H5T_COMPOUND )
403 {
404 GetDataTypesInGroup(m_poShared->GetHDF5(), osGroupFullName, oTypes);
405 }
406
407 m_dt = BuildDataType(m_hNativeDT, m_bHasVLenMember, m_bHasNonNativeDataType, oTypes);
408 for( auto& oPair: oTypes )
409 H5Tclose(oPair.second);
410 if( m_dt.GetClass() == GEDTC_NUMERIC &&
411 m_dt.GetNumericDataType() == GDT_Unknown )
412 {
413 CPLDebug("HDF5",
414 "Cannot map data type of %s to a type handled by GDAL",
415 osName.c_str());
416 }
417 }
418
419 protected:
420
421 bool IRead(const GUInt64* arrayStartIdx,
422 const size_t* count,
423 const GInt64* arrayStep,
424 const GPtrDiff_t* bufferStride,
425 const GDALExtendedDataType& bufferDataType,
426 void* pDstBuffer) const override;
427
428 public:
429 ~HDF5Attribute();
430
Create(const std::string & osGroupFullName,const std::string & osParentName,const std::string & osName,const std::shared_ptr<HDF5SharedResources> & poShared,hid_t hAttribute)431 static std::shared_ptr<HDF5Attribute> Create(
432 const std::string& osGroupFullName,
433 const std::string& osParentName,
434 const std::string& osName,
435 const std::shared_ptr<HDF5SharedResources>& poShared,
436 hid_t hAttribute)
437 {
438 auto ar(std::shared_ptr<HDF5Attribute>(new HDF5Attribute(
439 osGroupFullName, osParentName, osName, poShared, hAttribute)));
440 if( ar->m_dt.GetClass() == GEDTC_NUMERIC &&
441 ar->m_dt.GetNumericDataType() == GDT_Unknown )
442 {
443 return nullptr;
444 }
445 return ar;
446 }
447
GetDimensions() const448 const std::vector<std::shared_ptr<GDALDimension>>& GetDimensions() const override { return m_dims; }
449
GetDataType() const450 const GDALExtendedDataType &GetDataType() const override { return m_dt; }
451 };
452
453 /************************************************************************/
454 /* ~HDF5SharedResources() */
455 /************************************************************************/
456
~HDF5SharedResources()457 HDF5SharedResources::~HDF5SharedResources()
458 {
459 if( m_hHDF5 > 0 )
460 H5Fclose(m_hHDF5);
461 }
462
463 /************************************************************************/
464 /* GetDimensions() */
465 /************************************************************************/
466
GetDimensions(CSLConstList) const467 std::vector<std::shared_ptr<GDALDimension>> HDF5Group::GetDimensions(CSLConstList) const
468 {
469 if( m_bGotDims )
470 return m_cachedDims;
471
472 struct CallbackData
473 {
474 std::shared_ptr<HDF5SharedResources> poShared{};
475 std::string osFullName{};
476 std::vector<std::shared_ptr<GDALDimension>> oListDim{};
477 };
478
479 struct Callback
480 {
481 static herr_t f(hid_t hGroup, const char *pszObjName, void* user_data)
482 {
483 CallbackData* data = static_cast<CallbackData*>(user_data);
484 H5G_stat_t oStatbuf;
485
486 if( H5Gget_objinfo(hGroup, pszObjName, FALSE, &oStatbuf) < 0 )
487 return -1;
488
489 if( oStatbuf.type == H5G_DATASET )
490 {
491 auto hArray = H5Dopen(hGroup, pszObjName);
492 if( hArray >= 0 )
493 {
494 auto ar = HDF5Array::Create(data->osFullName, pszObjName,
495 data->poShared, hArray,
496 nullptr, true);
497 if( ar && ar->GetDimensionCount() == 1 )
498 {
499 auto attrCLASS = ar->GetAttribute("CLASS");
500 if( attrCLASS &&
501 attrCLASS->GetDimensionCount() == 0 &&
502 attrCLASS->GetDataType().GetClass() == GEDTC_STRING )
503 {
504 const char* pszStr = attrCLASS->ReadAsString();
505 if( pszStr && EQUAL(pszStr, "DIMENSION_SCALE") )
506 {
507 auto attrNAME = ar->GetAttribute("NAME");
508 if( attrNAME &&
509 attrNAME->GetDimensionCount() == 0 &&
510 attrNAME->GetDataType().GetClass() == GEDTC_STRING )
511 {
512 const char* pszName = attrNAME->ReadAsString();
513 if( pszName && STARTS_WITH(pszName,
514 "This is a netCDF dimension but not a netCDF variable") )
515 {
516 data->oListDim.emplace_back(
517 std::make_shared<GDALDimension>(
518 data->osFullName, pszObjName,
519 std::string(), std::string(),
520 ar->GetDimensions()[0]->GetSize()));
521 return 0;
522 }
523 }
524
525 data->oListDim.emplace_back(
526 std::make_shared<HDF5Dimension>(
527 data->osFullName, pszObjName,
528 std::string(), std::string(),
529 ar->GetDimensions()[0]->GetSize(),
530 data->poShared));
531 }
532 }
533 }
534 }
535 }
536
537 return 0;
538 }
539 };
540
541 CallbackData data;
542 data.poShared = m_poShared;
543 data.osFullName = GetFullName();
544 H5Giterate(m_poShared->GetHDF5(), GetFullName().c_str(), nullptr,
545 &(Callback::f), &data);
546 m_bGotDims = true;
547 m_cachedDims = data.oListDim;
548 return data.oListDim;
549 }
550
551 /************************************************************************/
552 /* GetGroupNamesCallback() */
553 /************************************************************************/
554
GetGroupNamesCallback(hid_t hGroup,const char * pszObjName,void * selfIn)555 herr_t HDF5Group::GetGroupNamesCallback( hid_t hGroup,
556 const char *pszObjName,
557 void *selfIn)
558 {
559 HDF5Group* self = static_cast<HDF5Group*>(selfIn);
560 H5G_stat_t oStatbuf;
561
562 if( H5Gget_objinfo(hGroup, pszObjName, FALSE, &oStatbuf) < 0 )
563 return -1;
564
565 if( oStatbuf.type == H5G_GROUP )
566 {
567 if( self->m_oSetParentIds.find(std::pair<unsigned long, unsigned long>(
568 oStatbuf.objno[0], oStatbuf.objno[1])) == self->m_oSetParentIds.end() )
569 {
570 self->m_osListSubGroups.push_back(pszObjName);
571 }
572 else
573 {
574 CPLDebug("HDF5",
575 "Group %s contains a link to group %s which is "
576 "itself, or one of its ancestor.",
577 self->GetFullName().c_str(),
578 pszObjName);
579 }
580 }
581 return 0;
582 }
583
584 /************************************************************************/
585 /* GetGroupNames() */
586 /************************************************************************/
587
GetGroupNames(CSLConstList) const588 std::vector<std::string> HDF5Group::GetGroupNames(CSLConstList) const
589 {
590 m_osListSubGroups.clear();
591 H5Giterate(m_poShared->GetHDF5(), GetFullName().c_str(), nullptr,
592 GetGroupNamesCallback,
593 const_cast<void*>(static_cast<const void*>(this)));
594 return m_osListSubGroups;
595 }
596
597
598 /************************************************************************/
599 /* OpenGroup() */
600 /************************************************************************/
601
OpenGroup(const std::string & osName,CSLConstList) const602 std::shared_ptr<GDALGroup> HDF5Group::OpenGroup(const std::string& osName, CSLConstList) const
603 {
604 if( m_osListSubGroups.empty() )
605 GetGroupNames(nullptr);
606 if( std::find(m_osListSubGroups.begin(), m_osListSubGroups.end(), osName) ==
607 m_osListSubGroups.end() )
608 {
609 return nullptr;
610 }
611
612 H5G_stat_t oStatbuf;
613 if( H5Gget_objinfo(m_hGroup, osName.c_str(), FALSE, &oStatbuf) < 0 )
614 return nullptr;
615
616 auto hSubGroup = H5Gopen(m_hGroup, osName.c_str());
617 if( hSubGroup < 0 )
618 {
619 return nullptr;
620 }
621 return std::make_shared<HDF5Group>(GetFullName(), osName,
622 m_poShared,
623 m_oSetParentIds,
624 hSubGroup,
625 oStatbuf.objno);
626 }
627
628 /************************************************************************/
629 /* GetArrayNamesCallback() */
630 /************************************************************************/
631
GetArrayNamesCallback(hid_t hGroup,const char * pszObjName,void * selfIn)632 herr_t HDF5Group::GetArrayNamesCallback( hid_t hGroup,
633 const char *pszObjName,
634 void *selfIn)
635 {
636 HDF5Group* self = static_cast<HDF5Group*>(selfIn);
637 H5G_stat_t oStatbuf;
638
639 if( H5Gget_objinfo(hGroup, pszObjName, FALSE, &oStatbuf) < 0 )
640 return -1;
641
642 if( oStatbuf.type == H5G_DATASET )
643 {
644 auto hArray = H5Dopen(hGroup, pszObjName);
645 if( hArray >= 0 )
646 {
647 auto ar(HDF5Array::Create(std::string(), pszObjName,
648 self->m_poShared, hArray, self, true));
649 if( ar )
650 {
651 auto attrNAME = ar->GetAttribute("NAME");
652 if( attrNAME &&
653 attrNAME->GetDimensionCount() == 0 &&
654 attrNAME->GetDataType().GetClass() == GEDTC_STRING )
655 {
656 const char* pszName = attrNAME->ReadAsString();
657 if( pszName && STARTS_WITH(pszName,
658 "This is a netCDF dimension but not a netCDF variable") )
659 {
660 return 0;
661 }
662 }
663 }
664 }
665 self->m_osListArrays.push_back(pszObjName);
666 }
667 return 0;
668 }
669
670 /************************************************************************/
671 /* GetMDArrayNames() */
672 /************************************************************************/
673
GetMDArrayNames(CSLConstList) const674 std::vector<std::string> HDF5Group::GetMDArrayNames(CSLConstList) const
675 {
676 m_osListArrays.clear();
677 H5Giterate(m_poShared->GetHDF5(), GetFullName().c_str(), nullptr,
678 GetArrayNamesCallback,
679 const_cast<void*>(static_cast<const void*>(this)));
680 return m_osListArrays;
681 }
682
683 /************************************************************************/
684 /* OpenMDArray() */
685 /************************************************************************/
686
OpenMDArray(const std::string & osName,CSLConstList) const687 std::shared_ptr<GDALMDArray> HDF5Group::OpenMDArray(
688 const std::string& osName, CSLConstList) const
689 {
690 if( m_osListArrays.empty() )
691 GetMDArrayNames(nullptr);
692 if( std::find(m_osListArrays.begin(), m_osListArrays.end(), osName) ==
693 m_osListArrays.end() )
694 {
695 return nullptr;
696 }
697 auto hArray = H5Dopen(m_hGroup, osName.c_str());
698 if( hArray < 0 )
699 {
700 return nullptr;
701 }
702 return HDF5Array::Create(GetFullName(), osName, m_poShared, hArray,
703 this, false);
704 }
705
706 /************************************************************************/
707 /* GetAttributesCallback() */
708 /************************************************************************/
709
GetAttributesCallback(hid_t hGroup,const char * pszObjName,void * selfIn)710 herr_t HDF5Group::GetAttributesCallback( hid_t hGroup,
711 const char *pszObjName,
712 void *selfIn)
713 {
714 HDF5Group* self = static_cast<HDF5Group*>(selfIn);
715 if( self->m_bShowAllAttributes ||
716 (!EQUAL(pszObjName, "_Netcdf4Dimid") &&
717 !EQUAL(pszObjName, "_NCProperties")) )
718 {
719 hid_t hAttr = H5Aopen_name(hGroup, pszObjName);
720 if( hAttr > 0 )
721 {
722 auto attr(HDF5Attribute::Create(self->GetFullName(),
723 self->GetFullName(),
724 pszObjName,
725 self->m_poShared, hAttr));
726 if( attr )
727 {
728 self->m_oListAttributes.emplace_back(attr);
729 }
730 }
731 }
732 return 0;
733 }
734
735 /************************************************************************/
736 /* GetAttributes() */
737 /************************************************************************/
738
GetAttributes(CSLConstList papszOptions) const739 std::vector<std::shared_ptr<GDALAttribute>> HDF5Group::GetAttributes(
740 CSLConstList papszOptions) const
741 {
742 m_oListAttributes.clear();
743 m_bShowAllAttributes =
744 CPLTestBool(CSLFetchNameValueDef(papszOptions, "SHOW_ALL", "NO"));
745 H5Aiterate(m_hGroup, nullptr,
746 GetAttributesCallback,
747 const_cast<void*>(static_cast<const void*>(this)));
748 return m_oListAttributes;
749 }
750
751 /************************************************************************/
752 /* ~HDF5Array() */
753 /************************************************************************/
754
~HDF5Array()755 HDF5Array::~HDF5Array()
756 {
757 if( m_hArray > 0 )
758 H5Dclose(m_hArray);
759 if( m_hNativeDT > 0 )
760 H5Tclose(m_hNativeDT);
761 if( m_hDataSpace > 0 )
762 H5Sclose(m_hDataSpace);
763 }
764
765 /************************************************************************/
766 /* HDF5Array() */
767 /************************************************************************/
768
HDF5Array(const std::string & osParentName,const std::string & osName,const std::shared_ptr<HDF5SharedResources> & poShared,hid_t hArray,const HDF5Group * poGroup,bool bSkipFullDimensionInstantiation)769 HDF5Array::HDF5Array(const std::string& osParentName,
770 const std::string& osName,
771 const std::shared_ptr<HDF5SharedResources>& poShared,
772 hid_t hArray,
773 const HDF5Group* poGroup,
774 bool bSkipFullDimensionInstantiation):
775 GDALAbstractMDArray(osParentName, osName),
776 GDALMDArray(osParentName, osName),
777 m_osGroupFullname(osParentName),
778 m_poShared(poShared),
779 m_hArray(hArray),
780 m_hDataSpace(H5Dget_space(hArray)),
781 m_nOffset(H5Dget_offset(hArray))
782 {
783 const hid_t hDataType = H5Dget_type(hArray);
784 m_hNativeDT = H5Tget_native_type(hDataType, H5T_DIR_ASCEND);
785 H5Tclose(hDataType);
786
787 std::vector<std::pair<std::string, hid_t>> oTypes;
788 if( !osParentName.empty() &&
789 H5Tget_class(m_hNativeDT) == H5T_COMPOUND )
790 {
791 GetDataTypesInGroup(m_poShared->GetHDF5(), osParentName, oTypes);
792 }
793
794 m_dt = BuildDataType(m_hNativeDT, m_bHasVLenMember, m_bHasNonNativeDataType, oTypes);
795 for( auto& oPair: oTypes )
796 H5Tclose(oPair.second);
797
798 if( m_dt.GetClass() == GEDTC_NUMERIC &&
799 m_dt.GetNumericDataType() == GDT_Unknown )
800 {
801 CPLDebug("HDF5",
802 "Cannot map data type of %s to a type handled by GDAL",
803 osName.c_str());
804 return;
805 }
806
807 HDF5Array::GetAttributes();
808
809 if( bSkipFullDimensionInstantiation )
810 {
811 const int nDims = H5Sget_simple_extent_ndims(m_hDataSpace);
812 std::vector<hsize_t> anDimSizes(nDims);
813 if( nDims )
814 {
815 H5Sget_simple_extent_dims(m_hDataSpace, &anDimSizes[0], nullptr);
816 for( int i = 0; i < nDims; ++i )
817 {
818 m_dims.emplace_back(std::make_shared<GDALDimension>(
819 std::string(),
820 CPLSPrintf("dim%d", i),
821 std::string(),
822 std::string(),
823 anDimSizes[i]));
824 }
825 }
826 }
827 else
828 {
829 InstantiateDimensions(osParentName, poGroup);
830 }
831 }
832
833 /************************************************************************/
834 /* InstantiateDimensions() */
835 /************************************************************************/
836
InstantiateDimensions(const std::string & osParentName,const HDF5Group * poGroup)837 void HDF5Array::InstantiateDimensions(const std::string& osParentName,
838 const HDF5Group* poGroup)
839 {
840 const int nDims = H5Sget_simple_extent_ndims(m_hDataSpace);
841 std::vector<hsize_t> anDimSizes(nDims);
842 if( nDims )
843 {
844 H5Sget_simple_extent_dims(m_hDataSpace, &anDimSizes[0], nullptr);
845 }
846
847 if( nDims == 1 )
848 {
849 auto attrCLASS = GetAttribute("CLASS");
850 if( attrCLASS &&
851 attrCLASS->GetDimensionCount() == 0 &&
852 attrCLASS->GetDataType().GetClass() == GEDTC_STRING )
853 {
854 const char* pszStr = attrCLASS->ReadAsString();
855 if( pszStr && EQUAL(pszStr, "DIMENSION_SCALE") )
856 {
857 auto attrName = GetAttribute("NAME");
858 if( attrName &&
859 attrName->GetDataType().GetClass() == GEDTC_STRING )
860 {
861 const char* pszName = attrName->ReadAsString();
862 if( pszName &&
863 STARTS_WITH(pszName,
864 "This is a netCDF dimension but not a netCDF variable") )
865 {
866 m_dims.emplace_back(
867 std::make_shared<GDALDimension>(
868 std::string(), GetName(),
869 std::string(), std::string(),
870 anDimSizes[0]));
871 return;
872 }
873 }
874
875 m_dims.emplace_back(
876 std::make_shared<HDF5Dimension>(
877 osParentName, GetName(),
878 std::string(), std::string(),
879 anDimSizes[0],
880 m_poShared));
881 return;
882 }
883 }
884 }
885
886 std::map<size_t, std::string> mapDimIndexToDimFullName;
887
888 if( m_bHasDimensionList )
889 {
890 hid_t hAttr = H5Aopen_name(m_hArray, "DIMENSION_LIST");
891 const hid_t hAttrDataType = H5Aget_type(hAttr);
892 const hid_t hAttrSpace = H5Aget_space(hAttr);
893 if( H5Tget_class(hAttrDataType) == H5T_VLEN &&
894 H5Sget_simple_extent_ndims(hAttrSpace) == 1 )
895 {
896 const hid_t hBaseType = H5Tget_super(hAttrDataType);
897 if( H5Tget_class(hBaseType) == H5T_REFERENCE )
898 {
899 hsize_t nSize = 0;
900 H5Sget_simple_extent_dims(hAttrSpace, &nSize, nullptr);
901 if( nSize == static_cast<hsize_t>(nDims) )
902 {
903 std::vector<hvl_t> aHvl(static_cast<size_t>(nSize));
904 H5Aread(hAttr, hAttrDataType, &aHvl[0]);
905 for( size_t i = 0; i < nSize; i++ )
906 {
907 if( aHvl[i].len == 1 &&
908 H5Rget_obj_type(m_hArray, H5R_OBJECT, aHvl[i].p) == H5G_DATASET )
909 {
910 std::string referenceName;
911 referenceName.resize(256);
912 auto ret = H5Rget_name(
913 m_poShared->GetHDF5(), H5R_OBJECT, aHvl[i].p,
914 &referenceName[0], referenceName.size());
915 if( ret > 0 )
916 {
917 referenceName.resize(ret);
918 mapDimIndexToDimFullName[i] = referenceName;
919 }
920 }
921 }
922 H5Dvlen_reclaim(hAttrDataType, hAttrSpace, H5P_DEFAULT, &aHvl[0]);
923 }
924 }
925 H5Tclose(hBaseType);
926 }
927 H5Tclose(hAttrDataType);
928 H5Sclose(hAttrSpace);
929 H5Aclose(hAttr);
930 }
931 else if( m_bHasDimensionLabels )
932 {
933 hid_t hAttr = H5Aopen_name(m_hArray, "DIMENSION_LABELS");
934 auto attr(HDF5Attribute::Create(m_osGroupFullname, GetFullName(),
935 "DIMENSION_LABELS", m_poShared,
936 hAttr));
937 if( attr && attr->GetDimensionCount() == 1 &&
938 attr->GetDataType().GetClass() == GEDTC_STRING )
939 {
940 auto aosList = attr->ReadAsStringArray();
941 if( aosList.size() == nDims )
942 {
943 for( int i = 0; i < nDims; ++i )
944 {
945 if( aosList[i][0] != '\0' )
946 {
947 mapDimIndexToDimFullName[i] = aosList[i];
948 }
949 }
950 }
951 }
952 }
953
954 std::map<std::string, std::shared_ptr<GDALDimension>> oMapFullNameToDim;
955 // cppcheck-suppress knownConditionTrueFalse
956 if( poGroup && !mapDimIndexToDimFullName.empty() )
957 {
958 auto groupDims = poGroup->GetDimensions();
959 for( auto dim: groupDims )
960 {
961 oMapFullNameToDim[dim->GetFullName()] = dim;
962 }
963 }
964
965 for( int i = 0; i < nDims; ++i )
966 {
967 auto oIter = mapDimIndexToDimFullName.find(static_cast<size_t>(i));
968 if( oIter != mapDimIndexToDimFullName.end() )
969 {
970 auto oIter2 = oMapFullNameToDim.find(oIter->second);
971 if( oIter2 != oMapFullNameToDim.end() )
972 {
973 m_dims.emplace_back(oIter2->second);
974 continue;
975 }
976
977 std::string osDimName(oIter->second);
978 auto nPos = osDimName.rfind('/');
979 if( nPos != std::string::npos )
980 {
981 std::string osDimParentName(osDimName.substr(0, nPos));
982 osDimName = osDimName.substr(nPos + 1);
983
984 m_dims.emplace_back(
985 std::make_shared<HDF5Dimension>(
986 osDimParentName.empty() ? "/" : osDimParentName,
987 osDimName,
988 std::string(), std::string(),
989 anDimSizes[i],
990 m_poShared));
991 }
992 else
993 {
994 m_dims.emplace_back(std::make_shared<GDALDimension>(
995 std::string(),
996 osDimName,
997 std::string(),
998 std::string(),
999 anDimSizes[i]));
1000 }
1001 }
1002 else
1003 {
1004 m_dims.emplace_back(std::make_shared<GDALDimension>(
1005 std::string(),
1006 CPLSPrintf("dim%d", i),
1007 std::string(),
1008 std::string(),
1009 anDimSizes[i]));
1010 }
1011 }
1012 }
1013
1014 /************************************************************************/
1015 /* GetAttributesCallback() */
1016 /************************************************************************/
1017
GetAttributesCallback(hid_t hArray,const char * pszObjName,void * selfIn)1018 herr_t HDF5Array::GetAttributesCallback( hid_t hArray,
1019 const char *pszObjName,
1020 void *selfIn)
1021 {
1022 HDF5Array* self = static_cast<HDF5Array*>(selfIn);
1023 if( self->m_bShowAllAttributes ||
1024 (strcmp(pszObjName, "_Netcdf4Dimid") != 0 &&
1025 strcmp(pszObjName, "_Netcdf4Coordinates") != 0 &&
1026 strcmp(pszObjName, "CLASS") != 0 &&
1027 strcmp(pszObjName, "NAME") != 0) )
1028 {
1029 if( EQUAL(pszObjName, "DIMENSION_LIST") )
1030 {
1031 self->m_bHasDimensionList = true;
1032 if( !self->m_bShowAllAttributes )
1033 return 0;
1034 }
1035 if( EQUAL(pszObjName, "DIMENSION_LABELS") )
1036 {
1037 self->m_bHasDimensionLabels = true;
1038 if( !self->m_bShowAllAttributes )
1039 return 0;
1040 }
1041
1042 hid_t hAttr = H5Aopen_name(hArray, pszObjName);
1043 if( hAttr > 0 )
1044 {
1045 auto attr(HDF5Attribute::Create(self->m_osGroupFullname,
1046 self->GetFullName(),
1047 pszObjName,
1048 self->m_poShared, hAttr));
1049 if( attr )
1050 {
1051 // Used by HDF5-EOS products
1052 if( EQUAL(pszObjName, "_FillValue") &&
1053 self->GetDataType() == attr->GetDataType() &&
1054 attr->GetDimensionCount() == 0 )
1055 {
1056 if( self->GetDataType().GetClass() == GEDTC_NUMERIC )
1057 {
1058 auto raw(attr->ReadAsRaw());
1059 if( raw.data() )
1060 {
1061 self->m_abyNoData.assign(raw.data(),
1062 raw.data() + raw.size());
1063 }
1064 }
1065 if( !self->m_bShowAllAttributes )
1066 return 0;
1067 }
1068
1069 if( EQUAL(pszObjName, "units") &&
1070 attr->GetDataType().GetClass() == GEDTC_STRING &&
1071 attr->GetDimensionCount() == 0 )
1072 {
1073 const char* pszStr = attr->ReadAsString();
1074 if( pszStr )
1075 {
1076 self->m_osUnit = pszStr;
1077 if( !self->m_bShowAllAttributes )
1078 return 0;
1079 }
1080 }
1081
1082 self->m_oListAttributes.emplace_back(attr);
1083 }
1084 }
1085 }
1086 return 0;
1087 }
1088
1089
1090 /************************************************************************/
1091 /* GetAttributeFromAttributes() */
1092 /************************************************************************/
1093
1094 /** Possible fallback implementation for GetAttribute() using GetAttributes().
1095 */
GetAttribute(const std::string & osName) const1096 std::shared_ptr<GDALAttribute> HDF5Array::GetAttribute(
1097 const std::string& osName) const
1098 {
1099 const char* const apszOptions[] = { "SHOW_ALL=YES", nullptr };
1100 if( !m_bShowAllAttributes )
1101 GetAttributes(apszOptions);
1102 for( const auto& attr: m_oListAttributes )
1103 {
1104 if( attr->GetName() == osName )
1105 return attr;
1106 }
1107 return nullptr;
1108 }
1109
1110 /************************************************************************/
1111 /* GetAttributes() */
1112 /************************************************************************/
1113
GetAttributes(CSLConstList papszOptions) const1114 std::vector<std::shared_ptr<GDALAttribute>> HDF5Array::GetAttributes(
1115 CSLConstList papszOptions) const
1116 {
1117 m_oListAttributes.clear();
1118 m_bShowAllAttributes =
1119 CPLTestBool(CSLFetchNameValueDef(papszOptions, "SHOW_ALL", "NO"));
1120 H5Aiterate(m_hArray, nullptr,
1121 GetAttributesCallback,
1122 const_cast<void*>(static_cast<const void*>(this)));
1123 return m_oListAttributes;
1124 }
1125
1126 /************************************************************************/
1127 /* CopyBuffer() */
1128 /************************************************************************/
1129
CopyBuffer(size_t nDims,const size_t * count,const GInt64 * arrayStep,const GPtrDiff_t * bufferStride,const GDALExtendedDataType & bufferDataType,GByte * pabySrc,void * pDstBuffer)1130 static void CopyBuffer(size_t nDims,
1131 const size_t* count,
1132 const GInt64* arrayStep,
1133 const GPtrDiff_t* bufferStride,
1134 const GDALExtendedDataType& bufferDataType,
1135 GByte* pabySrc,
1136 void* pDstBuffer)
1137 {
1138 const size_t nBufferDataTypeSize(bufferDataType.GetSize());
1139 std::vector<size_t> anStackCount(nDims);
1140 std::vector<GByte*> pabySrcBufferStack(nDims + 1);
1141 std::vector<GByte*> pabyDstBufferStack(nDims + 1);
1142 std::vector<GPtrDiff_t> anSrcStride(nDims);
1143 std::vector<size_t> anSrcOffset(nDims+1);
1144 size_t nCurStride = nBufferDataTypeSize;
1145 for( size_t i = nDims; i > 0; )
1146 {
1147 --i;
1148 anSrcStride[i] = arrayStep[i] > 0 ? nCurStride :
1149 -static_cast<GPtrDiff_t>(nCurStride);
1150 anSrcOffset[i] = arrayStep[i] > 0 ? 0 : (count[i] - 1) * nCurStride;
1151 nCurStride *= count[i];
1152 }
1153 pabySrcBufferStack[0] = pabySrc + anSrcOffset[0];
1154 pabyDstBufferStack[0] = static_cast<GByte*>(pDstBuffer);
1155 size_t iDim = 0;
1156 lbl_next_depth:
1157 if( iDim == nDims )
1158 {
1159 memcpy(pabyDstBufferStack[nDims], pabySrcBufferStack[nDims],
1160 nBufferDataTypeSize);
1161 }
1162 else
1163 {
1164 anStackCount[iDim] = count[iDim];
1165 while( true )
1166 {
1167 ++ iDim;
1168 pabySrcBufferStack[iDim] = pabySrcBufferStack[iDim-1] + anSrcOffset[iDim];
1169 pabyDstBufferStack[iDim] = pabyDstBufferStack[iDim-1];
1170 goto lbl_next_depth;
1171 lbl_return_to_caller_in_loop:
1172 -- iDim;
1173 -- anStackCount[iDim];
1174 if( anStackCount[iDim] == 0 )
1175 break;
1176 pabyDstBufferStack[iDim] += bufferStride[iDim] * nBufferDataTypeSize;
1177 pabySrcBufferStack[iDim] += anSrcStride[iDim];
1178 }
1179 }
1180 if( iDim > 0 )
1181 goto lbl_return_to_caller_in_loop;
1182 }
1183
1184 /************************************************************************/
1185 /* ReadSlow() */
1186 /************************************************************************/
1187
ReadSlow(const GUInt64 * arrayStartIdx,const size_t * count,const GInt64 * arrayStep,const GPtrDiff_t * bufferStride,const GDALExtendedDataType & bufferDataType,void * pDstBuffer) const1188 bool HDF5Array::ReadSlow(const GUInt64* arrayStartIdx,
1189 const size_t* count,
1190 const GInt64* arrayStep,
1191 const GPtrDiff_t* bufferStride,
1192 const GDALExtendedDataType& bufferDataType,
1193 void* pDstBuffer) const
1194 {
1195 const size_t nBufferDataTypeSize(bufferDataType.GetSize());
1196 if( nBufferDataTypeSize == 0 )
1197 return false;
1198 const size_t nDims(m_dims.size());
1199 size_t nEltCount = 1;
1200 for( size_t i = 0; i < nDims; ++i )
1201 {
1202 nEltCount *= count[i];
1203 }
1204
1205 // Only for testing
1206 const char* pszThreshold = CPLGetConfigOption(
1207 "GDAL_HDF5_TEMP_ARRAY_ALLOC_SIZE", "10000000");
1208 const GUIntBig nThreshold = CPLScanUIntBig(
1209 pszThreshold, static_cast<int>(strlen(pszThreshold)));
1210 if( nEltCount < nThreshold / nBufferDataTypeSize )
1211 {
1212 CPLDebug("HDF5", "Using slow path");
1213 std::vector<GByte> abyTemp;
1214 try
1215 {
1216 abyTemp.resize(nEltCount * nBufferDataTypeSize);
1217 }
1218 catch( const std::exception& e )
1219 {
1220 CPLError(CE_Failure, CPLE_OutOfMemory, "%s", e.what());
1221 return false;
1222 }
1223 std::vector<GUInt64> anStart(nDims);
1224 std::vector<GInt64> anStep(nDims);
1225 for( size_t i = 0; i < nDims; i++ )
1226 {
1227 if( arrayStep[i] >= 0 )
1228 {
1229 anStart[i] = arrayStartIdx[i];
1230 anStep[i] = arrayStep[i];
1231 }
1232 else
1233 {
1234 // Use double negation so that operations occur only on
1235 // positive quantities to avoid an artificial negative signed
1236 // integer to unsigned conversion.
1237 anStart[i] = arrayStartIdx[i] - ((-arrayStep[i]) * (count[i]-1));
1238 anStep[i] = -arrayStep[i];
1239 }
1240 }
1241 std::vector<GPtrDiff_t> anStride(nDims);
1242 size_t nCurStride = 1;
1243 for( size_t i = nDims; i > 0; )
1244 {
1245 --i;
1246 anStride[i] = nCurStride;
1247 nCurStride *= count[i];
1248 }
1249 if( !IRead(anStart.data(), count, anStep.data(), anStride.data(),
1250 bufferDataType, &abyTemp[0]) )
1251 {
1252 return false;
1253 }
1254 CopyBuffer(nDims, count, arrayStep, bufferStride, bufferDataType,
1255 &abyTemp[0], pDstBuffer);
1256 return true;
1257 }
1258
1259 CPLDebug("HDF5", "Using very slow path");
1260 std::vector<GUInt64> anStart(nDims);
1261 const std::vector<size_t> anCount(nDims, 1);
1262 const std::vector<GInt64> anStep(nDims, 1);
1263 const std::vector<GPtrDiff_t> anStride(nDims, 1);
1264
1265 std::vector<size_t> anStackCount(nDims);
1266 std::vector<GByte*> pabyDstBufferStack(nDims + 1);
1267 pabyDstBufferStack[0] = static_cast<GByte*>(pDstBuffer);
1268 size_t iDim = 0;
1269 lbl_next_depth:
1270 if( iDim == nDims )
1271 {
1272 if( !IRead(anStart.data(), anCount.data(), anStep.data(), anStride.data(),
1273 bufferDataType, pabyDstBufferStack[nDims]) )
1274 {
1275 return false;
1276 }
1277 }
1278 else
1279 {
1280 anStart[iDim] = arrayStartIdx[iDim];
1281 anStackCount[iDim] = count[iDim];
1282 while( true )
1283 {
1284 ++ iDim;
1285 pabyDstBufferStack[iDim] = pabyDstBufferStack[iDim-1];
1286 goto lbl_next_depth;
1287 lbl_return_to_caller_in_loop:
1288 -- iDim;
1289 -- anStackCount[iDim];
1290 if( anStackCount[iDim] == 0 )
1291 break;
1292 pabyDstBufferStack[iDim] += bufferStride[iDim] * nBufferDataTypeSize;
1293 anStart[iDim] = CPLUnsanitizedAdd<GUInt64>(anStart[iDim], arrayStep[iDim]);
1294 }
1295 }
1296 if( iDim > 0 )
1297 goto lbl_return_to_caller_in_loop;
1298 return true;
1299 }
1300
1301 /************************************************************************/
1302 /* IngestVariableStrings() */
1303 /************************************************************************/
1304
IngestVariableStrings(void * pDstBuffer,hid_t hBufferType,size_t nDims,const size_t * count,const GPtrDiff_t * bufferStride)1305 static void IngestVariableStrings(void* pDstBuffer, hid_t hBufferType,
1306 size_t nDims,
1307 const size_t* count,
1308 const GPtrDiff_t* bufferStride)
1309 {
1310 std::vector<hsize_t> anCountOne(nDims, 1);
1311 const hid_t hMemSpaceOne = nDims == 0 ?
1312 H5Screate(H5S_SCALAR) :
1313 H5Screate_simple(static_cast<int>(nDims), anCountOne.data(), nullptr);
1314 std::vector<size_t> anStackCount(nDims);
1315 std::vector<GByte*> pabyDstBufferStack(nDims + 1);
1316 pabyDstBufferStack[0] = static_cast<GByte*>(pDstBuffer);
1317 size_t iDim = 0;
1318 lbl_next_depth:
1319 if( iDim == nDims )
1320 {
1321 void* old_ptr = pabyDstBufferStack[nDims];
1322 const char* pszSrcStr = *static_cast<char**>(old_ptr);
1323 char* pszNewStr = pszSrcStr ? VSIStrdup(pszSrcStr) : nullptr;
1324 H5Dvlen_reclaim(hBufferType, hMemSpaceOne, H5P_DEFAULT, old_ptr);
1325 *static_cast<char**>(old_ptr) = pszNewStr;
1326 }
1327 else
1328 {
1329 anStackCount[iDim] = count[iDim];
1330 while( true )
1331 {
1332 ++ iDim;
1333 pabyDstBufferStack[iDim] = pabyDstBufferStack[iDim-1];
1334 goto lbl_next_depth;
1335 lbl_return_to_caller_in_loop:
1336 -- iDim;
1337 -- anStackCount[iDim];
1338 if( anStackCount[iDim] == 0 )
1339 break;
1340 pabyDstBufferStack[iDim] += bufferStride[iDim] * sizeof(char*);
1341 }
1342 }
1343 if( iDim > 0 )
1344 goto lbl_return_to_caller_in_loop;
1345 H5Sclose(hMemSpaceOne);
1346 }
1347
1348 /************************************************************************/
1349 /* IngestFixedLengthStrings() */
1350 /************************************************************************/
1351
IngestFixedLengthStrings(void * pDstBuffer,const void * pTemp,hid_t hBufferType,size_t nDims,const size_t * count,const GPtrDiff_t * bufferStride)1352 static void IngestFixedLengthStrings(void* pDstBuffer,
1353 const void* pTemp,
1354 hid_t hBufferType,
1355 size_t nDims,
1356 const size_t* count,
1357 const GPtrDiff_t* bufferStride)
1358 {
1359 const size_t nStringSize = H5Tget_size(hBufferType);
1360 std::vector<size_t> anStackCount(nDims);
1361 std::vector<GByte*> pabyDstBufferStack(nDims + 1);
1362 const GByte* pabySrcBuffer = static_cast<const GByte*>(pTemp);
1363 pabyDstBufferStack[0] = static_cast<GByte*>(pDstBuffer);
1364 size_t iDim = 0;
1365 const bool bSpacePad = H5Tget_strpad(hBufferType) == H5T_STR_SPACEPAD;
1366 lbl_next_depth:
1367 if( iDim == nDims )
1368 {
1369 char* pszStr = static_cast<char*>(VSIMalloc(nStringSize + 1));
1370 if( pszStr )
1371 {
1372 memcpy(pszStr, pabySrcBuffer, nStringSize);
1373 size_t nIter = nStringSize;
1374 if( bSpacePad )
1375 {
1376 while( nIter >= 1 && pszStr[nIter-1] == ' ' )
1377 {
1378 nIter --;
1379 }
1380 }
1381 pszStr[nIter] = 0;
1382 }
1383 void* ptr = pabyDstBufferStack[nDims];
1384 *static_cast<char**>(ptr) = pszStr;
1385 }
1386 else
1387 {
1388 anStackCount[iDim] = count[iDim];
1389 while( true )
1390 {
1391 ++ iDim;
1392 pabyDstBufferStack[iDim] = pabyDstBufferStack[iDim-1];
1393 goto lbl_next_depth;
1394 lbl_return_to_caller_in_loop:
1395 -- iDim;
1396 -- anStackCount[iDim];
1397 if( anStackCount[iDim] == 0 )
1398 break;
1399 pabyDstBufferStack[iDim] += bufferStride[iDim] * sizeof(char*);
1400 pabySrcBuffer += nStringSize;
1401 }
1402 }
1403 if( iDim > 0 )
1404 goto lbl_return_to_caller_in_loop;
1405 }
1406
1407
1408 /************************************************************************/
1409 /* GetHDF5DataTypeFromGDALDataType() */
1410 /************************************************************************/
1411
GetHDF5DataTypeFromGDALDataType(const GDALExtendedDataType & dt,hid_t hNativeDT,const GDALExtendedDataType & bufferDataType)1412 static hid_t GetHDF5DataTypeFromGDALDataType(const GDALExtendedDataType& dt,
1413 hid_t hNativeDT,
1414 const GDALExtendedDataType& bufferDataType)
1415 {
1416 hid_t hBufferType = H5I_INVALID_HID;
1417 switch( bufferDataType.GetNumericDataType() )
1418 {
1419 case GDT_Byte: hBufferType = H5Tcopy(H5T_NATIVE_UCHAR); break;
1420 case GDT_UInt16: hBufferType = H5Tcopy(H5T_NATIVE_USHORT); break;
1421 case GDT_Int16: hBufferType = H5Tcopy(H5T_NATIVE_SHORT); break;
1422 case GDT_UInt32: hBufferType = H5Tcopy(H5T_NATIVE_UINT); break;
1423 case GDT_Int32: hBufferType = H5Tcopy(H5T_NATIVE_INT); break;
1424 case GDT_Float32: hBufferType = H5Tcopy(H5T_NATIVE_FLOAT); break;
1425 case GDT_Float64: hBufferType = H5Tcopy(H5T_NATIVE_DOUBLE); break;
1426 case GDT_CInt16:
1427 case GDT_CInt32:
1428 case GDT_CFloat32:
1429 case GDT_CFloat64:
1430 if( bufferDataType != dt )
1431 {
1432 return H5I_INVALID_HID;
1433 }
1434 else
1435 {
1436 hBufferType = H5Tcopy(hNativeDT);
1437 break;
1438 }
1439 default:
1440 return H5I_INVALID_HID;
1441 }
1442 return hBufferType;
1443 }
1444
1445 /************************************************************************/
1446 /* FreeDynamicMemory() */
1447 /************************************************************************/
1448
FreeDynamicMemory(GByte * pabyPtr,hid_t hDataType)1449 static void FreeDynamicMemory(GByte* pabyPtr, hid_t hDataType)
1450 {
1451 const auto klass = H5Tget_class(hDataType);
1452 if( klass == H5T_STRING && H5Tis_variable_str(hDataType) )
1453 {
1454 auto hDataSpace = H5Screate(H5S_SCALAR);
1455 H5Dvlen_reclaim(hDataType, hDataSpace, H5P_DEFAULT, pabyPtr);
1456 H5Sclose(hDataSpace);
1457 }
1458 else if (klass == H5T_COMPOUND )
1459 {
1460 const unsigned nMembers = H5Tget_nmembers(hDataType);
1461 for(unsigned i = 0; i < nMembers; i++ )
1462 {
1463 const auto nOffset = H5Tget_member_offset(hDataType, i);
1464 auto hMemberType = H5Tget_member_type(hDataType, i);
1465 if( hMemberType < 0 )
1466 continue;
1467 FreeDynamicMemory(pabyPtr + nOffset, hMemberType);
1468 H5Tclose(hMemberType);
1469 }
1470 }
1471 }
1472
1473 /************************************************************************/
1474 /* CreateMapTargetComponentsToSrc() */
1475 /************************************************************************/
1476
CreateMapTargetComponentsToSrc(hid_t hSrcDataType,const GDALExtendedDataType & dstDataType)1477 static std::vector<unsigned> CreateMapTargetComponentsToSrc(
1478 hid_t hSrcDataType, const GDALExtendedDataType& dstDataType)
1479 {
1480 CPLAssert( H5Tget_class(hSrcDataType) == H5T_COMPOUND );
1481 CPLAssert( dstDataType.GetClass() == GEDTC_COMPOUND );
1482
1483 const unsigned nMembers = H5Tget_nmembers(hSrcDataType);
1484 std::map<std::string, unsigned> oMapSrcCompNameToIdx;
1485 for(unsigned i = 0; i < nMembers; i++ )
1486 {
1487 char* pszName = H5Tget_member_name(hSrcDataType, i);
1488 if( pszName )
1489 oMapSrcCompNameToIdx[pszName] = i;
1490 }
1491
1492 std::vector<unsigned> ret;
1493 const auto& comps = dstDataType.GetComponents();
1494 ret.reserve(comps.size());
1495 for( const auto& comp: comps )
1496 {
1497 auto oIter = oMapSrcCompNameToIdx.find(comp->GetName());
1498 CPLAssert( oIter != oMapSrcCompNameToIdx.end() );
1499 ret.emplace_back(oIter->second);
1500 }
1501 return ret;
1502 }
1503
1504 /************************************************************************/
1505 /* CopyValue() */
1506 /************************************************************************/
1507
CopyValue(const GByte * pabySrcBuffer,hid_t hSrcDataType,GByte * pabyDstBuffer,const GDALExtendedDataType & dstDataType,const std::vector<unsigned> & mapDstCompsToSrcComps)1508 static void CopyValue(const GByte* pabySrcBuffer, hid_t hSrcDataType,
1509 GByte* pabyDstBuffer, const GDALExtendedDataType& dstDataType,
1510 const std::vector<unsigned>& mapDstCompsToSrcComps)
1511 {
1512 const auto klass = H5Tget_class(hSrcDataType);
1513 if( klass == H5T_STRING )
1514 {
1515 if( H5Tis_variable_str(hSrcDataType) )
1516 {
1517 GDALExtendedDataType::CopyValue(
1518 pabySrcBuffer, GDALExtendedDataType::CreateString(),
1519 pabyDstBuffer, dstDataType);
1520 }
1521 else
1522 {
1523 size_t nStringSize = H5Tget_size(hSrcDataType);
1524 char* pszStr = static_cast<char*>(VSIMalloc(nStringSize + 1));
1525 if( pszStr )
1526 {
1527 memcpy(pszStr, pabySrcBuffer, nStringSize);
1528 pszStr[nStringSize] = 0;
1529 }
1530 GDALExtendedDataType::CopyValue(
1531 &pszStr, GDALExtendedDataType::CreateString(),
1532 pabyDstBuffer, dstDataType);
1533 CPLFree(pszStr);
1534 }
1535 }
1536 else if( klass == H5T_COMPOUND )
1537 {
1538 if( dstDataType.GetClass() != GEDTC_COMPOUND )
1539 {
1540 // Typically source is complex data type
1541 auto srcDataType(GDALExtendedDataType::Create(::HDF5Dataset::GetDataType(hSrcDataType)));
1542 if( srcDataType.GetClass() == GEDTC_NUMERIC &&
1543 srcDataType.GetNumericDataType() != GDT_Unknown )
1544 {
1545 GDALExtendedDataType::CopyValue(
1546 pabySrcBuffer, srcDataType,
1547 pabyDstBuffer, dstDataType);
1548 }
1549 }
1550 else
1551 {
1552 const auto& comps = dstDataType.GetComponents();
1553 CPLAssert( comps.size() == mapDstCompsToSrcComps.size() );
1554 const std::vector<unsigned> empty;
1555 for( size_t iDst = 0; iDst < comps.size(); ++iDst )
1556 {
1557 const unsigned iSrc = mapDstCompsToSrcComps[iDst];
1558 auto hMemberType = H5Tget_member_type(hSrcDataType, iSrc);
1559 CopyValue( pabySrcBuffer + H5Tget_member_offset(hSrcDataType, iSrc),
1560 hMemberType,
1561 pabyDstBuffer + comps[iDst]->GetOffset(),
1562 comps[iDst]->GetType(), empty );
1563 H5Tclose(hMemberType);
1564 }
1565 }
1566 }
1567 else if( klass == H5T_ENUM )
1568 {
1569 auto hParent = H5Tget_super(hSrcDataType);
1570 CopyValue(pabySrcBuffer, hParent, pabyDstBuffer, dstDataType, {});
1571 H5Tclose(hParent);
1572 }
1573 else
1574 {
1575 if( H5Tequal(H5T_NATIVE_SCHAR, hSrcDataType) )
1576 {
1577 const GInt16 nVal = *reinterpret_cast<const signed char*>(pabySrcBuffer);
1578 // coverity[overrun-buffer-val]
1579 GDALExtendedDataType::CopyValue(
1580 &nVal, GDALExtendedDataType::Create(GDT_Int16),
1581 pabyDstBuffer, dstDataType);
1582 }
1583 else if( H5Tequal(H5T_NATIVE_LLONG, hSrcDataType) )
1584 {
1585 const double dfVal = static_cast<double>(
1586 *reinterpret_cast<const GInt64*>(pabySrcBuffer));
1587 GDALExtendedDataType::CopyValue(
1588 &dfVal, GDALExtendedDataType::Create(GDT_Float64),
1589 pabyDstBuffer, dstDataType);
1590 }
1591 else if( H5Tequal(H5T_NATIVE_ULLONG, hSrcDataType) )
1592 {
1593 const double dfVal = static_cast<double>(
1594 *reinterpret_cast<const GUInt64*>(pabySrcBuffer));
1595 GDALExtendedDataType::CopyValue(
1596 &dfVal, GDALExtendedDataType::Create(GDT_Float64),
1597 pabyDstBuffer, dstDataType);
1598 }
1599 else
1600 {
1601 GDALDataType eDT = ::HDF5Dataset::GetDataType(hSrcDataType);
1602 GDALExtendedDataType::CopyValue(
1603 pabySrcBuffer, GDALExtendedDataType::Create(eDT),
1604 pabyDstBuffer, dstDataType);
1605 }
1606 }
1607 }
1608
1609 /************************************************************************/
1610 /* CopyToFinalBuffer() */
1611 /************************************************************************/
1612
CopyToFinalBuffer(void * pDstBuffer,const void * pTemp,size_t nDims,const size_t * count,const GPtrDiff_t * bufferStride,hid_t hSrcDataType,const GDALExtendedDataType & bufferDataType)1613 static void CopyToFinalBuffer(void* pDstBuffer,
1614 const void* pTemp,
1615 size_t nDims,
1616 const size_t* count,
1617 const GPtrDiff_t* bufferStride,
1618 hid_t hSrcDataType,
1619 const GDALExtendedDataType& bufferDataType)
1620 {
1621 const size_t nSrcDataTypeSize(H5Tget_size(hSrcDataType));
1622 std::vector<size_t> anStackCount(nDims);
1623 std::vector<GByte*> pabyDstBufferStack(nDims + 1);
1624 const GByte* pabySrcBuffer = static_cast<const GByte*>(pTemp);
1625 pabyDstBufferStack[0] = static_cast<GByte*>(pDstBuffer);
1626 size_t iDim = 0;
1627 const auto mapDstCompsToSrcComps =
1628 (H5Tget_class(hSrcDataType) == H5T_COMPOUND &&
1629 bufferDataType.GetClass() == GEDTC_COMPOUND) ?
1630 CreateMapTargetComponentsToSrc(hSrcDataType, bufferDataType) :
1631 std::vector<unsigned>();
1632
1633 lbl_next_depth:
1634 if( iDim == nDims )
1635 {
1636 CopyValue(
1637 pabySrcBuffer, hSrcDataType,
1638 pabyDstBufferStack[nDims], bufferDataType,
1639 mapDstCompsToSrcComps);
1640 }
1641 else
1642 {
1643 anStackCount[iDim] = count[iDim];
1644 while( true )
1645 {
1646 ++ iDim;
1647 pabyDstBufferStack[iDim] = pabyDstBufferStack[iDim-1];
1648 goto lbl_next_depth;
1649 lbl_return_to_caller_in_loop:
1650 -- iDim;
1651 -- anStackCount[iDim];
1652 if( anStackCount[iDim] == 0 )
1653 break;
1654 pabyDstBufferStack[iDim] += bufferStride[iDim] * bufferDataType.GetSize();
1655 pabySrcBuffer += nSrcDataTypeSize;
1656 }
1657 }
1658 if( iDim > 0 )
1659 goto lbl_return_to_caller_in_loop;
1660 }
1661
1662 /************************************************************************/
1663 /* IRead() */
1664 /************************************************************************/
1665
IRead(const GUInt64 * arrayStartIdx,const size_t * count,const GInt64 * arrayStep,const GPtrDiff_t * bufferStride,const GDALExtendedDataType & bufferDataType,void * pDstBuffer) const1666 bool HDF5Array::IRead(const GUInt64* arrayStartIdx,
1667 const size_t* count,
1668 const GInt64* arrayStep,
1669 const GPtrDiff_t* bufferStride,
1670 const GDALExtendedDataType& bufferDataType,
1671 void* pDstBuffer) const
1672 {
1673 const size_t nDims(m_dims.size());
1674 std::vector<H5OFFSET_TYPE> anOffset(nDims);
1675 std::vector<hsize_t> anCount(nDims);
1676 std::vector<hsize_t> anStep(nDims);
1677
1678 size_t nEltCount = 1;
1679 for( size_t i = 0; i < nDims; ++i )
1680 {
1681 if( count[i] != 1 && (arrayStep[i] < 0 || bufferStride[i] < 0) )
1682 {
1683 return ReadSlow(arrayStartIdx, count, arrayStep, bufferStride,
1684 bufferDataType, pDstBuffer);
1685 }
1686 anOffset[i] = static_cast<hsize_t>(arrayStartIdx[i]);
1687 anCount[i] = static_cast<hsize_t>(count[i]);
1688 anStep[i] = static_cast<hsize_t>(count[i] == 1 ? 1 : arrayStep[i]);
1689 nEltCount *= count[i];
1690 }
1691 size_t nCurStride = 1;
1692 for( size_t i = nDims; i > 0; )
1693 {
1694 --i;
1695 if( count[i] != 1 && static_cast<size_t>(bufferStride[i]) != nCurStride )
1696 {
1697 return ReadSlow(arrayStartIdx, count, arrayStep, bufferStride,
1698 bufferDataType, pDstBuffer);
1699 }
1700 nCurStride *= count[i];
1701 }
1702
1703 hid_t hBufferType = H5I_INVALID_HID;
1704 GByte* pabyTemp = nullptr;
1705 if( m_dt.GetClass() == GEDTC_STRING )
1706 {
1707 if( bufferDataType.GetClass() != GEDTC_STRING )
1708 {
1709 return false;
1710 }
1711 hBufferType = H5Tcopy(m_hNativeDT);
1712 if( !H5Tis_variable_str(m_hNativeDT) )
1713 {
1714 const size_t nStringSize = H5Tget_size(m_hNativeDT);
1715 pabyTemp = static_cast<GByte*>(
1716 VSI_MALLOC2_VERBOSE(nStringSize, nEltCount));
1717 if( pabyTemp == nullptr )
1718 return false;
1719 }
1720 }
1721 else if( bufferDataType.GetClass() == GEDTC_NUMERIC &&
1722 m_dt.GetClass() == GEDTC_NUMERIC &&
1723 !GDALDataTypeIsComplex(m_dt.GetNumericDataType()) &&
1724 !GDALDataTypeIsComplex(bufferDataType.GetNumericDataType()) )
1725 {
1726 // Compatibility with older libhdf5 that doesn't like requesting
1727 // an enum to an integer
1728 if (H5Tget_class(m_hNativeDT) == H5T_ENUM )
1729 {
1730 auto hParent = H5Tget_super(m_hNativeDT);
1731 if( H5Tequal(hParent, H5T_NATIVE_UCHAR) ||
1732 H5Tequal(hParent, H5T_NATIVE_USHORT) ||
1733 H5Tequal(hParent, H5T_NATIVE_SHORT) ||
1734 H5Tequal(hParent, H5T_NATIVE_UINT) ||
1735 H5Tequal(hParent, H5T_NATIVE_INT) )
1736 {
1737 hBufferType = H5Tcopy(m_hNativeDT);
1738 if( m_dt != bufferDataType )
1739 {
1740 const size_t nDataTypeSize = H5Tget_size(m_hNativeDT);
1741 pabyTemp = static_cast<GByte*>(
1742 VSI_MALLOC2_VERBOSE(nDataTypeSize, nEltCount));
1743 if( pabyTemp == nullptr )
1744 {
1745 H5Tclose(hBufferType);
1746 return false;
1747 }
1748 }
1749 }
1750 H5Tclose(hParent);
1751 }
1752 if( hBufferType == H5I_INVALID_HID )
1753 {
1754 hBufferType = GetHDF5DataTypeFromGDALDataType(
1755 m_dt, m_hNativeDT, bufferDataType);
1756 if( hBufferType == H5I_INVALID_HID )
1757 {
1758 VSIFree(pabyTemp);
1759 return false;
1760 }
1761 }
1762 }
1763 else
1764 {
1765 hBufferType = H5Tcopy(m_hNativeDT);
1766 if( m_dt != bufferDataType || m_bHasVLenMember || m_bHasNonNativeDataType )
1767 {
1768 const size_t nDataTypeSize = H5Tget_size(m_hNativeDT);
1769 pabyTemp = static_cast<GByte*>(
1770 VSI_MALLOC2_VERBOSE(nDataTypeSize, nEltCount));
1771 if( pabyTemp == nullptr )
1772 {
1773 H5Tclose(hBufferType);
1774 return false;
1775 }
1776 }
1777 }
1778
1779 // Select block from file space.
1780 herr_t status;
1781 if( nDims )
1782 {
1783 status = H5Sselect_hyperslab(m_hDataSpace,
1784 H5S_SELECT_SET,
1785 anOffset.data(),
1786 anStep.data(),
1787 anCount.data(),
1788 nullptr);
1789 if( status < 0 )
1790 {
1791 H5Tclose(hBufferType);
1792 VSIFree(pabyTemp);
1793 return false;
1794 }
1795 }
1796
1797 // Create memory data space
1798 const hid_t hMemSpace = nDims == 0 ?
1799 H5Screate(H5S_SCALAR) :
1800 H5Screate_simple(static_cast<int>(nDims), anCount.data(), nullptr);
1801 if( nDims )
1802 {
1803 std::vector<H5OFFSET_TYPE> anMemOffset(nDims);
1804 status = H5Sselect_hyperslab(hMemSpace,
1805 H5S_SELECT_SET,
1806 anMemOffset.data(),
1807 nullptr,
1808 anCount.data(),
1809 nullptr);
1810 if( status < 0 )
1811 {
1812 H5Tclose(hBufferType);
1813 H5Sclose(hMemSpace);
1814 VSIFree(pabyTemp);
1815 return false;
1816 }
1817 }
1818
1819 status = H5Dread(m_hArray, hBufferType, hMemSpace,
1820 m_hDataSpace, H5P_DEFAULT,
1821 pabyTemp ? pabyTemp : pDstBuffer);
1822
1823 if( status >= 0 )
1824 {
1825 if( H5Tis_variable_str(hBufferType) )
1826 {
1827 IngestVariableStrings(pDstBuffer, hBufferType,
1828 nDims, count, bufferStride);
1829 }
1830 else if( pabyTemp && bufferDataType.GetClass() == GEDTC_STRING )
1831 {
1832 IngestFixedLengthStrings(pDstBuffer, pabyTemp,
1833 hBufferType,
1834 nDims, count, bufferStride);
1835 }
1836 else if( pabyTemp )
1837 {
1838 CopyToFinalBuffer(pDstBuffer, pabyTemp,
1839 nDims, count, bufferStride,
1840 m_hNativeDT, bufferDataType);
1841
1842 if( m_bHasVLenMember )
1843 {
1844 const size_t nBufferTypeSize = H5Tget_size(hBufferType);
1845 GByte* pabyPtr = pabyTemp;
1846 for(size_t i = 0; i < nEltCount; ++i )
1847 {
1848 FreeDynamicMemory(pabyPtr, hBufferType);
1849 pabyPtr += nBufferTypeSize;
1850 }
1851 }
1852 }
1853 }
1854
1855 H5Tclose(hBufferType);
1856 H5Sclose(hMemSpace);
1857 VSIFree(pabyTemp);
1858
1859 return status >= 0;
1860 }
1861
1862 /************************************************************************/
1863 /* ~HDF5Attribute() */
1864 /************************************************************************/
1865
~HDF5Attribute()1866 HDF5Attribute::~HDF5Attribute()
1867 {
1868 if( m_hAttribute > 0 )
1869 H5Aclose(m_hAttribute);
1870 if( m_hNativeDT > 0 )
1871 H5Tclose(m_hNativeDT);
1872 if( m_hDataSpace > 0 )
1873 H5Sclose(m_hDataSpace);
1874 }
1875
1876 /************************************************************************/
1877 /* CopyAllAttrValuesInto() */
1878 /************************************************************************/
1879
CopyAllAttrValuesInto(size_t nDims,const GUInt64 * arrayStartIdx,const size_t * count,const GInt64 * arrayStep,const GPtrDiff_t * bufferStride,const GDALExtendedDataType & bufferDataType,void * pDstBuffer,hid_t hSrcBufferType,const void * pabySrcBuffer)1880 static void CopyAllAttrValuesInto(size_t nDims,
1881 const GUInt64* arrayStartIdx,
1882 const size_t* count,
1883 const GInt64* arrayStep,
1884 const GPtrDiff_t* bufferStride,
1885 const GDALExtendedDataType& bufferDataType,
1886 void* pDstBuffer,
1887 hid_t hSrcBufferType,
1888 const void* pabySrcBuffer)
1889 {
1890 const size_t nBufferDataTypeSize(bufferDataType.GetSize());
1891 const size_t nSrcDataTypeSize(H5Tget_size(hSrcBufferType));
1892 std::vector<size_t> anStackCount(nDims);
1893 std::vector<const GByte*> pabySrcBufferStack(nDims + 1);
1894 std::vector<GByte*> pabyDstBufferStack(nDims + 1);
1895 const auto mapDstCompsToSrcComps =
1896 (H5Tget_class(hSrcBufferType) == H5T_COMPOUND &&
1897 bufferDataType.GetClass() == GEDTC_COMPOUND) ?
1898 CreateMapTargetComponentsToSrc(hSrcBufferType, bufferDataType) :
1899 std::vector<unsigned>();
1900
1901 pabySrcBufferStack[0] = static_cast<const GByte*>(pabySrcBuffer);
1902 if( nDims > 0 )
1903 pabySrcBufferStack[0] += arrayStartIdx[0] * nSrcDataTypeSize;
1904 pabyDstBufferStack[0] = static_cast<GByte*>(pDstBuffer);
1905 size_t iDim = 0;
1906 lbl_next_depth:
1907 if( iDim == nDims )
1908 {
1909 CopyValue(
1910 pabySrcBufferStack[nDims], hSrcBufferType,
1911 pabyDstBufferStack[nDims], bufferDataType,
1912 mapDstCompsToSrcComps);
1913 }
1914 else
1915 {
1916 anStackCount[iDim] = count[iDim];
1917 while( true )
1918 {
1919 ++ iDim;
1920 pabyDstBufferStack[iDim] = pabyDstBufferStack[iDim-1];
1921 pabySrcBufferStack[iDim] = pabySrcBufferStack[iDim-1];
1922 if( iDim < nDims )
1923 {
1924 pabySrcBufferStack[iDim] += arrayStartIdx[iDim] * nSrcDataTypeSize;
1925 }
1926 goto lbl_next_depth;
1927 lbl_return_to_caller_in_loop:
1928 -- iDim;
1929 -- anStackCount[iDim];
1930 if( anStackCount[iDim] == 0 )
1931 break;
1932 pabyDstBufferStack[iDim] += bufferStride[iDim] * nBufferDataTypeSize;
1933 pabySrcBufferStack[iDim] += arrayStep[iDim] * nSrcDataTypeSize;
1934 }
1935 }
1936 if( iDim > 0 )
1937 goto lbl_return_to_caller_in_loop;
1938 }
1939
1940 /************************************************************************/
1941 /* IRead() */
1942 /************************************************************************/
1943
IRead(const GUInt64 * arrayStartIdx,const size_t * count,const GInt64 * arrayStep,const GPtrDiff_t * bufferStride,const GDALExtendedDataType & bufferDataType,void * pDstBuffer) const1944 bool HDF5Attribute::IRead(const GUInt64* arrayStartIdx,
1945 const size_t* count,
1946 const GInt64* arrayStep,
1947 const GPtrDiff_t* bufferStride,
1948 const GDALExtendedDataType& bufferDataType,
1949 void* pDstBuffer) const
1950 {
1951 const size_t nDims(m_dims.size());
1952 if( m_dt.GetClass() == GEDTC_STRING )
1953 {
1954 if( bufferDataType.GetClass() != GEDTC_STRING )
1955 {
1956 return false;
1957 }
1958
1959 if( !H5Tis_variable_str(m_hNativeDT) )
1960 {
1961 const size_t nStringSize = H5Tget_size(m_hNativeDT);
1962 GByte* pabyTemp = static_cast<GByte*>(
1963 VSI_CALLOC_VERBOSE(nStringSize, m_nElements));
1964 if( pabyTemp == nullptr )
1965 return false;
1966 if( H5Sget_simple_extent_type(m_hDataSpace) != H5S_NULL &&
1967 H5Aread(m_hAttribute, m_hNativeDT, pabyTemp) < 0 )
1968 {
1969 VSIFree(pabyTemp);
1970 return false;
1971 }
1972 CopyAllAttrValuesInto(nDims,
1973 arrayStartIdx, count, arrayStep, bufferStride,
1974 bufferDataType, pDstBuffer,
1975 m_hNativeDT, pabyTemp);
1976 VSIFree(pabyTemp);
1977 }
1978 else
1979 {
1980 void* pabyTemp =
1981 VSI_CALLOC_VERBOSE(sizeof(char*), m_nElements);
1982 if( pabyTemp == nullptr )
1983 return false;
1984 if( H5Sget_simple_extent_type(m_hDataSpace) != H5S_NULL &&
1985 H5Aread(m_hAttribute, m_hNativeDT, pabyTemp) < 0 )
1986 {
1987 VSIFree(pabyTemp);
1988 return false;
1989 }
1990 CopyAllAttrValuesInto(nDims,
1991 arrayStartIdx, count, arrayStep, bufferStride,
1992 bufferDataType, pDstBuffer,
1993 m_hNativeDT, pabyTemp);
1994 H5Dvlen_reclaim(m_hNativeDT, m_hDataSpace, H5P_DEFAULT,
1995 pabyTemp);
1996 VSIFree(pabyTemp);
1997 }
1998 return true;
1999 }
2000
2001 hid_t hBufferType = H5I_INVALID_HID;
2002 if( m_dt.GetClass() == GEDTC_NUMERIC &&
2003 bufferDataType.GetClass() == GEDTC_NUMERIC &&
2004 !GDALDataTypeIsComplex(m_dt.GetNumericDataType()) &&
2005 !GDALDataTypeIsComplex(bufferDataType.GetNumericDataType()) )
2006 {
2007 // Compatibility with older libhdf5 that doesn't like requesting
2008 // an enum to an integer
2009 if (H5Tget_class(m_hNativeDT) == H5T_ENUM )
2010 {
2011 auto hParent = H5Tget_super(m_hNativeDT);
2012 if( H5Tequal(hParent, H5T_NATIVE_UCHAR) ||
2013 H5Tequal(hParent, H5T_NATIVE_USHORT) ||
2014 H5Tequal(hParent, H5T_NATIVE_SHORT) ||
2015 H5Tequal(hParent, H5T_NATIVE_UINT) ||
2016 H5Tequal(hParent, H5T_NATIVE_INT) )
2017 {
2018 hBufferType = H5Tcopy(m_hNativeDT);
2019 }
2020 H5Tclose(hParent);
2021 }
2022 if( hBufferType == H5I_INVALID_HID )
2023 {
2024 hBufferType = GetHDF5DataTypeFromGDALDataType(
2025 m_dt, m_hNativeDT, bufferDataType);
2026 }
2027 }
2028 else
2029 {
2030 hBufferType = H5Tcopy(m_hNativeDT);
2031 }
2032
2033 if( hBufferType == H5I_INVALID_HID )
2034 return false;
2035
2036 const size_t nBufferTypeSize(H5Tget_size(hBufferType));
2037 GByte* pabyTemp = static_cast<GByte*>(
2038 VSI_MALLOC2_VERBOSE(nBufferTypeSize, m_nElements));
2039 if( pabyTemp == nullptr )
2040 {
2041 H5Tclose(hBufferType);
2042 return false;
2043 }
2044 if( H5Aread(m_hAttribute, hBufferType, pabyTemp) < 0 )
2045 {
2046 VSIFree(pabyTemp);
2047 return false;
2048 }
2049 CopyAllAttrValuesInto(nDims,
2050 arrayStartIdx, count, arrayStep, bufferStride,
2051 bufferDataType, pDstBuffer,
2052 hBufferType, pabyTemp);
2053 if( bufferDataType.GetClass() == GEDTC_COMPOUND && m_bHasVLenMember )
2054 {
2055 GByte* pabyPtr = pabyTemp;
2056 for(size_t i = 0; i < m_nElements; ++i )
2057 {
2058 FreeDynamicMemory(pabyPtr, hBufferType);
2059 pabyPtr += nBufferTypeSize;
2060 }
2061 }
2062 VSIFree(pabyTemp);
2063 H5Tclose(hBufferType);
2064 return true;
2065 }
2066
2067 /************************************************************************/
2068 /* GetIndexingVariable() */
2069 /************************************************************************/
2070
GetIndexingVariable() const2071 std::shared_ptr<GDALMDArray> HDF5Dimension::GetIndexingVariable() const
2072 {
2073 auto hGroup = H5Gopen(m_poShared->GetHDF5(), m_osGroupFullname.c_str());
2074 if( hGroup >= 0 )
2075 {
2076 auto hArray = H5Dopen(hGroup, GetName().c_str());
2077 H5Gclose(hGroup);
2078 if( hArray >= 0 )
2079 {
2080 auto ar(HDF5Array::Create(m_osGroupFullname, GetName(), m_poShared,
2081 hArray, nullptr, false));
2082 auto attrName = ar->GetAttribute("NAME");
2083 if( attrName &&
2084 attrName->GetDataType().GetClass() == GEDTC_STRING )
2085 {
2086 const char* pszName = attrName->ReadAsString();
2087 if( pszName &&
2088 STARTS_WITH(pszName,
2089 "This is a netCDF dimension but not a netCDF variable") )
2090 {
2091 return nullptr;
2092 }
2093 }
2094 return ar;
2095 }
2096 }
2097 return nullptr;
2098 }
2099
2100 } // namespace
2101
2102 /************************************************************************/
2103 /* OpenMultiDim() */
2104 /************************************************************************/
2105
OpenMultiDim(GDALOpenInfo * poOpenInfo)2106 GDALDataset *HDF5Dataset::OpenMultiDim( GDALOpenInfo *poOpenInfo )
2107 {
2108 const char* pszFilename =
2109 STARTS_WITH(poOpenInfo->pszFilename, "HDF5:") ?
2110 poOpenInfo->pszFilename + strlen("HDF5:") :
2111 poOpenInfo->pszFilename;
2112
2113 // Try opening the dataset.
2114 auto hHDF5 = GDAL_HDF5Open(pszFilename);
2115 if( hHDF5 < 0 )
2116 {
2117 return nullptr;
2118 }
2119
2120 auto poSharedResources = std::make_shared<GDAL::HDF5SharedResources>();
2121 poSharedResources->m_hHDF5 = hHDF5;
2122
2123 auto poGroup(OpenGroup(poSharedResources));
2124 if( poGroup == nullptr )
2125 {
2126 return nullptr;
2127 }
2128
2129 auto poDS(new HDF5Dataset());
2130 poDS->m_poRootGroup = poGroup;
2131
2132 poDS->SetDescription(poOpenInfo->pszFilename);
2133
2134 // Setup/check for pam .aux.xml.
2135 poDS->TryLoadXML();
2136
2137 return poDS;
2138 }
2139
2140 /************************************************************************/
2141 /* OpenGroup() */
2142 /************************************************************************/
2143
OpenGroup(std::shared_ptr<GDAL::HDF5SharedResources> poSharedResources)2144 std::shared_ptr<GDALGroup> HDF5Dataset::OpenGroup(std::shared_ptr<GDAL::HDF5SharedResources> poSharedResources)
2145 {
2146 H5G_stat_t oStatbuf;
2147 if( H5Gget_objinfo(poSharedResources->m_hHDF5, "/", FALSE, &oStatbuf) < 0 )
2148 {
2149 return nullptr;
2150 }
2151 auto hGroup = H5Gopen(poSharedResources->m_hHDF5, "/");
2152 if( hGroup < 0 )
2153 {
2154 return nullptr;
2155 }
2156
2157 return std::shared_ptr<GDALGroup>(new GDAL::HDF5Group(std::string(), "/",
2158 poSharedResources, {},
2159 hGroup,
2160 oStatbuf.objno));
2161 }
2162