1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 * Copyright by The HDF Group. * 3 * Copyright by the Board of Trustees of the University of Illinois. * 4 * All rights reserved. * 5 * * 6 * This file is part of HDF5. The full HDF5 copyright notice, including * 7 * terms governing use, modification, and redistribution, is contained in * 8 * the COPYING file, which can be found at the root of the source code * 9 * distribution tree, or in https://www.hdfgroup.org/licenses. * 10 * If you do not have access to either file, you may request a copy from * 11 * help@hdfgroup.org. * 12 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 13 14 /* 15 * Programmer: Robb Matzke 16 * Monday, July 26, 1999 17 */ 18 #ifndef H5FDpublic_H 19 #define H5FDpublic_H 20 21 #include "H5public.h" 22 #include "H5Fpublic.h" /*for H5F_close_degree_t */ 23 24 #define H5_HAVE_VFL 1 /*define a convenient app feature test*/ 25 #define H5FD_VFD_DEFAULT 0 /* Default VFL driver value */ 26 27 /* Types of allocation requests: see H5Fpublic.h */ 28 typedef enum H5F_mem_t H5FD_mem_t; 29 30 /* Map "fractal heap" header blocks to 'ohdr' type file memory, since its 31 * a fair amount of work to add a new kind of file memory and they are similar 32 * enough to object headers and probably too minor to deserve their own type. 33 * 34 * Map "fractal heap" indirect blocks to 'ohdr' type file memory, since they 35 * are similar to fractal heap header blocks. 36 * 37 * Map "fractal heap" direct blocks to 'lheap' type file memory, since they 38 * will be replacing local heaps. 39 * 40 * Map "fractal heap" 'huge' objects to 'draw' type file memory, since they 41 * represent large objects that are directly stored in the file. 42 * 43 * -QAK 44 */ 45 #define H5FD_MEM_FHEAP_HDR H5FD_MEM_OHDR 46 #define H5FD_MEM_FHEAP_IBLOCK H5FD_MEM_OHDR 47 #define H5FD_MEM_FHEAP_DBLOCK H5FD_MEM_LHEAP 48 #define H5FD_MEM_FHEAP_HUGE_OBJ H5FD_MEM_DRAW 49 50 /* Map "free space" header blocks to 'ohdr' type file memory, since its 51 * a fair amount of work to add a new kind of file memory and they are similar 52 * enough to object headers and probably too minor to deserve their own type. 53 * 54 * Map "free space" serialized sections to 'lheap' type file memory, since they 55 * are similar enough to local heap info. 56 * 57 * -QAK 58 */ 59 #define H5FD_MEM_FSPACE_HDR H5FD_MEM_OHDR 60 #define H5FD_MEM_FSPACE_SINFO H5FD_MEM_LHEAP 61 62 /* Map "shared object header message" master table to 'ohdr' type file memory, 63 * since its a fair amount of work to add a new kind of file memory and they are 64 * similar enough to object headers and probably too minor to deserve their own 65 * type. 66 * 67 * Map "shared object header message" indices to 'btree' type file memory, 68 * since they are similar enough to B-tree nodes. 69 * 70 * -QAK 71 */ 72 #define H5FD_MEM_SOHM_TABLE H5FD_MEM_OHDR 73 #define H5FD_MEM_SOHM_INDEX H5FD_MEM_BTREE 74 75 /* Map "extensible array" header blocks to 'ohdr' type file memory, since its 76 * a fair amount of work to add a new kind of file memory and they are similar 77 * enough to object headers and probably too minor to deserve their own type. 78 * 79 * Map "extensible array" index blocks to 'ohdr' type file memory, since they 80 * are similar to extensible array header blocks. 81 * 82 * Map "extensible array" super blocks to 'btree' type file memory, since they 83 * are similar enough to B-tree nodes. 84 * 85 * Map "extensible array" data blocks & pages to 'lheap' type file memory, since 86 * they are similar enough to local heap info. 87 * 88 * -QAK 89 */ 90 #define H5FD_MEM_EARRAY_HDR H5FD_MEM_OHDR 91 #define H5FD_MEM_EARRAY_IBLOCK H5FD_MEM_OHDR 92 #define H5FD_MEM_EARRAY_SBLOCK H5FD_MEM_BTREE 93 #define H5FD_MEM_EARRAY_DBLOCK H5FD_MEM_LHEAP 94 #define H5FD_MEM_EARRAY_DBLK_PAGE H5FD_MEM_LHEAP 95 96 /* Map "fixed array" header blocks to 'ohdr' type file memory, since its 97 * a fair amount of work to add a new kind of file memory and they are similar 98 * enough to object headers and probably too minor to deserve their own type. 99 * 100 * Map "fixed array" data blocks & pages to 'lheap' type file memory, since 101 * they are similar enough to local heap info. 102 * 103 */ 104 #define H5FD_MEM_FARRAY_HDR H5FD_MEM_OHDR 105 #define H5FD_MEM_FARRAY_DBLOCK H5FD_MEM_LHEAP 106 #define H5FD_MEM_FARRAY_DBLK_PAGE H5FD_MEM_LHEAP 107 108 /* 109 * A free-list map which maps all types of allocation requests to a single 110 * free list. This is useful for drivers that don't really care about 111 * keeping different requests segregated in the underlying file and which 112 * want to make most efficient reuse of freed memory. The use of the 113 * H5FD_MEM_SUPER free list is arbitrary. 114 */ 115 #define H5FD_FLMAP_SINGLE \ 116 { \ 117 H5FD_MEM_SUPER, /*default*/ \ 118 H5FD_MEM_SUPER, /*super*/ \ 119 H5FD_MEM_SUPER, /*btree*/ \ 120 H5FD_MEM_SUPER, /*draw*/ \ 121 H5FD_MEM_SUPER, /*gheap*/ \ 122 H5FD_MEM_SUPER, /*lheap*/ \ 123 H5FD_MEM_SUPER /*ohdr*/ \ 124 } 125 126 /* 127 * A free-list map which segregates requests into `raw' or `meta' data 128 * pools. 129 */ 130 #define H5FD_FLMAP_DICHOTOMY \ 131 { \ 132 H5FD_MEM_SUPER, /*default*/ \ 133 H5FD_MEM_SUPER, /*super*/ \ 134 H5FD_MEM_SUPER, /*btree*/ \ 135 H5FD_MEM_DRAW, /*draw*/ \ 136 H5FD_MEM_DRAW, /*gheap*/ \ 137 H5FD_MEM_SUPER, /*lheap*/ \ 138 H5FD_MEM_SUPER /*ohdr*/ \ 139 } 140 141 /* 142 * The default free list map which causes each request type to use it's own 143 * free-list. 144 */ 145 #define H5FD_FLMAP_DEFAULT \ 146 { \ 147 H5FD_MEM_DEFAULT, /*default*/ \ 148 H5FD_MEM_DEFAULT, /*super*/ \ 149 H5FD_MEM_DEFAULT, /*btree*/ \ 150 H5FD_MEM_DEFAULT, /*draw*/ \ 151 H5FD_MEM_DEFAULT, /*gheap*/ \ 152 H5FD_MEM_DEFAULT, /*lheap*/ \ 153 H5FD_MEM_DEFAULT /*ohdr*/ \ 154 } 155 156 /* Define VFL driver features that can be enabled on a per-driver basis */ 157 /* These are returned with the 'query' function pointer in H5FD_class_t */ 158 /* 159 * Defining H5FD_FEAT_AGGREGATE_METADATA for a VFL driver means that 160 * the library will attempt to allocate a larger block for metadata and 161 * then sub-allocate each metadata request from that larger block. 162 */ 163 #define H5FD_FEAT_AGGREGATE_METADATA 0x00000001 164 /* 165 * Defining H5FD_FEAT_ACCUMULATE_METADATA for a VFL driver means that 166 * the library will attempt to cache metadata as it is written to the file 167 * and build up a larger block of metadata to eventually pass to the VFL 168 * 'write' routine. 169 * 170 * Distinguish between updating the metadata accumulator on writes and 171 * reads. This is particularly (perhaps only, even) important for MPI-I/O 172 * where we guarantee that writes are collective, but reads may not be. 173 * If we were to allow the metadata accumulator to be written during a 174 * read operation, the application would hang. 175 */ 176 #define H5FD_FEAT_ACCUMULATE_METADATA_WRITE 0x00000002 177 #define H5FD_FEAT_ACCUMULATE_METADATA_READ 0x00000004 178 #define H5FD_FEAT_ACCUMULATE_METADATA \ 179 (H5FD_FEAT_ACCUMULATE_METADATA_WRITE | H5FD_FEAT_ACCUMULATE_METADATA_READ) 180 /* 181 * Defining H5FD_FEAT_DATA_SIEVE for a VFL driver means that 182 * the library will attempt to cache raw data as it is read from/written to 183 * a file in a "data seive" buffer. See Rajeev Thakur's papers: 184 * http://www.mcs.anl.gov/~thakur/papers/romio-coll.ps.gz 185 * http://www.mcs.anl.gov/~thakur/papers/mpio-high-perf.ps.gz 186 */ 187 #define H5FD_FEAT_DATA_SIEVE 0x00000008 188 /* 189 * Defining H5FD_FEAT_AGGREGATE_SMALLDATA for a VFL driver means that 190 * the library will attempt to allocate a larger block for "small" raw data 191 * and then sub-allocate "small" raw data requests from that larger block. 192 */ 193 #define H5FD_FEAT_AGGREGATE_SMALLDATA 0x00000010 194 /* 195 * Defining H5FD_FEAT_IGNORE_DRVRINFO for a VFL driver means that 196 * the library will ignore the driver info that is encoded in the file 197 * for the VFL driver. (This will cause the driver info to be eliminated 198 * from the file when it is flushed/closed, if the file is opened R/W). 199 */ 200 #define H5FD_FEAT_IGNORE_DRVRINFO 0x00000020 201 /* 202 * Defining the H5FD_FEAT_DIRTY_DRVRINFO_LOAD for a VFL driver means that 203 * the library will mark the driver info dirty when the file is opened 204 * R/W. This will cause the driver info to be re-encoded when the file 205 * is flushed/closed. 206 */ 207 #define H5FD_FEAT_DIRTY_DRVRINFO_LOAD 0x00000040 208 /* 209 * Defining H5FD_FEAT_POSIX_COMPAT_HANDLE for a VFL driver means that 210 * the handle for the VFD (returned with the 'get_handle' callback) is 211 * of type 'int' and is compatible with POSIX I/O calls. 212 */ 213 #define H5FD_FEAT_POSIX_COMPAT_HANDLE 0x00000080 214 /* 215 * Defining H5FD_FEAT_HAS_MPI for a VFL driver means that 216 * the driver makes use of MPI communication and code may retrieve 217 * communicator/rank information from it 218 */ 219 #define H5FD_FEAT_HAS_MPI 0x00000100 220 /* 221 * Defining the H5FD_FEAT_ALLOCATE_EARLY for a VFL driver will force 222 * the library to use the H5D_ALLOC_TIME_EARLY on dataset create 223 * instead of the default H5D_ALLOC_TIME_LATE 224 */ 225 #define H5FD_FEAT_ALLOCATE_EARLY 0x00000200 226 /* 227 * Defining H5FD_FEAT_ALLOW_FILE_IMAGE for a VFL driver means that 228 * the driver is able to use a file image in the fapl as the initial 229 * contents of a file. 230 */ 231 #define H5FD_FEAT_ALLOW_FILE_IMAGE 0x00000400 232 /* 233 * Defining H5FD_FEAT_CAN_USE_FILE_IMAGE_CALLBACKS for a VFL driver 234 * means that the driver is able to use callbacks to make a copy of the 235 * image to store in memory. 236 */ 237 #define H5FD_FEAT_CAN_USE_FILE_IMAGE_CALLBACKS 0x00000800 238 /* 239 * Defining H5FD_FEAT_SUPPORTS_SWMR_IO for a VFL driver means that the 240 * driver supports the single-writer/multiple-readers I/O pattern. 241 */ 242 #define H5FD_FEAT_SUPPORTS_SWMR_IO 0x00001000 243 /* 244 * Defining H5FD_FEAT_USE_ALLOC_SIZE for a VFL driver 245 * means that the library will just pass the allocation size to the 246 * the driver's allocation callback which will eventually handle alignment. 247 * This is specifically used for the multi/split driver. 248 */ 249 #define H5FD_FEAT_USE_ALLOC_SIZE 0x00002000 250 /* 251 * Defining H5FD_FEAT_PAGED_AGGR for a VFL driver 252 * means that the driver needs special file space mapping for paged aggregation. 253 * This is specifically used for the multi/split driver. 254 */ 255 #define H5FD_FEAT_PAGED_AGGR 0x00004000 256 /* 257 * Defining H5FD_FEAT_DEFAULT_VFD_COMPATIBLE for a VFL driver 258 * that creates a file which is compatible with the default VFD. 259 * Generally, this means that the VFD creates a single file that follows 260 * the canonical HDF5 file format. 261 * Regarding the Splitter VFD specifically, only drivers with this flag 262 * enabled may be used as the Write-Only (W/O) channel driver. 263 */ 264 #define H5FD_FEAT_DEFAULT_VFD_COMPATIBLE 0x00008000 265 266 /* Forward declaration */ 267 typedef struct H5FD_t H5FD_t; 268 269 /* Class information for each file driver */ 270 typedef struct H5FD_class_t { 271 const char * name; 272 haddr_t maxaddr; 273 H5F_close_degree_t fc_degree; 274 herr_t (*terminate)(void); 275 hsize_t (*sb_size)(H5FD_t *file); 276 herr_t (*sb_encode)(H5FD_t *file, char *name /*out*/, unsigned char *p /*out*/); 277 herr_t (*sb_decode)(H5FD_t *f, const char *name, const unsigned char *p); 278 size_t fapl_size; 279 void *(*fapl_get)(H5FD_t *file); 280 void *(*fapl_copy)(const void *fapl); 281 herr_t (*fapl_free)(void *fapl); 282 size_t dxpl_size; 283 void *(*dxpl_copy)(const void *dxpl); 284 herr_t (*dxpl_free)(void *dxpl); 285 H5FD_t *(*open)(const char *name, unsigned flags, hid_t fapl, haddr_t maxaddr); 286 herr_t (*close)(H5FD_t *file); 287 int (*cmp)(const H5FD_t *f1, const H5FD_t *f2); 288 herr_t (*query)(const H5FD_t *f1, unsigned long *flags); 289 herr_t (*get_type_map)(const H5FD_t *file, H5FD_mem_t *type_map); 290 haddr_t (*alloc)(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); 291 herr_t (*free)(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size); 292 haddr_t (*get_eoa)(const H5FD_t *file, H5FD_mem_t type); 293 herr_t (*set_eoa)(H5FD_t *file, H5FD_mem_t type, haddr_t addr); 294 haddr_t (*get_eof)(const H5FD_t *file, H5FD_mem_t type); 295 herr_t (*get_handle)(H5FD_t *file, hid_t fapl, void **file_handle); 296 herr_t (*read)(H5FD_t *file, H5FD_mem_t type, hid_t dxpl, haddr_t addr, size_t size, void *buffer); 297 herr_t (*write)(H5FD_t *file, H5FD_mem_t type, hid_t dxpl, haddr_t addr, size_t size, const void *buffer); 298 herr_t (*flush)(H5FD_t *file, hid_t dxpl_id, hbool_t closing); 299 herr_t (*truncate)(H5FD_t *file, hid_t dxpl_id, hbool_t closing); 300 herr_t (*lock)(H5FD_t *file, hbool_t rw); 301 herr_t (*unlock)(H5FD_t *file); 302 H5FD_mem_t fl_map[H5FD_MEM_NTYPES]; 303 } H5FD_class_t; 304 305 /* A free list is a singly-linked list of address/size pairs. */ 306 typedef struct H5FD_free_t { 307 haddr_t addr; 308 hsize_t size; 309 struct H5FD_free_t *next; 310 } H5FD_free_t; 311 312 /* 313 * The main datatype for each driver. Public fields common to all drivers 314 * are declared here and the driver appends private fields in memory. 315 */ 316 struct H5FD_t { 317 hid_t driver_id; /*driver ID for this file */ 318 const H5FD_class_t *cls; /*constant class info */ 319 unsigned long fileno; /* File 'serial' number */ 320 unsigned access_flags; /* File access flags (from create or open) */ 321 unsigned long feature_flags; /* VFL Driver feature Flags */ 322 haddr_t maxaddr; /* For this file, overrides class */ 323 haddr_t base_addr; /* Base address for HDF5 data w/in file */ 324 325 /* Space allocation management fields */ 326 hsize_t threshold; /* Threshold for alignment */ 327 hsize_t alignment; /* Allocation alignment */ 328 hbool_t paged_aggr; /* Paged aggregation for file space is enabled or not */ 329 }; 330 331 /** 332 * Define enum for the source of file image callbacks 333 */ 334 //! <!-- [H5FD_file_image_op_t_snip] --> 335 typedef enum { 336 H5FD_FILE_IMAGE_OP_NO_OP, 337 H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET, 338 /**< Passed to the \p image_malloc and \p image_memcpy callbacks when a 339 * file image buffer is to be copied while being set in a file access 340 * property list (FAPL)*/ 341 H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY, 342 /**< Passed to the \p image_malloc and \p image_memcpy callbacks 343 * when a file image buffer is to be copied when a FAPL is copied*/ 344 H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET, 345 /**<Passed to the \p image_malloc and \p image_memcpy callbacks when 346 * a file image buffer is to be copied while being retrieved from a FAPL*/ 347 H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE, 348 /**<Passed to the \p image_free callback when a file image 349 * buffer is to be released during a FAPL close operation*/ 350 H5FD_FILE_IMAGE_OP_FILE_OPEN, 351 /**<Passed to the \p image_malloc and 352 * \p image_memcpy callbackswhen a 353 * file image buffer is to be copied during a file open operation \n 354 * While the file image being opened will typically be copied from a 355 * FAPL, this need not always be the case. For example, the core file 356 * driver, also known as the memory file driver, takes its initial 357 * image from a file.*/ 358 H5FD_FILE_IMAGE_OP_FILE_RESIZE, 359 /**<Passed to the \p image_realloc callback when a file driver needs 360 * to resize an image buffer*/ 361 H5FD_FILE_IMAGE_OP_FILE_CLOSE 362 /**<Passed to the \p image_free callback when an image buffer is to 363 * be released during a file close operation*/ 364 } H5FD_file_image_op_t; 365 //! <!-- [H5FD_file_image_op_t_snip] --> 366 367 /** 368 * Define structure to hold file image callbacks 369 */ 370 //! <!-- [H5FD_file_image_callbacks_t_snip] --> 371 typedef struct { 372 /** 373 * \param[in] size Size in bytes of the file image buffer to allocate 374 * \param[in] file_image_op A value from H5FD_file_image_op_t indicating 375 * the operation being performed on the file image 376 * when this callback is invoked 377 * \param[in] udata Value passed in in the H5Pset_file_image_callbacks 378 * parameter \p udata 379 */ 380 //! <!-- [image_malloc_snip] --> 381 void *(*image_malloc)(size_t size, H5FD_file_image_op_t file_image_op, void *udata); 382 //! <!-- [image_malloc_snip] --> 383 /** 384 * \param[in] dest Address of the destination buffer 385 * \param[in] src Address of the source buffer 386 * \param[in] file_image_op A value from #H5FD_file_image_op_t indicating 387 * the operation being performed on the file image 388 * when this callback is invoked 389 * \param[in] udata Value passed in in the H5Pset_file_image_callbacks 390 * parameter \p udata 391 */ 392 //! <!-- [image_memcpy_snip] --> 393 void *(*image_memcpy)(void *dest, const void *src, size_t size, H5FD_file_image_op_t file_image_op, 394 void *udata); 395 //! <!-- [image_memcpy_snip] --> 396 /** 397 * \param[in] ptr Pointer to the buffer being reallocated 398 * \param[in] file_image_op A value from #H5FD_file_image_op_t indicating 399 * the operation being performed on the file image 400 * when this callback is invoked 401 * \param[in] udata Value passed in in the H5Pset_file_image_callbacks 402 * parameter \p udata 403 */ 404 //! <!-- [image_realloc_snip] --> 405 void *(*image_realloc)(void *ptr, size_t size, H5FD_file_image_op_t file_image_op, void *udata); 406 //! <!-- [image_realloc_snip] --> 407 /** 408 * \param[in] udata Value passed in in the H5Pset_file_image_callbacks 409 * parameter \p udata 410 */ 411 //! <!-- [image_free_snip] --> 412 herr_t (*image_free)(void *ptr, H5FD_file_image_op_t file_image_op, void *udata); 413 //! <!-- [image_free_snip] --> 414 /** 415 * \param[in] udata Value passed in in the H5Pset_file_image_callbacks 416 * parameter \p udata 417 */ 418 //! <!-- [udata_copy_snip] --> 419 void *(*udata_copy)(void *udata); 420 //! <!-- [udata_copy_snip] --> 421 /** 422 * \param[in] udata Value passed in in the H5Pset_file_image_callbacks 423 * parameter \p udata 424 */ 425 //! <!-- [udata_free_snip] --> 426 herr_t (*udata_free)(void *udata); 427 //! <!-- [udata_free_snip] --> 428 /** 429 * \brief The final field in the #H5FD_file_image_callbacks_t struct, 430 * provides a pointer to user-defined data. This pointer will be 431 * passed to the image_malloc, image_memcpy, image_realloc, and 432 * image_free callbacks. Define udata as NULL if no user-defined 433 * data is provided. 434 */ 435 void *udata; 436 } H5FD_file_image_callbacks_t; 437 //! <!-- [H5FD_file_image_callbacks_t_snip] --> 438 439 #ifdef __cplusplus 440 extern "C" { 441 #endif 442 443 /* Function prototypes */ 444 H5_DLL hid_t H5FDregister(const H5FD_class_t *cls); 445 H5_DLL herr_t H5FDunregister(hid_t driver_id); 446 H5_DLL H5FD_t *H5FDopen(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); 447 H5_DLL herr_t H5FDclose(H5FD_t *file); 448 H5_DLL int H5FDcmp(const H5FD_t *f1, const H5FD_t *f2); 449 H5_DLL int H5FDquery(const H5FD_t *f, unsigned long *flags); 450 H5_DLL haddr_t H5FDalloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); 451 H5_DLL herr_t H5FDfree(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size); 452 H5_DLL haddr_t H5FDget_eoa(H5FD_t *file, H5FD_mem_t type); 453 H5_DLL herr_t H5FDset_eoa(H5FD_t *file, H5FD_mem_t type, haddr_t eoa); 454 H5_DLL haddr_t H5FDget_eof(H5FD_t *file, H5FD_mem_t type); 455 H5_DLL herr_t H5FDget_vfd_handle(H5FD_t *file, hid_t fapl, void **file_handle); 456 H5_DLL herr_t H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, 457 void *buf /*out*/); 458 H5_DLL herr_t H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, 459 const void *buf); 460 H5_DLL herr_t H5FDflush(H5FD_t *file, hid_t dxpl_id, hbool_t closing); 461 H5_DLL herr_t H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing); 462 H5_DLL herr_t H5FDlock(H5FD_t *file, hbool_t rw); 463 H5_DLL herr_t H5FDunlock(H5FD_t *file); 464 465 /* Allows querying a VFD ID for features before the file is opened */ 466 H5_DLL herr_t H5FDdriver_query(hid_t driver_id, unsigned long *flags /*out*/); 467 468 #ifdef __cplusplus 469 } 470 #endif 471 #endif 472