1# This file is part of h5py, a Python interface to the HDF5 library. 2# 3# http://www.h5py.org 4# 5# Copyright 2008-2019 Andrew Collette and contributors 6# 7# License: Standard 3-clause BSD; see "license.txt" for full license terms 8# and contributor agreement. 9 10# This file contains code or comments from the HDF5 library. See the file 11# licenses/hdf5.txt for the full HDF5 software license. 12 13include "config.pxi" 14 15""" 16 File driver constants (H5FD*). 17""" 18 19# === Multi-file driver ======================================================= 20 21MEM_DEFAULT = H5FD_MEM_DEFAULT 22MEM_SUPER = H5FD_MEM_SUPER 23MEM_BTREE = H5FD_MEM_BTREE 24MEM_DRAW = H5FD_MEM_DRAW 25MEM_GHEAP = H5FD_MEM_GHEAP 26MEM_LHEAP = H5FD_MEM_LHEAP 27MEM_OHDR = H5FD_MEM_OHDR 28MEM_NTYPES = H5FD_MEM_NTYPES 29 30# === MPI driver ============================================================== 31 32MPIO_INDEPENDENT = H5FD_MPIO_INDEPENDENT 33MPIO_COLLECTIVE = H5FD_MPIO_COLLECTIVE 34 35# === Driver types ============================================================ 36 37CORE = H5FD_CORE 38FAMILY = H5FD_FAMILY 39LOG = H5FD_LOG 40MPIO = H5FD_MPIO 41MPIPOSIX = -1 42MULTI = H5FD_MULTI 43SEC2 = H5FD_SEC2 44STDIO = H5FD_STDIO 45IF HDF5_VERSION >= (1, 10, 6): 46 ROS3D = H5FD_ROS3 47IF UNAME_SYSNAME == "Windows": 48 WINDOWS = H5FD_WINDOWS 49ELSE: 50 WINDOWS = -1 51 52# === Logging driver ========================================================== 53 54LOG_LOC_READ = H5FD_LOG_LOC_READ # 0x0001 55LOG_LOC_WRITE = H5FD_LOG_LOC_WRITE # 0x0002 56LOG_LOC_SEEK = H5FD_LOG_LOC_SEEK # 0x0004 57LOG_LOC_IO = H5FD_LOG_LOC_IO # (H5FD_LOG_LOC_READ|H5FD_LOG_LOC_WRITE|H5FD_LOG_LOC_SEEK) 58 59# Flags for tracking number of times each byte is read/written 60LOG_FILE_READ = H5FD_LOG_FILE_READ # 0x0008 61LOG_FILE_WRITE= H5FD_LOG_FILE_WRITE # 0x0010 62LOG_FILE_IO = H5FD_LOG_FILE_IO # (H5FD_LOG_FILE_READ|H5FD_LOG_FILE_WRITE) 63 64# Flag for tracking "flavor" (type) of information stored at each byte 65LOG_FLAVOR = H5FD_LOG_FLAVOR # 0x0020 66 67# Flags for tracking total number of reads/writes/seeks 68LOG_NUM_READ = H5FD_LOG_NUM_READ # 0x0040 69LOG_NUM_WRITE = H5FD_LOG_NUM_WRITE # 0x0080 70LOG_NUM_SEEK = H5FD_LOG_NUM_SEEK # 0x0100 71LOG_NUM_IO = H5FD_LOG_NUM_IO # (H5FD_LOG_NUM_READ|H5FD_LOG_NUM_WRITE|H5FD_LOG_NUM_SEEK) 72 73# Flags for tracking time spent in open/read/write/seek/close 74LOG_TIME_OPEN = H5FD_LOG_TIME_OPEN # 0x0200 # Not implemented yet 75LOG_TIME_READ = H5FD_LOG_TIME_READ # 0x0400 # Not implemented yet 76LOG_TIME_WRITE= H5FD_LOG_TIME_WRITE # 0x0800 # Partially implemented (need to track total time) 77LOG_TIME_SEEK = H5FD_LOG_TIME_SEEK # 0x1000 # Partially implemented (need to track total time & track time for seeks during reading) 78LOG_TIME_CLOSE= H5FD_LOG_TIME_CLOSE # 0x2000 # Fully implemented 79LOG_TIME_IO = H5FD_LOG_TIME_IO # (H5FD_LOG_TIME_OPEN|H5FD_LOG_TIME_READ|H5FD_LOG_TIME_WRITE|H5FD_LOG_TIME_SEEK|H5FD_LOG_TIME_CLOSE) 80 81# Flag for tracking allocation of space in file 82LOG_ALLOC = H5FD_LOG_ALLOC # 0x4000 83LOG_ALL = H5FD_LOG_ALL # (H5FD_LOG_ALLOC|H5FD_LOG_TIME_IO|H5FD_LOG_NUM_IO|H5FD_LOG_FLAVOR|H5FD_LOG_FILE_IO|H5FD_LOG_LOC_IO) 84 85 86# Implementation of 'fileobj' Virtual File Driver: HDF5 Virtual File 87# Layer wrapper over Python file-like object. 88# https://support.hdfgroup.org/HDF5/doc1.8/TechNotes/VFL.html 89 90# HDF5 events (read, write, flush, ...) are dispatched via 91# H5FD_class_t (struct of callback pointers, H5FD_fileobj_*). This is 92# registered as the handler for 'fileobj' driver via H5FDregister. 93 94# File-like object is passed from Python side via FAPL with 95# PropFAID.set_fileobj_driver. Then H5FD_fileobj_open callback acts, 96# taking file-like object from FAPL and returning struct 97# H5FD_fileobj_t (descendant of base H5FD_t) which will hold file 98# state. Other callbacks receive H5FD_fileobj_t and operate on 99# f.fileobj. If successful, callbacks must return zero; otherwise 100# non-zero value. 101 102 103# H5FD_t of file-like object 104ctypedef struct H5FD_fileobj_t: 105 H5FD_t base # must be first 106 PyObject* fileobj 107 haddr_t eoa 108 109 110# A minimal subset of callbacks is implemented. Non-essential 111# parameters (dxpl, type) are ignored. 112 113from cpython cimport Py_INCREF, Py_DECREF 114from libc.stdlib cimport malloc as stdlib_malloc 115from libc.stdlib cimport free as stdlib_free 116cimport libc.stdio 117cimport libc.stdint 118 119 120cdef void *H5FD_fileobj_fapl_get(H5FD_fileobj_t *f) with gil: 121 Py_INCREF(<object>f.fileobj) 122 return f.fileobj 123 124cdef void *H5FD_fileobj_fapl_copy(PyObject *old_fa) with gil: 125 cdef PyObject *new_fa = old_fa 126 Py_INCREF(<object>new_fa) 127 return new_fa 128 129cdef herr_t H5FD_fileobj_fapl_free(PyObject *fa) except -1 with gil: 130 Py_DECREF(<object>fa) 131 return 0 132 133cdef H5FD_fileobj_t *H5FD_fileobj_open(const char *name, unsigned flags, hid_t fapl, haddr_t maxaddr) except * with gil: 134 cdef PyObject *fileobj = <PyObject *>H5Pget_driver_info(fapl) 135 f = <H5FD_fileobj_t *>stdlib_malloc(sizeof(H5FD_fileobj_t)) 136 f.fileobj = fileobj 137 Py_INCREF(<object>f.fileobj) 138 f.eoa = 0 139 return f 140 141cdef herr_t H5FD_fileobj_close(H5FD_fileobj_t *f) except -1 with gil: 142 Py_DECREF(<object>f.fileobj) 143 stdlib_free(f) 144 return 0 145 146cdef haddr_t H5FD_fileobj_get_eoa(const H5FD_fileobj_t *f, H5FD_mem_t type): 147 return f.eoa 148 149cdef herr_t H5FD_fileobj_set_eoa(H5FD_fileobj_t *f, H5FD_mem_t type, haddr_t addr): 150 f.eoa = addr 151 return 0 152 153cdef haddr_t H5FD_fileobj_get_eof(const H5FD_fileobj_t *f, H5FD_mem_t type) except -1 with gil: # HADDR_UNDEF 154 (<object>f.fileobj).seek(0, libc.stdio.SEEK_END) 155 return (<object>f.fileobj).tell() 156 157cdef herr_t H5FD_fileobj_read(H5FD_fileobj_t *f, H5FD_mem_t type, hid_t dxpl, haddr_t addr, size_t size, void *buf) except -1 with gil: 158 cdef unsigned char[:] mview 159 (<object>f.fileobj).seek(addr) 160 if hasattr(<object>f.fileobj, 'readinto'): 161 mview = <unsigned char[:size]>(buf) 162 (<object>f.fileobj).readinto(mview) 163 else: 164 b = (<object>f.fileobj).read(size) 165 if len(b) == size: 166 memcpy(buf, <unsigned char *>b, size) 167 else: 168 return 1 169 return 0 170 171cdef herr_t H5FD_fileobj_write(H5FD_fileobj_t *f, H5FD_mem_t type, hid_t dxpl, haddr_t addr, size_t size, void *buf) except -1 with gil: 172 cdef unsigned char[:] mview 173 (<object>f.fileobj).seek(addr) 174 mview = <unsigned char[:size]>buf 175 (<object>f.fileobj).write(mview) 176 return 0 177 178cdef herr_t H5FD_fileobj_truncate(H5FD_fileobj_t *f, hid_t dxpl, hbool_t closing) except -1 with gil: 179 (<object>f.fileobj).truncate(f.eoa) 180 return 0 181 182cdef herr_t H5FD_fileobj_flush(H5FD_fileobj_t *f, hid_t dxpl, hbool_t closing) except -1 with gil: 183 # TODO: avoid unneeded fileobj.flush() when closing for e.g. TemporaryFile 184 (<object>f.fileobj).flush() 185 return 0 186 187 188# Construct H5FD_class_t struct and register 'fileobj' driver. 189 190cdef H5FD_class_t info 191memset(&info, 0, sizeof(info)) 192 193info.name = 'fileobj' 194info.maxaddr = libc.stdint.SIZE_MAX - 1 195info.fc_degree = H5F_CLOSE_WEAK 196info.fapl_size = sizeof(PyObject *) 197info.fapl_get = <void *(*)(H5FD_t *)>H5FD_fileobj_fapl_get 198info.fapl_copy = <void *(*)(const void *)>H5FD_fileobj_fapl_copy 199info.fapl_free = <herr_t (*)(void *)>H5FD_fileobj_fapl_free 200info.open = <H5FD_t *(*)(const char *name, unsigned flags, hid_t fapl, haddr_t maxaddr)>H5FD_fileobj_open 201info.close = <herr_t (*)(H5FD_t *)>H5FD_fileobj_close 202info.get_eoa = <haddr_t (*)(const H5FD_t *, H5FD_mem_t)>H5FD_fileobj_get_eoa 203info.set_eoa = <herr_t (*)(H5FD_t *, H5FD_mem_t, haddr_t)>H5FD_fileobj_set_eoa 204info.get_eof = <haddr_t (*)(const H5FD_t *, H5FD_mem_t)>H5FD_fileobj_get_eof 205info.read = <herr_t (*)(H5FD_t *, H5FD_mem_t, hid_t, haddr_t, size_t, void *)>H5FD_fileobj_read 206info.write = <herr_t (*)(H5FD_t *, H5FD_mem_t, hid_t, haddr_t, size_t, const void *)>H5FD_fileobj_write 207info.truncate = <herr_t (*)(H5FD_t *, hid_t, hbool_t)>H5FD_fileobj_truncate 208info.flush = <herr_t (*)(H5FD_t *, hid_t, hbool_t)>H5FD_fileobj_flush 209# H5FD_FLMAP_DICHOTOMY 210info.fl_map = [H5FD_MEM_SUPER, # default 211 H5FD_MEM_SUPER, # super 212 H5FD_MEM_SUPER, # btree 213 H5FD_MEM_DRAW, # draw 214 H5FD_MEM_DRAW, # gheap 215 H5FD_MEM_SUPER, # lheap 216 H5FD_MEM_SUPER # ohdr 217 ] 218 219fileobj_driver = H5FDregister(&info) 220