1# This file is part of h5py, a Python interface to the HDF5 library.
2#
3# http://www.h5py.org
4#
5# Copyright 2008-2019 Andrew Collette and contributors
6#
7# License:  Standard 3-clause BSD; see "license.txt" for full license terms
8#           and contributor agreement.
9
10# This file contains code or comments from the HDF5 library.  See the file
11# licenses/hdf5.txt for the full HDF5 software license.
12
13include "config.pxi"
14
15"""
16    File driver constants (H5FD*).
17"""
18
19# === Multi-file driver =======================================================
20
21MEM_DEFAULT = H5FD_MEM_DEFAULT
22MEM_SUPER = H5FD_MEM_SUPER
23MEM_BTREE = H5FD_MEM_BTREE
24MEM_DRAW = H5FD_MEM_DRAW
25MEM_GHEAP = H5FD_MEM_GHEAP
26MEM_LHEAP = H5FD_MEM_LHEAP
27MEM_OHDR = H5FD_MEM_OHDR
28MEM_NTYPES = H5FD_MEM_NTYPES
29
30# === MPI driver ==============================================================
31
32MPIO_INDEPENDENT = H5FD_MPIO_INDEPENDENT
33MPIO_COLLECTIVE = H5FD_MPIO_COLLECTIVE
34
35# === Driver types ============================================================
36
37CORE = H5FD_CORE
38FAMILY = H5FD_FAMILY
39LOG = H5FD_LOG
40MPIO = H5FD_MPIO
41MPIPOSIX = -1
42MULTI = H5FD_MULTI
43SEC2 = H5FD_SEC2
44STDIO = H5FD_STDIO
45IF HDF5_VERSION >= (1, 10, 6):
46    ROS3D = H5FD_ROS3
47IF UNAME_SYSNAME == "Windows":
48    WINDOWS = H5FD_WINDOWS
49ELSE:
50    WINDOWS = -1
51
52# === Logging driver ==========================================================
53
54LOG_LOC_READ  = H5FD_LOG_LOC_READ   # 0x0001
55LOG_LOC_WRITE = H5FD_LOG_LOC_WRITE  # 0x0002
56LOG_LOC_SEEK  = H5FD_LOG_LOC_SEEK   # 0x0004
57LOG_LOC_IO    = H5FD_LOG_LOC_IO     # (H5FD_LOG_LOC_READ|H5FD_LOG_LOC_WRITE|H5FD_LOG_LOC_SEEK)
58
59# Flags for tracking number of times each byte is read/written
60LOG_FILE_READ = H5FD_LOG_FILE_READ  # 0x0008
61LOG_FILE_WRITE= H5FD_LOG_FILE_WRITE # 0x0010
62LOG_FILE_IO   = H5FD_LOG_FILE_IO    # (H5FD_LOG_FILE_READ|H5FD_LOG_FILE_WRITE)
63
64# Flag for tracking "flavor" (type) of information stored at each byte
65LOG_FLAVOR    = H5FD_LOG_FLAVOR     # 0x0020
66
67# Flags for tracking total number of reads/writes/seeks
68LOG_NUM_READ  = H5FD_LOG_NUM_READ   # 0x0040
69LOG_NUM_WRITE = H5FD_LOG_NUM_WRITE  # 0x0080
70LOG_NUM_SEEK  = H5FD_LOG_NUM_SEEK   # 0x0100
71LOG_NUM_IO    = H5FD_LOG_NUM_IO     # (H5FD_LOG_NUM_READ|H5FD_LOG_NUM_WRITE|H5FD_LOG_NUM_SEEK)
72
73# Flags for tracking time spent in open/read/write/seek/close
74LOG_TIME_OPEN = H5FD_LOG_TIME_OPEN  # 0x0200        # Not implemented yet
75LOG_TIME_READ = H5FD_LOG_TIME_READ  # 0x0400        # Not implemented yet
76LOG_TIME_WRITE= H5FD_LOG_TIME_WRITE # 0x0800        # Partially implemented (need to track total time)
77LOG_TIME_SEEK = H5FD_LOG_TIME_SEEK  # 0x1000        # Partially implemented (need to track total time & track time for seeks during reading)
78LOG_TIME_CLOSE= H5FD_LOG_TIME_CLOSE # 0x2000        # Fully implemented
79LOG_TIME_IO   = H5FD_LOG_TIME_IO    # (H5FD_LOG_TIME_OPEN|H5FD_LOG_TIME_READ|H5FD_LOG_TIME_WRITE|H5FD_LOG_TIME_SEEK|H5FD_LOG_TIME_CLOSE)
80
81# Flag for tracking allocation of space in file
82LOG_ALLOC     = H5FD_LOG_ALLOC      # 0x4000
83LOG_ALL       = H5FD_LOG_ALL        # (H5FD_LOG_ALLOC|H5FD_LOG_TIME_IO|H5FD_LOG_NUM_IO|H5FD_LOG_FLAVOR|H5FD_LOG_FILE_IO|H5FD_LOG_LOC_IO)
84
85
86# Implementation of 'fileobj' Virtual File Driver: HDF5 Virtual File
87# Layer wrapper over Python file-like object.
88# https://support.hdfgroup.org/HDF5/doc1.8/TechNotes/VFL.html
89
90# HDF5 events (read, write, flush, ...) are dispatched via
91# H5FD_class_t (struct of callback pointers, H5FD_fileobj_*). This is
92# registered as the handler for 'fileobj' driver via H5FDregister.
93
94# File-like object is passed from Python side via FAPL with
95# PropFAID.set_fileobj_driver. Then H5FD_fileobj_open callback acts,
96# taking file-like object from FAPL and returning struct
97# H5FD_fileobj_t (descendant of base H5FD_t) which will hold file
98# state. Other callbacks receive H5FD_fileobj_t and operate on
99# f.fileobj. If successful, callbacks must return zero; otherwise
100# non-zero value.
101
102
103# H5FD_t of file-like object
104ctypedef struct H5FD_fileobj_t:
105    H5FD_t base  # must be first
106    PyObject* fileobj
107    haddr_t eoa
108
109
110# A minimal subset of callbacks is implemented. Non-essential
111# parameters (dxpl, type) are ignored.
112
113from cpython cimport Py_INCREF, Py_DECREF
114from libc.stdlib cimport malloc as stdlib_malloc
115from libc.stdlib cimport free as stdlib_free
116cimport libc.stdio
117cimport libc.stdint
118
119
120cdef void *H5FD_fileobj_fapl_get(H5FD_fileobj_t *f) with gil:
121    Py_INCREF(<object>f.fileobj)
122    return f.fileobj
123
124cdef void *H5FD_fileobj_fapl_copy(PyObject *old_fa) with gil:
125    cdef PyObject *new_fa = old_fa
126    Py_INCREF(<object>new_fa)
127    return new_fa
128
129cdef herr_t H5FD_fileobj_fapl_free(PyObject *fa) except -1 with gil:
130    Py_DECREF(<object>fa)
131    return 0
132
133cdef H5FD_fileobj_t *H5FD_fileobj_open(const char *name, unsigned flags, hid_t fapl, haddr_t maxaddr) except * with gil:
134    cdef PyObject *fileobj = <PyObject *>H5Pget_driver_info(fapl)
135    f = <H5FD_fileobj_t *>stdlib_malloc(sizeof(H5FD_fileobj_t))
136    f.fileobj = fileobj
137    Py_INCREF(<object>f.fileobj)
138    f.eoa = 0
139    return f
140
141cdef herr_t H5FD_fileobj_close(H5FD_fileobj_t *f) except -1 with gil:
142    Py_DECREF(<object>f.fileobj)
143    stdlib_free(f)
144    return 0
145
146cdef haddr_t H5FD_fileobj_get_eoa(const H5FD_fileobj_t *f, H5FD_mem_t type):
147    return f.eoa
148
149cdef herr_t H5FD_fileobj_set_eoa(H5FD_fileobj_t *f, H5FD_mem_t type, haddr_t addr):
150    f.eoa = addr
151    return 0
152
153cdef haddr_t H5FD_fileobj_get_eof(const H5FD_fileobj_t *f, H5FD_mem_t type) except -1 with gil:  # HADDR_UNDEF
154    (<object>f.fileobj).seek(0, libc.stdio.SEEK_END)
155    return (<object>f.fileobj).tell()
156
157cdef herr_t H5FD_fileobj_read(H5FD_fileobj_t *f, H5FD_mem_t type, hid_t dxpl, haddr_t addr, size_t size, void *buf) except -1 with gil:
158    cdef unsigned char[:] mview
159    (<object>f.fileobj).seek(addr)
160    if hasattr(<object>f.fileobj, 'readinto'):
161        mview = <unsigned char[:size]>(buf)
162        (<object>f.fileobj).readinto(mview)
163    else:
164        b = (<object>f.fileobj).read(size)
165        if len(b) == size:
166            memcpy(buf, <unsigned char *>b, size)
167        else:
168            return 1
169    return 0
170
171cdef herr_t H5FD_fileobj_write(H5FD_fileobj_t *f, H5FD_mem_t type, hid_t dxpl, haddr_t addr, size_t size, void *buf) except -1 with gil:
172    cdef unsigned char[:] mview
173    (<object>f.fileobj).seek(addr)
174    mview = <unsigned char[:size]>buf
175    (<object>f.fileobj).write(mview)
176    return 0
177
178cdef herr_t H5FD_fileobj_truncate(H5FD_fileobj_t *f, hid_t dxpl, hbool_t closing) except -1 with gil:
179    (<object>f.fileobj).truncate(f.eoa)
180    return 0
181
182cdef herr_t H5FD_fileobj_flush(H5FD_fileobj_t *f, hid_t dxpl, hbool_t closing) except -1 with gil:
183    # TODO: avoid unneeded fileobj.flush() when closing for e.g. TemporaryFile
184    (<object>f.fileobj).flush()
185    return 0
186
187
188# Construct H5FD_class_t struct and register 'fileobj' driver.
189
190cdef H5FD_class_t info
191memset(&info, 0, sizeof(info))
192
193info.name = 'fileobj'
194info.maxaddr = libc.stdint.SIZE_MAX - 1
195info.fc_degree = H5F_CLOSE_WEAK
196info.fapl_size = sizeof(PyObject *)
197info.fapl_get = <void *(*)(H5FD_t *)>H5FD_fileobj_fapl_get
198info.fapl_copy = <void *(*)(const void *)>H5FD_fileobj_fapl_copy
199info.fapl_free = <herr_t (*)(void *)>H5FD_fileobj_fapl_free
200info.open = <H5FD_t *(*)(const char *name, unsigned flags, hid_t fapl, haddr_t maxaddr)>H5FD_fileobj_open
201info.close = <herr_t (*)(H5FD_t *)>H5FD_fileobj_close
202info.get_eoa = <haddr_t (*)(const H5FD_t *, H5FD_mem_t)>H5FD_fileobj_get_eoa
203info.set_eoa = <herr_t (*)(H5FD_t *, H5FD_mem_t, haddr_t)>H5FD_fileobj_set_eoa
204info.get_eof = <haddr_t (*)(const H5FD_t *, H5FD_mem_t)>H5FD_fileobj_get_eof
205info.read = <herr_t (*)(H5FD_t *, H5FD_mem_t, hid_t, haddr_t, size_t, void *)>H5FD_fileobj_read
206info.write = <herr_t (*)(H5FD_t *, H5FD_mem_t, hid_t, haddr_t, size_t, const void *)>H5FD_fileobj_write
207info.truncate = <herr_t (*)(H5FD_t *, hid_t, hbool_t)>H5FD_fileobj_truncate
208info.flush = <herr_t (*)(H5FD_t *, hid_t, hbool_t)>H5FD_fileobj_flush
209# H5FD_FLMAP_DICHOTOMY
210info.fl_map = [H5FD_MEM_SUPER,  # default
211               H5FD_MEM_SUPER,  # super
212               H5FD_MEM_SUPER,  # btree
213               H5FD_MEM_DRAW,   # draw
214               H5FD_MEM_DRAW,   # gheap
215               H5FD_MEM_SUPER,  # lheap
216               H5FD_MEM_SUPER   # ohdr
217	       ]
218
219fileobj_driver = H5FDregister(&info)
220