1# -*- coding: utf-8 -*-
2
3########################################################################
4#
5# License: BSD
6# Created: 2005-09-29
7# Author: Ivan Vilata i Balaguer - ivan@selidor.net
8#
9# $Id$
10#
11########################################################################
12
13"""Test module for compatibility with plain HDF files."""
14
15import os
16import shutil
17import tempfile
18
19import numpy
20
21import tables
22from tables.tests import common
23from tables.tests.common import allequal
24from tables.tests.common import unittest, test_filename
25from tables.tests.common import PyTablesTestCase as TestCase
26
27
28class PaddedArrayTestCase(common.TestFileMixin, TestCase):
29    """Test for H5T_COMPOUND (Table) datatype with padding.
30
31    Regression test for issue gh-734
32
33    itemsize.h5 was created with h5py with the array `expectedData` (see below)
34    in the table `/Test`:
35    'A' and 'B' are 4 + 4 bytes, with 8 bytes padding.
36
37    $ h5ls -v itemsize.h5
38    Test                     Dataset {3/3}
39    Location:  1:800
40    Links:     1
41    Storage:   48 logical bytes, 48 allocated bytes, 100.00% utilization
42    Type:      struct {
43                   "A"                +0    native unsigned int
44                   "B"                +4    native unsigned int
45               } 16 bytes
46
47    """
48    h5fname = test_filename('itemsize.h5')
49
50    def test(self):
51        arr = self.h5file.get_node('/Test')
52        data = arr.read()
53        expectedData = numpy.array(
54                [(1, 11), (2, 12), (3, 13)],
55                dtype={'names': ['A', 'B'], 'formats': ['<u4', '<u4'],
56                       'offsets': [0, 4], 'itemsize': 16})
57        self.assertTrue(common.areArraysEqual(data, expectedData))
58
59
60class EnumTestCase(common.TestFileMixin, TestCase):
61    """Test for enumerated datatype.
62
63    See ftp://ftp.hdfgroup.org/HDF5/current/src/unpacked/test/enum.c.
64
65    """
66
67    h5fname = test_filename('smpl_enum.h5')
68
69    def test(self):
70        self.assertIn('/EnumTest', self.h5file)
71
72        arr = self.h5file.get_node('/EnumTest')
73        self.assertIsInstance(arr, tables.Array)
74
75        enum = arr.get_enum()
76        expectedEnum = tables.Enum(['RED', 'GREEN', 'BLUE', 'WHITE', 'BLACK'])
77        self.assertEqual(enum, expectedEnum)
78
79        data = list(arr.read())
80        expectedData = [
81            enum[name] for name in
82            ['RED', 'GREEN', 'BLUE', 'WHITE', 'BLACK',
83             'RED', 'GREEN', 'BLUE', 'WHITE', 'BLACK']]
84        self.assertEqual(data, expectedData)
85
86
87class NumericTestCase(common.TestFileMixin, TestCase):
88    """Test for several numeric datatypes.
89
90    See
91    ftp://ftp.ncsa.uiuc.edu/HDF/files/hdf5/samples/[fiu]l?{8,16,32,64}{be,le}.c
92    (they seem to be no longer available).
93
94    """
95
96    def test(self):
97        self.assertIn('/TestArray', self.h5file)
98
99        arr = self.h5file.get_node('/TestArray')
100        self.assertIsInstance(arr, tables.Array)
101
102        self.assertEqual(arr.atom.type, self.type)
103        self.assertEqual(arr.byteorder, self.byteorder)
104        self.assertEqual(arr.shape, (6, 5))
105
106        data = arr.read()
107        expectedData = numpy.array([
108            [0, 1, 2, 3, 4],
109            [1, 2, 3, 4, 5],
110            [2, 3, 4, 5, 6],
111            [3, 4, 5, 6, 7],
112            [4, 5, 6, 7, 8],
113            [5, 6, 7, 8, 9]], dtype=self.type)
114        self.assertTrue(common.areArraysEqual(data, expectedData))
115
116
117class F64BETestCase(NumericTestCase):
118    h5fname = test_filename('smpl_f64be.h5')
119    type = 'float64'
120    byteorder = 'big'
121
122
123class F64LETestCase(NumericTestCase):
124    h5fname = test_filename('smpl_f64le.h5')
125    type = 'float64'
126    byteorder = 'little'
127
128
129class I64BETestCase(NumericTestCase):
130    h5fname = test_filename('smpl_i64be.h5')
131    type = 'int64'
132    byteorder = 'big'
133
134
135class I64LETestCase(NumericTestCase):
136    h5fname = test_filename('smpl_i64le.h5')
137    type = 'int64'
138    byteorder = 'little'
139
140
141class I32BETestCase(NumericTestCase):
142    h5fname = test_filename('smpl_i32be.h5')
143    type = 'int32'
144    byteorder = 'big'
145
146
147class I32LETestCase(NumericTestCase):
148    h5fname = test_filename('smpl_i32le.h5')
149    type = 'int32'
150    byteorder = 'little'
151
152
153class ChunkedCompoundTestCase(common.TestFileMixin, TestCase):
154    """Test for a more complex and chunked compound structure.
155
156    This is generated by a chunked version of the example in
157    ftp://ftp.ncsa.uiuc.edu/HDF/files/hdf5/samples/compound2.c.
158
159    """
160
161    h5fname = test_filename('smpl_compound_chunked.h5')
162
163    def test(self):
164        self.assertIn('/CompoundChunked', self.h5file)
165
166        tbl = self.h5file.get_node('/CompoundChunked')
167        self.assertIsInstance(tbl, tables.Table)
168
169        self.assertEqual(
170            tbl.colnames,
171            ['a_name', 'c_name', 'd_name', 'e_name', 'f_name', 'g_name'])
172
173        self.assertEqual(tbl.coltypes['a_name'], 'int32')
174        self.assertEqual(tbl.coldtypes['a_name'].shape, ())
175
176        self.assertEqual(tbl.coltypes['c_name'], 'string')
177        self.assertEqual(tbl.coldtypes['c_name'].shape, ())
178
179        self.assertEqual(tbl.coltypes['d_name'], 'int16')
180        self.assertEqual(tbl.coldtypes['d_name'].shape, (5, 10))
181
182        self.assertEqual(tbl.coltypes['e_name'], 'float32')
183        self.assertEqual(tbl.coldtypes['e_name'].shape, ())
184
185        self.assertEqual(tbl.coltypes['f_name'], 'float64')
186        self.assertEqual(tbl.coldtypes['f_name'].shape, (10,))
187
188        self.assertEqual(tbl.coltypes['g_name'], 'uint8')
189        self.assertEqual(tbl.coldtypes['g_name'].shape, ())
190
191        for m in range(len(tbl)):
192            row = tbl[m]
193        # This version of the loop seems to fail because of ``iterrows()``.
194        # for (m, row) in enumerate(tbl):
195            self.assertEqual(row['a_name'], m)
196            self.assertEqual(row['c_name'], b"Hello!")
197            dRow = row['d_name']
198            for n in range(5):
199                for o in range(10):
200                    self.assertEqual(dRow[n][o], m + n + o)
201            self.assertAlmostEqual(row['e_name'], m * 0.96, places=6)
202            fRow = row['f_name']
203            for n in range(10):
204                self.assertAlmostEqual(fRow[n], m * 1024.9637)
205            self.assertEqual(row['g_name'], ord('m'))
206
207
208class ContiguousCompoundTestCase(common.TestFileMixin, TestCase):
209    """Test for support of native contiguous compound datasets.
210
211    This example has been provided by Dav Clark.
212
213    """
214
215    h5fname = test_filename('non-chunked-table.h5')
216
217    def test(self):
218        self.assertIn('/test_var/structure variable', self.h5file)
219
220        tbl = self.h5file.get_node('/test_var/structure variable')
221        self.assertIsInstance(tbl, tables.Table)
222
223        self.assertEqual(
224            tbl.colnames,
225            ['a', 'b', 'c', 'd'])
226
227        self.assertEqual(tbl.coltypes['a'], 'float64')
228        self.assertEqual(tbl.coldtypes['a'].shape, ())
229
230        self.assertEqual(tbl.coltypes['b'], 'float64')
231        self.assertEqual(tbl.coldtypes['b'].shape, ())
232
233        self.assertEqual(tbl.coltypes['c'], 'float64')
234        self.assertEqual(tbl.coldtypes['c'].shape, (2,))
235
236        self.assertEqual(tbl.coltypes['d'], 'string')
237        self.assertEqual(tbl.coldtypes['d'].shape, ())
238
239        for row in tbl.iterrows():
240            self.assertEqual(row['a'], 3.0)
241            self.assertEqual(row['b'], 4.0)
242            self.assertTrue(allequal(row['c'], numpy.array([2.0, 3.0],
243                                                           dtype="float64")))
244            self.assertEqual(row['d'], b"d")
245
246        self.h5file.close()
247
248
249class ContiguousCompoundAppendTestCase(common.TestFileMixin, TestCase):
250    """Test for appending data to native contiguous compound datasets."""
251
252    h5fname = test_filename('non-chunked-table.h5')
253
254    def test(self):
255        self.assertIn('/test_var/structure variable', self.h5file)
256        self.h5file.close()
257        # Do a copy to a temporary to avoid modifying the original file
258        h5fname_copy = tempfile.mktemp(".h5")
259        shutil.copy(self.h5fname, h5fname_copy)
260        # Reopen in 'a'ppend mode
261        try:
262            self.h5file = tables.open_file(h5fname_copy, 'a')
263        except IOError:
264            # Problems for opening (probably not permisions to write the file)
265            return
266        tbl = self.h5file.get_node('/test_var/structure variable')
267        # Try to add rows to a non-chunked table (this should raise an error)
268        self.assertRaises(tables.HDF5ExtError, tbl.append,
269                          [(4.0, 5.0, [2.0, 3.0], 'd')])
270        # Appending using the Row interface
271        self.assertRaises(tables.HDF5ExtError, tbl.row.append)
272        # Remove the file copy
273        self.h5file.close()  # Close the handler first
274        os.remove(h5fname_copy)
275
276
277class ExtendibleTestCase(common.TestFileMixin, TestCase):
278    """Test for extendible datasets.
279
280    See the example programs in the Introduction to HDF5.
281
282    """
283
284    h5fname = test_filename('smpl_SDSextendible.h5')
285
286    def test(self):
287        self.assertIn('/ExtendibleArray', self.h5file)
288
289        arr = self.h5file.get_node('/ExtendibleArray')
290        self.assertIsInstance(arr, tables.EArray)
291
292        self.assertEqual(arr.byteorder, 'big')
293        self.assertEqual(arr.atom.type, 'int32')
294        self.assertEqual(arr.shape, (10, 5))
295        self.assertEqual(arr.extdim, 0)
296        self.assertEqual(len(arr), 10)
297
298        data = arr.read()
299        expectedData = numpy.array([
300            [1, 1, 1, 3, 3],
301            [1, 1, 1, 3, 3],
302            [1, 1, 1, 0, 0],
303            [2, 0, 0, 0, 0],
304            [2, 0, 0, 0, 0],
305            [2, 0, 0, 0, 0],
306            [2, 0, 0, 0, 0],
307            [2, 0, 0, 0, 0],
308            [2, 0, 0, 0, 0],
309            [2, 0, 0, 0, 0]], dtype=arr.atom.type)
310
311        self.assertTrue(common.areArraysEqual(data, expectedData))
312
313
314class SzipTestCase(common.TestFileMixin, TestCase):
315    """Test for native HDF5 files with datasets compressed with szip."""
316
317    h5fname = test_filename('test_szip.h5')
318
319    def test(self):
320        self.assertIn('/dset_szip', self.h5file)
321
322        arr = self.h5file.get_node('/dset_szip')
323        filters = ("Filters(complib='szip', shuffle=False, bitshuffle=False, "
324                   "fletcher32=False, least_significant_digit=None)")
325        self.assertEqual(repr(arr.filters), filters)
326
327
328# this demonstrates github #203
329class MatlabFileTestCase(common.TestFileMixin, TestCase):
330    h5fname = test_filename('matlab_file.mat')
331
332    def test_unicode(self):
333        array = self.h5file.get_node('/', 'a')
334        self.assertEqual(array.shape, (3, 1))
335
336    # in Python 3 this will be the same as the test above
337    def test_string(self):
338        array = self.h5file.get_node('/', 'a')
339        self.assertEqual(array.shape, (3, 1))
340
341    def test_numpy_str(self):
342        array = self.h5file.get_node(numpy.str_('/'), numpy.str_('a'))
343        self.assertEqual(array.shape, (3, 1))
344
345
346class ObjectReferenceTestCase(common.TestFileMixin, TestCase):
347    h5fname = test_filename('test_ref_array1.mat')
348
349    def test_node_var(self):
350        array = self.h5file.get_node('/ANN/my_arr')
351        self.assertEqual(array.shape, (1, 3))
352
353    def test_ref_utf_str(self):
354        array = self.h5file.get_node('/ANN/my_arr')
355
356        self.assertTrue(common.areArraysEqual(
357                        array[0][0][0],
358                        numpy.array([0, 0],
359                                    dtype=numpy.uint64)))
360
361
362class ObjectReferenceRecursiveTestCase(common.TestFileMixin, TestCase):
363    h5fname = test_filename('test_ref_array2.mat')
364
365    def test_var(self):
366        array = self.h5file.get_node('/var')
367        self.assertEqual(array.shape, (3, 1))
368
369    def test_ref_str(self):
370        array = self.h5file.get_node('/var')
371
372        self.assertTrue(common.areArraysEqual(
373                        array[1][0][0],
374                        numpy.array([[116], [101], [115], [116]],
375                                    dtype=numpy.uint16)))
376
377    def test_double_ref(self):
378        array = self.h5file.get_node('/var')
379        self.assertTrue(common.areArraysEqual(
380                        array[2][0][0][1][0],
381                        numpy.array([[105], [110], [115], [105], [100], [101]],
382                                    dtype=numpy.uint16)))
383
384
385def suite():
386    """Return a test suite consisting of all the test cases in the module."""
387
388    theSuite = unittest.TestSuite()
389    niter = 1
390
391    for i in range(niter):
392        theSuite.addTest(unittest.makeSuite(PaddedArrayTestCase))
393        theSuite.addTest(unittest.makeSuite(EnumTestCase))
394        theSuite.addTest(unittest.makeSuite(F64BETestCase))
395        theSuite.addTest(unittest.makeSuite(F64LETestCase))
396        theSuite.addTest(unittest.makeSuite(I64BETestCase))
397        theSuite.addTest(unittest.makeSuite(I64LETestCase))
398        theSuite.addTest(unittest.makeSuite(I32BETestCase))
399        theSuite.addTest(unittest.makeSuite(I32LETestCase))
400        theSuite.addTest(unittest.makeSuite(ChunkedCompoundTestCase))
401        theSuite.addTest(unittest.makeSuite(ContiguousCompoundTestCase))
402        theSuite.addTest(unittest.makeSuite(ContiguousCompoundAppendTestCase))
403        theSuite.addTest(unittest.makeSuite(ExtendibleTestCase))
404        theSuite.addTest(unittest.makeSuite(SzipTestCase))
405        theSuite.addTest(unittest.makeSuite(MatlabFileTestCase))
406        theSuite.addTest(unittest.makeSuite(ObjectReferenceTestCase))
407        theSuite.addTest(unittest.makeSuite(ObjectReferenceRecursiveTestCase))
408
409    return theSuite
410
411
412if __name__ == '__main__':
413    import sys
414    common.parse_argv(sys.argv)
415    common.print_versions()
416    unittest.main(defaultTest='suite')
417
418
419## Local Variables:
420## mode: python
421## py-indent-offset: 4
422## tab-width: 4
423## fill-column: 72
424## End:
425