1 /*************************************************************************** 2 * Copyright (c) Johan Mabille, Sylvain Corlay and Wolf Vollprecht * 3 * Copyright (c) QuantStack * 4 * * 5 * Distributed under the terms of the BSD 3-Clause License. * 6 * * 7 * The full license is in the file LICENSE, distributed with this software. * 8 ****************************************************************************/ 9 10 #include "gtest/gtest.h" 11 12 #include <xtensor/xbroadcast.hpp> 13 #include "xtensor-io/xchunk_store_manager.hpp" 14 #include "xtensor-io/xfile_array.hpp" 15 #include <xtensor-io/xio_binary.hpp> 16 #include "xtensor-io/xio_disk_handler.hpp" 17 18 namespace xt 19 { 20 namespace fs = ghc::filesystem; 21 22 template <class S> make_test_chunked_array(const S & shape,const S & chunk_shape,const std::string & chunk_dir,size_t pool_size,bool init=false,double init_value=0)23 inline auto make_test_chunked_array(const S& shape, 24 const S& chunk_shape, 25 const std::string& chunk_dir, 26 size_t pool_size, 27 bool init=false, 28 double init_value=0) 29 { 30 if (init) 31 { 32 return chunked_file_array<double, xio_disk_handler<xio_binary_config>>(shape, chunk_shape, chunk_dir, init_value, pool_size); 33 } 34 else 35 { 36 return chunked_file_array<double, xio_disk_handler<xio_binary_config>>(shape, chunk_shape, chunk_dir, pool_size); 37 } 38 } 39 TEST(xchunked_array,disk_array)40 TEST(xchunked_array, disk_array) 41 { 42 std::vector<size_t> shape = {4, 4}; 43 std::vector<size_t> chunk_shape = {2, 2}; 44 std::string chunk_dir = "files0"; 45 fs::create_directory(chunk_dir); 46 std::size_t pool_size = 2; 47 auto a1 = make_test_chunked_array(shape, chunk_shape, chunk_dir, pool_size); 48 std::vector<size_t> idx = {1, 2}; 49 double v1 = 3.4; 50 double v2 = 5.6; 51 double v3 = 7.8; 52 a1(2, 1) = v1; 53 a1[idx] = v2; 54 a1(0, 0) = v3; // this should unload chunk 1.0 55 EXPECT_EQ(a1(2, 1), v1); 56 EXPECT_EQ(a1[idx], v2); 57 EXPECT_EQ(a1(0, 0), v3); 58 59 std::ifstream in_file; 60 xt::xarray<double> data; 61 in_file.open(chunk_dir + "/1.0"); 62 auto i1 = xt::xistream_wrapper(in_file); 63 data = xt::load_bin<double>(i1); 64 EXPECT_EQ(data(1), v1); 65 in_file.close(); 66 67 a1.chunks().flush(); 68 in_file.open(chunk_dir + "/0.1"); 69 auto i2 = xt::xistream_wrapper(in_file); 70 data = xt::load_bin<double>(i2); 71 EXPECT_EQ(data(2), v2); 72 in_file.close(); 73 74 in_file.open(chunk_dir + "/0.0"); 75 auto i3 = xt::xistream_wrapper(in_file); 76 data = xt::load_bin<double>(i3); 77 EXPECT_EQ(data(0), v3); 78 in_file.close(); 79 } 80 TEST(xchunked_array,assign_disk_array)81 TEST(xchunked_array, assign_disk_array) 82 { 83 std::vector<size_t> shape = {4, 4}; 84 std::vector<size_t> chunk_shape = {2, 2}; 85 std::size_t pool_size = 2; 86 std::string chunk_dir1 = "files1"; 87 std::string chunk_dir2 = "files2"; 88 fs::create_directory(chunk_dir1); 89 fs::create_directory(chunk_dir2); 90 91 auto a1 = make_test_chunked_array(shape, chunk_shape, chunk_dir1, pool_size); 92 double v1[16u]; 93 double sv1 = 1.; 94 std::generate(v1, v1+16u, [&sv1]() { return sv1++ + 0.5; }); 95 std::copy(v1, v1+16u, a1.begin()); 96 a1.chunks().flush(); 97 98 auto a2 = make_test_chunked_array(shape, chunk_shape, chunk_dir2, pool_size); 99 double v2[16u]; 100 double sv2 = 1.; 101 std::generate(v2, v2+16u, [&sv2]() { return sv2++ + 0.5; }); 102 std::copy(v2, v2+16u, a2.begin()); 103 104 a2 = a1; 105 a2.chunks().flush(); 106 107 for (ptrdiff_t i = 0; i < 16; ++i) 108 { 109 EXPECT_EQ(a2.begin()[i], v1[i]); 110 } 111 } 112 TEST(xchunked_array,init_value)113 TEST(xchunked_array, init_value) 114 { 115 std::vector<size_t> shape = {4, 4}; 116 std::vector<size_t> chunk_shape = {2, 2}; 117 std::string chunk_dir = "files3"; 118 fs::create_directory(chunk_dir); 119 std::size_t pool_size = 1; 120 double init_value = 5.5; 121 auto a1 = make_test_chunked_array(shape, chunk_shape, chunk_dir, pool_size, true, init_value); 122 for (auto v: a1) 123 { 124 EXPECT_EQ(v, init_value); 125 } 126 } 127 TEST(xchunked_array,shape_initializer_list)128 TEST(xchunked_array, shape_initializer_list) 129 { 130 auto a = chunked_file_array<double, xio_disk_handler<xio_binary_config>>({4, 4}, {2, 2}, "files3", 5.5); 131 for (auto v: a) 132 { 133 EXPECT_EQ(v, 5.5); 134 } 135 } 136 TEST(xchunked_array,create_directories)137 TEST(xchunked_array, create_directories) 138 { 139 std::vector<size_t> shape = {4, 4}; 140 std::vector<size_t> chunk_shape = {2, 2}; 141 std::string chunk_dir = "doesntexist"; 142 std::size_t pool_size = 2; 143 auto a1 = make_test_chunked_array(shape, chunk_shape, chunk_dir, pool_size); 144 xio_binary_config format_config; 145 xio_disk_config io_config; 146 io_config.create_directories = false; 147 a1.chunks().configure(format_config, io_config); 148 a1(0, 0) = 1.2; 149 // the directory doesn't exist and we didn't set the create_directories option 150 // so this should throw an exception 151 EXPECT_THROW(a1.chunks().flush(), std::runtime_error); 152 // now we set the create_directories option, this should not throw 153 io_config.create_directories = true; 154 a1.chunks().configure(format_config, io_config); 155 a1.chunks().flush(); 156 } 157 } 158