1 #ifndef XTENSOR_CHUNK_STORE_MANAGER_HPP 2 #define XTENSOR_CHUNK_STORE_MANAGER_HPP 3 4 #include <vector> 5 #include <array> 6 7 #include "ghc/filesystem.hpp" 8 9 #include <xtl/xsequence.hpp> 10 11 #include "xtensor/xarray.hpp" 12 #include "xtensor/xchunked_array.hpp" 13 #include "xfile_array.hpp" 14 15 namespace xt 16 { 17 template <class EC, class IP> 18 class xchunk_store_manager; 19 20 /*************************** 21 * xindex_path declaration * 22 ***************************/ 23 24 class xindex_path 25 { 26 public: 27 28 std::string get_directory() const; 29 void set_directory(const std::string& directory); 30 31 template <class I> 32 void index_to_path(I, I, std::string&) const; 33 34 private: 35 36 std::string m_directory; 37 }; 38 39 /********************************* 40 * xchunked_assigner declaration * 41 *********************************/ 42 43 template <class T, class EC, class IP> 44 class xchunked_assigner<T, xchunk_store_manager<EC, IP>> 45 { 46 public: 47 48 using temporary_type = T; 49 50 template <class E, class DST> 51 void build_and_assign_temporary(const xexpression<E>& e, DST& dst); 52 }; 53 54 /************************************ 55 * xchunk_store_manager declaration * 56 ************************************/ 57 58 template <class EC, class IP> 59 struct xcontainer_inner_types<xchunk_store_manager<EC, IP>> 60 { 61 using storage_type = EC; 62 using reference = EC&; 63 using const_reference = const EC&; 64 using size_type = std::size_t; 65 using temporary_type = xchunk_store_manager<EC, IP>; 66 }; 67 68 template <class EC, class IP> 69 struct xiterable_inner_types<xchunk_store_manager<EC, IP>> 70 { 71 using inner_shape_type = std::vector<std::size_t>; 72 using stepper = xindexed_stepper<xchunk_store_manager<EC, IP>, false>; 73 using const_stepper = xindexed_stepper<xchunk_store_manager<EC, IP>, true>; 74 }; 75 76 /** 77 * @class xchunk_store_manager 78 * @brief Multidimensional chunk container and manager. 79 * 80 * The xchunk_store_manager class implements a multidimensional chunk container. 81 * Chunks are managed in a pool, allowing for a limited number of chunks 82 * that can simultaneously be hold in memory, by swapping chunks when the pool 83 * is full. Should not be used directly, instead use chunked_file_array factory 84 * functions. 85 * 86 * @tparam EC The type of a chunk (e.g. xfile_array) 87 * @tparam IP The type of the index-to-path transformer (default: xindex_path) 88 */ 89 template <class EC, class IP = xindex_path> 90 class xchunk_store_manager: public xaccessible<xchunk_store_manager<EC, IP>>, 91 public xiterable<xchunk_store_manager<EC, IP>> 92 { 93 public: 94 95 using self_type = xchunk_store_manager<EC, IP>; 96 using inner_types = xcontainer_inner_types<self_type>; 97 using storage_type = typename inner_types::storage_type; 98 using value_type = storage_type; 99 using reference = EC&; 100 using const_reference = const EC&; 101 using pointer = value_type*; 102 using const_pointer = const value_type*; 103 using size_type = typename inner_types::size_type; 104 using difference_type = std::ptrdiff_t; 105 using iterable_base = xconst_iterable<self_type>; 106 using stepper = typename iterable_base::stepper; 107 using const_stepper = typename iterable_base::const_stepper; 108 using shape_type = typename iterable_base::inner_shape_type; 109 110 template <class S> 111 xchunk_store_manager(S&& shape, 112 S&& chunk_shape, 113 const std::string& directory, 114 std::size_t pool_size, 115 layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); 116 117 template <class S, class T> 118 xchunk_store_manager(S&& shape, 119 S&& chunk_shape, 120 const std::string& directory, 121 std::size_t pool_size, 122 const T& init_value, 123 layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); 124 125 ~xchunk_store_manager() = default; 126 127 xchunk_store_manager(const xchunk_store_manager&) = default; 128 xchunk_store_manager& operator=(const xchunk_store_manager&) = default; 129 130 xchunk_store_manager(xchunk_store_manager&&) = default; 131 xchunk_store_manager& operator=(xchunk_store_manager&&) = default; 132 133 const shape_type& shape() const noexcept; 134 const shape_type& chunk_shape() const noexcept; 135 136 template <class... Idxs> 137 reference operator()(Idxs... idxs); 138 139 template <class... Idxs> 140 const_reference operator()(Idxs... idxs) const; 141 142 template <class It> 143 reference element(It first, It last); 144 145 template <class It> 146 const_reference element(It first, It last) const; 147 148 template <class O> 149 stepper stepper_begin(const O& shape) noexcept; 150 template <class O> 151 stepper stepper_end(const O& shape, layout_type) noexcept; 152 153 template <class O> 154 const_stepper stepper_begin(const O& shape) const noexcept; 155 template <class O> 156 const_stepper stepper_end(const O& shape, layout_type) const noexcept; 157 158 template <class S> 159 void resize(S&& shape); 160 161 size_type size() const; 162 const std::string& get_directory() const; 163 bool get_pool_size() const; 164 165 IP& get_index_path(); 166 void flush(); 167 168 template <class FC, class IOC> 169 void configure(FC& format_config, IOC& io_config); 170 171 template <class I> 172 reference map_file_array(I first, I last); 173 174 template <class I> 175 const_reference map_file_array(I first, I last) const; 176 177 std::string get_temporary_directory() const; 178 void reset_to_directory(const std::string& directory); 179 180 private: 181 182 template <class... Idxs> 183 std::array<std::size_t, sizeof...(Idxs)> get_indexes(Idxs... idxs) const; 184 185 template <class S, class T> 186 void initialize(S&& shape, 187 S&& chunk_shape, 188 const std::string& directory, 189 bool init, 190 const T& init_value, 191 std::size_t pool_size, 192 layout_type chunk_memory_layout); 193 194 using chunk_pool_type = std::vector<EC>; 195 using index_pool_type = std::vector<shape_type>; 196 197 shape_type m_shape; 198 shape_type m_chunk_shape; 199 chunk_pool_type m_chunk_pool; 200 index_pool_type m_index_pool; 201 std::size_t m_unload_index; 202 IP m_index_path; 203 }; 204 205 /** 206 * Creates a chunked file array. 207 * This function returns an uninitialized ``xchunked_array<xchunk_store_manager<xfile_array<T, IOH>>>``. 208 * 209 * @tparam T The type of the elements (e.g. double) 210 * @tparam IOH The type of the IO handler (e.g. xio_disk_handler) 211 * @tparam L The layout_type of the array 212 * @tparam IP The type of the index-to-path transformer (default: xindex_path) 213 * 214 * @param shape The shape of the array 215 * @param chunk_shape The shape of a chunk 216 * @param path The path to the chunk store 217 * @param pool_size The size of the chunk pool (default: 1) 218 * @param chunk_memory_layout The layout of each chunk (default: XTENSOR_DEFAULT_LAYOUT) 219 * 220 * @return returns a ``xchunked_array<xchunk_store_manager<xfile_array<T, IOH>>>`` with the given shape, chunk shape and memory layout. 221 */ 222 template <class T, class IOH, layout_type L = XTENSOR_DEFAULT_LAYOUT, class IP = xindex_path, class S> 223 xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>> 224 chunked_file_array(S&& shape, 225 S&& chunk_shape, 226 const std::string& path, 227 std::size_t pool_size = 1, 228 layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); 229 230 template <class T, class IOH, layout_type L = XTENSOR_DEFAULT_LAYOUT, class IP = xindex_path, class S> 231 xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>> 232 chunked_file_array(std::initializer_list<S> shape, 233 std::initializer_list<S> chunk_shape, 234 const std::string& path, 235 std::size_t pool_size = 1, 236 layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); 237 238 /** 239 * Creates a chunked file array. 240 * This function returns an uninitialized ``xchunked_array<xchunk_store_manager<xfile_array<T, IOH>>>``. 241 * 242 * @tparam T The type of the elements (e.g. double) 243 * @tparam IOH The type of the IO handler (e.g. xio_disk_handler) 244 * @tparam L The layout_type of the array 245 * @tparam IP The type of the index-to-path transformer (default: xindex_path) 246 * 247 * @param shape The shape of the array 248 * @param chunk_shape The shape of a chunk 249 * @param path The path to the chunk store 250 * @param init_value The value with which to initialize the chunks when they are not already stored 251 * @param pool_size The size of the chunk pool (default: 1) 252 * @param chunk_memory_layout The layout of each chunk (default: XTENSOR_DEFAULT_LAYOUT) 253 * 254 * @return returns a ``xchunked_array<xchunk_store_manager<xfile_array<T, IOH>>>`` with the given shape, chunk shape and memory layout. 255 */ 256 template <class T, class IOH, layout_type L = XTENSOR_DEFAULT_LAYOUT, class IP = xindex_path, class S> 257 xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>> 258 chunked_file_array(S&& shape, 259 S&& chunk_shape, 260 const std::string& path, 261 const T& init_value, 262 std::size_t pool_size = 1, 263 layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); 264 265 template <class T, class IOH, layout_type L = XTENSOR_DEFAULT_LAYOUT, class IP = xindex_path, class S> 266 xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>> 267 chunked_file_array(std::initializer_list<S> shape, 268 std::initializer_list<S> chunk_shape, 269 const std::string& path, 270 const T& init_value, 271 std::size_t pool_size = 1, 272 layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); 273 274 /** 275 * Creates a chunked file array. 276 * This function returns a ``xchunked_array<xchunk_store_manager<xfile_array<T, IOH>>>`` initialized from an expression. 277 * 278 * @tparam T The type of the elements (e.g. double) 279 * @tparam IOH The type of the IO handler (e.g. xio_disk_handler) 280 * @tparam L The layout_type of the array 281 * @tparam IP The type of the index-to-path transformer (default: xindex_path) 282 * 283 * @param e The expression to initialize the chunked array from 284 * @param chunk_shape The shape of a chunk 285 * @param path The path to the chunk store 286 * @param pool_size The size of the chunk pool (default: 1) 287 * @param chunk_memory_layout The layout of each chunk (default: XTENSOR_DEFAULT_LAYOUT) 288 * 289 * @return returns a ``xchunked_array<xchunk_store_manager<xfile_array<T, IOH>>>`` from the given expression, with the given chunk shape and memory layout. 290 */ 291 template <class IOH, layout_type L = XTENSOR_DEFAULT_LAYOUT, class IP = xindex_path, class E, class S> 292 xchunked_array<xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>> 293 chunked_file_array(const xexpression<E>& e, 294 S&& chunk_shape, 295 const std::string& path, 296 std::size_t pool_size = 1, 297 layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); 298 299 /** 300 * Creates a chunked file array. 301 * This function returns a ``xchunked_array<xchunk_store_manager<xfile_array<T, IOH>>>`` initialized from an expression. 302 * 303 * @tparam IOH The type of the IO handler (e.g. xio_disk_handler) 304 * @tparam L The layout_type of the array 305 * @tparam IP The type of the index-to-path transformer (default: xindex_path) 306 * 307 * @param e The expression to initialize the chunked array from 308 * @param path The path to the chunk store 309 * @param pool_size The size of the chunk pool (default: 1) 310 * @param chunk_memory_layout The layout of each chunk (default: XTENSOR_DEFAULT_LAYOUT) 311 * 312 * @return returns a ``xchunked_array<xchunk_store_manager<xfile_array<T, IOH>>>`` from the given expression, with the expression's chunk shape and the given memory layout. 313 */ 314 template <class IOH, layout_type L = XTENSOR_DEFAULT_LAYOUT, class IP = xindex_path, class E> 315 xchunked_array<xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>> 316 chunked_file_array(const xexpression<E>& e, 317 const std::string& path, 318 std::size_t pool_size = 1, 319 layout_type chunk_memory_layout = XTENSOR_DEFAULT_LAYOUT); 320 321 /****************************** 322 * xindex_path implementation * 323 ******************************/ 324 get_directory() const325 inline std::string xindex_path::get_directory() const 326 { 327 return m_directory; 328 } 329 set_directory(const std::string & directory)330 inline void xindex_path::set_directory(const std::string& directory) 331 { 332 m_directory = directory; 333 if (m_directory.back() != '/') 334 { 335 m_directory.push_back('/'); 336 } 337 } 338 339 template <class I> index_to_path(I first,I last,std::string & path) const340 void xindex_path::index_to_path(I first, I last, std::string& path) const 341 { 342 std::string fname; 343 for (auto it = first; it != last; ++it) 344 { 345 if (!fname.empty()) 346 { 347 fname.push_back('.'); 348 } 349 fname.append(std::to_string(*it)); 350 } 351 path = m_directory + fname; 352 } 353 354 /************************************ 355 * xchunked_assigner implementation * 356 ************************************/ 357 358 template <class T, class EC, class IP> 359 template <class E, class DST> build_and_assign_temporary(const xexpression<E> & e,DST & dst)360 inline void xchunked_assigner<T, xchunk_store_manager<EC, IP>>::build_and_assign_temporary(const xexpression<E>& e, 361 DST& dst) 362 { 363 using store_type = xchunk_store_manager<EC, IP>; 364 store_type store(e.derived_cast().shape(), dst.chunk_shape(), dst.chunks().get_temporary_directory(), dst.chunks().get_pool_size()); 365 temporary_type tmp(e, std::move(store), dst.chunk_shape()); 366 tmp.chunks().flush(); 367 dst.chunks().reset_to_directory(tmp.chunks().get_directory()); 368 } 369 370 /****************************************** 371 * xchunk_store_manager factory functions * 372 ******************************************/ 373 374 template <class T, class IOH, layout_type L, class IP, class S> 375 inline xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>> chunked_file_array(S && shape,S && chunk_shape,const std::string & path,std::size_t pool_size,layout_type chunk_memory_layout)376 chunked_file_array(S&& shape, S&& chunk_shape, const std::string& path, std::size_t pool_size, layout_type chunk_memory_layout) 377 { 378 using chunk_storage = xchunk_store_manager<xfile_array<T, IOH, L>, IP>; 379 chunk_storage chunks(shape, chunk_shape, path, pool_size, chunk_memory_layout); 380 return xchunked_array<chunk_storage>(std::move(chunks), std::forward<S>(shape), std::forward<S>(chunk_shape)); 381 } 382 383 template <class T, class IOH, layout_type L, class IP, class S> 384 inline xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>> chunked_file_array(std::initializer_list<S> shape,std::initializer_list<S> chunk_shape,const std::string & path,std::size_t pool_size,layout_type chunk_memory_layout)385 chunked_file_array(std::initializer_list<S> shape, std::initializer_list<S> chunk_shape, const std::string& path, std::size_t pool_size, layout_type chunk_memory_layout) 386 { 387 using sh_type = std::vector<std::size_t>; 388 auto sh = xtl::forward_sequence<sh_type, std::initializer_list<S>>(shape); 389 auto ch_sh = xtl::forward_sequence<sh_type, std::initializer_list<S>>(chunk_shape); 390 return chunked_file_array<T, IOH, L, IP, sh_type>(std::move(sh), std::move(ch_sh), path, pool_size, chunk_memory_layout); 391 } 392 393 template <class T, class IOH, layout_type L, class IP, class S> 394 inline xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>> chunked_file_array(S && shape,S && chunk_shape,const std::string & path,const T & init_value,std::size_t pool_size,layout_type chunk_memory_layout)395 chunked_file_array(S&& shape, S&& chunk_shape, const std::string& path, const T& init_value, std::size_t pool_size, layout_type chunk_memory_layout) 396 { 397 using chunk_storage = xchunk_store_manager<xfile_array<T, IOH, L>, IP>; 398 chunk_storage chunks(shape, chunk_shape, path, pool_size, init_value, chunk_memory_layout); 399 return xchunked_array<chunk_storage>(std::move(chunks), std::forward<S>(shape), std::forward<S>(chunk_shape)); 400 } 401 402 template <class T, class IOH, layout_type L, class IP, class S> 403 inline xchunked_array<xchunk_store_manager<xfile_array<T, IOH, L>, IP>> chunked_file_array(std::initializer_list<S> shape,std::initializer_list<S> chunk_shape,const std::string & path,const T & init_value,std::size_t pool_size,layout_type chunk_memory_layout)404 chunked_file_array(std::initializer_list<S> shape, std::initializer_list<S> chunk_shape, const std::string& path, const T& init_value, std::size_t pool_size, layout_type chunk_memory_layout) 405 { 406 using sh_type = std::vector<std::size_t>; 407 auto sh = xtl::forward_sequence<sh_type, std::initializer_list<S>>(shape); 408 auto ch_sh = xtl::forward_sequence<sh_type, std::initializer_list<S>>(chunk_shape); 409 return chunked_file_array<T, IOH, L, IP, sh_type>(std::move(sh), std::move(ch_sh), path, init_value, pool_size, chunk_memory_layout); 410 } 411 412 template <class IOH, layout_type L, class IP, class E, class S> 413 inline xchunked_array<xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>> chunked_file_array(const xexpression<E> & e,S && chunk_shape,const std::string & path,std::size_t pool_size,layout_type chunk_memory_layout)414 chunked_file_array(const xexpression<E>& e, S&& chunk_shape, const std::string& path, std::size_t pool_size, layout_type chunk_memory_layout) 415 { 416 using chunk_storage = xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>; 417 chunk_storage chunks(e.derived_cast().shape(), chunk_shape, path, pool_size, chunk_memory_layout); 418 return xchunked_array<chunk_storage>(e, chunk_storage(), std::forward<S>(chunk_shape)); 419 } 420 421 template <class IOH, layout_type L, class IP, class E> 422 inline xchunked_array<xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>> chunked_file_array(const xexpression<E> & e,const std::string & path,std::size_t pool_size,layout_type chunk_memory_layout)423 chunked_file_array(const xexpression<E>& e, const std::string& path, std::size_t pool_size, layout_type chunk_memory_layout) 424 { 425 using chunk_storage = xchunk_store_manager<xfile_array<typename E::value_type, IOH, L>, IP>; 426 chunk_storage chunks(e.derived_cast().shape(), detail::chunk_helper<E>::chunk_shape(e), path, pool_size, chunk_memory_layout); 427 return xchunked_array<chunk_storage>(e, chunk_storage()); 428 } 429 430 /*************************************** 431 * xchunk_store_manager implementation * 432 ***************************************/ 433 434 template <class EC, class IP> 435 template <class S> xchunk_store_manager(S && shape,S && chunk_shape,const std::string & directory,std::size_t pool_size,layout_type chunk_memory_layout)436 inline xchunk_store_manager<EC, IP>::xchunk_store_manager(S&& shape, 437 S&& chunk_shape, 438 const std::string& directory, 439 std::size_t pool_size, 440 layout_type chunk_memory_layout) 441 : m_shape(xtl::forward_sequence<shape_type, S>(shape)) 442 , m_chunk_shape(xtl::forward_sequence<shape_type, S>(chunk_shape)) 443 , m_unload_index(0u) 444 { 445 initialize(shape, chunk_shape, directory, false, 0, pool_size, chunk_memory_layout); 446 } 447 448 template <class EC, class IP> 449 template <class S, class T> xchunk_store_manager(S && shape,S && chunk_shape,const std::string & directory,std::size_t pool_size,const T & init_value,layout_type chunk_memory_layout)450 inline xchunk_store_manager<EC, IP>::xchunk_store_manager(S&& shape, 451 S&& chunk_shape, 452 const std::string& directory, 453 std::size_t pool_size, 454 const T& init_value, 455 layout_type chunk_memory_layout) 456 : m_shape(xtl::forward_sequence<shape_type, S>(shape)) 457 , m_chunk_shape(xtl::forward_sequence<shape_type, S>(chunk_shape)) 458 , m_unload_index(0u) 459 { 460 initialize(shape, chunk_shape, directory, true, init_value, pool_size, chunk_memory_layout); 461 } 462 463 template <class EC, class IP> 464 template <class S, class T> initialize(S && shape,S && chunk_shape,const std::string & directory,bool init,const T & init_value,std::size_t pool_size,layout_type chunk_memory_layout)465 inline void xchunk_store_manager<EC, IP>::initialize(S&& shape, 466 S&& chunk_shape, 467 const std::string& directory, 468 bool init, 469 const T& init_value, 470 std::size_t pool_size, 471 layout_type chunk_memory_layout) 472 { 473 if (pool_size == SIZE_MAX) 474 { 475 // as many "physical" chunks in the pool as there are "logical" chunks 476 pool_size = size(); 477 } 478 if (init) 479 { 480 m_chunk_pool.resize(pool_size, EC("", xfile_mode::init_on_fail, init_value)); 481 } 482 else 483 { 484 m_chunk_pool.resize(pool_size, EC("", xfile_mode::init_on_fail)); 485 } 486 m_index_pool.resize(pool_size); 487 // resize the pool chunks 488 for (auto& chunk: m_chunk_pool) 489 { 490 chunk.resize(chunk_shape, chunk_memory_layout); 491 } 492 m_index_path.set_directory(directory); 493 } 494 495 template <class EC, class IP> shape() const496 inline auto xchunk_store_manager<EC, IP>::shape() const noexcept -> const shape_type& 497 { 498 return m_shape; 499 } 500 501 template <class EC, class IP> chunk_shape() const502 inline auto xchunk_store_manager<EC, IP>::chunk_shape() const noexcept -> const shape_type& 503 { 504 return m_chunk_shape; 505 } 506 507 template <class EC, class IP> 508 template <class... Idxs> operator ()(Idxs...idxs)509 inline auto xchunk_store_manager<EC, IP>::operator()(Idxs... idxs) -> reference 510 { 511 auto index = get_indexes(idxs...); 512 return map_file_array(index.cbegin(), index.cend()); 513 } 514 515 template <class EC, class IP> 516 template <class... Idxs> operator ()(Idxs...idxs) const517 inline auto xchunk_store_manager<EC, IP>::operator()(Idxs... idxs) const -> const_reference 518 { 519 auto index = get_indexes(idxs...); 520 return map_file_array(index.cbegin(), index.cend()); 521 } 522 523 template <class EC, class IP> 524 template <class It> element(It first,It last)525 inline auto xchunk_store_manager<EC, IP>::element(It first, It last) -> reference 526 { 527 return map_file_array(first, last); 528 } 529 530 template <class EC, class IP> 531 template <class It> element(It first,It last) const532 inline auto xchunk_store_manager<EC, IP>::element(It first, It last) const -> const_reference 533 { 534 return map_file_array(first, last); 535 } 536 537 template <class EC, class IP> 538 template <class O> stepper_begin(const O & shape)539 inline auto xchunk_store_manager<EC, IP>::stepper_begin(const O& shape) noexcept -> stepper 540 { 541 size_type offset = shape.size() - this->dimension(); 542 return stepper(this, offset); 543 } 544 545 template <class EC, class IP> 546 template <class O> stepper_end(const O & shape,layout_type)547 inline auto xchunk_store_manager<EC, IP>::stepper_end(const O& shape, layout_type) noexcept -> stepper 548 { 549 size_type offset = shape.size() - this->dimension(); 550 return stepper(this, offset, true); 551 } 552 553 template <class EC, class IP> 554 template <class O> stepper_begin(const O & shape) const555 inline auto xchunk_store_manager<EC, IP>::stepper_begin(const O& shape) const noexcept -> const_stepper 556 { 557 size_type offset = shape.size() - this->dimension(); 558 return const_stepper(this, offset); 559 } 560 561 template <class EC, class IP> 562 template <class O> stepper_end(const O & shape,layout_type) const563 inline auto xchunk_store_manager<EC, IP>::stepper_end(const O& shape, layout_type) const noexcept -> const_stepper 564 { 565 size_type offset = shape.size() - this->dimension(); 566 return const_stepper(this, offset, true); 567 } 568 569 template <class EC, class IP> 570 template <class S> resize(S && shape)571 inline void xchunk_store_manager<EC, IP>::resize(S&& shape) 572 { 573 // don't resize according to total number of chunks 574 // instead the pool manages a number of in-memory chunks 575 m_shape = shape; 576 } 577 578 template <class EC, class IP> size() const579 inline auto xchunk_store_manager<EC, IP>::size() const -> size_type 580 { 581 return compute_size(m_shape); 582 } 583 584 template <class EC, class IP> get_directory() const585 inline const std::string& xchunk_store_manager<EC, IP>::get_directory() const 586 { 587 return m_index_path.get_directory(); 588 } 589 590 template <class EC, class IP> get_pool_size() const591 inline bool xchunk_store_manager<EC, IP>::get_pool_size() const 592 { 593 return m_chunk_pool.size(); 594 } 595 596 template <class EC, class IP> flush()597 inline void xchunk_store_manager<EC, IP>::flush() 598 { 599 for (auto& chunk: m_chunk_pool) 600 { 601 chunk.flush(); 602 } 603 } 604 605 template <class EC, class IP> 606 template <class FC, class IOC> configure(FC & format_config,IOC & io_config)607 void xchunk_store_manager<EC, IP>::configure(FC& format_config, IOC& io_config) 608 { 609 for (auto& chunk: m_chunk_pool) 610 { 611 chunk.configure(format_config, io_config); 612 } 613 } 614 615 template <class EC, class IP> get_index_path()616 IP& xchunk_store_manager<EC, IP>::get_index_path() 617 { 618 return m_index_path; 619 } 620 621 template <class EC, class IP> 622 template <class I> map_file_array(I first,I last)623 inline auto xchunk_store_manager<EC, IP>::map_file_array(I first, I last) -> reference 624 { 625 std::string path; 626 m_index_path.index_to_path(first, last, path); 627 if (first == last) 628 { 629 return m_chunk_pool[0]; 630 } 631 else 632 { 633 // check if the chunk is already loaded in memory 634 const auto it1 = std::find_if(m_index_pool.cbegin(), m_index_pool.cend(), [first, last](const auto& v) 635 { return std::equal(v.cbegin(), v.cend(), first, last); }); 636 std::size_t i; 637 if (it1 != m_index_pool.cend()) 638 { 639 i = static_cast<std::size_t>(std::distance(m_index_pool.cbegin(), it1)); 640 return m_chunk_pool[i]; 641 } 642 // if not, find a free chunk in the pool 643 std::vector<std::size_t> empty_index; 644 const auto it2 = std::find(m_index_pool.cbegin(), m_index_pool.cend(), empty_index); 645 if (it2 != m_index_pool.cend()) 646 { 647 i = static_cast<std::size_t>(std::distance(m_index_pool.cbegin(), it2)); 648 m_chunk_pool[i].set_path(path); 649 m_index_pool[i].resize(static_cast<size_t>(std::distance(first, last))); 650 std::copy(first, last, m_index_pool[i].begin()); 651 return m_chunk_pool[i]; 652 } 653 // no free chunk, take one (which will thus be unloaded) 654 // fairness is guaranteed through the use of a walking index 655 m_chunk_pool[m_unload_index].set_path(path); 656 m_index_pool[m_unload_index].resize(static_cast<size_t>(std::distance(first, last))); 657 std::copy(first, last, m_index_pool[m_unload_index].begin()); 658 auto& chunk = m_chunk_pool[m_unload_index]; 659 m_unload_index = (m_unload_index + 1) % m_index_pool.size(); 660 return chunk; 661 } 662 } 663 664 template <class EC, class IP> 665 template <class I> map_file_array(I first,I last) const666 inline auto xchunk_store_manager<EC, IP>::map_file_array(I first, I last) const -> const_reference 667 { 668 return const_cast<xchunk_store_manager<EC, IP>*>(this)->map_file_array(first, last); 669 } 670 671 template <class EC, class IP> get_temporary_directory() const672 inline std::string xchunk_store_manager<EC, IP>::get_temporary_directory() const 673 { 674 namespace fs = ghc::filesystem; 675 fs::path tmp_dir = fs::temp_directory_path(); 676 std::size_t count = 0; 677 while(fs::exists(tmp_dir / std::to_string(count))) 678 { 679 ++count; 680 } 681 return tmp_dir / std::to_string(count); 682 } 683 684 template <class EC, class IP> reset_to_directory(const std::string & directory)685 inline void xchunk_store_manager<EC, IP>::reset_to_directory(const std::string& directory) 686 { 687 namespace fs = ghc::filesystem; 688 fs::remove_all(get_directory()); 689 fs::rename(directory, get_directory()); 690 m_unload_index = 0u; 691 } 692 693 template <class EC, class IP> 694 template <class... Idxs> 695 inline std::array<std::size_t, sizeof...(Idxs)> get_indexes(Idxs...idxs) const696 xchunk_store_manager<EC, IP>::get_indexes(Idxs... idxs) const 697 { 698 std::array<std::size_t, sizeof...(Idxs)> indexes = {{idxs...}}; 699 return indexes; 700 } 701 } 702 703 #endif 704