1 #ifndef OSMIUM_IO_COMPRESSION_HPP 2 #define OSMIUM_IO_COMPRESSION_HPP 3 4 /* 5 6 This file is part of Osmium (https://osmcode.org/libosmium). 7 8 Copyright 2013-2021 Jochen Topf <jochen@topf.org> and others (see README). 9 10 Boost Software License - Version 1.0 - August 17th, 2003 11 12 Permission is hereby granted, free of charge, to any person or organization 13 obtaining a copy of the software and accompanying documentation covered by 14 this license (the "Software") to use, reproduce, display, distribute, 15 execute, and transmit the Software, and to prepare derivative works of the 16 Software, and to permit third-parties to whom the Software is furnished to 17 do so, all subject to the following: 18 19 The copyright notices in the Software and this entire statement, including 20 the above license grant, this restriction and the following disclaimer, 21 must be included in all copies of the Software, in whole or in part, and 22 all derivative works of the Software, unless such copies or derivative 23 works are solely in the form of machine-executable object code generated by 24 a source language processor. 25 26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE 30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 32 DEALINGS IN THE SOFTWARE. 33 34 */ 35 36 #include <osmium/io/detail/read_write.hpp> 37 #include <osmium/io/error.hpp> 38 #include <osmium/io/file_compression.hpp> 39 #include <osmium/io/writer_options.hpp> 40 #include <osmium/util/file.hpp> 41 42 #include <atomic> 43 #include <cerrno> 44 #include <cstddef> 45 #include <functional> 46 #include <map> 47 #include <memory> 48 #include <string> 49 #include <system_error> 50 #include <tuple> 51 #include <utility> 52 53 namespace osmium { 54 55 namespace io { 56 57 class Compressor { 58 59 fsync m_fsync; 60 61 protected: 62 do_fsync() const63 bool do_fsync() const noexcept { 64 return m_fsync == fsync::yes; 65 } 66 67 public: 68 Compressor(const fsync sync)69 explicit Compressor(const fsync sync) noexcept : 70 m_fsync(sync) { 71 } 72 73 Compressor(const Compressor&) = default; 74 Compressor& operator=(const Compressor&) = default; 75 76 Compressor(Compressor&&) noexcept = default; 77 Compressor& operator=(Compressor&&) noexcept = default; 78 79 virtual ~Compressor() noexcept = default; 80 81 virtual void write(const std::string& data) = 0; 82 83 virtual void close() = 0; 84 file_size() const85 virtual std::size_t file_size() const { 86 return 0; 87 } 88 89 }; // class Compressor 90 91 class Decompressor { 92 93 std::atomic<std::size_t>* m_offset_ptr{nullptr}; 94 95 std::atomic_bool m_want_buffered_pages_removed{false}; 96 97 public: 98 99 enum { 100 input_buffer_size = 1024U * 1024U 101 }; 102 103 Decompressor() = default; 104 105 Decompressor(const Decompressor&) = delete; 106 Decompressor& operator=(const Decompressor&) = delete; 107 108 Decompressor(Decompressor&&) = delete; 109 Decompressor& operator=(Decompressor&&) = delete; 110 111 virtual ~Decompressor() noexcept = default; 112 113 virtual std::string read() = 0; 114 115 virtual void close() = 0; 116 is_real() const117 virtual bool is_real() const noexcept { 118 return true; 119 } 120 set_offset_ptr(std::atomic<std::size_t> * offset_ptr)121 void set_offset_ptr(std::atomic<std::size_t>* offset_ptr) noexcept { 122 m_offset_ptr = offset_ptr; 123 } 124 set_offset(const std::size_t offset)125 void set_offset(const std::size_t offset) noexcept { 126 if (m_offset_ptr) { 127 *m_offset_ptr = offset; 128 } 129 } 130 want_buffered_pages_removed() const131 bool want_buffered_pages_removed() const noexcept { 132 return m_want_buffered_pages_removed; 133 } 134 set_want_buffered_pages_removed(bool value)135 void set_want_buffered_pages_removed(bool value) noexcept { 136 m_want_buffered_pages_removed = value; 137 } 138 139 }; // class Decompressor 140 141 /** 142 * This singleton factory class is used to register compression 143 * algorithms used for reading and writing OSM files. 144 * 145 * For each algorithm we store two functions that construct 146 * a compressor and decompressor object, respectively. 147 */ 148 class CompressionFactory { 149 150 public: 151 152 using create_compressor_type = std::function<osmium::io::Compressor*(int, fsync)>; 153 using create_decompressor_type_fd = std::function<osmium::io::Decompressor*(int)>; 154 using create_decompressor_type_buffer = std::function<osmium::io::Decompressor*(const char*, std::size_t)>; 155 156 private: 157 158 using callbacks_type = std::tuple<create_compressor_type, 159 create_decompressor_type_fd, 160 create_decompressor_type_buffer>; 161 162 using compression_map_type = std::map<const osmium::io::file_compression, callbacks_type>; 163 164 compression_map_type m_callbacks; 165 166 CompressionFactory() = default; 167 find_callbacks(const osmium::io::file_compression compression) const168 const callbacks_type& find_callbacks(const osmium::io::file_compression compression) const { 169 const auto it = m_callbacks.find(compression); 170 171 if (it != m_callbacks.end()) { 172 return it->second; 173 } 174 175 std::string error_message{"Support for compression '"}; 176 error_message += as_string(compression); 177 error_message += "' not compiled into this binary"; 178 throw unsupported_file_format_error{error_message}; 179 } 180 181 public: 182 183 CompressionFactory(const CompressionFactory&) = delete; 184 CompressionFactory& operator=(const CompressionFactory&) = delete; 185 186 CompressionFactory(CompressionFactory&&) = delete; 187 CompressionFactory& operator=(CompressionFactory&&) = delete; 188 189 ~CompressionFactory() noexcept = default; 190 instance()191 static CompressionFactory& instance() { 192 static CompressionFactory factory; 193 return factory; 194 } 195 register_compression(osmium::io::file_compression compression,const create_compressor_type & create_compressor,const create_decompressor_type_fd & create_decompressor_fd,const create_decompressor_type_buffer & create_decompressor_buffer)196 bool register_compression( 197 osmium::io::file_compression compression, 198 const create_compressor_type& create_compressor, 199 const create_decompressor_type_fd& create_decompressor_fd, 200 const create_decompressor_type_buffer& create_decompressor_buffer) { 201 202 compression_map_type::value_type cc{compression, 203 std::make_tuple(create_compressor, 204 create_decompressor_fd, 205 create_decompressor_buffer)}; 206 207 return m_callbacks.insert(cc).second; 208 } 209 210 template <typename... TArgs> create_compressor(const osmium::io::file_compression compression,TArgs &&...args) const211 std::unique_ptr<osmium::io::Compressor> create_compressor(const osmium::io::file_compression compression, TArgs&&... args) const { 212 const auto callbacks = find_callbacks(compression); 213 return std::unique_ptr<osmium::io::Compressor>(std::get<0>(callbacks)(std::forward<TArgs>(args)...)); 214 } 215 create_decompressor(const osmium::io::file_compression compression,const int fd) const216 std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const int fd) const { 217 const auto callbacks = find_callbacks(compression); 218 return std::unique_ptr<osmium::io::Decompressor>(std::get<1>(callbacks)(fd)); 219 } 220 create_decompressor(const osmium::io::file_compression compression,const char * buffer,const std::size_t size) const221 std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const char* buffer, const std::size_t size) const { 222 const auto callbacks = find_callbacks(compression); 223 return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(callbacks)(buffer, size)); 224 } 225 226 }; // class CompressionFactory 227 228 class NoCompressor final : public Compressor { 229 230 std::size_t m_file_size = 0; 231 int m_fd; 232 233 public: 234 NoCompressor(const int fd,const fsync sync)235 NoCompressor(const int fd, const fsync sync) : 236 Compressor(sync), 237 m_fd(fd) { 238 } 239 240 NoCompressor(const NoCompressor&) = delete; 241 NoCompressor& operator=(const NoCompressor&) = delete; 242 243 NoCompressor(NoCompressor&&) = delete; 244 NoCompressor& operator=(NoCompressor&&) = delete; 245 ~NoCompressor()246 ~NoCompressor() noexcept override { 247 try { 248 close(); 249 } catch (...) { 250 // Ignore any exceptions because destructor must not throw. 251 } 252 } 253 write(const std::string & data)254 void write(const std::string& data) override { 255 osmium::io::detail::reliable_write(m_fd, data.data(), data.size()); 256 m_file_size += data.size(); 257 } 258 close()259 void close() override { 260 if (m_fd >= 0) { 261 const int fd = m_fd; 262 m_fd = -1; 263 264 // Do not sync or close stdout 265 if (fd == 1) { 266 return; 267 } 268 269 if (do_fsync()) { 270 osmium::io::detail::reliable_fsync(fd); 271 } 272 osmium::io::detail::reliable_close(fd); 273 } 274 } 275 file_size() const276 std::size_t file_size() const override { 277 return m_file_size; 278 } 279 280 }; // class NoCompressor 281 282 /** 283 * The DummyDecompressor is used when reading PBF files. In that 284 * case the PBFParser class is responsible for reading from the 285 * file itself, and the DummyDecompressor does nothing. 286 */ 287 class DummyDecompressor final : public Decompressor { 288 public: 289 290 DummyDecompressor() = default; 291 292 DummyDecompressor(const DummyDecompressor&) = delete; 293 DummyDecompressor& operator=(const DummyDecompressor&) = delete; 294 295 DummyDecompressor(DummyDecompressor&&) = delete; 296 DummyDecompressor& operator=(DummyDecompressor&&) = delete; 297 298 ~DummyDecompressor() noexcept override = default; 299 read()300 std::string read() override { 301 return {}; 302 } 303 close()304 void close() override { 305 } 306 is_real() const307 bool is_real() const noexcept override { 308 return false; 309 } 310 311 }; // class DummyDecompressor 312 313 class NoDecompressor final : public Decompressor { 314 315 int m_fd = -1; 316 const char* m_buffer = nullptr; 317 std::size_t m_buffer_size = 0; 318 std::size_t m_offset = 0; 319 320 public: 321 NoDecompressor(const int fd)322 explicit NoDecompressor(const int fd) : 323 m_fd(fd) { 324 } 325 NoDecompressor(const char * buffer,const std::size_t size)326 NoDecompressor(const char* buffer, const std::size_t size) : 327 m_buffer(buffer), 328 m_buffer_size(size) { 329 } 330 331 NoDecompressor(const NoDecompressor&) = delete; 332 NoDecompressor& operator=(const NoDecompressor&) = delete; 333 334 NoDecompressor(NoDecompressor&&) = delete; 335 NoDecompressor& operator=(NoDecompressor&&) = delete; 336 ~NoDecompressor()337 ~NoDecompressor() noexcept override { 338 try { 339 close(); 340 } catch (...) { 341 // Ignore any exceptions because destructor must not throw. 342 } 343 } 344 read()345 std::string read() override { 346 std::string buffer; 347 348 if (m_buffer) { 349 if (m_buffer_size != 0) { 350 const std::size_t size = m_buffer_size; 351 m_buffer_size = 0; 352 buffer.append(m_buffer, size); 353 } 354 } else { 355 buffer.resize(osmium::io::Decompressor::input_buffer_size); 356 if (want_buffered_pages_removed()) { 357 osmium::io::detail::remove_buffered_pages(m_fd, m_offset); 358 } 359 const auto nread = detail::reliable_read(m_fd, &*buffer.begin(), osmium::io::Decompressor::input_buffer_size); 360 buffer.resize(std::string::size_type(nread)); 361 } 362 363 m_offset += buffer.size(); 364 set_offset(m_offset); 365 366 return buffer; 367 } 368 close()369 void close() override { 370 if (m_fd >= 0) { 371 if (want_buffered_pages_removed()) { 372 osmium::io::detail::remove_buffered_pages(m_fd); 373 } 374 const int fd = m_fd; 375 m_fd = -1; 376 osmium::io::detail::reliable_close(fd); 377 } 378 } 379 380 }; // class NoDecompressor 381 382 namespace detail { 383 384 // we want the register_compression() function to run, setting 385 // the variable is only a side-effect, it will never be used 386 const bool registered_no_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::none, __anon39bf54020202(const int fd, const fsync sync) 387 [](const int fd, const fsync sync) { return new osmium::io::NoCompressor{fd, sync}; }, __anon39bf54020302(const int fd) 388 [](const int fd) { return new osmium::io::NoDecompressor{fd}; }, __anon39bf54020402(const char* buffer, std::size_t size) 389 [](const char* buffer, std::size_t size) { return new osmium::io::NoDecompressor{buffer, size}; } 390 ); 391 392 // dummy function to silence the unused variable warning from above get_registered_no_compression()393 inline bool get_registered_no_compression() noexcept { 394 return registered_no_compression; 395 } 396 397 } // namespace detail 398 399 } // namespace io 400 401 } // namespace osmium 402 403 #endif // OSMIUM_IO_COMPRESSION_HPP 404