1 #ifndef OSMIUM_IO_COMPRESSION_HPP
2 #define OSMIUM_IO_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2021 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <osmium/io/detail/read_write.hpp>
37 #include <osmium/io/error.hpp>
38 #include <osmium/io/file_compression.hpp>
39 #include <osmium/io/writer_options.hpp>
40 #include <osmium/util/file.hpp>
41 
42 #include <atomic>
43 #include <cerrno>
44 #include <cstddef>
45 #include <functional>
46 #include <map>
47 #include <memory>
48 #include <string>
49 #include <system_error>
50 #include <tuple>
51 #include <utility>
52 
53 namespace osmium {
54 
55     namespace io {
56 
57         class Compressor {
58 
59             fsync m_fsync;
60 
61         protected:
62 
do_fsync() const63             bool do_fsync() const noexcept {
64                 return m_fsync == fsync::yes;
65             }
66 
67         public:
68 
Compressor(const fsync sync)69             explicit Compressor(const fsync sync) noexcept :
70                 m_fsync(sync) {
71             }
72 
73             Compressor(const Compressor&) = default;
74             Compressor& operator=(const Compressor&) = default;
75 
76             Compressor(Compressor&&) noexcept = default;
77             Compressor& operator=(Compressor&&) noexcept = default;
78 
79             virtual ~Compressor() noexcept = default;
80 
81             virtual void write(const std::string& data) = 0;
82 
83             virtual void close() = 0;
84 
file_size() const85             virtual std::size_t file_size() const {
86                 return 0;
87             }
88 
89         }; // class Compressor
90 
91         class Decompressor {
92 
93             std::atomic<std::size_t>* m_offset_ptr{nullptr};
94 
95             std::atomic_bool m_want_buffered_pages_removed{false};
96 
97         public:
98 
99             enum {
100                 input_buffer_size = 1024U * 1024U
101             };
102 
103             Decompressor() = default;
104 
105             Decompressor(const Decompressor&) = delete;
106             Decompressor& operator=(const Decompressor&) = delete;
107 
108             Decompressor(Decompressor&&) = delete;
109             Decompressor& operator=(Decompressor&&) = delete;
110 
111             virtual ~Decompressor() noexcept = default;
112 
113             virtual std::string read() = 0;
114 
115             virtual void close() = 0;
116 
is_real() const117             virtual bool is_real() const noexcept {
118                 return true;
119             }
120 
set_offset_ptr(std::atomic<std::size_t> * offset_ptr)121             void set_offset_ptr(std::atomic<std::size_t>* offset_ptr) noexcept {
122                 m_offset_ptr = offset_ptr;
123             }
124 
set_offset(const std::size_t offset)125             void set_offset(const std::size_t offset) noexcept {
126                 if (m_offset_ptr) {
127                     *m_offset_ptr = offset;
128                 }
129             }
130 
want_buffered_pages_removed() const131             bool want_buffered_pages_removed() const noexcept {
132                 return m_want_buffered_pages_removed;
133             }
134 
set_want_buffered_pages_removed(bool value)135             void set_want_buffered_pages_removed(bool value) noexcept {
136                 m_want_buffered_pages_removed = value;
137             }
138 
139         }; // class Decompressor
140 
141         /**
142          * This singleton factory class is used to register compression
143          * algorithms used for reading and writing OSM files.
144          *
145          * For each algorithm we store two functions that construct
146          * a compressor and decompressor object, respectively.
147          */
148         class CompressionFactory {
149 
150         public:
151 
152             using create_compressor_type          = std::function<osmium::io::Compressor*(int, fsync)>;
153             using create_decompressor_type_fd     = std::function<osmium::io::Decompressor*(int)>;
154             using create_decompressor_type_buffer = std::function<osmium::io::Decompressor*(const char*, std::size_t)>;
155 
156         private:
157 
158             using callbacks_type = std::tuple<create_compressor_type,
159                                               create_decompressor_type_fd,
160                                               create_decompressor_type_buffer>;
161 
162             using compression_map_type = std::map<const osmium::io::file_compression, callbacks_type>;
163 
164             compression_map_type m_callbacks;
165 
166             CompressionFactory() = default;
167 
find_callbacks(const osmium::io::file_compression compression) const168             const callbacks_type& find_callbacks(const osmium::io::file_compression compression) const {
169                 const auto it = m_callbacks.find(compression);
170 
171                 if (it != m_callbacks.end()) {
172                     return it->second;
173                 }
174 
175                 std::string error_message{"Support for compression '"};
176                 error_message += as_string(compression);
177                 error_message += "' not compiled into this binary";
178                 throw unsupported_file_format_error{error_message};
179             }
180 
181         public:
182 
183             CompressionFactory(const CompressionFactory&) = delete;
184             CompressionFactory& operator=(const CompressionFactory&) = delete;
185 
186             CompressionFactory(CompressionFactory&&) = delete;
187             CompressionFactory& operator=(CompressionFactory&&) = delete;
188 
189             ~CompressionFactory() noexcept = default;
190 
instance()191             static CompressionFactory& instance() {
192                 static CompressionFactory factory;
193                 return factory;
194             }
195 
register_compression(osmium::io::file_compression compression,const create_compressor_type & create_compressor,const create_decompressor_type_fd & create_decompressor_fd,const create_decompressor_type_buffer & create_decompressor_buffer)196             bool register_compression(
197                 osmium::io::file_compression compression,
198                 const create_compressor_type& create_compressor,
199                 const create_decompressor_type_fd& create_decompressor_fd,
200                 const create_decompressor_type_buffer& create_decompressor_buffer) {
201 
202                 compression_map_type::value_type cc{compression,
203                                                     std::make_tuple(create_compressor,
204                                                                     create_decompressor_fd,
205                                                                     create_decompressor_buffer)};
206 
207                 return m_callbacks.insert(cc).second;
208             }
209 
210             template <typename... TArgs>
create_compressor(const osmium::io::file_compression compression,TArgs &&...args) const211             std::unique_ptr<osmium::io::Compressor> create_compressor(const osmium::io::file_compression compression, TArgs&&... args) const {
212                 const auto callbacks = find_callbacks(compression);
213                 return std::unique_ptr<osmium::io::Compressor>(std::get<0>(callbacks)(std::forward<TArgs>(args)...));
214             }
215 
create_decompressor(const osmium::io::file_compression compression,const int fd) const216             std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const int fd) const {
217                 const auto callbacks = find_callbacks(compression);
218                 return std::unique_ptr<osmium::io::Decompressor>(std::get<1>(callbacks)(fd));
219             }
220 
create_decompressor(const osmium::io::file_compression compression,const char * buffer,const std::size_t size) const221             std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const char* buffer, const std::size_t size) const {
222                 const auto callbacks = find_callbacks(compression);
223                 return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(callbacks)(buffer, size));
224             }
225 
226         }; // class CompressionFactory
227 
228         class NoCompressor final : public Compressor {
229 
230             std::size_t m_file_size = 0;
231             int m_fd;
232 
233         public:
234 
NoCompressor(const int fd,const fsync sync)235             NoCompressor(const int fd, const fsync sync) :
236                 Compressor(sync),
237                 m_fd(fd) {
238             }
239 
240             NoCompressor(const NoCompressor&) = delete;
241             NoCompressor& operator=(const NoCompressor&) = delete;
242 
243             NoCompressor(NoCompressor&&) = delete;
244             NoCompressor& operator=(NoCompressor&&) = delete;
245 
~NoCompressor()246             ~NoCompressor() noexcept override {
247                 try {
248                     close();
249                 } catch (...) {
250                     // Ignore any exceptions because destructor must not throw.
251                 }
252             }
253 
write(const std::string & data)254             void write(const std::string& data) override {
255                 osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
256                 m_file_size += data.size();
257             }
258 
close()259             void close() override {
260                 if (m_fd >= 0) {
261                     const int fd = m_fd;
262                     m_fd = -1;
263 
264                     // Do not sync or close stdout
265                     if (fd == 1) {
266                         return;
267                     }
268 
269                     if (do_fsync()) {
270                         osmium::io::detail::reliable_fsync(fd);
271                     }
272                     osmium::io::detail::reliable_close(fd);
273                 }
274             }
275 
file_size() const276             std::size_t file_size() const override {
277                 return m_file_size;
278             }
279 
280         }; // class NoCompressor
281 
282         /**
283          * The DummyDecompressor is used when reading PBF files. In that
284          * case the PBFParser class is responsible for reading from the
285          * file itself, and the DummyDecompressor does nothing.
286          */
287         class DummyDecompressor final : public Decompressor {
288         public:
289 
290             DummyDecompressor() = default;
291 
292             DummyDecompressor(const DummyDecompressor&) = delete;
293             DummyDecompressor& operator=(const DummyDecompressor&) = delete;
294 
295             DummyDecompressor(DummyDecompressor&&) = delete;
296             DummyDecompressor& operator=(DummyDecompressor&&) = delete;
297 
298             ~DummyDecompressor() noexcept override = default;
299 
read()300             std::string read() override {
301                 return {};
302             }
303 
close()304             void close() override {
305             }
306 
is_real() const307             bool is_real() const noexcept override {
308                 return false;
309             }
310 
311         }; // class DummyDecompressor
312 
313         class NoDecompressor final : public Decompressor {
314 
315             int m_fd = -1;
316             const char* m_buffer = nullptr;
317             std::size_t m_buffer_size = 0;
318             std::size_t m_offset = 0;
319 
320         public:
321 
NoDecompressor(const int fd)322             explicit NoDecompressor(const int fd) :
323                 m_fd(fd) {
324             }
325 
NoDecompressor(const char * buffer,const std::size_t size)326             NoDecompressor(const char* buffer, const std::size_t size) :
327                 m_buffer(buffer),
328                 m_buffer_size(size) {
329             }
330 
331             NoDecompressor(const NoDecompressor&) = delete;
332             NoDecompressor& operator=(const NoDecompressor&) = delete;
333 
334             NoDecompressor(NoDecompressor&&) = delete;
335             NoDecompressor& operator=(NoDecompressor&&) = delete;
336 
~NoDecompressor()337             ~NoDecompressor() noexcept override {
338                 try {
339                     close();
340                 } catch (...) {
341                     // Ignore any exceptions because destructor must not throw.
342                 }
343             }
344 
read()345             std::string read() override {
346                 std::string buffer;
347 
348                 if (m_buffer) {
349                     if (m_buffer_size != 0) {
350                         const std::size_t size = m_buffer_size;
351                         m_buffer_size = 0;
352                         buffer.append(m_buffer, size);
353                     }
354                 } else {
355                     buffer.resize(osmium::io::Decompressor::input_buffer_size);
356                     if (want_buffered_pages_removed()) {
357                         osmium::io::detail::remove_buffered_pages(m_fd, m_offset);
358                     }
359                     const auto nread = detail::reliable_read(m_fd, &*buffer.begin(), osmium::io::Decompressor::input_buffer_size);
360                     buffer.resize(std::string::size_type(nread));
361                 }
362 
363                 m_offset += buffer.size();
364                 set_offset(m_offset);
365 
366                 return buffer;
367             }
368 
close()369             void close() override {
370                 if (m_fd >= 0) {
371                     if (want_buffered_pages_removed()) {
372                         osmium::io::detail::remove_buffered_pages(m_fd);
373                     }
374                     const int fd = m_fd;
375                     m_fd = -1;
376                     osmium::io::detail::reliable_close(fd);
377                 }
378             }
379 
380         }; // class NoDecompressor
381 
382         namespace detail {
383 
384             // we want the register_compression() function to run, setting
385             // the variable is only a side-effect, it will never be used
386             const bool registered_no_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::none,
__anon39bf54020202(const int fd, const fsync sync) 387                 [](const int fd, const fsync sync) { return new osmium::io::NoCompressor{fd, sync}; },
__anon39bf54020302(const int fd) 388                 [](const int fd) { return new osmium::io::NoDecompressor{fd}; },
__anon39bf54020402(const char* buffer, std::size_t size) 389                 [](const char* buffer, std::size_t size) { return new osmium::io::NoDecompressor{buffer, size}; }
390             );
391 
392             // dummy function to silence the unused variable warning from above
get_registered_no_compression()393             inline bool get_registered_no_compression() noexcept {
394                 return registered_no_compression;
395             }
396 
397         } // namespace detail
398 
399     } // namespace io
400 
401 } // namespace osmium
402 
403 #endif // OSMIUM_IO_COMPRESSION_HPP
404