1 #ifndef OSMIUM_IO_DETAIL_INPUT_FORMAT_HPP 2 #define OSMIUM_IO_DETAIL_INPUT_FORMAT_HPP 3 4 /* 5 6 This file is part of Osmium (https://osmcode.org/libosmium). 7 8 Copyright 2013-2021 Jochen Topf <jochen@topf.org> and others (see README). 9 10 Boost Software License - Version 1.0 - August 17th, 2003 11 12 Permission is hereby granted, free of charge, to any person or organization 13 obtaining a copy of the software and accompanying documentation covered by 14 this license (the "Software") to use, reproduce, display, distribute, 15 execute, and transmit the Software, and to prepare derivative works of the 16 Software, and to permit third-parties to whom the Software is furnished to 17 do so, all subject to the following: 18 19 The copyright notices in the Software and this entire statement, including 20 the above license grant, this restriction and the following disclaimer, 21 must be included in all copies of the Software, in whole or in part, and 22 all derivative works of the Software, unless such copies or derivative 23 works are solely in the form of machine-executable object code generated by 24 a source language processor. 25 26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE 30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 32 DEALINGS IN THE SOFTWARE. 33 34 */ 35 36 #include <osmium/io/detail/queue_util.hpp> 37 #include <osmium/io/error.hpp> 38 #include <osmium/io/file.hpp> 39 #include <osmium/io/file_format.hpp> 40 #include <osmium/io/header.hpp> 41 #include <osmium/memory/buffer.hpp> 42 #include <osmium/osm/entity_bits.hpp> 43 #include <osmium/thread/pool.hpp> 44 45 #include <array> 46 #include <exception> 47 #include <functional> 48 #include <future> 49 #include <memory> 50 #include <string> 51 #include <utility> 52 53 namespace osmium { 54 55 namespace io { 56 57 namespace detail { 58 59 struct parser_arguments { 60 osmium::thread::Pool& pool; 61 future_string_queue_type& input_queue; 62 future_buffer_queue_type& output_queue; 63 std::promise<osmium::io::Header>& header_promise; 64 osmium::osm_entity_bits::type read_which_entities; 65 osmium::io::read_meta read_metadata; 66 osmium::io::buffers_type buffers_kind; 67 }; 68 69 class Parser { 70 71 osmium::thread::Pool& m_pool; 72 future_buffer_queue_type& m_output_queue; 73 std::promise<osmium::io::Header>& m_header_promise; 74 queue_wrapper<std::string> m_input_queue; 75 osmium::osm_entity_bits::type m_read_which_entities; 76 osmium::io::read_meta m_read_metadata; 77 bool m_header_is_done; 78 79 protected: 80 get_pool()81 osmium::thread::Pool& get_pool() { 82 return m_pool; 83 } 84 read_types() const85 osmium::osm_entity_bits::type read_types() const noexcept { 86 return m_read_which_entities; 87 } 88 read_metadata() const89 osmium::io::read_meta read_metadata() const noexcept { 90 return m_read_metadata; 91 } 92 header_is_done() const93 bool header_is_done() const noexcept { 94 return m_header_is_done; 95 } 96 set_header_value(const osmium::io::Header & header)97 void set_header_value(const osmium::io::Header& header) { 98 if (!m_header_is_done) { 99 m_header_is_done = true; 100 m_header_promise.set_value(header); 101 } 102 } 103 set_header_exception(const std::exception_ptr & exception)104 void set_header_exception(const std::exception_ptr& exception) { 105 if (!m_header_is_done) { 106 m_header_is_done = true; 107 m_header_promise.set_exception(exception); 108 } 109 } 110 111 /** 112 * Wrap the buffer into a future and add it to the output queue. 113 */ send_to_output_queue(osmium::memory::Buffer && buffer)114 void send_to_output_queue(osmium::memory::Buffer&& buffer) { 115 add_to_queue(m_output_queue, std::move(buffer)); 116 } 117 send_to_output_queue(std::future<osmium::memory::Buffer> && future)118 void send_to_output_queue(std::future<osmium::memory::Buffer>&& future) { 119 m_output_queue.push(std::move(future)); 120 } 121 122 public: 123 Parser(parser_arguments & args)124 explicit Parser(parser_arguments& args) : 125 m_pool(args.pool), 126 m_output_queue(args.output_queue), 127 m_header_promise(args.header_promise), 128 m_input_queue(args.input_queue), 129 m_read_which_entities(args.read_which_entities), 130 m_read_metadata(args.read_metadata), 131 m_header_is_done(false) { 132 } 133 134 Parser(const Parser&) = delete; 135 Parser& operator=(const Parser&) = delete; 136 137 Parser(Parser&&) = delete; 138 Parser& operator=(Parser&&) = delete; 139 140 virtual ~Parser() noexcept = default; 141 142 virtual void run() = 0; 143 get_input()144 std::string get_input() { 145 return m_input_queue.pop(); 146 } 147 input_done() const148 bool input_done() const { 149 return m_input_queue.has_reached_end_of_data(); 150 } 151 parse()152 void parse() { 153 try { 154 run(); 155 } catch (...) { 156 std::exception_ptr exception = std::current_exception(); 157 set_header_exception(exception); 158 add_to_queue(m_output_queue, std::move(exception)); 159 } 160 161 add_end_of_data_to_queue(m_output_queue); 162 } 163 164 }; // class Parser 165 166 class ParserWithBuffer : public Parser { 167 168 enum { 169 initial_buffer_size = 1024UL * 1024UL 170 }; 171 172 osmium::memory::Buffer m_buffer{initial_buffer_size, 173 osmium::memory::Buffer::auto_grow::internal}; 174 175 osmium::io::buffers_type m_buffers_kind; 176 osmium::item_type m_last_type = osmium::item_type::undefined; 177 is_different_type(osmium::item_type current_type)178 bool is_different_type(osmium::item_type current_type) noexcept { 179 if (m_last_type == current_type) { 180 return false; 181 } 182 183 if (m_last_type == osmium::item_type::undefined) { 184 m_last_type = current_type; 185 return false; 186 } 187 188 m_last_type = current_type; 189 return true; 190 } 191 192 protected: 193 ParserWithBuffer(parser_arguments & args)194 explicit ParserWithBuffer(parser_arguments& args) : 195 Parser(args), 196 m_buffers_kind(args.buffers_kind) { 197 } 198 buffer()199 osmium::memory::Buffer& buffer() noexcept { 200 return m_buffer; 201 } 202 flush_nested_buffer()203 void flush_nested_buffer() { 204 if (m_buffer.has_nested_buffers()) { 205 std::unique_ptr<osmium::memory::Buffer> buffer_ptr{m_buffer.get_last_nested()}; 206 send_to_output_queue(std::move(*buffer_ptr)); 207 } 208 } 209 flush_final_buffer()210 void flush_final_buffer() { 211 if (m_buffer.committed() > 0) { 212 send_to_output_queue(std::move(m_buffer)); 213 } 214 } 215 maybe_new_buffer(osmium::item_type current_type)216 void maybe_new_buffer(osmium::item_type current_type) { 217 if (m_buffers_kind == buffers_type::any) { 218 return; 219 } 220 221 if (is_different_type(current_type) && m_buffer.committed() > 0) { 222 osmium::memory::Buffer new_buffer{initial_buffer_size, 223 osmium::memory::Buffer::auto_grow::internal}; 224 using std::swap; 225 swap(new_buffer, m_buffer); 226 send_to_output_queue(std::move(new_buffer)); 227 } 228 } 229 230 }; // class ParserWithBuffer 231 232 /** 233 * This factory class is used to create objects that decode OSM 234 * data written in a specified format. 235 * 236 * Do not use this class directly. Use the osmium::io::Reader 237 * class instead. 238 */ 239 class ParserFactory { 240 241 public: 242 243 using create_parser_type = std::function<std::unique_ptr<Parser>(parser_arguments&)>; 244 245 private: 246 247 std::array<create_parser_type, static_cast<std::size_t>(file_format::last) + 1> m_callbacks; 248 249 ParserFactory() noexcept = default; 250 callbacks(const osmium::io::file_format format)251 create_parser_type& callbacks(const osmium::io::file_format format) noexcept { 252 return m_callbacks[static_cast<std::size_t>(format)]; 253 } 254 callbacks(const osmium::io::file_format format) const255 const create_parser_type& callbacks(const osmium::io::file_format format) const noexcept { 256 return m_callbacks[static_cast<std::size_t>(format)]; 257 } 258 259 public: 260 instance()261 static ParserFactory& instance() noexcept { 262 static ParserFactory factory; 263 return factory; 264 } 265 register_parser(const osmium::io::file_format format,create_parser_type && create_function)266 bool register_parser(const osmium::io::file_format format, create_parser_type&& create_function) { 267 callbacks(format) = std::forward<create_parser_type>(create_function); 268 return true; 269 } 270 get_creator_function(const osmium::io::File & file) const271 create_parser_type get_creator_function(const osmium::io::File& file) const { 272 auto func = callbacks(file.format()); 273 if (func) { 274 return func; 275 } 276 throw unsupported_file_format_error{ 277 std::string{"Can not open file '"} + 278 file.filename() + 279 "' with type '" + 280 as_string(file.format()) + 281 "'. No support for reading this format in this program."}; 282 } 283 284 }; // class ParserFactory 285 286 } // namespace detail 287 288 } // namespace io 289 290 } // namespace osmium 291 292 #endif // OSMIUM_IO_DETAIL_INPUT_FORMAT_HPP 293