1 #ifndef OSM2PGSQL_FLEX_TABLE_HPP
2 #define OSM2PGSQL_FLEX_TABLE_HPP
3 
4 /**
5  * SPDX-License-Identifier: GPL-2.0-or-later
6  *
7  * This file is part of osm2pgsql (https://osm2pgsql.org/).
8  *
9  * Copyright (C) 2006-2021 by the osm2pgsql developer community.
10  * For a full list of authors see the git log.
11  */
12 
13 #include "db-copy-mgr.hpp"
14 #include "flex-table-column.hpp"
15 #include "osmium-builder.hpp"
16 #include "pgsql.hpp"
17 #include "thread-pool.hpp"
18 
19 #include <osmium/osm/item_type.hpp>
20 
21 #include <cstddef>
22 #include <limits>
23 #include <memory>
24 #include <string>
25 #include <utility>
26 #include <vector>
27 
28 /**
29  * An output table (in the SQL sense) for the flex backend.
30  */
31 class flex_table_t
32 {
33 
34 public:
35 
36     /**
37      * Table creation type: interim tables are created as UNLOGGED and with
38      * autovacuum disabled.
39      */
40     enum class table_type {
41         interim,
42         permanent
43     };
44 
flex_table_t(std::string name)45     explicit flex_table_t(std::string name) : m_name(std::move(name)) {}
46 
name() const47     std::string const &name() const noexcept { return m_name; }
48 
schema() const49     std::string const &schema() const noexcept { return m_schema; }
50 
cluster_by_geom() const51     bool cluster_by_geom() const noexcept
52     {
53         return has_geom_column() && m_cluster_by_geom;
54     }
55 
data_tablespace() const56     std::string const &data_tablespace() const noexcept
57     {
58         return m_data_tablespace;
59     }
60 
index_tablespace() const61     std::string const &index_tablespace() const noexcept
62     {
63         return m_index_tablespace;
64     }
65 
set_schema(std::string const & schema)66     void set_schema(std::string const &schema) noexcept { m_schema = schema; }
67 
set_cluster_by_geom(bool cluster)68     void set_cluster_by_geom(bool cluster) noexcept
69     {
70         m_cluster_by_geom = cluster;
71     }
72 
set_data_tablespace(std::string const & tablespace)73     void set_data_tablespace(std::string const &tablespace) noexcept
74     {
75         m_data_tablespace = tablespace;
76     }
77 
set_index_tablespace(std::string const & tablespace)78     void set_index_tablespace(std::string const &tablespace) noexcept
79     {
80         m_index_tablespace = tablespace;
81     }
82 
id_type() const83     osmium::item_type id_type() const noexcept { return m_id_type; }
84 
set_id_type(osmium::item_type type)85     void set_id_type(osmium::item_type type) noexcept { m_id_type = type; }
86 
has_id_column() const87     bool has_id_column() const noexcept
88     {
89         if (m_columns.empty()) {
90             return false;
91         }
92         return (m_columns[0].type() == table_column_type::id_type) ||
93                (m_columns[0].type() == table_column_type::id_num);
94     }
95 
num_columns() const96     std::size_t num_columns() const noexcept { return m_columns.size(); }
97 
begin() const98     std::vector<flex_table_column_t>::const_iterator begin() const noexcept
99     {
100         return m_columns.begin();
101     }
102 
end() const103     std::vector<flex_table_column_t>::const_iterator end() const noexcept
104     {
105         return m_columns.end();
106     }
107 
has_geom_column() const108     bool has_geom_column() const noexcept
109     {
110         return m_geom_column != std::numeric_limits<std::size_t>::max();
111     }
112 
113     // XXX should we allow several geometry columns?
geom_column() const114     flex_table_column_t const &geom_column() const noexcept
115     {
116         assert(has_geom_column());
117         return m_columns[m_geom_column];
118     }
119 
srid() const120     int srid() const noexcept
121     {
122         return has_geom_column() ? geom_column().srid() : 4326;
123     }
124 
125     std::string build_sql_prepare_get_wkb() const;
126 
127     std::string build_sql_create_table(table_type ttype,
128                                        std::string const &table_name) const;
129 
130     std::string build_sql_column_list() const;
131 
132     std::string build_sql_create_id_index() const;
133 
134     /// Does this table take objects of the specified type?
matches_type(osmium::item_type type) const135     bool matches_type(osmium::item_type type) const noexcept
136     {
137         // This table takes any type -> okay
138         if (m_id_type == osmium::item_type::undefined) {
139             return true;
140         }
141 
142         // Type and table type match -> okay
143         if (type == m_id_type) {
144             return true;
145         }
146 
147         // Relations can be written as linestrings into way tables -> okay
148         if (type == osmium::item_type::relation &&
149             m_id_type == osmium::item_type::way) {
150             return true;
151         }
152 
153         // Area tables can take ways or relations, but not nodes
154         return m_id_type == osmium::item_type::area &&
155                type != osmium::item_type::node;
156     }
157 
158     /// Map way/node/relation ID to id value used in database table column
map_id(osmium::item_type type,osmid_t id) const159     osmid_t map_id(osmium::item_type type, osmid_t id) const noexcept
160     {
161         if (m_id_type == osmium::item_type::undefined) {
162             if (has_multicolumn_id_index()) {
163                 return id;
164             }
165 
166             switch (type) {
167             case osmium::item_type::node:
168                 return id;
169             case osmium::item_type::way:
170                 return -id;
171             case osmium::item_type::relation:
172                 return -id - 100000000000000000LL;
173             default:
174                 assert(false);
175             }
176         }
177 
178         if (m_id_type != osmium::item_type::relation &&
179             type == osmium::item_type::relation) {
180             return -id;
181         }
182         return id;
183     }
184 
185     flex_table_column_t &add_column(std::string const &name,
186                                     std::string const &type,
187                                     std::string const &sql_type);
188 
189     bool has_multicolumn_id_index() const noexcept;
190     std::string id_column_names() const;
191     std::string full_name() const;
192     std::string full_tmp_name() const;
193 
194 private:
195     /// The name of the table
196     std::string m_name;
197 
198     /// The schema this table is in
199     std::string m_schema{"public"};
200 
201     /// The table space used for this table (empty for default tablespace)
202     std::string m_data_tablespace;
203 
204     /**
205      * The table space used for indexes on this table (empty for default
206      * tablespace)
207      */
208     std::string m_index_tablespace;
209 
210     /**
211      * The columns in this table (The first zero, one or two columns are always
212      * the id columns).
213      */
214     std::vector<flex_table_column_t> m_columns;
215 
216     /// Index of the geometry column in m_columns. Default means no geometry.
217     std::size_t m_geom_column = std::numeric_limits<std::size_t>::max();
218 
219     /**
220      * Type of Id stored in this table (node, way, relation, area, or
221      * undefined for any type).
222      */
223     osmium::item_type m_id_type = osmium::item_type::undefined;
224 
225     /// Cluster the table by geometry.
226     bool m_cluster_by_geom = true;
227 
228 }; // class flex_table_t
229 
230 class table_connection_t
231 {
232 public:
table_connection_t(flex_table_t * table,std::shared_ptr<db_copy_thread_t> const & copy_thread)233     table_connection_t(flex_table_t *table,
234                        std::shared_ptr<db_copy_thread_t> const &copy_thread)
235     : m_builder(reprojection::create_projection(table->srid())), m_table(table),
236       m_target(std::make_shared<db_target_descr_t>(
237           table->name(), table->id_column_names(),
238           table->build_sql_column_list())),
239       m_copy_mgr(copy_thread), m_db_connection(nullptr)
240     {
241         m_target->schema = table->schema();
242     }
243 
244     void connect(std::string const &conninfo);
245 
246     void start(bool append);
247 
248     void stop(bool updateable, bool append);
249 
table() const250     flex_table_t const &table() const noexcept { return *m_table; }
251 
teardown()252     void teardown() { m_db_connection.reset(); }
253 
254     void prepare();
255 
256     void create_id_index();
257 
258     pg_result_t get_geom_by_id(osmium::item_type type, osmid_t id) const;
259 
sync()260     void sync() { m_copy_mgr.sync(); }
261 
new_line()262     void new_line() { m_copy_mgr.new_line(m_target); }
263 
copy_mgr()264     db_copy_mgr_t<db_deleter_by_type_and_id_t> *copy_mgr() noexcept
265     {
266         return &m_copy_mgr;
267     }
268 
269     void delete_rows_with(osmium::item_type type, osmid_t id);
270 
get_builder()271     geom::osmium_builder_t *get_builder() { return &m_builder; }
272 
task_set(std::future<std::chrono::milliseconds> && future)273     void task_set(std::future<std::chrono::milliseconds> &&future)
274     {
275         m_task_result.set(std::move(future));
276     }
277 
278     void task_wait();
279 
280 private:
281     geom::osmium_builder_t m_builder;
282 
283     flex_table_t *m_table;
284 
285     std::shared_ptr<db_target_descr_t> m_target;
286 
287     /**
288      * The copy manager responsible for sending data through the COPY mechanism
289      * to the database server.
290      */
291     db_copy_mgr_t<db_deleter_by_type_and_id_t> m_copy_mgr;
292 
293     /// The connection to the database server.
294     std::unique_ptr<pg_conn_t> m_db_connection;
295 
296     task_result_t m_task_result;
297 
298     /// Has the Id index already been created?
299     bool m_id_index_created = false;
300 
301 }; // class table_connection_t
302 
303 char const *type_to_char(osmium::item_type type) noexcept;
304 
305 #endif // OSM2PGSQL_FLEX_TABLE_HPP
306