1 /******************************************************************************
2 *
3 * Project: OpenGIS Simple Features Reference Implementation
4 * Purpose: Implements OGROSMDataSource class.
5 * Author: Even Rouault, <even dot rouault at spatialys.com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2012-2014, Even Rouault <even dot rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29 #include "gpb.h"
30 #include "ogr_osm.h"
31
32 #include <cassert>
33 #include <cerrno>
34 #include <climits>
35 #include <cmath>
36 #include <cstddef>
37 #include <cstdio>
38 #include <cstdlib>
39 #include <cstring>
40 #include <ctime>
41 #include <algorithm>
42 #include <map>
43 #include <memory>
44 #include <set>
45 #include <string>
46 #include <unordered_set>
47 #include <utility>
48 #include <vector>
49
50 #include "cpl_conv.h"
51 #include "cpl_error.h"
52 #include "cpl_multiproc.h"
53 #include "cpl_port.h"
54 #include "cpl_progress.h"
55 #include "cpl_string.h"
56 #include "cpl_time.h"
57 #include "cpl_vsi.h"
58 #include "ogr_api.h"
59 #include "ogr_core.h"
60 #include "ogr_feature.h"
61 #include "ogr_geometry.h"
62 #include "ogr_p.h"
63 #include "ogrlayerdecorator.h"
64 #include "ogrsf_frmts.h"
65 #include "ogrsqliteexecutesql.h"
66 #include "osm_parser.h"
67 #include "ogr_swq.h"
68 #include "sqlite3.h"
69
70 #undef SQLITE_STATIC
71 #define SQLITE_STATIC ((sqlite3_destructor_type)nullptr)
72
73 constexpr int LIMIT_IDS_PER_REQUEST = 200;
74
75 constexpr int IDX_LYR_POINTS = 0;
76 constexpr int IDX_LYR_LINES = 1;
77 constexpr int IDX_LYR_MULTILINESTRINGS = 2;
78 constexpr int IDX_LYR_MULTIPOLYGONS = 3;
79 constexpr int IDX_LYR_OTHER_RELATIONS = 4;
80
DBL_TO_INT(double x)81 static int DBL_TO_INT( double x )
82 {
83 return static_cast<int>(floor(x * 1.0e7 + 0.5));
84 }
INT_TO_DBL(int x)85 static double INT_TO_DBL( int x ) { return x / 1.0e7; }
86
87 constexpr unsigned int MAX_COUNT_FOR_TAGS_IN_WAY = 255; // Must fit on 1 byte.
88
89 constexpr int NODE_PER_BUCKET = 65536;
90
VALID_ID_FOR_CUSTOM_INDEXING(GIntBig _id)91 static bool VALID_ID_FOR_CUSTOM_INDEXING( GIntBig _id )
92 {
93 return
94 _id >= 0 &&
95 _id / NODE_PER_BUCKET < INT_MAX;
96 }
97
98 // Minimum size of data written on disk, in *uncompressed* case.
99 constexpr int SECTOR_SIZE = 512;
100 // Which represents, 64 nodes
101 // constexpr int NODE_PER_SECTOR = SECTOR_SIZE / (2 * 4);
102 constexpr int NODE_PER_SECTOR = 64;
103 constexpr int NODE_PER_SECTOR_SHIFT = 6;
104
105 // Per bucket, we keep track of the absence/presence of sectors
106 // only, to reduce memory usage.
107 // #define BUCKET_BITMAP_SIZE NODE_PER_BUCKET / (8 * NODE_PER_SECTOR)
108 constexpr int BUCKET_BITMAP_SIZE = 128;
109
110 // #define BUCKET_SECTOR_SIZE_ARRAY_SIZE NODE_PER_BUCKET / NODE_PER_SECTOR
111 // Per bucket, we keep track of the real size of the sector. Each sector
112 // size is encoded in a single byte, whose value is:
113 // (sector_size in bytes - 8 ) / 2, minus 8. 252 means uncompressed
114 constexpr int BUCKET_SECTOR_SIZE_ARRAY_SIZE = 1024;
115
116 // Must be a multiple of both BUCKET_BITMAP_SIZE and
117 // BUCKET_SECTOR_SIZE_ARRAY_SIZE
118 constexpr int knPAGE_SIZE = 4096;
119
120 // compressSize should not be greater than 512, so COMPRESS_SIZE_TO_BYTE() fits
121 // on a byte.
COMPRESS_SIZE_TO_BYTE(size_t nCompressSize)122 static GByte COMPRESS_SIZE_TO_BYTE( size_t nCompressSize )
123 {
124 return static_cast<GByte>((nCompressSize - 8) / 2);
125 }
126
ROUND_COMPRESS_SIZE(T nCompressSize)127 template<typename T> static T ROUND_COMPRESS_SIZE( T nCompressSize )
128 {
129 return ((nCompressSize + 1) / 2) * 2;
130 }
COMPRESS_SIZE_FROM_BYTE(GByte byte_on_size)131 static int COMPRESS_SIZE_FROM_BYTE( GByte byte_on_size )
132 {
133 return static_cast<int>(byte_on_size) * 2 + 8;
134 }
135
136 // Max number of features that are accumulated in pasWayFeaturePairs.
137 constexpr int MAX_DELAYED_FEATURES = 75000;
138 // Max number of tags that are accumulated in pasAccumulatedTags.
139 constexpr int MAX_ACCUMULATED_TAGS = MAX_DELAYED_FEATURES * 5;
140 // Max size of the string with tag values that are accumulated in
141 // pabyNonRedundantValues.
142 constexpr int MAX_NON_REDUNDANT_VALUES = MAX_DELAYED_FEATURES * 10;
143 // Max number of features that are accumulated in panUnsortedReqIds
144 constexpr int MAX_ACCUMULATED_NODES = 1000000;
145
146 #ifdef ENABLE_NODE_LOOKUP_BY_HASHING
147 // Size of panHashedIndexes array. Must be in the list at
148 // http://planetmath.org/goodhashtableprimes , and greater than
149 // MAX_ACCUMULATED_NODES.
150 constexpr int HASHED_INDEXES_ARRAY_SIZE = 3145739;
151 // #define HASHED_INDEXES_ARRAY_SIZE 1572869
152 constexpr int COLLISION_BUCKET_ARRAY_SIZE =
153 (MAX_ACCUMULATED_NODES / 100) * 40;
154
155 // hash function = identity
156 #define HASH_ID_FUNC(x) ((GUIntBig)(x))
157 #endif // ENABLE_NODE_LOOKUP_BY_HASHING
158
159 // #define FAKE_LOOKUP_NODES
160
161 // #define DEBUG_MEM_USAGE
162 #ifdef DEBUG_MEM_USAGE
163 size_t GetMaxTotalAllocs();
164 #endif
165
166 static void WriteVarSInt64(GIntBig nSVal, GByte** ppabyData);
167
168 CPL_CVSID("$Id: ogrosmdatasource.cpp 09a48d5214b089c224b3b7afed5beee254d45614 2021-08-15 12:04:53 +0200 Even Rouault $")
169
170 class DSToBeOpened
171 {
172 public:
173 GIntBig nPID;
174 CPLString osDSName;
175 CPLString osInterestLayers;
176 };
177
178 static CPLMutex *hMutex = nullptr;
179 static std::vector<DSToBeOpened> oListDSToBeOpened;
180
181 /************************************************************************/
182 /* AddInterestLayersForDSName() */
183 /************************************************************************/
184
AddInterestLayersForDSName(const CPLString & osDSName,const CPLString & osInterestLayers)185 static void AddInterestLayersForDSName( const CPLString& osDSName,
186 const CPLString& osInterestLayers )
187 {
188 CPLMutexHolder oMutexHolder(&hMutex);
189 DSToBeOpened oDSToBeOpened;
190 oDSToBeOpened.nPID = CPLGetPID();
191 oDSToBeOpened.osDSName = osDSName;
192 oDSToBeOpened.osInterestLayers = osInterestLayers;
193 oListDSToBeOpened.push_back( oDSToBeOpened );
194 }
195
196 /************************************************************************/
197 /* GetInterestLayersForDSName() */
198 /************************************************************************/
199
GetInterestLayersForDSName(const CPLString & osDSName)200 static CPLString GetInterestLayersForDSName( const CPLString& osDSName )
201 {
202 CPLMutexHolder oMutexHolder(&hMutex);
203 GIntBig nPID = CPLGetPID();
204 for(int i = 0; i < (int)oListDSToBeOpened.size(); i++)
205 {
206 if( oListDSToBeOpened[i].nPID == nPID &&
207 oListDSToBeOpened[i].osDSName == osDSName )
208 {
209 CPLString osInterestLayers = oListDSToBeOpened[i].osInterestLayers;
210 oListDSToBeOpened.erase(oListDSToBeOpened.begin()+i);
211 return osInterestLayers;
212 }
213 }
214 return "";
215 }
216
217 /************************************************************************/
218 /* OGROSMDataSource() */
219 /************************************************************************/
220
OGROSMDataSource()221 OGROSMDataSource::OGROSMDataSource() :
222 nLayers(0),
223 papoLayers(nullptr),
224 pszName(nullptr),
225 bExtentValid(false),
226 bInterleavedReading(-1),
227 poCurrentLayer(nullptr),
228 psParser(nullptr),
229 bHasParsedFirstChunk(false),
230 bStopParsing(false),
231 pMyVFS(nullptr),
232 hDB(nullptr),
233 hInsertNodeStmt(nullptr),
234 hInsertWayStmt(nullptr),
235 hSelectNodeBetweenStmt(nullptr),
236 pahSelectNodeStmt(nullptr),
237 pahSelectWayStmt(nullptr),
238 hInsertPolygonsStandaloneStmt(nullptr),
239 hDeletePolygonsStandaloneStmt(nullptr),
240 hSelectPolygonsStandaloneStmt(nullptr),
241 bHasRowInPolygonsStandalone(false),
242 hDBForComputedAttributes(nullptr),
243 nMaxSizeForInMemoryDBInMB(0),
244 bInMemoryTmpDB(false),
245 bMustUnlink(true),
246 nNodesInTransaction(0),
247 nMinSizeKeysInSetClosedWaysArePolygons(0),
248 nMaxSizeKeysInSetClosedWaysArePolygons(0),
249 m_ignoredKeys({{"area","created_by","converted_by","note","todo","fixme","FIXME"}}),
250 bReportAllNodes(false),
251 bReportAllWays(false),
252 bFeatureAdded(false),
253 bInTransaction(false),
254 bIndexPoints(true),
255 bUsePointsIndex(true),
256 bIndexWays(true),
257 bUseWaysIndex(true),
258 poResultSetLayer(nullptr),
259 bIndexPointsBackup(false),
260 bUsePointsIndexBackup(false),
261 bIndexWaysBackup(false),
262 bUseWaysIndexBackup(false),
263 bIsFeatureCountEnabled(false),
264 bAttributeNameLaundering(true),
265 nWaysProcessed(0),
266 nRelationsProcessed(0),
267 bCustomIndexing(true),
268 bCompressNodes(false),
269 nUnsortedReqIds(0),
270 panUnsortedReqIds(nullptr),
271 nReqIds(0),
272 panReqIds(nullptr),
273 #ifdef ENABLE_NODE_LOOKUP_BY_HASHING
274 bEnableHashedIndex(true),
275 panHashedIndexes(nullptr),
276 psCollisionBuckets(nullptr),
277 bHashedIndexValid(false),
278 #endif
279 pasLonLatArray(nullptr),
280 pasAccumulatedTags(nullptr),
281 nAccumulatedTags(0),
282 pabyNonRedundantValues(nullptr),
283 nNonRedundantValuesLen(0),
284 pasWayFeaturePairs(nullptr),
285 nWayFeaturePairs(0),
286 nNextKeyIndex(0),
287 bInMemoryNodesFile(false),
288 bMustUnlinkNodesFile(true),
289 nNodesFileSize(0),
290 fpNodes(nullptr),
291 nPrevNodeId(-INT_MAX),
292 nBucketOld(-1),
293 nOffInBucketReducedOld(-1),
294 pabySector(nullptr),
295 bNeedsToSaveWayInfo(false),
296 m_nFileSize(FILESIZE_NOT_INIT)
297 {}
298
299 /************************************************************************/
300 /* ~OGROSMDataSource() */
301 /************************************************************************/
302
~OGROSMDataSource()303 OGROSMDataSource::~OGROSMDataSource()
304
305 {
306 for( int i=0; i<nLayers; i++ )
307 delete papoLayers[i];
308 CPLFree(papoLayers);
309
310 CPLFree(pszName);
311
312 if( psParser != nullptr )
313 CPLDebug( "OSM",
314 "Number of bytes read in file : " CPL_FRMT_GUIB,
315 OSM_GetBytesRead(psParser) );
316 OSM_Close(psParser);
317
318 if( hDB != nullptr )
319 CloseDB();
320
321 if( hDBForComputedAttributes != nullptr )
322 sqlite3_close(hDBForComputedAttributes);
323
324 if( pMyVFS )
325 {
326 sqlite3_vfs_unregister(pMyVFS);
327 CPLFree(pMyVFS->pAppData);
328 CPLFree(pMyVFS);
329 }
330
331 if( !osTmpDBName.empty() && bMustUnlink )
332 {
333 const char* pszVal = CPLGetConfigOption("OSM_UNLINK_TMPFILE", "YES");
334 if( !EQUAL(pszVal, "NOT_EVEN_AT_END") )
335 VSIUnlink(osTmpDBName);
336 }
337
338 CPLFree(panReqIds);
339 #ifdef ENABLE_NODE_LOOKUP_BY_HASHING
340 CPLFree(panHashedIndexes);
341 CPLFree(psCollisionBuckets);
342 #endif
343 CPLFree(pasLonLatArray);
344 CPLFree(panUnsortedReqIds);
345
346 for( int i = 0; i < nWayFeaturePairs; i++)
347 {
348 delete pasWayFeaturePairs[i].poFeature;
349 }
350 CPLFree(pasWayFeaturePairs);
351 CPLFree(pasAccumulatedTags);
352 CPLFree(pabyNonRedundantValues);
353
354 #ifdef OSM_DEBUG
355 FILE* f = fopen("keys.txt", "wt");
356 for( int i=0; i<startic_cast<int>(asKeys.size()); i++ )
357 {
358 KeyDesc* psKD = asKeys[i];
359 fprintf(f, "%08d idx=%d %s\n",
360 psKD->nOccurrences,
361 psKD->nKeyIndex,
362 psKD->pszK);
363 }
364 fclose(f);
365 #endif
366
367 for( int i=0; i<static_cast<int>(asKeys.size()); i++ )
368 {
369 KeyDesc* psKD = asKeys[i];
370 CPLFree(psKD->pszK);
371 for( int j=0; j<static_cast<int>(psKD->asValues.size());j++)
372 CPLFree(psKD->asValues[j]);
373 delete psKD;
374 }
375
376 if( fpNodes )
377 VSIFCloseL(fpNodes);
378 if( !osNodesFilename.empty() && bMustUnlinkNodesFile )
379 {
380 const char* pszVal = CPLGetConfigOption("OSM_UNLINK_TMPFILE", "YES");
381 if( !EQUAL(pszVal, "NOT_EVEN_AT_END") )
382 VSIUnlink(osNodesFilename);
383 }
384
385 CPLFree(pabySector);
386 std::map<int, Bucket>::iterator oIter = oMapBuckets.begin();
387 for( ; oIter != oMapBuckets.end(); ++oIter )
388 {
389 if( bCompressNodes )
390 {
391 int nRem = oIter->first % (knPAGE_SIZE / BUCKET_SECTOR_SIZE_ARRAY_SIZE);
392 if( nRem == 0 )
393 CPLFree(oIter->second.u.panSectorSize);
394 }
395 else
396 {
397 int nRem = oIter->first % (knPAGE_SIZE / BUCKET_BITMAP_SIZE);
398 if( nRem == 0 )
399 CPLFree(oIter->second.u.pabyBitmap);
400 }
401 }
402 }
403
404 /************************************************************************/
405 /* CloseDB() */
406 /************************************************************************/
407
CloseDB()408 void OGROSMDataSource::CloseDB()
409 {
410 if( hInsertNodeStmt != nullptr )
411 sqlite3_finalize( hInsertNodeStmt );
412 hInsertNodeStmt = nullptr;
413
414 if( hInsertWayStmt != nullptr )
415 sqlite3_finalize( hInsertWayStmt );
416 hInsertWayStmt = nullptr;
417
418 if( hInsertPolygonsStandaloneStmt != nullptr )
419 sqlite3_finalize( hInsertPolygonsStandaloneStmt );
420 hInsertPolygonsStandaloneStmt = nullptr;
421
422 if( hDeletePolygonsStandaloneStmt != nullptr )
423 sqlite3_finalize( hDeletePolygonsStandaloneStmt );
424 hDeletePolygonsStandaloneStmt = nullptr;
425
426 if( hSelectPolygonsStandaloneStmt != nullptr )
427 sqlite3_finalize( hSelectPolygonsStandaloneStmt );
428 hSelectPolygonsStandaloneStmt = nullptr;
429
430 if( pahSelectNodeStmt != nullptr )
431 {
432 for( int i = 0; i < LIMIT_IDS_PER_REQUEST; i++ )
433 {
434 if( pahSelectNodeStmt[i] != nullptr )
435 sqlite3_finalize( pahSelectNodeStmt[i] );
436 }
437 CPLFree(pahSelectNodeStmt);
438 pahSelectNodeStmt = nullptr;
439 }
440
441 if( pahSelectWayStmt != nullptr )
442 {
443 for( int i = 0; i < LIMIT_IDS_PER_REQUEST; i++ )
444 {
445 if( pahSelectWayStmt[i] != nullptr )
446 sqlite3_finalize( pahSelectWayStmt[i] );
447 }
448 CPLFree(pahSelectWayStmt);
449 pahSelectWayStmt = nullptr;
450 }
451
452 if( bInTransaction )
453 CommitTransactionCacheDB();
454
455 sqlite3_close(hDB);
456 hDB = nullptr;
457 }
458
459 /************************************************************************/
460 /* IndexPoint() */
461 /************************************************************************/
462
463 constexpr GByte abyBitsCount[] = {
464 0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,
465 1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,
466 1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,
467 2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,
468 1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,
469 2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,
470 2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,
471 3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,
472 1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,
473 2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,
474 2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,
475 3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,
476 2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,
477 3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,
478 3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,
479 4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8
480 };
481
IndexPoint(OSMNode * psNode)482 bool OGROSMDataSource::IndexPoint( OSMNode* psNode )
483 {
484 if( !bIndexPoints )
485 return true;
486
487 if( bCustomIndexing)
488 return IndexPointCustom(psNode);
489
490 return IndexPointSQLite(psNode);
491 }
492
493 /************************************************************************/
494 /* IndexPointSQLite() */
495 /************************************************************************/
496
IndexPointSQLite(OSMNode * psNode)497 bool OGROSMDataSource::IndexPointSQLite(OSMNode* psNode)
498 {
499 sqlite3_bind_int64( hInsertNodeStmt, 1, psNode->nID );
500
501 LonLat sLonLat;
502 sLonLat.nLon = DBL_TO_INT(psNode->dfLon);
503 sLonLat.nLat = DBL_TO_INT(psNode->dfLat);
504
505 sqlite3_bind_blob( hInsertNodeStmt, 2, &sLonLat, sizeof(sLonLat),
506 SQLITE_STATIC );
507
508 const int rc = sqlite3_step( hInsertNodeStmt );
509 sqlite3_reset( hInsertNodeStmt );
510 if( !(rc == SQLITE_OK || rc == SQLITE_DONE) )
511 {
512 CPLError( CE_Failure, CPLE_AppDefined,
513 "Failed inserting node " CPL_FRMT_GIB ": %s",
514 psNode->nID, sqlite3_errmsg(hDB));
515 }
516
517 return true;
518 }
519
520 /************************************************************************/
521 /* FlushCurrentSector() */
522 /************************************************************************/
523
FlushCurrentSector()524 bool OGROSMDataSource::FlushCurrentSector()
525 {
526 #ifndef FAKE_LOOKUP_NODES
527 if( bCompressNodes )
528 return FlushCurrentSectorCompressedCase();
529
530 return FlushCurrentSectorNonCompressedCase();
531 #else
532 return true;
533 #endif
534 }
535
536 /************************************************************************/
537 /* AllocBucket() */
538 /************************************************************************/
539
AllocBucket(int iBucket)540 Bucket* OGROSMDataSource::AllocBucket( int iBucket )
541 {
542 if( bCompressNodes )
543 {
544 const int nRem = iBucket % (knPAGE_SIZE / BUCKET_SECTOR_SIZE_ARRAY_SIZE);
545 Bucket* psPrevBucket = GetBucket(iBucket - nRem);
546 if( psPrevBucket->u.panSectorSize == nullptr )
547 psPrevBucket->u.panSectorSize =
548 static_cast<GByte*>(VSI_CALLOC_VERBOSE(1, knPAGE_SIZE));
549 GByte* panSectorSize = psPrevBucket->u.panSectorSize;
550 Bucket* psBucket = GetBucket( iBucket );
551 if( panSectorSize != nullptr )
552 {
553 psBucket->u.panSectorSize =
554 panSectorSize +
555 nRem * BUCKET_SECTOR_SIZE_ARRAY_SIZE;
556 return psBucket;
557 }
558 psBucket->u.panSectorSize = nullptr;
559 }
560 else
561 {
562 const int nRem = iBucket % (knPAGE_SIZE / BUCKET_BITMAP_SIZE);
563 Bucket* psPrevBucket = GetBucket(iBucket - nRem);
564 if( psPrevBucket->u.pabyBitmap == nullptr )
565 psPrevBucket->u.pabyBitmap =
566 reinterpret_cast<GByte *>(VSI_CALLOC_VERBOSE(1, knPAGE_SIZE));
567 GByte* pabyBitmap = psPrevBucket->u.pabyBitmap;
568 Bucket* psBucket = GetBucket( iBucket );
569 if( pabyBitmap != nullptr )
570 {
571 psBucket->u.pabyBitmap =
572 pabyBitmap +
573 nRem * BUCKET_BITMAP_SIZE;
574 return psBucket;
575 }
576 psBucket->u.pabyBitmap = nullptr;
577 }
578
579 // Out of memory.
580 CPLError( CE_Failure, CPLE_AppDefined,
581 "AllocBucket() failed. Use OSM_USE_CUSTOM_INDEXING=NO" );
582 bStopParsing = true;
583 return nullptr;
584 }
585
586 /************************************************************************/
587 /* GetBucket() */
588 /************************************************************************/
589
GetBucket(int nBucketId)590 Bucket* OGROSMDataSource::GetBucket(int nBucketId)
591 {
592 std::map<int, Bucket>::iterator oIter = oMapBuckets.find(nBucketId);
593 if( oIter == oMapBuckets.end() )
594 {
595 Bucket* psBucket = &oMapBuckets[nBucketId];
596 psBucket->nOff = -1;
597 if( bCompressNodes )
598 psBucket->u.panSectorSize = nullptr;
599 else
600 psBucket->u.pabyBitmap = nullptr;
601 return psBucket;
602 }
603 return &(oIter->second);
604 }
605
606 /************************************************************************/
607 /* FlushCurrentSectorCompressedCase() */
608 /************************************************************************/
609
FlushCurrentSectorCompressedCase()610 bool OGROSMDataSource::FlushCurrentSectorCompressedCase()
611 {
612 GByte abyOutBuffer[2 * SECTOR_SIZE];
613 GByte* pabyOut = abyOutBuffer;
614 LonLat* pasLonLatIn = (LonLat*)pabySector;
615 int nLastLon = 0;
616 int nLastLat = 0;
617 bool bLastValid = false;
618
619 CPLAssert((NODE_PER_SECTOR % 8) == 0);
620 memset(abyOutBuffer, 0, NODE_PER_SECTOR / 8);
621 pabyOut += NODE_PER_SECTOR / 8;
622 for( int i = 0; i < NODE_PER_SECTOR; i++)
623 {
624 if( pasLonLatIn[i].nLon || pasLonLatIn[i].nLat )
625 {
626 abyOutBuffer[i >> 3] |= (1 << (i % 8));
627 if( bLastValid )
628 {
629 const GIntBig nDiff64Lon =
630 static_cast<GIntBig>(pasLonLatIn[i].nLon) -
631 static_cast<GIntBig>(nLastLon);
632 const GIntBig nDiff64Lat = pasLonLatIn[i].nLat - nLastLat;
633 WriteVarSInt64(nDiff64Lon, &pabyOut);
634 WriteVarSInt64(nDiff64Lat, &pabyOut);
635 }
636 else
637 {
638 memcpy(pabyOut, &pasLonLatIn[i], sizeof(LonLat));
639 pabyOut += sizeof(LonLat);
640 }
641 bLastValid = true;
642
643 nLastLon = pasLonLatIn[i].nLon;
644 nLastLat = pasLonLatIn[i].nLat;
645 }
646 }
647
648 size_t nCompressSize = static_cast<size_t>(pabyOut - abyOutBuffer);
649 CPLAssert(nCompressSize < sizeof(abyOutBuffer) - 1);
650 abyOutBuffer[nCompressSize] = 0;
651
652 nCompressSize = ROUND_COMPRESS_SIZE(nCompressSize);
653 GByte* pabyToWrite = nullptr;
654 if( nCompressSize >= static_cast<size_t>(SECTOR_SIZE) )
655 {
656 nCompressSize = SECTOR_SIZE;
657 pabyToWrite = pabySector;
658 }
659 else
660 pabyToWrite = abyOutBuffer;
661
662 if( VSIFWriteL(pabyToWrite, 1, nCompressSize, fpNodes) == nCompressSize )
663 {
664 memset(pabySector, 0, SECTOR_SIZE);
665 nNodesFileSize += nCompressSize;
666
667 Bucket* psBucket = GetBucket(nBucketOld);
668 if( psBucket->u.panSectorSize == nullptr )
669 {
670 psBucket = AllocBucket(nBucketOld);
671 if( psBucket == nullptr )
672 return false;
673 }
674 CPLAssert( psBucket->u.panSectorSize != nullptr );
675 psBucket->u.panSectorSize[nOffInBucketReducedOld] =
676 COMPRESS_SIZE_TO_BYTE(nCompressSize);
677
678 return true;
679 }
680
681 CPLError( CE_Failure, CPLE_AppDefined,
682 "Cannot write in temporary node file %s : %s",
683 osNodesFilename.c_str(), VSIStrerror(errno));
684
685 return false;
686 }
687
688 /************************************************************************/
689 /* FlushCurrentSectorNonCompressedCase() */
690 /************************************************************************/
691
FlushCurrentSectorNonCompressedCase()692 bool OGROSMDataSource::FlushCurrentSectorNonCompressedCase()
693 {
694 if( VSIFWriteL(pabySector, 1, static_cast<size_t>(SECTOR_SIZE),
695 fpNodes) == static_cast<size_t>(SECTOR_SIZE) )
696 {
697 memset(pabySector, 0, SECTOR_SIZE);
698 nNodesFileSize += SECTOR_SIZE;
699 return true;
700 }
701
702 CPLError( CE_Failure, CPLE_AppDefined,
703 "Cannot write in temporary node file %s : %s",
704 osNodesFilename.c_str(), VSIStrerror(errno));
705
706 return false;
707 }
708
709 /************************************************************************/
710 /* IndexPointCustom() */
711 /************************************************************************/
712
IndexPointCustom(OSMNode * psNode)713 bool OGROSMDataSource::IndexPointCustom(OSMNode* psNode)
714 {
715 if( psNode->nID <= nPrevNodeId)
716 {
717 CPLError(CE_Failure, CPLE_AppDefined,
718 "Non increasing node id. Use OSM_USE_CUSTOM_INDEXING=NO");
719 bStopParsing = true;
720 return false;
721 }
722 if( !VALID_ID_FOR_CUSTOM_INDEXING(psNode->nID) )
723 {
724 CPLError( CE_Failure, CPLE_AppDefined,
725 "Unsupported node id value (" CPL_FRMT_GIB
726 "). Use OSM_USE_CUSTOM_INDEXING=NO",
727 psNode->nID);
728 bStopParsing = true;
729 return false;
730 }
731
732 const int nBucket = static_cast<int>(psNode->nID / NODE_PER_BUCKET);
733 const int nOffInBucket = static_cast<int>(psNode->nID % NODE_PER_BUCKET);
734 const int nOffInBucketReduced = nOffInBucket >> NODE_PER_SECTOR_SHIFT;
735 const int nOffInBucketReducedRemainder =
736 nOffInBucket & ((1 << NODE_PER_SECTOR_SHIFT) - 1);
737
738 Bucket* psBucket = GetBucket(nBucket);
739
740 if( !bCompressNodes )
741 {
742 const int nBitmapIndex = nOffInBucketReduced / 8;
743 const int nBitmapRemainder = nOffInBucketReduced % 8;
744 if( psBucket->u.pabyBitmap == nullptr )
745 {
746 psBucket = AllocBucket(nBucket);
747 if( psBucket == nullptr )
748 return false;
749 }
750 CPLAssert( psBucket->u.pabyBitmap != nullptr );
751 psBucket->u.pabyBitmap[nBitmapIndex] |= (1 << nBitmapRemainder);
752 }
753
754 if( nBucket != nBucketOld )
755 {
756 CPLAssert(nBucket > nBucketOld);
757 if( nBucketOld >= 0 )
758 {
759 if( !FlushCurrentSector() )
760 {
761 bStopParsing = true;
762 return false;
763 }
764 }
765 nBucketOld = nBucket;
766 nOffInBucketReducedOld = nOffInBucketReduced;
767 CPLAssert(psBucket->nOff == -1);
768 psBucket->nOff = VSIFTellL(fpNodes);
769 }
770 else if( nOffInBucketReduced != nOffInBucketReducedOld )
771 {
772 CPLAssert(nOffInBucketReduced > nOffInBucketReducedOld);
773 if( !FlushCurrentSector() )
774 {
775 bStopParsing = true;
776 return false;
777 }
778 nOffInBucketReducedOld = nOffInBucketReduced;
779 }
780
781 LonLat* psLonLat = reinterpret_cast<LonLat*>(
782 pabySector + sizeof(LonLat) * nOffInBucketReducedRemainder);
783 psLonLat->nLon = DBL_TO_INT(psNode->dfLon);
784 psLonLat->nLat = DBL_TO_INT(psNode->dfLat);
785
786 nPrevNodeId = psNode->nID;
787
788 return true;
789 }
790
791 /************************************************************************/
792 /* NotifyNodes() */
793 /************************************************************************/
794
NotifyNodes(unsigned int nNodes,OSMNode * pasNodes)795 void OGROSMDataSource::NotifyNodes( unsigned int nNodes, OSMNode* pasNodes )
796 {
797 const OGREnvelope* psEnvelope =
798 papoLayers[IDX_LYR_POINTS]->GetSpatialFilterEnvelope();
799
800 for( unsigned int i = 0; i < nNodes; i++ )
801 {
802 /* If the point doesn't fit into the envelope of the spatial filter */
803 /* then skip it */
804 if( psEnvelope != nullptr &&
805 !(pasNodes[i].dfLon >= psEnvelope->MinX &&
806 pasNodes[i].dfLon <= psEnvelope->MaxX &&
807 pasNodes[i].dfLat >= psEnvelope->MinY &&
808 pasNodes[i].dfLat <= psEnvelope->MaxY) )
809 continue;
810
811 if( !IndexPoint(&pasNodes[i]) )
812 break;
813
814 if( !papoLayers[IDX_LYR_POINTS]->IsUserInterested() )
815 continue;
816
817 bool bInterestingTag = bReportAllNodes;
818 OSMTag* pasTags = pasNodes[i].pasTags;
819
820 if( !bReportAllNodes )
821 {
822 for( unsigned int j = 0; j < pasNodes[i].nTags; j++)
823 {
824 const char* pszK = pasTags[j].pszK;
825 if( papoLayers[IDX_LYR_POINTS]->IsSignificantKey(pszK) )
826 {
827 bInterestingTag = true;
828 break;
829 }
830 }
831 }
832
833 if( bInterestingTag )
834 {
835 OGRFeature* poFeature = new OGRFeature(
836 papoLayers[IDX_LYR_POINTS]->GetLayerDefn());
837
838 poFeature->SetGeometryDirectly(
839 new OGRPoint(pasNodes[i].dfLon, pasNodes[i].dfLat));
840
841 papoLayers[IDX_LYR_POINTS]->SetFieldsFromTags(
842 poFeature, pasNodes[i].nID, false, pasNodes[i].nTags,
843 pasTags, &pasNodes[i].sInfo );
844
845 int bFilteredOut = FALSE;
846 if( !papoLayers[IDX_LYR_POINTS]->AddFeature(poFeature, FALSE,
847 &bFilteredOut,
848 !bFeatureAdded) )
849 {
850 bStopParsing = true;
851 break;
852 }
853 else if( !bFilteredOut )
854 bFeatureAdded = true;
855 }
856 }
857 }
858
OGROSMNotifyNodes(unsigned int nNodes,OSMNode * pasNodes,OSMContext *,void * user_data)859 static void OGROSMNotifyNodes ( unsigned int nNodes,
860 OSMNode *pasNodes,
861 OSMContext * /* psOSMContext */,
862 void *user_data)
863 {
864 static_cast<OGROSMDataSource *>(user_data)->NotifyNodes(nNodes, pasNodes);
865 }
866
867 /************************************************************************/
868 /* LookupNodes() */
869 /************************************************************************/
870
871 //#define DEBUG_COLLISIONS 1
872
LookupNodes()873 void OGROSMDataSource::LookupNodes( )
874 {
875 if( bCustomIndexing )
876 LookupNodesCustom();
877 else
878 LookupNodesSQLite();
879
880 #ifdef ENABLE_NODE_LOOKUP_BY_HASHING
881 if( nReqIds > 1 && bEnableHashedIndex )
882 {
883 memset(panHashedIndexes, 0xFF, HASHED_INDEXES_ARRAY_SIZE * sizeof(int));
884 bHashedIndexValid = true;
885 #ifdef DEBUG_COLLISIONS
886 int nCollisions = 0;
887 #endif
888 int iNextFreeBucket = 0;
889 for(unsigned int i = 0; i < nReqIds; i++)
890 {
891 int nIndInHashArray = static_cast<int>(HASH_ID_FUNC(panReqIds[i]) % HASHED_INDEXES_ARRAY_SIZE);
892 int nIdx = panHashedIndexes[nIndInHashArray];
893 if( nIdx == -1 )
894 {
895 panHashedIndexes[nIndInHashArray] = i;
896 }
897 else
898 {
899 #ifdef DEBUG_COLLISIONS
900 nCollisions ++;
901 #endif
902 int iBucket = 0;
903 if( nIdx >= 0 )
904 {
905 if(iNextFreeBucket == COLLISION_BUCKET_ARRAY_SIZE)
906 {
907 CPLDebug("OSM", "Too many collisions. Disabling hashed indexing");
908 bHashedIndexValid = false;
909 bEnableHashedIndex = false;
910 break;
911 }
912 iBucket = iNextFreeBucket;
913 psCollisionBuckets[iNextFreeBucket].nInd = nIdx;
914 psCollisionBuckets[iNextFreeBucket].nNext = -1;
915 panHashedIndexes[nIndInHashArray] = -iNextFreeBucket - 2;
916 iNextFreeBucket ++;
917 }
918 else
919 {
920 iBucket = -nIdx - 2;
921 }
922 if(iNextFreeBucket == COLLISION_BUCKET_ARRAY_SIZE)
923 {
924 CPLDebug("OSM", "Too many collisions. Disabling hashed indexing");
925 bHashedIndexValid = false;
926 bEnableHashedIndex = false;
927 break;
928 }
929 while( true )
930 {
931 int iNext = psCollisionBuckets[iBucket].nNext;
932 if( iNext < 0 )
933 {
934 psCollisionBuckets[iBucket].nNext = iNextFreeBucket;
935 psCollisionBuckets[iNextFreeBucket].nInd = i;
936 psCollisionBuckets[iNextFreeBucket].nNext = -1;
937 iNextFreeBucket ++;
938 break;
939 }
940 iBucket = iNext;
941 }
942 }
943 }
944 #ifdef DEBUG_COLLISIONS
945 /* Collision rate in practice is around 12% on France, Germany, ... */
946 /* Maximum seen ~ 15.9% on a planet file but often much smaller. */
947 CPLDebug("OSM", "nCollisions = %d/%d (%.1f %%), iNextFreeBucket = %d/%d",
948 nCollisions, nReqIds, nCollisions * 100.0 / nReqIds,
949 iNextFreeBucket, COLLISION_BUCKET_ARRAY_SIZE);
950 #endif
951 }
952 else
953 bHashedIndexValid = false;
954 #endif // ENABLE_NODE_LOOKUP_BY_HASHING
955 }
956
957 /************************************************************************/
958 /* LookupNodesSQLite() */
959 /************************************************************************/
960
LookupNodesSQLite()961 void OGROSMDataSource::LookupNodesSQLite( )
962 {
963 CPLAssert(
964 nUnsortedReqIds <= static_cast<unsigned int>(MAX_ACCUMULATED_NODES));
965
966 nReqIds = 0;
967 for( unsigned int i = 0; i < nUnsortedReqIds; i++)
968 {
969 GIntBig id = panUnsortedReqIds[i];
970 panReqIds[nReqIds++] = id;
971 }
972
973 std::sort(panReqIds, panReqIds + nReqIds);
974
975 /* Remove duplicates */
976 unsigned int j = 0;
977 for( unsigned int i = 0; i < nReqIds; i++)
978 {
979 if( !(i > 0 && panReqIds[i] == panReqIds[i-1]) )
980 panReqIds[j++] = panReqIds[i];
981 }
982 nReqIds = j;
983
984 unsigned int iCur = 0;
985 j = 0;
986 while( iCur < nReqIds )
987 {
988 unsigned int nToQuery = nReqIds - iCur;
989 if( nToQuery > static_cast<unsigned int>(LIMIT_IDS_PER_REQUEST) )
990 nToQuery = static_cast<unsigned int>(LIMIT_IDS_PER_REQUEST);
991
992 sqlite3_stmt* hStmt = pahSelectNodeStmt[nToQuery-1];
993 for( unsigned int i=iCur;i<iCur + nToQuery;i++)
994 {
995 sqlite3_bind_int64( hStmt, i - iCur +1, panReqIds[i] );
996 }
997 iCur += nToQuery;
998
999 while( sqlite3_step(hStmt) == SQLITE_ROW )
1000 {
1001 const GIntBig id = sqlite3_column_int64(hStmt, 0);
1002 LonLat* psLonLat = (LonLat*)sqlite3_column_blob(hStmt, 1);
1003
1004 panReqIds[j] = id;
1005 pasLonLatArray[j].nLon = psLonLat->nLon;
1006 pasLonLatArray[j].nLat = psLonLat->nLat;
1007 j++;
1008 }
1009
1010 sqlite3_reset(hStmt);
1011 }
1012 nReqIds = j;
1013 }
1014
1015 /************************************************************************/
1016 /* DecompressSector() */
1017 /************************************************************************/
1018
DecompressSector(const GByte * pabyIn,int nSectorSize,GByte * pabyOut)1019 static bool DecompressSector( const GByte* pabyIn, int nSectorSize,
1020 GByte* pabyOut )
1021 {
1022 const GByte* pabyPtr = pabyIn;
1023 LonLat* pasLonLatOut = (LonLat*) pabyOut;
1024 int nLastLon = 0;
1025 int nLastLat = 0;
1026 bool bLastValid = false;
1027
1028 pabyPtr += NODE_PER_SECTOR / 8;
1029 for( int i = 0; i < NODE_PER_SECTOR; i++)
1030 {
1031 if( pabyIn[i >> 3] & (1 << (i % 8)) )
1032 {
1033 if( bLastValid )
1034 {
1035 pasLonLatOut[i].nLon = (int)(nLastLon + ReadVarSInt64(&pabyPtr));
1036 pasLonLatOut[i].nLat = (int)(nLastLat + ReadVarSInt64(&pabyPtr));
1037 }
1038 else
1039 {
1040 bLastValid = true;
1041 memcpy(&(pasLonLatOut[i]), pabyPtr, sizeof(LonLat));
1042 pabyPtr += sizeof(LonLat);
1043 }
1044
1045 nLastLon = pasLonLatOut[i].nLon;
1046 nLastLat = pasLonLatOut[i].nLat;
1047 }
1048 else
1049 {
1050 pasLonLatOut[i].nLon = 0;
1051 pasLonLatOut[i].nLat = 0;
1052 }
1053 }
1054
1055 int nRead = (int)(pabyPtr - pabyIn);
1056 nRead = ROUND_COMPRESS_SIZE(nRead);
1057 return nRead == nSectorSize;
1058 }
1059
1060 /************************************************************************/
1061 /* LookupNodesCustom() */
1062 /************************************************************************/
1063
LookupNodesCustom()1064 void OGROSMDataSource::LookupNodesCustom( )
1065 {
1066 nReqIds = 0;
1067
1068 if( nBucketOld >= 0 )
1069 {
1070 if( !FlushCurrentSector() )
1071 {
1072 bStopParsing = true;
1073 return;
1074 }
1075
1076 nBucketOld = -1;
1077 }
1078
1079 CPLAssert(
1080 nUnsortedReqIds <= static_cast<unsigned int>(MAX_ACCUMULATED_NODES));
1081
1082 for( unsigned int i = 0; i < nUnsortedReqIds; i++ )
1083 {
1084 GIntBig id = panUnsortedReqIds[i];
1085
1086 if( !VALID_ID_FOR_CUSTOM_INDEXING(id) )
1087 continue;
1088
1089 int nBucket = static_cast<int>(id / NODE_PER_BUCKET);
1090 int nOffInBucket = static_cast<int>(id % NODE_PER_BUCKET);
1091 int nOffInBucketReduced = nOffInBucket >> NODE_PER_SECTOR_SHIFT;
1092
1093 std::map<int, Bucket>::const_iterator oIter = oMapBuckets.find(nBucket);
1094 if( oIter == oMapBuckets.end() )
1095 continue;
1096 const Bucket* psBucket = &(oIter->second);
1097
1098 if( bCompressNodes )
1099 {
1100 if( psBucket->u.panSectorSize == nullptr ||
1101 !(psBucket->u.panSectorSize[nOffInBucketReduced]) )
1102 continue;
1103 }
1104 else
1105 {
1106 int nBitmapIndex = nOffInBucketReduced / 8;
1107 int nBitmapRemainder = nOffInBucketReduced % 8;
1108 if( psBucket->u.pabyBitmap == nullptr ||
1109 !(psBucket->u.pabyBitmap[nBitmapIndex] & (1 << nBitmapRemainder)) )
1110 continue;
1111 }
1112
1113 panReqIds[nReqIds++] = id;
1114 }
1115
1116 std::sort(panReqIds, panReqIds + nReqIds);
1117
1118 /* Remove duplicates */
1119 unsigned int j = 0; // Used after for.
1120 for( unsigned int i = 0; i < nReqIds; i++)
1121 {
1122 if( !(i > 0 && panReqIds[i] == panReqIds[i-1]) )
1123 panReqIds[j++] = panReqIds[i];
1124 }
1125 nReqIds = j;
1126
1127 #ifdef FAKE_LOOKUP_NODES
1128 for( unsigned int i = 0; i < nReqIds; i++)
1129 {
1130 pasLonLatArray[i].nLon = 0;
1131 pasLonLatArray[i].nLat = 0;
1132 }
1133 #else
1134 if( bCompressNodes )
1135 LookupNodesCustomCompressedCase();
1136 else
1137 LookupNodesCustomNonCompressedCase();
1138 #endif
1139 }
1140
1141 /************************************************************************/
1142 /* LookupNodesCustomCompressedCase() */
1143 /************************************************************************/
1144
LookupNodesCustomCompressedCase()1145 void OGROSMDataSource::LookupNodesCustomCompressedCase()
1146 {
1147 constexpr int SECURITY_MARGIN = 8 + 8 + 2 * NODE_PER_SECTOR;
1148 GByte abyRawSector[SECTOR_SIZE + SECURITY_MARGIN];
1149 memset(abyRawSector + SECTOR_SIZE, 0, SECURITY_MARGIN);
1150
1151 int l_nBucketOld = -1;
1152 int l_nOffInBucketReducedOld = -1;
1153 int k = 0;
1154 int nOffFromBucketStart = 0;
1155
1156 unsigned int j = 0; // Used after for.
1157 for( unsigned int i = 0; i < nReqIds; i++ )
1158 {
1159 const GIntBig id = panReqIds[i];
1160 const int nBucket = static_cast<int>(id / NODE_PER_BUCKET);
1161 const int nOffInBucket = static_cast<int>(id % NODE_PER_BUCKET);
1162 const int nOffInBucketReduced = nOffInBucket >> NODE_PER_SECTOR_SHIFT;
1163 const int nOffInBucketReducedRemainder =
1164 nOffInBucket & ((1 << NODE_PER_SECTOR_SHIFT) - 1);
1165
1166 if( nBucket != l_nBucketOld )
1167 {
1168 l_nOffInBucketReducedOld = -1;
1169 k = 0;
1170 nOffFromBucketStart = 0;
1171 }
1172
1173 if( nOffInBucketReduced != l_nOffInBucketReducedOld )
1174 {
1175 std::map<int, Bucket>::const_iterator oIter = oMapBuckets.find(nBucket);
1176 if( oIter == oMapBuckets.end() )
1177 {
1178 CPLError(CE_Failure, CPLE_AppDefined,
1179 "Cannot read node " CPL_FRMT_GIB, id);
1180 continue;
1181 // FIXME ?
1182 }
1183 const Bucket* psBucket = &(oIter->second);
1184 if( psBucket->u.panSectorSize == nullptr )
1185 {
1186 CPLError(CE_Failure, CPLE_AppDefined,
1187 "Cannot read node " CPL_FRMT_GIB, id);
1188 continue;
1189 // FIXME ?
1190 }
1191 const int nSectorSize =
1192 COMPRESS_SIZE_FROM_BYTE(
1193 psBucket->u.panSectorSize[nOffInBucketReduced]);
1194
1195 /* If we stay in the same bucket, we can reuse the previously */
1196 /* computed offset, instead of starting from bucket start */
1197 for( ; k < nOffInBucketReduced; k++ )
1198 {
1199 if( psBucket->u.panSectorSize[k] )
1200 nOffFromBucketStart +=
1201 COMPRESS_SIZE_FROM_BYTE(psBucket->u.panSectorSize[k]);
1202 }
1203
1204 VSIFSeekL(fpNodes, psBucket->nOff + nOffFromBucketStart, SEEK_SET);
1205 if( nSectorSize == SECTOR_SIZE )
1206 {
1207 if( VSIFReadL(pabySector, 1,
1208 static_cast<size_t>(SECTOR_SIZE),
1209 fpNodes) != static_cast<size_t>(SECTOR_SIZE) )
1210 {
1211 CPLError(CE_Failure, CPLE_AppDefined,
1212 "Cannot read node " CPL_FRMT_GIB, id);
1213 continue;
1214 // FIXME ?
1215 }
1216 }
1217 else
1218 {
1219 if( static_cast<int>(VSIFReadL(abyRawSector, 1, nSectorSize,
1220 fpNodes)) != nSectorSize )
1221 {
1222 CPLError(CE_Failure, CPLE_AppDefined,
1223 "Cannot read sector for node " CPL_FRMT_GIB, id);
1224 continue;
1225 // FIXME ?
1226 }
1227 abyRawSector[nSectorSize] = 0;
1228
1229 if( !DecompressSector(abyRawSector, nSectorSize, pabySector) )
1230 {
1231 CPLError( CE_Failure, CPLE_AppDefined,
1232 "Error while uncompressing sector for node "
1233 CPL_FRMT_GIB, id );
1234 continue;
1235 // FIXME ?
1236 }
1237 }
1238
1239 l_nBucketOld = nBucket;
1240 l_nOffInBucketReducedOld = nOffInBucketReduced;
1241 }
1242
1243 panReqIds[j] = id;
1244 memcpy(pasLonLatArray + j,
1245 pabySector + nOffInBucketReducedRemainder * sizeof(LonLat),
1246 sizeof(LonLat));
1247
1248 if( pasLonLatArray[j].nLon || pasLonLatArray[j].nLat )
1249 j++;
1250 }
1251 nReqIds = j;
1252 }
1253
1254 /************************************************************************/
1255 /* LookupNodesCustomNonCompressedCase() */
1256 /************************************************************************/
1257
LookupNodesCustomNonCompressedCase()1258 void OGROSMDataSource::LookupNodesCustomNonCompressedCase()
1259 {
1260 unsigned int j = 0; // Used after for.
1261
1262 int l_nBucketOld = -1;
1263 const Bucket* psBucket = nullptr;
1264 // To be glibc friendly, we will do reads aligned on 4096 byte offsets
1265 const int knDISK_SECTOR_SIZE = 4096;
1266 CPL_STATIC_ASSERT( (knDISK_SECTOR_SIZE % SECTOR_SIZE) == 0 );
1267 GByte abyDiskSector[knDISK_SECTOR_SIZE];
1268 // Offset in the nodes files for which abyDiskSector was read
1269 GIntBig nOldOffset = -knDISK_SECTOR_SIZE-1;
1270 // Number of valid bytes in abyDiskSector
1271 size_t nValidBytes = 0;
1272 int k = 0;
1273 int nSectorBase = 0;
1274 for( unsigned int i = 0; i < nReqIds; i++ )
1275 {
1276 const GIntBig id = panReqIds[i];
1277 const int nBucket = static_cast<int>(id / NODE_PER_BUCKET);
1278 const int nOffInBucket = static_cast<int>(id % NODE_PER_BUCKET);
1279 const int nOffInBucketReduced = nOffInBucket >> NODE_PER_SECTOR_SHIFT;
1280 const int nOffInBucketReducedRemainder =
1281 nOffInBucket & ((1 << NODE_PER_SECTOR_SHIFT) - 1);
1282
1283 const int nBitmapIndex = nOffInBucketReduced / 8;
1284 const int nBitmapRemainder = nOffInBucketReduced % 8;
1285
1286 if( psBucket == nullptr || nBucket != l_nBucketOld )
1287 {
1288 std::map<int, Bucket>::const_iterator oIter = oMapBuckets.find(nBucket);
1289 if( oIter == oMapBuckets.end() )
1290 {
1291 CPLError(CE_Failure, CPLE_AppDefined,
1292 "Cannot read node " CPL_FRMT_GIB, id);
1293 continue;
1294 // FIXME ?
1295 }
1296 psBucket = &(oIter->second);
1297 if( psBucket->u.pabyBitmap == nullptr )
1298 {
1299 CPLError(CE_Failure, CPLE_AppDefined,
1300 "Cannot read node " CPL_FRMT_GIB, id);
1301 continue;
1302 // FIXME ?
1303 }
1304 l_nBucketOld = nBucket;
1305 nOldOffset = -knDISK_SECTOR_SIZE-1;
1306 k = 0;
1307 nSectorBase = 0;
1308 }
1309
1310 /* If we stay in the same bucket, we can reuse the previously */
1311 /* computed offset, instead of starting from bucket start */
1312 for( ; k < nBitmapIndex; k++ )
1313 {
1314 assert(psBucket->u.pabyBitmap);
1315 // psBucket->u.pabyBitmap cannot be NULL
1316 // coverity[var_deref_op]
1317 nSectorBase += abyBitsCount[psBucket->u.pabyBitmap[k]];
1318 }
1319 int nSector = nSectorBase;
1320 if( nBitmapRemainder )
1321 {
1322 assert(psBucket->u.pabyBitmap);
1323 nSector +=
1324 abyBitsCount[psBucket->u.pabyBitmap[nBitmapIndex] &
1325 ((1 << nBitmapRemainder) - 1)];
1326 }
1327
1328 const GIntBig nNewOffset = psBucket->nOff + nSector * SECTOR_SIZE;
1329 if( nNewOffset - nOldOffset >= knDISK_SECTOR_SIZE )
1330 {
1331 // Align on 4096 boundary to be glibc caching friendly
1332 const GIntBig nAlignedNewPos = nNewOffset &
1333 ~(static_cast<GIntBig>(knDISK_SECTOR_SIZE)-1);
1334 VSIFSeekL(fpNodes, nAlignedNewPos, SEEK_SET);
1335 nValidBytes =
1336 VSIFReadL(abyDiskSector, 1, knDISK_SECTOR_SIZE, fpNodes);
1337 nOldOffset = nAlignedNewPos;
1338 }
1339
1340 const size_t nOffsetInDiskSector =
1341 static_cast<size_t>(nNewOffset - nOldOffset) +
1342 nOffInBucketReducedRemainder * sizeof(LonLat);
1343 if( nValidBytes < sizeof(LonLat) ||
1344 nOffsetInDiskSector > nValidBytes - sizeof(LonLat) )
1345 {
1346 CPLError(CE_Failure, CPLE_AppDefined,
1347 "Cannot read node " CPL_FRMT_GIB, id);
1348 continue;
1349 }
1350 memcpy( &pasLonLatArray[j],
1351 abyDiskSector + nOffsetInDiskSector,
1352 sizeof(LonLat) );
1353
1354 panReqIds[j] = id;
1355 if( pasLonLatArray[j].nLon || pasLonLatArray[j].nLat )
1356 j++;
1357 }
1358 nReqIds = j;
1359 }
1360
1361 /************************************************************************/
1362 /* WriteVarInt() */
1363 /************************************************************************/
1364
WriteVarInt(unsigned int nVal,std::vector<GByte> & abyData)1365 static void WriteVarInt( unsigned int nVal, std::vector<GByte>& abyData )
1366 {
1367 while( true )
1368 {
1369 if( (nVal & (~0x7fU)) == 0 )
1370 {
1371 abyData.push_back((GByte)nVal);
1372 return;
1373 }
1374
1375 abyData.push_back(0x80 | (GByte)(nVal & 0x7f));
1376 nVal >>= 7;
1377 }
1378 }
1379
1380 /************************************************************************/
1381 /* WriteVarInt64() */
1382 /************************************************************************/
1383
WriteVarInt64(GUIntBig nVal,std::vector<GByte> & abyData)1384 static void WriteVarInt64( GUIntBig nVal, std::vector<GByte>& abyData )
1385 {
1386 while( true )
1387 {
1388 if( (((GUInt32)nVal) & (~0x7fU)) == 0 )
1389 {
1390 abyData.push_back((GByte)nVal);
1391 return;
1392 }
1393
1394 abyData.push_back(0x80 | (GByte)(nVal & 0x7f));
1395 nVal >>= 7;
1396 }
1397 }
1398
1399 /************************************************************************/
1400 /* WriteVarSInt64() */
1401 /************************************************************************/
1402
WriteVarSInt64(GIntBig nSVal,std::vector<GByte> & abyData)1403 static void WriteVarSInt64( GIntBig nSVal, std::vector<GByte>& abyData )
1404 {
1405 GIntBig nVal = nSVal >= 0
1406 ? nSVal << 1
1407 : ((-1-nSVal) << 1) + 1;
1408
1409 while( true )
1410 {
1411 if( (nVal & (~0x7f)) == 0 )
1412 {
1413 abyData.push_back((GByte)nVal);
1414 return;
1415 }
1416
1417 abyData.push_back(0x80 | (GByte)(nVal & 0x7f));
1418 nVal >>= 7;
1419 }
1420 }
1421
1422 /************************************************************************/
1423 /* WriteVarSInt64() */
1424 /************************************************************************/
1425
WriteVarSInt64(GIntBig nSVal,GByte ** ppabyData)1426 static void WriteVarSInt64( GIntBig nSVal, GByte** ppabyData )
1427 {
1428 GIntBig nVal = nSVal >= 0
1429 ? nSVal << 1
1430 : ((-1-nSVal) << 1) + 1;
1431
1432 GByte* pabyData = *ppabyData;
1433 while( true )
1434 {
1435 if( (nVal & (~0x7f)) == 0 )
1436 {
1437 *pabyData = (GByte)nVal;
1438 *ppabyData = pabyData + 1;
1439 return;
1440 }
1441
1442 *pabyData = 0x80 | (GByte)(nVal & 0x7f);
1443 nVal >>= 7;
1444 pabyData++;
1445 }
1446 }
1447
1448 /************************************************************************/
1449 /* CompressWay() */
1450 /************************************************************************/
1451
CompressWay(bool bIsArea,unsigned int nTags,IndexedKVP * pasTags,int nPoints,LonLat * pasLonLatPairs,OSMInfo * psInfo,std::vector<GByte> & abyCompressedWay)1452 void OGROSMDataSource::CompressWay ( bool bIsArea, unsigned int nTags,
1453 IndexedKVP* pasTags,
1454 int nPoints, LonLat* pasLonLatPairs,
1455 OSMInfo* psInfo,
1456 std::vector<GByte>& abyCompressedWay )
1457 {
1458 abyCompressedWay.clear();
1459 abyCompressedWay.push_back((bIsArea) ? 1 : 0);
1460 abyCompressedWay.push_back(0); // reserve space for tagCount
1461
1462 unsigned int nTagCount = 0;
1463 CPLAssert(nTags <= MAX_COUNT_FOR_TAGS_IN_WAY);
1464 for( unsigned int iTag = 0; iTag < nTags; iTag++ )
1465 {
1466 WriteVarInt(pasTags[iTag].nKeyIndex, abyCompressedWay);
1467
1468 // To fit in 2 bytes, the theoretical limit would be 127 * 128 + 127.
1469 if( pasTags[iTag].bVIsIndex )
1470 {
1471 WriteVarInt(pasTags[iTag].u.nValueIndex, abyCompressedWay);
1472 }
1473 else
1474 {
1475 const char* pszV = (const char*)pabyNonRedundantValues +
1476 pasTags[iTag].u.nOffsetInpabyNonRedundantValues;
1477
1478 abyCompressedWay.push_back(0);
1479
1480 abyCompressedWay.insert(abyCompressedWay.end(),
1481 reinterpret_cast<const GByte*>(pszV),
1482 reinterpret_cast<const GByte*>(pszV) + strlen(pszV) + 1);
1483 }
1484
1485 nTagCount ++;
1486 }
1487
1488 abyCompressedWay[1] = (GByte) nTagCount;
1489
1490 if( bNeedsToSaveWayInfo )
1491 {
1492 if( psInfo != nullptr )
1493 {
1494 abyCompressedWay.push_back(1);
1495 WriteVarInt64(psInfo->ts.nTimeStamp, abyCompressedWay);
1496 WriteVarInt64(psInfo->nChangeset,abyCompressedWay);
1497 WriteVarInt(psInfo->nVersion, abyCompressedWay);
1498 WriteVarInt(psInfo->nUID, abyCompressedWay);
1499 // FIXME : do something with pszUserSID
1500 }
1501 else
1502 {
1503 abyCompressedWay.push_back(0);
1504 }
1505 }
1506
1507 abyCompressedWay.insert(abyCompressedWay.end(),
1508 reinterpret_cast<const GByte*>(&(pasLonLatPairs[0])),
1509 reinterpret_cast<const GByte*>(&(pasLonLatPairs[0])) + sizeof(LonLat));
1510 for(int i=1;i<nPoints;i++)
1511 {
1512 GIntBig nDiff64 =
1513 (GIntBig)pasLonLatPairs[i].nLon - (GIntBig)pasLonLatPairs[i-1].nLon;
1514 WriteVarSInt64(nDiff64, abyCompressedWay);
1515
1516 nDiff64 = pasLonLatPairs[i].nLat - pasLonLatPairs[i-1].nLat;
1517 WriteVarSInt64(nDiff64,abyCompressedWay);
1518 }
1519 }
1520
1521 /************************************************************************/
1522 /* UncompressWay() */
1523 /************************************************************************/
1524
UncompressWay(int nBytes,const GByte * pabyCompressedWay,bool * pbIsArea,std::vector<LonLat> & asCoords,unsigned int * pnTags,OSMTag * pasTags,OSMInfo * psInfo)1525 void OGROSMDataSource::UncompressWay( int nBytes, const GByte* pabyCompressedWay,
1526 bool* pbIsArea,
1527 std::vector<LonLat>& asCoords,
1528 unsigned int* pnTags, OSMTag* pasTags,
1529 OSMInfo* psInfo )
1530 {
1531 asCoords.clear();
1532 const GByte* pabyPtr = pabyCompressedWay;
1533 if( pbIsArea )
1534 *pbIsArea = (*pabyPtr == 1) ? true : false;
1535 pabyPtr ++;
1536 unsigned int nTags = *pabyPtr;
1537 pabyPtr ++;
1538
1539 if( pnTags )
1540 *pnTags = nTags;
1541
1542 // TODO: Some additional safety checks.
1543 for(unsigned int iTag = 0; iTag < nTags; iTag++)
1544 {
1545 int nK = ReadVarInt32(&pabyPtr);
1546 int nV = ReadVarInt32(&pabyPtr);
1547 const GByte* pszV = nullptr;
1548 if( nV == 0 )
1549 {
1550 pszV = pabyPtr;
1551 while(*pabyPtr != '\0')
1552 pabyPtr ++;
1553 pabyPtr ++;
1554 }
1555
1556 if( pasTags )
1557 {
1558 CPLAssert(nK >= 0 && nK < (int)asKeys.size());
1559 pasTags[iTag].pszK = asKeys[nK]->pszK;
1560 CPLAssert(nV == 0 ||
1561 (nV > 0 && nV < (int)asKeys[nK]->asValues.size()));
1562 pasTags[iTag].pszV =
1563 nV ? asKeys[nK]->asValues[nV] : (const char*) pszV;
1564 }
1565 }
1566
1567 if( bNeedsToSaveWayInfo )
1568 {
1569 if( *pabyPtr )
1570 {
1571 pabyPtr ++;
1572
1573 OSMInfo sInfo;
1574 if( psInfo == nullptr )
1575 psInfo = &sInfo;
1576
1577 psInfo->ts.nTimeStamp = ReadVarInt64(&pabyPtr);
1578 psInfo->nChangeset = ReadVarInt64(&pabyPtr);
1579 psInfo->nVersion = ReadVarInt32(&pabyPtr);
1580 psInfo->nUID = ReadVarInt32(&pabyPtr);
1581
1582 psInfo->bTimeStampIsStr = false;
1583 psInfo->pszUserSID = ""; // FIXME
1584 }
1585 else
1586 pabyPtr ++;
1587 }
1588
1589 LonLat lonLat;
1590 memcpy(&lonLat.nLon, pabyPtr, sizeof(int));
1591 memcpy(&lonLat.nLat, pabyPtr + sizeof(int), sizeof(int));
1592 asCoords.emplace_back(lonLat);
1593 pabyPtr += 2 * sizeof(int);
1594 do
1595 {
1596 lonLat.nLon = (int)(lonLat.nLon + ReadVarSInt64(&pabyPtr));
1597 lonLat.nLat = (int)(lonLat.nLat + ReadVarSInt64(&pabyPtr));
1598 asCoords.emplace_back(lonLat);
1599 } while (pabyPtr < pabyCompressedWay + nBytes);
1600 }
1601
1602 /************************************************************************/
1603 /* IndexWay() */
1604 /************************************************************************/
1605
IndexWay(GIntBig nWayID,bool bIsArea,unsigned int nTags,IndexedKVP * pasTags,LonLat * pasLonLatPairs,int nPairs,OSMInfo * psInfo)1606 void OGROSMDataSource::IndexWay(GIntBig nWayID, bool bIsArea,
1607 unsigned int nTags, IndexedKVP* pasTags,
1608 LonLat* pasLonLatPairs, int nPairs,
1609 OSMInfo* psInfo)
1610 {
1611 if( !bIndexWays )
1612 return;
1613
1614 sqlite3_bind_int64( hInsertWayStmt, 1, nWayID );
1615
1616 const unsigned nTagsClamped = std::min(nTags, MAX_COUNT_FOR_TAGS_IN_WAY);
1617 if( nTagsClamped < nTags )
1618 {
1619 CPLDebug("OSM", "Too many tags for way " CPL_FRMT_GIB ": %u. "
1620 "Clamping to %u",
1621 nWayID, nTags, nTagsClamped);
1622 }
1623 CompressWay (bIsArea, nTagsClamped, pasTags, nPairs, pasLonLatPairs, psInfo,
1624 m_abyWayBuffer);
1625 sqlite3_bind_blob( hInsertWayStmt, 2,
1626 m_abyWayBuffer.data(),
1627 static_cast<int>(m_abyWayBuffer.size()),
1628 SQLITE_STATIC );
1629
1630 int rc = sqlite3_step( hInsertWayStmt );
1631 sqlite3_reset( hInsertWayStmt );
1632 if( !(rc == SQLITE_OK || rc == SQLITE_DONE) )
1633 {
1634 CPLError(CE_Failure, CPLE_AppDefined,
1635 "Failed inserting way " CPL_FRMT_GIB ": %s",
1636 nWayID, sqlite3_errmsg(hDB));
1637 }
1638 }
1639
1640 /************************************************************************/
1641 /* FindNode() */
1642 /************************************************************************/
1643
FindNode(GIntBig nID)1644 int OGROSMDataSource::FindNode(GIntBig nID)
1645 {
1646 if( nReqIds == 0 )
1647 return -1;
1648 int iFirst = 0;
1649 int iLast = nReqIds - 1;
1650 while(iFirst < iLast)
1651 {
1652 int iMid = (iFirst + iLast) / 2;
1653 if( nID > panReqIds[iMid])
1654 iFirst = iMid + 1;
1655 else
1656 iLast = iMid;
1657 }
1658 if( iFirst == iLast && nID == panReqIds[iFirst] )
1659 return iFirst;
1660 return -1;
1661 }
1662
1663 /************************************************************************/
1664 /* ProcessWaysBatch() */
1665 /************************************************************************/
1666
ProcessWaysBatch()1667 void OGROSMDataSource::ProcessWaysBatch()
1668 {
1669 if( nWayFeaturePairs == 0 ) return;
1670
1671 //printf("nodes = %d, features = %d\n", nUnsortedReqIds, nWayFeaturePairs);
1672 LookupNodes();
1673
1674 for( int iPair = 0; iPair < nWayFeaturePairs; iPair ++)
1675 {
1676 WayFeaturePair* psWayFeaturePairs = &pasWayFeaturePairs[iPair];
1677
1678 const EMULATED_BOOL bIsArea = psWayFeaturePairs->bIsArea;
1679 m_asLonLatCache.clear();
1680
1681 #ifdef ENABLE_NODE_LOOKUP_BY_HASHING
1682 if( bHashedIndexValid )
1683 {
1684 for( unsigned int i=0;i<psWayFeaturePairs->nRefs;i++)
1685 {
1686 int nIndInHashArray = static_cast<int>(
1687 HASH_ID_FUNC(psWayFeaturePairs->panNodeRefs[i]) %
1688 HASHED_INDEXES_ARRAY_SIZE);
1689 int nIdx = panHashedIndexes[nIndInHashArray];
1690 if( nIdx < -1 )
1691 {
1692 int iBucket = -nIdx - 2;
1693 while( true )
1694 {
1695 nIdx = psCollisionBuckets[iBucket].nInd;
1696 if( panReqIds[nIdx] ==
1697 psWayFeaturePairs->panNodeRefs[i] )
1698 break;
1699 iBucket = psCollisionBuckets[iBucket].nNext;
1700 if( iBucket < 0 )
1701 {
1702 nIdx = -1;
1703 break;
1704 }
1705 }
1706 }
1707 else if( nIdx >= 0 &&
1708 panReqIds[nIdx] != psWayFeaturePairs->panNodeRefs[i] )
1709 nIdx = -1;
1710
1711 if( nIdx >= 0 )
1712 {
1713 m_asLonLatCache.push_back(pasLonLatArray[nIdx]);
1714 }
1715 }
1716 }
1717 else
1718 #endif // ENABLE_NODE_LOOKUP_BY_HASHING
1719 {
1720 int nIdx = -1;
1721 for( unsigned int i=0;i<psWayFeaturePairs->nRefs;i++)
1722 {
1723 if( nIdx >= 0 && psWayFeaturePairs->panNodeRefs[i] ==
1724 psWayFeaturePairs->panNodeRefs[i-1] + 1 )
1725 {
1726 if( nIdx+1 < (int)nReqIds && panReqIds[nIdx+1] ==
1727 psWayFeaturePairs->panNodeRefs[i] )
1728 nIdx ++;
1729 else
1730 nIdx = -1;
1731 }
1732 else
1733 nIdx = FindNode( psWayFeaturePairs->panNodeRefs[i] );
1734 if( nIdx >= 0 )
1735 {
1736 m_asLonLatCache.push_back(pasLonLatArray[nIdx]);
1737 }
1738 }
1739 }
1740
1741 if( !m_asLonLatCache.empty() && bIsArea )
1742 {
1743 m_asLonLatCache.push_back(m_asLonLatCache[0]);
1744 }
1745
1746 if( m_asLonLatCache.size() < 2 )
1747 {
1748 CPLDebug("OSM", "Way " CPL_FRMT_GIB " with %d nodes that could be found. Discarding it",
1749 psWayFeaturePairs->nWayID, static_cast<int>(m_asLonLatCache.size()));
1750 delete psWayFeaturePairs->poFeature;
1751 psWayFeaturePairs->poFeature = nullptr;
1752 psWayFeaturePairs->bIsArea = false;
1753 continue;
1754 }
1755
1756 if( bIsArea && papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() )
1757 {
1758 IndexWay(psWayFeaturePairs->nWayID,
1759 bIsArea != 0,
1760 psWayFeaturePairs->nTags,
1761 psWayFeaturePairs->pasTags,
1762 m_asLonLatCache.data(),
1763 static_cast<int>(m_asLonLatCache.size()),
1764 &psWayFeaturePairs->sInfo);
1765 }
1766 else
1767 IndexWay(psWayFeaturePairs->nWayID, bIsArea != 0, 0, nullptr,
1768 m_asLonLatCache.data(),
1769 static_cast<int>(m_asLonLatCache.size()),
1770 nullptr);
1771
1772 if( psWayFeaturePairs->poFeature == nullptr )
1773 {
1774 continue;
1775 }
1776
1777 OGRLineString* poLS = new OGRLineString();
1778 OGRGeometry* poGeom = poLS;
1779
1780 const int nPoints = static_cast<int>(m_asLonLatCache.size());
1781 poLS->setNumPoints(nPoints);
1782 for(int i=0;i<nPoints;i++)
1783 {
1784 poLS->setPoint(i,
1785 INT_TO_DBL(m_asLonLatCache[i].nLon),
1786 INT_TO_DBL(m_asLonLatCache[i].nLat));
1787 }
1788
1789 psWayFeaturePairs->poFeature->SetGeometryDirectly(poGeom);
1790
1791 if( m_asLonLatCache.size() != psWayFeaturePairs->nRefs )
1792 CPLDebug("OSM", "For way " CPL_FRMT_GIB ", got only %d nodes instead of %d",
1793 psWayFeaturePairs->nWayID,
1794 nPoints,
1795 psWayFeaturePairs->nRefs);
1796
1797 int bFilteredOut = FALSE;
1798 if( !papoLayers[IDX_LYR_LINES]->AddFeature(psWayFeaturePairs->poFeature,
1799 psWayFeaturePairs->bAttrFilterAlreadyEvaluated,
1800 &bFilteredOut,
1801 !bFeatureAdded) )
1802 bStopParsing = true;
1803 else if( !bFilteredOut )
1804 bFeatureAdded = true;
1805 }
1806
1807 if( papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() )
1808 {
1809 for( int iPair = 0; iPair < nWayFeaturePairs; iPair ++)
1810 {
1811 WayFeaturePair* psWayFeaturePairs = &pasWayFeaturePairs[iPair];
1812
1813 if( psWayFeaturePairs->bIsArea &&
1814 (psWayFeaturePairs->nTags || bReportAllWays) )
1815 {
1816 sqlite3_bind_int64( hInsertPolygonsStandaloneStmt , 1, psWayFeaturePairs->nWayID );
1817
1818 int rc = sqlite3_step( hInsertPolygonsStandaloneStmt );
1819 sqlite3_reset( hInsertPolygonsStandaloneStmt );
1820 if( !(rc == SQLITE_OK || rc == SQLITE_DONE) )
1821 {
1822 CPLError(CE_Failure, CPLE_AppDefined,
1823 "Failed inserting into polygons_standalone " CPL_FRMT_GIB ": %s",
1824 psWayFeaturePairs->nWayID, sqlite3_errmsg(hDB));
1825 }
1826 }
1827 }
1828 }
1829
1830 nWayFeaturePairs = 0;
1831 nUnsortedReqIds = 0;
1832
1833 nAccumulatedTags = 0;
1834 nNonRedundantValuesLen = 0;
1835 }
1836
1837 /************************************************************************/
1838 /* IsClosedWayTaggedAsPolygon() */
1839 /************************************************************************/
1840
IsClosedWayTaggedAsPolygon(unsigned int nTags,const OSMTag * pasTags)1841 bool OGROSMDataSource::IsClosedWayTaggedAsPolygon( unsigned int nTags, const OSMTag* pasTags )
1842 {
1843 bool bIsArea = false;
1844 const int nSizeArea = 4;
1845 const int nStrnlenK = std::max(nSizeArea,
1846 nMaxSizeKeysInSetClosedWaysArePolygons)+1;
1847 std::string oTmpStr;
1848 oTmpStr.reserve(nMaxSizeKeysInSetClosedWaysArePolygons);
1849 for( unsigned int i=0;i<nTags;i++)
1850 {
1851 const char* pszK = pasTags[i].pszK;
1852 const int nKLen = static_cast<int>(CPLStrnlen(pszK, nStrnlenK));
1853 if( nKLen > nMaxSizeKeysInSetClosedWaysArePolygons )
1854 continue;
1855
1856 if( nKLen == nSizeArea && strcmp(pszK, "area") == 0 )
1857 {
1858 const char* pszV = pasTags[i].pszV;
1859 if( strcmp(pszV, "yes") == 0 )
1860 {
1861 bIsArea = true;
1862 // final true. We can't have several area tags...
1863 break;
1864 }
1865 else if( strcmp(pszV, "no") == 0 )
1866 {
1867 bIsArea = false;
1868 break;
1869 }
1870 }
1871 if( bIsArea )
1872 continue;
1873
1874 if( nKLen >= nMinSizeKeysInSetClosedWaysArePolygons )
1875 {
1876 oTmpStr.assign(pszK, nKLen);
1877 if( aoSetClosedWaysArePolygons.find(oTmpStr) !=
1878 aoSetClosedWaysArePolygons.end() )
1879 {
1880 bIsArea = true;
1881 continue;
1882 }
1883 }
1884
1885 const char* pszV = pasTags[i].pszV;
1886 const int nVLen = static_cast<int>(CPLStrnlen(pszV, nStrnlenK));
1887 if( nKLen + 1 + nVLen >= nMinSizeKeysInSetClosedWaysArePolygons &&
1888 nKLen + 1 + nVLen <= nMaxSizeKeysInSetClosedWaysArePolygons )
1889 {
1890 oTmpStr.assign(pszK, nKLen);
1891 oTmpStr.append(1, '=');
1892 oTmpStr.append(pszV, nVLen);
1893 if( aoSetClosedWaysArePolygons.find(oTmpStr) !=
1894 aoSetClosedWaysArePolygons.end() )
1895 {
1896 bIsArea = true;
1897 continue;
1898 }
1899 }
1900 }
1901 return bIsArea;
1902 }
1903
1904 /************************************************************************/
1905 /* NotifyWay() */
1906 /************************************************************************/
1907
NotifyWay(OSMWay * psWay)1908 void OGROSMDataSource::NotifyWay( OSMWay* psWay )
1909 {
1910 nWaysProcessed++;
1911 if( nWaysProcessed % 10000 == 0 )
1912 {
1913 CPLDebug("OSM", "Ways processed : %d", nWaysProcessed);
1914 #ifdef DEBUG_MEM_USAGE
1915 CPLDebug("OSM", "GetMaxTotalAllocs() = " CPL_FRMT_GUIB,
1916 static_cast<GUIntBig>(GetMaxTotalAllocs()));
1917 #endif
1918 }
1919
1920 if( !bUsePointsIndex )
1921 return;
1922
1923 //printf("way %d : %d nodes\n", (int)psWay->nID, (int)psWay->nRefs);
1924
1925 if( psWay->nRefs < 2 )
1926 {
1927 CPLDebug("OSM", "Way " CPL_FRMT_GIB " with %d nodes. Discarding it",
1928 psWay->nID, psWay->nRefs);
1929 return;
1930 }
1931
1932 /* Is a closed way a polygon ? */
1933 bool bIsArea = false;
1934 if( psWay->panNodeRefs[0] == psWay->panNodeRefs[psWay->nRefs - 1] )
1935 {
1936 bIsArea = IsClosedWayTaggedAsPolygon(psWay->nTags, psWay->pasTags);
1937 }
1938
1939 bool bInterestingTag = bReportAllWays;
1940 if( !bIsArea && !bReportAllWays )
1941 {
1942 for( unsigned int i=0;i<psWay->nTags;i++)
1943 {
1944 const char* pszK = psWay->pasTags[i].pszK;
1945 if( papoLayers[IDX_LYR_LINES]->IsSignificantKey(pszK) )
1946 {
1947 bInterestingTag = true;
1948 break;
1949 }
1950 }
1951 }
1952
1953 OGRFeature* poFeature = nullptr;
1954 bool bAttrFilterAlreadyEvaluated = false;
1955 if( !bIsArea && papoLayers[IDX_LYR_LINES]->IsUserInterested() &&
1956 bInterestingTag )
1957 {
1958 poFeature = new OGRFeature(papoLayers[IDX_LYR_LINES]->GetLayerDefn());
1959
1960 papoLayers[IDX_LYR_LINES]->SetFieldsFromTags(
1961 poFeature, psWay->nID, false, psWay->nTags, psWay->pasTags,
1962 &psWay->sInfo );
1963
1964 // Optimization: if we have an attribute filter, that does not require
1965 // geometry, and if we don't need to index ways, then we can just
1966 // evaluate the attribute filter without the geometry.
1967 if( papoLayers[IDX_LYR_LINES]->HasAttributeFilter() &&
1968 !papoLayers[IDX_LYR_LINES]->
1969 AttributeFilterEvaluationNeedsGeometry() &&
1970 !bIndexWays )
1971 {
1972 if( !papoLayers[IDX_LYR_LINES]->EvaluateAttributeFilter(poFeature) )
1973 {
1974 delete poFeature;
1975 return;
1976 }
1977 bAttrFilterAlreadyEvaluated = true;
1978 }
1979 }
1980 else if( !bIndexWays )
1981 {
1982 return;
1983 }
1984
1985 if( nUnsortedReqIds + psWay->nRefs >
1986 static_cast<unsigned int>(MAX_ACCUMULATED_NODES) ||
1987 nWayFeaturePairs == MAX_DELAYED_FEATURES ||
1988 nAccumulatedTags + psWay->nTags >
1989 static_cast<unsigned int>(MAX_ACCUMULATED_TAGS) ||
1990 nNonRedundantValuesLen + 1024 > MAX_NON_REDUNDANT_VALUES )
1991 {
1992 ProcessWaysBatch();
1993 }
1994
1995 WayFeaturePair* psWayFeaturePairs = &pasWayFeaturePairs[nWayFeaturePairs];
1996
1997 psWayFeaturePairs->nWayID = psWay->nID;
1998 psWayFeaturePairs->nRefs = psWay->nRefs - (bIsArea ? 1 : 0);
1999 psWayFeaturePairs->panNodeRefs = panUnsortedReqIds + nUnsortedReqIds;
2000 psWayFeaturePairs->poFeature = poFeature;
2001 psWayFeaturePairs->bIsArea = bIsArea;
2002 psWayFeaturePairs->bAttrFilterAlreadyEvaluated =
2003 bAttrFilterAlreadyEvaluated;
2004
2005 if( bIsArea && papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() )
2006 {
2007 unsigned int nTagCount = 0;
2008
2009 if( bNeedsToSaveWayInfo )
2010 {
2011 if( !psWay->sInfo.bTimeStampIsStr )
2012 psWayFeaturePairs->sInfo.ts.nTimeStamp =
2013 psWay->sInfo.ts.nTimeStamp;
2014 else
2015 {
2016 OGRField sField;
2017 if( OGRParseXMLDateTime(psWay->sInfo.ts.pszTimeStamp, &sField) )
2018 {
2019 struct tm brokendown;
2020 memset(&brokendown, 0, sizeof(brokendown));
2021 brokendown.tm_year = sField.Date.Year - 1900;
2022 brokendown.tm_mon = sField.Date.Month - 1;
2023 brokendown.tm_mday = sField.Date.Day;
2024 brokendown.tm_hour = sField.Date.Hour;
2025 brokendown.tm_min = sField.Date.Minute;
2026 brokendown.tm_sec = (int)(sField.Date.Second + .5);
2027 psWayFeaturePairs->sInfo.ts.nTimeStamp =
2028 CPLYMDHMSToUnixTime(&brokendown);
2029 }
2030 else
2031 psWayFeaturePairs->sInfo.ts.nTimeStamp = 0;
2032 }
2033 psWayFeaturePairs->sInfo.nChangeset = psWay->sInfo.nChangeset;
2034 psWayFeaturePairs->sInfo.nVersion = psWay->sInfo.nVersion;
2035 psWayFeaturePairs->sInfo.nUID = psWay->sInfo.nUID;
2036 psWayFeaturePairs->sInfo.bTimeStampIsStr = false;
2037 psWayFeaturePairs->sInfo.pszUserSID = ""; // FIXME
2038 }
2039 else
2040 {
2041 psWayFeaturePairs->sInfo.ts.nTimeStamp = 0;
2042 psWayFeaturePairs->sInfo.nChangeset = 0;
2043 psWayFeaturePairs->sInfo.nVersion = 0;
2044 psWayFeaturePairs->sInfo.nUID = 0;
2045 psWayFeaturePairs->sInfo.bTimeStampIsStr = false;
2046 psWayFeaturePairs->sInfo.pszUserSID = "";
2047 }
2048
2049 psWayFeaturePairs->pasTags = pasAccumulatedTags + nAccumulatedTags;
2050
2051 for(unsigned int iTag = 0; iTag < psWay->nTags; iTag++)
2052 {
2053 const char* pszK = psWay->pasTags[iTag].pszK;
2054 const char* pszV = psWay->pasTags[iTag].pszV;
2055
2056 if( std::any_of(begin(m_ignoredKeys), end(m_ignoredKeys),
2057 [pszK](const char* pszIgnoredKey) { return strcmp(pszK, pszIgnoredKey) == 0; }) )
2058 {
2059 continue;
2060 }
2061
2062 std::map<const char*, KeyDesc*, ConstCharComp>::iterator oIterK =
2063 aoMapIndexedKeys.find(pszK);
2064 KeyDesc* psKD = nullptr;
2065 if( oIterK == aoMapIndexedKeys.end() )
2066 {
2067 if( nNextKeyIndex >= 32768 ) /* somewhat arbitrary */
2068 {
2069 if( nNextKeyIndex == 32768 )
2070 {
2071 CPLError(CE_Failure, CPLE_AppDefined,
2072 "Too many different keys in file");
2073 nNextKeyIndex ++; /* to avoid next warnings */
2074 }
2075 continue;
2076 }
2077 psKD = new KeyDesc();
2078 psKD->pszK = CPLStrdup(pszK);
2079 psKD->nKeyIndex = nNextKeyIndex ++;
2080 //CPLDebug("OSM", "nNextKeyIndex=%d", nNextKeyIndex);
2081 psKD->nOccurrences = 0;
2082 psKD->asValues.push_back(CPLStrdup(""));
2083 aoMapIndexedKeys[psKD->pszK] = psKD;
2084 asKeys.push_back(psKD);
2085 }
2086 else
2087 psKD = oIterK->second;
2088 psKD->nOccurrences ++;
2089
2090 pasAccumulatedTags[nAccumulatedTags].nKeyIndex = (short)psKD->nKeyIndex;
2091
2092 /* to fit in 2 bytes, the theoretical limit would be 127 * 128 + 127 */
2093 if( psKD->asValues.size() < 1024 )
2094 {
2095 std::map<const char*, int, ConstCharComp>::iterator oIterV;
2096 oIterV = psKD->anMapV.find(pszV);
2097 int nValueIndex = 0;
2098 if( oIterV == psKD->anMapV.end() )
2099 {
2100 char* pszVDup = CPLStrdup(pszV);
2101 nValueIndex = (int)psKD->asValues.size();
2102 psKD->anMapV[pszVDup] = nValueIndex;
2103 psKD->asValues.push_back(pszVDup);
2104 }
2105 else
2106 nValueIndex = oIterV->second;
2107
2108 pasAccumulatedTags[nAccumulatedTags].bVIsIndex = TRUE;
2109 pasAccumulatedTags[nAccumulatedTags].u.nValueIndex = nValueIndex;
2110 }
2111 else
2112 {
2113 const int nLenV = static_cast<int>(strlen(pszV)) + 1;
2114
2115 if( psKD->asValues.size() == 1024 )
2116 {
2117 CPLDebug( "OSM", "More than %d different values for tag %s",
2118 1024, pszK);
2119 // To avoid next warnings.
2120 psKD->asValues.push_back(CPLStrdup(""));
2121 }
2122
2123 CPLAssert( nNonRedundantValuesLen + nLenV <=
2124 MAX_NON_REDUNDANT_VALUES );
2125 memcpy( pabyNonRedundantValues + nNonRedundantValuesLen, pszV,
2126 nLenV);
2127 pasAccumulatedTags[nAccumulatedTags].bVIsIndex = FALSE;
2128 pasAccumulatedTags[nAccumulatedTags].u.nOffsetInpabyNonRedundantValues = nNonRedundantValuesLen;
2129 nNonRedundantValuesLen += nLenV;
2130 }
2131 nAccumulatedTags ++;
2132
2133 nTagCount ++;
2134 if( nTagCount == MAX_COUNT_FOR_TAGS_IN_WAY )
2135 break;
2136 }
2137
2138 psWayFeaturePairs->nTags = nTagCount;
2139 }
2140 else
2141 {
2142 psWayFeaturePairs->sInfo.ts.nTimeStamp = 0;
2143 psWayFeaturePairs->sInfo.nChangeset = 0;
2144 psWayFeaturePairs->sInfo.nVersion = 0;
2145 psWayFeaturePairs->sInfo.nUID = 0;
2146 psWayFeaturePairs->sInfo.bTimeStampIsStr = false;
2147 psWayFeaturePairs->sInfo.pszUserSID = "";
2148
2149 psWayFeaturePairs->nTags = 0;
2150 psWayFeaturePairs->pasTags = nullptr;
2151 }
2152
2153 nWayFeaturePairs++;
2154
2155 memcpy( panUnsortedReqIds + nUnsortedReqIds,
2156 psWay->panNodeRefs, sizeof(GIntBig) * (psWay->nRefs - (bIsArea ? 1 : 0)));
2157 nUnsortedReqIds += (psWay->nRefs - (bIsArea ? 1 : 0));
2158 }
2159
OGROSMNotifyWay(OSMWay * psWay,OSMContext *,void * user_data)2160 static void OGROSMNotifyWay ( OSMWay *psWay,
2161 OSMContext * /* psOSMContext */,
2162 void *user_data)
2163 {
2164 static_cast<OGROSMDataSource *>(user_data)->NotifyWay(psWay);
2165 }
2166
2167 /************************************************************************/
2168 /* LookupWays() */
2169 /************************************************************************/
2170
LookupWays(std::map<GIntBig,std::pair<int,void * >> & aoMapWays,OSMRelation * psRelation)2171 unsigned int OGROSMDataSource::LookupWays( std::map< GIntBig,
2172 std::pair<int,void*> >& aoMapWays,
2173 OSMRelation* psRelation )
2174 {
2175 unsigned int nFound = 0;
2176 unsigned int iCur = 0;
2177
2178 while( iCur < psRelation->nMembers )
2179 {
2180 unsigned int nToQuery = 0;
2181 unsigned int i = iCur; // Used after for.
2182 for( ; i < psRelation->nMembers; i++ )
2183 {
2184 if( psRelation->pasMembers[i].eType == MEMBER_WAY &&
2185 strcmp(psRelation->pasMembers[i].pszRole, "subarea") != 0 )
2186 {
2187 nToQuery ++;
2188 if( nToQuery ==
2189 static_cast<unsigned int>(LIMIT_IDS_PER_REQUEST) )
2190 {
2191 break;
2192 }
2193 }
2194 }
2195
2196 if( nToQuery == 0)
2197 break;
2198
2199 unsigned int iLastI = (i == psRelation->nMembers) ? i : i + 1;
2200
2201 sqlite3_stmt* hStmt = pahSelectWayStmt[nToQuery-1];
2202 unsigned int nBindIndex = 1;
2203 for(i=iCur;i<iLastI;i++)
2204 {
2205 if( psRelation->pasMembers[i].eType == MEMBER_WAY &&
2206 strcmp(psRelation->pasMembers[i].pszRole, "subarea") != 0 )
2207 {
2208 sqlite3_bind_int64( hStmt, nBindIndex,
2209 psRelation->pasMembers[i].nID );
2210 nBindIndex ++;
2211 }
2212 }
2213 iCur = iLastI;
2214
2215 while( sqlite3_step(hStmt) == SQLITE_ROW )
2216 {
2217 GIntBig id = sqlite3_column_int64(hStmt, 0);
2218 if( aoMapWays.find(id) == aoMapWays.end() )
2219 {
2220 int nBlobSize = sqlite3_column_bytes(hStmt, 1);
2221 const void* blob = sqlite3_column_blob(hStmt, 1);
2222 void* blob_dup = CPLMalloc(nBlobSize);
2223 memcpy(blob_dup, blob, nBlobSize);
2224 aoMapWays[id] = std::pair<int,void*>(nBlobSize, blob_dup);
2225 }
2226 nFound++;
2227 }
2228
2229 sqlite3_reset(hStmt);
2230 }
2231
2232 return nFound;
2233 }
2234
2235 /************************************************************************/
2236 /* BuildMultiPolygon() */
2237 /************************************************************************/
2238
BuildMultiPolygon(OSMRelation * psRelation,unsigned int * pnTags,OSMTag * pasTags)2239 OGRGeometry* OGROSMDataSource::BuildMultiPolygon(OSMRelation* psRelation,
2240 unsigned int* pnTags,
2241 OSMTag* pasTags)
2242 {
2243 std::map< GIntBig, std::pair<int,void*> > aoMapWays;
2244 LookupWays( aoMapWays, psRelation );
2245
2246 bool bMissing = false;
2247
2248 for( unsigned int i = 0; i < psRelation->nMembers; i++ )
2249 {
2250 if( psRelation->pasMembers[i].eType == MEMBER_WAY &&
2251 strcmp(psRelation->pasMembers[i].pszRole, "subarea") != 0 )
2252 {
2253 if( aoMapWays.find( psRelation->pasMembers[i].nID ) == aoMapWays.end() )
2254 {
2255 CPLDebug("OSM", "Relation " CPL_FRMT_GIB " has missing ways. Ignoring it",
2256 psRelation->nID);
2257 bMissing = true;
2258 break;
2259 }
2260 }
2261 }
2262
2263 if( bMissing )
2264 {
2265 std::map< GIntBig, std::pair<int,void*> >::iterator oIter;
2266 for( oIter = aoMapWays.begin(); oIter != aoMapWays.end(); ++oIter )
2267 CPLFree(oIter->second.second);
2268
2269 return nullptr;
2270 }
2271
2272 OGRMultiLineString* poMLS = new OGRMultiLineString();
2273 OGRGeometry** papoPolygons = static_cast<OGRGeometry**>( CPLMalloc(
2274 sizeof(OGRGeometry*) * psRelation->nMembers) );
2275 int nPolys = 0;
2276
2277 if( pnTags != nullptr )
2278 *pnTags = 0;
2279
2280 for( unsigned int i = 0; i < psRelation->nMembers; i++ )
2281 {
2282 if( psRelation->pasMembers[i].eType == MEMBER_WAY &&
2283 strcmp(psRelation->pasMembers[i].pszRole, "subarea") != 0 )
2284 {
2285 const std::pair<int, void*>& oGeom = aoMapWays[ psRelation->pasMembers[i].nID ];
2286
2287 if( pnTags != nullptr && *pnTags == 0 &&
2288 strcmp(psRelation->pasMembers[i].pszRole, "outer") == 0 )
2289 {
2290 // This backup in m_abyWayBuffer is crucial for safe memory
2291 // usage, as pasTags[].pszV will point to it !
2292 m_abyWayBuffer.clear();
2293 m_abyWayBuffer.insert(m_abyWayBuffer.end(),
2294 static_cast<const GByte*>(oGeom.second),
2295 static_cast<const GByte*>(oGeom.second) + oGeom.first);
2296
2297 UncompressWay( oGeom.first,
2298 m_abyWayBuffer.data(),
2299 nullptr, m_asLonLatCache,
2300 pnTags, pasTags, nullptr );
2301 }
2302 else
2303 {
2304 UncompressWay( oGeom.first,
2305 static_cast<const GByte*>(oGeom.second),
2306 nullptr, m_asLonLatCache,
2307 nullptr, nullptr, nullptr );
2308 }
2309
2310 OGRLineString* poLS = nullptr;
2311
2312 if( !m_asLonLatCache.empty() &&
2313 m_asLonLatCache.front().nLon == m_asLonLatCache.back().nLon &&
2314 m_asLonLatCache.front().nLat == m_asLonLatCache.back().nLat )
2315 {
2316 OGRPolygon* poPoly = new OGRPolygon();
2317 OGRLinearRing* poRing = new OGRLinearRing();
2318 poPoly->addRingDirectly(poRing);
2319 papoPolygons[nPolys ++] = poPoly;
2320 poLS = poRing;
2321
2322 if( strcmp(psRelation->pasMembers[i].pszRole, "outer") == 0 )
2323 {
2324 sqlite3_bind_int64( hDeletePolygonsStandaloneStmt, 1, psRelation->pasMembers[i].nID );
2325 CPL_IGNORE_RET_VAL(sqlite3_step( hDeletePolygonsStandaloneStmt ));
2326 sqlite3_reset( hDeletePolygonsStandaloneStmt );
2327 }
2328 }
2329 else
2330 {
2331 poLS = new OGRLineString();
2332 poMLS->addGeometryDirectly(poLS);
2333 }
2334
2335 const int nPoints = static_cast<int>(m_asLonLatCache.size());
2336 poLS->setNumPoints(nPoints);
2337 for(int j=0;j<nPoints;j++)
2338 {
2339 poLS->setPoint( j,
2340 INT_TO_DBL(m_asLonLatCache[j].nLon),
2341 INT_TO_DBL(m_asLonLatCache[j].nLat) );
2342 }
2343 }
2344 }
2345
2346 if( poMLS->getNumGeometries() > 0 )
2347 {
2348 OGRGeometryH hPoly = OGRBuildPolygonFromEdges( (OGRGeometryH) poMLS,
2349 TRUE,
2350 FALSE,
2351 0,
2352 nullptr );
2353 if( hPoly != nullptr && OGR_G_GetGeometryType(hPoly) == wkbPolygon )
2354 {
2355 OGRPolygon* poSuperPoly = reinterpret_cast<OGRGeometry*>(hPoly)->toPolygon();
2356 for( unsigned int i = 0;
2357 i < 1 + (unsigned int)poSuperPoly->getNumInteriorRings();
2358 i++ )
2359 {
2360 OGRLinearRing* poRing = (i == 0) ? poSuperPoly->getExteriorRing() :
2361 poSuperPoly->getInteriorRing(i - 1);
2362 if( poRing != nullptr && poRing->getNumPoints() >= 4 &&
2363 poRing->getX(0) == poRing->getX(poRing->getNumPoints() -1) &&
2364 poRing->getY(0) == poRing->getY(poRing->getNumPoints() -1) )
2365 {
2366 OGRPolygon* poPoly = new OGRPolygon();
2367 poPoly->addRing( poRing );
2368 papoPolygons[nPolys ++] = poPoly;
2369 }
2370 }
2371 }
2372
2373 OGR_G_DestroyGeometry(hPoly);
2374 }
2375 delete poMLS;
2376
2377 OGRGeometry* poRet = nullptr;
2378
2379 if( nPolys > 0 )
2380 {
2381 int bIsValidGeometry = FALSE;
2382 const char* apszOptions[2] = { "METHOD=DEFAULT", nullptr };
2383 OGRGeometry* poGeom = OGRGeometryFactory::organizePolygons(
2384 papoPolygons, nPolys, &bIsValidGeometry, apszOptions );
2385
2386 if( poGeom != nullptr && poGeom->getGeometryType() == wkbPolygon )
2387 {
2388 OGRMultiPolygon* poMulti = new OGRMultiPolygon();
2389 poMulti->addGeometryDirectly(poGeom);
2390 poGeom = poMulti;
2391 }
2392
2393 if( poGeom != nullptr && poGeom->getGeometryType() == wkbMultiPolygon )
2394 {
2395 poRet = poGeom;
2396 }
2397 else
2398 {
2399 CPLDebug( "OSM",
2400 "Relation " CPL_FRMT_GIB
2401 ": Geometry has incompatible type : %s",
2402 psRelation->nID,
2403 poGeom != nullptr ?
2404 OGR_G_GetGeometryName(
2405 reinterpret_cast<OGRGeometryH>(poGeom)) : "null" );
2406 delete poGeom;
2407 }
2408 }
2409
2410 CPLFree(papoPolygons);
2411
2412 std::map< GIntBig, std::pair<int,void*> >::iterator oIter;
2413 for( oIter = aoMapWays.begin(); oIter != aoMapWays.end(); ++oIter )
2414 CPLFree(oIter->second.second);
2415
2416 return poRet;
2417 }
2418
2419 /************************************************************************/
2420 /* BuildGeometryCollection() */
2421 /************************************************************************/
2422
BuildGeometryCollection(OSMRelation * psRelation,int bMultiLineString)2423 OGRGeometry* OGROSMDataSource::BuildGeometryCollection(OSMRelation* psRelation,
2424 int bMultiLineString)
2425 {
2426 std::map< GIntBig, std::pair<int,void*> > aoMapWays;
2427 LookupWays( aoMapWays, psRelation );
2428
2429 OGRGeometryCollection* poColl = ( bMultiLineString ) ?
2430 new OGRMultiLineString() : new OGRGeometryCollection();
2431
2432 for( unsigned int i = 0; i < psRelation->nMembers; i ++ )
2433 {
2434 if( psRelation->pasMembers[i].eType ==
2435 MEMBER_NODE && !bMultiLineString )
2436 {
2437 nUnsortedReqIds = 1;
2438 panUnsortedReqIds[0] = psRelation->pasMembers[i].nID;
2439 LookupNodes();
2440 if( nReqIds == 1 )
2441 {
2442 poColl->addGeometryDirectly(new OGRPoint(
2443 INT_TO_DBL(pasLonLatArray[0].nLon),
2444 INT_TO_DBL(pasLonLatArray[0].nLat)));
2445 }
2446 }
2447 else if( psRelation->pasMembers[i].eType == MEMBER_WAY &&
2448 strcmp(psRelation->pasMembers[i].pszRole, "subarea") != 0 &&
2449 aoMapWays.find( psRelation->pasMembers[i].nID ) != aoMapWays.end() )
2450 {
2451 const std::pair<int, void*>& oGeom = aoMapWays[ psRelation->pasMembers[i].nID ];
2452
2453 bool bIsArea = false;
2454 UncompressWay(
2455 oGeom.first,
2456 reinterpret_cast<GByte *>(oGeom.second),
2457 &bIsArea, m_asLonLatCache, nullptr, nullptr, nullptr );
2458 OGRLineString* poLS = nullptr;
2459 if( bIsArea && !bMultiLineString )
2460 {
2461 OGRLinearRing* poLR = new OGRLinearRing();
2462 OGRPolygon* poPoly = new OGRPolygon();
2463 poPoly->addRingDirectly(poLR);
2464 poColl->addGeometryDirectly(poPoly);
2465 poLS = poLR;
2466 }
2467 else
2468 {
2469 poLS = new OGRLineString();
2470 poColl->addGeometryDirectly(poLS);
2471 }
2472
2473 const int nPoints = static_cast<int>(m_asLonLatCache.size());
2474 poLS->setNumPoints(nPoints);
2475 for(int j=0;j<nPoints;j++)
2476 {
2477 poLS->setPoint( j,
2478 INT_TO_DBL(m_asLonLatCache[j].nLon),
2479 INT_TO_DBL(m_asLonLatCache[j].nLat) );
2480 }
2481 }
2482 }
2483
2484 if( poColl->getNumGeometries() == 0 )
2485 {
2486 delete poColl;
2487 poColl = nullptr;
2488 }
2489
2490 std::map< GIntBig, std::pair<int,void*> >::iterator oIter;
2491 for( oIter = aoMapWays.begin(); oIter != aoMapWays.end(); ++oIter )
2492 CPLFree(oIter->second.second);
2493
2494 return poColl;
2495 }
2496
2497 /************************************************************************/
2498 /* NotifyRelation() */
2499 /************************************************************************/
2500
NotifyRelation(OSMRelation * psRelation)2501 void OGROSMDataSource::NotifyRelation (OSMRelation* psRelation)
2502 {
2503 if( nWayFeaturePairs != 0 )
2504 ProcessWaysBatch();
2505
2506 nRelationsProcessed++;
2507 if( (nRelationsProcessed % 10000) == 0 )
2508 {
2509 CPLDebug( "OSM", "Relations processed : %d", nRelationsProcessed );
2510 #ifdef DEBUG_MEM_USAGE
2511 CPLDebug( "OSM",
2512 "GetMaxTotalAllocs() = " CPL_FRMT_GUIB,
2513 static_cast<GUIntBig>(GetMaxTotalAllocs()) );
2514 #endif
2515 }
2516
2517 if( !bUseWaysIndex )
2518 return;
2519
2520 bool bMultiPolygon = false;
2521 bool bMultiLineString = false;
2522 bool bInterestingTagFound = false;
2523 const char* pszTypeV = nullptr;
2524 for( unsigned int i = 0; i < psRelation->nTags; i ++ )
2525 {
2526 const char* pszK = psRelation->pasTags[i].pszK;
2527 if( strcmp(pszK, "type") == 0 )
2528 {
2529 const char* pszV = psRelation->pasTags[i].pszV;
2530 pszTypeV = pszV;
2531 if( strcmp(pszV, "multipolygon") == 0 ||
2532 strcmp(pszV, "boundary") == 0)
2533 {
2534 bMultiPolygon = true;
2535 }
2536 else if( strcmp(pszV, "multilinestring") == 0 ||
2537 strcmp(pszV, "route") == 0 )
2538 {
2539 bMultiLineString = true;
2540 }
2541 }
2542 else if( strcmp(pszK, "created_by") != 0 )
2543 bInterestingTagFound = true;
2544 }
2545
2546 // Optimization: If we have an attribute filter, that does not require
2547 // geometry, then we can just evaluate the attribute filter without the
2548 // geometry.
2549 const int iCurLayer =
2550 bMultiPolygon ? IDX_LYR_MULTIPOLYGONS :
2551 bMultiLineString ? IDX_LYR_MULTILINESTRINGS :
2552 IDX_LYR_OTHER_RELATIONS;
2553 if( !papoLayers[iCurLayer]->IsUserInterested() )
2554 return;
2555
2556 OGRFeature* poFeature = nullptr;
2557
2558 if( !(bMultiPolygon && !bInterestingTagFound) &&
2559 // We cannot do early filtering for multipolygon that has no
2560 // interesting tag, since we may fetch attributes from ways.
2561 papoLayers[iCurLayer]->HasAttributeFilter() &&
2562 !papoLayers[iCurLayer]->AttributeFilterEvaluationNeedsGeometry() )
2563 {
2564 poFeature = new OGRFeature(papoLayers[iCurLayer]->GetLayerDefn());
2565
2566 papoLayers[iCurLayer]->SetFieldsFromTags( poFeature,
2567 psRelation->nID,
2568 false,
2569 psRelation->nTags,
2570 psRelation->pasTags,
2571 &psRelation->sInfo);
2572
2573 if( !papoLayers[iCurLayer]->EvaluateAttributeFilter(poFeature) )
2574 {
2575 delete poFeature;
2576 return;
2577 }
2578 }
2579
2580 OGRGeometry* poGeom = nullptr;
2581
2582 unsigned int nExtraTags = 0;
2583 OSMTag pasExtraTags[1 + MAX_COUNT_FOR_TAGS_IN_WAY];
2584
2585 if( bMultiPolygon )
2586 {
2587 if( !bInterestingTagFound )
2588 {
2589 poGeom = BuildMultiPolygon(psRelation, &nExtraTags, pasExtraTags);
2590 CPLAssert(nExtraTags <= MAX_COUNT_FOR_TAGS_IN_WAY);
2591 pasExtraTags[nExtraTags].pszK = "type";
2592 pasExtraTags[nExtraTags].pszV = pszTypeV;
2593 nExtraTags ++;
2594 }
2595 else
2596 poGeom = BuildMultiPolygon(psRelation, nullptr, nullptr);
2597 }
2598 else
2599 poGeom = BuildGeometryCollection(psRelation, bMultiLineString);
2600
2601 if( poGeom != nullptr )
2602 {
2603 bool bAttrFilterAlreadyEvaluated = true;
2604 if( poFeature == nullptr )
2605 {
2606 poFeature = new OGRFeature(papoLayers[iCurLayer]->GetLayerDefn());
2607
2608 papoLayers[iCurLayer]->SetFieldsFromTags(
2609 poFeature,
2610 psRelation->nID,
2611 false,
2612 nExtraTags ? nExtraTags : psRelation->nTags,
2613 nExtraTags ? pasExtraTags : psRelation->pasTags,
2614 &psRelation->sInfo);
2615
2616 bAttrFilterAlreadyEvaluated = false;
2617 }
2618
2619 poFeature->SetGeometryDirectly(poGeom);
2620
2621 int bFilteredOut = FALSE;
2622 if( !papoLayers[iCurLayer]->AddFeature( poFeature,
2623 bAttrFilterAlreadyEvaluated,
2624 &bFilteredOut,
2625 !bFeatureAdded ) )
2626 bStopParsing = true;
2627 else if( !bFilteredOut )
2628 bFeatureAdded = true;
2629 }
2630 else
2631 {
2632 delete poFeature;
2633 }
2634 }
2635
OGROSMNotifyRelation(OSMRelation * psRelation,OSMContext *,void * user_data)2636 static void OGROSMNotifyRelation ( OSMRelation *psRelation,
2637 OSMContext * /* psOSMContext */,
2638 void *user_data)
2639 {
2640 static_cast<OGROSMDataSource *>(user_data)->NotifyRelation(psRelation);
2641 }
2642
2643 /************************************************************************/
2644 /* ProcessPolygonsStandalone() */
2645 /************************************************************************/
2646
ProcessPolygonsStandalone()2647 void OGROSMDataSource::ProcessPolygonsStandalone()
2648 {
2649 unsigned int nTags = 0;
2650 OSMTag pasTags[MAX_COUNT_FOR_TAGS_IN_WAY];
2651 OSMInfo sInfo;
2652
2653 sInfo.ts.nTimeStamp = 0;
2654 sInfo.nChangeset = 0;
2655 sInfo.nVersion = 0;
2656 sInfo.nUID = 0;
2657 sInfo.bTimeStampIsStr = false;
2658 sInfo.pszUserSID = "";
2659
2660 if( !bHasRowInPolygonsStandalone )
2661 bHasRowInPolygonsStandalone =
2662 sqlite3_step(hSelectPolygonsStandaloneStmt) == SQLITE_ROW;
2663
2664 bool bFirst = true;
2665
2666 while( bHasRowInPolygonsStandalone &&
2667 papoLayers[IDX_LYR_MULTIPOLYGONS]->nFeatureArraySize < 10000 )
2668 {
2669 if( bFirst )
2670 {
2671 CPLDebug( "OSM", "Remaining standalone polygons" );
2672 bFirst = false;
2673 }
2674
2675 GIntBig id = sqlite3_column_int64(hSelectPolygonsStandaloneStmt, 0);
2676
2677 sqlite3_bind_int64( pahSelectWayStmt[0], 1, id );
2678 if( sqlite3_step(pahSelectWayStmt[0]) == SQLITE_ROW )
2679 {
2680 int nBlobSize = sqlite3_column_bytes(pahSelectWayStmt[0], 1);
2681 const void* blob = sqlite3_column_blob(pahSelectWayStmt[0], 1);
2682
2683 // coverity[tainted_data]
2684 UncompressWay(
2685 nBlobSize, static_cast<const GByte*>(blob),
2686 nullptr, m_asLonLatCache, &nTags, pasTags, &sInfo );
2687 CPLAssert(nTags <= MAX_COUNT_FOR_TAGS_IN_WAY);
2688
2689 OGRMultiPolygon* poMulti = new OGRMultiPolygon();
2690 OGRPolygon* poPoly = new OGRPolygon();
2691 OGRLinearRing* poRing = new OGRLinearRing();
2692 poMulti->addGeometryDirectly(poPoly);
2693 poPoly->addRingDirectly(poRing);
2694 OGRLineString* poLS = poRing;
2695
2696 poLS->setNumPoints(static_cast<int>(m_asLonLatCache.size()));
2697 for(int j=0;j<static_cast<int>(m_asLonLatCache.size());j++)
2698 {
2699 poLS->setPoint( j,
2700 INT_TO_DBL(m_asLonLatCache[j].nLon),
2701 INT_TO_DBL(m_asLonLatCache[j].nLat) );
2702 }
2703
2704 OGRFeature* poFeature =
2705 new OGRFeature(
2706 papoLayers[IDX_LYR_MULTIPOLYGONS]->GetLayerDefn());
2707
2708 papoLayers[IDX_LYR_MULTIPOLYGONS]->SetFieldsFromTags( poFeature,
2709 id,
2710 true,
2711 nTags,
2712 pasTags,
2713 &sInfo);
2714
2715 poFeature->SetGeometryDirectly(poMulti);
2716
2717 int bFilteredOut = FALSE;
2718 if( !papoLayers[IDX_LYR_MULTIPOLYGONS]->AddFeature( poFeature,
2719 FALSE,
2720 &bFilteredOut,
2721 !bFeatureAdded ) )
2722 {
2723 bStopParsing = true;
2724 break;
2725 }
2726 else if( !bFilteredOut )
2727 {
2728 bFeatureAdded = true;
2729 }
2730 }
2731 else
2732 {
2733 CPLAssert(false);
2734 }
2735
2736 sqlite3_reset(pahSelectWayStmt[0]);
2737
2738 bHasRowInPolygonsStandalone =
2739 sqlite3_step(hSelectPolygonsStandaloneStmt) == SQLITE_ROW;
2740 }
2741 }
2742
2743 /************************************************************************/
2744 /* NotifyBounds() */
2745 /************************************************************************/
2746
NotifyBounds(double dfXMin,double dfYMin,double dfXMax,double dfYMax)2747 void OGROSMDataSource::NotifyBounds ( double dfXMin, double dfYMin,
2748 double dfXMax, double dfYMax )
2749 {
2750 sExtent.MinX = dfXMin;
2751 sExtent.MinY = dfYMin;
2752 sExtent.MaxX = dfXMax;
2753 sExtent.MaxY = dfYMax;
2754 bExtentValid = true;
2755
2756 CPLDebug( "OSM", "Got bounds : minx=%f, miny=%f, maxx=%f, maxy=%f",
2757 dfXMin, dfYMin, dfXMax, dfYMax );
2758 }
2759
OGROSMNotifyBounds(double dfXMin,double dfYMin,double dfXMax,double dfYMax,OSMContext *,void * user_data)2760 static void OGROSMNotifyBounds( double dfXMin, double dfYMin,
2761 double dfXMax, double dfYMax,
2762 OSMContext* /* psCtxt */,
2763 void* user_data )
2764 {
2765 static_cast<OGROSMDataSource *>(user_data)->NotifyBounds( dfXMin, dfYMin,
2766 dfXMax, dfYMax );
2767 }
2768
2769 /************************************************************************/
2770 /* Open() */
2771 /************************************************************************/
2772
Open(const char * pszFilename,char ** papszOpenOptionsIn)2773 int OGROSMDataSource::Open( const char * pszFilename,
2774 char** papszOpenOptionsIn )
2775
2776 {
2777 pszName = CPLStrdup( pszFilename );
2778
2779 psParser = OSM_Open( pszName,
2780 OGROSMNotifyNodes,
2781 OGROSMNotifyWay,
2782 OGROSMNotifyRelation,
2783 OGROSMNotifyBounds,
2784 this );
2785 if( psParser == nullptr )
2786 return FALSE;
2787
2788 if( CPLFetchBool(papszOpenOptionsIn, "INTERLEAVED_READING", false) )
2789 bInterleavedReading = TRUE;
2790
2791 /* The following 4 config options are only useful for debugging */
2792 bIndexPoints = CPLTestBool(CPLGetConfigOption("OSM_INDEX_POINTS", "YES"));
2793 bUsePointsIndex = CPLTestBool(
2794 CPLGetConfigOption("OSM_USE_POINTS_INDEX", "YES"));
2795 bIndexWays = CPLTestBool(CPLGetConfigOption("OSM_INDEX_WAYS", "YES"));
2796 bUseWaysIndex = CPLTestBool(
2797 CPLGetConfigOption("OSM_USE_WAYS_INDEX", "YES"));
2798
2799 bCustomIndexing = CPLTestBool(CSLFetchNameValueDef(
2800 papszOpenOptionsIn, "USE_CUSTOM_INDEXING",
2801 CPLGetConfigOption("OSM_USE_CUSTOM_INDEXING", "YES")));
2802 if( !bCustomIndexing )
2803 CPLDebug("OSM", "Using SQLite indexing for points");
2804 bCompressNodes = CPLTestBool(CSLFetchNameValueDef(
2805 papszOpenOptionsIn, "COMPRESS_NODES",
2806 CPLGetConfigOption("OSM_COMPRESS_NODES", "NO")));
2807 if( bCompressNodes )
2808 CPLDebug("OSM", "Using compression for nodes DB");
2809
2810 nLayers = 5;
2811 papoLayers = static_cast<OGROSMLayer **>(
2812 CPLMalloc(nLayers * sizeof(OGROSMLayer*)) );
2813
2814 papoLayers[IDX_LYR_POINTS] =
2815 new OGROSMLayer(this, IDX_LYR_POINTS, "points");
2816 papoLayers[IDX_LYR_POINTS]->GetLayerDefn()->SetGeomType(wkbPoint);
2817
2818 papoLayers[IDX_LYR_LINES] = new OGROSMLayer(this, IDX_LYR_LINES, "lines");
2819 papoLayers[IDX_LYR_LINES]->GetLayerDefn()->SetGeomType(wkbLineString);
2820
2821 papoLayers[IDX_LYR_MULTILINESTRINGS] =
2822 new OGROSMLayer(this, IDX_LYR_MULTILINESTRINGS, "multilinestrings");
2823 papoLayers[IDX_LYR_MULTILINESTRINGS]->GetLayerDefn()->
2824 SetGeomType(wkbMultiLineString);
2825
2826 papoLayers[IDX_LYR_MULTIPOLYGONS] =
2827 new OGROSMLayer(this, IDX_LYR_MULTIPOLYGONS, "multipolygons");
2828 papoLayers[IDX_LYR_MULTIPOLYGONS]->GetLayerDefn()->
2829 SetGeomType(wkbMultiPolygon);
2830
2831 papoLayers[IDX_LYR_OTHER_RELATIONS] =
2832 new OGROSMLayer(this, IDX_LYR_OTHER_RELATIONS, "other_relations");
2833 papoLayers[IDX_LYR_OTHER_RELATIONS]->GetLayerDefn()->
2834 SetGeomType(wkbGeometryCollection);
2835
2836 if( !ParseConf(papszOpenOptionsIn) )
2837 {
2838 CPLError( CE_Failure, CPLE_AppDefined,
2839 "Could not parse configuration file for OSM import");
2840 return FALSE;
2841 }
2842
2843 bNeedsToSaveWayInfo =
2844 ( papoLayers[IDX_LYR_MULTIPOLYGONS]->HasTimestamp() ||
2845 papoLayers[IDX_LYR_MULTIPOLYGONS]->HasChangeset() ||
2846 papoLayers[IDX_LYR_MULTIPOLYGONS]->HasVersion() ||
2847 papoLayers[IDX_LYR_MULTIPOLYGONS]->HasUID() ||
2848 papoLayers[IDX_LYR_MULTIPOLYGONS]->HasUser() );
2849
2850 panReqIds = static_cast<GIntBig*>(
2851 VSI_MALLOC_VERBOSE(MAX_ACCUMULATED_NODES * sizeof(GIntBig)));
2852 #ifdef ENABLE_NODE_LOOKUP_BY_HASHING
2853 panHashedIndexes = static_cast<int*>(
2854 VSI_MALLOC_VERBOSE(HASHED_INDEXES_ARRAY_SIZE * sizeof(int)));
2855 psCollisionBuckets = static_cast<CollisionBucket*>(
2856 VSI_MALLOC_VERBOSE(COLLISION_BUCKET_ARRAY_SIZE *
2857 sizeof(CollisionBucket)));
2858 #endif
2859 pasLonLatArray = static_cast<LonLat*>(
2860 VSI_MALLOC_VERBOSE(MAX_ACCUMULATED_NODES * sizeof(LonLat)));
2861 panUnsortedReqIds = static_cast<GIntBig*>(
2862 VSI_MALLOC_VERBOSE(MAX_ACCUMULATED_NODES * sizeof(GIntBig)));
2863 pasWayFeaturePairs = static_cast<WayFeaturePair*>(
2864 VSI_MALLOC_VERBOSE(MAX_DELAYED_FEATURES * sizeof(WayFeaturePair)));
2865 pasAccumulatedTags = static_cast<IndexedKVP*>(
2866 VSI_MALLOC_VERBOSE(MAX_ACCUMULATED_TAGS * sizeof(IndexedKVP)) );
2867 pabyNonRedundantValues = static_cast<GByte*>(
2868 VSI_MALLOC_VERBOSE(MAX_NON_REDUNDANT_VALUES) );
2869
2870 if( panReqIds == nullptr ||
2871 pasLonLatArray == nullptr ||
2872 panUnsortedReqIds == nullptr ||
2873 pasWayFeaturePairs == nullptr ||
2874 pasAccumulatedTags == nullptr ||
2875 pabyNonRedundantValues == nullptr )
2876 {
2877 return FALSE;
2878 }
2879
2880 nMaxSizeForInMemoryDBInMB = atoi(CSLFetchNameValueDef(papszOpenOptionsIn,
2881 "MAX_TMPFILE_SIZE", CPLGetConfigOption("OSM_MAX_TMPFILE_SIZE", "100")));
2882 GIntBig nSize =
2883 static_cast<GIntBig>(nMaxSizeForInMemoryDBInMB) * 1024 * 1024;
2884 if( nSize < 0 || (GIntBig)(size_t)nSize != nSize )
2885 {
2886 CPLError( CE_Failure, CPLE_AppDefined,
2887 "Invalid value for OSM_MAX_TMPFILE_SIZE. Using 100 instead." );
2888 nMaxSizeForInMemoryDBInMB = 100;
2889 nSize = static_cast<GIntBig>(nMaxSizeForInMemoryDBInMB) * 1024 * 1024;
2890 }
2891
2892 if( bCustomIndexing )
2893 {
2894 pabySector = static_cast<GByte *>(VSI_CALLOC_VERBOSE(1, SECTOR_SIZE));
2895
2896 if( pabySector == nullptr )
2897 {
2898 return FALSE;
2899 }
2900
2901 bInMemoryNodesFile = true;
2902 osNodesFilename.Printf("/vsimem/osm_importer/osm_temp_nodes_%p", this);
2903 fpNodes = VSIFOpenL(osNodesFilename, "wb+");
2904 if( fpNodes == nullptr )
2905 {
2906 return FALSE;
2907 }
2908
2909 CPLPushErrorHandler(CPLQuietErrorHandler);
2910 bool bSuccess =
2911 VSIFSeekL(fpNodes, (vsi_l_offset) (nSize * 3 / 4), SEEK_SET) == 0;
2912 CPLPopErrorHandler();
2913
2914 if( bSuccess )
2915 {
2916 VSIFSeekL(fpNodes, 0, SEEK_SET);
2917 VSIFTruncateL(fpNodes, 0);
2918 }
2919 else
2920 {
2921 CPLDebug( "OSM",
2922 "Not enough memory for in-memory file. "
2923 "Using disk temporary file instead." );
2924
2925 VSIFCloseL(fpNodes);
2926 fpNodes = nullptr;
2927 VSIUnlink(osNodesFilename);
2928
2929 bInMemoryNodesFile = false;
2930 osNodesFilename = CPLGenerateTempFilename("osm_tmp_nodes");
2931
2932 fpNodes = VSIFOpenL(osNodesFilename, "wb+");
2933 if( fpNodes == nullptr )
2934 {
2935 return FALSE;
2936 }
2937
2938 /* On Unix filesystems, you can remove a file even if it */
2939 /* opened */
2940 const char* pszVal = CPLGetConfigOption("OSM_UNLINK_TMPFILE", "YES");
2941 if( EQUAL(pszVal, "YES") )
2942 {
2943 CPLPushErrorHandler(CPLQuietErrorHandler);
2944 bMustUnlinkNodesFile = VSIUnlink( osNodesFilename ) != 0;
2945 CPLPopErrorHandler();
2946 }
2947
2948 return FALSE;
2949 }
2950 }
2951
2952 const bool bRet = CreateTempDB();
2953 if( bRet )
2954 {
2955 CPLString osInterestLayers = GetInterestLayersForDSName(GetName());
2956 if( !osInterestLayers.empty() )
2957 {
2958 delete ExecuteSQL( osInterestLayers, nullptr, nullptr );
2959 }
2960 }
2961 return bRet;
2962 }
2963
2964 /************************************************************************/
2965 /* CreateTempDB() */
2966 /************************************************************************/
2967
CreateTempDB()2968 bool OGROSMDataSource::CreateTempDB()
2969 {
2970 char* pszErrMsg = nullptr;
2971
2972 int rc = 0;
2973 bool bIsExisting = false;
2974 bool bSuccess = false;
2975
2976 const char* pszExistingTmpFile = CPLGetConfigOption("OSM_EXISTING_TMPFILE", nullptr);
2977 if( pszExistingTmpFile != nullptr )
2978 {
2979 bSuccess = true;
2980 bIsExisting = true;
2981 rc = sqlite3_open_v2( pszExistingTmpFile, &hDB,
2982 SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX,
2983 nullptr );
2984 }
2985 else
2986 {
2987 osTmpDBName.Printf("/vsimem/osm_importer/osm_temp_%p.sqlite", this);
2988
2989 // On 32 bit, the virtual memory space is scarce, so we need to
2990 // reserve it right now. Will not hurt on 64 bit either.
2991 VSILFILE* fp = VSIFOpenL(osTmpDBName, "wb");
2992 if( fp )
2993 {
2994 GIntBig nSize =
2995 static_cast<GIntBig>(nMaxSizeForInMemoryDBInMB) * 1024 * 1024;
2996 if( bCustomIndexing && bInMemoryNodesFile )
2997 nSize = nSize / 4;
2998
2999 CPLPushErrorHandler(CPLQuietErrorHandler);
3000 bSuccess =
3001 VSIFSeekL(fp, static_cast<vsi_l_offset>(nSize), SEEK_SET) == 0;
3002 CPLPopErrorHandler();
3003
3004 if( bSuccess )
3005 bSuccess = VSIFTruncateL(fp, 0) == 0;
3006
3007 VSIFCloseL(fp);
3008
3009 if( !bSuccess )
3010 {
3011 CPLDebug( "OSM",
3012 "Not enough memory for in-memory file. "
3013 "Using disk temporary file instead.");
3014 VSIUnlink(osTmpDBName);
3015 }
3016 }
3017
3018 if( bSuccess )
3019 {
3020 bInMemoryTmpDB = true;
3021 pMyVFS = OGRSQLiteCreateVFS(nullptr, this);
3022 sqlite3_vfs_register(pMyVFS, 0);
3023 rc = sqlite3_open_v2(
3024 osTmpDBName.c_str(), &hDB,
3025 SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
3026 SQLITE_OPEN_NOMUTEX,
3027 pMyVFS->zName );
3028 }
3029 }
3030
3031 if( !bSuccess )
3032 {
3033 osTmpDBName = CPLGenerateTempFilename("osm_tmp");
3034 rc = sqlite3_open( osTmpDBName.c_str(), &hDB );
3035
3036 /* On Unix filesystems, you can remove a file even if it */
3037 /* opened */
3038 if( rc == SQLITE_OK )
3039 {
3040 const char* pszVal =
3041 CPLGetConfigOption("OSM_UNLINK_TMPFILE", "YES");
3042 if( EQUAL(pszVal, "YES") )
3043 {
3044 CPLPushErrorHandler(CPLQuietErrorHandler);
3045 bMustUnlink = VSIUnlink( osTmpDBName ) != 0;
3046 CPLPopErrorHandler();
3047 }
3048 }
3049 }
3050
3051 if( rc != SQLITE_OK )
3052 {
3053 CPLError( CE_Failure, CPLE_OpenFailed,
3054 "sqlite3_open(%s) failed: %s",
3055 osTmpDBName.c_str(), sqlite3_errmsg( hDB ) );
3056 return false;
3057 }
3058
3059 if( !SetDBOptions() )
3060 {
3061 return false;
3062 }
3063
3064 if( !bIsExisting )
3065 {
3066 rc = sqlite3_exec(
3067 hDB,
3068 "CREATE TABLE nodes (id INTEGER PRIMARY KEY, coords BLOB)",
3069 nullptr, nullptr, &pszErrMsg );
3070 if( rc != SQLITE_OK )
3071 {
3072 CPLError( CE_Failure, CPLE_AppDefined,
3073 "Unable to create table nodes : %s", pszErrMsg );
3074 sqlite3_free( pszErrMsg );
3075 return false;
3076 }
3077
3078 rc = sqlite3_exec(
3079 hDB,
3080 "CREATE TABLE ways (id INTEGER PRIMARY KEY, data BLOB)",
3081 nullptr, nullptr, &pszErrMsg );
3082 if( rc != SQLITE_OK )
3083 {
3084 CPLError( CE_Failure, CPLE_AppDefined,
3085 "Unable to create table ways : %s", pszErrMsg );
3086 sqlite3_free( pszErrMsg );
3087 return false;
3088 }
3089
3090 rc = sqlite3_exec(
3091 hDB,
3092 "CREATE TABLE polygons_standalone (id INTEGER PRIMARY KEY)",
3093 nullptr, nullptr, &pszErrMsg );
3094 if( rc != SQLITE_OK )
3095 {
3096 CPLError( CE_Failure, CPLE_AppDefined,
3097 "Unable to create table polygons_standalone : %s",
3098 pszErrMsg );
3099 sqlite3_free( pszErrMsg );
3100 return false;
3101 }
3102 }
3103
3104 return CreatePreparedStatements();
3105 }
3106
3107 /************************************************************************/
3108 /* SetDBOptions() */
3109 /************************************************************************/
3110
SetDBOptions()3111 bool OGROSMDataSource::SetDBOptions()
3112 {
3113 char* pszErrMsg = nullptr;
3114 int rc =
3115 sqlite3_exec( hDB, "PRAGMA synchronous = OFF", nullptr, nullptr, &pszErrMsg );
3116 if( rc != SQLITE_OK )
3117 {
3118 CPLError( CE_Failure, CPLE_AppDefined,
3119 "Unable to run PRAGMA synchronous : %s",
3120 pszErrMsg );
3121 sqlite3_free( pszErrMsg );
3122 return false;
3123 }
3124
3125 rc = sqlite3_exec(
3126 hDB, "PRAGMA journal_mode = OFF", nullptr, nullptr, &pszErrMsg );
3127 if( rc != SQLITE_OK )
3128 {
3129 CPLError( CE_Failure, CPLE_AppDefined,
3130 "Unable to run PRAGMA journal_mode : %s",
3131 pszErrMsg );
3132 sqlite3_free( pszErrMsg );
3133 return false;
3134 }
3135
3136 rc = sqlite3_exec(
3137 hDB, "PRAGMA temp_store = MEMORY", nullptr, nullptr, &pszErrMsg );
3138 if( rc != SQLITE_OK )
3139 {
3140 CPLError( CE_Failure, CPLE_AppDefined,
3141 "Unable to run PRAGMA temp_store : %s",
3142 pszErrMsg );
3143 sqlite3_free( pszErrMsg );
3144 return false;
3145 }
3146
3147 if( !SetCacheSize() )
3148 return false;
3149
3150 if( !StartTransactionCacheDB() )
3151 return false;
3152
3153 return true;
3154 }
3155
3156 /************************************************************************/
3157 /* SetCacheSize() */
3158 /************************************************************************/
3159
SetCacheSize()3160 bool OGROSMDataSource::SetCacheSize()
3161 {
3162 const char* pszSqliteCacheMB = CPLGetConfigOption("OSM_SQLITE_CACHE", nullptr);
3163
3164 if( pszSqliteCacheMB == nullptr )
3165 return true;
3166
3167 char* pszErrMsg = nullptr;
3168 char **papszResult = nullptr;
3169 int nRowCount = 0;
3170 int nColCount = 0;
3171 int iSqlitePageSize = -1;
3172 const GIntBig iSqliteCacheBytes =
3173 static_cast<GIntBig>(atoi( pszSqliteCacheMB )) * 1024 * 1024;
3174
3175 /* querying the current PageSize */
3176 int rc = sqlite3_get_table( hDB, "PRAGMA page_size",
3177 &papszResult, &nRowCount, &nColCount,
3178 &pszErrMsg );
3179 if( rc == SQLITE_OK )
3180 {
3181 for( int iRow = 1; iRow <= nRowCount; iRow++ )
3182 {
3183 iSqlitePageSize = atoi( papszResult[(iRow * nColCount) + 0] );
3184 }
3185 sqlite3_free_table(papszResult);
3186 }
3187 if( iSqlitePageSize < 0 )
3188 {
3189 CPLError( CE_Failure, CPLE_AppDefined,
3190 "Unable to run PRAGMA page_size : %s",
3191 pszErrMsg ? pszErrMsg : sqlite3_errmsg(hDB) );
3192 sqlite3_free( pszErrMsg );
3193 return true;
3194 }
3195 if( iSqlitePageSize == 0 )
3196 return true;
3197
3198 /* computing the CacheSize as #Pages */
3199 const int iSqliteCachePages = static_cast<int>(
3200 iSqliteCacheBytes / iSqlitePageSize);
3201 if( iSqliteCachePages <= 0)
3202 return true;
3203
3204 rc = sqlite3_exec( hDB, CPLSPrintf( "PRAGMA cache_size = %d",
3205 iSqliteCachePages ),
3206 nullptr, nullptr, &pszErrMsg );
3207 if( rc != SQLITE_OK )
3208 {
3209 CPLError( CE_Warning, CPLE_AppDefined,
3210 "Unrecognized value for PRAGMA cache_size : %s",
3211 pszErrMsg );
3212 sqlite3_free( pszErrMsg );
3213 }
3214
3215 return true;
3216 }
3217
3218 /************************************************************************/
3219 /* CreatePreparedStatements() */
3220 /************************************************************************/
3221
CreatePreparedStatements()3222 bool OGROSMDataSource::CreatePreparedStatements()
3223 {
3224 int rc =
3225 sqlite3_prepare_v2( hDB,
3226 "INSERT INTO nodes (id, coords) VALUES (?,?)", -1,
3227 &hInsertNodeStmt, nullptr );
3228 if( rc != SQLITE_OK )
3229 {
3230 CPLError( CE_Failure, CPLE_AppDefined,
3231 "sqlite3_prepare_v2() failed : %s", sqlite3_errmsg(hDB) );
3232 return false;
3233 }
3234
3235 pahSelectNodeStmt = static_cast<sqlite3_stmt**>(
3236 CPLCalloc(sizeof(sqlite3_stmt*), LIMIT_IDS_PER_REQUEST) );
3237
3238 char szTmp[LIMIT_IDS_PER_REQUEST*2 + 128];
3239 strcpy(szTmp, "SELECT id, coords FROM nodes WHERE id IN (");
3240 int nLen = static_cast<int>(strlen(szTmp));
3241 for(int i=0;i<LIMIT_IDS_PER_REQUEST;i++)
3242 {
3243 if(i == 0)
3244 {
3245 strcpy(szTmp + nLen, "?) ORDER BY id ASC");
3246 nLen += 2;
3247 }
3248 else
3249 {
3250 strcpy(szTmp + nLen -1, ",?) ORDER BY id ASC");
3251 nLen += 2;
3252 }
3253 rc = sqlite3_prepare_v2( hDB, szTmp, -1, &pahSelectNodeStmt[i], nullptr );
3254 if( rc != SQLITE_OK )
3255 {
3256 CPLError( CE_Failure, CPLE_AppDefined,
3257 "sqlite3_prepare_v2() failed : %s", sqlite3_errmsg(hDB) );
3258 return false;
3259 }
3260 }
3261
3262 rc = sqlite3_prepare_v2( hDB, "INSERT INTO ways (id, data) VALUES (?,?)", -1,
3263 &hInsertWayStmt, nullptr );
3264 if( rc != SQLITE_OK )
3265 {
3266 CPLError( CE_Failure, CPLE_AppDefined,
3267 "sqlite3_prepare_v2() failed : %s", sqlite3_errmsg(hDB) );
3268 return false;
3269 }
3270
3271 pahSelectWayStmt = static_cast<sqlite3_stmt**>(
3272 CPLCalloc(sizeof(sqlite3_stmt*), LIMIT_IDS_PER_REQUEST) );
3273
3274 strcpy(szTmp, "SELECT id, data FROM ways WHERE id IN (");
3275 nLen = static_cast<int>(strlen(szTmp));
3276 for(int i=0;i<LIMIT_IDS_PER_REQUEST;i++)
3277 {
3278 if(i == 0)
3279 {
3280 strcpy(szTmp + nLen, "?)");
3281 nLen += 2;
3282 }
3283 else
3284 {
3285 strcpy(szTmp + nLen -1, ",?)");
3286 nLen += 2;
3287 }
3288 rc = sqlite3_prepare_v2( hDB, szTmp, -1, &pahSelectWayStmt[i], nullptr );
3289 if( rc != SQLITE_OK )
3290 {
3291 CPLError( CE_Failure, CPLE_AppDefined,
3292 "sqlite3_prepare_v2() failed : %s", sqlite3_errmsg(hDB) );
3293 return false;
3294 }
3295 }
3296
3297 rc = sqlite3_prepare_v2(
3298 hDB, "INSERT INTO polygons_standalone (id) VALUES (?)", -1,
3299 &hInsertPolygonsStandaloneStmt, nullptr );
3300 if( rc != SQLITE_OK )
3301 {
3302 CPLError( CE_Failure, CPLE_AppDefined,
3303 "sqlite3_prepare_v2() failed : %s", sqlite3_errmsg(hDB) );
3304 return false;
3305 }
3306
3307 rc = sqlite3_prepare_v2(
3308 hDB, "DELETE FROM polygons_standalone WHERE id = ?", -1,
3309 &hDeletePolygonsStandaloneStmt, nullptr );
3310 if( rc != SQLITE_OK )
3311 {
3312 CPLError( CE_Failure, CPLE_AppDefined,
3313 "sqlite3_prepare_v2() failed : %s", sqlite3_errmsg(hDB) );
3314 return false;
3315 }
3316
3317 rc = sqlite3_prepare_v2(
3318 hDB, "SELECT id FROM polygons_standalone ORDER BY id", -1,
3319 &hSelectPolygonsStandaloneStmt, nullptr );
3320 if( rc != SQLITE_OK )
3321 {
3322 CPLError( CE_Failure, CPLE_AppDefined,
3323 "sqlite3_prepare_v2() failed : %s", sqlite3_errmsg(hDB) );
3324 return false;
3325 }
3326
3327 return true;
3328 }
3329
3330 /************************************************************************/
3331 /* StartTransactionCacheDB() */
3332 /************************************************************************/
3333
StartTransactionCacheDB()3334 bool OGROSMDataSource::StartTransactionCacheDB()
3335 {
3336 if( bInTransaction )
3337 return false;
3338
3339 char* pszErrMsg = nullptr;
3340 int rc = sqlite3_exec( hDB, "BEGIN", nullptr, nullptr, &pszErrMsg );
3341 if( rc != SQLITE_OK )
3342 {
3343 CPLError( CE_Failure, CPLE_AppDefined,
3344 "Unable to start transaction : %s", pszErrMsg );
3345 sqlite3_free( pszErrMsg );
3346 return false;
3347 }
3348
3349 bInTransaction = true;
3350
3351 return true;
3352 }
3353
3354 /************************************************************************/
3355 /* CommitTransactionCacheDB() */
3356 /************************************************************************/
3357
CommitTransactionCacheDB()3358 bool OGROSMDataSource::CommitTransactionCacheDB()
3359 {
3360 if( !bInTransaction )
3361 return false;
3362
3363 bInTransaction = false;
3364
3365 char* pszErrMsg = nullptr;
3366 int rc = sqlite3_exec( hDB, "COMMIT", nullptr, nullptr, &pszErrMsg );
3367 if( rc != SQLITE_OK )
3368 {
3369 CPLError( CE_Failure, CPLE_AppDefined,
3370 "Unable to commit transaction : %s", pszErrMsg );
3371 sqlite3_free( pszErrMsg );
3372 return false;
3373 }
3374
3375 return true;
3376 }
3377
3378 /************************************************************************/
3379 /* AddComputedAttributes() */
3380 /************************************************************************/
3381
AddComputedAttributes(int iCurLayer,const std::vector<OGROSMComputedAttribute> & oAttributes)3382 void OGROSMDataSource::AddComputedAttributes(
3383 int iCurLayer,
3384 const std::vector<OGROSMComputedAttribute>& oAttributes)
3385 {
3386 for(size_t i=0; i<oAttributes.size();i++)
3387 {
3388 if( !oAttributes[i].osSQL.empty() )
3389 {
3390 papoLayers[iCurLayer]->AddComputedAttribute(oAttributes[i].osName,
3391 oAttributes[i].eType,
3392 oAttributes[i].osSQL);
3393 }
3394 }
3395 }
3396
3397 /************************************************************************/
3398 /* ParseConf() */
3399 /************************************************************************/
3400
ParseConf(char ** papszOpenOptionsIn)3401 bool OGROSMDataSource::ParseConf( char** papszOpenOptionsIn )
3402 {
3403 const char *pszFilename =
3404 CSLFetchNameValueDef(papszOpenOptionsIn, "CONFIG_FILE",
3405 CPLGetConfigOption("OSM_CONFIG_FILE", nullptr));
3406 if( pszFilename == nullptr )
3407 pszFilename = CPLFindFile( "gdal", "osmconf.ini" );
3408 if( pszFilename == nullptr )
3409 {
3410 CPLError( CE_Warning, CPLE_AppDefined,
3411 "Cannot find osmconf.ini configuration file");
3412 return false;
3413 }
3414
3415 VSILFILE* fpConf = VSIFOpenL(pszFilename, "rb");
3416 if( fpConf == nullptr )
3417 return false;
3418
3419 const char* pszLine = nullptr;
3420 int iCurLayer = -1;
3421 std::vector<OGROSMComputedAttribute> oAttributes;
3422
3423 while((pszLine = CPLReadLine2L(fpConf, -1, nullptr)) != nullptr)
3424 {
3425 if(pszLine[0] == '#')
3426 continue;
3427 if(pszLine[0] == '[' && pszLine[strlen(pszLine)-1] == ']' )
3428 {
3429 if( iCurLayer >= 0 )
3430 AddComputedAttributes(iCurLayer, oAttributes);
3431 oAttributes.resize(0);
3432
3433 iCurLayer = -1;
3434 pszLine ++;
3435 ((char*)pszLine)[strlen(pszLine)-1] = '\0'; /* Evil but OK */
3436 for(int i = 0; i < nLayers; i++)
3437 {
3438 if( strcmp(pszLine, papoLayers[i]->GetName()) == 0 )
3439 {
3440 iCurLayer = i;
3441 break;
3442 }
3443 }
3444 if( iCurLayer < 0 )
3445 {
3446 CPLError(CE_Warning, CPLE_AppDefined,
3447 "Layer '%s' mentioned in %s is unknown to the driver",
3448 pszLine, pszFilename);
3449 }
3450 continue;
3451 }
3452
3453 if( STARTS_WITH(pszLine, "closed_ways_are_polygons="))
3454 {
3455 char** papszTokens2 = CSLTokenizeString2(
3456 pszLine + strlen("closed_ways_are_polygons="), ",", 0);
3457 nMinSizeKeysInSetClosedWaysArePolygons = INT_MAX;
3458 nMaxSizeKeysInSetClosedWaysArePolygons = 0;
3459 for(int i=0;papszTokens2[i] != nullptr;i++)
3460 {
3461 const int nTokenSize = static_cast<int>(strlen(papszTokens2[i]));
3462 aoSetClosedWaysArePolygons.insert(papszTokens2[i]);
3463 nMinSizeKeysInSetClosedWaysArePolygons = std::min(
3464 nMinSizeKeysInSetClosedWaysArePolygons, nTokenSize);
3465 nMaxSizeKeysInSetClosedWaysArePolygons = std::max(
3466 nMinSizeKeysInSetClosedWaysArePolygons, nTokenSize);
3467 }
3468 CSLDestroy(papszTokens2);
3469 }
3470
3471 else if(STARTS_WITH(pszLine, "report_all_tags="))
3472 {
3473 if( strcmp(pszLine + strlen("report_all_tags="), "yes") == 0 )
3474 {
3475 std::fill( begin(m_ignoredKeys), end(m_ignoredKeys), "" );
3476 }
3477 }
3478
3479 else if(STARTS_WITH(pszLine, "report_all_nodes="))
3480 {
3481 if( strcmp(pszLine + strlen("report_all_nodes="), "no") == 0 )
3482 {
3483 bReportAllNodes = false;
3484 }
3485 else if( strcmp(pszLine + strlen("report_all_nodes="), "yes") == 0 )
3486 {
3487 bReportAllNodes = true;
3488 }
3489 }
3490
3491 else if(STARTS_WITH(pszLine, "report_all_ways="))
3492 {
3493 if( strcmp(pszLine + strlen("report_all_ways="), "no") == 0 )
3494 {
3495 bReportAllWays = false;
3496 }
3497 else if( strcmp(pszLine + strlen("report_all_ways="), "yes") == 0 )
3498 {
3499 bReportAllWays = true;
3500 }
3501 }
3502
3503 else if(STARTS_WITH(pszLine, "attribute_name_laundering="))
3504 {
3505 if( strcmp(pszLine + strlen("attribute_name_laundering="), "no") == 0 )
3506 {
3507 bAttributeNameLaundering = false;
3508 }
3509 else if( strcmp(pszLine + strlen("attribute_name_laundering="), "yes") == 0 )
3510 {
3511 bAttributeNameLaundering = true;
3512 }
3513 }
3514
3515 else if( iCurLayer >= 0 )
3516 {
3517 char** papszTokens = CSLTokenizeString2(pszLine, "=", 0);
3518 if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "other_tags") == 0 )
3519 {
3520 if( strcmp(papszTokens[1], "no") == 0 )
3521 papoLayers[iCurLayer]->SetHasOtherTags(false);
3522 else if( strcmp(papszTokens[1], "yes") == 0 )
3523 papoLayers[iCurLayer]->SetHasOtherTags(true);
3524 }
3525 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "all_tags") == 0 )
3526 {
3527 if( strcmp(papszTokens[1], "no") == 0 )
3528 papoLayers[iCurLayer]->SetHasAllTags(false);
3529 else if( strcmp(papszTokens[1], "yes") == 0 )
3530 papoLayers[iCurLayer]->SetHasAllTags(true);
3531 }
3532 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_id") == 0 )
3533 {
3534 if( strcmp(papszTokens[1], "no") == 0 )
3535 papoLayers[iCurLayer]->SetHasOSMId(false);
3536 else if( strcmp(papszTokens[1], "yes") == 0 )
3537 {
3538 papoLayers[iCurLayer]->SetHasOSMId(true);
3539 papoLayers[iCurLayer]->AddField("osm_id", OFTString);
3540
3541 if( iCurLayer == IDX_LYR_MULTIPOLYGONS )
3542 papoLayers[iCurLayer]->AddField("osm_way_id", OFTString);
3543 }
3544 }
3545 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_version") == 0 )
3546 {
3547 if( strcmp(papszTokens[1], "no") == 0 )
3548 papoLayers[iCurLayer]->SetHasVersion(false);
3549 else if( strcmp(papszTokens[1], "yes") == 0 )
3550 {
3551 papoLayers[iCurLayer]->SetHasVersion(true);
3552 papoLayers[iCurLayer]->AddField("osm_version", OFTInteger);
3553 }
3554 }
3555 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_timestamp") == 0 )
3556 {
3557 if( strcmp(papszTokens[1], "no") == 0 )
3558 papoLayers[iCurLayer]->SetHasTimestamp(false);
3559 else if( strcmp(papszTokens[1], "yes") == 0 )
3560 {
3561 papoLayers[iCurLayer]->SetHasTimestamp(true);
3562 papoLayers[iCurLayer]->AddField("osm_timestamp", OFTDateTime);
3563 }
3564 }
3565 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_uid") == 0 )
3566 {
3567 if( strcmp(papszTokens[1], "no") == 0 )
3568 papoLayers[iCurLayer]->SetHasUID(false);
3569 else if( strcmp(papszTokens[1], "yes") == 0 )
3570 {
3571 papoLayers[iCurLayer]->SetHasUID(true);
3572 papoLayers[iCurLayer]->AddField("osm_uid", OFTInteger);
3573 }
3574 }
3575 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_user") == 0 )
3576 {
3577 if( strcmp(papszTokens[1], "no") == 0 )
3578 papoLayers[iCurLayer]->SetHasUser(false);
3579 else if( strcmp(papszTokens[1], "yes") == 0 )
3580 {
3581 papoLayers[iCurLayer]->SetHasUser(true);
3582 papoLayers[iCurLayer]->AddField("osm_user", OFTString);
3583 }
3584 }
3585 else if( CSLCount(papszTokens) == 2 &&
3586 strcmp(papszTokens[0], "osm_changeset") == 0 )
3587 {
3588 if( strcmp(papszTokens[1], "no") == 0 )
3589 papoLayers[iCurLayer]->SetHasChangeset(false);
3590 else if( strcmp(papszTokens[1], "yes") == 0 )
3591 {
3592 papoLayers[iCurLayer]->SetHasChangeset(true);
3593 papoLayers[iCurLayer]->AddField("osm_changeset",
3594 OFTInteger);
3595 }
3596 }
3597 else if( CSLCount(papszTokens) == 2 &&
3598 strcmp(papszTokens[0], "attributes") == 0 )
3599 {
3600 char** papszTokens2 =
3601 CSLTokenizeString2(papszTokens[1], ",", 0);
3602 for(int i=0;papszTokens2[i] != nullptr;i++)
3603 {
3604 papoLayers[iCurLayer]->AddField(papszTokens2[i], OFTString);
3605 for( const char*& pszIgnoredKey : m_ignoredKeys )
3606 {
3607 if( strcmp( papszTokens2[i], pszIgnoredKey ) == 0 )
3608 pszIgnoredKey = "";
3609 }
3610 }
3611 CSLDestroy(papszTokens2);
3612 }
3613 else if( CSLCount(papszTokens) == 2 &&
3614 (strcmp(papszTokens[0], "unsignificant") == 0 ||
3615 strcmp(papszTokens[0], "insignificant") == 0) )
3616 {
3617 char** papszTokens2 =
3618 CSLTokenizeString2(papszTokens[1], ",", 0);
3619 for(int i=0;papszTokens2[i] != nullptr;i++)
3620 {
3621 papoLayers[iCurLayer]->AddInsignificantKey(papszTokens2[i]);
3622 }
3623 CSLDestroy(papszTokens2);
3624 }
3625 else if( CSLCount(papszTokens) == 2 &&
3626 strcmp(papszTokens[0], "ignore") == 0 )
3627 {
3628 char** papszTokens2 =
3629 CSLTokenizeString2(papszTokens[1], ",", 0);
3630 for(int i=0;papszTokens2[i] != nullptr;i++)
3631 {
3632 papoLayers[iCurLayer]->AddIgnoreKey(papszTokens2[i]);
3633 papoLayers[iCurLayer]->AddWarnKey(papszTokens2[i]);
3634 }
3635 CSLDestroy(papszTokens2);
3636 }
3637 else if( CSLCount(papszTokens) == 2 &&
3638 strcmp(papszTokens[0], "computed_attributes") == 0 )
3639 {
3640 char** papszTokens2 =
3641 CSLTokenizeString2(papszTokens[1], ",", 0);
3642 oAttributes.resize(0);
3643 for(int i=0;papszTokens2[i] != nullptr;i++)
3644 {
3645 oAttributes.push_back(
3646 OGROSMComputedAttribute(papszTokens2[i]));
3647 }
3648 CSLDestroy(papszTokens2);
3649 }
3650 else if( CSLCount(papszTokens) == 2 &&
3651 strlen(papszTokens[0]) >= 5 &&
3652 strcmp(papszTokens[0] + strlen(papszTokens[0]) - 5,
3653 "_type") == 0 )
3654 {
3655 CPLString osName(papszTokens[0]);
3656 osName.resize(strlen(papszTokens[0]) - 5);
3657 const char* pszType = papszTokens[1];
3658 bool bFound = false;
3659 OGRFieldType eType = OFTString;
3660 if( EQUAL(pszType, "Integer") )
3661 eType = OFTInteger;
3662 else if( EQUAL(pszType, "Integer64") )
3663 eType = OFTInteger64;
3664 else if( EQUAL(pszType, "Real") )
3665 eType = OFTReal;
3666 else if( EQUAL(pszType, "String") )
3667 eType = OFTString;
3668 else if( EQUAL(pszType, "DateTime") )
3669 eType = OFTDateTime;
3670 else
3671 CPLError(CE_Warning, CPLE_AppDefined,
3672 "Unhandled type (%s) for attribute %s",
3673 pszType, osName.c_str());
3674 for(size_t i = 0; i < oAttributes.size(); i++ )
3675 {
3676 if( oAttributes[i].osName == osName )
3677 {
3678 bFound = true;
3679 oAttributes[i].eType = eType;
3680 break;
3681 }
3682 }
3683 if( !bFound )
3684 {
3685 const int idx =
3686 papoLayers[iCurLayer]->
3687 GetLayerDefn()->GetFieldIndex(osName);
3688 if( idx >= 0 )
3689 {
3690 papoLayers[iCurLayer]->
3691 GetLayerDefn()->GetFieldDefn(idx)->SetType(eType);
3692 bFound = true;
3693 }
3694 }
3695 if( !bFound )
3696 {
3697 CPLError(CE_Warning, CPLE_AppDefined,
3698 "Undeclared attribute : %s",
3699 osName.c_str());
3700 }
3701 }
3702 else if( CSLCount(papszTokens) >= 2 &&
3703 strlen(papszTokens[0]) >= 4 &&
3704 strcmp(papszTokens[0] + strlen(papszTokens[0]) - 4,
3705 "_sql") == 0 )
3706 {
3707 CPLString osName(papszTokens[0]);
3708 osName.resize(strlen(papszTokens[0]) - 4);
3709 size_t i = 0; // Used after for.
3710 for( ; i < oAttributes.size(); i++ )
3711 {
3712 if( oAttributes[i].osName == osName )
3713 {
3714 const char* pszSQL = strchr(pszLine, '=') + 1;
3715 while( *pszSQL == ' ' )
3716 pszSQL ++;
3717 bool bInQuotes = false;
3718 if( *pszSQL == '"' )
3719 {
3720 bInQuotes = true;
3721 pszSQL ++;
3722 }
3723 oAttributes[i].osSQL = pszSQL;
3724 if( bInQuotes && oAttributes[i].osSQL.size() > 1 &&
3725 oAttributes[i].osSQL.back() == '"' )
3726 oAttributes[i].osSQL.resize(oAttributes[i].osSQL.size()-1);
3727 break;
3728 }
3729 }
3730 if( i == oAttributes.size() )
3731 {
3732 CPLError( CE_Warning, CPLE_AppDefined,
3733 "Undeclared attribute : %s",
3734 osName.c_str());
3735 }
3736 }
3737 CSLDestroy(papszTokens);
3738 }
3739 }
3740
3741 if( iCurLayer >= 0 )
3742 AddComputedAttributes(iCurLayer, oAttributes);
3743
3744 for(int i=0;i<nLayers;i++)
3745 {
3746 if( papoLayers[i]->HasAllTags() )
3747 {
3748 papoLayers[i]->AddField("all_tags", OFTString);
3749 if( papoLayers[i]->HasOtherTags() )
3750 {
3751 papoLayers[i]->SetHasOtherTags(false);
3752 }
3753 }
3754 else if( papoLayers[i]->HasOtherTags() )
3755 papoLayers[i]->AddField("other_tags", OFTString);
3756 }
3757
3758 VSIFCloseL(fpConf);
3759
3760 return true;
3761 }
3762
3763 /************************************************************************/
3764 /* MyResetReading() */
3765 /************************************************************************/
3766
MyResetReading()3767 int OGROSMDataSource::MyResetReading()
3768 {
3769 if( hDB == nullptr )
3770 return FALSE;
3771 if( bCustomIndexing && fpNodes == nullptr )
3772 return FALSE;
3773
3774 OSM_ResetReading(psParser);
3775
3776 char* pszErrMsg = nullptr;
3777 int rc = sqlite3_exec( hDB, "DELETE FROM nodes", nullptr, nullptr, &pszErrMsg );
3778 if( rc != SQLITE_OK )
3779 {
3780 CPLError( CE_Failure, CPLE_AppDefined,
3781 "Unable to DELETE FROM nodes : %s", pszErrMsg );
3782 sqlite3_free( pszErrMsg );
3783 return FALSE;
3784 }
3785
3786 rc = sqlite3_exec( hDB, "DELETE FROM ways", nullptr, nullptr, &pszErrMsg );
3787 if( rc != SQLITE_OK )
3788 {
3789 CPLError( CE_Failure, CPLE_AppDefined,
3790 "Unable to DELETE FROM ways : %s", pszErrMsg );
3791 sqlite3_free( pszErrMsg );
3792 return FALSE;
3793 }
3794
3795 rc = sqlite3_exec( hDB, "DELETE FROM polygons_standalone", nullptr, nullptr,
3796 &pszErrMsg );
3797 if( rc != SQLITE_OK )
3798 {
3799 CPLError( CE_Failure, CPLE_AppDefined,
3800 "Unable to DELETE FROM polygons_standalone : %s", pszErrMsg );
3801 sqlite3_free( pszErrMsg );
3802 return FALSE;
3803 }
3804 bHasRowInPolygonsStandalone = false;
3805
3806 if( hSelectPolygonsStandaloneStmt != nullptr )
3807 sqlite3_reset( hSelectPolygonsStandaloneStmt );
3808
3809 {
3810 for( int i = 0; i < nWayFeaturePairs; i++)
3811 {
3812 delete pasWayFeaturePairs[i].poFeature;
3813 }
3814 nWayFeaturePairs = 0;
3815 nUnsortedReqIds = 0;
3816 nReqIds = 0;
3817 nAccumulatedTags = 0;
3818 nNonRedundantValuesLen = 0;
3819
3820 for( int i=0;i<static_cast<int>(asKeys.size()); i++ )
3821 {
3822 KeyDesc* psKD = asKeys[i];
3823 CPLFree(psKD->pszK);
3824 for(int j=0;j<(int)psKD->asValues.size();j++)
3825 CPLFree(psKD->asValues[j]);
3826 delete psKD;
3827 }
3828 asKeys.resize(0);
3829 aoMapIndexedKeys.clear();
3830 nNextKeyIndex = 0;
3831 }
3832
3833 if( bCustomIndexing )
3834 {
3835 nPrevNodeId = -1;
3836 nBucketOld = -1;
3837 nOffInBucketReducedOld = -1;
3838
3839 VSIFSeekL(fpNodes, 0, SEEK_SET);
3840 VSIFTruncateL(fpNodes, 0);
3841 nNodesFileSize = 0;
3842
3843 memset(pabySector, 0, SECTOR_SIZE);
3844
3845 std::map<int, Bucket>::iterator oIter = oMapBuckets.begin();
3846 for( ; oIter != oMapBuckets.end(); ++oIter )
3847 {
3848 Bucket* psBucket = &(oIter->second);
3849 psBucket->nOff = -1;
3850 if( bCompressNodes )
3851 {
3852 if( psBucket->u.panSectorSize )
3853 memset(psBucket->u.panSectorSize, 0, BUCKET_SECTOR_SIZE_ARRAY_SIZE);
3854 }
3855 else
3856 {
3857 if( psBucket->u.pabyBitmap )
3858 memset(psBucket->u.pabyBitmap, 0, BUCKET_BITMAP_SIZE);
3859 }
3860 }
3861 }
3862
3863 for(int i=0;i<nLayers;i++)
3864 {
3865 papoLayers[i]->ForceResetReading();
3866 }
3867
3868 bStopParsing = false;
3869 poCurrentLayer = nullptr;
3870
3871 return TRUE;
3872 }
3873
3874 /************************************************************************/
3875 /* ResetReading() */
3876 /************************************************************************/
3877
ResetReading()3878 void OGROSMDataSource::ResetReading()
3879 {
3880 MyResetReading();
3881 }
3882
3883 /************************************************************************/
3884 /* GetNextFeature() */
3885 /************************************************************************/
3886
GetNextFeature(OGRLayer ** ppoBelongingLayer,double * pdfProgressPct,GDALProgressFunc pfnProgress,void * pProgressData)3887 OGRFeature* OGROSMDataSource::GetNextFeature( OGRLayer** ppoBelongingLayer,
3888 double* pdfProgressPct,
3889 GDALProgressFunc pfnProgress,
3890 void* pProgressData )
3891 {
3892 bInterleavedReading = TRUE;
3893
3894 if( poCurrentLayer == nullptr )
3895 {
3896 poCurrentLayer = papoLayers[0];
3897 }
3898 if( pdfProgressPct != nullptr || pfnProgress != nullptr )
3899 {
3900 if( m_nFileSize == FILESIZE_NOT_INIT )
3901 {
3902 VSIStatBufL sStat;
3903 if( VSIStatL( pszName, &sStat ) == 0 )
3904 {
3905 m_nFileSize = static_cast<GIntBig>(sStat.st_size);
3906 }
3907 else
3908 {
3909 m_nFileSize = FILESIZE_INVALID;
3910 }
3911 }
3912 }
3913
3914 while( true )
3915 {
3916 OGROSMLayer* poNewCurLayer = nullptr;
3917 CPLAssert( poCurrentLayer != nullptr );
3918 OGRFeature* poFeature = poCurrentLayer->MyGetNextFeature(&poNewCurLayer,
3919 pfnProgress,
3920 pProgressData);
3921 poCurrentLayer = poNewCurLayer;
3922 if( poFeature == nullptr)
3923 {
3924 if( poCurrentLayer != nullptr )
3925 continue;
3926 if( ppoBelongingLayer != nullptr )
3927 *ppoBelongingLayer = nullptr;
3928 if( pdfProgressPct != nullptr )
3929 *pdfProgressPct = 1.0;
3930 return nullptr;
3931 }
3932 if( ppoBelongingLayer != nullptr )
3933 *ppoBelongingLayer = poCurrentLayer;
3934 if( pdfProgressPct != nullptr )
3935 {
3936 if( m_nFileSize != FILESIZE_INVALID )
3937 {
3938 *pdfProgressPct = 1.0 * OSM_GetBytesRead(psParser) /
3939 m_nFileSize;
3940 }
3941 else
3942 {
3943 *pdfProgressPct = -1.0;
3944 }
3945 }
3946
3947 return poFeature;
3948 }
3949 }
3950
3951 /************************************************************************/
3952 /* ParseNextChunk() */
3953 /************************************************************************/
3954
ParseNextChunk(int nIdxLayer,GDALProgressFunc pfnProgress,void * pProgressData)3955 bool OGROSMDataSource::ParseNextChunk( int nIdxLayer,
3956 GDALProgressFunc pfnProgress,
3957 void* pProgressData )
3958 {
3959 if( bStopParsing )
3960 return false;
3961
3962 bHasParsedFirstChunk = true;
3963 bFeatureAdded = false;
3964 while( true )
3965 {
3966 #ifdef DEBUG_MEM_USAGE
3967 static int counter = 0;
3968 counter ++;
3969 if( (counter % 1000) == 0 )
3970 CPLDebug("OSM", "GetMaxTotalAllocs() = " CPL_FRMT_GUIB,
3971 static_cast<GUIntBig>(GetMaxTotalAllocs()));
3972 #endif
3973
3974 OSMRetCode eRet = OSM_ProcessBlock(psParser);
3975 if( pfnProgress != nullptr )
3976 {
3977 double dfPct = -1.0;
3978 if( m_nFileSize != FILESIZE_INVALID )
3979 {
3980 dfPct = 1.0 * OSM_GetBytesRead(psParser) / m_nFileSize;
3981 }
3982 if( !pfnProgress( dfPct, "", pProgressData ) )
3983 {
3984 bStopParsing = true;
3985 for(int i=0;i<nLayers;i++)
3986 {
3987 papoLayers[i]->ForceResetReading();
3988 }
3989 return false;
3990 }
3991 }
3992
3993 if( eRet == OSM_EOF || eRet == OSM_ERROR )
3994 {
3995 if( eRet == OSM_EOF )
3996 {
3997 if( nWayFeaturePairs != 0 )
3998 ProcessWaysBatch();
3999
4000 ProcessPolygonsStandalone();
4001
4002 if( !bHasRowInPolygonsStandalone )
4003 bStopParsing = true;
4004
4005 if( !bInterleavedReading && !bFeatureAdded &&
4006 bHasRowInPolygonsStandalone &&
4007 nIdxLayer != IDX_LYR_MULTIPOLYGONS )
4008 {
4009 return false;
4010 }
4011
4012 return bFeatureAdded || bHasRowInPolygonsStandalone;
4013 }
4014 else
4015 {
4016 CPLError(CE_Failure, CPLE_AppDefined,
4017 "An error occurred during the parsing of data "
4018 "around byte " CPL_FRMT_GUIB,
4019 OSM_GetBytesRead(psParser));
4020
4021 bStopParsing = true;
4022 return false;
4023 }
4024 }
4025 else
4026 {
4027 if( bInMemoryTmpDB )
4028 {
4029 if( !TransferToDiskIfNecesserary() )
4030 return false;
4031 }
4032
4033 if( bFeatureAdded )
4034 break;
4035 }
4036 }
4037
4038 return true;
4039 }
4040
4041 /************************************************************************/
4042 /* TransferToDiskIfNecesserary() */
4043 /************************************************************************/
4044
TransferToDiskIfNecesserary()4045 bool OGROSMDataSource::TransferToDiskIfNecesserary()
4046 {
4047 if( bInMemoryNodesFile )
4048 {
4049 if( nNodesFileSize / 1024 / 1024 > 3 * nMaxSizeForInMemoryDBInMB / 4 )
4050 {
4051 bInMemoryNodesFile = false;
4052
4053 VSIFCloseL(fpNodes);
4054 fpNodes = nullptr;
4055
4056 CPLString osNewTmpDBName;
4057 osNewTmpDBName = CPLGenerateTempFilename("osm_tmp_nodes");
4058
4059 CPLDebug("OSM", "%s too big for RAM. Transferring it onto disk in %s",
4060 osNodesFilename.c_str(), osNewTmpDBName.c_str());
4061
4062 if( CPLCopyFile( osNewTmpDBName, osNodesFilename ) != 0 )
4063 {
4064 CPLError(CE_Failure, CPLE_AppDefined,
4065 "Cannot copy %s to %s",
4066 osNodesFilename.c_str(), osNewTmpDBName.c_str() );
4067 VSIUnlink(osNewTmpDBName);
4068 bStopParsing = true;
4069 return false;
4070 }
4071
4072 VSIUnlink(osNodesFilename);
4073
4074 if( bInMemoryTmpDB )
4075 {
4076 /* Try to grow the sqlite in memory-db to the full space now */
4077 /* it has been freed. */
4078 VSILFILE* fp = VSIFOpenL(osTmpDBName, "rb+");
4079 if( fp )
4080 {
4081 VSIFSeekL(fp, 0, SEEK_END);
4082 vsi_l_offset nCurSize = VSIFTellL(fp);
4083 GIntBig nNewSize =
4084 static_cast<GIntBig>(nMaxSizeForInMemoryDBInMB) *
4085 1024 * 1024;
4086 CPLPushErrorHandler(CPLQuietErrorHandler);
4087 const bool bSuccess =
4088 VSIFSeekL(fp, (vsi_l_offset) nNewSize, SEEK_SET) == 0;
4089 CPLPopErrorHandler();
4090
4091 if( bSuccess )
4092 VSIFTruncateL(fp, nCurSize);
4093
4094 VSIFCloseL(fp);
4095 }
4096 }
4097
4098 osNodesFilename = osNewTmpDBName;
4099
4100 fpNodes = VSIFOpenL(osNodesFilename, "rb+");
4101 if( fpNodes == nullptr )
4102 {
4103 bStopParsing = true;
4104 return false;
4105 }
4106
4107 VSIFSeekL(fpNodes, 0, SEEK_END);
4108
4109 /* On Unix filesystems, you can remove a file even if it */
4110 /* opened */
4111 const char* pszVal = CPLGetConfigOption("OSM_UNLINK_TMPFILE", "YES");
4112 if( EQUAL(pszVal, "YES") )
4113 {
4114 CPLPushErrorHandler(CPLQuietErrorHandler);
4115 bMustUnlinkNodesFile = VSIUnlink( osNodesFilename ) != 0;
4116 CPLPopErrorHandler();
4117 }
4118 }
4119 }
4120
4121 if( bInMemoryTmpDB )
4122 {
4123 VSIStatBufL sStat;
4124
4125 int nLimitMB = nMaxSizeForInMemoryDBInMB;
4126 if( bCustomIndexing && bInMemoryNodesFile )
4127 nLimitMB = nLimitMB * 1 / 4;
4128
4129 if( VSIStatL( osTmpDBName, &sStat ) == 0 &&
4130 sStat.st_size / 1024 / 1024 > nLimitMB )
4131 {
4132 bInMemoryTmpDB = false;
4133
4134 CloseDB();
4135
4136 CPLString osNewTmpDBName;
4137
4138 osNewTmpDBName = CPLGenerateTempFilename("osm_tmp");
4139
4140 CPLDebug("OSM", "%s too big for RAM. Transferring it onto disk in %s",
4141 osTmpDBName.c_str(), osNewTmpDBName.c_str());
4142
4143 if( CPLCopyFile( osNewTmpDBName, osTmpDBName ) != 0 )
4144 {
4145 CPLError(CE_Failure, CPLE_AppDefined,
4146 "Cannot copy %s to %s",
4147 osTmpDBName.c_str(), osNewTmpDBName.c_str() );
4148 VSIUnlink(osNewTmpDBName);
4149 bStopParsing = true;
4150 return false;
4151 }
4152
4153 VSIUnlink(osTmpDBName);
4154
4155 osTmpDBName = osNewTmpDBName;
4156
4157 const int rc =
4158 sqlite3_open_v2( osTmpDBName.c_str(), &hDB,
4159 SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX,
4160 nullptr );
4161 if( rc != SQLITE_OK )
4162 {
4163 CPLError( CE_Failure, CPLE_OpenFailed,
4164 "sqlite3_open(%s) failed: %s",
4165 osTmpDBName.c_str(), sqlite3_errmsg( hDB ) );
4166 bStopParsing = true;
4167 CloseDB();
4168 return false;
4169 }
4170
4171 /* On Unix filesystems, you can remove a file even if it */
4172 /* opened */
4173 const char* pszVal =
4174 CPLGetConfigOption("OSM_UNLINK_TMPFILE", "YES");
4175 if( EQUAL(pszVal, "YES") )
4176 {
4177 CPLPushErrorHandler(CPLQuietErrorHandler);
4178 bMustUnlink = VSIUnlink( osTmpDBName ) != 0;
4179 CPLPopErrorHandler();
4180 }
4181
4182 if( !SetDBOptions() || !CreatePreparedStatements() )
4183 {
4184 bStopParsing = true;
4185 CloseDB();
4186 return false;
4187 }
4188 }
4189 }
4190
4191 return true;
4192 }
4193
4194 /************************************************************************/
4195 /* TestCapability() */
4196 /************************************************************************/
4197
TestCapability(const char * pszCap)4198 int OGROSMDataSource::TestCapability( const char * pszCap )
4199 {
4200 return EQUAL(pszCap, ODsCRandomLayerRead);
4201 }
4202
4203 /************************************************************************/
4204 /* GetLayer() */
4205 /************************************************************************/
4206
GetLayer(int iLayer)4207 OGRLayer *OGROSMDataSource::GetLayer( int iLayer )
4208
4209 {
4210 if( iLayer < 0 || iLayer >= nLayers )
4211 return nullptr;
4212
4213 return papoLayers[iLayer];
4214 }
4215
4216 /************************************************************************/
4217 /* GetExtent() */
4218 /************************************************************************/
4219
GetExtent(OGREnvelope * psExtent)4220 OGRErr OGROSMDataSource::GetExtent( OGREnvelope *psExtent )
4221 {
4222 if( !bHasParsedFirstChunk )
4223 {
4224 bHasParsedFirstChunk = true;
4225 OSM_ProcessBlock(psParser);
4226 }
4227
4228 if( bExtentValid )
4229 {
4230 *psExtent = sExtent;
4231 return OGRERR_NONE;
4232 }
4233
4234 return OGRERR_FAILURE;
4235 }
4236
4237 /************************************************************************/
4238 /* OGROSMSingleFeatureLayer */
4239 /************************************************************************/
4240
4241 class OGROSMSingleFeatureLayer final: public OGRLayer
4242 {
4243 private:
4244 int nVal;
4245 char *pszVal;
4246 OGRFeatureDefn *poFeatureDefn;
4247 int iNextShapeId;
4248
4249 public:
4250 OGROSMSingleFeatureLayer( const char* pszLayerName,
4251 int nVal );
4252 OGROSMSingleFeatureLayer( const char* pszLayerName,
4253 const char *pszVal );
4254 virtual ~OGROSMSingleFeatureLayer();
4255
ResetReading()4256 virtual void ResetReading() override { iNextShapeId = 0; }
4257 virtual OGRFeature *GetNextFeature() override;
GetLayerDefn()4258 virtual OGRFeatureDefn *GetLayerDefn() override { return poFeatureDefn; }
TestCapability(const char *)4259 virtual int TestCapability( const char * ) override { return FALSE; }
4260 };
4261
4262 /************************************************************************/
4263 /* OGROSMSingleFeatureLayer() */
4264 /************************************************************************/
4265
OGROSMSingleFeatureLayer(const char * pszLayerName,int nValIn)4266 OGROSMSingleFeatureLayer::OGROSMSingleFeatureLayer( const char* pszLayerName,
4267 int nValIn ) :
4268 nVal(nValIn),
4269 pszVal(nullptr),
4270 poFeatureDefn(new OGRFeatureDefn( "SELECT" )),
4271 iNextShapeId(0)
4272 {
4273 poFeatureDefn->Reference();
4274 OGRFieldDefn oField( pszLayerName, OFTInteger );
4275 poFeatureDefn->AddFieldDefn( &oField );
4276 }
4277
4278 /************************************************************************/
4279 /* OGROSMSingleFeatureLayer() */
4280 /************************************************************************/
4281
OGROSMSingleFeatureLayer(const char * pszLayerName,const char * pszValIn)4282 OGROSMSingleFeatureLayer::OGROSMSingleFeatureLayer( const char* pszLayerName,
4283 const char *pszValIn ) :
4284 nVal(0),
4285 pszVal(CPLStrdup(pszValIn)),
4286 poFeatureDefn(new OGRFeatureDefn( "SELECT" )),
4287 iNextShapeId(0)
4288 {
4289 poFeatureDefn->Reference();
4290 OGRFieldDefn oField( pszLayerName, OFTString );
4291 poFeatureDefn->AddFieldDefn( &oField );
4292 }
4293
4294 /************************************************************************/
4295 /* ~OGROSMSingleFeatureLayer() */
4296 /************************************************************************/
4297
~OGROSMSingleFeatureLayer()4298 OGROSMSingleFeatureLayer::~OGROSMSingleFeatureLayer()
4299 {
4300 poFeatureDefn->Release();
4301 CPLFree(pszVal);
4302 }
4303
4304 /************************************************************************/
4305 /* GetNextFeature() */
4306 /************************************************************************/
4307
GetNextFeature()4308 OGRFeature * OGROSMSingleFeatureLayer::GetNextFeature()
4309 {
4310 if( iNextShapeId != 0 )
4311 return nullptr;
4312
4313 OGRFeature* poFeature = new OGRFeature(poFeatureDefn);
4314 if( pszVal )
4315 poFeature->SetField(0, pszVal);
4316 else
4317 poFeature->SetField(0, nVal);
4318 poFeature->SetFID(iNextShapeId ++);
4319 return poFeature;
4320 }
4321
4322 /************************************************************************/
4323 /* OGROSMResultLayerDecorator */
4324 /************************************************************************/
4325
4326 class OGROSMResultLayerDecorator final: public OGRLayerDecorator
4327 {
4328 CPLString osDSName;
4329 CPLString osInterestLayers;
4330
4331 public:
OGROSMResultLayerDecorator(OGRLayer * poLayer,CPLString osDSNameIn,CPLString osInterestLayersIn)4332 OGROSMResultLayerDecorator(OGRLayer* poLayer,
4333 CPLString osDSNameIn,
4334 CPLString osInterestLayersIn) :
4335 OGRLayerDecorator(poLayer, TRUE),
4336 osDSName(osDSNameIn),
4337 osInterestLayers(osInterestLayersIn) {}
4338
GetFeatureCount(int bForce=TRUE)4339 virtual GIntBig GetFeatureCount( int bForce = TRUE ) override
4340 {
4341 /* When we run GetFeatureCount() with SQLite SQL dialect, */
4342 /* the OSM dataset will be re-opened. Make sure that it is */
4343 /* re-opened with the same interest layers */
4344 AddInterestLayersForDSName(osDSName, osInterestLayers);
4345 return OGRLayerDecorator::GetFeatureCount(bForce);
4346 }
4347 };
4348
4349 /************************************************************************/
4350 /* ExecuteSQL() */
4351 /************************************************************************/
4352
ExecuteSQL(const char * pszSQLCommand,OGRGeometry * poSpatialFilter,const char * pszDialect)4353 OGRLayer * OGROSMDataSource::ExecuteSQL( const char *pszSQLCommand,
4354 OGRGeometry *poSpatialFilter,
4355 const char *pszDialect )
4356
4357 {
4358 /* -------------------------------------------------------------------- */
4359 /* Special GetBytesRead() command */
4360 /* -------------------------------------------------------------------- */
4361 if( strcmp(pszSQLCommand, "GetBytesRead()") == 0 )
4362 {
4363 char szVal[64] = {};
4364 snprintf( szVal, sizeof(szVal), CPL_FRMT_GUIB,
4365 OSM_GetBytesRead(psParser) );
4366 return new OGROSMSingleFeatureLayer( "GetBytesRead", szVal );
4367 }
4368
4369 if( poResultSetLayer != nullptr )
4370 {
4371 CPLError(CE_Failure, CPLE_NotSupported,
4372 "A SQL result layer is still in use. Please delete it first");
4373 return nullptr;
4374 }
4375
4376 /* -------------------------------------------------------------------- */
4377 /* Special SET interest_layers = command */
4378 /* -------------------------------------------------------------------- */
4379 if( STARTS_WITH(pszSQLCommand, "SET interest_layers =") )
4380 {
4381 char** papszTokens =
4382 CSLTokenizeString2(pszSQLCommand + 21, ",",
4383 CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES);
4384 for( int i=0; i < nLayers; i++ )
4385 {
4386 papoLayers[i]->SetDeclareInterest(FALSE);
4387 }
4388
4389 for( int i=0; papszTokens[i] != nullptr; i++ )
4390 {
4391 OGROSMLayer* poLayer = reinterpret_cast<OGROSMLayer *>(
4392 GetLayerByName(papszTokens[i]) );
4393 if( poLayer != nullptr )
4394 {
4395 poLayer->SetDeclareInterest(TRUE);
4396 }
4397 }
4398
4399 if( papoLayers[IDX_LYR_POINTS]->IsUserInterested() &&
4400 !papoLayers[IDX_LYR_LINES]->IsUserInterested() &&
4401 !papoLayers[IDX_LYR_MULTILINESTRINGS]->IsUserInterested() &&
4402 !papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() &&
4403 !papoLayers[IDX_LYR_OTHER_RELATIONS]->IsUserInterested())
4404 {
4405 if( CPLGetConfigOption("OSM_INDEX_POINTS", nullptr) == nullptr )
4406 {
4407 CPLDebug("OSM", "Disabling indexing of nodes");
4408 bIndexPoints = false;
4409 }
4410 if( CPLGetConfigOption("OSM_USE_POINTS_INDEX", nullptr) == nullptr )
4411 {
4412 bUsePointsIndex = false;
4413 }
4414 if( CPLGetConfigOption("OSM_INDEX_WAYS", nullptr) == nullptr )
4415 {
4416 CPLDebug("OSM", "Disabling indexing of ways");
4417 bIndexWays = false;
4418 }
4419 if( CPLGetConfigOption("OSM_USE_WAYS_INDEX", nullptr) == nullptr )
4420 {
4421 bUseWaysIndex = false;
4422 }
4423 }
4424 else if( papoLayers[IDX_LYR_LINES]->IsUserInterested() &&
4425 !papoLayers[IDX_LYR_MULTILINESTRINGS]->IsUserInterested() &&
4426 !papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() &&
4427 !papoLayers[IDX_LYR_OTHER_RELATIONS]->IsUserInterested() )
4428 {
4429 if( CPLGetConfigOption("OSM_INDEX_WAYS", nullptr) == nullptr )
4430 {
4431 CPLDebug("OSM", "Disabling indexing of ways");
4432 bIndexWays = false;
4433 }
4434 if( CPLGetConfigOption("OSM_USE_WAYS_INDEX", nullptr) == nullptr )
4435 {
4436 bUseWaysIndex = false;
4437 }
4438 }
4439
4440 CSLDestroy(papszTokens);
4441
4442 return nullptr;
4443 }
4444
4445 while(*pszSQLCommand == ' ')
4446 pszSQLCommand ++;
4447
4448 /* Try to analyse the SQL command to get the interest table */
4449 if( STARTS_WITH_CI(pszSQLCommand, "SELECT") )
4450 {
4451 bool bLayerAlreadyAdded = false;
4452 CPLString osInterestLayers = "SET interest_layers =";
4453
4454 if( pszDialect != nullptr && EQUAL(pszDialect, "SQLITE") )
4455 {
4456 std::set<LayerDesc> oSetLayers =
4457 OGRSQLiteGetReferencedLayers(pszSQLCommand);
4458 std::set<LayerDesc>::iterator oIter = oSetLayers.begin();
4459 for(; oIter != oSetLayers.end(); ++oIter)
4460 {
4461 const LayerDesc& oLayerDesc = *oIter;
4462 if( oLayerDesc.osDSName.empty() )
4463 {
4464 if( bLayerAlreadyAdded ) osInterestLayers += ",";
4465 bLayerAlreadyAdded = true;
4466 osInterestLayers += oLayerDesc.osLayerName;
4467 }
4468 }
4469 }
4470 else
4471 {
4472 swq_select sSelectInfo;
4473
4474 CPLPushErrorHandler(CPLQuietErrorHandler);
4475 CPLErr eErr = sSelectInfo.preparse( pszSQLCommand );
4476 CPLPopErrorHandler();
4477
4478 if( eErr == CE_None )
4479 {
4480 swq_select* pCurSelect = &sSelectInfo;
4481 while(pCurSelect != nullptr)
4482 {
4483 for( int iTable = 0; iTable < pCurSelect->table_count;
4484 iTable++ )
4485 {
4486 swq_table_def *psTableDef =
4487 pCurSelect->table_defs + iTable;
4488 if( psTableDef->data_source == nullptr )
4489 {
4490 if( bLayerAlreadyAdded ) osInterestLayers += ",";
4491 bLayerAlreadyAdded = true;
4492 osInterestLayers += psTableDef->table_name;
4493 }
4494 }
4495 pCurSelect = pCurSelect->poOtherSelect;
4496 }
4497 }
4498 }
4499
4500 if( bLayerAlreadyAdded )
4501 {
4502 /* Backup current optimization parameters */
4503 abSavedDeclaredInterest.resize(0);
4504 for(int i=0; i < nLayers; i++)
4505 {
4506 abSavedDeclaredInterest.push_back(papoLayers[i]->IsUserInterested());
4507 }
4508 bIndexPointsBackup = bIndexPoints;
4509 bUsePointsIndexBackup = bUsePointsIndex;
4510 bIndexWaysBackup = bIndexWays;
4511 bUseWaysIndexBackup = bUseWaysIndex;
4512
4513 /* Update optimization parameters */
4514 delete ExecuteSQL(osInterestLayers, nullptr, nullptr);
4515
4516 MyResetReading();
4517
4518 /* Run the request */
4519 poResultSetLayer = OGRDataSource::ExecuteSQL( pszSQLCommand,
4520 poSpatialFilter,
4521 pszDialect );
4522
4523 /* If the user explicitly run a COUNT() request, then do it ! */
4524 if( poResultSetLayer )
4525 {
4526 if( pszDialect != nullptr && EQUAL(pszDialect, "SQLITE") )
4527 {
4528 poResultSetLayer = new OGROSMResultLayerDecorator(
4529 poResultSetLayer, GetName(), osInterestLayers);
4530 }
4531 bIsFeatureCountEnabled = true;
4532 }
4533
4534 return poResultSetLayer;
4535 }
4536 }
4537
4538 return OGRDataSource::ExecuteSQL( pszSQLCommand,
4539 poSpatialFilter,
4540 pszDialect );
4541 }
4542
4543 /************************************************************************/
4544 /* ReleaseResultSet() */
4545 /************************************************************************/
4546
ReleaseResultSet(OGRLayer * poLayer)4547 void OGROSMDataSource::ReleaseResultSet( OGRLayer * poLayer )
4548
4549 {
4550 if( poLayer != nullptr && poLayer == poResultSetLayer )
4551 {
4552 poResultSetLayer = nullptr;
4553
4554 bIsFeatureCountEnabled = false;
4555
4556 /* Restore backup'ed optimization parameters */
4557 for(int i=0; i < nLayers; i++)
4558 {
4559 papoLayers[i]->SetDeclareInterest(abSavedDeclaredInterest[i]);
4560 }
4561 if( bIndexPointsBackup && !bIndexPoints )
4562 CPLDebug("OSM", "Re-enabling indexing of nodes");
4563 bIndexPoints = bIndexPointsBackup;
4564 bUsePointsIndex = bUsePointsIndexBackup;
4565 if( bIndexWaysBackup && !bIndexWays )
4566 CPLDebug("OSM", "Re-enabling indexing of ways");
4567 bIndexWays = bIndexWaysBackup;
4568 bUseWaysIndex = bUseWaysIndexBackup;
4569 abSavedDeclaredInterest.resize(0);
4570 }
4571
4572 delete poLayer;
4573 }
4574
4575 /************************************************************************/
4576 /* IsInterleavedReading() */
4577 /************************************************************************/
4578
IsInterleavedReading()4579 int OGROSMDataSource::IsInterleavedReading()
4580 {
4581 if( bInterleavedReading < 0 )
4582 {
4583 bInterleavedReading = CPLTestBool(
4584 CPLGetConfigOption("OGR_INTERLEAVED_READING", "NO"));
4585 CPLDebug("OSM", "OGR_INTERLEAVED_READING = %d", bInterleavedReading);
4586 }
4587 return bInterleavedReading;
4588 }
4589