1 /******************************************************************************
2 * $Id: ogrosmdatasource.cpp 29242 2015-05-24 10:59:41Z rouault $
3 *
4 * Project: OpenGIS Simple Features Reference Implementation
5 * Purpose: Implements OGROSMDataSource class.
6 * Author: Even Rouault, <even dot rouault at mines dash paris dot org>
7 *
8 ******************************************************************************
9 * Copyright (c) 2012-2014, Even Rouault <even dot rouault at mines-paris dot org>
10 *
11 * Permission is hereby granted, free of charge, to any person obtaining a
12 * copy of this software and associated documentation files (the "Software"),
13 * to deal in the Software without restriction, including without limitation
14 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 * and/or sell copies of the Software, and to permit persons to whom the
16 * Software is furnished to do so, subject to the following conditions:
17 *
18 * The above copyright notice and this permission notice shall be included
19 * in all copies or substantial portions of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 * DEALINGS IN THE SOFTWARE.
28 ****************************************************************************/
29
30 #include "ogr_osm.h"
31 #include "cpl_conv.h"
32 #include "cpl_time.h"
33 #include "ogr_p.h"
34 #include "ogr_api.h"
35 #include "swq.h"
36 #include "gpb.h"
37 #include "ogrlayerdecorator.h"
38 #include "ogrsqliteexecutesql.h"
39 #include "cpl_multiproc.h"
40
41 #include <algorithm>
42
43 #define LIMIT_IDS_PER_REQUEST 200
44
45 #define MAX_NODES_PER_WAY 2000
46
47 #define IDX_LYR_POINTS 0
48 #define IDX_LYR_LINES 1
49 #define IDX_LYR_MULTILINESTRINGS 2
50 #define IDX_LYR_MULTIPOLYGONS 3
51 #define IDX_LYR_OTHER_RELATIONS 4
52
53 #define DBL_TO_INT(x) (int)floor((x) * 1e7 + 0.5)
54 #define INT_TO_DBL(x) ((x) / 1e7)
55
56 #define MAX_COUNT_FOR_TAGS_IN_WAY 255 /* must fit on 1 byte */
57 #define MAX_SIZE_FOR_TAGS_IN_WAY 1024
58
59 /* 5 bytes for encoding a int : really the worst case scenario ! */
60 #define WAY_BUFFER_SIZE (1 + MAX_NODES_PER_WAY * 2 * 5 + MAX_SIZE_FOR_TAGS_IN_WAY)
61
62 #define NODE_PER_BUCKET 65536
63
64 /* Initial Maximum count of buckets */
65 #define INIT_BUCKET_COUNT 65536
66
67 #define VALID_ID_FOR_CUSTOM_INDEXING(_id) ((_id) >= 0 && (_id / NODE_PER_BUCKET) < INT_MAX)
68
69 /* Minimum size of data written on disk, in *uncompressed* case */
70 #define SECTOR_SIZE 512
71 /* Which represents, 64 nodes */
72 /* #define NODE_PER_SECTOR SECTOR_SIZE / (2 * 4) */
73 #define NODE_PER_SECTOR 64
74 #define NODE_PER_SECTOR_SHIFT 6
75
76 /* Per bucket, we keep track of the absence/presence of sectors */
77 /* only, to reduce memory usage */
78 /* #define BUCKET_BITMAP_SIZE NODE_PER_BUCKET / (8 * NODE_PER_SECTOR) */
79 #define BUCKET_BITMAP_SIZE 128
80
81 /* #define BUCKET_SECTOR_SIZE_ARRAY_SIZE NODE_PER_BUCKET / NODE_PER_SECTOR */
82 /* Per bucket, we keep track of the real size of the sector. Each sector */
83 /* size is encoded in a single byte, whose value is : */
84 /* (sector_size in bytes - 8 ) / 2, minus 8. 252 means uncompressed */
85 #define BUCKET_SECTOR_SIZE_ARRAY_SIZE 1024
86
87 /* Must be a multiple of both BUCKET_BITMAP_SIZE and BUCKET_SECTOR_SIZE_ARRAY_SIZE */
88 #define PAGE_SIZE 4096
89
90 /* compressSize should not be greater than 512, so COMPRESS_SIZE_TO_BYTE() fits on a byte */
91 #define COMPRESS_SIZE_TO_BYTE(nCompressSize) (GByte)(((nCompressSize) - 8) / 2)
92 #define ROUND_COMPRESS_SIZE(nCompressSize) (((nCompressSize) + 1) / 2) * 2;
93 #define COMPRESS_SIZE_FROM_BYTE(byte_on_size) ((byte_on_size) * 2 + 8)
94
95 /* Max number of features that are accumulated in pasWayFeaturePairs */
96 #define MAX_DELAYED_FEATURES 75000
97 /* Max number of tags that are accumulated in pasAccumulatedTags */
98 #define MAX_ACCUMULATED_TAGS MAX_DELAYED_FEATURES * 5
99 /* Max size of the string with tag values that are accumulated in pabyNonRedundantValues */
100 #define MAX_NON_REDUNDANT_VALUES MAX_DELAYED_FEATURES * 10
101 /* Max number of features that are accumulated in panUnsortedReqIds */
102 #define MAX_ACCUMULATED_NODES 1000000
103
104 #ifdef ENABLE_NODE_LOOKUP_BY_HASHING
105 /* Size of panHashedIndexes array. Must be in the list at */
106 /* http://planetmath.org/goodhashtableprimes , and greater than MAX_ACCUMULATED_NODES */
107 #define HASHED_INDEXES_ARRAY_SIZE 3145739
108 //#define HASHED_INDEXES_ARRAY_SIZE 1572869
109 #define COLLISION_BUCKET_ARRAY_SIZE ((MAX_ACCUMULATED_NODES / 100) * 40)
110 /* hash function = identity ! */
111 #define HASH_ID_FUNC(x) ((GUIntBig)(x))
112 #endif // ENABLE_NODE_LOOKUP_BY_HASHING
113
114 //#define FAKE_LOOKUP_NODES
115
116 //#define DEBUG_MEM_USAGE
117 #ifdef DEBUG_MEM_USAGE
118 size_t GetMaxTotalAllocs();
119 #endif
120
121 static void WriteVarInt64(GUIntBig nSVal, GByte** ppabyData);
122 static void WriteVarSInt64(GIntBig nSVal, GByte** ppabyData);
123
124 CPL_CVSID("$Id: ogrosmdatasource.cpp 29242 2015-05-24 10:59:41Z rouault $");
125
126 class DSToBeOpened
127 {
128 public:
129 GIntBig nPID;
130 CPLString osDSName;
131 CPLString osInterestLayers;
132 };
133
134 static CPLMutex *hMutex = NULL;
135 static std::vector<DSToBeOpened> oListDSToBeOpened;
136
137 /************************************************************************/
138 /* AddInterestLayersForDSName() */
139 /************************************************************************/
140
AddInterestLayersForDSName(const CPLString & osDSName,const CPLString & osInterestLayers)141 static void AddInterestLayersForDSName(const CPLString& osDSName,
142 const CPLString& osInterestLayers)
143 {
144 CPLMutexHolder oMutexHolder(&hMutex);
145 DSToBeOpened oDSToBeOpened;
146 oDSToBeOpened.nPID = CPLGetPID();
147 oDSToBeOpened.osDSName = osDSName;
148 oDSToBeOpened.osInterestLayers = osInterestLayers;
149 oListDSToBeOpened.push_back( oDSToBeOpened );
150 }
151
152 /************************************************************************/
153 /* GetInterestLayersForDSName() */
154 /************************************************************************/
155
GetInterestLayersForDSName(const CPLString & osDSName)156 static CPLString GetInterestLayersForDSName(const CPLString& osDSName)
157 {
158 CPLMutexHolder oMutexHolder(&hMutex);
159 GIntBig nPID = CPLGetPID();
160 for(int i = 0; i < (int)oListDSToBeOpened.size(); i++)
161 {
162 if( oListDSToBeOpened[i].nPID == nPID &&
163 oListDSToBeOpened[i].osDSName == osDSName )
164 {
165 CPLString osInterestLayers = oListDSToBeOpened[i].osInterestLayers;
166 oListDSToBeOpened.erase(oListDSToBeOpened.begin()+i);
167 return osInterestLayers;
168 }
169 }
170 return "";
171 }
172
173 /************************************************************************/
174 /* OGROSMDataSource() */
175 /************************************************************************/
176
OGROSMDataSource()177 OGROSMDataSource::OGROSMDataSource()
178
179 {
180 nLayers = 0;
181 papoLayers = NULL;
182 pszName = NULL;
183 bExtentValid = FALSE;
184 bInterleavedReading = -1;
185 poCurrentLayer = NULL;
186 psParser = NULL;
187 bHasParsedFirstChunk = FALSE;
188 bStopParsing = FALSE;
189 #ifdef HAVE_SQLITE_VFS
190 pMyVFS = NULL;
191 #endif
192 hDB = NULL;
193 hInsertNodeStmt = NULL;
194 hInsertWayStmt = NULL;
195 hInsertPolygonsStandaloneStmt = NULL;
196 hDeletePolygonsStandaloneStmt = NULL;
197 hSelectPolygonsStandaloneStmt = NULL;
198 bHasRowInPolygonsStandalone = FALSE;
199
200 hDBForComputedAttributes = NULL;
201
202 nNodesInTransaction = 0;
203 bInTransaction = FALSE;
204 pahSelectNodeStmt = NULL;
205 pahSelectWayStmt = NULL;
206 pasLonLatCache = NULL;
207 bInMemoryTmpDB = FALSE;
208 bMustUnlink = TRUE;
209 nMaxSizeForInMemoryDBInMB = 0;
210 bReportAllNodes = FALSE;
211 bReportAllWays = FALSE;
212 bFeatureAdded = FALSE;
213
214 bIndexPoints = TRUE;
215 bUsePointsIndex = TRUE;
216 bIndexWays = TRUE;
217 bUseWaysIndex = TRUE;
218
219 poResultSetLayer = NULL;
220 bIndexPointsBackup = FALSE;
221 bUsePointsIndexBackup = FALSE;
222 bIndexWaysBackup = FALSE;
223 bUseWaysIndexBackup = FALSE;
224
225 bIsFeatureCountEnabled = FALSE;
226
227 bAttributeNameLaundering = TRUE;
228
229 pabyWayBuffer = NULL;
230
231 nWaysProcessed = 0;
232 nRelationsProcessed = 0;
233
234 bCustomIndexing = TRUE;
235 bCompressNodes = FALSE;
236
237 bInMemoryNodesFile = FALSE;
238 bMustUnlinkNodesFile = TRUE;
239 fpNodes = NULL;
240 nNodesFileSize = 0;
241
242 nPrevNodeId = -INT_MAX;
243 nBucketOld = -1;
244 nOffInBucketReducedOld = -1;
245 pabySector = NULL;
246 papsBuckets = NULL;
247 nBuckets = 0;
248
249 nReqIds = 0;
250 panReqIds = NULL;
251 #ifdef ENABLE_NODE_LOOKUP_BY_HASHING
252 bEnableHashedIndex = TRUE;
253 panHashedIndexes = NULL;
254 psCollisionBuckets = NULL;
255 bHashedIndexValid = FALSE;
256 #endif
257 pasLonLatArray = NULL;
258 nUnsortedReqIds = 0;
259 panUnsortedReqIds = NULL;
260 nWayFeaturePairs = 0;
261 pasWayFeaturePairs = NULL;
262 nAccumulatedTags = 0;
263 pasAccumulatedTags = NULL;
264 nNonRedundantValuesLen = 0;
265 pabyNonRedundantValues = NULL;
266 nNextKeyIndex = 0;
267
268 bNeedsToSaveWayInfo = FALSE;
269 }
270
271 /************************************************************************/
272 /* ~OGROSMDataSource() */
273 /************************************************************************/
274
~OGROSMDataSource()275 OGROSMDataSource::~OGROSMDataSource()
276
277 {
278 int i;
279 for(i=0;i<nLayers;i++)
280 delete papoLayers[i];
281 CPLFree(papoLayers);
282
283 CPLFree(pszName);
284
285 if( psParser != NULL )
286 CPLDebug("OSM", "Number of bytes read in file : " CPL_FRMT_GUIB, OSM_GetBytesRead(psParser));
287 OSM_Close(psParser);
288
289 CPLFree(pasLonLatCache);
290 CPLFree(pabyWayBuffer);
291
292 if( hDB != NULL )
293 CloseDB();
294
295 if( hDBForComputedAttributes != NULL )
296 sqlite3_close(hDBForComputedAttributes);
297
298 #ifdef HAVE_SQLITE_VFS
299 if (pMyVFS)
300 {
301 sqlite3_vfs_unregister(pMyVFS);
302 CPLFree(pMyVFS->pAppData);
303 CPLFree(pMyVFS);
304 }
305 #endif
306
307 if( osTmpDBName.size() && bMustUnlink )
308 {
309 const char* pszVal = CPLGetConfigOption("OSM_UNLINK_TMPFILE", "YES");
310 if( !EQUAL(pszVal, "NOT_EVEN_AT_END") )
311 VSIUnlink(osTmpDBName);
312 }
313
314 CPLFree(panReqIds);
315 #ifdef ENABLE_NODE_LOOKUP_BY_HASHING
316 CPLFree(panHashedIndexes);
317 CPLFree(psCollisionBuckets);
318 #endif
319 CPLFree(pasLonLatArray);
320 CPLFree(panUnsortedReqIds);
321
322 for( i = 0; i < nWayFeaturePairs; i++)
323 {
324 delete pasWayFeaturePairs[i].poFeature;
325 }
326 CPLFree(pasWayFeaturePairs);
327 CPLFree(pasAccumulatedTags);
328 CPLFree(pabyNonRedundantValues);
329
330 #ifdef OSM_DEBUG
331 FILE* f;
332 f = fopen("keys.txt", "wt");
333 for(i=0;i<(int)asKeys.size();i++)
334 {
335 KeyDesc* psKD = asKeys[i];
336 fprintf(f, "%08d idx=%d %s\n",
337 psKD->nOccurences,
338 psKD->nKeyIndex,
339 psKD->pszK);
340 }
341 fclose(f);
342 #endif
343
344 for(i=0;i<(int)asKeys.size();i++)
345 {
346 KeyDesc* psKD = asKeys[i];
347 CPLFree(psKD->pszK);
348 for(int j=0;j<(int)psKD->asValues.size();j++)
349 CPLFree(psKD->asValues[j]);
350 delete psKD;
351 }
352
353 if( fpNodes )
354 VSIFCloseL(fpNodes);
355 if( osNodesFilename.size() && bMustUnlinkNodesFile )
356 {
357 const char* pszVal = CPLGetConfigOption("OSM_UNLINK_TMPFILE", "YES");
358 if( !EQUAL(pszVal, "NOT_EVEN_AT_END") )
359 VSIUnlink(osNodesFilename);
360 }
361
362 CPLFree(pabySector);
363 if( papsBuckets )
364 {
365 for( i = 0; i < nBuckets; i++)
366 {
367 if( bCompressNodes )
368 {
369 int nRem = i % (PAGE_SIZE / BUCKET_SECTOR_SIZE_ARRAY_SIZE);
370 if( nRem == 0 )
371 CPLFree(papsBuckets[i].u.panSectorSize);
372 }
373 else
374 {
375 int nRem = i % (PAGE_SIZE / BUCKET_BITMAP_SIZE);
376 if( nRem == 0 )
377 CPLFree(papsBuckets[i].u.pabyBitmap);
378 }
379 }
380 CPLFree(papsBuckets);
381 }
382 }
383
384 /************************************************************************/
385 /* CloseDB() */
386 /************************************************************************/
387
CloseDB()388 void OGROSMDataSource::CloseDB()
389 {
390 int i;
391
392 if( hInsertNodeStmt != NULL )
393 sqlite3_finalize( hInsertNodeStmt );
394 hInsertNodeStmt = NULL;
395
396 if( hInsertWayStmt != NULL )
397 sqlite3_finalize( hInsertWayStmt );
398 hInsertWayStmt = NULL;
399
400 if( hInsertPolygonsStandaloneStmt != NULL )
401 sqlite3_finalize( hInsertPolygonsStandaloneStmt );
402 hInsertPolygonsStandaloneStmt = NULL;
403
404 if( hDeletePolygonsStandaloneStmt != NULL )
405 sqlite3_finalize( hDeletePolygonsStandaloneStmt );
406 hDeletePolygonsStandaloneStmt = NULL;
407
408 if( hSelectPolygonsStandaloneStmt != NULL )
409 sqlite3_finalize( hSelectPolygonsStandaloneStmt );
410 hSelectPolygonsStandaloneStmt = NULL;
411
412 if( pahSelectNodeStmt != NULL )
413 {
414 for(i = 0; i < LIMIT_IDS_PER_REQUEST; i++)
415 {
416 if( pahSelectNodeStmt[i] != NULL )
417 sqlite3_finalize( pahSelectNodeStmt[i] );
418 }
419 CPLFree(pahSelectNodeStmt);
420 pahSelectNodeStmt = NULL;
421 }
422
423 if( pahSelectWayStmt != NULL )
424 {
425 for(i = 0; i < LIMIT_IDS_PER_REQUEST; i++)
426 {
427 if( pahSelectWayStmt[i] != NULL )
428 sqlite3_finalize( pahSelectWayStmt[i] );
429 }
430 CPLFree(pahSelectWayStmt);
431 pahSelectWayStmt = NULL;
432 }
433
434 if( bInTransaction )
435 CommitTransactionCacheDB();
436
437 sqlite3_close(hDB);
438 hDB = NULL;
439 }
440
441 /************************************************************************/
442 /* IndexPoint() */
443 /************************************************************************/
444
445 static const GByte abyBitsCount[] = {
446 0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,
447 1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,
448 1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,
449 2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,
450 1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,
451 2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,
452 2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,
453 3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,
454 1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,
455 2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,
456 2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,
457 3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,
458 2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,
459 3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,
460 3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,
461 4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8
462 };
463
IndexPoint(OSMNode * psNode)464 int OGROSMDataSource::IndexPoint(OSMNode* psNode)
465 {
466 if( !bIndexPoints )
467 return TRUE;
468
469 if( bCustomIndexing)
470 return IndexPointCustom(psNode);
471 else
472 return IndexPointSQLite(psNode);
473 }
474
475 /************************************************************************/
476 /* IndexPointSQLite() */
477 /************************************************************************/
478
IndexPointSQLite(OSMNode * psNode)479 int OGROSMDataSource::IndexPointSQLite(OSMNode* psNode)
480 {
481 sqlite3_bind_int64( hInsertNodeStmt, 1, psNode->nID );
482
483 LonLat sLonLat;
484 sLonLat.nLon = DBL_TO_INT(psNode->dfLon);
485 sLonLat.nLat = DBL_TO_INT(psNode->dfLat);
486
487 sqlite3_bind_blob( hInsertNodeStmt, 2, &sLonLat, sizeof(sLonLat), SQLITE_STATIC );
488
489 int rc = sqlite3_step( hInsertNodeStmt );
490 sqlite3_reset( hInsertNodeStmt );
491 if( !(rc == SQLITE_OK || rc == SQLITE_DONE) )
492 {
493 CPLError(CE_Failure, CPLE_AppDefined, "Failed inserting node " CPL_FRMT_GIB ": %s",
494 psNode->nID, sqlite3_errmsg(hDB));
495 }
496
497 return TRUE;
498 }
499
500 /************************************************************************/
501 /* FlushCurrentSector() */
502 /************************************************************************/
503
FlushCurrentSector()504 int OGROSMDataSource::FlushCurrentSector()
505 {
506 #ifndef FAKE_LOOKUP_NODES
507 if( bCompressNodes )
508 return FlushCurrentSectorCompressedCase();
509 else
510 return FlushCurrentSectorNonCompressedCase();
511 #else
512 return TRUE;
513 #endif
514 }
515
516 /************************************************************************/
517 /* AllocBucket() */
518 /************************************************************************/
519
AllocBucket(int iBucket)520 int OGROSMDataSource::AllocBucket(int iBucket)
521 {
522 int bOOM = FALSE;
523 if( bCompressNodes )
524 {
525 int nRem = iBucket % (PAGE_SIZE / BUCKET_SECTOR_SIZE_ARRAY_SIZE);
526 if( papsBuckets[iBucket - nRem].u.panSectorSize == NULL )
527 papsBuckets[iBucket - nRem].u.panSectorSize = (GByte*)VSICalloc(1, PAGE_SIZE);
528 if( papsBuckets[iBucket - nRem].u.panSectorSize == NULL )
529 {
530 papsBuckets[iBucket].u.panSectorSize = NULL;
531 bOOM = TRUE;
532 }
533 else
534 papsBuckets[iBucket].u.panSectorSize = papsBuckets[iBucket - nRem].u.panSectorSize + nRem * BUCKET_SECTOR_SIZE_ARRAY_SIZE;
535 }
536 else
537 {
538 int nRem = iBucket % (PAGE_SIZE / BUCKET_BITMAP_SIZE);
539 if( papsBuckets[iBucket - nRem].u.pabyBitmap == NULL )
540 papsBuckets[iBucket - nRem].u.pabyBitmap = (GByte*)VSICalloc(1, PAGE_SIZE);
541 if( papsBuckets[iBucket - nRem].u.pabyBitmap == NULL )
542 {
543 papsBuckets[iBucket].u.pabyBitmap = NULL;
544 bOOM = TRUE;
545 }
546 else
547 papsBuckets[iBucket].u.pabyBitmap = papsBuckets[iBucket - nRem].u.pabyBitmap + nRem * BUCKET_BITMAP_SIZE;
548 }
549
550 if( bOOM )
551 {
552 CPLError(CE_Failure, CPLE_AppDefined, "AllocBucket() failed. Use OSM_USE_CUSTOM_INDEXING=NO");
553 bStopParsing = TRUE;
554 return FALSE;
555 }
556
557 return TRUE;
558 }
559
560 /************************************************************************/
561 /* AllocMoreBuckets() */
562 /************************************************************************/
563
AllocMoreBuckets(int nNewBucketIdx,int bAllocBucket)564 int OGROSMDataSource::AllocMoreBuckets(int nNewBucketIdx, int bAllocBucket)
565 {
566 CPLAssert(nNewBucketIdx >= nBuckets);
567
568 int nNewBuckets = MAX(nBuckets + nBuckets / 2, nNewBucketIdx);
569
570 size_t nNewSize = sizeof(Bucket) * nNewBuckets;
571 if( (GUIntBig)nNewSize != sizeof(Bucket) * (GUIntBig)nNewBuckets )
572 {
573 CPLError(CE_Failure, CPLE_AppDefined, "AllocMoreBuckets() failed. Use OSM_USE_CUSTOM_INDEXING=NO");
574 bStopParsing = TRUE;
575 return FALSE;
576 }
577
578 Bucket* papsNewBuckets = (Bucket*) VSIRealloc(papsBuckets, nNewSize);
579 if( papsNewBuckets == NULL )
580 {
581 CPLError(CE_Failure, CPLE_AppDefined, "AllocMoreBuckets() failed. Use OSM_USE_CUSTOM_INDEXING=NO");
582 bStopParsing = TRUE;
583 return FALSE;
584 }
585 papsBuckets = papsNewBuckets;
586
587 int bOOM = FALSE;
588 int i;
589 for(i = nBuckets; i < nNewBuckets && !bOOM; i++)
590 {
591 papsBuckets[i].nOff = -1;
592 if( bAllocBucket )
593 {
594 if( !AllocBucket(i) )
595 bOOM = TRUE;
596 }
597 else
598 {
599 if( bCompressNodes )
600 papsBuckets[i].u.panSectorSize = NULL;
601 else
602 papsBuckets[i].u.pabyBitmap = NULL;
603 }
604 }
605 nBuckets = i;
606
607 if( bOOM )
608 {
609 CPLError(CE_Failure, CPLE_AppDefined, "AllocMoreBuckets() failed. Use OSM_USE_CUSTOM_INDEXING=NO");
610 bStopParsing = TRUE;
611 return FALSE;
612 }
613
614 return TRUE;
615 }
616
617 /************************************************************************/
618 /* FlushCurrentSectorCompressedCase() */
619 /************************************************************************/
620
FlushCurrentSectorCompressedCase()621 int OGROSMDataSource::FlushCurrentSectorCompressedCase()
622 {
623 GByte abyOutBuffer[2 * SECTOR_SIZE];
624 GByte* pabyOut = abyOutBuffer;
625 LonLat* pasLonLatIn = (LonLat*)pabySector;
626 int nLastLon = 0, nLastLat = 0;
627 int bLastValid = FALSE;
628 int i;
629
630 CPLAssert((NODE_PER_SECTOR % 8) == 0);
631 memset(abyOutBuffer, 0, NODE_PER_SECTOR / 8);
632 pabyOut += NODE_PER_SECTOR / 8;
633 for(i = 0; i < NODE_PER_SECTOR; i++)
634 {
635 if( pasLonLatIn[i].nLon || pasLonLatIn[i].nLat )
636 {
637 abyOutBuffer[i >> 3] |= (1 << (i % 8));
638 if( bLastValid )
639 {
640 GIntBig nDiff64Lon = (GIntBig)pasLonLatIn[i].nLon - (GIntBig)nLastLon;
641 GIntBig nDiff64Lat = pasLonLatIn[i].nLat - nLastLat;
642 WriteVarSInt64(nDiff64Lon, &pabyOut);
643 WriteVarSInt64(nDiff64Lat, &pabyOut);
644 }
645 else
646 {
647 memcpy(pabyOut, &pasLonLatIn[i], sizeof(LonLat));
648 pabyOut += sizeof(LonLat);
649 }
650 bLastValid = TRUE;
651
652 nLastLon = pasLonLatIn[i].nLon;
653 nLastLat = pasLonLatIn[i].nLat;
654 }
655 }
656
657 size_t nCompressSize = (size_t)(pabyOut - abyOutBuffer);
658 CPLAssert(nCompressSize < sizeof(abyOutBuffer) - 1);
659 abyOutBuffer[nCompressSize] = 0;
660
661 nCompressSize = ROUND_COMPRESS_SIZE(nCompressSize);
662 GByte* pabyToWrite;
663 if(nCompressSize >= SECTOR_SIZE)
664 {
665 nCompressSize = SECTOR_SIZE;
666 pabyToWrite = pabySector;
667 }
668 else
669 pabyToWrite = abyOutBuffer;
670
671 if( VSIFWriteL(pabyToWrite, 1, nCompressSize, fpNodes) == nCompressSize )
672 {
673 memset(pabySector, 0, SECTOR_SIZE);
674 nNodesFileSize += nCompressSize;
675
676 if( nBucketOld >= nBuckets )
677 {
678 if( !AllocMoreBuckets(nBucketOld + 1) )
679 return FALSE;
680 }
681 Bucket* psBucket = &papsBuckets[nBucketOld];
682 if( psBucket->u.panSectorSize == NULL && !AllocBucket(nBucketOld) )
683 return FALSE;
684 psBucket->u.panSectorSize[nOffInBucketReducedOld] =
685 COMPRESS_SIZE_TO_BYTE(nCompressSize);
686
687 return TRUE;
688 }
689 else
690 {
691 CPLError(CE_Failure, CPLE_AppDefined,
692 "Cannot write in temporary node file %s : %s",
693 osNodesFilename.c_str(), VSIStrerror(errno));
694 }
695
696 return FALSE;
697 }
698
699 /************************************************************************/
700 /* FlushCurrentSectorNonCompressedCase() */
701 /************************************************************************/
702
FlushCurrentSectorNonCompressedCase()703 int OGROSMDataSource::FlushCurrentSectorNonCompressedCase()
704 {
705 if( VSIFWriteL(pabySector, 1, SECTOR_SIZE, fpNodes) == SECTOR_SIZE )
706 {
707 memset(pabySector, 0, SECTOR_SIZE);
708 nNodesFileSize += SECTOR_SIZE;
709 return TRUE;
710 }
711 else
712 {
713 CPLError(CE_Failure, CPLE_AppDefined,
714 "Cannot write in temporary node file %s : %s",
715 osNodesFilename.c_str(), VSIStrerror(errno));
716 }
717
718 return FALSE;
719 }
720
721 /************************************************************************/
722 /* IndexPointCustom() */
723 /************************************************************************/
724
IndexPointCustom(OSMNode * psNode)725 int OGROSMDataSource::IndexPointCustom(OSMNode* psNode)
726 {
727 if( psNode->nID <= nPrevNodeId)
728 {
729 CPLError(CE_Failure, CPLE_AppDefined,
730 "Non increasing node id. Use OSM_USE_CUSTOM_INDEXING=NO");
731 bStopParsing = TRUE;
732 return FALSE;
733 }
734 if( !VALID_ID_FOR_CUSTOM_INDEXING(psNode->nID) )
735 {
736 CPLError(CE_Failure, CPLE_AppDefined,
737 "Unsupported node id value (" CPL_FRMT_GIB "). Use OSM_USE_CUSTOM_INDEXING=NO",
738 psNode->nID);
739 bStopParsing = TRUE;
740 return FALSE;
741 }
742
743 int nBucket = (int)(psNode->nID / NODE_PER_BUCKET);
744 int nOffInBucket = psNode->nID % NODE_PER_BUCKET;
745 int nOffInBucketReduced = nOffInBucket >> NODE_PER_SECTOR_SHIFT;
746 int nOffInBucketReducedRemainer = nOffInBucket & ((1 << NODE_PER_SECTOR_SHIFT) - 1);
747
748 if( nBucket >= nBuckets )
749 {
750 if( !AllocMoreBuckets(nBucket + 1) )
751 return FALSE;
752 }
753 Bucket* psBucket = &papsBuckets[nBucket];
754
755 if( !bCompressNodes )
756 {
757 int nBitmapIndex = nOffInBucketReduced / 8;
758 int nBitmapRemainer = nOffInBucketReduced % 8;
759 if( psBucket->u.pabyBitmap == NULL && !AllocBucket(nBucket) )
760 return FALSE;
761 psBucket->u.pabyBitmap[nBitmapIndex] |= (1 << nBitmapRemainer);
762 }
763
764 if( nBucket != nBucketOld )
765 {
766 CPLAssert(nBucket > nBucketOld);
767 if( nBucketOld >= 0 )
768 {
769 if( !FlushCurrentSector() )
770 {
771 bStopParsing = TRUE;
772 return FALSE;
773 }
774 }
775 nBucketOld = nBucket;
776 nOffInBucketReducedOld = nOffInBucketReduced;
777 CPLAssert(psBucket->nOff == -1);
778 psBucket->nOff = VSIFTellL(fpNodes);
779 }
780 else if( nOffInBucketReduced != nOffInBucketReducedOld )
781 {
782 CPLAssert(nOffInBucketReduced > nOffInBucketReducedOld);
783 if( !FlushCurrentSector() )
784 {
785 bStopParsing = TRUE;
786 return FALSE;
787 }
788 nOffInBucketReducedOld = nOffInBucketReduced;
789 }
790
791 LonLat* psLonLat = (LonLat*)(pabySector + sizeof(LonLat) * nOffInBucketReducedRemainer);
792 psLonLat->nLon = DBL_TO_INT(psNode->dfLon);
793 psLonLat->nLat = DBL_TO_INT(psNode->dfLat);
794
795 nPrevNodeId = psNode->nID;
796
797 return TRUE;
798 }
799
800 /************************************************************************/
801 /* NotifyNodes() */
802 /************************************************************************/
803
NotifyNodes(unsigned int nNodes,OSMNode * pasNodes)804 void OGROSMDataSource::NotifyNodes(unsigned int nNodes, OSMNode* pasNodes)
805 {
806 unsigned int i;
807
808 const OGREnvelope* psEnvelope =
809 papoLayers[IDX_LYR_POINTS]->GetSpatialFilterEnvelope();
810
811 for(i = 0; i < nNodes; i++)
812 {
813 /* If the point doesn't fit into the envelope of the spatial filter */
814 /* then skip it */
815 if( psEnvelope != NULL &&
816 !(pasNodes[i].dfLon >= psEnvelope->MinX &&
817 pasNodes[i].dfLon <= psEnvelope->MaxX &&
818 pasNodes[i].dfLat >= psEnvelope->MinY &&
819 pasNodes[i].dfLat <= psEnvelope->MaxY) )
820 continue;
821
822 if( !IndexPoint(&pasNodes[i]) )
823 break;
824
825 if( !papoLayers[IDX_LYR_POINTS]->IsUserInterested() )
826 continue;
827
828 unsigned int j;
829 int bInterestingTag = bReportAllNodes;
830 OSMTag* pasTags = pasNodes[i].pasTags;
831
832 if( !bReportAllNodes )
833 {
834 for(j = 0; j < pasNodes[i].nTags; j++)
835 {
836 const char* pszK = pasTags[j].pszK;
837 if( papoLayers[IDX_LYR_POINTS]->IsSignificantKey(pszK) )
838 {
839 bInterestingTag = TRUE;
840 break;
841 }
842 }
843 }
844
845 if( bInterestingTag )
846 {
847 OGRFeature* poFeature = new OGRFeature(
848 papoLayers[IDX_LYR_POINTS]->GetLayerDefn());
849
850 poFeature->SetGeometryDirectly(
851 new OGRPoint(pasNodes[i].dfLon, pasNodes[i].dfLat));
852
853 papoLayers[IDX_LYR_POINTS]->SetFieldsFromTags(
854 poFeature, pasNodes[i].nID, FALSE, pasNodes[i].nTags, pasTags, &pasNodes[i].sInfo );
855
856 int bFilteredOut = FALSE;
857 if( !papoLayers[IDX_LYR_POINTS]->AddFeature(poFeature, FALSE,
858 &bFilteredOut,
859 !bFeatureAdded) )
860 {
861 bStopParsing = TRUE;
862 break;
863 }
864 else if (!bFilteredOut)
865 bFeatureAdded = TRUE;
866 }
867 }
868 }
869
OGROSMNotifyNodes(unsigned int nNodes,OSMNode * pasNodes,CPL_UNUSED OSMContext * psOSMContext,void * user_data)870 static void OGROSMNotifyNodes (unsigned int nNodes,
871 OSMNode* pasNodes,
872 CPL_UNUSED OSMContext* psOSMContext,
873 void* user_data)
874 {
875 ((OGROSMDataSource*) user_data)->NotifyNodes(nNodes, pasNodes);
876 }
877
878 /************************************************************************/
879 /* LookupNodes() */
880 /************************************************************************/
881
882 //#define DEBUG_COLLISIONS 1
883
LookupNodes()884 void OGROSMDataSource::LookupNodes( )
885 {
886 if( bCustomIndexing )
887 LookupNodesCustom();
888 else
889 LookupNodesSQLite();
890
891 #ifdef ENABLE_NODE_LOOKUP_BY_HASHING
892 if( nReqIds > 1 && bEnableHashedIndex )
893 {
894 memset(panHashedIndexes, 0xFF, HASHED_INDEXES_ARRAY_SIZE * sizeof(int));
895 bHashedIndexValid = TRUE;
896 #ifdef DEBUG_COLLISIONS
897 int nCollisions = 0;
898 #endif
899 int iNextFreeBucket = 0;
900 for(unsigned int i = 0; i < nReqIds; i++)
901 {
902 int nIndInHashArray = HASH_ID_FUNC(panReqIds[i]) % HASHED_INDEXES_ARRAY_SIZE;
903 int nIdx = panHashedIndexes[nIndInHashArray];
904 if( nIdx == -1 )
905 {
906 panHashedIndexes[nIndInHashArray] = i;
907 }
908 else
909 {
910 #ifdef DEBUG_COLLISIONS
911 nCollisions ++;
912 #endif
913 int iBucket;
914 if( nIdx >= 0 )
915 {
916 if(iNextFreeBucket == COLLISION_BUCKET_ARRAY_SIZE)
917 {
918 CPLDebug("OSM", "Too many collisions. Disabling hashed indexing");
919 bHashedIndexValid = FALSE;
920 bEnableHashedIndex = FALSE;
921 break;
922 }
923 iBucket = iNextFreeBucket;
924 psCollisionBuckets[iNextFreeBucket].nInd = nIdx;
925 psCollisionBuckets[iNextFreeBucket].nNext = -1;
926 panHashedIndexes[nIndInHashArray] = -iNextFreeBucket - 2;
927 iNextFreeBucket ++;
928 }
929 else
930 iBucket = -nIdx - 2;
931 if(iNextFreeBucket == COLLISION_BUCKET_ARRAY_SIZE)
932 {
933 CPLDebug("OSM", "Too many collisions. Disabling hashed indexing");
934 bHashedIndexValid = FALSE;
935 bEnableHashedIndex = FALSE;
936 break;
937 }
938 while( TRUE )
939 {
940 int iNext = psCollisionBuckets[iBucket].nNext;
941 if( iNext < 0 )
942 {
943 psCollisionBuckets[iBucket].nNext = iNextFreeBucket;
944 psCollisionBuckets[iNextFreeBucket].nInd = i;
945 psCollisionBuckets[iNextFreeBucket].nNext = -1;
946 iNextFreeBucket ++;
947 break;
948 }
949 iBucket = iNext;
950 }
951 }
952 }
953 #ifdef DEBUG_COLLISIONS
954 /* Collision rate in practice is around 12% on France, Germany, ... */
955 /* Maximum seen ~ 15.9% on a planet file but often much smaller. */
956 CPLDebug("OSM", "nCollisions = %d/%d (%.1f %%), iNextFreeBucket = %d/%d",
957 nCollisions, nReqIds, nCollisions * 100.0 / nReqIds,
958 iNextFreeBucket, COLLISION_BUCKET_ARRAY_SIZE);
959 #endif
960 }
961 else
962 bHashedIndexValid = FALSE;
963 #endif // ENABLE_NODE_LOOKUP_BY_HASHING
964 }
965
966 /************************************************************************/
967 /* LookupNodesSQLite() */
968 /************************************************************************/
969
LookupNodesSQLite()970 void OGROSMDataSource::LookupNodesSQLite( )
971 {
972 unsigned int iCur;
973 unsigned int i;
974
975 CPLAssert(nUnsortedReqIds <= MAX_ACCUMULATED_NODES);
976
977 nReqIds = 0;
978 for(i = 0; i < nUnsortedReqIds; i++)
979 {
980 GIntBig id = panUnsortedReqIds[i];
981 panReqIds[nReqIds++] = id;
982 }
983
984 std::sort(panReqIds, panReqIds + nReqIds);
985
986 /* Remove duplicates */
987 unsigned int j = 0;
988 for(i = 0; i < nReqIds; i++)
989 {
990 if (!(i > 0 && panReqIds[i] == panReqIds[i-1]))
991 panReqIds[j++] = panReqIds[i];
992 }
993 nReqIds = j;
994
995 iCur = 0;
996 j = 0;
997 while( iCur < nReqIds )
998 {
999 unsigned int nToQuery = nReqIds - iCur;
1000 if( nToQuery > LIMIT_IDS_PER_REQUEST )
1001 nToQuery = LIMIT_IDS_PER_REQUEST;
1002
1003 sqlite3_stmt* hStmt = pahSelectNodeStmt[nToQuery-1];
1004 for(i=iCur;i<iCur + nToQuery;i++)
1005 {
1006 sqlite3_bind_int64( hStmt, i - iCur +1, panReqIds[i] );
1007 }
1008 iCur += nToQuery;
1009
1010 while( sqlite3_step(hStmt) == SQLITE_ROW )
1011 {
1012 GIntBig id = sqlite3_column_int64(hStmt, 0);
1013 LonLat* psLonLat = (LonLat*)sqlite3_column_blob(hStmt, 1);
1014
1015 panReqIds[j] = id;
1016 pasLonLatArray[j].nLon = psLonLat->nLon;
1017 pasLonLatArray[j].nLat = psLonLat->nLat;
1018 j++;
1019 }
1020
1021 sqlite3_reset(hStmt);
1022 }
1023 nReqIds = j;
1024 }
1025
1026 /************************************************************************/
1027 /* ReadVarSInt64() */
1028 /************************************************************************/
1029
ReadVarSInt64(GByte ** ppabyPtr)1030 static GIntBig ReadVarSInt64(GByte** ppabyPtr)
1031 {
1032 GIntBig nSVal64 = ReadVarInt64(ppabyPtr);
1033 GIntBig nDiff64 = ((nSVal64 & 1) == 0) ? (((GUIntBig)nSVal64) >> 1) : -(((GUIntBig)nSVal64) >> 1)-1;
1034 return nDiff64;
1035 }
1036
1037 /************************************************************************/
1038 /* DecompressSector() */
1039 /************************************************************************/
1040
DecompressSector(GByte * pabyIn,int nSectorSize,GByte * pabyOut)1041 static int DecompressSector(GByte* pabyIn, int nSectorSize, GByte* pabyOut)
1042 {
1043 GByte* pabyPtr = pabyIn;
1044 LonLat* pasLonLatOut = (LonLat*) pabyOut;
1045 int nLastLon = 0, nLastLat = 0;
1046 int bLastValid = FALSE;
1047 int i;
1048
1049 pabyPtr += NODE_PER_SECTOR / 8;
1050 for(i = 0; i < NODE_PER_SECTOR; i++)
1051 {
1052 if( pabyIn[i >> 3] & (1 << (i % 8)) )
1053 {
1054 if( bLastValid )
1055 {
1056 pasLonLatOut[i].nLon = (int)(nLastLon + ReadVarSInt64(&pabyPtr));
1057 pasLonLatOut[i].nLat = (int)(nLastLat + ReadVarSInt64(&pabyPtr));
1058 }
1059 else
1060 {
1061 bLastValid = TRUE;
1062 memcpy(&(pasLonLatOut[i]), pabyPtr, sizeof(LonLat));
1063 pabyPtr += sizeof(LonLat);
1064 }
1065
1066 nLastLon = pasLonLatOut[i].nLon;
1067 nLastLat = pasLonLatOut[i].nLat;
1068 }
1069 else
1070 {
1071 pasLonLatOut[i].nLon = 0;
1072 pasLonLatOut[i].nLat = 0;
1073 }
1074 }
1075
1076 int nRead = (int)(pabyPtr - pabyIn);
1077 nRead = ROUND_COMPRESS_SIZE(nRead);
1078 return( nRead == nSectorSize );
1079 }
1080
1081 /************************************************************************/
1082 /* LookupNodesCustom() */
1083 /************************************************************************/
1084
LookupNodesCustom()1085 void OGROSMDataSource::LookupNodesCustom( )
1086 {
1087 nReqIds = 0;
1088
1089 if( nBucketOld >= 0 )
1090 {
1091 if( !FlushCurrentSector() )
1092 {
1093 bStopParsing = TRUE;
1094 return;
1095 }
1096
1097 nBucketOld = -1;
1098 }
1099
1100 unsigned int i;
1101
1102 CPLAssert(nUnsortedReqIds <= MAX_ACCUMULATED_NODES);
1103
1104 for(i = 0; i < nUnsortedReqIds; i++)
1105 {
1106 GIntBig id = panUnsortedReqIds[i];
1107
1108 if( !VALID_ID_FOR_CUSTOM_INDEXING(id) )
1109 continue;
1110
1111 int nBucket = (int)(id / NODE_PER_BUCKET);
1112 int nOffInBucket = id % NODE_PER_BUCKET;
1113 int nOffInBucketReduced = nOffInBucket >> NODE_PER_SECTOR_SHIFT;
1114
1115 if( nBucket >= nBuckets )
1116 continue;
1117 Bucket* psBucket = &papsBuckets[nBucket];
1118
1119 if( bCompressNodes )
1120 {
1121 if( psBucket->u.panSectorSize == NULL ||
1122 !(psBucket->u.panSectorSize[nOffInBucketReduced]) )
1123 continue;
1124 }
1125 else
1126 {
1127 int nBitmapIndex = nOffInBucketReduced / 8;
1128 int nBitmapRemainer = nOffInBucketReduced % 8;
1129 if( psBucket->u.pabyBitmap == NULL ||
1130 !(psBucket->u.pabyBitmap[nBitmapIndex] & (1 << nBitmapRemainer)) )
1131 continue;
1132 }
1133
1134 panReqIds[nReqIds++] = id;
1135 }
1136
1137 std::sort(panReqIds, panReqIds + nReqIds);
1138
1139 /* Remove duplicates */
1140 unsigned int j = 0;
1141 for(i = 0; i < nReqIds; i++)
1142 {
1143 if (!(i > 0 && panReqIds[i] == panReqIds[i-1]))
1144 panReqIds[j++] = panReqIds[i];
1145 }
1146 nReqIds = j;
1147
1148 #ifdef FAKE_LOOKUP_NODES
1149 for(i = 0; i < nReqIds; i++)
1150 {
1151 pasLonLatArray[i].nLon = 0;
1152 pasLonLatArray[i].nLat = 0;
1153 }
1154 #else
1155 if( bCompressNodes )
1156 LookupNodesCustomCompressedCase();
1157 else
1158 LookupNodesCustomNonCompressedCase();
1159 #endif
1160 }
1161
1162 /************************************************************************/
1163 /* LookupNodesCustomCompressedCase() */
1164 /************************************************************************/
1165
LookupNodesCustomCompressedCase()1166 void OGROSMDataSource::LookupNodesCustomCompressedCase()
1167 {
1168 unsigned int i;
1169 unsigned int j = 0;
1170 #define SECURITY_MARGIN (8 + 8 + 2 * NODE_PER_SECTOR)
1171 GByte abyRawSector[SECTOR_SIZE + SECURITY_MARGIN];
1172 memset(abyRawSector + SECTOR_SIZE, 0, SECURITY_MARGIN);
1173
1174 int nBucketOld = -1;
1175 int nOffInBucketReducedOld = -1;
1176 int k = 0;
1177 int nOffFromBucketStart = 0;
1178
1179 for(i = 0; i < nReqIds; i++)
1180 {
1181 GIntBig id = panReqIds[i];
1182
1183 int nBucket = (int)(id / NODE_PER_BUCKET);
1184 int nOffInBucket = id % NODE_PER_BUCKET;
1185 int nOffInBucketReduced = nOffInBucket >> NODE_PER_SECTOR_SHIFT;
1186 int nOffInBucketReducedRemainer = nOffInBucket & ((1 << NODE_PER_SECTOR_SHIFT) - 1);
1187
1188 if( nBucket != nBucketOld )
1189 {
1190 nOffInBucketReducedOld = -1;
1191 k = 0;
1192 nOffFromBucketStart = 0;
1193 }
1194
1195 if ( nOffInBucketReduced != nOffInBucketReducedOld )
1196 {
1197 if( nBucket >= nBuckets )
1198 {
1199 CPLError(CE_Failure, CPLE_AppDefined,
1200 "Cannot read node " CPL_FRMT_GIB, id);
1201 continue;
1202 // FIXME ?
1203 }
1204 Bucket* psBucket = &papsBuckets[nBucket];
1205 if( psBucket->u.panSectorSize == NULL )
1206 {
1207 CPLError(CE_Failure, CPLE_AppDefined,
1208 "Cannot read node " CPL_FRMT_GIB, id);
1209 continue;
1210 // FIXME ?
1211 }
1212 int nSectorSize = COMPRESS_SIZE_FROM_BYTE(psBucket->u.panSectorSize[nOffInBucketReduced]);
1213
1214 /* If we stay in the same bucket, we can reuse the previously */
1215 /* computed offset, instead of starting from bucket start */
1216 for(; k < nOffInBucketReduced; k++)
1217 {
1218 if( psBucket->u.panSectorSize[k] )
1219 nOffFromBucketStart += COMPRESS_SIZE_FROM_BYTE(psBucket->u.panSectorSize[k]);
1220 }
1221
1222 VSIFSeekL(fpNodes, psBucket->nOff + nOffFromBucketStart, SEEK_SET);
1223 if( nSectorSize == SECTOR_SIZE )
1224 {
1225 if( VSIFReadL(pabySector, 1, SECTOR_SIZE, fpNodes) != SECTOR_SIZE )
1226 {
1227 CPLError(CE_Failure, CPLE_AppDefined,
1228 "Cannot read node " CPL_FRMT_GIB, id);
1229 continue;
1230 // FIXME ?
1231 }
1232 }
1233 else
1234 {
1235 if( (int)VSIFReadL(abyRawSector, 1, nSectorSize, fpNodes) != nSectorSize )
1236 {
1237 CPLError(CE_Failure, CPLE_AppDefined,
1238 "Cannot read sector for node " CPL_FRMT_GIB, id);
1239 continue;
1240 // FIXME ?
1241 }
1242 abyRawSector[nSectorSize] = 0;
1243
1244 if( !DecompressSector(abyRawSector, nSectorSize, pabySector) )
1245 {
1246 CPLError(CE_Failure, CPLE_AppDefined,
1247 "Error while uncompressing sector for node " CPL_FRMT_GIB, id);
1248 continue;
1249 // FIXME ?
1250 }
1251 }
1252
1253 nBucketOld = nBucket;
1254 nOffInBucketReducedOld = nOffInBucketReduced;
1255 }
1256
1257 panReqIds[j] = id;
1258 memcpy(pasLonLatArray + j,
1259 pabySector + nOffInBucketReducedRemainer * sizeof(LonLat),
1260 sizeof(LonLat));
1261
1262 if( pasLonLatArray[j].nLon || pasLonLatArray[j].nLat )
1263 j++;
1264 }
1265 nReqIds = j;
1266 }
1267
1268 /************************************************************************/
1269 /* LookupNodesCustomNonCompressedCase() */
1270 /************************************************************************/
1271
LookupNodesCustomNonCompressedCase()1272 void OGROSMDataSource::LookupNodesCustomNonCompressedCase()
1273 {
1274 unsigned int i;
1275 unsigned int j = 0;
1276
1277 for(i = 0; i < nReqIds; i++)
1278 {
1279 GIntBig id = panReqIds[i];
1280
1281 int nBucket = (int)(id / NODE_PER_BUCKET);
1282 int nOffInBucket = id % NODE_PER_BUCKET;
1283 int nOffInBucketReduced = nOffInBucket >> NODE_PER_SECTOR_SHIFT;
1284 int nOffInBucketReducedRemainer = nOffInBucket & ((1 << NODE_PER_SECTOR_SHIFT) - 1);
1285
1286 int nBitmapIndex = nOffInBucketReduced / 8;
1287 int nBitmapRemainer = nOffInBucketReduced % 8;
1288
1289 if( nBucket >= nBuckets )
1290 {
1291 CPLError(CE_Failure, CPLE_AppDefined,
1292 "Cannot read node " CPL_FRMT_GIB, id);
1293 continue;
1294 // FIXME ?
1295 }
1296 Bucket* psBucket = &papsBuckets[nBucket];
1297 if( psBucket->u.pabyBitmap == NULL )
1298 {
1299 CPLError(CE_Failure, CPLE_AppDefined,
1300 "Cannot read node " CPL_FRMT_GIB, id);
1301 continue;
1302 // FIXME ?
1303 }
1304
1305 int k;
1306 int nSector = 0;
1307 for(k = 0; k < nBitmapIndex; k++)
1308 nSector += abyBitsCount[psBucket->u.pabyBitmap[k]];
1309 if (nBitmapRemainer)
1310 nSector += abyBitsCount[psBucket->u.pabyBitmap[nBitmapIndex] & ((1 << nBitmapRemainer) - 1)];
1311
1312 VSIFSeekL(fpNodes, psBucket->nOff + nSector * SECTOR_SIZE + nOffInBucketReducedRemainer * sizeof(LonLat), SEEK_SET);
1313 if( VSIFReadL(pasLonLatArray + j, 1, sizeof(LonLat), fpNodes) != sizeof(LonLat) )
1314 {
1315 CPLError(CE_Failure, CPLE_AppDefined,
1316 "Cannot read node " CPL_FRMT_GIB, id);
1317 // FIXME ?
1318 }
1319 else
1320 {
1321 panReqIds[j] = id;
1322 if( pasLonLatArray[j].nLon || pasLonLatArray[j].nLat )
1323 j++;
1324 }
1325 }
1326 nReqIds = j;
1327 }
1328
1329 /************************************************************************/
1330 /* WriteVarInt() */
1331 /************************************************************************/
1332
WriteVarInt(unsigned int nVal,GByte ** ppabyData)1333 static void WriteVarInt(unsigned int nVal, GByte** ppabyData)
1334 {
1335 GByte* pabyData = *ppabyData;
1336 while(TRUE)
1337 {
1338 if( (nVal & (~0x7f)) == 0 )
1339 {
1340 *pabyData = (GByte)nVal;
1341 *ppabyData = pabyData + 1;
1342 return;
1343 }
1344
1345 *pabyData = 0x80 | (GByte)(nVal & 0x7f);
1346 nVal >>= 7;
1347 pabyData ++;
1348 }
1349 }
1350
1351 /************************************************************************/
1352 /* WriteVarInt64() */
1353 /************************************************************************/
1354
WriteVarInt64(GUIntBig nVal,GByte ** ppabyData)1355 static void WriteVarInt64(GUIntBig nVal, GByte** ppabyData)
1356 {
1357 GByte* pabyData = *ppabyData;
1358 while(TRUE)
1359 {
1360 if( (nVal & (~0x7f)) == 0 )
1361 {
1362 *pabyData = (GByte)nVal;
1363 *ppabyData = pabyData + 1;
1364 return;
1365 }
1366
1367 *pabyData = 0x80 | (GByte)(nVal & 0x7f);
1368 nVal >>= 7;
1369 pabyData ++;
1370 }
1371 }
1372
1373 /************************************************************************/
1374 /* WriteVarSInt64() */
1375 /************************************************************************/
1376
WriteVarSInt64(GIntBig nSVal,GByte ** ppabyData)1377 static void WriteVarSInt64(GIntBig nSVal, GByte** ppabyData)
1378 {
1379 GIntBig nVal;
1380 if( nSVal >= 0 )
1381 nVal = nSVal << 1;
1382 else
1383 nVal = ((-1-nSVal) << 1) + 1;
1384
1385 GByte* pabyData = *ppabyData;
1386 while(TRUE)
1387 {
1388 if( (nVal & (~0x7f)) == 0 )
1389 {
1390 *pabyData = (GByte)nVal;
1391 *ppabyData = pabyData + 1;
1392 return;
1393 }
1394
1395 *pabyData = 0x80 | (GByte)(nVal & 0x7f);
1396 nVal >>= 7;
1397 pabyData ++;
1398 }
1399 }
1400
1401 /************************************************************************/
1402 /* CompressWay() */
1403 /************************************************************************/
1404
CompressWay(unsigned int nTags,IndexedKVP * pasTags,int nPoints,LonLat * pasLonLatPairs,OSMInfo * psInfo,GByte * pabyCompressedWay)1405 int OGROSMDataSource::CompressWay ( unsigned int nTags, IndexedKVP* pasTags,
1406 int nPoints, LonLat* pasLonLatPairs,
1407 OSMInfo* psInfo,
1408 GByte* pabyCompressedWay )
1409 {
1410 GByte* pabyPtr = pabyCompressedWay;
1411 pabyPtr ++;
1412
1413 int nTagCount = 0;
1414 CPLAssert(nTags < MAX_COUNT_FOR_TAGS_IN_WAY);
1415 for(unsigned int iTag = 0; iTag < nTags; iTag++)
1416 {
1417 if ((int)(pabyPtr - pabyCompressedWay) + 2 >= MAX_SIZE_FOR_TAGS_IN_WAY)
1418 {
1419 break;
1420 }
1421
1422 WriteVarInt(pasTags[iTag].nKeyIndex, &pabyPtr);
1423
1424 /* to fit in 2 bytes, the theoretical limit would be 127 * 128 + 127 */
1425 if( pasTags[iTag].bVIsIndex )
1426 {
1427 if ((int)(pabyPtr - pabyCompressedWay) + 2 >= MAX_SIZE_FOR_TAGS_IN_WAY)
1428 {
1429 break;
1430 }
1431
1432 WriteVarInt(pasTags[iTag].u.nValueIndex, &pabyPtr);
1433 }
1434 else
1435 {
1436 const char* pszV = (const char*)pabyNonRedundantValues +
1437 pasTags[iTag].u.nOffsetInpabyNonRedundantValues;
1438
1439 int nLenV = strlen(pszV) + 1;
1440 if ((int)(pabyPtr - pabyCompressedWay) + 2 + nLenV >= MAX_SIZE_FOR_TAGS_IN_WAY)
1441 {
1442 break;
1443 }
1444
1445 WriteVarInt(0, &pabyPtr);
1446
1447 memcpy(pabyPtr, pszV, nLenV);
1448 pabyPtr += nLenV;
1449 }
1450
1451 nTagCount ++;
1452 }
1453
1454 pabyCompressedWay[0] = (GByte) nTagCount;
1455
1456 if( bNeedsToSaveWayInfo )
1457 {
1458 if( psInfo != NULL )
1459 {
1460 *pabyPtr = 1;
1461 pabyPtr ++;
1462
1463 WriteVarInt64(psInfo->ts.nTimeStamp, &pabyPtr);
1464 WriteVarInt64(psInfo->nChangeset, &pabyPtr);
1465 WriteVarInt(psInfo->nVersion, &pabyPtr);
1466 WriteVarInt(psInfo->nUID, &pabyPtr);
1467 // FIXME : do something with pszUserSID
1468 }
1469 else
1470 {
1471 *pabyPtr = 0;
1472 pabyPtr ++;
1473 }
1474 }
1475
1476 memcpy(pabyPtr, &(pasLonLatPairs[0]), sizeof(LonLat));
1477 pabyPtr += sizeof(LonLat);
1478 for(int i=1;i<nPoints;i++)
1479 {
1480 GIntBig nDiff64;
1481
1482 nDiff64 = (GIntBig)pasLonLatPairs[i].nLon - (GIntBig)pasLonLatPairs[i-1].nLon;
1483 WriteVarSInt64(nDiff64, &pabyPtr);
1484
1485 nDiff64 = pasLonLatPairs[i].nLat - pasLonLatPairs[i-1].nLat;
1486 WriteVarSInt64(nDiff64, &pabyPtr);
1487 }
1488 int nBufferSize = (int)(pabyPtr - pabyCompressedWay);
1489 return nBufferSize;
1490 }
1491
1492 /************************************************************************/
1493 /* UncompressWay() */
1494 /************************************************************************/
1495
UncompressWay(int nBytes,GByte * pabyCompressedWay,LonLat * pasCoords,unsigned int * pnTags,OSMTag * pasTags,OSMInfo * psInfo)1496 int OGROSMDataSource::UncompressWay( int nBytes, GByte* pabyCompressedWay,
1497 LonLat* pasCoords,
1498 unsigned int* pnTags, OSMTag* pasTags,
1499 OSMInfo* psInfo )
1500 {
1501 GByte* pabyPtr = pabyCompressedWay;
1502 unsigned int nTags = *pabyPtr;
1503 pabyPtr ++;
1504
1505 if (pnTags)
1506 *pnTags = nTags;
1507
1508 /* TODO? : some additional safety checks */
1509 for(unsigned int iTag = 0; iTag < nTags; iTag++)
1510 {
1511 int nK = ReadVarInt32(&pabyPtr);
1512 int nV = ReadVarInt32(&pabyPtr);
1513 GByte* pszV = NULL;
1514 if( nV == 0 )
1515 {
1516 pszV = pabyPtr;
1517 while(*pabyPtr != '\0')
1518 pabyPtr ++;
1519 pabyPtr ++;
1520 }
1521
1522 if( pasTags )
1523 {
1524 CPLAssert(nK >= 0 && nK < (int)asKeys.size());
1525 pasTags[iTag].pszK = asKeys[nK]->pszK;
1526 CPLAssert(nV == 0 || (nV > 0 && nV < (int)asKeys[nK]->asValues.size()));
1527 pasTags[iTag].pszV = nV ? asKeys[nK]->asValues[nV] : (const char*) pszV;
1528 }
1529 }
1530
1531 if( bNeedsToSaveWayInfo )
1532 {
1533 if( *pabyPtr )
1534 {
1535 pabyPtr ++;
1536
1537 OSMInfo sInfo;
1538 if( psInfo == NULL )
1539 psInfo = &sInfo;
1540
1541 psInfo->ts.nTimeStamp = ReadVarInt64(&pabyPtr);
1542 psInfo->nChangeset = ReadVarInt64(&pabyPtr);
1543 psInfo->nVersion = ReadVarInt32(&pabyPtr);
1544 psInfo->nUID = ReadVarInt32(&pabyPtr);
1545
1546 psInfo->bTimeStampIsStr = FALSE;
1547 psInfo->pszUserSID = ""; // FIXME
1548 }
1549 else
1550 pabyPtr ++;
1551 }
1552
1553 memcpy(&pasCoords[0].nLon, pabyPtr, sizeof(int));
1554 memcpy(&pasCoords[0].nLat, pabyPtr + sizeof(int), sizeof(int));
1555 pabyPtr += 2 * sizeof(int);
1556 int nPoints = 1;
1557 do
1558 {
1559 pasCoords[nPoints].nLon = (int)(pasCoords[nPoints-1].nLon + ReadVarSInt64(&pabyPtr));
1560 pasCoords[nPoints].nLat = (int)(pasCoords[nPoints-1].nLat + ReadVarSInt64(&pabyPtr));
1561
1562 nPoints ++;
1563 } while (pabyPtr < pabyCompressedWay + nBytes);
1564
1565 return nPoints;
1566 }
1567
1568 /************************************************************************/
1569 /* IndexWay() */
1570 /************************************************************************/
1571
IndexWay(GIntBig nWayID,unsigned int nTags,IndexedKVP * pasTags,LonLat * pasLonLatPairs,int nPairs,OSMInfo * psInfo)1572 void OGROSMDataSource::IndexWay(GIntBig nWayID,
1573 unsigned int nTags, IndexedKVP* pasTags,
1574 LonLat* pasLonLatPairs, int nPairs,
1575 OSMInfo* psInfo)
1576 {
1577 if( !bIndexWays )
1578 return;
1579
1580 sqlite3_bind_int64( hInsertWayStmt, 1, nWayID );
1581
1582 int nBufferSize = CompressWay (nTags, pasTags, nPairs, pasLonLatPairs, psInfo, pabyWayBuffer);
1583 CPLAssert(nBufferSize <= WAY_BUFFER_SIZE);
1584 sqlite3_bind_blob( hInsertWayStmt, 2, pabyWayBuffer,
1585 nBufferSize, SQLITE_STATIC );
1586
1587 int rc = sqlite3_step( hInsertWayStmt );
1588 sqlite3_reset( hInsertWayStmt );
1589 if( !(rc == SQLITE_OK || rc == SQLITE_DONE) )
1590 {
1591 CPLError(CE_Failure, CPLE_AppDefined,
1592 "Failed inserting way " CPL_FRMT_GIB ": %s",
1593 nWayID, sqlite3_errmsg(hDB));
1594 }
1595 }
1596
1597 /************************************************************************/
1598 /* FindNode() */
1599 /************************************************************************/
1600
FindNode(GIntBig nID)1601 int OGROSMDataSource::FindNode(GIntBig nID)
1602 {
1603 int iFirst = 0;
1604 int iLast = nReqIds - 1;
1605 while(iFirst < iLast)
1606 {
1607 int iMid = (iFirst + iLast) / 2;
1608 if( nID > panReqIds[iMid])
1609 iFirst = iMid + 1;
1610 else
1611 iLast = iMid;
1612 }
1613 if( iFirst == iLast && nID == panReqIds[iFirst] )
1614 return iFirst;
1615 return -1;
1616 }
1617
1618 /************************************************************************/
1619 /* ProcessWaysBatch() */
1620 /************************************************************************/
1621
ProcessWaysBatch()1622 void OGROSMDataSource::ProcessWaysBatch()
1623 {
1624 if( nWayFeaturePairs == 0 ) return;
1625
1626 //printf("nodes = %d, features = %d\n", nUnsortedReqIds, nWayFeaturePairs);
1627 LookupNodes();
1628
1629 int iPair;
1630 for(iPair = 0; iPair < nWayFeaturePairs; iPair ++)
1631 {
1632 WayFeaturePair* psWayFeaturePairs = &pasWayFeaturePairs[iPair];
1633
1634 int bIsArea = psWayFeaturePairs->bIsArea;
1635
1636 unsigned int nFound = 0;
1637 unsigned int i;
1638
1639 #ifdef ENABLE_NODE_LOOKUP_BY_HASHING
1640 if( bHashedIndexValid )
1641 {
1642 for(i=0;i<psWayFeaturePairs->nRefs;i++)
1643 {
1644 int nIndInHashArray =
1645 HASH_ID_FUNC(psWayFeaturePairs->panNodeRefs[i]) %
1646 HASHED_INDEXES_ARRAY_SIZE;
1647 int nIdx = panHashedIndexes[nIndInHashArray];
1648 if( nIdx < -1 )
1649 {
1650 int iBucket = -nIdx - 2;
1651 while( TRUE )
1652 {
1653 nIdx = psCollisionBuckets[iBucket].nInd;
1654 if( panReqIds[nIdx] == psWayFeaturePairs->panNodeRefs[i] )
1655 break;
1656 iBucket = psCollisionBuckets[iBucket].nNext;
1657 if( iBucket < 0 )
1658 {
1659 nIdx = -1;
1660 break;
1661 }
1662 }
1663 }
1664 else if( nIdx >= 0 &&
1665 panReqIds[nIdx] != psWayFeaturePairs->panNodeRefs[i] )
1666 nIdx = -1;
1667
1668 if (nIdx >= 0)
1669 {
1670 pasLonLatCache[nFound].nLon = pasLonLatArray[nIdx].nLon;
1671 pasLonLatCache[nFound].nLat = pasLonLatArray[nIdx].nLat;
1672 nFound ++;
1673 }
1674 }
1675 }
1676 else
1677 #endif // ENABLE_NODE_LOOKUP_BY_HASHING
1678 {
1679 int nIdx = -1;
1680 for(i=0;i<psWayFeaturePairs->nRefs;i++)
1681 {
1682 if( nIdx >= 0 && psWayFeaturePairs->panNodeRefs[i] ==
1683 psWayFeaturePairs->panNodeRefs[i-1] + 1 )
1684 {
1685 if( nIdx+1 < (int)nReqIds && panReqIds[nIdx+1] ==
1686 psWayFeaturePairs->panNodeRefs[i] )
1687 nIdx ++;
1688 else
1689 nIdx = -1;
1690 }
1691 else
1692 nIdx = FindNode( psWayFeaturePairs->panNodeRefs[i] );
1693 if (nIdx >= 0)
1694 {
1695 pasLonLatCache[nFound].nLon = pasLonLatArray[nIdx].nLon;
1696 pasLonLatCache[nFound].nLat = pasLonLatArray[nIdx].nLat;
1697 nFound ++;
1698 }
1699 }
1700 }
1701
1702 if( nFound > 0 && bIsArea )
1703 {
1704 pasLonLatCache[nFound].nLon = pasLonLatCache[0].nLon;
1705 pasLonLatCache[nFound].nLat = pasLonLatCache[0].nLat;
1706 nFound ++;
1707 }
1708
1709 if( nFound < 2 )
1710 {
1711 CPLDebug("OSM", "Way " CPL_FRMT_GIB " with %d nodes that could be found. Discarding it",
1712 psWayFeaturePairs->nWayID, nFound);
1713 delete psWayFeaturePairs->poFeature;
1714 psWayFeaturePairs->poFeature = NULL;
1715 psWayFeaturePairs->bIsArea = FALSE;
1716 continue;
1717 }
1718
1719 if( bIsArea && papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() )
1720 {
1721 IndexWay(psWayFeaturePairs->nWayID,
1722 psWayFeaturePairs->nTags,
1723 psWayFeaturePairs->pasTags,
1724 pasLonLatCache, (int)nFound,
1725 &psWayFeaturePairs->sInfo);
1726 }
1727 else
1728 IndexWay(psWayFeaturePairs->nWayID, 0, NULL,
1729 pasLonLatCache, (int)nFound, NULL);
1730
1731 if( psWayFeaturePairs->poFeature == NULL )
1732 {
1733 continue;
1734 }
1735
1736 OGRLineString* poLS;
1737 OGRGeometry* poGeom;
1738
1739 poLS = new OGRLineString();
1740 poGeom = poLS;
1741
1742 poLS->setNumPoints((int)nFound);
1743 for(i=0;i<nFound;i++)
1744 {
1745 poLS->setPoint(i,
1746 INT_TO_DBL(pasLonLatCache[i].nLon),
1747 INT_TO_DBL(pasLonLatCache[i].nLat));
1748 }
1749
1750 psWayFeaturePairs->poFeature->SetGeometryDirectly(poGeom);
1751
1752 if( nFound != psWayFeaturePairs->nRefs )
1753 CPLDebug("OSM", "For way " CPL_FRMT_GIB ", got only %d nodes instead of %d",
1754 psWayFeaturePairs->nWayID, nFound,
1755 psWayFeaturePairs->nRefs);
1756
1757 int bFilteredOut = FALSE;
1758 if( !papoLayers[IDX_LYR_LINES]->AddFeature(psWayFeaturePairs->poFeature,
1759 psWayFeaturePairs->bAttrFilterAlreadyEvaluated,
1760 &bFilteredOut,
1761 !bFeatureAdded) )
1762 bStopParsing = TRUE;
1763 else if (!bFilteredOut)
1764 bFeatureAdded = TRUE;
1765 }
1766
1767 if( papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() )
1768 {
1769 for(iPair = 0; iPair < nWayFeaturePairs; iPair ++)
1770 {
1771 WayFeaturePair* psWayFeaturePairs = &pasWayFeaturePairs[iPair];
1772
1773 if( psWayFeaturePairs->bIsArea &&
1774 (psWayFeaturePairs->nTags || bReportAllWays) )
1775 {
1776 sqlite3_bind_int64( hInsertPolygonsStandaloneStmt , 1, psWayFeaturePairs->nWayID );
1777
1778 int rc = sqlite3_step( hInsertPolygonsStandaloneStmt );
1779 sqlite3_reset( hInsertPolygonsStandaloneStmt );
1780 if( !(rc == SQLITE_OK || rc == SQLITE_DONE) )
1781 {
1782 CPLError(CE_Failure, CPLE_AppDefined,
1783 "Failed inserting into polygons_standalone " CPL_FRMT_GIB ": %s",
1784 psWayFeaturePairs->nWayID, sqlite3_errmsg(hDB));
1785 }
1786 }
1787 }
1788 }
1789
1790 nWayFeaturePairs = 0;
1791 nUnsortedReqIds = 0;
1792
1793 nAccumulatedTags = 0;
1794 nNonRedundantValuesLen = 0;
1795 }
1796
1797 /************************************************************************/
1798 /* NotifyWay() */
1799 /************************************************************************/
1800
NotifyWay(OSMWay * psWay)1801 void OGROSMDataSource::NotifyWay (OSMWay* psWay)
1802 {
1803 unsigned int i;
1804
1805 nWaysProcessed++;
1806 if( (nWaysProcessed % 10000) == 0 )
1807 {
1808 CPLDebug("OSM", "Ways processed : %d", nWaysProcessed);
1809 #ifdef DEBUG_MEM_USAGE
1810 CPLDebug("OSM", "GetMaxTotalAllocs() = " CPL_FRMT_GUIB, (GUIntBig)GetMaxTotalAllocs());
1811 #endif
1812 }
1813
1814 if( !bUsePointsIndex )
1815 return;
1816
1817 //printf("way %d : %d nodes\n", (int)psWay->nID, (int)psWay->nRefs);
1818 if( psWay->nRefs > MAX_NODES_PER_WAY )
1819 {
1820 CPLError(CE_Failure, CPLE_NotSupported,
1821 "Ways with more than %d nodes are not supported",
1822 MAX_NODES_PER_WAY);
1823 return;
1824 }
1825
1826 if( psWay->nRefs < 2 )
1827 {
1828 CPLDebug("OSM", "Way " CPL_FRMT_GIB " with %d nodes. Discarding it",
1829 psWay->nID, psWay->nRefs);
1830 return;
1831 }
1832
1833 /* Is a closed way a polygon ? */
1834 int bIsArea = FALSE;
1835 if( psWay->panNodeRefs[0] == psWay->panNodeRefs[psWay->nRefs - 1] )
1836 {
1837 for(i=0;i<psWay->nTags;i++)
1838 {
1839 const char* pszK = psWay->pasTags[i].pszK;
1840 if( strcmp(pszK, "area") == 0 )
1841 {
1842 if( strcmp(psWay->pasTags[i].pszV, "yes") == 0 )
1843 {
1844 bIsArea = TRUE;
1845 }
1846 else if( strcmp(psWay->pasTags[i].pszV, "no") == 0 )
1847 {
1848 bIsArea = FALSE;
1849 break;
1850 }
1851 }
1852 else if( aoSetClosedWaysArePolygons.find(pszK) !=
1853 aoSetClosedWaysArePolygons.end() )
1854 {
1855 bIsArea = TRUE;
1856 }
1857 }
1858 }
1859
1860 OGRFeature* poFeature = NULL;
1861
1862 int bInterestingTag = bReportAllWays;
1863 if( !bIsArea && !bReportAllWays )
1864 {
1865 for(i=0;i<psWay->nTags;i++)
1866 {
1867 const char* pszK = psWay->pasTags[i].pszK;
1868 if( papoLayers[IDX_LYR_LINES]->IsSignificantKey(pszK) )
1869 {
1870 bInterestingTag = TRUE;
1871 break;
1872 }
1873 }
1874 }
1875
1876 int bAttrFilterAlreadyEvaluated = FALSE;
1877 if( !bIsArea && papoLayers[IDX_LYR_LINES]->IsUserInterested() && bInterestingTag )
1878 {
1879 poFeature = new OGRFeature(papoLayers[IDX_LYR_LINES]->GetLayerDefn());
1880
1881 papoLayers[IDX_LYR_LINES]->SetFieldsFromTags(
1882 poFeature, psWay->nID, FALSE, psWay->nTags, psWay->pasTags, &psWay->sInfo );
1883
1884 /* Optimization : if we have an attribute filter, that does not require geometry, */
1885 /* and if we don't need to index ways, then we can just evaluate the attribute */
1886 /* filter without the geometry */
1887 if( papoLayers[IDX_LYR_LINES]->HasAttributeFilter() &&
1888 !papoLayers[IDX_LYR_LINES]->AttributeFilterEvaluationNeedsGeometry() &&
1889 !bIndexWays )
1890 {
1891 if( !papoLayers[IDX_LYR_LINES]->EvaluateAttributeFilter(poFeature) )
1892 {
1893 delete poFeature;
1894 return;
1895 }
1896 bAttrFilterAlreadyEvaluated = TRUE;
1897 }
1898 }
1899 else if( !bIndexWays )
1900 {
1901 return;
1902 }
1903
1904 if( nUnsortedReqIds + psWay->nRefs > MAX_ACCUMULATED_NODES ||
1905 nWayFeaturePairs == MAX_DELAYED_FEATURES ||
1906 nAccumulatedTags + psWay->nTags > MAX_ACCUMULATED_TAGS ||
1907 nNonRedundantValuesLen + 1024 > MAX_NON_REDUNDANT_VALUES )
1908 {
1909 ProcessWaysBatch();
1910 }
1911
1912 WayFeaturePair* psWayFeaturePairs = &pasWayFeaturePairs[nWayFeaturePairs];
1913
1914 psWayFeaturePairs->nWayID = psWay->nID;
1915 psWayFeaturePairs->nRefs = psWay->nRefs - bIsArea;
1916 psWayFeaturePairs->panNodeRefs = panUnsortedReqIds + nUnsortedReqIds;
1917 psWayFeaturePairs->poFeature = poFeature;
1918 psWayFeaturePairs->bIsArea = bIsArea;
1919 psWayFeaturePairs->bAttrFilterAlreadyEvaluated = bAttrFilterAlreadyEvaluated;
1920
1921 if( bIsArea && papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() )
1922 {
1923 int nTagCount = 0;
1924
1925 if( bNeedsToSaveWayInfo )
1926 {
1927 if( !psWay->sInfo.bTimeStampIsStr )
1928 psWayFeaturePairs->sInfo.ts.nTimeStamp =
1929 psWay->sInfo.ts.nTimeStamp;
1930 else
1931 {
1932 OGRField sField;
1933 if (OGRParseXMLDateTime(psWay->sInfo.ts.pszTimeStamp, &sField))
1934 {
1935 struct tm brokendown;
1936 brokendown.tm_year = sField.Date.Year - 1900;
1937 brokendown.tm_mon = sField.Date.Month - 1;
1938 brokendown.tm_mday = sField.Date.Day;
1939 brokendown.tm_hour = sField.Date.Hour;
1940 brokendown.tm_min = sField.Date.Minute;
1941 brokendown.tm_sec = (int)(sField.Date.Second + .5);
1942 psWayFeaturePairs->sInfo.ts.nTimeStamp =
1943 CPLYMDHMSToUnixTime(&brokendown);
1944 }
1945 else
1946 psWayFeaturePairs->sInfo.ts.nTimeStamp = 0;
1947 }
1948 psWayFeaturePairs->sInfo.nChangeset = psWay->sInfo.nChangeset;
1949 psWayFeaturePairs->sInfo.nVersion = psWay->sInfo.nVersion;
1950 psWayFeaturePairs->sInfo.nUID = psWay->sInfo.nUID;
1951 psWayFeaturePairs->sInfo.bTimeStampIsStr = FALSE;
1952 psWayFeaturePairs->sInfo.pszUserSID = ""; // FIXME
1953 }
1954 else
1955 {
1956 psWayFeaturePairs->sInfo.ts.nTimeStamp = 0;
1957 psWayFeaturePairs->sInfo.nChangeset = 0;
1958 psWayFeaturePairs->sInfo.nVersion = 0;
1959 psWayFeaturePairs->sInfo.nUID = 0;
1960 psWayFeaturePairs->sInfo.bTimeStampIsStr = FALSE;
1961 psWayFeaturePairs->sInfo.pszUserSID = "";
1962 }
1963
1964 psWayFeaturePairs->pasTags = pasAccumulatedTags + nAccumulatedTags;
1965
1966 for(unsigned int iTag = 0; iTag < psWay->nTags; iTag++)
1967 {
1968 const char* pszK = psWay->pasTags[iTag].pszK;
1969 const char* pszV = psWay->pasTags[iTag].pszV;
1970
1971 if (strcmp(pszK, "area") == 0)
1972 continue;
1973 if (strcmp(pszK, "created_by") == 0)
1974 continue;
1975 if (strcmp(pszK, "converted_by") == 0)
1976 continue;
1977 if (strcmp(pszK, "note") == 0)
1978 continue;
1979 if (strcmp(pszK, "todo") == 0)
1980 continue;
1981 if (strcmp(pszK, "fixme") == 0)
1982 continue;
1983 if (strcmp(pszK, "FIXME") == 0)
1984 continue;
1985
1986 std::map<const char*, KeyDesc*, ConstCharComp>::iterator oIterK =
1987 aoMapIndexedKeys.find(pszK);
1988 KeyDesc* psKD;
1989 if (oIterK == aoMapIndexedKeys.end())
1990 {
1991 if( nNextKeyIndex >= 32768 ) /* somewhat arbitrary */
1992 {
1993 if( nNextKeyIndex == 32768 )
1994 {
1995 CPLError(CE_Failure, CPLE_AppDefined,
1996 "Too many different keys in file");
1997 nNextKeyIndex ++; /* to avoid next warnings */
1998 }
1999 continue;
2000 }
2001 psKD = new KeyDesc();
2002 psKD->pszK = CPLStrdup(pszK);
2003 psKD->nKeyIndex = nNextKeyIndex ++;
2004 //CPLDebug("OSM", "nNextKeyIndex=%d", nNextKeyIndex);
2005 psKD->nOccurences = 0;
2006 psKD->asValues.push_back(CPLStrdup(""));
2007 aoMapIndexedKeys[psKD->pszK] = psKD;
2008 asKeys.push_back(psKD);
2009 }
2010 else
2011 psKD = oIterK->second;
2012 psKD->nOccurences ++;
2013
2014 pasAccumulatedTags[nAccumulatedTags].nKeyIndex = (short)psKD->nKeyIndex;
2015
2016 /* to fit in 2 bytes, the theoretical limit would be 127 * 128 + 127 */
2017 if( psKD->asValues.size() < 1024 )
2018 {
2019 std::map<const char*, int, ConstCharComp>::iterator oIterV;
2020 oIterV = psKD->anMapV.find(pszV);
2021 int nValueIndex;
2022 if (oIterV == psKD->anMapV.end())
2023 {
2024 char* pszVDup = CPLStrdup(pszV);
2025 nValueIndex = (int)psKD->asValues.size();
2026 psKD->anMapV[pszVDup] = nValueIndex;
2027 psKD->asValues.push_back(pszVDup);
2028 }
2029 else
2030 nValueIndex = oIterV->second;
2031
2032 pasAccumulatedTags[nAccumulatedTags].bVIsIndex = TRUE;
2033 pasAccumulatedTags[nAccumulatedTags].u.nValueIndex = nValueIndex;
2034 }
2035 else
2036 {
2037 int nLenV = strlen(pszV) + 1;
2038
2039 if( psKD->asValues.size() == 1024 )
2040 {
2041 CPLDebug("OSM", "More than %d different values for tag %s",
2042 1024, pszK);
2043 psKD->asValues.push_back(CPLStrdup("")); /* to avoid next warnings */
2044 }
2045
2046 CPLAssert( nNonRedundantValuesLen + nLenV <= MAX_NON_REDUNDANT_VALUES );
2047 memcpy(pabyNonRedundantValues + nNonRedundantValuesLen, pszV, nLenV);
2048 pasAccumulatedTags[nAccumulatedTags].bVIsIndex = FALSE;
2049 pasAccumulatedTags[nAccumulatedTags].u.nOffsetInpabyNonRedundantValues = nNonRedundantValuesLen;
2050 nNonRedundantValuesLen += nLenV;
2051 }
2052 nAccumulatedTags ++;
2053
2054 nTagCount ++;
2055 if( nTagCount == MAX_COUNT_FOR_TAGS_IN_WAY )
2056 break;
2057 }
2058
2059 psWayFeaturePairs->nTags = nTagCount;
2060 }
2061 else
2062 {
2063 psWayFeaturePairs->sInfo.ts.nTimeStamp = 0;
2064 psWayFeaturePairs->sInfo.nChangeset = 0;
2065 psWayFeaturePairs->sInfo.nVersion = 0;
2066 psWayFeaturePairs->sInfo.nUID = 0;
2067 psWayFeaturePairs->sInfo.bTimeStampIsStr = FALSE;
2068 psWayFeaturePairs->sInfo.pszUserSID = "";
2069
2070 psWayFeaturePairs->nTags = 0;
2071 psWayFeaturePairs->pasTags = NULL;
2072 }
2073
2074 nWayFeaturePairs++;
2075
2076 memcpy(panUnsortedReqIds + nUnsortedReqIds, psWay->panNodeRefs, sizeof(GIntBig) * (psWay->nRefs - bIsArea));
2077 nUnsortedReqIds += (psWay->nRefs - bIsArea);
2078 }
2079
OGROSMNotifyWay(OSMWay * psWay,CPL_UNUSED OSMContext * psOSMContext,void * user_data)2080 static void OGROSMNotifyWay (OSMWay* psWay,
2081 CPL_UNUSED OSMContext* psOSMContext,
2082 void* user_data)
2083 {
2084 ((OGROSMDataSource*) user_data)->NotifyWay(psWay);
2085 }
2086
2087 /************************************************************************/
2088 /* LookupWays() */
2089 /************************************************************************/
2090
LookupWays(std::map<GIntBig,std::pair<int,void * >> & aoMapWays,OSMRelation * psRelation)2091 unsigned int OGROSMDataSource::LookupWays( std::map< GIntBig, std::pair<int,void*> >& aoMapWays,
2092 OSMRelation* psRelation )
2093 {
2094 unsigned int nFound = 0;
2095 unsigned int iCur = 0;
2096 unsigned int i;
2097
2098 while( iCur < psRelation->nMembers )
2099 {
2100 unsigned int nToQuery = 0;
2101 for(i=iCur;i<psRelation->nMembers;i++)
2102 {
2103 if( psRelation->pasMembers[i].eType == MEMBER_WAY &&
2104 strcmp(psRelation->pasMembers[i].pszRole, "subarea") != 0 )
2105 {
2106 nToQuery ++;
2107 if( nToQuery == LIMIT_IDS_PER_REQUEST )
2108 break;
2109 }
2110 }
2111
2112 if( nToQuery == 0)
2113 break;
2114
2115 unsigned int iLastI = (i == psRelation->nMembers) ? i : i + 1;
2116
2117 sqlite3_stmt* hStmt = pahSelectWayStmt[nToQuery-1];
2118 unsigned int nBindIndex = 1;
2119 for(i=iCur;i<iLastI;i++)
2120 {
2121 if( psRelation->pasMembers[i].eType == MEMBER_WAY &&
2122 strcmp(psRelation->pasMembers[i].pszRole, "subarea") != 0 )
2123 {
2124 sqlite3_bind_int64( hStmt, nBindIndex,
2125 psRelation->pasMembers[i].nID );
2126 nBindIndex ++;
2127 }
2128 }
2129 iCur = iLastI;
2130
2131 while( sqlite3_step(hStmt) == SQLITE_ROW )
2132 {
2133 GIntBig id = sqlite3_column_int64(hStmt, 0);
2134 if( aoMapWays.find(id) == aoMapWays.end() )
2135 {
2136 int nBlobSize = sqlite3_column_bytes(hStmt, 1);
2137 const void* blob = sqlite3_column_blob(hStmt, 1);
2138 void* blob_dup = CPLMalloc(nBlobSize);
2139 memcpy(blob_dup, blob, nBlobSize);
2140 aoMapWays[id] = std::pair<int,void*>(nBlobSize, blob_dup);
2141 }
2142 nFound++;
2143 }
2144
2145 sqlite3_reset(hStmt);
2146 }
2147
2148 return nFound;
2149 }
2150
2151 /************************************************************************/
2152 /* BuildMultiPolygon() */
2153 /************************************************************************/
2154
BuildMultiPolygon(OSMRelation * psRelation,unsigned int * pnTags,OSMTag * pasTags)2155 OGRGeometry* OGROSMDataSource::BuildMultiPolygon(OSMRelation* psRelation,
2156 unsigned int* pnTags,
2157 OSMTag* pasTags)
2158 {
2159
2160 std::map< GIntBig, std::pair<int,void*> > aoMapWays;
2161 LookupWays( aoMapWays, psRelation );
2162
2163 int bMissing = FALSE;
2164 unsigned int i;
2165 for(i = 0; i < psRelation->nMembers; i ++ )
2166 {
2167 if( psRelation->pasMembers[i].eType == MEMBER_WAY &&
2168 strcmp(psRelation->pasMembers[i].pszRole, "subarea") != 0 )
2169 {
2170 if( aoMapWays.find( psRelation->pasMembers[i].nID ) == aoMapWays.end() )
2171 {
2172 CPLDebug("OSM", "Relation " CPL_FRMT_GIB " has missing ways. Ignoring it",
2173 psRelation->nID);
2174 bMissing = TRUE;
2175 break;
2176 }
2177 }
2178 }
2179
2180 OGRGeometry* poRet = NULL;
2181 OGRMultiLineString* poMLS = NULL;
2182 OGRGeometry** papoPolygons = NULL;
2183 int nPolys = 0;
2184
2185 if( bMissing )
2186 goto cleanup;
2187
2188 poMLS = new OGRMultiLineString();
2189 papoPolygons = (OGRGeometry**) CPLMalloc(
2190 sizeof(OGRGeometry*) * psRelation->nMembers);
2191 nPolys = 0;
2192
2193 if( pnTags != NULL )
2194 *pnTags = 0;
2195
2196 for(i = 0; i < psRelation->nMembers; i ++ )
2197 {
2198 if( psRelation->pasMembers[i].eType == MEMBER_WAY &&
2199 strcmp(psRelation->pasMembers[i].pszRole, "subarea") != 0 )
2200 {
2201 const std::pair<int, void*>& oGeom = aoMapWays[ psRelation->pasMembers[i].nID ];
2202
2203 LonLat* pasCoords = (LonLat*) pasLonLatCache;
2204 int nPoints;
2205
2206 if( pnTags != NULL && *pnTags == 0 &&
2207 strcmp(psRelation->pasMembers[i].pszRole, "outer") == 0 )
2208 {
2209 int nCompressedWaySize = oGeom.first;
2210 GByte* pabyCompressedWay = (GByte*) oGeom.second;
2211
2212 memcpy(pabyWayBuffer, pabyCompressedWay, nCompressedWaySize);
2213
2214 nPoints = UncompressWay (nCompressedWaySize, pabyWayBuffer,
2215 pasCoords,
2216 pnTags, pasTags, NULL );
2217 }
2218 else
2219 {
2220 nPoints = UncompressWay (oGeom.first, (GByte*) oGeom.second, pasCoords,
2221 NULL, NULL, NULL);
2222 }
2223
2224 OGRLineString* poLS;
2225
2226 if ( pasCoords[0].nLon == pasCoords[nPoints - 1].nLon &&
2227 pasCoords[0].nLat == pasCoords[nPoints - 1].nLat )
2228 {
2229 OGRPolygon* poPoly = new OGRPolygon();
2230 OGRLinearRing* poRing = new OGRLinearRing();
2231 poPoly->addRingDirectly(poRing);
2232 papoPolygons[nPolys ++] = poPoly;
2233 poLS = poRing;
2234
2235 if( strcmp(psRelation->pasMembers[i].pszRole, "outer") == 0 )
2236 {
2237 sqlite3_bind_int64( hDeletePolygonsStandaloneStmt, 1, psRelation->pasMembers[i].nID );
2238 sqlite3_step( hDeletePolygonsStandaloneStmt );
2239 sqlite3_reset( hDeletePolygonsStandaloneStmt );
2240 }
2241 }
2242 else
2243 {
2244 poLS = new OGRLineString();
2245 poMLS->addGeometryDirectly(poLS);
2246 }
2247
2248 poLS->setNumPoints(nPoints);
2249 for(int j=0;j<nPoints;j++)
2250 {
2251 poLS->setPoint( j,
2252 INT_TO_DBL(pasCoords[j].nLon),
2253 INT_TO_DBL(pasCoords[j].nLat) );
2254 }
2255
2256 }
2257 }
2258
2259 if( poMLS->getNumGeometries() > 0 )
2260 {
2261 OGRGeometryH hPoly = OGRBuildPolygonFromEdges( (OGRGeometryH) poMLS,
2262 TRUE,
2263 FALSE,
2264 0,
2265 NULL );
2266 if( hPoly != NULL && OGR_G_GetGeometryType(hPoly) == wkbPolygon )
2267 {
2268 OGRPolygon* poSuperPoly = (OGRPolygon* ) hPoly;
2269 for(i = 0; i < 1 + (unsigned int)poSuperPoly->getNumInteriorRings(); i++)
2270 {
2271 OGRPolygon* poPoly = new OGRPolygon();
2272 OGRLinearRing* poRing = (i == 0) ? poSuperPoly->getExteriorRing() :
2273 poSuperPoly->getInteriorRing(i - 1);
2274 if( poRing != NULL && poRing->getNumPoints() >= 4 &&
2275 poRing->getX(0) == poRing->getX(poRing->getNumPoints() -1) &&
2276 poRing->getY(0) == poRing->getY(poRing->getNumPoints() -1) )
2277 {
2278 poPoly->addRing( poRing );
2279 papoPolygons[nPolys ++] = poPoly;
2280 }
2281 }
2282 }
2283
2284 OGR_G_DestroyGeometry(hPoly);
2285 }
2286 delete poMLS;
2287
2288 if( nPolys > 0 )
2289 {
2290 int bIsValidGeometry = FALSE;
2291 const char* apszOptions[2] = { "METHOD=DEFAULT", NULL };
2292 OGRGeometry* poGeom = OGRGeometryFactory::organizePolygons(
2293 papoPolygons, nPolys, &bIsValidGeometry, apszOptions );
2294
2295 if( poGeom != NULL && poGeom->getGeometryType() == wkbPolygon )
2296 {
2297 OGRMultiPolygon* poMulti = new OGRMultiPolygon();
2298 poMulti->addGeometryDirectly(poGeom);
2299 poGeom = poMulti;
2300 }
2301
2302 if( poGeom != NULL && poGeom->getGeometryType() == wkbMultiPolygon )
2303 {
2304 poRet = poGeom;
2305 }
2306 else
2307 {
2308 CPLDebug("OSM", "Relation " CPL_FRMT_GIB ": Geometry has incompatible type : %s",
2309 psRelation->nID,
2310 poGeom != NULL ? OGR_G_GetGeometryName((OGRGeometryH)poGeom) : "null" );
2311 delete poGeom;
2312 }
2313 }
2314
2315 CPLFree(papoPolygons);
2316
2317 cleanup:
2318 /* Cleanup */
2319 std::map< GIntBig, std::pair<int,void*> >::iterator oIter;
2320 for( oIter = aoMapWays.begin(); oIter != aoMapWays.end(); ++oIter )
2321 CPLFree(oIter->second.second);
2322
2323 return poRet;
2324 }
2325
2326 /************************************************************************/
2327 /* BuildGeometryCollection() */
2328 /************************************************************************/
2329
BuildGeometryCollection(OSMRelation * psRelation,int bMultiLineString)2330 OGRGeometry* OGROSMDataSource::BuildGeometryCollection(OSMRelation* psRelation, int bMultiLineString)
2331 {
2332 std::map< GIntBig, std::pair<int,void*> > aoMapWays;
2333 LookupWays( aoMapWays, psRelation );
2334
2335 unsigned int i;
2336
2337 OGRGeometryCollection* poColl;
2338 if( bMultiLineString )
2339 poColl = new OGRMultiLineString();
2340 else
2341 poColl = new OGRGeometryCollection();
2342
2343 for(i = 0; i < psRelation->nMembers; i ++ )
2344 {
2345 if( psRelation->pasMembers[i].eType == MEMBER_NODE && !bMultiLineString )
2346 {
2347 nUnsortedReqIds = 1;
2348 panUnsortedReqIds[0] = psRelation->pasMembers[i].nID;
2349 LookupNodes();
2350 if( nReqIds == 1 )
2351 {
2352 poColl->addGeometryDirectly(new OGRPoint(
2353 INT_TO_DBL(pasLonLatArray[0].nLon),
2354 INT_TO_DBL(pasLonLatArray[0].nLat)));
2355 }
2356 }
2357 else if( psRelation->pasMembers[i].eType == MEMBER_WAY &&
2358 strcmp(psRelation->pasMembers[i].pszRole, "subarea") != 0 &&
2359 aoMapWays.find( psRelation->pasMembers[i].nID ) != aoMapWays.end() )
2360 {
2361 const std::pair<int, void*>& oGeom = aoMapWays[ psRelation->pasMembers[i].nID ];
2362
2363 LonLat* pasCoords = (LonLat*) pasLonLatCache;
2364 int nPoints = UncompressWay (oGeom.first, (GByte*) oGeom.second, pasCoords, NULL, NULL, NULL);
2365
2366 OGRLineString* poLS;
2367
2368 poLS = new OGRLineString();
2369 poColl->addGeometryDirectly(poLS);
2370
2371 poLS->setNumPoints(nPoints);
2372 for(int j=0;j<nPoints;j++)
2373 {
2374 poLS->setPoint( j,
2375 INT_TO_DBL(pasCoords[j].nLon),
2376 INT_TO_DBL(pasCoords[j].nLat) );
2377 }
2378
2379 }
2380 }
2381
2382 if( poColl->getNumGeometries() == 0 )
2383 {
2384 delete poColl;
2385 poColl = NULL;
2386 }
2387
2388 /* Cleanup */
2389 std::map< GIntBig, std::pair<int,void*> >::iterator oIter;
2390 for( oIter = aoMapWays.begin(); oIter != aoMapWays.end(); ++oIter )
2391 CPLFree(oIter->second.second);
2392
2393 return poColl;
2394 }
2395
2396 /************************************************************************/
2397 /* NotifyRelation() */
2398 /************************************************************************/
2399
NotifyRelation(OSMRelation * psRelation)2400 void OGROSMDataSource::NotifyRelation (OSMRelation* psRelation)
2401 {
2402 unsigned int i;
2403
2404 if( nWayFeaturePairs != 0 )
2405 ProcessWaysBatch();
2406
2407 nRelationsProcessed++;
2408 if( (nRelationsProcessed % 10000) == 0 )
2409 {
2410 CPLDebug("OSM", "Relations processed : %d", nRelationsProcessed);
2411 #ifdef DEBUG_MEM_USAGE
2412 CPLDebug("OSM", "GetMaxTotalAllocs() = " CPL_FRMT_GUIB, (GUIntBig)GetMaxTotalAllocs());
2413 #endif
2414 }
2415
2416 if( !bUseWaysIndex )
2417 return;
2418
2419 int bMultiPolygon = FALSE;
2420 int bMultiLineString = FALSE;
2421 int bInterestingTagFound = FALSE;
2422 const char* pszTypeV = NULL;
2423 for(i = 0; i < psRelation->nTags; i ++ )
2424 {
2425 const char* pszK = psRelation->pasTags[i].pszK;
2426 if( strcmp(pszK, "type") == 0 )
2427 {
2428 const char* pszV = psRelation->pasTags[i].pszV;
2429 pszTypeV = pszV;
2430 if( strcmp(pszV, "multipolygon") == 0 ||
2431 strcmp(pszV, "boundary") == 0)
2432 {
2433 bMultiPolygon = TRUE;
2434 }
2435 else if( strcmp(pszV, "multilinestring") == 0 ||
2436 strcmp(pszV, "route") == 0 )
2437 {
2438 bMultiLineString = TRUE;
2439 }
2440 }
2441 else if ( strcmp(pszK, "created_by") != 0 )
2442 bInterestingTagFound = TRUE;
2443 }
2444
2445 /* Optimization : if we have an attribute filter, that does not require geometry, */
2446 /* then we can just evaluate the attribute filter without the geometry */
2447 int iCurLayer = (bMultiPolygon) ? IDX_LYR_MULTIPOLYGONS :
2448 (bMultiLineString) ? IDX_LYR_MULTILINESTRINGS :
2449 IDX_LYR_OTHER_RELATIONS;
2450 if( !papoLayers[iCurLayer]->IsUserInterested() )
2451 return;
2452
2453 OGRFeature* poFeature = NULL;
2454
2455 if( !(bMultiPolygon && !bInterestingTagFound) && /* we cannot do early filtering for multipolygon that has no interesting tag, since we may fetch attributes from ways */
2456 papoLayers[iCurLayer]->HasAttributeFilter() &&
2457 !papoLayers[iCurLayer]->AttributeFilterEvaluationNeedsGeometry() )
2458 {
2459 poFeature = new OGRFeature(papoLayers[iCurLayer]->GetLayerDefn());
2460
2461 papoLayers[iCurLayer]->SetFieldsFromTags( poFeature,
2462 psRelation->nID,
2463 FALSE,
2464 psRelation->nTags,
2465 psRelation->pasTags,
2466 &psRelation->sInfo);
2467
2468 if( !papoLayers[iCurLayer]->EvaluateAttributeFilter(poFeature) )
2469 {
2470 delete poFeature;
2471 return;
2472 }
2473 }
2474
2475 OGRGeometry* poGeom;
2476
2477 unsigned int nExtraTags = 0;
2478 OSMTag pasExtraTags[1 + MAX_COUNT_FOR_TAGS_IN_WAY];
2479
2480 if( bMultiPolygon )
2481 {
2482 if( !bInterestingTagFound )
2483 {
2484 poGeom = BuildMultiPolygon(psRelation, &nExtraTags, pasExtraTags);
2485 CPLAssert(nExtraTags <= MAX_COUNT_FOR_TAGS_IN_WAY);
2486 pasExtraTags[nExtraTags].pszK = "type";
2487 pasExtraTags[nExtraTags].pszV = pszTypeV;
2488 nExtraTags ++;
2489 }
2490 else
2491 poGeom = BuildMultiPolygon(psRelation, NULL, NULL);
2492 }
2493 else
2494 poGeom = BuildGeometryCollection(psRelation, bMultiLineString);
2495
2496 if( poGeom != NULL )
2497 {
2498 int bAttrFilterAlreadyEvaluated;
2499 if( poFeature == NULL )
2500 {
2501 poFeature = new OGRFeature(papoLayers[iCurLayer]->GetLayerDefn());
2502
2503 papoLayers[iCurLayer]->SetFieldsFromTags( poFeature,
2504 psRelation->nID,
2505 FALSE,
2506 nExtraTags ? nExtraTags : psRelation->nTags,
2507 nExtraTags ? pasExtraTags : psRelation->pasTags,
2508 &psRelation->sInfo);
2509
2510 bAttrFilterAlreadyEvaluated = FALSE;
2511 }
2512 else
2513 bAttrFilterAlreadyEvaluated = TRUE;
2514
2515 poFeature->SetGeometryDirectly(poGeom);
2516
2517 int bFilteredOut = FALSE;
2518 if( !papoLayers[iCurLayer]->AddFeature( poFeature,
2519 bAttrFilterAlreadyEvaluated,
2520 &bFilteredOut,
2521 !bFeatureAdded ) )
2522 bStopParsing = TRUE;
2523 else if (!bFilteredOut)
2524 bFeatureAdded = TRUE;
2525 }
2526 else
2527 delete poFeature;
2528 }
2529
OGROSMNotifyRelation(OSMRelation * psRelation,CPL_UNUSED OSMContext * psOSMContext,void * user_data)2530 static void OGROSMNotifyRelation (OSMRelation* psRelation,
2531 CPL_UNUSED OSMContext* psOSMContext,
2532 void* user_data)
2533 {
2534 ((OGROSMDataSource*) user_data)->NotifyRelation(psRelation);
2535 }
2536
2537
2538 /************************************************************************/
2539 /* ProcessPolygonsStandalone() */
2540 /************************************************************************/
2541
ProcessPolygonsStandalone()2542 void OGROSMDataSource::ProcessPolygonsStandalone()
2543 {
2544 unsigned int nTags = 0;
2545 OSMTag pasTags[MAX_COUNT_FOR_TAGS_IN_WAY];
2546 OSMInfo sInfo;
2547 int bFirst = TRUE;
2548
2549 sInfo.ts.nTimeStamp = 0;
2550 sInfo.nChangeset = 0;
2551 sInfo.nVersion = 0;
2552 sInfo.nUID = 0;
2553 sInfo.bTimeStampIsStr = FALSE;
2554 sInfo.pszUserSID = "";
2555
2556 if( !bHasRowInPolygonsStandalone )
2557 bHasRowInPolygonsStandalone = (sqlite3_step(hSelectPolygonsStandaloneStmt) == SQLITE_ROW);
2558
2559 while( bHasRowInPolygonsStandalone &&
2560 papoLayers[IDX_LYR_MULTIPOLYGONS]->nFeatureArraySize < 10000 )
2561 {
2562 if( bFirst )
2563 {
2564 CPLDebug("OSM", "Remaining standalone polygons");
2565 bFirst = FALSE;
2566 }
2567
2568 GIntBig id = sqlite3_column_int64(hSelectPolygonsStandaloneStmt, 0);
2569
2570 sqlite3_bind_int64( pahSelectWayStmt[0], 1, id );
2571 if( sqlite3_step(pahSelectWayStmt[0]) == SQLITE_ROW )
2572 {
2573 int nBlobSize = sqlite3_column_bytes(pahSelectWayStmt[0], 1);
2574 const void* blob = sqlite3_column_blob(pahSelectWayStmt[0], 1);
2575
2576 LonLat* pasCoords = (LonLat*) pasLonLatCache;
2577
2578 int nPoints = UncompressWay (nBlobSize, (GByte*) blob,
2579 pasCoords,
2580 &nTags, pasTags, &sInfo );
2581 CPLAssert(nTags <= MAX_COUNT_FOR_TAGS_IN_WAY);
2582
2583 OGRLineString* poLS;
2584
2585 OGRMultiPolygon* poMulti = new OGRMultiPolygon();
2586 OGRPolygon* poPoly = new OGRPolygon();
2587 OGRLinearRing* poRing = new OGRLinearRing();
2588 poMulti->addGeometryDirectly(poPoly);
2589 poPoly->addRingDirectly(poRing);
2590 poLS = poRing;
2591
2592 poLS->setNumPoints(nPoints);
2593 for(int j=0;j<nPoints;j++)
2594 {
2595 poLS->setPoint( j,
2596 INT_TO_DBL(pasCoords[j].nLon),
2597 INT_TO_DBL(pasCoords[j].nLat) );
2598 }
2599
2600 OGRFeature* poFeature = new OGRFeature(papoLayers[IDX_LYR_MULTIPOLYGONS]->GetLayerDefn());
2601
2602 papoLayers[IDX_LYR_MULTIPOLYGONS]->SetFieldsFromTags( poFeature,
2603 id,
2604 TRUE,
2605 nTags,
2606 pasTags,
2607 &sInfo);
2608
2609 poFeature->SetGeometryDirectly(poMulti);
2610
2611 int bFilteredOut = FALSE;
2612 if( !papoLayers[IDX_LYR_MULTIPOLYGONS]->AddFeature( poFeature,
2613 FALSE,
2614 &bFilteredOut,
2615 !bFeatureAdded ) )
2616 {
2617 bStopParsing = TRUE;
2618 break;
2619 }
2620 else if (!bFilteredOut)
2621 bFeatureAdded = TRUE;
2622
2623 }
2624 else {
2625 CPLAssert(FALSE);
2626 }
2627
2628 sqlite3_reset(pahSelectWayStmt[0]);
2629
2630 bHasRowInPolygonsStandalone = (sqlite3_step(hSelectPolygonsStandaloneStmt) == SQLITE_ROW);
2631 }
2632 }
2633
2634 /************************************************************************/
2635 /* NotifyBounds() */
2636 /************************************************************************/
2637
NotifyBounds(double dfXMin,double dfYMin,double dfXMax,double dfYMax)2638 void OGROSMDataSource::NotifyBounds (double dfXMin, double dfYMin,
2639 double dfXMax, double dfYMax)
2640 {
2641 sExtent.MinX = dfXMin;
2642 sExtent.MinY = dfYMin;
2643 sExtent.MaxX = dfXMax;
2644 sExtent.MaxY = dfYMax;
2645 bExtentValid = TRUE;
2646
2647 CPLDebug("OSM", "Got bounds : minx=%f, miny=%f, maxx=%f, maxy=%f",
2648 dfXMin, dfYMin, dfXMax, dfYMax);
2649 }
2650
OGROSMNotifyBounds(double dfXMin,double dfYMin,double dfXMax,double dfYMax,CPL_UNUSED OSMContext * psCtxt,void * user_data)2651 static void OGROSMNotifyBounds( double dfXMin, double dfYMin,
2652 double dfXMax, double dfYMax,
2653 CPL_UNUSED OSMContext* psCtxt,
2654 void* user_data )
2655 {
2656 ((OGROSMDataSource*) user_data)->NotifyBounds(dfXMin, dfYMin,
2657 dfXMax, dfYMax);
2658 }
2659
2660 /************************************************************************/
2661 /* Open() */
2662 /************************************************************************/
2663
Open(const char * pszFilename,char ** papszOpenOptions)2664 int OGROSMDataSource::Open( const char * pszFilename, char** papszOpenOptions )
2665
2666 {
2667 pszName = CPLStrdup( pszFilename );
2668
2669 psParser = OSM_Open( pszName,
2670 OGROSMNotifyNodes,
2671 OGROSMNotifyWay,
2672 OGROSMNotifyRelation,
2673 OGROSMNotifyBounds,
2674 this );
2675 if( psParser == NULL )
2676 return FALSE;
2677
2678 if( CSLFetchBoolean(papszOpenOptions, "INTERLEAVED_READING", FALSE) )
2679 bInterleavedReading = TRUE;
2680
2681 /* The following 4 config options are only useful for debugging */
2682 bIndexPoints = CSLTestBoolean(CPLGetConfigOption("OSM_INDEX_POINTS", "YES"));
2683 bUsePointsIndex = CSLTestBoolean(CPLGetConfigOption("OSM_USE_POINTS_INDEX", "YES"));
2684 bIndexWays = CSLTestBoolean(CPLGetConfigOption("OSM_INDEX_WAYS", "YES"));
2685 bUseWaysIndex = CSLTestBoolean(CPLGetConfigOption("OSM_USE_WAYS_INDEX", "YES"));
2686
2687 bCustomIndexing = CSLTestBoolean(CSLFetchNameValueDef(
2688 papszOpenOptions, "USE_CUSTOM_INDEXING",
2689 CPLGetConfigOption("OSM_USE_CUSTOM_INDEXING", "YES")));
2690 if( !bCustomIndexing )
2691 CPLDebug("OSM", "Using SQLite indexing for points");
2692 bCompressNodes = CSLTestBoolean(CSLFetchNameValueDef(
2693 papszOpenOptions, "COMPRESS_NODES",
2694 CPLGetConfigOption("OSM_COMPRESS_NODES", "NO")));
2695 if( bCompressNodes )
2696 CPLDebug("OSM", "Using compression for nodes DB");
2697
2698 nLayers = 5;
2699 papoLayers = (OGROSMLayer**) CPLMalloc(nLayers * sizeof(OGROSMLayer*));
2700
2701 papoLayers[IDX_LYR_POINTS] = new OGROSMLayer(this, IDX_LYR_POINTS, "points");
2702 papoLayers[IDX_LYR_POINTS]->GetLayerDefn()->SetGeomType(wkbPoint);
2703
2704 papoLayers[IDX_LYR_LINES] = new OGROSMLayer(this, IDX_LYR_LINES, "lines");
2705 papoLayers[IDX_LYR_LINES]->GetLayerDefn()->SetGeomType(wkbLineString);
2706
2707 papoLayers[IDX_LYR_MULTILINESTRINGS] = new OGROSMLayer(this, IDX_LYR_MULTILINESTRINGS, "multilinestrings");
2708 papoLayers[IDX_LYR_MULTILINESTRINGS]->GetLayerDefn()->SetGeomType(wkbMultiLineString);
2709
2710 papoLayers[IDX_LYR_MULTIPOLYGONS] = new OGROSMLayer(this, IDX_LYR_MULTIPOLYGONS, "multipolygons");
2711 papoLayers[IDX_LYR_MULTIPOLYGONS]->GetLayerDefn()->SetGeomType(wkbMultiPolygon);
2712
2713 papoLayers[IDX_LYR_OTHER_RELATIONS] = new OGROSMLayer(this, IDX_LYR_OTHER_RELATIONS, "other_relations");
2714 papoLayers[IDX_LYR_OTHER_RELATIONS]->GetLayerDefn()->SetGeomType(wkbGeometryCollection);
2715
2716 if( !ParseConf(papszOpenOptions) )
2717 {
2718 CPLError(CE_Failure, CPLE_AppDefined,
2719 "Could not parse configuration file for OSM import");
2720 return FALSE;
2721 }
2722
2723 bNeedsToSaveWayInfo =
2724 ( papoLayers[IDX_LYR_MULTIPOLYGONS]->HasTimestamp() ||
2725 papoLayers[IDX_LYR_MULTIPOLYGONS]->HasChangeset() ||
2726 papoLayers[IDX_LYR_MULTIPOLYGONS]->HasVersion() ||
2727 papoLayers[IDX_LYR_MULTIPOLYGONS]->HasUID() ||
2728 papoLayers[IDX_LYR_MULTIPOLYGONS]->HasUser() );
2729
2730 pasLonLatCache = (LonLat*)VSIMalloc(MAX_NODES_PER_WAY * sizeof(LonLat));
2731 pabyWayBuffer = (GByte*)VSIMalloc(WAY_BUFFER_SIZE);
2732
2733 panReqIds = (GIntBig*)VSIMalloc(MAX_ACCUMULATED_NODES * sizeof(GIntBig));
2734 #ifdef ENABLE_NODE_LOOKUP_BY_HASHING
2735 panHashedIndexes = (int*)VSIMalloc(HASHED_INDEXES_ARRAY_SIZE * sizeof(int));
2736 psCollisionBuckets = (CollisionBucket*)VSIMalloc(COLLISION_BUCKET_ARRAY_SIZE * sizeof(CollisionBucket));
2737 #endif
2738 pasLonLatArray = (LonLat*)VSIMalloc(MAX_ACCUMULATED_NODES * sizeof(LonLat));
2739 panUnsortedReqIds = (GIntBig*)VSIMalloc(MAX_ACCUMULATED_NODES * sizeof(GIntBig));
2740 pasWayFeaturePairs = (WayFeaturePair*)VSIMalloc(MAX_DELAYED_FEATURES * sizeof(WayFeaturePair));
2741 pasAccumulatedTags = (IndexedKVP*) VSIMalloc(MAX_ACCUMULATED_TAGS * sizeof(IndexedKVP));
2742 pabyNonRedundantValues = (GByte*) VSIMalloc(MAX_NON_REDUNDANT_VALUES);
2743
2744 if( pasLonLatCache == NULL ||
2745 pabyWayBuffer == NULL ||
2746 panReqIds == NULL ||
2747 pasLonLatArray == NULL ||
2748 panUnsortedReqIds == NULL ||
2749 pasWayFeaturePairs == NULL ||
2750 pasAccumulatedTags == NULL ||
2751 pabyNonRedundantValues == NULL )
2752 {
2753 CPLError(CE_Failure, CPLE_OutOfMemory,
2754 "Out-of-memory when allocating one of the buffer used for the processing.");
2755 return FALSE;
2756 }
2757
2758 nMaxSizeForInMemoryDBInMB = atoi(CSLFetchNameValueDef(papszOpenOptions,
2759 "MAX_TMPFILE_SIZE", CPLGetConfigOption("OSM_MAX_TMPFILE_SIZE", "100")));
2760 GIntBig nSize = (GIntBig)nMaxSizeForInMemoryDBInMB * 1024 * 1024;
2761 if (nSize < 0 || (GIntBig)(size_t)nSize != nSize)
2762 {
2763 CPLError(CE_Failure, CPLE_AppDefined, "Invalid value for OSM_MAX_TMPFILE_SIZE. Using 100 instead.");
2764 nMaxSizeForInMemoryDBInMB = 100;
2765 nSize = (GIntBig)nMaxSizeForInMemoryDBInMB * 1024 * 1024;
2766 }
2767
2768 if( bCustomIndexing )
2769 {
2770 pabySector = (GByte*) VSICalloc(1, SECTOR_SIZE);
2771
2772 if( pabySector == NULL || !AllocMoreBuckets(INIT_BUCKET_COUNT) )
2773 {
2774 CPLError(CE_Failure, CPLE_OutOfMemory,
2775 "Out-of-memory when allocating one of the buffer used for the processing.");
2776 return FALSE;
2777 }
2778
2779 bInMemoryNodesFile = TRUE;
2780 osNodesFilename.Printf("/vsimem/osm_importer/osm_temp_nodes_%p", this);
2781 fpNodes = VSIFOpenL(osNodesFilename, "wb+");
2782 if( fpNodes == NULL )
2783 {
2784 return FALSE;
2785 }
2786
2787 CPLPushErrorHandler(CPLQuietErrorHandler);
2788 int bSuccess = VSIFSeekL(fpNodes, (vsi_l_offset) (nSize * 3 / 4), SEEK_SET) == 0;
2789 CPLPopErrorHandler();
2790
2791 if( bSuccess )
2792 {
2793 VSIFSeekL(fpNodes, 0, SEEK_SET);
2794 VSIFTruncateL(fpNodes, 0);
2795 }
2796 else
2797 {
2798 CPLDebug("OSM", "Not enough memory for in-memory file. Using disk temporary file instead.");
2799
2800 VSIFCloseL(fpNodes);
2801 fpNodes = NULL;
2802 VSIUnlink(osNodesFilename);
2803
2804 bInMemoryNodesFile = FALSE;
2805 osNodesFilename = CPLGenerateTempFilename("osm_tmp_nodes");
2806
2807 fpNodes = VSIFOpenL(osNodesFilename, "wb+");
2808 if( fpNodes == NULL )
2809 {
2810 return FALSE;
2811 }
2812
2813 /* On Unix filesystems, you can remove a file even if it */
2814 /* opened */
2815 const char* pszVal = CPLGetConfigOption("OSM_UNLINK_TMPFILE", "YES");
2816 if( EQUAL(pszVal, "YES") )
2817 {
2818 CPLPushErrorHandler(CPLQuietErrorHandler);
2819 bMustUnlinkNodesFile = VSIUnlink( osNodesFilename ) != 0;
2820 CPLPopErrorHandler();
2821 }
2822
2823 return FALSE;
2824 }
2825 }
2826
2827 int bRet = CreateTempDB();
2828 if( bRet )
2829 {
2830 CPLString osInterestLayers = GetInterestLayersForDSName(GetName());
2831 if( osInterestLayers.size() )
2832 {
2833 ExecuteSQL( osInterestLayers, NULL, NULL );
2834 }
2835 }
2836 return bRet;
2837 }
2838
2839 /************************************************************************/
2840 /* CreateTempDB() */
2841 /************************************************************************/
2842
CreateTempDB()2843 int OGROSMDataSource::CreateTempDB()
2844 {
2845 char* pszErrMsg = NULL;
2846
2847 int rc = 0;
2848 int bIsExisting = FALSE;
2849 int bSuccess = FALSE;
2850
2851 #ifdef HAVE_SQLITE_VFS
2852 const char* pszExistingTmpFile = CPLGetConfigOption("OSM_EXISTING_TMPFILE", NULL);
2853 if ( pszExistingTmpFile != NULL )
2854 {
2855 bSuccess = TRUE;
2856 bIsExisting = TRUE;
2857 rc = sqlite3_open_v2( pszExistingTmpFile, &hDB,
2858 SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX,
2859 NULL );
2860 }
2861 else
2862 {
2863 osTmpDBName.Printf("/vsimem/osm_importer/osm_temp_%p.sqlite", this);
2864
2865 /* On 32 bit, the virtual memory space is scarce, so we need to reserve it right now */
2866 /* Won't hurt on 64 bit either. */
2867 VSILFILE* fp = VSIFOpenL(osTmpDBName, "wb");
2868 if( fp )
2869 {
2870 GIntBig nSize = (GIntBig)nMaxSizeForInMemoryDBInMB * 1024 * 1024;
2871 if( bCustomIndexing && bInMemoryNodesFile )
2872 nSize = nSize * 1 / 4;
2873
2874 CPLPushErrorHandler(CPLQuietErrorHandler);
2875 bSuccess = VSIFSeekL(fp, (vsi_l_offset) nSize, SEEK_SET) == 0;
2876 CPLPopErrorHandler();
2877
2878 if( bSuccess )
2879 VSIFTruncateL(fp, 0);
2880
2881 VSIFCloseL(fp);
2882
2883 if( !bSuccess )
2884 {
2885 CPLDebug("OSM", "Not enough memory for in-memory file. Using disk temporary file instead.");
2886 VSIUnlink(osTmpDBName);
2887 }
2888 }
2889
2890 if( bSuccess )
2891 {
2892 bInMemoryTmpDB = TRUE;
2893 pMyVFS = OGRSQLiteCreateVFS(NULL, this);
2894 sqlite3_vfs_register(pMyVFS, 0);
2895 rc = sqlite3_open_v2( osTmpDBName.c_str(), &hDB,
2896 SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX,
2897 pMyVFS->zName );
2898 }
2899 }
2900 #endif
2901
2902 if( !bSuccess )
2903 {
2904 osTmpDBName = CPLGenerateTempFilename("osm_tmp");
2905 rc = sqlite3_open( osTmpDBName.c_str(), &hDB );
2906
2907 /* On Unix filesystems, you can remove a file even if it */
2908 /* opened */
2909 if( rc == SQLITE_OK )
2910 {
2911 const char* pszVal = CPLGetConfigOption("OSM_UNLINK_TMPFILE", "YES");
2912 if( EQUAL(pszVal, "YES") )
2913 {
2914 CPLPushErrorHandler(CPLQuietErrorHandler);
2915 bMustUnlink = VSIUnlink( osTmpDBName ) != 0;
2916 CPLPopErrorHandler();
2917 }
2918 }
2919 }
2920
2921 if( rc != SQLITE_OK )
2922 {
2923 CPLError( CE_Failure, CPLE_OpenFailed,
2924 "sqlite3_open(%s) failed: %s",
2925 osTmpDBName.c_str(), sqlite3_errmsg( hDB ) );
2926 return FALSE;
2927 }
2928
2929 if( !SetDBOptions() )
2930 {
2931 return FALSE;
2932 }
2933
2934 if( !bIsExisting )
2935 {
2936 rc = sqlite3_exec( hDB,
2937 "CREATE TABLE nodes (id INTEGER PRIMARY KEY, coords BLOB)",
2938 NULL, NULL, &pszErrMsg );
2939 if( rc != SQLITE_OK )
2940 {
2941 CPLError( CE_Failure, CPLE_AppDefined,
2942 "Unable to create table nodes : %s", pszErrMsg );
2943 sqlite3_free( pszErrMsg );
2944 return FALSE;
2945 }
2946
2947 rc = sqlite3_exec( hDB,
2948 "CREATE TABLE ways (id INTEGER PRIMARY KEY, data BLOB)",
2949 NULL, NULL, &pszErrMsg );
2950 if( rc != SQLITE_OK )
2951 {
2952 CPLError( CE_Failure, CPLE_AppDefined,
2953 "Unable to create table ways : %s", pszErrMsg );
2954 sqlite3_free( pszErrMsg );
2955 return FALSE;
2956 }
2957
2958 rc = sqlite3_exec( hDB,
2959 "CREATE TABLE polygons_standalone (id INTEGER PRIMARY KEY)",
2960 NULL, NULL, &pszErrMsg );
2961 if( rc != SQLITE_OK )
2962 {
2963 CPLError( CE_Failure, CPLE_AppDefined,
2964 "Unable to create table polygons_standalone : %s", pszErrMsg );
2965 sqlite3_free( pszErrMsg );
2966 return FALSE;
2967 }
2968 }
2969
2970 return CreatePreparedStatements();
2971 }
2972 /************************************************************************/
2973 /* SetDBOptions() */
2974 /************************************************************************/
2975
SetDBOptions()2976 int OGROSMDataSource::SetDBOptions()
2977 {
2978 char* pszErrMsg = NULL;
2979 int rc;
2980
2981 rc = sqlite3_exec( hDB, "PRAGMA synchronous = OFF", NULL, NULL, &pszErrMsg );
2982 if( rc != SQLITE_OK )
2983 {
2984 CPLError( CE_Failure, CPLE_AppDefined,
2985 "Unable to run PRAGMA synchronous : %s",
2986 pszErrMsg );
2987 sqlite3_free( pszErrMsg );
2988 return FALSE;
2989 }
2990
2991 rc = sqlite3_exec( hDB, "PRAGMA journal_mode = OFF", NULL, NULL, &pszErrMsg );
2992 if( rc != SQLITE_OK )
2993 {
2994 CPLError( CE_Failure, CPLE_AppDefined,
2995 "Unable to run PRAGMA journal_mode : %s",
2996 pszErrMsg );
2997 sqlite3_free( pszErrMsg );
2998 return FALSE;
2999 }
3000
3001 rc = sqlite3_exec( hDB, "PRAGMA temp_store = MEMORY", NULL, NULL, &pszErrMsg );
3002 if( rc != SQLITE_OK )
3003 {
3004 CPLError( CE_Failure, CPLE_AppDefined,
3005 "Unable to run PRAGMA temp_store : %s",
3006 pszErrMsg );
3007 sqlite3_free( pszErrMsg );
3008 return FALSE;
3009 }
3010
3011 if( !SetCacheSize() )
3012 return FALSE;
3013
3014 if( !StartTransactionCacheDB() )
3015 return FALSE;
3016
3017 return TRUE;
3018 }
3019
3020 /************************************************************************/
3021 /* SetCacheSize() */
3022 /************************************************************************/
3023
SetCacheSize()3024 int OGROSMDataSource::SetCacheSize()
3025 {
3026 int rc;
3027 const char* pszSqliteCacheMB = CPLGetConfigOption("OSM_SQLITE_CACHE", NULL);
3028 if (pszSqliteCacheMB != NULL)
3029 {
3030 char* pszErrMsg = NULL;
3031 char **papszResult;
3032 int nRowCount, nColCount;
3033 int iSqliteCachePages;
3034 int iSqlitePageSize = -1;
3035 int iSqliteCacheBytes = atoi( pszSqliteCacheMB ) * 1024 * 1024;
3036
3037 /* querying the current PageSize */
3038 rc = sqlite3_get_table( hDB, "PRAGMA page_size",
3039 &papszResult, &nRowCount, &nColCount,
3040 &pszErrMsg );
3041 if( rc == SQLITE_OK )
3042 {
3043 int iRow;
3044 for (iRow = 1; iRow <= nRowCount; iRow++)
3045 {
3046 iSqlitePageSize = atoi( papszResult[(iRow * nColCount) + 0] );
3047 }
3048 sqlite3_free_table(papszResult);
3049 }
3050 if( iSqlitePageSize < 0 )
3051 {
3052 CPLError( CE_Failure, CPLE_AppDefined,
3053 "Unable to run PRAGMA page_size : %s",
3054 pszErrMsg ? pszErrMsg : sqlite3_errmsg(hDB) );
3055 sqlite3_free( pszErrMsg );
3056 return TRUE;
3057 }
3058
3059 /* computing the CacheSize as #Pages */
3060 iSqliteCachePages = iSqliteCacheBytes / iSqlitePageSize;
3061 if( iSqliteCachePages <= 0)
3062 return TRUE;
3063
3064 rc = sqlite3_exec( hDB, CPLSPrintf( "PRAGMA cache_size = %d",
3065 iSqliteCachePages ),
3066 NULL, NULL, &pszErrMsg );
3067 if( rc != SQLITE_OK )
3068 {
3069 CPLError( CE_Warning, CPLE_AppDefined,
3070 "Unrecognized value for PRAGMA cache_size : %s",
3071 pszErrMsg );
3072 sqlite3_free( pszErrMsg );
3073 rc = SQLITE_OK;
3074 }
3075 }
3076 return TRUE;
3077 }
3078
3079 /************************************************************************/
3080 /* CreatePreparedStatements() */
3081 /************************************************************************/
3082
CreatePreparedStatements()3083 int OGROSMDataSource::CreatePreparedStatements()
3084 {
3085 int rc;
3086
3087 rc = sqlite3_prepare( hDB, "INSERT INTO nodes (id, coords) VALUES (?,?)", -1,
3088 &hInsertNodeStmt, NULL );
3089 if( rc != SQLITE_OK )
3090 {
3091 CPLError( CE_Failure, CPLE_AppDefined,
3092 "sqlite3_prepare() failed : %s", sqlite3_errmsg(hDB) );
3093 return FALSE;
3094 }
3095
3096 pahSelectNodeStmt = (sqlite3_stmt**) CPLCalloc(sizeof(sqlite3_stmt*), LIMIT_IDS_PER_REQUEST);
3097
3098 char szTmp[LIMIT_IDS_PER_REQUEST*2 + 128];
3099 strcpy(szTmp, "SELECT id, coords FROM nodes WHERE id IN (");
3100 int nLen = strlen(szTmp);
3101 for(int i=0;i<LIMIT_IDS_PER_REQUEST;i++)
3102 {
3103 if(i == 0)
3104 {
3105 strcpy(szTmp + nLen, "?) ORDER BY id ASC");
3106 nLen += 2;
3107 }
3108 else
3109 {
3110 strcpy(szTmp + nLen -1, ",?) ORDER BY id ASC");
3111 nLen += 2;
3112 }
3113 rc = sqlite3_prepare( hDB, szTmp, -1, &pahSelectNodeStmt[i], NULL );
3114 if( rc != SQLITE_OK )
3115 {
3116 CPLError( CE_Failure, CPLE_AppDefined,
3117 "sqlite3_prepare() failed : %s", sqlite3_errmsg(hDB) );
3118 return FALSE;
3119 }
3120 }
3121
3122 rc = sqlite3_prepare( hDB, "INSERT INTO ways (id, data) VALUES (?,?)", -1,
3123 &hInsertWayStmt, NULL );
3124 if( rc != SQLITE_OK )
3125 {
3126 CPLError( CE_Failure, CPLE_AppDefined,
3127 "sqlite3_prepare() failed : %s", sqlite3_errmsg(hDB) );
3128 return FALSE;
3129 }
3130
3131 pahSelectWayStmt = (sqlite3_stmt**) CPLCalloc(sizeof(sqlite3_stmt*), LIMIT_IDS_PER_REQUEST);
3132
3133 strcpy(szTmp, "SELECT id, data FROM ways WHERE id IN (");
3134 nLen = strlen(szTmp);
3135 for(int i=0;i<LIMIT_IDS_PER_REQUEST;i++)
3136 {
3137 if(i == 0)
3138 {
3139 strcpy(szTmp + nLen, "?)");
3140 nLen += 2;
3141 }
3142 else
3143 {
3144 strcpy(szTmp + nLen -1, ",?)");
3145 nLen += 2;
3146 }
3147 rc = sqlite3_prepare( hDB, szTmp, -1, &pahSelectWayStmt[i], NULL );
3148 if( rc != SQLITE_OK )
3149 {
3150 CPLError( CE_Failure, CPLE_AppDefined,
3151 "sqlite3_prepare() failed : %s", sqlite3_errmsg(hDB) );
3152 return FALSE;
3153 }
3154 }
3155
3156 rc = sqlite3_prepare( hDB, "INSERT INTO polygons_standalone (id) VALUES (?)", -1,
3157 &hInsertPolygonsStandaloneStmt, NULL );
3158 if( rc != SQLITE_OK )
3159 {
3160 CPLError( CE_Failure, CPLE_AppDefined,
3161 "sqlite3_prepare() failed : %s", sqlite3_errmsg(hDB) );
3162 return FALSE;
3163 }
3164
3165 rc = sqlite3_prepare( hDB, "DELETE FROM polygons_standalone WHERE id = ?", -1,
3166 &hDeletePolygonsStandaloneStmt, NULL );
3167 if( rc != SQLITE_OK )
3168 {
3169 CPLError( CE_Failure, CPLE_AppDefined,
3170 "sqlite3_prepare() failed : %s", sqlite3_errmsg(hDB) );
3171 return FALSE;
3172 }
3173
3174 rc = sqlite3_prepare( hDB, "SELECT id FROM polygons_standalone ORDER BY id", -1,
3175 &hSelectPolygonsStandaloneStmt, NULL );
3176 if( rc != SQLITE_OK )
3177 {
3178 CPLError( CE_Failure, CPLE_AppDefined,
3179 "sqlite3_prepare() failed : %s", sqlite3_errmsg(hDB) );
3180 return FALSE;
3181 }
3182
3183 return TRUE;
3184 }
3185
3186 /************************************************************************/
3187 /* StartTransactionCacheDB() */
3188 /************************************************************************/
3189
StartTransactionCacheDB()3190 int OGROSMDataSource::StartTransactionCacheDB()
3191 {
3192 if( bInTransaction )
3193 return FALSE;
3194
3195 char* pszErrMsg = NULL;
3196 int rc = sqlite3_exec( hDB, "BEGIN", NULL, NULL, &pszErrMsg );
3197 if( rc != SQLITE_OK )
3198 {
3199 CPLError( CE_Failure, CPLE_AppDefined,
3200 "Unable to start transaction : %s", pszErrMsg );
3201 sqlite3_free( pszErrMsg );
3202 return FALSE;
3203 }
3204
3205 bInTransaction = TRUE;
3206
3207 return TRUE;
3208 }
3209
3210 /************************************************************************/
3211 /* CommitTransactionCacheDB() */
3212 /************************************************************************/
3213
CommitTransactionCacheDB()3214 int OGROSMDataSource::CommitTransactionCacheDB()
3215 {
3216 if( !bInTransaction )
3217 return FALSE;
3218
3219 bInTransaction = FALSE;
3220
3221 char* pszErrMsg = NULL;
3222 int rc = sqlite3_exec( hDB, "COMMIT", NULL, NULL, &pszErrMsg );
3223 if( rc != SQLITE_OK )
3224 {
3225 CPLError( CE_Failure, CPLE_AppDefined,
3226 "Unable to commit transaction : %s", pszErrMsg );
3227 sqlite3_free( pszErrMsg );
3228 return FALSE;
3229 }
3230
3231 return TRUE;
3232 }
3233
3234 /************************************************************************/
3235 /* AddComputedAttributes() */
3236 /************************************************************************/
3237
AddComputedAttributes(int iCurLayer,const std::vector<OGROSMComputedAttribute> & oAttributes)3238 void OGROSMDataSource::AddComputedAttributes(int iCurLayer,
3239 const std::vector<OGROSMComputedAttribute>& oAttributes)
3240 {
3241 for(size_t i=0; i<oAttributes.size();i++)
3242 {
3243 if( oAttributes[i].osSQL.size() )
3244 {
3245 papoLayers[iCurLayer]->AddComputedAttribute(oAttributes[i].osName,
3246 oAttributes[i].eType,
3247 oAttributes[i].osSQL);
3248 }
3249 }
3250 }
3251
3252 /************************************************************************/
3253 /* ParseConf() */
3254 /************************************************************************/
3255
ParseConf(char ** papszOpenOptions)3256 int OGROSMDataSource::ParseConf(char** papszOpenOptions)
3257 {
3258 const char *pszFilename =
3259 CSLFetchNameValueDef(papszOpenOptions, "CONFIG_FILE",
3260 CPLGetConfigOption("OSM_CONFIG_FILE", NULL));
3261 if( pszFilename == NULL )
3262 pszFilename = CPLFindFile( "gdal", "osmconf.ini" );
3263 if( pszFilename == NULL )
3264 {
3265 CPLError(CE_Warning, CPLE_AppDefined, "Cannot find osmconf.ini configuration file");
3266 return FALSE;
3267 }
3268
3269 VSILFILE* fpConf = VSIFOpenL(pszFilename, "rb");
3270 if( fpConf == NULL )
3271 return FALSE;
3272
3273 const char* pszLine;
3274 int iCurLayer = -1;
3275 std::vector<OGROSMComputedAttribute> oAttributes;
3276
3277 int i;
3278
3279 while((pszLine = CPLReadLine2L(fpConf, -1, NULL)) != NULL)
3280 {
3281 if(pszLine[0] == '#')
3282 continue;
3283 if(pszLine[0] == '[' && pszLine[strlen(pszLine)-1] == ']' )
3284 {
3285 if( iCurLayer >= 0 )
3286 AddComputedAttributes(iCurLayer, oAttributes);
3287 oAttributes.resize(0);
3288
3289 iCurLayer = -1;
3290 pszLine ++;
3291 ((char*)pszLine)[strlen(pszLine)-1] = '\0'; /* Evil but OK */
3292 for(i = 0; i < nLayers; i++)
3293 {
3294 if( strcmp(pszLine, papoLayers[i]->GetName()) == 0 )
3295 {
3296 iCurLayer = i;
3297 break;
3298 }
3299 }
3300 if( iCurLayer < 0 )
3301 {
3302 CPLError(CE_Warning, CPLE_AppDefined,
3303 "Layer '%s' mentionned in %s is unknown to the driver",
3304 pszLine, pszFilename);
3305 }
3306 continue;
3307 }
3308
3309 if( strncmp(pszLine, "closed_ways_are_polygons=",
3310 strlen("closed_ways_are_polygons=")) == 0)
3311 {
3312 char** papszTokens = CSLTokenizeString2(pszLine, "=", 0);
3313 if( CSLCount(papszTokens) == 2)
3314 {
3315 char** papszTokens2 = CSLTokenizeString2(papszTokens[1], ",", 0);
3316 for(int i=0;papszTokens2[i] != NULL;i++)
3317 {
3318 aoSetClosedWaysArePolygons.insert(papszTokens2[i]);
3319 }
3320 CSLDestroy(papszTokens2);
3321 }
3322 CSLDestroy(papszTokens);
3323 }
3324
3325 else if(strncmp(pszLine, "report_all_nodes=", strlen("report_all_nodes=")) == 0)
3326 {
3327 if( strcmp(pszLine + strlen("report_all_nodes="), "no") == 0 )
3328 {
3329 bReportAllNodes = FALSE;
3330 }
3331 else if( strcmp(pszLine + strlen("report_all_nodes="), "yes") == 0 )
3332 {
3333 bReportAllNodes = TRUE;
3334 }
3335 }
3336
3337 else if(strncmp(pszLine, "report_all_ways=", strlen("report_all_ways=")) == 0)
3338 {
3339 if( strcmp(pszLine + strlen("report_all_ways="), "no") == 0 )
3340 {
3341 bReportAllWays = FALSE;
3342 }
3343 else if( strcmp(pszLine + strlen("report_all_ways="), "yes") == 0 )
3344 {
3345 bReportAllWays = TRUE;
3346 }
3347 }
3348
3349 else if(strncmp(pszLine, "attribute_name_laundering=", strlen("attribute_name_laundering=")) == 0)
3350 {
3351 if( strcmp(pszLine + strlen("attribute_name_laundering="), "no") == 0 )
3352 {
3353 bAttributeNameLaundering = FALSE;
3354 }
3355 else if( strcmp(pszLine + strlen("attribute_name_laundering="), "yes") == 0 )
3356 {
3357 bAttributeNameLaundering = TRUE;
3358 }
3359 }
3360
3361 else if( iCurLayer >= 0 )
3362 {
3363 char** papszTokens = CSLTokenizeString2(pszLine, "=", 0);
3364 if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "other_tags") == 0 )
3365 {
3366 if( strcmp(papszTokens[1], "no") == 0 )
3367 papoLayers[iCurLayer]->SetHasOtherTags(FALSE);
3368 else if( strcmp(papszTokens[1], "yes") == 0 )
3369 papoLayers[iCurLayer]->SetHasOtherTags(TRUE);
3370 }
3371 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "all_tags") == 0 )
3372 {
3373 if( strcmp(papszTokens[1], "no") == 0 )
3374 papoLayers[iCurLayer]->SetHasAllTags(FALSE);
3375 else if( strcmp(papszTokens[1], "yes") == 0 )
3376 papoLayers[iCurLayer]->SetHasAllTags(TRUE);
3377 }
3378 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_id") == 0 )
3379 {
3380 if( strcmp(papszTokens[1], "no") == 0 )
3381 papoLayers[iCurLayer]->SetHasOSMId(FALSE);
3382 else if( strcmp(papszTokens[1], "yes") == 0 )
3383 {
3384 papoLayers[iCurLayer]->SetHasOSMId(TRUE);
3385 papoLayers[iCurLayer]->AddField("osm_id", OFTString);
3386
3387 if( iCurLayer == IDX_LYR_MULTIPOLYGONS )
3388 papoLayers[iCurLayer]->AddField("osm_way_id", OFTString);
3389 }
3390 }
3391 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_version") == 0 )
3392 {
3393 if( strcmp(papszTokens[1], "no") == 0 )
3394 papoLayers[iCurLayer]->SetHasVersion(FALSE);
3395 else if( strcmp(papszTokens[1], "yes") == 0 )
3396 {
3397 papoLayers[iCurLayer]->SetHasVersion(TRUE);
3398 papoLayers[iCurLayer]->AddField("osm_version", OFTInteger);
3399 }
3400 }
3401 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_timestamp") == 0 )
3402 {
3403 if( strcmp(papszTokens[1], "no") == 0 )
3404 papoLayers[iCurLayer]->SetHasTimestamp(FALSE);
3405 else if( strcmp(papszTokens[1], "yes") == 0 )
3406 {
3407 papoLayers[iCurLayer]->SetHasTimestamp(TRUE);
3408 papoLayers[iCurLayer]->AddField("osm_timestamp", OFTDateTime);
3409 }
3410 }
3411 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_uid") == 0 )
3412 {
3413 if( strcmp(papszTokens[1], "no") == 0 )
3414 papoLayers[iCurLayer]->SetHasUID(FALSE);
3415 else if( strcmp(papszTokens[1], "yes") == 0 )
3416 {
3417 papoLayers[iCurLayer]->SetHasUID(TRUE);
3418 papoLayers[iCurLayer]->AddField("osm_uid", OFTInteger);
3419 }
3420 }
3421 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_user") == 0 )
3422 {
3423 if( strcmp(papszTokens[1], "no") == 0 )
3424 papoLayers[iCurLayer]->SetHasUser(FALSE);
3425 else if( strcmp(papszTokens[1], "yes") == 0 )
3426 {
3427 papoLayers[iCurLayer]->SetHasUser(TRUE);
3428 papoLayers[iCurLayer]->AddField("osm_user", OFTString);
3429 }
3430 }
3431 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "osm_changeset") == 0 )
3432 {
3433 if( strcmp(papszTokens[1], "no") == 0 )
3434 papoLayers[iCurLayer]->SetHasChangeset(FALSE);
3435 else if( strcmp(papszTokens[1], "yes") == 0 )
3436 {
3437 papoLayers[iCurLayer]->SetHasChangeset(TRUE);
3438 papoLayers[iCurLayer]->AddField("osm_changeset", OFTInteger);
3439 }
3440 }
3441 else if( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "attributes") == 0 )
3442 {
3443 char** papszTokens2 = CSLTokenizeString2(papszTokens[1], ",", 0);
3444 for(int i=0;papszTokens2[i] != NULL;i++)
3445 {
3446 papoLayers[iCurLayer]->AddField(papszTokens2[i], OFTString);
3447 }
3448 CSLDestroy(papszTokens2);
3449 }
3450 else if ( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "unsignificant") == 0 )
3451 {
3452 char** papszTokens2 = CSLTokenizeString2(papszTokens[1], ",", 0);
3453 for(int i=0;papszTokens2[i] != NULL;i++)
3454 {
3455 papoLayers[iCurLayer]->AddUnsignificantKey(papszTokens2[i]);
3456 }
3457 CSLDestroy(papszTokens2);
3458 }
3459 else if ( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "ignore") == 0 )
3460 {
3461 char** papszTokens2 = CSLTokenizeString2(papszTokens[1], ",", 0);
3462 for(int i=0;papszTokens2[i] != NULL;i++)
3463 {
3464 papoLayers[iCurLayer]->AddIgnoreKey(papszTokens2[i]);
3465 papoLayers[iCurLayer]->AddWarnKey(papszTokens2[i]);
3466 }
3467 CSLDestroy(papszTokens2);
3468 }
3469 else if ( CSLCount(papszTokens) == 2 && strcmp(papszTokens[0], "computed_attributes") == 0 )
3470 {
3471 char** papszTokens2 = CSLTokenizeString2(papszTokens[1], ",", 0);
3472 oAttributes.resize(0);
3473 for(int i=0;papszTokens2[i] != NULL;i++)
3474 {
3475 oAttributes.push_back(OGROSMComputedAttribute(papszTokens2[i]));
3476 }
3477 CSLDestroy(papszTokens2);
3478 }
3479 else if ( CSLCount(papszTokens) == 2 && strlen(papszTokens[0]) >= 5 &&
3480 strcmp(papszTokens[0] + strlen(papszTokens[0]) - 5, "_type") == 0 )
3481 {
3482 CPLString osName(papszTokens[0]);
3483 osName.resize(strlen(papszTokens[0]) - 5);
3484 const char* pszType = papszTokens[1];
3485 int bFound = FALSE;
3486 OGRFieldType eType = OFTString;
3487 if( EQUAL(pszType, "Integer") )
3488 eType = OFTInteger;
3489 else if( EQUAL(pszType, "Integer64") )
3490 eType = OFTInteger64;
3491 else if( EQUAL(pszType, "Real") )
3492 eType = OFTReal;
3493 else if( EQUAL(pszType, "String") )
3494 eType = OFTString;
3495 else if( EQUAL(pszType, "DateTime") )
3496 eType = OFTDateTime;
3497 else
3498 CPLError(CE_Warning, CPLE_AppDefined,
3499 "Unhandled type (%s) for attribute %s",
3500 pszType, osName.c_str());
3501 for(size_t i = 0; i < oAttributes.size(); i++ )
3502 {
3503 if( oAttributes[i].osName == osName )
3504 {
3505 bFound = TRUE;
3506 oAttributes[i].eType = eType;
3507 break;
3508 }
3509 }
3510 if( !bFound )
3511 {
3512 int idx = papoLayers[iCurLayer]->GetLayerDefn()->GetFieldIndex(osName);
3513 if( idx >= 0 )
3514 {
3515 papoLayers[iCurLayer]->GetLayerDefn()->GetFieldDefn(idx)->SetType(eType);
3516 bFound = TRUE;
3517 }
3518 }
3519 if( !bFound )
3520 {
3521 CPLError(CE_Warning, CPLE_AppDefined, "Undeclared attribute : %s",
3522 osName.c_str());
3523 }
3524 }
3525 else if ( CSLCount(papszTokens) >= 2 && strlen(papszTokens[0]) >= 4 &&
3526 strcmp(papszTokens[0] + strlen(papszTokens[0]) - 4, "_sql") == 0 )
3527 {
3528 CPLString osName(papszTokens[0]);
3529 osName.resize(strlen(papszTokens[0]) - 4);
3530 size_t i;
3531 for(i = 0; i < oAttributes.size(); i++ )
3532 {
3533 if( oAttributes[i].osName == osName )
3534 {
3535 const char* pszSQL = strchr(pszLine, '=') + 1;
3536 while( *pszSQL == ' ' )
3537 pszSQL ++;
3538 int bInQuotes = FALSE;
3539 if( *pszSQL == '"' )
3540 {
3541 bInQuotes = TRUE;
3542 pszSQL ++;
3543 }
3544 oAttributes[i].osSQL = pszSQL;
3545 if( bInQuotes && oAttributes[i].osSQL.size() > 1 &&
3546 oAttributes[i].osSQL[oAttributes[i].osSQL.size()-1] == '"' )
3547 oAttributes[i].osSQL.resize(oAttributes[i].osSQL.size()-1);
3548 break;
3549 }
3550 }
3551 if( i == oAttributes.size() )
3552 {
3553 CPLError(CE_Warning, CPLE_AppDefined, "Undeclared attribute : %s",
3554 osName.c_str());
3555 }
3556 }
3557 CSLDestroy(papszTokens);
3558 }
3559 }
3560
3561 if( iCurLayer >= 0 )
3562 AddComputedAttributes(iCurLayer, oAttributes);
3563
3564 for(i=0;i<nLayers;i++)
3565 {
3566 if( papoLayers[i]->HasAllTags() )
3567 {
3568 papoLayers[i]->AddField("all_tags", OFTString);
3569 if( papoLayers[i]->HasOtherTags() )
3570 {
3571 papoLayers[i]->SetHasOtherTags(FALSE);
3572 }
3573 }
3574 else if( papoLayers[i]->HasOtherTags() )
3575 papoLayers[i]->AddField("other_tags", OFTString);
3576 }
3577
3578 VSIFCloseL(fpConf);
3579
3580 return TRUE;
3581 }
3582
3583 /************************************************************************/
3584 /* ResetReading() */
3585 /************************************************************************/
3586
ResetReading()3587 int OGROSMDataSource::ResetReading()
3588 {
3589 if( hDB == NULL )
3590 return FALSE;
3591 if( bCustomIndexing && fpNodes == NULL )
3592 return FALSE;
3593
3594 OSM_ResetReading(psParser);
3595
3596 char* pszErrMsg = NULL;
3597 int rc = sqlite3_exec( hDB, "DELETE FROM nodes", NULL, NULL, &pszErrMsg );
3598 if( rc != SQLITE_OK )
3599 {
3600 CPLError( CE_Failure, CPLE_AppDefined,
3601 "Unable to DELETE FROM nodes : %s", pszErrMsg );
3602 sqlite3_free( pszErrMsg );
3603 return FALSE;
3604 }
3605
3606 rc = sqlite3_exec( hDB, "DELETE FROM ways", NULL, NULL, &pszErrMsg );
3607 if( rc != SQLITE_OK )
3608 {
3609 CPLError( CE_Failure, CPLE_AppDefined,
3610 "Unable to DELETE FROM ways : %s", pszErrMsg );
3611 sqlite3_free( pszErrMsg );
3612 return FALSE;
3613 }
3614
3615 rc = sqlite3_exec( hDB, "DELETE FROM polygons_standalone", NULL, NULL, &pszErrMsg );
3616 if( rc != SQLITE_OK )
3617 {
3618 CPLError( CE_Failure, CPLE_AppDefined,
3619 "Unable to DELETE FROM polygons_standalone : %s", pszErrMsg );
3620 sqlite3_free( pszErrMsg );
3621 return FALSE;
3622 }
3623 bHasRowInPolygonsStandalone = FALSE;
3624
3625 {
3626 int i;
3627 for( i = 0; i < nWayFeaturePairs; i++)
3628 {
3629 delete pasWayFeaturePairs[i].poFeature;
3630 }
3631 nWayFeaturePairs = 0;
3632 nUnsortedReqIds = 0;
3633 nReqIds = 0;
3634 nAccumulatedTags = 0;
3635 nNonRedundantValuesLen = 0;
3636
3637 for(i=0;i<(int)asKeys.size();i++)
3638 {
3639 KeyDesc* psKD = asKeys[i];
3640 CPLFree(psKD->pszK);
3641 for(int j=0;j<(int)psKD->asValues.size();j++)
3642 CPLFree(psKD->asValues[j]);
3643 delete psKD;
3644 }
3645 asKeys.resize(0);
3646 aoMapIndexedKeys.clear();
3647 nNextKeyIndex = 0;
3648 }
3649
3650 if( bCustomIndexing )
3651 {
3652 nPrevNodeId = -1;
3653 nBucketOld = -1;
3654 nOffInBucketReducedOld = -1;
3655
3656 VSIFSeekL(fpNodes, 0, SEEK_SET);
3657 VSIFTruncateL(fpNodes, 0);
3658 nNodesFileSize = 0;
3659
3660 memset(pabySector, 0, SECTOR_SIZE);
3661 for(int i = 0; i < nBuckets; i++)
3662 {
3663 papsBuckets[i].nOff = -1;
3664 if( bCompressNodes )
3665 {
3666 if( papsBuckets[i].u.panSectorSize )
3667 memset(papsBuckets[i].u.panSectorSize, 0, BUCKET_SECTOR_SIZE_ARRAY_SIZE);
3668 }
3669 else
3670 {
3671 if( papsBuckets[i].u.pabyBitmap )
3672 memset(papsBuckets[i].u.pabyBitmap, 0, BUCKET_BITMAP_SIZE);
3673 }
3674 }
3675 }
3676
3677 for(int i=0;i<nLayers;i++)
3678 {
3679 papoLayers[i]->ForceResetReading();
3680 }
3681
3682 bStopParsing = FALSE;
3683
3684 return TRUE;
3685 }
3686
3687 /************************************************************************/
3688 /* ParseNextChunk() */
3689 /************************************************************************/
3690
ParseNextChunk(int nIdxLayer)3691 int OGROSMDataSource::ParseNextChunk(int nIdxLayer)
3692 {
3693 if( bStopParsing )
3694 return FALSE;
3695
3696 bHasParsedFirstChunk = TRUE;
3697 bFeatureAdded = FALSE;
3698 while( TRUE )
3699 {
3700 #ifdef DEBUG_MEM_USAGE
3701 static int counter = 0;
3702 counter ++;
3703 if ((counter % 1000) == 0)
3704 CPLDebug("OSM", "GetMaxTotalAllocs() = " CPL_FRMT_GUIB, (GUIntBig)GetMaxTotalAllocs());
3705 #endif
3706
3707 OSMRetCode eRet = OSM_ProcessBlock(psParser);
3708 if( eRet == OSM_EOF || eRet == OSM_ERROR )
3709 {
3710 if( eRet == OSM_EOF )
3711 {
3712 if( nWayFeaturePairs != 0 )
3713 ProcessWaysBatch();
3714
3715 ProcessPolygonsStandalone();
3716
3717 if( !bHasRowInPolygonsStandalone )
3718 bStopParsing = TRUE;
3719
3720 if( !bInterleavedReading && !bFeatureAdded &&
3721 bHasRowInPolygonsStandalone &&
3722 nIdxLayer != IDX_LYR_MULTIPOLYGONS )
3723 {
3724 return FALSE;
3725 }
3726
3727 return bFeatureAdded || bHasRowInPolygonsStandalone;
3728 }
3729 else
3730 {
3731 CPLError(CE_Failure, CPLE_AppDefined,
3732 "An error occured during the parsing of data around byte " CPL_FRMT_GUIB,
3733 OSM_GetBytesRead(psParser));
3734
3735 bStopParsing = TRUE;
3736 return FALSE;
3737 }
3738 }
3739 else
3740 {
3741 if( bInMemoryTmpDB )
3742 {
3743 if( !TransferToDiskIfNecesserary() )
3744 return FALSE;
3745 }
3746
3747 if( bFeatureAdded )
3748 break;
3749 }
3750 }
3751
3752 return TRUE;
3753 }
3754
3755 /************************************************************************/
3756 /* TransferToDiskIfNecesserary() */
3757 /************************************************************************/
3758
TransferToDiskIfNecesserary()3759 int OGROSMDataSource::TransferToDiskIfNecesserary()
3760 {
3761 if( bInMemoryNodesFile )
3762 {
3763 if( nNodesFileSize / 1024 / 1024 > 3 * nMaxSizeForInMemoryDBInMB / 4 )
3764 {
3765 bInMemoryNodesFile = FALSE;
3766
3767 VSIFCloseL(fpNodes);
3768 fpNodes = NULL;
3769
3770 CPLString osNewTmpDBName;
3771 osNewTmpDBName = CPLGenerateTempFilename("osm_tmp_nodes");
3772
3773 CPLDebug("OSM", "%s too big for RAM. Transferring it onto disk in %s",
3774 osNodesFilename.c_str(), osNewTmpDBName.c_str());
3775
3776 if( CPLCopyFile( osNewTmpDBName, osNodesFilename ) != 0 )
3777 {
3778 CPLError(CE_Failure, CPLE_AppDefined,
3779 "Cannot copy %s to %s",
3780 osNodesFilename.c_str(), osNewTmpDBName.c_str() );
3781 VSIUnlink(osNewTmpDBName);
3782 bStopParsing = TRUE;
3783 return FALSE;
3784 }
3785
3786 VSIUnlink(osNodesFilename);
3787
3788 if( bInMemoryTmpDB )
3789 {
3790 /* Try to grow the sqlite in memory-db to the full space now */
3791 /* it has been freed. */
3792 VSILFILE* fp = VSIFOpenL(osTmpDBName, "rb+");
3793 if( fp )
3794 {
3795 VSIFSeekL(fp, 0, SEEK_END);
3796 vsi_l_offset nCurSize = VSIFTellL(fp);
3797 GIntBig nNewSize = ((GIntBig)nMaxSizeForInMemoryDBInMB) * 1024 * 1024;
3798 CPLPushErrorHandler(CPLQuietErrorHandler);
3799 int bSuccess = VSIFSeekL(fp, (vsi_l_offset) nNewSize, SEEK_SET) == 0;
3800 CPLPopErrorHandler();
3801
3802 if( bSuccess )
3803 VSIFTruncateL(fp, nCurSize);
3804
3805 VSIFCloseL(fp);
3806 }
3807 }
3808
3809 osNodesFilename = osNewTmpDBName;
3810
3811 fpNodes = VSIFOpenL(osNodesFilename, "rb+");
3812 if( fpNodes == NULL )
3813 {
3814 bStopParsing = TRUE;
3815 return FALSE;
3816 }
3817
3818 VSIFSeekL(fpNodes, 0, SEEK_END);
3819
3820 /* On Unix filesystems, you can remove a file even if it */
3821 /* opened */
3822 const char* pszVal = CPLGetConfigOption("OSM_UNLINK_TMPFILE", "YES");
3823 if( EQUAL(pszVal, "YES") )
3824 {
3825 CPLPushErrorHandler(CPLQuietErrorHandler);
3826 bMustUnlinkNodesFile = VSIUnlink( osNodesFilename ) != 0;
3827 CPLPopErrorHandler();
3828 }
3829 }
3830 }
3831
3832 if( bInMemoryTmpDB )
3833 {
3834 VSIStatBufL sStat;
3835
3836 int nLimitMB = nMaxSizeForInMemoryDBInMB;
3837 if( bCustomIndexing && bInMemoryNodesFile )
3838 nLimitMB = nLimitMB * 1 / 4;
3839
3840 if( VSIStatL( osTmpDBName, &sStat ) == 0 &&
3841 sStat.st_size / 1024 / 1024 > nLimitMB )
3842 {
3843 bInMemoryTmpDB = FALSE;
3844
3845 CloseDB();
3846
3847 CPLString osNewTmpDBName;
3848 int rc;
3849
3850 osNewTmpDBName = CPLGenerateTempFilename("osm_tmp");
3851
3852 CPLDebug("OSM", "%s too big for RAM. Transferring it onto disk in %s",
3853 osTmpDBName.c_str(), osNewTmpDBName.c_str());
3854
3855 if( CPLCopyFile( osNewTmpDBName, osTmpDBName ) != 0 )
3856 {
3857 CPLError(CE_Failure, CPLE_AppDefined,
3858 "Cannot copy %s to %s",
3859 osTmpDBName.c_str(), osNewTmpDBName.c_str() );
3860 VSIUnlink(osNewTmpDBName);
3861 bStopParsing = TRUE;
3862 return FALSE;
3863 }
3864
3865 VSIUnlink(osTmpDBName);
3866
3867 osTmpDBName = osNewTmpDBName;
3868
3869 #ifdef HAVE_SQLITE_VFS
3870 rc = sqlite3_open_v2( osTmpDBName.c_str(), &hDB,
3871 SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX,
3872 NULL );
3873 #else
3874 rc = sqlite3_open( osTmpDBName.c_str(), &hDB );
3875 #endif
3876 if( rc != SQLITE_OK )
3877 {
3878 CPLError( CE_Failure, CPLE_OpenFailed,
3879 "sqlite3_open(%s) failed: %s",
3880 osTmpDBName.c_str(), sqlite3_errmsg( hDB ) );
3881 bStopParsing = TRUE;
3882 CloseDB();
3883 return FALSE;
3884 }
3885
3886 /* On Unix filesystems, you can remove a file even if it */
3887 /* opened */
3888 const char* pszVal = CPLGetConfigOption("OSM_UNLINK_TMPFILE", "YES");
3889 if( EQUAL(pszVal, "YES") )
3890 {
3891 CPLPushErrorHandler(CPLQuietErrorHandler);
3892 bMustUnlink = VSIUnlink( osTmpDBName ) != 0;
3893 CPLPopErrorHandler();
3894 }
3895
3896 if( !SetDBOptions() || !CreatePreparedStatements() )
3897 {
3898 bStopParsing = TRUE;
3899 CloseDB();
3900 return FALSE;
3901 }
3902 }
3903 }
3904
3905 return TRUE;
3906 }
3907
3908 /************************************************************************/
3909 /* TestCapability() */
3910 /************************************************************************/
3911
TestCapability(CPL_UNUSED const char * pszCap)3912 int OGROSMDataSource::TestCapability( CPL_UNUSED const char * pszCap )
3913 {
3914 return FALSE;
3915 }
3916
3917 /************************************************************************/
3918 /* GetLayer() */
3919 /************************************************************************/
3920
GetLayer(int iLayer)3921 OGRLayer *OGROSMDataSource::GetLayer( int iLayer )
3922
3923 {
3924 if( iLayer < 0 || iLayer >= nLayers )
3925 return NULL;
3926 else
3927 return papoLayers[iLayer];
3928 }
3929
3930 /************************************************************************/
3931 /* GetExtent() */
3932 /************************************************************************/
3933
GetExtent(OGREnvelope * psExtent)3934 OGRErr OGROSMDataSource::GetExtent( OGREnvelope *psExtent )
3935 {
3936 if (!bHasParsedFirstChunk)
3937 {
3938 bHasParsedFirstChunk = TRUE;
3939 OSM_ProcessBlock(psParser);
3940 }
3941
3942 if (bExtentValid)
3943 {
3944 memcpy(psExtent, &sExtent, sizeof(sExtent));
3945 return OGRERR_NONE;
3946 }
3947
3948 return OGRERR_FAILURE;
3949 }
3950
3951
3952 /************************************************************************/
3953 /* OGROSMSingleFeatureLayer */
3954 /************************************************************************/
3955
3956 class OGROSMSingleFeatureLayer : public OGRLayer
3957 {
3958 private:
3959 int nVal;
3960 char *pszVal;
3961 OGRFeatureDefn *poFeatureDefn;
3962 int iNextShapeId;
3963
3964 public:
3965 OGROSMSingleFeatureLayer( const char* pszLayerName,
3966 int nVal );
3967 OGROSMSingleFeatureLayer( const char* pszLayerName,
3968 const char *pszVal );
3969 ~OGROSMSingleFeatureLayer();
3970
ResetReading()3971 virtual void ResetReading() { iNextShapeId = 0; }
3972 virtual OGRFeature *GetNextFeature();
GetLayerDefn()3973 virtual OGRFeatureDefn *GetLayerDefn() { return poFeatureDefn; }
TestCapability(const char *)3974 virtual int TestCapability( const char * ) { return FALSE; }
3975 };
3976
3977
3978 /************************************************************************/
3979 /* OGROSMSingleFeatureLayer() */
3980 /************************************************************************/
3981
OGROSMSingleFeatureLayer(const char * pszLayerName,int nVal)3982 OGROSMSingleFeatureLayer::OGROSMSingleFeatureLayer( const char* pszLayerName,
3983 int nVal )
3984 {
3985 poFeatureDefn = new OGRFeatureDefn( "SELECT" );
3986 poFeatureDefn->Reference();
3987 OGRFieldDefn oField( pszLayerName, OFTInteger );
3988 poFeatureDefn->AddFieldDefn( &oField );
3989
3990 iNextShapeId = 0;
3991 this->nVal = nVal;
3992 pszVal = NULL;
3993 }
3994
3995 /************************************************************************/
3996 /* OGROSMSingleFeatureLayer() */
3997 /************************************************************************/
3998
OGROSMSingleFeatureLayer(const char * pszLayerName,const char * pszVal)3999 OGROSMSingleFeatureLayer::OGROSMSingleFeatureLayer( const char* pszLayerName,
4000 const char *pszVal )
4001 {
4002 poFeatureDefn = new OGRFeatureDefn( "SELECT" );
4003 poFeatureDefn->Reference();
4004 OGRFieldDefn oField( pszLayerName, OFTString );
4005 poFeatureDefn->AddFieldDefn( &oField );
4006
4007 iNextShapeId = 0;
4008 nVal = 0;
4009 this->pszVal = CPLStrdup(pszVal);
4010 }
4011
4012 /************************************************************************/
4013 /* ~OGROSMSingleFeatureLayer() */
4014 /************************************************************************/
4015
~OGROSMSingleFeatureLayer()4016 OGROSMSingleFeatureLayer::~OGROSMSingleFeatureLayer()
4017 {
4018 poFeatureDefn->Release();
4019 CPLFree(pszVal);
4020 }
4021
4022 /************************************************************************/
4023 /* GetNextFeature() */
4024 /************************************************************************/
4025
GetNextFeature()4026 OGRFeature * OGROSMSingleFeatureLayer::GetNextFeature()
4027 {
4028 if (iNextShapeId != 0)
4029 return NULL;
4030
4031 OGRFeature* poFeature = new OGRFeature(poFeatureDefn);
4032 if (pszVal)
4033 poFeature->SetField(0, pszVal);
4034 else
4035 poFeature->SetField(0, nVal);
4036 poFeature->SetFID(iNextShapeId ++);
4037 return poFeature;
4038 }
4039
4040 /************************************************************************/
4041 /* OGROSMResultLayerDecorator */
4042 /************************************************************************/
4043
4044 class OGROSMResultLayerDecorator : public OGRLayerDecorator
4045 {
4046 CPLString osDSName;
4047 CPLString osInterestLayers;
4048
4049 public:
OGROSMResultLayerDecorator(OGRLayer * poLayer,CPLString osDSName,CPLString osInterestLayers)4050 OGROSMResultLayerDecorator(OGRLayer* poLayer,
4051 CPLString osDSName,
4052 CPLString osInterestLayers) :
4053 OGRLayerDecorator(poLayer, TRUE),
4054 osDSName(osDSName),
4055 osInterestLayers(osInterestLayers) {}
4056
GetFeatureCount(int bForce=TRUE)4057 virtual GIntBig GetFeatureCount( int bForce = TRUE )
4058 {
4059 /* When we run GetFeatureCount() with SQLite SQL dialect, */
4060 /* the OSM dataset will be re-opened. Make sure that it is */
4061 /* re-opened with the same interest layers */
4062 AddInterestLayersForDSName(osDSName, osInterestLayers);
4063 return OGRLayerDecorator::GetFeatureCount(bForce);
4064 }
4065 };
4066
4067 /************************************************************************/
4068 /* ExecuteSQL() */
4069 /************************************************************************/
4070
ExecuteSQL(const char * pszSQLCommand,OGRGeometry * poSpatialFilter,const char * pszDialect)4071 OGRLayer * OGROSMDataSource::ExecuteSQL( const char *pszSQLCommand,
4072 OGRGeometry *poSpatialFilter,
4073 const char *pszDialect )
4074
4075 {
4076 /* -------------------------------------------------------------------- */
4077 /* Special GetBytesRead() command */
4078 /* -------------------------------------------------------------------- */
4079 if (strcmp(pszSQLCommand, "GetBytesRead()") == 0)
4080 {
4081 char szVal[64];
4082 sprintf(szVal, CPL_FRMT_GUIB, OSM_GetBytesRead(psParser));
4083 return new OGROSMSingleFeatureLayer( "GetBytesRead", szVal );
4084 }
4085
4086 if( poResultSetLayer != NULL )
4087 {
4088 CPLError(CE_Failure, CPLE_NotSupported,
4089 "A SQL result layer is still in use. Please delete it first");
4090 return NULL;
4091 }
4092
4093 /* -------------------------------------------------------------------- */
4094 /* Special SET interest_layers = command */
4095 /* -------------------------------------------------------------------- */
4096 if (strncmp(pszSQLCommand, "SET interest_layers =", 21) == 0)
4097 {
4098 char** papszTokens = CSLTokenizeString2(pszSQLCommand + 21, ",", CSLT_STRIPLEADSPACES | CSLT_STRIPENDSPACES);
4099 int i;
4100
4101 for(i=0; i < nLayers; i++)
4102 {
4103 papoLayers[i]->SetDeclareInterest(FALSE);
4104 }
4105
4106 for(i=0; papszTokens[i] != NULL; i++)
4107 {
4108 OGROSMLayer* poLayer = (OGROSMLayer*) GetLayerByName(papszTokens[i]);
4109 if( poLayer != NULL )
4110 {
4111 poLayer->SetDeclareInterest(TRUE);
4112 }
4113 }
4114
4115 if( papoLayers[IDX_LYR_POINTS]->IsUserInterested() &&
4116 !papoLayers[IDX_LYR_LINES]->IsUserInterested() &&
4117 !papoLayers[IDX_LYR_MULTILINESTRINGS]->IsUserInterested() &&
4118 !papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() &&
4119 !papoLayers[IDX_LYR_OTHER_RELATIONS]->IsUserInterested())
4120 {
4121 if( CPLGetConfigOption("OSM_INDEX_POINTS", NULL) == NULL )
4122 {
4123 CPLDebug("OSM", "Disabling indexing of nodes");
4124 bIndexPoints = FALSE;
4125 }
4126 if( CPLGetConfigOption("OSM_USE_POINTS_INDEX", NULL) == NULL )
4127 {
4128 bUsePointsIndex = FALSE;
4129 }
4130 if( CPLGetConfigOption("OSM_INDEX_WAYS", NULL) == NULL )
4131 {
4132 CPLDebug("OSM", "Disabling indexing of ways");
4133 bIndexWays = FALSE;
4134 }
4135 if( CPLGetConfigOption("OSM_USE_WAYS_INDEX", NULL) == NULL )
4136 {
4137 bUseWaysIndex = FALSE;
4138 }
4139 }
4140 else if( papoLayers[IDX_LYR_LINES]->IsUserInterested() &&
4141 !papoLayers[IDX_LYR_MULTILINESTRINGS]->IsUserInterested() &&
4142 !papoLayers[IDX_LYR_MULTIPOLYGONS]->IsUserInterested() &&
4143 !papoLayers[IDX_LYR_OTHER_RELATIONS]->IsUserInterested() )
4144 {
4145 if( CPLGetConfigOption("OSM_INDEX_WAYS", NULL) == NULL )
4146 {
4147 CPLDebug("OSM", "Disabling indexing of ways");
4148 bIndexWays = FALSE;
4149 }
4150 if( CPLGetConfigOption("OSM_USE_WAYS_INDEX", NULL) == NULL )
4151 {
4152 bUseWaysIndex = FALSE;
4153 }
4154 }
4155
4156 CSLDestroy(papszTokens);
4157
4158 return NULL;
4159 }
4160
4161 while(*pszSQLCommand == ' ')
4162 pszSQLCommand ++;
4163
4164 /* Try to analyse the SQL command to get the interest table */
4165 if( EQUALN(pszSQLCommand, "SELECT", 5) )
4166 {
4167 int bLayerAlreadyAdded = FALSE;
4168 CPLString osInterestLayers = "SET interest_layers =";
4169
4170 if( pszDialect != NULL && EQUAL(pszDialect, "SQLITE") )
4171 {
4172 std::set<LayerDesc> oSetLayers = OGRSQLiteGetReferencedLayers(pszSQLCommand);
4173 std::set<LayerDesc>::iterator oIter = oSetLayers.begin();
4174 for(; oIter != oSetLayers.end(); ++oIter)
4175 {
4176 const LayerDesc& oLayerDesc = *oIter;
4177 if( oLayerDesc.osDSName.size() == 0 )
4178 {
4179 if( bLayerAlreadyAdded ) osInterestLayers += ",";
4180 bLayerAlreadyAdded = TRUE;
4181 osInterestLayers += oLayerDesc.osLayerName;
4182 }
4183 }
4184 }
4185 else
4186 {
4187 swq_select sSelectInfo;
4188
4189 CPLPushErrorHandler(CPLQuietErrorHandler);
4190 CPLErr eErr = sSelectInfo.preparse( pszSQLCommand );
4191 CPLPopErrorHandler();
4192
4193 if( eErr == CPLE_None )
4194 {
4195 swq_select* pCurSelect = &sSelectInfo;
4196 while(pCurSelect != NULL)
4197 {
4198 for( int iTable = 0; iTable < pCurSelect->table_count; iTable++ )
4199 {
4200 swq_table_def *psTableDef = pCurSelect->table_defs + iTable;
4201 if( psTableDef->data_source == NULL )
4202 {
4203 if( bLayerAlreadyAdded ) osInterestLayers += ",";
4204 bLayerAlreadyAdded = TRUE;
4205 osInterestLayers += psTableDef->table_name;
4206 }
4207 }
4208 pCurSelect = pCurSelect->poOtherSelect;
4209 }
4210 }
4211 }
4212
4213 if( bLayerAlreadyAdded )
4214 {
4215 /* Backup current optimization parameters */
4216 abSavedDeclaredInterest.resize(0);
4217 for(int i=0; i < nLayers; i++)
4218 {
4219 abSavedDeclaredInterest.push_back(papoLayers[i]->IsUserInterested());
4220 }
4221 bIndexPointsBackup = bIndexPoints;
4222 bUsePointsIndexBackup = bUsePointsIndex;
4223 bIndexWaysBackup = bIndexWays;
4224 bUseWaysIndexBackup = bUseWaysIndex;
4225
4226 /* Update optimization parameters */
4227 ExecuteSQL(osInterestLayers, NULL, NULL);
4228
4229 ResetReading();
4230
4231 /* Run the request */
4232 poResultSetLayer = OGRDataSource::ExecuteSQL( pszSQLCommand,
4233 poSpatialFilter,
4234 pszDialect );
4235
4236 /* If the user explicitly run a COUNT() request, then do it ! */
4237 if( poResultSetLayer )
4238 {
4239 if( pszDialect != NULL && EQUAL(pszDialect, "SQLITE") )
4240 {
4241 poResultSetLayer = new OGROSMResultLayerDecorator(
4242 poResultSetLayer, GetName(), osInterestLayers);
4243 }
4244 bIsFeatureCountEnabled = TRUE;
4245 }
4246
4247 return poResultSetLayer;
4248 }
4249 }
4250
4251 return OGRDataSource::ExecuteSQL( pszSQLCommand,
4252 poSpatialFilter,
4253 pszDialect );
4254 }
4255
4256 /************************************************************************/
4257 /* ReleaseResultSet() */
4258 /************************************************************************/
4259
ReleaseResultSet(OGRLayer * poLayer)4260 void OGROSMDataSource::ReleaseResultSet( OGRLayer * poLayer )
4261
4262 {
4263 if( poLayer != NULL && poLayer == poResultSetLayer )
4264 {
4265 poResultSetLayer = NULL;
4266
4267 bIsFeatureCountEnabled = FALSE;
4268
4269 /* Restore backup'ed optimization parameters */
4270 for(int i=0; i < nLayers; i++)
4271 {
4272 papoLayers[i]->SetDeclareInterest(abSavedDeclaredInterest[i]);
4273 }
4274 if( bIndexPointsBackup && !bIndexPoints )
4275 CPLDebug("OSM", "Re-enabling indexing of nodes");
4276 bIndexPoints = bIndexPointsBackup;
4277 bUsePointsIndex = bUsePointsIndexBackup;
4278 if( bIndexWaysBackup && !bIndexWays )
4279 CPLDebug("OSM", "Re-enabling indexing of ways");
4280 bIndexWays = bIndexWaysBackup;
4281 bUseWaysIndex = bUseWaysIndexBackup;
4282 abSavedDeclaredInterest.resize(0);
4283 }
4284
4285 delete poLayer;
4286 }
4287
4288 /************************************************************************/
4289 /* IsInterleavedReading() */
4290 /************************************************************************/
4291
IsInterleavedReading()4292 int OGROSMDataSource::IsInterleavedReading()
4293 {
4294 if( bInterleavedReading < 0 )
4295 {
4296 bInterleavedReading = CSLTestBoolean(
4297 CPLGetConfigOption("OGR_INTERLEAVED_READING", "NO"));
4298 CPLDebug("OSM", "OGR_INTERLEAVED_READING = %d", bInterleavedReading);
4299 }
4300 return bInterleavedReading;
4301 }
4302