1 /****************************************************************************** 2 * 3 * Project: FMEObjects Translator 4 * Purpose: Implement the OGRFMECacheIndex class, a mechanism to manage a 5 * persistent index list of cached datasets. 6 * Author: Frank Warmerdam, warmerdam@pobox.com 7 * 8 ****************************************************************************** 9 * Copyright (c) 2002 Safe Software Inc. 10 * 11 * Permission is hereby granted, free of charge, to any person obtaining a 12 * copy of this software and associated documentation files (the "Software"), 13 * to deal in the Software without restriction, including without limitation 14 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 15 * and/or sell copies of the Software, and to permit persons to whom the 16 * Software is furnished to do so, subject to the following conditions: 17 * 18 * The above copyright notice and this permission notice shall be included 19 * in all copies or substantial portions of the Software. 20 * 21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 24 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 27 * DEALINGS IN THE SOFTWARE. 28 ****************************************************************************/ 29 30 #include "fme2ogr.h" 31 #include "cpl_multiproc.h" 32 #include "cpl_conv.h" 33 #include "cpl_minixml.h" 34 #include "cpl_string.h" 35 #include <time.h> 36 37 CPL_CVSID("$Id: ogrfmecacheindex.cpp 002b050d9a9ef403a732c1210784736ef97216d4 2018-04-09 21:34:55 +0200 Even Rouault $") 38 39 /************************************************************************/ 40 /* OGRFMECacheIndex() */ 41 /************************************************************************/ 42 43 OGRFMECacheIndex::OGRFMECacheIndex( const char * pszPathIn ) 44 45 { 46 psTree = NULL; 47 pszPath = CPLStrdup( pszPathIn ); 48 hLock = NULL; 49 } 50 51 /************************************************************************/ 52 /* ~OGRFMECacheIndex() */ 53 /************************************************************************/ 54 55 OGRFMECacheIndex::~OGRFMECacheIndex() 56 57 { 58 if( psTree != NULL ) 59 { 60 Unlock(); 61 CPLDestroyXMLNode( psTree ); 62 psTree = NULL; 63 } 64 CPLFree( pszPath ); 65 } 66 67 /************************************************************************/ 68 /* Lock() */ 69 /************************************************************************/ 70 71 int OGRFMECacheIndex::Lock() 72 73 { 74 if( pszPath == NULL ) 75 return FALSE; 76 77 hLock = CPLLockFile( pszPath, 5.0 ); 78 79 return hLock != NULL; 80 } 81 82 /************************************************************************/ 83 /* Unlock() */ 84 /************************************************************************/ 85 86 int OGRFMECacheIndex::Unlock() 87 88 { 89 if( pszPath == NULL || hLock == NULL ) 90 return FALSE; 91 92 CPLUnlockFile( hLock ); 93 hLock = NULL; 94 95 return TRUE; 96 } 97 98 /************************************************************************/ 99 /* Load() */ 100 /************************************************************************/ 101 102 int OGRFMECacheIndex::Load() 103 104 { 105 /* -------------------------------------------------------------------- */ 106 /* Lock the cache index file if not already locked. */ 107 /* -------------------------------------------------------------------- */ 108 if( hLock == NULL && !Lock() ) 109 return FALSE; 110 111 if( psTree != NULL ) 112 { 113 CPLDestroyXMLNode( psTree ); 114 psTree = NULL; 115 } 116 117 /* -------------------------------------------------------------------- */ 118 /* Open the index file. If we don't get it, we assume it is */ 119 /* because it doesn't exist, and we create a "stub" tree in */ 120 /* memory. */ 121 /* -------------------------------------------------------------------- */ 122 FILE *fpIndex; 123 int nLength; 124 char *pszIndexBuffer; 125 126 fpIndex = VSIFOpen( GetPath(), "rb" ); 127 if( fpIndex == NULL ) 128 { 129 psTree = CPLCreateXMLNode( NULL, CXT_Element, "OGRFMECacheIndex" ); 130 return TRUE; 131 } 132 133 /* -------------------------------------------------------------------- */ 134 /* Load the data from the file. */ 135 /* -------------------------------------------------------------------- */ 136 VSIFSeek( fpIndex, 0, SEEK_END ); 137 nLength = VSIFTell( fpIndex ); 138 VSIFSeek( fpIndex, 0, SEEK_SET ); 139 140 pszIndexBuffer = (char *) CPLMalloc(nLength+1); 141 if( (int) VSIFRead( pszIndexBuffer, 1, nLength, fpIndex ) != nLength ) 142 { 143 CPLError( CE_Failure, CPLE_FileIO, 144 "Read of %d byte index file failed.", nLength ); 145 return FALSE; 146 } 147 VSIFClose( fpIndex ); 148 149 /* -------------------------------------------------------------------- */ 150 /* Parse the result into an inmemory XML tree. */ 151 /* -------------------------------------------------------------------- */ 152 pszIndexBuffer[nLength] = '\0'; 153 psTree = CPLParseXMLString( pszIndexBuffer ); 154 155 CPLFree( pszIndexBuffer ); 156 157 return psTree != NULL; 158 } 159 160 /************************************************************************/ 161 /* Save() */ 162 /************************************************************************/ 163 164 int OGRFMECacheIndex::Save() 165 166 { 167 if( hLock == NULL ) 168 return FALSE; 169 170 /* -------------------------------------------------------------------- */ 171 /* Convert the XML tree into one big character buffer, and */ 172 /* write it out. */ 173 /* -------------------------------------------------------------------- */ 174 char *pszIndexBuffer = CPLSerializeXMLTree( psTree ); 175 176 if( pszIndexBuffer == NULL ) 177 return FALSE; 178 179 FILE *fpIndex = VSIFOpen( GetPath(), "wb" ); 180 if( fpIndex == NULL ) 181 return FALSE; 182 183 VSIFWrite( pszIndexBuffer, 1, strlen(pszIndexBuffer), fpIndex ); 184 CPLFree( pszIndexBuffer ); 185 186 VSIFClose( fpIndex ); 187 188 Unlock(); 189 190 return TRUE; 191 } 192 193 /************************************************************************/ 194 /* FindMatch() */ 195 /* */ 196 /* Find a DataSource subtree that matches the passed in */ 197 /* component values. */ 198 /************************************************************************/ 199 200 CPLXMLNode *OGRFMECacheIndex::FindMatch( const char *pszDriver, 201 const char *pszDataset, 202 IFMEStringArray &oUserDirectives ) 203 204 { 205 CPLXMLNode *psCDS; 206 207 if( psTree == NULL ) 208 return NULL; 209 210 for( psCDS = psTree->psChild; psCDS != NULL; psCDS = psCDS->psNext ) 211 { 212 if( !EQUAL(pszDriver,CPLGetXMLValue(psCDS,"Driver","")) ) 213 continue; 214 215 if( !EQUAL(pszDataset,CPLGetXMLValue(psCDS,"DSName","")) ) 216 continue; 217 218 CPLXMLNode *psDirective; 219 int bMatch = TRUE; 220 int iDir; 221 222 psDirective = CPLGetXMLNode( psCDS, "UserDirectives.Directive" ); 223 for( iDir = 0; 224 iDir < (int)oUserDirectives.entries() && bMatch; 225 iDir++ ) 226 { 227 if( psDirective == NULL || psDirective->psChild == NULL ) 228 bMatch = FALSE; 229 else if( !EQUAL(psDirective->psChild->pszValue, 230 oUserDirectives(iDir)) ) 231 bMatch = FALSE; 232 else 233 psDirective = psDirective->psNext; 234 } 235 236 if( iDir < (int) oUserDirectives.entries() || !bMatch 237 || (psDirective != NULL && psDirective->psNext != NULL) ) 238 continue; 239 240 return psCDS; 241 } 242 243 return NULL; 244 } 245 246 /************************************************************************/ 247 /* Touch() */ 248 /* */ 249 /* Update the LastUseTime on the passed datasource. */ 250 /************************************************************************/ 251 252 void OGRFMECacheIndex::Touch( CPLXMLNode *psDSNode ) 253 254 { 255 if( psDSNode == NULL || !EQUAL(psDSNode->pszValue,"DataSource") ) 256 return; 257 258 /* -------------------------------------------------------------------- */ 259 /* Prepare the new time value to use. */ 260 /* -------------------------------------------------------------------- */ 261 char szNewTime[32]; 262 263 sprintf( szNewTime, "%lu", (unsigned long) time(NULL) ); 264 265 /* -------------------------------------------------------------------- */ 266 /* Set or insert LastUseTime into dataset. */ 267 /* -------------------------------------------------------------------- */ 268 CPLSetXMLValue( psDSNode, "LastUseTime", szNewTime ); 269 } 270 271 /************************************************************************/ 272 /* Reference() */ 273 /************************************************************************/ 274 275 void OGRFMECacheIndex::Reference( CPLXMLNode *psDSNode ) 276 277 { 278 if( psDSNode == NULL || !EQUAL(psDSNode->pszValue,"DataSource") ) 279 return; 280 281 char szNewRefCount[32]; 282 283 sprintf( szNewRefCount, "%d", 284 atoi(CPLGetXMLValue(psDSNode, "RefCount", "0")) + 1 ); 285 286 CPLSetXMLValue( psDSNode, "RefCount", szNewRefCount ); 287 288 Touch( psDSNode ); 289 } 290 291 /************************************************************************/ 292 /* Dereference() */ 293 /************************************************************************/ 294 295 void OGRFMECacheIndex::Dereference( CPLXMLNode *psDSNode ) 296 297 { 298 if( psDSNode == NULL 299 || !EQUAL(psDSNode->pszValue,"DataSource") 300 || CPLGetXMLNode(psDSNode,"RefCount") == NULL ) 301 return; 302 303 char szNewRefCount[32]; 304 int nRefCount = atoi(CPLGetXMLValue(psDSNode, "RefCount", "1")); 305 if( nRefCount < 1 ) 306 nRefCount = 1; 307 308 sprintf( szNewRefCount, "%d", nRefCount-1 ); 309 310 CPLSetXMLValue( psDSNode, "RefCount", szNewRefCount ); 311 312 Touch( psDSNode ); 313 } 314 315 /************************************************************************/ 316 /* Add() */ 317 /* */ 318 /* Note that Add() takes over ownership of the passed tree. */ 319 /************************************************************************/ 320 321 void OGRFMECacheIndex::Add( CPLXMLNode *psDSNode ) 322 323 { 324 if( psTree == NULL ) 325 { 326 CPLAssert( false ); 327 return; 328 } 329 330 if( psDSNode == NULL || !EQUAL(psDSNode->pszValue,"DataSource") ) 331 return; 332 333 psDSNode->psNext = psTree->psChild; 334 psTree->psChild = psDSNode; 335 336 337 /* -------------------------------------------------------------------- */ 338 /* Prepare the creation time value to use. */ 339 /* -------------------------------------------------------------------- */ 340 char szNewTime[32]; 341 342 sprintf( szNewTime, "%lu", (unsigned long) time(NULL) ); 343 344 /* -------------------------------------------------------------------- */ 345 /* Set or insert CreationTime into dataset. */ 346 /* -------------------------------------------------------------------- */ 347 CPLSetXMLValue( psDSNode, "CreationTime", szNewTime ); 348 } 349 350 /************************************************************************/ 351 /* ExpireOldCaches() */ 352 /* */ 353 /* Make a pass over all the cache index entries. Remove (and */ 354 /* free the associated FME spatial caches) for any entries that */ 355 /* haven't been touched for a long time. Note that two */ 356 /* different timeouts apply. One is for layers with a RefCount */ 357 /* of 0 and the other (longer time) is for those with a */ 358 /* non-zero refcount. Even if the RefCount is non-zero we */ 359 /* assume this may because a program crashed during its run. */ 360 /************************************************************************/ 361 362 int OGRFMECacheIndex::ExpireOldCaches( IFMESession *poSession ) 363 364 { 365 CPLXMLNode *psDSNode, *psLastDSNode = NULL; 366 unsigned long nCurTime = time(NULL); 367 int bChangeMade = FALSE; 368 369 if( psTree == NULL ) 370 return FALSE; 371 372 for( psLastDSNode = NULL; true; psLastDSNode = psDSNode ) 373 { 374 if( psLastDSNode != NULL ) 375 psDSNode = psLastDSNode->psNext; 376 else 377 psDSNode = psTree->psChild; 378 if( psDSNode == NULL ) 379 break; 380 381 if( !EQUAL(psDSNode->pszValue,"DataSource") ) 382 continue; 383 384 /* -------------------------------------------------------------------- */ 385 /* When was this datasource last accessed? */ 386 /* -------------------------------------------------------------------- */ 387 unsigned long nLastUseTime = 0; 388 389 sscanf( CPLGetXMLValue( psDSNode, "LastUseTime", "0" ), 390 "%lu", &nLastUseTime ); 391 392 /* -------------------------------------------------------------------- */ 393 /* When was this datasource created. */ 394 /* -------------------------------------------------------------------- */ 395 unsigned long nCreationTime = 0; 396 397 sscanf( CPLGetXMLValue( psDSNode, "CreationTime", "0" ), 398 "%lu", &nCreationTime ); 399 400 /* -------------------------------------------------------------------- */ 401 /* Do we want to delete this datasource according to our */ 402 /* retention and ref timeout rules? */ 403 /* -------------------------------------------------------------------- */ 404 int bCleanup = FALSE; 405 406 // Do we want to cleanup this node? 407 if( atoi(CPLGetXMLValue( psDSNode, "RefCount", "0" )) > 0 408 && nLastUseTime + FMECACHE_REF_TIMEOUT < nCurTime ) 409 bCleanup = TRUE; 410 411 if( atoi(CPLGetXMLValue( psDSNode, "RefCount", "0" )) < 1 412 && nLastUseTime + FMECACHE_RETENTION < nCurTime ) 413 bCleanup = TRUE; 414 415 if( atoi(CPLGetXMLValue( psDSNode, "RefCount", "0" )) < 1 416 && nCreationTime + FMECACHE_MAX_RETENTION < nCurTime ) 417 bCleanup = TRUE; 418 419 if( !bCleanup ) 420 continue; 421 422 bChangeMade = TRUE; 423 424 CPLDebug( "OGRFMECacheIndex", 425 "ExpireOldCaches() cleaning up data source %s - %ds since last use, %ds old.", 426 CPLGetXMLValue( psDSNode, "DSName", "<missing name>" ), 427 nCurTime - nLastUseTime, 428 nCurTime - nCreationTime ); 429 430 /* -------------------------------------------------------------------- */ 431 /* Loop over all the layers, to delete the spatial caches on */ 432 /* disk. */ 433 /* -------------------------------------------------------------------- */ 434 CPLXMLNode *psLayerN; 435 436 for( psLayerN = psDSNode->psChild; 437 psLayerN != NULL; 438 psLayerN = psLayerN->psNext ) 439 { 440 IFMESpatialIndex *poIndex; 441 442 if( !EQUAL(psLayerN->pszValue,"OGRLayer") ) 443 continue; 444 445 const char *pszBase; 446 447 pszBase = CPLGetXMLValue( psLayerN, "SpatialCacheName", "" ); 448 if( EQUAL(pszBase,"") ) 449 continue; 450 451 // open, and then delete the index on close. 452 poIndex = poSession->createSpatialIndex( pszBase, "READ", NULL ); 453 454 if( poIndex == NULL ) 455 continue; 456 457 if( poIndex->open() != 0 ) 458 { 459 CPLDebug( "OGRFMECacheIndex", "Failed to open FME index %s.", 460 pszBase ); 461 462 poSession->destroySpatialIndex( poIndex ); 463 continue; 464 } 465 466 poIndex->close( FME_TRUE ); 467 poSession->destroySpatialIndex( poIndex ); 468 } 469 470 /* -------------------------------------------------------------------- */ 471 /* Remove the datasource from the tree. */ 472 /* -------------------------------------------------------------------- */ 473 if( psLastDSNode == NULL ) 474 psTree->psChild = psDSNode->psNext; 475 else 476 psLastDSNode->psNext = psDSNode->psNext; 477 478 psDSNode->psNext = NULL; 479 CPLDestroyXMLNode( psDSNode ); 480 psDSNode = psLastDSNode; 481 } 482 483 return bChangeMade; 484 } 485