1 /******************************************************************************
2 *
3 * Project: WMS Client Driver
4 * Purpose: Implementation of Dataset and RasterBand classes for WMS
5 * and other similar services.
6 * Author: Adam Nowacki, nowak@xpam.de
7 *
8 ******************************************************************************
9 * Copyright (c) 2007, Adam Nowacki
10 * Copyright (c) 2017, Dmitry Baryshnikov, <polimax@mail.ru>
11 * Copyright (c) 2017, NextGIS, <info@nextgis.com>
12 *
13 * Permission is hereby granted, free of charge, to any person obtaining a
14 * copy of this software and associated documentation files (the "Software"),
15 * to deal in the Software without restriction, including without limitation
16 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
17 * and/or sell copies of the Software, and to permit persons to whom the
18 * Software is furnished to do so, subject to the following conditions:
19 *
20 * The above copyright notice and this permission notice shall be included
21 * in all copies or substantial portions of the Software.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
24 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
26 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
29 * DEALINGS IN THE SOFTWARE.
30 ****************************************************************************/
31
32 #include "cpl_md5.h"
33 #include "wmsdriver.h"
34
35 CPL_CVSID("$Id: gdalwmscache.cpp 816ac3854ae287862bcafe49aa66f201dda93274 2020-09-24 20:05:33 +0300 Idan Miara $")
36
37
CleanCacheThread(void * pData)38 static void CleanCacheThread( void *pData )
39 {
40 GDALWMSCache *pCache = static_cast<GDALWMSCache *>(pData);
41 pCache->Clean();
42 }
43
44 //------------------------------------------------------------------------------
45 // GDALWMSFileCache
46 //------------------------------------------------------------------------------
47 class GDALWMSFileCache : public GDALWMSCacheImpl
48 {
49 public:
GDALWMSFileCache(const CPLString & soPath,CPLXMLNode * pConfig)50 GDALWMSFileCache(const CPLString& soPath, CPLXMLNode *pConfig) :
51 GDALWMSCacheImpl(soPath, pConfig),
52 m_osPostfix(""),
53 m_nDepth(2),
54 m_nExpires(604800), // 7 days
55 m_nMaxSize(67108864), // 64 Mb
56 m_nCleanThreadRunTimeout(120) // 3 min
57 {
58 const char *pszCacheDepth = CPLGetXMLValue( pConfig, "Depth", "2" );
59 if( pszCacheDepth != nullptr )
60 m_nDepth = atoi( pszCacheDepth );
61
62 const char *pszCacheExtension = CPLGetXMLValue( pConfig, "Extension", nullptr );
63 if( pszCacheExtension != nullptr )
64 m_osPostfix = pszCacheExtension;
65
66 const char *pszCacheExpires = CPLGetXMLValue( pConfig, "Expires", nullptr );
67 if( pszCacheExpires != nullptr )
68 {
69 m_nExpires = atoi( pszCacheExpires );
70 CPLDebug("WMS", "Cache expires in %d sec", m_nExpires);
71 }
72
73 const char *pszCacheMaxSize = CPLGetXMLValue( pConfig, "MaxSize", nullptr );
74 if( pszCacheMaxSize != nullptr )
75 m_nMaxSize = atol( pszCacheMaxSize );
76
77 const char *pszCleanThreadRunTimeout = CPLGetXMLValue( pConfig, "CleanTimeout", nullptr );
78 if( pszCleanThreadRunTimeout != nullptr )
79 {
80 m_nCleanThreadRunTimeout = atoi( pszCleanThreadRunTimeout );
81 CPLDebug("WMS", "Clean Thread Run Timeout is %d sec", m_nCleanThreadRunTimeout);
82 }
83 }
84
GetCleanThreadRunTimeout()85 virtual int GetCleanThreadRunTimeout() override
86 {
87 return m_nCleanThreadRunTimeout;
88 }
89
Insert(const char * pszKey,const CPLString & osFileName)90 virtual CPLErr Insert(const char *pszKey, const CPLString &osFileName) override
91 {
92 // Warns if it fails to write, but returns success
93 CPLString soFilePath = GetFilePath( pszKey );
94 MakeDirs( CPLGetDirname(soFilePath) );
95 if ( CPLCopyFile( soFilePath, osFileName ) == CE_None)
96 return CE_None;
97 // Warn if it fails after folder creation
98 CPLError( CE_Warning, CPLE_FileIO, "Error writing to WMS cache %s",
99 m_soPath.c_str() );
100 return CE_None;
101 }
102
GetItemStatus(const char * pszKey) const103 virtual enum GDALWMSCacheItemStatus GetItemStatus(const char *pszKey) const override
104 {
105 VSIStatBufL sStatBuf;
106 if( VSIStatL( GetFilePath(pszKey), &sStatBuf ) == 0 )
107 {
108 long seconds = static_cast<long>( time( nullptr ) - sStatBuf.st_mtime );
109 return seconds < m_nExpires ? CACHE_ITEM_OK : CACHE_ITEM_EXPIRED;
110 }
111 return CACHE_ITEM_NOT_FOUND;
112 }
113
GetDataset(const char * pszKey,char ** papszOpenOptions) const114 virtual GDALDataset* GetDataset(const char *pszKey, char **papszOpenOptions) const override
115 {
116 return reinterpret_cast<GDALDataset*>(
117 GDALOpenEx( GetFilePath( pszKey ), GDAL_OF_RASTER |
118 GDAL_OF_READONLY | GDAL_OF_VERBOSE_ERROR, nullptr,
119 papszOpenOptions, nullptr ) );
120 }
121
Clean()122 virtual void Clean() override
123 {
124 char **papszList = VSIReadDirRecursive( m_soPath );
125 if( papszList == nullptr )
126 {
127 return;
128 }
129
130 int counter = 0;
131 std::vector<int> toDelete;
132 long nSize = 0;
133 time_t nTime = time( nullptr );
134 while( papszList[counter] != nullptr )
135 {
136 const char* pszPath = CPLFormFilename( m_soPath, papszList[counter], nullptr );
137 VSIStatBufL sStatBuf;
138 if( VSIStatL( pszPath, &sStatBuf ) == 0 )
139 {
140 if( !VSI_ISDIR( sStatBuf.st_mode ) )
141 {
142 long seconds = static_cast<long>( nTime - sStatBuf.st_mtime );
143 if(seconds > m_nExpires)
144 {
145 toDelete.push_back(counter);
146 }
147
148 nSize += static_cast<long>( sStatBuf.st_size );
149 }
150 }
151 counter++;
152 }
153
154 if( nSize > m_nMaxSize )
155 {
156 CPLDebug( "WMS", "Delete %u items from cache",
157 static_cast<unsigned int>(toDelete.size()) );
158 for( size_t i = 0; i < toDelete.size(); ++i )
159 {
160 const char* pszPath = CPLFormFilename( m_soPath,
161 papszList[toDelete[i]],
162 nullptr );
163 VSIUnlink( pszPath );
164 }
165 }
166
167 CSLDestroy(papszList);
168 }
169
170 private:
GetFilePath(const char * pszKey) const171 CPLString GetFilePath(const char* pszKey) const
172 {
173 CPLString soHash( CPLMD5String( pszKey ) );
174 CPLString soCacheFile( m_soPath );
175
176 if( !soCacheFile.empty() && soCacheFile.back() != '/' )
177 {
178 soCacheFile.append(1, '/');
179 }
180
181 for( int i = 0; i < m_nDepth; ++i )
182 {
183 soCacheFile.append( 1, soHash[i] );
184 soCacheFile.append( 1, '/' );
185 }
186 soCacheFile.append( soHash );
187 soCacheFile.append( m_osPostfix );
188 return soCacheFile;
189 }
190
MakeDirs(const char * pszPath)191 static void MakeDirs(const char *pszPath)
192 {
193 if( IsPathExists( pszPath ) )
194 {
195 return;
196 }
197 // Recursive makedirs, ignoring errors
198 const char *pszDirPath = CPLGetDirname( pszPath );
199 MakeDirs( pszDirPath );
200
201 VSIMkdir( pszPath, 0744 );
202 }
203
IsPathExists(const char * pszPath)204 static bool IsPathExists(const char *pszPath)
205 {
206 VSIStatBufL sbuf;
207 return VSIStatL( pszPath, &sbuf ) == 0;
208 }
209
210 private:
211 CPLString m_osPostfix;
212 int m_nDepth;
213 int m_nExpires;
214 long m_nMaxSize;
215 int m_nCleanThreadRunTimeout;
216 };
217
218 //------------------------------------------------------------------------------
219 // GDALWMSCache
220 //------------------------------------------------------------------------------
221
GDALWMSCache()222 GDALWMSCache::GDALWMSCache() :
223 m_osCachePath("./gdalwmscache"),
224 m_bIsCleanThreadRunning(false),
225 m_nCleanThreadLastRunTime(0),
226 m_poCache(nullptr),
227 m_hThread(nullptr)
228 {
229
230 }
231
~GDALWMSCache()232 GDALWMSCache::~GDALWMSCache()
233 {
234 if( m_hThread )
235 CPLJoinThread(m_hThread);
236 delete m_poCache;
237 }
238
Initialize(const char * pszUrl,CPLXMLNode * pConfig)239 CPLErr GDALWMSCache::Initialize(const char *pszUrl, CPLXMLNode *pConfig) {
240 const char *pszXmlCachePath = CPLGetXMLValue( pConfig, "Path", nullptr );
241 const char *pszUserCachePath = CPLGetConfigOption( "GDAL_DEFAULT_WMS_CACHE_PATH",
242 nullptr );
243 if( pszXmlCachePath != nullptr )
244 {
245 m_osCachePath = pszXmlCachePath;
246 }
247 else if( pszUserCachePath != nullptr )
248 {
249 m_osCachePath = pszUserCachePath;
250 }
251
252 // Separate folder for each unique dataset url
253 if( CPLTestBool( CPLGetXMLValue( pConfig, "Unique", "True" ) ) )
254 {
255 m_osCachePath = CPLFormFilename( m_osCachePath, CPLMD5String( pszUrl ), nullptr );
256 }
257
258 // TODO: Add sqlite db cache type
259 const char *pszType = CPLGetXMLValue( pConfig, "Type", "file" );
260 if( EQUAL(pszType, "file") )
261 {
262 m_poCache = new GDALWMSFileCache(m_osCachePath, pConfig);
263 }
264
265 return CE_None;
266 }
267
Insert(const char * pszKey,const CPLString & soFileName)268 CPLErr GDALWMSCache::Insert(const char *pszKey, const CPLString &soFileName)
269 {
270 if( m_poCache != nullptr && pszKey != nullptr )
271 {
272 // Add file to cache
273 CPLErr result = m_poCache->Insert(pszKey, soFileName);
274 if( result == CE_None )
275 {
276 // Start clean thread
277 int cleanThreadRunTimeout = m_poCache->GetCleanThreadRunTimeout();
278 if( cleanThreadRunTimeout > 0 &&
279 !m_bIsCleanThreadRunning &&
280 time(nullptr) - m_nCleanThreadLastRunTime > cleanThreadRunTimeout )
281 {
282 if( m_hThread )
283 CPLJoinThread(m_hThread);
284 m_bIsCleanThreadRunning = true;
285 m_hThread = CPLCreateJoinableThread(CleanCacheThread, this);
286 }
287 }
288 return result;
289 }
290
291 return CE_Failure;
292 }
293
GetItemStatus(const char * pszKey) const294 enum GDALWMSCacheItemStatus GDALWMSCache::GetItemStatus(const char *pszKey) const
295 {
296 if( m_poCache != nullptr )
297 {
298 return m_poCache->GetItemStatus(pszKey);
299 }
300 return CACHE_ITEM_NOT_FOUND;
301 }
302
GetDataset(const char * pszKey,char ** papszOpenOptions) const303 GDALDataset* GDALWMSCache::GetDataset(const char *pszKey,
304 char **papszOpenOptions) const
305 {
306 if( m_poCache != nullptr )
307 {
308 return m_poCache->GetDataset(pszKey, papszOpenOptions);
309 }
310 return nullptr;
311 }
312
Clean()313 void GDALWMSCache::Clean()
314 {
315 if( m_poCache != nullptr )
316 {
317 CPLDebug("WMS", "Clean cache");
318 m_poCache->Clean();
319 }
320
321 m_nCleanThreadLastRunTime = time( nullptr );
322 m_bIsCleanThreadRunning = false;
323 }
324