1 /******************************************************************************
2 * $Id: cpl_vsil_curl_streaming.cpp 28459 2015-02-12 13:48:21Z rouault $
3 *
4 * Project: CPL - Common Portability Library
5 * Purpose: Implement VSI large file api for HTTP/FTP files in streaming mode
6 * Author: Even Rouault <even dot rouault at mines dash paris.org>
7 *
8 ******************************************************************************
9 * Copyright (c) 2012-2013, Even Rouault <even dot rouault at mines-paris dot org>
10 *
11 * Permission is hereby granted, free of charge, to any person obtaining a
12 * copy of this software and associated documentation files (the "Software"),
13 * to deal in the Software without restriction, including without limitation
14 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 * and/or sell copies of the Software, and to permit persons to whom the
16 * Software is furnished to do so, subject to the following conditions:
17 *
18 * The above copyright notice and this permission notice shall be included
19 * in all copies or substantial portions of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 * DEALINGS IN THE SOFTWARE.
28 ****************************************************************************/
29
30 #include "cpl_vsi_virtual.h"
31 #include "cpl_string.h"
32 #include "cpl_multiproc.h"
33 #include "cpl_hash_set.h"
34 #include "cpl_time.h"
35
36 CPL_CVSID("$Id: cpl_vsil_curl_streaming.cpp 28459 2015-02-12 13:48:21Z rouault $");
37
38 #if !defined(HAVE_CURL) || defined(CPL_MULTIPROC_STUB)
39
VSIInstallCurlStreamingFileHandler(void)40 void VSIInstallCurlStreamingFileHandler(void)
41 {
42 /* not supported */
43 }
44
45 #else
46
47 #include <curl/curl.h>
48
49 void VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL);
50
51 #include <map>
52
53 #define ENABLE_DEBUG 0
54
55 #define N_MAX_REGIONS 10
56
57 #define BKGND_BUFFER_SIZE (1024 * 1024)
58
59
60 /************************************************************************/
61 /* RingBuffer */
62 /************************************************************************/
63
64 class RingBuffer
65 {
66 GByte* pabyBuffer;
67 size_t nCapacity;
68 size_t nOffset;
69 size_t nLength;
70
71 public:
72 RingBuffer(size_t nCapacity = BKGND_BUFFER_SIZE);
73 ~RingBuffer();
74
GetCapacity() const75 size_t GetCapacity() const { return nCapacity; }
GetSize() const76 size_t GetSize() const { return nLength; }
77
78 void Reset();
79 void Write(void* pBuffer, size_t nSize);
80 void Read(void* pBuffer, size_t nSize);
81 };
82
RingBuffer(size_t nCapacityIn)83 RingBuffer::RingBuffer(size_t nCapacityIn)
84 {
85 pabyBuffer = (GByte*)CPLMalloc(nCapacityIn);
86 nCapacity = nCapacityIn;
87 nOffset = 0;
88 nLength = 0;
89 }
90
~RingBuffer()91 RingBuffer::~RingBuffer()
92 {
93 CPLFree(pabyBuffer);
94 }
95
Reset()96 void RingBuffer::Reset()
97 {
98 nOffset = 0;
99 nLength = 0;
100 }
101
Write(void * pBuffer,size_t nSize)102 void RingBuffer::Write(void* pBuffer, size_t nSize)
103 {
104 CPLAssert(nLength + nSize <= nCapacity);
105
106 size_t nEndOffset = (nOffset + nLength) % nCapacity;
107 size_t nSz = MIN(nSize, nCapacity - nEndOffset);
108 memcpy(pabyBuffer + nEndOffset, pBuffer, nSz);
109 if (nSz < nSize)
110 memcpy(pabyBuffer, (GByte*)pBuffer + nSz, nSize - nSz);
111
112 nLength += nSize;
113 }
114
Read(void * pBuffer,size_t nSize)115 void RingBuffer::Read(void* pBuffer, size_t nSize)
116 {
117 CPLAssert(nSize <= nLength);
118
119 if (pBuffer)
120 {
121 size_t nSz = MIN(nSize, nCapacity - nOffset);
122 memcpy(pBuffer, pabyBuffer + nOffset, nSz);
123 if (nSz < nSize)
124 memcpy((GByte*)pBuffer + nSz, pabyBuffer, nSize - nSz);
125 }
126
127 nOffset = (nOffset + nSize) % nCapacity;
128 nLength -= nSize;
129 }
130
131 /************************************************************************/
132
133 typedef enum
134 {
135 EXIST_UNKNOWN = -1,
136 EXIST_NO,
137 EXIST_YES,
138 } ExistStatus;
139
140 typedef struct
141 {
142 ExistStatus eExists;
143 int bHastComputedFileSize;
144 vsi_l_offset fileSize;
145 int bIsDirectory;
146 #ifdef notdef
147 unsigned int nChecksumOfFirst1024Bytes;
148 #endif
149 } CachedFileProp;
150
151 /************************************************************************/
152 /* VSICurlStreamingFSHandler */
153 /************************************************************************/
154
155 class VSICurlStreamingFSHandler : public VSIFilesystemHandler
156 {
157 CPLMutex *hMutex;
158
159 std::map<CPLString, CachedFileProp*> cacheFileSize;
160
161 public:
162 VSICurlStreamingFSHandler();
163 ~VSICurlStreamingFSHandler();
164
165 virtual VSIVirtualHandle *Open( const char *pszFilename,
166 const char *pszAccess);
167 virtual int Stat( const char *pszFilename, VSIStatBufL *pStatBuf, int nFlags );
168
169 void AcquireMutex();
170 void ReleaseMutex();
171
172 CachedFileProp* GetCachedFileProp(const char* pszURL);
173 };
174
175 /************************************************************************/
176 /* VSICurlStreamingHandle */
177 /************************************************************************/
178
179 class VSICurlStreamingHandle : public VSIVirtualHandle
180 {
181 private:
182 VSICurlStreamingFSHandler* poFS;
183
184 char* pszURL;
185
186 #ifdef notdef
187 unsigned int nRecomputedChecksumOfFirst1024Bytes;
188 #endif
189 vsi_l_offset curOffset;
190 vsi_l_offset fileSize;
191 int bHastComputedFileSize;
192 ExistStatus eExists;
193 int bIsDirectory;
194
195 int bCanTrustCandidateFileSize;
196 int bHasCandidateFileSize;
197 vsi_l_offset nCandidateFileSize;
198
199 int bEOF;
200
201 size_t nCachedSize;
202 GByte *pCachedData;
203
204 CURL* hCurlHandle;
205
206 volatile int bDownloadInProgress;
207 volatile int bDownloadStopped;
208 volatile int bAskDownloadEnd;
209 vsi_l_offset nRingBufferFileOffset;
210 CPLJoinableThread *hThread;
211 CPLMutex *hRingBufferMutex;
212 CPLCond *hCondProducer;
213 CPLCond *hCondConsumer;
214 RingBuffer oRingBuffer;
215 void StartDownload();
216 void StopDownload();
217 void PutRingBufferInCache();
218
219 GByte *pabyHeaderData;
220 size_t nHeaderSize;
221 vsi_l_offset nBodySize;
222 int nHTTPCode;
223
224 void AcquireMutex();
225 void ReleaseMutex();
226
227 void AddRegion( vsi_l_offset nFileOffsetStart,
228 size_t nSize,
229 GByte *pData );
230 public:
231
232 VSICurlStreamingHandle(VSICurlStreamingFSHandler* poFS, const char* pszURL);
233 ~VSICurlStreamingHandle();
234
235 virtual int Seek( vsi_l_offset nOffset, int nWhence );
236 virtual vsi_l_offset Tell();
237 virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
238 virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
239 virtual int Eof();
240 virtual int Flush();
241 virtual int Close();
242
243 void DownloadInThread();
244 int ReceivedBytes(GByte *buffer, size_t count, size_t nmemb);
245 int ReceivedBytesHeader(GByte *buffer, size_t count, size_t nmemb);
246
IsKnownFileSize() const247 int IsKnownFileSize() const { return bHastComputedFileSize; }
248 vsi_l_offset GetFileSize();
249 int Exists();
IsDirectory() const250 int IsDirectory() const { return bIsDirectory; }
251 };
252
253 /************************************************************************/
254 /* VSICurlStreamingHandle() */
255 /************************************************************************/
256
VSICurlStreamingHandle(VSICurlStreamingFSHandler * poFS,const char * pszURL)257 VSICurlStreamingHandle::VSICurlStreamingHandle(VSICurlStreamingFSHandler* poFS, const char* pszURL)
258 {
259 this->poFS = poFS;
260 this->pszURL = CPLStrdup(pszURL);
261
262 #ifdef notdef
263 nRecomputedChecksumOfFirst1024Bytes = 0;
264 #endif
265 curOffset = 0;
266
267 poFS->AcquireMutex();
268 CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
269 eExists = cachedFileProp->eExists;
270 fileSize = cachedFileProp->fileSize;
271 bHastComputedFileSize = cachedFileProp->bHastComputedFileSize;
272 bIsDirectory = cachedFileProp->bIsDirectory;
273 poFS->ReleaseMutex();
274
275 bCanTrustCandidateFileSize = TRUE;
276 bHasCandidateFileSize = FALSE;
277 nCandidateFileSize = 0;
278
279 nCachedSize = 0;
280 pCachedData = NULL;
281
282 bEOF = FALSE;
283
284 hCurlHandle = NULL;
285
286 hThread = NULL;
287 hRingBufferMutex = CPLCreateMutex();
288 ReleaseMutex();
289 hCondProducer = CPLCreateCond();
290 hCondConsumer = CPLCreateCond();
291
292 bDownloadInProgress = FALSE;
293 bDownloadStopped = FALSE;
294 bAskDownloadEnd = FALSE;
295 nRingBufferFileOffset = 0;
296
297 pabyHeaderData = NULL;
298 nHeaderSize = 0;
299 nBodySize = 0;
300 nHTTPCode = 0;
301 }
302
303 /************************************************************************/
304 /* ~VSICurlStreamingHandle() */
305 /************************************************************************/
306
~VSICurlStreamingHandle()307 VSICurlStreamingHandle::~VSICurlStreamingHandle()
308 {
309 StopDownload();
310
311 CPLFree(pszURL);
312 if (hCurlHandle != NULL)
313 curl_easy_cleanup(hCurlHandle);
314
315 CPLFree(pCachedData);
316
317 CPLFree(pabyHeaderData);
318
319 CPLDestroyMutex( hRingBufferMutex );
320 CPLDestroyCond( hCondProducer );
321 CPLDestroyCond( hCondConsumer );
322 }
323
324 /************************************************************************/
325 /* AcquireMutex() */
326 /************************************************************************/
327
AcquireMutex()328 void VSICurlStreamingHandle::AcquireMutex()
329 {
330 CPLAcquireMutex(hRingBufferMutex, 1000.0);
331 }
332
333 /************************************************************************/
334 /* ReleaseMutex() */
335 /************************************************************************/
336
ReleaseMutex()337 void VSICurlStreamingHandle::ReleaseMutex()
338 {
339 CPLReleaseMutex(hRingBufferMutex);
340 }
341
342 /************************************************************************/
343 /* Seek() */
344 /************************************************************************/
345
Seek(vsi_l_offset nOffset,int nWhence)346 int VSICurlStreamingHandle::Seek( vsi_l_offset nOffset, int nWhence )
347 {
348 if( curOffset >= BKGND_BUFFER_SIZE )
349 {
350 if (ENABLE_DEBUG)
351 CPLDebug("VSICURL", "Invalidating cache and file size due to Seek() beyond caching zone");
352 CPLFree(pCachedData);
353 pCachedData = NULL;
354 nCachedSize = 0;
355 AcquireMutex();
356 bHastComputedFileSize = FALSE;
357 fileSize = 0;
358 ReleaseMutex();
359 }
360
361 if (nWhence == SEEK_SET)
362 {
363 curOffset = nOffset;
364 }
365 else if (nWhence == SEEK_CUR)
366 {
367 curOffset = curOffset + nOffset;
368 }
369 else
370 {
371 curOffset = GetFileSize() + nOffset;
372 }
373 bEOF = FALSE;
374 return 0;
375 }
376
377 typedef struct
378 {
379 char* pBuffer;
380 size_t nSize;
381 int bIsHTTP;
382 int bIsInHeader;
383 int nHTTPCode;
384 int bDownloadHeaderOnly;
385 } WriteFuncStruct;
386
387 /************************************************************************/
388 /* VSICURLStreamingInitWriteFuncStruct() */
389 /************************************************************************/
390
VSICURLStreamingInitWriteFuncStruct(WriteFuncStruct * psStruct)391 static void VSICURLStreamingInitWriteFuncStruct(WriteFuncStruct *psStruct)
392 {
393 psStruct->pBuffer = NULL;
394 psStruct->nSize = 0;
395 psStruct->bIsHTTP = FALSE;
396 psStruct->bIsInHeader = TRUE;
397 psStruct->nHTTPCode = 0;
398 psStruct->bDownloadHeaderOnly = FALSE;
399 }
400
401 /************************************************************************/
402 /* VSICurlStreamingHandleWriteFuncForHeader() */
403 /************************************************************************/
404
VSICurlStreamingHandleWriteFuncForHeader(void * buffer,size_t count,size_t nmemb,void * req)405 static int VSICurlStreamingHandleWriteFuncForHeader(void *buffer, size_t count, size_t nmemb, void *req)
406 {
407 WriteFuncStruct* psStruct = (WriteFuncStruct*) req;
408 size_t nSize = count * nmemb;
409
410 char* pNewBuffer = (char*) VSIRealloc(psStruct->pBuffer,
411 psStruct->nSize + nSize + 1);
412 if (pNewBuffer)
413 {
414 psStruct->pBuffer = pNewBuffer;
415 memcpy(psStruct->pBuffer + psStruct->nSize, buffer, nSize);
416 psStruct->pBuffer[psStruct->nSize + nSize] = '\0';
417 if (psStruct->bIsHTTP && psStruct->bIsInHeader)
418 {
419 char* pszLine = psStruct->pBuffer + psStruct->nSize;
420 if (EQUALN(pszLine, "HTTP/1.0 ", 9) ||
421 EQUALN(pszLine, "HTTP/1.1 ", 9))
422 psStruct->nHTTPCode = atoi(pszLine + 9);
423
424 if (pszLine[0] == '\r' || pszLine[0] == '\n')
425 {
426 if (psStruct->bDownloadHeaderOnly)
427 {
428 /* If moved permanently/temporarily, go on. Otherwise stop now*/
429 if (!(psStruct->nHTTPCode == 301 || psStruct->nHTTPCode == 302))
430 return 0;
431 }
432 else
433 {
434 psStruct->bIsInHeader = FALSE;
435 }
436 }
437 }
438 psStruct->nSize += nSize;
439 return nmemb;
440 }
441 else
442 {
443 return 0;
444 }
445 }
446
447
448 /************************************************************************/
449 /* GetFileSize() */
450 /************************************************************************/
451
GetFileSize()452 vsi_l_offset VSICurlStreamingHandle::GetFileSize()
453 {
454 WriteFuncStruct sWriteFuncData;
455 WriteFuncStruct sWriteFuncHeaderData;
456
457 AcquireMutex();
458 if (bHastComputedFileSize)
459 {
460 vsi_l_offset nRet = fileSize;
461 ReleaseMutex();
462 return nRet;
463 }
464 ReleaseMutex();
465
466 #if LIBCURL_VERSION_NUM < 0x070B00
467 /* Curl 7.10.X doesn't manage to unset the CURLOPT_RANGE that would have been */
468 /* previously set, so we have to reinit the connection handle */
469 if (hCurlHandle)
470 {
471 curl_easy_cleanup(hCurlHandle);
472 hCurlHandle = curl_easy_init();
473 }
474 #endif
475
476 CURL* hLocalHandle = curl_easy_init();
477
478 VSICurlSetOptions(hLocalHandle, pszURL);
479
480 VSICURLStreamingInitWriteFuncStruct(&sWriteFuncHeaderData);
481
482 /* HACK for mbtiles driver: proper fix would be to auto-detect servers that don't accept HEAD */
483 /* http://a.tiles.mapbox.com/v3/ doesn't accept HEAD, so let's start a GET */
484 /* and interrupt is as soon as the header is found */
485 if (strstr(pszURL, ".tiles.mapbox.com/") != NULL)
486 {
487 curl_easy_setopt(hLocalHandle, CURLOPT_HEADERDATA, &sWriteFuncHeaderData);
488 curl_easy_setopt(hLocalHandle, CURLOPT_HEADERFUNCTION, VSICurlStreamingHandleWriteFuncForHeader);
489
490 sWriteFuncHeaderData.bIsHTTP = strncmp(pszURL, "http", 4) == 0;
491 sWriteFuncHeaderData.bDownloadHeaderOnly = TRUE;
492 }
493 else
494 {
495 curl_easy_setopt(hLocalHandle, CURLOPT_NOBODY, 1);
496 curl_easy_setopt(hLocalHandle, CURLOPT_HTTPGET, 0);
497 curl_easy_setopt(hLocalHandle, CURLOPT_HEADER, 1);
498 }
499
500 /* We need that otherwise OSGEO4W's libcurl issue a dummy range request */
501 /* when doing a HEAD when recycling connections */
502 curl_easy_setopt(hLocalHandle, CURLOPT_RANGE, NULL);
503
504 /* Bug with older curl versions (<=7.16.4) and FTP. See http://curl.haxx.se/mail/lib-2007-08/0312.html */
505 VSICURLStreamingInitWriteFuncStruct(&sWriteFuncData);
506 curl_easy_setopt(hLocalHandle, CURLOPT_WRITEDATA, &sWriteFuncData);
507 curl_easy_setopt(hLocalHandle, CURLOPT_WRITEFUNCTION, VSICurlStreamingHandleWriteFuncForHeader);
508
509 char szCurlErrBuf[CURL_ERROR_SIZE+1];
510 szCurlErrBuf[0] = '\0';
511 curl_easy_setopt(hLocalHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
512
513 double dfSize = 0;
514 curl_easy_perform(hLocalHandle);
515
516 AcquireMutex();
517
518 eExists = EXIST_UNKNOWN;
519 bHastComputedFileSize = TRUE;
520
521 if (strncmp(pszURL, "ftp", 3) == 0)
522 {
523 if (sWriteFuncData.pBuffer != NULL &&
524 strncmp(sWriteFuncData.pBuffer, "Content-Length: ", strlen( "Content-Length: ")) == 0)
525 {
526 const char* pszBuffer = sWriteFuncData.pBuffer + strlen("Content-Length: ");
527 eExists = EXIST_YES;
528 fileSize = CPLScanUIntBig(pszBuffer, sWriteFuncData.nSize - strlen("Content-Length: "));
529 if (ENABLE_DEBUG)
530 CPLDebug("VSICURL", "GetFileSize(%s)=" CPL_FRMT_GUIB,
531 pszURL, fileSize);
532 }
533 }
534
535 if (eExists != EXIST_YES)
536 {
537 CURLcode code = curl_easy_getinfo(hLocalHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dfSize );
538 if (code == 0)
539 {
540 eExists = EXIST_YES;
541 if (dfSize < 0)
542 fileSize = 0;
543 else
544 fileSize = (GUIntBig)dfSize;
545 }
546 else
547 {
548 eExists = EXIST_NO;
549 fileSize = 0;
550 CPLError(CE_Failure, CPLE_AppDefined,
551 "VSICurlStreamingHandle::GetFileSize failed");
552 }
553
554 long response_code = 0;
555 curl_easy_getinfo(hLocalHandle, CURLINFO_HTTP_CODE, &response_code);
556 if (response_code != 200)
557 {
558 eExists = EXIST_NO;
559 fileSize = 0;
560 }
561
562 /* Try to guess if this is a directory. Generally if this is a directory, */
563 /* curl will retry with an URL with slash added */
564 char *pszEffectiveURL = NULL;
565 curl_easy_getinfo(hLocalHandle, CURLINFO_EFFECTIVE_URL, &pszEffectiveURL);
566 if (pszEffectiveURL != NULL &&
567 strncmp(pszURL, pszEffectiveURL, strlen(pszURL)) == 0 &&
568 pszEffectiveURL[strlen(pszURL)] == '/')
569 {
570 eExists = EXIST_YES;
571 fileSize = 0;
572 bIsDirectory = TRUE;
573 }
574
575 if (ENABLE_DEBUG)
576 CPLDebug("VSICURL", "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d",
577 pszURL, fileSize, (int)response_code);
578 }
579
580 CPLFree(sWriteFuncData.pBuffer);
581 CPLFree(sWriteFuncHeaderData.pBuffer);
582
583 poFS->AcquireMutex();
584 CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
585 cachedFileProp->bHastComputedFileSize = TRUE;
586 #ifdef notdef
587 cachedFileProp->nChecksumOfFirst1024Bytes = nRecomputedChecksumOfFirst1024Bytes;
588 #endif
589 cachedFileProp->fileSize = fileSize;
590 cachedFileProp->eExists = eExists;
591 cachedFileProp->bIsDirectory = bIsDirectory;
592 poFS->ReleaseMutex();
593
594 vsi_l_offset nRet = fileSize;
595 ReleaseMutex();
596
597 if (hCurlHandle == NULL)
598 hCurlHandle = hLocalHandle;
599 else
600 curl_easy_cleanup(hLocalHandle);
601
602 return nRet;
603 }
604
605 /************************************************************************/
606 /* Exists() */
607 /************************************************************************/
608
Exists()609 int VSICurlStreamingHandle::Exists()
610 {
611 if (eExists == EXIST_UNKNOWN)
612 {
613 /* Consider that only the files whose extension ends up with one that is */
614 /* listed in CPL_VSIL_CURL_ALLOWED_EXTENSIONS exist on the server */
615 /* This can speeds up dramatically open experience, in case the server */
616 /* cannot return a file list */
617 /* For example : */
618 /* gdalinfo --config CPL_VSIL_CURL_ALLOWED_EXTENSIONS ".tif" /vsicurl_streaming/http://igskmncngs506.cr.usgs.gov/gmted/Global_tiles_GMTED/075darcsec/bln/W030/30N030W_20101117_gmted_bln075.tif */
619 const char* pszAllowedExtensions =
620 CPLGetConfigOption("CPL_VSIL_CURL_ALLOWED_EXTENSIONS", NULL);
621 if (pszAllowedExtensions)
622 {
623 char** papszExtensions = CSLTokenizeString2( pszAllowedExtensions, ", ", 0 );
624 int nURLLen = strlen(pszURL);
625 int bFound = FALSE;
626 for(int i=0;papszExtensions[i] != NULL;i++)
627 {
628 int nExtensionLen = strlen(papszExtensions[i]);
629 if (nURLLen > nExtensionLen &&
630 EQUAL(pszURL + nURLLen - nExtensionLen, papszExtensions[i]))
631 {
632 bFound = TRUE;
633 break;
634 }
635 }
636
637 if (!bFound)
638 {
639 eExists = EXIST_NO;
640 fileSize = 0;
641
642 poFS->AcquireMutex();
643 CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
644 cachedFileProp->bHastComputedFileSize = TRUE;
645 cachedFileProp->fileSize = fileSize;
646 cachedFileProp->eExists = eExists;
647 poFS->ReleaseMutex();
648
649 CSLDestroy(papszExtensions);
650
651 return 0;
652 }
653
654 CSLDestroy(papszExtensions);
655 }
656
657 char chFirstByte;
658 int bExists = (Read(&chFirstByte, 1, 1) == 1);
659
660 AcquireMutex();
661 poFS->AcquireMutex();
662 CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
663 cachedFileProp->eExists = eExists = bExists ? EXIST_YES : EXIST_NO;
664 poFS->ReleaseMutex();
665 ReleaseMutex();
666
667 Seek(0, SEEK_SET);
668 }
669
670 return eExists == EXIST_YES;
671 }
672
673 /************************************************************************/
674 /* Tell() */
675 /************************************************************************/
676
Tell()677 vsi_l_offset VSICurlStreamingHandle::Tell()
678 {
679 return curOffset;
680 }
681
682 /************************************************************************/
683 /* ReceivedBytes() */
684 /************************************************************************/
685
ReceivedBytes(GByte * buffer,size_t count,size_t nmemb)686 int VSICurlStreamingHandle::ReceivedBytes(GByte *buffer, size_t count, size_t nmemb)
687 {
688 size_t nSize = count * nmemb;
689 nBodySize += nSize;
690
691 if (ENABLE_DEBUG)
692 CPLDebug("VSICURL", "Receiving %d bytes...", (int)nSize);
693
694 if( bHasCandidateFileSize && bCanTrustCandidateFileSize && !bHastComputedFileSize )
695 {
696 poFS->AcquireMutex();
697 CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
698 cachedFileProp->fileSize = fileSize = nCandidateFileSize;
699 cachedFileProp->bHastComputedFileSize = bHastComputedFileSize = TRUE;
700 if (ENABLE_DEBUG)
701 CPLDebug("VSICURL", "File size = " CPL_FRMT_GUIB, fileSize);
702 poFS->ReleaseMutex();
703 }
704
705 AcquireMutex();
706 if (eExists == EXIST_UNKNOWN)
707 {
708 poFS->AcquireMutex();
709 CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
710 cachedFileProp->eExists = eExists = EXIST_YES;
711 poFS->ReleaseMutex();
712 }
713 else if (eExists == EXIST_NO)
714 {
715 ReleaseMutex();
716 return 0;
717 }
718
719 while(TRUE)
720 {
721 size_t nFree = oRingBuffer.GetCapacity() - oRingBuffer.GetSize();
722 if (nSize <= nFree)
723 {
724 oRingBuffer.Write(buffer, nSize);
725
726 /* Signal to the consumer that we have added bytes to the buffer */
727 CPLCondSignal(hCondProducer);
728
729 if (bAskDownloadEnd)
730 {
731 if (ENABLE_DEBUG)
732 CPLDebug("VSICURL", "Download interruption asked");
733
734 ReleaseMutex();
735 return 0;
736 }
737 break;
738 }
739 else
740 {
741 oRingBuffer.Write(buffer, nFree);
742 buffer += nFree;
743 nSize -= nFree;
744
745 /* Signal to the consumer that we have added bytes to the buffer */
746 CPLCondSignal(hCondProducer);
747
748 if (ENABLE_DEBUG)
749 CPLDebug("VSICURL", "Waiting for reader to consume some bytes...");
750
751 while(oRingBuffer.GetSize() == oRingBuffer.GetCapacity() && !bAskDownloadEnd)
752 {
753 CPLCondWait(hCondConsumer, hRingBufferMutex);
754 }
755
756 if (bAskDownloadEnd)
757 {
758 if (ENABLE_DEBUG)
759 CPLDebug("VSICURL", "Download interruption asked");
760
761 ReleaseMutex();
762 return 0;
763 }
764 }
765 }
766
767 ReleaseMutex();
768
769 return nmemb;
770 }
771
772 /************************************************************************/
773 /* VSICurlStreamingHandleReceivedBytes() */
774 /************************************************************************/
775
VSICurlStreamingHandleReceivedBytes(void * buffer,size_t count,size_t nmemb,void * req)776 static int VSICurlStreamingHandleReceivedBytes(void *buffer, size_t count, size_t nmemb, void *req)
777 {
778 return ((VSICurlStreamingHandle*)req)->ReceivedBytes((GByte*)buffer, count, nmemb);
779 }
780
781
782 /************************************************************************/
783 /* VSICurlStreamingHandleReceivedBytesHeader() */
784 /************************************************************************/
785
786 #define HEADER_SIZE 32768
787
ReceivedBytesHeader(GByte * buffer,size_t count,size_t nmemb)788 int VSICurlStreamingHandle::ReceivedBytesHeader(GByte *buffer, size_t count, size_t nmemb)
789 {
790 size_t nSize = count * nmemb;
791 if (ENABLE_DEBUG)
792 CPLDebug("VSICURL", "Receiving %d bytes for header...", (int)nSize);
793
794 /* Reset buffer if we have followed link after a redirect */
795 if (nSize >=9 && (nHTTPCode == 301 || nHTTPCode == 302) &&
796 (EQUALN((const char*)buffer, "HTTP/1.0 ", 9) ||
797 EQUALN((const char*)buffer, "HTTP/1.1 ", 9)))
798 {
799 nHeaderSize = 0;
800 nHTTPCode = 0;
801 }
802
803 if (nHeaderSize < HEADER_SIZE)
804 {
805 size_t nSz = MIN(nSize, HEADER_SIZE - nHeaderSize);
806 memcpy(pabyHeaderData + nHeaderSize, buffer, nSz);
807 pabyHeaderData[nHeaderSize + nSz] = '\0';
808 nHeaderSize += nSz;
809
810 //CPLDebug("VSICURL", "Header : %s", pabyHeaderData);
811
812 AcquireMutex();
813
814 if (eExists == EXIST_UNKNOWN && nHTTPCode == 0 &&
815 strchr((const char*)pabyHeaderData, '\n') != NULL &&
816 (EQUALN((const char*)pabyHeaderData, "HTTP/1.0 ", 9) ||
817 EQUALN((const char*)pabyHeaderData, "HTTP/1.1 ", 9)))
818 {
819 nHTTPCode = atoi((const char*)pabyHeaderData + 9);
820 if (ENABLE_DEBUG)
821 CPLDebug("VSICURL", "HTTP code = %d", nHTTPCode);
822
823 /* If moved permanently/temporarily, go on */
824 if( !(nHTTPCode == 301 || nHTTPCode == 302) )
825 {
826 poFS->AcquireMutex();
827 CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
828 cachedFileProp->eExists = eExists = (nHTTPCode == 200) ? EXIST_YES : EXIST_NO;
829 poFS->ReleaseMutex();
830 }
831 }
832
833 if ( !(nHTTPCode == 301 || nHTTPCode == 302) && !bHastComputedFileSize)
834 {
835 /* Caution: when gzip compression is enabled, the content-length is the compressed */
836 /* size, which we are not interested in, so we must not take it into account. */
837
838 const char* pszContentLength = strstr((const char*)pabyHeaderData, "Content-Length: ");
839 const char* pszEndOfLine = pszContentLength ? strchr(pszContentLength, '\n') : NULL;
840 if( bCanTrustCandidateFileSize && pszEndOfLine != NULL )
841 {
842 const char* pszVal = pszContentLength + strlen("Content-Length: ");
843 bHasCandidateFileSize = TRUE;
844 nCandidateFileSize = CPLScanUIntBig(pszVal, pszEndOfLine - pszVal);
845 if (ENABLE_DEBUG)
846 CPLDebug("VSICURL", "Has found candidate file size = " CPL_FRMT_GUIB, nCandidateFileSize);
847 }
848
849 const char* pszContentEncoding = strstr((const char*)pabyHeaderData, "Content-Encoding: ");
850 pszEndOfLine = pszContentEncoding ? strchr(pszContentEncoding, '\n') : NULL;
851 if( bHasCandidateFileSize && pszEndOfLine != NULL )
852 {
853 const char* pszVal = pszContentEncoding + strlen("Content-Encoding: ");
854 if( strncmp(pszVal, "gzip", 4) == 0 )
855 {
856 if (ENABLE_DEBUG)
857 CPLDebug("VSICURL", "GZip compression enabled --> cannot trust candidate file size");
858 bCanTrustCandidateFileSize = FALSE;
859 }
860 }
861 }
862
863 ReleaseMutex();
864 }
865
866 return nmemb;
867 }
868
869 /************************************************************************/
870 /* VSICurlStreamingHandleReceivedBytesHeader() */
871 /************************************************************************/
872
VSICurlStreamingHandleReceivedBytesHeader(void * buffer,size_t count,size_t nmemb,void * req)873 static int VSICurlStreamingHandleReceivedBytesHeader(void *buffer, size_t count, size_t nmemb, void *req)
874 {
875 return ((VSICurlStreamingHandle*)req)->ReceivedBytesHeader((GByte*)buffer, count, nmemb);
876 }
877 /************************************************************************/
878 /* DownloadInThread() */
879 /************************************************************************/
880
DownloadInThread()881 void VSICurlStreamingHandle::DownloadInThread()
882 {
883 VSICurlSetOptions(hCurlHandle, pszURL);
884
885 static int bHasCheckVersion = FALSE;
886 static int bSupportGZip = FALSE;
887 if (!bHasCheckVersion)
888 {
889 bSupportGZip = strstr(curl_version(), "zlib/") != NULL;
890 bHasCheckVersion = TRUE;
891 }
892 if (bSupportGZip && CSLTestBoolean(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
893 {
894 curl_easy_setopt(hCurlHandle, CURLOPT_ENCODING, "gzip");
895 }
896
897 if (pabyHeaderData == NULL)
898 pabyHeaderData = (GByte*) CPLMalloc(HEADER_SIZE + 1);
899 nHeaderSize = 0;
900 nBodySize = 0;
901 nHTTPCode = 0;
902
903 curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, this);
904 curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, VSICurlStreamingHandleReceivedBytesHeader);
905
906 curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, this);
907 curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, VSICurlStreamingHandleReceivedBytes);
908
909 char szCurlErrBuf[CURL_ERROR_SIZE+1];
910 szCurlErrBuf[0] = '\0';
911 curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
912
913 CURLcode eRet = curl_easy_perform(hCurlHandle);
914
915 curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, NULL);
916 curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, NULL);
917 curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, NULL);
918 curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, NULL);
919
920 AcquireMutex();
921 if (!bAskDownloadEnd && eRet == 0 && !bHastComputedFileSize)
922 {
923 poFS->AcquireMutex();
924 CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
925 cachedFileProp->fileSize = fileSize = nBodySize;
926 cachedFileProp->bHastComputedFileSize = bHastComputedFileSize = TRUE;
927 if (ENABLE_DEBUG)
928 CPLDebug("VSICURL", "File size = " CPL_FRMT_GUIB, fileSize);
929 poFS->ReleaseMutex();
930 }
931
932 bDownloadInProgress = FALSE;
933 bDownloadStopped = TRUE;
934
935 /* Signal to the consumer that the download has ended */
936 CPLCondSignal(hCondProducer);
937 ReleaseMutex();
938 }
939
VSICurlDownloadInThread(void * pArg)940 static void VSICurlDownloadInThread(void* pArg)
941 {
942 ((VSICurlStreamingHandle*)pArg)->DownloadInThread();
943 }
944
945 /************************************************************************/
946 /* StartDownload() */
947 /************************************************************************/
948
StartDownload()949 void VSICurlStreamingHandle::StartDownload()
950 {
951 if (bDownloadInProgress || bDownloadStopped)
952 return;
953
954 //if (ENABLE_DEBUG)
955 CPLDebug("VSICURL", "Start download for %s", pszURL);
956
957 if (hCurlHandle == NULL)
958 hCurlHandle = curl_easy_init();
959 oRingBuffer.Reset();
960 bDownloadInProgress = TRUE;
961 nRingBufferFileOffset = 0;
962 hThread = CPLCreateJoinableThread(VSICurlDownloadInThread, this);
963 }
964
965 /************************************************************************/
966 /* StopDownload() */
967 /************************************************************************/
968
StopDownload()969 void VSICurlStreamingHandle::StopDownload()
970 {
971 if (hThread)
972 {
973 //if (ENABLE_DEBUG)
974 CPLDebug("VSICURL", "Stop download for %s", pszURL);
975
976 AcquireMutex();
977 /* Signal to the producer that we ask for download interruption */
978 bAskDownloadEnd = TRUE;
979 CPLCondSignal(hCondConsumer);
980
981 /* Wait for the producer to have finished */
982 while(bDownloadInProgress)
983 CPLCondWait(hCondProducer, hRingBufferMutex);
984
985 bAskDownloadEnd = FALSE;
986
987 ReleaseMutex();
988
989 CPLJoinThread(hThread);
990 hThread = NULL;
991
992 curl_easy_cleanup(hCurlHandle);
993 hCurlHandle = NULL;
994 }
995
996 oRingBuffer.Reset();
997 bDownloadStopped = FALSE;
998 }
999
1000 /************************************************************************/
1001 /* PutRingBufferInCache() */
1002 /************************************************************************/
1003
PutRingBufferInCache()1004 void VSICurlStreamingHandle::PutRingBufferInCache()
1005 {
1006 if (nRingBufferFileOffset >= BKGND_BUFFER_SIZE)
1007 return;
1008
1009 AcquireMutex();
1010
1011 /* Cache any remaining bytes available in the ring buffer */
1012 size_t nBufSize = oRingBuffer.GetSize();
1013 if ( nBufSize > 0 )
1014 {
1015 if (nRingBufferFileOffset + nBufSize > BKGND_BUFFER_SIZE)
1016 nBufSize = (size_t) (BKGND_BUFFER_SIZE - nRingBufferFileOffset);
1017 GByte* pabyTmp = (GByte*) CPLMalloc(nBufSize);
1018 oRingBuffer.Read(pabyTmp, nBufSize);
1019
1020 /* Signal to the producer that we have ingested some bytes */
1021 CPLCondSignal(hCondConsumer);
1022
1023 AddRegion(nRingBufferFileOffset, nBufSize, pabyTmp);
1024 nRingBufferFileOffset += nBufSize;
1025 CPLFree(pabyTmp);
1026 }
1027
1028 ReleaseMutex();
1029 }
1030
1031 /************************************************************************/
1032 /* Read() */
1033 /************************************************************************/
1034
Read(void * pBuffer,size_t nSize,size_t nMemb)1035 size_t VSICurlStreamingHandle::Read( void *pBuffer, size_t nSize, size_t nMemb )
1036 {
1037 GByte* pabyBuffer = (GByte*)pBuffer;
1038 size_t nBufferRequestSize = nSize * nMemb;
1039 if (nBufferRequestSize == 0)
1040 return 0;
1041 size_t nRemaining = nBufferRequestSize;
1042
1043 AcquireMutex();
1044 int bHastComputedFileSizeLocal = bHastComputedFileSize;
1045 vsi_l_offset fileSizeLocal = fileSize;
1046 ReleaseMutex();
1047
1048 if (bHastComputedFileSizeLocal && curOffset >= fileSizeLocal)
1049 {
1050 CPLDebug("VSICURL", "Read attempt beyond end of file");
1051 bEOF = TRUE;
1052 }
1053 if (bEOF)
1054 return 0;
1055
1056 if (curOffset < nRingBufferFileOffset)
1057 PutRingBufferInCache();
1058
1059 if (ENABLE_DEBUG)
1060 CPLDebug("VSICURL", "Read [" CPL_FRMT_GUIB ", " CPL_FRMT_GUIB "[ in %s",
1061 curOffset, curOffset + nBufferRequestSize, pszURL);
1062
1063 #ifdef notdef
1064 if( pCachedData != NULL && nCachedSize >= 1024 &&
1065 nRecomputedChecksumOfFirst1024Bytes == 0 )
1066 {
1067 for(size_t i = 0; i < 1024 / sizeof(int); i ++)
1068 {
1069 int nVal;
1070 memcpy(&nVal, pCachedData + i * sizeof(int), sizeof(int));
1071 nRecomputedChecksumOfFirst1024Bytes += nVal;
1072 }
1073
1074 if( bHastComputedFileSizeLocal )
1075 {
1076 poFS->AcquireMutex();
1077 CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
1078 if( cachedFileProp->nChecksumOfFirst1024Bytes == 0 )
1079 {
1080 cachedFileProp->nChecksumOfFirst1024Bytes = nRecomputedChecksumOfFirst1024Bytes;
1081 }
1082 else if( nRecomputedChecksumOfFirst1024Bytes != cachedFileProp->nChecksumOfFirst1024Bytes )
1083 {
1084 CPLDebug("VSICURL", "Invalidating previously cached file size. First bytes of file have changed!");
1085 AcquireMutex();
1086 bHastComputedFileSize = FALSE;
1087 cachedFileProp->bHastComputedFileSize = FALSE;
1088 cachedFileProp->nChecksumOfFirst1024Bytes = 0;
1089 ReleaseMutex();
1090 }
1091 poFS->ReleaseMutex();
1092 }
1093 }
1094 #endif
1095
1096 /* Can we use the cache ? */
1097 if( pCachedData != NULL && curOffset < nCachedSize )
1098 {
1099 size_t nSz = MIN(nRemaining, (size_t)(nCachedSize - curOffset));
1100 if (ENABLE_DEBUG)
1101 CPLDebug("VSICURL", "Using cache for [%d, %d[ in %s",
1102 (int)curOffset, (int)(curOffset + nSz), pszURL);
1103 memcpy(pabyBuffer, pCachedData + curOffset, nSz);
1104 pabyBuffer += nSz;
1105 curOffset += nSz;
1106 nRemaining -= nSz;
1107 }
1108
1109 /* Is the request partially covered by the cache and going beyond file size ? */
1110 if ( pCachedData != NULL && bHastComputedFileSizeLocal &&
1111 curOffset <= nCachedSize &&
1112 curOffset + nRemaining > fileSizeLocal &&
1113 fileSize == nCachedSize )
1114 {
1115 size_t nSz = (size_t) (nCachedSize - curOffset);
1116 if (ENABLE_DEBUG && nSz != 0)
1117 CPLDebug("VSICURL", "Using cache for [%d, %d[ in %s",
1118 (int)curOffset, (int)(curOffset + nSz), pszURL);
1119 memcpy(pabyBuffer, pCachedData + curOffset, nSz);
1120 pabyBuffer += nSz;
1121 curOffset += nSz;
1122 nRemaining -= nSz;
1123 bEOF = TRUE;
1124 }
1125
1126 /* Has a Seek() being done since the last Read() ? */
1127 if (!bEOF && nRemaining > 0 && curOffset != nRingBufferFileOffset)
1128 {
1129 /* Backward seek : we need to restart the download from the start */
1130 if (curOffset < nRingBufferFileOffset)
1131 StopDownload();
1132
1133 StartDownload();
1134
1135 #define SKIP_BUFFER_SIZE 32768
1136 GByte* pabyTmp = (GByte*)CPLMalloc(SKIP_BUFFER_SIZE);
1137
1138 CPLAssert(curOffset >= nRingBufferFileOffset);
1139 vsi_l_offset nBytesToSkip = curOffset - nRingBufferFileOffset;
1140 while(nBytesToSkip > 0)
1141 {
1142 vsi_l_offset nBytesToRead = nBytesToSkip;
1143
1144 AcquireMutex();
1145 if (nBytesToRead > oRingBuffer.GetSize())
1146 nBytesToRead = oRingBuffer.GetSize();
1147 if (nBytesToRead > SKIP_BUFFER_SIZE)
1148 nBytesToRead = SKIP_BUFFER_SIZE;
1149 oRingBuffer.Read(pabyTmp, (size_t)nBytesToRead);
1150
1151 /* Signal to the producer that we have ingested some bytes */
1152 CPLCondSignal(hCondConsumer);
1153 ReleaseMutex();
1154
1155 if (nBytesToRead)
1156 AddRegion(nRingBufferFileOffset, (size_t)nBytesToRead, pabyTmp);
1157
1158 nBytesToSkip -= nBytesToRead;
1159 nRingBufferFileOffset += nBytesToRead;
1160
1161 if (nBytesToRead == 0 && nBytesToSkip != 0)
1162 {
1163 if (ENABLE_DEBUG)
1164 CPLDebug("VSICURL", "Waiting for writer to produce some bytes...");
1165
1166 AcquireMutex();
1167 while(oRingBuffer.GetSize() == 0 && bDownloadInProgress)
1168 CPLCondWait(hCondProducer, hRingBufferMutex);
1169 int bBufferEmpty = (oRingBuffer.GetSize() == 0);
1170 ReleaseMutex();
1171
1172 if (bBufferEmpty && !bDownloadInProgress)
1173 break;
1174 }
1175 }
1176
1177 CPLFree(pabyTmp);
1178
1179 if (nBytesToSkip != 0)
1180 {
1181 bEOF = TRUE;
1182 return 0;
1183 }
1184 }
1185
1186 if (!bEOF && nRemaining > 0)
1187 {
1188 StartDownload();
1189 CPLAssert(curOffset == nRingBufferFileOffset);
1190 }
1191
1192 /* Fill the destination buffer from the ring buffer */
1193 while(!bEOF && nRemaining > 0)
1194 {
1195 AcquireMutex();
1196 size_t nToRead = oRingBuffer.GetSize();
1197 if (nToRead > nRemaining)
1198 nToRead = nRemaining;
1199 oRingBuffer.Read(pabyBuffer, nToRead);
1200
1201 /* Signal to the producer that we have ingested some bytes */
1202 CPLCondSignal(hCondConsumer);
1203 ReleaseMutex();
1204
1205 if (nToRead)
1206 AddRegion(curOffset, nToRead, pabyBuffer);
1207
1208 nRemaining -= nToRead;
1209 pabyBuffer += nToRead;
1210 curOffset += nToRead;
1211 nRingBufferFileOffset += nToRead;
1212
1213 if (nToRead == 0 && nRemaining != 0)
1214 {
1215 if (ENABLE_DEBUG)
1216 CPLDebug("VSICURL", "Waiting for writer to produce some bytes...");
1217
1218 AcquireMutex();
1219 while(oRingBuffer.GetSize() == 0 && bDownloadInProgress)
1220 CPLCondWait(hCondProducer, hRingBufferMutex);
1221 int bBufferEmpty = (oRingBuffer.GetSize() == 0);
1222 ReleaseMutex();
1223
1224 if (bBufferEmpty && !bDownloadInProgress)
1225 break;
1226 }
1227 }
1228
1229 if (ENABLE_DEBUG)
1230 CPLDebug("VSICURL", "Read(%d) = %d",
1231 (int)nBufferRequestSize, (int)(nBufferRequestSize - nRemaining));
1232 size_t nRet = (nBufferRequestSize - nRemaining) / nSize;
1233 if (nRet < nMemb)
1234 bEOF = TRUE;
1235
1236 return nRet;
1237 }
1238
1239 /************************************************************************/
1240 /* AddRegion() */
1241 /************************************************************************/
1242
AddRegion(vsi_l_offset nFileOffsetStart,size_t nSize,GByte * pData)1243 void VSICurlStreamingHandle::AddRegion( vsi_l_offset nFileOffsetStart,
1244 size_t nSize,
1245 GByte *pData )
1246 {
1247 if (nFileOffsetStart >= BKGND_BUFFER_SIZE)
1248 return;
1249
1250 if (pCachedData == NULL)
1251 pCachedData = (GByte*) CPLMalloc(BKGND_BUFFER_SIZE);
1252
1253 if (nFileOffsetStart <= nCachedSize &&
1254 nFileOffsetStart + nSize > nCachedSize)
1255 {
1256 size_t nSz = MIN(nSize, (size_t) (BKGND_BUFFER_SIZE - nFileOffsetStart));
1257 if (ENABLE_DEBUG)
1258 CPLDebug("VSICURL", "Writing [%d, %d[ in cache for %s",
1259 (int)nFileOffsetStart, (int)(nFileOffsetStart + nSz), pszURL);
1260 memcpy(pCachedData + nFileOffsetStart, pData, nSz);
1261 nCachedSize = (size_t) (nFileOffsetStart + nSz);
1262 }
1263 }
1264 /************************************************************************/
1265 /* Write() */
1266 /************************************************************************/
1267
Write(CPL_UNUSED const void * pBuffer,CPL_UNUSED size_t nSize,CPL_UNUSED size_t nMemb)1268 size_t VSICurlStreamingHandle::Write( CPL_UNUSED const void *pBuffer,
1269 CPL_UNUSED size_t nSize,
1270 CPL_UNUSED size_t nMemb )
1271 {
1272 return 0;
1273 }
1274
1275 /************************************************************************/
1276 /* Eof() */
1277 /************************************************************************/
1278
1279
Eof()1280 int VSICurlStreamingHandle::Eof()
1281 {
1282 return bEOF;
1283 }
1284
1285 /************************************************************************/
1286 /* Flush() */
1287 /************************************************************************/
1288
Flush()1289 int VSICurlStreamingHandle::Flush()
1290 {
1291 return 0;
1292 }
1293
1294 /************************************************************************/
1295 /* Close() */
1296 /************************************************************************/
1297
Close()1298 int VSICurlStreamingHandle::Close()
1299 {
1300 return 0;
1301 }
1302
1303
1304 /************************************************************************/
1305 /* VSICurlStreamingFSHandler() */
1306 /************************************************************************/
1307
VSICurlStreamingFSHandler()1308 VSICurlStreamingFSHandler::VSICurlStreamingFSHandler()
1309 {
1310 hMutex = CPLCreateMutex();
1311 CPLReleaseMutex(hMutex);
1312 }
1313
1314 /************************************************************************/
1315 /* ~VSICurlStreamingFSHandler() */
1316 /************************************************************************/
1317
~VSICurlStreamingFSHandler()1318 VSICurlStreamingFSHandler::~VSICurlStreamingFSHandler()
1319 {
1320 std::map<CPLString, CachedFileProp*>::const_iterator iterCacheFileSize;
1321
1322 for( iterCacheFileSize = cacheFileSize.begin();
1323 iterCacheFileSize != cacheFileSize.end();
1324 iterCacheFileSize++ )
1325 {
1326 CPLFree(iterCacheFileSize->second);
1327 }
1328
1329 CPLDestroyMutex( hMutex );
1330 hMutex = NULL;
1331 }
1332
1333 /************************************************************************/
1334 /* AcquireMutex() */
1335 /************************************************************************/
1336
AcquireMutex()1337 void VSICurlStreamingFSHandler::AcquireMutex()
1338 {
1339 CPLAcquireMutex(hMutex, 1000.0);
1340 }
1341
1342 /************************************************************************/
1343 /* ReleaseMutex() */
1344 /************************************************************************/
1345
ReleaseMutex()1346 void VSICurlStreamingFSHandler::ReleaseMutex()
1347 {
1348 CPLReleaseMutex(hMutex);
1349 }
1350
1351 /************************************************************************/
1352 /* GetCachedFileProp() */
1353 /************************************************************************/
1354
1355 /* Should be called under the FS Lock */
1356
GetCachedFileProp(const char * pszURL)1357 CachedFileProp* VSICurlStreamingFSHandler::GetCachedFileProp(const char* pszURL)
1358 {
1359 CachedFileProp* cachedFileProp = cacheFileSize[pszURL];
1360 if (cachedFileProp == NULL)
1361 {
1362 cachedFileProp = (CachedFileProp*) CPLMalloc(sizeof(CachedFileProp));
1363 cachedFileProp->eExists = EXIST_UNKNOWN;
1364 cachedFileProp->bHastComputedFileSize = FALSE;
1365 cachedFileProp->fileSize = 0;
1366 cachedFileProp->bIsDirectory = FALSE;
1367 #ifdef notdef
1368 cachedFileProp->nChecksumOfFirst1024Bytes = 0;
1369 #endif
1370 cacheFileSize[pszURL] = cachedFileProp;
1371 }
1372
1373 return cachedFileProp;
1374 }
1375
1376 /************************************************************************/
1377 /* Open() */
1378 /************************************************************************/
1379
Open(const char * pszFilename,const char * pszAccess)1380 VSIVirtualHandle* VSICurlStreamingFSHandler::Open( const char *pszFilename,
1381 const char *pszAccess )
1382 {
1383 if (strchr(pszAccess, 'w') != NULL ||
1384 strchr(pszAccess, '+') != NULL)
1385 {
1386 CPLError(CE_Failure, CPLE_AppDefined,
1387 "Only read-only mode is supported for /vsicurl_streaming");
1388 return NULL;
1389 }
1390
1391 VSICurlStreamingHandle* poHandle = new VSICurlStreamingHandle(
1392 this, pszFilename + strlen("/vsicurl_streaming/"));
1393 /* If we didn't get a filelist, check that the file really exists */
1394 if (!poHandle->Exists())
1395 {
1396 delete poHandle;
1397 poHandle = NULL;
1398 }
1399
1400 if( CSLTestBoolean( CPLGetConfigOption( "VSI_CACHE", "FALSE" ) ) )
1401 return VSICreateCachedFile( poHandle );
1402 else
1403 return poHandle;
1404 }
1405
1406 /************************************************************************/
1407 /* Stat() */
1408 /************************************************************************/
1409
Stat(const char * pszFilename,VSIStatBufL * pStatBuf,int nFlags)1410 int VSICurlStreamingFSHandler::Stat( const char *pszFilename,
1411 VSIStatBufL *pStatBuf,
1412 int nFlags )
1413 {
1414 CPLString osFilename(pszFilename);
1415
1416 memset(pStatBuf, 0, sizeof(VSIStatBufL));
1417
1418 VSICurlStreamingHandle oHandle( this, osFilename + strlen("/vsicurl_streaming/"));
1419
1420 if ( oHandle.IsKnownFileSize() ||
1421 ((nFlags & VSI_STAT_SIZE_FLAG) && !oHandle.IsDirectory() &&
1422 CSLTestBoolean(CPLGetConfigOption("CPL_VSIL_CURL_SLOW_GET_SIZE", "YES"))) )
1423 pStatBuf->st_size = oHandle.GetFileSize();
1424
1425 int nRet = (oHandle.Exists()) ? 0 : -1;
1426 pStatBuf->st_mode = oHandle.IsDirectory() ? S_IFDIR : S_IFREG;
1427 return nRet;
1428 }
1429
1430 /************************************************************************/
1431 /* VSIInstallCurlFileHandler() */
1432 /************************************************************************/
1433
1434 /**
1435 * \brief Install /vsicurl_streaming/ HTTP/FTP file system handler (requires libcurl)
1436 *
1437 * A special file handler is installed that allows on-the-fly reading of files
1438 * streamed through HTTP/FTP web protocols (typically dynamically generated files),
1439 * without downloading the entire file.
1440 *
1441 * Although this file handler is able seek to random offsets in the file, this will not
1442 * be efficient. If you need efficient random access and that the server supports range
1443 * dowloading, you should use the /vsicurl/ file system handler instead.
1444 *
1445 * Recognized filenames are of the form /vsicurl_streaming/http://path/to/remote/resource or
1446 * /vsicurl_streaming/ftp://path/to/remote/resource where path/to/remote/resource is the
1447 * URL of a remote resource.
1448 *
1449 * The GDAL_HTTP_PROXY, GDAL_HTTP_PROXYUSERPWD and GDAL_PROXY_AUTH configuration options can be
1450 * used to define a proxy server. The syntax to use is the one of Curl CURLOPT_PROXY,
1451 * CURLOPT_PROXYUSERPWD and CURLOPT_PROXYAUTH options.
1452 *
1453 * The file can be cached in RAM by setting the configuration option
1454 * VSI_CACHE to TRUE. The cache size defaults to 25 MB, but can be modified by setting
1455 * the configuration option VSI_CACHE_SIZE (in bytes).
1456 *
1457 * VSIStatL() will return the size in st_size member and file
1458 * nature- file or directory - in st_mode member (the later only reliable with FTP
1459 * resources for now).
1460 *
1461 * @since GDAL 1.10
1462 */
VSIInstallCurlStreamingFileHandler(void)1463 void VSIInstallCurlStreamingFileHandler(void)
1464 {
1465 VSIFileManager::InstallHandler( "/vsicurl_streaming/", new VSICurlStreamingFSHandler );
1466 }
1467
1468 #ifdef AS_PLUGIN
1469 CPL_C_START
1470 void CPL_DLL GDALRegisterMe();
GDALRegisterMe()1471 void GDALRegisterMe()
1472 {
1473 VSIInstallCurlStreamingFileHandler();
1474 }
1475 CPL_C_END
1476 #endif
1477
1478 #endif /* !defined(HAVE_CURL) || defined(CPL_MULTIPROC_STUB) */
1479