1 /******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Implement VSI large file api for HTTP/FTP files in streaming mode
5 * Author: Even Rouault <even dot rouault at spatialys.com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2012-2015, Even Rouault <even dot rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29 #include "cpl_port.h"
30 #include "cpl_vsi.h"
31 #include "cpl_vsi_virtual.h"
32 #include "cpl_vsil_curl_class.h"
33
34 #include <algorithm>
35 #include <map>
36
37 #include "cpl_aws.h"
38 #include "cpl_google_cloud.h"
39 #include "cpl_azure.h"
40 #include "cpl_alibaba_oss.h"
41 #include "cpl_swift.h"
42 #include "cpl_hash_set.h"
43 #include "cpl_http.h"
44 #include "cpl_multiproc.h"
45 #include "cpl_string.h"
46 #include "cpl_time.h"
47
48 CPL_CVSID("$Id: cpl_vsil_curl_streaming.cpp 86933038c3926cd4dc3ff37c431b317abb69e602 2021-03-27 23:20:49 +0100 Even Rouault $")
49
50 #if !defined(HAVE_CURL) || defined(CPL_MULTIPROC_STUB)
51
VSIInstallCurlStreamingFileHandler(void)52 void VSIInstallCurlStreamingFileHandler(void)
53 {
54 // Not supported.
55 }
56
VSIInstallS3StreamingFileHandler(void)57 void VSIInstallS3StreamingFileHandler(void)
58 {
59 // Not supported.
60 }
61
VSIInstallGSStreamingFileHandler(void)62 void VSIInstallGSStreamingFileHandler(void)
63 {
64 // Not supported.
65 }
66
VSIInstallAzureStreamingFileHandler(void)67 void VSIInstallAzureStreamingFileHandler(void)
68 {
69 // Not supported
70 }
71
VSIInstallOSSStreamingFileHandler(void)72 void VSIInstallOSSStreamingFileHandler(void)
73 {
74 // Not supported
75 }
76
VSIInstallSwiftStreamingFileHandler(void)77 void VSIInstallSwiftStreamingFileHandler(void)
78 {
79 // Not supported
80 }
81
82 #ifdef HAVE_CURL
VSICurlStreamingClearCache(void)83 void VSICurlStreamingClearCache( void )
84 {
85 // Not supported
86 }
87 #endif
88
89 #else
90
91 //! @cond Doxygen_Suppress
92
93 #include <curl/curl.h>
94
95 #define ENABLE_DEBUG 0
96
97 #define N_MAX_REGIONS 10
98
99 #define BKGND_BUFFER_SIZE (1024 * 1024)
100
101 /************************************************************************/
102 /* RingBuffer */
103 /************************************************************************/
104
105 class RingBuffer
106 {
107 CPL_DISALLOW_COPY_ASSIGN(RingBuffer)
108
109 GByte* pabyBuffer = nullptr;
110 size_t nCapacity = 0;
111 size_t nOffset = 0;
112 size_t nLength = 0;
113
114 public:
115 explicit RingBuffer(size_t nCapacity = BKGND_BUFFER_SIZE);
116 ~RingBuffer();
117
118 size_t GetCapacity() const { return nCapacity; }
119 size_t GetSize() const { return nLength; }
120
121 void Reset();
122 void Write(void* pBuffer, size_t nSize);
123 void Read(void* pBuffer, size_t nSize);
124 };
125
126 RingBuffer::RingBuffer( size_t nCapacityIn ) :
127 pabyBuffer(static_cast<GByte*>(CPLMalloc(nCapacityIn))),
128 nCapacity(nCapacityIn)
129 {}
130
131 RingBuffer::~RingBuffer()
132 {
133 CPLFree(pabyBuffer);
134 }
135
136 void RingBuffer::Reset()
137 {
138 nOffset = 0;
139 nLength = 0;
140 }
141
142 void RingBuffer::Write( void* pBuffer, size_t nSize )
143 {
144 CPLAssert(nLength + nSize <= nCapacity);
145
146 const size_t nEndOffset = (nOffset + nLength) % nCapacity;
147 const size_t nSz = std::min(nSize, nCapacity - nEndOffset);
148 memcpy(pabyBuffer + nEndOffset, pBuffer, nSz);
149 if( nSz < nSize )
150 memcpy(pabyBuffer, static_cast<GByte *>(pBuffer) + nSz, nSize - nSz);
151
152 nLength += nSize;
153 }
154
155 void RingBuffer::Read( void* pBuffer, size_t nSize )
156 {
157 CPLAssert(nSize <= nLength);
158
159 if( pBuffer )
160 {
161 const size_t nSz = std::min(nSize, nCapacity - nOffset);
162 memcpy(pBuffer, pabyBuffer + nOffset, nSz);
163 if( nSz < nSize )
164 memcpy(static_cast<GByte *>(pBuffer) + nSz, pabyBuffer, nSize - nSz);
165 }
166
167 nOffset = (nOffset + nSize) % nCapacity;
168 nLength -= nSize;
169 }
170
171 /************************************************************************/
172
173 namespace {
174
175 typedef enum
176 {
177 EXIST_UNKNOWN = -1,
178 EXIST_NO,
179 EXIST_YES,
180 } ExistStatus;
181
182 typedef struct
183 {
184 ExistStatus eExists;
185 int bHasComputedFileSize;
186 vsi_l_offset fileSize;
187 int bIsDirectory;
188 #ifdef notdef
189 unsigned int nChecksumOfFirst1024Bytes;
190 #endif
191 } CachedFileProp;
192
193 typedef struct
194 {
195 char* pBuffer;
196 size_t nSize;
197 int bIsHTTP;
198 int bIsInHeader;
199 int nHTTPCode;
200 int bDownloadHeaderOnly;
201 } WriteFuncStruct;
202
203 /************************************************************************/
204 /* VSICurlStreamingFSHandler */
205 /************************************************************************/
206
207 class VSICurlStreamingHandle;
208
209 class VSICurlStreamingFSHandler : public VSIFilesystemHandler
210 {
211 CPL_DISALLOW_COPY_ASSIGN(VSICurlStreamingFSHandler)
212
213 std::map<CPLString, CachedFileProp*> cacheFileSize{};
214
215 protected:
216 CPLMutex *hMutex = nullptr;
217
218 virtual CPLString GetFSPrefix() { return "/vsicurl_streaming/"; }
219 virtual VSICurlStreamingHandle* CreateFileHandle(const char* pszURL);
220
221 public:
222 VSICurlStreamingFSHandler();
223 virtual ~VSICurlStreamingFSHandler();
224
225 virtual VSIVirtualHandle *Open( const char *pszFilename,
226 const char *pszAccess,
227 bool bSetError,
228 CSLConstList /* papszOptions */ ) override;
229 virtual int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
230 int nFlags ) override;
231
232 const char* GetActualURL(const char* pszFilename) override;
233
234 const char* GetOptions() override
235 { return VSIGetFileSystemOptions("/vsicurl/"); }
236
237 void AcquireMutex();
238 void ReleaseMutex();
239
240 CachedFileProp* GetCachedFileProp(const char* pszURL);
241
242 virtual void ClearCache();
243 };
244
245 /************************************************************************/
246 /* VSICurlStreamingHandle */
247 /************************************************************************/
248
249 class VSICurlStreamingHandle : public VSIVirtualHandle
250 {
251 CPL_DISALLOW_COPY_ASSIGN(VSICurlStreamingHandle)
252
253 protected:
254 VSICurlStreamingFSHandler* m_poFS = nullptr;
255 char** m_papszHTTPOptions = nullptr;
256
257 private:
258 char* m_pszURL = nullptr;
259
260 #ifdef notdef
261 unsigned int nRecomputedChecksumOfFirst1024Bytes = 0;
262 #endif
263 vsi_l_offset curOffset = 0;
264 vsi_l_offset fileSize = 0;
265 int bHasComputedFileSize = 0;
266 ExistStatus eExists = EXIST_UNKNOWN;
267 int bIsDirectory = 0;
268
269 bool bCanTrustCandidateFileSize = true;
270 bool bHasCandidateFileSize = false;
271 vsi_l_offset nCandidateFileSize = 0;
272
273 bool bEOF = false;
274
275 size_t nCachedSize = 0;
276 GByte *pCachedData = nullptr;
277
278 volatile int bDownloadInProgress = FALSE;
279 volatile int bDownloadStopped = FALSE;
280 volatile int bAskDownloadEnd = FALSE;
281 vsi_l_offset nRingBufferFileOffset = 0;
282 CPLJoinableThread *hThread = nullptr;
283 CPLMutex *hRingBufferMutex = nullptr;
284 CPLCond *hCondProducer = nullptr;
285 CPLCond *hCondConsumer = nullptr;
286 RingBuffer oRingBuffer{};
287 void StartDownload();
288 void StopDownload();
289 void PutRingBufferInCache();
290
291 GByte *pabyHeaderData = nullptr;
292 size_t nHeaderSize = 0;
293 vsi_l_offset nBodySize = 0;
294 int nHTTPCode = 0;
295
296 void AcquireMutex();
297 void ReleaseMutex();
298
299 void AddRegion( vsi_l_offset nFileOffsetStart,
300 size_t nSize,
301 GByte *pData );
302
303 protected:
304 virtual struct curl_slist* GetCurlHeaders(const CPLString&,
305 const struct curl_slist* /* psExistingHeaders */ )
306 { return nullptr; }
307 virtual bool StopReceivingBytesOnError() { return true; }
308 virtual bool CanRestartOnError( const char* /*pszErrorMsg*/,
309 const char* /*pszHeaders*/,
310 bool /*bSetError*/ ) { return false; }
311 virtual bool InterpretRedirect() { return true; }
312 void SetURL( const char* pszURL );
313
314 public:
315 VSICurlStreamingHandle( VSICurlStreamingFSHandler* poFS,
316 const char* pszURL );
317 ~VSICurlStreamingHandle() override;
318
319 int Seek( vsi_l_offset nOffset, int nWhence ) override;
320 vsi_l_offset Tell() override;
321 size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
322 size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
323 int Eof() override;
324 int Flush() override;
325 int Close() override;
326
327 void DownloadInThread();
328 size_t ReceivedBytes( GByte *buffer, size_t count,
329 size_t nmemb);
330 size_t ReceivedBytesHeader( GByte *buffer, size_t count,
331 size_t nmemb );
332
333 int IsKnownFileSize() const
334 { return bHasComputedFileSize; }
335 vsi_l_offset GetFileSize();
336 int Exists();
337 int IsDirectory() const { return bIsDirectory; }
338
339 const char *GetURL() const { return m_pszURL; }
340 };
341
342 /************************************************************************/
343 /* VSICurlStreamingHandle() */
344 /************************************************************************/
345
346 VSICurlStreamingHandle::VSICurlStreamingHandle( VSICurlStreamingFSHandler* poFS,
347 const char* pszURL ):
348 m_poFS(poFS),
349 m_papszHTTPOptions(CPLHTTPGetOptionsFromEnv()),
350 m_pszURL(CPLStrdup(pszURL))
351 {
352
353 poFS->AcquireMutex();
354 CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
355 eExists = cachedFileProp->eExists;
356 fileSize = cachedFileProp->fileSize;
357 bHasComputedFileSize = cachedFileProp->bHasComputedFileSize;
358 bIsDirectory = cachedFileProp->bIsDirectory;
359 poFS->ReleaseMutex();
360
361 hRingBufferMutex = CPLCreateMutex();
362 ReleaseMutex();
363 hCondProducer = CPLCreateCond();
364 hCondConsumer = CPLCreateCond();
365 }
366
367 /************************************************************************/
368 /* ~VSICurlStreamingHandle() */
369 /************************************************************************/
370
371 VSICurlStreamingHandle::~VSICurlStreamingHandle()
372 {
373 StopDownload();
374
375 CPLFree(m_pszURL);
376 CSLDestroy( m_papszHTTPOptions );
377
378 CPLFree(pCachedData);
379
380 CPLFree(pabyHeaderData);
381
382 CPLDestroyMutex( hRingBufferMutex );
383 CPLDestroyCond( hCondProducer );
384 CPLDestroyCond( hCondConsumer );
385 }
386
387 /************************************************************************/
388 /* SetURL() */
389 /************************************************************************/
390
391 void VSICurlStreamingHandle::SetURL(const char* pszURLIn)
392 {
393 CPLFree(m_pszURL);
394 m_pszURL = CPLStrdup(pszURLIn);
395 }
396
397 /************************************************************************/
398 /* AcquireMutex() */
399 /************************************************************************/
400
401 void VSICurlStreamingHandle::AcquireMutex()
402 {
403 CPLAcquireMutex(hRingBufferMutex, 1000.0);
404 }
405
406 /************************************************************************/
407 /* ReleaseMutex() */
408 /************************************************************************/
409
410 void VSICurlStreamingHandle::ReleaseMutex()
411 {
412 CPLReleaseMutex(hRingBufferMutex);
413 }
414
415 /************************************************************************/
416 /* Seek() */
417 /************************************************************************/
418
419 int VSICurlStreamingHandle::Seek( vsi_l_offset nOffset, int nWhence )
420 {
421 if( curOffset >= BKGND_BUFFER_SIZE )
422 {
423 if( ENABLE_DEBUG )
424 CPLDebug("VSICURL",
425 "Invalidating cache and file size due to Seek() "
426 "beyond caching zone");
427 CPLFree(pCachedData);
428 pCachedData = nullptr;
429 nCachedSize = 0;
430 AcquireMutex();
431 bHasComputedFileSize = FALSE;
432 fileSize = 0;
433 ReleaseMutex();
434 }
435
436 if( nWhence == SEEK_SET )
437 {
438 curOffset = nOffset;
439 }
440 else if( nWhence == SEEK_CUR )
441 {
442 curOffset = curOffset + nOffset;
443 }
444 else
445 {
446 curOffset = GetFileSize() + nOffset;
447 }
448 bEOF = false;
449 return 0;
450 }
451
452 /************************************************************************/
453 /* VSICURLStreamingInitWriteFuncStruct() */
454 /************************************************************************/
455
456 static void VSICURLStreamingInitWriteFuncStruct( WriteFuncStruct *psStruct )
457 {
458 psStruct->pBuffer = nullptr;
459 psStruct->nSize = 0;
460 psStruct->bIsHTTP = FALSE;
461 psStruct->bIsInHeader = TRUE;
462 psStruct->nHTTPCode = 0;
463 psStruct->bDownloadHeaderOnly = FALSE;
464 }
465
466 /************************************************************************/
467 /* VSICurlStreamingHandleWriteFuncForHeader() */
468 /************************************************************************/
469
470 static size_t
471 VSICurlStreamingHandleWriteFuncForHeader( void *buffer, size_t count,
472 size_t nmemb, void *req )
473 {
474 WriteFuncStruct* psStruct = static_cast<WriteFuncStruct *>(req);
475 const size_t nSize = count * nmemb;
476
477 char* pNewBuffer = static_cast<char*>(
478 VSIRealloc(psStruct->pBuffer, psStruct->nSize + nSize + 1));
479 if( pNewBuffer )
480 {
481 psStruct->pBuffer = pNewBuffer;
482 memcpy(psStruct->pBuffer + psStruct->nSize, buffer, nSize);
483 psStruct->pBuffer[psStruct->nSize + nSize] = '\0';
484 if( psStruct->bIsHTTP && psStruct->bIsInHeader )
485 {
486 char* pszLine = psStruct->pBuffer + psStruct->nSize;
487 if( STARTS_WITH_CI(pszLine, "HTTP/") )
488 {
489 const char* pszSpace = strchr(
490 const_cast<const char*>(pszLine), ' ');
491 if( pszSpace )
492 psStruct->nHTTPCode = atoi(pszSpace + 1);
493 }
494
495 if( pszLine[0] == '\r' || pszLine[0] == '\n' )
496 {
497 if( psStruct->bDownloadHeaderOnly )
498 {
499 // If moved permanently/temporarily, go on.
500 // Otherwise stop now.
501 if( !(psStruct->nHTTPCode == 301 ||
502 psStruct->nHTTPCode == 302) )
503 return 0;
504 }
505 else
506 {
507 psStruct->bIsInHeader = FALSE;
508 }
509 }
510 }
511 psStruct->nSize += nSize;
512 return nmemb;
513 }
514 else
515 {
516 return 0;
517 }
518 }
519
520 /************************************************************************/
521 /* GetFileSize() */
522 /************************************************************************/
523
524 vsi_l_offset VSICurlStreamingHandle::GetFileSize()
525 {
526 WriteFuncStruct sWriteFuncData;
527 WriteFuncStruct sWriteFuncHeaderData;
528
529 AcquireMutex();
530 if( bHasComputedFileSize )
531 {
532 const vsi_l_offset nRet = fileSize;
533 ReleaseMutex();
534 return nRet;
535 }
536 ReleaseMutex();
537
538 CURL* hLocalHandle = curl_easy_init();
539
540 struct curl_slist* headers =
541 VSICurlSetOptions(hLocalHandle, m_pszURL, m_papszHTTPOptions);
542
543 VSICURLStreamingInitWriteFuncStruct(&sWriteFuncHeaderData);
544
545 // HACK for mbtiles driver: Proper fix would be to auto-detect servers that
546 // don't accept HEAD http://a.tiles.mapbox.com/v3/ doesn't accept HEAD, so
547 // let's start a GET and interrupt is as soon as the header is found.
548 CPLString osVerb;
549 if( strstr(m_pszURL, ".tiles.mapbox.com/") != nullptr )
550 {
551 curl_easy_setopt(hLocalHandle, CURLOPT_HEADERDATA,
552 &sWriteFuncHeaderData);
553 curl_easy_setopt(hLocalHandle, CURLOPT_HEADERFUNCTION,
554 VSICurlStreamingHandleWriteFuncForHeader);
555
556 sWriteFuncHeaderData.bIsHTTP = STARTS_WITH(m_pszURL, "http");
557 sWriteFuncHeaderData.bDownloadHeaderOnly = TRUE;
558 osVerb = "GET";
559 }
560 else
561 {
562 curl_easy_setopt(hLocalHandle, CURLOPT_NOBODY, 1);
563 curl_easy_setopt(hLocalHandle, CURLOPT_HTTPGET, 0);
564 curl_easy_setopt(hLocalHandle, CURLOPT_HEADER, 1);
565 osVerb = "HEAD";
566 }
567
568 headers = VSICurlMergeHeaders(headers, GetCurlHeaders(osVerb, headers));
569 curl_easy_setopt(hLocalHandle, CURLOPT_HTTPHEADER, headers);
570
571 // We need that otherwise OSGEO4W's libcurl issue a dummy range request
572 // when doing a HEAD when recycling connections.
573 curl_easy_setopt(hLocalHandle, CURLOPT_RANGE, nullptr);
574
575 // Bug with older curl versions (<=7.16.4) and FTP.
576 // See http://curl.haxx.se/mail/lib-2007-08/0312.html
577 VSICURLStreamingInitWriteFuncStruct(&sWriteFuncData);
578 curl_easy_setopt(hLocalHandle, CURLOPT_WRITEDATA, &sWriteFuncData);
579 curl_easy_setopt(hLocalHandle, CURLOPT_WRITEFUNCTION,
580 VSICurlStreamingHandleWriteFuncForHeader);
581
582 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
583 curl_easy_setopt(hLocalHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
584
585 void* old_handler = CPLHTTPIgnoreSigPipe();
586 curl_easy_perform(hLocalHandle);
587 CPLHTTPRestoreSigPipeHandler(old_handler);
588 if( headers != nullptr )
589 curl_slist_free_all(headers);
590
591 AcquireMutex();
592
593 eExists = EXIST_UNKNOWN;
594 bHasComputedFileSize = TRUE;
595
596 if( STARTS_WITH(m_pszURL, "ftp") )
597 {
598 if( sWriteFuncData.pBuffer != nullptr &&
599 STARTS_WITH_CI(sWriteFuncData.pBuffer, "Content-Length: ") )
600 {
601 const char* pszBuffer =
602 sWriteFuncData.pBuffer + strlen("Content-Length: ");
603 eExists = EXIST_YES;
604 fileSize =
605 CPLScanUIntBig(pszBuffer,
606 static_cast<int>(sWriteFuncData.nSize -
607 strlen("Content-Length: ")));
608 if( ENABLE_DEBUG )
609 CPLDebug("VSICURL", "GetFileSize(%s)=" CPL_FRMT_GUIB,
610 m_pszURL, fileSize);
611 }
612 }
613
614 double dfSize = 0;
615 if( eExists != EXIST_YES )
616 {
617 const CURLcode code =
618 curl_easy_getinfo(hLocalHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD,
619 &dfSize );
620 if( code == 0 )
621 {
622 eExists = EXIST_YES;
623 if( dfSize < 0 )
624 fileSize = 0;
625 else
626 fileSize = static_cast<GUIntBig>(dfSize);
627 }
628 else
629 {
630 eExists = EXIST_NO;
631 fileSize = 0;
632 CPLError(CE_Failure, CPLE_AppDefined,
633 "VSICurlStreamingHandle::GetFileSize failed");
634 }
635
636 long response_code = 0;
637 curl_easy_getinfo(hLocalHandle, CURLINFO_HTTP_CODE, &response_code);
638 if( response_code != 200 )
639 {
640 eExists = EXIST_NO;
641 fileSize = 0;
642 }
643
644 // Try to guess if this is a directory. Generally if this is a
645 // directory, curl will retry with an URL with slash added.
646 char *pszEffectiveURL = nullptr;
647 curl_easy_getinfo(hLocalHandle, CURLINFO_EFFECTIVE_URL,
648 &pszEffectiveURL);
649 if( pszEffectiveURL != nullptr &&
650 strncmp(m_pszURL, pszEffectiveURL, strlen(m_pszURL)) == 0 &&
651 pszEffectiveURL[strlen(m_pszURL)] == '/' )
652 {
653 eExists = EXIST_YES;
654 fileSize = 0;
655 bIsDirectory = TRUE;
656 }
657
658 if( ENABLE_DEBUG )
659 CPLDebug("VSICURL",
660 "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d",
661 m_pszURL, fileSize, static_cast<int>(response_code));
662 }
663
664 CPLFree(sWriteFuncData.pBuffer);
665 CPLFree(sWriteFuncHeaderData.pBuffer);
666
667 m_poFS->AcquireMutex();
668 CachedFileProp* cachedFileProp = m_poFS->GetCachedFileProp(m_pszURL);
669 cachedFileProp->bHasComputedFileSize = TRUE;
670 #ifdef notdef
671 cachedFileProp->nChecksumOfFirst1024Bytes =
672 nRecomputedChecksumOfFirst1024Bytes;
673 #endif
674 cachedFileProp->fileSize = fileSize;
675 cachedFileProp->eExists = eExists;
676 cachedFileProp->bIsDirectory = bIsDirectory;
677 m_poFS->ReleaseMutex();
678
679 const vsi_l_offset nRet = fileSize;
680 ReleaseMutex();
681
682 curl_easy_cleanup(hLocalHandle);
683
684 return nRet;
685 }
686
687 /************************************************************************/
688 /* Exists() */
689 /************************************************************************/
690
691 int VSICurlStreamingHandle::Exists()
692 {
693 if( eExists == EXIST_UNKNOWN )
694 {
695 // Consider that only the files whose extension ends up with one that is
696 // listed in CPL_VSIL_CURL_ALLOWED_EXTENSIONS exist on the server.
697 // This can speeds up dramatically open experience, in case the server
698 // cannot return a file list.
699 // For example:
700 // gdalinfo --config CPL_VSIL_CURL_ALLOWED_EXTENSIONS ".tif" /vsicurl_streaming/http://igskmncngs506.cr.usgs.gov/gmted/Global_tiles_GMTED/075darcsec/bln/W030/30N030W_20101117_gmted_bln075.tif */
701 const char* pszAllowedExtensions =
702 CPLGetConfigOption("CPL_VSIL_CURL_ALLOWED_EXTENSIONS", nullptr);
703 if( pszAllowedExtensions )
704 {
705 char** papszExtensions =
706 CSLTokenizeString2( pszAllowedExtensions, ", ", 0 );
707 const size_t nURLLen = strlen(m_pszURL);
708 bool bFound = false;
709 for( int i = 0; papszExtensions[i] != nullptr; i++ )
710 {
711 const size_t nExtensionLen = strlen(papszExtensions[i]);
712 if( nURLLen > nExtensionLen &&
713 EQUAL(m_pszURL + nURLLen - nExtensionLen,
714 papszExtensions[i]) )
715 {
716 bFound = true;
717 break;
718 }
719 }
720
721 if( !bFound )
722 {
723 eExists = EXIST_NO;
724 fileSize = 0;
725
726 m_poFS->AcquireMutex();
727 CachedFileProp* cachedFileProp =
728 m_poFS->GetCachedFileProp(m_pszURL);
729 cachedFileProp->bHasComputedFileSize = TRUE;
730 cachedFileProp->fileSize = fileSize;
731 cachedFileProp->eExists = eExists;
732 m_poFS->ReleaseMutex();
733
734 CSLDestroy(papszExtensions);
735
736 return 0;
737 }
738
739 CSLDestroy(papszExtensions);
740 }
741
742 char chFirstByte = '\0';
743 int bExists = (Read(&chFirstByte, 1, 1) == 1);
744
745 AcquireMutex();
746 m_poFS->AcquireMutex();
747 CachedFileProp* cachedFileProp = m_poFS->GetCachedFileProp(m_pszURL);
748 cachedFileProp->eExists = eExists = bExists ? EXIST_YES : EXIST_NO;
749 m_poFS->ReleaseMutex();
750 ReleaseMutex();
751
752 Seek(0, SEEK_SET);
753 }
754
755 return eExists == EXIST_YES;
756 }
757
758 /************************************************************************/
759 /* Tell() */
760 /************************************************************************/
761
762 vsi_l_offset VSICurlStreamingHandle::Tell()
763 {
764 return curOffset;
765 }
766
767 /************************************************************************/
768 /* ReceivedBytes() */
769 /************************************************************************/
770
771 size_t VSICurlStreamingHandle::ReceivedBytes( GByte *buffer, size_t count,
772 size_t nmemb )
773 {
774 size_t nSize = count * nmemb;
775 nBodySize += nSize;
776
777 if( ENABLE_DEBUG )
778 CPLDebug("VSICURL", "Receiving %d bytes...", static_cast<int>(nSize));
779
780 if( bHasCandidateFileSize && bCanTrustCandidateFileSize &&
781 !bHasComputedFileSize )
782 {
783 m_poFS->AcquireMutex();
784 CachedFileProp* cachedFileProp = m_poFS->GetCachedFileProp(m_pszURL);
785 cachedFileProp->fileSize = fileSize = nCandidateFileSize;
786 cachedFileProp->bHasComputedFileSize = bHasComputedFileSize = TRUE;
787 if( ENABLE_DEBUG )
788 CPLDebug("VSICURL", "File size = " CPL_FRMT_GUIB, fileSize);
789 m_poFS->ReleaseMutex();
790 }
791
792 AcquireMutex();
793 if( eExists == EXIST_UNKNOWN )
794 {
795 m_poFS->AcquireMutex();
796 CachedFileProp* cachedFileProp = m_poFS->GetCachedFileProp(m_pszURL);
797 cachedFileProp->eExists = eExists = EXIST_YES;
798 m_poFS->ReleaseMutex();
799 }
800 else if( eExists == EXIST_NO && StopReceivingBytesOnError() )
801 {
802 ReleaseMutex();
803 return 0;
804 }
805
806 while( true )
807 {
808 const size_t nFree = oRingBuffer.GetCapacity() - oRingBuffer.GetSize();
809 if( nSize <= nFree )
810 {
811 oRingBuffer.Write(buffer, nSize);
812
813 // Signal to the consumer that we have added bytes to the buffer.
814 CPLCondSignal(hCondProducer);
815
816 if( bAskDownloadEnd )
817 {
818 if( ENABLE_DEBUG )
819 CPLDebug("VSICURL", "Download interruption asked");
820
821 ReleaseMutex();
822 return 0;
823 }
824 break;
825 }
826 else
827 {
828 oRingBuffer.Write(buffer, nFree);
829 buffer += nFree;
830 nSize -= nFree;
831
832 // Signal to the consumer that we have added bytes to the buffer.
833 CPLCondSignal(hCondProducer);
834
835 if( ENABLE_DEBUG )
836 CPLDebug("VSICURL",
837 "Waiting for reader to consume some bytes...");
838
839 while( oRingBuffer.GetSize() == oRingBuffer.GetCapacity() &&
840 !bAskDownloadEnd )
841 {
842 CPLCondWait(hCondConsumer, hRingBufferMutex);
843 }
844
845 if( bAskDownloadEnd )
846 {
847 if( ENABLE_DEBUG )
848 CPLDebug("VSICURL", "Download interruption asked");
849
850 ReleaseMutex();
851 return 0;
852 }
853 }
854 }
855
856 ReleaseMutex();
857
858 return nmemb;
859 }
860
861 /************************************************************************/
862 /* VSICurlStreamingHandleReceivedBytes() */
863 /************************************************************************/
864
865 static size_t VSICurlStreamingHandleReceivedBytes( void *buffer, size_t count,
866 size_t nmemb, void *req )
867 {
868 return
869 static_cast<VSICurlStreamingHandle *>(req)->
870 ReceivedBytes(static_cast<GByte *>(buffer), count, nmemb);
871 }
872
873 /************************************************************************/
874 /* VSICurlStreamingHandleReceivedBytesHeader() */
875 /************************************************************************/
876
877 #define HEADER_SIZE 32768
878
879 size_t VSICurlStreamingHandle::ReceivedBytesHeader( GByte *buffer, size_t count,
880 size_t nmemb )
881 {
882 const size_t nSize = count * nmemb;
883 if( ENABLE_DEBUG )
884 CPLDebug("VSICURL", "Receiving %d bytes for header...",
885 static_cast<int>(nSize));
886
887 // Reset buffer if we have followed link after a redirect.
888 if( nSize >= 9 && InterpretRedirect() &&
889 (nHTTPCode == 301 || nHTTPCode == 302) &&
890 STARTS_WITH_CI(reinterpret_cast<char *>(buffer), "HTTP/") )
891 {
892 nHeaderSize = 0;
893 nHTTPCode = 0;
894 }
895
896 if( nHeaderSize < HEADER_SIZE )
897 {
898 const size_t nSz = std::min(nSize, HEADER_SIZE - nHeaderSize);
899 memcpy(pabyHeaderData + nHeaderSize, buffer, nSz);
900 pabyHeaderData[nHeaderSize + nSz] = '\0';
901 nHeaderSize += nSz;
902
903 #if DEBUG_VERBOSE
904 CPLDebug("VSICURL", "Header : %s", pabyHeaderData);
905 #endif
906
907 AcquireMutex();
908
909 if( eExists == EXIST_UNKNOWN && nHTTPCode == 0 &&
910 strchr(reinterpret_cast<char *>(pabyHeaderData), '\n') != nullptr &&
911 STARTS_WITH_CI(reinterpret_cast<char *>(pabyHeaderData),
912 "HTTP/") )
913 {
914 nHTTPCode = 0;
915 const char* pszSpace = strchr(
916 const_cast<const char*>(
917 reinterpret_cast<char *>(pabyHeaderData)), ' ');
918 if( pszSpace )
919 nHTTPCode = atoi(pszSpace + 1);
920 if( ENABLE_DEBUG )
921 CPLDebug("VSICURL", "HTTP code = %d", nHTTPCode);
922
923 // If moved permanently/temporarily, go on.
924 if( !(InterpretRedirect() &&
925 (nHTTPCode == 301 || nHTTPCode == 302)) )
926 {
927 m_poFS->AcquireMutex();
928 CachedFileProp* cachedFileProp =
929 m_poFS->GetCachedFileProp(m_pszURL);
930 eExists = nHTTPCode == 200 ? EXIST_YES : EXIST_NO;
931 cachedFileProp->eExists = eExists;
932 m_poFS->ReleaseMutex();
933 }
934 }
935
936 if( !(InterpretRedirect() && (nHTTPCode == 301 || nHTTPCode == 302)) &&
937 !bHasComputedFileSize )
938 {
939 // Caution: When gzip compression is enabled, the content-length is
940 // the compressed size, which we are not interested in, so we must
941 // not take it into account.
942
943 const char* pszContentLength =
944 strstr(reinterpret_cast<char *>(pabyHeaderData),
945 "Content-Length: ");
946 const char* pszEndOfLine =
947 pszContentLength ? strchr(pszContentLength, '\n') : nullptr;
948 if( bCanTrustCandidateFileSize && pszEndOfLine != nullptr )
949 {
950 const char* pszVal =
951 pszContentLength + strlen("Content-Length: ");
952 bHasCandidateFileSize = true;
953 nCandidateFileSize =
954 CPLScanUIntBig(pszVal,
955 static_cast<int>(pszEndOfLine - pszVal));
956 if( ENABLE_DEBUG )
957 CPLDebug("VSICURL",
958 "Has found candidate file size = " CPL_FRMT_GUIB,
959 nCandidateFileSize);
960 }
961
962 const char* pszContentEncoding =
963 strstr(reinterpret_cast<char *>(pabyHeaderData),
964 "Content-Encoding: ");
965 pszEndOfLine =
966 pszContentEncoding ? strchr(pszContentEncoding, '\n') : nullptr;
967 if( bHasCandidateFileSize && pszEndOfLine != nullptr )
968 {
969 const char* pszVal =
970 pszContentEncoding + strlen("Content-Encoding: ");
971 if( STARTS_WITH(pszVal, "gzip") )
972 {
973 if( ENABLE_DEBUG )
974 CPLDebug("VSICURL",
975 "GZip compression enabled --> "
976 "cannot trust candidate file size");
977 bCanTrustCandidateFileSize = false;
978 }
979 }
980 }
981
982 ReleaseMutex();
983 }
984
985 return nmemb;
986 }
987
988 /************************************************************************/
989 /* VSICurlStreamingHandleReceivedBytesHeader() */
990 /************************************************************************/
991
992 static size_t
993 VSICurlStreamingHandleReceivedBytesHeader( void *buffer, size_t count,
994 size_t nmemb, void *req )
995 {
996 return
997 static_cast<VSICurlStreamingHandle *>(req)->
998 ReceivedBytesHeader(static_cast<GByte *>(buffer), count, nmemb);
999 }
1000
1001 /************************************************************************/
1002 /* DownloadInThread() */
1003 /************************************************************************/
1004
1005 void VSICurlStreamingHandle::DownloadInThread()
1006 {
1007 CURL* hCurlHandle = curl_easy_init();
1008
1009 struct curl_slist* headers =
1010 VSICurlSetOptions(hCurlHandle, m_pszURL, m_papszHTTPOptions);
1011 headers = VSICurlMergeHeaders(headers, GetCurlHeaders("GET", headers));
1012 curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers);
1013
1014 static bool bHasCheckVersion = false;
1015 static bool bSupportGZip = false;
1016 if( !bHasCheckVersion )
1017 {
1018 bSupportGZip = strstr(curl_version(), "zlib/") != nullptr;
1019 bHasCheckVersion = true;
1020 }
1021 if( bSupportGZip &&
1022 CPLTestBool(CPLGetConfigOption("CPL_CURL_GZIP", "YES")) )
1023 {
1024 curl_easy_setopt(hCurlHandle, CURLOPT_ENCODING, "gzip");
1025 }
1026
1027 if( pabyHeaderData == nullptr )
1028 pabyHeaderData = static_cast<GByte *>(CPLMalloc(HEADER_SIZE + 1));
1029 nHeaderSize = 0;
1030 nBodySize = 0;
1031 nHTTPCode = 0;
1032
1033 curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, this);
1034 curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION,
1035 VSICurlStreamingHandleReceivedBytesHeader);
1036
1037 curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, this);
1038 curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION,
1039 VSICurlStreamingHandleReceivedBytes);
1040
1041 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
1042 szCurlErrBuf[0] = '\0';
1043 curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
1044
1045 void* old_handler = CPLHTTPIgnoreSigPipe();
1046 CURLcode eRet = curl_easy_perform(hCurlHandle);
1047 CPLHTTPRestoreSigPipeHandler(old_handler);
1048 if( headers != nullptr )
1049 curl_slist_free_all(headers);
1050
1051 curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr);
1052 curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr);
1053 curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, nullptr);
1054 curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, nullptr);
1055
1056 AcquireMutex();
1057 if( !bAskDownloadEnd && eRet == 0 && !bHasComputedFileSize )
1058 {
1059 m_poFS->AcquireMutex();
1060 CachedFileProp* cachedFileProp = m_poFS->GetCachedFileProp(m_pszURL);
1061 cachedFileProp->fileSize = fileSize = nBodySize;
1062 cachedFileProp->bHasComputedFileSize = bHasComputedFileSize = TRUE;
1063 if( ENABLE_DEBUG )
1064 CPLDebug("VSICURL", "File size = " CPL_FRMT_GUIB, fileSize);
1065 m_poFS->ReleaseMutex();
1066 }
1067
1068 bDownloadInProgress = FALSE;
1069 bDownloadStopped = TRUE;
1070
1071 // Signal to the consumer that the download has ended.
1072 CPLCondSignal(hCondProducer);
1073 ReleaseMutex();
1074
1075 curl_easy_cleanup(hCurlHandle);
1076 }
1077
1078 static void VSICurlDownloadInThread( void* pArg )
1079 {
1080 static_cast<VSICurlStreamingHandle *>(pArg)->DownloadInThread();
1081 }
1082
1083 /************************************************************************/
1084 /* StartDownload() */
1085 /************************************************************************/
1086
1087 void VSICurlStreamingHandle::StartDownload()
1088 {
1089 if( bDownloadInProgress || bDownloadStopped )
1090 return;
1091
1092 CPLDebug("VSICURL", "Start download for %s", m_pszURL);
1093
1094 oRingBuffer.Reset();
1095 bDownloadInProgress = TRUE;
1096 nRingBufferFileOffset = 0;
1097 hThread = CPLCreateJoinableThread(VSICurlDownloadInThread, this);
1098 }
1099
1100 /************************************************************************/
1101 /* StopDownload() */
1102 /************************************************************************/
1103
1104 void VSICurlStreamingHandle::StopDownload()
1105 {
1106 if( hThread )
1107 {
1108 CPLDebug("VSICURL", "Stop download for %s", m_pszURL);
1109
1110 AcquireMutex();
1111 // Signal to the producer that we ask for download interruption.
1112 bAskDownloadEnd = TRUE;
1113 CPLCondSignal(hCondConsumer);
1114
1115 // Wait for the producer to have finished.
1116 while( bDownloadInProgress )
1117 CPLCondWait(hCondProducer, hRingBufferMutex);
1118
1119 bAskDownloadEnd = FALSE;
1120
1121 ReleaseMutex();
1122
1123 CPLJoinThread(hThread);
1124 hThread = nullptr;
1125 }
1126
1127 oRingBuffer.Reset();
1128 bDownloadStopped = FALSE;
1129 }
1130
1131 /************************************************************************/
1132 /* PutRingBufferInCache() */
1133 /************************************************************************/
1134
1135 void VSICurlStreamingHandle::PutRingBufferInCache()
1136 {
1137 if( nRingBufferFileOffset >= BKGND_BUFFER_SIZE )
1138 return;
1139
1140 AcquireMutex();
1141
1142 // Cache any remaining bytes available in the ring buffer.
1143 size_t nBufSize = oRingBuffer.GetSize();
1144 if( nBufSize > 0 )
1145 {
1146 if( nRingBufferFileOffset + nBufSize > BKGND_BUFFER_SIZE )
1147 nBufSize =
1148 static_cast<size_t>(BKGND_BUFFER_SIZE - nRingBufferFileOffset);
1149 GByte* pabyTmp = static_cast<GByte *>(CPLMalloc(nBufSize));
1150 oRingBuffer.Read(pabyTmp, nBufSize);
1151
1152 // Signal to the producer that we have ingested some bytes.
1153 CPLCondSignal(hCondConsumer);
1154
1155 AddRegion(nRingBufferFileOffset, nBufSize, pabyTmp);
1156 nRingBufferFileOffset += nBufSize;
1157 CPLFree(pabyTmp);
1158 }
1159
1160 ReleaseMutex();
1161 }
1162
1163 /************************************************************************/
1164 /* Read() */
1165 /************************************************************************/
1166
1167 size_t VSICurlStreamingHandle::Read( void * const pBuffer, size_t const nSize,
1168 size_t const nMemb )
1169 {
1170 GByte* pabyBuffer = static_cast<GByte *>(pBuffer);
1171 const size_t nBufferRequestSize = nSize * nMemb;
1172 const vsi_l_offset curOffsetOri = curOffset;
1173 const vsi_l_offset nRingBufferFileOffsetOri = nRingBufferFileOffset;
1174 if( nBufferRequestSize == 0 )
1175 return 0;
1176 size_t nRemaining = nBufferRequestSize;
1177
1178 AcquireMutex();
1179 // fileSize might be set wrongly to 0, such as
1180 // /vsicurl_streaming/https://query.data.world/s/jgsghstpphjhicstradhy5kpjwrnfy
1181 const int bHasComputedFileSizeLocal = bHasComputedFileSize && fileSize > 0;
1182 const vsi_l_offset fileSizeLocal = fileSize;
1183 ReleaseMutex();
1184
1185 if( bHasComputedFileSizeLocal && curOffset >= fileSizeLocal )
1186 {
1187 CPLDebug("VSICURL", "Read attempt beyond end of file");
1188 bEOF = true;
1189 }
1190 if( bEOF )
1191 return 0;
1192
1193 if( curOffset < nRingBufferFileOffset )
1194 PutRingBufferInCache();
1195
1196 if( ENABLE_DEBUG )
1197 CPLDebug("VSICURL", "Read [" CPL_FRMT_GUIB ", " CPL_FRMT_GUIB "[ in %s",
1198 curOffset, curOffset + nBufferRequestSize, m_pszURL);
1199
1200 #ifdef notdef
1201 if( pCachedData != nullptr && nCachedSize >= 1024 &&
1202 nRecomputedChecksumOfFirst1024Bytes == 0 )
1203 {
1204 for( size_t i = 0; i < 1024 / sizeof(int); i++ )
1205 {
1206 int nVal = 0;
1207 memcpy(&nVal, pCachedData + i * sizeof(int), sizeof(int));
1208 nRecomputedChecksumOfFirst1024Bytes += nVal;
1209 }
1210
1211 if( bHasComputedFileSizeLocal )
1212 {
1213 poFS->AcquireMutex();
1214 CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
1215 if( cachedFileProp->nChecksumOfFirst1024Bytes == 0 )
1216 {
1217 cachedFileProp->nChecksumOfFirst1024Bytes =
1218 nRecomputedChecksumOfFirst1024Bytes;
1219 }
1220 else if( nRecomputedChecksumOfFirst1024Bytes !=
1221 cachedFileProp->nChecksumOfFirst1024Bytes )
1222 {
1223 CPLDebug("VSICURL",
1224 "Invalidating previously cached file size. "
1225 "First bytes of file have changed!");
1226 AcquireMutex();
1227 bHasComputedFileSize = FALSE;
1228 cachedFileProp->bHasComputedFileSize = FALSE;
1229 cachedFileProp->nChecksumOfFirst1024Bytes = 0;
1230 ReleaseMutex();
1231 }
1232 poFS->ReleaseMutex();
1233 }
1234 }
1235 #endif
1236
1237 // Can we use the cache?
1238 if( pCachedData != nullptr && curOffset < nCachedSize )
1239 {
1240 const size_t nSz =
1241 std::min(nRemaining, static_cast<size_t>(nCachedSize - curOffset));
1242 if( ENABLE_DEBUG )
1243 CPLDebug("VSICURL", "Using cache for [%d, %d[ in %s",
1244 static_cast<int>(curOffset),
1245 static_cast<int>(curOffset + nSz), m_pszURL);
1246 memcpy(pabyBuffer, pCachedData + curOffset, nSz);
1247 pabyBuffer += nSz;
1248 curOffset += nSz;
1249 nRemaining -= nSz;
1250 }
1251
1252 // Is the request partially covered by the cache and going beyond file size?
1253 if( pCachedData != nullptr && bHasComputedFileSizeLocal &&
1254 curOffset <= nCachedSize &&
1255 curOffset + nRemaining > fileSizeLocal &&
1256 fileSize == nCachedSize )
1257 {
1258 size_t nSz = static_cast<size_t>(nCachedSize - curOffset);
1259 if( ENABLE_DEBUG && nSz != 0 )
1260 CPLDebug("VSICURL", "Using cache for [%d, %d[ in %s",
1261 static_cast<int>(curOffset),
1262 static_cast<int>(curOffset + nSz), m_pszURL);
1263 memcpy(pabyBuffer, pCachedData + curOffset, nSz);
1264 pabyBuffer += nSz;
1265 curOffset += nSz;
1266 nRemaining -= nSz;
1267 bEOF = true;
1268 }
1269
1270 // Has a Seek() being done since the last Read()?
1271 if( !bEOF && nRemaining > 0 && curOffset != nRingBufferFileOffset )
1272 {
1273 // Backward seek: Need to restart the download from the beginning.
1274 if( curOffset < nRingBufferFileOffset )
1275 StopDownload();
1276
1277 StartDownload();
1278
1279 const vsi_l_offset SKIP_BUFFER_SIZE = 32768;
1280 GByte* pabyTmp = static_cast<GByte *>(CPLMalloc(SKIP_BUFFER_SIZE));
1281
1282 CPLAssert(curOffset >= nRingBufferFileOffset);
1283 vsi_l_offset nBytesToSkip = curOffset - nRingBufferFileOffset;
1284 while(nBytesToSkip > 0)
1285 {
1286 vsi_l_offset nBytesToRead = nBytesToSkip;
1287
1288 AcquireMutex();
1289 if( nBytesToRead > oRingBuffer.GetSize() )
1290 nBytesToRead = oRingBuffer.GetSize();
1291 if( nBytesToRead > SKIP_BUFFER_SIZE )
1292 nBytesToRead = SKIP_BUFFER_SIZE;
1293 oRingBuffer.Read(pabyTmp, static_cast<size_t>(nBytesToRead));
1294
1295 // Signal to the producer that we have ingested some bytes.
1296 CPLCondSignal(hCondConsumer);
1297 ReleaseMutex();
1298
1299 if( nBytesToRead )
1300 AddRegion(nRingBufferFileOffset,
1301 static_cast<size_t>(nBytesToRead), pabyTmp);
1302
1303 nBytesToSkip -= nBytesToRead;
1304 nRingBufferFileOffset += nBytesToRead;
1305
1306 if( nBytesToRead == 0 && nBytesToSkip != 0 )
1307 {
1308 if( ENABLE_DEBUG )
1309 CPLDebug("VSICURL",
1310 "Waiting for writer to produce some bytes...");
1311
1312 AcquireMutex();
1313 while(oRingBuffer.GetSize() == 0 && bDownloadInProgress)
1314 CPLCondWait(hCondProducer, hRingBufferMutex);
1315 const int bBufferEmpty = (oRingBuffer.GetSize() == 0);
1316 ReleaseMutex();
1317
1318 if( bBufferEmpty && !bDownloadInProgress )
1319 break;
1320 }
1321 }
1322
1323 CPLFree(pabyTmp);
1324
1325 if( nBytesToSkip != 0 )
1326 {
1327 bEOF = true;
1328 return 0;
1329 }
1330 }
1331
1332 if( !bEOF && nRemaining > 0 )
1333 {
1334 StartDownload();
1335 CPLAssert(curOffset == nRingBufferFileOffset);
1336 }
1337
1338 // Fill the destination buffer from the ring buffer.
1339 while(!bEOF && nRemaining > 0)
1340 {
1341 AcquireMutex();
1342 size_t nToRead = oRingBuffer.GetSize();
1343 if( nToRead > nRemaining )
1344 nToRead = nRemaining;
1345 oRingBuffer.Read(pabyBuffer, nToRead);
1346
1347 // Signal to the producer that we have ingested some bytes.
1348 CPLCondSignal(hCondConsumer);
1349 ReleaseMutex();
1350
1351 if( nToRead )
1352 AddRegion(curOffset, nToRead, pabyBuffer);
1353
1354 nRemaining -= nToRead;
1355 pabyBuffer += nToRead;
1356 curOffset += nToRead;
1357 nRingBufferFileOffset += nToRead;
1358
1359 if( nToRead == 0 && nRemaining != 0 )
1360 {
1361 if( ENABLE_DEBUG )
1362 CPLDebug("VSICURL",
1363 "Waiting for writer to produce some bytes...");
1364
1365 AcquireMutex();
1366 while(oRingBuffer.GetSize() == 0 && bDownloadInProgress)
1367 CPLCondWait(hCondProducer, hRingBufferMutex);
1368 const bool bBufferEmpty = oRingBuffer.GetSize() == 0;
1369 ReleaseMutex();
1370
1371 if( bBufferEmpty && !bDownloadInProgress )
1372 break;
1373 }
1374 }
1375
1376 if( ENABLE_DEBUG )
1377 CPLDebug("VSICURL", "Read(%d) = %d",
1378 static_cast<int>(nBufferRequestSize),
1379 static_cast<int>(nBufferRequestSize - nRemaining));
1380 size_t nRet = (nBufferRequestSize - nRemaining) / nSize;
1381 if( nRet < nMemb )
1382 bEOF = true;
1383
1384 // Give a chance to specialized filesystem to deal with errors to redirect
1385 // elsewhere.
1386 if( curOffsetOri == 0 && nRingBufferFileOffsetOri == 0 &&
1387 !StopReceivingBytesOnError() &&
1388 eExists == EXIST_NO && nRemaining < nBufferRequestSize )
1389 {
1390 const size_t nErrorBufferMaxSize = 4096;
1391 GByte* pabyErrorBuffer =
1392 static_cast<GByte *>(CPLMalloc(nErrorBufferMaxSize + 1));
1393 size_t nRead = nBufferRequestSize - nRemaining;
1394 size_t nErrorBufferSize = std::min(nErrorBufferMaxSize, nRead);
1395 memcpy( pabyErrorBuffer, pBuffer, nErrorBufferSize );
1396 if( nRead < nErrorBufferMaxSize )
1397 nErrorBufferSize += Read(pabyErrorBuffer + nRead, 1,
1398 nErrorBufferMaxSize - nRead);
1399 pabyErrorBuffer[nErrorBufferSize] = 0;
1400 StopDownload();
1401 if( CanRestartOnError(reinterpret_cast<char *>(pabyErrorBuffer),
1402 reinterpret_cast<char *>(pabyHeaderData),
1403 true) )
1404 {
1405 curOffset = 0;
1406 nRingBufferFileOffset = 0;
1407 bEOF = false;
1408 AcquireMutex();
1409 eExists = EXIST_UNKNOWN;
1410 bHasComputedFileSize = FALSE;
1411 fileSize = 0;
1412 ReleaseMutex();
1413 nCachedSize = 0;
1414 m_poFS->AcquireMutex();
1415 CachedFileProp* cachedFileProp =
1416 m_poFS->GetCachedFileProp(m_pszURL);
1417 cachedFileProp->bHasComputedFileSize = FALSE;
1418 cachedFileProp->fileSize = 0;
1419 cachedFileProp->eExists = EXIST_UNKNOWN;
1420 m_poFS->ReleaseMutex();
1421 nRet = Read(pBuffer, nSize, nMemb);
1422 }
1423 else
1424 {
1425 CPLDebug("VSICURL", "Error buffer: %s",
1426 reinterpret_cast<char *>(pabyErrorBuffer));
1427 nRet = 0;
1428 }
1429
1430 CPLFree(pabyErrorBuffer);
1431 }
1432
1433 return nRet;
1434 }
1435
1436 /************************************************************************/
1437 /* AddRegion() */
1438 /************************************************************************/
1439
1440 void VSICurlStreamingHandle::AddRegion( vsi_l_offset nFileOffsetStart,
1441 size_t nSize,
1442 GByte *pData )
1443 {
1444 if( nFileOffsetStart >= BKGND_BUFFER_SIZE )
1445 return;
1446
1447 if( pCachedData == nullptr )
1448 pCachedData = static_cast<GByte *>(CPLMalloc(BKGND_BUFFER_SIZE));
1449
1450 if( nFileOffsetStart <= nCachedSize &&
1451 nFileOffsetStart + nSize > nCachedSize )
1452 {
1453 const size_t nSz =
1454 std::min(nSize,
1455 static_cast<size_t>(BKGND_BUFFER_SIZE - nFileOffsetStart));
1456 if( ENABLE_DEBUG )
1457 CPLDebug("VSICURL", "Writing [%d, %d[ in cache for %s",
1458 static_cast<int>(nFileOffsetStart),
1459 static_cast<int>(nFileOffsetStart + nSz), m_pszURL);
1460 memcpy(pCachedData + nFileOffsetStart, pData, nSz);
1461 nCachedSize = static_cast<size_t>(nFileOffsetStart + nSz);
1462 }
1463 }
1464 /************************************************************************/
1465 /* Write() */
1466 /************************************************************************/
1467
1468 size_t VSICurlStreamingHandle::Write( const void * /* pBuffer */,
1469 size_t /* nSize */,
1470 size_t /* nMemb */ )
1471 {
1472 return 0;
1473 }
1474
1475 /************************************************************************/
1476 /* Eof() */
1477 /************************************************************************/
1478
1479 int VSICurlStreamingHandle::Eof()
1480 {
1481 return bEOF;
1482 }
1483
1484 /************************************************************************/
1485 /* Flush() */
1486 /************************************************************************/
1487
1488 int VSICurlStreamingHandle::Flush()
1489 {
1490 return 0;
1491 }
1492
1493 /************************************************************************/
1494 /* Close() */
1495 /************************************************************************/
1496
1497 int VSICurlStreamingHandle::Close()
1498 {
1499 return 0;
1500 }
1501
1502 /************************************************************************/
1503 /* VSICurlStreamingFSHandler() */
1504 /************************************************************************/
1505
1506 VSICurlStreamingFSHandler::VSICurlStreamingFSHandler()
1507 {
1508 hMutex = CPLCreateMutex();
1509 CPLReleaseMutex(hMutex);
1510 }
1511
1512 /************************************************************************/
1513 /* ~VSICurlStreamingFSHandler() */
1514 /************************************************************************/
1515
1516 VSICurlStreamingFSHandler::~VSICurlStreamingFSHandler()
1517 {
1518 VSICurlStreamingFSHandler::ClearCache();
1519
1520 CPLDestroyMutex( hMutex );
1521 hMutex = nullptr;
1522 }
1523
1524 /************************************************************************/
1525 /* ClearCache() */
1526 /************************************************************************/
1527
1528 void VSICurlStreamingFSHandler::ClearCache()
1529 {
1530 CPLMutexHolder oHolder( &hMutex );
1531
1532 for( auto& kv: cacheFileSize )
1533 {
1534 CPLFree(kv.second);
1535 }
1536 cacheFileSize.clear();
1537 }
1538
1539 /************************************************************************/
1540 /* AcquireMutex() */
1541 /************************************************************************/
1542
1543 void VSICurlStreamingFSHandler::AcquireMutex()
1544 {
1545 CPLAcquireMutex(hMutex, 1000.0);
1546 }
1547
1548 /************************************************************************/
1549 /* ReleaseMutex() */
1550 /************************************************************************/
1551
1552 void VSICurlStreamingFSHandler::ReleaseMutex()
1553 {
1554 CPLReleaseMutex(hMutex);
1555 }
1556
1557 /************************************************************************/
1558 /* GetCachedFileProp() */
1559 /************************************************************************/
1560
1561 /* Should be called under the FS Lock */
1562
1563 CachedFileProp *
1564 VSICurlStreamingFSHandler::GetCachedFileProp( const char* pszURL )
1565 {
1566 CachedFileProp* cachedFileProp = cacheFileSize[pszURL];
1567 if( cachedFileProp == nullptr )
1568 {
1569 cachedFileProp =
1570 static_cast<CachedFileProp *>(CPLMalloc(sizeof(CachedFileProp)));
1571 cachedFileProp->eExists = EXIST_UNKNOWN;
1572 cachedFileProp->bHasComputedFileSize = FALSE;
1573 cachedFileProp->fileSize = 0;
1574 cachedFileProp->bIsDirectory = FALSE;
1575 #ifdef notdef
1576 cachedFileProp->nChecksumOfFirst1024Bytes = 0;
1577 #endif
1578 cacheFileSize[pszURL] = cachedFileProp;
1579 }
1580
1581 return cachedFileProp;
1582 }
1583
1584 /************************************************************************/
1585 /* CreateFileHandle() */
1586 /************************************************************************/
1587
1588 VSICurlStreamingHandle *
1589 VSICurlStreamingFSHandler::CreateFileHandle( const char* pszURL )
1590 {
1591 return new VSICurlStreamingHandle(this, pszURL);
1592 }
1593
1594 /************************************************************************/
1595 /* Open() */
1596 /************************************************************************/
1597
1598 VSIVirtualHandle* VSICurlStreamingFSHandler::Open( const char *pszFilename,
1599 const char *pszAccess,
1600 bool /* bSetError */,
1601 CSLConstList /* papszOptions */ )
1602 {
1603 if( !STARTS_WITH_CI(pszFilename, GetFSPrefix()) )
1604 return nullptr;
1605
1606 if( strchr(pszAccess, 'w') != nullptr ||
1607 strchr(pszAccess, '+') != nullptr )
1608 {
1609 CPLError(CE_Failure, CPLE_AppDefined,
1610 "Only read-only mode is supported for %s",
1611 GetFSPrefix().c_str());
1612 return nullptr;
1613 }
1614
1615 VSICurlStreamingHandle* poHandle =
1616 CreateFileHandle(pszFilename + GetFSPrefix().size());
1617 // If we didn't get a filelist, check that the file really exists.
1618 if( poHandle == nullptr || !poHandle->Exists() )
1619 {
1620 delete poHandle;
1621 return nullptr;
1622 }
1623
1624 if( CPLTestBool( CPLGetConfigOption( "VSI_CACHE", "FALSE" ) ) )
1625 return VSICreateCachedFile( poHandle );
1626
1627 return poHandle;
1628 }
1629
1630 /************************************************************************/
1631 /* Stat() */
1632 /************************************************************************/
1633
1634 int VSICurlStreamingFSHandler::Stat( const char *pszFilename,
1635 VSIStatBufL *pStatBuf,
1636 int nFlags )
1637 {
1638 if( !STARTS_WITH_CI(pszFilename, GetFSPrefix()) )
1639 return -1;
1640
1641 memset(pStatBuf, 0, sizeof(VSIStatBufL));
1642
1643 VSICurlStreamingHandle* poHandle =
1644 CreateFileHandle(pszFilename + GetFSPrefix().size());
1645 if( poHandle == nullptr )
1646 {
1647 return -1;
1648 }
1649 if( poHandle->IsKnownFileSize() ||
1650 ((nFlags & VSI_STAT_SIZE_FLAG) && !poHandle->IsDirectory() &&
1651 CPLTestBool(CPLGetConfigOption("CPL_VSIL_CURL_SLOW_GET_SIZE",
1652 "YES"))) )
1653 {
1654 pStatBuf->st_size = poHandle->GetFileSize();
1655 }
1656
1657 int nRet = (poHandle->Exists()) ? 0 : -1;
1658 pStatBuf->st_mode = poHandle->IsDirectory() ? S_IFDIR : S_IFREG;
1659
1660 delete poHandle;
1661 return nRet;
1662 }
1663
1664 /************************************************************************/
1665 /* GetActualURL() */
1666 /************************************************************************/
1667
1668 const char* VSICurlStreamingFSHandler::GetActualURL(const char* pszFilename)
1669 {
1670 if( !STARTS_WITH_CI(pszFilename, GetFSPrefix()) )
1671 return pszFilename;
1672 auto poHandle = std::unique_ptr<VSICurlStreamingHandle>(
1673 CreateFileHandle(pszFilename + GetFSPrefix().size()));
1674 if( poHandle == nullptr )
1675 return pszFilename;
1676 return CPLSPrintf("%s", poHandle->GetURL());
1677 }
1678
1679 /************************************************************************/
1680 /* IVSIS3LikeStreamingFSHandler */
1681 /************************************************************************/
1682
1683 class IVSIS3LikeStreamingFSHandler: public VSICurlStreamingFSHandler
1684 {
1685 CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeStreamingFSHandler)
1686
1687 public:
1688 IVSIS3LikeStreamingFSHandler() = default;
1689
1690 virtual void UpdateMapFromHandle( IVSIS3LikeHandleHelper * /*poHandleHelper*/ ) {}
1691 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * /*poHandleHelper*/ ) {}
1692 };
1693
1694 /************************************************************************/
1695 /* VSIS3StreamingFSHandler */
1696 /************************************************************************/
1697
1698 class VSIS3StreamingFSHandler final: public IVSIS3LikeStreamingFSHandler
1699 {
1700 CPL_DISALLOW_COPY_ASSIGN(VSIS3StreamingFSHandler)
1701
1702 protected:
1703 CPLString GetFSPrefix() override { return "/vsis3_streaming/"; }
1704 VSICurlStreamingHandle* CreateFileHandle( const char* pszURL ) override;
1705
1706 public:
1707 VSIS3StreamingFSHandler() = default;
1708 ~VSIS3StreamingFSHandler() override = default;
1709
1710 const char* GetOptions() override
1711 { return VSIGetFileSystemOptions("/vsis3/"); }
1712
1713 void UpdateMapFromHandle( IVSIS3LikeHandleHelper * poHandleHelper ) override;
1714 void UpdateHandleFromMap( IVSIS3LikeHandleHelper * poHandleHelper ) override;
1715
1716 void ClearCache() override
1717 {
1718 IVSIS3LikeStreamingFSHandler::ClearCache();
1719 VSIS3UpdateParams::ClearCache();
1720 }
1721 };
1722
1723 /************************************************************************/
1724 /* UpdateMapFromHandle() */
1725 /************************************************************************/
1726
1727 void VSIS3StreamingFSHandler::UpdateMapFromHandle(
1728 IVSIS3LikeHandleHelper * poHandleHelper )
1729 {
1730 VSIS3UpdateParams::UpdateMapFromHandle(poHandleHelper);
1731 }
1732
1733 /************************************************************************/
1734 /* UpdateHandleFromMap() */
1735 /************************************************************************/
1736
1737 void VSIS3StreamingFSHandler::UpdateHandleFromMap(
1738 IVSIS3LikeHandleHelper * poHandleHelper )
1739 {
1740 VSIS3UpdateParams::UpdateHandleFromMap(poHandleHelper);
1741 }
1742
1743 /************************************************************************/
1744 /* VSIS3LikeStreamingHandle */
1745 /************************************************************************/
1746
1747 class VSIS3LikeStreamingHandle final: public VSICurlStreamingHandle
1748 {
1749 CPL_DISALLOW_COPY_ASSIGN(VSIS3LikeStreamingHandle)
1750
1751 IVSIS3LikeHandleHelper* m_poS3HandleHelper = nullptr;
1752
1753 protected:
1754 struct curl_slist* GetCurlHeaders(
1755 const CPLString& osVerb,
1756 const struct curl_slist* psExistingHeaders) override;
1757 bool StopReceivingBytesOnError() override { return false; }
1758 bool CanRestartOnError( const char* pszErrorMsg,
1759 const char* pszHeaders,
1760 bool bSetError ) override;
1761 bool InterpretRedirect() override { return false; }
1762
1763 public:
1764 VSIS3LikeStreamingHandle( IVSIS3LikeStreamingFSHandler* poFS,
1765 IVSIS3LikeHandleHelper* poS3HandleHelper );
1766 ~VSIS3LikeStreamingHandle() override;
1767 };
1768
1769 /************************************************************************/
1770 /* CreateFileHandle() */
1771 /************************************************************************/
1772
1773 VSICurlStreamingHandle *
1774 VSIS3StreamingFSHandler::CreateFileHandle( const char* pszURL )
1775 {
1776 VSIS3HandleHelper* poS3HandleHelper =
1777 VSIS3HandleHelper::BuildFromURI(pszURL, GetFSPrefix().c_str(),
1778 false);
1779 if( poS3HandleHelper )
1780 {
1781 UpdateHandleFromMap(poS3HandleHelper);
1782 return new VSIS3LikeStreamingHandle(this, poS3HandleHelper);
1783 }
1784 return nullptr;
1785 }
1786
1787 /************************************************************************/
1788 /* VSIS3LikeStreamingHandle() */
1789 /************************************************************************/
1790
1791 VSIS3LikeStreamingHandle::VSIS3LikeStreamingHandle(
1792 IVSIS3LikeStreamingFSHandler* poFS,
1793 IVSIS3LikeHandleHelper* poS3HandleHelper) :
1794 VSICurlStreamingHandle(poFS, poS3HandleHelper->GetURL()),
1795 m_poS3HandleHelper(poS3HandleHelper)
1796 {}
1797
1798 /************************************************************************/
1799 /* ~VSIS3LikeStreamingHandle() */
1800 /************************************************************************/
1801
1802 VSIS3LikeStreamingHandle::~VSIS3LikeStreamingHandle()
1803 {
1804 delete m_poS3HandleHelper;
1805 }
1806
1807 /************************************************************************/
1808 /* GetCurlHeaders() */
1809 /************************************************************************/
1810
1811 struct curl_slist*
1812 VSIS3LikeStreamingHandle::GetCurlHeaders( const CPLString& osVerb,
1813 const struct curl_slist* psExistingHeaders )
1814 {
1815 return m_poS3HandleHelper->GetCurlHeaders(osVerb, psExistingHeaders);
1816 }
1817
1818 /************************************************************************/
1819 /* CanRestartOnError() */
1820 /************************************************************************/
1821
1822 bool VSIS3LikeStreamingHandle::CanRestartOnError( const char* pszErrorMsg,
1823 const char* pszHeaders,
1824 bool bSetError )
1825 {
1826 if( m_poS3HandleHelper->CanRestartOnError(pszErrorMsg, pszHeaders,
1827 bSetError) )
1828 {
1829 static_cast<IVSIS3LikeStreamingFSHandler*>(m_poFS)->
1830 UpdateMapFromHandle(m_poS3HandleHelper);
1831
1832 SetURL(m_poS3HandleHelper->GetURL());
1833 return true;
1834 }
1835 return false;
1836 }
1837
1838
1839
1840 /************************************************************************/
1841 /* VSIGSStreamingFSHandler */
1842 /************************************************************************/
1843
1844 class VSIGSStreamingFSHandler final: public IVSIS3LikeStreamingFSHandler
1845 {
1846 protected:
1847 CPLString GetFSPrefix() override { return "/vsigs_streaming/"; }
1848 VSICurlStreamingHandle* CreateFileHandle( const char* pszURL ) override;
1849
1850 public:
1851 VSIGSStreamingFSHandler() {}
1852 ~VSIGSStreamingFSHandler() override {}
1853
1854 const char* GetOptions() override
1855 { return VSIGetFileSystemOptions("/vsigs/"); }
1856 };
1857
1858 /************************************************************************/
1859 /* CreateFileHandle() */
1860 /************************************************************************/
1861
1862 VSICurlStreamingHandle *
1863 VSIGSStreamingFSHandler::CreateFileHandle( const char* pszURL )
1864 {
1865 VSIGSHandleHelper* poGCHandleHelper =
1866 VSIGSHandleHelper::BuildFromURI(pszURL, GetFSPrefix().c_str());
1867 if( poGCHandleHelper )
1868 {
1869 return new VSIS3LikeStreamingHandle(this, poGCHandleHelper);
1870 }
1871 return nullptr;
1872 }
1873
1874
1875 /************************************************************************/
1876 /* VSIAzureStreamingFSHandler */
1877 /************************************************************************/
1878
1879 class VSIAzureStreamingFSHandler final: public IVSIS3LikeStreamingFSHandler
1880 {
1881 protected:
1882 CPLString GetFSPrefix() override { return "/vsiaz_streaming/"; }
1883 VSICurlStreamingHandle* CreateFileHandle( const char* pszURL ) override;
1884
1885 public:
1886 VSIAzureStreamingFSHandler() {}
1887 ~VSIAzureStreamingFSHandler() override {}
1888
1889 const char* GetOptions() override
1890 { return VSIGetFileSystemOptions("/vsiaz/"); }
1891 };
1892
1893 /************************************************************************/
1894 /* CreateFileHandle() */
1895 /************************************************************************/
1896
1897 VSICurlStreamingHandle *
1898 VSIAzureStreamingFSHandler::CreateFileHandle( const char* pszURL )
1899 {
1900 VSIAzureBlobHandleHelper* poHandleHelper =
1901 VSIAzureBlobHandleHelper::BuildFromURI(pszURL, GetFSPrefix().c_str());
1902 if( poHandleHelper )
1903 {
1904 return new VSIS3LikeStreamingHandle(this, poHandleHelper);
1905 }
1906 return nullptr;
1907 }
1908
1909
1910 /************************************************************************/
1911 /* VSIOSSStreamingFSHandler */
1912 /************************************************************************/
1913
1914 class VSIOSSStreamingFSHandler final: public IVSIS3LikeStreamingFSHandler
1915 {
1916 CPL_DISALLOW_COPY_ASSIGN(VSIOSSStreamingFSHandler)
1917
1918 std::map< CPLString, VSIOSSUpdateParams > oMapBucketsToOSSParams{};
1919
1920 protected:
1921 CPLString GetFSPrefix() override { return "/vsioss_streaming/"; }
1922 VSICurlStreamingHandle* CreateFileHandle( const char* pszURL ) override;
1923
1924 public:
1925 VSIOSSStreamingFSHandler() = default;
1926 ~VSIOSSStreamingFSHandler() override = default;
1927
1928 const char* GetOptions() override
1929 { return VSIGetFileSystemOptions("/vsioss/"); }
1930
1931 void UpdateMapFromHandle( IVSIS3LikeHandleHelper * poHandleHelper ) override;
1932 void UpdateHandleFromMap( IVSIS3LikeHandleHelper * poHandleHelper ) override;
1933 };
1934
1935 /************************************************************************/
1936 /* UpdateMapFromHandle() */
1937 /************************************************************************/
1938
1939 void VSIOSSStreamingFSHandler::UpdateMapFromHandle(
1940 IVSIS3LikeHandleHelper * poHandleHelper )
1941 {
1942 CPLMutexHolder oHolder( &hMutex );
1943
1944 VSIOSSHandleHelper * poOSSHandleHelper =
1945 cpl::down_cast<VSIOSSHandleHelper *>(poHandleHelper);
1946 oMapBucketsToOSSParams[ poOSSHandleHelper->GetBucket() ] =
1947 VSIOSSUpdateParams ( poOSSHandleHelper );
1948 }
1949
1950 /************************************************************************/
1951 /* UpdateHandleFromMap() */
1952 /************************************************************************/
1953
1954 void VSIOSSStreamingFSHandler::UpdateHandleFromMap(
1955 IVSIS3LikeHandleHelper * poHandleHelper )
1956 {
1957 CPLMutexHolder oHolder( &hMutex );
1958
1959 VSIOSSHandleHelper * poOSSHandleHelper =
1960 cpl::down_cast<VSIOSSHandleHelper *>(poHandleHelper);
1961 std::map< CPLString, VSIOSSUpdateParams>::iterator oIter =
1962 oMapBucketsToOSSParams.find(poOSSHandleHelper->GetBucket());
1963 if( oIter != oMapBucketsToOSSParams.end() )
1964 {
1965 oIter->second.UpdateHandlerHelper(poOSSHandleHelper);
1966 }
1967 }
1968
1969 /************************************************************************/
1970 /* CreateFileHandle() */
1971 /************************************************************************/
1972
1973 VSICurlStreamingHandle *
1974 VSIOSSStreamingFSHandler::CreateFileHandle( const char* pszURL )
1975 {
1976 VSIOSSHandleHelper* poOSSHandleHelper =
1977 VSIOSSHandleHelper::BuildFromURI(pszURL, GetFSPrefix().c_str(),
1978 false);
1979 if( poOSSHandleHelper )
1980 {
1981 UpdateHandleFromMap(poOSSHandleHelper);
1982 return new VSIS3LikeStreamingHandle(this, poOSSHandleHelper);
1983 }
1984 return nullptr;
1985 }
1986
1987 /************************************************************************/
1988 /* VSISwiftStreamingFSHandler */
1989 /************************************************************************/
1990
1991 class VSISwiftStreamingFSHandler final: public IVSIS3LikeStreamingFSHandler
1992 {
1993 protected:
1994 CPLString GetFSPrefix() override { return "/vsiswift_streaming/"; }
1995 VSICurlStreamingHandle* CreateFileHandle( const char* pszURL ) override;
1996
1997 public:
1998 VSISwiftStreamingFSHandler() {}
1999 ~VSISwiftStreamingFSHandler() override {}
2000
2001 const char* GetOptions() override
2002 { return VSIGetFileSystemOptions("/vsiswift/"); }
2003 };
2004
2005 /************************************************************************/
2006 /* CreateFileHandle() */
2007 /************************************************************************/
2008
2009 VSICurlStreamingHandle *
2010 VSISwiftStreamingFSHandler::CreateFileHandle( const char* pszURL )
2011 {
2012 VSISwiftHandleHelper* poHandleHelper =
2013 VSISwiftHandleHelper::BuildFromURI(pszURL, GetFSPrefix().c_str());
2014 if( poHandleHelper )
2015 {
2016 return new VSIS3LikeStreamingHandle(this, poHandleHelper);
2017 }
2018 return nullptr;
2019 }
2020
2021
2022
2023 //! @endcond
2024
2025 } /* end of anonymous namespace */
2026
2027 /************************************************************************/
2028 /* VSIInstallCurlStreamingFileHandler() */
2029 /************************************************************************/
2030
2031 /**
2032 * \brief Install /vsicurl_streaming/ HTTP/FTP file system handler (requires
2033 * libcurl).
2034 *
2035 * @see <a href="gdal_virtual_file_systems.html#gdal_virtual_file_systems_vsicurl_streaming">/vsicurl_streaming/ documentation</a>
2036 *
2037 * @since GDAL 1.10
2038 */
2039 void VSIInstallCurlStreamingFileHandler(void)
2040 {
2041 VSIFileManager::InstallHandler( "/vsicurl_streaming/",
2042 new VSICurlStreamingFSHandler );
2043 }
2044
2045 /************************************************************************/
2046 /* VSIInstallS3StreamingFileHandler() */
2047 /************************************************************************/
2048
2049 /**
2050 * \brief Install /vsis3_streaming/ Amazon S3 file system handler (requires
2051 * libcurl).
2052 *
2053 * @see <a href="gdal_virtual_file_systems.html#gdal_virtual_file_systems_vsis3_streaming">/vsis3_streaming/ documentation</a>
2054 *
2055 * @since GDAL 2.1
2056 */
2057 void VSIInstallS3StreamingFileHandler(void)
2058 {
2059 VSIFileManager::InstallHandler( "/vsis3_streaming/",
2060 new VSIS3StreamingFSHandler );
2061 }
2062
2063 /************************************************************************/
2064 /* VSIInstallGSStreamingFileHandler() */
2065 /************************************************************************/
2066
2067 /**
2068 * \brief Install /vsigs_streaming/ Google Cloud Storage file system handler
2069 * (requires libcurl)
2070 *
2071 * @see <a href="gdal_virtual_file_systems.html#gdal_virtual_file_systems_vsigs_streaming">/vsigs_streaming/ documentation</a>
2072 *
2073 * @since GDAL 2.2
2074 */
2075
2076 void VSIInstallGSStreamingFileHandler( void )
2077 {
2078 VSIFileManager::InstallHandler( "/vsigs_streaming/", new VSIGSStreamingFSHandler );
2079 }
2080
2081 /************************************************************************/
2082 /* VSIInstallAzureStreamingFileHandler() */
2083 /************************************************************************/
2084
2085 /**
2086 * \brief Install /vsiaz_streaming/ Microsoft Azure Blob file system handler
2087 * (requires libcurl)
2088 *
2089 * @see <a href="gdal_virtual_file_systems.html#gdal_virtual_file_systems_vsiaz_streaming">/vsiaz_streaming/ documentation</a>1
2090 *
2091 * @since GDAL 2.3
2092 */
2093
2094 void VSIInstallAzureStreamingFileHandler( void )
2095 {
2096 VSIFileManager::InstallHandler( "/vsiaz_streaming/",
2097 new VSIAzureStreamingFSHandler );
2098 }
2099
2100 /************************************************************************/
2101 /* VSIInstallOSSStreamingFileHandler() */
2102 /************************************************************************/
2103
2104 /**
2105 * \brief Install /vsioss_streaming/ Alibaba Cloud Object Storage Service (OSS)
2106 * file system handler (requires libcurl)
2107 *
2108 * @see <a href="gdal_virtual_file_systems.html#gdal_virtual_file_systems_vsioss_streaming">/vsioss_streaming/ documentation</a>
2109 *
2110 * @since GDAL 2.3
2111 */
2112
2113 void VSIInstallOSSStreamingFileHandler( void )
2114 {
2115 VSIFileManager::InstallHandler( "/vsioss_streaming/",
2116 new VSIOSSStreamingFSHandler );
2117 }
2118
2119 /************************************************************************/
2120 /* VSIInstallSwiftStreamingFileHandler() */
2121 /************************************************************************/
2122
2123 /**
2124 * \brief Install /vsiswift_streamin/ OpenStack Swif Object Storage (Swift) file
2125 * system handler (requires libcurl)
2126 *
2127 * @see <a href="gdal_virtual_file_systems.html#gdal_virtual_file_systems_vsiswift_streaming">/vsiswift_streaming/ documentation</a>
2128 *
2129 * @since GDAL 2.3
2130 */
2131
2132 void VSIInstallSwiftStreamingFileHandler( void )
2133 {
2134 VSIFileManager::InstallHandler( "/vsiswift_streaming/",
2135 new VSISwiftStreamingFSHandler );
2136 }
2137 //! @cond Doxygen_Suppress
2138
2139 /************************************************************************/
2140 /* VSICurlStreamingClearCache() */
2141 /************************************************************************/
2142
2143 void VSICurlStreamingClearCache( void )
2144 {
2145 // FIXME ? Currently we have different filesystem instances for
2146 // vsicurl/, /vsis3/, /vsigs/ . So each one has its own cache of regions,
2147 // file size, etc.
2148 CSLConstList papszPrefix = VSIFileManager::GetPrefixes();
2149 for( size_t i = 0; papszPrefix && papszPrefix[i]; ++i )
2150 {
2151 auto poFSHandler =
2152 dynamic_cast<VSICurlStreamingFSHandler*>(
2153 VSIFileManager::GetHandler( papszPrefix[i] ));
2154
2155 if( poFSHandler )
2156 poFSHandler->ClearCache();
2157 }
2158 }
2159
2160 //! @endcond
2161
2162 #endif // !defined(HAVE_CURL) || defined(CPL_MULTIPROC_STUB)
2163