1 /******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Implement VSI large file api for OpenStack Swift
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2017-2018, Even Rouault <even.rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29 #include "cpl_json.h"
30 #include "cpl_port.h"
31 #include "cpl_http.h"
32 #include "cpl_time.h"
33 #include "cpl_vsil_curl_priv.h"
34 #include "cpl_vsil_curl_class.h"
35
36 #include <errno.h>
37
38 #include <algorithm>
39 #include <set>
40 #include <map>
41 #include <memory>
42
43 #include "cpl_swift.h"
44
45 CPL_CVSID("$Id: cpl_vsil_swift.cpp 978dfdb07e7a2ee3e7fc742b8cad366a264c0e65 2021-06-14 23:53:42 +0200 Even Rouault $")
46
47 #ifndef HAVE_CURL
48
VSIInstallSwiftFileHandler(void)49 void VSIInstallSwiftFileHandler( void )
50 {
51 // Not supported
52 }
53
54 #else
55
56 //! @cond Doxygen_Suppress
57 #ifndef DOXYGEN_SKIP
58
59 #define ENABLE_DEBUG 0
60
61 namespace cpl {
62
63 /************************************************************************/
64 /* AnalyseSwiftFileList() */
65 /************************************************************************/
66
67 void VSICurlFilesystemHandler::AnalyseSwiftFileList(
68 const CPLString& osBaseURL,
69 const CPLString& osPrefix,
70 const char* pszJson,
71 CPLStringList& osFileList,
72 int nMaxFilesThisQuery,
73 int nMaxFiles,
74 bool& bIsTruncated,
75 CPLString& osNextMarker )
76 {
77 #if DEBUG_VERBOSE
78 CPLDebug("SWIFT", "%s", pszJson);
79 #endif
80 osNextMarker = "";
81 bIsTruncated = false;
82
83 CPLJSONDocument oDoc;
84 if( !oDoc.LoadMemory(reinterpret_cast<const GByte*>(pszJson)) )
85 return;
86
87 std::vector< std::pair<CPLString, FileProp> > aoProps;
88 // Count the number of occurrences of a path. Can be 1 or 2. 2 in the case
89 // that both a filename and directory exist
90 std::map<CPLString, int> aoNameCount;
91
92 CPLJSONArray oArray = oDoc.GetRoot().ToArray();
93 for( int i = 0; i < oArray.Size(); i++ )
94 {
95 CPLJSONObject oItem = oArray[i];
96 std::string osName = oItem.GetString("name");
97 GInt64 nSize = oItem.GetLong("bytes");
98 std::string osLastModified = oItem.GetString("last_modified");
99 CPLString osSubdir = oItem.GetString("subdir");
100 bool bHasCount = oItem.GetLong("count", -1) >= 0;
101 if( !osName.empty() )
102 {
103 osNextMarker = osName;
104 if( osName.size() > osPrefix.size() &&
105 osName.substr(0, osPrefix.size()) == osPrefix )
106 {
107 if( bHasCount )
108 {
109 // Case when listing /vsiswift/
110 FileProp prop;
111 prop.eExists = EXIST_YES;
112 prop.bIsDirectory = true;
113 prop.bHasComputedFileSize = true;
114 prop.fileSize = 0;
115 prop.mTime = 0;
116
117 aoProps.push_back(
118 std::pair<CPLString, FileProp>
119 (osName, prop));
120 aoNameCount[ osName ] ++;
121 }
122 else
123 {
124 FileProp prop;
125 prop.eExists = EXIST_YES;
126 prop.bHasComputedFileSize = true;
127 prop.fileSize = static_cast<GUIntBig>(nSize);
128 prop.bIsDirectory = false;
129 prop.mTime = 0;
130 int nYear, nMonth, nDay, nHour, nMin, nSec;
131 if( sscanf( osLastModified.c_str(),
132 "%04d-%02d-%02dT%02d:%02d:%02d",
133 &nYear, &nMonth, &nDay,
134 &nHour, &nMin, &nSec ) == 6 )
135 {
136 struct tm brokendowntime;
137 brokendowntime.tm_year = nYear - 1900;
138 brokendowntime.tm_mon = nMonth - 1;
139 brokendowntime.tm_mday = nDay;
140 brokendowntime.tm_hour = nHour;
141 brokendowntime.tm_min = nMin;
142 brokendowntime.tm_sec = nSec;
143 prop.mTime =
144 static_cast<time_t>(
145 CPLYMDHMSToUnixTime(&brokendowntime));
146 }
147
148 aoProps.push_back(
149 std::pair<CPLString, FileProp>
150 (osName.substr(osPrefix.size()), prop));
151 aoNameCount[ osName.substr(osPrefix.size()) ] ++;
152 }
153 }
154 }
155 else if( !osSubdir.empty() )
156 {
157 osNextMarker = osSubdir;
158 if( osSubdir.back() == '/' )
159 osSubdir.resize( osSubdir.size() - 1 );
160 if( osSubdir.find(osPrefix) == 0 )
161 {
162
163 FileProp prop;
164 prop.eExists = EXIST_YES;
165 prop.bIsDirectory = true;
166 prop.bHasComputedFileSize = true;
167 prop.fileSize = 0;
168 prop.mTime = 0;
169
170 aoProps.push_back(
171 std::pair<CPLString, FileProp>
172 (osSubdir.substr(osPrefix.size()), prop));
173 aoNameCount[ osSubdir.substr(osPrefix.size()) ] ++;
174 }
175 }
176
177 if( nMaxFiles > 0 && aoProps.size() > static_cast<unsigned>(nMaxFiles) )
178 break;
179 }
180
181 bIsTruncated =
182 aoProps.size() >= static_cast<unsigned>(nMaxFilesThisQuery);
183 if( !bIsTruncated )
184 {
185 osNextMarker.clear();
186 }
187
188 for( size_t i = 0; i < aoProps.size(); i++ )
189 {
190 CPLString osSuffix;
191 if( aoNameCount[aoProps[i].first] == 2 &&
192 aoProps[i].second.bIsDirectory )
193 {
194 // Add a / suffix to disambiguish the situation
195 // Normally we don't suffix directories with /, but we have
196 // no alternative here
197 osSuffix = "/";
198 }
199 if( nMaxFiles != 1 )
200 {
201 CPLString osCachedFilename =
202 osBaseURL + "/" + CPLAWSURLEncode(osPrefix,false) +
203 CPLAWSURLEncode(aoProps[i].first,false) + osSuffix;
204 #if DEBUG_VERBOSE
205 CPLDebug("SWIFT", "Cache %s", osCachedFilename.c_str());
206 #endif
207 SetCachedFileProp(osCachedFilename, aoProps[i].second);
208 }
209 osFileList.AddString( (aoProps[i].first + osSuffix).c_str() );
210 }
211 }
212
213 /************************************************************************/
214 /* VSISwiftFSHandler */
215 /************************************************************************/
216
217 class VSISwiftFSHandler final : public IVSIS3LikeFSHandler
218 {
219 CPL_DISALLOW_COPY_ASSIGN(VSISwiftFSHandler)
220
221 protected:
222 VSICurlHandle* CreateFileHandle( const char* pszFilename ) override;
223 CPLString GetURLFromFilename( const CPLString& osFilename ) override;
224
225 const char* GetDebugKey() const override { return "SWIFT"; }
226
227 IVSIS3LikeHandleHelper* CreateHandleHelper(
228 const char* pszURI, bool bAllowNoObject) override;
229
230 CPLString GetFSPrefix() const override { return "/vsiswift/"; }
231
232 char** GetFileList( const char *pszFilename,
233 int nMaxFiles,
234 bool* pbGotFileList ) override;
235
236 void ClearCache() override;
237
238 public:
239 VSISwiftFSHandler() = default;
240 ~VSISwiftFSHandler() override;
241
242 VSIVirtualHandle *Open( const char *pszFilename,
243 const char *pszAccess,
244 bool bSetError,
245 CSLConstList papszOptions ) override;
246
247 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
248 int nFlags ) override;
249
250 VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
251 const char* const *papszOptions) override
252 {
253 return VSICurlFilesystemHandler::OpenDir(pszPath, nRecurseDepth,
254 papszOptions);
255 }
256
257 const char* GetOptions() override;
258 };
259
260 /************************************************************************/
261 /* VSISwiftHandle */
262 /************************************************************************/
263
264 class VSISwiftHandle final : public IVSIS3LikeHandle
265 {
266 CPL_DISALLOW_COPY_ASSIGN(VSISwiftHandle)
267
268 VSISwiftHandleHelper* m_poHandleHelper = nullptr;
269
270 protected:
271 struct curl_slist* GetCurlHeaders(
272 const CPLString& osVerb,
273 const struct curl_slist* psExistingHeaders ) override;
274 virtual bool Authenticate() override;
275
276 public:
277 VSISwiftHandle( VSISwiftFSHandler* poFS,
278 const char* pszFilename,
279 VSISwiftHandleHelper* poHandleHelper );
280 ~VSISwiftHandle() override;
281 };
282
283 /************************************************************************/
284 /* Open() */
285 /************************************************************************/
286
287 VSIVirtualHandle* VSISwiftFSHandler::Open( const char *pszFilename,
288 const char *pszAccess,
289 bool bSetError,
290 CSLConstList papszOptions )
291 {
292 if( !STARTS_WITH_CI(pszFilename, GetFSPrefix()) )
293 return nullptr;
294
295 if( strchr(pszAccess, 'w') != nullptr || strchr(pszAccess, 'a') != nullptr )
296 {
297 if( strchr(pszAccess, '+') != nullptr &&
298 !CPLTestBool(CPLGetConfigOption("CPL_VSIL_USE_TEMP_FILE_FOR_RANDOM_WRITE", "NO")) )
299 {
300 CPLError(CE_Failure, CPLE_AppDefined,
301 "w+ not supported for /vsiswift, unless "
302 "CPL_VSIL_USE_TEMP_FILE_FOR_RANDOM_WRITE is set to YES");
303 errno = EACCES;
304 return nullptr;
305 }
306
307 VSISwiftHandleHelper* poHandleHelper =
308 VSISwiftHandleHelper::BuildFromURI(pszFilename + GetFSPrefix().size(),
309 GetFSPrefix().c_str());
310 if( poHandleHelper == nullptr )
311 return nullptr;
312 UpdateHandleFromMap(poHandleHelper);
313 VSIS3WriteHandle* poHandle =
314 new VSIS3WriteHandle(this, pszFilename, poHandleHelper, true, papszOptions);
315 if( !poHandle->IsOK() )
316 {
317 delete poHandle;
318 return nullptr;
319 }
320 if( strchr(pszAccess, '+') != nullptr)
321 {
322 return VSICreateUploadOnCloseFile(poHandle);
323 }
324 return poHandle;
325 }
326
327 return
328 VSICurlFilesystemHandler::Open(pszFilename, pszAccess, bSetError, papszOptions);
329 }
330
331 /************************************************************************/
332 /* ~VSISwiftFSHandler() */
333 /************************************************************************/
334
335 VSISwiftFSHandler::~VSISwiftFSHandler()
336 {
337 VSISwiftFSHandler::ClearCache();
338 VSISwiftHandleHelper::CleanMutex();
339 }
340
341 /************************************************************************/
342 /* ClearCache() */
343 /************************************************************************/
344
345 void VSISwiftFSHandler::ClearCache()
346 {
347 VSICurlFilesystemHandler::ClearCache();
348
349 VSISwiftHandleHelper::ClearCache();
350 }
351
352 /************************************************************************/
353 /* GetOptions() */
354 /************************************************************************/
355
356 const char* VSISwiftFSHandler::GetOptions()
357 {
358 static CPLString osOptions(
359 CPLString("<Options>") +
360 " <Option name='SWIFT_STORAGE_URL' type='string' "
361 "description='Storage URL. To use with SWIFT_AUTH_TOKEN'/>"
362 " <Option name='SWIFT_AUTH_TOKEN' type='string' "
363 "description='Authorization token'/>"
364 " <Option name='SWIFT_AUTH_V1_URL' type='string' "
365 "description='Authentication V1 URL. To use with SWIFT_USER and "
366 "SWIFT_KEY'/>"
367 " <Option name='SWIFT_USER' type='string' "
368 "description='User name to use with authentication V1'/>"
369 " <Option name='SWIFT_KEY' type='string' "
370 "description='Key/password to use with authentication V1'/>"
371 " <Option name='OS_IDENTITY_API_VERSION' type='string' "
372 "description='OpenStack identity API version'/>"
373 " <Option name='OS_AUTH_TYPE' type='string' "
374 "description='Authentication URL'/>"
375 " <Option name='OS_USERNAME' type='string' "
376 "description='User name'/>"
377 " <Option name='OS_PASSWORD' type='string' "
378 "description='Password'/>"
379 " <Option name='OS_USER_DOMAIN_NAME' type='string' "
380 "description='User domain name'/>"
381 " <Option name='OS_PROJECT_NAME' type='string' "
382 "description='Project name'/>"
383 " <Option name='OS_PROJECT_DOMAIN_NAME' type='string' "
384 "description='Project domain name'/>"
385 " <Option name='OS_REGION_NAME' type='string' "
386 "description='Region name'/>"
387 + VSICurlFilesystemHandler::GetOptionsStatic() +
388 "</Options>");
389 return osOptions.c_str();
390 }
391
392 /************************************************************************/
393 /* CreateFileHandle() */
394 /************************************************************************/
395
396 VSICurlHandle* VSISwiftFSHandler::CreateFileHandle(const char* pszFilename)
397 {
398 VSISwiftHandleHelper* poHandleHelper =
399 VSISwiftHandleHelper::BuildFromURI(pszFilename + GetFSPrefix().size(),
400 GetFSPrefix().c_str());
401 if( poHandleHelper )
402 {
403 UpdateHandleFromMap(poHandleHelper);
404 return new VSISwiftHandle(this, pszFilename, poHandleHelper);
405 }
406 return nullptr;
407 }
408
409 /************************************************************************/
410 /* GetURLFromFilename() */
411 /************************************************************************/
412
413 CPLString VSISwiftFSHandler::GetURLFromFilename( const CPLString& osFilename )
414 {
415 CPLString osFilenameWithoutPrefix = osFilename.substr(GetFSPrefix().size());
416
417 VSISwiftHandleHelper* poHandleHelper =
418 VSISwiftHandleHelper::BuildFromURI(osFilenameWithoutPrefix,
419 GetFSPrefix().c_str());
420 if( poHandleHelper == nullptr )
421 {
422 return "";
423 }
424 CPLString osBaseURL(poHandleHelper->GetURL());
425 if( !osBaseURL.empty() && osBaseURL.back() == '/' )
426 osBaseURL.resize(osBaseURL.size()-1);
427 delete poHandleHelper;
428
429 return osBaseURL;
430 }
431
432 /************************************************************************/
433 /* CreateHandleHelper() */
434 /************************************************************************/
435
436 IVSIS3LikeHandleHelper* VSISwiftFSHandler::CreateHandleHelper(const char* pszURI,
437 bool)
438 {
439 return VSISwiftHandleHelper::BuildFromURI(
440 pszURI, GetFSPrefix().c_str());
441 }
442
443 /************************************************************************/
444 /* Stat() */
445 /************************************************************************/
446
447 int VSISwiftFSHandler::Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
448 int nFlags )
449 {
450 if( !STARTS_WITH_CI(pszFilename, GetFSPrefix()) )
451 return -1;
452
453 CPLString osFilename(pszFilename);
454 if( osFilename.back() == '/' )
455 osFilename.resize( osFilename.size() - 1 );
456
457 memset(pStatBuf, 0, sizeof(VSIStatBufL));
458
459 if( VSICurlFilesystemHandler::Stat(pszFilename, pStatBuf, nFlags) == 0 )
460 {
461 // if querying /vsiswift/container_name, the GET will succeed and
462 // we would consider this as a file whereas it should be exposed as
463 // a directory
464 if( std::count(osFilename.begin(), osFilename.end(), '/') <= 2 )
465 {
466
467 IVSIS3LikeHandleHelper* poS3HandleHelper =
468 CreateHandleHelper(pszFilename + GetFSPrefix().size(), true);
469 CPLString osURL(poS3HandleHelper->GetURL());
470 delete poS3HandleHelper;
471
472 FileProp cachedFileProp;
473 cachedFileProp.eExists = EXIST_YES;
474 cachedFileProp.bHasComputedFileSize = false;
475 cachedFileProp.fileSize = 0;
476 cachedFileProp.bIsDirectory = true;
477 cachedFileProp.mTime = 0;
478 SetCachedFileProp(osURL, cachedFileProp);
479
480 pStatBuf->st_size = 0;
481 pStatBuf->st_mode = S_IFDIR;
482 }
483 return 0;
484 }
485
486 // In the case of a directory, a GET on it will not work, so we have to
487 // query the upper directory contents
488 if( std::count(osFilename.begin(), osFilename.end(), '/') < 2 )
489 return -1;
490
491 char** papszContents = VSIReadDir( CPLGetPath(osFilename) );
492 int nRet = CSLFindStringCaseSensitive(papszContents,
493 CPLGetFilename(osFilename)) >= 0 ? 0 : -1;
494 CSLDestroy(papszContents);
495 if( nRet == 0 )
496 {
497 pStatBuf->st_mode = S_IFDIR;
498 }
499 return nRet;
500 }
501
502 /************************************************************************/
503 /* GetFileList() */
504 /************************************************************************/
505
506 char** VSISwiftFSHandler::GetFileList( const char *pszDirname,
507 int nMaxFiles,
508 bool* pbGotFileList )
509 {
510 if( ENABLE_DEBUG )
511 CPLDebug(GetDebugKey(), "GetFileList(%s)" , pszDirname);
512 *pbGotFileList = false;
513 CPLAssert( strlen(pszDirname) >= GetFSPrefix().size() );
514 CPLString osDirnameWithoutPrefix = pszDirname + GetFSPrefix().size();
515 if( !osDirnameWithoutPrefix.empty() &&
516 osDirnameWithoutPrefix.back() == '/' )
517 {
518 osDirnameWithoutPrefix.resize(osDirnameWithoutPrefix.size()-1);
519 }
520
521 CPLString osBucket(osDirnameWithoutPrefix);
522 CPLString osObjectKey;
523 size_t nSlashPos = osDirnameWithoutPrefix.find('/');
524 if( nSlashPos != std::string::npos )
525 {
526 osBucket = osDirnameWithoutPrefix.substr(0, nSlashPos);
527 osObjectKey = osDirnameWithoutPrefix.substr(nSlashPos+1);
528 }
529
530 IVSIS3LikeHandleHelper* poS3HandleHelper =
531 CreateHandleHelper(osBucket, true);
532 if( poS3HandleHelper == nullptr )
533 {
534 return nullptr;
535 }
536
537 WriteFuncStruct sWriteFuncData;
538
539 CPLStringList osFileList; // must be left in this scope !
540 CPLString osNextMarker; // must be left in this scope !
541
542 CPLString osMaxKeys = CPLGetConfigOption("SWIFT_MAX_KEYS", "10000");
543 int nMaxFilesThisQuery = atoi(osMaxKeys);
544 if( nMaxFiles > 0 && nMaxFiles <= 100 && nMaxFiles < nMaxFilesThisQuery )
545 {
546 nMaxFilesThisQuery = nMaxFiles+1;
547 }
548 const CPLString osPrefix(osObjectKey.empty() ? CPLString():
549 osObjectKey + "/");
550
551 while( true )
552 {
553 bool bRetry;
554 int nRetryCount = 0;
555 const int nMaxRetry = atoi(CPLGetConfigOption("GDAL_HTTP_MAX_RETRY",
556 CPLSPrintf("%d",CPL_HTTP_MAX_RETRY)));
557 // coverity[tainted_data]
558 double dfRetryDelay = CPLAtof(CPLGetConfigOption("GDAL_HTTP_RETRY_DELAY",
559 CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY)));
560 do
561 {
562 bRetry = false;
563 poS3HandleHelper->ResetQueryParameters();
564 CPLString osBaseURL(poS3HandleHelper->GetURL());
565
566 CURLM* hCurlMultiHandle = GetCurlMultiHandleFor(osBaseURL);
567 CURL* hCurlHandle = curl_easy_init();
568
569 if( !osBucket.empty() )
570 {
571 poS3HandleHelper->AddQueryParameter("delimiter", "/");
572 if( !osNextMarker.empty() )
573 poS3HandleHelper->AddQueryParameter("marker", osNextMarker);
574 poS3HandleHelper->AddQueryParameter("limit",
575 CPLSPrintf("%d", nMaxFilesThisQuery));
576 if( !osPrefix.empty() )
577 poS3HandleHelper->AddQueryParameter("prefix", osPrefix);
578 }
579
580 struct curl_slist* headers =
581 VSICurlSetOptions(hCurlHandle, poS3HandleHelper->GetURL(), nullptr);
582 // Disable automatic redirection
583 curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 0 );
584
585 curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, nullptr);
586
587 VSICURLInitWriteFuncStruct(&sWriteFuncData, nullptr, nullptr, nullptr);
588 curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData);
589 curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION,
590 VSICurlHandleWriteFunc);
591
592 WriteFuncStruct sWriteFuncHeaderData;
593 VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, nullptr, nullptr, nullptr);
594 curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &sWriteFuncHeaderData);
595 curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION,
596 VSICurlHandleWriteFunc);
597
598 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
599 curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
600
601 headers = VSICurlMergeHeaders(headers,
602 poS3HandleHelper->GetCurlHeaders("GET", headers));
603 curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers);
604
605 MultiPerform(hCurlMultiHandle, hCurlHandle);
606
607 VSICURLResetHeaderAndWriterFunctions(hCurlHandle);
608
609 if( headers != nullptr )
610 curl_slist_free_all(headers);
611
612 if( sWriteFuncData.pBuffer == nullptr)
613 {
614 delete poS3HandleHelper;
615 curl_easy_cleanup(hCurlHandle);
616 CPLFree(sWriteFuncHeaderData.pBuffer);
617 return nullptr;
618 }
619
620 long response_code = 0;
621 curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code);
622 if( response_code != 200 )
623 {
624 // Look if we should attempt a retry
625 const double dfNewRetryDelay = CPLHTTPGetNewRetryDelay(
626 static_cast<int>(response_code), dfRetryDelay,
627 sWriteFuncHeaderData.pBuffer, szCurlErrBuf);
628 if( dfNewRetryDelay > 0 &&
629 nRetryCount < nMaxRetry )
630 {
631 CPLError(CE_Warning, CPLE_AppDefined,
632 "HTTP error code: %d - %s. "
633 "Retrying again in %.1f secs",
634 static_cast<int>(response_code),
635 poS3HandleHelper->GetURL().c_str(),
636 dfRetryDelay);
637 CPLSleep(dfRetryDelay);
638 dfRetryDelay = dfNewRetryDelay;
639 nRetryCount++;
640 bRetry = true;
641 CPLFree(sWriteFuncData.pBuffer);
642 CPLFree(sWriteFuncHeaderData.pBuffer);
643 }
644 else
645 {
646 CPLDebug(GetDebugKey(), "%s",
647 sWriteFuncData.pBuffer
648 ? sWriteFuncData.pBuffer : "(null)");
649 CPLFree(sWriteFuncData.pBuffer);
650 CPLFree(sWriteFuncHeaderData.pBuffer);
651 delete poS3HandleHelper;
652 curl_easy_cleanup(hCurlHandle);
653 return nullptr;
654 }
655 }
656 else
657 {
658 *pbGotFileList = true;
659 bool bIsTruncated;
660 AnalyseSwiftFileList( osBaseURL,
661 osPrefix,
662 sWriteFuncData.pBuffer,
663 osFileList,
664 nMaxFilesThisQuery,
665 nMaxFiles,
666 bIsTruncated,
667 osNextMarker );
668
669 CPLFree(sWriteFuncData.pBuffer);
670 CPLFree(sWriteFuncHeaderData.pBuffer);
671
672 if( osNextMarker.empty() )
673 {
674 delete poS3HandleHelper;
675 curl_easy_cleanup(hCurlHandle);
676 return osFileList.StealList();
677 }
678 }
679
680 curl_easy_cleanup(hCurlHandle);
681 }
682 while(bRetry);
683 }
684 }
685
686 /************************************************************************/
687 /* VSISwiftHandle() */
688 /************************************************************************/
689
690 VSISwiftHandle::VSISwiftHandle( VSISwiftFSHandler* poFSIn,
691 const char* pszFilename,
692 VSISwiftHandleHelper* poHandleHelper ) :
693 IVSIS3LikeHandle(poFSIn, pszFilename, poHandleHelper->GetURL()),
694 m_poHandleHelper(poHandleHelper)
695 {
696 }
697
698 /************************************************************************/
699 /* ~VSISwiftHandle() */
700 /************************************************************************/
701
702 VSISwiftHandle::~VSISwiftHandle()
703 {
704 delete m_poHandleHelper;
705 }
706
707 /************************************************************************/
708 /* GetCurlHeaders() */
709 /************************************************************************/
710
711 struct curl_slist* VSISwiftHandle::GetCurlHeaders( const CPLString& osVerb,
712 const struct curl_slist* psExistingHeaders )
713 {
714 return m_poHandleHelper->GetCurlHeaders(osVerb, psExistingHeaders);
715 }
716
717 /************************************************************************/
718 /* Authenticate() */
719 /************************************************************************/
720
721 bool VSISwiftHandle::Authenticate()
722 {
723 return m_poHandleHelper->Authenticate();
724 }
725
726
727
728 } /* end of namespace cpl */
729
730
731 #endif // DOXYGEN_SKIP
732 //! @endcond
733
734 /************************************************************************/
735 /* VSIInstallSwiftFileHandler() */
736 /************************************************************************/
737
738 /**
739 * \brief Install /vsiswift/ OpenStack Swif Object Storage (Swift) file
740 * system handler (requires libcurl)
741 *
742 * @see <a href="gdal_virtual_file_systems.html#gdal_virtual_file_systems_vsiswift">/vsiswift/ documentation</a>
743 *
744 * @since GDAL 2.3
745 */
746 void VSIInstallSwiftFileHandler( void )
747 {
748 VSIFileManager::InstallHandler( "/vsiswift/", new cpl::VSISwiftFSHandler );
749 }
750
751 #endif /* HAVE_CURL */
752