1 /******************************************************************************
2 *
3 * Project: CPL - Common Portability Library
4 * Purpose: Implement VSI large file api for Microsoft Azure Data Lake Storage Gen2
5 * Author: Even Rouault, even.rouault at spatialys.com
6 *
7 ******************************************************************************
8 * Copyright (c) 2020, Even Rouault <even.rouault at spatialys.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the "Software"),
12 * to deal in the Software without restriction, including without limitation
13 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 * and/or sell copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included
18 * in all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 ****************************************************************************/
28
29 #include "cpl_port.h"
30 #include "cpl_http.h"
31 #include "cpl_json.h"
32 #include "cpl_time.h"
33 #include "cpl_vsil_curl_priv.h"
34 #include "cpl_vsil_curl_class.h"
35
36 #include <errno.h>
37
38 #include <algorithm>
39 #include <set>
40 #include <map>
41 #include <memory>
42
43 #include "cpl_azure.h"
44
45 CPL_CVSID("$Id: cpl_vsil_adls.cpp 52eeac2ffc46e32ea8ff7024c81c44bdbb0b0709 2021-03-06 00:13:31 +0100 Even Rouault $")
46
47 #ifndef HAVE_CURL
48
VSIInstallADLSFileHandler(void)49 void VSIInstallADLSFileHandler( void )
50 {
51 // Not supported
52 }
53
54 #else
55
56 //! @cond Doxygen_Suppress
57 #ifndef DOXYGEN_SKIP
58
59 #define ENABLE_DEBUG 0
60
61 namespace cpl {
62
63 /************************************************************************/
64 /* GetContinuationToken() */
65 /************************************************************************/
66
67 static CPLString GetContinuationToken(const char* pszHeaders)
68 {
69 CPLString osContinuation;
70 if( pszHeaders )
71 {
72 const char* pszContinuation = strstr(pszHeaders, "x-ms-continuation: ");
73 if( pszContinuation )
74 {
75 pszContinuation += strlen("x-ms-continuation: ");
76 const char* pszEOL = strstr(pszContinuation, "\r\n");
77 if( pszEOL )
78 {
79 osContinuation.assign(pszContinuation,
80 pszEOL - pszContinuation);
81 }
82 }
83 }
84 return osContinuation;
85 }
86
87 /************************************************************************/
88 /* RemoveTrailingSlash() */
89 /************************************************************************/
90
91 static CPLString RemoveTrailingSlash(const CPLString& osFilename)
92 {
93 CPLString osWithoutSlash(osFilename);
94 if( !osWithoutSlash.empty() && osWithoutSlash.back() == '/' )
95 osWithoutSlash.resize( osWithoutSlash.size() - 1 );
96 return osWithoutSlash;
97 }
98
99 /************************************************************************/
100 /* VSIDIRADLS */
101 /************************************************************************/
102
103 class VSIADLSFSHandler;
104
105 struct VSIDIRADLS: public VSIDIR
106 {
107 CPLString m_osRootPath{};
108 int m_nRecurseDepth = 0;
109
110 struct Iterator
111 {
112 CPLString m_osNextMarker{};
113 std::vector<std::unique_ptr<VSIDIREntry>> m_aoEntries{};
114 int m_nPos = 0;
115
116 void clear()
117 {
118 m_osNextMarker.clear();
119 m_nPos = 0;
120 m_aoEntries.clear();
121 }
122 };
123
124 Iterator m_oIterWithinFilesystem{};
125 Iterator m_oIterFromRoot{};
126
127 // Backup file system listing when doing a recursive OpenDir() from
128 // the account root
129 bool m_bRecursiveRequestFromAccountRoot = false;
130
131 CPLString m_osFilesystem{};
132 CPLString m_osObjectKey{};
133 VSIADLSFSHandler* m_poFS = nullptr;
134 int m_nMaxFiles = 0;
135 bool m_bCacheEntries = true;
136
137 explicit VSIDIRADLS(VSIADLSFSHandler *poFSIn): m_poFS(poFSIn) {}
138
139 VSIDIRADLS(const VSIDIRADLS&) = delete;
140 VSIDIRADLS& operator=(const VSIDIRADLS&) = delete;
141
142 const VSIDIREntry* NextDirEntry() override;
143
144 bool IssueListDir();
145 bool AnalysePathList( const CPLString& osBaseURL, const char* pszJSON );
146 bool AnalyseFilesystemList( const CPLString& osBaseURL, const char* pszJSON );
147 void clear();
148 };
149
150 /************************************************************************/
151 /* VSIADLSFSHandler */
152 /************************************************************************/
153
154 class VSIADLSFSHandler final : public IVSIS3LikeFSHandler
155 {
156 CPL_DISALLOW_COPY_ASSIGN(VSIADLSFSHandler)
157
158 protected:
159 VSICurlHandle* CreateFileHandle( const char* pszFilename ) override;
160 CPLString GetURLFromFilename( const CPLString& osFilename ) override;
161
162 char** GetFileList( const char *pszFilename,
163 int nMaxFiles,
164 bool* pbGotFileList ) override;
165
166 int CopyObject( const char *oldpath, const char *newpath,
167 CSLConstList papszMetadata ) override;
168 int MkdirInternal( const char *pszDirname, long nMode, bool bDoStatCheck ) override;
169 int RmdirInternal( const char * pszDirname, bool bRecursive );
170
171 void ClearCache() override;
172
173 public:
174 VSIADLSFSHandler() = default;
175 ~VSIADLSFSHandler() override = default;
176
177 CPLString GetFSPrefix() const override { return "/vsiadls/"; }
178 const char* GetDebugKey() const override { return "ADLS"; }
179
180 VSIVirtualHandle *Open( const char *pszFilename,
181 const char *pszAccess,
182 bool bSetError,
183 CSLConstList papszOptions ) override;
184
185 int Rename( const char *oldpath, const char *newpath ) override;
186 int Unlink( const char *pszFilename ) override;
187 int Mkdir( const char *, long ) override;
188 int Rmdir( const char * ) override;
189 int RmdirRecursive( const char *pszDirname ) override;
190 int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
191 int nFlags ) override;
192
193 char** GetFileMetadata( const char * pszFilename, const char* pszDomain,
194 CSLConstList papszOptions ) override;
195
196 bool SetFileMetadata( const char * pszFilename,
197 CSLConstList papszMetadata,
198 const char* pszDomain,
199 CSLConstList papszOptions ) override;
200
201 const char* GetOptions() override;
202
203 char* GetSignedURL( const char* pszFilename, CSLConstList papszOptions ) override;
204
205 char** GetFileList( const char *pszFilename,
206 int nMaxFiles,
207 bool bCacheEntries,
208 bool* pbGotFileList );
209
210 VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
211 const char* const *papszOptions) override;
212
213 enum class Event
214 {
215 CREATE_FILE,
216 APPEND_DATA,
217 FLUSH
218 };
219
220 // Block list upload
221 bool UploadFile(const CPLString& osFilename,
222 Event event,
223 vsi_l_offset nPosition,
224 const void* pabyBuffer,
225 size_t nBufferSize,
226 IVSIS3LikeHandleHelper *poS3HandleHelper,
227 int nMaxRetry,
228 double dfRetryDelay);
229
230 // Multipart upload (mapping of S3 interface)
231 bool SupportsParallelMultipartUpload() const override { return true; }
232
233 CPLString InitiateMultipartUpload(
234 const std::string& osFilename,
235 IVSIS3LikeHandleHelper * poS3HandleHelper,
236 int nMaxRetry,
237 double dfRetryDelay,
238 CSLConstList /* papszOptions */) override {
239 return UploadFile(osFilename, Event::CREATE_FILE, 0, nullptr, 0,
240 poS3HandleHelper, nMaxRetry, dfRetryDelay) ?
241 std::string("dummy") : std::string();
242 }
243
244 CPLString UploadPart(const CPLString& osFilename,
245 int /* nPartNumber */,
246 const std::string& /* osUploadID */,
247 vsi_l_offset nPosition,
248 const void* pabyBuffer,
249 size_t nBufferSize,
250 IVSIS3LikeHandleHelper *poS3HandleHelper,
251 int nMaxRetry,
252 double dfRetryDelay) override
253 {
254 return UploadFile(osFilename, Event::APPEND_DATA,
255 nPosition, pabyBuffer, nBufferSize,
256 poS3HandleHelper, nMaxRetry, dfRetryDelay) ?
257 std::string("dummy") : std::string();
258 }
259
260 bool CompleteMultipart(const CPLString& osFilename,
261 const CPLString& /* osUploadID */,
262 const std::vector<CPLString>& /* aosEtags */,
263 vsi_l_offset nTotalSize,
264 IVSIS3LikeHandleHelper *poS3HandleHelper,
265 int nMaxRetry,
266 double dfRetryDelay) override
267 {
268 return UploadFile(osFilename, Event::FLUSH, nTotalSize, nullptr, 0,
269 poS3HandleHelper, nMaxRetry, dfRetryDelay);
270 }
271
272 bool AbortMultipart(const CPLString& /* osFilename */,
273 const CPLString& /* osUploadID */,
274 IVSIS3LikeHandleHelper * /*poS3HandleHelper */,
275 int /* nMaxRetry */,
276 double /* dfRetryDelay */) override { return true; }
277
278 CPLString GetStreamingPath( const char* pszFilename ) const override;
279
280 IVSIS3LikeHandleHelper* CreateHandleHelper(
281 const char* pszURI, bool bAllowNoObject) override;
282 };
283
284 /************************************************************************/
285 /* clear() */
286 /************************************************************************/
287
288 void VSIDIRADLS::clear()
289 {
290 if( !m_osFilesystem.empty() )
291 m_oIterWithinFilesystem.clear();
292 else
293 m_oIterFromRoot.clear();
294 }
295
296 /************************************************************************/
297 /* GetUnixTimeFromRFC822() */
298 /************************************************************************/
299
300 static GIntBig GetUnixTimeFromRFC822(const char* pszRFC822DateTime)
301 {
302 int nYear, nMonth, nDay, nHour, nMinute, nSecond;
303 if( CPLParseRFC822DateTime(pszRFC822DateTime,
304 &nYear,
305 &nMonth,
306 &nDay,
307 &nHour,
308 &nMinute,
309 &nSecond,
310 nullptr,
311 nullptr ) )
312 {
313 struct tm brokendowntime;
314 brokendowntime.tm_year = nYear - 1900;
315 brokendowntime.tm_mon = nMonth - 1;
316 brokendowntime.tm_mday = nDay;
317 brokendowntime.tm_hour = nHour;
318 brokendowntime.tm_min = nMinute;
319 brokendowntime.tm_sec = nSecond < 0 ? 0 : nSecond;
320 return CPLYMDHMSToUnixTime(&brokendowntime);
321 }
322 return GINTBIG_MIN;
323 }
324
325 /************************************************************************/
326 /* AnalysePathList() */
327 /************************************************************************/
328
329 bool VSIDIRADLS::AnalysePathList(
330 const CPLString& osBaseURL,
331 const char* pszJSON)
332 {
333 #if DEBUG_VERBOSE
334 CPLDebug(m_poFS->GetDebugKey(), "%s", pszJSON);
335 #endif
336
337 CPLJSONDocument oDoc;
338 if( !oDoc.LoadMemory(pszJSON) )
339 return false;
340
341 auto oPaths = oDoc.GetRoot().GetArray("paths");
342 if( !oPaths.IsValid() )
343 {
344 CPLError(CE_Failure, CPLE_AppDefined, "Cannot find paths[]");
345 return false;
346 }
347
348 for( const auto& oPath: oPaths )
349 {
350 m_oIterWithinFilesystem.m_aoEntries.push_back(
351 std::unique_ptr<VSIDIREntry>(new VSIDIREntry()));
352 auto& entry = m_oIterWithinFilesystem.m_aoEntries.back();
353
354 // Returns relative path to the filesystem, so for example "mydir/foo.bin"
355 // for https://{account}.dfs.core.windows.net/{filesystem}/mydir/foo.bin
356 const CPLString osName(oPath.GetString("name"));
357 if( !m_osObjectKey.empty() && STARTS_WITH(osName, (m_osObjectKey + "/").c_str()) )
358 entry->pszName = CPLStrdup(osName.substr(m_osObjectKey.size() + 1).c_str());
359 else if (m_bRecursiveRequestFromAccountRoot && !m_osFilesystem.empty() )
360 entry->pszName = CPLStrdup((m_osFilesystem + '/' + osName).c_str());
361 else
362 entry->pszName = CPLStrdup(osName.c_str());
363 entry->nSize = static_cast<GUIntBig>(oPath.GetLong("contentLength"));
364 entry->bSizeKnown = true;
365 entry->nMode = oPath.GetString("isDirectory") == "true" ? S_IFDIR : S_IFREG;
366 entry->nMode |= VSICurlParseUnixPermissions(oPath.GetString("permissions").c_str());
367 entry->bModeKnown = true;
368
369 CPLString ETag = oPath.GetString("etag");
370 if( !ETag.empty() )
371 {
372 entry->papszExtra = CSLSetNameValue(
373 entry->papszExtra, "ETag", ETag.c_str());
374 }
375
376 const GIntBig nMTime = GetUnixTimeFromRFC822(oPath.GetString("lastModified").c_str());
377 if( nMTime != GINTBIG_MIN )
378 {
379 entry->nMTime = nMTime;
380 entry->bMTimeKnown = true;
381 }
382
383 if( m_bCacheEntries )
384 {
385 FileProp prop;
386 prop.eExists = EXIST_YES;
387 prop.bHasComputedFileSize = true;
388 prop.fileSize = entry->nSize;
389 prop.bIsDirectory = CPL_TO_BOOL(VSI_ISDIR(entry->nMode));
390 prop.nMode = entry->nMode;
391 prop.mTime = static_cast<time_t>(entry->nMTime);
392 prop.ETag = ETag;
393
394 CPLString osCachedFilename =
395 osBaseURL + "/" +
396 CPLAWSURLEncode(osName, false);
397 #if DEBUG_VERBOSE
398 CPLDebug(m_poFS->GetDebugKey(), "Cache %s", osCachedFilename.c_str());
399 #endif
400 m_poFS->SetCachedFileProp(osCachedFilename, prop);
401 }
402
403 if( m_nMaxFiles > 0 && m_oIterWithinFilesystem.m_aoEntries.size() >
404 static_cast<unsigned>(m_nMaxFiles) )
405 {
406 break;
407 }
408 }
409
410 return true;
411 }
412
413 /************************************************************************/
414 /* AnalysePathList() */
415 /************************************************************************/
416
417 bool VSIDIRADLS::AnalyseFilesystemList (
418 const CPLString& osBaseURL,
419 const char* pszJSON)
420 {
421 #if DEBUG_VERBOSE
422 CPLDebug(m_poFS->GetDebugKey(), "%s", pszJSON);
423 #endif
424
425 CPLJSONDocument oDoc;
426 if( !oDoc.LoadMemory(pszJSON) )
427 return false;
428
429 auto oPaths = oDoc.GetRoot().GetArray("filesystems");
430 if( !oPaths.IsValid() )
431 {
432 CPLError(CE_Failure, CPLE_AppDefined, "Cannot find filesystems[]");
433 return false;
434 }
435
436 for( const auto& oPath: oPaths )
437 {
438 m_oIterFromRoot.m_aoEntries.push_back(
439 std::unique_ptr<VSIDIREntry>(new VSIDIREntry()));
440 auto& entry = m_oIterFromRoot.m_aoEntries.back();
441
442 const CPLString osName(oPath.GetString("name"));
443 entry->pszName = CPLStrdup(osName.c_str());
444 entry->nSize = 0;
445 entry->bSizeKnown = true;
446 entry->nMode = S_IFDIR;
447 entry->bModeKnown = true;
448
449 CPLString ETag = oPath.GetString("etag");
450 if( !ETag.empty() )
451 {
452 entry->papszExtra = CSLSetNameValue(
453 entry->papszExtra, "ETag", ETag.c_str());
454 }
455
456 const GIntBig nMTime = GetUnixTimeFromRFC822(oPath.GetString("lastModified").c_str());
457 if( nMTime != GINTBIG_MIN )
458 {
459 entry->nMTime = nMTime;
460 entry->bMTimeKnown = true;
461 }
462
463 if( m_bCacheEntries )
464 {
465 FileProp prop;
466 prop.eExists = EXIST_YES;
467 prop.bHasComputedFileSize = true;
468 prop.fileSize = 0;
469 prop.bIsDirectory = true;
470 prop.mTime = static_cast<time_t>(entry->nMTime);
471 prop.ETag = ETag;
472
473 CPLString osCachedFilename =
474 osBaseURL + CPLAWSURLEncode(osName, false);
475 #if DEBUG_VERBOSE
476 CPLDebug(m_poFS->GetDebugKey(), "Cache %s", osCachedFilename.c_str());
477 #endif
478 m_poFS->SetCachedFileProp(osCachedFilename, prop);
479 }
480
481 if( m_nMaxFiles > 0 && m_oIterFromRoot.m_aoEntries.size() >
482 static_cast<unsigned>(m_nMaxFiles) )
483 {
484 break;
485 }
486
487 }
488
489 return true;
490 }
491
492 /************************************************************************/
493 /* IssueListDir() */
494 /************************************************************************/
495
496 bool VSIDIRADLS::IssueListDir()
497 {
498 WriteFuncStruct sWriteFuncData;
499
500 auto& oIter = !m_osFilesystem.empty() ? m_oIterWithinFilesystem : m_oIterFromRoot;
501 const CPLString l_osNextMarker(oIter.m_osNextMarker);
502 clear();
503
504 NetworkStatisticsFileSystem oContextFS(m_poFS->GetFSPrefix());
505 NetworkStatisticsAction oContextAction("ListBucket");
506
507 CPLString osMaxKeys = CPLGetConfigOption("AZURE_MAX_RESULTS", "");
508 const int AZURE_SERVER_LIMIT_SINGLE_REQUEST = 5000;
509 if( m_nMaxFiles > 0 && m_nMaxFiles < AZURE_SERVER_LIMIT_SINGLE_REQUEST &&
510 (osMaxKeys.empty() || m_nMaxFiles < atoi(osMaxKeys)) )
511 {
512 osMaxKeys.Printf("%d", m_nMaxFiles);
513 }
514
515
516 auto poHandleHelper = std::unique_ptr<IVSIS3LikeHandleHelper>(
517 m_poFS->CreateHandleHelper(m_osFilesystem, true));
518 if( poHandleHelper == nullptr )
519 {
520 return false;
521 }
522
523 const CPLString osBaseURL(poHandleHelper->GetURLNoKVP());
524
525 CURL* hCurlHandle = curl_easy_init();
526
527 if( !l_osNextMarker.empty() )
528 poHandleHelper->AddQueryParameter("continuation", l_osNextMarker);
529 if( !osMaxKeys.empty() )
530 poHandleHelper->AddQueryParameter("maxresults", osMaxKeys);
531 if( !m_osFilesystem.empty() )
532 {
533 poHandleHelper->AddQueryParameter("resource", "filesystem");
534 poHandleHelper->AddQueryParameter("recursive",
535 m_nRecurseDepth == 0 ? "false" : "true");
536 if( !m_osObjectKey.empty() )
537 poHandleHelper->AddQueryParameter("directory", m_osObjectKey);
538 }
539 else
540 {
541 poHandleHelper->AddQueryParameter("resource", "account");
542 }
543
544 struct curl_slist* headers =
545 VSICurlSetOptions(hCurlHandle, poHandleHelper->GetURL(), nullptr);
546 headers = VSICurlMergeHeaders(headers,
547 poHandleHelper->GetCurlHeaders("GET", headers));
548 curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers);
549
550 CurlRequestHelper requestHelper;
551 const long response_code =
552 requestHelper.perform(hCurlHandle, headers, m_poFS, poHandleHelper.get());
553
554 NetworkStatisticsLogger::LogGET(sWriteFuncData.nSize);
555
556 bool ret = false;
557 if( response_code != 200 )
558 {
559 CPLDebug(m_poFS->GetDebugKey(), "%s",
560 requestHelper.sWriteFuncData.pBuffer
561 ? requestHelper.sWriteFuncData.pBuffer : "(null)");
562 }
563 else
564 {
565 if( !m_osFilesystem.empty() )
566 {
567 // https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/list
568 ret = AnalysePathList( osBaseURL, requestHelper.sWriteFuncData.pBuffer );
569 }
570 else
571 {
572 // https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/filesystem/list
573 ret = AnalyseFilesystemList( osBaseURL, requestHelper.sWriteFuncData.pBuffer );
574 }
575
576 // Get continuation token for response headers
577 oIter.m_osNextMarker = GetContinuationToken(requestHelper.sWriteFuncHeaderData.pBuffer);
578 }
579
580 curl_easy_cleanup(hCurlHandle);
581 return ret;
582 }
583
584 /************************************************************************/
585 /* NextDirEntry() */
586 /************************************************************************/
587
588 const VSIDIREntry* VSIDIRADLS::NextDirEntry()
589 {
590 while( true )
591 {
592 auto& oIter = !m_osFilesystem.empty() ? m_oIterWithinFilesystem : m_oIterFromRoot;
593 if( oIter.m_nPos < static_cast<int>(oIter.m_aoEntries.size()) )
594 {
595 auto& entry = oIter.m_aoEntries[oIter.m_nPos];
596 oIter.m_nPos ++;
597 if( m_bRecursiveRequestFromAccountRoot )
598 {
599 // If we just read an entry from the account root, it is a
600 // filesystem name, and we want the next iteration to read
601 // into it.
602 if( m_osFilesystem.empty() )
603 {
604 m_osFilesystem = entry->pszName;
605 if( !IssueListDir() )
606 {
607 return nullptr;
608 }
609 }
610 }
611 return entry.get();
612 }
613 if( oIter.m_osNextMarker.empty() )
614 {
615 if( m_bRecursiveRequestFromAccountRoot )
616 {
617 // If we have no more entries at the filesystem level, go back
618 // to the root level.
619 if( !m_osFilesystem.empty() )
620 {
621 m_osFilesystem.clear();
622 continue;
623 }
624 }
625 return nullptr;
626 }
627 if( !IssueListDir() )
628 {
629 return nullptr;
630 }
631 }
632 }
633
634 /************************************************************************/
635 /* VSIADLSHandle */
636 /************************************************************************/
637
638 class VSIADLSHandle final : public VSICurlHandle
639 {
640 CPL_DISALLOW_COPY_ASSIGN(VSIADLSHandle)
641
642 std::unique_ptr<VSIAzureBlobHandleHelper> m_poHandleHelper{};
643
644 protected:
645 virtual struct curl_slist* GetCurlHeaders( const CPLString& osVerb,
646 const struct curl_slist* psExistingHeaders ) override;
647
648 public:
649 VSIADLSHandle( VSIADLSFSHandler* poFS, const char* pszFilename,
650 VSIAzureBlobHandleHelper* poHandleHelper);
651 };
652
653 /************************************************************************/
654 /* CreateFileHandle() */
655 /************************************************************************/
656
657 VSICurlHandle* VSIADLSFSHandler::CreateFileHandle(const char* pszFilename)
658 {
659 VSIAzureBlobHandleHelper* poHandleHelper =
660 VSIAzureBlobHandleHelper::BuildFromURI( pszFilename + GetFSPrefix().size(),
661 GetFSPrefix() );
662 if( poHandleHelper == nullptr )
663 return nullptr;
664 return new VSIADLSHandle(this, pszFilename, poHandleHelper);
665 }
666
667 /************************************************************************/
668 /* Stat() */
669 /************************************************************************/
670
671 int VSIADLSFSHandler::Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
672 int nFlags )
673 {
674 if( !STARTS_WITH_CI(pszFilename, GetFSPrefix()) )
675 return -1;
676
677 const CPLString osFilenameWithoutSlash(RemoveTrailingSlash(pszFilename));
678
679 // Stat("/vsiadls/") ?
680 if( osFilenameWithoutSlash + "/" == GetFSPrefix() )
681 {
682 // List file systems (stop at the first one), to confirm that the
683 // account is correct
684 bool bGotFileList = false;
685 CSLDestroy(GetFileList(GetFSPrefix(), 1, false, &bGotFileList));
686 if( bGotFileList )
687 {
688 memset(pStatBuf, 0, sizeof(VSIStatBufL));
689 pStatBuf->st_mode = S_IFDIR;
690 return 0;
691 }
692 return -1;
693 }
694
695 // Stat("/vsiadls/filesystem") ?
696 if( osFilenameWithoutSlash.size() > GetFSPrefix().size() &&
697 osFilenameWithoutSlash.substr(GetFSPrefix().size()).find('/') == std::string::npos )
698 {
699 // Use https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/filesystem/getproperties
700
701 NetworkStatisticsFileSystem oContextFS(GetFSPrefix());
702 NetworkStatisticsAction oContextAction("GetProperties");
703
704 const CPLString osFilesystem(
705 osFilenameWithoutSlash.substr(GetFSPrefix().size()));
706 auto poHandleHelper =
707 std::unique_ptr<IVSIS3LikeHandleHelper>(CreateHandleHelper(osFilesystem, true));
708 if( poHandleHelper == nullptr )
709 {
710 return -1;
711 }
712
713 CURL* hCurlHandle = curl_easy_init();
714
715 poHandleHelper->AddQueryParameter("resource", "filesystem");
716
717 struct curl_slist* headers =
718 VSICurlSetOptions(hCurlHandle, poHandleHelper->GetURL(), nullptr);
719
720 headers = VSICurlMergeHeaders(headers,
721 poHandleHelper->GetCurlHeaders("HEAD", headers));
722 curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers);
723
724 curl_easy_setopt(hCurlHandle, CURLOPT_NOBODY, 1);
725
726 CurlRequestHelper requestHelper;
727 const long response_code =
728 requestHelper.perform(hCurlHandle, headers, this, poHandleHelper.get());
729
730 NetworkStatisticsLogger::LogHEAD();
731
732 if( response_code != 200 || requestHelper.sWriteFuncHeaderData.pBuffer == nullptr )
733 {
734 curl_easy_cleanup(hCurlHandle);
735 return -1;
736 }
737
738 memset(pStatBuf, 0, sizeof(VSIStatBufL));
739 pStatBuf->st_mode = S_IFDIR;
740
741 const char* pszLastModified =
742 strstr(requestHelper.sWriteFuncHeaderData.pBuffer, "Last-Modified: ");
743 if( pszLastModified )
744 {
745 pszLastModified += strlen("Last-Modified: ");
746 const char* pszEOL = strstr(pszLastModified, "\r\n");
747 if( pszEOL )
748 {
749 CPLString osLastModified;
750 osLastModified.assign(pszLastModified,
751 pszEOL - pszLastModified);
752
753 const GIntBig nMTime = GetUnixTimeFromRFC822(osLastModified.c_str());
754 if( nMTime != GINTBIG_MIN )
755 {
756 pStatBuf->st_mtime = static_cast<time_t>(nMTime);
757 }
758 }
759 }
760
761 curl_easy_cleanup(hCurlHandle);
762
763 return 0;
764 }
765
766 return VSICurlFilesystemHandler::Stat(osFilenameWithoutSlash, pStatBuf, nFlags);
767 }
768
769 /************************************************************************/
770 /* GetFileMetadata() */
771 /************************************************************************/
772
773 char** VSIADLSFSHandler::GetFileMetadata( const char* pszFilename,
774 const char* pszDomain,
775 CSLConstList papszOptions )
776 {
777 if( !STARTS_WITH_CI(pszFilename, GetFSPrefix()) )
778 return nullptr;
779
780 if( pszDomain == nullptr || (!EQUAL(pszDomain, "STATUS") && !EQUAL(pszDomain, "ACL")) )
781 {
782 return VSICurlFilesystemHandler::GetFileMetadata(
783 pszFilename, pszDomain, papszOptions);
784 }
785
786 auto poHandleHelper = std::unique_ptr<IVSIS3LikeHandleHelper>(
787 CreateHandleHelper(pszFilename + GetFSPrefix().size(), false));
788 if( poHandleHelper == nullptr )
789 {
790 return nullptr;
791 }
792
793 NetworkStatisticsFileSystem oContextFS(GetFSPrefix());
794 NetworkStatisticsAction oContextAction("GetFileMetadata");
795
796 bool bRetry;
797 // coverity[tainted_data]
798 double dfRetryDelay = CPLAtof(CPLGetConfigOption("GDAL_HTTP_RETRY_DELAY",
799 CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY)));
800 const int nMaxRetry = atoi(CPLGetConfigOption("GDAL_HTTP_MAX_RETRY",
801 CPLSPrintf("%d",CPL_HTTP_MAX_RETRY)));
802 int nRetryCount = 0;
803 bool bError = true;
804
805 CPLStringList aosMetadata;
806 do
807 {
808 bRetry = false;
809 CURL* hCurlHandle = curl_easy_init();
810 poHandleHelper->AddQueryParameter("action",
811 EQUAL(pszDomain, "STATUS") ? "getStatus" : "getAccessControl");
812
813 struct curl_slist* headers =
814 VSICurlSetOptions(hCurlHandle, poHandleHelper->GetURL(), nullptr);
815
816 headers = VSICurlMergeHeaders(headers,
817 poHandleHelper->GetCurlHeaders("HEAD", headers));
818 curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers);
819
820 curl_easy_setopt(hCurlHandle, CURLOPT_NOBODY, 1);
821
822 CurlRequestHelper requestHelper;
823 const long response_code =
824 requestHelper.perform(hCurlHandle, headers, this, poHandleHelper.get());
825
826 NetworkStatisticsLogger::LogHEAD();
827
828 if( response_code != 200 || requestHelper.sWriteFuncHeaderData.pBuffer == nullptr )
829 {
830 // Look if we should attempt a retry
831 const double dfNewRetryDelay = CPLHTTPGetNewRetryDelay(
832 static_cast<int>(response_code), dfRetryDelay,
833 requestHelper.sWriteFuncHeaderData.pBuffer, requestHelper.szCurlErrBuf);
834 if( dfNewRetryDelay > 0 &&
835 nRetryCount < nMaxRetry )
836 {
837 CPLError(CE_Warning, CPLE_AppDefined,
838 "HTTP error code: %d - %s. "
839 "Retrying again in %.1f secs",
840 static_cast<int>(response_code),
841 poHandleHelper->GetURL().c_str(),
842 dfRetryDelay);
843 CPLSleep(dfRetryDelay);
844 dfRetryDelay = dfNewRetryDelay;
845 nRetryCount++;
846 bRetry = true;
847 }
848 else
849 {
850 CPLDebug(GetDebugKey(), "GetFileMetadata failed on %s: %s",
851 pszFilename,
852 requestHelper.sWriteFuncData.pBuffer
853 ? requestHelper.sWriteFuncData.pBuffer
854 : "(null)");
855 }
856 }
857 else
858 {
859 char** papszHeaders = CSLTokenizeString2(requestHelper.sWriteFuncHeaderData.pBuffer, "\r\n", 0);
860 for( int i = 0; papszHeaders[i]; ++i )
861 {
862 char* pszKey = nullptr;
863 const char* pszValue = CPLParseNameValue(papszHeaders[i], &pszKey);
864 if( pszKey && pszValue && !EQUAL(pszKey, "Server") && !EQUAL(pszKey, "Date") )
865 {
866 aosMetadata.SetNameValue(pszKey, pszValue);
867 }
868 CPLFree(pszKey);
869 }
870 CSLDestroy(papszHeaders);
871 bError = false;
872 }
873
874 curl_easy_cleanup(hCurlHandle);
875 }
876 while( bRetry );
877 return bError ? nullptr : CSLDuplicate(aosMetadata.List());
878 }
879
880 /************************************************************************/
881 /* SetFileMetadata() */
882 /************************************************************************/
883
884 bool VSIADLSFSHandler::SetFileMetadata( const char * pszFilename,
885 CSLConstList papszMetadata,
886 const char* pszDomain,
887 CSLConstList papszOptions )
888 {
889 if( !STARTS_WITH_CI(pszFilename, GetFSPrefix()) )
890 return false;
891
892 if( pszDomain == nullptr ||
893 !(EQUAL(pszDomain, "PROPERTIES") || EQUAL(pszDomain, "ACL")) )
894 {
895 CPLError(CE_Failure, CPLE_NotSupported,
896 "Only PROPERTIES and ACL domain are supported");
897 return false;
898 }
899
900 auto poHandleHelper = std::unique_ptr<IVSIS3LikeHandleHelper>(
901 CreateHandleHelper(pszFilename + GetFSPrefix().size(), false));
902 if( poHandleHelper == nullptr )
903 {
904 return false;
905 }
906
907 const bool bRecursive = CPLTestBool(
908 CSLFetchNameValueDef(papszOptions, "RECURSIVE", "FALSE"));
909 const char* pszMode = CSLFetchNameValue(papszOptions, "MODE");
910 if( !EQUAL(pszDomain, "PROPERTIES") && bRecursive && pszMode == nullptr )
911 {
912 CPLError(CE_Failure, CPLE_AppDefined,
913 "For setAccessControlRecursive, the MODE option should be set "
914 "to: 'set', 'modify' or 'remove'");
915 return false;
916 }
917
918 NetworkStatisticsFileSystem oContextFS(GetFSPrefix());
919 NetworkStatisticsAction oContextAction("SetFileMetadata");
920
921 bool bRetry;
922 // coverity[tainted_data]
923 double dfRetryDelay = CPLAtof(CPLGetConfigOption("GDAL_HTTP_RETRY_DELAY",
924 CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY)));
925 const int nMaxRetry = atoi(CPLGetConfigOption("GDAL_HTTP_MAX_RETRY",
926 CPLSPrintf("%d",CPL_HTTP_MAX_RETRY)));
927 int nRetryCount = 0;
928
929 bool bRet = false;
930
931 do
932 {
933 bRetry = false;
934 CURL* hCurlHandle = curl_easy_init();
935 poHandleHelper->AddQueryParameter("action",
936 EQUAL(pszDomain, "PROPERTIES") ? "setProperties" :
937 bRecursive ? "setAccessControlRecursive" : "setAccessControl");
938 if( pszMode )
939 {
940 poHandleHelper->AddQueryParameter("mode", CPLString(pszMode).tolower());
941 }
942 curl_easy_setopt(hCurlHandle, CURLOPT_CUSTOMREQUEST, "PATCH");
943
944 struct curl_slist* headers = static_cast<struct curl_slist*>(
945 CPLHTTPSetOptions(hCurlHandle,
946 poHandleHelper->GetURL().c_str(),
947 nullptr));
948
949 CPLStringList aosList;
950 for( CSLConstList papszIter = papszMetadata; papszIter && *papszIter; ++papszIter )
951 {
952 char* pszKey = nullptr;
953 const char* pszValue = CPLParseNameValue(*papszIter, &pszKey);
954 if( pszKey && pszValue )
955 {
956 if( (EQUAL(pszDomain, "PROPERTIES") &&
957 (EQUAL(pszKey, "x-ms-lease-id") ||
958 EQUAL(pszKey, "x-ms-cache-control") ||
959 EQUAL(pszKey, "x-ms-content-type") ||
960 EQUAL(pszKey, "x-ms-content-disposition") ||
961 EQUAL(pszKey, "x-ms-content-encoding") ||
962 EQUAL(pszKey, "x-ms-content-language") ||
963 EQUAL(pszKey, "x-ms-content-md5") ||
964 EQUAL(pszKey, "x-ms-properties") ||
965 EQUAL(pszKey, "x-ms-client-request-id") ||
966 STARTS_WITH_CI(pszKey, "If-"))) ||
967 (!EQUAL(pszDomain, "PROPERTIES") && !bRecursive &&
968 (EQUAL(pszKey, "x-ms-lease-id") ||
969 EQUAL(pszKey, "x-ms-owner") ||
970 EQUAL(pszKey, "x-ms-group") ||
971 EQUAL(pszKey, "x-ms-permissions") ||
972 EQUAL(pszKey, "x-ms-acl") ||
973 EQUAL(pszKey, "x-ms-client-request-id") ||
974 STARTS_WITH_CI(pszKey, "If-"))) ||
975 (!EQUAL(pszDomain, "PROPERTIES") && bRecursive &&
976 (EQUAL(pszKey, "x-ms-lease-id") ||
977 EQUAL(pszKey, "x-ms-acl") ||
978 EQUAL(pszKey, "x-ms-client-request-id") ||
979 STARTS_WITH_CI(pszKey, "If-"))) )
980 {
981 char* pszHeader = CPLStrdup(CPLSPrintf("%s: %s", pszKey, pszValue));
982 aosList.AddStringDirectly(pszHeader);
983 headers = curl_slist_append(headers, pszHeader);
984 }
985 else
986 {
987 CPLDebug(GetDebugKey(), "Ignorizing metadata item %s", *papszIter);
988 }
989 }
990 CPLFree(pszKey);
991 }
992
993 headers = VSICurlMergeHeaders(headers,
994 poHandleHelper->GetCurlHeaders("PATCH", headers));
995 curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers);
996
997 NetworkStatisticsLogger::LogPUT(0);
998
999 CurlRequestHelper requestHelper;
1000 const long response_code =
1001 requestHelper.perform(hCurlHandle, headers, this, poHandleHelper.get());
1002
1003 if( response_code != 200 && response_code != 202 )
1004 {
1005 // Look if we should attempt a retry
1006 const double dfNewRetryDelay = CPLHTTPGetNewRetryDelay(
1007 static_cast<int>(response_code), dfRetryDelay,
1008 requestHelper.sWriteFuncHeaderData.pBuffer, requestHelper.szCurlErrBuf);
1009 if( dfNewRetryDelay > 0 &&
1010 nRetryCount < nMaxRetry )
1011 {
1012 CPLError(CE_Warning, CPLE_AppDefined,
1013 "HTTP error code: %d - %s. "
1014 "Retrying again in %.1f secs",
1015 static_cast<int>(response_code),
1016 poHandleHelper->GetURL().c_str(),
1017 dfRetryDelay);
1018 CPLSleep(dfRetryDelay);
1019 dfRetryDelay = dfNewRetryDelay;
1020 nRetryCount++;
1021 bRetry = true;
1022 }
1023 else
1024 {
1025 CPLDebug(GetDebugKey(), "SetFileMetadata on %s failed: %s",
1026 pszFilename,
1027 requestHelper.sWriteFuncData.pBuffer
1028 ? requestHelper.sWriteFuncData.pBuffer
1029 : "(null)");
1030 }
1031 }
1032 else
1033 {
1034 bRet = true;
1035 }
1036
1037 curl_easy_cleanup(hCurlHandle);
1038 }
1039 while( bRetry );
1040 return bRet;
1041 }
1042
1043 /************************************************************************/
1044 /* VSIADLSWriteHandle */
1045 /************************************************************************/
1046
1047 class VSIADLSWriteHandle final : public VSIAppendWriteHandle
1048 {
1049 CPL_DISALLOW_COPY_ASSIGN(VSIADLSWriteHandle)
1050
1051 std::unique_ptr<VSIAzureBlobHandleHelper> m_poHandleHelper{};
1052 bool m_bCreated = false;
1053
1054 bool Send(bool bIsLastBlock) override;
1055
1056 bool SendInternal(VSIADLSFSHandler::Event event);
1057
1058 void InvalidateParentDirectory();
1059
1060 public:
1061 VSIADLSWriteHandle( VSIADLSFSHandler* poFS,
1062 const char* pszFilename,
1063 VSIAzureBlobHandleHelper* poHandleHelper );
1064 virtual ~VSIADLSWriteHandle();
1065
1066 bool CreateFile();
1067 };
1068
1069 /************************************************************************/
1070 /* VSIADLSWriteHandle() */
1071 /************************************************************************/
1072
1073 VSIADLSWriteHandle::VSIADLSWriteHandle( VSIADLSFSHandler* poFS,
1074 const char* pszFilename,
1075 VSIAzureBlobHandleHelper* poHandleHelper) :
1076 VSIAppendWriteHandle(poFS, poFS->GetFSPrefix(), pszFilename, GetAzureBufferSize()),
1077 m_poHandleHelper(poHandleHelper)
1078 {
1079 }
1080
1081 /************************************************************************/
1082 /* ~VSIADLSWriteHandle() */
1083 /************************************************************************/
1084
1085 VSIADLSWriteHandle::~VSIADLSWriteHandle()
1086 {
1087 Close();
1088 }
1089
1090 /************************************************************************/
1091 /* InvalidateParentDirectory() */
1092 /************************************************************************/
1093
1094 void VSIADLSWriteHandle::InvalidateParentDirectory()
1095 {
1096 m_poFS->InvalidateCachedData(
1097 m_poHandleHelper->GetURLNoKVP().c_str() );
1098
1099 const CPLString osFilenameWithoutSlash(RemoveTrailingSlash(m_osFilename));
1100 m_poFS->InvalidateDirContent( CPLGetDirname(osFilenameWithoutSlash) );
1101 }
1102
1103 /************************************************************************/
1104 /* CreateFile() */
1105 /************************************************************************/
1106
1107 bool VSIADLSWriteHandle::CreateFile()
1108 {
1109 m_bCreated = SendInternal(VSIADLSFSHandler::Event::CREATE_FILE);
1110 return m_bCreated;
1111 }
1112
1113 /************************************************************************/
1114 /* Send() */
1115 /************************************************************************/
1116
1117 bool VSIADLSWriteHandle::Send(bool bIsLastBlock)
1118 {
1119 if( !m_bCreated )
1120 return false;
1121 // If we have a non-empty buffer, append it
1122 if( m_nBufferOff != 0 && !SendInternal(VSIADLSFSHandler::Event::APPEND_DATA) )
1123 return false;
1124 // If we are the last block, send the flush event
1125 if( bIsLastBlock && !SendInternal(VSIADLSFSHandler::Event::FLUSH) )
1126 return false;
1127 return true;
1128 }
1129
1130 /************************************************************************/
1131 /* SendInternal() */
1132 /************************************************************************/
1133
1134 bool VSIADLSWriteHandle::SendInternal(VSIADLSFSHandler::Event event)
1135 {
1136 // coverity[tainted_data]
1137 const int nMaxRetry = atoi(CPLGetConfigOption("GDAL_HTTP_MAX_RETRY",
1138 CPLSPrintf("%d",CPL_HTTP_MAX_RETRY)));
1139 // coverity[tainted_data]
1140 double dfRetryDelay = CPLAtof(CPLGetConfigOption("GDAL_HTTP_RETRY_DELAY",
1141 CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY)));
1142
1143 return cpl::down_cast<VSIADLSFSHandler*>(m_poFS)->UploadFile(
1144 m_osFilename, event,
1145 event == VSIADLSFSHandler::Event::CREATE_FILE ? 0 :
1146 event == VSIADLSFSHandler::Event::APPEND_DATA ? m_nCurOffset - m_nBufferOff :
1147 m_nCurOffset,
1148 m_pabyBuffer, m_nBufferOff, m_poHandleHelper.get(),
1149 nMaxRetry, dfRetryDelay);
1150 }
1151
1152 /************************************************************************/
1153 /* ClearCache() */
1154 /************************************************************************/
1155
1156 void VSIADLSFSHandler::ClearCache()
1157 {
1158 IVSIS3LikeFSHandler::ClearCache();
1159
1160 VSIAzureBlobHandleHelper::ClearCache();
1161 }
1162
1163 /************************************************************************/
1164 /* Open() */
1165 /************************************************************************/
1166
1167 VSIVirtualHandle* VSIADLSFSHandler::Open( const char *pszFilename,
1168 const char *pszAccess,
1169 bool bSetError,
1170 CSLConstList papszOptions )
1171 {
1172 if( !STARTS_WITH_CI(pszFilename, GetFSPrefix()) )
1173 return nullptr;
1174
1175 if( strchr(pszAccess, 'w') != nullptr || strchr(pszAccess, 'a') != nullptr )
1176 {
1177 if( strchr(pszAccess, '+') != nullptr &&
1178 !CPLTestBool(CPLGetConfigOption("CPL_VSIL_USE_TEMP_FILE_FOR_RANDOM_WRITE", "NO")) )
1179 {
1180 CPLError(CE_Failure, CPLE_AppDefined,
1181 "w+ not supported for /vsiadls, unless "
1182 "CPL_VSIL_USE_TEMP_FILE_FOR_RANDOM_WRITE is set to YES");
1183 errno = EACCES;
1184 return nullptr;
1185 }
1186
1187 VSIAzureBlobHandleHelper* poHandleHelper =
1188 VSIAzureBlobHandleHelper::BuildFromURI(pszFilename + GetFSPrefix().size(),
1189 GetFSPrefix().c_str());
1190 if( poHandleHelper == nullptr )
1191 return nullptr;
1192 auto poHandle = std::unique_ptr<VSIADLSWriteHandle>(
1193 new VSIADLSWriteHandle(this, pszFilename, poHandleHelper));
1194 if( !poHandle->CreateFile() )
1195 {
1196 return nullptr;
1197 }
1198 if( strchr(pszAccess, '+') != nullptr)
1199 {
1200 return VSICreateUploadOnCloseFile(poHandle.release());
1201 }
1202 return poHandle.release();
1203 }
1204
1205 return
1206 VSICurlFilesystemHandler::Open(pszFilename, pszAccess, bSetError, papszOptions);
1207 }
1208
1209 /************************************************************************/
1210 /* GetURLFromFilename() */
1211 /************************************************************************/
1212
1213 CPLString VSIADLSFSHandler::GetURLFromFilename( const CPLString& osFilename )
1214 {
1215 CPLString osFilenameWithoutPrefix = osFilename.substr(GetFSPrefix().size());
1216 VSIAzureBlobHandleHelper* poHandleHelper =
1217 VSIAzureBlobHandleHelper::BuildFromURI( osFilenameWithoutPrefix, GetFSPrefix() );
1218 if( poHandleHelper == nullptr )
1219 return CPLString();
1220 CPLString osURL( poHandleHelper->GetURLNoKVP() );
1221 delete poHandleHelper;
1222 return osURL;
1223 }
1224
1225 /************************************************************************/
1226 /* CreateHandleHelper() */
1227 /************************************************************************/
1228
1229 IVSIS3LikeHandleHelper* VSIADLSFSHandler::CreateHandleHelper(const char* pszURI,
1230 bool)
1231 {
1232 return VSIAzureBlobHandleHelper::BuildFromURI(pszURI, GetFSPrefix().c_str());
1233 }
1234
1235 /************************************************************************/
1236 /* Rename() */
1237 /************************************************************************/
1238
1239 int VSIADLSFSHandler::Rename( const char *oldpath, const char *newpath )
1240 {
1241 if( !STARTS_WITH_CI(oldpath, GetFSPrefix()) )
1242 return -1;
1243 if( !STARTS_WITH_CI(newpath, GetFSPrefix()) )
1244 return -1;
1245
1246 NetworkStatisticsFileSystem oContextFS(GetFSPrefix());
1247 NetworkStatisticsAction oContextAction("Rename");
1248
1249 VSIStatBufL sStat;
1250 if( VSIStatL(oldpath, &sStat) != 0 )
1251 {
1252 CPLDebug(GetDebugKey(), "%s is not a object", oldpath);
1253 errno = ENOENT;
1254 return -1;
1255 }
1256
1257 // POSIX says renaming on the same file is OK
1258 if( strcmp(oldpath, newpath) == 0 )
1259 return 0;
1260
1261 auto poHandleHelper =
1262 std::unique_ptr<IVSIS3LikeHandleHelper>(
1263 CreateHandleHelper(newpath + GetFSPrefix().size(), false));
1264 if( poHandleHelper == nullptr )
1265 {
1266 return -1;
1267 }
1268
1269
1270 CPLString osContinuation;
1271 int nRet = 0;
1272 bool bRetry;
1273
1274 const int nMaxRetry = atoi(CPLGetConfigOption("GDAL_HTTP_MAX_RETRY",
1275 CPLSPrintf("%d",CPL_HTTP_MAX_RETRY)));
1276 // coverity[tainted_data]
1277 double dfRetryDelay = CPLAtof(CPLGetConfigOption("GDAL_HTTP_RETRY_DELAY",
1278 CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY)));
1279 int nRetryCount = 0;
1280
1281 InvalidateCachedData( GetURLFromFilename(oldpath) );
1282 InvalidateCachedData( GetURLFromFilename(newpath) );
1283 InvalidateDirContent( CPLGetDirname(oldpath) );
1284
1285 do
1286 {
1287 bRetry = false;
1288
1289 CURL* hCurlHandle = curl_easy_init();
1290 curl_easy_setopt(hCurlHandle, CURLOPT_CUSTOMREQUEST, "PUT");
1291
1292 poHandleHelper->ResetQueryParameters();
1293 if( !osContinuation.empty() )
1294 poHandleHelper->AddQueryParameter("continuation", osContinuation);
1295
1296 struct curl_slist* headers = static_cast<struct curl_slist*>(
1297 CPLHTTPSetOptions(hCurlHandle,
1298 poHandleHelper->GetURL().c_str(),
1299 nullptr));
1300 headers = curl_slist_append(headers, "Content-Length: 0");
1301 CPLString osRenameSource("x-ms-rename-source: /");
1302 osRenameSource += CPLAWSURLEncode(oldpath + GetFSPrefix().size(), false);
1303 headers = curl_slist_append(headers, osRenameSource.c_str());
1304 headers = VSICurlMergeHeaders(headers,
1305 poHandleHelper->GetCurlHeaders("PUT", headers));
1306 curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers);
1307
1308 CurlRequestHelper requestHelper;
1309 const long response_code =
1310 requestHelper.perform(hCurlHandle, headers, this, poHandleHelper.get());
1311
1312 NetworkStatisticsLogger::LogPUT(0);
1313
1314 if( response_code != 201)
1315 {
1316 // Look if we should attempt a retry
1317 const double dfNewRetryDelay = CPLHTTPGetNewRetryDelay(
1318 static_cast<int>(response_code), dfRetryDelay,
1319 requestHelper.sWriteFuncHeaderData.pBuffer,
1320 requestHelper.szCurlErrBuf);
1321 if( dfNewRetryDelay > 0 &&
1322 nRetryCount < nMaxRetry )
1323 {
1324 CPLError(CE_Warning, CPLE_AppDefined,
1325 "HTTP error code: %d - %s. "
1326 "Retrying again in %.1f secs",
1327 static_cast<int>(response_code),
1328 poHandleHelper->GetURL().c_str(),
1329 dfRetryDelay);
1330 CPLSleep(dfRetryDelay);
1331 dfRetryDelay = dfNewRetryDelay;
1332 nRetryCount++;
1333 bRetry = true;
1334 }
1335 else
1336 {
1337 CPLDebug(GetDebugKey(), "Renaming of %s failed: %s",
1338 oldpath,
1339 requestHelper.sWriteFuncData.pBuffer
1340 ? requestHelper.sWriteFuncData.pBuffer
1341 : "(null)");
1342 nRet = -1;
1343 }
1344 }
1345 else
1346 {
1347 // Get continuation token for response headers
1348 osContinuation = GetContinuationToken(requestHelper.sWriteFuncHeaderData.pBuffer);
1349 if( !osContinuation.empty() )
1350 {
1351 nRetryCount = 0;
1352 bRetry = true;
1353 }
1354 }
1355
1356 curl_easy_cleanup(hCurlHandle);
1357 }
1358 while( bRetry );
1359
1360 return nRet;
1361 }
1362
1363 /************************************************************************/
1364 /* Unlink() */
1365 /************************************************************************/
1366
1367 int VSIADLSFSHandler::Unlink( const char *pszFilename )
1368 {
1369 return IVSIS3LikeFSHandler::Unlink(pszFilename);
1370 }
1371
1372 /************************************************************************/
1373 /* Mkdir() */
1374 /************************************************************************/
1375
1376 int VSIADLSFSHandler::MkdirInternal( const char *pszDirname, long nMode, bool bDoStatCheck )
1377 {
1378 if( !STARTS_WITH_CI(pszDirname, GetFSPrefix()) )
1379 return -1;
1380
1381 NetworkStatisticsFileSystem oContextFS(GetFSPrefix());
1382 NetworkStatisticsAction oContextAction("Mkdir");
1383
1384 const CPLString osDirname(pszDirname);
1385
1386 if( bDoStatCheck )
1387 {
1388 VSIStatBufL sStat;
1389 if( VSIStatL(osDirname, &sStat) == 0 )
1390 {
1391 CPLDebug(GetDebugKey(), "Directory or file %s already exists", osDirname.c_str());
1392 errno = EEXIST;
1393 return -1;
1394 }
1395 }
1396
1397 const CPLString osDirnameWithoutEndSlash(RemoveTrailingSlash(osDirname));
1398 auto poHandleHelper =
1399 std::unique_ptr<IVSIS3LikeHandleHelper>(
1400 CreateHandleHelper(osDirnameWithoutEndSlash.c_str() + GetFSPrefix().size(), false));
1401 if( poHandleHelper == nullptr )
1402 {
1403 return -1;
1404 }
1405
1406 InvalidateCachedData( GetURLFromFilename(osDirname) );
1407 InvalidateCachedData( GetURLFromFilename(osDirnameWithoutEndSlash) );
1408 InvalidateDirContent( CPLGetDirname(osDirnameWithoutEndSlash) );
1409
1410 int nRet = 0;
1411
1412 bool bRetry;
1413
1414 const int nMaxRetry = atoi(CPLGetConfigOption("GDAL_HTTP_MAX_RETRY",
1415 CPLSPrintf("%d",CPL_HTTP_MAX_RETRY)));
1416 // coverity[tainted_data]
1417 double dfRetryDelay = CPLAtof(CPLGetConfigOption("GDAL_HTTP_RETRY_DELAY",
1418 CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY)));
1419 int nRetryCount = 0;
1420
1421 do
1422 {
1423 bRetry = false;
1424 CURL* hCurlHandle = curl_easy_init();
1425 curl_easy_setopt(hCurlHandle, CURLOPT_CUSTOMREQUEST, "PUT");
1426
1427 poHandleHelper->ResetQueryParameters();
1428 poHandleHelper->AddQueryParameter("resource",
1429 osDirnameWithoutEndSlash.find('/', GetFSPrefix().size())
1430 == std::string::npos ? "filesystem" : "directory");
1431
1432 struct curl_slist* headers = static_cast<struct curl_slist*>(
1433 CPLHTTPSetOptions(hCurlHandle,
1434 poHandleHelper->GetURL().c_str(),
1435 nullptr));
1436 headers = curl_slist_append(headers, "Content-Length: 0");
1437 CPLString osPermissions; // keep in this scope
1438 if( (nMode & 0777) != 0 )
1439 {
1440 osPermissions.Printf("x-ms-permissions: 0%03o", static_cast<int>(nMode));
1441 headers = curl_slist_append(headers, osPermissions.c_str());
1442 }
1443 if( bDoStatCheck )
1444 {
1445 headers = curl_slist_append(headers, "If-None-Match: \"*\"");
1446 }
1447
1448 headers = VSICurlMergeHeaders(headers,
1449 poHandleHelper->GetCurlHeaders("PUT", headers));
1450 curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers);
1451
1452 CurlRequestHelper requestHelper;
1453 const long response_code =
1454 requestHelper.perform(hCurlHandle, headers, this, poHandleHelper.get());
1455
1456 NetworkStatisticsLogger::LogPUT(0);
1457
1458 if( response_code != 201 )
1459 {
1460 // Look if we should attempt a retry
1461 const double dfNewRetryDelay = CPLHTTPGetNewRetryDelay(
1462 static_cast<int>(response_code), dfRetryDelay,
1463 requestHelper.sWriteFuncHeaderData.pBuffer,
1464 requestHelper.szCurlErrBuf);
1465 if( dfNewRetryDelay > 0 &&
1466 nRetryCount < nMaxRetry )
1467 {
1468 CPLError(CE_Warning, CPLE_AppDefined,
1469 "HTTP error code: %d - %s. "
1470 "Retrying again in %.1f secs",
1471 static_cast<int>(response_code),
1472 poHandleHelper->GetURL().c_str(),
1473 dfRetryDelay);
1474 CPLSleep(dfRetryDelay);
1475 dfRetryDelay = dfNewRetryDelay;
1476 nRetryCount++;
1477 bRetry = true;
1478 }
1479 else
1480 {
1481 CPLDebug(GetDebugKey(), "Creation of %s failed: %s",
1482 osDirname.c_str(),
1483 requestHelper.sWriteFuncData.pBuffer
1484 ? requestHelper.sWriteFuncData.pBuffer
1485 : "(null)");
1486 nRet = -1;
1487 }
1488 }
1489
1490 curl_easy_cleanup(hCurlHandle);
1491 }
1492 while( bRetry );
1493
1494 return nRet;
1495 }
1496
1497 int VSIADLSFSHandler::Mkdir( const char * pszDirname, long nMode )
1498 {
1499 return MkdirInternal(pszDirname, nMode, true);
1500 }
1501
1502 /************************************************************************/
1503 /* RmdirInternal() */
1504 /************************************************************************/
1505
1506 int VSIADLSFSHandler::RmdirInternal( const char * pszDirname, bool bRecursive )
1507 {
1508 const CPLString osDirname(pszDirname);
1509 const CPLString osDirnameWithoutEndSlash(RemoveTrailingSlash(osDirname));
1510
1511 const bool bIsFileSystem =
1512 osDirnameWithoutEndSlash.find('/', GetFSPrefix().size()) == std::string::npos;
1513
1514 if( !bRecursive && bIsFileSystem )
1515 {
1516 // List content, to confirm it is empty first, as filesystem deletion
1517 // is recursive by default.
1518 bool bGotFileList = false;
1519 CSLDestroy(GetFileList(osDirnameWithoutEndSlash, 1, false, &bGotFileList));
1520 if( bGotFileList )
1521 {
1522 CPLDebug(GetDebugKey(),
1523 "Cannot delete filesystem with non-recursive method as it is not empty");
1524 errno = ENOTEMPTY;
1525 return -1;
1526 }
1527 }
1528
1529 if( !bIsFileSystem )
1530 {
1531 VSIStatBufL sStat;
1532 if( VSIStatL(osDirname, &sStat) != 0 )
1533 {
1534 CPLDebug(GetDebugKey(), "Object %s does not exist", osDirname.c_str());
1535 errno = ENOENT;
1536 return -1;
1537 }
1538 if( !VSI_ISDIR(sStat.st_mode) )
1539 {
1540 CPLDebug(GetDebugKey(), "Object %s is not a directory", osDirname.c_str());
1541 errno = ENOTDIR;
1542 return -1;
1543 }
1544 }
1545
1546 auto poHandleHelper =
1547 std::unique_ptr<IVSIS3LikeHandleHelper>(
1548 CreateHandleHelper(osDirnameWithoutEndSlash.c_str() + GetFSPrefix().size(), false));
1549 if( poHandleHelper == nullptr )
1550 {
1551 return -1;
1552 }
1553
1554 InvalidateCachedData( GetURLFromFilename(osDirname) );
1555 InvalidateCachedData( GetURLFromFilename(osDirnameWithoutEndSlash) );
1556 InvalidateDirContent( CPLGetDirname(osDirnameWithoutEndSlash) );
1557 if( bRecursive )
1558 {
1559 PartialClearCache(osDirnameWithoutEndSlash);
1560 }
1561
1562 CPLString osContinuation;
1563 int nRet = 0;
1564 bool bRetry;
1565
1566 const int nMaxRetry = atoi(CPLGetConfigOption("GDAL_HTTP_MAX_RETRY",
1567 CPLSPrintf("%d",CPL_HTTP_MAX_RETRY)));
1568 // coverity[tainted_data]
1569 double dfRetryDelay = CPLAtof(CPLGetConfigOption("GDAL_HTTP_RETRY_DELAY",
1570 CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY)));
1571 int nRetryCount = 0;
1572 do
1573 {
1574 bRetry = false;
1575 CURL* hCurlHandle = curl_easy_init();
1576 curl_easy_setopt(hCurlHandle, CURLOPT_CUSTOMREQUEST, "DELETE");
1577
1578 poHandleHelper->ResetQueryParameters();
1579 if( bIsFileSystem )
1580 {
1581 poHandleHelper->AddQueryParameter("resource", "filesystem");
1582 }
1583 else
1584 {
1585 poHandleHelper->AddQueryParameter("recursive", bRecursive ? "true" : "false");
1586 if( !osContinuation.empty() )
1587 poHandleHelper->AddQueryParameter("continuation", osContinuation);
1588 }
1589
1590 struct curl_slist* headers = static_cast<struct curl_slist*>(
1591 CPLHTTPSetOptions(hCurlHandle,
1592 poHandleHelper->GetURL().c_str(),
1593 nullptr));
1594 headers = VSICurlMergeHeaders(headers,
1595 poHandleHelper->GetCurlHeaders("DELETE", headers));
1596
1597 CurlRequestHelper requestHelper;
1598 const long response_code =
1599 requestHelper.perform(hCurlHandle, headers, this, poHandleHelper.get());
1600
1601 NetworkStatisticsLogger::LogDELETE();
1602
1603 // 200 for path deletion
1604 // 202 for filesystem deletion
1605 if( response_code != 200 && response_code != 202 )
1606 {
1607 // Look if we should attempt a retry
1608 const double dfNewRetryDelay = CPLHTTPGetNewRetryDelay(
1609 static_cast<int>(response_code), dfRetryDelay,
1610 requestHelper.sWriteFuncHeaderData.pBuffer, requestHelper.szCurlErrBuf);
1611 if( dfNewRetryDelay > 0 &&
1612 nRetryCount < nMaxRetry )
1613 {
1614 CPLError(CE_Warning, CPLE_AppDefined,
1615 "HTTP error code: %d - %s. "
1616 "Retrying again in %.1f secs",
1617 static_cast<int>(response_code),
1618 poHandleHelper->GetURL().c_str(),
1619 dfRetryDelay);
1620 CPLSleep(dfRetryDelay);
1621 dfRetryDelay = dfNewRetryDelay;
1622 nRetryCount++;
1623 bRetry = true;
1624 }
1625 else
1626 {
1627 CPLDebug(GetDebugKey(), "Delete of %s failed: %s",
1628 osDirname.c_str(),
1629 requestHelper.sWriteFuncData.pBuffer
1630 ? requestHelper.sWriteFuncData.pBuffer
1631 : "(null)");
1632 if( requestHelper.sWriteFuncData.pBuffer != nullptr )
1633 {
1634 VSIError(VSIE_AWSError, "%s", requestHelper.sWriteFuncData.pBuffer);
1635 if( strstr(requestHelper.sWriteFuncData.pBuffer, "PathNotFound") )
1636 {
1637 errno = ENOENT;
1638 }
1639 else if( strstr(requestHelper.sWriteFuncData.pBuffer, "DirectoryNotEmpty") )
1640 {
1641 errno = ENOTEMPTY;
1642 }
1643 }
1644 nRet = -1;
1645 }
1646 }
1647 else
1648 {
1649 // Get continuation token for response headers
1650 osContinuation = GetContinuationToken(requestHelper.sWriteFuncHeaderData.pBuffer);
1651 if( !osContinuation.empty() )
1652 {
1653 nRetryCount = 0;
1654 bRetry = true;
1655 }
1656 }
1657
1658 curl_easy_cleanup(hCurlHandle);
1659 }
1660 while( bRetry );
1661
1662 return nRet;
1663 }
1664
1665 /************************************************************************/
1666 /* Rmdir() */
1667 /************************************************************************/
1668
1669 int VSIADLSFSHandler::Rmdir( const char * pszDirname )
1670 {
1671 if( !STARTS_WITH_CI(pszDirname, GetFSPrefix()) )
1672 return -1;
1673
1674 NetworkStatisticsFileSystem oContextFS(GetFSPrefix());
1675 NetworkStatisticsAction oContextAction("Rmdir");
1676
1677 return RmdirInternal(pszDirname, false);
1678 }
1679
1680 /************************************************************************/
1681 /* RmdirRecursive() */
1682 /************************************************************************/
1683
1684 int VSIADLSFSHandler::RmdirRecursive( const char * pszDirname )
1685 {
1686 if( !STARTS_WITH_CI(pszDirname, GetFSPrefix()) )
1687 return -1;
1688
1689 NetworkStatisticsFileSystem oContextFS(GetFSPrefix());
1690 NetworkStatisticsAction oContextAction("RmdirRecursive");
1691
1692 return RmdirInternal(pszDirname, true);
1693 }
1694
1695 /************************************************************************/
1696 /* CopyObject() */
1697 /************************************************************************/
1698
1699 int VSIADLSFSHandler::CopyObject( const char *oldpath, const char *newpath,
1700 CSLConstList /* papszMetadata */ )
1701 {
1702 // There is no CopyObject in ADLS... So use the base Azure blob one...
1703
1704 NetworkStatisticsFileSystem oContextFS(GetFSPrefix());
1705 NetworkStatisticsAction oContextAction("CopyObject");
1706
1707 CPLString osTargetNameWithoutPrefix = newpath + GetFSPrefix().size();
1708 auto poAzHandleHelper =
1709 std::unique_ptr<IVSIS3LikeHandleHelper>(
1710 VSIAzureBlobHandleHelper::BuildFromURI(osTargetNameWithoutPrefix, "/vsiaz/"));
1711 if( poAzHandleHelper == nullptr )
1712 {
1713 return -1;
1714 }
1715
1716 CPLString osSourceNameWithoutPrefix = oldpath + GetFSPrefix().size();
1717 auto poAzHandleHelperSource =
1718 std::unique_ptr<IVSIS3LikeHandleHelper>(
1719 VSIAzureBlobHandleHelper::BuildFromURI(osSourceNameWithoutPrefix, "/vsiaz/"));
1720 if( poAzHandleHelperSource == nullptr )
1721 {
1722 return -1;
1723 }
1724
1725 CPLString osSourceHeader("x-ms-copy-source: ");
1726 osSourceHeader += poAzHandleHelperSource->GetURLNoKVP();
1727
1728 int nRet = 0;
1729
1730 bool bRetry;
1731
1732 const int nMaxRetry = atoi(CPLGetConfigOption("GDAL_HTTP_MAX_RETRY",
1733 CPLSPrintf("%d",CPL_HTTP_MAX_RETRY)));
1734 // coverity[tainted_data]
1735 double dfRetryDelay = CPLAtof(CPLGetConfigOption("GDAL_HTTP_RETRY_DELAY",
1736 CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY)));
1737 int nRetryCount = 0;
1738
1739 do
1740 {
1741 bRetry = false;
1742 CURL* hCurlHandle = curl_easy_init();
1743 curl_easy_setopt(hCurlHandle, CURLOPT_CUSTOMREQUEST, "PUT");
1744
1745 struct curl_slist* headers = static_cast<struct curl_slist*>(
1746 CPLHTTPSetOptions(hCurlHandle,
1747 poAzHandleHelper->GetURL().c_str(),
1748 nullptr));
1749 headers = curl_slist_append(headers, osSourceHeader.c_str());
1750 headers = curl_slist_append(headers, "Content-Length: 0");
1751 headers = VSICurlSetContentTypeFromExt(headers, newpath);
1752 headers = VSICurlMergeHeaders(headers,
1753 poAzHandleHelper->GetCurlHeaders("PUT", headers));
1754 curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers);
1755
1756 CurlRequestHelper requestHelper;
1757 const long response_code =
1758 requestHelper.perform(hCurlHandle, headers, this, poAzHandleHelper.get());
1759
1760 NetworkStatisticsLogger::LogPUT(0);
1761
1762 if( response_code != 202)
1763 {
1764 // Look if we should attempt a retry
1765 const double dfNewRetryDelay = CPLHTTPGetNewRetryDelay(
1766 static_cast<int>(response_code), dfRetryDelay,
1767 requestHelper.sWriteFuncHeaderData.pBuffer, requestHelper.szCurlErrBuf);
1768 if( dfNewRetryDelay > 0 &&
1769 nRetryCount < nMaxRetry )
1770 {
1771 CPLError(CE_Warning, CPLE_AppDefined,
1772 "HTTP error code: %d - %s. "
1773 "Retrying again in %.1f secs",
1774 static_cast<int>(response_code),
1775 poAzHandleHelper->GetURL().c_str(),
1776 dfRetryDelay);
1777 CPLSleep(dfRetryDelay);
1778 dfRetryDelay = dfNewRetryDelay;
1779 nRetryCount++;
1780 bRetry = true;
1781 }
1782 else
1783 {
1784 CPLDebug(GetDebugKey(), "%s",
1785 requestHelper.sWriteFuncData.pBuffer
1786 ? requestHelper.sWriteFuncData.pBuffer
1787 : "(null)");
1788 CPLError(CE_Failure, CPLE_AppDefined, "Copy of %s to %s failed",
1789 oldpath, newpath);
1790 nRet = -1;
1791 }
1792 }
1793 else
1794 {
1795 auto poADLSHandleHelper =
1796 std::unique_ptr<IVSIS3LikeHandleHelper>(
1797 VSIAzureBlobHandleHelper::BuildFromURI(osTargetNameWithoutPrefix, GetFSPrefix()));
1798 if( poADLSHandleHelper != nullptr )
1799 InvalidateCachedData(poADLSHandleHelper->GetURLNoKVP().c_str());
1800
1801 const CPLString osFilenameWithoutSlash(RemoveTrailingSlash(newpath));
1802 InvalidateDirContent( CPLGetDirname(osFilenameWithoutSlash) );
1803 }
1804
1805 curl_easy_cleanup(hCurlHandle);
1806 }
1807 while( bRetry );
1808
1809 return nRet;
1810 }
1811
1812 /************************************************************************/
1813 /* UploadFile() */
1814 /************************************************************************/
1815
1816 bool VSIADLSFSHandler::UploadFile(const CPLString& osFilename,
1817 Event event,
1818 vsi_l_offset nPosition,
1819 const void* pabyBuffer,
1820 size_t nBufferSize,
1821 IVSIS3LikeHandleHelper *poHandleHelper,
1822 int nMaxRetry,
1823 double dfRetryDelay)
1824 {
1825 NetworkStatisticsFileSystem oContextFS(GetFSPrefix());
1826 NetworkStatisticsFile oContextFile(osFilename);
1827 NetworkStatisticsAction oContextAction("UploadFile");
1828
1829 if( event == Event::CREATE_FILE )
1830 {
1831 InvalidateCachedData(poHandleHelper->GetURLNoKVP().c_str());
1832 InvalidateDirContent( CPLGetDirname(osFilename) );
1833 }
1834
1835 bool bSuccess = true;
1836 int nRetryCount = 0;
1837 bool bRetry;
1838 do
1839 {
1840 bRetry = false;
1841
1842 CURL* hCurlHandle = curl_easy_init();
1843
1844 poHandleHelper->ResetQueryParameters();
1845 if( event == Event::CREATE_FILE )
1846 {
1847 poHandleHelper->AddQueryParameter("resource", "file");
1848 }
1849 else if( event == Event::APPEND_DATA )
1850 {
1851 poHandleHelper->AddQueryParameter("action", "append");
1852 poHandleHelper->AddQueryParameter("position",
1853 CPLSPrintf(CPL_FRMT_GUIB, static_cast<GUIntBig>(nPosition)));
1854 }
1855 else
1856 {
1857 poHandleHelper->AddQueryParameter("action", "flush");
1858 poHandleHelper->AddQueryParameter("close", "true");
1859 poHandleHelper->AddQueryParameter("position",
1860 CPLSPrintf(CPL_FRMT_GUIB, static_cast<GUIntBig>(nPosition)));
1861 }
1862
1863 curl_easy_setopt(hCurlHandle, CURLOPT_UPLOAD, 1L);
1864 curl_easy_setopt(hCurlHandle, CURLOPT_READFUNCTION,
1865 PutData::ReadCallBackBuffer);
1866 PutData putData;
1867 putData.pabyData = static_cast<const GByte*>(pabyBuffer);
1868 putData.nOff = 0;
1869 putData.nTotalSize = nBufferSize;
1870 curl_easy_setopt(hCurlHandle, CURLOPT_READDATA, &putData);
1871
1872 struct curl_slist* headers = static_cast<struct curl_slist*>(
1873 CPLHTTPSetOptions(hCurlHandle,
1874 poHandleHelper->GetURL().c_str(),
1875 nullptr));
1876 headers = VSICurlSetContentTypeFromExt(headers, osFilename.c_str());
1877
1878 CPLString osContentLength; // leave it in this scope
1879
1880 if( event == Event::APPEND_DATA )
1881 {
1882 curl_easy_setopt(hCurlHandle, CURLOPT_INFILESIZE,
1883 static_cast<int>(nBufferSize));
1884 // Disable "Expect: 100-continue" which doesn't hurt, but is not
1885 // needed
1886 headers = curl_slist_append(headers, "Expect:");
1887 osContentLength.Printf("Content-Length: %d",
1888 static_cast<int>(nBufferSize));
1889 headers = curl_slist_append(headers, osContentLength.c_str());
1890 }
1891 else
1892 {
1893 curl_easy_setopt(hCurlHandle, CURLOPT_INFILESIZE, 0);
1894 headers = curl_slist_append(headers, "Content-Length: 0");
1895 }
1896
1897 const char* pszVerb = (event == Event::CREATE_FILE) ? "PUT" : "PATCH";
1898 curl_easy_setopt(hCurlHandle, CURLOPT_CUSTOMREQUEST, pszVerb);
1899 headers = VSICurlMergeHeaders(headers,
1900 poHandleHelper->GetCurlHeaders(pszVerb, headers));
1901 curl_easy_setopt(hCurlHandle, CURLOPT_HTTPHEADER, headers);
1902
1903 CurlRequestHelper requestHelper;
1904 const long response_code =
1905 requestHelper.perform(hCurlHandle, headers, this, poHandleHelper);
1906
1907 NetworkStatisticsLogger::LogPUT( event == Event::APPEND_DATA ? nBufferSize : 0 );
1908
1909 // 200 for PATCH flush
1910 // 201 for PUT create
1911 // 202 for PATCH append
1912 if( response_code != 200 && response_code != 201 && response_code != 202 )
1913 {
1914 // Look if we should attempt a retry
1915 const double dfNewRetryDelay = CPLHTTPGetNewRetryDelay(
1916 static_cast<int>(response_code), dfRetryDelay,
1917 requestHelper.sWriteFuncHeaderData.pBuffer, requestHelper.szCurlErrBuf);
1918 if( dfNewRetryDelay > 0 &&
1919 nRetryCount < nMaxRetry )
1920 {
1921 CPLError(CE_Warning, CPLE_AppDefined,
1922 "HTTP error code: %d - %s. "
1923 "Retrying again in %.1f secs",
1924 static_cast<int>(response_code),
1925 poHandleHelper->GetURL().c_str(),
1926 dfRetryDelay);
1927 CPLSleep(dfRetryDelay);
1928 dfRetryDelay = dfNewRetryDelay;
1929 nRetryCount++;
1930 bRetry = true;
1931 }
1932 else
1933 {
1934 CPLDebug(GetDebugKey(),
1935 "%s of %s failed: %s",
1936 pszVerb,
1937 osFilename.c_str(),
1938 requestHelper.sWriteFuncData.pBuffer
1939 ? requestHelper.sWriteFuncData.pBuffer
1940 : "(null)");
1941 bSuccess = false;
1942 }
1943 }
1944
1945 curl_easy_cleanup(hCurlHandle);
1946 } while( bRetry );
1947
1948 return bSuccess;
1949 }
1950
1951 /************************************************************************/
1952 /* GetFileList() */
1953 /************************************************************************/
1954
1955 char** VSIADLSFSHandler::GetFileList( const char *pszDirname,
1956 int nMaxFiles,
1957 bool* pbGotFileList )
1958 {
1959 return GetFileList(pszDirname, nMaxFiles, true, pbGotFileList);
1960 }
1961
1962
1963 char** VSIADLSFSHandler::GetFileList( const char *pszDirname,
1964 int nMaxFiles,
1965 bool bCacheEntries,
1966 bool* pbGotFileList )
1967 {
1968 if( ENABLE_DEBUG )
1969 CPLDebug(GetDebugKey(), "GetFileList(%s)" , pszDirname);
1970
1971 *pbGotFileList = false;
1972
1973 char** papszOptions = CSLSetNameValue(nullptr,
1974 "MAXFILES", CPLSPrintf("%d", nMaxFiles));
1975 papszOptions = CSLSetNameValue(papszOptions,
1976 "CACHE_ENTRIES", bCacheEntries ? "YES" : "NO");
1977 auto dir = OpenDir(pszDirname, 0, papszOptions);
1978 CSLDestroy(papszOptions);
1979 if( !dir )
1980 {
1981 return nullptr;
1982 }
1983 CPLStringList aosFileList;
1984 while( true )
1985 {
1986 auto entry = dir->NextDirEntry();
1987 if( !entry )
1988 {
1989 break;
1990 }
1991 aosFileList.AddString(entry->pszName);
1992
1993 if( nMaxFiles > 0 && aosFileList.size() >= nMaxFiles )
1994 break;
1995 }
1996 delete dir;
1997 *pbGotFileList = true;
1998 return aosFileList.StealList();
1999 }
2000
2001
2002 /************************************************************************/
2003 /* GetOptions() */
2004 /************************************************************************/
2005
2006 const char* VSIADLSFSHandler::GetOptions()
2007 {
2008 static CPLString osOptions(
2009 CPLString("<Options>") +
2010 " <Option name='AZURE_STORAGE_CONNECTION_STRING' type='string' "
2011 "description='Connection string that contains account name and "
2012 "secret key'/>"
2013 " <Option name='AZURE_STORAGE_ACCOUNT' type='string' "
2014 "description='Storage account. To use with AZURE_STORAGE_ACCESS_KEY'/>"
2015 " <Option name='AZURE_STORAGE_ACCESS_KEY' type='string' "
2016 "description='Secret key'/>"
2017 " <Option name='VSIAZ_CHUNK_SIZE' type='int' "
2018 "description='Size in MB for chunks of files that are uploaded' "
2019 "default='4' min='1' max='4'/>" +
2020 VSICurlFilesystemHandler::GetOptionsStatic() +
2021 "</Options>");
2022 return osOptions.c_str();
2023 }
2024
2025 /************************************************************************/
2026 /* GetSignedURL() */
2027 /************************************************************************/
2028
2029 char* VSIADLSFSHandler::GetSignedURL(const char* pszFilename, CSLConstList papszOptions )
2030 {
2031 if( !STARTS_WITH_CI(pszFilename, GetFSPrefix()) )
2032 return nullptr;
2033
2034 auto poHandleHelper = std::unique_ptr<VSIAzureBlobHandleHelper>(
2035 VSIAzureBlobHandleHelper::BuildFromURI(pszFilename + GetFSPrefix().size(),
2036 "/vsiaz/", // use Azure blob
2037 papszOptions));
2038 if( poHandleHelper == nullptr )
2039 {
2040 return nullptr;
2041 }
2042
2043 CPLString osRet(poHandleHelper->GetSignedURL(papszOptions));
2044
2045 return CPLStrdup(osRet);
2046 }
2047
2048 /************************************************************************/
2049 /* OpenDir() */
2050 /************************************************************************/
2051
2052 VSIDIR* VSIADLSFSHandler::OpenDir( const char *pszPath,
2053 int nRecurseDepth,
2054 const char* const *papszOptions)
2055 {
2056 if( nRecurseDepth > 0)
2057 {
2058 return VSIFilesystemHandler::OpenDir(pszPath, nRecurseDepth, papszOptions);
2059 }
2060
2061 if( !STARTS_WITH_CI(pszPath, GetFSPrefix()) )
2062 return nullptr;
2063
2064 NetworkStatisticsFileSystem oContextFS(GetFSPrefix());
2065 NetworkStatisticsAction oContextAction("OpenDir");
2066
2067 const CPLString osDirnameWithoutPrefix =
2068 RemoveTrailingSlash(pszPath + GetFSPrefix().size());
2069 CPLString osFilesystem(osDirnameWithoutPrefix);
2070 CPLString osObjectKey;
2071 size_t nSlashPos = osDirnameWithoutPrefix.find('/');
2072 if( nSlashPos != std::string::npos )
2073 {
2074 osFilesystem = osDirnameWithoutPrefix.substr(0, nSlashPos);
2075 osObjectKey = osDirnameWithoutPrefix.substr(nSlashPos+1);
2076 }
2077
2078 VSIDIRADLS* dir = new VSIDIRADLS(this);
2079 dir->m_nRecurseDepth = nRecurseDepth;
2080 dir->m_poFS = this;
2081 dir->m_bRecursiveRequestFromAccountRoot = osFilesystem.empty() && nRecurseDepth < 0;
2082 dir->m_osFilesystem = osFilesystem;
2083 dir->m_osObjectKey = osObjectKey;
2084 dir->m_nMaxFiles = atoi(CSLFetchNameValueDef(papszOptions, "MAXFILES", "0"));
2085 dir->m_bCacheEntries = CPLTestBool(
2086 CSLFetchNameValueDef(papszOptions, "CACHE_ENTRIES", "YES"));
2087 if( !dir->IssueListDir() )
2088 {
2089 delete dir;
2090 return nullptr;
2091 }
2092
2093 return dir;
2094 }
2095
2096 /************************************************************************/
2097 /* GetStreamingPath() */
2098 /************************************************************************/
2099
2100 CPLString VSIADLSFSHandler::GetStreamingPath( const char* pszFilename ) const
2101 {
2102 const CPLString osPrefix(GetFSPrefix());
2103 if( !STARTS_WITH_CI(pszFilename, osPrefix) )
2104 return CPLString();
2105
2106 // Transform /vsiadls/foo into /vsiaz_streaming/foo
2107 const size_t nPrefixLen = osPrefix.size();
2108 return CPLString("/vsiaz_streaming/") + (pszFilename + nPrefixLen);
2109 }
2110
2111 /************************************************************************/
2112 /* VSIADLSHandle() */
2113 /************************************************************************/
2114
2115 VSIADLSHandle::VSIADLSHandle( VSIADLSFSHandler* poFSIn,
2116 const char* pszFilename,
2117 VSIAzureBlobHandleHelper* poHandleHelper ) :
2118 VSICurlHandle(poFSIn, pszFilename, poHandleHelper->GetURLNoKVP()),
2119 m_poHandleHelper(poHandleHelper)
2120 {
2121 }
2122
2123 /************************************************************************/
2124 /* GetCurlHeaders() */
2125 /************************************************************************/
2126
2127 struct curl_slist* VSIADLSHandle::GetCurlHeaders( const CPLString& osVerb,
2128 const struct curl_slist* psExistingHeaders )
2129 {
2130 return m_poHandleHelper->GetCurlHeaders( osVerb, psExistingHeaders );
2131 }
2132
2133 } /* end of namespace cpl */
2134
2135
2136 #endif // DOXYGEN_SKIP
2137 //! @endcond
2138
2139 /************************************************************************/
2140 /* VSIInstallADLSFileHandler() */
2141 /************************************************************************/
2142
2143 /**
2144 * \brief Install /vsiadls/ Microsoft Azure Data Lake Storage Gen2 file system handler
2145 * (requires libcurl)
2146 *
2147 * @see <a href="gdal_virtual_file_systems.html#gdal_virtual_file_systems_vsiadls">/vsiadls/ documentation</a>
2148 *
2149 * @since GDAL 3.3
2150 */
2151
2152 void VSIInstallADLSFileHandler( void )
2153 {
2154 VSIFileManager::InstallHandler( "/vsiadls/", new cpl::VSIADLSFSHandler );
2155 }
2156
2157 #endif /* HAVE_CURL */
2158