1 /**********************************************************************
2 *
3 * Name: cpl_aws.cpp
4 * Project: CPL - Common Portability Library
5 * Purpose: Amazon Web Services routines
6 * Author: Even Rouault <even.rouault at spatialys.com>
7 *
8 **********************************************************************
9 * Copyright (c) 2015, Even Rouault <even.rouault at spatialys.com>
10 *
11 * Permission is hereby granted, free of charge, to any person obtaining a
12 * copy of this software and associated documentation files (the "Software"),
13 * to deal in the Software without restriction, including without limitation
14 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 * and/or sell copies of the Software, and to permit persons to whom the
16 * Software is furnished to do so, subject to the following conditions:
17 *
18 * The above copyright notice and this permission notice shall be included
19 * in all copies or substantial portions of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 * DEALINGS IN THE SOFTWARE.
28 ****************************************************************************/
29
30 //! @cond Doxygen_Suppress
31
32 #include "cpl_aws.h"
33 #include "cpl_vsi_error.h"
34 #include "cpl_sha256.h"
35 #include "cpl_time.h"
36 #include "cpl_minixml.h"
37 #include "cpl_multiproc.h"
38 #include "cpl_http.h"
39 #include <algorithm>
40
41 CPL_CVSID("$Id: cpl_aws.cpp 86933038c3926cd4dc3ff37c431b317abb69e602 2021-03-27 23:20:49 +0100 Even Rouault $")
42
43 // #define DEBUG_VERBOSE 1
44
45 #ifdef WIN32
46 #if defined(_MSC_VER)
47 bool CPLFetchWindowsProductUUID(CPLString &osStr); // defined in cpl_aws_win32.cpp
48 #endif
49 const char* CPLGetWineVersion(); // defined in cpl_vsil_win32.cpp
50 #endif
51
52 #ifdef HAVE_CURL
53 static CPLMutex *ghMutex = nullptr;
54 static CPLString gosIAMRole;
55 static CPLString gosGlobalAccessKeyId;
56 static CPLString gosGlobalSecretAccessKey;
57 static CPLString gosGlobalSessionToken;
58 static GIntBig gnGlobalExpiration = 0;
59
60 /************************************************************************/
61 /* CPLGetLowerCaseHex() */
62 /************************************************************************/
63
CPLGetLowerCaseHex(const GByte * pabyData,size_t nBytes)64 static CPLString CPLGetLowerCaseHex( const GByte *pabyData, size_t nBytes )
65
66 {
67 CPLString osRet;
68 osRet.resize(nBytes * 2);
69
70 constexpr char achHex[] = "0123456789abcdef";
71
72 for( size_t i = 0; i < nBytes; ++i )
73 {
74 const int nLow = pabyData[i] & 0x0f;
75 const int nHigh = (pabyData[i] & 0xf0) >> 4;
76
77 osRet[i*2] = achHex[nHigh];
78 osRet[i*2+1] = achHex[nLow];
79 }
80
81 return osRet;
82 }
83
84 /************************************************************************/
85 /* CPLGetLowerCaseHexSHA256() */
86 /************************************************************************/
87
CPLGetLowerCaseHexSHA256(const void * pabyData,size_t nBytes)88 CPLString CPLGetLowerCaseHexSHA256( const void *pabyData, size_t nBytes )
89 {
90 GByte hash[CPL_SHA256_HASH_SIZE] = {};
91 CPL_SHA256(static_cast<const GByte *>(pabyData), nBytes, hash);
92 return CPLGetLowerCaseHex(hash, CPL_SHA256_HASH_SIZE);
93 }
94
95 /************************************************************************/
96 /* CPLGetLowerCaseHexSHA256() */
97 /************************************************************************/
98
CPLGetLowerCaseHexSHA256(const CPLString & osStr)99 CPLString CPLGetLowerCaseHexSHA256( const CPLString& osStr )
100 {
101 return CPLGetLowerCaseHexSHA256(osStr.c_str(), osStr.size());
102 }
103
104 /************************************************************************/
105 /* CPLAWSURLEncode() */
106 /************************************************************************/
107
CPLAWSURLEncode(const CPLString & osURL,bool bEncodeSlash)108 CPLString CPLAWSURLEncode( const CPLString& osURL, bool bEncodeSlash )
109 {
110 CPLString osRet;
111 for( size_t i = 0; i < osURL.size(); i++ )
112 {
113 char ch = osURL[i];
114 if( (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') ||
115 (ch >= '0' && ch <= '9') ||
116 ch == '_' || ch == '-' || ch == '~' || ch == '.' )
117 {
118 osRet += ch;
119 }
120 else if( ch == '/' )
121 {
122 if( bEncodeSlash )
123 osRet += "%2F";
124 else
125 osRet += ch;
126 }
127 else
128 {
129 osRet += CPLSPrintf("%%%02X", static_cast<unsigned char>(ch));
130 }
131 }
132 return osRet;
133 }
134
135
136 /************************************************************************/
137 /* CPLAWSGetHeaderVal() */
138 /************************************************************************/
139
CPLAWSGetHeaderVal(const struct curl_slist * psExistingHeaders,const char * pszKey)140 CPLString CPLAWSGetHeaderVal(const struct curl_slist* psExistingHeaders,
141 const char* pszKey)
142 {
143 CPLString osKey(pszKey);
144 osKey += ":";
145 const struct curl_slist* psIter = psExistingHeaders;
146 for(; psIter != nullptr; psIter = psIter->next)
147 {
148 if( STARTS_WITH(psIter->data, osKey.c_str()) )
149 return CPLString(psIter->data + osKey.size()).Trim();
150 }
151 return CPLString();
152 }
153
154
155 /************************************************************************/
156 /* CPLGetAWS_SIGN4_Signature() */
157 /************************************************************************/
158
159 // See:
160 // http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
161 CPLString
CPLGetAWS_SIGN4_Signature(const CPLString & osSecretAccessKey,const CPLString & osAccessToken,const CPLString & osRegion,const CPLString & osRequestPayer,const CPLString & osService,const CPLString & osVerb,const struct curl_slist * psExistingHeaders,const CPLString & osHost,const CPLString & osCanonicalURI,const CPLString & osCanonicalQueryString,const CPLString & osXAMZContentSHA256,const CPLString & osTimestamp,CPLString & osSignedHeaders)162 CPLGetAWS_SIGN4_Signature( const CPLString& osSecretAccessKey,
163 const CPLString& osAccessToken,
164 const CPLString& osRegion,
165 const CPLString& osRequestPayer,
166 const CPLString& osService,
167 const CPLString& osVerb,
168 const struct curl_slist* psExistingHeaders,
169 const CPLString& osHost,
170 const CPLString& osCanonicalURI,
171 const CPLString& osCanonicalQueryString,
172 const CPLString& osXAMZContentSHA256,
173 const CPLString& osTimestamp,
174 CPLString& osSignedHeaders )
175 {
176 /* -------------------------------------------------------------------- */
177 /* Compute canonical request string. */
178 /* -------------------------------------------------------------------- */
179 CPLString osCanonicalRequest = osVerb + "\n";
180
181 osCanonicalRequest += osCanonicalURI + "\n";
182
183 osCanonicalRequest += osCanonicalQueryString + "\n";
184
185 std::map<CPLString, CPLString> oSortedMapHeaders;
186 oSortedMapHeaders["host"] = osHost;
187 if( osXAMZContentSHA256 != "UNSIGNED-PAYLOAD" )
188 {
189 oSortedMapHeaders["x-amz-content-sha256"] = osXAMZContentSHA256;
190 oSortedMapHeaders["x-amz-date"] = osTimestamp;
191 }
192 if( !osRequestPayer.empty() )
193 oSortedMapHeaders["x-amz-request-payer"] = osRequestPayer;
194 if( !osAccessToken.empty() )
195 oSortedMapHeaders["x-amz-security-token"] = osAccessToken;
196 CPLString osCanonicalizedHeaders(
197 IVSIS3LikeHandleHelper::BuildCanonicalizedHeaders(
198 oSortedMapHeaders,
199 psExistingHeaders,
200 "x-amz-"));
201
202 osCanonicalRequest += osCanonicalizedHeaders + "\n";
203
204 osSignedHeaders.clear();
205 std::map<CPLString, CPLString>::const_iterator oIter = oSortedMapHeaders.begin();
206 for(; oIter != oSortedMapHeaders.end(); ++oIter )
207 {
208 if( !osSignedHeaders.empty() )
209 osSignedHeaders += ";";
210 osSignedHeaders += oIter->first;
211 }
212
213 osCanonicalRequest += osSignedHeaders + "\n";
214
215 osCanonicalRequest += osXAMZContentSHA256;
216
217 #ifdef DEBUG_VERBOSE
218 CPLDebug("S3", "osCanonicalRequest='%s'", osCanonicalRequest.c_str());
219 #endif
220
221 /* -------------------------------------------------------------------- */
222 /* Compute StringToSign . */
223 /* -------------------------------------------------------------------- */
224 CPLString osStringToSign = "AWS4-HMAC-SHA256\n";
225 osStringToSign += osTimestamp + "\n";
226
227 CPLString osYYMMDD(osTimestamp);
228 osYYMMDD.resize(8);
229
230 CPLString osScope = osYYMMDD + "/";
231 osScope += osRegion;
232 osScope += "/";
233 osScope += osService;
234 osScope += "/aws4_request";
235 osStringToSign += osScope + "\n";
236 osStringToSign += CPLGetLowerCaseHexSHA256(osCanonicalRequest);
237
238 #ifdef DEBUG_VERBOSE
239 CPLDebug("S3", "osStringToSign='%s'", osStringToSign.c_str());
240 #endif
241
242 /* -------------------------------------------------------------------- */
243 /* Compute signing key. */
244 /* -------------------------------------------------------------------- */
245 GByte abySigningKeyIn[CPL_SHA256_HASH_SIZE] = {};
246 GByte abySigningKeyOut[CPL_SHA256_HASH_SIZE] = {};
247
248 CPLString osFirstKey(CPLString("AWS4") + osSecretAccessKey);
249 CPL_HMAC_SHA256( osFirstKey.c_str(), osFirstKey.size(),
250 osYYMMDD, osYYMMDD.size(),
251 abySigningKeyOut );
252 memcpy(abySigningKeyIn, abySigningKeyOut, CPL_SHA256_HASH_SIZE);
253
254 CPL_HMAC_SHA256( abySigningKeyIn, CPL_SHA256_HASH_SIZE,
255 osRegion.c_str(), osRegion.size(),
256 abySigningKeyOut );
257 memcpy(abySigningKeyIn, abySigningKeyOut, CPL_SHA256_HASH_SIZE);
258
259 CPL_HMAC_SHA256( abySigningKeyIn, CPL_SHA256_HASH_SIZE,
260 osService.c_str(), osService.size(),
261 abySigningKeyOut );
262 memcpy(abySigningKeyIn, abySigningKeyOut, CPL_SHA256_HASH_SIZE);
263
264 CPL_HMAC_SHA256( abySigningKeyIn, CPL_SHA256_HASH_SIZE,
265 "aws4_request", strlen("aws4_request"),
266 abySigningKeyOut );
267 memcpy(abySigningKeyIn, abySigningKeyOut, CPL_SHA256_HASH_SIZE);
268
269 #ifdef DEBUG_VERBOSE
270 CPLString osSigningKey(CPLGetLowerCaseHex(abySigningKeyIn,
271 CPL_SHA256_HASH_SIZE));
272 CPLDebug("S3", "osSigningKey='%s'", osSigningKey.c_str());
273 #endif
274
275 /* -------------------------------------------------------------------- */
276 /* Compute signature. */
277 /* -------------------------------------------------------------------- */
278 GByte abySignature[CPL_SHA256_HASH_SIZE] = {};
279 CPL_HMAC_SHA256( abySigningKeyIn, CPL_SHA256_HASH_SIZE,
280 osStringToSign, osStringToSign.size(),
281 abySignature);
282 CPLString osSignature(CPLGetLowerCaseHex(abySignature,
283 CPL_SHA256_HASH_SIZE));
284
285 #ifdef DEBUG_VERBOSE
286 CPLDebug("S3", "osSignature='%s'", osSignature.c_str());
287 #endif
288
289 return osSignature;
290 }
291
292 /************************************************************************/
293 /* CPLGetAWS_SIGN4_Authorization() */
294 /************************************************************************/
295
296 CPLString
CPLGetAWS_SIGN4_Authorization(const CPLString & osSecretAccessKey,const CPLString & osAccessKeyId,const CPLString & osAccessToken,const CPLString & osRegion,const CPLString & osRequestPayer,const CPLString & osService,const CPLString & osVerb,const struct curl_slist * psExistingHeaders,const CPLString & osHost,const CPLString & osCanonicalURI,const CPLString & osCanonicalQueryString,const CPLString & osXAMZContentSHA256,const CPLString & osTimestamp)297 CPLGetAWS_SIGN4_Authorization( const CPLString& osSecretAccessKey,
298 const CPLString& osAccessKeyId,
299 const CPLString& osAccessToken,
300 const CPLString& osRegion,
301 const CPLString& osRequestPayer,
302 const CPLString& osService,
303 const CPLString& osVerb,
304 const struct curl_slist* psExistingHeaders,
305 const CPLString& osHost,
306 const CPLString& osCanonicalURI,
307 const CPLString& osCanonicalQueryString,
308 const CPLString& osXAMZContentSHA256,
309 const CPLString& osTimestamp )
310 {
311 CPLString osSignedHeaders;
312 CPLString osSignature(CPLGetAWS_SIGN4_Signature(osSecretAccessKey,
313 osAccessToken,
314 osRegion,
315 osRequestPayer,
316 osService,
317 osVerb,
318 psExistingHeaders,
319 osHost,
320 osCanonicalURI,
321 osCanonicalQueryString,
322 osXAMZContentSHA256,
323 osTimestamp,
324 osSignedHeaders));
325
326 CPLString osYYMMDD(osTimestamp);
327 osYYMMDD.resize(8);
328
329 /* -------------------------------------------------------------------- */
330 /* Build authorization header. */
331 /* -------------------------------------------------------------------- */
332 CPLString osAuthorization;
333 osAuthorization = "AWS4-HMAC-SHA256 Credential=";
334 osAuthorization += osAccessKeyId;
335 osAuthorization += "/";
336 osAuthorization += osYYMMDD;
337 osAuthorization += "/";
338 osAuthorization += osRegion;
339 osAuthorization += "/";
340 osAuthorization += osService;
341 osAuthorization += "/";
342 osAuthorization += "aws4_request";
343 osAuthorization += ",";
344 osAuthorization += "SignedHeaders=";
345 osAuthorization += osSignedHeaders;
346 osAuthorization += ",";
347 osAuthorization += "Signature=";
348 osAuthorization += osSignature;
349
350 #ifdef DEBUG_VERBOSE
351 CPLDebug("S3", "osAuthorization='%s'", osAuthorization.c_str());
352 #endif
353
354 return osAuthorization;
355 }
356
357 /************************************************************************/
358 /* CPLGetAWS_SIGN4_Timestamp() */
359 /************************************************************************/
360
CPLGetAWS_SIGN4_Timestamp()361 CPLString CPLGetAWS_SIGN4_Timestamp()
362 {
363 struct tm brokenDown;
364 CPLUnixTimeToYMDHMS(time(nullptr), &brokenDown);
365
366 char szTimeStamp[80] = {};
367 snprintf(szTimeStamp, sizeof(szTimeStamp), "%04d%02d%02dT%02d%02d%02dZ",
368 brokenDown.tm_year + 1900,
369 brokenDown.tm_mon + 1,
370 brokenDown.tm_mday,
371 brokenDown.tm_hour,
372 brokenDown.tm_min,
373 brokenDown.tm_sec);
374 return szTimeStamp;
375 }
376
377
378 /************************************************************************/
379 /* VSIS3HandleHelper() */
380 /************************************************************************/
VSIS3HandleHelper(const CPLString & osSecretAccessKey,const CPLString & osAccessKeyId,const CPLString & osSessionToken,const CPLString & osEndpoint,const CPLString & osRegion,const CPLString & osRequestPayer,const CPLString & osBucket,const CPLString & osObjectKey,bool bUseHTTPS,bool bUseVirtualHosting,bool bFromEC2)381 VSIS3HandleHelper::VSIS3HandleHelper( const CPLString& osSecretAccessKey,
382 const CPLString& osAccessKeyId,
383 const CPLString& osSessionToken,
384 const CPLString& osEndpoint,
385 const CPLString& osRegion,
386 const CPLString& osRequestPayer,
387 const CPLString& osBucket,
388 const CPLString& osObjectKey,
389 bool bUseHTTPS,
390 bool bUseVirtualHosting,
391 bool bFromEC2 ) :
392 m_osURL(BuildURL(osEndpoint, osBucket, osObjectKey, bUseHTTPS,
393 bUseVirtualHosting)),
394 m_osSecretAccessKey(osSecretAccessKey),
395 m_osAccessKeyId(osAccessKeyId),
396 m_osSessionToken(osSessionToken),
397 m_osEndpoint(osEndpoint),
398 m_osRegion(osRegion),
399 m_osRequestPayer(osRequestPayer),
400 m_osBucket(osBucket),
401 m_osObjectKey(osObjectKey),
402 m_bUseHTTPS(bUseHTTPS),
403 m_bUseVirtualHosting(bUseVirtualHosting),
404 m_bFromEC2(bFromEC2)
405 {}
406
407 /************************************************************************/
408 /* ~VSIS3HandleHelper() */
409 /************************************************************************/
410
~VSIS3HandleHelper()411 VSIS3HandleHelper::~VSIS3HandleHelper()
412 {
413 for( size_t i = 0; i < m_osSecretAccessKey.size(); i++ )
414 m_osSecretAccessKey[i] = 0;
415 }
416
417 /************************************************************************/
418 /* BuildURL() */
419 /************************************************************************/
420
BuildURL(const CPLString & osEndpoint,const CPLString & osBucket,const CPLString & osObjectKey,bool bUseHTTPS,bool bUseVirtualHosting)421 CPLString VSIS3HandleHelper::BuildURL(const CPLString& osEndpoint,
422 const CPLString& osBucket,
423 const CPLString& osObjectKey,
424 bool bUseHTTPS, bool bUseVirtualHosting)
425 {
426 const char* pszProtocol = (bUseHTTPS) ? "https" : "http";
427 if( osBucket.empty() )
428 return CPLSPrintf("%s://%s", pszProtocol,
429 osEndpoint.c_str());
430 else if( bUseVirtualHosting )
431 return CPLSPrintf("%s://%s.%s/%s", pszProtocol,
432 osBucket.c_str(),
433 osEndpoint.c_str(),
434 CPLAWSURLEncode(osObjectKey, false).c_str());
435 else
436 return CPLSPrintf("%s://%s/%s/%s", pszProtocol,
437 osEndpoint.c_str(),
438 osBucket.c_str(),
439 CPLAWSURLEncode(osObjectKey, false).c_str());
440 }
441
442 /************************************************************************/
443 /* RebuildURL() */
444 /************************************************************************/
445
RebuildURL()446 void VSIS3HandleHelper::RebuildURL()
447 {
448 m_osURL = BuildURL(m_osEndpoint, m_osBucket, m_osObjectKey,
449 m_bUseHTTPS, m_bUseVirtualHosting);
450 m_osURL += GetQueryString(false);
451 }
452
453 /************************************************************************/
454 /* GetBucketAndObjectKey() */
455 /************************************************************************/
456
GetBucketAndObjectKey(const char * pszURI,const char * pszFSPrefix,bool bAllowNoObject,CPLString & osBucket,CPLString & osObjectKey)457 bool IVSIS3LikeHandleHelper::GetBucketAndObjectKey( const char* pszURI,
458 const char* pszFSPrefix,
459 bool bAllowNoObject,
460 CPLString &osBucket,
461 CPLString &osObjectKey )
462 {
463 osBucket = pszURI;
464 if( osBucket.empty() )
465 {
466 return false;
467 }
468 size_t nPos = osBucket.find('/');
469 if( nPos == std::string::npos )
470 {
471 if( bAllowNoObject )
472 {
473 osObjectKey = "";
474 return true;
475 }
476 CPLError(CE_Failure, CPLE_AppDefined,
477 "Filename should be of the form %sbucket/key", pszFSPrefix);
478 return false;
479 }
480 osBucket.resize(nPos);
481 osObjectKey = pszURI + nPos + 1;
482 return true;
483 }
484
485 /************************************************************************/
486 /* BuildCanonicalizedHeaders() */
487 /************************************************************************/
488
BuildCanonicalizedHeaders(std::map<CPLString,CPLString> & oSortedMapHeaders,const struct curl_slist * psExistingHeaders,const char * pszHeaderPrefix)489 CPLString IVSIS3LikeHandleHelper::BuildCanonicalizedHeaders(
490 std::map<CPLString, CPLString>& oSortedMapHeaders,
491 const struct curl_slist* psExistingHeaders,
492 const char* pszHeaderPrefix)
493 {
494 const struct curl_slist* psIter = psExistingHeaders;
495 for(; psIter != nullptr; psIter = psIter->next)
496 {
497 if( STARTS_WITH_CI(psIter->data, pszHeaderPrefix) ||
498 STARTS_WITH_CI(psIter->data, "Content-MD5") )
499 {
500 const char* pszColumn = strstr(psIter->data, ":");
501 if( pszColumn )
502 {
503 CPLString osKey(psIter->data);
504 osKey.resize( pszColumn - psIter->data);
505 oSortedMapHeaders[osKey.tolower()] =
506 CPLString(pszColumn + strlen(":")).Trim();
507 }
508 }
509 }
510
511 CPLString osCanonicalizedHeaders;
512 std::map<CPLString, CPLString>::const_iterator oIter =
513 oSortedMapHeaders.begin();
514 for(; oIter != oSortedMapHeaders.end(); ++oIter )
515 {
516 osCanonicalizedHeaders += oIter->first + ":" + oIter->second + "\n";
517 }
518 return osCanonicalizedHeaders;
519 }
520
521 /************************************************************************/
522 /* GetRFC822DateTime() */
523 /************************************************************************/
524
GetRFC822DateTime()525 CPLString IVSIS3LikeHandleHelper::GetRFC822DateTime()
526 {
527 char szDate[64];
528 time_t nNow = time(nullptr);
529 struct tm tm;
530 CPLUnixTimeToYMDHMS(nNow, &tm);
531 int nRet = CPLPrintTime(szDate, sizeof(szDate)-1,
532 "%a, %d %b %Y %H:%M:%S GMT", &tm, "C");
533 szDate[nRet] = 0;
534 return szDate;
535 }
536
537 /************************************************************************/
538 /* ParseSimpleJson() */
539 /* */
540 /* Return a string list of name/value pairs extracted from a */
541 /* JSON doc. The EC2 IAM web service returns simple JSON */
542 /* responses. The parsing as done currently is very fragile */
543 /* and depends on JSON documents being in a very very simple */
544 /* form. */
545 /************************************************************************/
546
ParseSimpleJson(const char * pszJson)547 static CPLStringList ParseSimpleJson(const char *pszJson)
548
549 {
550 /* -------------------------------------------------------------------- */
551 /* We are expecting simple documents like the following with no */
552 /* hierarchy or complex structure. */
553 /* -------------------------------------------------------------------- */
554 /*
555 {
556 "Code" : "Success",
557 "LastUpdated" : "2017-07-03T16:20:17Z",
558 "Type" : "AWS-HMAC",
559 "AccessKeyId" : "bla",
560 "SecretAccessKey" : "bla",
561 "Token" : "bla",
562 "Expiration" : "2017-07-03T22:42:58Z"
563 }
564 */
565
566 CPLStringList oWords(
567 CSLTokenizeString2(pszJson, " \n\t,:{}", CSLT_HONOURSTRINGS ));
568 CPLStringList oNameValue;
569
570 for( int i=0; i < oWords.size(); i += 2 )
571 {
572 oNameValue.SetNameValue(oWords[i], oWords[i+1]);
573 }
574
575 return oNameValue;
576 }
577
578 /************************************************************************/
579 /* Iso8601ToUnixTime() */
580 /************************************************************************/
581
Iso8601ToUnixTime(const char * pszDT,GIntBig * pnUnixTime)582 static bool Iso8601ToUnixTime(const char* pszDT, GIntBig* pnUnixTime)
583 {
584 int nYear;
585 int nMonth;
586 int nDay;
587 int nHour;
588 int nMinute;
589 int nSecond;
590 if( sscanf(pszDT, "%04d-%02d-%02dT%02d:%02d:%02d",
591 &nYear, &nMonth, &nDay, &nHour, &nMinute, &nSecond) == 6 )
592 {
593 struct tm brokendowntime;
594 brokendowntime.tm_year = nYear - 1900;
595 brokendowntime.tm_mon = nMonth - 1;
596 brokendowntime.tm_mday = nDay;
597 brokendowntime.tm_hour = nHour;
598 brokendowntime.tm_min = nMinute;
599 brokendowntime.tm_sec = nSecond;
600 *pnUnixTime = CPLYMDHMSToUnixTime(&brokendowntime);
601 return true;
602 }
603 return false;
604 }
605
606 /************************************************************************/
607 /* IsMachinePotentiallyEC2Instance() */
608 /************************************************************************/
609
IsMachinePotentiallyEC2Instance()610 static bool IsMachinePotentiallyEC2Instance()
611 {
612 #if defined(__linux) || defined(WIN32)
613 const auto IsMachinePotentiallyEC2InstanceFromLinuxHost = []()
614 {
615 // On the newer Nitro Hypervisor (C5, M5, H1, T3), use
616 // /sys/devices/virtual/dmi/id/sys_vendor = 'Amazon EC2' instead.
617
618 // On older Xen hypervisor EC2 instances, a /sys/hypervisor/uuid file will
619 // exist with a string beginning with 'ec2'.
620
621 // If the files exist but don't contain the correct content, then we're not EC2 and
622 // do not attempt any network access
623
624 // Check for Xen Hypervisor instances
625 // This file doesn't exist on Nitro instances
626 VSILFILE* fp = VSIFOpenL("/sys/hypervisor/uuid", "rb");
627 if( fp != nullptr )
628 {
629 char uuid[36+1] = { 0 };
630 VSIFReadL( uuid, 1, sizeof(uuid)-1, fp );
631 VSIFCloseL(fp);
632 return EQUALN( uuid, "ec2", 3 );
633 }
634
635 // Check for Nitro Hypervisor instances
636 // This file may exist on Xen instances with a value of 'Xen'
637 // (but that doesn't mean we're on EC2)
638 fp = VSIFOpenL("/sys/devices/virtual/dmi/id/sys_vendor", "rb");
639 if( fp != nullptr )
640 {
641 char buf[10+1] = { 0 };
642 VSIFReadL( buf, 1, sizeof(buf)-1, fp );
643 VSIFCloseL(fp);
644 return EQUALN( buf, "Amazon EC2", 10 );
645 }
646
647 // Fallback: Check via the network
648 return true;
649 };
650 #endif
651
652 #ifdef __linux
653 // Optimization on Linux to avoid the network request
654 // See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html
655 // Skip if either:
656 // - CPL_AWS_AUTODETECT_EC2=NO
657 // - CPL_AWS_CHECK_HYPERVISOR_UUID=NO (deprecated)
658
659 if( ! CPLTestBool(CPLGetConfigOption("CPL_AWS_AUTODETECT_EC2", "YES")) )
660 {
661 return true;
662 }
663 else
664 {
665 CPLString opt = CPLGetConfigOption("CPL_AWS_CHECK_HYPERVISOR_UUID", "");
666 if ( ! opt.empty() )
667 {
668 CPLDebug("AWS", "CPL_AWS_CHECK_HYPERVISOR_UUID is deprecated. Use CPL_AWS_AUTODETECT_EC2 instead");
669 if ( ! CPLTestBool(opt) )
670 {
671 return true;
672 }
673 }
674 }
675
676 return IsMachinePotentiallyEC2InstanceFromLinuxHost();
677
678 #elif defined(WIN32)
679 if( ! CPLTestBool(CPLGetConfigOption("CPL_AWS_AUTODETECT_EC2", "YES")) )
680 {
681 return true;
682 }
683
684 // Regular UUID is not valid for WINE, fetch from sysfs instead.
685 if( CPLGetWineVersion() != nullptr )
686 {
687 return IsMachinePotentiallyEC2InstanceFromLinuxHost();
688 }
689 else
690 {
691 #if defined(_MSC_VER)
692 CPLString osMachineUUID;
693 if( CPLFetchWindowsProductUUID(osMachineUUID) )
694 {
695 if( osMachineUUID.length() >= 3 && EQUALN(osMachineUUID.c_str(), "EC2", 3) )
696 {
697 return true;
698 }
699 else if( osMachineUUID.length() >= 8 && osMachineUUID[4] == '2' &&
700 osMachineUUID[6] == 'E' && osMachineUUID[7] == 'C' )
701 {
702 return true;
703 }
704 else
705 {
706 return false;
707 }
708 }
709 #endif
710 }
711
712 // Fallback: Check via the network
713 return true;
714 #else
715 // At time of writing EC2 instances can be only Linux or Windows
716 return false;
717 #endif
718 }
719
720 /************************************************************************/
721 /* GetConfigurationFromEC2() */
722 /************************************************************************/
723
GetConfigurationFromEC2(CPLString & osSecretAccessKey,CPLString & osAccessKeyId,CPLString & osSessionToken)724 bool VSIS3HandleHelper::GetConfigurationFromEC2(CPLString& osSecretAccessKey,
725 CPLString& osAccessKeyId,
726 CPLString& osSessionToken)
727 {
728 CPLMutexHolder oHolder( &ghMutex );
729 time_t nCurTime;
730 time(&nCurTime);
731 // Try to reuse credentials if they are still valid, but
732 // keep one minute of margin...
733 if( !gosGlobalAccessKeyId.empty() && nCurTime < gnGlobalExpiration - 60 )
734 {
735 osAccessKeyId = gosGlobalAccessKeyId;
736 osSecretAccessKey = gosGlobalSecretAccessKey;
737 osSessionToken = gosGlobalSessionToken;
738 return true;
739 }
740
741 CPLString osURLRefreshCredentials;
742 const CPLString osEC2DefaultURL("http://169.254.169.254");
743 // coverity[tainted_data]
744 const CPLString osEC2RootURL(
745 CPLGetConfigOption("CPL_AWS_EC2_API_ROOT_URL", osEC2DefaultURL));
746 // coverity[tainted_data]
747 const CPLString osECSRelativeURI(
748 CPLGetConfigOption("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", ""));
749 CPLString osToken;
750 if( osEC2RootURL == osEC2DefaultURL && !osECSRelativeURI.empty() )
751 {
752 // See https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html
753 osURLRefreshCredentials = "http://169.254.170.2" + osECSRelativeURI;
754 }
755 else
756 {
757 if( !IsMachinePotentiallyEC2Instance() )
758 return false;
759
760 // Use IMDSv2 protocol:
761 // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html
762
763 // Retrieve IMDSv2 token
764 {
765 const CPLString osEC2_IMDSv2_api_token_URL =
766 osEC2RootURL + "/latest/api/token";
767 CPLStringList aosOptions;
768 aosOptions.SetNameValue("TIMEOUT", "1");
769 aosOptions.SetNameValue("CUSTOMREQUEST", "PUT");
770 aosOptions.SetNameValue("HEADERS",
771 "X-aws-ec2-metadata-token-ttl-seconds: 10");
772 CPLPushErrorHandler(CPLQuietErrorHandler);
773 CPLHTTPResult* psResult =
774 CPLHTTPFetch( osEC2_IMDSv2_api_token_URL, aosOptions.List() );
775 CPLPopErrorHandler();
776 if( psResult )
777 {
778 if( psResult->nStatus == 0 && psResult->pabyData != nullptr )
779 {
780 osToken = reinterpret_cast<char*>(psResult->pabyData);
781 }
782 else
783 {
784 // Failure: either we are not running on EC2 (or something emulating it)
785 // or this doesn't implement yet IDMSv2
786 // Go on trying IDMSv1
787 }
788 CPLHTTPDestroyResult(psResult);
789 }
790 }
791
792 // If we don't know yet the IAM role, fetch it
793 const CPLString osEC2CredentialsURL =
794 osEC2RootURL + "/latest/meta-data/iam/security-credentials/";
795 if( gosIAMRole.empty() )
796 {
797 CPLStringList aosOptions;
798 aosOptions.SetNameValue("TIMEOUT", "1");
799 if( !osToken.empty() )
800 {
801 aosOptions.SetNameValue("HEADERS",
802 ("X-aws-ec2-metadata-token: " + osToken).c_str());
803 }
804 CPLPushErrorHandler(CPLQuietErrorHandler);
805 CPLHTTPResult* psResult =
806 CPLHTTPFetch( osEC2CredentialsURL, aosOptions.List() );
807 CPLPopErrorHandler();
808 if( psResult )
809 {
810 if( psResult->nStatus == 0 && psResult->pabyData != nullptr )
811 {
812 gosIAMRole = reinterpret_cast<char*>(psResult->pabyData);
813 }
814 CPLHTTPDestroyResult(psResult);
815 }
816 if( gosIAMRole.empty() )
817 {
818 // We didn't get the IAM role. We are definitely not running
819 // on EC2 or an emulation of it.
820 return false;
821 }
822 }
823 osURLRefreshCredentials = osEC2CredentialsURL + gosIAMRole;
824 }
825
826 // Now fetch the refreshed credentials
827 CPLStringList oResponse;
828 CPLStringList aosOptions;
829 if( !osToken.empty() )
830 {
831 aosOptions.SetNameValue("HEADERS",
832 ("X-aws-ec2-metadata-token: " + osToken).c_str());
833 }
834 CPLHTTPResult* psResult = CPLHTTPFetch(osURLRefreshCredentials.c_str(), aosOptions.List() );
835 if( psResult )
836 {
837 if( psResult->nStatus == 0 && psResult->pabyData != nullptr )
838 {
839 const CPLString osJSon =
840 reinterpret_cast<char*>(psResult->pabyData);
841 oResponse = ParseSimpleJson(osJSon);
842 }
843 CPLHTTPDestroyResult(psResult);
844 }
845 osAccessKeyId = oResponse.FetchNameValueDef("AccessKeyId", "");
846 osSecretAccessKey =
847 oResponse.FetchNameValueDef("SecretAccessKey", "");
848 osSessionToken = oResponse.FetchNameValueDef("Token", "");
849 const CPLString osExpiration =
850 oResponse.FetchNameValueDef("Expiration", "");
851 GIntBig nExpirationUnix = 0;
852 if( !osAccessKeyId.empty() &&
853 !osSecretAccessKey.empty() &&
854 Iso8601ToUnixTime(osExpiration, &nExpirationUnix) )
855 {
856 gosGlobalAccessKeyId = osAccessKeyId;
857 gosGlobalSecretAccessKey = osSecretAccessKey;
858 gosGlobalSessionToken = osSessionToken;
859 gnGlobalExpiration = nExpirationUnix;
860 CPLDebug("AWS", "Storing AIM credentials until %s",
861 osExpiration.c_str());
862 }
863 return !osAccessKeyId.empty() && !osSecretAccessKey.empty();
864 }
865
866
867 /************************************************************************/
868 /* UpdateAndWarnIfInconsistent() */
869 /************************************************************************/
870
871 static
UpdateAndWarnIfInconsistent(const char * pszKeyword,CPLString & osVal,const CPLString & osNewVal,const CPLString & osCredentials,const CPLString & osConfig)872 void UpdateAndWarnIfInconsistent(const char* pszKeyword,
873 CPLString& osVal,
874 const CPLString& osNewVal,
875 const CPLString& osCredentials,
876 const CPLString& osConfig)
877 {
878 // nominally defined in ~/.aws/credentials but can
879 // be set here too. If both values exist, credentials
880 // has the priority
881 if( osVal.empty() )
882 {
883 osVal = osNewVal;
884 }
885 else if( osVal != osNewVal )
886 {
887 CPLError(CE_Warning, CPLE_AppDefined,
888 "%s defined in both %s "
889 "and %s. The one of %s will be used",
890 pszKeyword,
891 osCredentials.c_str(),
892 osConfig.c_str(),
893 osCredentials.c_str());
894 }
895 }
896
897 /************************************************************************/
898 /* GetConfigurationFromAWSConfigFiles() */
899 /************************************************************************/
900
GetConfigurationFromAWSConfigFiles(CPLString & osSecretAccessKey,CPLString & osAccessKeyId,CPLString & osSessionToken,CPLString & osRegion,CPLString & osCredentials)901 bool VSIS3HandleHelper::GetConfigurationFromAWSConfigFiles(
902 CPLString& osSecretAccessKey,
903 CPLString& osAccessKeyId,
904 CPLString& osSessionToken,
905 CPLString& osRegion,
906 CPLString& osCredentials)
907 {
908 // See http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html
909 // If AWS_DEFAULT_PROFILE is set (obsolete, no longer documented), use it in priority
910 // Otherwise use AWS_PROFILE
911 // Otherwise fallback to "default"
912 const char* pszProfile = CPLGetConfigOption("AWS_DEFAULT_PROFILE",
913 CPLGetConfigOption("AWS_PROFILE", ""));
914 const CPLString osProfile(pszProfile[0] != '\0' ? pszProfile : "default");
915
916 #ifdef WIN32
917 const char* pszHome = CPLGetConfigOption("USERPROFILE", nullptr);
918 constexpr char SEP_STRING[] = "\\";
919 #else
920 const char* pszHome = CPLGetConfigOption("HOME", nullptr);
921 constexpr char SEP_STRING[] = "/";
922 #endif
923
924 CPLString osDotAws( pszHome ? pszHome : "" );
925 osDotAws += SEP_STRING;
926 osDotAws += ".aws";
927
928 // Read first ~/.aws/credential file
929
930 // GDAL specific config option (mostly for testing purpose, but also
931 // used in production in some cases)
932 const char* pszCredentials =
933 CPLGetConfigOption( "CPL_AWS_CREDENTIALS_FILE", nullptr );
934 if( pszCredentials )
935 {
936 osCredentials = pszCredentials;
937 }
938 else
939 {
940 osCredentials = osDotAws;
941 osCredentials += SEP_STRING;
942 osCredentials += "credentials";
943 }
944 VSILFILE* fp = VSIFOpenL( osCredentials, "rb" );
945 if( fp != nullptr )
946 {
947 const char* pszLine;
948 bool bInProfile = false;
949 const CPLString osBracketedProfile("[" + osProfile + "]");
950 while( (pszLine = CPLReadLineL(fp)) != nullptr )
951 {
952 if( pszLine[0] == '[' )
953 {
954 if( bInProfile )
955 break;
956 if( CPLString(pszLine) == osBracketedProfile )
957 bInProfile = true;
958 }
959 else if( bInProfile )
960 {
961 char* pszKey = nullptr;
962 const char* pszValue = CPLParseNameValue(pszLine, &pszKey);
963 if( pszKey && pszValue )
964 {
965 if( EQUAL(pszKey, "aws_access_key_id") )
966 osAccessKeyId = pszValue;
967 else if( EQUAL(pszKey, "aws_secret_access_key") )
968 osSecretAccessKey = pszValue;
969 else if( EQUAL(pszKey, "aws_session_token") )
970 osSessionToken = pszValue;
971 }
972 CPLFree(pszKey);
973 }
974 }
975 VSIFCloseL(fp);
976 }
977
978 // And then ~/.aws/config file (unless AWS_CONFIG_FILE is defined)
979 const char* pszAWSConfigFileEnv =
980 CPLGetConfigOption( "AWS_CONFIG_FILE", nullptr );
981 CPLString osConfig;
982 if( pszAWSConfigFileEnv )
983 {
984 osConfig = pszAWSConfigFileEnv;
985 }
986 else
987 {
988 osConfig = osDotAws;
989 osConfig += SEP_STRING;
990 osConfig += "config";
991 }
992 fp = VSIFOpenL( osConfig, "rb" );
993 if( fp != nullptr )
994 {
995 const char* pszLine;
996 bool bInProfile = false;
997 const CPLString osBracketedProfile("[" + osProfile + "]");
998 const CPLString osBracketedProfileProfile("[profile " + osProfile + "]");
999 while( (pszLine = CPLReadLineL(fp)) != nullptr )
1000 {
1001 if( pszLine[0] == '[' )
1002 {
1003 if( bInProfile )
1004 break;
1005 // In config file, the section name is nominally [profile foo]
1006 // for the non default profile.
1007 if( CPLString(pszLine) == osBracketedProfile ||
1008 CPLString(pszLine) == osBracketedProfileProfile )
1009 {
1010 bInProfile = true;
1011 }
1012 }
1013 else if( bInProfile )
1014 {
1015 char* pszKey = nullptr;
1016 const char* pszValue = CPLParseNameValue(pszLine, &pszKey);
1017 if( pszKey && pszValue )
1018 {
1019 if( EQUAL(pszKey, "aws_access_key_id") )
1020 {
1021 UpdateAndWarnIfInconsistent(pszKey,
1022 osAccessKeyId,
1023 pszValue,
1024 osCredentials,
1025 osConfig);
1026 }
1027 else if( EQUAL(pszKey, "aws_secret_access_key") )
1028 {
1029 UpdateAndWarnIfInconsistent(pszKey,
1030 osSecretAccessKey,
1031 pszValue,
1032 osCredentials,
1033 osConfig);
1034 }
1035 else if( EQUAL(pszKey, "aws_session_token") )
1036 {
1037 UpdateAndWarnIfInconsistent(pszKey,
1038 osSessionToken,
1039 pszValue,
1040 osCredentials,
1041 osConfig);
1042 }
1043 else if( EQUAL(pszKey, "region") )
1044 {
1045 osRegion = pszValue;
1046 }
1047 }
1048 CPLFree(pszKey);
1049 }
1050 }
1051 VSIFCloseL(fp);
1052 }
1053 else if( pszAWSConfigFileEnv != nullptr )
1054 {
1055 if( pszAWSConfigFileEnv[0] != '\0' )
1056 {
1057 CPLError(CE_Warning, CPLE_AppDefined,
1058 "%s does not exist or cannot be open",
1059 pszAWSConfigFileEnv);
1060 }
1061 }
1062
1063 return !osAccessKeyId.empty() && !osSecretAccessKey.empty();
1064 }
1065
1066 /************************************************************************/
1067 /* GetConfiguration() */
1068 /************************************************************************/
1069
GetConfiguration(CSLConstList papszOptions,CPLString & osSecretAccessKey,CPLString & osAccessKeyId,CPLString & osSessionToken,CPLString & osRegion,bool & bFromEC2)1070 bool VSIS3HandleHelper::GetConfiguration(CSLConstList papszOptions,
1071 CPLString& osSecretAccessKey,
1072 CPLString& osAccessKeyId,
1073 CPLString& osSessionToken,
1074 CPLString& osRegion,
1075 bool& bFromEC2)
1076 {
1077 bFromEC2 = false;
1078
1079 // AWS_REGION is GDAL specific. Later overloaded by standard
1080 // AWS_DEFAULT_REGION
1081 osRegion = CSLFetchNameValueDef(papszOptions, "AWS_REGION",
1082 CPLGetConfigOption("AWS_REGION", "us-east-1"));
1083
1084 if( CPLTestBool(CPLGetConfigOption("AWS_NO_SIGN_REQUEST", "NO")) )
1085 {
1086 osSecretAccessKey.clear();
1087 osAccessKeyId.clear();
1088 osSessionToken.clear();
1089 return true;
1090 }
1091
1092 osSecretAccessKey = CSLFetchNameValueDef(papszOptions,
1093 "AWS_SECRET_ACCESS_KEY",
1094 CPLGetConfigOption("AWS_SECRET_ACCESS_KEY", ""));
1095 if( !osSecretAccessKey.empty() )
1096 {
1097 osAccessKeyId = CPLGetConfigOption("AWS_ACCESS_KEY_ID", "");
1098 if( osAccessKeyId.empty() )
1099 {
1100 VSIError(VSIE_AWSInvalidCredentials,
1101 "AWS_ACCESS_KEY_ID configuration option not defined");
1102 return false;
1103 }
1104
1105 osSessionToken = CSLFetchNameValueDef(papszOptions,
1106 "AWS_SESSION_TOKEN",
1107 CPLGetConfigOption("AWS_SESSION_TOKEN", ""));
1108 return true;
1109 }
1110
1111 // Next try reading from ~/.aws/credentials and ~/.aws/config
1112 CPLString osCredentials;
1113 // coverity[tainted_data]
1114 if( GetConfigurationFromAWSConfigFiles(osSecretAccessKey, osAccessKeyId,
1115 osSessionToken, osRegion,
1116 osCredentials) )
1117 {
1118 return true;
1119 }
1120
1121 // Last method: use IAM role security credentials on EC2 instances
1122 if( GetConfigurationFromEC2(osSecretAccessKey, osAccessKeyId,
1123 osSessionToken) )
1124 {
1125 bFromEC2 = true;
1126 return true;
1127 }
1128
1129 VSIError(VSIE_AWSInvalidCredentials,
1130 "AWS_SECRET_ACCESS_KEY and AWS_NO_SIGN_REQUEST configuration "
1131 "options not defined, and %s not filled",
1132 osCredentials.c_str());
1133 return false;
1134 }
1135
1136 /************************************************************************/
1137 /* CleanMutex() */
1138 /************************************************************************/
1139
CleanMutex()1140 void VSIS3HandleHelper::CleanMutex()
1141 {
1142 if( ghMutex != nullptr )
1143 CPLDestroyMutex( ghMutex );
1144 ghMutex = nullptr;
1145 }
1146
1147 /************************************************************************/
1148 /* ClearCache() */
1149 /************************************************************************/
1150
ClearCache()1151 void VSIS3HandleHelper::ClearCache()
1152 {
1153 CPLMutexHolder oHolder( &ghMutex );
1154
1155 gosIAMRole.clear();
1156 gosGlobalAccessKeyId.clear();
1157 gosGlobalSecretAccessKey.clear();
1158 gosGlobalSessionToken.clear();
1159 gnGlobalExpiration = 0;
1160 }
1161
1162 /************************************************************************/
1163 /* BuildFromURI() */
1164 /************************************************************************/
1165
BuildFromURI(const char * pszURI,const char * pszFSPrefix,bool bAllowNoObject,CSLConstList papszOptions)1166 VSIS3HandleHelper* VSIS3HandleHelper::BuildFromURI( const char* pszURI,
1167 const char* pszFSPrefix,
1168 bool bAllowNoObject,
1169 CSLConstList papszOptions )
1170 {
1171 CPLString osSecretAccessKey;
1172 CPLString osAccessKeyId;
1173 CPLString osSessionToken;
1174 CPLString osRegion;
1175 bool bFromEC2 = false;
1176 if( !GetConfiguration(papszOptions,
1177 osSecretAccessKey, osAccessKeyId,
1178 osSessionToken, osRegion, bFromEC2) )
1179 {
1180 return nullptr;
1181 }
1182
1183 // According to http://docs.aws.amazon.com/cli/latest/userguide/cli-environment.html
1184 // " This variable overrides the default region of the in-use profile, if set."
1185 const CPLString osDefaultRegion = CSLFetchNameValueDef(
1186 papszOptions, "AWS_DEFAULT_REGION",
1187 CPLGetConfigOption("AWS_DEFAULT_REGION", ""));
1188 if( !osDefaultRegion.empty() )
1189 {
1190 osRegion = osDefaultRegion;
1191 }
1192
1193 const CPLString osEndpoint =
1194 CPLGetConfigOption("AWS_S3_ENDPOINT", "s3.amazonaws.com");
1195 const CPLString osRequestPayer =
1196 CPLGetConfigOption("AWS_REQUEST_PAYER", "");
1197 CPLString osBucket;
1198 CPLString osObjectKey;
1199 if( pszURI != nullptr && pszURI[0] != '\0' &&
1200 !GetBucketAndObjectKey(pszURI, pszFSPrefix, bAllowNoObject,
1201 osBucket, osObjectKey) )
1202 {
1203 return nullptr;
1204 }
1205 const bool bUseHTTPS = CPLTestBool(CPLGetConfigOption("AWS_HTTPS", "YES"));
1206 const bool bIsValidNameForVirtualHosting =
1207 osBucket.find('.') == std::string::npos;
1208 const bool bUseVirtualHosting = CPLTestBool(
1209 CSLFetchNameValueDef(papszOptions, "AWS_VIRTUAL_HOSTING",
1210 CPLGetConfigOption("AWS_VIRTUAL_HOSTING",
1211 bIsValidNameForVirtualHosting ? "TRUE" : "FALSE")));
1212 return new VSIS3HandleHelper(osSecretAccessKey, osAccessKeyId,
1213 osSessionToken,
1214 osEndpoint, osRegion,
1215 osRequestPayer,
1216 osBucket, osObjectKey, bUseHTTPS,
1217 bUseVirtualHosting, bFromEC2);
1218 }
1219
1220 /************************************************************************/
1221 /* GetQueryString() */
1222 /************************************************************************/
1223
GetQueryString(bool bAddEmptyValueAfterEqual) const1224 CPLString IVSIS3LikeHandleHelper::GetQueryString(bool bAddEmptyValueAfterEqual) const
1225 {
1226 CPLString osQueryString;
1227 std::map<CPLString, CPLString>::const_iterator oIter =
1228 m_oMapQueryParameters.begin();
1229 for( ; oIter != m_oMapQueryParameters.end(); ++oIter )
1230 {
1231 if( oIter == m_oMapQueryParameters.begin() )
1232 osQueryString += "?";
1233 else
1234 osQueryString += "&";
1235 osQueryString += oIter->first;
1236 if( !oIter->second.empty() || bAddEmptyValueAfterEqual )
1237 {
1238 osQueryString += "=";
1239 osQueryString += CPLAWSURLEncode(oIter->second);
1240 }
1241 }
1242 return osQueryString;
1243 }
1244
1245 /************************************************************************/
1246 /* ResetQueryParameters() */
1247 /************************************************************************/
1248
ResetQueryParameters()1249 void IVSIS3LikeHandleHelper::ResetQueryParameters()
1250 {
1251 m_oMapQueryParameters.clear();
1252 RebuildURL();
1253 }
1254
1255 /************************************************************************/
1256 /* AddQueryParameter() */
1257 /************************************************************************/
1258
AddQueryParameter(const CPLString & osKey,const CPLString & osValue)1259 void IVSIS3LikeHandleHelper::AddQueryParameter( const CPLString& osKey,
1260 const CPLString& osValue )
1261 {
1262 m_oMapQueryParameters[osKey] = osValue;
1263 RebuildURL();
1264 }
1265
1266 /************************************************************************/
1267 /* GetURLNoKVP() */
1268 /************************************************************************/
1269
GetURLNoKVP() const1270 CPLString IVSIS3LikeHandleHelper::GetURLNoKVP() const
1271 {
1272 CPLString osURL(GetURL());
1273 const auto nPos = osURL.find('?');
1274 if( nPos != std::string::npos )
1275 osURL.resize(nPos);
1276 return osURL;
1277 }
1278
1279 /************************************************************************/
1280 /* GetCurlHeaders() */
1281 /************************************************************************/
1282
1283 struct curl_slist *
GetCurlHeaders(const CPLString & osVerb,const struct curl_slist * psExistingHeaders,const void * pabyDataContent,size_t nBytesContent) const1284 VSIS3HandleHelper::GetCurlHeaders( const CPLString& osVerb,
1285 const struct curl_slist* psExistingHeaders,
1286 const void *pabyDataContent,
1287 size_t nBytesContent ) const
1288 {
1289 if( m_bFromEC2 )
1290 {
1291 CPLString osSecretAccessKey, osAccessKeyId, osSessionToken;
1292 if( GetConfigurationFromEC2(osSecretAccessKey,
1293 osAccessKeyId,
1294 osSessionToken) )
1295 {
1296 m_osSecretAccessKey = osSecretAccessKey;
1297 m_osAccessKeyId = osAccessKeyId;
1298 m_osSessionToken = osSessionToken;
1299 }
1300 }
1301
1302 CPLString osXAMZDate = CPLGetConfigOption("AWS_TIMESTAMP", "");
1303 if( osXAMZDate.empty() )
1304 osXAMZDate = CPLGetAWS_SIGN4_Timestamp();
1305
1306 const CPLString osXAMZContentSHA256 =
1307 CPLGetLowerCaseHexSHA256(pabyDataContent, nBytesContent);
1308
1309 CPLString osCanonicalQueryString(GetQueryString(true));
1310 if( !osCanonicalQueryString.empty() )
1311 osCanonicalQueryString = osCanonicalQueryString.substr(1);
1312
1313 const CPLString osHost(m_bUseVirtualHosting && !m_osBucket.empty()
1314 ? CPLString(m_osBucket + "." + m_osEndpoint) : m_osEndpoint);
1315 const CPLString osAuthorization = m_osSecretAccessKey.empty() ? CPLString():
1316 CPLGetAWS_SIGN4_Authorization(
1317 m_osSecretAccessKey,
1318 m_osAccessKeyId,
1319 m_osSessionToken,
1320 m_osRegion,
1321 m_osRequestPayer,
1322 "s3",
1323 osVerb,
1324 psExistingHeaders,
1325 osHost,
1326 m_bUseVirtualHosting
1327 ? CPLAWSURLEncode("/" + m_osObjectKey, false).c_str() :
1328 CPLAWSURLEncode("/" + m_osBucket + "/" + m_osObjectKey, false).c_str(),
1329 osCanonicalQueryString,
1330 osXAMZContentSHA256,
1331 osXAMZDate);
1332
1333 struct curl_slist *headers=nullptr;
1334 headers = curl_slist_append(
1335 headers, CPLSPrintf("x-amz-date: %s", osXAMZDate.c_str()));
1336 headers = curl_slist_append(
1337 headers, CPLSPrintf("x-amz-content-sha256: %s",
1338 osXAMZContentSHA256.c_str()));
1339 if( !m_osSessionToken.empty() )
1340 headers = curl_slist_append(
1341 headers,
1342 CPLSPrintf("X-Amz-Security-Token: %s", m_osSessionToken.c_str()));
1343 if( !m_osRequestPayer.empty() )
1344 headers = curl_slist_append(
1345 headers,
1346 CPLSPrintf("x-amz-request-payer: %s", m_osRequestPayer.c_str()));
1347 if( !osAuthorization.empty() )
1348 {
1349 headers = curl_slist_append(
1350 headers, CPLSPrintf("Authorization: %s", osAuthorization.c_str()));
1351 }
1352 return headers;
1353 }
1354
1355 /************************************************************************/
1356 /* CanRestartOnError() */
1357 /************************************************************************/
1358
CanRestartOnError(const char * pszErrorMsg,const char * pszHeaders,bool bSetError,bool * pbUpdateMap)1359 bool VSIS3HandleHelper::CanRestartOnError( const char* pszErrorMsg,
1360 const char* pszHeaders,
1361 bool bSetError, bool* pbUpdateMap )
1362 {
1363 #ifdef DEBUG_VERBOSE
1364 CPLDebug("S3", "%s", pszErrorMsg);
1365 CPLDebug("S3", "%s", pszHeaders ? pszHeaders : "");
1366 #endif
1367
1368 if( pbUpdateMap != nullptr )
1369 *pbUpdateMap = true;
1370
1371 if( !STARTS_WITH(pszErrorMsg, "<?xml") &&
1372 !STARTS_WITH(pszErrorMsg, "<Error>") )
1373 {
1374 if( bSetError )
1375 {
1376 VSIError(VSIE_AWSError, "Invalid AWS response: %s", pszErrorMsg);
1377 }
1378 return false;
1379 }
1380
1381 CPLXMLNode* psTree = CPLParseXMLString(pszErrorMsg);
1382 if( psTree == nullptr )
1383 {
1384 if( bSetError )
1385 {
1386 VSIError(VSIE_AWSError,
1387 "Malformed AWS XML response: %s", pszErrorMsg);
1388 }
1389 return false;
1390 }
1391
1392 const char* pszCode = CPLGetXMLValue(psTree, "=Error.Code", nullptr);
1393 if( pszCode == nullptr )
1394 {
1395 CPLDestroyXMLNode(psTree);
1396 if( bSetError )
1397 {
1398 VSIError(VSIE_AWSError,
1399 "Malformed AWS XML response: %s", pszErrorMsg);
1400 }
1401 return false;
1402 }
1403
1404 if( EQUAL(pszCode, "AuthorizationHeaderMalformed") )
1405 {
1406 const char* pszRegion = CPLGetXMLValue(psTree, "=Error.Region", nullptr);
1407 if( pszRegion == nullptr )
1408 {
1409 CPLDestroyXMLNode(psTree);
1410 if( bSetError )
1411 {
1412 VSIError(VSIE_AWSError,
1413 "Malformed AWS XML response: %s", pszErrorMsg);
1414 }
1415 return false;
1416 }
1417 SetRegion(pszRegion);
1418 CPLDebug("S3", "Switching to region %s", m_osRegion.c_str());
1419 CPLDestroyXMLNode(psTree);
1420 return true;
1421 }
1422
1423 if( EQUAL(pszCode, "PermanentRedirect") || EQUAL(pszCode, "TemporaryRedirect") )
1424 {
1425 const bool bIsTemporaryRedirect = EQUAL(pszCode, "TemporaryRedirect");
1426 const char* pszEndpoint =
1427 CPLGetXMLValue(psTree, "=Error.Endpoint", nullptr);
1428 if( pszEndpoint == nullptr ||
1429 (m_bUseVirtualHosting &&
1430 (strncmp(pszEndpoint, m_osBucket.c_str(),
1431 m_osBucket.size()) != 0 ||
1432 pszEndpoint[m_osBucket.size()] != '.')) )
1433 {
1434 CPLDestroyXMLNode(psTree);
1435 if( bSetError )
1436 {
1437 VSIError(VSIE_AWSError,
1438 "Malformed AWS XML response: %s", pszErrorMsg);
1439 }
1440 return false;
1441 }
1442 if( !m_bUseVirtualHosting &&
1443 strncmp(pszEndpoint, m_osBucket.c_str(), m_osBucket.size()) == 0 &&
1444 pszEndpoint[m_osBucket.size()] == '.' )
1445 {
1446 /* If we have a body with
1447 <Error><Code>PermanentRedirect</Code><Message>The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.</Message><Bucket>bucket.with.dot</Bucket><Endpoint>bucket.with.dot.s3.amazonaws.com</Endpoint></Error>
1448 and headers like
1449 x-amz-bucket-region: eu-west-1
1450 and the bucket name has dot in it,
1451 then we must use s3.$(x-amz-bucket-region).amazon.com as endpoint.
1452 See #7154 */
1453 const char* pszRegionPtr = (pszHeaders != nullptr) ?
1454 strstr(pszHeaders, "x-amz-bucket-region: "): nullptr;
1455 if( strchr(m_osBucket.c_str(), '.') != nullptr && pszRegionPtr != nullptr )
1456 {
1457 CPLString osRegion(pszRegionPtr + strlen("x-amz-bucket-region: "));
1458 size_t nPos = osRegion.find('\r');
1459 if( nPos != std::string::npos )
1460 osRegion.resize(nPos);
1461 SetEndpoint( CPLSPrintf("s3.%s.amazonaws.com", osRegion.c_str()) );
1462 SetRegion(osRegion.c_str());
1463 CPLDebug("S3", "Switching to endpoint %s", m_osEndpoint.c_str());
1464 CPLDebug("S3", "Switching to region %s", m_osRegion.c_str());
1465 CPLDestroyXMLNode(psTree);
1466 if( bIsTemporaryRedirect && pbUpdateMap != nullptr)
1467 *pbUpdateMap = false;
1468 return true;
1469 }
1470
1471 m_bUseVirtualHosting = true;
1472 CPLDebug("S3", "Switching to virtual hosting");
1473 }
1474 SetEndpoint(
1475 m_bUseVirtualHosting
1476 ? pszEndpoint + m_osBucket.size() + 1
1477 : pszEndpoint);
1478 CPLDebug("S3", "Switching to endpoint %s", m_osEndpoint.c_str());
1479 CPLDestroyXMLNode(psTree);
1480
1481 if( bIsTemporaryRedirect && pbUpdateMap != nullptr)
1482 *pbUpdateMap = false;
1483
1484 return true;
1485 }
1486
1487 if( bSetError )
1488 {
1489 // Translate AWS errors into VSI errors.
1490 const char* pszMessage = CPLGetXMLValue(psTree, "=Error.Message", nullptr);
1491
1492 if( pszMessage == nullptr ) {
1493 VSIError(VSIE_AWSError, "%s", pszErrorMsg);
1494 } else if( EQUAL(pszCode, "AccessDenied") ) {
1495 VSIError(VSIE_AWSAccessDenied, "%s", pszMessage);
1496 } else if( EQUAL(pszCode, "NoSuchBucket") ) {
1497 VSIError(VSIE_AWSBucketNotFound, "%s", pszMessage);
1498 } else if( EQUAL(pszCode, "NoSuchKey") ) {
1499 VSIError(VSIE_AWSObjectNotFound, "%s", pszMessage);
1500 } else if( EQUAL(pszCode, "SignatureDoesNotMatch") ) {
1501 VSIError(VSIE_AWSSignatureDoesNotMatch, "%s", pszMessage);
1502 } else {
1503 VSIError(VSIE_AWSError, "%s", pszMessage);
1504 }
1505 }
1506
1507 CPLDestroyXMLNode(psTree);
1508
1509 return false;
1510 }
1511
1512 /************************************************************************/
1513 /* SetEndpoint() */
1514 /************************************************************************/
1515
SetEndpoint(const CPLString & osStr)1516 void VSIS3HandleHelper::SetEndpoint( const CPLString &osStr )
1517 {
1518 m_osEndpoint = osStr;
1519 RebuildURL();
1520 }
1521
1522 /************************************************************************/
1523 /* SetRegion() */
1524 /************************************************************************/
1525
SetRegion(const CPLString & osStr)1526 void VSIS3HandleHelper::SetRegion( const CPLString &osStr )
1527 {
1528 m_osRegion = osStr;
1529 }
1530
1531 /************************************************************************/
1532 /* SetRequestPayer() */
1533 /************************************************************************/
1534
SetRequestPayer(const CPLString & osStr)1535 void VSIS3HandleHelper::SetRequestPayer( const CPLString &osStr )
1536 {
1537 m_osRequestPayer = osStr;
1538 }
1539
1540 /************************************************************************/
1541 /* SetVirtualHosting() */
1542 /************************************************************************/
1543
SetVirtualHosting(bool b)1544 void VSIS3HandleHelper::SetVirtualHosting( bool b )
1545 {
1546 m_bUseVirtualHosting = b;
1547 RebuildURL();
1548 }
1549
1550 /************************************************************************/
1551 /* GetSignedURL() */
1552 /************************************************************************/
1553
GetSignedURL(CSLConstList papszOptions)1554 CPLString VSIS3HandleHelper::GetSignedURL(CSLConstList papszOptions)
1555 {
1556 CPLString osXAMZDate = CSLFetchNameValueDef(papszOptions, "START_DATE",
1557 CPLGetConfigOption("AWS_TIMESTAMP", ""));
1558 if( osXAMZDate.empty() )
1559 osXAMZDate = CPLGetAWS_SIGN4_Timestamp();
1560 CPLString osDate(osXAMZDate);
1561 osDate.resize(8);
1562
1563 CPLString osXAMZExpires =
1564 CSLFetchNameValueDef(papszOptions, "EXPIRATION_DELAY", "3600");
1565
1566 CPLString osVerb(CSLFetchNameValueDef(papszOptions, "VERB", "GET"));
1567
1568 ResetQueryParameters();
1569 AddQueryParameter("X-Amz-Algorithm", "AWS4-HMAC-SHA256");
1570 AddQueryParameter("X-Amz-Credential",
1571 m_osAccessKeyId + "/" + osDate + "/" + m_osRegion + "/s3/aws4_request");
1572 AddQueryParameter("X-Amz-Date", osXAMZDate);
1573 AddQueryParameter("X-Amz-Expires", osXAMZExpires);
1574 AddQueryParameter("X-Amz-SignedHeaders", "host");
1575
1576 CPLString osCanonicalQueryString(GetQueryString(true).substr(1));
1577
1578 const CPLString osHost(m_bUseVirtualHosting && !m_osBucket.empty()
1579 ? CPLString(m_osBucket + "." + m_osEndpoint) : m_osEndpoint);
1580 CPLString osSignedHeaders;
1581 const CPLString osSignature =
1582 CPLGetAWS_SIGN4_Signature(
1583 m_osSecretAccessKey,
1584 m_osSessionToken,
1585 m_osRegion,
1586 m_osRequestPayer,
1587 "s3",
1588 osVerb,
1589 nullptr, /* existing headers */
1590 osHost,
1591 m_bUseVirtualHosting
1592 ? CPLAWSURLEncode("/" + m_osObjectKey, false).c_str() :
1593 CPLAWSURLEncode("/" + m_osBucket + "/" + m_osObjectKey, false).c_str(),
1594 osCanonicalQueryString,
1595 "UNSIGNED-PAYLOAD",
1596 osXAMZDate,
1597 osSignedHeaders);
1598
1599 AddQueryParameter("X-Amz-Signature", osSignature);
1600 return m_osURL;
1601 }
1602
1603 /************************************************************************/
1604 /* UpdateMapFromHandle() */
1605 /************************************************************************/
1606
1607 std::mutex VSIS3UpdateParams::gsMutex{};
1608 std::map< CPLString, VSIS3UpdateParams > VSIS3UpdateParams::goMapBucketsToS3Params{};
1609
UpdateMapFromHandle(IVSIS3LikeHandleHelper * poHandleHelper)1610 void VSIS3UpdateParams::UpdateMapFromHandle( IVSIS3LikeHandleHelper* poHandleHelper )
1611 {
1612 std::lock_guard<std::mutex> guard(gsMutex);
1613
1614 VSIS3HandleHelper * poS3HandleHelper =
1615 cpl::down_cast<VSIS3HandleHelper *>(poHandleHelper);
1616 goMapBucketsToS3Params[ poS3HandleHelper->GetBucket() ] =
1617 VSIS3UpdateParams ( poS3HandleHelper );
1618 }
1619
1620 /************************************************************************/
1621 /* UpdateHandleFromMap() */
1622 /************************************************************************/
1623
UpdateHandleFromMap(IVSIS3LikeHandleHelper * poHandleHelper)1624 void VSIS3UpdateParams::UpdateHandleFromMap( IVSIS3LikeHandleHelper* poHandleHelper )
1625 {
1626 std::lock_guard<std::mutex> guard(gsMutex);
1627
1628 VSIS3HandleHelper * poS3HandleHelper =
1629 cpl::down_cast<VSIS3HandleHelper *>(poHandleHelper);
1630 std::map< CPLString, VSIS3UpdateParams>::iterator oIter =
1631 goMapBucketsToS3Params.find(poS3HandleHelper->GetBucket());
1632 if( oIter != goMapBucketsToS3Params.end() )
1633 {
1634 oIter->second.UpdateHandlerHelper(poS3HandleHelper);
1635 }
1636 }
1637
1638 /************************************************************************/
1639 /* ClearCache() */
1640 /************************************************************************/
1641
ClearCache()1642 void VSIS3UpdateParams::ClearCache()
1643 {
1644 std::lock_guard<std::mutex> guard(gsMutex);
1645
1646 goMapBucketsToS3Params.clear();
1647 }
1648
1649 #endif
1650
1651 //! @endcond
1652