1 /**********************************************************************
2  *
3  * Name:     cpl_aws.cpp
4  * Project:  CPL - Common Portability Library
5  * Purpose:  Amazon Web Services routines
6  * Author:   Even Rouault <even.rouault at spatialys.com>
7  *
8  **********************************************************************
9  * Copyright (c) 2015, Even Rouault <even.rouault at spatialys.com>
10  *
11  * Permission is hereby granted, free of charge, to any person obtaining a
12  * copy of this software and associated documentation files (the "Software"),
13  * to deal in the Software without restriction, including without limitation
14  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15  * and/or sell copies of the Software, and to permit persons to whom the
16  * Software is furnished to do so, subject to the following conditions:
17  *
18  * The above copyright notice and this permission notice shall be included
19  * in all copies or substantial portions of the Software.
20  *
21  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27  * DEALINGS IN THE SOFTWARE.
28  ****************************************************************************/
29 
30 //! @cond Doxygen_Suppress
31 
32 #include "cpl_aws.h"
33 #include "cpl_vsi_error.h"
34 #include "cpl_sha256.h"
35 #include "cpl_time.h"
36 #include "cpl_minixml.h"
37 #include "cpl_multiproc.h"
38 #include "cpl_http.h"
39 #include <algorithm>
40 
41 CPL_CVSID("$Id: cpl_aws.cpp 86933038c3926cd4dc3ff37c431b317abb69e602 2021-03-27 23:20:49 +0100 Even Rouault $")
42 
43 // #define DEBUG_VERBOSE 1
44 
45 #ifdef WIN32
46 #if defined(_MSC_VER)
47 bool CPLFetchWindowsProductUUID(CPLString &osStr); // defined in cpl_aws_win32.cpp
48 #endif
49 const char* CPLGetWineVersion(); // defined in cpl_vsil_win32.cpp
50 #endif
51 
52 #ifdef HAVE_CURL
53 static CPLMutex *ghMutex = nullptr;
54 static CPLString gosIAMRole;
55 static CPLString gosGlobalAccessKeyId;
56 static CPLString gosGlobalSecretAccessKey;
57 static CPLString gosGlobalSessionToken;
58 static GIntBig gnGlobalExpiration = 0;
59 
60 /************************************************************************/
61 /*                         CPLGetLowerCaseHex()                         */
62 /************************************************************************/
63 
CPLGetLowerCaseHex(const GByte * pabyData,size_t nBytes)64 static CPLString CPLGetLowerCaseHex( const GByte *pabyData, size_t nBytes )
65 
66 {
67     CPLString osRet;
68     osRet.resize(nBytes * 2);
69 
70     constexpr char achHex[] = "0123456789abcdef";
71 
72     for( size_t i = 0; i < nBytes; ++i )
73     {
74         const int nLow = pabyData[i] & 0x0f;
75         const int nHigh = (pabyData[i] & 0xf0) >> 4;
76 
77         osRet[i*2] = achHex[nHigh];
78         osRet[i*2+1] = achHex[nLow];
79     }
80 
81     return osRet;
82 }
83 
84 /************************************************************************/
85 /*                       CPLGetLowerCaseHexSHA256()                     */
86 /************************************************************************/
87 
CPLGetLowerCaseHexSHA256(const void * pabyData,size_t nBytes)88 CPLString CPLGetLowerCaseHexSHA256( const void *pabyData, size_t nBytes )
89 {
90     GByte hash[CPL_SHA256_HASH_SIZE] = {};
91     CPL_SHA256(static_cast<const GByte *>(pabyData), nBytes, hash);
92     return CPLGetLowerCaseHex(hash, CPL_SHA256_HASH_SIZE);
93 }
94 
95 /************************************************************************/
96 /*                       CPLGetLowerCaseHexSHA256()                     */
97 /************************************************************************/
98 
CPLGetLowerCaseHexSHA256(const CPLString & osStr)99 CPLString CPLGetLowerCaseHexSHA256( const CPLString& osStr )
100 {
101     return CPLGetLowerCaseHexSHA256(osStr.c_str(), osStr.size());
102 }
103 
104 /************************************************************************/
105 /*                       CPLAWSURLEncode()                              */
106 /************************************************************************/
107 
CPLAWSURLEncode(const CPLString & osURL,bool bEncodeSlash)108 CPLString CPLAWSURLEncode( const CPLString& osURL, bool bEncodeSlash )
109 {
110     CPLString osRet;
111     for( size_t i = 0; i < osURL.size(); i++ )
112     {
113         char ch = osURL[i];
114         if( (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') ||
115             (ch >= '0' && ch <= '9') ||
116             ch == '_' || ch == '-' || ch == '~' || ch == '.' )
117         {
118             osRet += ch;
119         }
120         else if( ch == '/' )
121         {
122             if( bEncodeSlash )
123                 osRet += "%2F";
124             else
125                 osRet += ch;
126         }
127         else
128         {
129             osRet += CPLSPrintf("%%%02X", static_cast<unsigned char>(ch));
130         }
131     }
132     return osRet;
133 }
134 
135 
136 /************************************************************************/
137 /*                         CPLAWSGetHeaderVal()                         */
138 /************************************************************************/
139 
CPLAWSGetHeaderVal(const struct curl_slist * psExistingHeaders,const char * pszKey)140 CPLString CPLAWSGetHeaderVal(const struct curl_slist* psExistingHeaders,
141                              const char* pszKey)
142 {
143     CPLString osKey(pszKey);
144     osKey += ":";
145     const struct curl_slist* psIter = psExistingHeaders;
146     for(; psIter != nullptr; psIter = psIter->next)
147     {
148         if( STARTS_WITH(psIter->data, osKey.c_str()) )
149             return CPLString(psIter->data + osKey.size()).Trim();
150     }
151     return CPLString();
152 }
153 
154 
155 /************************************************************************/
156 /*                 CPLGetAWS_SIGN4_Signature()                          */
157 /************************************************************************/
158 
159 // See:
160 // http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
161 CPLString
CPLGetAWS_SIGN4_Signature(const CPLString & osSecretAccessKey,const CPLString & osAccessToken,const CPLString & osRegion,const CPLString & osRequestPayer,const CPLString & osService,const CPLString & osVerb,const struct curl_slist * psExistingHeaders,const CPLString & osHost,const CPLString & osCanonicalURI,const CPLString & osCanonicalQueryString,const CPLString & osXAMZContentSHA256,const CPLString & osTimestamp,CPLString & osSignedHeaders)162 CPLGetAWS_SIGN4_Signature( const CPLString& osSecretAccessKey,
163                                const CPLString& osAccessToken,
164                                const CPLString& osRegion,
165                                const CPLString& osRequestPayer,
166                                const CPLString& osService,
167                                const CPLString& osVerb,
168                                const struct curl_slist* psExistingHeaders,
169                                const CPLString& osHost,
170                                const CPLString& osCanonicalURI,
171                                const CPLString& osCanonicalQueryString,
172                                const CPLString& osXAMZContentSHA256,
173                                const CPLString& osTimestamp,
174                                CPLString& osSignedHeaders )
175 {
176 /* -------------------------------------------------------------------- */
177 /*      Compute canonical request string.                               */
178 /* -------------------------------------------------------------------- */
179     CPLString osCanonicalRequest = osVerb + "\n";
180 
181     osCanonicalRequest += osCanonicalURI + "\n";
182 
183     osCanonicalRequest += osCanonicalQueryString + "\n";
184 
185     std::map<CPLString, CPLString> oSortedMapHeaders;
186     oSortedMapHeaders["host"] = osHost;
187     if( osXAMZContentSHA256 != "UNSIGNED-PAYLOAD" )
188     {
189         oSortedMapHeaders["x-amz-content-sha256"] = osXAMZContentSHA256;
190         oSortedMapHeaders["x-amz-date"] = osTimestamp;
191     }
192     if( !osRequestPayer.empty() )
193         oSortedMapHeaders["x-amz-request-payer"] = osRequestPayer;
194     if( !osAccessToken.empty() )
195         oSortedMapHeaders["x-amz-security-token"] = osAccessToken;
196     CPLString osCanonicalizedHeaders(
197         IVSIS3LikeHandleHelper::BuildCanonicalizedHeaders(
198                             oSortedMapHeaders,
199                             psExistingHeaders,
200                             "x-amz-"));
201 
202     osCanonicalRequest += osCanonicalizedHeaders + "\n";
203 
204     osSignedHeaders.clear();
205     std::map<CPLString, CPLString>::const_iterator oIter = oSortedMapHeaders.begin();
206     for(; oIter != oSortedMapHeaders.end(); ++oIter )
207     {
208         if( !osSignedHeaders.empty() )
209             osSignedHeaders += ";";
210         osSignedHeaders += oIter->first;
211     }
212 
213     osCanonicalRequest += osSignedHeaders + "\n";
214 
215     osCanonicalRequest += osXAMZContentSHA256;
216 
217 #ifdef DEBUG_VERBOSE
218     CPLDebug("S3", "osCanonicalRequest='%s'", osCanonicalRequest.c_str());
219 #endif
220 
221 /* -------------------------------------------------------------------- */
222 /*      Compute StringToSign .                                          */
223 /* -------------------------------------------------------------------- */
224     CPLString osStringToSign = "AWS4-HMAC-SHA256\n";
225     osStringToSign += osTimestamp + "\n";
226 
227     CPLString osYYMMDD(osTimestamp);
228     osYYMMDD.resize(8);
229 
230     CPLString osScope = osYYMMDD + "/";
231     osScope += osRegion;
232     osScope += "/";
233     osScope += osService;
234     osScope += "/aws4_request";
235     osStringToSign += osScope + "\n";
236     osStringToSign += CPLGetLowerCaseHexSHA256(osCanonicalRequest);
237 
238 #ifdef DEBUG_VERBOSE
239     CPLDebug("S3", "osStringToSign='%s'", osStringToSign.c_str());
240 #endif
241 
242 /* -------------------------------------------------------------------- */
243 /*      Compute signing key.                                            */
244 /* -------------------------------------------------------------------- */
245     GByte abySigningKeyIn[CPL_SHA256_HASH_SIZE] = {};
246     GByte abySigningKeyOut[CPL_SHA256_HASH_SIZE] = {};
247 
248     CPLString osFirstKey(CPLString("AWS4") + osSecretAccessKey);
249     CPL_HMAC_SHA256( osFirstKey.c_str(), osFirstKey.size(),
250                      osYYMMDD, osYYMMDD.size(),
251                      abySigningKeyOut );
252     memcpy(abySigningKeyIn, abySigningKeyOut, CPL_SHA256_HASH_SIZE);
253 
254     CPL_HMAC_SHA256( abySigningKeyIn, CPL_SHA256_HASH_SIZE,
255                      osRegion.c_str(), osRegion.size(),
256                      abySigningKeyOut );
257     memcpy(abySigningKeyIn, abySigningKeyOut, CPL_SHA256_HASH_SIZE);
258 
259     CPL_HMAC_SHA256( abySigningKeyIn, CPL_SHA256_HASH_SIZE,
260                      osService.c_str(), osService.size(),
261                      abySigningKeyOut );
262     memcpy(abySigningKeyIn, abySigningKeyOut, CPL_SHA256_HASH_SIZE);
263 
264     CPL_HMAC_SHA256( abySigningKeyIn, CPL_SHA256_HASH_SIZE,
265                      "aws4_request", strlen("aws4_request"),
266                      abySigningKeyOut );
267     memcpy(abySigningKeyIn, abySigningKeyOut, CPL_SHA256_HASH_SIZE);
268 
269 #ifdef DEBUG_VERBOSE
270     CPLString osSigningKey(CPLGetLowerCaseHex(abySigningKeyIn,
271                                               CPL_SHA256_HASH_SIZE));
272     CPLDebug("S3", "osSigningKey='%s'", osSigningKey.c_str());
273 #endif
274 
275 /* -------------------------------------------------------------------- */
276 /*      Compute signature.                                              */
277 /* -------------------------------------------------------------------- */
278     GByte abySignature[CPL_SHA256_HASH_SIZE] = {};
279     CPL_HMAC_SHA256( abySigningKeyIn, CPL_SHA256_HASH_SIZE,
280                      osStringToSign, osStringToSign.size(),
281                      abySignature);
282     CPLString osSignature(CPLGetLowerCaseHex(abySignature,
283                                              CPL_SHA256_HASH_SIZE));
284 
285 #ifdef DEBUG_VERBOSE
286     CPLDebug("S3", "osSignature='%s'", osSignature.c_str());
287 #endif
288 
289     return osSignature;
290 }
291 
292 /************************************************************************/
293 /*                CPLGetAWS_SIGN4_Authorization()                       */
294 /************************************************************************/
295 
296 CPLString
CPLGetAWS_SIGN4_Authorization(const CPLString & osSecretAccessKey,const CPLString & osAccessKeyId,const CPLString & osAccessToken,const CPLString & osRegion,const CPLString & osRequestPayer,const CPLString & osService,const CPLString & osVerb,const struct curl_slist * psExistingHeaders,const CPLString & osHost,const CPLString & osCanonicalURI,const CPLString & osCanonicalQueryString,const CPLString & osXAMZContentSHA256,const CPLString & osTimestamp)297 CPLGetAWS_SIGN4_Authorization( const CPLString& osSecretAccessKey,
298                                const CPLString& osAccessKeyId,
299                                const CPLString& osAccessToken,
300                                const CPLString& osRegion,
301                                const CPLString& osRequestPayer,
302                                const CPLString& osService,
303                                const CPLString& osVerb,
304                                const struct curl_slist* psExistingHeaders,
305                                const CPLString& osHost,
306                                const CPLString& osCanonicalURI,
307                                const CPLString& osCanonicalQueryString,
308                                const CPLString& osXAMZContentSHA256,
309                                const CPLString& osTimestamp )
310 {
311     CPLString osSignedHeaders;
312     CPLString osSignature(CPLGetAWS_SIGN4_Signature(osSecretAccessKey,
313                                                     osAccessToken,
314                                                     osRegion,
315                                                     osRequestPayer,
316                                                     osService,
317                                                     osVerb,
318                                                     psExistingHeaders,
319                                                     osHost,
320                                                     osCanonicalURI,
321                                                     osCanonicalQueryString,
322                                                     osXAMZContentSHA256,
323                                                     osTimestamp,
324                                                     osSignedHeaders));
325 
326     CPLString osYYMMDD(osTimestamp);
327     osYYMMDD.resize(8);
328 
329 /* -------------------------------------------------------------------- */
330 /*      Build authorization header.                                     */
331 /* -------------------------------------------------------------------- */
332     CPLString osAuthorization;
333     osAuthorization = "AWS4-HMAC-SHA256 Credential=";
334     osAuthorization += osAccessKeyId;
335     osAuthorization += "/";
336     osAuthorization += osYYMMDD;
337     osAuthorization += "/";
338     osAuthorization += osRegion;
339     osAuthorization += "/";
340     osAuthorization += osService;
341     osAuthorization += "/";
342     osAuthorization += "aws4_request";
343     osAuthorization += ",";
344     osAuthorization += "SignedHeaders=";
345     osAuthorization += osSignedHeaders;
346     osAuthorization += ",";
347     osAuthorization += "Signature=";
348     osAuthorization += osSignature;
349 
350 #ifdef DEBUG_VERBOSE
351     CPLDebug("S3", "osAuthorization='%s'", osAuthorization.c_str());
352 #endif
353 
354     return osAuthorization;
355 }
356 
357 /************************************************************************/
358 /*                        CPLGetAWS_SIGN4_Timestamp()                   */
359 /************************************************************************/
360 
CPLGetAWS_SIGN4_Timestamp()361 CPLString CPLGetAWS_SIGN4_Timestamp()
362 {
363     struct tm brokenDown;
364     CPLUnixTimeToYMDHMS(time(nullptr), &brokenDown);
365 
366     char szTimeStamp[80] = {};
367     snprintf(szTimeStamp, sizeof(szTimeStamp), "%04d%02d%02dT%02d%02d%02dZ",
368             brokenDown.tm_year + 1900,
369             brokenDown.tm_mon + 1,
370             brokenDown.tm_mday,
371             brokenDown.tm_hour,
372             brokenDown.tm_min,
373             brokenDown.tm_sec);
374     return szTimeStamp;
375 }
376 
377 
378 /************************************************************************/
379 /*                         VSIS3HandleHelper()                          */
380 /************************************************************************/
VSIS3HandleHelper(const CPLString & osSecretAccessKey,const CPLString & osAccessKeyId,const CPLString & osSessionToken,const CPLString & osEndpoint,const CPLString & osRegion,const CPLString & osRequestPayer,const CPLString & osBucket,const CPLString & osObjectKey,bool bUseHTTPS,bool bUseVirtualHosting,bool bFromEC2)381 VSIS3HandleHelper::VSIS3HandleHelper( const CPLString& osSecretAccessKey,
382                                       const CPLString& osAccessKeyId,
383                                       const CPLString& osSessionToken,
384                                       const CPLString& osEndpoint,
385                                       const CPLString& osRegion,
386                                       const CPLString& osRequestPayer,
387                                       const CPLString& osBucket,
388                                       const CPLString& osObjectKey,
389                                       bool bUseHTTPS,
390                                       bool bUseVirtualHosting,
391                                       bool bFromEC2 ) :
392     m_osURL(BuildURL(osEndpoint, osBucket, osObjectKey, bUseHTTPS,
393                      bUseVirtualHosting)),
394     m_osSecretAccessKey(osSecretAccessKey),
395     m_osAccessKeyId(osAccessKeyId),
396     m_osSessionToken(osSessionToken),
397     m_osEndpoint(osEndpoint),
398     m_osRegion(osRegion),
399     m_osRequestPayer(osRequestPayer),
400     m_osBucket(osBucket),
401     m_osObjectKey(osObjectKey),
402     m_bUseHTTPS(bUseHTTPS),
403     m_bUseVirtualHosting(bUseVirtualHosting),
404     m_bFromEC2(bFromEC2)
405 {}
406 
407 /************************************************************************/
408 /*                        ~VSIS3HandleHelper()                          */
409 /************************************************************************/
410 
~VSIS3HandleHelper()411 VSIS3HandleHelper::~VSIS3HandleHelper()
412 {
413     for( size_t i = 0; i < m_osSecretAccessKey.size(); i++ )
414         m_osSecretAccessKey[i] = 0;
415 }
416 
417 /************************************************************************/
418 /*                           BuildURL()                                 */
419 /************************************************************************/
420 
BuildURL(const CPLString & osEndpoint,const CPLString & osBucket,const CPLString & osObjectKey,bool bUseHTTPS,bool bUseVirtualHosting)421 CPLString VSIS3HandleHelper::BuildURL(const CPLString& osEndpoint,
422                                       const CPLString& osBucket,
423                                       const CPLString& osObjectKey,
424                                       bool bUseHTTPS, bool bUseVirtualHosting)
425 {
426     const char* pszProtocol = (bUseHTTPS) ? "https" : "http";
427     if( osBucket.empty()  )
428         return CPLSPrintf("%s://%s", pszProtocol,
429                                         osEndpoint.c_str());
430     else if( bUseVirtualHosting )
431         return CPLSPrintf("%s://%s.%s/%s", pszProtocol,
432                                         osBucket.c_str(),
433                                         osEndpoint.c_str(),
434                                         CPLAWSURLEncode(osObjectKey, false).c_str());
435     else
436         return CPLSPrintf("%s://%s/%s/%s", pszProtocol,
437                                         osEndpoint.c_str(),
438                                         osBucket.c_str(),
439                                         CPLAWSURLEncode(osObjectKey, false).c_str());
440 }
441 
442 /************************************************************************/
443 /*                           RebuildURL()                               */
444 /************************************************************************/
445 
RebuildURL()446 void VSIS3HandleHelper::RebuildURL()
447 {
448     m_osURL = BuildURL(m_osEndpoint, m_osBucket, m_osObjectKey,
449                        m_bUseHTTPS, m_bUseVirtualHosting);
450     m_osURL += GetQueryString(false);
451 }
452 
453 /************************************************************************/
454 /*                        GetBucketAndObjectKey()                       */
455 /************************************************************************/
456 
GetBucketAndObjectKey(const char * pszURI,const char * pszFSPrefix,bool bAllowNoObject,CPLString & osBucket,CPLString & osObjectKey)457 bool IVSIS3LikeHandleHelper::GetBucketAndObjectKey( const char* pszURI,
458                                                const char* pszFSPrefix,
459                                                bool bAllowNoObject,
460                                                CPLString &osBucket,
461                                                CPLString &osObjectKey )
462 {
463     osBucket = pszURI;
464     if( osBucket.empty() )
465     {
466         return false;
467     }
468     size_t nPos = osBucket.find('/');
469     if( nPos == std::string::npos )
470     {
471         if( bAllowNoObject )
472         {
473             osObjectKey = "";
474             return true;
475         }
476         CPLError(CE_Failure, CPLE_AppDefined,
477                  "Filename should be of the form %sbucket/key", pszFSPrefix);
478         return false;
479     }
480     osBucket.resize(nPos);
481     osObjectKey = pszURI + nPos + 1;
482     return true;
483 }
484 
485 /************************************************************************/
486 /*                      BuildCanonicalizedHeaders()                    */
487 /************************************************************************/
488 
BuildCanonicalizedHeaders(std::map<CPLString,CPLString> & oSortedMapHeaders,const struct curl_slist * psExistingHeaders,const char * pszHeaderPrefix)489 CPLString IVSIS3LikeHandleHelper::BuildCanonicalizedHeaders(
490                             std::map<CPLString, CPLString>& oSortedMapHeaders,
491                             const struct curl_slist* psExistingHeaders,
492                             const char* pszHeaderPrefix)
493 {
494     const struct curl_slist* psIter = psExistingHeaders;
495     for(; psIter != nullptr; psIter = psIter->next)
496     {
497         if( STARTS_WITH_CI(psIter->data, pszHeaderPrefix) ||
498             STARTS_WITH_CI(psIter->data, "Content-MD5") )
499         {
500             const char* pszColumn = strstr(psIter->data, ":");
501             if( pszColumn )
502             {
503                 CPLString osKey(psIter->data);
504                 osKey.resize( pszColumn - psIter->data);
505                 oSortedMapHeaders[osKey.tolower()] =
506                     CPLString(pszColumn + strlen(":")).Trim();
507             }
508         }
509     }
510 
511     CPLString osCanonicalizedHeaders;
512     std::map<CPLString, CPLString>::const_iterator oIter =
513         oSortedMapHeaders.begin();
514     for(; oIter != oSortedMapHeaders.end(); ++oIter )
515     {
516         osCanonicalizedHeaders += oIter->first + ":" + oIter->second + "\n";
517     }
518     return osCanonicalizedHeaders;
519 }
520 
521 /************************************************************************/
522 /*                         GetRFC822DateTime()                          */
523 /************************************************************************/
524 
GetRFC822DateTime()525 CPLString IVSIS3LikeHandleHelper::GetRFC822DateTime()
526 {
527     char szDate[64];
528     time_t nNow = time(nullptr);
529     struct tm tm;
530     CPLUnixTimeToYMDHMS(nNow, &tm);
531     int nRet = CPLPrintTime(szDate, sizeof(szDate)-1,
532                     "%a, %d %b %Y %H:%M:%S GMT", &tm, "C");
533     szDate[nRet] = 0;
534     return szDate;
535 }
536 
537 /************************************************************************/
538 /*                          ParseSimpleJson()                           */
539 /*                                                                      */
540 /*      Return a string list of name/value pairs extracted from a       */
541 /*      JSON doc.  The EC2 IAM web service returns simple JSON          */
542 /*      responses.  The parsing as done currently is very fragile       */
543 /*      and depends on JSON documents being in a very very simple       */
544 /*      form.                                                           */
545 /************************************************************************/
546 
ParseSimpleJson(const char * pszJson)547 static CPLStringList ParseSimpleJson(const char *pszJson)
548 
549 {
550 /* -------------------------------------------------------------------- */
551 /*      We are expecting simple documents like the following with no    */
552 /*      hierarchy or complex structure.                                 */
553 /* -------------------------------------------------------------------- */
554 /*
555     {
556     "Code" : "Success",
557     "LastUpdated" : "2017-07-03T16:20:17Z",
558     "Type" : "AWS-HMAC",
559     "AccessKeyId" : "bla",
560     "SecretAccessKey" : "bla",
561     "Token" : "bla",
562     "Expiration" : "2017-07-03T22:42:58Z"
563     }
564 */
565 
566     CPLStringList oWords(
567         CSLTokenizeString2(pszJson, " \n\t,:{}", CSLT_HONOURSTRINGS ));
568     CPLStringList oNameValue;
569 
570     for( int i=0; i < oWords.size(); i += 2 )
571     {
572         oNameValue.SetNameValue(oWords[i], oWords[i+1]);
573     }
574 
575     return oNameValue;
576 }
577 
578 /************************************************************************/
579 /*                        Iso8601ToUnixTime()                           */
580 /************************************************************************/
581 
Iso8601ToUnixTime(const char * pszDT,GIntBig * pnUnixTime)582 static bool Iso8601ToUnixTime(const char* pszDT, GIntBig* pnUnixTime)
583 {
584     int nYear;
585     int nMonth;
586     int nDay;
587     int nHour;
588     int nMinute;
589     int nSecond;
590     if( sscanf(pszDT, "%04d-%02d-%02dT%02d:%02d:%02d",
591                 &nYear, &nMonth, &nDay, &nHour, &nMinute, &nSecond) == 6 )
592     {
593         struct tm brokendowntime;
594         brokendowntime.tm_year = nYear - 1900;
595         brokendowntime.tm_mon = nMonth - 1;
596         brokendowntime.tm_mday = nDay;
597         brokendowntime.tm_hour = nHour;
598         brokendowntime.tm_min = nMinute;
599         brokendowntime.tm_sec = nSecond;
600         *pnUnixTime = CPLYMDHMSToUnixTime(&brokendowntime);
601         return true;
602     }
603     return false;
604 }
605 
606 /************************************************************************/
607 /*                  IsMachinePotentiallyEC2Instance()                   */
608 /************************************************************************/
609 
IsMachinePotentiallyEC2Instance()610 static bool IsMachinePotentiallyEC2Instance()
611 {
612 #if defined(__linux) || defined(WIN32)
613     const auto IsMachinePotentiallyEC2InstanceFromLinuxHost = []()
614     {
615         // On the newer Nitro Hypervisor (C5, M5, H1, T3), use
616         // /sys/devices/virtual/dmi/id/sys_vendor = 'Amazon EC2' instead.
617 
618         // On older Xen hypervisor EC2 instances, a /sys/hypervisor/uuid file will
619         // exist with a string beginning with 'ec2'.
620 
621         // If the files exist but don't contain the correct content, then we're not EC2 and
622         // do not attempt any network access
623 
624         // Check for Xen Hypervisor instances
625         // This file doesn't exist on Nitro instances
626         VSILFILE* fp = VSIFOpenL("/sys/hypervisor/uuid", "rb");
627         if( fp != nullptr )
628         {
629             char uuid[36+1] = { 0 };
630             VSIFReadL( uuid, 1, sizeof(uuid)-1, fp );
631             VSIFCloseL(fp);
632             return EQUALN( uuid, "ec2", 3 );
633         }
634 
635         // Check for Nitro Hypervisor instances
636         // This file may exist on Xen instances with a value of 'Xen'
637         // (but that doesn't mean we're on EC2)
638         fp = VSIFOpenL("/sys/devices/virtual/dmi/id/sys_vendor", "rb");
639         if( fp != nullptr )
640         {
641             char buf[10+1] = { 0 };
642             VSIFReadL( buf, 1, sizeof(buf)-1, fp );
643             VSIFCloseL(fp);
644             return EQUALN( buf, "Amazon EC2", 10 );
645         }
646 
647         // Fallback: Check via the network
648         return true;
649     };
650 #endif
651 
652 #ifdef __linux
653     // Optimization on Linux to avoid the network request
654     // See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html
655     // Skip if either:
656     // - CPL_AWS_AUTODETECT_EC2=NO
657     // - CPL_AWS_CHECK_HYPERVISOR_UUID=NO (deprecated)
658 
659     if( ! CPLTestBool(CPLGetConfigOption("CPL_AWS_AUTODETECT_EC2", "YES")) )
660     {
661         return true;
662     }
663     else
664     {
665         CPLString opt = CPLGetConfigOption("CPL_AWS_CHECK_HYPERVISOR_UUID", "");
666         if ( ! opt.empty() )
667         {
668             CPLDebug("AWS", "CPL_AWS_CHECK_HYPERVISOR_UUID is deprecated. Use CPL_AWS_AUTODETECT_EC2 instead");
669             if ( ! CPLTestBool(opt) )
670             {
671                 return true;
672             }
673         }
674     }
675 
676     return IsMachinePotentiallyEC2InstanceFromLinuxHost();
677 
678 #elif defined(WIN32)
679     if( ! CPLTestBool(CPLGetConfigOption("CPL_AWS_AUTODETECT_EC2", "YES")) )
680     {
681         return true;
682     }
683 
684     // Regular UUID is not valid for WINE, fetch from sysfs instead.
685     if( CPLGetWineVersion() != nullptr )
686     {
687         return IsMachinePotentiallyEC2InstanceFromLinuxHost();
688     }
689     else
690     {
691 #if defined(_MSC_VER)
692         CPLString osMachineUUID;
693         if( CPLFetchWindowsProductUUID(osMachineUUID) )
694         {
695             if( osMachineUUID.length() >= 3 && EQUALN(osMachineUUID.c_str(), "EC2", 3) )
696             {
697                 return true;
698             }
699             else if( osMachineUUID.length() >= 8 && osMachineUUID[4] == '2' &&
700                      osMachineUUID[6] == 'E' && osMachineUUID[7] == 'C' )
701             {
702                 return true;
703             }
704             else
705             {
706                 return false;
707             }
708         }
709 #endif
710     }
711 
712     // Fallback: Check via the network
713     return true;
714 #else
715     // At time of writing EC2 instances can be only Linux or Windows
716     return false;
717 #endif
718 }
719 
720 /************************************************************************/
721 /*                      GetConfigurationFromEC2()                       */
722 /************************************************************************/
723 
GetConfigurationFromEC2(CPLString & osSecretAccessKey,CPLString & osAccessKeyId,CPLString & osSessionToken)724 bool VSIS3HandleHelper::GetConfigurationFromEC2(CPLString& osSecretAccessKey,
725                                                 CPLString& osAccessKeyId,
726                                                 CPLString& osSessionToken)
727 {
728     CPLMutexHolder oHolder( &ghMutex );
729     time_t nCurTime;
730     time(&nCurTime);
731     // Try to reuse credentials if they are still valid, but
732     // keep one minute of margin...
733     if( !gosGlobalAccessKeyId.empty() && nCurTime < gnGlobalExpiration - 60 )
734     {
735         osAccessKeyId = gosGlobalAccessKeyId;
736         osSecretAccessKey = gosGlobalSecretAccessKey;
737         osSessionToken = gosGlobalSessionToken;
738         return true;
739     }
740 
741     CPLString osURLRefreshCredentials;
742     const CPLString osEC2DefaultURL("http://169.254.169.254");
743     // coverity[tainted_data]
744     const CPLString osEC2RootURL(
745         CPLGetConfigOption("CPL_AWS_EC2_API_ROOT_URL", osEC2DefaultURL));
746     // coverity[tainted_data]
747     const CPLString osECSRelativeURI(
748         CPLGetConfigOption("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", ""));
749     CPLString osToken;
750     if( osEC2RootURL == osEC2DefaultURL && !osECSRelativeURI.empty() )
751     {
752         // See https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html
753         osURLRefreshCredentials = "http://169.254.170.2" + osECSRelativeURI;
754     }
755     else
756     {
757         if( !IsMachinePotentiallyEC2Instance() )
758             return false;
759 
760         // Use IMDSv2 protocol:
761         // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html
762 
763         // Retrieve IMDSv2 token
764         {
765             const CPLString osEC2_IMDSv2_api_token_URL =
766                 osEC2RootURL + "/latest/api/token";
767             CPLStringList aosOptions;
768             aosOptions.SetNameValue("TIMEOUT", "1");
769             aosOptions.SetNameValue("CUSTOMREQUEST", "PUT");
770             aosOptions.SetNameValue("HEADERS",
771                                     "X-aws-ec2-metadata-token-ttl-seconds: 10");
772             CPLPushErrorHandler(CPLQuietErrorHandler);
773             CPLHTTPResult* psResult =
774                         CPLHTTPFetch( osEC2_IMDSv2_api_token_URL, aosOptions.List() );
775             CPLPopErrorHandler();
776             if( psResult )
777             {
778                 if( psResult->nStatus == 0 && psResult->pabyData != nullptr )
779                 {
780                     osToken = reinterpret_cast<char*>(psResult->pabyData);
781                 }
782                 else
783                 {
784                     // Failure: either we are not running on EC2 (or something emulating it)
785                     // or this doesn't implement yet IDMSv2
786                     // Go on trying IDMSv1
787                 }
788                 CPLHTTPDestroyResult(psResult);
789             }
790         }
791 
792         // If we don't know yet the IAM role, fetch it
793         const CPLString osEC2CredentialsURL =
794             osEC2RootURL + "/latest/meta-data/iam/security-credentials/";
795         if( gosIAMRole.empty() )
796         {
797             CPLStringList aosOptions;
798             aosOptions.SetNameValue("TIMEOUT", "1");
799             if( !osToken.empty() )
800             {
801                 aosOptions.SetNameValue("HEADERS",
802                                 ("X-aws-ec2-metadata-token: " + osToken).c_str());
803             }
804             CPLPushErrorHandler(CPLQuietErrorHandler);
805             CPLHTTPResult* psResult =
806                         CPLHTTPFetch( osEC2CredentialsURL, aosOptions.List() );
807             CPLPopErrorHandler();
808             if( psResult )
809             {
810                 if( psResult->nStatus == 0 && psResult->pabyData != nullptr )
811                 {
812                     gosIAMRole = reinterpret_cast<char*>(psResult->pabyData);
813                 }
814                 CPLHTTPDestroyResult(psResult);
815             }
816             if( gosIAMRole.empty() )
817             {
818                 // We didn't get the IAM role. We are definitely not running
819                 // on EC2 or an emulation of it.
820                 return false;
821             }
822         }
823         osURLRefreshCredentials = osEC2CredentialsURL + gosIAMRole;
824     }
825 
826     // Now fetch the refreshed credentials
827     CPLStringList oResponse;
828     CPLStringList aosOptions;
829     if( !osToken.empty() )
830     {
831         aosOptions.SetNameValue("HEADERS",
832                             ("X-aws-ec2-metadata-token: " + osToken).c_str());
833     }
834     CPLHTTPResult* psResult = CPLHTTPFetch(osURLRefreshCredentials.c_str(), aosOptions.List() );
835     if( psResult )
836     {
837         if( psResult->nStatus == 0 && psResult->pabyData != nullptr )
838         {
839             const CPLString osJSon =
840                     reinterpret_cast<char*>(psResult->pabyData);
841             oResponse = ParseSimpleJson(osJSon);
842         }
843         CPLHTTPDestroyResult(psResult);
844     }
845     osAccessKeyId = oResponse.FetchNameValueDef("AccessKeyId", "");
846     osSecretAccessKey =
847                 oResponse.FetchNameValueDef("SecretAccessKey", "");
848     osSessionToken = oResponse.FetchNameValueDef("Token", "");
849     const CPLString osExpiration =
850         oResponse.FetchNameValueDef("Expiration", "");
851     GIntBig nExpirationUnix = 0;
852     if( !osAccessKeyId.empty() &&
853         !osSecretAccessKey.empty() &&
854         Iso8601ToUnixTime(osExpiration, &nExpirationUnix) )
855     {
856         gosGlobalAccessKeyId = osAccessKeyId;
857         gosGlobalSecretAccessKey = osSecretAccessKey;
858         gosGlobalSessionToken = osSessionToken;
859         gnGlobalExpiration = nExpirationUnix;
860         CPLDebug("AWS", "Storing AIM credentials until %s",
861                 osExpiration.c_str());
862     }
863     return !osAccessKeyId.empty() && !osSecretAccessKey.empty();
864 }
865 
866 
867 /************************************************************************/
868 /*                      UpdateAndWarnIfInconsistent()                   */
869 /************************************************************************/
870 
871 static
UpdateAndWarnIfInconsistent(const char * pszKeyword,CPLString & osVal,const CPLString & osNewVal,const CPLString & osCredentials,const CPLString & osConfig)872 void UpdateAndWarnIfInconsistent(const char* pszKeyword,
873                                  CPLString& osVal,
874                                  const CPLString& osNewVal,
875                                  const CPLString& osCredentials,
876                                  const CPLString& osConfig)
877 {
878     // nominally defined in ~/.aws/credentials but can
879     // be set here too. If both values exist, credentials
880     // has the priority
881     if( osVal.empty() )
882     {
883         osVal = osNewVal;
884     }
885     else if( osVal != osNewVal )
886     {
887         CPLError(CE_Warning, CPLE_AppDefined,
888                     "%s defined in both %s "
889                     "and %s. The one of %s will be used",
890                     pszKeyword,
891                     osCredentials.c_str(),
892                     osConfig.c_str(),
893                     osCredentials.c_str());
894     }
895 }
896 
897 /************************************************************************/
898 /*                GetConfigurationFromAWSConfigFiles()                  */
899 /************************************************************************/
900 
GetConfigurationFromAWSConfigFiles(CPLString & osSecretAccessKey,CPLString & osAccessKeyId,CPLString & osSessionToken,CPLString & osRegion,CPLString & osCredentials)901 bool VSIS3HandleHelper::GetConfigurationFromAWSConfigFiles(
902                                                 CPLString& osSecretAccessKey,
903                                                 CPLString& osAccessKeyId,
904                                                 CPLString& osSessionToken,
905                                                 CPLString& osRegion,
906                                                 CPLString& osCredentials)
907 {
908     // See http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html
909     // If AWS_DEFAULT_PROFILE is set (obsolete, no longer documented), use it in priority
910     // Otherwise use AWS_PROFILE
911     // Otherwise fallback to "default"
912     const char* pszProfile = CPLGetConfigOption("AWS_DEFAULT_PROFILE",
913         CPLGetConfigOption("AWS_PROFILE", ""));
914     const CPLString osProfile(pszProfile[0] != '\0' ? pszProfile : "default");
915 
916 #ifdef WIN32
917     const char* pszHome = CPLGetConfigOption("USERPROFILE", nullptr);
918     constexpr char SEP_STRING[] = "\\";
919 #else
920     const char* pszHome = CPLGetConfigOption("HOME", nullptr);
921     constexpr char SEP_STRING[] = "/";
922 #endif
923 
924     CPLString osDotAws( pszHome ? pszHome : "" );
925     osDotAws += SEP_STRING;
926     osDotAws += ".aws";
927 
928     // Read first ~/.aws/credential file
929 
930     // GDAL specific config option (mostly for testing purpose, but also
931     // used in production in some cases)
932     const char* pszCredentials =
933                     CPLGetConfigOption( "CPL_AWS_CREDENTIALS_FILE", nullptr );
934     if( pszCredentials )
935     {
936         osCredentials = pszCredentials;
937     }
938     else
939     {
940         osCredentials = osDotAws;
941         osCredentials += SEP_STRING;
942         osCredentials += "credentials";
943     }
944     VSILFILE* fp = VSIFOpenL( osCredentials, "rb" );
945     if( fp != nullptr )
946     {
947         const char* pszLine;
948         bool bInProfile = false;
949         const CPLString osBracketedProfile("[" + osProfile + "]");
950         while( (pszLine = CPLReadLineL(fp)) != nullptr )
951         {
952             if( pszLine[0] == '[' )
953             {
954                 if( bInProfile )
955                     break;
956                 if( CPLString(pszLine) == osBracketedProfile )
957                     bInProfile = true;
958             }
959             else if( bInProfile )
960             {
961                 char* pszKey = nullptr;
962                 const char* pszValue = CPLParseNameValue(pszLine, &pszKey);
963                 if( pszKey && pszValue )
964                 {
965                     if( EQUAL(pszKey, "aws_access_key_id") )
966                         osAccessKeyId = pszValue;
967                     else if( EQUAL(pszKey, "aws_secret_access_key") )
968                         osSecretAccessKey = pszValue;
969                     else if( EQUAL(pszKey, "aws_session_token") )
970                         osSessionToken = pszValue;
971                 }
972                 CPLFree(pszKey);
973             }
974         }
975         VSIFCloseL(fp);
976     }
977 
978     // And then ~/.aws/config file (unless AWS_CONFIG_FILE is defined)
979     const char* pszAWSConfigFileEnv =
980                             CPLGetConfigOption( "AWS_CONFIG_FILE", nullptr );
981     CPLString osConfig;
982     if( pszAWSConfigFileEnv )
983     {
984         osConfig = pszAWSConfigFileEnv;
985     }
986     else
987     {
988         osConfig = osDotAws;
989         osConfig += SEP_STRING;
990         osConfig += "config";
991     }
992     fp = VSIFOpenL( osConfig, "rb" );
993     if( fp != nullptr )
994     {
995         const char* pszLine;
996         bool bInProfile = false;
997         const CPLString osBracketedProfile("[" + osProfile + "]");
998         const CPLString osBracketedProfileProfile("[profile " + osProfile + "]");
999         while( (pszLine = CPLReadLineL(fp)) != nullptr )
1000         {
1001             if( pszLine[0] == '[' )
1002             {
1003                 if( bInProfile )
1004                     break;
1005                 // In config file, the section name is nominally [profile foo]
1006                 // for the non default profile.
1007                 if( CPLString(pszLine) == osBracketedProfile ||
1008                     CPLString(pszLine) == osBracketedProfileProfile )
1009                 {
1010                     bInProfile = true;
1011                 }
1012             }
1013             else if( bInProfile )
1014             {
1015                 char* pszKey = nullptr;
1016                 const char* pszValue = CPLParseNameValue(pszLine, &pszKey);
1017                 if( pszKey && pszValue )
1018                 {
1019                     if( EQUAL(pszKey, "aws_access_key_id") )
1020                     {
1021                         UpdateAndWarnIfInconsistent(pszKey,
1022                                                     osAccessKeyId,
1023                                                     pszValue,
1024                                                     osCredentials,
1025                                                     osConfig);
1026                     }
1027                     else if( EQUAL(pszKey, "aws_secret_access_key") )
1028                     {
1029                         UpdateAndWarnIfInconsistent(pszKey,
1030                                                     osSecretAccessKey,
1031                                                     pszValue,
1032                                                     osCredentials,
1033                                                     osConfig);
1034                     }
1035                     else if( EQUAL(pszKey, "aws_session_token") )
1036                     {
1037                         UpdateAndWarnIfInconsistent(pszKey,
1038                                                     osSessionToken,
1039                                                     pszValue,
1040                                                     osCredentials,
1041                                                     osConfig);
1042                     }
1043                     else if( EQUAL(pszKey, "region") )
1044                     {
1045                         osRegion = pszValue;
1046                     }
1047                 }
1048                 CPLFree(pszKey);
1049             }
1050         }
1051         VSIFCloseL(fp);
1052     }
1053     else if( pszAWSConfigFileEnv != nullptr )
1054     {
1055         if( pszAWSConfigFileEnv[0] != '\0' )
1056         {
1057             CPLError(CE_Warning, CPLE_AppDefined,
1058                      "%s does not exist or cannot be open",
1059                      pszAWSConfigFileEnv);
1060         }
1061     }
1062 
1063     return !osAccessKeyId.empty() && !osSecretAccessKey.empty();
1064 }
1065 
1066 /************************************************************************/
1067 /*                        GetConfiguration()                            */
1068 /************************************************************************/
1069 
GetConfiguration(CSLConstList papszOptions,CPLString & osSecretAccessKey,CPLString & osAccessKeyId,CPLString & osSessionToken,CPLString & osRegion,bool & bFromEC2)1070 bool VSIS3HandleHelper::GetConfiguration(CSLConstList papszOptions,
1071                                          CPLString& osSecretAccessKey,
1072                                          CPLString& osAccessKeyId,
1073                                          CPLString& osSessionToken,
1074                                          CPLString& osRegion,
1075                                          bool& bFromEC2)
1076 {
1077     bFromEC2 = false;
1078 
1079     // AWS_REGION is GDAL specific. Later overloaded by standard
1080     // AWS_DEFAULT_REGION
1081     osRegion = CSLFetchNameValueDef(papszOptions, "AWS_REGION",
1082                             CPLGetConfigOption("AWS_REGION", "us-east-1"));
1083 
1084     if( CPLTestBool(CPLGetConfigOption("AWS_NO_SIGN_REQUEST", "NO")) )
1085     {
1086         osSecretAccessKey.clear();
1087         osAccessKeyId.clear();
1088         osSessionToken.clear();
1089         return true;
1090     }
1091 
1092     osSecretAccessKey = CSLFetchNameValueDef(papszOptions,
1093         "AWS_SECRET_ACCESS_KEY",
1094         CPLGetConfigOption("AWS_SECRET_ACCESS_KEY", ""));
1095     if( !osSecretAccessKey.empty() )
1096     {
1097         osAccessKeyId = CPLGetConfigOption("AWS_ACCESS_KEY_ID", "");
1098         if( osAccessKeyId.empty() )
1099         {
1100             VSIError(VSIE_AWSInvalidCredentials,
1101                     "AWS_ACCESS_KEY_ID configuration option not defined");
1102             return false;
1103         }
1104 
1105         osSessionToken = CSLFetchNameValueDef(papszOptions,
1106             "AWS_SESSION_TOKEN",
1107             CPLGetConfigOption("AWS_SESSION_TOKEN", ""));
1108         return true;
1109     }
1110 
1111     // Next try reading from ~/.aws/credentials and ~/.aws/config
1112     CPLString osCredentials;
1113     // coverity[tainted_data]
1114     if( GetConfigurationFromAWSConfigFiles(osSecretAccessKey, osAccessKeyId,
1115                                            osSessionToken, osRegion,
1116                                            osCredentials) )
1117     {
1118         return true;
1119     }
1120 
1121     // Last method: use IAM role security credentials on EC2 instances
1122     if( GetConfigurationFromEC2(osSecretAccessKey, osAccessKeyId,
1123                                 osSessionToken) )
1124     {
1125         bFromEC2 = true;
1126         return true;
1127     }
1128 
1129     VSIError(VSIE_AWSInvalidCredentials,
1130                 "AWS_SECRET_ACCESS_KEY and AWS_NO_SIGN_REQUEST configuration "
1131                 "options not defined, and %s not filled",
1132                 osCredentials.c_str());
1133     return false;
1134 }
1135 
1136 /************************************************************************/
1137 /*                          CleanMutex()                                */
1138 /************************************************************************/
1139 
CleanMutex()1140 void VSIS3HandleHelper::CleanMutex()
1141 {
1142     if( ghMutex != nullptr )
1143         CPLDestroyMutex( ghMutex );
1144     ghMutex = nullptr;
1145 }
1146 
1147 /************************************************************************/
1148 /*                          ClearCache()                                */
1149 /************************************************************************/
1150 
ClearCache()1151 void VSIS3HandleHelper::ClearCache()
1152 {
1153     CPLMutexHolder oHolder( &ghMutex );
1154 
1155     gosIAMRole.clear();
1156     gosGlobalAccessKeyId.clear();
1157     gosGlobalSecretAccessKey.clear();
1158     gosGlobalSessionToken.clear();
1159     gnGlobalExpiration = 0;
1160 }
1161 
1162 /************************************************************************/
1163 /*                          BuildFromURI()                              */
1164 /************************************************************************/
1165 
BuildFromURI(const char * pszURI,const char * pszFSPrefix,bool bAllowNoObject,CSLConstList papszOptions)1166 VSIS3HandleHelper* VSIS3HandleHelper::BuildFromURI( const char* pszURI,
1167                                                     const char* pszFSPrefix,
1168                                                     bool bAllowNoObject,
1169                                                     CSLConstList papszOptions )
1170 {
1171     CPLString osSecretAccessKey;
1172     CPLString osAccessKeyId;
1173     CPLString osSessionToken;
1174     CPLString osRegion;
1175     bool bFromEC2 = false;
1176     if( !GetConfiguration(papszOptions,
1177                           osSecretAccessKey, osAccessKeyId,
1178                           osSessionToken, osRegion, bFromEC2) )
1179     {
1180         return nullptr;
1181     }
1182 
1183     // According to http://docs.aws.amazon.com/cli/latest/userguide/cli-environment.html
1184     // " This variable overrides the default region of the in-use profile, if set."
1185     const CPLString osDefaultRegion = CSLFetchNameValueDef(
1186         papszOptions, "AWS_DEFAULT_REGION",
1187         CPLGetConfigOption("AWS_DEFAULT_REGION", ""));
1188     if( !osDefaultRegion.empty() )
1189     {
1190         osRegion = osDefaultRegion;
1191     }
1192 
1193     const CPLString osEndpoint =
1194         CPLGetConfigOption("AWS_S3_ENDPOINT", "s3.amazonaws.com");
1195     const CPLString osRequestPayer =
1196         CPLGetConfigOption("AWS_REQUEST_PAYER", "");
1197     CPLString osBucket;
1198     CPLString osObjectKey;
1199     if( pszURI != nullptr && pszURI[0] != '\0' &&
1200         !GetBucketAndObjectKey(pszURI, pszFSPrefix, bAllowNoObject,
1201                                osBucket, osObjectKey) )
1202     {
1203         return nullptr;
1204     }
1205     const bool bUseHTTPS = CPLTestBool(CPLGetConfigOption("AWS_HTTPS", "YES"));
1206     const bool bIsValidNameForVirtualHosting =
1207         osBucket.find('.') == std::string::npos;
1208     const bool bUseVirtualHosting = CPLTestBool(
1209         CSLFetchNameValueDef(papszOptions, "AWS_VIRTUAL_HOSTING",
1210                 CPLGetConfigOption("AWS_VIRTUAL_HOSTING",
1211                            bIsValidNameForVirtualHosting ? "TRUE" : "FALSE")));
1212     return new VSIS3HandleHelper(osSecretAccessKey, osAccessKeyId,
1213                                  osSessionToken,
1214                                  osEndpoint, osRegion,
1215                                  osRequestPayer,
1216                                  osBucket, osObjectKey, bUseHTTPS,
1217                                  bUseVirtualHosting, bFromEC2);
1218 }
1219 
1220 /************************************************************************/
1221 /*                          GetQueryString()                            */
1222 /************************************************************************/
1223 
GetQueryString(bool bAddEmptyValueAfterEqual) const1224 CPLString IVSIS3LikeHandleHelper::GetQueryString(bool bAddEmptyValueAfterEqual) const
1225 {
1226     CPLString osQueryString;
1227     std::map<CPLString, CPLString>::const_iterator oIter =
1228         m_oMapQueryParameters.begin();
1229     for( ; oIter != m_oMapQueryParameters.end(); ++oIter )
1230     {
1231         if( oIter == m_oMapQueryParameters.begin() )
1232             osQueryString += "?";
1233         else
1234             osQueryString += "&";
1235         osQueryString += oIter->first;
1236         if( !oIter->second.empty() || bAddEmptyValueAfterEqual )
1237         {
1238             osQueryString += "=";
1239             osQueryString += CPLAWSURLEncode(oIter->second);
1240         }
1241     }
1242     return osQueryString;
1243 }
1244 
1245 /************************************************************************/
1246 /*                       ResetQueryParameters()                         */
1247 /************************************************************************/
1248 
ResetQueryParameters()1249 void IVSIS3LikeHandleHelper::ResetQueryParameters()
1250 {
1251     m_oMapQueryParameters.clear();
1252     RebuildURL();
1253 }
1254 
1255 /************************************************************************/
1256 /*                         AddQueryParameter()                          */
1257 /************************************************************************/
1258 
AddQueryParameter(const CPLString & osKey,const CPLString & osValue)1259 void IVSIS3LikeHandleHelper::AddQueryParameter( const CPLString& osKey,
1260                                                 const CPLString& osValue )
1261 {
1262     m_oMapQueryParameters[osKey] = osValue;
1263     RebuildURL();
1264 }
1265 
1266 /************************************************************************/
1267 /*                           GetURLNoKVP()                              */
1268 /************************************************************************/
1269 
GetURLNoKVP() const1270 CPLString IVSIS3LikeHandleHelper::GetURLNoKVP() const
1271 {
1272     CPLString osURL(GetURL());
1273     const auto nPos = osURL.find('?');
1274     if( nPos != std::string::npos )
1275         osURL.resize(nPos);
1276     return osURL;
1277 }
1278 
1279 /************************************************************************/
1280 /*                           GetCurlHeaders()                           */
1281 /************************************************************************/
1282 
1283 struct curl_slist *
GetCurlHeaders(const CPLString & osVerb,const struct curl_slist * psExistingHeaders,const void * pabyDataContent,size_t nBytesContent) const1284 VSIS3HandleHelper::GetCurlHeaders( const CPLString& osVerb,
1285                                    const struct curl_slist* psExistingHeaders,
1286                                    const void *pabyDataContent,
1287                                    size_t nBytesContent ) const
1288 {
1289     if( m_bFromEC2 )
1290     {
1291         CPLString osSecretAccessKey, osAccessKeyId, osSessionToken;
1292         if( GetConfigurationFromEC2(osSecretAccessKey,
1293                                     osAccessKeyId,
1294                                     osSessionToken) )
1295         {
1296             m_osSecretAccessKey = osSecretAccessKey;
1297             m_osAccessKeyId = osAccessKeyId;
1298             m_osSessionToken = osSessionToken;
1299         }
1300     }
1301 
1302     CPLString osXAMZDate = CPLGetConfigOption("AWS_TIMESTAMP", "");
1303     if( osXAMZDate.empty() )
1304         osXAMZDate = CPLGetAWS_SIGN4_Timestamp();
1305 
1306     const CPLString osXAMZContentSHA256 =
1307         CPLGetLowerCaseHexSHA256(pabyDataContent, nBytesContent);
1308 
1309     CPLString osCanonicalQueryString(GetQueryString(true));
1310     if( !osCanonicalQueryString.empty() )
1311         osCanonicalQueryString = osCanonicalQueryString.substr(1);
1312 
1313     const CPLString osHost(m_bUseVirtualHosting && !m_osBucket.empty()
1314         ? CPLString(m_osBucket + "." + m_osEndpoint) : m_osEndpoint);
1315     const CPLString osAuthorization = m_osSecretAccessKey.empty() ? CPLString():
1316       CPLGetAWS_SIGN4_Authorization(
1317         m_osSecretAccessKey,
1318         m_osAccessKeyId,
1319         m_osSessionToken,
1320         m_osRegion,
1321         m_osRequestPayer,
1322         "s3",
1323         osVerb,
1324         psExistingHeaders,
1325         osHost,
1326         m_bUseVirtualHosting
1327         ? CPLAWSURLEncode("/" + m_osObjectKey, false).c_str() :
1328         CPLAWSURLEncode("/" + m_osBucket + "/" + m_osObjectKey, false).c_str(),
1329         osCanonicalQueryString,
1330         osXAMZContentSHA256,
1331         osXAMZDate);
1332 
1333     struct curl_slist *headers=nullptr;
1334     headers = curl_slist_append(
1335         headers, CPLSPrintf("x-amz-date: %s", osXAMZDate.c_str()));
1336     headers = curl_slist_append(
1337         headers, CPLSPrintf("x-amz-content-sha256: %s",
1338                             osXAMZContentSHA256.c_str()));
1339     if( !m_osSessionToken.empty() )
1340         headers = curl_slist_append(
1341             headers,
1342             CPLSPrintf("X-Amz-Security-Token: %s", m_osSessionToken.c_str()));
1343     if( !m_osRequestPayer.empty() )
1344         headers = curl_slist_append(
1345             headers,
1346             CPLSPrintf("x-amz-request-payer: %s", m_osRequestPayer.c_str()));
1347     if( !osAuthorization.empty() )
1348     {
1349         headers = curl_slist_append(
1350             headers, CPLSPrintf("Authorization: %s", osAuthorization.c_str()));
1351     }
1352     return headers;
1353 }
1354 
1355 /************************************************************************/
1356 /*                          CanRestartOnError()                         */
1357 /************************************************************************/
1358 
CanRestartOnError(const char * pszErrorMsg,const char * pszHeaders,bool bSetError,bool * pbUpdateMap)1359 bool VSIS3HandleHelper::CanRestartOnError( const char* pszErrorMsg,
1360                                            const char* pszHeaders,
1361                                            bool bSetError, bool* pbUpdateMap )
1362 {
1363 #ifdef DEBUG_VERBOSE
1364     CPLDebug("S3", "%s", pszErrorMsg);
1365     CPLDebug("S3", "%s", pszHeaders ? pszHeaders : "");
1366 #endif
1367 
1368     if( pbUpdateMap != nullptr )
1369         *pbUpdateMap = true;
1370 
1371     if( !STARTS_WITH(pszErrorMsg, "<?xml") &&
1372         !STARTS_WITH(pszErrorMsg, "<Error>") )
1373     {
1374         if( bSetError )
1375         {
1376             VSIError(VSIE_AWSError, "Invalid AWS response: %s", pszErrorMsg);
1377         }
1378         return false;
1379     }
1380 
1381     CPLXMLNode* psTree = CPLParseXMLString(pszErrorMsg);
1382     if( psTree == nullptr )
1383     {
1384         if( bSetError )
1385         {
1386             VSIError(VSIE_AWSError,
1387                      "Malformed AWS XML response: %s", pszErrorMsg);
1388         }
1389         return false;
1390     }
1391 
1392     const char* pszCode = CPLGetXMLValue(psTree, "=Error.Code", nullptr);
1393     if( pszCode == nullptr )
1394     {
1395         CPLDestroyXMLNode(psTree);
1396         if( bSetError )
1397         {
1398             VSIError(VSIE_AWSError,
1399                      "Malformed AWS XML response: %s", pszErrorMsg);
1400         }
1401         return false;
1402     }
1403 
1404     if( EQUAL(pszCode, "AuthorizationHeaderMalformed") )
1405     {
1406         const char* pszRegion = CPLGetXMLValue(psTree, "=Error.Region", nullptr);
1407         if( pszRegion == nullptr )
1408         {
1409             CPLDestroyXMLNode(psTree);
1410             if( bSetError )
1411             {
1412                 VSIError(VSIE_AWSError,
1413                          "Malformed AWS XML response: %s", pszErrorMsg);
1414             }
1415             return false;
1416         }
1417         SetRegion(pszRegion);
1418         CPLDebug("S3", "Switching to region %s", m_osRegion.c_str());
1419         CPLDestroyXMLNode(psTree);
1420         return true;
1421     }
1422 
1423     if( EQUAL(pszCode, "PermanentRedirect") || EQUAL(pszCode, "TemporaryRedirect") )
1424     {
1425         const bool bIsTemporaryRedirect = EQUAL(pszCode, "TemporaryRedirect");
1426         const char* pszEndpoint =
1427             CPLGetXMLValue(psTree, "=Error.Endpoint", nullptr);
1428         if( pszEndpoint == nullptr ||
1429             (m_bUseVirtualHosting &&
1430              (strncmp(pszEndpoint, m_osBucket.c_str(),
1431                       m_osBucket.size()) != 0 ||
1432               pszEndpoint[m_osBucket.size()] != '.')) )
1433         {
1434             CPLDestroyXMLNode(psTree);
1435             if( bSetError )
1436             {
1437                 VSIError(VSIE_AWSError,
1438                          "Malformed AWS XML response: %s", pszErrorMsg);
1439             }
1440             return false;
1441         }
1442         if( !m_bUseVirtualHosting &&
1443             strncmp(pszEndpoint, m_osBucket.c_str(), m_osBucket.size()) == 0 &&
1444             pszEndpoint[m_osBucket.size()] == '.' )
1445         {
1446             /* If we have a body with
1447             <Error><Code>PermanentRedirect</Code><Message>The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.</Message><Bucket>bucket.with.dot</Bucket><Endpoint>bucket.with.dot.s3.amazonaws.com</Endpoint></Error>
1448             and headers like
1449             x-amz-bucket-region: eu-west-1
1450             and the bucket name has dot in it,
1451             then we must use s3.$(x-amz-bucket-region).amazon.com as endpoint.
1452             See #7154 */
1453             const char* pszRegionPtr = (pszHeaders != nullptr) ?
1454                 strstr(pszHeaders, "x-amz-bucket-region: "): nullptr;
1455             if( strchr(m_osBucket.c_str(), '.') != nullptr && pszRegionPtr != nullptr )
1456             {
1457                 CPLString osRegion(pszRegionPtr + strlen("x-amz-bucket-region: "));
1458                 size_t nPos = osRegion.find('\r');
1459                 if( nPos != std::string::npos )
1460                     osRegion.resize(nPos);
1461                 SetEndpoint( CPLSPrintf("s3.%s.amazonaws.com", osRegion.c_str()) );
1462                 SetRegion(osRegion.c_str());
1463                 CPLDebug("S3", "Switching to endpoint %s", m_osEndpoint.c_str());
1464                 CPLDebug("S3", "Switching to region %s", m_osRegion.c_str());
1465                 CPLDestroyXMLNode(psTree);
1466                 if( bIsTemporaryRedirect && pbUpdateMap != nullptr)
1467                     *pbUpdateMap = false;
1468                 return true;
1469             }
1470 
1471             m_bUseVirtualHosting = true;
1472             CPLDebug("S3", "Switching to virtual hosting");
1473         }
1474         SetEndpoint(
1475             m_bUseVirtualHosting
1476             ? pszEndpoint + m_osBucket.size() + 1
1477             : pszEndpoint);
1478         CPLDebug("S3", "Switching to endpoint %s", m_osEndpoint.c_str());
1479         CPLDestroyXMLNode(psTree);
1480 
1481         if( bIsTemporaryRedirect && pbUpdateMap != nullptr)
1482             *pbUpdateMap = false;
1483 
1484         return true;
1485     }
1486 
1487     if( bSetError )
1488     {
1489         // Translate AWS errors into VSI errors.
1490         const char* pszMessage = CPLGetXMLValue(psTree, "=Error.Message", nullptr);
1491 
1492         if( pszMessage == nullptr ) {
1493             VSIError(VSIE_AWSError, "%s", pszErrorMsg);
1494         } else if( EQUAL(pszCode, "AccessDenied") ) {
1495             VSIError(VSIE_AWSAccessDenied, "%s", pszMessage);
1496         } else if( EQUAL(pszCode, "NoSuchBucket") ) {
1497             VSIError(VSIE_AWSBucketNotFound, "%s", pszMessage);
1498         } else if( EQUAL(pszCode, "NoSuchKey") ) {
1499             VSIError(VSIE_AWSObjectNotFound, "%s", pszMessage);
1500         } else if( EQUAL(pszCode, "SignatureDoesNotMatch") ) {
1501             VSIError(VSIE_AWSSignatureDoesNotMatch, "%s", pszMessage);
1502         } else {
1503             VSIError(VSIE_AWSError, "%s", pszMessage);
1504         }
1505     }
1506 
1507     CPLDestroyXMLNode(psTree);
1508 
1509     return false;
1510 }
1511 
1512 /************************************************************************/
1513 /*                          SetEndpoint()                          */
1514 /************************************************************************/
1515 
SetEndpoint(const CPLString & osStr)1516 void VSIS3HandleHelper::SetEndpoint( const CPLString &osStr )
1517 {
1518     m_osEndpoint = osStr;
1519     RebuildURL();
1520 }
1521 
1522 /************************************************************************/
1523 /*                           SetRegion()                             */
1524 /************************************************************************/
1525 
SetRegion(const CPLString & osStr)1526 void VSIS3HandleHelper::SetRegion( const CPLString &osStr )
1527 {
1528     m_osRegion = osStr;
1529 }
1530 
1531 /************************************************************************/
1532 /*                           SetRequestPayer()                          */
1533 /************************************************************************/
1534 
SetRequestPayer(const CPLString & osStr)1535 void VSIS3HandleHelper::SetRequestPayer( const CPLString &osStr )
1536 {
1537     m_osRequestPayer = osStr;
1538 }
1539 
1540 /************************************************************************/
1541 /*                         SetVirtualHosting()                          */
1542 /************************************************************************/
1543 
SetVirtualHosting(bool b)1544 void VSIS3HandleHelper::SetVirtualHosting( bool b )
1545 {
1546     m_bUseVirtualHosting = b;
1547     RebuildURL();
1548 }
1549 
1550 /************************************************************************/
1551 /*                           GetSignedURL()                             */
1552 /************************************************************************/
1553 
GetSignedURL(CSLConstList papszOptions)1554 CPLString VSIS3HandleHelper::GetSignedURL(CSLConstList papszOptions)
1555 {
1556     CPLString osXAMZDate = CSLFetchNameValueDef(papszOptions, "START_DATE",
1557                                     CPLGetConfigOption("AWS_TIMESTAMP", ""));
1558     if( osXAMZDate.empty() )
1559         osXAMZDate = CPLGetAWS_SIGN4_Timestamp();
1560     CPLString osDate(osXAMZDate);
1561     osDate.resize(8);
1562 
1563     CPLString osXAMZExpires =
1564         CSLFetchNameValueDef(papszOptions, "EXPIRATION_DELAY", "3600");
1565 
1566     CPLString osVerb(CSLFetchNameValueDef(papszOptions, "VERB", "GET"));
1567 
1568     ResetQueryParameters();
1569     AddQueryParameter("X-Amz-Algorithm", "AWS4-HMAC-SHA256");
1570     AddQueryParameter("X-Amz-Credential",
1571         m_osAccessKeyId + "/" + osDate + "/" + m_osRegion + "/s3/aws4_request");
1572     AddQueryParameter("X-Amz-Date", osXAMZDate);
1573     AddQueryParameter("X-Amz-Expires", osXAMZExpires);
1574     AddQueryParameter("X-Amz-SignedHeaders", "host");
1575 
1576     CPLString osCanonicalQueryString(GetQueryString(true).substr(1));
1577 
1578     const CPLString osHost(m_bUseVirtualHosting && !m_osBucket.empty()
1579         ? CPLString(m_osBucket + "." + m_osEndpoint) : m_osEndpoint);
1580     CPLString osSignedHeaders;
1581     const CPLString osSignature =
1582       CPLGetAWS_SIGN4_Signature(
1583         m_osSecretAccessKey,
1584         m_osSessionToken,
1585         m_osRegion,
1586         m_osRequestPayer,
1587         "s3",
1588         osVerb,
1589         nullptr, /* existing headers */
1590         osHost,
1591         m_bUseVirtualHosting
1592         ? CPLAWSURLEncode("/" + m_osObjectKey, false).c_str() :
1593         CPLAWSURLEncode("/" + m_osBucket + "/" + m_osObjectKey, false).c_str(),
1594         osCanonicalQueryString,
1595         "UNSIGNED-PAYLOAD",
1596         osXAMZDate,
1597         osSignedHeaders);
1598 
1599     AddQueryParameter("X-Amz-Signature", osSignature);
1600     return m_osURL;
1601 }
1602 
1603 /************************************************************************/
1604 /*                        UpdateMapFromHandle()                         */
1605 /************************************************************************/
1606 
1607 std::mutex VSIS3UpdateParams::gsMutex{};
1608 std::map< CPLString, VSIS3UpdateParams > VSIS3UpdateParams::goMapBucketsToS3Params{};
1609 
UpdateMapFromHandle(IVSIS3LikeHandleHelper * poHandleHelper)1610 void VSIS3UpdateParams::UpdateMapFromHandle( IVSIS3LikeHandleHelper* poHandleHelper )
1611 {
1612     std::lock_guard<std::mutex> guard(gsMutex);
1613 
1614     VSIS3HandleHelper * poS3HandleHelper =
1615         cpl::down_cast<VSIS3HandleHelper *>(poHandleHelper);
1616     goMapBucketsToS3Params[ poS3HandleHelper->GetBucket() ] =
1617         VSIS3UpdateParams ( poS3HandleHelper );
1618 }
1619 
1620 /************************************************************************/
1621 /*                         UpdateHandleFromMap()                        */
1622 /************************************************************************/
1623 
UpdateHandleFromMap(IVSIS3LikeHandleHelper * poHandleHelper)1624 void VSIS3UpdateParams::UpdateHandleFromMap( IVSIS3LikeHandleHelper* poHandleHelper )
1625 {
1626     std::lock_guard<std::mutex> guard(gsMutex);
1627 
1628     VSIS3HandleHelper * poS3HandleHelper =
1629         cpl::down_cast<VSIS3HandleHelper *>(poHandleHelper);
1630     std::map< CPLString, VSIS3UpdateParams>::iterator oIter =
1631         goMapBucketsToS3Params.find(poS3HandleHelper->GetBucket());
1632     if( oIter != goMapBucketsToS3Params.end() )
1633     {
1634         oIter->second.UpdateHandlerHelper(poS3HandleHelper);
1635     }
1636 }
1637 
1638 /************************************************************************/
1639 /*                            ClearCache()                              */
1640 /************************************************************************/
1641 
ClearCache()1642 void VSIS3UpdateParams::ClearCache()
1643 {
1644     std::lock_guard<std::mutex> guard(gsMutex);
1645 
1646     goMapBucketsToS3Params.clear();
1647 }
1648 
1649 #endif
1650 
1651 //! @endcond
1652