1 /**********************************************************************
2 *
3 * Name: cpl_alibaba_oss.h
4 * Project: CPL - Common Portability Library
5 * Purpose: Alibaba Cloud Object Storage Service
6 * Author: Even Rouault <even.rouault at spatialys.com>
7 *
8 **********************************************************************
9 * Copyright (c) 2017, Even Rouault <even.rouault at spatialys.com>
10 *
11 * Permission is hereby granted, free of charge, to any person obtaining a
12 * copy of this software and associated documentation files (the "Software"),
13 * to deal in the Software without restriction, including without limitation
14 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 * and/or sell copies of the Software, and to permit persons to whom the
16 * Software is furnished to do so, subject to the following conditions:
17 *
18 * The above copyright notice and this permission notice shall be included
19 * in all copies or substantial portions of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 * DEALINGS IN THE SOFTWARE.
28 ****************************************************************************/
29
30 //! @cond Doxygen_Suppress
31
32 #include "cpl_alibaba_oss.h"
33 #include "cpl_vsi_error.h"
34 #include "cpl_time.h"
35 #include "cpl_minixml.h"
36 #include "cpl_multiproc.h"
37 #include "cpl_http.h"
38 #include "cpl_sha1.h"
39 #include <algorithm>
40
41 CPL_CVSID("$Id: cpl_alibaba_oss.cpp 5318f6d39d2006a10cb6c1410334c56d76a74aa6 2018-06-20 16:38:42 +0200 Even Rouault $")
42
43 // #define DEBUG_VERBOSE 1
44
45 #ifdef HAVE_CURL
46
47 /************************************************************************/
48 /* GetSignature() */
49 /************************************************************************/
50
GetSignature(const CPLString & osStringToSign,const CPLString & osSecretAccessKey)51 static CPLString GetSignature(const CPLString& osStringToSign,
52 const CPLString& osSecretAccessKey )
53 {
54
55 /* -------------------------------------------------------------------- */
56 /* Compute signature. */
57 /* -------------------------------------------------------------------- */
58 GByte abySignature[CPL_SHA1_HASH_SIZE] = {};
59 CPL_HMAC_SHA1( osSecretAccessKey.c_str(), osSecretAccessKey.size(),
60 osStringToSign, osStringToSign.size(),
61 abySignature);
62 char* pszBase64 = CPLBase64Encode( sizeof(abySignature), abySignature );
63 CPLString osSignature(pszBase64);
64 CPLFree(pszBase64);
65
66 return osSignature;
67 }
68
69 /************************************************************************/
70 /* CPLGetOSSHeaders() */
71 /************************************************************************/
72
73 // See:
74 // https://www.alibabacloud.com/help/doc-detail/31951.htm?spm=a3c0i.o31982en.b99.178.5HUTqV
75 static struct curl_slist*
CPLGetOSSHeaders(const CPLString & osSecretAccessKey,const CPLString & osAccessKeyId,const CPLString & osVerb,const struct curl_slist * psExistingHeaders,const CPLString & osCanonicalizedResource)76 CPLGetOSSHeaders( const CPLString& osSecretAccessKey,
77 const CPLString& osAccessKeyId,
78 const CPLString& osVerb,
79 const struct curl_slist* psExistingHeaders,
80 const CPLString& osCanonicalizedResource )
81 {
82 CPLString osDate = CPLGetConfigOption("CPL_OSS_TIMESTAMP", "");
83 if( osDate.empty() )
84 {
85 osDate = IVSIS3LikeHandleHelper::GetRFC822DateTime();
86 }
87
88 std::map<CPLString, CPLString> oSortedMapHeaders;
89 CPLString osCanonicalizedHeaders(
90 IVSIS3LikeHandleHelper::BuildCanonicalizedHeaders(
91 oSortedMapHeaders,
92 psExistingHeaders,
93 "x-oss-"));
94
95 CPLString osStringToSign;
96 osStringToSign += osVerb + "\n";
97 osStringToSign += CPLAWSGetHeaderVal(psExistingHeaders, "Content-MD5") + "\n";
98 osStringToSign += CPLAWSGetHeaderVal(psExistingHeaders, "Content-Type") + "\n";
99 osStringToSign += osDate + "\n";
100 osStringToSign += osCanonicalizedHeaders;
101 osStringToSign += osCanonicalizedResource;
102 #ifdef DEBUG_VERBOSE
103 CPLDebug("OSS", "osStringToSign = %s", osStringToSign.c_str());
104 #endif
105
106 /* -------------------------------------------------------------------- */
107 /* Build authorization header. */
108 /* -------------------------------------------------------------------- */
109
110 CPLString osAuthorization("OSS ");
111 osAuthorization += osAccessKeyId;
112 osAuthorization += ":";
113 osAuthorization += GetSignature(osStringToSign, osSecretAccessKey);
114
115 #ifdef DEBUG_VERBOSE
116 CPLDebug("OSS", "osAuthorization='%s'", osAuthorization.c_str());
117 #endif
118
119 struct curl_slist *headers=nullptr;
120 headers = curl_slist_append(
121 headers, CPLSPrintf("Date: %s", osDate.c_str()));
122 headers = curl_slist_append(
123 headers, CPLSPrintf("Authorization: %s", osAuthorization.c_str()));
124 return headers;
125 }
126
127 /************************************************************************/
128 /* VSIOSSHandleHelper() */
129 /************************************************************************/
VSIOSSHandleHelper(const CPLString & osSecretAccessKey,const CPLString & osAccessKeyId,const CPLString & osEndpoint,const CPLString & osBucket,const CPLString & osObjectKey,bool bUseHTTPS,bool bUseVirtualHosting)130 VSIOSSHandleHelper::VSIOSSHandleHelper( const CPLString& osSecretAccessKey,
131 const CPLString& osAccessKeyId,
132 const CPLString& osEndpoint,
133 const CPLString& osBucket,
134 const CPLString& osObjectKey,
135 bool bUseHTTPS,
136 bool bUseVirtualHosting ) :
137 m_osURL(BuildURL(osEndpoint, osBucket, osObjectKey, bUseHTTPS,
138 bUseVirtualHosting)),
139 m_osSecretAccessKey(osSecretAccessKey),
140 m_osAccessKeyId(osAccessKeyId),
141 m_osEndpoint(osEndpoint),
142 m_osBucket(osBucket),
143 m_osObjectKey(osObjectKey),
144 m_bUseHTTPS(bUseHTTPS),
145 m_bUseVirtualHosting(bUseVirtualHosting)
146 {}
147
148 /************************************************************************/
149 /* ~VSIOSSHandleHelper() */
150 /************************************************************************/
151
~VSIOSSHandleHelper()152 VSIOSSHandleHelper::~VSIOSSHandleHelper()
153 {
154 for( size_t i = 0; i < m_osSecretAccessKey.size(); i++ )
155 m_osSecretAccessKey[i] = 0;
156 }
157
158 /************************************************************************/
159 /* BuildURL() */
160 /************************************************************************/
161
BuildURL(const CPLString & osEndpoint,const CPLString & osBucket,const CPLString & osObjectKey,bool bUseHTTPS,bool bUseVirtualHosting)162 CPLString VSIOSSHandleHelper::BuildURL(const CPLString& osEndpoint,
163 const CPLString& osBucket,
164 const CPLString& osObjectKey,
165 bool bUseHTTPS, bool bUseVirtualHosting)
166 {
167 const char* pszProtocol = (bUseHTTPS) ? "https" : "http";
168 if( osBucket.empty() )
169 {
170 return CPLSPrintf("%s://%s", pszProtocol,
171 osEndpoint.c_str());
172 }
173 else if( bUseVirtualHosting )
174 return CPLSPrintf("%s://%s.%s/%s", pszProtocol,
175 osBucket.c_str(),
176 osEndpoint.c_str(),
177 CPLAWSURLEncode(osObjectKey, false).c_str());
178 else
179 return CPLSPrintf("%s://%s/%s/%s", pszProtocol,
180 osEndpoint.c_str(),
181 osBucket.c_str(),
182 CPLAWSURLEncode(osObjectKey, false).c_str());
183 }
184
185 /************************************************************************/
186 /* RebuildURL() */
187 /************************************************************************/
188
RebuildURL()189 void VSIOSSHandleHelper::RebuildURL()
190 {
191 m_osURL = BuildURL(m_osEndpoint, m_osBucket, m_osObjectKey,
192 m_bUseHTTPS, m_bUseVirtualHosting);
193 m_osURL += GetQueryString(false);
194 }
195
196 /************************************************************************/
197 /* GetConfiguration() */
198 /************************************************************************/
199
GetConfiguration(CSLConstList papszOptions,CPLString & osSecretAccessKey,CPLString & osAccessKeyId)200 bool VSIOSSHandleHelper::GetConfiguration(CSLConstList papszOptions,
201 CPLString& osSecretAccessKey,
202 CPLString& osAccessKeyId)
203 {
204 osSecretAccessKey = CSLFetchNameValueDef(papszOptions,
205 "OSS_SECRET_ACCESS_KEY",
206 CPLGetConfigOption("OSS_SECRET_ACCESS_KEY", ""));
207
208 if( !osSecretAccessKey.empty() )
209 {
210 osAccessKeyId = CSLFetchNameValueDef(papszOptions,
211 "OSS_ACCESS_KEY_ID",
212 CPLGetConfigOption("OSS_ACCESS_KEY_ID", ""));
213 if( osAccessKeyId.empty() )
214 {
215 VSIError(VSIE_AWSInvalidCredentials,
216 "OSS_ACCESS_KEY_ID configuration option not defined");
217 return false;
218 }
219
220 return true;
221 }
222
223 VSIError(VSIE_AWSInvalidCredentials,
224 "OSS_SECRET_ACCESS_KEY configuration option not defined");
225 return false;
226 }
227
228 /************************************************************************/
229 /* BuildFromURI() */
230 /************************************************************************/
231
BuildFromURI(const char * pszURI,const char * pszFSPrefix,bool bAllowNoObject,CSLConstList papszOptions)232 VSIOSSHandleHelper* VSIOSSHandleHelper::BuildFromURI( const char* pszURI,
233 const char* pszFSPrefix,
234 bool bAllowNoObject,
235 CSLConstList papszOptions )
236 {
237 CPLString osSecretAccessKey;
238 CPLString osAccessKeyId;
239 if( !GetConfiguration(papszOptions, osSecretAccessKey, osAccessKeyId) )
240 {
241 return nullptr;
242 }
243
244 const CPLString osEndpoint = CSLFetchNameValueDef(papszOptions,
245 "OSS_ENDPOINT",
246 CPLGetConfigOption("OSS_ENDPOINT", "oss-us-east-1.aliyuncs.com"));
247 CPLString osBucket;
248 CPLString osObjectKey;
249 if( pszURI != nullptr && pszURI[0] != '\0' &&
250 !GetBucketAndObjectKey(pszURI, pszFSPrefix, bAllowNoObject,
251 osBucket, osObjectKey) )
252 {
253 return nullptr;
254 }
255 const bool bUseHTTPS = CPLTestBool(CPLGetConfigOption("OSS_HTTPS", "YES"));
256 const bool bIsValidNameForVirtualHosting =
257 osBucket.find('.') == std::string::npos;
258 const bool bUseVirtualHosting = CPLTestBool(
259 CPLGetConfigOption("OSS_VIRTUAL_HOSTING",
260 bIsValidNameForVirtualHosting ? "TRUE" : "FALSE"));
261 return new VSIOSSHandleHelper(osSecretAccessKey, osAccessKeyId,
262 osEndpoint,
263 osBucket, osObjectKey, bUseHTTPS,
264 bUseVirtualHosting);
265 }
266
267 /************************************************************************/
268 /* GetCurlHeaders() */
269 /************************************************************************/
270
271 struct curl_slist *
GetCurlHeaders(const CPLString & osVerb,const struct curl_slist * psExistingHeaders,const void *,size_t) const272 VSIOSSHandleHelper::GetCurlHeaders( const CPLString& osVerb,
273 const struct curl_slist* psExistingHeaders,
274 const void * /*pabyDataContent*/,
275 size_t /*nBytesContent*/ ) const
276 {
277 CPLString osCanonicalQueryString;
278 if( !m_osObjectKey.empty() )
279 {
280 osCanonicalQueryString = GetQueryString(false);
281 }
282
283 CPLString osCanonicalizedResource( m_osBucket.empty() ? CPLString("/") :
284 "/" + m_osBucket + "/" + m_osObjectKey );
285 osCanonicalizedResource += osCanonicalQueryString;
286
287 return CPLGetOSSHeaders(
288 m_osSecretAccessKey,
289 m_osAccessKeyId,
290 osVerb,
291 psExistingHeaders,
292 osCanonicalizedResource);
293 }
294
295 /************************************************************************/
296 /* CanRestartOnError() */
297 /************************************************************************/
298
CanRestartOnError(const char * pszErrorMsg,const char *,bool bSetError,bool * pbUpdateMap)299 bool VSIOSSHandleHelper::CanRestartOnError( const char* pszErrorMsg,
300 const char*,
301 bool bSetError,
302 bool* pbUpdateMap )
303 {
304 #ifdef DEBUG_VERBOSE
305 CPLDebug("OSS", "%s", pszErrorMsg);
306 #endif
307
308 if( pbUpdateMap != nullptr )
309 *pbUpdateMap = true;
310
311 if( !STARTS_WITH(pszErrorMsg, "<?xml") )
312 {
313 if( bSetError )
314 {
315 VSIError(VSIE_AWSError, "Invalid AWS response: %s", pszErrorMsg);
316 }
317 return false;
318 }
319
320 CPLXMLNode* psTree = CPLParseXMLString(pszErrorMsg);
321 if( psTree == nullptr )
322 {
323 if( bSetError )
324 {
325 VSIError(VSIE_AWSError,
326 "Malformed AWS XML response: %s", pszErrorMsg);
327 }
328 return false;
329 }
330
331 const char* pszCode = CPLGetXMLValue(psTree, "=Error.Code", nullptr);
332 if( pszCode == nullptr )
333 {
334 CPLDestroyXMLNode(psTree);
335 if( bSetError )
336 {
337 VSIError(VSIE_AWSError,
338 "Malformed AWS XML response: %s", pszErrorMsg);
339 }
340 return false;
341 }
342
343 if( EQUAL(pszCode, "AccessDenied") )
344 {
345 const char* pszEndpoint =
346 CPLGetXMLValue(psTree, "=Error.Endpoint", nullptr);
347 if( pszEndpoint && pszEndpoint != m_osEndpoint )
348 {
349 SetEndpoint(pszEndpoint);
350 CPLDebug("OSS", "Switching to endpoint %s", m_osEndpoint.c_str());
351 CPLDestroyXMLNode(psTree);
352 return true;
353 }
354 }
355
356 if( bSetError )
357 {
358 // Translate AWS errors into VSI errors.
359 const char* pszMessage = CPLGetXMLValue(psTree, "=Error.Message", nullptr);
360
361 if( pszMessage == nullptr ) {
362 VSIError(VSIE_AWSError, "%s", pszErrorMsg);
363 } else if( EQUAL(pszCode, "AccessDenied") ) {
364 VSIError(VSIE_AWSAccessDenied, "%s", pszMessage);
365 } else if( EQUAL(pszCode, "NoSuchBucket") ) {
366 VSIError(VSIE_AWSBucketNotFound, "%s", pszMessage);
367 } else if( EQUAL(pszCode, "NoSuchKey") ) {
368 VSIError(VSIE_AWSObjectNotFound, "%s", pszMessage);
369 } else if( EQUAL(pszCode, "SignatureDoesNotMatch") ) {
370 VSIError(VSIE_AWSSignatureDoesNotMatch, "%s", pszMessage);
371 } else {
372 VSIError(VSIE_AWSError, "%s", pszMessage);
373 }
374 }
375
376 CPLDestroyXMLNode(psTree);
377
378 return false;
379 }
380
381 /************************************************************************/
382 /* SetEndpoint() */
383 /************************************************************************/
384
SetEndpoint(const CPLString & osStr)385 void VSIOSSHandleHelper::SetEndpoint( const CPLString &osStr )
386 {
387 m_osEndpoint = osStr;
388 RebuildURL();
389 }
390
391 /************************************************************************/
392 /* GetSignedURL() */
393 /************************************************************************/
394
GetSignedURL(CSLConstList papszOptions)395 CPLString VSIOSSHandleHelper::GetSignedURL(CSLConstList papszOptions)
396 {
397 GIntBig nStartDate = static_cast<GIntBig>(time(nullptr));
398 const char* pszStartDate = CSLFetchNameValue(papszOptions, "START_DATE");
399 if( pszStartDate )
400 {
401 int nYear, nMonth, nDay, nHour, nMin, nSec;
402 if( sscanf(pszStartDate, "%04d%02d%02dT%02d%02d%02dZ",
403 &nYear, &nMonth, &nDay, &nHour, &nMin, &nSec) == 6 )
404 {
405 struct tm brokendowntime;
406 brokendowntime.tm_year = nYear - 1900;
407 brokendowntime.tm_mon = nMonth - 1;
408 brokendowntime.tm_mday = nDay;
409 brokendowntime.tm_hour = nHour;
410 brokendowntime.tm_min = nMin;
411 brokendowntime.tm_sec = nSec;
412 nStartDate = CPLYMDHMSToUnixTime(&brokendowntime);
413 }
414 }
415 GIntBig nExpiresIn = nStartDate + atoi(
416 CSLFetchNameValueDef(papszOptions, "EXPIRATION_DELAY", "3600"));
417 CPLString osExpires(CSLFetchNameValueDef(papszOptions, "EXPIRES",
418 CPLSPrintf(CPL_FRMT_GIB, nExpiresIn)));
419
420 CPLString osVerb(CSLFetchNameValueDef(papszOptions, "VERB", "GET"));
421
422 CPLString osCanonicalizedResource( m_osBucket.empty() ? CPLString("/") :
423 "/" + m_osBucket + "/" + m_osObjectKey );
424
425 CPLString osStringToSign;
426 osStringToSign += osVerb + "\n";
427 osStringToSign += "\n";
428 osStringToSign += "\n";
429 osStringToSign += osExpires + "\n";
430 // osStringToSign += ; // osCanonicalizedHeaders;
431 osStringToSign += osCanonicalizedResource;
432 #ifdef DEBUG_VERBOSE
433 CPLDebug("OSS", "osStringToSign = %s", osStringToSign.c_str());
434 #endif
435
436 CPLString osSignature(GetSignature(osStringToSign, m_osSecretAccessKey));
437
438 ResetQueryParameters();
439 // Note: https://www.alibabacloud.com/help/doc-detail/31952.htm?spm=a3c0i.o32002en.b99.294.6d70a0fc7cRJfJ is wrong on the name of the OSSAccessKeyId parameter !
440 AddQueryParameter("OSSAccessKeyId", m_osAccessKeyId);
441 AddQueryParameter("Expires", osExpires);
442 AddQueryParameter("Signature", osSignature);
443 return m_osURL;
444 }
445
446 #endif // HAVE_CURL
447
448 //! @endcond
449