1 /***********************************************************************************************************************************
2 S3 Storage
3 ***********************************************************************************************************************************/
4 #include "build.auto.h"
5
6 #include <string.h>
7
8 #include "common/crypto/hash.h"
9 #include "common/debug.h"
10 #include "common/io/http/client.h"
11 #include "common/io/http/common.h"
12 #include "common/io/socket/client.h"
13 #include "common/io/tls/client.h"
14 #include "common/log.h"
15 #include "common/memContext.h"
16 #include "common/regExp.h"
17 #include "common/type/object.h"
18 #include "common/type/json.h"
19 #include "common/type/xml.h"
20 #include "storage/s3/read.h"
21 #include "storage/s3/storage.intern.h"
22 #include "storage/s3/write.h"
23
24 /***********************************************************************************************************************************
25 Defaults
26 ***********************************************************************************************************************************/
27 #define STORAGE_S3_DELETE_MAX 1000
28
29 /***********************************************************************************************************************************
30 S3 HTTP headers
31 ***********************************************************************************************************************************/
32 STRING_STATIC(S3_HEADER_CONTENT_SHA256_STR, "x-amz-content-sha256");
33 STRING_STATIC(S3_HEADER_DATE_STR, "x-amz-date");
34 STRING_STATIC(S3_HEADER_TOKEN_STR, "x-amz-security-token");
35
36 /***********************************************************************************************************************************
37 S3 query tokens
38 ***********************************************************************************************************************************/
39 STRING_STATIC(S3_QUERY_CONTINUATION_TOKEN_STR, "continuation-token");
40 STRING_STATIC(S3_QUERY_DELETE_STR, "delete");
41 STRING_STATIC(S3_QUERY_DELIMITER_STR, "delimiter");
42 STRING_STATIC(S3_QUERY_LIST_TYPE_STR, "list-type");
43 STRING_STATIC(S3_QUERY_PREFIX_STR, "prefix");
44
45 STRING_STATIC(S3_QUERY_VALUE_LIST_TYPE_2_STR, "2");
46
47 /***********************************************************************************************************************************
48 XML tags
49 ***********************************************************************************************************************************/
50 STRING_STATIC(S3_XML_TAG_CODE_STR, "Code");
51 STRING_STATIC(S3_XML_TAG_COMMON_PREFIXES_STR, "CommonPrefixes");
52 STRING_STATIC(S3_XML_TAG_CONTENTS_STR, "Contents");
53 STRING_STATIC(S3_XML_TAG_DELETE_STR, "Delete");
54 STRING_STATIC(S3_XML_TAG_ERROR_STR, "Error");
55 STRING_STATIC(S3_XML_TAG_KEY_STR, "Key");
56 STRING_STATIC(S3_XML_TAG_LAST_MODIFIED_STR, "LastModified");
57 STRING_STATIC(S3_XML_TAG_MESSAGE_STR, "Message");
58 STRING_STATIC(S3_XML_TAG_NEXT_CONTINUATION_TOKEN_STR, "NextContinuationToken");
59 STRING_STATIC(S3_XML_TAG_OBJECT_STR, "Object");
60 STRING_STATIC(S3_XML_TAG_PREFIX_STR, "Prefix");
61 STRING_STATIC(S3_XML_TAG_QUIET_STR, "Quiet");
62 STRING_STATIC(S3_XML_TAG_SIZE_STR, "Size");
63
64 /***********************************************************************************************************************************
65 Constants for automatically fetching the current role and credentials
66
67 Documentation for the response format is found at: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
68 ***********************************************************************************************************************************/
69 STRING_STATIC(S3_CREDENTIAL_HOST_STR, "169.254.169.254");
70 #define S3_CREDENTIAL_PORT 80
71 #define S3_CREDENTIAL_PATH "/latest/meta-data/iam/security-credentials"
72 #define S3_CREDENTIAL_RENEW_SEC (5 * 60)
73
74 VARIANT_STRDEF_STATIC(S3_JSON_TAG_ACCESS_KEY_ID_VAR, "AccessKeyId");
75 VARIANT_STRDEF_STATIC(S3_JSON_TAG_CODE_VAR, "Code");
76 VARIANT_STRDEF_STATIC(S3_JSON_TAG_EXPIRATION_VAR, "Expiration");
77 VARIANT_STRDEF_STATIC(S3_JSON_TAG_SECRET_ACCESS_KEY_VAR, "SecretAccessKey");
78 VARIANT_STRDEF_STATIC(S3_JSON_TAG_TOKEN_VAR, "Token");
79
80 VARIANT_STRDEF_STATIC(S3_JSON_VALUE_SUCCESS_VAR, "Success");
81
82 /***********************************************************************************************************************************
83 AWS authentication v4 constants
84 ***********************************************************************************************************************************/
85 #define S3 "s3"
86 BUFFER_STRDEF_STATIC(S3_BUF, S3);
87 #define AWS4 "AWS4"
88 #define AWS4_REQUEST "aws4_request"
89 BUFFER_STRDEF_STATIC(AWS4_REQUEST_BUF, AWS4_REQUEST);
90 #define AWS4_HMAC_SHA256 "AWS4-HMAC-SHA256"
91
92 /***********************************************************************************************************************************
93 Starting date for signing string so it will be regenerated on the first request
94 ***********************************************************************************************************************************/
95 STRING_STATIC(YYYYMMDD_STR, "YYYYMMDD");
96
97 /***********************************************************************************************************************************
98 Object type
99 ***********************************************************************************************************************************/
100 struct StorageS3
101 {
102 STORAGE_COMMON_MEMBER;
103 MemContext *memContext;
104 HttpClient *httpClient; // HTTP client to service requests
105 StringList *headerRedactList; // List of headers to redact from logging
106
107 const String *bucket; // Bucket to store data in
108 const String *region; // e.g. us-east-1
109 StorageS3KeyType keyType; // Key type (e.g. storageS3KeyTypeShared)
110 String *accessKey; // Access key
111 String *secretAccessKey; // Secret access key
112 String *securityToken; // Security token, if any
113 size_t partSize; // Part size for multi-part upload
114 unsigned int deleteMax; // Maximum objects that can be deleted in one request
115 StorageS3UriStyle uriStyle; // Path or host style URIs
116 const String *bucketEndpoint; // Set to {bucket}.{endpoint}
117
118 // For retrieving temporary security credentials
119 HttpClient *credHttpClient; // HTTP client to service credential requests
120 const String *credHost; // Credentials host
121 const String *credRole; // Role to use for credential requests
122 time_t credExpirationTime; // Time the temporary credentials expire
123
124 // Current signing key and date it is valid for
125 const String *signingKeyDate; // Date of cached signing key (so we know when to regenerate)
126 const Buffer *signingKey; // Cached signing key
127 };
128
129 /***********************************************************************************************************************************
130 Expected ISO-8601 data/time size
131 ***********************************************************************************************************************************/
132 #define ISO_8601_DATE_TIME_SIZE 16
133
134 /***********************************************************************************************************************************
135 Format ISO-8601 date/time for authentication
136 ***********************************************************************************************************************************/
137 static String *
storageS3DateTime(time_t authTime)138 storageS3DateTime(time_t authTime)
139 {
140 FUNCTION_TEST_BEGIN();
141 FUNCTION_TEST_PARAM(TIME, authTime);
142 FUNCTION_TEST_END();
143
144 struct tm timePart;
145 char buffer[ISO_8601_DATE_TIME_SIZE + 1];
146
147 THROW_ON_SYS_ERROR(
148 strftime(buffer, sizeof(buffer), "%Y%m%dT%H%M%SZ", gmtime_r(&authTime, &timePart)) != ISO_8601_DATE_TIME_SIZE, AssertError,
149 "unable to format date");
150
151 FUNCTION_TEST_RETURN(strNewZ(buffer));
152 }
153
154 /***********************************************************************************************************************************
155 Generate authorization header and add it to the supplied header list
156
157 Based on the excellent documentation at http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html.
158 ***********************************************************************************************************************************/
159 static void
storageS3Auth(StorageS3 * this,const String * verb,const String * path,const HttpQuery * query,const String * dateTime,HttpHeader * httpHeader,const String * payloadHash)160 storageS3Auth(
161 StorageS3 *this, const String *verb, const String *path, const HttpQuery *query, const String *dateTime,
162 HttpHeader *httpHeader, const String *payloadHash)
163 {
164 FUNCTION_TEST_BEGIN();
165 FUNCTION_TEST_PARAM(STORAGE_S3, this);
166 FUNCTION_TEST_PARAM(STRING, verb);
167 FUNCTION_TEST_PARAM(STRING, path);
168 FUNCTION_TEST_PARAM(HTTP_QUERY, query);
169 FUNCTION_TEST_PARAM(STRING, dateTime);
170 FUNCTION_TEST_PARAM(KEY_VALUE, httpHeader);
171 FUNCTION_TEST_PARAM(STRING, payloadHash);
172 FUNCTION_TEST_END();
173
174 ASSERT(verb != NULL);
175 ASSERT(path != NULL);
176 ASSERT(dateTime != NULL);
177 ASSERT(httpHeader != NULL);
178 ASSERT(payloadHash != NULL);
179
180 MEM_CONTEXT_TEMP_BEGIN()
181 {
182 // Get date from datetime
183 const String *date = strSubN(dateTime, 0, 8);
184
185 // Set required headers
186 httpHeaderPut(httpHeader, S3_HEADER_CONTENT_SHA256_STR, payloadHash);
187 httpHeaderPut(httpHeader, S3_HEADER_DATE_STR, dateTime);
188 httpHeaderPut(httpHeader, HTTP_HEADER_HOST_STR, this->bucketEndpoint);
189
190 if (this->securityToken != NULL)
191 httpHeaderPut(httpHeader, S3_HEADER_TOKEN_STR, this->securityToken);
192
193 // Generate canonical request and signed headers
194 const StringList *headerList = strLstSort(strLstDup(httpHeaderList(httpHeader)), sortOrderAsc);
195 String *signedHeaders = NULL;
196
197 String *canonicalRequest = strNewFmt(
198 "%s\n%s\n%s\n", strZ(verb), strZ(path), query == NULL ? "" : strZ(httpQueryRenderP(query)));
199
200 for (unsigned int headerIdx = 0; headerIdx < strLstSize(headerList); headerIdx++)
201 {
202 const String *headerKey = strLstGet(headerList, headerIdx);
203 const String *headerKeyLower = strLower(strDup(headerKey));
204
205 // Skip the authorization (exists on retry) and content-length headers since they do not need to be signed
206 if (strEq(headerKeyLower, HTTP_HEADER_AUTHORIZATION_STR) || strEq(headerKeyLower, HTTP_HEADER_CONTENT_LENGTH_STR))
207 continue;
208
209 strCatFmt(canonicalRequest, "%s:%s\n", strZ(headerKeyLower), strZ(httpHeaderGet(httpHeader, headerKey)));
210
211 if (signedHeaders == NULL)
212 signedHeaders = strDup(headerKeyLower);
213 else
214 strCatFmt(signedHeaders, ";%s", strZ(headerKeyLower));
215 }
216
217 strCatFmt(canonicalRequest, "\n%s\n%s", strZ(signedHeaders), strZ(payloadHash));
218
219 // Generate string to sign
220 const String *stringToSign = strNewFmt(
221 AWS4_HMAC_SHA256 "\n%s\n%s/%s/" S3 "/" AWS4_REQUEST "\n%s", strZ(dateTime), strZ(date), strZ(this->region),
222 strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA256_STR, BUFSTR(canonicalRequest)))));
223
224 // Generate signing key. This key only needs to be regenerated every seven days but we'll do it once a day to keep the
225 // logic simple. It's a relatively expensive operation so we'd rather not do it for every request.
226 // If the cached signing key has expired (or has none been generated) then regenerate it
227 if (!strEq(date, this->signingKeyDate))
228 {
229 const Buffer *dateKey = cryptoHmacOne(
230 HASH_TYPE_SHA256_STR, BUFSTR(strNewFmt(AWS4 "%s", strZ(this->secretAccessKey))), BUFSTR(date));
231 const Buffer *regionKey = cryptoHmacOne(HASH_TYPE_SHA256_STR, dateKey, BUFSTR(this->region));
232 const Buffer *serviceKey = cryptoHmacOne(HASH_TYPE_SHA256_STR, regionKey, S3_BUF);
233
234 // Switch to the object context so signing key and date are not lost
235 MEM_CONTEXT_BEGIN(this->memContext)
236 {
237 this->signingKey = cryptoHmacOne(HASH_TYPE_SHA256_STR, serviceKey, AWS4_REQUEST_BUF);
238 this->signingKeyDate = strDup(date);
239 }
240 MEM_CONTEXT_END();
241 }
242
243 // Generate authorization header
244 const String *authorization = strNewFmt(
245 AWS4_HMAC_SHA256 " Credential=%s/%s/%s/" S3 "/" AWS4_REQUEST ",SignedHeaders=%s,Signature=%s",
246 strZ(this->accessKey), strZ(date), strZ(this->region), strZ(signedHeaders),
247 strZ(bufHex(cryptoHmacOne(HASH_TYPE_SHA256_STR, this->signingKey, BUFSTR(stringToSign)))));
248
249 httpHeaderPut(httpHeader, HTTP_HEADER_AUTHORIZATION_STR, authorization);
250 }
251 MEM_CONTEXT_TEMP_END();
252
253 FUNCTION_TEST_RETURN_VOID();
254 }
255
256 /***********************************************************************************************************************************
257 Process S3 request
258 ***********************************************************************************************************************************/
259 // Helper to convert YYYY-MM-DDTHH:MM:SS.MSECZ format to time_t. This format is very nearly ISO-8601 except for the inclusion of
260 // milliseconds, which are discarded here.
261 static time_t
storageS3CvtTime(const String * time)262 storageS3CvtTime(const String *time)
263 {
264 FUNCTION_TEST_BEGIN();
265 FUNCTION_TEST_PARAM(STRING, time);
266 FUNCTION_TEST_END();
267
268 FUNCTION_TEST_RETURN(
269 epochFromParts(
270 cvtZToInt(strZ(strSubN(time, 0, 4))), cvtZToInt(strZ(strSubN(time, 5, 2))),
271 cvtZToInt(strZ(strSubN(time, 8, 2))), cvtZToInt(strZ(strSubN(time, 11, 2))),
272 cvtZToInt(strZ(strSubN(time, 14, 2))), cvtZToInt(strZ(strSubN(time, 17, 2))), 0));
273 }
274
275 HttpRequest *
storageS3RequestAsync(StorageS3 * this,const String * verb,const String * path,StorageS3RequestAsyncParam param)276 storageS3RequestAsync(StorageS3 *this, const String *verb, const String *path, StorageS3RequestAsyncParam param)
277 {
278 FUNCTION_LOG_BEGIN(logLevelDebug);
279 FUNCTION_LOG_PARAM(STORAGE_S3, this);
280 FUNCTION_LOG_PARAM(STRING, verb);
281 FUNCTION_LOG_PARAM(STRING, path);
282 FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
283 FUNCTION_LOG_PARAM(BUFFER, param.content);
284 FUNCTION_LOG_END();
285
286 ASSERT(this != NULL);
287 ASSERT(verb != NULL);
288 ASSERT(path != NULL);
289
290 HttpRequest *result = NULL;
291
292 MEM_CONTEXT_TEMP_BEGIN()
293 {
294 HttpHeader *requestHeader = httpHeaderNew(this->headerRedactList);
295
296 // Set content length
297 httpHeaderAdd(
298 requestHeader, HTTP_HEADER_CONTENT_LENGTH_STR,
299 param.content == NULL || bufEmpty(param.content) ? ZERO_STR : strNewFmt("%zu", bufUsed(param.content)));
300
301 // Calculate content-md5 header if there is content
302 if (param.content != NULL)
303 {
304 httpHeaderAdd(
305 requestHeader, HTTP_HEADER_CONTENT_MD5_STR,
306 strNewEncode(encodeBase64, cryptoHashOne(HASH_TYPE_MD5_STR, param.content)));
307 }
308
309 // When using path-style URIs the bucket name needs to be prepended
310 if (this->uriStyle == storageS3UriStylePath)
311 path = strNewFmt("/%s%s", strZ(this->bucket), strZ(path));
312
313 // If temp crendentials will be expiring soon then renew them
314 if (this->keyType == storageS3KeyTypeAuto && (this->credExpirationTime - time(NULL)) < S3_CREDENTIAL_RENEW_SEC)
315 {
316 // Set content-length and host headers
317 HttpHeader *credHeader = httpHeaderNew(NULL);
318 httpHeaderAdd(credHeader, HTTP_HEADER_CONTENT_LENGTH_STR, ZERO_STR);
319 httpHeaderAdd(credHeader, HTTP_HEADER_HOST_STR, this->credHost);
320
321 // If the role was not set explicitly or retrieved previously then retrieve it
322 if (this->credRole == NULL)
323 {
324 // Request the role
325 HttpRequest *request = httpRequestNewP(
326 this->credHttpClient, HTTP_VERB_GET_STR, STRDEF(S3_CREDENTIAL_PATH), .header = credHeader);
327 HttpResponse *response = httpRequestResponse(request, true);
328
329 // Not found likely means no role is associated with this instance
330 if (httpResponseCode(response) == HTTP_RESPONSE_CODE_NOT_FOUND)
331 {
332 THROW(
333 ProtocolError,
334 "role to retrieve temporary credentials not found\n"
335 "HINT: is a valid IAM role associated with this instance?");
336 }
337 // Else an error that we can't handle
338 else if (!httpResponseCodeOk(response))
339 httpRequestError(request, response);
340
341 // Get role from the text response
342 MEM_CONTEXT_BEGIN(this->memContext)
343 {
344 this->credRole = strNewBuf(httpResponseContent(response));
345 }
346 MEM_CONTEXT_END();
347 }
348
349 // Retrieve the temp credentials
350 HttpRequest *request = httpRequestNewP(
351 this->credHttpClient, HTTP_VERB_GET_STR, strNewFmt(S3_CREDENTIAL_PATH "/%s", strZ(this->credRole)),
352 .header = credHeader);
353 HttpResponse *response = httpRequestResponse(request, true);
354
355 // Not found likely means the role is not valid
356 if (httpResponseCode(response) == HTTP_RESPONSE_CODE_NOT_FOUND)
357 {
358 THROW_FMT(
359 ProtocolError,
360 "role '%s' not found\n"
361 "HINT: is '%s' a valid IAM role associated with this instance?",
362 strZ(this->credRole), strZ(this->credRole));
363 }
364 // Else an error that we can't handle
365 else if (!httpResponseCodeOk(response))
366 httpRequestError(request, response);
367
368 // Free old credentials
369 strFree(this->accessKey);
370 strFree(this->secretAccessKey);
371 strFree(this->securityToken);
372
373 // Get credentials from the JSON response
374 KeyValue *credential = jsonToKv(strNewBuf(httpResponseContent(response)));
375
376 MEM_CONTEXT_BEGIN(this->memContext)
377 {
378 // Check the code field for errors
379 const Variant *code = kvGetDefault(credential, S3_JSON_TAG_CODE_VAR, VARSTRDEF("code field is missing"));
380 CHECK(code != NULL);
381
382 if (!varEq(code, S3_JSON_VALUE_SUCCESS_VAR))
383 THROW_FMT(FormatError, "unable to retrieve temporary credentials: %s", strZ(varStr(code)));
384
385 // Make sure the required values are present
386 CHECK(kvGet(credential, S3_JSON_TAG_ACCESS_KEY_ID_VAR) != NULL);
387 CHECK(kvGet(credential, S3_JSON_TAG_SECRET_ACCESS_KEY_VAR) != NULL);
388 CHECK(kvGet(credential, S3_JSON_TAG_TOKEN_VAR) != NULL);
389
390 // Copy credentials
391 this->accessKey = strDup(varStr(kvGet(credential, S3_JSON_TAG_ACCESS_KEY_ID_VAR)));
392 this->secretAccessKey = strDup(varStr(kvGet(credential, S3_JSON_TAG_SECRET_ACCESS_KEY_VAR)));
393 this->securityToken = strDup(varStr(kvGet(credential, S3_JSON_TAG_TOKEN_VAR)));
394 }
395 MEM_CONTEXT_END();
396
397 // Update expiration time
398 CHECK(kvGet(credential, S3_JSON_TAG_EXPIRATION_VAR) != NULL);
399 this->credExpirationTime = storageS3CvtTime(varStr(kvGet(credential, S3_JSON_TAG_EXPIRATION_VAR)));
400
401 // Reset the signing key date so the signing key gets regenerated
402 this->signingKeyDate = YYYYMMDD_STR;
403 }
404
405 // Encode path
406 path = httpUriEncode(path, true);
407
408 // Generate authorization header
409 storageS3Auth(
410 this, verb, path, param.query, storageS3DateTime(time(NULL)), requestHeader,
411 param.content == NULL || bufEmpty(param.content) ?
412 HASH_TYPE_SHA256_ZERO_STR : bufHex(cryptoHashOne(HASH_TYPE_SHA256_STR, param.content)));
413
414 // Send request
415 MEM_CONTEXT_PRIOR_BEGIN()
416 {
417 result = httpRequestNewP(
418 this->httpClient, verb, path, .query = param.query, .header = requestHeader, .content = param.content);
419 }
420 MEM_CONTEXT_END();
421 }
422 MEM_CONTEXT_TEMP_END();
423
424 FUNCTION_LOG_RETURN(HTTP_REQUEST, result);
425 }
426
427 HttpResponse *
storageS3Response(HttpRequest * request,StorageS3ResponseParam param)428 storageS3Response(HttpRequest *request, StorageS3ResponseParam param)
429 {
430 FUNCTION_LOG_BEGIN(logLevelDebug);
431 FUNCTION_LOG_PARAM(HTTP_REQUEST, request);
432 FUNCTION_LOG_PARAM(BOOL, param.allowMissing);
433 FUNCTION_LOG_PARAM(BOOL, param.contentIo);
434 FUNCTION_LOG_END();
435
436 ASSERT(request != NULL);
437
438 HttpResponse *result = NULL;
439
440 MEM_CONTEXT_TEMP_BEGIN()
441 {
442 // Get response
443 result = httpRequestResponse(request, !param.contentIo);
444
445 // Error if the request was not successful
446 if (!httpResponseCodeOk(result) && (!param.allowMissing || httpResponseCode(result) != HTTP_RESPONSE_CODE_NOT_FOUND))
447 httpRequestError(request, result);
448
449 // Move response to the prior context
450 httpResponseMove(result, memContextPrior());
451 }
452 MEM_CONTEXT_TEMP_END();
453
454 FUNCTION_LOG_RETURN(HTTP_RESPONSE, result);
455 }
456
457 HttpResponse *
storageS3Request(StorageS3 * this,const String * verb,const String * path,StorageS3RequestParam param)458 storageS3Request(StorageS3 *this, const String *verb, const String *path, StorageS3RequestParam param)
459 {
460 FUNCTION_LOG_BEGIN(logLevelDebug);
461 FUNCTION_LOG_PARAM(STORAGE_S3, this);
462 FUNCTION_LOG_PARAM(STRING, verb);
463 FUNCTION_LOG_PARAM(STRING, path);
464 FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
465 FUNCTION_LOG_PARAM(BUFFER, param.content);
466 FUNCTION_LOG_PARAM(BOOL, param.allowMissing);
467 FUNCTION_LOG_PARAM(BOOL, param.contentIo);
468 FUNCTION_LOG_END();
469
470 FUNCTION_LOG_RETURN(
471 HTTP_RESPONSE,
472 storageS3ResponseP(
473 storageS3RequestAsyncP(this, verb, path, .query = param.query, .content = param.content),
474 .allowMissing = param.allowMissing, .contentIo = param.contentIo));
475 }
476
477 /***********************************************************************************************************************************
478 General function for listing files to be used by other list routines
479 ***********************************************************************************************************************************/
480 static void
storageS3ListInternal(StorageS3 * this,const String * path,StorageInfoLevel level,const String * expression,bool recurse,StorageInfoListCallback callback,void * callbackData)481 storageS3ListInternal(
482 StorageS3 *this, const String *path, StorageInfoLevel level, const String *expression, bool recurse,
483 StorageInfoListCallback callback, void *callbackData)
484 {
485 FUNCTION_LOG_BEGIN(logLevelDebug);
486 FUNCTION_LOG_PARAM(STORAGE_S3, this);
487 FUNCTION_LOG_PARAM(STRING, path);
488 FUNCTION_LOG_PARAM(ENUM, level);
489 FUNCTION_LOG_PARAM(STRING, expression);
490 FUNCTION_LOG_PARAM(BOOL, recurse);
491 FUNCTION_LOG_PARAM(FUNCTIONP, callback);
492 FUNCTION_LOG_PARAM_P(VOID, callbackData);
493 FUNCTION_LOG_END();
494
495 ASSERT(this != NULL);
496 ASSERT(path != NULL);
497
498 MEM_CONTEXT_TEMP_BEGIN()
499 {
500 // Build the base prefix by stripping off the initial /
501 const String *basePrefix;
502
503 if (strSize(path) == 1)
504 basePrefix = EMPTY_STR;
505 else
506 basePrefix = strNewFmt("%s/", strZ(strSub(path, 1)));
507
508 // Get the expression prefix when possible to limit initial results
509 const String *expressionPrefix = regExpPrefix(expression);
510
511 // If there is an expression prefix then use it to build the query prefix, otherwise query prefix is base prefix
512 const String *queryPrefix;
513
514 if (expressionPrefix == NULL)
515 queryPrefix = basePrefix;
516 else
517 {
518 if (strEmpty(basePrefix))
519 queryPrefix = expressionPrefix;
520 else
521 queryPrefix = strNewFmt("%s%s", strZ(basePrefix), strZ(expressionPrefix));
522 }
523
524 // Create query
525 HttpQuery *query = httpQueryNewP();
526
527 // Add the delimiter to not recurse
528 if (!recurse)
529 httpQueryAdd(query, S3_QUERY_DELIMITER_STR, FSLASH_STR);
530
531 // Use list type 2
532 httpQueryAdd(query, S3_QUERY_LIST_TYPE_STR, S3_QUERY_VALUE_LIST_TYPE_2_STR);
533
534 // Don't specify empty prefix because it is the default
535 if (!strEmpty(queryPrefix))
536 httpQueryAdd(query, S3_QUERY_PREFIX_STR, queryPrefix);
537
538 // Loop as long as a continuation token returned
539 HttpRequest *request = NULL;
540
541 do
542 {
543 // Use an inner mem context here because we could potentially be retrieving millions of files so it is a good idea to
544 // free memory at regular intervals
545 MEM_CONTEXT_TEMP_BEGIN()
546 {
547 HttpResponse *response = NULL;
548
549 // If there is an outstanding async request then wait for the response
550 if (request != NULL)
551 {
552 response = storageS3ResponseP(request);
553
554 httpRequestFree(request);
555 request = NULL;
556 }
557 // Else get the response immediately from a sync request
558 else
559 response = storageS3RequestP(this, HTTP_VERB_GET_STR, FSLASH_STR, query);
560
561 XmlNode *xmlRoot = xmlDocumentRoot(xmlDocumentNewBuf(httpResponseContent(response)));
562
563 // If a continuation token exists then send an async request to get more data
564 const String *continuationToken = xmlNodeContent(
565 xmlNodeChild(xmlRoot, S3_XML_TAG_NEXT_CONTINUATION_TOKEN_STR, false));
566
567 if (continuationToken != NULL)
568 {
569 httpQueryPut(query, S3_QUERY_CONTINUATION_TOKEN_STR, continuationToken);
570
571 // Store request in the outer temp context
572 MEM_CONTEXT_PRIOR_BEGIN()
573 {
574 request = storageS3RequestAsyncP(this, HTTP_VERB_GET_STR, FSLASH_STR, query);
575 }
576 MEM_CONTEXT_PRIOR_END();
577 }
578
579 // Get prefix list
580 XmlNodeList *subPathList = xmlNodeChildList(xmlRoot, S3_XML_TAG_COMMON_PREFIXES_STR);
581
582 for (unsigned int subPathIdx = 0; subPathIdx < xmlNodeLstSize(subPathList); subPathIdx++)
583 {
584 const XmlNode *subPathNode = xmlNodeLstGet(subPathList, subPathIdx);
585
586 // Get path name
587 StorageInfo info =
588 {
589 .level = level,
590 .name = xmlNodeContent(xmlNodeChild(subPathNode, S3_XML_TAG_PREFIX_STR, true)),
591 .exists = true,
592 };
593
594 // Strip off base prefix and final /
595 info.name = strSubN(info.name, strSize(basePrefix), strSize(info.name) - strSize(basePrefix) - 1);
596
597 // Add type info if requested
598 if (level >= storageInfoLevelType)
599 info.type = storageTypePath;
600
601 // Callback with info
602 callback(callbackData, &info);
603 }
604
605 // Get file list
606 XmlNodeList *fileList = xmlNodeChildList(xmlRoot, S3_XML_TAG_CONTENTS_STR);
607
608 for (unsigned int fileIdx = 0; fileIdx < xmlNodeLstSize(fileList); fileIdx++)
609 {
610 const XmlNode *fileNode = xmlNodeLstGet(fileList, fileIdx);
611
612 // Get file name
613 StorageInfo info =
614 {
615 .level = level,
616 .name = xmlNodeContent(xmlNodeChild(fileNode, S3_XML_TAG_KEY_STR, true)),
617 .exists = true,
618 };
619
620 // Strip off the base prefix when present
621 if (!strEmpty(basePrefix))
622 info.name = strSub(info.name, strSize(basePrefix));
623
624 // Add basic info if requested (no need to add type info since file is default type)
625 if (level >= storageInfoLevelBasic)
626 {
627 info.size = cvtZToUInt64(strZ(xmlNodeContent(xmlNodeChild(fileNode, S3_XML_TAG_SIZE_STR, true))));
628 info.timeModified = storageS3CvtTime(
629 xmlNodeContent(xmlNodeChild(fileNode, S3_XML_TAG_LAST_MODIFIED_STR, true)));
630 }
631
632 // Callback with info
633 callback(callbackData, &info);
634 }
635 }
636 MEM_CONTEXT_TEMP_END();
637 }
638 while (request != NULL);
639 }
640 MEM_CONTEXT_TEMP_END();
641
642 FUNCTION_LOG_RETURN_VOID();
643 }
644
645 /**********************************************************************************************************************************/
646 static StorageInfo
storageS3Info(THIS_VOID,const String * file,StorageInfoLevel level,StorageInterfaceInfoParam param)647 storageS3Info(THIS_VOID, const String *file, StorageInfoLevel level, StorageInterfaceInfoParam param)
648 {
649 THIS(StorageS3);
650
651 FUNCTION_LOG_BEGIN(logLevelTrace);
652 FUNCTION_LOG_PARAM(STORAGE_S3, this);
653 FUNCTION_LOG_PARAM(STRING, file);
654 FUNCTION_LOG_PARAM(ENUM, level);
655 (void)param; // No parameters are used
656 FUNCTION_LOG_END();
657
658 ASSERT(this != NULL);
659 ASSERT(file != NULL);
660
661 // Attempt to get file info
662 HttpResponse *httpResponse = storageS3RequestP(this, HTTP_VERB_HEAD_STR, file, .allowMissing = true);
663
664 // Does the file exist?
665 StorageInfo result = {.level = level, .exists = httpResponseCodeOk(httpResponse)};
666
667 // Add basic level info if requested and the file exists (no need to add type info since file is default type)
668 if (result.level >= storageInfoLevelBasic && result.exists)
669 {
670 const HttpHeader *httpHeader = httpResponseHeader(httpResponse);
671
672 const String *const contentLength = httpHeaderGet(httpHeader, HTTP_HEADER_CONTENT_LENGTH_STR);
673 CHECK(contentLength != NULL);
674 result.size = cvtZToUInt64(strZ(contentLength));
675
676 const String *const lastModified = httpHeaderGet(httpHeader, HTTP_HEADER_LAST_MODIFIED_STR);
677 CHECK(lastModified != NULL);
678 result.timeModified = httpDateToTime(lastModified);
679 }
680
681 FUNCTION_LOG_RETURN(STORAGE_INFO, result);
682 }
683
684 /**********************************************************************************************************************************/
685 static bool
storageS3InfoList(THIS_VOID,const String * path,StorageInfoLevel level,StorageInfoListCallback callback,void * callbackData,StorageInterfaceInfoListParam param)686 storageS3InfoList(
687 THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
688 StorageInterfaceInfoListParam param)
689 {
690 THIS(StorageS3);
691
692 FUNCTION_LOG_BEGIN(logLevelTrace);
693 FUNCTION_LOG_PARAM(STORAGE_S3, this);
694 FUNCTION_LOG_PARAM(STRING, path);
695 FUNCTION_LOG_PARAM(ENUM, level);
696 FUNCTION_LOG_PARAM(FUNCTIONP, callback);
697 FUNCTION_LOG_PARAM_P(VOID, callbackData);
698 FUNCTION_LOG_PARAM(STRING, param.expression);
699 FUNCTION_LOG_END();
700
701 ASSERT(this != NULL);
702 ASSERT(path != NULL);
703 ASSERT(callback != NULL);
704
705 storageS3ListInternal(this, path, level, param.expression, false, callback, callbackData);
706
707 FUNCTION_LOG_RETURN(BOOL, true);
708 }
709
710 /**********************************************************************************************************************************/
711 static StorageRead *
storageS3NewRead(THIS_VOID,const String * file,bool ignoreMissing,StorageInterfaceNewReadParam param)712 storageS3NewRead(THIS_VOID, const String *file, bool ignoreMissing, StorageInterfaceNewReadParam param)
713 {
714 THIS(StorageS3);
715
716 FUNCTION_LOG_BEGIN(logLevelDebug);
717 FUNCTION_LOG_PARAM(STORAGE_S3, this);
718 FUNCTION_LOG_PARAM(STRING, file);
719 FUNCTION_LOG_PARAM(BOOL, ignoreMissing);
720 (void)param; // No parameters are used
721 FUNCTION_LOG_END();
722
723 ASSERT(this != NULL);
724 ASSERT(file != NULL);
725
726 FUNCTION_LOG_RETURN(STORAGE_READ, storageReadS3New(this, file, ignoreMissing));
727 }
728
729 /**********************************************************************************************************************************/
730 static StorageWrite *
storageS3NewWrite(THIS_VOID,const String * file,StorageInterfaceNewWriteParam param)731 storageS3NewWrite(THIS_VOID, const String *file, StorageInterfaceNewWriteParam param)
732 {
733 THIS(StorageS3);
734
735 FUNCTION_LOG_BEGIN(logLevelDebug);
736 FUNCTION_LOG_PARAM(STORAGE_S3, this);
737 FUNCTION_LOG_PARAM(STRING, file);
738 (void)param; // No parameters are used
739 FUNCTION_LOG_END();
740
741 ASSERT(this != NULL);
742 ASSERT(file != NULL);
743 ASSERT(param.createPath);
744 ASSERT(param.user == NULL);
745 ASSERT(param.group == NULL);
746 ASSERT(param.timeModified == 0);
747
748 FUNCTION_LOG_RETURN(STORAGE_WRITE, storageWriteS3New(this, file, this->partSize));
749 }
750
751 /**********************************************************************************************************************************/
752 typedef struct StorageS3PathRemoveData
753 {
754 StorageS3 *this; // Storage object
755 MemContext *memContext; // Mem context to create xml document in
756 unsigned int size; // Size of delete request
757 HttpRequest *request; // Async delete request
758 XmlDocument *xml; // Delete xml
759 const String *path; // Root path of remove
760 } StorageS3PathRemoveData;
761
762 static HttpRequest *
storageS3PathRemoveInternal(StorageS3 * this,HttpRequest * request,XmlDocument * xml)763 storageS3PathRemoveInternal(StorageS3 *this, HttpRequest *request, XmlDocument *xml)
764 {
765 FUNCTION_TEST_BEGIN();
766 FUNCTION_TEST_PARAM(STORAGE_S3, this);
767 FUNCTION_TEST_PARAM(HTTP_REQUEST, request);
768 FUNCTION_TEST_PARAM(XML_DOCUMENT, xml);
769 FUNCTION_TEST_END();
770
771 ASSERT(this != NULL);
772
773 // Get response for async request
774 if (request != NULL)
775 {
776 const Buffer *response = httpResponseContent(storageS3ResponseP(request));
777
778 // Nothing is returned when there are no errors
779 if (!bufEmpty(response))
780 {
781 XmlNodeList *errorList = xmlNodeChildList(xmlDocumentRoot(xmlDocumentNewBuf(response)), S3_XML_TAG_ERROR_STR);
782
783 if (xmlNodeLstSize(errorList) > 0)
784 {
785 XmlNode *error = xmlNodeLstGet(errorList, 0);
786
787 THROW_FMT(
788 FileRemoveError, STORAGE_ERROR_PATH_REMOVE_FILE ": [%s] %s",
789 strZ(xmlNodeContent(xmlNodeChild(error, S3_XML_TAG_KEY_STR, true))),
790 strZ(xmlNodeContent(xmlNodeChild(error, S3_XML_TAG_CODE_STR, true))),
791 strZ(xmlNodeContent(xmlNodeChild(error, S3_XML_TAG_MESSAGE_STR, true))));
792 }
793 }
794
795 httpRequestFree(request);
796 }
797
798 // Send new async request if there is more to remove
799 HttpRequest *result = NULL;
800
801 if (xml != NULL)
802 {
803 result = storageS3RequestAsyncP(
804 this, HTTP_VERB_POST_STR, FSLASH_STR, .query = httpQueryAdd(httpQueryNewP(), S3_QUERY_DELETE_STR, EMPTY_STR),
805 .content = xmlDocumentBuf(xml));
806 }
807
808 FUNCTION_TEST_RETURN(result);
809 }
810
811 static void
storageS3PathRemoveCallback(void * callbackData,const StorageInfo * info)812 storageS3PathRemoveCallback(void *callbackData, const StorageInfo *info)
813 {
814 FUNCTION_TEST_BEGIN();
815 FUNCTION_TEST_PARAM_P(VOID, callbackData);
816 FUNCTION_TEST_PARAM(STORAGE_INFO, info);
817 FUNCTION_TEST_END();
818
819 ASSERT(callbackData != NULL);
820 ASSERT(info != NULL);
821
822 // Only delete files since paths don't really exist
823 if (info->type == storageTypeFile)
824 {
825 StorageS3PathRemoveData *data = (StorageS3PathRemoveData *)callbackData;
826
827 // If there is something to delete then create the request
828 if (data->xml == NULL)
829 {
830 MEM_CONTEXT_BEGIN(data->memContext)
831 {
832 data->xml = xmlDocumentNew(S3_XML_TAG_DELETE_STR);
833 xmlNodeContentSet(xmlNodeAdd(xmlDocumentRoot(data->xml), S3_XML_TAG_QUIET_STR), TRUE_STR);
834 }
835 MEM_CONTEXT_END();
836 }
837
838 // Add to delete list
839 xmlNodeContentSet(
840 xmlNodeAdd(xmlNodeAdd(xmlDocumentRoot(data->xml), S3_XML_TAG_OBJECT_STR), S3_XML_TAG_KEY_STR),
841 strNewFmt("%s%s", strZ(data->path), strZ(info->name)));
842 data->size++;
843
844 // Delete list when it is full
845 if (data->size == data->this->deleteMax)
846 {
847 MEM_CONTEXT_BEGIN(data->memContext)
848 {
849 data->request = storageS3PathRemoveInternal(data->this, data->request, data->xml);
850 }
851 MEM_CONTEXT_END();
852
853 xmlDocumentFree(data->xml);
854 data->xml = NULL;
855 data->size = 0;
856 }
857 }
858
859 FUNCTION_TEST_RETURN_VOID();
860 }
861
862 static bool
storageS3PathRemove(THIS_VOID,const String * path,bool recurse,StorageInterfacePathRemoveParam param)863 storageS3PathRemove(THIS_VOID, const String *path, bool recurse, StorageInterfacePathRemoveParam param)
864 {
865 THIS(StorageS3);
866
867 FUNCTION_LOG_BEGIN(logLevelDebug);
868 FUNCTION_LOG_PARAM(STORAGE_S3, this);
869 FUNCTION_LOG_PARAM(STRING, path);
870 FUNCTION_LOG_PARAM(BOOL, recurse);
871 (void)param; // No parameters are used
872 FUNCTION_LOG_END();
873
874 ASSERT(this != NULL);
875 ASSERT(path != NULL);
876
877 MEM_CONTEXT_TEMP_BEGIN()
878 {
879 StorageS3PathRemoveData data =
880 {
881 .this = this,
882 .memContext = memContextCurrent(),
883 .path = strEq(path, FSLASH_STR) ? EMPTY_STR : strNewFmt("%s/", strZ(strSub(path, 1))),
884 };
885
886 storageS3ListInternal(this, path, storageInfoLevelType, NULL, true, storageS3PathRemoveCallback, &data);
887
888 // Call if there is more to be removed
889 if (data.xml != NULL)
890 data.request = storageS3PathRemoveInternal(this, data.request, data.xml);
891
892 // Check response on last async request
893 storageS3PathRemoveInternal(this, data.request, NULL);
894 }
895 MEM_CONTEXT_TEMP_END();
896
897 FUNCTION_LOG_RETURN(BOOL, true);
898 }
899
900 /**********************************************************************************************************************************/
901 static void
storageS3Remove(THIS_VOID,const String * file,StorageInterfaceRemoveParam param)902 storageS3Remove(THIS_VOID, const String *file, StorageInterfaceRemoveParam param)
903 {
904 THIS(StorageS3);
905
906 FUNCTION_LOG_BEGIN(logLevelDebug);
907 FUNCTION_LOG_PARAM(STORAGE_S3, this);
908 FUNCTION_LOG_PARAM(STRING, file);
909 FUNCTION_LOG_PARAM(BOOL, param.errorOnMissing);
910 FUNCTION_LOG_END();
911
912 ASSERT(this != NULL);
913 ASSERT(file != NULL);
914 ASSERT(!param.errorOnMissing);
915
916 storageS3RequestP(this, HTTP_VERB_DELETE_STR, file);
917
918 FUNCTION_LOG_RETURN_VOID();
919 }
920
921 /**********************************************************************************************************************************/
922 static const StorageInterface storageInterfaceS3 =
923 {
924 .info = storageS3Info,
925 .infoList = storageS3InfoList,
926 .newRead = storageS3NewRead,
927 .newWrite = storageS3NewWrite,
928 .pathRemove = storageS3PathRemove,
929 .remove = storageS3Remove,
930 };
931
932 Storage *
storageS3New(const String * path,bool write,StoragePathExpressionCallback pathExpressionFunction,const String * bucket,const String * endPoint,StorageS3UriStyle uriStyle,const String * region,StorageS3KeyType keyType,const String * accessKey,const String * secretAccessKey,const String * securityToken,const String * credRole,size_t partSize,const String * host,unsigned int port,TimeMSec timeout,bool verifyPeer,const String * caFile,const String * caPath)933 storageS3New(
934 const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
935 const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
936 const String *secretAccessKey, const String *securityToken, const String *credRole, size_t partSize, const String *host,
937 unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath)
938 {
939 FUNCTION_LOG_BEGIN(logLevelDebug);
940 FUNCTION_LOG_PARAM(STRING, path);
941 FUNCTION_LOG_PARAM(BOOL, write);
942 FUNCTION_LOG_PARAM(FUNCTIONP, pathExpressionFunction);
943 FUNCTION_LOG_PARAM(STRING, bucket);
944 FUNCTION_LOG_PARAM(STRING, endPoint);
945 FUNCTION_LOG_PARAM(STRING_ID, uriStyle);
946 FUNCTION_LOG_PARAM(STRING, region);
947 FUNCTION_LOG_PARAM(STRING_ID, keyType);
948 FUNCTION_TEST_PARAM(STRING, accessKey);
949 FUNCTION_TEST_PARAM(STRING, secretAccessKey);
950 FUNCTION_TEST_PARAM(STRING, securityToken);
951 FUNCTION_TEST_PARAM(STRING, credRole);
952 FUNCTION_LOG_PARAM(SIZE, partSize);
953 FUNCTION_LOG_PARAM(STRING, host);
954 FUNCTION_LOG_PARAM(UINT, port);
955 FUNCTION_LOG_PARAM(TIME_MSEC, timeout);
956 FUNCTION_LOG_PARAM(BOOL, verifyPeer);
957 FUNCTION_LOG_PARAM(STRING, caFile);
958 FUNCTION_LOG_PARAM(STRING, caPath);
959 FUNCTION_LOG_END();
960
961 ASSERT(path != NULL);
962 ASSERT(bucket != NULL);
963 ASSERT(endPoint != NULL);
964 ASSERT(region != NULL);
965 ASSERT(
966 (keyType == storageS3KeyTypeShared && accessKey != NULL && secretAccessKey != NULL) ||
967 (keyType == storageS3KeyTypeAuto && accessKey == NULL && secretAccessKey == NULL && securityToken == NULL));
968 ASSERT(partSize != 0);
969
970 Storage *this = NULL;
971
972 MEM_CONTEXT_NEW_BEGIN("StorageS3")
973 {
974 StorageS3 *driver = memNew(sizeof(StorageS3));
975
976 *driver = (StorageS3)
977 {
978 .memContext = MEM_CONTEXT_NEW(),
979 .interface = storageInterfaceS3,
980 .bucket = strDup(bucket),
981 .region = strDup(region),
982 .keyType = keyType,
983 .accessKey = strDup(accessKey),
984 .secretAccessKey = strDup(secretAccessKey),
985 .securityToken = strDup(securityToken),
986 .partSize = partSize,
987 .deleteMax = STORAGE_S3_DELETE_MAX,
988 .uriStyle = uriStyle,
989 .bucketEndpoint = uriStyle == storageS3UriStyleHost ?
990 strNewFmt("%s.%s", strZ(bucket), strZ(endPoint)) : strDup(endPoint),
991 .credHost = S3_CREDENTIAL_HOST_STR,
992 .credRole = strDup(credRole),
993
994 // Force the signing key to be generated on the first run
995 .signingKeyDate = YYYYMMDD_STR,
996 };
997
998 // Create the HTTP client used to service requests
999 if (host == NULL)
1000 host = driver->bucketEndpoint;
1001
1002 driver->httpClient = httpClientNew(
1003 tlsClientNew(sckClientNew(host, port, timeout), host, timeout, verifyPeer, caFile, caPath), timeout);
1004
1005 // Create the HTTP client used to retreive temporary security credentials
1006 if (driver->keyType == storageS3KeyTypeAuto)
1007 driver->credHttpClient = httpClientNew(sckClientNew(driver->credHost, S3_CREDENTIAL_PORT, timeout), timeout);
1008
1009 // Create list of redacted headers
1010 driver->headerRedactList = strLstNew();
1011 strLstAdd(driver->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
1012 strLstAdd(driver->headerRedactList, S3_HEADER_DATE_STR);
1013 strLstAdd(driver->headerRedactList, S3_HEADER_TOKEN_STR);
1014
1015 this = storageNew(STORAGE_S3_TYPE, path, 0, 0, write, pathExpressionFunction, driver, driver->interface);
1016 }
1017 MEM_CONTEXT_NEW_END();
1018
1019 FUNCTION_LOG_RETURN(STORAGE, this);
1020 }
1021