1 /***********************************************************************************************************************************
2 Test S3 Storage
3 ***********************************************************************************************************************************/
4 #include <unistd.h>
5
6 #include "common/io/fdRead.h"
7 #include "common/io/fdWrite.h"
8 #include "version.h"
9
10 #include "common/harnessConfig.h"
11 #include "common/harnessFork.h"
12 #include "common/harnessServer.h"
13 #include "common/harnessStorage.h"
14
15 /***********************************************************************************************************************************
16 Constants
17 ***********************************************************************************************************************************/
18 #define S3_TEST_HOST "s3.amazonaws.com"
19
20 /***********************************************************************************************************************************
21 Helper to build test requests
22 ***********************************************************************************************************************************/
23 typedef struct TestRequestParam
24 {
25 VAR_PARAM_HEADER;
26 const char *content;
27 const char *accessKey;
28 const char *securityToken;
29 } TestRequestParam;
30
31 #define testRequestP(write, s3, verb, path, ...) \
32 testRequest(write, s3, verb, path, (TestRequestParam){VAR_PARAM_INIT, __VA_ARGS__})
33
34 static void
testRequest(IoWrite * write,Storage * s3,const char * verb,const char * path,TestRequestParam param)35 testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, TestRequestParam param)
36 {
37 // Get security token from param
38 const char *securityToken = param.securityToken == NULL ? NULL : param.securityToken;
39
40 // If s3 storage is set then get the driver
41 StorageS3 *driver = NULL;
42
43 if (s3 != NULL)
44 {
45 driver = (StorageS3 *)storageDriver(s3);
46
47 // Also update the security token if it is not already set
48 if (param.securityToken == NULL)
49 securityToken = strZNull(driver->securityToken);
50 }
51
52 // Add request
53 String *request = strNewFmt("%s %s HTTP/1.1\r\nuser-agent:" PROJECT_NAME "/" PROJECT_VERSION "\r\n", verb, path);
54
55 // Add authorization header when s3 service
56 if (s3 != NULL)
57 {
58 strCatFmt(
59 request,
60 "authorization:AWS4-HMAC-SHA256 Credential=%s/\?\?\?\?\?\?\?\?/us-east-1/s3/aws4_request,SignedHeaders=",
61 param.accessKey == NULL ? strZ(driver->accessKey) : param.accessKey);
62
63 if (param.content != NULL)
64 strCatZ(request, "content-md5;");
65
66 strCatZ(request, "host;x-amz-content-sha256;x-amz-date");
67
68 if (securityToken != NULL)
69 strCatZ(request, ";x-amz-security-token");
70
71 strCatZ(request, ",Signature=????????????????????????????????????????????????????????????????\r\n");
72 }
73
74 // Add content-length
75 strCatFmt(request, "content-length:%zu\r\n", param.content != NULL ? strlen(param.content) : 0);
76
77 // Add md5
78 if (param.content != NULL)
79 {
80 strCatFmt(
81 request, "content-md5:%s\r\n",
82 strZ(strNewEncode(encodeBase64, cryptoHashOne(HASH_TYPE_MD5_STR, BUFSTRZ(param.content)))));
83 }
84
85 // Add host
86 if (s3 != NULL)
87 {
88 if (driver->uriStyle == storageS3UriStyleHost)
89 strCatFmt(request, "host:bucket." S3_TEST_HOST "\r\n");
90 else
91 strCatFmt(request, "host:" S3_TEST_HOST "\r\n");
92 }
93 else
94 strCatFmt(request, "host:%s\r\n", strZ(hrnServerHost()));
95
96 // Add content checksum and date if s3 service
97 if (s3 != NULL)
98 {
99 // Add content sha256 and date
100 strCatFmt(
101 request,
102 "x-amz-content-sha256:%s\r\n"
103 "x-amz-date:????????T??????Z" "\r\n",
104 param.content == NULL ? HASH_TYPE_SHA256_ZERO : strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA256_STR,
105 BUFSTRZ(param.content)))));
106
107 // Add security token
108 if (securityToken != NULL)
109 strCatFmt(request, "x-amz-security-token:%s\r\n", securityToken);
110 }
111
112 // Add final \r\n
113 strCatZ(request, "\r\n");
114
115 // Add content
116 if (param.content != NULL)
117 strCatZ(request, param.content);
118
119 hrnServerScriptExpect(write, request);
120 }
121
122 /***********************************************************************************************************************************
123 Helper to build test responses
124 ***********************************************************************************************************************************/
125 typedef struct TestResponseParam
126 {
127 VAR_PARAM_HEADER;
128 unsigned int code;
129 const char *http;
130 const char *header;
131 const char *content;
132 } TestResponseParam;
133
134 #define testResponseP(write, ...) \
135 testResponse(write, (TestResponseParam){VAR_PARAM_INIT, __VA_ARGS__})
136
137 static void
testResponse(IoWrite * write,TestResponseParam param)138 testResponse(IoWrite *write, TestResponseParam param)
139 {
140 // Set code to 200 if not specified
141 param.code = param.code == 0 ? 200 : param.code;
142
143 // Output header and code
144 String *response = strNewFmt("HTTP/%s %u ", param.http == NULL ? "1.1" : param.http, param.code);
145
146 // Add reason for some codes
147 switch (param.code)
148 {
149 case 200:
150 strCatZ(response, "OK");
151 break;
152
153 case 403:
154 strCatZ(response, "Forbidden");
155 break;
156 }
157
158 // End header
159 strCatZ(response, "\r\n");
160
161 // Headers
162 if (param.header != NULL)
163 strCatFmt(response, "%s\r\n", param.header);
164
165 // Content
166 if (param.content != NULL)
167 {
168 strCatFmt(
169 response,
170 "content-length:%zu\r\n"
171 "\r\n"
172 "%s",
173 strlen(param.content), param.content);
174 }
175 else
176 strCatZ(response, "\r\n");
177
178 hrnServerScriptReply(write, response);
179 }
180
181 /***********************************************************************************************************************************
182 Format ISO-8601 date with - and :
183 ***********************************************************************************************************************************/
184 static String *
testS3DateTime(time_t time)185 testS3DateTime(time_t time)
186 {
187 FUNCTION_HARNESS_BEGIN();
188 FUNCTION_HARNESS_PARAM(TIME, time);
189 FUNCTION_HARNESS_END();
190
191 char buffer[21];
192
193 THROW_ON_SYS_ERROR(
194 strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%SZ", gmtime(&time)) != sizeof(buffer) - 1, AssertError,
195 "unable to format date");
196
197 FUNCTION_HARNESS_RETURN(STRING, strNewZ(buffer));
198 }
199
200 /***********************************************************************************************************************************
201 Test Run
202 ***********************************************************************************************************************************/
203 void
testRun(void)204 testRun(void)
205 {
206 FUNCTION_HARNESS_VOID();
207
208 // Test strings
209 const String *path = STRDEF("/");
210 const String *bucket = STRDEF("bucket");
211 const String *region = STRDEF("us-east-1");
212 const String *endPoint = STRDEF("s3.amazonaws.com");
213 const String *host = hrnServerHost();
214 const unsigned int port = hrnServerPort(0);
215 const unsigned int authPort = hrnServerPort(1);
216 const String *accessKey = STRDEF("AKIAIOSFODNN7EXAMPLE");
217 const String *secretAccessKey = STRDEF("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY");
218 const String *securityToken = STRDEF(
219 "AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/q"
220 "kPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xV"
221 "qr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA==");
222 const String *credRole = STRDEF("credrole");
223
224 // Config settings that are required for every test (without endpoint for special tests)
225 StringList *commonArgWithoutEndpointList = strLstNew();
226 hrnCfgArgRawZ(commonArgWithoutEndpointList, cfgOptStanza, "db");
227 hrnCfgArgRawZ(commonArgWithoutEndpointList, cfgOptRepoType, "s3");
228 hrnCfgArgRaw(commonArgWithoutEndpointList, cfgOptRepoPath, path);
229 hrnCfgArgRaw(commonArgWithoutEndpointList, cfgOptRepoS3Bucket, bucket);
230 hrnCfgArgRaw(commonArgWithoutEndpointList, cfgOptRepoS3Region, region);
231
232 // TLS can only be verified in a container
233 if (!TEST_IN_CONTAINER)
234 hrnCfgArgRawBool(commonArgWithoutEndpointList, cfgOptRepoStorageVerifyTls, false);
235
236 // Config settings that are required for every test (with endpoint)
237 StringList *commonArgList = strLstDup(commonArgWithoutEndpointList);
238 hrnCfgArgRaw(commonArgList, cfgOptRepoS3Endpoint, endPoint);
239
240 // Secure options must be loaded into environment variables
241 hrnCfgEnvRaw(cfgOptRepoS3Key, accessKey);
242 hrnCfgEnvRaw(cfgOptRepoS3KeySecret, secretAccessKey);
243
244 // *****************************************************************************************************************************
245 if (testBegin("storageS3DateTime() and storageS3Auth()"))
246 {
247 TEST_RESULT_STR_Z(storageS3DateTime(1491267845), "20170404T010405Z", "static date");
248
249 // -------------------------------------------------------------------------------------------------------------------------
250 TEST_TITLE("config without token");
251
252 StringList *argList = strLstDup(commonArgList);
253 HRN_CFG_LOAD(cfgCmdArchivePush, argList);
254
255 StorageS3 *driver = (StorageS3 *)storageDriver(storageRepoGet(0, false));
256
257 TEST_RESULT_STR(driver->bucket, bucket, "check bucket");
258 TEST_RESULT_STR(driver->region, region, "check region");
259 TEST_RESULT_STR(driver->bucketEndpoint, strNewFmt("%s.%s", strZ(bucket), strZ(endPoint)), "check host");
260 TEST_RESULT_STR(driver->accessKey, accessKey, "check access key");
261 TEST_RESULT_STR(driver->secretAccessKey, secretAccessKey, "check secret access key");
262 TEST_RESULT_STR(driver->securityToken, NULL, "check security token");
263 TEST_RESULT_STR(
264 httpClientToLog(driver->httpClient),
265 strNewFmt(
266 "{ioClient: {type: tls, driver: {ioClient: {type: socket, driver: {host: bucket.s3.amazonaws.com, port: 443"
267 ", timeout: 60000}}, timeout: 60000, verifyPeer: %s}}, reusable: 0, timeout: 60000}",
268 cvtBoolToConstZ(TEST_IN_CONTAINER)),
269 "check http client");
270
271 // -------------------------------------------------------------------------------------------------------------------------
272 TEST_TITLE("auth with token");
273
274 HttpHeader *header = httpHeaderNew(NULL);
275 HttpQuery *query = httpQueryNewP();
276 httpQueryAdd(query, STRDEF("list-type"), STRDEF("2"));
277
278 TEST_RESULT_VOID(
279 storageS3Auth(driver, STRDEF("GET"), STRDEF("/"), query, STRDEF("20170606T121212Z"), header, HASH_TYPE_SHA256_ZERO_STR),
280 "generate authorization");
281 TEST_RESULT_STR_Z(
282 httpHeaderGet(header, STRDEF("authorization")),
283 "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request,"
284 "SignedHeaders=host;x-amz-content-sha256;x-amz-date,"
285 "Signature=cb03bf1d575c1f8904dabf0e573990375340ab293ef7ad18d049fc1338fd89b3",
286 "check authorization header");
287
288 // Test again to be sure cache signing key is used
289 const Buffer *lastSigningKey = driver->signingKey;
290
291 TEST_RESULT_VOID(
292 storageS3Auth(driver, STRDEF("GET"), STRDEF("/"), query, STRDEF("20170606T121212Z"), header, HASH_TYPE_SHA256_ZERO_STR),
293 "generate authorization");
294 TEST_RESULT_STR_Z(
295 httpHeaderGet(header, STRDEF("authorization")),
296 "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request,"
297 "SignedHeaders=host;x-amz-content-sha256;x-amz-date,"
298 "Signature=cb03bf1d575c1f8904dabf0e573990375340ab293ef7ad18d049fc1338fd89b3",
299 "check authorization header");
300 TEST_RESULT_BOOL(driver->signingKey == lastSigningKey, true, "check signing key was reused");
301
302 // -------------------------------------------------------------------------------------------------------------------------
303 TEST_TITLE("change date to generate new signing key");
304
305 TEST_RESULT_VOID(
306 storageS3Auth(driver, STRDEF("GET"), STRDEF("/"), query, STRDEF("20180814T080808Z"), header, HASH_TYPE_SHA256_ZERO_STR),
307 "generate authorization");
308 TEST_RESULT_STR_Z(
309 httpHeaderGet(header, STRDEF("authorization")),
310 "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20180814/us-east-1/s3/aws4_request,"
311 "SignedHeaders=host;x-amz-content-sha256;x-amz-date,"
312 "Signature=d0fa9c36426eb94cdbaf287a7872c7a3b6c913f523163d0d7debba0758e36f49",
313 "check authorization header");
314 TEST_RESULT_BOOL(driver->signingKey != lastSigningKey, true, "check signing key was regenerated");
315
316 // -------------------------------------------------------------------------------------------------------------------------
317 TEST_TITLE("config with token, endpoint with custom port, and ca-file/path");
318
319 argList = strLstDup(commonArgWithoutEndpointList);
320 hrnCfgArgRawZ(argList, cfgOptRepoS3Endpoint, "custom.endpoint:333");
321 hrnCfgArgRawZ(argList, cfgOptRepoStorageCaPath, "/path/to/cert");
322 hrnCfgArgRawZ(argList, cfgOptRepoStorageCaFile, HRN_PATH_REPO "/" HRN_SERVER_CERT_PREFIX ".crt");
323 hrnCfgEnvRaw(cfgOptRepoS3Token, securityToken);
324 HRN_CFG_LOAD(cfgCmdArchivePush, argList);
325
326 driver = (StorageS3 *)storageDriver(storageRepoGet(0, false));
327
328 TEST_RESULT_STR(driver->securityToken, securityToken, "check security token");
329 TEST_RESULT_STR(
330 httpClientToLog(driver->httpClient),
331 strNewFmt(
332 "{ioClient: {type: tls, driver: {ioClient: {type: socket, driver: {host: bucket.custom.endpoint, port: 333"
333 ", timeout: 60000}}, timeout: 60000, verifyPeer: %s}}, reusable: 0, timeout: 60000}",
334 cvtBoolToConstZ(TEST_IN_CONTAINER)),
335 "check http client");
336
337 // -------------------------------------------------------------------------------------------------------------------------
338 TEST_TITLE("auth with token");
339
340 TEST_RESULT_VOID(
341 storageS3Auth(driver, STRDEF("GET"), STRDEF("/"), query, STRDEF("20170606T121212Z"), header, HASH_TYPE_SHA256_ZERO_STR),
342 "generate authorization");
343 TEST_RESULT_STR_Z(
344 httpHeaderGet(header, STRDEF("authorization")),
345 "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20170606/us-east-1/s3/aws4_request,"
346 "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,"
347 "Signature=85278841678ccbc0f137759265030d7b5e237868dd36eea658426b18344d1685",
348 "check authorization header");
349 }
350
351 // *****************************************************************************************************************************
352 if (testBegin("storageS3*(), StorageReadS3, and StorageWriteS3"))
353 {
354 HRN_FORK_BEGIN()
355 {
356 HRN_FORK_CHILD_BEGIN(.prefix = "s3 server", .timeout = 5000)
357 {
358 TEST_RESULT_VOID(hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolTls, .port = port), "s3 server run");
359 }
360 HRN_FORK_CHILD_END();
361
362 HRN_FORK_CHILD_BEGIN(.prefix = "auth server", .timeout = 5000)
363 {
364 TEST_RESULT_VOID(
365 hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolSocket, .port = authPort), "auth server run");
366 }
367 HRN_FORK_CHILD_END();
368
369 HRN_FORK_PARENT_BEGIN()
370 {
371 // Do not use HRN_FORK_PARENT_WRITE() here so individual names can be assigned to help with debugging
372 IoWrite *service = hrnServerScriptBegin(
373 ioFdWriteNewOpen(STRDEF("s3 client write"), HRN_FORK_PARENT_WRITE_FD(0), 2000));
374 IoWrite *auth = hrnServerScriptBegin(
375 ioFdWriteNewOpen(STRDEF("auth client write"), HRN_FORK_PARENT_WRITE_FD(1), 2000));
376
377 // -----------------------------------------------------------------------------------------------------------------
378 TEST_TITLE("config with keys, token, and host with custom port");
379
380 StringList *argList = strLstDup(commonArgList);
381 hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "%s:%u", strZ(host), port);
382 hrnCfgEnvRaw(cfgOptRepoS3Token, securityToken);
383 HRN_CFG_LOAD(cfgCmdArchivePush, argList);
384
385 Storage *s3 = storageRepoGet(0, true);
386 StorageS3 *driver = (StorageS3 *)storageDriver(s3);
387
388 TEST_RESULT_STR(s3->path, path, "check path");
389 TEST_RESULT_BOOL(storageFeature(s3, storageFeaturePath), false, "check path feature");
390 TEST_RESULT_BOOL(storageFeature(s3, storageFeatureCompress), false, "check compress feature");
391
392 // -----------------------------------------------------------------------------------------------------------------
393 TEST_TITLE("coverage for noop functions");
394
395 TEST_RESULT_VOID(storagePathSyncP(s3, STRDEF("path")), "path sync is a noop");
396
397 // -----------------------------------------------------------------------------------------------------------------
398 TEST_TITLE("ignore missing file");
399
400 hrnServerScriptAccept(service);
401 testRequestP(service, s3, HTTP_VERB_GET, "/fi%26le.txt");
402 testResponseP(service, .code = 404);
403
404 TEST_RESULT_PTR(storageGetP(storageNewReadP(s3, STRDEF("fi&le.txt"), .ignoreMissing = true)), NULL, "get file");
405
406 // -----------------------------------------------------------------------------------------------------------------
407 TEST_TITLE("error on missing file");
408
409 testRequestP(service, s3, HTTP_VERB_GET, "/file.txt");
410 testResponseP(service, .code = 404);
411
412 TEST_ERROR(
413 storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), FileMissingError,
414 "unable to open missing file '/file.txt' for read");
415
416 // -----------------------------------------------------------------------------------------------------------------
417 TEST_TITLE("get file");
418
419 testRequestP(service, s3, HTTP_VERB_GET, "/file.txt");
420 testResponseP(service, .content = "this is a sample file");
421
422 TEST_RESULT_STR_Z(
423 strNewBuf(storageGetP(storageNewReadP(s3, STRDEF("file.txt")))), "this is a sample file", "get file");
424
425 // -----------------------------------------------------------------------------------------------------------------
426 TEST_TITLE("get zero-length file");
427
428 testRequestP(service, s3, HTTP_VERB_GET, "/file0.txt");
429 testResponseP(service);
430
431 TEST_RESULT_STR_Z(strNewBuf(storageGetP(storageNewReadP(s3, STRDEF("file0.txt")))), "", "get zero-length file");
432
433 // -----------------------------------------------------------------------------------------------------------------
434 TEST_TITLE("switch to temp credentials");
435
436 hrnServerScriptClose(service);
437
438 argList = strLstDup(commonArgList);
439 hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "%s:%u", strZ(host), port);
440 hrnCfgArgRaw(argList, cfgOptRepoS3Role, credRole);
441 hrnCfgArgRawStrId(argList, cfgOptRepoS3KeyType, storageS3KeyTypeAuto);
442 HRN_CFG_LOAD(cfgCmdArchivePush, argList);
443
444 s3 = storageRepoGet(0, true);
445 driver = (StorageS3 *)storageDriver(s3);
446
447 TEST_RESULT_STR(s3->path, path, "check path");
448 TEST_RESULT_STR(driver->credRole, credRole, "check role");
449 TEST_RESULT_BOOL(storageFeature(s3, storageFeaturePath), false, "check path feature");
450 TEST_RESULT_BOOL(storageFeature(s3, storageFeatureCompress), false, "check compress feature");
451
452 // Set partSize to a small value for testing
453 driver->partSize = 16;
454
455 // Testing requires the auth http client to be redirected
456 driver->credHost = hrnServerHost();
457 driver->credHttpClient = httpClientNew(sckClientNew(host, authPort, 5000), 5000);
458
459 // Now that we have checked the role when set explicitly, null it out to make sure it is retrieved automatically
460 driver->credRole = NULL;
461
462 hrnServerScriptAccept(service);
463
464 // -----------------------------------------------------------------------------------------------------------------
465 TEST_TITLE("error when retrieving role");
466
467 hrnServerScriptAccept(auth);
468
469 testRequestP(auth, NULL, HTTP_VERB_GET, S3_CREDENTIAL_PATH);
470 testResponseP(auth, .http = "1.0", .code = 301);
471
472 hrnServerScriptClose(auth);
473
474 TEST_ERROR_FMT(
475 storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), ProtocolError,
476 "HTTP request failed with 301:\n"
477 "*** Path/Query ***:\n"
478 "/latest/meta-data/iam/security-credentials\n"
479 "*** Request Headers ***:\n"
480 "content-length: 0\n"
481 "host: %s",
482 strZ(hrnServerHost()));
483
484 // -----------------------------------------------------------------------------------------------------------------
485 TEST_TITLE("missing role");
486
487 hrnServerScriptAccept(auth);
488
489 testRequestP(auth, NULL, HTTP_VERB_GET, S3_CREDENTIAL_PATH);
490 testResponseP(auth, .http = "1.0", .code = 404);
491
492 hrnServerScriptClose(auth);
493
494 TEST_ERROR(
495 storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), ProtocolError,
496 "role to retrieve temporary credentials not found\n"
497 "HINT: is a valid IAM role associated with this instance?");
498
499 // -----------------------------------------------------------------------------------------------------------------
500 TEST_TITLE("error when retrieving temp credentials");
501
502 hrnServerScriptAccept(auth);
503
504 testRequestP(auth, NULL, HTTP_VERB_GET, S3_CREDENTIAL_PATH);
505 testResponseP(auth, .http = "1.0", .content = strZ(credRole));
506
507 hrnServerScriptClose(auth);
508 hrnServerScriptAccept(auth);
509
510 testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_PATH "/%s", strZ(credRole))));
511 testResponseP(auth, .http = "1.0", .code = 300);
512
513 hrnServerScriptClose(auth);
514
515 TEST_ERROR_FMT(
516 storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), ProtocolError,
517 "HTTP request failed with 300:\n"
518 "*** Path/Query ***:\n"
519 "/latest/meta-data/iam/security-credentials/credrole\n"
520 "*** Request Headers ***:\n"
521 "content-length: 0\n"
522 "host: %s",
523 strZ(hrnServerHost()));
524
525 // -----------------------------------------------------------------------------------------------------------------
526 TEST_TITLE("invalid temp credentials role");
527
528 hrnServerScriptAccept(auth);
529
530 testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_PATH "/%s", strZ(credRole))));
531 testResponseP(auth, .http = "1.0", .code = 404);
532
533 hrnServerScriptClose(auth);
534
535 TEST_ERROR_FMT(
536 storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), ProtocolError,
537 "role '%s' not found\n"
538 "HINT: is '%s' a valid IAM role associated with this instance?",
539 strZ(credRole), strZ(credRole));
540
541 // -----------------------------------------------------------------------------------------------------------------
542 TEST_TITLE("invalid code when retrieving temp credentials");
543
544 hrnServerScriptAccept(auth);
545
546 testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_PATH "/%s", strZ(credRole))));
547 testResponseP(auth, .http = "1.0", .content = "{\"Code\":\"IAM role is not configured\"}");
548
549 hrnServerScriptClose(auth);
550
551 TEST_ERROR(
552 storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), FormatError,
553 "unable to retrieve temporary credentials: IAM role is not configured");
554
555 // -----------------------------------------------------------------------------------------------------------------
556 TEST_TITLE("non-404 error");
557
558 hrnServerScriptAccept(auth);
559
560 testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_PATH "/%s", strZ(credRole))));
561 testResponseP(
562 auth,
563 .content = strZ(
564 strNewFmt(
565 "{\"Code\":\"Success\",\"AccessKeyId\":\"x\",\"SecretAccessKey\":\"y\",\"Token\":\"z\""
566 ",\"Expiration\":\"%s\"}",
567 strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC - 1))))));
568
569 hrnServerScriptClose(auth);
570
571 testRequestP(service, s3, HTTP_VERB_GET, "/file.txt", .accessKey = "x", .securityToken = "z");
572 testResponseP(service, .code = 303, .content = "CONTENT");
573
574 StorageRead *read = NULL;
575 TEST_ASSIGN(read, storageNewReadP(s3, STRDEF("file.txt"), .ignoreMissing = true), "new read file");
576 TEST_RESULT_BOOL(storageReadIgnoreMissing(read), true, "check ignore missing");
577 TEST_RESULT_STR_Z(storageReadName(read), "/file.txt", "check name");
578
579 TEST_ERROR(
580 ioReadOpen(storageReadIo(read)), ProtocolError,
581 "HTTP request failed with 303:\n"
582 "*** Path/Query ***:\n"
583 "/file.txt\n"
584 "*** Request Headers ***:\n"
585 "authorization: <redacted>\n"
586 "content-length: 0\n"
587 "host: bucket." S3_TEST_HOST "\n"
588 "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
589 "x-amz-date: <redacted>\n"
590 "x-amz-security-token: <redacted>\n"
591 "*** Response Headers ***:\n"
592 "content-length: 7\n"
593 "*** Response Content ***:\n"
594 "CONTENT")
595
596 // Check that temp credentials were set
597 TEST_RESULT_STR_Z(driver->accessKey, "x", "check access key");
598 TEST_RESULT_STR_Z(driver->secretAccessKey, "y", "check secret access key");
599 TEST_RESULT_STR_Z(driver->securityToken, "z", "check security token");
600
601 // -----------------------------------------------------------------------------------------------------------------
602 TEST_TITLE("write file in one part");
603
604 hrnServerScriptAccept(auth);
605
606 testRequestP(auth, NULL, HTTP_VERB_GET, strZ(strNewFmt(S3_CREDENTIAL_PATH "/%s", strZ(credRole))));
607 testResponseP(
608 auth,
609 .content = strZ(
610 strNewFmt(
611 "{\"Code\":\"Success\",\"AccessKeyId\":\"xx\",\"SecretAccessKey\":\"yy\",\"Token\":\"zz\""
612 ",\"Expiration\":\"%s\"}",
613 strZ(testS3DateTime(time(NULL) + (S3_CREDENTIAL_RENEW_SEC * 2))))));
614
615 hrnServerScriptClose(auth);
616
617 testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD", .accessKey = "xx", .securityToken = "zz");
618 testResponseP(service);
619
620 // Make a copy of the signing key to verify that it gets changed when the keys are updated
621 const Buffer *oldSigningKey = bufDup(driver->signingKey);
622
623 StorageWrite *write = NULL;
624 TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
625 TEST_RESULT_VOID(storagePutP(write, BUFSTRDEF("ABCD")), "write");
626
627 TEST_RESULT_BOOL(storageWriteAtomic(write), true, "write is atomic");
628 TEST_RESULT_BOOL(storageWriteCreatePath(write), true, "path will be created");
629 TEST_RESULT_UINT(storageWriteModeFile(write), 0, "file mode is 0");
630 TEST_RESULT_UINT(storageWriteModePath(write), 0, "path mode is 0");
631 TEST_RESULT_STR_Z(storageWriteName(write), "/file.txt", "check file name");
632 TEST_RESULT_BOOL(storageWriteSyncFile(write), true, "file is synced");
633 TEST_RESULT_BOOL(storageWriteSyncPath(write), true, "path is synced");
634
635 TEST_RESULT_VOID(storageWriteS3Close(write->driver), "close file again");
636
637 // Check that temp credentials were changed
638 TEST_RESULT_STR_Z(driver->accessKey, "xx", "check access key");
639 TEST_RESULT_STR_Z(driver->secretAccessKey, "yy", "check secret access key");
640 TEST_RESULT_STR_Z(driver->securityToken, "zz", "check security token");
641
642 // Check that the signing key changed
643 TEST_RESULT_BOOL(bufEq(driver->signingKey, oldSigningKey), false, "signing key changed");
644
645 // Auth service no longer needed
646 hrnServerScriptEnd(auth);
647
648 // -----------------------------------------------------------------------------------------------------------------
649 TEST_TITLE("write zero-length file");
650
651 testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt", .content = "");
652 testResponseP(service);
653
654 TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
655 TEST_RESULT_VOID(storagePutP(write, NULL), "write");
656
657 // -----------------------------------------------------------------------------------------------------------------
658 TEST_TITLE("write file in chunks with nothing left over on close");
659
660 testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=");
661 testResponseP(
662 service,
663 .content =
664 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
665 "<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
666 "<Bucket>bucket</Bucket>"
667 "<Key>file.txt</Key>"
668 "<UploadId>WxRt</UploadId>"
669 "</InitiateMultipartUploadResult>");
670
671 testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", .content = "1234567890123456");
672 testResponseP(service, .header = "etag:WxRt1");
673
674 testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", .content = "7890123456789012");
675 testResponseP(service, .header = "eTag:WxRt2");
676
677 testRequestP(
678 service, s3, HTTP_VERB_POST, "/file.txt?uploadId=WxRt",
679 .content =
680 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
681 "<CompleteMultipartUpload>"
682 "<Part><PartNumber>1</PartNumber><ETag>WxRt1</ETag></Part>"
683 "<Part><PartNumber>2</PartNumber><ETag>WxRt2</ETag></Part>"
684 "</CompleteMultipartUpload>\n");
685 testResponseP(
686 service,
687 .content =
688 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
689 "<CompleteMultipartUploadResult><ETag>XXX</ETag></CompleteMultipartUploadResult>");
690
691 TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
692 TEST_RESULT_VOID(storagePutP(write, BUFSTRDEF("12345678901234567890123456789012")), "write");
693
694 // -----------------------------------------------------------------------------------------------------------------
695 TEST_TITLE("error in success response of multipart upload");
696
697 testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=");
698 testResponseP(
699 service,
700 .content =
701 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
702 "<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
703 "<Bucket>bucket</Bucket>"
704 "<Key>file.txt</Key>"
705 "<UploadId>WxRt</UploadId>"
706 "</InitiateMultipartUploadResult>");
707
708 testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", .content = "1234567890123456");
709 testResponseP(service, .header = "etag:WxRt1");
710
711 testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", .content = "7890123456789012");
712 testResponseP(service, .header = "eTag:WxRt2");
713
714 testRequestP(
715 service, s3, HTTP_VERB_POST, "/file.txt?uploadId=WxRt",
716 .content =
717 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
718 "<CompleteMultipartUpload>"
719 "<Part><PartNumber>1</PartNumber><ETag>WxRt1</ETag></Part>"
720 "<Part><PartNumber>2</PartNumber><ETag>WxRt2</ETag></Part>"
721 "</CompleteMultipartUpload>\n");
722 testResponseP(
723 service,
724 .content =
725 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
726 "<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>");
727
728 TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
729 TEST_ERROR(
730 storagePutP(write, BUFSTRDEF("12345678901234567890123456789012")), ProtocolError,
731 "HTTP request failed with 200 (OK):\n"
732 "*** Path/Query ***:\n"
733 "/file.txt?uploadId=WxRt\n"
734 "*** Request Headers ***:\n"
735 "authorization: <redacted>\n"
736 "content-length: 205\n"
737 "content-md5: 37smUM6Ah2/EjZbp420dPw==\n"
738 "host: bucket.s3.amazonaws.com\n"
739 "x-amz-content-sha256: 0838a79dfbddc2128d28fb4fa8d605e0a8e6d1355094000f39b6eb3feff4641f\n"
740 "x-amz-date: <redacted>\n"
741 "x-amz-security-token: <redacted>\n"
742 "*** Response Headers ***:\n"
743 "content-length: 110\n"
744 "*** Response Content ***:\n"
745 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
746 "<Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>");
747
748 // -----------------------------------------------------------------------------------------------------------------
749 TEST_TITLE("write file in chunks with something left over on close");
750
751 testRequestP(service, s3, HTTP_VERB_POST, "/file.txt?uploads=");
752 testResponseP(
753 service,
754 .content =
755 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
756 "<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
757 "<Bucket>bucket</Bucket>"
758 "<Key>file.txt</Key>"
759 "<UploadId>RR55</UploadId>"
760 "</InitiateMultipartUploadResult>");
761
762 testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", .content = "1234567890123456");
763 testResponseP(service, .header = "etag:RR551");
764
765 testRequestP(service, s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", .content = "7890");
766 testResponseP(service, .header = "eTag:RR552");
767
768 testRequestP(
769 service, s3, HTTP_VERB_POST, "/file.txt?uploadId=RR55",
770 .content =
771 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
772 "<CompleteMultipartUpload>"
773 "<Part><PartNumber>1</PartNumber><ETag>RR551</ETag></Part>"
774 "<Part><PartNumber>2</PartNumber><ETag>RR552</ETag></Part>"
775 "</CompleteMultipartUpload>\n");
776 testResponseP(
777 service,
778 .content =
779 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
780 "<CompleteMultipartUploadResult><ETag>XXX</ETag></CompleteMultipartUploadResult>");
781
782 TEST_ASSIGN(write, storageNewWriteP(s3, STRDEF("file.txt")), "new write");
783 TEST_RESULT_VOID(storagePutP(write, BUFSTRDEF("12345678901234567890")), "write");
784
785 // -----------------------------------------------------------------------------------------------------------------
786 TEST_TITLE("file missing");
787
788 testRequestP(service, s3, HTTP_VERB_HEAD, "/BOGUS");
789 testResponseP(service, .code = 404);
790
791 TEST_RESULT_BOOL(storageExistsP(s3, STRDEF("BOGUS")), false, "check");
792
793 // -----------------------------------------------------------------------------------------------------------------
794 TEST_TITLE("info for / does not exist");
795
796 TEST_RESULT_BOOL(storageInfoP(s3, NULL, .ignoreMissing = true).exists, false, "info for /");
797
798 // -----------------------------------------------------------------------------------------------------------------
799 TEST_TITLE("info for missing file");
800
801 // File missing
802 testRequestP(service, s3, HTTP_VERB_HEAD, "/BOGUS");
803 testResponseP(service, .code = 404);
804
805 TEST_RESULT_BOOL(storageInfoP(s3, STRDEF("BOGUS"), .ignoreMissing = true).exists, false, "file does not exist");
806
807 // -----------------------------------------------------------------------------------------------------------------
808 TEST_TITLE("info for file");
809
810 testRequestP(service, s3, HTTP_VERB_HEAD, "/subdir/file1.txt");
811 testResponseP(service, .header = "content-length:9999\r\nLast-Modified: Wed, 21 Oct 2015 07:28:00 GMT");
812
813 StorageInfo info;
814 TEST_ASSIGN(info, storageInfoP(s3, STRDEF("subdir/file1.txt")), "file exists");
815 TEST_RESULT_BOOL(info.exists, true, "check exists");
816 TEST_RESULT_UINT(info.type, storageTypeFile, "check type");
817 TEST_RESULT_UINT(info.size, 9999, "check exists");
818 TEST_RESULT_INT(info.timeModified, 1445412480, "check time");
819
820 // -----------------------------------------------------------------------------------------------------------------
821 TEST_TITLE("info check existence only");
822
823 testRequestP(service, s3, HTTP_VERB_HEAD, "/subdir/file2.txt");
824 testResponseP(service, .header = "content-length:777\r\nLast-Modified: Wed, 22 Oct 2015 07:28:00 GMT");
825
826 TEST_ASSIGN(info, storageInfoP(s3, STRDEF("subdir/file2.txt"), .level = storageInfoLevelExists), "file exists");
827 TEST_RESULT_BOOL(info.exists, true, "check exists");
828 TEST_RESULT_UINT(info.type, storageTypeFile, "check type");
829 TEST_RESULT_UINT(info.size, 0, "check exists");
830 TEST_RESULT_INT(info.timeModified, 0, "check time");
831
832 // -----------------------------------------------------------------------------------------------------------------
833 TEST_TITLE("errorOnMissing invalid because there are no paths");
834
835 TEST_ERROR(
836 storageListP(s3, STRDEF("/"), .errorOnMissing = true), AssertError,
837 "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");
838
839 // -----------------------------------------------------------------------------------------------------------------
840 TEST_TITLE("error without xml");
841
842 testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2");
843 testResponseP(service, .code = 344);
844
845 TEST_ERROR(storageListP(s3, STRDEF("/")), ProtocolError,
846 "HTTP request failed with 344:\n"
847 "*** Path/Query ***:\n"
848 "/?delimiter=%2F&list-type=2\n"
849 "*** Request Headers ***:\n"
850 "authorization: <redacted>\n"
851 "content-length: 0\n"
852 "host: bucket." S3_TEST_HOST "\n"
853 "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
854 "x-amz-date: <redacted>\n"
855 "x-amz-security-token: <redacted>");
856
857 // -----------------------------------------------------------------------------------------------------------------
858 TEST_TITLE("error with xml");
859
860 testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2");
861 testResponseP(
862 service, .code = 344,
863 .content =
864 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
865 "<Error>"
866 "<Code>SomeOtherCode</Code>"
867 "</Error>");
868
869 TEST_ERROR(storageListP(s3, STRDEF("/")), ProtocolError,
870 "HTTP request failed with 344:\n"
871 "*** Path/Query ***:\n"
872 "/?delimiter=%2F&list-type=2\n"
873 "*** Request Headers ***:\n"
874 "authorization: <redacted>\n"
875 "content-length: 0\n"
876 "host: bucket." S3_TEST_HOST "\n"
877 "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n"
878 "x-amz-date: <redacted>\n"
879 "x-amz-security-token: <redacted>\n"
880 "*** Response Headers ***:\n"
881 "content-length: 79\n"
882 "*** Response Content ***:\n"
883 "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error><Code>SomeOtherCode</Code></Error>");
884
885 // -----------------------------------------------------------------------------------------------------------------
886 TEST_TITLE("list basic level");
887
888 testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F");
889 testResponseP(
890 service,
891 .content =
892 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
893 "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
894 " <Contents>"
895 " <Key>path/to/test_file</Key>"
896 " <LastModified>2009-10-12T17:50:30.000Z</LastModified>"
897 " <Size>787</Size>"
898 " </Contents>"
899 " <CommonPrefixes>"
900 " <Prefix>path/to/test_path/</Prefix>"
901 " </CommonPrefixes>"
902 "</ListBucketResult>");
903
904 HarnessStorageInfoListCallbackData callbackData =
905 {
906 .content = strNew(),
907 };
908
909 TEST_ERROR(
910 storageInfoListP(s3, STRDEF("/"), hrnStorageInfoListCallback, NULL, .errorOnMissing = true),
911 AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");
912
913 TEST_RESULT_VOID(
914 storageInfoListP(s3, STRDEF("/path/to"), hrnStorageInfoListCallback, &callbackData), "list");
915 TEST_RESULT_STR_Z(
916 callbackData.content,
917 "test_path {path}\n"
918 "test_file {file, s=787, t=1255369830}\n",
919 "check");
920
921 // -----------------------------------------------------------------------------------------------------------------
922 TEST_TITLE("list exists level");
923
924 testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2");
925 testResponseP(
926 service,
927 .content =
928 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
929 "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
930 " <Contents>"
931 " <Key>test1.txt</Key>"
932 " </Contents>"
933 " <CommonPrefixes>"
934 " <Prefix>path1/</Prefix>"
935 " </CommonPrefixes>"
936 "</ListBucketResult>");
937
938 callbackData.content = strNew();
939
940 TEST_RESULT_VOID(
941 storageInfoListP(s3, STRDEF("/"), hrnStorageInfoListCallback, &callbackData, .level = storageInfoLevelExists),
942 "list");
943 TEST_RESULT_STR_Z(
944 callbackData.content,
945 "path1 {}\n"
946 "test1.txt {}\n",
947 "check");
948
949 // -----------------------------------------------------------------------------------------------------------------
950 TEST_TITLE("list a file in root with expression");
951
952 testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=test");
953 testResponseP(
954 service,
955 .content =
956 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
957 "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
958 " <Contents>"
959 " <Key>test1.txt</Key>"
960 " </Contents>"
961 "</ListBucketResult>");
962
963 callbackData.content = strNew();
964
965 TEST_RESULT_VOID(
966 storageInfoListP(
967 s3, STRDEF("/"), hrnStorageInfoListCallback, &callbackData, .expression = STRDEF("^test.*$"),
968 .level = storageInfoLevelExists),
969 "list");
970 TEST_RESULT_STR_Z(
971 callbackData.content,
972 "test1.txt {}\n",
973 "check");
974
975 // -----------------------------------------------------------------------------------------------------------------
976 TEST_TITLE("list files with continuation");
977
978 testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F");
979 testResponseP(
980 service,
981 .content =
982 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
983 "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
984 " <NextContinuationToken>1ueGcxLPRx1Tr/XYExHnhbYLgveDs2J/wm36Hy4vbOwM=</NextContinuationToken>"
985 " <Contents>"
986 " <Key>path/to/test1.txt</Key>"
987 " </Contents>"
988 " <Contents>"
989 " <Key>path/to/test2.txt</Key>"
990 " </Contents>"
991 " <CommonPrefixes>"
992 " <Prefix>path/to/path1/</Prefix>"
993 " </CommonPrefixes>"
994 "</ListBucketResult>");
995
996 testRequestP(
997 service, s3, HTTP_VERB_GET,
998 "/?continuation-token=1ueGcxLPRx1Tr%2FXYExHnhbYLgveDs2J%2Fwm36Hy4vbOwM%3D&delimiter=%2F&list-type=2"
999 "&prefix=path%2Fto%2F");
1000 testResponseP(
1001 service,
1002 .content =
1003 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
1004 "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
1005 " <Contents>"
1006 " <Key>path/to/test3.txt</Key>"
1007 " </Contents>"
1008 " <CommonPrefixes>"
1009 " <Prefix>path/to/path2/</Prefix>"
1010 " </CommonPrefixes>"
1011 "</ListBucketResult>");
1012
1013 callbackData.content = strNew();
1014
1015 TEST_RESULT_VOID(
1016 storageInfoListP(
1017 s3, STRDEF("/path/to"), hrnStorageInfoListCallback, &callbackData, .level = storageInfoLevelExists),
1018 "list");
1019 TEST_RESULT_STR_Z(
1020 callbackData.content,
1021 "path1 {}\n"
1022 "test1.txt {}\n"
1023 "test2.txt {}\n"
1024 "path2 {}\n"
1025 "test3.txt {}\n",
1026 "check");
1027
1028 // -----------------------------------------------------------------------------------------------------------------
1029 TEST_TITLE("list files with expression");
1030
1031 testRequestP(service, s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2Ftest");
1032 testResponseP(
1033 service,
1034 .content =
1035 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
1036 "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
1037 " <Contents>"
1038 " <Key>path/to/test1.txt</Key>"
1039 " </Contents>"
1040 " <Contents>"
1041 " <Key>path/to/test2.txt</Key>"
1042 " </Contents>"
1043 " <Contents>"
1044 " <Key>path/to/test3.txt</Key>"
1045 " </Contents>"
1046 " <CommonPrefixes>"
1047 " <Prefix>path/to/test1.path/</Prefix>"
1048 " </CommonPrefixes>"
1049 " <CommonPrefixes>"
1050 " <Prefix>path/to/test2.path/</Prefix>"
1051 " </CommonPrefixes>"
1052 "</ListBucketResult>");
1053
1054 callbackData.content = strNew();
1055
1056 TEST_RESULT_VOID(
1057 storageInfoListP(
1058 s3, STRDEF("/path/to"), hrnStorageInfoListCallback, &callbackData, .expression = STRDEF("^test(1|3)"),
1059 .level = storageInfoLevelExists),
1060 "list");
1061 TEST_RESULT_STR_Z(
1062 callbackData.content,
1063 "test1.path {}\n"
1064 "test1.txt {}\n"
1065 "test3.txt {}\n",
1066 "check");
1067
1068 // -----------------------------------------------------------------------------------------------------------------
1069 TEST_TITLE("switch to path-style URIs");
1070
1071 hrnServerScriptClose(service);
1072
1073 argList = strLstDup(commonArgList);
1074 hrnCfgArgRawStrId(argList, cfgOptRepoS3UriStyle, storageS3UriStylePath);
1075 hrnCfgArgRaw(argList, cfgOptRepoStorageHost, host);
1076 hrnCfgArgRawFmt(argList, cfgOptRepoStoragePort, "%u", port);
1077 hrnCfgEnvRemoveRaw(cfgOptRepoS3Token);
1078 HRN_CFG_LOAD(cfgCmdArchivePush, argList);
1079
1080 s3 = storageRepoGet(0, true);
1081 driver = (StorageS3 *)storageDriver(s3);
1082
1083 // Set deleteMax to a small value for testing
1084 driver->deleteMax = 2;
1085
1086 hrnServerScriptAccept(service);
1087
1088 // -----------------------------------------------------------------------------------------------------------------
1089 TEST_TITLE("error when no recurse because there are no paths");
1090
1091 TEST_ERROR(
1092 storagePathRemoveP(s3, STRDEF("/")), AssertError,
1093 "assertion 'param.recurse || storageFeature(this, storageFeaturePath)' failed");
1094
1095 // -----------------------------------------------------------------------------------------------------------------
1096 TEST_TITLE("remove files from root");
1097
1098 testRequestP(service, s3, HTTP_VERB_GET, "/bucket/?list-type=2");
1099 testResponseP(
1100 service,
1101 .content =
1102 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
1103 "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
1104 " <Contents>"
1105 " <Key>test1.txt</Key>"
1106 " </Contents>"
1107 " <Contents>"
1108 " <Key>path1/xxx.zzz</Key>"
1109 " </Contents>"
1110 "</ListBucketResult>");
1111
1112 testRequestP(
1113 service, s3, HTTP_VERB_POST, "/bucket/?delete=",
1114 .content =
1115 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
1116 "<Delete><Quiet>true</Quiet>"
1117 "<Object><Key>test1.txt</Key></Object>"
1118 "<Object><Key>path1/xxx.zzz</Key></Object>"
1119 "</Delete>\n");
1120 testResponseP(
1121 service, .content = "<DeleteResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"></DeleteResult>");
1122
1123 TEST_RESULT_VOID(storagePathRemoveP(s3, STRDEF("/"), .recurse = true), "remove");
1124
1125 // -----------------------------------------------------------------------------------------------------------------
1126 TEST_TITLE("remove files in empty subpath (nothing to do)");
1127
1128 testRequestP(service, s3, HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2F");
1129 testResponseP(
1130 service,
1131 .content =
1132 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
1133 "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
1134 "</ListBucketResult>");
1135
1136 TEST_RESULT_VOID(storagePathRemoveP(s3, STRDEF("/path"), .recurse = true), "remove");
1137
1138 // -----------------------------------------------------------------------------------------------------------------
1139 TEST_TITLE("remove files with continuation");
1140
1141 testRequestP(service, s3, HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2Fto%2F");
1142 testResponseP(
1143 service,
1144 .content =
1145 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
1146 "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
1147 " <NextContinuationToken>continue</NextContinuationToken>"
1148 " <CommonPrefixes>"
1149 " <Prefix>path/to/test3/</Prefix>"
1150 " </CommonPrefixes>"
1151 " <Contents>"
1152 " <Key>path/to/test1.txt</Key>"
1153 " </Contents>"
1154 "</ListBucketResult>");
1155
1156 testRequestP(service, s3, HTTP_VERB_GET, "/bucket/?continuation-token=continue&list-type=2&prefix=path%2Fto%2F");
1157 testResponseP(
1158 service,
1159 .content =
1160 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
1161 "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
1162 " <Contents>"
1163 " <Key>path/to/test3.txt</Key>"
1164 " </Contents>"
1165 " <Contents>"
1166 " <Key>path/to/test2.txt</Key>"
1167 " </Contents>"
1168 "</ListBucketResult>");
1169
1170 testRequestP(
1171 service, s3, HTTP_VERB_POST, "/bucket/?delete=",
1172 .content =
1173 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
1174 "<Delete><Quiet>true</Quiet>"
1175 "<Object><Key>path/to/test1.txt</Key></Object>"
1176 "<Object><Key>path/to/test3.txt</Key></Object>"
1177 "</Delete>\n");
1178 testResponseP(service);
1179
1180 testRequestP(
1181 service, s3, HTTP_VERB_POST, "/bucket/?delete=",
1182 .content =
1183 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
1184 "<Delete><Quiet>true</Quiet>"
1185 "<Object><Key>path/to/test2.txt</Key></Object>"
1186 "</Delete>\n");
1187 testResponseP(service);
1188
1189 TEST_RESULT_VOID(storagePathRemoveP(s3, STRDEF("/path/to"), .recurse = true), "remove");
1190
1191 // -----------------------------------------------------------------------------------------------------------------
1192 TEST_TITLE("remove error");
1193
1194 testRequestP(service, s3, HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2F");
1195 testResponseP(
1196 service,
1197 .content =
1198 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
1199 "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
1200 " <Contents>"
1201 " <Key>path/sample.txt</Key>"
1202 " </Contents>"
1203 " <Contents>"
1204 " <Key>path/sample2.txt</Key>"
1205 " </Contents>"
1206 "</ListBucketResult>");
1207
1208 testRequestP(
1209 service, s3, HTTP_VERB_POST, "/bucket/?delete=",
1210 .content =
1211 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
1212 "<Delete><Quiet>true</Quiet>"
1213 "<Object><Key>path/sample.txt</Key></Object>"
1214 "<Object><Key>path/sample2.txt</Key></Object>"
1215 "</Delete>\n");
1216 testResponseP(
1217 service,
1218 .content =
1219 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
1220 "<DeleteResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
1221 "<Error><Key>sample2.txt</Key><Code>AccessDenied</Code><Message>Access Denied</Message></Error>"
1222 "</DeleteResult>");
1223
1224 TEST_ERROR(
1225 storagePathRemoveP(s3, STRDEF("/path"), .recurse = true), FileRemoveError,
1226 "unable to remove file 'sample2.txt': [AccessDenied] Access Denied");
1227
1228 // -----------------------------------------------------------------------------------------------------------------
1229 TEST_TITLE("remove file");
1230
1231 testRequestP(service, s3, HTTP_VERB_DELETE, "/bucket/path/to/test.txt");
1232 testResponseP(service, .code = 204);
1233
1234 TEST_RESULT_VOID(storageRemoveP(s3, STRDEF("/path/to/test.txt")), "remove");
1235
1236 // -----------------------------------------------------------------------------------------------------------------
1237 hrnServerScriptEnd(service);
1238 }
1239 HRN_FORK_PARENT_END();
1240 }
1241 HRN_FORK_END();
1242 }
1243
1244 FUNCTION_HARNESS_RETURN_VOID();
1245 }
1246