1 /*
2 * s3fs - FUSE-based file system backed by Amazon S3
3 *
4 * Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version 2
9 * of the License, or (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
20
21 #include <cstdio>
22 #include <cstdlib>
23 #include <ctime>
24 #include <unistd.h>
25 #include <fstream>
26 #include <sstream>
27 #include <algorithm>
28
29 #include "common.h"
30 #include "s3fs.h"
31 #include "curl.h"
32 #include "curl_multi.h"
33 #include "curl_util.h"
34 #include "s3fs_auth.h"
35 #include "autolock.h"
36 #include "s3fs_util.h"
37 #include "string_util.h"
38 #include "addhead.h"
39
40 //-------------------------------------------------------------------
41 // Symbols
42 //-------------------------------------------------------------------
43 static const char EMPTY_PAYLOAD_HASH[] = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
44 static const char EMPTY_MD5_BASE64_HASH[] = "1B2M2Y8AsgTpgAmY7PhCfg==";
45
46 //-------------------------------------------------------------------
47 // Class S3fsCurl
48 //-------------------------------------------------------------------
49 static const int MULTIPART_SIZE = 10 * 1024 * 1024;
50 static const int GET_OBJECT_RESPONSE_LIMIT = 1024;
51
52 static const int IAM_EXPIRE_MERGIN = 20 * 60; // update timing
53 static const char ECS_IAM_ENV_VAR[] = "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI";
54 static const char IAMCRED_ACCESSKEYID[] = "AccessKeyId";
55 static const char IAMCRED_SECRETACCESSKEY[] = "SecretAccessKey";
56 static const char IAMCRED_ROLEARN[] = "RoleArn";
57
58 // [NOTE] about default mime.types file
59 // If no mime.types file is specified in the mime option, s3fs
60 // will look for /etc/mime.types on all operating systems and
61 // load mime information.
62 // However, in the case of macOS, when this file does not exist,
63 // it tries to detect the /etc/apache2/mime.types file.
64 // The reason for this is that apache2 is preinstalled on macOS,
65 // and the mime.types file is expected to exist in this path.
66 // If the mime.types file is not found, s3fs will exit with an
67 // error.
68 //
69 static const char DEFAULT_MIME_FILE[] = "/etc/mime.types";
70 static const char SPECIAL_DARWIN_MIME_FILE[] = "/etc/apache2/mime.types";
71
72 // [NOTICE]
73 // This symbol is for libcurl under 7.23.0
74 #ifndef CURLSHE_NOT_BUILT_IN
75 #define CURLSHE_NOT_BUILT_IN 5
76 #endif
77
78 //-------------------------------------------------------------------
79 // Class S3fsCurl
80 //-------------------------------------------------------------------
81 const long S3fsCurl::S3FSCURL_RESPONSECODE_NOTSET;
82 const long S3fsCurl::S3FSCURL_RESPONSECODE_FATAL_ERROR;
83 const int S3fsCurl::S3FSCURL_PERFORM_RESULT_NOTSET;
84 pthread_mutex_t S3fsCurl::curl_warnings_lock;
85 pthread_mutex_t S3fsCurl::curl_handles_lock;
86 S3fsCurl::callback_locks_t S3fsCurl::callback_locks;
87 bool S3fsCurl::is_initglobal_done = false;
88 CurlHandlerPool* S3fsCurl::sCurlPool = NULL;
89 int S3fsCurl::sCurlPoolSize = 32;
90 CURLSH* S3fsCurl::hCurlShare = NULL;
91 bool S3fsCurl::is_cert_check = true; // default
92 bool S3fsCurl::is_dns_cache = true; // default
93 bool S3fsCurl::is_ssl_session_cache= true; // default
94 long S3fsCurl::connect_timeout = 300; // default
95 time_t S3fsCurl::readwrite_timeout = 120; // default
96 int S3fsCurl::retries = 5; // default
97 bool S3fsCurl::is_public_bucket = false;
98 acl_t S3fsCurl::default_acl = acl_t::PRIVATE;
99 std::string S3fsCurl::storage_class = "STANDARD";
100 sseckeylist_t S3fsCurl::sseckeys;
101 std::string S3fsCurl::ssekmsid;
102 sse_type_t S3fsCurl::ssetype = sse_type_t::SSE_DISABLE;
103 bool S3fsCurl::is_content_md5 = false;
104 bool S3fsCurl::is_verbose = false;
105 bool S3fsCurl::is_dump_body = false;
106 std::string S3fsCurl::AWSAccessKeyId;
107 std::string S3fsCurl::AWSSecretAccessKey;
108 std::string S3fsCurl::AWSAccessToken;
109 time_t S3fsCurl::AWSAccessTokenExpire= 0;
110 bool S3fsCurl::is_ecs = false;
111 bool S3fsCurl::is_ibm_iam_auth = false;
112 std::string S3fsCurl::IAM_cred_url = "http://169.254.169.254/latest/meta-data/iam/security-credentials/";
113 std::string S3fsCurl::IAMv2_token_url = "http://169.254.169.254/latest/api/token";
114 std::string S3fsCurl::IAMv2_token_ttl_hdr = "X-aws-ec2-metadata-token-ttl-seconds";
115 std::string S3fsCurl::IAMv2_token_hdr = "X-aws-ec2-metadata-token";
116 int S3fsCurl::IAMv2_token_ttl = 21600;
117 size_t S3fsCurl::IAM_field_count = 4;
118 std::string S3fsCurl::IAM_token_field = "Token";
119 std::string S3fsCurl::IAM_expiry_field = "Expiration";
120 std::string S3fsCurl::IAM_role;
121 std::string S3fsCurl::IAMv2_api_token;
122 int S3fsCurl::IAM_api_version = 2;
123 long S3fsCurl::ssl_verify_hostname = 1; // default(original code...)
124
125 // protected by curl_warnings_lock
126 bool S3fsCurl::curl_warnings_once = false;
127
128 // protected by curl_handles_lock
129 curltime_t S3fsCurl::curl_times;
130 curlprogress_t S3fsCurl::curl_progress;
131
132 std::string S3fsCurl::curl_ca_bundle;
133 mimes_t S3fsCurl::mimeTypes;
134 std::string S3fsCurl::userAgent;
135 int S3fsCurl::max_parallel_cnt = 5; // default
136 int S3fsCurl::max_multireq = 20; // default
137 off_t S3fsCurl::multipart_size = MULTIPART_SIZE; // default
138 off_t S3fsCurl::multipart_copy_size = 512 * 1024 * 1024; // default
139 signature_type_t S3fsCurl::signature_type = V2_OR_V4; // default
140 bool S3fsCurl::is_ua = true; // default
141 bool S3fsCurl::listobjectsv2 = false; // default
142 bool S3fsCurl::is_use_session_token= false; // default
143 bool S3fsCurl::requester_pays = false; // default
144
145 //-------------------------------------------------------------------
146 // Class methods for S3fsCurl
147 //-------------------------------------------------------------------
InitS3fsCurl()148 bool S3fsCurl::InitS3fsCurl()
149 {
150 pthread_mutexattr_t attr;
151 pthread_mutexattr_init(&attr);
152 #if S3FS_PTHREAD_ERRORCHECK
153 pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
154 #endif
155 if(0 != pthread_mutex_init(&S3fsCurl::curl_warnings_lock, &attr)){
156 return false;
157 }
158 if(0 != pthread_mutex_init(&S3fsCurl::curl_handles_lock, &attr)){
159 return false;
160 }
161 if(0 != pthread_mutex_init(&S3fsCurl::callback_locks.dns, &attr)){
162 return false;
163 }
164 if(0 != pthread_mutex_init(&S3fsCurl::callback_locks.ssl_session, &attr)){
165 return false;
166 }
167 if(!S3fsCurl::InitGlobalCurl()){
168 return false;
169 }
170 if(!S3fsCurl::InitShareCurl()){
171 return false;
172 }
173 if(!S3fsCurl::InitCryptMutex()){
174 return false;
175 }
176 // [NOTE]
177 // sCurlPoolSize must be over parallel(or multireq) count.
178 //
179 if(sCurlPoolSize < std::max(GetMaxParallelCount(), GetMaxMultiRequest())){
180 sCurlPoolSize = std::max(GetMaxParallelCount(), GetMaxMultiRequest());
181 }
182 sCurlPool = new CurlHandlerPool(sCurlPoolSize);
183 if (!sCurlPool->Init()) {
184 return false;
185 }
186 return true;
187 }
188
DestroyS3fsCurl()189 bool S3fsCurl::DestroyS3fsCurl()
190 {
191 bool result = true;
192
193 if(!S3fsCurl::DestroyCryptMutex()){
194 result = false;
195 }
196 if(!sCurlPool->Destroy()){
197 result = false;
198 }
199 delete sCurlPool;
200 sCurlPool = NULL;
201 if(!S3fsCurl::DestroyShareCurl()){
202 result = false;
203 }
204 if(!S3fsCurl::DestroyGlobalCurl()){
205 result = false;
206 }
207 if(0 != pthread_mutex_destroy(&S3fsCurl::callback_locks.dns)){
208 result = false;
209 }
210 if(0 != pthread_mutex_destroy(&S3fsCurl::callback_locks.ssl_session)){
211 result = false;
212 }
213 if(0 != pthread_mutex_destroy(&S3fsCurl::curl_handles_lock)){
214 result = false;
215 }
216 if(0 != pthread_mutex_destroy(&S3fsCurl::curl_warnings_lock)){
217 result = false;
218 }
219 return result;
220 }
221
InitGlobalCurl()222 bool S3fsCurl::InitGlobalCurl()
223 {
224 if(S3fsCurl::is_initglobal_done){
225 return false;
226 }
227 if(CURLE_OK != curl_global_init(CURL_GLOBAL_ALL)){
228 S3FS_PRN_ERR("init_curl_global_all returns error.");
229 return false;
230 }
231 S3fsCurl::is_initglobal_done = true;
232 return true;
233 }
234
DestroyGlobalCurl()235 bool S3fsCurl::DestroyGlobalCurl()
236 {
237 if(!S3fsCurl::is_initglobal_done){
238 return false;
239 }
240 curl_global_cleanup();
241 S3fsCurl::is_initglobal_done = false;
242 return true;
243 }
244
InitShareCurl()245 bool S3fsCurl::InitShareCurl()
246 {
247 CURLSHcode nSHCode;
248
249 if(!S3fsCurl::is_dns_cache && !S3fsCurl::is_ssl_session_cache){
250 S3FS_PRN_INFO("Curl does not share DNS data.");
251 return true;
252 }
253 if(S3fsCurl::hCurlShare){
254 S3FS_PRN_WARN("already initiated.");
255 return false;
256 }
257 if(NULL == (S3fsCurl::hCurlShare = curl_share_init())){
258 S3FS_PRN_ERR("curl_share_init failed");
259 return false;
260 }
261 if(CURLSHE_OK != (nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_LOCKFUNC, S3fsCurl::LockCurlShare))){
262 S3FS_PRN_ERR("curl_share_setopt(LOCKFUNC) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
263 return false;
264 }
265 if(CURLSHE_OK != (nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_UNLOCKFUNC, S3fsCurl::UnlockCurlShare))){
266 S3FS_PRN_ERR("curl_share_setopt(UNLOCKFUNC) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
267 return false;
268 }
269 if(S3fsCurl::is_dns_cache){
270 nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
271 if(CURLSHE_OK != nSHCode && CURLSHE_BAD_OPTION != nSHCode && CURLSHE_NOT_BUILT_IN != nSHCode){
272 S3FS_PRN_ERR("curl_share_setopt(DNS) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
273 return false;
274 }else if(CURLSHE_BAD_OPTION == nSHCode || CURLSHE_NOT_BUILT_IN == nSHCode){
275 S3FS_PRN_WARN("curl_share_setopt(DNS) returns %d(%s), but continue without shared dns data.", nSHCode, curl_share_strerror(nSHCode));
276 }
277 }
278 if(S3fsCurl::is_ssl_session_cache){
279 nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
280 if(CURLSHE_OK != nSHCode && CURLSHE_BAD_OPTION != nSHCode && CURLSHE_NOT_BUILT_IN != nSHCode){
281 S3FS_PRN_ERR("curl_share_setopt(SSL SESSION) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
282 return false;
283 }else if(CURLSHE_BAD_OPTION == nSHCode || CURLSHE_NOT_BUILT_IN == nSHCode){
284 S3FS_PRN_WARN("curl_share_setopt(SSL SESSION) returns %d(%s), but continue without shared ssl session data.", nSHCode, curl_share_strerror(nSHCode));
285 }
286 }
287 if(CURLSHE_OK != (nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_USERDATA, &S3fsCurl::callback_locks))){
288 S3FS_PRN_ERR("curl_share_setopt(USERDATA) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
289 return false;
290 }
291 return true;
292 }
293
DestroyShareCurl()294 bool S3fsCurl::DestroyShareCurl()
295 {
296 if(!S3fsCurl::hCurlShare){
297 if(!S3fsCurl::is_dns_cache && !S3fsCurl::is_ssl_session_cache){
298 return true;
299 }
300 S3FS_PRN_WARN("already destroy share curl.");
301 return false;
302 }
303 if(CURLSHE_OK != curl_share_cleanup(S3fsCurl::hCurlShare)){
304 return false;
305 }
306 S3fsCurl::hCurlShare = NULL;
307 return true;
308 }
309
LockCurlShare(CURL * handle,curl_lock_data nLockData,curl_lock_access laccess,void * useptr)310 void S3fsCurl::LockCurlShare(CURL* handle, curl_lock_data nLockData, curl_lock_access laccess, void* useptr)
311 {
312 if(!hCurlShare){
313 return;
314 }
315 S3fsCurl::callback_locks_t* locks = static_cast<S3fsCurl::callback_locks_t*>(useptr);
316 int result;
317 if(CURL_LOCK_DATA_DNS == nLockData){
318 if(0 != (result = pthread_mutex_lock(&locks->dns))){
319 S3FS_PRN_CRIT("pthread_mutex_lock returned: %d", result);
320 abort();
321 }
322 }else if(CURL_LOCK_DATA_SSL_SESSION == nLockData){
323 if(0 != (result = pthread_mutex_lock(&locks->ssl_session))){
324 S3FS_PRN_CRIT("pthread_mutex_lock returned: %d", result);
325 abort();
326 }
327 }
328 }
329
UnlockCurlShare(CURL * handle,curl_lock_data nLockData,void * useptr)330 void S3fsCurl::UnlockCurlShare(CURL* handle, curl_lock_data nLockData, void* useptr)
331 {
332 if(!hCurlShare){
333 return;
334 }
335 S3fsCurl::callback_locks_t* locks = static_cast<S3fsCurl::callback_locks_t*>(useptr);
336 int result;
337 if(CURL_LOCK_DATA_DNS == nLockData){
338 if(0 != (result = pthread_mutex_unlock(&locks->dns))){
339 S3FS_PRN_CRIT("pthread_mutex_unlock returned: %d", result);
340 abort();
341 }
342 }else if(CURL_LOCK_DATA_SSL_SESSION == nLockData){
343 if(0 != (result = pthread_mutex_unlock(&locks->ssl_session))){
344 S3FS_PRN_CRIT("pthread_mutex_unlock returned: %d", result);
345 abort();
346 }
347 }
348 }
349
InitCryptMutex()350 bool S3fsCurl::InitCryptMutex()
351 {
352 return s3fs_init_crypt_mutex();
353 }
354
DestroyCryptMutex()355 bool S3fsCurl::DestroyCryptMutex()
356 {
357 return s3fs_destroy_crypt_mutex();
358 }
359
360 // homegrown timeout mechanism
CurlProgress(void * clientp,double dltotal,double dlnow,double ultotal,double ulnow)361 int S3fsCurl::CurlProgress(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
362 {
363 CURL* curl = static_cast<CURL*>(clientp);
364 time_t now = time(0);
365 progress_t p(dlnow, ulnow);
366
367 AutoLock lock(&S3fsCurl::curl_handles_lock);
368
369 // any progress?
370 if(p != S3fsCurl::curl_progress[curl]){
371 // yes!
372 S3fsCurl::curl_times[curl] = now;
373 S3fsCurl::curl_progress[curl] = p;
374 }else{
375 // timeout?
376 if(now - S3fsCurl::curl_times[curl] > readwrite_timeout){
377 S3FS_PRN_ERR("timeout now: %lld, curl_times[curl]: %lld, readwrite_timeout: %lld",
378 static_cast<long long>(now), static_cast<long long>((S3fsCurl::curl_times[curl])), static_cast<long long>(readwrite_timeout));
379 return CURLE_ABORTED_BY_CALLBACK;
380 }
381 }
382 return 0;
383 }
384
InitMimeType(const std::string & strFile)385 bool S3fsCurl::InitMimeType(const std::string& strFile)
386 {
387 std::string MimeFile;
388 if(!strFile.empty()){
389 MimeFile = strFile;
390 }else{
391 // search default mime.types
392 std::string errPaths = DEFAULT_MIME_FILE;
393 struct stat st;
394 if(0 == stat(DEFAULT_MIME_FILE, &st)){
395 MimeFile = DEFAULT_MIME_FILE;
396 }else if(compare_sysname("Darwin")){
397 // for macOS, search another default file.
398 if(0 == stat(SPECIAL_DARWIN_MIME_FILE, &st)){
399 MimeFile = SPECIAL_DARWIN_MIME_FILE;
400 }else{
401 errPaths += " and ";
402 errPaths += SPECIAL_DARWIN_MIME_FILE;
403 }
404 }
405 if(MimeFile.empty()){
406 S3FS_PRN_WARN("Could not find mime.types files, you have to create file(%s) or specify mime option for existing mime.types file.", errPaths.c_str());
407 return false;
408 }
409 }
410 S3FS_PRN_DBG("Try to load mime types from %s file.", MimeFile.c_str());
411
412 std::string line;
413 std::ifstream MT(MimeFile.c_str());
414 if(MT.good()){
415 S3FS_PRN_DBG("The old mime types are cleared to load new mime types.");
416 S3fsCurl::mimeTypes.clear();
417
418 while(getline(MT, line)){
419 if(line.empty()){
420 continue;
421 }
422 if(line[0]=='#'){
423 continue;
424 }
425
426 std::istringstream tmp(line);
427 std::string mimeType;
428 tmp >> mimeType;
429 std::string ext;
430 while(tmp >> ext){
431 S3fsCurl::mimeTypes[ext] = mimeType;
432 }
433 }
434 S3FS_PRN_INIT_INFO("Loaded mime information from %s", MimeFile.c_str());
435 }else{
436 S3FS_PRN_WARN("Could not load mime types from %s, please check the existence and permissions of this file.", MimeFile.c_str());
437 return false;
438 }
439 return true;
440 }
441
InitUserAgent()442 void S3fsCurl::InitUserAgent()
443 {
444 if(S3fsCurl::userAgent.empty()){
445 S3fsCurl::userAgent = "s3fs/";
446 S3fsCurl::userAgent += VERSION;
447 S3fsCurl::userAgent += " (commit hash ";
448 S3fsCurl::userAgent += COMMIT_HASH_VAL;
449 S3fsCurl::userAgent += "; ";
450 S3fsCurl::userAgent += s3fs_crypt_lib_name();
451 S3fsCurl::userAgent += ")";
452 S3fsCurl::userAgent += instance_name;
453 }
454 }
455
456 //
457 // @param s e.g., "index.html"
458 // @return e.g., "text/html"
459 //
LookupMimeType(const std::string & name)460 std::string S3fsCurl::LookupMimeType(const std::string& name)
461 {
462 if(!name.empty() && name[name.size() - 1] == '/'){
463 return "application/x-directory";
464 }
465
466 std::string result("application/octet-stream");
467 std::string::size_type last_pos = name.find_last_of('.');
468 std::string::size_type first_pos = name.find_first_of('.');
469 std::string prefix, ext, ext2;
470
471 // No dots in name, just return
472 if(last_pos == std::string::npos){
473 return result;
474 }
475 // extract the last extension
476 ext = name.substr(1+last_pos, std::string::npos);
477
478 if (last_pos != std::string::npos) {
479 // one dot was found, now look for another
480 if (first_pos != std::string::npos && first_pos < last_pos) {
481 prefix = name.substr(0, last_pos);
482 // Now get the second to last file extension
483 std::string::size_type next_pos = prefix.find_last_of('.');
484 if (next_pos != std::string::npos) {
485 ext2 = prefix.substr(1+next_pos, std::string::npos);
486 }
487 }
488 }
489
490 // if we get here, then we have an extension (ext)
491 mimes_t::const_iterator iter = S3fsCurl::mimeTypes.find(ext);
492 // if the last extension matches a mimeType, then return
493 // that mime type
494 if (iter != S3fsCurl::mimeTypes.end()) {
495 result = (*iter).second;
496 return result;
497 }
498
499 // return with the default result if there isn't a second extension
500 if(first_pos == last_pos){
501 return result;
502 }
503
504 // Didn't find a mime-type for the first extension
505 // Look for second extension in mimeTypes, return if found
506 iter = S3fsCurl::mimeTypes.find(ext2);
507 if (iter != S3fsCurl::mimeTypes.end()) {
508 result = (*iter).second;
509 return result;
510 }
511
512 // neither the last extension nor the second-to-last extension
513 // matched a mimeType, return the default mime type
514 return result;
515 }
516
LocateBundle()517 bool S3fsCurl::LocateBundle()
518 {
519 // See if environment variable CURL_CA_BUNDLE is set
520 // if so, check it, if it is a good path, then set the
521 // curl_ca_bundle variable to it
522 if(S3fsCurl::curl_ca_bundle.empty()){
523 char* CURL_CA_BUNDLE = getenv("CURL_CA_BUNDLE");
524 if(CURL_CA_BUNDLE != NULL) {
525 // check for existence and readability of the file
526 std::ifstream BF(CURL_CA_BUNDLE);
527 if(!BF.good()){
528 S3FS_PRN_ERR("%s: file specified by CURL_CA_BUNDLE environment variable is not readable", program_name.c_str());
529 return false;
530 }
531 BF.close();
532 S3fsCurl::curl_ca_bundle = CURL_CA_BUNDLE;
533 return true;
534 }
535 }else{
536 // Already set ca bundle variable
537 return true;
538 }
539
540 // not set via environment variable, look in likely locations
541
542 ///////////////////////////////////////////
543 // following comment from curl's (7.21.2) acinclude.m4 file
544 ///////////////////////////////////////////
545 // dnl CURL_CHECK_CA_BUNDLE
546 // dnl -------------------------------------------------
547 // dnl Check if a default ca-bundle should be used
548 // dnl
549 // dnl regarding the paths this will scan:
550 // dnl /etc/ssl/certs/ca-certificates.crt Debian systems
551 // dnl /etc/pki/tls/certs/ca-bundle.crt Redhat and Mandriva
552 // dnl /usr/share/ssl/certs/ca-bundle.crt old(er) Redhat
553 // dnl /usr/local/share/certs/ca-root.crt FreeBSD
554 // dnl /etc/ssl/cert.pem OpenBSD
555 // dnl /etc/ssl/certs/ (ca path) SUSE
556 ///////////////////////////////////////////
557 // Within CURL the above path should have been checked
558 // according to the OS. Thus, although we do not need
559 // to check files here, we will only examine some files.
560 //
561 std::ifstream BF("/etc/pki/tls/certs/ca-bundle.crt");
562 if(BF.good()){
563 BF.close();
564 S3fsCurl::curl_ca_bundle = "/etc/pki/tls/certs/ca-bundle.crt";
565 }else{
566 BF.open("/etc/ssl/certs/ca-certificates.crt");
567 if(BF.good()){
568 BF.close();
569 S3fsCurl::curl_ca_bundle = "/etc/ssl/certs/ca-certificates.crt";
570 }else{
571 BF.open("/usr/share/ssl/certs/ca-bundle.crt");
572 if(BF.good()){
573 BF.close();
574 S3fsCurl::curl_ca_bundle = "/usr/share/ssl/certs/ca-bundle.crt";
575 }else{
576 BF.open("/usr/local/share/certs/ca-root.crt");
577 if(BF.good()){
578 BF.close();
579 S3fsCurl::curl_ca_bundle = "/usr/share/ssl/certs/ca-bundle.crt";
580 }else{
581 S3FS_PRN_ERR("%s: /.../ca-bundle.crt is not readable", program_name.c_str());
582 return false;
583 }
584 }
585 }
586 }
587 return true;
588 }
589
WriteMemoryCallback(void * ptr,size_t blockSize,size_t numBlocks,void * data)590 size_t S3fsCurl::WriteMemoryCallback(void* ptr, size_t blockSize, size_t numBlocks, void* data)
591 {
592 BodyData* body = static_cast<BodyData*>(data);
593
594 if(!body->Append(ptr, blockSize, numBlocks)){
595 S3FS_PRN_CRIT("BodyData.Append() returned false.");
596 S3FS_FUSE_EXIT();
597 return -1;
598 }
599 return (blockSize * numBlocks);
600 }
601
ReadCallback(void * ptr,size_t size,size_t nmemb,void * userp)602 size_t S3fsCurl::ReadCallback(void* ptr, size_t size, size_t nmemb, void* userp)
603 {
604 S3fsCurl* pCurl = static_cast<S3fsCurl*>(userp);
605
606 if(1 > (size * nmemb)){
607 return 0;
608 }
609 if(0 >= pCurl->postdata_remaining){
610 return 0;
611 }
612 size_t copysize = std::min(static_cast<off_t>(size * nmemb), pCurl->postdata_remaining);
613 memcpy(ptr, pCurl->postdata, copysize);
614
615 pCurl->postdata_remaining = (pCurl->postdata_remaining > static_cast<off_t>(copysize) ? (pCurl->postdata_remaining - copysize) : 0);
616 pCurl->postdata += static_cast<size_t>(copysize);
617
618 return copysize;
619 }
620
HeaderCallback(void * data,size_t blockSize,size_t numBlocks,void * userPtr)621 size_t S3fsCurl::HeaderCallback(void* data, size_t blockSize, size_t numBlocks, void* userPtr)
622 {
623 headers_t* headers = static_cast<headers_t*>(userPtr);
624 std::string header(static_cast<char*>(data), blockSize * numBlocks);
625 std::string key;
626 std::istringstream ss(header);
627
628 if(getline(ss, key, ':')){
629 // Force to lower, only "x-amz"
630 std::string lkey = key;
631 transform(lkey.begin(), lkey.end(), lkey.begin(), static_cast<int (*)(int)>(std::tolower));
632 if(is_prefix(lkey.c_str(), "x-amz")){
633 key = lkey;
634 }
635 std::string value;
636 getline(ss, value);
637 (*headers)[key] = trim(value);
638 }
639 return blockSize * numBlocks;
640 }
641
UploadReadCallback(void * ptr,size_t size,size_t nmemb,void * userp)642 size_t S3fsCurl::UploadReadCallback(void* ptr, size_t size, size_t nmemb, void* userp)
643 {
644 S3fsCurl* pCurl = static_cast<S3fsCurl*>(userp);
645
646 if(1 > (size * nmemb)){
647 return 0;
648 }
649 if(-1 == pCurl->partdata.fd || 0 >= pCurl->partdata.size){
650 return 0;
651 }
652 // read size
653 ssize_t copysize = (size * nmemb) < (size_t)pCurl->partdata.size ? (size * nmemb) : (size_t)pCurl->partdata.size;
654 ssize_t readbytes;
655 ssize_t totalread;
656 // read and set
657 for(totalread = 0, readbytes = 0; totalread < copysize; totalread += readbytes){
658 readbytes = pread(pCurl->partdata.fd, &((char*)ptr)[totalread], (copysize - totalread), pCurl->partdata.startpos + totalread);
659 if(0 == readbytes){
660 // eof
661 break;
662 }else if(-1 == readbytes){
663 // error
664 S3FS_PRN_ERR("read file error(%d).", errno);
665 return 0;
666 }
667 }
668 pCurl->partdata.startpos += totalread;
669 pCurl->partdata.size -= totalread;
670
671 return totalread;
672 }
673
DownloadWriteCallback(void * ptr,size_t size,size_t nmemb,void * userp)674 size_t S3fsCurl::DownloadWriteCallback(void* ptr, size_t size, size_t nmemb, void* userp)
675 {
676 S3fsCurl* pCurl = static_cast<S3fsCurl*>(userp);
677
678 if(1 > (size * nmemb)){
679 return 0;
680 }
681 if(-1 == pCurl->partdata.fd || 0 >= pCurl->partdata.size){
682 return 0;
683 }
684
685 // Buffer initial bytes in case it is an XML error response.
686 if(pCurl->bodydata.size() < GET_OBJECT_RESPONSE_LIMIT){
687 pCurl->bodydata.Append(ptr, std::min(size * nmemb, GET_OBJECT_RESPONSE_LIMIT - pCurl->bodydata.size()));
688 }
689
690 // write size
691 ssize_t copysize = (size * nmemb) < (size_t)pCurl->partdata.size ? (size * nmemb) : (size_t)pCurl->partdata.size;
692 ssize_t writebytes;
693 ssize_t totalwrite;
694
695 // write
696 for(totalwrite = 0, writebytes = 0; totalwrite < copysize; totalwrite += writebytes){
697 writebytes = pwrite(pCurl->partdata.fd, &((char*)ptr)[totalwrite], (copysize - totalwrite), pCurl->partdata.startpos + totalwrite);
698 if(0 == writebytes){
699 // eof?
700 break;
701 }else if(-1 == writebytes){
702 // error
703 S3FS_PRN_ERR("write file error(%d).", errno);
704 return 0;
705 }
706 }
707 pCurl->partdata.startpos += totalwrite;
708 pCurl->partdata.size -= totalwrite;
709
710 return totalwrite;
711 }
712
SetCheckCertificate(bool isCertCheck)713 bool S3fsCurl::SetCheckCertificate(bool isCertCheck)
714 {
715 bool old = S3fsCurl::is_cert_check;
716 S3fsCurl::is_cert_check = isCertCheck;
717 return old;
718 }
719
SetDnsCache(bool isCache)720 bool S3fsCurl::SetDnsCache(bool isCache)
721 {
722 bool old = S3fsCurl::is_dns_cache;
723 S3fsCurl::is_dns_cache = isCache;
724 return old;
725 }
726
ResetOffset(S3fsCurl * pCurl)727 void S3fsCurl::ResetOffset(S3fsCurl* pCurl)
728 {
729 pCurl->partdata.startpos = pCurl->b_partdata_startpos;
730 pCurl->partdata.size = pCurl->b_partdata_size;
731 }
732
SetSslSessionCache(bool isCache)733 bool S3fsCurl::SetSslSessionCache(bool isCache)
734 {
735 bool old = S3fsCurl::is_ssl_session_cache;
736 S3fsCurl::is_ssl_session_cache = isCache;
737 return old;
738 }
739
SetConnectTimeout(long timeout)740 long S3fsCurl::SetConnectTimeout(long timeout)
741 {
742 long old = S3fsCurl::connect_timeout;
743 S3fsCurl::connect_timeout = timeout;
744 return old;
745 }
746
SetReadwriteTimeout(time_t timeout)747 time_t S3fsCurl::SetReadwriteTimeout(time_t timeout)
748 {
749 time_t old = S3fsCurl::readwrite_timeout;
750 S3fsCurl::readwrite_timeout = timeout;
751 return old;
752 }
753
SetRetries(int count)754 int S3fsCurl::SetRetries(int count)
755 {
756 int old = S3fsCurl::retries;
757 S3fsCurl::retries = count;
758 return old;
759 }
760
SetPublicBucket(bool flag)761 bool S3fsCurl::SetPublicBucket(bool flag)
762 {
763 bool old = S3fsCurl::is_public_bucket;
764 S3fsCurl::is_public_bucket = flag;
765 return old;
766 }
767
SetDefaultAcl(acl_t acl)768 acl_t S3fsCurl::SetDefaultAcl(acl_t acl)
769 {
770 acl_t old = S3fsCurl::default_acl;
771 S3fsCurl::default_acl = acl;
772 return old;
773 }
774
GetDefaultAcl()775 acl_t S3fsCurl::GetDefaultAcl()
776 {
777 return S3fsCurl::default_acl;
778 }
779
SetStorageClass(const std::string & storage_class)780 std::string S3fsCurl::SetStorageClass(const std::string& storage_class)
781 {
782 std::string old = S3fsCurl::storage_class;
783 S3fsCurl::storage_class = storage_class;
784 // AWS requires uppercase storage class values
785 transform(S3fsCurl::storage_class.begin(), S3fsCurl::storage_class.end(), S3fsCurl::storage_class.begin(), ::toupper);
786 return old;
787 }
788
PushbackSseKeys(const std::string & input)789 bool S3fsCurl::PushbackSseKeys(const std::string& input)
790 {
791 std::string onekey = trim(input);
792 if(onekey.empty()){
793 return false;
794 }
795 if('#' == onekey[0]){
796 return false;
797 }
798 // make base64 if the key is short enough, otherwise assume it is already so
799 std::string base64_key;
800 std::string raw_key;
801 if(onekey.length() > 256 / 8){
802 char* p_key;
803 size_t keylength;
804
805 if(NULL != (p_key = (char *)s3fs_decode64(onekey.c_str(), &keylength))) {
806 raw_key = std::string(p_key, keylength);
807 base64_key = onekey;
808 delete[] p_key;
809 } else {
810 S3FS_PRN_ERR("Failed to convert base64 to SSE-C key %s", onekey.c_str());
811 return false;
812 }
813 } else {
814 char* pbase64_key;
815
816 if(NULL != (pbase64_key = s3fs_base64((unsigned char*)onekey.c_str(), onekey.length()))) {
817 raw_key = onekey;
818 base64_key = pbase64_key;
819 delete[] pbase64_key;
820 } else {
821 S3FS_PRN_ERR("Failed to convert base64 from SSE-C key %s", onekey.c_str());
822 return false;
823 }
824 }
825
826 // make MD5
827 std::string strMd5;
828 if(!make_md5_from_binary(raw_key.c_str(), raw_key.length(), strMd5)){
829 S3FS_PRN_ERR("Could not make MD5 from SSE-C keys(%s).", raw_key.c_str());
830 return false;
831 }
832 // mapped MD5 = SSE Key
833 sseckeymap_t md5map;
834 md5map.clear();
835 md5map[strMd5] = base64_key;
836 S3fsCurl::sseckeys.push_back(md5map);
837
838 return true;
839 }
840
SetSseType(sse_type_t type)841 sse_type_t S3fsCurl::SetSseType(sse_type_t type)
842 {
843 sse_type_t old = S3fsCurl::ssetype;
844 S3fsCurl::ssetype = type;
845 return old;
846 }
847
SetSseCKeys(const char * filepath)848 bool S3fsCurl::SetSseCKeys(const char* filepath)
849 {
850 if(!filepath){
851 S3FS_PRN_ERR("SSE-C keys filepath is empty.");
852 return false;
853 }
854 struct stat st;
855 if(0 != stat(filepath, &st)){
856 S3FS_PRN_ERR("could not open use_sse keys file(%s).", filepath);
857 return false;
858 }
859 if(st.st_mode & (S_IXUSR | S_IRWXG | S_IRWXO)){
860 S3FS_PRN_ERR("use_sse keys file %s should be 0600 permissions.", filepath);
861 return false;
862 }
863
864 S3fsCurl::sseckeys.clear();
865
866 std::ifstream ssefs(filepath);
867 if(!ssefs.good()){
868 S3FS_PRN_ERR("Could not open SSE-C keys file(%s).", filepath);
869 return false;
870 }
871
872 std::string line;
873 while(getline(ssefs, line)){
874 S3fsCurl::PushbackSseKeys(line);
875 }
876 if(S3fsCurl::sseckeys.empty()){
877 S3FS_PRN_ERR("There is no SSE Key in file(%s).", filepath);
878 return false;
879 }
880 return true;
881 }
882
SetSseKmsid(const char * kmsid)883 bool S3fsCurl::SetSseKmsid(const char* kmsid)
884 {
885 if(!kmsid || '\0' == kmsid[0]){
886 S3FS_PRN_ERR("SSE-KMS kms id is empty.");
887 return false;
888 }
889 S3fsCurl::ssekmsid = kmsid;
890 return true;
891 }
892
893 // [NOTE]
894 // Because SSE is set by some options and environment,
895 // this function check the integrity of the SSE data finally.
FinalCheckSse()896 bool S3fsCurl::FinalCheckSse()
897 {
898 switch(S3fsCurl::ssetype){
899 case sse_type_t::SSE_DISABLE:
900 S3fsCurl::ssekmsid.erase();
901 return true;
902 case sse_type_t::SSE_S3:
903 S3fsCurl::ssekmsid.erase();
904 return true;
905 case sse_type_t::SSE_C:
906 if(S3fsCurl::sseckeys.empty()){
907 S3FS_PRN_ERR("sse type is SSE-C, but there is no custom key.");
908 return false;
909 }
910 S3fsCurl::ssekmsid.erase();
911 return true;
912 case sse_type_t::SSE_KMS:
913 if(S3fsCurl::ssekmsid.empty()){
914 S3FS_PRN_ERR("sse type is SSE-KMS, but there is no specified kms id.");
915 return false;
916 }
917 if(S3fsCurl::GetSignatureType() == V2_ONLY){
918 S3FS_PRN_ERR("sse type is SSE-KMS, but signature type is not v4. SSE-KMS require signature v4.");
919 return false;
920 }
921 return true;
922 }
923 S3FS_PRN_ERR("sse type is unknown(%d).", static_cast<int>(S3fsCurl::ssetype));
924
925 return false;
926 }
927
LoadEnvSseCKeys()928 bool S3fsCurl::LoadEnvSseCKeys()
929 {
930 char* envkeys = getenv("AWSSSECKEYS");
931 if(NULL == envkeys){
932 // nothing to do
933 return true;
934 }
935 S3fsCurl::sseckeys.clear();
936
937 std::istringstream fullkeys(envkeys);
938 std::string onekey;
939 while(getline(fullkeys, onekey, ':')){
940 S3fsCurl::PushbackSseKeys(onekey);
941 }
942 if(S3fsCurl::sseckeys.empty()){
943 S3FS_PRN_ERR("There is no SSE Key in environment(AWSSSECKEYS=%s).", envkeys);
944 return false;
945 }
946 return true;
947 }
948
LoadEnvSseKmsid()949 bool S3fsCurl::LoadEnvSseKmsid()
950 {
951 char* envkmsid = getenv("AWSSSEKMSID");
952 if(NULL == envkmsid){
953 // nothing to do
954 return true;
955 }
956 return S3fsCurl::SetSseKmsid(envkmsid);
957 }
958
959 //
960 // If md5 is empty, returns first(current) sse key.
961 //
GetSseKey(std::string & md5,std::string & ssekey)962 bool S3fsCurl::GetSseKey(std::string& md5, std::string& ssekey)
963 {
964 for(sseckeylist_t::const_iterator iter = S3fsCurl::sseckeys.begin(); iter != S3fsCurl::sseckeys.end(); ++iter){
965 if(md5.empty() || md5 == (*iter).begin()->first){
966 md5 = iter->begin()->first;
967 ssekey = iter->begin()->second;
968 return true;
969 }
970 }
971 return false;
972 }
973
GetSseKeyMd5(size_t pos,std::string & md5)974 bool S3fsCurl::GetSseKeyMd5(size_t pos, std::string& md5)
975 {
976 if(S3fsCurl::sseckeys.size() <= static_cast<size_t>(pos)){
977 return false;
978 }
979 size_t cnt = 0;
980 for(sseckeylist_t::const_iterator iter = S3fsCurl::sseckeys.begin(); iter != S3fsCurl::sseckeys.end(); ++iter, ++cnt){
981 if(pos == cnt){
982 md5 = iter->begin()->first;
983 return true;
984 }
985 }
986 return false;
987 }
988
GetSseKeyCount()989 size_t S3fsCurl::GetSseKeyCount()
990 {
991 return S3fsCurl::sseckeys.size();
992 }
993
SetContentMd5(bool flag)994 bool S3fsCurl::SetContentMd5(bool flag)
995 {
996 bool old = S3fsCurl::is_content_md5;
997 S3fsCurl::is_content_md5 = flag;
998 return old;
999 }
1000
SetVerbose(bool flag)1001 bool S3fsCurl::SetVerbose(bool flag)
1002 {
1003 bool old = S3fsCurl::is_verbose;
1004 S3fsCurl::is_verbose = flag;
1005 return old;
1006 }
1007
SetDumpBody(bool flag)1008 bool S3fsCurl::SetDumpBody(bool flag)
1009 {
1010 bool old = S3fsCurl::is_dump_body;
1011 S3fsCurl::is_dump_body = flag;
1012 return old;
1013 }
1014
SetAccessKey(const char * AccessKeyId,const char * SecretAccessKey)1015 bool S3fsCurl::SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey)
1016 {
1017 if((!S3fsCurl::is_ibm_iam_auth && (!AccessKeyId || '\0' == AccessKeyId[0])) || !SecretAccessKey || '\0' == SecretAccessKey[0]){
1018 return false;
1019 }
1020 AWSAccessKeyId = AccessKeyId;
1021 AWSSecretAccessKey = SecretAccessKey;
1022 return true;
1023 }
1024
SetAccessKeyWithSessionToken(const char * AccessKeyId,const char * SecretAccessKey,const char * SessionToken)1025 bool S3fsCurl::SetAccessKeyWithSessionToken(const char* AccessKeyId, const char* SecretAccessKey, const char* SessionToken)
1026 {
1027 bool access_key_is_empty = !AccessKeyId || '\0' == AccessKeyId[0];
1028 bool secret_access_key_is_empty = !SecretAccessKey || '\0' == SecretAccessKey[0];
1029 bool session_token_is_empty = !SessionToken || '\0' == SessionToken[0];
1030
1031 if((!S3fsCurl::is_ibm_iam_auth && access_key_is_empty) || secret_access_key_is_empty || session_token_is_empty){
1032 return false;
1033 }
1034 AWSAccessKeyId = AccessKeyId;
1035 AWSSecretAccessKey = SecretAccessKey;
1036 AWSAccessToken = SessionToken;
1037 S3fsCurl::is_use_session_token = true;
1038 return true;
1039 }
1040
SetSslVerifyHostname(long value)1041 long S3fsCurl::SetSslVerifyHostname(long value)
1042 {
1043 if(0 != value && 1 != value){
1044 return -1;
1045 }
1046 long old = S3fsCurl::ssl_verify_hostname;
1047 S3fsCurl::ssl_verify_hostname = value;
1048 return old;
1049 }
1050
SetIsIBMIAMAuth(bool flag)1051 bool S3fsCurl::SetIsIBMIAMAuth(bool flag)
1052 {
1053 bool old = S3fsCurl::is_ibm_iam_auth;
1054 S3fsCurl::is_ibm_iam_auth = flag;
1055 return old;
1056 }
1057
SetIsECS(bool flag)1058 bool S3fsCurl::SetIsECS(bool flag)
1059 {
1060 bool old = S3fsCurl::is_ecs;
1061 S3fsCurl::is_ecs = flag;
1062 return old;
1063 }
1064
SetIAMRole(const char * role)1065 std::string S3fsCurl::SetIAMRole(const char* role)
1066 {
1067 std::string old = S3fsCurl::IAM_role;
1068 S3fsCurl::IAM_role = role ? role : "";
1069 return old;
1070 }
1071
SetIAMFieldCount(size_t field_count)1072 size_t S3fsCurl::SetIAMFieldCount(size_t field_count)
1073 {
1074 size_t old = S3fsCurl::IAM_field_count;
1075 S3fsCurl::IAM_field_count = field_count;
1076 return old;
1077 }
1078
SetIAMCredentialsURL(const char * url)1079 std::string S3fsCurl::SetIAMCredentialsURL(const char* url)
1080 {
1081 std::string old = S3fsCurl::IAM_cred_url;
1082 S3fsCurl::IAM_cred_url = url ? url : "";
1083 return old;
1084 }
1085
SetIAMTokenField(const char * token_field)1086 std::string S3fsCurl::SetIAMTokenField(const char* token_field)
1087 {
1088 std::string old = S3fsCurl::IAM_token_field;
1089 S3fsCurl::IAM_token_field = token_field ? token_field : "";
1090 return old;
1091 }
1092
SetIAMExpiryField(const char * expiry_field)1093 std::string S3fsCurl::SetIAMExpiryField(const char* expiry_field)
1094 {
1095 std::string old = S3fsCurl::IAM_expiry_field;
1096 S3fsCurl::IAM_expiry_field = expiry_field ? expiry_field : "";
1097 return old;
1098 }
1099
SetIMDSVersion(int version)1100 bool S3fsCurl::SetIMDSVersion(int version)
1101 {
1102 S3fsCurl::IAM_api_version = version;
1103 return true;
1104 }
1105
SetMultipartSize(off_t size)1106 bool S3fsCurl::SetMultipartSize(off_t size)
1107 {
1108 size = size * 1024 * 1024;
1109 if(size < MIN_MULTIPART_SIZE){
1110 return false;
1111 }
1112 S3fsCurl::multipart_size = size;
1113 return true;
1114 }
1115
SetMultipartCopySize(off_t size)1116 bool S3fsCurl::SetMultipartCopySize(off_t size)
1117 {
1118 size = size * 1024 * 1024;
1119 if(size < MIN_MULTIPART_SIZE){
1120 return false;
1121 }
1122 S3fsCurl::multipart_copy_size = size;
1123 return true;
1124 }
1125
SetMaxParallelCount(int value)1126 int S3fsCurl::SetMaxParallelCount(int value)
1127 {
1128 int old = S3fsCurl::max_parallel_cnt;
1129 S3fsCurl::max_parallel_cnt = value;
1130 return old;
1131 }
1132
SetMaxMultiRequest(int max)1133 int S3fsCurl::SetMaxMultiRequest(int max)
1134 {
1135 int old = S3fsCurl::max_multireq;
1136 S3fsCurl::max_multireq = max;
1137 return old;
1138 }
1139
UploadMultipartPostCallback(S3fsCurl * s3fscurl)1140 bool S3fsCurl::UploadMultipartPostCallback(S3fsCurl* s3fscurl)
1141 {
1142 if(!s3fscurl){
1143 return false;
1144 }
1145
1146 return s3fscurl->UploadMultipartPostComplete();
1147 }
1148
MixMultipartPostCallback(S3fsCurl * s3fscurl)1149 bool S3fsCurl::MixMultipartPostCallback(S3fsCurl* s3fscurl)
1150 {
1151 if(!s3fscurl){
1152 return false;
1153 }
1154
1155 return s3fscurl->MixMultipartPostComplete();
1156 }
1157
UploadMultipartPostRetryCallback(S3fsCurl * s3fscurl)1158 S3fsCurl* S3fsCurl::UploadMultipartPostRetryCallback(S3fsCurl* s3fscurl)
1159 {
1160 if(!s3fscurl){
1161 return NULL;
1162 }
1163 // parse and get part_num, upload_id.
1164 std::string upload_id;
1165 std::string part_num_str;
1166 int part_num;
1167 off_t tmp_part_num = 0;
1168 if(!get_keyword_value(s3fscurl->url, "uploadId", upload_id)){
1169 return NULL;
1170 }
1171 if(!get_keyword_value(s3fscurl->url, "partNumber", part_num_str)){
1172 return NULL;
1173 }
1174 if(!s3fs_strtoofft(&tmp_part_num, part_num_str.c_str(), /*base=*/ 10)){
1175 return NULL;
1176 }
1177 part_num = static_cast<int>(tmp_part_num);
1178
1179 if(s3fscurl->retry_count >= S3fsCurl::retries){
1180 S3FS_PRN_ERR("Over retry count(%d) limit(%s:%d).", s3fscurl->retry_count, s3fscurl->path.c_str(), part_num);
1181 return NULL;
1182 }
1183
1184 // duplicate request
1185 S3fsCurl* newcurl = new S3fsCurl(s3fscurl->IsUseAhbe());
1186 newcurl->partdata.petag = s3fscurl->partdata.petag;
1187 newcurl->partdata.fd = s3fscurl->partdata.fd;
1188 newcurl->partdata.startpos = s3fscurl->b_partdata_startpos;
1189 newcurl->partdata.size = s3fscurl->b_partdata_size;
1190 newcurl->b_partdata_startpos = s3fscurl->b_partdata_startpos;
1191 newcurl->b_partdata_size = s3fscurl->b_partdata_size;
1192 newcurl->retry_count = s3fscurl->retry_count + 1;
1193 newcurl->op = s3fscurl->op;
1194 newcurl->type = s3fscurl->type;
1195
1196 // setup new curl object
1197 if(0 != newcurl->UploadMultipartPostSetup(s3fscurl->path.c_str(), part_num, upload_id)){
1198 S3FS_PRN_ERR("Could not duplicate curl object(%s:%d).", s3fscurl->path.c_str(), part_num);
1199 delete newcurl;
1200 return NULL;
1201 }
1202 return newcurl;
1203 }
1204
CopyMultipartPostRetryCallback(S3fsCurl * s3fscurl)1205 S3fsCurl* S3fsCurl::CopyMultipartPostRetryCallback(S3fsCurl* s3fscurl)
1206 {
1207 if(!s3fscurl){
1208 return NULL;
1209 }
1210 // parse and get part_num, upload_id.
1211 std::string upload_id;
1212 std::string part_num_str;
1213 int part_num;
1214 off_t tmp_part_num = 0;
1215 if(!get_keyword_value(s3fscurl->url, "uploadId", upload_id)){
1216 return NULL;
1217 }
1218 if(!get_keyword_value(s3fscurl->url, "partNumber", part_num_str)){
1219 return NULL;
1220 }
1221 if(!s3fs_strtoofft(&tmp_part_num, part_num_str.c_str(), /*base=*/ 10)){
1222 return NULL;
1223 }
1224 part_num = static_cast<int>(tmp_part_num);
1225
1226 if(s3fscurl->retry_count >= S3fsCurl::retries){
1227 S3FS_PRN_ERR("Over retry count(%d) limit(%s:%d).", s3fscurl->retry_count, s3fscurl->path.c_str(), part_num);
1228 return NULL;
1229 }
1230
1231 // duplicate request
1232 S3fsCurl* newcurl = new S3fsCurl(s3fscurl->IsUseAhbe());
1233 newcurl->partdata.petag = s3fscurl->partdata.petag;
1234 newcurl->b_from = s3fscurl->b_from;
1235 newcurl->b_meta = s3fscurl->b_meta;
1236 newcurl->retry_count = s3fscurl->retry_count + 1;
1237 newcurl->op = s3fscurl->op;
1238 newcurl->type = s3fscurl->type;
1239
1240 // setup new curl object
1241 if(0 != newcurl->CopyMultipartPostSetup(s3fscurl->b_from.c_str(), s3fscurl->path.c_str(), part_num, upload_id, s3fscurl->b_meta)){
1242 S3FS_PRN_ERR("Could not duplicate curl object(%s:%d).", s3fscurl->path.c_str(), part_num);
1243 delete newcurl;
1244 return NULL;
1245 }
1246 return newcurl;
1247 }
1248
MixMultipartPostRetryCallback(S3fsCurl * s3fscurl)1249 S3fsCurl* S3fsCurl::MixMultipartPostRetryCallback(S3fsCurl* s3fscurl)
1250 {
1251 if(!s3fscurl){
1252 return NULL;
1253 }
1254
1255 S3fsCurl* pcurl;
1256 if(-1 == s3fscurl->partdata.fd){
1257 pcurl = S3fsCurl::CopyMultipartPostRetryCallback(s3fscurl);
1258 }else{
1259 pcurl = S3fsCurl::UploadMultipartPostRetryCallback(s3fscurl);
1260 }
1261 return pcurl;
1262 }
1263
MapPutErrorResponse(int result)1264 int S3fsCurl::MapPutErrorResponse(int result)
1265 {
1266 if(result != 0){
1267 return result;
1268 }
1269 // PUT returns 200 status code with something error, thus
1270 // we need to check body.
1271 //
1272 // example error body:
1273 // <?xml version="1.0" encoding="UTF-8"?>
1274 // <Error>
1275 // <Code>AccessDenied</Code>
1276 // <Message>Access Denied</Message>
1277 // <RequestId>E4CA6F6767D6685C</RequestId>
1278 // <HostId>BHzLOATeDuvN8Es1wI8IcERq4kl4dc2A9tOB8Yqr39Ys6fl7N4EJ8sjGiVvu6wLP</HostId>
1279 // </Error>
1280 //
1281 const char* pstrbody = bodydata.str();
1282 if(!pstrbody || NULL != strcasestr(pstrbody, "<Error>")){
1283 S3FS_PRN_ERR("Put request get 200 status response, but it included error body(or NULL). The request failed during copying the object in S3.");
1284 S3FS_PRN_DBG("Put request Response Body : %s", (pstrbody ? pstrbody : "(null)"));
1285 // TODO: parse more specific error from <Code>
1286 result = -EIO;
1287 }
1288 return result;
1289 }
1290
ParallelMultipartUploadRequest(const char * tpath,headers_t & meta,int fd)1291 int S3fsCurl::ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd)
1292 {
1293 int result;
1294 std::string upload_id;
1295 struct stat st;
1296 int fd2;
1297 etaglist_t list;
1298 off_t remaining_bytes;
1299 S3fsCurl s3fscurl(true);
1300
1301 S3FS_PRN_INFO3("[tpath=%s][fd=%d]", SAFESTRPTR(tpath), fd);
1302
1303 // duplicate fd
1304 if(-1 == (fd2 = dup(fd)) || 0 != lseek(fd2, 0, SEEK_SET)){
1305 S3FS_PRN_ERR("Could not duplicate file descriptor(errno=%d)", errno);
1306 if(-1 != fd2){
1307 close(fd2);
1308 }
1309 return -errno;
1310 }
1311 if(-1 == fstat(fd2, &st)){
1312 S3FS_PRN_ERR("Invalid file descriptor(errno=%d)", errno);
1313 close(fd2);
1314 return -errno;
1315 }
1316
1317 if(0 != (result = s3fscurl.PreMultipartPostRequest(tpath, meta, upload_id, false))){
1318 close(fd2);
1319 return result;
1320 }
1321 s3fscurl.DestroyCurlHandle();
1322
1323 // Initialize S3fsMultiCurl
1324 S3fsMultiCurl curlmulti(GetMaxParallelCount());
1325 curlmulti.SetSuccessCallback(S3fsCurl::UploadMultipartPostCallback);
1326 curlmulti.SetRetryCallback(S3fsCurl::UploadMultipartPostRetryCallback);
1327
1328 // cycle through open fd, pulling off 10MB chunks at a time
1329 for(remaining_bytes = st.st_size; 0 < remaining_bytes; ){
1330 off_t chunk = remaining_bytes > S3fsCurl::multipart_size ? S3fsCurl::multipart_size : remaining_bytes;
1331
1332 // s3fscurl sub object
1333 S3fsCurl* s3fscurl_para = new S3fsCurl(true);
1334 s3fscurl_para->partdata.fd = fd2;
1335 s3fscurl_para->partdata.startpos = st.st_size - remaining_bytes;
1336 s3fscurl_para->partdata.size = chunk;
1337 s3fscurl_para->b_partdata_startpos = s3fscurl_para->partdata.startpos;
1338 s3fscurl_para->b_partdata_size = s3fscurl_para->partdata.size;
1339 s3fscurl_para->partdata.add_etag_list(&list);
1340
1341 // initiate upload part for parallel
1342 if(0 != (result = s3fscurl_para->UploadMultipartPostSetup(tpath, static_cast<int>(list.size()), upload_id))){
1343 S3FS_PRN_ERR("failed uploading part setup(%d)", result);
1344 close(fd2);
1345 delete s3fscurl_para;
1346 return result;
1347 }
1348
1349 // set into parallel object
1350 if(!curlmulti.SetS3fsCurlObject(s3fscurl_para)){
1351 S3FS_PRN_ERR("Could not make curl object into multi curl(%s).", tpath);
1352 close(fd2);
1353 delete s3fscurl_para;
1354 return -EIO;
1355 }
1356 remaining_bytes -= chunk;
1357 }
1358
1359 // Multi request
1360 if(0 != (result = curlmulti.Request())){
1361 S3FS_PRN_ERR("error occurred in multi request(errno=%d).", result);
1362
1363 S3fsCurl s3fscurl_abort(true);
1364 int result2 = s3fscurl_abort.AbortMultipartUpload(tpath, upload_id);
1365 s3fscurl_abort.DestroyCurlHandle();
1366 if(result2 != 0){
1367 S3FS_PRN_ERR("error aborting multipart upload(errno=%d).", result2);
1368 }
1369
1370 return result;
1371 }
1372
1373 close(fd2);
1374
1375 if(0 != (result = s3fscurl.CompleteMultipartPostRequest(tpath, upload_id, list))){
1376 return result;
1377 }
1378 return 0;
1379 }
1380
ParallelMixMultipartUploadRequest(const char * tpath,headers_t & meta,int fd,const fdpage_list_t & mixuppages)1381 int S3fsCurl::ParallelMixMultipartUploadRequest(const char* tpath, headers_t& meta, int fd, const fdpage_list_t& mixuppages)
1382 {
1383 int result;
1384 std::string upload_id;
1385 struct stat st;
1386 int fd2;
1387 etaglist_t list;
1388 S3fsCurl s3fscurl(true);
1389
1390 S3FS_PRN_INFO3("[tpath=%s][fd=%d]", SAFESTRPTR(tpath), fd);
1391
1392 // duplicate fd
1393 if(-1 == (fd2 = dup(fd)) || 0 != lseek(fd2, 0, SEEK_SET)){
1394 S3FS_PRN_ERR("Could not duplicate file descriptor(errno=%d)", errno);
1395 if(-1 != fd2){
1396 close(fd2);
1397 }
1398 return -errno;
1399 }
1400 if(-1 == fstat(fd2, &st)){
1401 S3FS_PRN_ERR("Invalid file descriptor(errno=%d)", errno);
1402 close(fd2);
1403 return -errno;
1404 }
1405
1406 if(0 != (result = s3fscurl.PreMultipartPostRequest(tpath, meta, upload_id, true))){
1407 close(fd2);
1408 return result;
1409 }
1410 s3fscurl.DestroyCurlHandle();
1411
1412 // for copy multipart
1413 std::string srcresource;
1414 std::string srcurl;
1415 MakeUrlResource(get_realpath(tpath).c_str(), srcresource, srcurl);
1416 meta["Content-Type"] = S3fsCurl::LookupMimeType(std::string(tpath));
1417 meta["x-amz-copy-source"] = srcresource;
1418
1419 // Initialize S3fsMultiCurl
1420 S3fsMultiCurl curlmulti(GetMaxParallelCount());
1421 curlmulti.SetSuccessCallback(S3fsCurl::MixMultipartPostCallback);
1422 curlmulti.SetRetryCallback(S3fsCurl::MixMultipartPostRetryCallback);
1423
1424 for(fdpage_list_t::const_iterator iter = mixuppages.begin(); iter != mixuppages.end(); ++iter){
1425 if(iter->modified){
1426 // Multipart upload
1427 S3fsCurl* s3fscurl_para = new S3fsCurl(true);
1428
1429 s3fscurl_para->partdata.fd = fd2;
1430 s3fscurl_para->partdata.startpos = iter->offset;
1431 s3fscurl_para->partdata.size = iter->bytes;
1432 s3fscurl_para->b_partdata_startpos = s3fscurl_para->partdata.startpos;
1433 s3fscurl_para->b_partdata_size = s3fscurl_para->partdata.size;
1434 s3fscurl_para->partdata.add_etag_list(&list);
1435
1436 S3FS_PRN_INFO3("Upload Part [tpath=%s][start=%lld][size=%lld][part=%zu]", SAFESTRPTR(tpath), static_cast<long long>(iter->offset), static_cast<long long>(iter->bytes), list.size());
1437
1438 // initiate upload part for parallel
1439 if(0 != (result = s3fscurl_para->UploadMultipartPostSetup(tpath, static_cast<int>(list.size()), upload_id))){
1440 S3FS_PRN_ERR("failed uploading part setup(%d)", result);
1441 close(fd2);
1442 delete s3fscurl_para;
1443 return result;
1444 }
1445
1446 // set into parallel object
1447 if(!curlmulti.SetS3fsCurlObject(s3fscurl_para)){
1448 S3FS_PRN_ERR("Could not make curl object into multi curl(%s).", tpath);
1449 close(fd2);
1450 delete s3fscurl_para;
1451 return -EIO;
1452 }
1453 }else{
1454 // Multipart copy
1455 for(off_t i = 0; i < iter->bytes; i += GetMultipartCopySize()){
1456 S3fsCurl* s3fscurl_para = new S3fsCurl(true);
1457
1458 off_t bytes = std::min(static_cast<off_t>(GetMultipartCopySize()), iter->bytes - i);
1459 std::ostringstream strrange;
1460 strrange << "bytes=" << (iter->offset + i) << "-" << (iter->offset + i + bytes - 1);
1461 meta["x-amz-copy-source-range"] = strrange.str();
1462
1463 s3fscurl_para->b_from = SAFESTRPTR(tpath);
1464 s3fscurl_para->b_meta = meta;
1465 s3fscurl_para->partdata.add_etag_list(&list);
1466
1467 S3FS_PRN_INFO3("Copy Part [tpath=%s][start=%lld][size=%lld][part=%zu]", SAFESTRPTR(tpath), static_cast<long long>(iter->offset + i), static_cast<long long>(bytes), list.size());
1468
1469 // initiate upload part for parallel
1470 if(0 != (result = s3fscurl_para->CopyMultipartPostSetup(tpath, tpath, static_cast<int>(list.size()), upload_id, meta))){
1471 S3FS_PRN_ERR("failed uploading part setup(%d)", result);
1472 close(fd2);
1473 delete s3fscurl_para;
1474 return result;
1475 }
1476
1477 // set into parallel object
1478 if(!curlmulti.SetS3fsCurlObject(s3fscurl_para)){
1479 S3FS_PRN_ERR("Could not make curl object into multi curl(%s).", tpath);
1480 close(fd2);
1481 delete s3fscurl_para;
1482 return -EIO;
1483 }
1484 }
1485 }
1486 }
1487
1488 // Multi request
1489 if(0 != (result = curlmulti.Request())){
1490 S3FS_PRN_ERR("error occurred in multi request(errno=%d).", result);
1491
1492 S3fsCurl s3fscurl_abort(true);
1493 int result2 = s3fscurl_abort.AbortMultipartUpload(tpath, upload_id);
1494 s3fscurl_abort.DestroyCurlHandle();
1495 if(result2 != 0){
1496 S3FS_PRN_ERR("error aborting multipart upload(errno=%d).", result2);
1497 }
1498 close(fd2);
1499 return result;
1500 }
1501 close(fd2);
1502
1503 if(0 != (result = s3fscurl.CompleteMultipartPostRequest(tpath, upload_id, list))){
1504 return result;
1505 }
1506 return 0;
1507 }
1508
ParallelGetObjectRetryCallback(S3fsCurl * s3fscurl)1509 S3fsCurl* S3fsCurl::ParallelGetObjectRetryCallback(S3fsCurl* s3fscurl)
1510 {
1511 int result;
1512
1513 if(!s3fscurl){
1514 return NULL;
1515 }
1516 if(s3fscurl->retry_count >= S3fsCurl::retries){
1517 S3FS_PRN_ERR("Over retry count(%d) limit(%s).", s3fscurl->retry_count, s3fscurl->path.c_str());
1518 return NULL;
1519 }
1520
1521 // duplicate request(setup new curl object)
1522 S3fsCurl* newcurl = new S3fsCurl(s3fscurl->IsUseAhbe());
1523
1524 if(0 != (result = newcurl->PreGetObjectRequest(s3fscurl->path.c_str(), s3fscurl->partdata.fd, s3fscurl->partdata.startpos, s3fscurl->partdata.size, s3fscurl->b_ssetype, s3fscurl->b_ssevalue))){
1525 S3FS_PRN_ERR("failed downloading part setup(%d)", result);
1526 delete newcurl;
1527 return NULL;;
1528 }
1529 newcurl->retry_count = s3fscurl->retry_count + 1;
1530
1531 return newcurl;
1532 }
1533
ParallelGetObjectRequest(const char * tpath,int fd,off_t start,off_t size)1534 int S3fsCurl::ParallelGetObjectRequest(const char* tpath, int fd, off_t start, off_t size)
1535 {
1536 S3FS_PRN_INFO3("[tpath=%s][fd=%d]", SAFESTRPTR(tpath), fd);
1537
1538 sse_type_t ssetype = sse_type_t::SSE_DISABLE;
1539 std::string ssevalue;
1540 if(!get_object_sse_type(tpath, ssetype, ssevalue)){
1541 S3FS_PRN_WARN("Failed to get SSE type for file(%s).", SAFESTRPTR(tpath));
1542 }
1543 int result = 0;
1544 off_t remaining_bytes;
1545
1546 // cycle through open fd, pulling off 10MB chunks at a time
1547 for(remaining_bytes = size; 0 < remaining_bytes; ){
1548 S3fsMultiCurl curlmulti(GetMaxParallelCount());
1549 int para_cnt;
1550 off_t chunk;
1551
1552 // Initialize S3fsMultiCurl
1553 //curlmulti.SetSuccessCallback(NULL); // not need to set success callback
1554 curlmulti.SetRetryCallback(S3fsCurl::ParallelGetObjectRetryCallback);
1555
1556 // Loop for setup parallel upload(multipart) request.
1557 for(para_cnt = 0; para_cnt < S3fsCurl::max_parallel_cnt && 0 < remaining_bytes; para_cnt++, remaining_bytes -= chunk){
1558 // chunk size
1559 chunk = remaining_bytes > S3fsCurl::multipart_size ? S3fsCurl::multipart_size : remaining_bytes;
1560
1561 // s3fscurl sub object
1562 S3fsCurl* s3fscurl_para = new S3fsCurl();
1563 if(0 != (result = s3fscurl_para->PreGetObjectRequest(tpath, fd, (start + size - remaining_bytes), chunk, ssetype, ssevalue))){
1564 S3FS_PRN_ERR("failed downloading part setup(%d)", result);
1565 delete s3fscurl_para;
1566 return result;
1567 }
1568
1569 // set into parallel object
1570 if(!curlmulti.SetS3fsCurlObject(s3fscurl_para)){
1571 S3FS_PRN_ERR("Could not make curl object into multi curl(%s).", tpath);
1572 delete s3fscurl_para;
1573 return -EIO;
1574 }
1575 }
1576
1577 // Multi request
1578 if(0 != (result = curlmulti.Request())){
1579 S3FS_PRN_ERR("error occurred in multi request(errno=%d).", result);
1580 break;
1581 }
1582
1583 // reinit for loop.
1584 curlmulti.Clear();
1585 }
1586 return result;
1587 }
1588
UploadMultipartPostSetCurlOpts(S3fsCurl * s3fscurl)1589 bool S3fsCurl::UploadMultipartPostSetCurlOpts(S3fsCurl* s3fscurl)
1590 {
1591 if(!s3fscurl){
1592 return false;
1593 }
1594 if(!s3fscurl->CreateCurlHandle()){
1595 return false;
1596 }
1597 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_URL, s3fscurl->url.c_str());
1598 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_UPLOAD, true); // HTTP PUT
1599 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEDATA, (void*)(&s3fscurl->bodydata));
1600 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
1601 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERDATA, (void*)&(s3fscurl->responseHeaders));
1602 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback);
1603 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(s3fscurl->partdata.size)); // Content-Length
1604 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_READFUNCTION, UploadReadCallback);
1605 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_READDATA, (void*)s3fscurl);
1606 S3fsCurl::AddUserAgent(s3fscurl->hCurl); // put User-Agent
1607
1608 return true;
1609 }
1610
CopyMultipartPostSetCurlOpts(S3fsCurl * s3fscurl)1611 bool S3fsCurl::CopyMultipartPostSetCurlOpts(S3fsCurl* s3fscurl)
1612 {
1613 if(!s3fscurl){
1614 return false;
1615 }
1616 if(!s3fscurl->CreateCurlHandle()){
1617 return false;
1618 }
1619
1620 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_URL, s3fscurl->url.c_str());
1621 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_UPLOAD, true); // HTTP PUT
1622 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEDATA, (void*)(&s3fscurl->bodydata));
1623 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
1624 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERDATA, (void*)(&s3fscurl->headdata));
1625 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERFUNCTION, WriteMemoryCallback);
1626 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_INFILESIZE, 0); // Content-Length
1627 S3fsCurl::AddUserAgent(s3fscurl->hCurl); // put User-Agent
1628
1629 return true;
1630 }
1631
PreGetObjectRequestSetCurlOpts(S3fsCurl * s3fscurl)1632 bool S3fsCurl::PreGetObjectRequestSetCurlOpts(S3fsCurl* s3fscurl)
1633 {
1634 if(!s3fscurl){
1635 return false;
1636 }
1637 if(!s3fscurl->CreateCurlHandle()){
1638 return false;
1639 }
1640
1641 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_URL, s3fscurl->url.c_str());
1642 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEFUNCTION, DownloadWriteCallback);
1643 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEDATA, (void*)s3fscurl);
1644 S3fsCurl::AddUserAgent(s3fscurl->hCurl); // put User-Agent
1645
1646 return true;
1647 }
1648
PreHeadRequestSetCurlOpts(S3fsCurl * s3fscurl)1649 bool S3fsCurl::PreHeadRequestSetCurlOpts(S3fsCurl* s3fscurl)
1650 {
1651 if(!s3fscurl){
1652 return false;
1653 }
1654 if(!s3fscurl->CreateCurlHandle()){
1655 return false;
1656 }
1657
1658 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_URL, s3fscurl->url.c_str());
1659 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_NOBODY, true); // HEAD
1660 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_FILETIME, true); // Last-Modified
1661
1662 // responseHeaders
1663 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERDATA, (void*)&(s3fscurl->responseHeaders));
1664 curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback);
1665 S3fsCurl::AddUserAgent(s3fscurl->hCurl); // put User-Agent
1666
1667 return true;
1668 }
1669
ParseIAMCredentialResponse(const char * response,iamcredmap_t & keyval)1670 bool S3fsCurl::ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval)
1671 {
1672 if(!response){
1673 return false;
1674 }
1675 std::istringstream sscred(response);
1676 std::string oneline;
1677 keyval.clear();
1678 while(getline(sscred, oneline, ',')){
1679 std::string::size_type pos;
1680 std::string key;
1681 std::string val;
1682 if(std::string::npos != (pos = oneline.find(IAMCRED_ACCESSKEYID))){
1683 key = IAMCRED_ACCESSKEYID;
1684 }else if(std::string::npos != (pos = oneline.find(IAMCRED_SECRETACCESSKEY))){
1685 key = IAMCRED_SECRETACCESSKEY;
1686 }else if(std::string::npos != (pos = oneline.find(S3fsCurl::IAM_token_field))){
1687 key = S3fsCurl::IAM_token_field;
1688 }else if(std::string::npos != (pos = oneline.find(S3fsCurl::IAM_expiry_field))){
1689 key = S3fsCurl::IAM_expiry_field;
1690 }else if(std::string::npos != (pos = oneline.find(IAMCRED_ROLEARN))){
1691 key = IAMCRED_ROLEARN;
1692 }else{
1693 continue;
1694 }
1695 if(std::string::npos == (pos = oneline.find(':', pos + key.length()))){
1696 continue;
1697 }
1698
1699 if(S3fsCurl::is_ibm_iam_auth && key == S3fsCurl::IAM_expiry_field){
1700 // parse integer value
1701 if(std::string::npos == (pos = oneline.find_first_of("0123456789", pos))){
1702 continue;
1703 }
1704 oneline.erase(0, pos);
1705 if(std::string::npos == (pos = oneline.find_last_of("0123456789"))){
1706 continue;
1707 }
1708 val = oneline.substr(0, pos+1);
1709 }else{
1710 // parse std::string value (starts and ends with quotes)
1711 if(std::string::npos == (pos = oneline.find('\"', pos))){
1712 continue;
1713 }
1714 oneline.erase(0, pos+1);
1715 if(std::string::npos == (pos = oneline.find('\"'))){
1716 continue;
1717 }
1718 val = oneline.substr(0, pos);
1719 }
1720 keyval[key] = val;
1721 }
1722 return true;
1723 }
1724
SetIAMv2APIToken(const char * response)1725 bool S3fsCurl::SetIAMv2APIToken(const char* response)
1726 {
1727 S3FS_PRN_INFO3("Setting AWS IMDSv2 API token to %s", response);
1728 S3fsCurl::IAMv2_api_token = std::string(response);
1729 return true;
1730 }
1731
SetIAMCredentials(const char * response)1732 bool S3fsCurl::SetIAMCredentials(const char* response)
1733 {
1734 S3FS_PRN_INFO3("IAM credential response = \"%s\"", response);
1735
1736 iamcredmap_t keyval;
1737
1738 if(!ParseIAMCredentialResponse(response, keyval)){
1739 return false;
1740 }
1741
1742 if(S3fsCurl::IAM_field_count != keyval.size()){
1743 return false;
1744 }
1745
1746 S3fsCurl::AWSAccessToken = keyval[std::string(S3fsCurl::IAM_token_field)];
1747
1748 if(S3fsCurl::is_ibm_iam_auth){
1749 off_t tmp_expire = 0;
1750 if(!s3fs_strtoofft(&tmp_expire, keyval[std::string(S3fsCurl::IAM_expiry_field)].c_str(), /*base=*/ 10)){
1751 return false;
1752 }
1753 S3fsCurl::AWSAccessTokenExpire = static_cast<time_t>(tmp_expire);
1754 }else{
1755 S3fsCurl::AWSAccessKeyId = keyval[std::string(IAMCRED_ACCESSKEYID)];
1756 S3fsCurl::AWSSecretAccessKey = keyval[std::string(IAMCRED_SECRETACCESSKEY)];
1757 S3fsCurl::AWSAccessTokenExpire = cvtIAMExpireStringToTime(keyval[S3fsCurl::IAM_expiry_field].c_str());
1758 }
1759 return true;
1760 }
1761
CheckIAMCredentialUpdate()1762 bool S3fsCurl::CheckIAMCredentialUpdate()
1763 {
1764 if(S3fsCurl::IAM_role.empty() && !S3fsCurl::is_ecs && !S3fsCurl::is_ibm_iam_auth){
1765 return true;
1766 }
1767 if(time(NULL) + IAM_EXPIRE_MERGIN <= S3fsCurl::AWSAccessTokenExpire){
1768 return true;
1769 }
1770 S3FS_PRN_INFO("IAM Access Token refreshing...");
1771 // update
1772 S3fsCurl s3fscurl;
1773 if(0 != s3fscurl.GetIAMCredentials()){
1774 S3FS_PRN_ERR("IAM Access Token refresh failed");
1775 return false;
1776 }
1777 S3FS_PRN_INFO("IAM Access Token refreshed");
1778 return true;
1779 }
1780
ParseIAMRoleFromMetaDataResponse(const char * response,std::string & rolename)1781 bool S3fsCurl::ParseIAMRoleFromMetaDataResponse(const char* response, std::string& rolename)
1782 {
1783 if(!response){
1784 return false;
1785 }
1786 // [NOTE]
1787 // expected following strings.
1788 //
1789 // myrolename
1790 //
1791 std::istringstream ssrole(response);
1792 std::string oneline;
1793 if (getline(ssrole, oneline, '\n')){
1794 rolename = oneline;
1795 return !rolename.empty();
1796 }
1797 return false;
1798 }
1799
SetIAMRoleFromMetaData(const char * response)1800 bool S3fsCurl::SetIAMRoleFromMetaData(const char* response)
1801 {
1802 S3FS_PRN_INFO3("IAM role name response = \"%s\"", response);
1803
1804 std::string rolename;
1805
1806 if(!S3fsCurl::ParseIAMRoleFromMetaDataResponse(response, rolename)){
1807 return false;
1808 }
1809
1810 SetIAMRole(rolename.c_str());
1811 return true;
1812 }
1813
AddUserAgent(CURL * hCurl)1814 bool S3fsCurl::AddUserAgent(CURL* hCurl)
1815 {
1816 if(!hCurl){
1817 return false;
1818 }
1819 if(S3fsCurl::IsUserAgentFlag()){
1820 curl_easy_setopt(hCurl, CURLOPT_USERAGENT, S3fsCurl::userAgent.c_str());
1821 }
1822 return true;
1823 }
1824
CurlDebugFunc(CURL * hcurl,curl_infotype type,char * data,size_t size,void * userptr)1825 int S3fsCurl::CurlDebugFunc(CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr)
1826 {
1827 return S3fsCurl::RawCurlDebugFunc(hcurl, type, data, size, userptr, CURLINFO_END);
1828 }
1829
CurlDebugBodyInFunc(CURL * hcurl,curl_infotype type,char * data,size_t size,void * userptr)1830 int S3fsCurl::CurlDebugBodyInFunc(CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr)
1831 {
1832 return S3fsCurl::RawCurlDebugFunc(hcurl, type, data, size, userptr, CURLINFO_DATA_IN);
1833 }
1834
CurlDebugBodyOutFunc(CURL * hcurl,curl_infotype type,char * data,size_t size,void * userptr)1835 int S3fsCurl::CurlDebugBodyOutFunc(CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr)
1836 {
1837 return S3fsCurl::RawCurlDebugFunc(hcurl, type, data, size, userptr, CURLINFO_DATA_OUT);
1838 }
1839
RawCurlDebugFunc(CURL * hcurl,curl_infotype type,char * data,size_t size,void * userptr,curl_infotype datatype)1840 int S3fsCurl::RawCurlDebugFunc(CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr, curl_infotype datatype)
1841 {
1842 if(!hcurl){
1843 // something wrong...
1844 return 0;
1845 }
1846
1847 switch(type){
1848 case CURLINFO_TEXT:
1849 // Swap tab indentation with spaces so it stays pretty in syslog
1850 int indent;
1851 indent = 0;
1852 while (*data == '\t' && size > 0) {
1853 indent += 4;
1854 size--;
1855 data++;
1856 }
1857 if(foreground && 0 < size && '\n' == data[size - 1]){
1858 size--;
1859 }
1860 S3FS_PRN_CURL("* %*s%.*s", indent, "", (int)size, data);
1861 break;
1862
1863 case CURLINFO_DATA_IN:
1864 case CURLINFO_DATA_OUT:
1865 if(type != datatype || !S3fsCurl::is_dump_body){
1866 // not put
1867 break;
1868 }
1869 case CURLINFO_HEADER_IN:
1870 case CURLINFO_HEADER_OUT:
1871 size_t remaining;
1872 char* p;
1873
1874 // Print each line individually for tidy output
1875 remaining = size;
1876 p = data;
1877 do {
1878 char* eol = (char*)memchr(p, '\n', remaining);
1879 int newline = 0;
1880 if (eol == NULL) {
1881 eol = (char*)memchr(p, '\r', remaining);
1882 } else {
1883 if (eol > p && *(eol - 1) == '\r') {
1884 newline++;
1885 }
1886 newline++;
1887 eol++;
1888 }
1889 size_t length = eol - p;
1890 S3FS_PRN_CURL("%s %.*s", getCurlDebugHead(type), (int)length - newline, p);
1891 remaining -= length;
1892 p = eol;
1893 } while (p != NULL && remaining > 0);
1894 break;
1895
1896 case CURLINFO_SSL_DATA_IN:
1897 case CURLINFO_SSL_DATA_OUT:
1898 // not put
1899 break;
1900 default:
1901 // why
1902 break;
1903 }
1904 return 0;
1905 }
1906
1907 //-------------------------------------------------------------------
1908 // Methods for S3fsCurl
1909 //-------------------------------------------------------------------
S3fsCurl(bool ahbe)1910 S3fsCurl::S3fsCurl(bool ahbe) :
1911 hCurl(NULL), type(REQTYPE_UNSET), requestHeaders(NULL),
1912 LastResponseCode(S3FSCURL_RESPONSECODE_NOTSET), postdata(NULL), postdata_remaining(0), is_use_ahbe(ahbe),
1913 retry_count(0), b_infile(NULL), b_postdata(NULL), b_postdata_remaining(0), b_partdata_startpos(0), b_partdata_size(0),
1914 b_ssekey_pos(-1), b_ssetype(sse_type_t::SSE_DISABLE),
1915 sem(NULL), completed_tids_lock(NULL), completed_tids(NULL), fpLazySetup(NULL), curlCode(CURLE_OK)
1916 {
1917 }
1918
~S3fsCurl()1919 S3fsCurl::~S3fsCurl()
1920 {
1921 DestroyCurlHandle();
1922 }
1923
ResetHandle(bool lock_already_held)1924 bool S3fsCurl::ResetHandle(bool lock_already_held)
1925 {
1926 bool run_once;
1927 {
1928 AutoLock lock(&S3fsCurl::curl_warnings_lock);
1929 run_once = curl_warnings_once;
1930 curl_warnings_once = true;
1931 }
1932
1933 curl_easy_reset(hCurl);
1934 curl_easy_setopt(hCurl, CURLOPT_NOSIGNAL, 1);
1935 curl_easy_setopt(hCurl, CURLOPT_FOLLOWLOCATION, true);
1936 curl_easy_setopt(hCurl, CURLOPT_CONNECTTIMEOUT, S3fsCurl::connect_timeout);
1937 curl_easy_setopt(hCurl, CURLOPT_NOPROGRESS, 0);
1938 curl_easy_setopt(hCurl, CURLOPT_PROGRESSFUNCTION, S3fsCurl::CurlProgress);
1939 curl_easy_setopt(hCurl, CURLOPT_PROGRESSDATA, hCurl);
1940 // curl_easy_setopt(hCurl, CURLOPT_FORBID_REUSE, 1);
1941 if(CURLE_OK != curl_easy_setopt(hCurl, S3FS_CURLOPT_TCP_KEEPALIVE, 1) && !run_once){
1942 S3FS_PRN_WARN("The CURLOPT_TCP_KEEPALIVE option could not be set. For maximize performance you need to enable this option and you should use libcurl 7.25.0 or later.");
1943 }
1944 if(CURLE_OK != curl_easy_setopt(hCurl, S3FS_CURLOPT_SSL_ENABLE_ALPN, 0) && !run_once){
1945 S3FS_PRN_WARN("The CURLOPT_SSL_ENABLE_ALPN option could not be unset. S3 server does not support ALPN, then this option should be disabled to maximize performance. you need to use libcurl 7.36.0 or later.");
1946 }
1947 if(CURLE_OK != curl_easy_setopt(hCurl, S3FS_CURLOPT_KEEP_SENDING_ON_ERROR, 1) && !run_once){
1948 S3FS_PRN_WARN("The S3FS_CURLOPT_KEEP_SENDING_ON_ERROR option could not be set. For maximize performance you need to enable this option and you should use libcurl 7.51.0 or later.");
1949 }
1950
1951 if(type != REQTYPE_IAMCRED && type != REQTYPE_IAMROLE){
1952 // REQTYPE_IAMCRED and REQTYPE_IAMROLE are always HTTP
1953 if(0 == S3fsCurl::ssl_verify_hostname){
1954 curl_easy_setopt(hCurl, CURLOPT_SSL_VERIFYHOST, 0);
1955 }
1956 if(!S3fsCurl::curl_ca_bundle.empty()){
1957 curl_easy_setopt(hCurl, CURLOPT_CAINFO, S3fsCurl::curl_ca_bundle.c_str());
1958 }
1959 }
1960 if((S3fsCurl::is_dns_cache || S3fsCurl::is_ssl_session_cache) && S3fsCurl::hCurlShare){
1961 curl_easy_setopt(hCurl, CURLOPT_SHARE, S3fsCurl::hCurlShare);
1962 }
1963 if(!S3fsCurl::is_cert_check) {
1964 S3FS_PRN_DBG("'no_check_certificate' option in effect.");
1965 S3FS_PRN_DBG("The server certificate won't be checked against the available certificate authorities.");
1966 curl_easy_setopt(hCurl, CURLOPT_SSL_VERIFYPEER, false);
1967 }
1968 if(S3fsCurl::is_verbose){
1969 curl_easy_setopt(hCurl, CURLOPT_VERBOSE, true);
1970 curl_easy_setopt(hCurl, CURLOPT_DEBUGFUNCTION, S3fsCurl::CurlDebugFunc);
1971 }
1972 if(!cipher_suites.empty()) {
1973 curl_easy_setopt(hCurl, CURLOPT_SSL_CIPHER_LIST, cipher_suites.c_str());
1974 }
1975
1976 AutoLock lock(&S3fsCurl::curl_handles_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
1977 S3fsCurl::curl_times[hCurl] = time(0);
1978 S3fsCurl::curl_progress[hCurl] = progress_t(-1, -1);
1979
1980 return true;
1981 }
1982
CreateCurlHandle(bool only_pool,bool remake)1983 bool S3fsCurl::CreateCurlHandle(bool only_pool, bool remake)
1984 {
1985 AutoLock lock(&S3fsCurl::curl_handles_lock);
1986
1987 if(hCurl && remake){
1988 if(!DestroyCurlHandle(false)){
1989 S3FS_PRN_ERR("could not destroy handle.");
1990 return false;
1991 }
1992 S3FS_PRN_INFO3("already has handle, so destroyed it or restored it to pool.");
1993 }
1994
1995 if(!hCurl){
1996 if(NULL == (hCurl = sCurlPool->GetHandler(only_pool))){
1997 if(!only_pool){
1998 S3FS_PRN_ERR("Failed to create handle.");
1999 return false;
2000 }else{
2001 // [NOTE]
2002 // Further initialization processing is left to lazy processing to be executed later.
2003 // (Currently we do not use only_pool=true, but this code is remained for the future)
2004 return true;
2005 }
2006 }
2007 }
2008 ResetHandle(/*lock_already_held=*/ true);
2009
2010 return true;
2011 }
2012
DestroyCurlHandle(bool restore_pool,bool clear_internal_data)2013 bool S3fsCurl::DestroyCurlHandle(bool restore_pool, bool clear_internal_data)
2014 {
2015 // [NOTE]
2016 // If type is REQTYPE_IAMCRED or REQTYPE_IAMROLE, do not clear type.
2017 // Because that type only uses HTTP protocol, then the special
2018 // logic in ResetHandle function.
2019 //
2020 if(type != REQTYPE_IAMCRED && type != REQTYPE_IAMROLE){
2021 type = REQTYPE_UNSET;
2022 }
2023
2024 if(clear_internal_data){
2025 ClearInternalData();
2026 }
2027
2028 if(hCurl){
2029 AutoLock lock(&S3fsCurl::curl_handles_lock);
2030
2031 S3fsCurl::curl_times.erase(hCurl);
2032 S3fsCurl::curl_progress.erase(hCurl);
2033 sCurlPool->ReturnHandler(hCurl, restore_pool);
2034 hCurl = NULL;
2035 }else{
2036 return false;
2037 }
2038 return true;
2039 }
2040
ClearInternalData()2041 bool S3fsCurl::ClearInternalData()
2042 {
2043 // Always clear internal data
2044 //
2045 type = REQTYPE_UNSET;
2046 path = "";
2047 base_path = "";
2048 saved_path = "";
2049 url = "";
2050 op = "";
2051 query_string= "";
2052 if(requestHeaders){
2053 curl_slist_free_all(requestHeaders);
2054 requestHeaders = NULL;
2055 }
2056 responseHeaders.clear();
2057 bodydata.Clear();
2058 headdata.Clear();
2059 LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET;
2060 postdata = NULL;
2061 postdata_remaining = 0;
2062 retry_count = 0;
2063 b_infile = NULL;
2064 b_postdata = NULL;
2065 b_postdata_remaining = 0;
2066 b_partdata_startpos = 0;
2067 b_partdata_size = 0;
2068 partdata.clear();
2069
2070 fpLazySetup = NULL;
2071
2072 S3FS_MALLOCTRIM(0);
2073
2074 return true;
2075 }
2076
SetUseAhbe(bool ahbe)2077 bool S3fsCurl::SetUseAhbe(bool ahbe)
2078 {
2079 bool old = is_use_ahbe;
2080 is_use_ahbe = ahbe;
2081 return old;
2082 }
2083
GetResponseCode(long & responseCode,bool from_curl_handle)2084 bool S3fsCurl::GetResponseCode(long& responseCode, bool from_curl_handle)
2085 {
2086 responseCode = -1;
2087
2088 if(!from_curl_handle){
2089 responseCode = LastResponseCode;
2090 }else{
2091 if(!hCurl){
2092 return false;
2093 }
2094 if(CURLE_OK != curl_easy_getinfo(hCurl, CURLINFO_RESPONSE_CODE, &LastResponseCode)){
2095 return false;
2096 }
2097 responseCode = LastResponseCode;
2098 }
2099 return true;
2100 }
2101
2102 //
2103 // Reset all options for retrying
2104 //
RemakeHandle()2105 bool S3fsCurl::RemakeHandle()
2106 {
2107 S3FS_PRN_INFO3("Retry request. [type=%d][url=%s][path=%s]", type, url.c_str(), path.c_str());
2108
2109 if(REQTYPE_UNSET == type){
2110 return false;
2111 }
2112
2113 // rewind file
2114 struct stat st;
2115 if(b_infile){
2116 rewind(b_infile);
2117 if(-1 == fstat(fileno(b_infile), &st)){
2118 S3FS_PRN_WARN("Could not get file stat(fd=%d)", fileno(b_infile));
2119 return false;
2120 }
2121 }
2122
2123 // reinitialize internal data
2124 requestHeaders = curl_slist_remove(requestHeaders, "Authorization");
2125 responseHeaders.clear();
2126 bodydata.Clear();
2127 headdata.Clear();
2128 LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET;
2129
2130 // count up(only use for multipart)
2131 retry_count++;
2132
2133 // set from backup
2134 postdata = b_postdata;
2135 postdata_remaining = b_postdata_remaining;
2136 partdata.startpos = b_partdata_startpos;
2137 partdata.size = b_partdata_size;
2138
2139 // reset handle
2140 ResetHandle();
2141
2142 // set options
2143 switch(type){
2144 case REQTYPE_DELETE:
2145 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2146 curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE");
2147 break;
2148
2149 case REQTYPE_HEAD:
2150 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2151 curl_easy_setopt(hCurl, CURLOPT_NOBODY, true);
2152 curl_easy_setopt(hCurl, CURLOPT_FILETIME, true);
2153 // responseHeaders
2154 curl_easy_setopt(hCurl, CURLOPT_HEADERDATA, (void*)&responseHeaders);
2155 curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback);
2156 break;
2157
2158 case REQTYPE_PUTHEAD:
2159 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2160 curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true);
2161 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2162 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2163 curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0);
2164 break;
2165
2166 case REQTYPE_PUT:
2167 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2168 curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true);
2169 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2170 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2171 if(b_infile){
2172 curl_easy_setopt(hCurl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(st.st_size));
2173 curl_easy_setopt(hCurl, CURLOPT_INFILE, b_infile);
2174 }else{
2175 curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0);
2176 }
2177 break;
2178
2179 case REQTYPE_GET:
2180 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2181 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, S3fsCurl::DownloadWriteCallback);
2182 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)this);
2183 break;
2184
2185 case REQTYPE_CHKBUCKET:
2186 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2187 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2188 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2189 break;
2190
2191 case REQTYPE_LISTBUCKET:
2192 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2193 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2194 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2195 break;
2196
2197 case REQTYPE_PREMULTIPOST:
2198 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2199 curl_easy_setopt(hCurl, CURLOPT_POST, true);
2200 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2201 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2202 curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, 0);
2203 break;
2204
2205 case REQTYPE_COMPLETEMULTIPOST:
2206 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2207 curl_easy_setopt(hCurl, CURLOPT_POST, true);
2208 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2209 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2210 curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, static_cast<curl_off_t>(postdata_remaining));
2211 curl_easy_setopt(hCurl, CURLOPT_READDATA, (void*)this);
2212 curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::ReadCallback);
2213 break;
2214
2215 case REQTYPE_UPLOADMULTIPOST:
2216 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2217 curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true);
2218 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2219 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2220 curl_easy_setopt(hCurl, CURLOPT_HEADERDATA, (void*)&responseHeaders);
2221 curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback);
2222 curl_easy_setopt(hCurl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(partdata.size));
2223 curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::UploadReadCallback);
2224 curl_easy_setopt(hCurl, CURLOPT_READDATA, (void*)this);
2225 break;
2226
2227 case REQTYPE_COPYMULTIPOST:
2228 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2229 curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true);
2230 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2231 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2232 curl_easy_setopt(hCurl, CURLOPT_HEADERDATA, (void*)&headdata);
2233 curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, WriteMemoryCallback);
2234 curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0);
2235 break;
2236
2237 case REQTYPE_MULTILIST:
2238 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2239 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2240 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2241 break;
2242
2243 case REQTYPE_IAMCRED:
2244 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2245 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2246 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2247 if(S3fsCurl::is_ibm_iam_auth){
2248 curl_easy_setopt(hCurl, CURLOPT_POST, true);
2249 curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, static_cast<curl_off_t>(postdata_remaining));
2250 curl_easy_setopt(hCurl, CURLOPT_READDATA, (void*)this);
2251 curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::ReadCallback);
2252 }
2253 break;
2254
2255 case REQTYPE_ABORTMULTIUPLOAD:
2256 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2257 curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE");
2258 break;
2259
2260 case REQTYPE_IAMROLE:
2261 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2262 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2263 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2264 break;
2265
2266 default:
2267 S3FS_PRN_ERR("request type is unknown(%d)", type);
2268 return false;
2269 }
2270 S3fsCurl::AddUserAgent(hCurl); // put User-Agent
2271
2272 return true;
2273 }
2274
2275 //
2276 // returns curl return code
2277 //
RequestPerform(bool dontAddAuthHeaders)2278 int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
2279 {
2280 if(S3fsLog::IsS3fsLogDbg()){
2281 char* ptr_url = NULL;
2282 curl_easy_getinfo(hCurl, CURLINFO_EFFECTIVE_URL , &ptr_url);
2283 S3FS_PRN_DBG("connecting to URL %s", SAFESTRPTR(ptr_url));
2284 }
2285
2286 LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET;
2287 long responseCode = S3FSCURL_RESPONSECODE_NOTSET;
2288 int result = S3FSCURL_PERFORM_RESULT_NOTSET;
2289
2290 // 1 attempt + retries...
2291 for(int retrycnt = 0; S3FSCURL_PERFORM_RESULT_NOTSET == result && retrycnt < S3fsCurl::retries; ++retrycnt){
2292 // Reset response code
2293 responseCode = S3FSCURL_RESPONSECODE_NOTSET;
2294
2295 // Insert headers
2296 if(!dontAddAuthHeaders) {
2297 insertAuthHeaders();
2298 }
2299
2300 curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
2301
2302 // Requests
2303 curlCode = curl_easy_perform(hCurl);
2304
2305 // Check result
2306 switch(curlCode){
2307 case CURLE_OK:
2308 // Need to look at the HTTP response code
2309 if(0 != curl_easy_getinfo(hCurl, CURLINFO_RESPONSE_CODE, &responseCode)){
2310 S3FS_PRN_ERR("curl_easy_getinfo failed while trying to retrieve HTTP response code");
2311 responseCode = S3FSCURL_RESPONSECODE_FATAL_ERROR;
2312 result = -EIO;
2313 break;
2314 }
2315 if(responseCode >= 200 && responseCode < 300){
2316 S3FS_PRN_INFO3("HTTP response code %ld", responseCode);
2317 result = 0;
2318 break;
2319 }
2320
2321 {
2322 // Try to parse more specific AWS error code otherwise fall back to HTTP error code.
2323 std::string value;
2324 if(simple_parse_xml(bodydata.str(), bodydata.size(), "Code", value)){
2325 // TODO: other error codes
2326 if(value == "EntityTooLarge"){
2327 result = -EFBIG;
2328 break;
2329 }else if(value == "InvalidObjectState"){
2330 result = -EREMOTE;
2331 break;
2332 }else if(value == "KeyTooLongError"){
2333 result = -ENAMETOOLONG;
2334 break;
2335 }
2336 }
2337 }
2338
2339 // Service response codes which are >= 300 && < 500
2340 switch(responseCode){
2341 case 301:
2342 case 307:
2343 S3FS_PRN_ERR("HTTP response code 301(Moved Permanently: also happens when bucket's region is incorrect), returning EIO. Body Text: %s", bodydata.str());
2344 S3FS_PRN_ERR("The options of url and endpoint may be useful for solving, please try to use both options.");
2345 result = -EIO;
2346 break;
2347
2348 case 400:
2349 if(op == "HEAD"){
2350 if(path.size() > 1024){
2351 S3FS_PRN_ERR("HEAD HTTP response code %ld with path longer than 1024, returning ENAMETOOLONG.", responseCode);
2352 return -ENAMETOOLONG;
2353 }
2354 S3FS_PRN_ERR("HEAD HTTP response code %ld, returning EPERM.", responseCode);
2355 result = -EPERM;
2356 }else{
2357 S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.str());
2358 result = -EIO;
2359 }
2360 break;
2361
2362 case 403:
2363 S3FS_PRN_ERR("HTTP response code %ld, returning EPERM. Body Text: %s", responseCode, bodydata.str());
2364 result = -EPERM;
2365 break;
2366
2367 case 404:
2368 S3FS_PRN_INFO3("HTTP response code 404 was returned, returning ENOENT");
2369 S3FS_PRN_DBG("Body Text: %s", bodydata.str());
2370 result = -ENOENT;
2371 break;
2372
2373 case 416:
2374 S3FS_PRN_INFO3("HTTP response code 416 was returned, returning EIO");
2375 result = -EIO;
2376 break;
2377
2378 case 501:
2379 S3FS_PRN_INFO3("HTTP response code 501 was returned, returning ENOTSUP");
2380 S3FS_PRN_DBG("Body Text: %s", bodydata.str());
2381 result = -ENOTSUP;
2382 break;
2383
2384 case 500:
2385 case 503: {
2386 S3FS_PRN_INFO3("HTTP response code %ld was returned, slowing down", responseCode);
2387 S3FS_PRN_DBG("Body Text: %s", bodydata.str());
2388 // Add jitter to avoid thundering herd.
2389 unsigned int sleep_time = 2 << retry_count;
2390 sleep(sleep_time + random() % sleep_time);
2391 break;
2392 }
2393 default:
2394 S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.str());
2395 result = -EIO;
2396 break;
2397 }
2398 break;
2399
2400 case CURLE_WRITE_ERROR:
2401 S3FS_PRN_ERR("### CURLE_WRITE_ERROR");
2402 sleep(2);
2403 break;
2404
2405 case CURLE_OPERATION_TIMEDOUT:
2406 S3FS_PRN_ERR("### CURLE_OPERATION_TIMEDOUT");
2407 sleep(2);
2408 break;
2409
2410 case CURLE_COULDNT_RESOLVE_HOST:
2411 S3FS_PRN_ERR("### CURLE_COULDNT_RESOLVE_HOST");
2412 sleep(2);
2413 break;
2414
2415 case CURLE_COULDNT_CONNECT:
2416 S3FS_PRN_ERR("### CURLE_COULDNT_CONNECT");
2417 sleep(4);
2418 break;
2419
2420 case CURLE_GOT_NOTHING:
2421 S3FS_PRN_ERR("### CURLE_GOT_NOTHING");
2422 sleep(4);
2423 break;
2424
2425 case CURLE_ABORTED_BY_CALLBACK:
2426 S3FS_PRN_ERR("### CURLE_ABORTED_BY_CALLBACK");
2427 sleep(4);
2428 {
2429 AutoLock lock(&S3fsCurl::curl_handles_lock);
2430 S3fsCurl::curl_times[hCurl] = time(0);
2431 }
2432 break;
2433
2434 case CURLE_PARTIAL_FILE:
2435 S3FS_PRN_ERR("### CURLE_PARTIAL_FILE");
2436 sleep(4);
2437 break;
2438
2439 case CURLE_SEND_ERROR:
2440 S3FS_PRN_ERR("### CURLE_SEND_ERROR");
2441 sleep(2);
2442 break;
2443
2444 case CURLE_RECV_ERROR:
2445 S3FS_PRN_ERR("### CURLE_RECV_ERROR");
2446 sleep(2);
2447 break;
2448
2449 case CURLE_SSL_CONNECT_ERROR:
2450 S3FS_PRN_ERR("### CURLE_SSL_CONNECT_ERROR");
2451 sleep(2);
2452 break;
2453
2454 case CURLE_SSL_CACERT:
2455 S3FS_PRN_ERR("### CURLE_SSL_CACERT");
2456
2457 // try to locate cert, if successful, then set the
2458 // option and continue
2459 if(S3fsCurl::curl_ca_bundle.empty()){
2460 if(!S3fsCurl::LocateBundle()){
2461 S3FS_PRN_ERR("could not get CURL_CA_BUNDLE.");
2462 result = -EIO;
2463 }
2464 // retry with CAINFO
2465 }else{
2466 S3FS_PRN_ERR("curlCode: %d msg: %s", curlCode, curl_easy_strerror(curlCode));
2467 result = -EIO;
2468 }
2469 break;
2470
2471 #ifdef CURLE_PEER_FAILED_VERIFICATION
2472 case CURLE_PEER_FAILED_VERIFICATION:
2473 S3FS_PRN_ERR("### CURLE_PEER_FAILED_VERIFICATION");
2474
2475 first_pos = bucket.find_first_of('.');
2476 if(first_pos != std::string::npos){
2477 S3FS_PRN_INFO("curl returned a CURL_PEER_FAILED_VERIFICATION error");
2478 S3FS_PRN_INFO("security issue found: buckets with periods in their name are incompatible with http");
2479 S3FS_PRN_INFO("This check can be over-ridden by using the -o ssl_verify_hostname=0");
2480 S3FS_PRN_INFO("The certificate will still be checked but the hostname will not be verified.");
2481 S3FS_PRN_INFO("A more secure method would be to use a bucket name without periods.");
2482 }else{
2483 S3FS_PRN_INFO("my_curl_easy_perform: curlCode: %d -- %s", curlCode, curl_easy_strerror(curlCode));
2484 }
2485 result = -EIO;
2486 break;
2487 #endif
2488
2489 // This should be invalid since curl option HTTP FAILONERROR is now off
2490 case CURLE_HTTP_RETURNED_ERROR:
2491 S3FS_PRN_ERR("### CURLE_HTTP_RETURNED_ERROR");
2492
2493 if(0 != curl_easy_getinfo(hCurl, CURLINFO_RESPONSE_CODE, &responseCode)){
2494 result = -EIO;
2495 }else{
2496 S3FS_PRN_INFO3("HTTP response code =%ld", responseCode);
2497
2498 // Let's try to retrieve the
2499 if(404 == responseCode){
2500 result = -ENOENT;
2501 }else if(500 > responseCode){
2502 result = -EIO;
2503 }
2504 }
2505 break;
2506
2507 // Unknown CURL return code
2508 default:
2509 S3FS_PRN_ERR("###curlCode: %d msg: %s", curlCode, curl_easy_strerror(curlCode));
2510 result = -EIO;
2511 break;
2512 } // switch
2513
2514 if(S3FSCURL_PERFORM_RESULT_NOTSET == result){
2515 S3FS_PRN_INFO("### retrying...");
2516
2517 if(!RemakeHandle()){
2518 S3FS_PRN_INFO("Failed to reset handle and internal data for retrying.");
2519 result = -EIO;
2520 break;
2521 }
2522 }
2523 } // for
2524
2525 // set last response code
2526 if(S3FSCURL_RESPONSECODE_NOTSET == responseCode){
2527 LastResponseCode = S3FSCURL_RESPONSECODE_FATAL_ERROR;
2528 }else{
2529 LastResponseCode = responseCode;
2530 }
2531
2532 if(S3FSCURL_PERFORM_RESULT_NOTSET == result){
2533 S3FS_PRN_ERR("### giving up");
2534 result = -EIO;
2535 }
2536 return result;
2537 }
2538
2539 //
2540 // Returns the Amazon AWS signature for the given parameters.
2541 //
2542 // @param method e.g., "GET"
2543 // @param content_type e.g., "application/x-directory"
2544 // @param date e.g., get_date_rfc850()
2545 // @param resource e.g., "/pub"
2546 //
CalcSignatureV2(const std::string & method,const std::string & strMD5,const std::string & content_type,const std::string & date,const std::string & resource)2547 std::string S3fsCurl::CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource)
2548 {
2549 std::string Signature;
2550 std::string StringToSign;
2551
2552 if(!S3fsCurl::IAM_role.empty() || S3fsCurl::is_ecs || S3fsCurl::is_use_session_token){
2553 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", S3fsCurl::AWSAccessToken.c_str());
2554 }
2555
2556 StringToSign += method + "\n";
2557 StringToSign += strMD5 + "\n"; // md5
2558 StringToSign += content_type + "\n";
2559 StringToSign += date + "\n";
2560 StringToSign += get_canonical_headers(requestHeaders, true);
2561 StringToSign += resource;
2562
2563 const void* key = S3fsCurl::AWSSecretAccessKey.data();
2564 size_t key_len = S3fsCurl::AWSSecretAccessKey.size();
2565 const unsigned char* sdata = reinterpret_cast<const unsigned char*>(StringToSign.data());
2566 size_t sdata_len = StringToSign.size();
2567 unsigned char* md = NULL;
2568 unsigned int md_len = 0;;
2569
2570 s3fs_HMAC(key, key_len, sdata, sdata_len, &md, &md_len);
2571
2572 char* base64;
2573 if(NULL == (base64 = s3fs_base64(md, md_len))){
2574 delete[] md;
2575 return std::string(""); // ENOMEM
2576 }
2577 delete[] md;
2578
2579 Signature = base64;
2580 delete[] base64;
2581
2582 return Signature;
2583 }
2584
CalcSignature(const std::string & method,const std::string & canonical_uri,const std::string & query_string,const std::string & strdate,const std::string & payload_hash,const std::string & date8601)2585 std::string S3fsCurl::CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601)
2586 {
2587 std::string Signature, StringCQ, StringToSign;
2588 std::string uriencode;
2589
2590 if(!S3fsCurl::IAM_role.empty() || S3fsCurl::is_ecs || S3fsCurl::is_use_session_token){
2591 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", S3fsCurl::AWSAccessToken.c_str());
2592 }
2593
2594 uriencode = urlEncode(canonical_uri);
2595 StringCQ = method + "\n";
2596 if(method == "HEAD" || method == "PUT" || method == "DELETE"){
2597 StringCQ += uriencode + "\n";
2598 }else if(method == "GET" && uriencode.empty()){
2599 StringCQ +="/\n";
2600 }else if(method == "GET" && is_prefix(uriencode.c_str(), "/")){
2601 StringCQ += uriencode +"\n";
2602 }else if(method == "GET" && !is_prefix(uriencode.c_str(), "/")){
2603 StringCQ += "/\n" + urlEncode2(canonical_uri) +"\n";
2604 }else if(method == "POST"){
2605 StringCQ += uriencode + "\n";
2606 }
2607 StringCQ += urlEncode2(query_string) + "\n";
2608 StringCQ += get_canonical_headers(requestHeaders) + "\n";
2609 StringCQ += get_sorted_header_keys(requestHeaders) + "\n";
2610 StringCQ += payload_hash;
2611
2612 std::string kSecret = "AWS4" + S3fsCurl::AWSSecretAccessKey;
2613 unsigned char *kDate, *kRegion, *kService, *kSigning, *sRequest = NULL;
2614 unsigned int kDate_len,kRegion_len, kService_len, kSigning_len, sRequest_len = 0;
2615
2616 s3fs_HMAC256(kSecret.c_str(), kSecret.size(), reinterpret_cast<const unsigned char*>(strdate.data()), strdate.size(), &kDate, &kDate_len);
2617 s3fs_HMAC256(kDate, kDate_len, reinterpret_cast<const unsigned char*>(endpoint.c_str()), endpoint.size(), &kRegion, &kRegion_len);
2618 s3fs_HMAC256(kRegion, kRegion_len, reinterpret_cast<const unsigned char*>("s3"), sizeof("s3") - 1, &kService, &kService_len);
2619 s3fs_HMAC256(kService, kService_len, reinterpret_cast<const unsigned char*>("aws4_request"), sizeof("aws4_request") - 1, &kSigning, &kSigning_len);
2620 delete[] kDate;
2621 delete[] kRegion;
2622 delete[] kService;
2623
2624 const unsigned char* cRequest = reinterpret_cast<const unsigned char*>(StringCQ.c_str());
2625 size_t cRequest_len = StringCQ.size();
2626 s3fs_sha256(cRequest, cRequest_len, &sRequest, &sRequest_len);
2627
2628 StringToSign = "AWS4-HMAC-SHA256\n";
2629 StringToSign += date8601 + "\n";
2630 StringToSign += strdate + "/" + endpoint + "/s3/aws4_request\n";
2631 StringToSign += s3fs_hex_lower(sRequest, sRequest_len);
2632 delete[] sRequest;
2633
2634 const unsigned char* cscope = reinterpret_cast<const unsigned char*>(StringToSign.c_str());
2635 size_t cscope_len = StringToSign.size();
2636 unsigned char* md = NULL;
2637 unsigned int md_len = 0;
2638
2639 s3fs_HMAC256(kSigning, kSigning_len, cscope, cscope_len, &md, &md_len);
2640 delete[] kSigning;
2641
2642 Signature = s3fs_hex_lower(md, md_len);
2643 delete[] md;
2644
2645 return Signature;
2646 }
2647
insertV4Headers()2648 void S3fsCurl::insertV4Headers()
2649 {
2650 std::string server_path = type == REQTYPE_LISTBUCKET ? "/" : path;
2651 std::string payload_hash;
2652 switch (type) {
2653 case REQTYPE_PUT:
2654 payload_hash = s3fs_sha256_hex_fd(b_infile == NULL ? -1 : fileno(b_infile), 0, -1);
2655 break;
2656
2657 case REQTYPE_COMPLETEMULTIPOST:
2658 {
2659 size_t cRequest_len = strlen(reinterpret_cast<const char *>(b_postdata));
2660 unsigned char* sRequest = NULL;
2661 unsigned int sRequest_len = 0;
2662 s3fs_sha256(b_postdata, cRequest_len, &sRequest, &sRequest_len);
2663 payload_hash = s3fs_hex_lower(sRequest, sRequest_len);
2664 delete[] sRequest;
2665 break;
2666 }
2667
2668 case REQTYPE_UPLOADMULTIPOST:
2669 payload_hash = s3fs_sha256_hex_fd(partdata.fd, partdata.startpos, partdata.size);
2670 break;
2671 default:
2672 break;
2673 }
2674
2675 if(b_infile != NULL && payload_hash.empty()){
2676 S3FS_PRN_ERR("Failed to make SHA256.");
2677 // TODO: propagate error
2678 }
2679
2680 S3FS_PRN_INFO3("computing signature [%s] [%s] [%s] [%s]", op.c_str(), server_path.c_str(), query_string.c_str(), payload_hash.c_str());
2681 std::string strdate;
2682 std::string date8601;
2683 get_date_sigv3(strdate, date8601);
2684
2685 std::string contentSHA256 = payload_hash.empty() ? EMPTY_PAYLOAD_HASH : payload_hash;
2686 const std::string realpath = pathrequeststyle ? "/" + bucket + server_path : server_path;
2687
2688 //string canonical_headers, signed_headers;
2689 requestHeaders = curl_slist_sort_insert(requestHeaders, "host", get_bucket_host().c_str());
2690 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", contentSHA256.c_str());
2691 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str());
2692
2693 if (S3fsCurl::IsRequesterPays()) {
2694 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-request-payer", "requester");
2695 }
2696
2697 if(!S3fsCurl::IsPublicBucket()){
2698 std::string Signature = CalcSignature(op, realpath, query_string + (type == REQTYPE_PREMULTIPOST || type == REQTYPE_MULTILIST ? "=" : ""), strdate, contentSHA256, date8601);
2699 std::string auth = "AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature;
2700 requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", auth.c_str());
2701 }
2702 }
2703
insertV2Headers()2704 void S3fsCurl::insertV2Headers()
2705 {
2706 std::string resource;
2707 std::string turl;
2708 std::string server_path = type == REQTYPE_LISTBUCKET ? "/" : path;
2709 MakeUrlResource(server_path.c_str(), resource, turl);
2710 if(!query_string.empty() && type != REQTYPE_CHKBUCKET && type != REQTYPE_LISTBUCKET){
2711 resource += "?" + query_string;
2712 }
2713
2714 std::string date = get_date_rfc850();
2715 requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str());
2716 if(op != "PUT" && op != "POST"){
2717 requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", NULL);
2718 }
2719
2720 if(!S3fsCurl::IsPublicBucket()){
2721 std::string Signature = CalcSignatureV2(op, get_header_value(requestHeaders, "Content-MD5"), get_header_value(requestHeaders, "Content-Type"), date, resource);
2722 requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", std::string("AWS " + AWSAccessKeyId + ":" + Signature).c_str());
2723 }
2724 }
2725
insertIBMIAMHeaders()2726 void S3fsCurl::insertIBMIAMHeaders()
2727 {
2728 requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", ("Bearer " + S3fsCurl::AWSAccessToken).c_str());
2729
2730 if(op == "PUT" && path == mount_prefix + "/"){
2731 // ibm-service-instance-id header is required for bucket creation requests
2732 requestHeaders = curl_slist_sort_insert(requestHeaders, "ibm-service-instance-id", S3fsCurl::AWSAccessKeyId.c_str());
2733 }
2734 }
2735
insertAuthHeaders()2736 void S3fsCurl::insertAuthHeaders()
2737 {
2738 if(!S3fsCurl::CheckIAMCredentialUpdate()){
2739 S3FS_PRN_ERR("An error occurred in checking IAM credential.");
2740 return; // do not insert auth headers on error
2741 }
2742
2743 if(S3fsCurl::is_ibm_iam_auth){
2744 insertIBMIAMHeaders();
2745 }else if(S3fsCurl::signature_type == V2_ONLY){
2746 insertV2Headers();
2747 }else{
2748 insertV4Headers();
2749 }
2750 }
2751
DeleteRequest(const char * tpath)2752 int S3fsCurl::DeleteRequest(const char* tpath)
2753 {
2754 S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath));
2755
2756 if(!tpath){
2757 return -EINVAL;
2758 }
2759 if(!CreateCurlHandle()){
2760 return -EIO;
2761 }
2762 std::string resource;
2763 std::string turl;
2764 MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
2765
2766 url = prepare_url(turl.c_str());
2767 path = get_realpath(tpath);
2768 requestHeaders = NULL;
2769 responseHeaders.clear();
2770
2771 op = "DELETE";
2772 type = REQTYPE_DELETE;
2773
2774 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2775 curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE");
2776 S3fsCurl::AddUserAgent(hCurl); // put User-Agent
2777
2778 return RequestPerform();
2779 }
2780
2781 //
2782 // Get the token that we need to pass along with AWS IMDSv2 API requests
2783 //
GetIAMv2ApiToken()2784 int S3fsCurl::GetIAMv2ApiToken()
2785 {
2786 url = std::string(S3fsCurl::IAMv2_token_url);
2787 if(!CreateCurlHandle()){
2788 return -EIO;
2789 }
2790 requestHeaders = NULL;
2791 responseHeaders.clear();
2792 bodydata.Clear();
2793
2794 std::string ttlstr = str(S3fsCurl::IAMv2_token_ttl);
2795 requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCurl::IAMv2_token_ttl_hdr.c_str(),
2796 ttlstr.c_str());
2797 curl_easy_setopt(hCurl, CURLOPT_PUT, true);
2798 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2799 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2800 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2801 S3fsCurl::AddUserAgent(hCurl);
2802
2803 int result = RequestPerform(true);
2804
2805 if(0 == result && !S3fsCurl::SetIAMv2APIToken(bodydata.str())){
2806 S3FS_PRN_ERR("Error storing IMDSv2 API token.");
2807 result = -EIO;
2808 }
2809 bodydata.Clear();
2810 curl_easy_setopt(hCurl, CURLOPT_PUT, false);
2811
2812 return result;
2813 }
2814
2815 //
2816 // Get AccessKeyId/SecretAccessKey/AccessToken/Expiration by IAM role,
2817 // and Set these value to class variable.
2818 //
GetIAMCredentials()2819 int S3fsCurl::GetIAMCredentials()
2820 {
2821 if (!S3fsCurl::is_ecs && !S3fsCurl::is_ibm_iam_auth) {
2822 S3FS_PRN_INFO3("[IAM role=%s]", S3fsCurl::IAM_role.c_str());
2823
2824 if(S3fsCurl::IAM_role.empty()) {
2825 S3FS_PRN_ERR("IAM role name is empty.");
2826 return -EIO;
2827 }
2828 }
2829
2830 // at first set type for handle
2831 type = REQTYPE_IAMCRED;
2832
2833 if(!CreateCurlHandle()){
2834 return -EIO;
2835 }
2836
2837 // url
2838 if(is_ecs){
2839 const char *env = std::getenv(ECS_IAM_ENV_VAR);
2840 if(env == NULL){
2841 S3FS_PRN_ERR("%s is not set.", ECS_IAM_ENV_VAR);
2842 return -EIO;
2843 }
2844 url = std::string(S3fsCurl::IAM_cred_url) + env;
2845 }else{
2846 if(S3fsCurl::IAM_api_version > 1){
2847 int result = GetIAMv2ApiToken();
2848 if(-ENOENT == result){
2849 // If we get a 404 back when requesting the token service,
2850 // then it's highly likely we're running in an environment
2851 // that doesn't support the AWS IMDSv2 API, so we'll skip
2852 // the token retrieval in the future.
2853 SetIMDSVersion(1);
2854 }else if(result != 0){
2855 // If we get an unexpected error when retrieving the API
2856 // token, log it but continue. Requirement for including
2857 // an API token with the metadata request may or may not
2858 // be required, so we should not abort here.
2859 S3FS_PRN_ERR("AWS IMDSv2 token retrieval failed: %d", result);
2860 }
2861 }
2862
2863 url = std::string(S3fsCurl::IAM_cred_url) + S3fsCurl::IAM_role;
2864 }
2865
2866 requestHeaders = NULL;
2867 responseHeaders.clear();
2868 bodydata.Clear();
2869 std::string postContent;
2870
2871 if(S3fsCurl::is_ibm_iam_auth){
2872 url = std::string(S3fsCurl::IAM_cred_url);
2873
2874 // make contents
2875 postContent += "grant_type=urn:ibm:params:oauth:grant-type:apikey";
2876 postContent += "&response_type=cloud_iam";
2877 postContent += "&apikey=" + S3fsCurl::AWSSecretAccessKey;
2878
2879 // set postdata
2880 postdata = reinterpret_cast<const unsigned char*>(postContent.c_str());
2881 b_postdata = postdata;
2882 postdata_remaining = postContent.size(); // without null
2883 b_postdata_remaining = postdata_remaining;
2884
2885 requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", "Basic Yng6Yng=");
2886
2887 curl_easy_setopt(hCurl, CURLOPT_POST, true); // POST
2888 curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, static_cast<curl_off_t>(postdata_remaining));
2889 curl_easy_setopt(hCurl, CURLOPT_READDATA, (void*)this);
2890 curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::ReadCallback);
2891 }
2892
2893 if(S3fsCurl::IAM_api_version > 1){
2894 requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCurl::IAMv2_token_hdr.c_str(), S3fsCurl::IAMv2_api_token.c_str());
2895 }
2896
2897 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2898 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2899 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2900 S3fsCurl::AddUserAgent(hCurl); // put User-Agent
2901
2902 int result = RequestPerform(true);
2903
2904 // analyzing response
2905 if(0 == result && !S3fsCurl::SetIAMCredentials(bodydata.str())){
2906 S3FS_PRN_ERR("Something error occurred, could not get IAM credential.");
2907 result = -EIO;
2908 }
2909 bodydata.Clear();
2910
2911 return result;
2912 }
2913
2914 //
2915 // Get IAM role name automatically.
2916 //
LoadIAMRoleFromMetaData()2917 bool S3fsCurl::LoadIAMRoleFromMetaData()
2918 {
2919 S3FS_PRN_INFO3("Get IAM Role name");
2920
2921 // at first set type for handle
2922 type = REQTYPE_IAMROLE;
2923
2924 if(!CreateCurlHandle()){
2925 return false;
2926 }
2927
2928 // url
2929 url = std::string(S3fsCurl::IAM_cred_url);
2930 requestHeaders = NULL;
2931 responseHeaders.clear();
2932 bodydata.Clear();
2933
2934 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
2935 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
2936 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
2937 S3fsCurl::AddUserAgent(hCurl); // put User-Agent
2938
2939 int result = RequestPerform(true);
2940
2941 // analyzing response
2942 if(0 == result && !S3fsCurl::SetIAMRoleFromMetaData(bodydata.str())){
2943 S3FS_PRN_ERR("Something error occurred, could not get IAM role name.");
2944 result = -EIO;
2945 }
2946 bodydata.Clear();
2947
2948 return (0 == result);
2949 }
2950
AddSseRequestHead(sse_type_t ssetype,const std::string & input,bool is_only_c,bool is_copy)2951 bool S3fsCurl::AddSseRequestHead(sse_type_t ssetype, const std::string& input, bool is_only_c, bool is_copy)
2952 {
2953 std::string ssevalue = input;
2954 switch(ssetype){
2955 case sse_type_t::SSE_DISABLE:
2956 return true;
2957 case sse_type_t::SSE_S3:
2958 if(!is_only_c){
2959 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256");
2960 }
2961 return true;
2962 case sse_type_t::SSE_C:
2963 {
2964 std::string sseckey;
2965 if(S3fsCurl::GetSseKey(ssevalue, sseckey)){
2966 if(is_copy){
2967 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-algorithm", "AES256");
2968 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-key", sseckey.c_str());
2969 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-key-md5", ssevalue.c_str());
2970 }else{
2971 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-algorithm", "AES256");
2972 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-key", sseckey.c_str());
2973 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-key-md5", ssevalue.c_str());
2974 }
2975 }else{
2976 S3FS_PRN_WARN("Failed to insert SSE-C header.");
2977 }
2978 return true;
2979 }
2980 case sse_type_t::SSE_KMS:
2981 if(!is_only_c){
2982 if(ssevalue.empty()){
2983 ssevalue = S3fsCurl::GetSseKmsId();
2984 }
2985 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "aws:kms");
2986 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-aws-kms-key-id", ssevalue.c_str());
2987 }
2988 return true;
2989 }
2990 S3FS_PRN_ERR("sse type is unknown(%d).", static_cast<int>(S3fsCurl::ssetype));
2991
2992 return false;
2993 }
2994
2995 //
2996 // tpath : target path for head request
2997 // bpath : saved into base_path
2998 // savedpath : saved into saved_path
2999 // ssekey_pos : -1 means "not" SSE-C type
3000 // 0 - X means SSE-C type and position for SSE-C key(0 is latest key)
3001 //
PreHeadRequest(const char * tpath,const char * bpath,const char * savedpath,size_t ssekey_pos)3002 bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char* savedpath, size_t ssekey_pos)
3003 {
3004 S3FS_PRN_INFO3("[tpath=%s][bpath=%s][save=%s][sseckeypos=%zu]", SAFESTRPTR(tpath), SAFESTRPTR(bpath), SAFESTRPTR(savedpath), ssekey_pos);
3005
3006 if(!tpath){
3007 return false;
3008 }
3009 std::string resource;
3010 std::string turl;
3011 MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
3012
3013 // libcurl 7.17 does deep copy of url, deep copy "stable" url
3014 url = prepare_url(turl.c_str());
3015 path = get_realpath(tpath);
3016 base_path = SAFESTRPTR(bpath);
3017 saved_path = SAFESTRPTR(savedpath);
3018 requestHeaders = NULL;
3019 responseHeaders.clear();
3020
3021 // requestHeaders
3022 if(0 == ssekey_pos){
3023 std::string md5;
3024 if(!S3fsCurl::GetSseKeyMd5(ssekey_pos, md5) || !AddSseRequestHead(sse_type_t::SSE_C, md5, true, false)){
3025 S3FS_PRN_ERR("Failed to set SSE-C headers for sse-c key pos(%zu)(=md5(%s)).", ssekey_pos, md5.c_str());
3026 return false;
3027 }
3028 }
3029 b_ssekey_pos = ssekey_pos;
3030
3031 op = "HEAD";
3032 type = REQTYPE_HEAD;
3033
3034 // set lazy function
3035 fpLazySetup = PreHeadRequestSetCurlOpts;
3036
3037 return true;
3038 }
3039
HeadRequest(const char * tpath,headers_t & meta)3040 int S3fsCurl::HeadRequest(const char* tpath, headers_t& meta)
3041 {
3042 int result = -1;
3043
3044 S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath));
3045
3046 // At first, try to get without SSE-C headers
3047 if(!PreHeadRequest(tpath) || !fpLazySetup || !fpLazySetup(this) || 0 != (result = RequestPerform())){
3048 // If has SSE-C keys, try to get with all SSE-C keys.
3049 for(size_t pos = 0; pos < S3fsCurl::sseckeys.size(); pos++){
3050 if(!DestroyCurlHandle()){
3051 break;
3052 }
3053 if(!PreHeadRequest(tpath, NULL, NULL, pos)){
3054 break;
3055 }
3056 if(!fpLazySetup || !fpLazySetup(this)){
3057 S3FS_PRN_ERR("Failed to lazy setup in single head request.");
3058 break;
3059 }
3060 if(0 == (result = RequestPerform())){
3061 break;
3062 }
3063 }
3064 if(0 != result){
3065 DestroyCurlHandle(); // not check result.
3066 return result;
3067 }
3068 }
3069
3070 // file exists in s3
3071 // fixme: clean this up.
3072 meta.clear();
3073 for(headers_t::iterator iter = responseHeaders.begin(); iter != responseHeaders.end(); ++iter){
3074 std::string key = lower(iter->first);
3075 std::string value = iter->second;
3076 if(key == "content-type"){
3077 meta[iter->first] = value;
3078 }else if(key == "content-length"){
3079 meta[iter->first] = value;
3080 }else if(key == "etag"){
3081 meta[iter->first] = value;
3082 }else if(key == "last-modified"){
3083 meta[iter->first] = value;
3084 }else if(is_prefix(key.c_str(), "x-amz")){
3085 meta[key] = value; // key is lower case for "x-amz"
3086 }
3087 }
3088 return 0;
3089 }
3090
PutHeadRequest(const char * tpath,headers_t & meta,bool is_copy)3091 int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
3092 {
3093 S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath));
3094
3095 if(!tpath){
3096 return -EINVAL;
3097 }
3098 if(!CreateCurlHandle()){
3099 return -EIO;
3100 }
3101 std::string resource;
3102 std::string turl;
3103 MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
3104
3105 url = prepare_url(turl.c_str());
3106 path = get_realpath(tpath);
3107 requestHeaders = NULL;
3108 responseHeaders.clear();
3109 bodydata.Clear();
3110
3111 std::string contype = S3fsCurl::LookupMimeType(std::string(tpath));
3112 requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
3113
3114 // Make request headers
3115 for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
3116 std::string key = lower(iter->first);
3117 std::string value = iter->second;
3118 if(is_prefix(key.c_str(), "x-amz-acl")){
3119 // not set value, but after set it.
3120 }else if(is_prefix(key.c_str(), "x-amz-meta")){
3121 requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
3122 }else if(key == "x-amz-copy-source"){
3123 requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
3124 }else if(key == "x-amz-server-side-encryption" && value != "aws:kms"){
3125 // Only copy mode.
3126 if(is_copy && !AddSseRequestHead(sse_type_t::SSE_S3, value, false, true)){
3127 S3FS_PRN_WARN("Failed to insert SSE-S3 header.");
3128 }
3129 }else if(key == "x-amz-server-side-encryption-aws-kms-key-id"){
3130 // Only copy mode.
3131 if(is_copy && !value.empty() && !AddSseRequestHead(sse_type_t::SSE_KMS, value, false, true)){
3132 S3FS_PRN_WARN("Failed to insert SSE-KMS header.");
3133 }
3134 }else if(key == "x-amz-server-side-encryption-customer-key-md5"){
3135 // Only copy mode.
3136 if(is_copy){
3137 if(!AddSseRequestHead(sse_type_t::SSE_C, value, true, true) || !AddSseRequestHead(sse_type_t::SSE_C, value, true, false)){
3138 S3FS_PRN_WARN("Failed to insert SSE-C header.");
3139 }
3140 }
3141 }
3142 }
3143
3144 // "x-amz-acl", storage class, sse
3145 if(S3fsCurl::default_acl != acl_t::PRIVATE){
3146 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-acl", S3fsCurl::default_acl.str());
3147 }
3148 if(strcasecmp(GetStorageClass().c_str(), "STANDARD") != 0){
3149 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", GetStorageClass().c_str());
3150 }
3151 // SSE
3152 if(!is_copy){
3153 std::string ssevalue;
3154 if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false, false)){
3155 S3FS_PRN_WARN("Failed to set SSE header, but continue...");
3156 }
3157 }
3158 if(is_use_ahbe){
3159 // set additional header by ahbe conf
3160 requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath);
3161 }
3162
3163 op = "PUT";
3164 type = REQTYPE_PUTHEAD;
3165
3166 // setopt
3167 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
3168 curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true); // HTTP PUT
3169 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
3170 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
3171 curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0); // Content-Length
3172 S3fsCurl::AddUserAgent(hCurl); // put User-Agent
3173
3174 S3FS_PRN_INFO3("copying... [path=%s]", tpath);
3175
3176 int result = RequestPerform();
3177 result = MapPutErrorResponse(result);
3178 bodydata.Clear();
3179
3180 return result;
3181 }
3182
PutRequest(const char * tpath,headers_t & meta,int fd)3183 int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
3184 {
3185 struct stat st;
3186 FILE* file = NULL;
3187
3188 S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath));
3189
3190 if(!tpath){
3191 return -EINVAL;
3192 }
3193 if(-1 != fd){
3194 // duplicate fd
3195 int fd2;
3196 if(-1 == (fd2 = dup(fd)) || -1 == fstat(fd2, &st) || 0 != lseek(fd2, 0, SEEK_SET) || NULL == (file = fdopen(fd2, "rb"))){
3197 S3FS_PRN_ERR("Could not duplicate file descriptor(errno=%d)", errno);
3198 if(-1 != fd2){
3199 close(fd2);
3200 }
3201 return -errno;
3202 }
3203 b_infile = file;
3204 }else{
3205 // This case is creating zero byte object.(calling by create_file_object())
3206 S3FS_PRN_INFO3("create zero byte file object.");
3207 }
3208
3209 if(!CreateCurlHandle()){
3210 if(file){
3211 fclose(file);
3212 }
3213 return -EIO;
3214 }
3215 std::string resource;
3216 std::string turl;
3217 MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
3218
3219 url = prepare_url(turl.c_str());
3220 path = get_realpath(tpath);
3221 requestHeaders = NULL;
3222 responseHeaders.clear();
3223 bodydata.Clear();
3224
3225 // Make request headers
3226 std::string strMD5;
3227 if(S3fsCurl::is_content_md5){
3228 if(-1 != fd){
3229 strMD5 = s3fs_get_content_md5(fd);
3230 if(0 == strMD5.length()){
3231 S3FS_PRN_ERR("Failed to make MD5.");
3232 return -EIO;
3233 }
3234 }else{
3235 strMD5 = EMPTY_MD5_BASE64_HASH;
3236 }
3237 requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-MD5", strMD5.c_str());
3238 }
3239
3240 std::string contype = S3fsCurl::LookupMimeType(std::string(tpath));
3241 requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
3242
3243 for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
3244 std::string key = lower(iter->first);
3245 std::string value = iter->second;
3246 if(is_prefix(key.c_str(), "x-amz-acl")){
3247 // not set value, but after set it.
3248 }else if(is_prefix(key.c_str(), "x-amz-meta")){
3249 requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
3250 }else if(key == "x-amz-server-side-encryption" && value != "aws:kms"){
3251 // skip this header, because this header is specified after logic.
3252 }else if(key == "x-amz-server-side-encryption-aws-kms-key-id"){
3253 // skip this header, because this header is specified after logic.
3254 }else if(key == "x-amz-server-side-encryption-customer-key-md5"){
3255 // skip this header, because this header is specified after logic.
3256 }
3257 }
3258 // "x-amz-acl", storage class, sse
3259 if(S3fsCurl::default_acl != acl_t::PRIVATE){
3260 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-acl", S3fsCurl::default_acl.str());
3261 }
3262 if(strcasecmp(GetStorageClass().c_str(), "STANDARD") != 0){
3263 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", GetStorageClass().c_str());
3264 }
3265 // SSE
3266 std::string ssevalue;
3267 // do not add SSE for create bucket
3268 if(0 != strcmp(tpath, "/")){
3269 if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false, false)){
3270 S3FS_PRN_WARN("Failed to set SSE header, but continue...");
3271 }
3272 }
3273 if(is_use_ahbe){
3274 // set additional header by ahbe conf
3275 requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath);
3276 }
3277
3278 op = "PUT";
3279 type = REQTYPE_PUT;
3280
3281 // setopt
3282 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
3283 curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true); // HTTP PUT
3284 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
3285 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
3286 if(file){
3287 curl_easy_setopt(hCurl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(st.st_size)); // Content-Length
3288 curl_easy_setopt(hCurl, CURLOPT_INFILE, file);
3289 }else{
3290 curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0); // Content-Length: 0
3291 }
3292 S3fsCurl::AddUserAgent(hCurl); // put User-Agent
3293
3294 S3FS_PRN_INFO3("uploading... [path=%s][fd=%d][size=%lld]", tpath, fd, static_cast<long long int>(-1 != fd ? st.st_size : 0));
3295
3296 int result = RequestPerform();
3297 result = MapPutErrorResponse(result);
3298 bodydata.Clear();
3299 if(file){
3300 fclose(file);
3301 }
3302 return result;
3303 }
3304
PreGetObjectRequest(const char * tpath,int fd,off_t start,off_t size,sse_type_t ssetype,const std::string & ssevalue)3305 int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, off_t size, sse_type_t ssetype, const std::string& ssevalue)
3306 {
3307 S3FS_PRN_INFO3("[tpath=%s][start=%lld][size=%lld]", SAFESTRPTR(tpath), static_cast<long long>(start), static_cast<long long>(size));
3308
3309 if(!tpath || -1 == fd || 0 > start || 0 > size){
3310 return -EINVAL;
3311 }
3312
3313 std::string resource;
3314 std::string turl;
3315 MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
3316
3317 url = prepare_url(turl.c_str());
3318 path = get_realpath(tpath);
3319 requestHeaders = NULL;
3320 responseHeaders.clear();
3321
3322 if(0 < size){
3323 std::string range = "bytes=";
3324 range += str(start);
3325 range += "-";
3326 range += str(start + size - 1);
3327 requestHeaders = curl_slist_sort_insert(requestHeaders, "Range", range.c_str());
3328 }
3329 // SSE
3330 if(!AddSseRequestHead(ssetype, ssevalue, true, false)){
3331 S3FS_PRN_WARN("Failed to set SSE header, but continue...");
3332 }
3333
3334 op = "GET";
3335 type = REQTYPE_GET;
3336
3337 // set lazy function
3338 fpLazySetup = PreGetObjectRequestSetCurlOpts;
3339
3340 // set info for callback func.
3341 // (use only fd, startpos and size, other member is not used.)
3342 partdata.clear();
3343 partdata.fd = fd;
3344 partdata.startpos = start;
3345 partdata.size = size;
3346 b_partdata_startpos = start;
3347 b_partdata_size = size;
3348 b_ssetype = ssetype;
3349 b_ssevalue = ssevalue;
3350 b_ssekey_pos = -1; // not use this value for get object.
3351
3352 return 0;
3353 }
3354
GetObjectRequest(const char * tpath,int fd,off_t start,off_t size)3355 int S3fsCurl::GetObjectRequest(const char* tpath, int fd, off_t start, off_t size)
3356 {
3357 int result;
3358
3359 S3FS_PRN_INFO3("[tpath=%s][start=%lld][size=%lld]", SAFESTRPTR(tpath), static_cast<long long>(start), static_cast<long long>(size));
3360
3361 if(!tpath){
3362 return -EINVAL;
3363 }
3364 sse_type_t ssetype = sse_type_t::SSE_DISABLE;
3365 std::string ssevalue;
3366 if(!get_object_sse_type(tpath, ssetype, ssevalue)){
3367 S3FS_PRN_WARN("Failed to get SSE type for file(%s).", SAFESTRPTR(tpath));
3368 }
3369
3370 if(0 != (result = PreGetObjectRequest(tpath, fd, start, size, ssetype, ssevalue))){
3371 return result;
3372 }
3373 if(!fpLazySetup || !fpLazySetup(this)){
3374 S3FS_PRN_ERR("Failed to lazy setup in single get object request.");
3375 return -EIO;
3376 }
3377
3378 S3FS_PRN_INFO3("downloading... [path=%s][fd=%d]", tpath, fd);
3379
3380 result = RequestPerform();
3381 partdata.clear();
3382
3383 return result;
3384 }
3385
CheckBucket()3386 int S3fsCurl::CheckBucket()
3387 {
3388 S3FS_PRN_INFO3("check a bucket.");
3389
3390 if(!CreateCurlHandle()){
3391 return -EIO;
3392 }
3393 std::string urlargs;
3394 if(S3fsCurl::IsListObjectsV2()){
3395 query_string = "list-type=2";
3396 urlargs = "?" + query_string;
3397 }
3398 std::string resource;
3399 std::string turl;
3400 MakeUrlResource("/", resource, turl);
3401
3402 turl += urlargs;
3403 url = prepare_url(turl.c_str());
3404 path = "/"; // Only check the presence of the bucket, not the entire virtual path.
3405 requestHeaders = NULL;
3406 responseHeaders.clear();
3407 bodydata.Clear();
3408
3409 op = "GET";
3410 type = REQTYPE_CHKBUCKET;
3411
3412 // setopt
3413 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
3414 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
3415 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
3416 curl_easy_setopt(hCurl, CURLOPT_UNRESTRICTED_AUTH, 1L);
3417 S3fsCurl::AddUserAgent(hCurl); // put User-Agent
3418
3419 int result = RequestPerform();
3420 if (result != 0) {
3421 S3FS_PRN_ERR("Check bucket failed, S3 response: %s", bodydata.str());
3422 }
3423 return result;
3424 }
3425
ListBucketRequest(const char * tpath,const char * query)3426 int S3fsCurl::ListBucketRequest(const char* tpath, const char* query)
3427 {
3428 S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath));
3429
3430 if(!tpath){
3431 return -EINVAL;
3432 }
3433 if(!CreateCurlHandle()){
3434 return -EIO;
3435 }
3436 std::string resource;
3437 std::string turl;
3438 MakeUrlResource("", resource, turl); // NOTICE: path is "".
3439 if(query){
3440 turl += "?";
3441 turl += query;
3442 query_string = query;
3443 }
3444
3445 url = prepare_url(turl.c_str());
3446 path = get_realpath(tpath);
3447 requestHeaders = NULL;
3448 responseHeaders.clear();
3449 bodydata.Clear();
3450
3451 op = "GET";
3452 type = REQTYPE_LISTBUCKET;
3453
3454 // setopt
3455 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
3456 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
3457 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
3458 if(S3fsCurl::is_verbose){
3459 curl_easy_setopt(hCurl, CURLOPT_DEBUGFUNCTION, S3fsCurl::CurlDebugBodyInFunc); // replace debug function
3460 }
3461 S3fsCurl::AddUserAgent(hCurl); // put User-Agent
3462
3463 return RequestPerform();
3464 }
3465
3466 //
3467 // Initialize multipart upload
3468 //
3469 // Example :
3470 // POST /example-object?uploads HTTP/1.1
3471 // Host: example-bucket.s3.amazonaws.com
3472 // Date: Mon, 1 Nov 2010 20:34:56 GMT
3473 // Authorization: AWS VGhpcyBtZXNzYWdlIHNpZ25lZCBieSBlbHZpbmc=
3474 //
PreMultipartPostRequest(const char * tpath,headers_t & meta,std::string & upload_id,bool is_copy)3475 int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, std::string& upload_id, bool is_copy)
3476 {
3477 S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath));
3478
3479 if(!tpath){
3480 return -EINVAL;
3481 }
3482 if(!CreateCurlHandle()){
3483 return -EIO;
3484 }
3485 std::string resource;
3486 std::string turl;
3487 MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
3488
3489 query_string = "uploads";
3490 turl += "?" + query_string;
3491 url = prepare_url(turl.c_str());
3492 path = get_realpath(tpath);
3493 requestHeaders = NULL;
3494 bodydata.Clear();
3495 responseHeaders.clear();
3496
3497 std::string contype = S3fsCurl::LookupMimeType(std::string(tpath));
3498
3499 for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
3500 std::string key = lower(iter->first);
3501 std::string value = iter->second;
3502 if(is_prefix(key.c_str(), "x-amz-acl")){
3503 // not set value, but after set it.
3504 }else if(is_prefix(key.c_str(), "x-amz-meta")){
3505 requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
3506 }else if(key == "x-amz-server-side-encryption" && value != "aws:kms"){
3507 // Only copy mode.
3508 if(is_copy && !AddSseRequestHead(sse_type_t::SSE_S3, value, false, true)){
3509 S3FS_PRN_WARN("Failed to insert SSE-S3 header.");
3510 }
3511 }else if(key == "x-amz-server-side-encryption-aws-kms-key-id"){
3512 // Only copy mode.
3513 if(is_copy && !value.empty() && !AddSseRequestHead(sse_type_t::SSE_KMS, value, false, true)){
3514 S3FS_PRN_WARN("Failed to insert SSE-KMS header.");
3515 }
3516 }else if(key == "x-amz-server-side-encryption-customer-key-md5"){
3517 // Only copy mode.
3518 if(is_copy){
3519 if(!AddSseRequestHead(sse_type_t::SSE_C, value, true, true) || !AddSseRequestHead(sse_type_t::SSE_C, value, true, false)){
3520 S3FS_PRN_WARN("Failed to insert SSE-C header.");
3521 }
3522 }
3523 }
3524 }
3525 // "x-amz-acl", storage class, sse
3526 if(S3fsCurl::default_acl != acl_t::PRIVATE){
3527 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-acl", S3fsCurl::default_acl.str());
3528 }
3529 if(strcasecmp(GetStorageClass().c_str(), "STANDARD") != 0){
3530 requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", GetStorageClass().c_str());
3531 }
3532 // SSE
3533 if(!is_copy){
3534 std::string ssevalue;
3535 if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false, false)){
3536 S3FS_PRN_WARN("Failed to set SSE header, but continue...");
3537 }
3538 }
3539 if(is_use_ahbe){
3540 // set additional header by ahbe conf
3541 requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath);
3542 }
3543
3544 requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL);
3545 requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
3546
3547 op = "POST";
3548 type = REQTYPE_PREMULTIPOST;
3549
3550 // setopt
3551 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
3552 curl_easy_setopt(hCurl, CURLOPT_POST, true); // POST
3553 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
3554 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
3555 curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, 0);
3556 curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0); // Content-Length
3557 S3fsCurl::AddUserAgent(hCurl); // put User-Agent
3558
3559 // request
3560 int result;
3561 if(0 != (result = RequestPerform())){
3562 bodydata.Clear();
3563 return result;
3564 }
3565
3566 if(!simple_parse_xml(bodydata.str(), bodydata.size(), "UploadId", upload_id)){
3567 bodydata.Clear();
3568 return -EIO;
3569 }
3570
3571 bodydata.Clear();
3572 return 0;
3573 }
3574
CompleteMultipartPostRequest(const char * tpath,const std::string & upload_id,etaglist_t & parts)3575 int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, const std::string& upload_id, etaglist_t& parts)
3576 {
3577 S3FS_PRN_INFO3("[tpath=%s][parts=%zu]", SAFESTRPTR(tpath), parts.size());
3578
3579 if(!tpath){
3580 return -EINVAL;
3581 }
3582
3583 // make contents
3584 std::string postContent;
3585 postContent += "<CompleteMultipartUpload>\n";
3586 int cnt = 0;
3587 for(etaglist_t::iterator it = parts.begin(); it != parts.end(); ++it, ++cnt){
3588 if(it->empty()){
3589 S3FS_PRN_ERR("%d file part is not finished uploading.", cnt + 1);
3590 return -EIO;
3591 }
3592 postContent += "<Part>\n";
3593 postContent += " <PartNumber>" + str(cnt + 1) + "</PartNumber>\n";
3594 postContent += " <ETag>" + *it + "</ETag>\n";
3595 postContent += "</Part>\n";
3596 }
3597 postContent += "</CompleteMultipartUpload>\n";
3598
3599 // set postdata
3600 postdata = reinterpret_cast<const unsigned char*>(postContent.c_str());
3601 b_postdata = postdata;
3602 postdata_remaining = postContent.size(); // without null
3603 b_postdata_remaining = postdata_remaining;
3604
3605 if(!CreateCurlHandle()){
3606 postdata = NULL;
3607 b_postdata = NULL;
3608 return -EIO;
3609 }
3610 std::string resource;
3611 std::string turl;
3612 MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
3613
3614 query_string = "uploadId=" + upload_id;
3615 turl += "?" + query_string;
3616 url = prepare_url(turl.c_str());
3617 path = get_realpath(tpath);
3618 requestHeaders = NULL;
3619 bodydata.Clear();
3620 responseHeaders.clear();
3621 std::string contype = "application/xml";
3622
3623 requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL);
3624 requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
3625
3626 op = "POST";
3627 type = REQTYPE_COMPLETEMULTIPOST;
3628
3629 // setopt
3630 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
3631 curl_easy_setopt(hCurl, CURLOPT_POST, true); // POST
3632 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
3633 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
3634 curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, static_cast<curl_off_t>(postdata_remaining));
3635 curl_easy_setopt(hCurl, CURLOPT_READDATA, (void*)this);
3636 curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::ReadCallback);
3637 if(S3fsCurl::is_verbose){
3638 curl_easy_setopt(hCurl, CURLOPT_DEBUGFUNCTION, S3fsCurl::CurlDebugBodyOutFunc); // replace debug function
3639 }
3640 S3fsCurl::AddUserAgent(hCurl); // put User-Agent
3641
3642 // request
3643 int result = RequestPerform();
3644 bodydata.Clear();
3645 postdata = NULL;
3646 b_postdata = NULL;
3647
3648 return result;
3649 }
3650
MultipartListRequest(std::string & body)3651 int S3fsCurl::MultipartListRequest(std::string& body)
3652 {
3653 S3FS_PRN_INFO3("list request(multipart)");
3654
3655 if(!CreateCurlHandle()){
3656 return -EIO;
3657 }
3658 std::string resource;
3659 std::string turl;
3660 path = get_realpath("/");
3661 MakeUrlResource(path.c_str(), resource, turl);
3662
3663 query_string = "uploads";
3664 turl += "?" + query_string;
3665 url = prepare_url(turl.c_str());
3666 requestHeaders = NULL;
3667 responseHeaders.clear();
3668 bodydata.Clear();
3669
3670 requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL);
3671
3672 op = "GET";
3673 type = REQTYPE_MULTILIST;
3674
3675 // setopt
3676 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
3677 curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
3678 curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
3679 S3fsCurl::AddUserAgent(hCurl); // put User-Agent
3680
3681 int result;
3682 if(0 == (result = RequestPerform()) && 0 < bodydata.size()){
3683 body = bodydata.str();
3684 }else{
3685 body = "";
3686 }
3687 bodydata.Clear();
3688
3689 return result;
3690 }
3691
AbortMultipartUpload(const char * tpath,const std::string & upload_id)3692 int S3fsCurl::AbortMultipartUpload(const char* tpath, const std::string& upload_id)
3693 {
3694 S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath));
3695
3696 if(!tpath){
3697 return -EINVAL;
3698 }
3699 if(!CreateCurlHandle()){
3700 return -EIO;
3701 }
3702 std::string resource;
3703 std::string turl;
3704 MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
3705
3706 query_string = "uploadId=" + upload_id;
3707 turl += "?" + query_string;
3708 url = prepare_url(turl.c_str());
3709 path = get_realpath(tpath);
3710 requestHeaders = NULL;
3711 responseHeaders.clear();
3712
3713 op = "DELETE";
3714 type = REQTYPE_ABORTMULTIUPLOAD;
3715
3716 curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
3717 curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE");
3718 S3fsCurl::AddUserAgent(hCurl); // put User-Agent
3719
3720 return RequestPerform();
3721 }
3722
3723 //
3724 // PUT /ObjectName?partNumber=PartNumber&uploadId=UploadId HTTP/1.1
3725 // Host: BucketName.s3.amazonaws.com
3726 // Date: date
3727 // Content-Length: Size
3728 // Authorization: Signature
3729 //
3730 // PUT /my-movie.m2ts?partNumber=1&uploadId=VCVsb2FkIElEIGZvciBlbZZpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZR HTTP/1.1
3731 // Host: example-bucket.s3.amazonaws.com
3732 // Date: Mon, 1 Nov 2010 20:34:56 GMT
3733 // Content-Length: 10485760
3734 // Content-MD5: pUNXr/BjKK5G2UKvaRRrOA==
3735 // Authorization: AWS VGhpcyBtZXNzYWdlIHNpZ25lZGGieSRlbHZpbmc=
3736 //
UploadMultipartPostSetup(const char * tpath,int part_num,const std::string & upload_id)3737 int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const std::string& upload_id)
3738 {
3739 S3FS_PRN_INFO3("[tpath=%s][start=%lld][size=%lld][part=%d]", SAFESTRPTR(tpath), static_cast<long long int>(partdata.startpos), static_cast<long long int>(partdata.size), part_num);
3740
3741 if(-1 == partdata.fd || -1 == partdata.startpos || -1 == partdata.size){
3742 return -EINVAL;
3743 }
3744
3745 requestHeaders = NULL;
3746
3747 // make md5 and file pointer
3748 if(S3fsCurl::is_content_md5){
3749 unsigned char *md5raw = s3fs_md5_fd(partdata.fd, partdata.startpos, partdata.size);
3750 if(md5raw == NULL){
3751 S3FS_PRN_ERR("Could not make md5 for file(part %d)", part_num);
3752 return -EIO;
3753 }
3754 partdata.etag = s3fs_hex_lower(md5raw, get_md5_digest_length());
3755 char* md5base64p = s3fs_base64(md5raw, get_md5_digest_length());
3756 requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-MD5", md5base64p);
3757 delete[] md5base64p;
3758 delete[] md5raw;
3759 }
3760
3761 // make request
3762 query_string = "partNumber=" + str(part_num) + "&uploadId=" + upload_id;
3763 std::string urlargs = "?" + query_string;
3764 std::string resource;
3765 std::string turl;
3766 MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
3767
3768 turl += urlargs;
3769 url = prepare_url(turl.c_str());
3770 path = get_realpath(tpath);
3771 bodydata.Clear();
3772 headdata.Clear();
3773 responseHeaders.clear();
3774
3775 // SSE
3776 if(sse_type_t::SSE_C == S3fsCurl::GetSseType()){
3777 std::string ssevalue;
3778 if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false, false)){
3779 S3FS_PRN_WARN("Failed to set SSE header, but continue...");
3780 }
3781 }
3782
3783 requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL);
3784
3785 op = "PUT";
3786 type = REQTYPE_UPLOADMULTIPOST;
3787
3788 // set lazy function
3789 fpLazySetup = UploadMultipartPostSetCurlOpts;
3790
3791 return 0;
3792 }
3793
UploadMultipartPostRequest(const char * tpath,int part_num,const std::string & upload_id)3794 int S3fsCurl::UploadMultipartPostRequest(const char* tpath, int part_num, const std::string& upload_id)
3795 {
3796 int result;
3797
3798 S3FS_PRN_INFO3("[tpath=%s][start=%lld][size=%lld][part=%d]", SAFESTRPTR(tpath), static_cast<long long int>(partdata.startpos), static_cast<long long int>(partdata.size), part_num);
3799
3800 // setup
3801 if(0 != (result = S3fsCurl::UploadMultipartPostSetup(tpath, part_num, upload_id))){
3802 return result;
3803 }
3804
3805 if(!fpLazySetup || !fpLazySetup(this)){
3806 S3FS_PRN_ERR("Failed to lazy setup in multipart upload post request.");
3807 return -EIO;
3808 }
3809
3810 // request
3811 if(0 == (result = RequestPerform())){
3812 // UploadMultipartPostComplete returns true on success -> convert to 0
3813 result = !UploadMultipartPostComplete();
3814 }
3815
3816 // closing
3817 bodydata.Clear();
3818 headdata.Clear();
3819
3820 return result;
3821 }
3822
CopyMultipartPostSetup(const char * from,const char * to,int part_num,const std::string & upload_id,headers_t & meta)3823 int S3fsCurl::CopyMultipartPostSetup(const char* from, const char* to, int part_num, const std::string& upload_id, headers_t& meta)
3824 {
3825 S3FS_PRN_INFO3("[from=%s][to=%s][part=%d]", SAFESTRPTR(from), SAFESTRPTR(to), part_num);
3826
3827 if(!from || !to){
3828 return -EINVAL;
3829 }
3830 query_string = "partNumber=" + str(part_num) + "&uploadId=" + upload_id;
3831 std::string urlargs = "?" + query_string;
3832 std::string resource;
3833 std::string turl;
3834 MakeUrlResource(get_realpath(to).c_str(), resource, turl);
3835
3836 turl += urlargs;
3837 url = prepare_url(turl.c_str());
3838 path = get_realpath(to);
3839 requestHeaders = NULL;
3840 responseHeaders.clear();
3841 bodydata.Clear();
3842 headdata.Clear();
3843
3844 std::string contype = S3fsCurl::LookupMimeType(std::string(to));
3845 requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
3846
3847 // Make request headers
3848 for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
3849 std::string key = lower(iter->first);
3850 std::string value = iter->second;
3851 if(key == "x-amz-copy-source"){
3852 requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
3853 }else if(key == "x-amz-copy-source-range"){
3854 requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
3855 }
3856 // NOTICE: x-amz-acl, x-amz-server-side-encryption is not set!
3857 }
3858
3859 op = "PUT";
3860 type = REQTYPE_COPYMULTIPOST;
3861
3862 // set lazy function
3863 fpLazySetup = CopyMultipartPostSetCurlOpts;
3864
3865 // request
3866 S3FS_PRN_INFO3("copying... [from=%s][to=%s][part=%d]", from, to, part_num);
3867
3868 return 0;
3869 }
3870
UploadMultipartPostComplete()3871 bool S3fsCurl::UploadMultipartPostComplete()
3872 {
3873 headers_t::iterator it = responseHeaders.find("ETag");
3874 if (it == responseHeaders.end()) {
3875 return false;
3876 }
3877
3878 // check etag(md5);
3879 //
3880 // The ETAG when using SSE_C and SSE_KMS does not reflect the MD5 we sent
3881 // SSE_C: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
3882 // SSE_KMS is ignored in the above, but in the following it states the same in the highlights:
3883 // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
3884 //
3885 if(S3fsCurl::is_content_md5 && sse_type_t::SSE_C != S3fsCurl::GetSseType() && sse_type_t::SSE_KMS != S3fsCurl::GetSseType()){
3886 if(!etag_equals(it->second, partdata.etag)){
3887 return false;
3888 }
3889 }
3890 (*partdata.petag) = it->second;
3891 partdata.uploaded = true;
3892
3893 return true;
3894 }
3895
CopyMultipartPostCallback(S3fsCurl * s3fscurl)3896 bool S3fsCurl::CopyMultipartPostCallback(S3fsCurl* s3fscurl)
3897 {
3898 if(!s3fscurl){
3899 return false;
3900 }
3901
3902 return s3fscurl->CopyMultipartPostComplete();
3903 }
3904
CopyMultipartPostComplete()3905 bool S3fsCurl::CopyMultipartPostComplete()
3906 {
3907 std::string etag;
3908 partdata.uploaded = simple_parse_xml(bodydata.str(), bodydata.size(), "ETag", etag);
3909 if(etag.size() >= 2 && *etag.begin() == '"' && *etag.rbegin() == '"'){
3910 etag.erase(etag.size() - 1);
3911 etag.erase(0, 1);
3912 }
3913 (*partdata.petag) = etag;
3914
3915 bodydata.Clear();
3916 headdata.Clear();
3917
3918 return true;
3919 }
3920
MixMultipartPostComplete()3921 bool S3fsCurl::MixMultipartPostComplete()
3922 {
3923 bool result;
3924 if(-1 == partdata.fd){
3925 result = CopyMultipartPostComplete();
3926 }else{
3927 result = UploadMultipartPostComplete();
3928 }
3929 return result;
3930 }
3931
MultipartHeadRequest(const char * tpath,off_t size,headers_t & meta,bool is_copy)3932 int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta, bool is_copy)
3933 {
3934 int result;
3935 std::string upload_id;
3936 off_t chunk;
3937 off_t bytes_remaining;
3938 etaglist_t list;
3939
3940 S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath));
3941
3942 if(0 != (result = PreMultipartPostRequest(tpath, meta, upload_id, is_copy))){
3943 return result;
3944 }
3945 DestroyCurlHandle();
3946
3947 // Initialize S3fsMultiCurl
3948 S3fsMultiCurl curlmulti(GetMaxParallelCount());
3949 curlmulti.SetSuccessCallback(S3fsCurl::CopyMultipartPostCallback);
3950 curlmulti.SetRetryCallback(S3fsCurl::CopyMultipartPostRetryCallback);
3951
3952 for(bytes_remaining = size, chunk = 0; 0 < bytes_remaining; bytes_remaining -= chunk){
3953 chunk = bytes_remaining > GetMultipartCopySize() ? GetMultipartCopySize() : bytes_remaining;
3954
3955 std::ostringstream strrange;
3956 strrange << "bytes=" << (size - bytes_remaining) << "-" << (size - bytes_remaining + chunk - 1);
3957 meta["x-amz-copy-source-range"] = strrange.str();
3958
3959 // s3fscurl sub object
3960 S3fsCurl* s3fscurl_para = new S3fsCurl(true);
3961 s3fscurl_para->b_from = SAFESTRPTR(tpath);
3962 s3fscurl_para->b_meta = meta;
3963 s3fscurl_para->partdata.add_etag_list(&list);
3964
3965 // initiate upload part for parallel
3966 if(0 != (result = s3fscurl_para->CopyMultipartPostSetup(tpath, tpath, static_cast<int>(list.size()), upload_id, meta))){
3967 S3FS_PRN_ERR("failed uploading part setup(%d)", result);
3968 delete s3fscurl_para;
3969 return result;
3970 }
3971
3972 // set into parallel object
3973 if(!curlmulti.SetS3fsCurlObject(s3fscurl_para)){
3974 S3FS_PRN_ERR("Could not make curl object into multi curl(%s).", tpath);
3975 delete s3fscurl_para;
3976 return -EIO;
3977 }
3978 }
3979
3980 // Multi request
3981 if(0 != (result = curlmulti.Request())){
3982 S3FS_PRN_ERR("error occurred in multi request(errno=%d).", result);
3983
3984 S3fsCurl s3fscurl_abort(true);
3985 int result2 = s3fscurl_abort.AbortMultipartUpload(tpath, upload_id);
3986 s3fscurl_abort.DestroyCurlHandle();
3987 if(result2 != 0){
3988 S3FS_PRN_ERR("error aborting multipart upload(errno=%d).", result2);
3989 }
3990 return result;
3991 }
3992
3993 if(0 != (result = CompleteMultipartPostRequest(tpath, upload_id, list))){
3994 return result;
3995 }
3996 return 0;
3997 }
3998
MultipartUploadRequest(const std::string & upload_id,const char * tpath,int fd,off_t offset,off_t size,int part_num,std::string * petag)3999 int S3fsCurl::MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, int part_num, std::string* petag)
4000 {
4001 S3FS_PRN_INFO3("[upload_id=%s][tpath=%s][fd=%d][offset=%lld][size=%lld]", upload_id.c_str(), SAFESTRPTR(tpath), fd, static_cast<long long int>(offset), static_cast<long long int>(size));
4002
4003 // duplicate fd
4004 int fd2;
4005 if(-1 == (fd2 = dup(fd)) || 0 != lseek(fd2, 0, SEEK_SET)){
4006 S3FS_PRN_ERR("Could not duplicate file descriptor(errno=%d)", errno);
4007 if(-1 != fd2){
4008 close(fd2);
4009 }
4010 return -errno;
4011 }
4012
4013 // set
4014 partdata.fd = fd2;
4015 partdata.startpos = offset;
4016 partdata.size = size;
4017 b_partdata_startpos = partdata.startpos;
4018 b_partdata_size = partdata.size;
4019 partdata.add_etag(petag);
4020
4021 // upload part
4022 int result;
4023 if(0 != (result = UploadMultipartPostRequest(tpath, part_num, upload_id))){
4024 S3FS_PRN_ERR("failed uploading %d part by error(%d)", part_num, result);
4025 close(fd2);
4026 return result;
4027 }
4028 DestroyCurlHandle();
4029 close(fd2);
4030
4031 return 0;
4032 }
4033
MultipartRenameRequest(const char * from,const char * to,headers_t & meta,off_t size)4034 int S3fsCurl::MultipartRenameRequest(const char* from, const char* to, headers_t& meta, off_t size)
4035 {
4036 int result;
4037 std::string upload_id;
4038 off_t chunk;
4039 off_t bytes_remaining;
4040 etaglist_t list;
4041
4042 S3FS_PRN_INFO3("[from=%s][to=%s]", SAFESTRPTR(from), SAFESTRPTR(to));
4043
4044 std::string srcresource;
4045 std::string srcurl;
4046 MakeUrlResource(get_realpath(from).c_str(), srcresource, srcurl);
4047
4048 meta["Content-Type"] = S3fsCurl::LookupMimeType(std::string(to));
4049 meta["x-amz-copy-source"] = srcresource;
4050
4051 if(0 != (result = PreMultipartPostRequest(to, meta, upload_id, true))){
4052 return result;
4053 }
4054 DestroyCurlHandle();
4055
4056 // Initialize S3fsMultiCurl
4057 S3fsMultiCurl curlmulti(GetMaxParallelCount());
4058 curlmulti.SetSuccessCallback(S3fsCurl::CopyMultipartPostCallback);
4059 curlmulti.SetRetryCallback(S3fsCurl::CopyMultipartPostRetryCallback);
4060
4061 for(bytes_remaining = size, chunk = 0; 0 < bytes_remaining; bytes_remaining -= chunk){
4062 chunk = bytes_remaining > GetMultipartCopySize() ? GetMultipartCopySize() : bytes_remaining;
4063
4064 std::ostringstream strrange;
4065 strrange << "bytes=" << (size - bytes_remaining) << "-" << (size - bytes_remaining + chunk - 1);
4066 meta["x-amz-copy-source-range"] = strrange.str();
4067
4068 // s3fscurl sub object
4069 S3fsCurl* s3fscurl_para = new S3fsCurl(true);
4070 s3fscurl_para->b_from = SAFESTRPTR(from);
4071 s3fscurl_para->b_meta = meta;
4072 s3fscurl_para->partdata.add_etag_list(&list);
4073
4074 // initiate upload part for parallel
4075 if(0 != (result = s3fscurl_para->CopyMultipartPostSetup(from, to, static_cast<int>(list.size()), upload_id, meta))){
4076 S3FS_PRN_ERR("failed uploading part setup(%d)", result);
4077 delete s3fscurl_para;
4078 return result;
4079 }
4080
4081 // set into parallel object
4082 if(!curlmulti.SetS3fsCurlObject(s3fscurl_para)){
4083 S3FS_PRN_ERR("Could not make curl object into multi curl(%s).", to);
4084 delete s3fscurl_para;
4085 return -EIO;
4086 }
4087 }
4088
4089 // Multi request
4090 if(0 != (result = curlmulti.Request())){
4091 S3FS_PRN_ERR("error occurred in multi request(errno=%d).", result);
4092
4093 S3fsCurl s3fscurl_abort(true);
4094 int result2 = s3fscurl_abort.AbortMultipartUpload(to, upload_id);
4095 s3fscurl_abort.DestroyCurlHandle();
4096 if(result2 != 0){
4097 S3FS_PRN_ERR("error aborting multipart upload(errno=%d).", result2);
4098 }
4099 return result;
4100 }
4101
4102 if(0 != (result = CompleteMultipartPostRequest(to, upload_id, list))){
4103 return result;
4104 }
4105 return 0;
4106 }
4107
4108 /*
4109 * Local variables:
4110 * tab-width: 4
4111 * c-basic-offset: 4
4112 * End:
4113 * vim600: expandtab sw=4 ts=4 fdm=marker
4114 * vim<600: expandtab sw=4 ts=4
4115 */
4116